From 5d38f72a0e7029318d49b17d8d943d7ba38cd205 Mon Sep 17 00:00:00 2001 From: Ishaan Bhardwaj Date: Thu, 5 Jun 2025 21:01:33 +0000 Subject: [PATCH 01/13] Add stress and scalability test framework for EFS CSI Driver --- test/e2e/stress-scale-tests/README.md | 91 + .../__pycache__/cluster_setup.cpython-312.pyc | Bin 0 -> 31301 bytes .../cleanup_test_resources.py | 231 ++ .../cluster-config-code/cluster_setup.py | 756 +++++ .../efs-csi-test-1749045745.yaml | 51 + .../efs-csi-test-1749045783.yaml | 51 + .../efs-csi-test-1749048222.yaml | 51 + .../efs-csi-test-1749048235.yaml | 51 + .../efs-csi-test-1749048734.yaml | 51 + .../efs-csi-test-1749051234.yaml | 51 + .../efs-csi-test-1749051325.yaml | 51 + .../efs-csi-test-1749051374.yaml | 51 + .../efs-csi-test-1749051392.yaml | 51 + .../config/orchestrator_config.yaml | 124 + .../logs/cleanup_20250605_190904.log | 129 + .../stress-scale-tests/logs/cluster_setup.log | 383 +++ .../e2e/stress-scale-tests/logs/efs_tests.log | 2869 +++++++++++++++++ ...creation-error-efs-csi-test-1749045783.log | 8 + ...creation-error-efs-csi-test-1749048222.log | 8 + ...creation-error-efs-csi-test-1749048235.log | 17 + ...creation-error-efs-csi-test-1749048734.log | 17 + ...creation-error-efs-csi-test-1749051325.log | 17 + ...creation-error-efs-csi-test-1749051374.log | 17 + ...creation-error-efs-csi-test-1749051392.log | 254 ++ .../logs/orchestrator_20250603_145210.log | 147 + .../logs/orchestrator_20250603_154721.log | 8 + .../logs/orchestrator_20250603_154805.log | 455 +++ .../logs/orchestrator_20250603_174810.log | 8 + .../logs/orchestrator_20250603_175820.log | 77 + .../logs/orchestrator_20250603_204418.log | 8 + .../logs/orchestrator_20250603_204433.log | 94 + .../logs/orchestrator_20250603_204821.log | 175 + .../logs/orchestrator_20250603_210011.log | 158 + .../logs/orchestrator_20250604_204301.log | 125 + .../logs/orchestrator_20250604_211255.log | 43 + .../logs/orchestrator_20250604_211353.log | 80 + .../logs/orchestrator_20250605_132802.log | 202 ++ .../logs/orchestrator_20250605_135002.log | 123 + .../logs/orchestrator_20250605_140408.log | 138 + .../logs/orchestrator_20250605_141321.log | 7 + .../logs/orchestrator_20250605_141434.log | 142 + .../logs/orchestrator_20250605_143926.log | 73 + .../logs/orchestrator_20250605_145133.log | 111 + .../logs/orchestrator_20250605_145937.log | 134 + .../logs/orchestrator_20250605_150619.log | 147 + .../logs/orchestrator_20250605_151529.log | 7 + .../logs/orchestrator_20250605_151604.log | 42 + .../logs/orchestrator_20250605_151742.log | 17 + .../logs/orchestrator_20250605_153750.log | 249 ++ .../logs/orchestrator_20250605_154546.log | 254 ++ .../logs/orchestrator_20250605_155948.log | 280 ++ .../logs/orchestrator_20250605_160844.log | 281 ++ .../logs/orchestrator_20250605_162134.log | 7 + .../logs/orchestrator_20250605_162155.log | 52 + .../logs/orchestrator_20250605_162320.log | 27 + .../logs/orchestrator_20250605_162354.log | 247 ++ .../logs/orchestrator_20250605_163114.log | 67 + .../logs/orchestrator_20250605_163340.log | 42 + .../logs/orchestrator_20250605_163501.log | 205 ++ .../logs/orchestrator_20250605_164709.log | 67 + .../logs/orchestrator_20250605_164934.log | 246 ++ .../logs/orchestrator_20250605_170306.log | 17 + .../logs/orchestrator_20250605_170342.log | 17 + .../logs/orchestrator_20250605_170405.log | 39 + .../logs/orchestrator_20250605_170502.log | 12 + .../logs/orchestrator_20250605_170526.log | 234 ++ .../logs/orchestrator_20250605_182216.log | 7 + .../logs/orchestrator_20250605_182254.log | 207 ++ .../logs/orchestrator_20250605_183835.log | 983 ++++++ .../logs/orchestrator_20250605_191215.log | 158 + .../logs/orchestrator_20250605_192824.log | 7 + .../logs/orchestrator_20250605_192851.log | 207 ++ .../logs/orchestrator_20250605_195225.log | 335 ++ .../logs/orchestrator_20250605_200645.log | 348 ++ .../chaos/efs_chaos_20250603_135616.json | 29 + ...tests_20250603_145849_20250603_145849.json | 104 + ...0250603_145849_20250603_145849_summary.txt | 61 + ...tests_20250603_160604_20250603_160604.json | 104 + ...0250603_160604_20250603_160604_summary.txt | 61 + ...tests_20250603_180130_20250603_180130.json | 104 + ...0250603_180130_20250603_180130_summary.txt | 61 + ...tests_20250603_204448_20250603_204448.json | 104 + ...0250603_204448_20250603_204448_summary.txt | 61 + ...tests_20250603_205333_20250603_205333.json | 104 + ...0250603_205333_20250603_205333_summary.txt | 61 + ...tests_20250603_210938_20250603_210938.json | 104 + ...0250603_210938_20250603_210938_summary.txt | 61 + ...tests_20250604_204834_20250604_204834.json | 104 + ...tests_20250604_211347_20250604_211347.json | 104 + ...tests_20250604_211457_20250604_211457.json | 104 + ...tests_20250605_133650_20250605_133650.json | 104 + ...tests_20250605_135626_20250605_135626.json | 104 + ...tests_20250605_141131_20250605_141131.json | 104 + ...tests_20250605_142059_20250605_142059.json | 104 + ...tests_20250605_145750_20250605_145750.json | 104 + ...tests_20250605_150603_20250605_150603.json | 104 + ...tests_20250605_151246_20250605_151246.json | 104 + ...tests_20250605_154402_20250605_154402.json | 104 + ...tests_20250605_155202_20250605_155202.json | 104 + ...tests_20250605_160701_20250605_160701.json | 104 + ...tests_20250605_161509_20250605_161509.json | 104 + ...tests_20250605_163010_20250605_163010.json | 104 + ...tests_20250605_164017_20250605_164017.json | 104 + ...tests_20250605_165552_20250605_165552.json | 104 + ...tests_20250605_171216_20250605_171216.json | 104 + ...tests_20250605_182932_20250605_182932.json | 104 + ...tests_20250605_193407_20250605_193407.json | 104 + ...tests_20250605_200124_20250605_200124.json | 104 + ...tests_20250605_201716_20250605_201716.json | 104 + .../efs_orchestrator_20250603_145849.json | 59 + .../efs_orchestrator_20250603_160604.json | 59 + .../efs_orchestrator_20250603_180130.json | 59 + .../efs_orchestrator_20250603_204448.json | 59 + .../efs_orchestrator_20250603_205333.json | 59 + .../efs_orchestrator_20250603_210938.json | 59 + .../efs_orchestrator_20250604_204834.json | 59 + .../efs_orchestrator_20250604_211347.json | 59 + .../efs_orchestrator_20250604_211457.json | 59 + .../efs_orchestrator_20250605_133650.json | 59 + .../efs_orchestrator_20250605_135626.json | 59 + .../efs_orchestrator_20250605_141131.json | 59 + .../efs_orchestrator_20250605_142059.json | 59 + .../efs_orchestrator_20250605_145750.json | 59 + .../efs_orchestrator_20250605_150603.json | 59 + .../efs_orchestrator_20250605_151246.json | 59 + .../efs_orchestrator_20250605_154402.json | 59 + .../efs_orchestrator_20250605_155202.json | 59 + .../efs_orchestrator_20250605_160701.json | 59 + .../efs_orchestrator_20250605_161509.json | 59 + .../efs_orchestrator_20250605_163010.json | 59 + .../efs_orchestrator_20250605_164017.json | 59 + .../efs_orchestrator_20250605_165552.json | 59 + .../efs_orchestrator_20250605_171216.json | 59 + .../efs_orchestrator_20250605_182932.json | 59 + .../efs_orchestrator_20250605_193407.json | 59 + .../efs_orchestrator_20250605_200124.json | 59 + .../efs_orchestrator_20250605_201716.json | 59 + ...tests_20250602_195427_20250602_195427.json | 100 + ...0250602_195427_20250602_195427_summary.txt | 22 + ...tests_20250602_183718_20250602_183718.json | 58 + ...0250602_183718_20250602_183718_summary.txt | 40 + ...tests_20250602_201648_20250602_201648.json | 58 + ...0250602_201648_20250602_201648_summary.txt | 40 + ...tests_20250602_210512_20250602_210512.json | 79 + ...0250602_210512_20250602_210512_summary.txt | 40 + ...tests_20250603_135616_20250603_135616.json | 216 ++ ...0250603_135616_20250603_135616_summary.txt | 52 + test/e2e/stress-scale-tests/requirements.txt | 6 + test/e2e/stress-scale-tests/run_tests.py | 327 ++ test/e2e/stress-scale-tests/tests/__init__.py | 0 .../__pycache__/__init__.cpython-312.pyc | Bin 0 -> 200 bytes .../__pycache__/base_test.cpython-312.pyc | Bin 0 -> 8668 bytes .../__pycache__/chaos_test.cpython-312.pyc | Bin 0 -> 28455 bytes .../concurrent_api_test.cpython-312.pyc | Bin 0 -> 21034 bytes .../__pycache__/io_tests.cpython-312.pyc | Bin 0 -> 16448 bytes .../__pycache__/mount_tests.cpython-312.pyc | Bin 0 -> 7627 bytes .../__pycache__/orchestrator.cpython-312.pyc | Bin 0 -> 75796 bytes .../run_stress_tests.cpython-312.pyc | Bin 0 -> 11283 bytes .../test_access_point_scaling.cpython-312.pyc | Bin 0 -> 16665 bytes ...ic_operations.cpython-312-pytest-8.3.5.pyc | Bin 0 -> 10279 bytes .../test_scalability.cpython-312.pyc | Bin 0 -> 21309 bytes .../test_statefulset_scaling.cpython-312.pyc | Bin 0 -> 13283 bytes ...ss_operations.cpython-312-pytest-8.3.5.pyc | Bin 0 -> 20282 bytes .../test_stress_operations.cpython-312.pyc | Bin 0 -> 48424 bytes .../stress-scale-tests/tests/orchestrator.py | 1663 ++++++++++ test/e2e/stress-scale-tests/utils/__init__.py | 0 .../__pycache__/__init__.cpython-312.pyc | Bin 0 -> 200 bytes .../metrics_collector.cpython-312.pyc | Bin 0 -> 17194 bytes .../report_generator.cpython-312.pyc | Bin 0 -> 18806 bytes .../utils/metrics_collector.py | 420 +++ .../utils/report_generator.py | 352 ++ 171 files changed, 20928 insertions(+) create mode 100644 test/e2e/stress-scale-tests/README.md create mode 100644 test/e2e/stress-scale-tests/__pycache__/cluster_setup.cpython-312.pyc create mode 100755 test/e2e/stress-scale-tests/cleanup_test_resources.py create mode 100644 test/e2e/stress-scale-tests/cluster-config-code/cluster_setup.py create mode 100644 test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749045745.yaml create mode 100644 test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749045783.yaml create mode 100644 test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749048222.yaml create mode 100644 test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749048235.yaml create mode 100644 test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749048734.yaml create mode 100644 test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749051234.yaml create mode 100644 test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749051325.yaml create mode 100644 test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749051374.yaml create mode 100644 test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749051392.yaml create mode 100644 test/e2e/stress-scale-tests/config/orchestrator_config.yaml create mode 100644 test/e2e/stress-scale-tests/logs/cleanup_20250605_190904.log create mode 100644 test/e2e/stress-scale-tests/logs/cluster_setup.log create mode 100644 test/e2e/stress-scale-tests/logs/efs_tests.log create mode 100644 test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749045783.log create mode 100644 test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749048222.log create mode 100644 test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749048235.log create mode 100644 test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749048734.log create mode 100644 test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749051325.log create mode 100644 test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749051374.log create mode 100644 test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749051392.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250603_145210.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250603_154721.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250603_154805.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250603_174810.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250603_175820.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250603_204418.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250603_204433.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250603_204821.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250603_210011.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250604_204301.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250604_211255.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250604_211353.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_132802.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_135002.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_140408.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_141321.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_141434.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_143926.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_145133.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_145937.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_150619.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_151529.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_151604.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_151742.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_153750.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_154546.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_155948.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_160844.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_162134.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_162155.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_162320.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_162354.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_163114.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_163340.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_163501.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_164709.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_164934.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_170306.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_170342.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_170405.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_170502.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_170526.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_182216.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_182254.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_183835.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_191215.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_192824.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_192851.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_195225.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_200645.log create mode 100644 test/e2e/stress-scale-tests/reports/chaos/efs_chaos_20250603_135616.json create mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_145849_20250603_145849.json create mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_145849_20250603_145849_summary.txt create mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_160604_20250603_160604.json create mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_160604_20250603_160604_summary.txt create mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_180130_20250603_180130.json create mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_180130_20250603_180130_summary.txt create mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_204448_20250603_204448.json create mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_204448_20250603_204448_summary.txt create mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_205333_20250603_205333.json create mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_205333_20250603_205333_summary.txt create mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_210938_20250603_210938.json create mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_210938_20250603_210938_summary.txt create mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250604_204834_20250604_204834.json create mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250604_211347_20250604_211347.json create mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250604_211457_20250604_211457.json create mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_133650_20250605_133650.json create mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_135626_20250605_135626.json create mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_141131_20250605_141131.json create mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_142059_20250605_142059.json create mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_145750_20250605_145750.json create mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_150603_20250605_150603.json create mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_151246_20250605_151246.json create mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_154402_20250605_154402.json create mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_155202_20250605_155202.json create mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_160701_20250605_160701.json create mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_161509_20250605_161509.json create mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_163010_20250605_163010.json create mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_164017_20250605_164017.json create mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_165552_20250605_165552.json create mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_171216_20250605_171216.json create mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_182932_20250605_182932.json create mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_193407_20250605_193407.json create mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_200124_20250605_200124.json create mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_201716_20250605_201716.json create mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_145849.json create mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_160604.json create mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_180130.json create mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_204448.json create mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_205333.json create mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_210938.json create mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250604_204834.json create mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250604_211347.json create mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250604_211457.json create mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_133650.json create mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_135626.json create mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_141131.json create mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_142059.json create mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_145750.json create mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_150603.json create mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_151246.json create mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_154402.json create mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_155202.json create mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_160701.json create mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_161509.json create mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_163010.json create mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_164017.json create mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_165552.json create mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_171216.json create mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_182932.json create mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_193407.json create mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_200124.json create mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_201716.json create mode 100644 test/e2e/stress-scale-tests/reports/scalability/efs_tests_20250602_195427_20250602_195427.json create mode 100644 test/e2e/stress-scale-tests/reports/scalability/efs_tests_20250602_195427_20250602_195427_summary.txt create mode 100644 test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_183718_20250602_183718.json create mode 100644 test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_183718_20250602_183718_summary.txt create mode 100644 test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_201648_20250602_201648.json create mode 100644 test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_201648_20250602_201648_summary.txt create mode 100644 test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_210512_20250602_210512.json create mode 100644 test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_210512_20250602_210512_summary.txt create mode 100644 test/e2e/stress-scale-tests/reports/stress/efs_tests_20250603_135616_20250603_135616.json create mode 100644 test/e2e/stress-scale-tests/reports/stress/efs_tests_20250603_135616_20250603_135616_summary.txt create mode 100644 test/e2e/stress-scale-tests/requirements.txt create mode 100644 test/e2e/stress-scale-tests/run_tests.py create mode 100644 test/e2e/stress-scale-tests/tests/__init__.py create mode 100644 test/e2e/stress-scale-tests/tests/__pycache__/__init__.cpython-312.pyc create mode 100644 test/e2e/stress-scale-tests/tests/__pycache__/base_test.cpython-312.pyc create mode 100644 test/e2e/stress-scale-tests/tests/__pycache__/chaos_test.cpython-312.pyc create mode 100644 test/e2e/stress-scale-tests/tests/__pycache__/concurrent_api_test.cpython-312.pyc create mode 100644 test/e2e/stress-scale-tests/tests/__pycache__/io_tests.cpython-312.pyc create mode 100644 test/e2e/stress-scale-tests/tests/__pycache__/mount_tests.cpython-312.pyc create mode 100644 test/e2e/stress-scale-tests/tests/__pycache__/orchestrator.cpython-312.pyc create mode 100644 test/e2e/stress-scale-tests/tests/__pycache__/run_stress_tests.cpython-312.pyc create mode 100644 test/e2e/stress-scale-tests/tests/__pycache__/test_access_point_scaling.cpython-312.pyc create mode 100644 test/e2e/stress-scale-tests/tests/__pycache__/test_basic_operations.cpython-312-pytest-8.3.5.pyc create mode 100644 test/e2e/stress-scale-tests/tests/__pycache__/test_scalability.cpython-312.pyc create mode 100644 test/e2e/stress-scale-tests/tests/__pycache__/test_statefulset_scaling.cpython-312.pyc create mode 100644 test/e2e/stress-scale-tests/tests/__pycache__/test_stress_operations.cpython-312-pytest-8.3.5.pyc create mode 100644 test/e2e/stress-scale-tests/tests/__pycache__/test_stress_operations.cpython-312.pyc create mode 100644 test/e2e/stress-scale-tests/tests/orchestrator.py create mode 100644 test/e2e/stress-scale-tests/utils/__init__.py create mode 100644 test/e2e/stress-scale-tests/utils/__pycache__/__init__.cpython-312.pyc create mode 100644 test/e2e/stress-scale-tests/utils/__pycache__/metrics_collector.cpython-312.pyc create mode 100644 test/e2e/stress-scale-tests/utils/__pycache__/report_generator.cpython-312.pyc create mode 100644 test/e2e/stress-scale-tests/utils/metrics_collector.py create mode 100644 test/e2e/stress-scale-tests/utils/report_generator.py diff --git a/test/e2e/stress-scale-tests/README.md b/test/e2e/stress-scale-tests/README.md new file mode 100644 index 000000000..fb76aa278 --- /dev/null +++ b/test/e2e/stress-scale-tests/README.md @@ -0,0 +1,91 @@ +# AWS EFS CSI Driver Stress and Scalability Tests + +This framework provides comprehensive stress and scalability testing for the AWS EFS CSI Driver in Kubernetes environments. + +## Overview + +The test framework automatically generates load on the EFS CSI Driver by creating and managing PVCs and Pods according to configurable parameters. It tests various access patterns and scenarios to ensure reliability under stress. + +## Features + +- **Orchestrated Testing**: Random sequence of volume and pod operations with configurable weights +- **Scenario Testing**: Specialized test scenarios including: + - Many-to-one (multiple pods using a single PVC) + - One-to-one (one pod per PVC) + - Concurrent PVC operations +- **Shared Volume Testing**: Verifies read/write operations between pods sharing volumes +- **Comprehensive Reporting**: Detailed logs and metrics in JSON and summary formats +- **Configurable Parameters**: Adjust test duration, operation rates, resource limits, and more + +## Requirements + +- Python 3.8+ +- Kubernetes cluster with EFS CSI Driver installed +- `kubectl` configured to access the cluster + +## Quick Start + +1. Install dependencies: + ``` + pip install -r requirements.txt + ``` + +2. Configure the test parameters in `config/orchestrator_config.yaml` + +3. Run the tests: + ``` + python run_tests.py + ``` + +## Configuration Options + +The main configuration file is located at `config/orchestrator_config.yaml`. Key parameters include: + +- `test.duration`: Test duration in seconds (default: 3600) +- `test.namespace`: Kubernetes namespace for test resources (default: efs-stress-test) +- `test.operation_interval`: Time between operations in seconds (default: 3) +- `resource_limits`: Controls maximum PVCs and pods to create +- `operation_weights`: Adjust relative frequency of different operations + +## Running Tests + +Basic test with default parameters: +``` +python run_tests.py +``` + +Run with custom duration (e.g., 2 hours): +``` +python run_tests.py --duration 7200 +``` + +Run with custom operation rate (operations per second): +``` +python run_tests.py --rate 10 +``` + +## Cleanup + +To clean up resources created by tests: +``` +python cleanup_test_resources.py +``` + +## Reports + +Test reports are stored in: +- `reports/orchestrator/`: Orchestrator test reports (JSON) +- `reports/general/`: General test summary reports + +## Architecture + +- `tests/orchestrator.py`: Main test orchestration engine +- `utils/metrics_collector.py`: Collects performance metrics +- `utils/report_generator.py`: Generates test reports +- `run_tests.py`: CLI for running tests +- `cleanup_test_resources.py`: Utility for cleaning up test resources + +## Notes + +- The tests use `kubectl` subprocess calls for pod exec operations to avoid WebSocket protocol issues +- All tests run in the namespace specified in the config (default: `efs-stress-test`) diff --git a/test/e2e/stress-scale-tests/__pycache__/cluster_setup.cpython-312.pyc b/test/e2e/stress-scale-tests/__pycache__/cluster_setup.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8ebc0830c3ba48a034f969d81d6bf419ae60175b GIT binary patch literal 31301 zcmb__d2kz7dM7~KK!Ybpf~UY!1WAx0MP1Y_Qk1C68cVjvvOOG#4N{Omg3|ycvxgda zj;hc*N?F=XO7zAPb8-!*##PZDo2)jwRjWCz*(9}q2n867va7B5AOAvfxOeE-;i3$F?$(?8LTayaFSPiM_0(=8KkI%nd| zd`ZlF&dh#G&XurV%Q*}CwVt!GU)wnw`?a65vtP$K2m5uNbK=($bIrKVxy@9UHRhS| zob#AXC8mh|qF-rP`$cNKcBfqNHkQ9pxz~Pq`%lnUI#`Zo=~{n|lw1tx@1<*{)O5)Q zj-ObhG`Ux;Pv_>HO8#?|C8i@L-t|2b@BT@Nl=EB_@A;nToG((%bC zl}13`%=rlQi$`gc2*={HkpQ|HotcXZNhF_NV0B~MNu*C+z^@QSLw)X#=@p*}n)BAv z;p7GUI(fh{8J&rULmge=#3Wq?6M;~mqiYsmkS}AA%aNGweunxsji+KwEIys+m*3(< zf+j;Bl7g1JjTOqXOVi#6ft3Wn`micSy%EU z-!o%bTQH$*ycK6VZ^PNa+i`aC4xC-Q6K6Lqf^#0;jWfpI5y@}x2*8YiJK zTGF3(;(W`Llo5oS0pHhq2v}n3$ zFDyrQ7hO-M@s_AbDRB) zE2P1+LERkh)#X-4zd`O&zARa-w4bb1t}fOnSK4ney8I*44^-#_LY$KGOT342-FvVu zS`FVr1z)Kfk5_HQICmBA)8#IGPVVZ0cU<qzEWZ~y=pS!?}qvM*Wa|fYPw<$){0xlXQRnzI2ILwFB!R+3kyI_ zAU}z~ln|c@OvY!YqSNz0aVBb*R&U~qi8Ya;D3voVgy%34b^w^OosIGG8j!BM)1k7bwts5$JUY1n;~hI-AT} z3C-s@1rgH-@*ebFfr&s9U#O!K@jUq|C8yNd#2ko6zFZ-R6Y|rbEpJUkVpFJ!b&8EA zZ=OQGBq~24ch)OGN4`xEE(y@_WU4qIg#sm-~j?rc+Ex~XsV+Fk2f(_u=iM_cV(bQQ?8cV*jmq}z9-0y~zh z51dV(_W)pNZ`+fE2QdEqhB3S4S9aT2xnokLg-9oQL5u6tB~VX2oxG^0rpQ1CG4vl; z6fDQ+o&n7iT7ZdE$|9Advx3E0Ih8sXJ)iAJs+jDeig&e~yj7vftS%LMYB>tpQt_qs zT&O@FtF(LNN-H;e3ORgAy7mh$RO_2cwY2BJ>oR=JrX;Ovc4#8Xcg(u-Mq_grx65dr zS~K!AD0OSUTJQ0^NlRC*wCB(1gDUDU_!s|$@t7%3qCKw`eSkYFkFp+XfF71ENo1G3W*A#aV&PQ~+eAgj@- z1?4G~gTO?@pAjv!UeE{~G$dMPiwHto$h%(-$L1r4*>#OVl_@`wL795{b7O4f>;q@_BTre@ z(|q64oa4%~TxXi=%y3=ox;4$UKH%Cv;ws-c{`T>d|44>A${uv5x$X?t%SyJ~_q3=b zA8=ids%o=U9rvp`9+ubK;%;)Orn8yymmbyxQZ4*iO(azwc_`OczwI`k-F`H^{U``i z%6}}w9e?EHvd*@&vn}K7$kBC6+S!tEwteI*r5>kzyED!`j~WBn#!$L3l&h(~b?N4% zTx(~xbyvD|SFWig+tid!T7$~N?*8+tzR*vsAPCcDf1DaLLp!1u&mGkm|u@I7z7 zYSv^UBb*O;sQ~BmkWR36tyFX2fmBuc(4HVwl^e8F-u|8iGL(adJn~)%@18dEp7%;{ z=5Xgl85PHBWss_J`dVsI#r}M$aSpyzL~RPy+8|Gvh${JV9y~u^g+BQBil~{d=4<%c zX%p}N6Ek1O*Q3w1d;@w}$2am#d~?){Je%ILA~nELTliKjb=tzWy;p)9?eDqpwBtP+ zq)oNs7Y(SA5k*=wqsIo&{v#9LsYyXsEt)K~XyG?2qlJWP)yY-{%C^OlMXQn<>bWlZ z)9#b1BF>1O@WDk(5nZxC+ECxfqUDUKSl+naElV5SV%rGAqJhXIXhUKuzLxN_EMkZX zM;nro&k9=rLa|aR(-)2RUmgg@<}QS{<-IZj7-1}N&>|4kkuW69da+vJV^LF5h{vEJ z2_?c$JS3=uB)wFygcX830zN%47oLoWd&9!)Fu361Xn1CrUChN}(aDAWL`1k8os5J~ zQ>6dkOjwN1LOB7w-6&H{oMyR#B|yZHWEi@iaK1d8Ook^fNKX#Fcq&0pNrCmk_~=N! zOs??YBolcOqIGsUH1IJYx{rw$`}j0YdB=2QHX?*$5G;iG2r95r`=X1+7EL7PBVs9u zP18brevWl5&*@tg-Gf^O215f|LIXSUwu7-){7TR&c1D11lhH{XAsUU4Zt;8sy#diV z9fvHF5IZ)`HwH=Y4Abo;o>kpRlIG5mc!%nmB9nu|YD1U9(OCF=G!{)RoP(MzkvDfk zw`D&(H3fQ;w;yCgC+~bwR?>-WPj^O|Pr5GW-Di>ln1U10OA)b?-r|dqZ_Gn}IUAnt z*926WMw60s(%+r8#jngp_@GsI5das?(CI9J-i3+9Lp+A)YS8v*gt!!a+nm();cv`? zaS%64txFolXEcvj>UddFq2^s@lF%8>L}rsgmozasMn7jKBWEEi=gas=A}U1qkuatz z;FfpHL}$-Hc*jNfO?KhF9FNV~83>jOgH(Jgor*XVon(yO<*iJ#wO+*3 z5{Q=-NVA`}^7Auu!gjjfL8lI!!0Zss&$}fQFafSc=%xfk`lXtLAW}2|m$3%I5Yoif z0>(gQH00MU^Fd)M`&U?MZeEFJahZd23i=5VEwwKtFh*jrX3ZP>y}*&rJsaOo9o?6rJqfem}={8P2Zfp6}uT*8NS_~9UMsy zj-&=g?;cL|A6u(AzT|o6tpM2%x3=@wi>$)BjXuA0GDSx7wOt(P#-g|VbozlT*+a4L{0 zpPkc=njKVgA!SagvgSoqH>owD8Mn-)*)vI1r+87-9BO${zl-WuMhvb->yd%n&$|~( zE~=VdtsdUPa|)*cZpOPou2Oxj9~4Ud23iEZj4z*7wf3ZA|59+SP(}smT^p6r8?mG2 ze;u5Ix)zM!R*4oEq%Rui8I+k>^p>?g8`W5aS#H1s0{T8YQF%S>9pOp!5A+(1XV+h& zK@asAznxm%&)4Cp)`)t=*MqxuD0N(gUS9w1ZkPq94q`^{q){pR+_%?+x{cnRHa?^J zo4;`Vo4#<)z!%Qh@`ZD@YB~9~4c~eD4=h)0NmWm@XyZGS^$5AazG%O=Nqsfy1>XtC zu~~_^u~`R2-O7AQig-3ugNwGJS^R}@m2N&5&G&8%4jatJ9=?|k@qNZH;`@e$wXo!STm?!NUwG971;2>?13AE zg(lJ0*9Z340_>;0Z z2P}Am?+cucMZyVK$}dDFFOfP+kQ&#Ue)^s`m_}zfkhdq2s9q4p(Ez)N&nLwyNykOa zg(ZOoIUQh+NdU+@=4UU>g3T5=$$$?NyFfL)gx_(PS|sUCU}D<=Sv;JLCj(O`4ZA&z z_PDtM`vd_7Dy+GP5L;0DE$f(Mxw20zmutXS$uj{;wrU1}LKI zGhx{Gp+^f0Yzqx;34ptvCl?1;sujH)@(M&x1)8!?N=j#)ibkF$Pms zM3Q!OEsA9fh5;TDVlNcwtdY zgTVN~lK~8U27({Pl@Nyl0eD@Ukp$DgH|FEXZ~|TpkU62$?0WijCLlTnMVN!&WN|8z zB!H53lqvWwL!N+A6B1=Y3q*kX6MHB`()|->C?pvogB485V8VhX zdE~hXVSX0!!ORT&4uoM=CA0*2D-_JI>s?I5XW@AP%V0tvGzm2%M9n#!&_hB^-gZTZ z!g{(B*FlFQ#<;L%C)mr#+t^CYbJ7Hr6JZ}s%*T>>7aKi{kJFNDLkNy+cQ_>JW*ZX3 z<^t*FDRu78ktgx@So@G{%Br$uo9~xxRuz}vwlzlEH<)p5&H4OUUq{;4aliBUn(xGt z>ygKIpQ)uF5dB@7G)vu=8wx?Qmr0RBNxLpsqvUiTY zee}jxe*gGKKL4#_H;*kZtok9ttogPs9aiZ=E_*7&j6@PQ>wT09T;saqW-A?R3o@YucMsaIc5ac_KTDXDZX*&)1fl{b3Ro=qzQYo2y> z1UoX^PGto9SGV8pO84web?-^l?agre)LcUuZku|uC&TSkZ(hi7r_~#2)bgtv&2Wd& z!G_&;hO)a)q<5c44WGPsHsybDY5ZYD^^NFCRi*;q%#~N%a@=$*2Uo^ccc$BiQf=FA zzj1f_y{`1gnbaZVe<@S`GDNk?PEwT9q-fZAr|ym?RYwhcq)dV9#}xqjwr89>(6Gy! zb=IYwbvdr$#gE%}mcJXeowO86d}QTj5?=r^8I$;A*$nCG0eQ?^B#)IadCbDx_xh4V zFWGUC>MJ_*7RYDkgtTPSOH1Gg+`1A*T#M??2rgFZg;7oiYIhpQdMf8_q(_GbkSeP# zxPu;XNMbVJ$p#J170`Y7y~$*N=oUH89ao(>6r~TC4+q1l$(W4)Nbi^s<^U;)*Jzh9 zlgwiPNrynUk%u?}C>5jUW07GYf&AjgNPIrVGpf!Qk_Dv|sX$-gEIjohpuf`>lH`(S zK&1MFZ&1quorn`T6|@WAq8m+@L^xcyMyI!Mg06Qne)1mHCkgi^2;2&KC7dmG7NA$D za|XsRxYL*taTEZWt*Mu$^Zl-owVKf-PtH?wV=?V%Up3$N1Zhz+v(ml{w_m9(Dz)?KN(;S9H1hrxscXtTh;L&|qH`IljKfk7X) z(qW|D(5o$;lM}8+8<-)W@;5U4fTh=o1jwUKN~K4n{NQao9N3KO=AFhh^DZ8aX*@UW zQ6@4|r+C@&EoJJIZrx~%1i3P$B$n8-g^hA8Cl*@;wQ*5Zk7y%N82$nikAXsouUsrC z6m~X*3>?bH0A&mtssm;x5~WK3i+UX_NagWmV9}tB@QcGDS*48K(711rAf?pIMEOR7 z-BmIk8qb6(GO0lb-NXlYGNiTf?b@g|QmbJGb?}}1W{8|YzMF?Q_Y8P#G@l55UbU|f ze7_X@LQ2UmGnbwNKYJ1U9E%RVkHOEmh*3Z_qO)+eJtzMb-B_qOb{L7k z{D78|AH3?~ORu^Y-HWbEz@WmV4c~DGzm*?agwdDZ##XROimrMVJzrpTanI@h_C>C6 zyo(&agAsJ|Y+%vNmy1~KSKXRp&hy5mj>&xe61#pK<9i-<*{Q+ii?65+VOGWO;)nU& z65ash_OSJ?LC*V}x#TUHOQnmYqoy~80Ee>YKy&Y6SrIfhTDu+mKK5SsFP8BK7H!jN z@64v_+pW)|LI-v5diJb4q~&~S*8O-x2zKxztar4|HT7cobKnMN0P2j=B0tL3Uj={o zYUN@DAPBMlYSm&DKSt@8Wl-ZM%-1>o$kj?H(al!VVx`db90(uPsozx9GZbs%Kd>n6 zqOLuc`)s)CR zuPWsNqK9ghhT@@7)U1p74o4-|AP)&&j%#Gp!?ui~^oPoM#_z$7lpzFMI~ zWO5$bgM|>Y=8IfNx?t)Hkt2&gbAe7wheU5EB#mT_*{6dPCZQA$u;L6H6jda7Tm`5o z;3%T&7z0#>!nbMAo3#E)s+FQ@@(yMX9_K|0*0>NCU~MkyXb9DkW3cE}It-sTTthdJ zj8Dd6c}sF~PJ|kUVrcT#!7W3(=x!2S5M5jP*kAt^fMO~kR!W@+(F~D&G{j1jj0h~) z1%~j@3`B~2sq7YUdR~}==1r^!g{=7LT?mI&OFYGg^pGniL?|5J#lI)c{RN=-1g*1SO+MvYL1l=|a zOmci;5^$m1D}0xp(qb3BLnpFt2(36lgBgoN=JJ*p!XF%0;P-^8^G%WlRd$q^Aa^C5 zqi4{^glJ)j%G>eg;opj&iRlShhv~p2^L?`60*~BSW6CopUDHF!Ths%m5Ilks6}~?n z1{*X=)&ehr?=W|ja1eK0nm1}uhkT#+!ca1yzNoxiPNx|` z`qkn?(zouyQ)6!u8i}^%y`&8IbJ)DzGO60HEg9#4rfaPk&^tvOPn|xSJ^fnx^lSI7 zrhIQ?oL|dTRLlB9NlVsiq$SI7)v4NDq%^ayd+hCS_Ow@epi(T?REqa?D#iW`w?)+) zTtEJ(qITJtu4u_tY))5fUaJT$*<~f>jhU6Pm7OVHZ^jvd!mheb(g@26yMY{6zZ_c~ z%J%F@_v}e^?@cxC%W(~78VP(y&?wj4qudckA zZrZX|Ik4o&mDeu2(&ep7wx`vGdvctAxkEnrZcGt;E&&RpHnc-+SZRV5rn)m--6?mw zFUJK^t%Esk(@H%ur~E^=&t|ucrMHcxhK{6aSm&AUQ5t~P(S-pxYCKDB)Wy}@h^0LU zO1bapQnjkZ1E{LKHGFe;`65)^&4X)ITbG(EP zye`!+oGY(i9$RG(tVA-+fd18KE<2JcPQf=c{Yg2otnaI)-3?8wYhBR)MuQjw&B*r zn-^D_VP0t2wN^8HXJ5MJ*wPVzps8i)UWel`DU`XQeN-gzJ3Y>9Fk_8cTxF*{;NU%Mg2GMPKS53 z8f}ao7sU7rjB3oo+{%zsoM7zyMICJ}v*b{3T{1$^Y) zTDWPRqYg+e#pAqWFYc1M4aZZ~=Z!i$90)okwuZ4pe@?Bb9;O2My9hL6n1d+Lfc@2)#vDF}FE;f}*(9rWG!Cd{w7ZM(ADj-Ysgxb=DJoJ)7)CoDZC5Ki`BLDetZ)pZn#D1o@@?Wz9&1VLMUCUyn@yy<{yTDrj5P;pck z@n_(zsX=bRh4{PZbMYCm z=_ScG2xx`1Ah{+!!K;)50p z_cw$)K0t0we1H?QFFSBBJ#a9!<ly5BKJR*g#r9I6no#0CuOC-mx<-8SHZy@ar ztc>0Fb{juAdfy8^46$#4G#7ZlwJN4ZnDKqtT_@7JPNa68Ox2yra4+ciJ5`=x?${Ko z)|Ga4A)fAOE)2!z&qihk z{P*e%KIDsC%9OZevhmQWbkNY(TW1n{Huu7-DqXBUtxf~HfSo4faf4f`3i`D5MXRZ3 z2*3*xT4nMy9#gQ15WXT~5c1!p1qP?oJ}8ch@r)66xhxL#^O3n& zd_kAHKMsN#3oj(Fw@4W7LH&_Qp`^g>pbEt= zffy)1AqIvMG$$n^#9)=2l$^wmOkm`arj=vekTstPqAaAblth)EQjed3GN`7H2xa_N zWMn9#=5=%G^{;1NkELIS3+Bwa3D(Q=CHV18mMl4eQ&ew8Ec~(Mqbq!>_aLIe2qly- zl%Vrs)S-ligLk{`)~D*8))6VLoR3W<-BgL+v&50G<2P{TlpLjGmr|WSp)p<|oWk%* zA1=O(`xCa`vxtYDyDlU~Fd(s`hWkp4NS^OY{2slw}bE4vW z$*|t3xs*;Js#!(Z8Xp*-mKfsCo85T_Hk_f(h@H@MDLh_LW@g@#J1yGBSTG90C*+ik zN`XmNhbPuS&XXurqEFU+^5#AV4u=#{JOmQ_=r=8CTCRVy99Luo)nhAImF>h5!j?YQzEck0GS-6z{#VY zAShxxp6F~O?;;_VMK%-%V*i|m(p8AEHkO?&z(olwCL*+FN{>Z{A5q@rjVtDKdCN@>Muh4;5`7xD?`hnqepxuaF}3{q@+pcS@Ey-M zPoP!Bga)3R#$aJ%a2Je0RSq71fS?1#NqEl#Ztv4LTh-6s1LodPKi{p>E)$k7ItpBW zReXHWXs;GEniiN@*4zWvdx|Q=El;k%PbU{eA|sE1ru?E2hOz2)X@rz+mSd`$n;Ay4P48s<|7*52GMd;_L=%ns-^_1XEYbC! z+)b7KcL595KWAnKSX(Xnwic>&We5~S2j9xdv=x?l$}00WvQ?&C?3Y}K80nO016?@?QB~+D9i}CGBuBaUN72Lz-azU=>p0 zAt~)7T*SmmxwG)}14tz~aSDx}lBo0FQ}>w2r!o>^vnKZ$fmke)2P?m&&0@`F`CSwR zB(eZOPVN9}U-UC(iT2KxlhsO`o25PM67Z5L4vQ(un>TL`dUd8Q0|`=Srh!ohbT%+T z(h>{6s5>VKnXbg7Y^%#Va7pSjIN3~M4^m|hSZOx=Pb&St==4uGvApUIERuZcm1(== z1(YCkPXelPNP_BKq>3#CJb?t|33-%~QLvMkf&U>f0~Eoin9^7vVv%U6sHbZV-1TJ- zo=G1(lR9uV<$EdPe3_BqTgPu6zxBe+7gk=l{oEd#Gqhq~bxJ-b{+9Q> z?|N5a*jm)TW66tB!BDnqINde8KBET}vmI`_fzxCF|SXZ+f>WwRy){-Oi=b-+1bBq1|{2V_r9& zZt8k}=G_^{)@#j&QqG3oU~{^G(WR3Jl1p=acwV)6$wB+3Y+l})@giW5d^Bj^l$n*U zWVp>_zzcv$QH1rC@zpDLtaqkUZDVT{N0#i5JQZ@Fp-x1v+m+#lbu5y?!o*jmS6)r^ zji!8uGtM!zt$B68;bM7qbsF+Nxo?z^`v$Q_b<2%uPvz5Pj!EB&J@qw)*dho=+e26E_&oPxfBN9rF>Rq(ej4H@#Jy47k#yum`h0aHv zl$mZuW&k06J5M#^e}nM8j7p<20L|3GJGVBFywXd(Nn9jo%l9a=-qq zFPeq#BPSV7iVoo#((8*7aSNwpHPH~>^~Uv~WsAP$4I$7VZB}Sp3)mvl#YVLQ>V-B3 zpKCe}7}s13ndgjBw`Ivfb(<;aQ@YZQjl{0nb)}6MJl~?dP>KI4Slg@iZ`rZEd8u@D z#eBu2IK~vz-um3y>o%(W`tSbLb+>F-_q)Y)2iwH)5p2ydOP{n*1b+I01kd)KWBLc3 z*A8h|B#sJpdg(hA`n@L_&cNxF`K7~AJUojk!In{6F3jelpKWJH93rVgbd^e0d)i<@ z_(MAVF`fR1PVdm^+jLSHD5i-K-bY&At8=G@|96dAzuw~+Ufs0TSB})9w?2D;MBYB*BXYr*9m-@u$Q}xqTY35^qpe1Ol;2 zt^mFcw6}Q_KK8uWV%UHNIofqZ9klK}1cWODAG_2r+eSnegTm=^ip_>?OiC^VZ1<4D zHwix}Y4c=0Sf-sKDiFRIT>M0xIk>8w6#TmuF7#z zwzDXedJbldeWphE50v4r==2|P%DW`jYI6QIM-n9Z$y^{~0_np6Z1Rb|LPk(ZjS%nm zb)0m1?17APkhwlIt$CVO+Ow@Y*IL4Uy~caooNMWj zc2&>$>Xx@*JBXaGbJZ&Gb~#`3iXHBqLzU1lwN~Ar&HQ|=E4}He{@VlhtA-vnx4wVz z-HWTu*d-!J>{+&XEZsbI_gkq~UR`TGcjL%EwFcq)+}^#qKi$52+4a!hO#2w5I``ap z_0I8B`_YVlJg4p+*x0hNE!Dk0*Vuo%Zrx_-ssCq_rLF;9p>5rtRhp_Bh)YccPGo#1 zAIke@2!C*D-RD zhxLs&E?-!6<$R5-7VKY70JQ$d^F!LYfjVJDC)`RW3cSVlo=!D>B~$()`Y7-4=j(sq z-1586wwZiQk4<)bX;Ve^y3aZJpg`VMVBGGO|J z=b+_i!1OEbVA*)7>B9ln(I(4>!~Uaw%RRph*Y^UhagXI*&_8as{Mv5A^{-1^FyTINtMYpvSvT~DR23} z`V*ObCcCY z7cGk$^~(7kb|?QR{+O_Ify3I3=!)e>&L28gtNzrxd@bec&p5Yy zRN0uVY)x0T=D4zlRZX|{-`p?%V4u^^T(&Crk^>T3z4UrAzi3kXtLW zo~AT#J@)u%RrKUdhwkpkj=Yo}c`0@1L?E zN^DJ!FIsDCFPR^2DYNxFZmqSAnI8{I=Mvj%=Eq!%ZQtWD^IqFA^QYf3@30MgDwwNn z5*S#6lW0plRFw7C7KXBbxTM3-rOy0;Y9hEwQm;+Pt^G>Qc^}xd@9b@mk8KJ6f&01n zJdl*RFObiRe5FPCR0XuwKLw4^P9jE#RCwjtpUg-i8eBSzHg?zKR9!6wTdP7AZvh(J z0M&lcHmArI>|gf|;~ROU$ua7l^!uY%Dw=6qx-OdHwJr?=)Pd>z($ncWA$f}H@L{K6*o zV*6;Eg}Jh4g|aPuV@%or5f$3Q(uYUvfIUULOG@TYd^WZqcD#VIwA0jr!6#rK3KMCi zn0`KTetuf?9iAlzGe~l{qa6mPLBv4~U<)#dJu;~)+#ORDrV2M31o4Bv75l1{CT|sF z0WJ4Qk{geBeu$my<0MFJJD9}w?vSvMrU>ETDG+L9&?JhNjmSbB_>VYt$vh?bs#WKm zh>3=39{Ws|9u%gr$@^^bG~Ei=@}K7?@RX&opqfO4?h|<%8!zJ?@=kQ)1buEY?~uoz zw~Zb?^wJT5@FSxM%y^LZ$l81Yv_#`}PzFZ))^UZM3Kc<-wv2})$V57c!jVZL=*Zhy zPw`b5)(waJiAt#s3LX|drHZ_A(Fr!zr=>;f3?{VT$NT>);yL~d7my6S)mbmTbk&|3 zcp>9G{npqct2=A0OCgsG zE7;Svf~Oof6==a5*F4Q{je_@Za=#~jN2H+7vbW3LbC6*mj~Xwq#@SZo_>~1EN{8Tu++o z$#5Y>V)Gx)aAUbD9~!Kzz148D;omi>vAn68q1!LtKAQ6F%{cev%F2KDk;nJg4JjYmZAkghW1;IR z8%hzMM)P1|B3LV!F$JVg4qT&I)Jmss(}{GNLK!{%9bLI_1x^?EDi%F2fT=MFB-rFV z=fjDpyqQPdI(};Gg}i4B%Acdqo?$E9ycZvFh=gb4TgD>iUD%4?1p7#m#D)_xXOmL+ zKXmmLooGWijjhW&_elKYJ|Tby_-9J|HBOLd&E`+6ZnO1MzscXznks&4 zD*H#1{~t|Fc)4cZTaK)$Ds8IDxhvl3eY-d3Irzxgu;j=(8`92(RO8T^b6d{YLj>m`f0h?QU4A&ARH-uKMNuYp$NR?4Mg-aGTwqnCSfYyl;=W nVtvtUvT{q}eM?==%Dpx6dq=;0^zF0XfBAvchY4z}XXE<6YR`E% literal 0 HcmV?d00001 diff --git a/test/e2e/stress-scale-tests/cleanup_test_resources.py b/test/e2e/stress-scale-tests/cleanup_test_resources.py new file mode 100755 index 000000000..68d6743d7 --- /dev/null +++ b/test/e2e/stress-scale-tests/cleanup_test_resources.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 + +import argparse +import logging +import os +import time +from datetime import datetime +from kubernetes import client, config +from kubernetes.client.rest import ApiException + +""" +EFS CSI Driver Test Cleanup Script +This script deletes all test-related resources to ensure a clean environment +""" + +# Configure logging +os.makedirs('logs', exist_ok=True) +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s', + handlers=[ + logging.FileHandler(f'logs/cleanup_{datetime.now().strftime("%Y%m%d_%H%M%S")}.log'), + logging.StreamHandler() + ] +) +logger = logging.getLogger(__name__) + +def delete_resource(api_instance, resource_type, name, namespace, force=False): + """Delete a specific Kubernetes resource with proper error handling""" + try: + logger.info(f"Deleting {resource_type}/{name} in namespace {namespace}") + + # Set deletion options + body = client.V1DeleteOptions( + grace_period_seconds=0 if force else None, + propagation_policy="Background" if force else "Foreground" + ) + + if resource_type == "pod": + api_instance.delete_namespaced_pod(name=name, namespace=namespace, body=body) + elif resource_type == "pvc": + api_instance.delete_namespaced_persistent_volume_claim(name=name, namespace=namespace, body=body) + + return True + except ApiException as e: + if e.status == 404: + logger.warning(f"{resource_type}/{name} already deleted or not found") + return True + else: + logger.error(f"Failed to delete {resource_type}/{name}: {e}") + return False + +def cleanup_test_resources(namespaces=None, pod_prefixes=None, pvc_prefixes=None, force=True, wait=True): + """Clean up all test-related resources""" + # Load kube config + try: + config.load_kube_config() + except Exception as e: + logger.error(f"Failed to load kubeconfig: {e}") + return False + + core_v1 = client.CoreV1Api() + + # Default prefixes if none provided + if not pod_prefixes: + pod_prefixes = ["test-pod-", "efs-scale-test-", "efs-app", "efs-sanity-pod"] + + if not pvc_prefixes: + pvc_prefixes = ["test-pvc-", "concurrent-pvc-", "many2one-", "one2one-", + "chaos-pvc-", "chaos-ap-", "efs-volume-", "scale-test-pvc"] + + # Get list of namespaces to clean up + try: + if not namespaces: + namespaces_list = core_v1.list_namespace() + namespaces = [ns.metadata.name for ns in namespaces_list.items] + # Filter only to likely test namespaces to avoid touching system namespaces + namespaces = [ns for ns in namespaces if ns in + ["default", "efs-stress-test", "efs-test", "test"]] + + logger.info(f"Cleaning up resources in namespaces: {namespaces}") + except ApiException as e: + logger.error(f"Failed to list namespaces: {e}") + return False + + # Track deleted resources and failures + deleted_resources = { + "pods": [], + "pvcs": [] + } + failed_deletions = { + "pods": [], + "pvcs": [] + } + + # Delete pods that match the prefixes in each namespace + for namespace in namespaces: + try: + # Get all pods in the namespace + pods = core_v1.list_namespaced_pod(namespace=namespace) + + # Filter pods that match the prefixes + for pod in pods.items: + pod_name = pod.metadata.name + if any(pod_name.startswith(prefix) for prefix in pod_prefixes): + success = delete_resource(core_v1, "pod", pod_name, namespace, force) + if success: + deleted_resources["pods"].append(f"{namespace}/{pod_name}") + else: + failed_deletions["pods"].append(f"{namespace}/{pod_name}") + except ApiException as e: + logger.error(f"Failed to list pods in namespace {namespace}: {e}") + + # Wait briefly for pods to start terminating + if wait: + logger.info("Waiting 5 seconds for pods to start terminating before deleting PVCs...") + time.sleep(5) + + # Delete PVCs that match the prefixes in each namespace + for namespace in namespaces: + try: + # Get all PVCs in the namespace + pvcs = core_v1.list_namespaced_persistent_volume_claim(namespace=namespace) + + # Filter PVCs that match the prefixes + for pvc in pvcs.items: + pvc_name = pvc.metadata.name + if any(pvc_name.startswith(prefix) for prefix in pvc_prefixes): + success = delete_resource(core_v1, "pvc", pvc_name, namespace, force) + if success: + deleted_resources["pvcs"].append(f"{namespace}/{pvc_name}") + else: + failed_deletions["pvcs"].append(f"{namespace}/{pvc_name}") + except ApiException as e: + logger.error(f"Failed to list PVCs in namespace {namespace}: {e}") + + # Print summary + logger.info("Cleanup Summary:") + logger.info(f"Deleted {len(deleted_resources['pods'])} pods and {len(deleted_resources['pvcs'])} PVCs") + + if failed_deletions["pods"] or failed_deletions["pvcs"]: + logger.warning("Failed deletions:") + for pod in failed_deletions["pods"]: + logger.warning(f" - Pod: {pod}") + for pvc in failed_deletions["pvcs"]: + logger.warning(f" - PVC: {pvc}") + return False + else: + logger.info("All resources deleted successfully") + return True + +def verify_resources_deleted(namespaces=None, pod_prefixes=None, pvc_prefixes=None, timeout=60): + """Verify that resources have been completely deleted""" + if not namespaces: + namespaces = ["default", "efs-stress-test", "efs-test", "test"] + + if not pod_prefixes: + pod_prefixes = ["test-pod-", "efs-scale-test-", "efs-app", "efs-sanity-pod"] + + if not pvc_prefixes: + pvc_prefixes = ["test-pvc-", "concurrent-pvc-", "many2one-", "one2one-", + "chaos-pvc-", "chaos-ap-", "efs-volume-", "scale-test-pvc"] + + logger.info(f"Verifying resource deletion for up to {timeout} seconds...") + + start_time = time.time() + core_v1 = client.CoreV1Api() + + while time.time() - start_time < timeout: + remaining_resources = [] + + # Check each namespace for remaining resources + for namespace in namespaces: + try: + # Check for remaining pods + pods = core_v1.list_namespaced_pod(namespace=namespace) + for pod in pods.items: + if any(pod.metadata.name.startswith(prefix) for prefix in pod_prefixes): + remaining_resources.append(f"pod/{namespace}/{pod.metadata.name}") + + # Check for remaining PVCs + pvcs = core_v1.list_namespaced_persistent_volume_claim(namespace=namespace) + for pvc in pvcs.items: + if any(pvc.metadata.name.startswith(prefix) for prefix in pvc_prefixes): + remaining_resources.append(f"pvc/{namespace}/{pvc.metadata.name}") + + except ApiException as e: + logger.error(f"Error checking namespace {namespace}: {e}") + + if not remaining_resources: + logger.info(f"All resources deleted successfully after {time.time() - start_time:.1f} seconds") + return True + + logger.info(f"Still waiting on {len(remaining_resources)} resources to be deleted...") + time.sleep(5) + + # If we get here, we timed out waiting for deletion + logger.error(f"Timed out waiting for resource deletion. Remaining resources:") + for resource in remaining_resources: + logger.error(f" - {resource}") + + return False + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Clean up EFS CSI Driver test resources") + parser.add_argument("--namespaces", "-n", type=str, nargs="+", + help="Namespaces to clean up (default: default, efs-stress-test)") + parser.add_argument("--force", "-f", action="store_true", default=True, + help="Force deletion with grace period 0 (default: True)") + parser.add_argument("--verify", "-v", action="store_true", default=True, + help="Verify that all resources are deleted (default: True)") + parser.add_argument("--verify-timeout", "-t", type=int, default=60, + help="Timeout in seconds for verification (default: 60)") + + args = parser.parse_args() + + # Start the cleanup process + logger.info("Starting EFS CSI Driver test resource cleanup") + success = cleanup_test_resources( + namespaces=args.namespaces, + force=args.force + ) + + # Verify deletion if requested + if args.verify and success: + verify_resources_deleted( + namespaces=args.namespaces, + timeout=args.verify_timeout + ) + + logger.info("Cleanup process completed") diff --git a/test/e2e/stress-scale-tests/cluster-config-code/cluster_setup.py b/test/e2e/stress-scale-tests/cluster-config-code/cluster_setup.py new file mode 100644 index 000000000..b41437803 --- /dev/null +++ b/test/e2e/stress-scale-tests/cluster-config-code/cluster_setup.py @@ -0,0 +1,756 @@ +#!/usr/bin/env python3 +""" +EFS CSI Driver Cluster Setup Script + +This script automates the provisioning of an EKS cluster and installation of the EFS CSI Driver: +1. Creates an EKS cluster with the specified Kubernetes version +2. Creates an EFS filesystem in the same VPC (or uses an existing one) +3. Installs the EFS CSI Driver with the specified version +4. Sets up the storage class and other necessary resources + +This can be used standalone or imported by the test orchestrator. +""" + +import os +import sys +import time +import json +import yaml +import logging +import subprocess +import argparse +from pathlib import Path +from datetime import datetime + +# Set up logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + handlers=[ + logging.FileHandler("logs/cluster_setup.log"), + logging.StreamHandler(sys.stdout) + ] +) +logger = logging.getLogger("cluster_setup") + +class ClusterSetup: + """Class to automate EKS cluster creation and EFS CSI driver installation""" + + def __init__(self, config_file='config/orchestrator_config.yaml'): + """Initialize setup parameters from configuration file""" + # Load configuration + with open(config_file, 'r') as f: + self.config = yaml.safe_load(f) + + # Extract cluster configuration + cluster_config = self.config.get('cluster', {}) + self.create_cluster = cluster_config.get('create', True) + self.kubernetes_version = cluster_config.get('kubernetes_version', '1.26') + self.region = cluster_config.get('region', 'us-west-2') + self.node_type = cluster_config.get('node_type', 't3.large') + self.node_count = cluster_config.get('node_count', 3) + self.cluster_name = cluster_config.get('name', f"efs-csi-test-{int(time.time())}") + + # Extract driver configuration + driver_config = self.config.get('driver', {}) + self.csi_version = driver_config.get('version', '2.4.8') + self.install_method = driver_config.get('install_method', 'helm') + self.create_filesystem = driver_config.get('create_filesystem', True) + self.filesystem_id = driver_config.get('filesystem_id', None) + + # Track resources + self.cluster_config_path = None + self.existing_cluster = not self.create_cluster + + def setup(self): + """Run the complete setup process""" + # Create directories for logs and configs + os.makedirs("logs", exist_ok=True) + + if self.create_cluster: + self.create_cluster_config() + self.provision_cluster() + else: + logger.info("Using existing cluster as specified in config") + self.verify_cluster_connection() + + if self.create_filesystem: + self.create_efs_filesystem() + else: + if not self.filesystem_id: + logger.error("No filesystem ID provided but create_filesystem=false") + raise ValueError("When create_filesystem is false, you must specify a filesystem_id") + logger.info(f"Using existing filesystem: {self.filesystem_id}") + + self.install_efs_csi_driver() + self.create_storage_class() + + logger.info("Cluster setup completed successfully!") + return { + "cluster_name": self.cluster_name, + "region": self.region, + "kubernetes_version": self.kubernetes_version, + "filesystem_id": self.filesystem_id, + "csi_version": self.csi_version + } + + def create_cluster_config(self): + """Create eksctl cluster configuration""" + config_dir = "cluster-config" + os.makedirs(config_dir, exist_ok=True) + + # Generate cluster configuration for eksctl + cluster_config = { + "apiVersion": "eksctl.io/v1alpha5", + "kind": "ClusterConfig", + "metadata": { + "name": self.cluster_name, + "region": self.region, + "version": self.kubernetes_version + }, + "iam": { + "withOIDC": True, + "serviceAccounts": [ + { + "metadata": { + "name": "efs-csi-controller-sa", + "namespace": "kube-system" + }, + "attachPolicyARNs": [ + "arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy" + ] + } + ] + }, + "nodeGroups": [ + { + "name": "ng-1", + "instanceType": self.node_type, + "desiredCapacity": self.node_count, + "minSize": max(1, self.node_count - 1), + "maxSize": self.node_count + 2, + "volumeSize": 80, + "labels": { + "role": "general", + "efs-issue": "true" + }, + "tags": { + "nodegroup-role": "general" + }, + "iam": { + "attachPolicy": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "elasticfilesystem:DescribeAccessPoints", + "elasticfilesystem:DescribeFileSystems", + "elasticfilesystem:DescribeMountTargets", + "ec2:DescribeAvailabilityZones" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "elasticfilesystem:CreateAccessPoint" + ], + "Resource": "*", + "Condition": { + "StringLike": { + "aws:RequestTag/kubernetes.io/cluster/*": "owned" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "elasticfilesystem:DeleteAccessPoint" + ], + "Resource": "*", + "Condition": { + "StringEquals": { + "aws:ResourceTag/kubernetes.io/cluster/*": "owned" + } + } + } + ] + } + } + } + ] + } + + # Write config to file + config_path = os.path.join(config_dir, f"{self.cluster_name}.yaml") + with open(config_path, 'w') as f: + yaml.dump(cluster_config, f) + + logger.info(f"Created cluster config at {config_path}") + self.cluster_config_path = config_path + + return config_path + + def provision_cluster(self): + """Create EKS cluster using eksctl""" + if not self.cluster_config_path: + self.create_cluster_config() + + # Verify AWS credentials before attempting to create cluster + logger.info("Verifying AWS credentials...") + aws_check = self._run_command(["aws", "sts", "get-caller-identity"]) + + if aws_check['returncode'] != 0: + logger.error("AWS credentials verification failed. Please check your AWS configuration.") + logger.error(f"Error: {aws_check['stderr']}") + sys.exit(1) + else: + ident = json.loads(aws_check['stdout']) + logger.info(f"Using AWS account: {ident.get('Account', 'unknown')} with user: {ident.get('UserId', 'unknown')}") + + # Verify eksctl installation + eksctl_check = self._run_command(["eksctl", "version"]) + if eksctl_check['returncode'] != 0: + logger.error("eksctl not found or not working properly. Please install eksctl.") + logger.error(f"Error: {eksctl_check['stderr']}") + sys.exit(1) + else: + logger.info(f"eksctl version: {eksctl_check['stdout'].strip()}") + + logger.info(f"Creating EKS cluster {self.cluster_name} in {self.region}...") + logger.info("This may take 15-20 minutes to complete...") + + # Use verbose flag for more detailed output + result = self._run_command( + ["eksctl", "create", "cluster", "-f", self.cluster_config_path, "--verbose", "4"] + ) + + if result['returncode'] != 0: + logger.error("Failed to create EKS cluster") + logger.error(f"Error details: {result['stderr']}") + logger.error("Common causes of failure:") + logger.error(" 1. Insufficient IAM permissions") + logger.error(" 2. Service quotas reached (check your AWS console)") + logger.error(" 3. Network connectivity issues") + logger.error(" 4. Region availability zone limitations") + + # Save the detailed output to a log file for debugging + error_log = f"logs/eks-cluster-creation-error-{self.cluster_name}.log" + with open(error_log, 'w') as f: + f.write(f"STDOUT:\n{result['stdout']}\n\nSTDERR:\n{result['stderr']}") + logger.error(f"Detailed error log saved to: {error_log}") + + sys.exit(1) + + logger.info("EKS cluster created successfully") + + # Update kubeconfig + self._run_command([ + "aws", "eks", "update-kubeconfig", + "--name", self.cluster_name, + "--region", self.region + ]) + + # Verify cluster is accessible + self.verify_cluster_connection() + + def verify_cluster_connection(self): + """Verify that we can connect to the Kubernetes cluster""" + nodes_check = self._run_command(["kubectl", "get", "nodes"]) + if nodes_check['returncode'] == 0: + logger.info("Cluster verified - nodes accessible:") + logger.info(nodes_check['stdout']) + else: + logger.error("Could not verify cluster nodes. There might be connectivity issues.") + logger.error(f"Error: {nodes_check['stderr']}") + sys.exit(1) + + def create_efs_filesystem(self): + """Create EFS filesystem in the same VPC as the cluster""" + logger.info("Creating EFS filesystem...") + + # Get VPC ID from cluster + logger.info(f"Getting VPC ID for cluster {self.cluster_name} in region {self.region}") + result = self._run_command([ + "aws", "eks", "describe-cluster", + "--name", self.cluster_name, + "--region", self.region, + "--query", "cluster.resourcesVpcConfig.vpcId", + "--output", "text" + ]) + + if result['returncode'] != 0: + logger.error("Failed to get VPC ID") + sys.exit(1) + + vpc_id = result['stdout'].strip() + logger.info(f"Cluster VPC: {vpc_id}") + + # Create security group for EFS + sg_result = self._run_command([ + "aws", "ec2", "create-security-group", + "--group-name", f"efs-test-sg-{self.cluster_name}", + "--description", "EFS security group for CSI driver testing", + "--vpc-id", vpc_id, + "--output", "json" + ]) + + if sg_result['returncode'] != 0: + logger.error("Failed to create security group") + sys.exit(1) + + sg_data = json.loads(sg_result['stdout']) + sg_id = sg_data['GroupId'] + + # Add inbound rule for NFS + self._run_command([ + "aws", "ec2", "authorize-security-group-ingress", + "--group-id", sg_id, + "--protocol", "tcp", + "--port", "2049", + "--cidr", "0.0.0.0/0" + ]) + + # Create EFS filesystem + efs_result = self._run_command([ + "aws", "efs", "create-file-system", + "--performance-mode", "generalPurpose", + "--throughput-mode", "bursting", + "--tags", f"Key=Name,Value=efs-csi-test-{self.cluster_name}", + "--region", self.region, + "--output", "json" + ]) + + if efs_result['returncode'] != 0: + logger.error("Failed to create EFS filesystem") + sys.exit(1) + + efs_data = json.loads(efs_result['stdout']) + filesystem_id = efs_data['FileSystemId'] + self.filesystem_id = filesystem_id + logger.info(f"Created EFS filesystem: {filesystem_id}") + + # Wait for filesystem creation to complete + logger.info("Waiting for EFS filesystem to be available...") + time.sleep(10) # Initial wait + + max_retries = 12 + retry_count = 0 + while retry_count < max_retries: + status_result = self._run_command([ + "aws", "efs", "describe-file-systems", + "--file-system-id", filesystem_id, + "--query", "FileSystems[0].LifeCycleState", + "--output", "text" + ]) + + status = status_result['stdout'].strip() + logger.info(f"EFS status: {status}") + + if status == "available": + break + + retry_count += 1 + time.sleep(10) + + if retry_count >= max_retries: + logger.error("EFS filesystem failed to become available in time") + sys.exit(1) + + # Get subnet IDs + subnet_result = self._run_command([ + "aws", "ec2", "describe-subnets", + "--filters", f"Name=vpc-id,Values={vpc_id}", + "--query", "Subnets[*].SubnetId", + "--output", "json" + ]) + + subnets = json.loads(subnet_result['stdout']) + + # Create mount targets in each subnet + for subnet_id in subnets: + logger.info(f"Creating mount target in subnet {subnet_id}") + self._run_command([ + "aws", "efs", "create-mount-target", + "--file-system-id", filesystem_id, + "--subnet-id", subnet_id, + "--security-groups", sg_id + ]) + + # Wait for mount targets to be available + logger.info("Waiting for mount targets to be available...") + time.sleep(20) # Initial wait + + max_retries = 12 + retry_count = 0 + while retry_count < max_retries: + targets_result = self._run_command([ + "aws", "efs", "describe-mount-targets", + "--file-system-id", filesystem_id, + "--output", "json" + ]) + + targets = json.loads(targets_result['stdout'])['MountTargets'] + + # Check if all targets are available + all_available = True + for target in targets: + if target['LifeCycleState'] != 'available': + all_available = False + break + + if all_available: + logger.info(f"All {len(targets)} mount targets are available") + break + + retry_count += 1 + logger.info(f"Waiting for mount targets... (retry {retry_count}/{max_retries})") + time.sleep(10) + + if retry_count >= max_retries: + logger.warning("Some mount targets may not be available yet, but continuing...") + + # Update config file with filesystem ID + self.update_config_with_filesystem() + return filesystem_id + + def update_config_with_filesystem(self): + """Update the configuration file with filesystem ID""" + if not self.filesystem_id: + logger.warning("No filesystem ID to update in config") + return + + # Update the original config file + with open('config/orchestrator_config.yaml', 'r') as f: + config_data = yaml.safe_load(f) + + # Update the driver section + if 'driver' in config_data: + config_data['driver']['filesystem_id'] = self.filesystem_id + + # Update the storage_class section + if 'storage_class' in config_data and 'parameters' in config_data['storage_class']: + config_data['storage_class']['parameters']['fileSystemId'] = self.filesystem_id + + # Write updated config + with open('config/orchestrator_config.yaml', 'w') as f: + yaml.dump(config_data, f) + + logger.info(f"Updated config file with filesystem ID: {self.filesystem_id}") + + def install_efs_csi_driver(self): + """Install EFS CSI driver on the cluster""" + logger.info(f"Installing EFS CSI Driver {self.csi_version}...") + + if self.install_method == "helm": + self._install_via_helm() + elif self.install_method == "yaml": + self._install_via_yaml() + elif self.install_method == "custom": + self._install_custom() + else: + logger.error(f"Unknown installation method: {self.install_method}") + sys.exit(1) + + logger.info("Verifying EFS CSI Driver installation...") + self._verify_driver_installation() + + def _install_via_helm(self): + """Install the EFS CSI Driver using Helm""" + # Install driver using Helm + self._run_command([ + "helm", "repo", "add", "aws-efs-csi-driver", + "https://kubernetes-sigs.github.io/aws-efs-csi-driver/" + ]) + + self._run_command(["helm", "repo", "update"]) + + result = self._run_command([ + "helm", "upgrade", "--install", "aws-efs-csi-driver", + "aws-efs-csi-driver/aws-efs-csi-driver", + "--namespace", "kube-system", + "--version", self.csi_version, + "--set", "controller.serviceAccount.create=false", + "--set", "controller.serviceAccount.name=efs-csi-controller-sa" + ]) + + if result['returncode'] != 0: + logger.error("Failed to install EFS CSI driver via Helm") + sys.exit(1) + + logger.info("EFS CSI driver installed successfully via Helm") + + def _install_via_yaml(self): + """Install the EFS CSI Driver using kubectl and YAML manifests""" + # Download the driver YAML for the specified version + yaml_url = f"https://raw.githubusercontent.com/kubernetes-sigs/aws-efs-csi-driver/v{self.csi_version}/deploy/kubernetes/overlays/stable/ecr/manifest.yaml" + + # Download the manifest + result = self._run_command([ + "curl", "-sSL", yaml_url, "-o", "efs-csi-driver.yaml" + ]) + + if result['returncode'] != 0: + logger.error(f"Failed to download YAML manifest from {yaml_url}") + sys.exit(1) + + # Apply the manifest + apply_result = self._run_command([ + "kubectl", "apply", "-f", "efs-csi-driver.yaml" + ]) + + if apply_result['returncode'] != 0: + logger.error("Failed to apply EFS CSI driver YAML") + sys.exit(1) + + logger.info("EFS CSI driver installed successfully via YAML") + + def _install_custom(self): + """Install the EFS CSI Driver using custom method specified in config""" + logger.info("Using custom installation method - this must be implemented by user") + # This is a placeholder for custom installation logic + # Users would implement this in their own subclass or modify this method + + def _verify_driver_installation(self): + """Verify that the EFS CSI Driver is installed and running""" + pod_check = self._run_command([ + "kubectl", "get", "pods", "-n", "kube-system", + "-l", "app=efs-csi-controller", + "--no-headers" + ]) + + if pod_check['returncode'] != 0 or not pod_check['stdout'].strip(): + logger.error("Could not find EFS CSI controller pods") + sys.exit(1) + + # Count running pods + running_pods = 0 + for line in pod_check['stdout'].strip().split('\n'): + if line and "Running" in line: + running_pods += 1 + + if running_pods == 0: + logger.error("No EFS CSI controller pods are running") + sys.exit(1) + + logger.info(f"Found {running_pods} running EFS CSI controller pods") + + # Check CSI driver registration + csi_check = self._run_command([ + "kubectl", "get", "csidrivers.storage.k8s.io", + "efs.csi.aws.com", "--no-headers" + ]) + + if csi_check['returncode'] != 0: + logger.warning("CSI driver not registered as a custom resource") + else: + logger.info("CSI driver registered successfully") + + def create_storage_class(self): + """Create the storage class for the EFS CSI Driver""" + if not self.filesystem_id: + logger.error("Cannot create storage class without filesystem ID") + sys.exit(1) + + logger.info(f"Creating StorageClass with filesystem ID: {self.filesystem_id}") + + # Get storage class configuration from config + sc_config = self.config.get('storage_class', {}) + sc_name = sc_config.get('name', 'efs-sc') + sc_parameters = sc_config.get('parameters', {}) + + # Ensure filesystem ID is set + sc_parameters['fileSystemId'] = self.filesystem_id + + storage_class = { + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass", + "metadata": {"name": sc_name}, + "provisioner": "efs.csi.aws.com", + "parameters": sc_parameters + } + + # Add mount options if specified + mount_options = sc_config.get('mount_options') + if mount_options: + storage_class["mountOptions"] = mount_options + + # Add reclaim policy if specified + reclaim_policy = sc_config.get('reclaim_policy') + if reclaim_policy: + storage_class["reclaimPolicy"] = reclaim_policy + + # Add volume binding mode if specified + volume_binding_mode = sc_config.get('volume_binding_mode') + if volume_binding_mode: + storage_class["volumeBindingMode"] = volume_binding_mode + + # Write storage class to file + sc_path = "storage-class.yaml" + with open(sc_path, 'w') as f: + yaml.dump(storage_class, f) + + # Apply storage class + result = self._run_command(["kubectl", "apply", "-f", sc_path]) + + if result['returncode'] != 0: + logger.error("Failed to create storage class") + sys.exit(1) + + logger.info(f"Storage class '{sc_name}' created successfully") + + # Verify storage class + verify_result = self._run_command([ + "kubectl", "get", "storageclass", sc_name + ]) + + if verify_result['returncode'] == 0: + logger.info(f"Verified storage class '{sc_name}' exists") + else: + logger.warning(f"Could not verify storage class '{sc_name}'") + + def cleanup(self, delete_cluster=True): + """Clean up resources - delete EFS filesystem and optionally the EKS cluster""" + logger.info("Starting cleanup process...") + + if self.filesystem_id and self.create_filesystem: + logger.info(f"Deleting EFS filesystem {self.filesystem_id}...") + + # First delete mount targets + try: + targets_result = self._run_command([ + "aws", "efs", "describe-mount-targets", + "--file-system-id", self.filesystem_id, + "--output", "json" + ]) + + if targets_result['returncode'] == 0: + targets = json.loads(targets_result['stdout']).get('MountTargets', []) + + for target in targets: + target_id = target['MountTargetId'] + logger.info(f"Deleting mount target {target_id}") + self._run_command([ + "aws", "efs", "delete-mount-target", + "--mount-target-id", target_id + ]) + + # Wait for mount targets to be deleted + logger.info("Waiting for mount targets to be deleted...") + time.sleep(30) + except Exception as e: + logger.warning(f"Error while deleting mount targets: {e}") + + # Now delete the filesystem + try: + self._run_command([ + "aws", "efs", "delete-file-system", + "--file-system-id", self.filesystem_id + ]) + logger.info(f"EFS filesystem {self.filesystem_id} deletion initiated") + except Exception as e: + logger.warning(f"Error while deleting filesystem: {e}") + + if delete_cluster and self.cluster_name and self.create_cluster: + logger.info(f"Deleting EKS cluster {self.cluster_name}...") + + try: + self._run_command([ + "eksctl", "delete", "cluster", + "--name", self.cluster_name, + "--region", self.region + ]) + logger.info(f"EKS cluster {self.cluster_name} deletion initiated") + except Exception as e: + logger.warning(f"Error while deleting cluster: {e}") + + logger.info("Cleanup process completed") + + def _run_command(self, cmd): + """Run a command and return stdout, stderr, and return code""" + logger.info(f"Running: {' '.join(cmd)}") + + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True + ) + + stdout, stderr = process.communicate() + + if process.returncode != 0: + logger.warning(f"Command returned non-zero exit code: {process.returncode}") + logger.warning(f"stderr: {stderr}") + + return { + "stdout": stdout, + "stderr": stderr, + "returncode": process.returncode + } + +def main(): + """Main function when run as a script""" + parser = argparse.ArgumentParser(description='EFS CSI Driver Cluster Setup') + + parser.add_argument( + '--config', + default='config/orchestrator_config.yaml', + help='Path to configuration file' + ) + + parser.add_argument( + '--no-cleanup', + action='store_true', + help='Do not clean up resources on failure' + ) + + parser.add_argument( + '--verify-only', + action='store_true', + help='Only verify existing cluster and exit' + ) + + parser.add_argument( + '--debug', + action='store_true', + help='Enable debug logging' + ) + + args = parser.parse_args() + + # Set debug logging if requested + if args.debug: + logger.setLevel(logging.DEBUG) + for handler in logger.handlers: + handler.setLevel(logging.DEBUG) + + try: + # Initialize setup + setup = ClusterSetup(config_file=args.config) + + if args.verify_only: + setup.verify_cluster_connection() + logger.info("Cluster verification completed successfully") + return 0 + + # Run setup + result = setup.setup() + logger.info(f"Setup completed successfully: {result}") + return 0 + + except Exception as e: + logger.exception(f"Error during setup: {e}") + + if not args.no_cleanup: + try: + logger.info("Attempting cleanup after failure...") + setup.cleanup() + except Exception as cleanup_error: + logger.error(f"Cleanup error: {cleanup_error}") + + return 1 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749045745.yaml b/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749045745.yaml new file mode 100644 index 000000000..49d66df21 --- /dev/null +++ b/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749045745.yaml @@ -0,0 +1,51 @@ +apiVersion: eksctl.io/v1alpha5 +iam: + serviceAccounts: + - attachPolicyARNs: + - arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy + metadata: + name: efs-csi-controller-sa + namespace: kube-system + withOIDC: true +kind: ClusterConfig +metadata: + name: efs-csi-test-1749045745 + region: us-west-2 + version: '1.25' +nodeGroups: +- desiredCapacity: 3 + iam: + attachPolicy: + Statement: + - Action: + - elasticfilesystem:DescribeAccessPoints + - elasticfilesystem:DescribeFileSystems + - elasticfilesystem:DescribeMountTargets + - ec2:DescribeAvailabilityZones + Effect: Allow + Resource: '*' + - Action: + - elasticfilesystem:CreateAccessPoint + Condition: + StringLike: + aws:RequestTag/kubernetes.io/cluster/*: owned + Effect: Allow + Resource: '*' + - Action: + - elasticfilesystem:DeleteAccessPoint + Condition: + StringEquals: + aws:ResourceTag/kubernetes.io/cluster/*: owned + Effect: Allow + Resource: '*' + Version: '2012-10-17' + instanceType: t3.large + labels: + efs-issue: 'true' + role: general + maxSize: 5 + minSize: 2 + name: ng-1 + tags: + nodegroup-role: general + volumeSize: 80 diff --git a/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749045783.yaml b/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749045783.yaml new file mode 100644 index 000000000..6da348e66 --- /dev/null +++ b/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749045783.yaml @@ -0,0 +1,51 @@ +apiVersion: eksctl.io/v1alpha5 +iam: + serviceAccounts: + - attachPolicyARNs: + - arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy + metadata: + name: efs-csi-controller-sa + namespace: kube-system + withOIDC: true +kind: ClusterConfig +metadata: + name: efs-csi-test-1749045783 + region: us-west-2 + version: '1.25' +nodeGroups: +- desiredCapacity: 3 + iam: + attachPolicy: + Statement: + - Action: + - elasticfilesystem:DescribeAccessPoints + - elasticfilesystem:DescribeFileSystems + - elasticfilesystem:DescribeMountTargets + - ec2:DescribeAvailabilityZones + Effect: Allow + Resource: '*' + - Action: + - elasticfilesystem:CreateAccessPoint + Condition: + StringLike: + aws:RequestTag/kubernetes.io/cluster/*: owned + Effect: Allow + Resource: '*' + - Action: + - elasticfilesystem:DeleteAccessPoint + Condition: + StringEquals: + aws:ResourceTag/kubernetes.io/cluster/*: owned + Effect: Allow + Resource: '*' + Version: '2012-10-17' + instanceType: t3.large + labels: + efs-issue: 'true' + role: general + maxSize: 5 + minSize: 2 + name: ng-1 + tags: + nodegroup-role: general + volumeSize: 80 diff --git a/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749048222.yaml b/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749048222.yaml new file mode 100644 index 000000000..b4f93b86e --- /dev/null +++ b/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749048222.yaml @@ -0,0 +1,51 @@ +apiVersion: eksctl.io/v1alpha5 +iam: + serviceAccounts: + - attachPolicyARNs: + - arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy + metadata: + name: efs-csi-controller-sa + namespace: kube-system + withOIDC: true +kind: ClusterConfig +metadata: + name: efs-csi-test-1749048222 + region: us-west-2 + version: '1.25' +nodeGroups: +- desiredCapacity: 3 + iam: + attachPolicy: + Statement: + - Action: + - elasticfilesystem:DescribeAccessPoints + - elasticfilesystem:DescribeFileSystems + - elasticfilesystem:DescribeMountTargets + - ec2:DescribeAvailabilityZones + Effect: Allow + Resource: '*' + - Action: + - elasticfilesystem:CreateAccessPoint + Condition: + StringLike: + aws:RequestTag/kubernetes.io/cluster/*: owned + Effect: Allow + Resource: '*' + - Action: + - elasticfilesystem:DeleteAccessPoint + Condition: + StringEquals: + aws:ResourceTag/kubernetes.io/cluster/*: owned + Effect: Allow + Resource: '*' + Version: '2012-10-17' + instanceType: t3.large + labels: + efs-issue: 'true' + role: general + maxSize: 5 + minSize: 2 + name: ng-1 + tags: + nodegroup-role: general + volumeSize: 80 diff --git a/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749048235.yaml b/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749048235.yaml new file mode 100644 index 000000000..3defffefe --- /dev/null +++ b/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749048235.yaml @@ -0,0 +1,51 @@ +apiVersion: eksctl.io/v1alpha5 +iam: + serviceAccounts: + - attachPolicyARNs: + - arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy + metadata: + name: efs-csi-controller-sa + namespace: kube-system + withOIDC: true +kind: ClusterConfig +metadata: + name: efs-csi-test-1749048235 + region: us-west-2 + version: '1.33' +nodeGroups: +- desiredCapacity: 3 + iam: + attachPolicy: + Statement: + - Action: + - elasticfilesystem:DescribeAccessPoints + - elasticfilesystem:DescribeFileSystems + - elasticfilesystem:DescribeMountTargets + - ec2:DescribeAvailabilityZones + Effect: Allow + Resource: '*' + - Action: + - elasticfilesystem:CreateAccessPoint + Condition: + StringLike: + aws:RequestTag/kubernetes.io/cluster/*: owned + Effect: Allow + Resource: '*' + - Action: + - elasticfilesystem:DeleteAccessPoint + Condition: + StringEquals: + aws:ResourceTag/kubernetes.io/cluster/*: owned + Effect: Allow + Resource: '*' + Version: '2012-10-17' + instanceType: t3.large + labels: + efs-issue: 'true' + role: general + maxSize: 5 + minSize: 2 + name: ng-1 + tags: + nodegroup-role: general + volumeSize: 80 diff --git a/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749048734.yaml b/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749048734.yaml new file mode 100644 index 000000000..dfcfaea6b --- /dev/null +++ b/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749048734.yaml @@ -0,0 +1,51 @@ +apiVersion: eksctl.io/v1alpha5 +iam: + serviceAccounts: + - attachPolicyARNs: + - arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy + metadata: + name: efs-csi-controller-sa + namespace: kube-system + withOIDC: true +kind: ClusterConfig +metadata: + name: efs-csi-test-1749048734 + region: us-west-2 + version: '1.33' +nodeGroups: +- desiredCapacity: 3 + iam: + attachPolicy: + Statement: + - Action: + - elasticfilesystem:DescribeAccessPoints + - elasticfilesystem:DescribeFileSystems + - elasticfilesystem:DescribeMountTargets + - ec2:DescribeAvailabilityZones + Effect: Allow + Resource: '*' + - Action: + - elasticfilesystem:CreateAccessPoint + Condition: + StringLike: + aws:RequestTag/kubernetes.io/cluster/*: owned + Effect: Allow + Resource: '*' + - Action: + - elasticfilesystem:DeleteAccessPoint + Condition: + StringEquals: + aws:ResourceTag/kubernetes.io/cluster/*: owned + Effect: Allow + Resource: '*' + Version: '2012-10-17' + instanceType: t3.large + labels: + efs-issue: 'true' + role: general + maxSize: 5 + minSize: 2 + name: ng-1 + tags: + nodegroup-role: general + volumeSize: 80 diff --git a/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749051234.yaml b/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749051234.yaml new file mode 100644 index 000000000..dfb706581 --- /dev/null +++ b/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749051234.yaml @@ -0,0 +1,51 @@ +apiVersion: eksctl.io/v1alpha5 +iam: + serviceAccounts: + - attachPolicyARNs: + - arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy + metadata: + name: efs-csi-controller-sa + namespace: kube-system + withOIDC: true +kind: ClusterConfig +metadata: + name: efs-csi-test-1749051234 + region: us-west-2 + version: '1.33' +nodeGroups: +- desiredCapacity: 3 + iam: + attachPolicy: + Statement: + - Action: + - elasticfilesystem:DescribeAccessPoints + - elasticfilesystem:DescribeFileSystems + - elasticfilesystem:DescribeMountTargets + - ec2:DescribeAvailabilityZones + Effect: Allow + Resource: '*' + - Action: + - elasticfilesystem:CreateAccessPoint + Condition: + StringLike: + aws:RequestTag/kubernetes.io/cluster/*: owned + Effect: Allow + Resource: '*' + - Action: + - elasticfilesystem:DeleteAccessPoint + Condition: + StringEquals: + aws:ResourceTag/kubernetes.io/cluster/*: owned + Effect: Allow + Resource: '*' + Version: '2012-10-17' + instanceType: t3.large + labels: + efs-issue: 'true' + role: general + maxSize: 5 + minSize: 2 + name: ng-1 + tags: + nodegroup-role: general + volumeSize: 80 diff --git a/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749051325.yaml b/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749051325.yaml new file mode 100644 index 000000000..e185004f3 --- /dev/null +++ b/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749051325.yaml @@ -0,0 +1,51 @@ +apiVersion: eksctl.io/v1alpha5 +iam: + serviceAccounts: + - attachPolicyARNs: + - arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy + metadata: + name: efs-csi-controller-sa + namespace: kube-system + withOIDC: true +kind: ClusterConfig +metadata: + name: efs-csi-test-1749051325 + region: us-west-2 + version: '1.33' +nodeGroups: +- desiredCapacity: 3 + iam: + attachPolicy: + Statement: + - Action: + - elasticfilesystem:DescribeAccessPoints + - elasticfilesystem:DescribeFileSystems + - elasticfilesystem:DescribeMountTargets + - ec2:DescribeAvailabilityZones + Effect: Allow + Resource: '*' + - Action: + - elasticfilesystem:CreateAccessPoint + Condition: + StringLike: + aws:RequestTag/kubernetes.io/cluster/*: owned + Effect: Allow + Resource: '*' + - Action: + - elasticfilesystem:DeleteAccessPoint + Condition: + StringEquals: + aws:ResourceTag/kubernetes.io/cluster/*: owned + Effect: Allow + Resource: '*' + Version: '2012-10-17' + instanceType: t3.large + labels: + efs-issue: 'true' + role: general + maxSize: 5 + minSize: 2 + name: ng-1 + tags: + nodegroup-role: general + volumeSize: 80 diff --git a/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749051374.yaml b/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749051374.yaml new file mode 100644 index 000000000..c1d32de98 --- /dev/null +++ b/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749051374.yaml @@ -0,0 +1,51 @@ +apiVersion: eksctl.io/v1alpha5 +iam: + serviceAccounts: + - attachPolicyARNs: + - arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy + metadata: + name: efs-csi-controller-sa + namespace: kube-system + withOIDC: true +kind: ClusterConfig +metadata: + name: efs-csi-test-1749051374 + region: us-west-2 + version: '1.33' +nodeGroups: +- desiredCapacity: 3 + iam: + attachPolicy: + Statement: + - Action: + - elasticfilesystem:DescribeAccessPoints + - elasticfilesystem:DescribeFileSystems + - elasticfilesystem:DescribeMountTargets + - ec2:DescribeAvailabilityZones + Effect: Allow + Resource: '*' + - Action: + - elasticfilesystem:CreateAccessPoint + Condition: + StringLike: + aws:RequestTag/kubernetes.io/cluster/*: owned + Effect: Allow + Resource: '*' + - Action: + - elasticfilesystem:DeleteAccessPoint + Condition: + StringEquals: + aws:ResourceTag/kubernetes.io/cluster/*: owned + Effect: Allow + Resource: '*' + Version: '2012-10-17' + instanceType: t3.large + labels: + efs-issue: 'true' + role: general + maxSize: 5 + minSize: 2 + name: ng-1 + tags: + nodegroup-role: general + volumeSize: 80 diff --git a/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749051392.yaml b/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749051392.yaml new file mode 100644 index 000000000..e4fc85ea9 --- /dev/null +++ b/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749051392.yaml @@ -0,0 +1,51 @@ +apiVersion: eksctl.io/v1alpha5 +iam: + serviceAccounts: + - attachPolicyARNs: + - arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy + metadata: + name: efs-csi-controller-sa + namespace: kube-system + withOIDC: true +kind: ClusterConfig +metadata: + name: efs-csi-test-1749051392 + region: us-west-2 + version: '1.28' +nodeGroups: +- desiredCapacity: 3 + iam: + attachPolicy: + Statement: + - Action: + - elasticfilesystem:DescribeAccessPoints + - elasticfilesystem:DescribeFileSystems + - elasticfilesystem:DescribeMountTargets + - ec2:DescribeAvailabilityZones + Effect: Allow + Resource: '*' + - Action: + - elasticfilesystem:CreateAccessPoint + Condition: + StringLike: + aws:RequestTag/kubernetes.io/cluster/*: owned + Effect: Allow + Resource: '*' + - Action: + - elasticfilesystem:DeleteAccessPoint + Condition: + StringEquals: + aws:ResourceTag/kubernetes.io/cluster/*: owned + Effect: Allow + Resource: '*' + Version: '2012-10-17' + instanceType: t3.large + labels: + efs-issue: 'true' + role: general + maxSize: 5 + minSize: 2 + name: ng-1 + tags: + nodegroup-role: general + volumeSize: 80 diff --git a/test/e2e/stress-scale-tests/config/orchestrator_config.yaml b/test/e2e/stress-scale-tests/config/orchestrator_config.yaml new file mode 100644 index 000000000..2b34d2e52 --- /dev/null +++ b/test/e2e/stress-scale-tests/config/orchestrator_config.yaml @@ -0,0 +1,124 @@ +cluster: + create: true + kubernetes_version: '1.28' + node_count: 3 + node_type: t3.large + region: us-west-2 +driver: + create_filesystem: true + filesystem_id: fs-024e09bcfd37c74aa + install_method: helm + version: 2.1.0 +driver_resources: + controller: + delete_access_point_root_dir: true + log_level: 5 + resources: + limits: + cpu: 1000m + memory: 2.5Gi + requests: + cpu: 200m + memory: 1Gi + node: + log_level: 5 + resources: + limits: + cpu: 1000m + memory: 1Gi + requests: + cpu: 100m + memory: 128Mi + sidecars: + liveness_probe: + resources: + limits: + cpu: 100m + memory: 128Mi + requests: + cpu: 10m + memory: 20Mi + node_driver_registrar: + resources: + limits: + cpu: 100m + memory: 128Mi + requests: + cpu: 10m + memory: 20Mi +logging: + console_enabled: true + file_enabled: true + level: INFO +metrics: + collect_operation_durations: true + collect_system_metrics: true +operation_weights: + attach_pod: 15 + create_pvc: 25 + delete_pod: 5 + delete_pvc: 10 + run_specific_scenario: 30 + verify_readwrite: 25 +pod_config: + args: + - touch /data/pod-ready && while true; do sleep 30; done + command: + - /bin/sh + - -c + image: alpine + # Temporarily commenting out node selector to fix pod scheduling issues + # node_selector: + # efs-issue: 'false' + readiness_probe: + initial_delay_seconds: 5 + period_seconds: 5 + tolerations: + - effect: NoSchedule + key: instance + operator: Equal + value: core +reporting: + json_report: true + report_directory: reports/orchestrator + summary_report: true +resource_limits: + max_pods_per_pvc: 50 + max_pvcs: 100 + total_max_pods: 30 +retries: + max_attempts: 3 + retry_delay: 1 +scenarios: + concurrent_pvc: + enabled: true + max_pvcs: 7 + min_pvcs: 3 + many_to_one: + enabled: true + max_pods: 5 + min_pods: 3 + one_to_one: + enabled: true + max_pairs: 5 + min_pairs: 3 +storage_class: + mount_options: + - tls + - hard + - nfsvers=4.1 + name: efs-sc + parameters: + basePath: /dynamic_provisioning_test + deleteAccessPointRootDir: 'true' + directoryPerms: '700' + fileSystemId: fs-024e09bcfd37c74aa + gidRangeEnd: '2000' + gidRangeStart: '1500' + provisioningMode: efs-ap + reclaim_policy: Delete + volume_binding_mode: Immediate +test: + duration: 300 + namespace: efs-stress-test + operation_interval: 3 diff --git a/test/e2e/stress-scale-tests/logs/cleanup_20250605_190904.log b/test/e2e/stress-scale-tests/logs/cleanup_20250605_190904.log new file mode 100644 index 000000000..1bd96f257 --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/cleanup_20250605_190904.log @@ -0,0 +1,129 @@ +2025-06-05 19:09:04,981 - INFO - Starting EFS CSI Driver test resource cleanup +2025-06-05 19:09:05,918 - INFO - Cleaning up resources in namespaces: ['default', 'efs-stress-test'] +2025-06-05 19:09:06,196 - INFO - Deleting pod/efs-scale-test-0 in namespace default +2025-06-05 19:09:06,287 - INFO - Deleting pod/efs-scale-test-1 in namespace default +2025-06-05 19:09:06,368 - INFO - Deleting pod/efs-scale-test-10 in namespace default +2025-06-05 19:09:06,463 - INFO - Deleting pod/efs-scale-test-11 in namespace default +2025-06-05 19:09:06,555 - INFO - Deleting pod/efs-scale-test-12 in namespace default +2025-06-05 19:09:06,645 - INFO - Deleting pod/efs-scale-test-13 in namespace default +2025-06-05 19:09:06,738 - INFO - Deleting pod/efs-scale-test-14 in namespace default +2025-06-05 19:09:06,820 - INFO - Deleting pod/efs-scale-test-15 in namespace default +2025-06-05 19:09:06,900 - INFO - Deleting pod/efs-scale-test-16 in namespace default +2025-06-05 19:09:06,983 - INFO - Deleting pod/efs-scale-test-17 in namespace default +2025-06-05 19:09:07,064 - INFO - Deleting pod/efs-scale-test-18 in namespace default +2025-06-05 19:09:07,151 - INFO - Deleting pod/efs-scale-test-19 in namespace default +2025-06-05 19:09:07,231 - INFO - Deleting pod/efs-scale-test-2 in namespace default +2025-06-05 19:09:07,310 - INFO - Deleting pod/efs-scale-test-3 in namespace default +2025-06-05 19:09:07,391 - INFO - Deleting pod/efs-scale-test-4 in namespace default +2025-06-05 19:09:07,471 - INFO - Deleting pod/efs-scale-test-5 in namespace default +2025-06-05 19:09:07,562 - INFO - Deleting pod/efs-scale-test-6 in namespace default +2025-06-05 19:09:07,650 - INFO - Deleting pod/efs-scale-test-7 in namespace default +2025-06-05 19:09:07,729 - INFO - Deleting pod/efs-scale-test-8 in namespace default +2025-06-05 19:09:07,813 - INFO - Deleting pod/efs-scale-test-9 in namespace default +2025-06-05 19:09:07,968 - INFO - Deleting pod/test-pod-3cc2d48d in namespace efs-stress-test +2025-06-05 19:09:08,055 - INFO - Deleting pod/test-pod-564344d0 in namespace efs-stress-test +2025-06-05 19:09:08,144 - INFO - Deleting pod/test-pod-a9935180 in namespace efs-stress-test +2025-06-05 19:09:08,225 - INFO - Deleting pod/test-pod-e1337987 in namespace efs-stress-test +2025-06-05 19:09:08,311 - INFO - Waiting 5 seconds for pods to start terminating before deleting PVCs... +2025-06-05 19:09:13,444 - INFO - Deleting pvc/efs-volume-efs-scale-test-0 in namespace default +2025-06-05 19:09:13,514 - INFO - Deleting pvc/efs-volume-efs-scale-test-1 in namespace default +2025-06-05 19:09:13,585 - INFO - Deleting pvc/efs-volume-efs-scale-test-10 in namespace default +2025-06-05 19:09:13,658 - INFO - Deleting pvc/efs-volume-efs-scale-test-11 in namespace default +2025-06-05 19:09:13,729 - INFO - Deleting pvc/efs-volume-efs-scale-test-12 in namespace default +2025-06-05 19:09:13,803 - INFO - Deleting pvc/efs-volume-efs-scale-test-13 in namespace default +2025-06-05 19:09:13,874 - INFO - Deleting pvc/efs-volume-efs-scale-test-14 in namespace default +2025-06-05 19:09:13,944 - INFO - Deleting pvc/efs-volume-efs-scale-test-15 in namespace default +2025-06-05 19:09:14,015 - INFO - Deleting pvc/efs-volume-efs-scale-test-16 in namespace default +2025-06-05 19:09:14,088 - INFO - Deleting pvc/efs-volume-efs-scale-test-17 in namespace default +2025-06-05 19:09:14,159 - INFO - Deleting pvc/efs-volume-efs-scale-test-18 in namespace default +2025-06-05 19:09:14,232 - INFO - Deleting pvc/efs-volume-efs-scale-test-19 in namespace default +2025-06-05 19:09:14,308 - INFO - Deleting pvc/efs-volume-efs-scale-test-2 in namespace default +2025-06-05 19:09:14,378 - INFO - Deleting pvc/efs-volume-efs-scale-test-3 in namespace default +2025-06-05 19:09:14,449 - INFO - Deleting pvc/efs-volume-efs-scale-test-4 in namespace default +2025-06-05 19:09:14,520 - INFO - Deleting pvc/efs-volume-efs-scale-test-5 in namespace default +2025-06-05 19:09:14,592 - INFO - Deleting pvc/efs-volume-efs-scale-test-6 in namespace default +2025-06-05 19:09:14,663 - INFO - Deleting pvc/efs-volume-efs-scale-test-7 in namespace default +2025-06-05 19:09:14,735 - INFO - Deleting pvc/efs-volume-efs-scale-test-8 in namespace default +2025-06-05 19:09:14,806 - INFO - Deleting pvc/efs-volume-efs-scale-test-9 in namespace default +2025-06-05 19:09:15,013 - INFO - Deleting pvc/test-pvc-0b5af805 in namespace efs-stress-test +2025-06-05 19:09:15,088 - INFO - Deleting pvc/test-pvc-0bf22cd9 in namespace efs-stress-test +2025-06-05 19:09:15,161 - INFO - Deleting pvc/test-pvc-1bb798a3 in namespace efs-stress-test +2025-06-05 19:09:15,237 - INFO - Deleting pvc/test-pvc-29a67853 in namespace efs-stress-test +2025-06-05 19:09:15,308 - INFO - Deleting pvc/test-pvc-36370699 in namespace efs-stress-test +2025-06-05 19:09:15,382 - INFO - Deleting pvc/test-pvc-3691f2a0 in namespace efs-stress-test +2025-06-05 19:09:15,454 - INFO - Deleting pvc/test-pvc-3d16a983 in namespace efs-stress-test +2025-06-05 19:09:15,526 - INFO - Deleting pvc/test-pvc-4184bff1 in namespace efs-stress-test +2025-06-05 19:09:15,600 - INFO - Deleting pvc/test-pvc-41d09b77 in namespace efs-stress-test +2025-06-05 19:09:15,672 - INFO - Deleting pvc/test-pvc-455abf8b in namespace efs-stress-test +2025-06-05 19:09:15,746 - INFO - Deleting pvc/test-pvc-51dbaa96 in namespace efs-stress-test +2025-06-05 19:09:15,820 - INFO - Deleting pvc/test-pvc-601432ce in namespace efs-stress-test +2025-06-05 19:09:15,891 - INFO - Deleting pvc/test-pvc-60b59274 in namespace efs-stress-test +2025-06-05 19:09:15,962 - INFO - Deleting pvc/test-pvc-63ada5ce in namespace efs-stress-test +2025-06-05 19:09:16,034 - INFO - Deleting pvc/test-pvc-64bf8174 in namespace efs-stress-test +2025-06-05 19:09:16,105 - INFO - Deleting pvc/test-pvc-6e8e6345 in namespace efs-stress-test +2025-06-05 19:09:16,179 - INFO - Deleting pvc/test-pvc-7ae6d5e7 in namespace efs-stress-test +2025-06-05 19:09:16,251 - INFO - Deleting pvc/test-pvc-b84d5df8 in namespace efs-stress-test +2025-06-05 19:09:16,322 - INFO - Deleting pvc/test-pvc-c196bb79 in namespace efs-stress-test +2025-06-05 19:09:16,400 - INFO - Deleting pvc/test-pvc-c1c2eedb in namespace efs-stress-test +2025-06-05 19:09:16,481 - INFO - Deleting pvc/test-pvc-c6976eae in namespace efs-stress-test +2025-06-05 19:09:16,552 - INFO - Deleting pvc/test-pvc-d0c53501 in namespace efs-stress-test +2025-06-05 19:09:16,630 - INFO - Deleting pvc/test-pvc-d3ee5325 in namespace efs-stress-test +2025-06-05 19:09:16,702 - INFO - Deleting pvc/test-pvc-faea1890 in namespace efs-stress-test +2025-06-05 19:09:16,775 - INFO - Deleting pvc/test-pvc-fe0fe649 in namespace efs-stress-test +2025-06-05 19:09:16,849 - INFO - Cleanup Summary: +2025-06-05 19:09:16,849 - INFO - Deleted 24 pods and 45 PVCs +2025-06-05 19:09:16,849 - INFO - All resources deleted successfully +2025-06-05 19:09:16,850 - INFO - Verifying resource deletion for up to 60 seconds... +2025-06-05 19:09:17,649 - INFO - Still waiting on 40 resources to be deleted... +2025-06-05 19:09:23,324 - INFO - Still waiting on 40 resources to be deleted... +2025-06-05 19:09:28,995 - INFO - Still waiting on 40 resources to be deleted... +2025-06-05 19:09:34,677 - INFO - Still waiting on 40 resources to be deleted... +2025-06-05 19:09:40,425 - INFO - Still waiting on 40 resources to be deleted... +2025-06-05 19:09:46,178 - INFO - Still waiting on 40 resources to be deleted... +2025-06-05 19:09:51,931 - INFO - Still waiting on 40 resources to be deleted... +2025-06-05 19:09:57,697 - INFO - Still waiting on 40 resources to be deleted... +2025-06-05 19:10:03,442 - INFO - Still waiting on 40 resources to be deleted... +2025-06-05 19:10:09,190 - INFO - Still waiting on 40 resources to be deleted... +2025-06-05 19:10:14,938 - INFO - Still waiting on 40 resources to be deleted... +2025-06-05 19:10:19,938 - ERROR - Timed out waiting for resource deletion. Remaining resources: +2025-06-05 19:10:19,939 - ERROR - - pod/default/efs-scale-test-0 +2025-06-05 19:10:19,939 - ERROR - - pod/default/efs-scale-test-1 +2025-06-05 19:10:19,939 - ERROR - - pod/default/efs-scale-test-10 +2025-06-05 19:10:19,939 - ERROR - - pod/default/efs-scale-test-11 +2025-06-05 19:10:19,939 - ERROR - - pod/default/efs-scale-test-12 +2025-06-05 19:10:19,939 - ERROR - - pod/default/efs-scale-test-13 +2025-06-05 19:10:19,939 - ERROR - - pod/default/efs-scale-test-14 +2025-06-05 19:10:19,939 - ERROR - - pod/default/efs-scale-test-15 +2025-06-05 19:10:19,939 - ERROR - - pod/default/efs-scale-test-16 +2025-06-05 19:10:19,939 - ERROR - - pod/default/efs-scale-test-17 +2025-06-05 19:10:19,939 - ERROR - - pod/default/efs-scale-test-18 +2025-06-05 19:10:19,939 - ERROR - - pod/default/efs-scale-test-19 +2025-06-05 19:10:19,939 - ERROR - - pod/default/efs-scale-test-2 +2025-06-05 19:10:19,939 - ERROR - - pod/default/efs-scale-test-3 +2025-06-05 19:10:19,939 - ERROR - - pod/default/efs-scale-test-4 +2025-06-05 19:10:19,939 - ERROR - - pod/default/efs-scale-test-5 +2025-06-05 19:10:19,939 - ERROR - - pod/default/efs-scale-test-6 +2025-06-05 19:10:19,939 - ERROR - - pod/default/efs-scale-test-7 +2025-06-05 19:10:19,939 - ERROR - - pod/default/efs-scale-test-8 +2025-06-05 19:10:19,939 - ERROR - - pod/default/efs-scale-test-9 +2025-06-05 19:10:19,939 - ERROR - - pvc/default/efs-volume-efs-scale-test-0 +2025-06-05 19:10:19,939 - ERROR - - pvc/default/efs-volume-efs-scale-test-1 +2025-06-05 19:10:19,939 - ERROR - - pvc/default/efs-volume-efs-scale-test-10 +2025-06-05 19:10:19,939 - ERROR - - pvc/default/efs-volume-efs-scale-test-11 +2025-06-05 19:10:19,939 - ERROR - - pvc/default/efs-volume-efs-scale-test-12 +2025-06-05 19:10:19,939 - ERROR - - pvc/default/efs-volume-efs-scale-test-13 +2025-06-05 19:10:19,939 - ERROR - - pvc/default/efs-volume-efs-scale-test-14 +2025-06-05 19:10:19,940 - ERROR - - pvc/default/efs-volume-efs-scale-test-15 +2025-06-05 19:10:19,940 - ERROR - - pvc/default/efs-volume-efs-scale-test-16 +2025-06-05 19:10:19,940 - ERROR - - pvc/default/efs-volume-efs-scale-test-17 +2025-06-05 19:10:19,940 - ERROR - - pvc/default/efs-volume-efs-scale-test-18 +2025-06-05 19:10:19,940 - ERROR - - pvc/default/efs-volume-efs-scale-test-19 +2025-06-05 19:10:19,940 - ERROR - - pvc/default/efs-volume-efs-scale-test-2 +2025-06-05 19:10:19,940 - ERROR - - pvc/default/efs-volume-efs-scale-test-3 +2025-06-05 19:10:19,940 - ERROR - - pvc/default/efs-volume-efs-scale-test-4 +2025-06-05 19:10:19,940 - ERROR - - pvc/default/efs-volume-efs-scale-test-5 +2025-06-05 19:10:19,940 - ERROR - - pvc/default/efs-volume-efs-scale-test-6 +2025-06-05 19:10:19,940 - ERROR - - pvc/default/efs-volume-efs-scale-test-7 +2025-06-05 19:10:19,940 - ERROR - - pvc/default/efs-volume-efs-scale-test-8 +2025-06-05 19:10:19,940 - ERROR - - pvc/default/efs-volume-efs-scale-test-9 +2025-06-05 19:10:19,940 - INFO - Cleanup process completed diff --git a/test/e2e/stress-scale-tests/logs/cluster_setup.log b/test/e2e/stress-scale-tests/logs/cluster_setup.log new file mode 100644 index 000000000..8a879c89f --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/cluster_setup.log @@ -0,0 +1,383 @@ +2025-06-04 14:02:25,306 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-04 14:02:25,306 - __main__ - INFO - Verifying AWS credentials +2025-06-04 14:02:25,307 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-04 14:02:25,308 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-04 14:02:25,308 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-04 14:02:25,309 - __main__ - INFO - Setting up EKS cluster and EFS CSI driver +2025-06-04 14:02:25,309 - __main__ - INFO - Using Kubernetes version from command line: 1.25 +2025-06-04 14:02:25,309 - __main__ - INFO - Using EFS CSI Driver version from command line: 2.1.0 +2025-06-04 14:02:25,322 - cluster_setup - INFO - Created cluster config at cluster-config/efs-csi-test-1749045745.yaml +2025-06-04 14:02:25,322 - cluster_setup - INFO - Verifying AWS credentials... +2025-06-04 14:02:25,323 - cluster_setup - INFO - Running: aws sts get-caller-identity +2025-06-04 14:02:26,166 - cluster_setup - WARNING - Command returned non-zero exit code: 254 +2025-06-04 14:02:26,167 - cluster_setup - WARNING - stderr: +An error occurred (ExpiredToken) when calling the GetCallerIdentity operation: The security token included in the request is expired + +2025-06-04 14:02:26,167 - cluster_setup - ERROR - AWS credentials verification failed. Please check your AWS configuration. +2025-06-04 14:02:26,167 - cluster_setup - ERROR - Error: +An error occurred (ExpiredToken) when calling the GetCallerIdentity operation: The security token included in the request is expired + +2025-06-04 14:03:03,790 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-04 14:03:03,790 - __main__ - INFO - Verifying AWS credentials +2025-06-04 14:03:03,791 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-04 14:03:03,791 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-04 14:03:03,792 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-04 14:03:03,792 - __main__ - INFO - Setting up EKS cluster and EFS CSI driver +2025-06-04 14:03:03,792 - __main__ - INFO - Using Kubernetes version from command line: 1.25 +2025-06-04 14:03:03,792 - __main__ - INFO - Using EFS CSI Driver version from command line: 2.1.0 +2025-06-04 14:03:03,807 - cluster_setup - INFO - Created cluster config at cluster-config/efs-csi-test-1749045783.yaml +2025-06-04 14:03:03,807 - cluster_setup - INFO - Verifying AWS credentials... +2025-06-04 14:03:03,807 - cluster_setup - INFO - Running: aws sts get-caller-identity +2025-06-04 14:03:04,654 - cluster_setup - INFO - Using AWS account: 745939127895 with user: AROA23LLPZZLQCLTVMVDC:ibhard-Isengard +2025-06-04 14:03:04,655 - cluster_setup - INFO - Running: eksctl version +2025-06-04 14:03:04,697 - cluster_setup - INFO - eksctl version: 0.208.0 +2025-06-04 14:03:04,697 - cluster_setup - INFO - Creating EKS cluster efs-csi-test-1749045783 in us-west-2... +2025-06-04 14:03:04,697 - cluster_setup - INFO - This may take 15-20 minutes to complete... +2025-06-04 14:03:04,697 - cluster_setup - INFO - Running: eksctl create cluster -f cluster-config/efs-csi-test-1749045783.yaml --verbose 4 +2025-06-04 14:03:05,304 - cluster_setup - WARNING - Command returned non-zero exit code: 1 +2025-06-04 14:03:05,304 - cluster_setup - WARNING - stderr: Error: resolving cluster version: invalid version, 1.25 is no longer supported, supported values: 1.26, 1.27, 1.28, 1.29, 1.30, 1.31, 1.32, 1.33 +see also: https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html + +2025-06-04 14:03:05,304 - cluster_setup - ERROR - Failed to create EKS cluster +2025-06-04 14:03:05,304 - cluster_setup - ERROR - Error details: Error: resolving cluster version: invalid version, 1.25 is no longer supported, supported values: 1.26, 1.27, 1.28, 1.29, 1.30, 1.31, 1.32, 1.33 +see also: https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html + +2025-06-04 14:03:05,304 - cluster_setup - ERROR - Common causes of failure: +2025-06-04 14:03:05,304 - cluster_setup - ERROR - 1. Insufficient IAM permissions +2025-06-04 14:03:05,304 - cluster_setup - ERROR - 2. Service quotas reached (check your AWS console) +2025-06-04 14:03:05,304 - cluster_setup - ERROR - 3. Network connectivity issues +2025-06-04 14:03:05,304 - cluster_setup - ERROR - 4. Region availability zone limitations +2025-06-04 14:03:05,304 - cluster_setup - ERROR - Detailed error log saved to: logs/eks-cluster-creation-error-efs-csi-test-1749045783.log +2025-06-04 14:43:42,712 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-04 14:43:42,712 - __main__ - INFO - Verifying AWS credentials +2025-06-04 14:43:42,713 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-04 14:43:42,714 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-04 14:43:42,714 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-04 14:43:42,715 - __main__ - INFO - Setting up EKS cluster and EFS CSI driver +2025-06-04 14:43:42,715 - __main__ - INFO - Using Kubernetes version from command line: 1.25 +2025-06-04 14:43:42,715 - __main__ - INFO - Using EFS CSI Driver version from command line: 2.1.0 +2025-06-04 14:43:42,728 - cluster_setup - INFO - Created cluster config at cluster-config/efs-csi-test-1749048222.yaml +2025-06-04 14:43:42,728 - cluster_setup - INFO - Verifying AWS credentials... +2025-06-04 14:43:42,728 - cluster_setup - INFO - Running: aws sts get-caller-identity +2025-06-04 14:43:43,584 - cluster_setup - INFO - Using AWS account: 745939127895 with user: AROA23LLPZZLQCLTVMVDC:ibhard-Isengard +2025-06-04 14:43:43,584 - cluster_setup - INFO - Running: eksctl version +2025-06-04 14:43:43,619 - cluster_setup - INFO - eksctl version: 0.208.0 +2025-06-04 14:43:43,619 - cluster_setup - INFO - Creating EKS cluster efs-csi-test-1749048222 in us-west-2... +2025-06-04 14:43:43,619 - cluster_setup - INFO - This may take 15-20 minutes to complete... +2025-06-04 14:43:43,619 - cluster_setup - INFO - Running: eksctl create cluster -f cluster-config/efs-csi-test-1749048222.yaml --verbose 4 +2025-06-04 14:43:44,231 - cluster_setup - WARNING - Command returned non-zero exit code: 1 +2025-06-04 14:43:44,231 - cluster_setup - WARNING - stderr: Error: resolving cluster version: invalid version, 1.25 is no longer supported, supported values: 1.26, 1.27, 1.28, 1.29, 1.30, 1.31, 1.32, 1.33 +see also: https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html + +2025-06-04 14:43:44,231 - cluster_setup - ERROR - Failed to create EKS cluster +2025-06-04 14:43:44,231 - cluster_setup - ERROR - Error details: Error: resolving cluster version: invalid version, 1.25 is no longer supported, supported values: 1.26, 1.27, 1.28, 1.29, 1.30, 1.31, 1.32, 1.33 +see also: https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html + +2025-06-04 14:43:44,231 - cluster_setup - ERROR - Common causes of failure: +2025-06-04 14:43:44,231 - cluster_setup - ERROR - 1. Insufficient IAM permissions +2025-06-04 14:43:44,231 - cluster_setup - ERROR - 2. Service quotas reached (check your AWS console) +2025-06-04 14:43:44,231 - cluster_setup - ERROR - 3. Network connectivity issues +2025-06-04 14:43:44,231 - cluster_setup - ERROR - 4. Region availability zone limitations +2025-06-04 14:43:44,231 - cluster_setup - ERROR - Detailed error log saved to: logs/eks-cluster-creation-error-efs-csi-test-1749048222.log +2025-06-04 14:43:55,254 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-04 14:43:55,254 - __main__ - INFO - Verifying AWS credentials +2025-06-04 14:43:55,255 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-04 14:43:55,255 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-04 14:43:55,256 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-04 14:43:55,256 - __main__ - INFO - Setting up EKS cluster and EFS CSI driver +2025-06-04 14:43:55,256 - __main__ - INFO - Using Kubernetes version from command line: 1.33 +2025-06-04 14:43:55,256 - __main__ - INFO - Using EFS CSI Driver version from command line: 2.1.0 +2025-06-04 14:43:55,269 - cluster_setup - INFO - Created cluster config at cluster-config/efs-csi-test-1749048235.yaml +2025-06-04 14:43:55,269 - cluster_setup - INFO - Verifying AWS credentials... +2025-06-04 14:43:55,269 - cluster_setup - INFO - Running: aws sts get-caller-identity +2025-06-04 14:43:56,102 - cluster_setup - INFO - Using AWS account: 745939127895 with user: AROA23LLPZZLQCLTVMVDC:ibhard-Isengard +2025-06-04 14:43:56,102 - cluster_setup - INFO - Running: eksctl version +2025-06-04 14:43:56,138 - cluster_setup - INFO - eksctl version: 0.208.0 +2025-06-04 14:43:56,138 - cluster_setup - INFO - Creating EKS cluster efs-csi-test-1749048235 in us-west-2... +2025-06-04 14:43:56,138 - cluster_setup - INFO - This may take 15-20 minutes to complete... +2025-06-04 14:43:56,138 - cluster_setup - INFO - Running: eksctl create cluster -f cluster-config/efs-csi-test-1749048235.yaml --verbose 4 +2025-06-04 14:43:57,393 - cluster_setup - WARNING - Command returned non-zero exit code: 1 +2025-06-04 14:43:57,393 - cluster_setup - WARNING - stderr: Error: unable to determine AMI to use: error getting AMI from SSM Parameter Store: operation error SSM: GetParameter, https response error StatusCode: 400, RequestID: a443a5ad-af0e-4afb-b46f-a021532755a5, ParameterNotFound: . please verify that AMI Family is supported + +2025-06-04 14:43:57,393 - cluster_setup - ERROR - Failed to create EKS cluster +2025-06-04 14:43:57,393 - cluster_setup - ERROR - Error details: Error: unable to determine AMI to use: error getting AMI from SSM Parameter Store: operation error SSM: GetParameter, https response error StatusCode: 400, RequestID: a443a5ad-af0e-4afb-b46f-a021532755a5, ParameterNotFound: . please verify that AMI Family is supported + +2025-06-04 14:43:57,393 - cluster_setup - ERROR - Common causes of failure: +2025-06-04 14:43:57,393 - cluster_setup - ERROR - 1. Insufficient IAM permissions +2025-06-04 14:43:57,393 - cluster_setup - ERROR - 2. Service quotas reached (check your AWS console) +2025-06-04 14:43:57,393 - cluster_setup - ERROR - 3. Network connectivity issues +2025-06-04 14:43:57,393 - cluster_setup - ERROR - 4. Region availability zone limitations +2025-06-04 14:43:57,393 - cluster_setup - ERROR - Detailed error log saved to: logs/eks-cluster-creation-error-efs-csi-test-1749048235.log +2025-06-04 14:52:14,794 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-04 14:52:14,794 - __main__ - INFO - Verifying AWS credentials +2025-06-04 14:52:14,795 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-04 14:52:14,795 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-04 14:52:14,796 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-04 14:52:14,796 - __main__ - INFO - Setting up EKS cluster and EFS CSI driver +2025-06-04 14:52:14,796 - __main__ - INFO - Using Kubernetes version from command line: 1.33 +2025-06-04 14:52:14,796 - __main__ - INFO - Using EFS CSI Driver version from command line: 2.1.0 +2025-06-04 14:52:14,809 - cluster_setup - INFO - Created cluster config at cluster-config/efs-csi-test-1749048734.yaml +2025-06-04 14:52:14,809 - cluster_setup - INFO - Verifying AWS credentials... +2025-06-04 14:52:14,809 - cluster_setup - INFO - Running: aws sts get-caller-identity +2025-06-04 14:52:15,649 - cluster_setup - INFO - Using AWS account: 745939127895 with user: AROA23LLPZZLQCLTVMVDC:ibhard-Isengard +2025-06-04 14:52:15,649 - cluster_setup - INFO - Running: eksctl version +2025-06-04 14:52:15,683 - cluster_setup - INFO - eksctl version: 0.208.0 +2025-06-04 14:52:15,683 - cluster_setup - INFO - Creating EKS cluster efs-csi-test-1749048734 in us-west-2... +2025-06-04 14:52:15,683 - cluster_setup - INFO - This may take 15-20 minutes to complete... +2025-06-04 14:52:15,683 - cluster_setup - INFO - Running: eksctl create cluster -f cluster-config/efs-csi-test-1749048734.yaml --verbose 4 +2025-06-04 14:52:16,955 - cluster_setup - WARNING - Command returned non-zero exit code: 1 +2025-06-04 14:52:16,955 - cluster_setup - WARNING - stderr: Error: unable to determine AMI to use: error getting AMI from SSM Parameter Store: operation error SSM: GetParameter, https response error StatusCode: 400, RequestID: 4650ea1c-e137-440e-835d-9214a82b7beb, ParameterNotFound: . please verify that AMI Family is supported + +2025-06-04 14:52:16,955 - cluster_setup - ERROR - Failed to create EKS cluster +2025-06-04 14:52:16,955 - cluster_setup - ERROR - Error details: Error: unable to determine AMI to use: error getting AMI from SSM Parameter Store: operation error SSM: GetParameter, https response error StatusCode: 400, RequestID: 4650ea1c-e137-440e-835d-9214a82b7beb, ParameterNotFound: . please verify that AMI Family is supported + +2025-06-04 14:52:16,955 - cluster_setup - ERROR - Common causes of failure: +2025-06-04 14:52:16,955 - cluster_setup - ERROR - 1. Insufficient IAM permissions +2025-06-04 14:52:16,956 - cluster_setup - ERROR - 2. Service quotas reached (check your AWS console) +2025-06-04 14:52:16,956 - cluster_setup - ERROR - 3. Network connectivity issues +2025-06-04 14:52:16,956 - cluster_setup - ERROR - 4. Region availability zone limitations +2025-06-04 14:52:16,956 - cluster_setup - ERROR - Detailed error log saved to: logs/eks-cluster-creation-error-efs-csi-test-1749048734.log +2025-06-04 15:33:53,994 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-04 15:33:53,994 - __main__ - INFO - Verifying AWS credentials +2025-06-04 15:33:53,995 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-04 15:33:53,995 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-04 15:33:53,996 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-04 15:33:53,996 - __main__ - INFO - Setting up EKS cluster and EFS CSI driver +2025-06-04 15:33:53,996 - __main__ - INFO - Using Kubernetes version from command line: 1.33 +2025-06-04 15:33:53,996 - __main__ - INFO - Using EFS CSI Driver version from command line: 2.1.0 +2025-06-04 15:33:54,008 - cluster_setup - INFO - Created cluster config at cluster-config/efs-csi-test-1749051234.yaml +2025-06-04 15:33:54,008 - cluster_setup - INFO - Verifying AWS credentials... +2025-06-04 15:33:54,008 - cluster_setup - INFO - Running: aws sts get-caller-identity +2025-06-04 15:33:54,759 - cluster_setup - WARNING - Command returned non-zero exit code: 254 +2025-06-04 15:33:54,759 - cluster_setup - WARNING - stderr: +An error occurred (ExpiredToken) when calling the GetCallerIdentity operation: The security token included in the request is expired + +2025-06-04 15:33:54,760 - cluster_setup - ERROR - AWS credentials verification failed. Please check your AWS configuration. +2025-06-04 15:33:54,760 - cluster_setup - ERROR - Error: +An error occurred (ExpiredToken) when calling the GetCallerIdentity operation: The security token included in the request is expired + +2025-06-04 15:35:25,473 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-04 15:35:25,474 - __main__ - INFO - Verifying AWS credentials +2025-06-04 15:35:25,475 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-04 15:35:25,475 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-04 15:35:25,475 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-04 15:35:25,476 - __main__ - INFO - Setting up EKS cluster and EFS CSI driver +2025-06-04 15:35:25,476 - __main__ - INFO - Using Kubernetes version from command line: 1.33 +2025-06-04 15:35:25,476 - __main__ - INFO - Using EFS CSI Driver version from command line: 2.1.0 +2025-06-04 15:35:25,488 - cluster_setup - INFO - Created cluster config at cluster-config/efs-csi-test-1749051325.yaml +2025-06-04 15:35:25,489 - cluster_setup - INFO - Verifying AWS credentials... +2025-06-04 15:35:25,489 - cluster_setup - INFO - Running: aws sts get-caller-identity +2025-06-04 15:35:26,251 - cluster_setup - INFO - Using AWS account: 745939127895 with user: AROA23LLPZZLQCLTVMVDC:ibhard-Isengard +2025-06-04 15:35:26,251 - cluster_setup - INFO - Running: eksctl version +2025-06-04 15:35:26,284 - cluster_setup - INFO - eksctl version: 0.208.0 +2025-06-04 15:35:26,285 - cluster_setup - INFO - Creating EKS cluster efs-csi-test-1749051325 in us-west-2... +2025-06-04 15:35:26,285 - cluster_setup - INFO - This may take 15-20 minutes to complete... +2025-06-04 15:35:26,285 - cluster_setup - INFO - Running: eksctl create cluster -f cluster-config/efs-csi-test-1749051325.yaml --verbose 4 +2025-06-04 15:35:27,569 - cluster_setup - WARNING - Command returned non-zero exit code: 1 +2025-06-04 15:35:27,569 - cluster_setup - WARNING - stderr: Error: unable to determine AMI to use: error getting AMI from SSM Parameter Store: operation error SSM: GetParameter, https response error StatusCode: 400, RequestID: 59f5b73c-2035-46bb-9074-986f681c13f8, ParameterNotFound: . please verify that AMI Family is supported + +2025-06-04 15:35:27,569 - cluster_setup - ERROR - Failed to create EKS cluster +2025-06-04 15:35:27,569 - cluster_setup - ERROR - Error details: Error: unable to determine AMI to use: error getting AMI from SSM Parameter Store: operation error SSM: GetParameter, https response error StatusCode: 400, RequestID: 59f5b73c-2035-46bb-9074-986f681c13f8, ParameterNotFound: . please verify that AMI Family is supported + +2025-06-04 15:35:27,570 - cluster_setup - ERROR - Common causes of failure: +2025-06-04 15:35:27,570 - cluster_setup - ERROR - 1. Insufficient IAM permissions +2025-06-04 15:35:27,570 - cluster_setup - ERROR - 2. Service quotas reached (check your AWS console) +2025-06-04 15:35:27,570 - cluster_setup - ERROR - 3. Network connectivity issues +2025-06-04 15:35:27,570 - cluster_setup - ERROR - 4. Region availability zone limitations +2025-06-04 15:35:27,570 - cluster_setup - ERROR - Detailed error log saved to: logs/eks-cluster-creation-error-efs-csi-test-1749051325.log +2025-06-04 15:36:14,552 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-04 15:36:14,552 - __main__ - INFO - Verifying AWS credentials +2025-06-04 15:36:14,553 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-04 15:36:14,553 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-04 15:36:14,554 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-04 15:36:14,554 - __main__ - INFO - Setting up EKS cluster and EFS CSI driver +2025-06-04 15:36:14,554 - __main__ - INFO - Using Kubernetes version from command line: 1.33 +2025-06-04 15:36:14,554 - __main__ - INFO - Using EFS CSI Driver version from command line: 2.1.0 +2025-06-04 15:36:14,567 - cluster_setup - INFO - Created cluster config at cluster-config/efs-csi-test-1749051374.yaml +2025-06-04 15:36:14,567 - cluster_setup - INFO - Verifying AWS credentials... +2025-06-04 15:36:14,567 - cluster_setup - INFO - Running: aws sts get-caller-identity +2025-06-04 15:36:15,333 - cluster_setup - INFO - Using AWS account: 745939127895 with user: AROA23LLPZZLQCLTVMVDC:ibhard-Isengard +2025-06-04 15:36:15,333 - cluster_setup - INFO - Running: eksctl version +2025-06-04 15:36:15,366 - cluster_setup - INFO - eksctl version: 0.208.0 +2025-06-04 15:36:15,366 - cluster_setup - INFO - Creating EKS cluster efs-csi-test-1749051374 in us-west-2... +2025-06-04 15:36:15,366 - cluster_setup - INFO - This may take 15-20 minutes to complete... +2025-06-04 15:36:15,366 - cluster_setup - INFO - Running: eksctl create cluster -f cluster-config/efs-csi-test-1749051374.yaml --verbose 4 +2025-06-04 15:36:16,734 - cluster_setup - WARNING - Command returned non-zero exit code: 1 +2025-06-04 15:36:16,734 - cluster_setup - WARNING - stderr: Error: unable to determine AMI to use: error getting AMI from SSM Parameter Store: operation error SSM: GetParameter, https response error StatusCode: 400, RequestID: 1189c451-cdbd-4c3b-9495-a627bfe2113d, ParameterNotFound: . please verify that AMI Family is supported + +2025-06-04 15:36:16,734 - cluster_setup - ERROR - Failed to create EKS cluster +2025-06-04 15:36:16,734 - cluster_setup - ERROR - Error details: Error: unable to determine AMI to use: error getting AMI from SSM Parameter Store: operation error SSM: GetParameter, https response error StatusCode: 400, RequestID: 1189c451-cdbd-4c3b-9495-a627bfe2113d, ParameterNotFound: . please verify that AMI Family is supported + +2025-06-04 15:36:16,734 - cluster_setup - ERROR - Common causes of failure: +2025-06-04 15:36:16,734 - cluster_setup - ERROR - 1. Insufficient IAM permissions +2025-06-04 15:36:16,734 - cluster_setup - ERROR - 2. Service quotas reached (check your AWS console) +2025-06-04 15:36:16,734 - cluster_setup - ERROR - 3. Network connectivity issues +2025-06-04 15:36:16,734 - cluster_setup - ERROR - 4. Region availability zone limitations +2025-06-04 15:36:16,734 - cluster_setup - ERROR - Detailed error log saved to: logs/eks-cluster-creation-error-efs-csi-test-1749051374.log +2025-06-04 15:36:31,995 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-04 15:36:31,996 - __main__ - INFO - Verifying AWS credentials +2025-06-04 15:36:31,997 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-04 15:36:31,997 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-04 15:36:31,997 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-04 15:36:31,998 - __main__ - INFO - Setting up EKS cluster and EFS CSI driver +2025-06-04 15:36:31,998 - __main__ - INFO - Using Kubernetes version from command line: 1.28 +2025-06-04 15:36:31,998 - __main__ - INFO - Using EFS CSI Driver version from command line: 2.1.0 +2025-06-04 15:36:32,010 - cluster_setup - INFO - Created cluster config at cluster-config/efs-csi-test-1749051392.yaml +2025-06-04 15:36:32,010 - cluster_setup - INFO - Verifying AWS credentials... +2025-06-04 15:36:32,011 - cluster_setup - INFO - Running: aws sts get-caller-identity +2025-06-04 15:36:32,772 - cluster_setup - INFO - Using AWS account: 745939127895 with user: AROA23LLPZZLQCLTVMVDC:ibhard-Isengard +2025-06-04 15:36:32,772 - cluster_setup - INFO - Running: eksctl version +2025-06-04 15:36:32,804 - cluster_setup - INFO - eksctl version: 0.208.0 +2025-06-04 15:36:32,804 - cluster_setup - INFO - Creating EKS cluster efs-csi-test-1749051392 in us-west-2... +2025-06-04 15:36:32,804 - cluster_setup - INFO - This may take 15-20 minutes to complete... +2025-06-04 15:36:32,804 - cluster_setup - INFO - Running: eksctl create cluster -f cluster-config/efs-csi-test-1749051392.yaml --verbose 4 +2025-06-04 15:37:05,360 - cluster_setup - WARNING - Command returned non-zero exit code: 1 +2025-06-04 15:37:05,360 - cluster_setup - WARNING - stderr: Error: failed to create cluster "efs-csi-test-1749051392" + +2025-06-04 15:37:05,360 - cluster_setup - ERROR - Failed to create EKS cluster +2025-06-04 15:37:05,360 - cluster_setup - ERROR - Error details: Error: failed to create cluster "efs-csi-test-1749051392" + +2025-06-04 15:37:05,360 - cluster_setup - ERROR - Common causes of failure: +2025-06-04 15:37:05,360 - cluster_setup - ERROR - 1. Insufficient IAM permissions +2025-06-04 15:37:05,360 - cluster_setup - ERROR - 2. Service quotas reached (check your AWS console) +2025-06-04 15:37:05,360 - cluster_setup - ERROR - 3. Network connectivity issues +2025-06-04 15:37:05,361 - cluster_setup - ERROR - 4. Region availability zone limitations +2025-06-04 15:37:05,361 - cluster_setup - ERROR - Detailed error log saved to: logs/eks-cluster-creation-error-efs-csi-test-1749051392.log +2025-06-04 20:43:00,283 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-04 20:43:00,283 - __main__ - INFO - Verifying AWS credentials +2025-06-04 20:43:00,284 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-04 20:43:00,285 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-04 20:43:00,285 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-04 20:43:00,285 - __main__ - INFO - Running orchestrator stress test suite +2025-06-04 20:43:00,285 - __main__ - INFO - Starting orchestrator for 300 seconds +2025-06-04 20:43:00,285 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-04 20:43:01,074 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-04 20:43:01,299 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-04 20:43:01,299 - __main__ - INFO - Test duration overridden to 300 seconds +2025-06-04 20:43:01,299 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) +2025-06-04 20:43:01,299 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-04 20:43:01,528 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-04 20:43:01,529 - tests.orchestrator - INFO - Selected operation: _verify_readwrite +2025-06-04 20:43:01,529 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test +2025-06-04 20:43:02,529 - tests.orchestrator - INFO - Selected operation: _attach_pod +2025-06-04 20:43:02,529 - tests.orchestrator - INFO - No PVCs available, skipping pod attachment +2025-06-04 20:43:03,529 - tests.orchestrator - INFO - Selected operation: _create_pvc +2025-06-04 20:43:03,530 - tests.orchestrator - INFO - Creating PVC: test-pvc-a8738d91 +2025-06-04 20:43:03,612 - tests.orchestrator - INFO - Created PVC: test-pvc-a8738d91 +2025-06-04 20:43:03,613 - tests.orchestrator - INFO - Waiting for PVC test-pvc-a8738d91 to be bound +2025-06-04 20:43:05,748 - tests.orchestrator - INFO - PVC test-pvc-a8738d91 is bound +2025-06-04 20:43:06,749 - tests.orchestrator - INFO - Selected operation: _create_pvc +2025-06-04 20:43:06,749 - tests.orchestrator - INFO - Creating PVC: test-pvc-ba881b7a +2025-06-04 20:43:06,823 - tests.orchestrator - INFO - Created PVC: test-pvc-ba881b7a +2025-06-04 20:43:06,824 - tests.orchestrator - INFO - Waiting for PVC test-pvc-ba881b7a to be bound +2025-06-04 20:43:08,959 - tests.orchestrator - INFO - PVC test-pvc-ba881b7a is bound +2025-06-04 20:43:09,959 - tests.orchestrator - INFO - Selected operation: _delete_pod +2025-06-04 20:43:09,959 - tests.orchestrator - INFO - No pods to delete +2025-06-04 20:43:10,959 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario +2025-06-04 20:43:10,960 - tests.orchestrator - INFO - Running specific scenario: _scenario_one_to_one +2025-06-04 20:43:10,960 - tests.orchestrator - INFO - Running scenario: One pod per PVC +2025-06-04 20:43:10,960 - tests.orchestrator - INFO - Creating 4 PVC-pod pairs +2025-06-04 20:43:11,032 - tests.orchestrator - INFO - Waiting for PVC one2one-43f0888a to be bound +2025-06-04 20:43:13,170 - tests.orchestrator - INFO - PVC one2one-43f0888a is bound +2025-06-04 20:43:13,170 - tests.orchestrator - INFO - Creating pod test-pod-f3b3a319 on PVC one2one-43f0888a +2025-06-04 20:43:13,279 - tests.orchestrator - INFO - Created pod: test-pod-f3b3a319 using PVC: one2one-43f0888a +2025-06-04 20:43:13,279 - tests.orchestrator - INFO - Waiting for pod test-pod-f3b3a319 to be ready +2025-06-04 20:44:15,334 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-f3b3a319 to be ready after 60s +2025-06-04 20:44:15,334 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-f3b3a319 to be ready +2025-06-04 20:44:15,411 - tests.orchestrator - INFO - Waiting for PVC one2one-aac689b5 to be bound +2025-06-04 20:44:17,545 - tests.orchestrator - INFO - PVC one2one-aac689b5 is bound +2025-06-04 20:44:17,545 - tests.orchestrator - INFO - Creating pod test-pod-80037404 on PVC one2one-aac689b5 +2025-06-04 20:44:17,620 - tests.orchestrator - INFO - Created pod: test-pod-80037404 using PVC: one2one-aac689b5 +2025-06-04 20:44:17,620 - tests.orchestrator - INFO - Waiting for pod test-pod-80037404 to be ready +2025-06-04 20:45:17,743 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-80037404 to be ready after 60s +2025-06-04 20:45:17,744 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-80037404 to be ready +2025-06-04 20:45:17,826 - tests.orchestrator - INFO - Waiting for PVC one2one-116b8e13 to be bound +2025-06-04 20:45:19,959 - tests.orchestrator - INFO - PVC one2one-116b8e13 is bound +2025-06-04 20:45:19,959 - tests.orchestrator - INFO - Creating pod test-pod-725477e8 on PVC one2one-116b8e13 +2025-06-04 20:45:20,032 - tests.orchestrator - INFO - Created pod: test-pod-725477e8 using PVC: one2one-116b8e13 +2025-06-04 20:45:20,032 - tests.orchestrator - INFO - Waiting for pod test-pod-725477e8 to be ready +2025-06-04 20:46:20,115 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-725477e8 to be ready after 60s +2025-06-04 20:46:20,116 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-725477e8 to be ready +2025-06-04 20:46:20,195 - tests.orchestrator - INFO - Waiting for PVC one2one-4e963a5b to be bound +2025-06-04 20:46:22,332 - tests.orchestrator - INFO - PVC one2one-4e963a5b is bound +2025-06-04 20:46:22,332 - tests.orchestrator - INFO - Creating pod test-pod-b778d0e5 on PVC one2one-4e963a5b +2025-06-04 20:46:22,407 - tests.orchestrator - INFO - Created pod: test-pod-b778d0e5 using PVC: one2one-4e963a5b +2025-06-04 20:46:22,408 - tests.orchestrator - INFO - Waiting for pod test-pod-b778d0e5 to be ready +2025-06-04 20:47:24,472 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b778d0e5 to be ready after 60s +2025-06-04 20:47:24,472 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b778d0e5 to be ready +2025-06-04 20:47:24,472 - tests.orchestrator - WARNING - Failed to create enough PVC-pod pairs, only created 0 +2025-06-04 20:47:25,473 - tests.orchestrator - INFO - Selected operation: _delete_pod +2025-06-04 20:47:25,473 - tests.orchestrator - INFO - Deleting pod: test-pod-725477e8 from PVC: one2one-116b8e13 +2025-06-04 20:47:25,551 - tests.orchestrator - INFO - Waiting for pod test-pod-725477e8 to be deleted +2025-06-04 20:47:25,616 - tests.orchestrator - INFO - Pod test-pod-725477e8 has been deleted +2025-06-04 20:47:25,616 - tests.orchestrator - INFO - Deleted pod: test-pod-725477e8 +2025-06-04 20:47:26,616 - tests.orchestrator - INFO - Selected operation: _delete_pod +2025-06-04 20:47:26,616 - tests.orchestrator - INFO - Deleting pod: test-pod-f3b3a319 from PVC: one2one-43f0888a +2025-06-04 20:47:26,699 - tests.orchestrator - INFO - Waiting for pod test-pod-f3b3a319 to be deleted +2025-06-04 20:47:26,766 - tests.orchestrator - INFO - Pod test-pod-f3b3a319 has been deleted +2025-06-04 20:47:26,766 - tests.orchestrator - INFO - Deleted pod: test-pod-f3b3a319 +2025-06-04 20:47:27,767 - tests.orchestrator - INFO - Selected operation: _delete_pod +2025-06-04 20:47:27,767 - tests.orchestrator - INFO - Deleting pod: test-pod-80037404 from PVC: one2one-aac689b5 +2025-06-04 20:47:27,853 - tests.orchestrator - INFO - Waiting for pod test-pod-80037404 to be deleted +2025-06-04 20:47:27,919 - tests.orchestrator - INFO - Pod test-pod-80037404 has been deleted +2025-06-04 20:47:27,919 - tests.orchestrator - INFO - Deleted pod: test-pod-80037404 +2025-06-04 20:47:28,919 - tests.orchestrator - INFO - Selected operation: _verify_readwrite +2025-06-04 20:47:28,920 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test +2025-06-04 20:47:29,920 - tests.orchestrator - INFO - Selected operation: _attach_pod +2025-06-04 20:47:29,920 - tests.orchestrator - INFO - Creating pod test-pod-715d3677 on PVC one2one-116b8e13 +2025-06-04 20:47:30,008 - tests.orchestrator - INFO - Created pod: test-pod-715d3677 using PVC: one2one-116b8e13 +2025-06-04 20:47:30,008 - tests.orchestrator - INFO - Waiting for pod test-pod-715d3677 to be ready +2025-06-04 20:48:32,062 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-715d3677 to be ready after 60s +2025-06-04 20:48:32,062 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-715d3677 to be ready +2025-06-04 20:48:33,062 - tests.orchestrator - INFO - Test completed in 331.76 seconds +2025-06-04 20:48:33,063 - tests.orchestrator - INFO - Cleaning up resources... +2025-06-04 20:48:33,063 - tests.orchestrator - INFO - Deleting pod: test-pod-715d3677 from PVC: one2one-116b8e13 +2025-06-04 20:48:33,144 - tests.orchestrator - INFO - Waiting for pod test-pod-715d3677 to be deleted +2025-06-04 20:48:33,208 - tests.orchestrator - INFO - Pod test-pod-715d3677 has been deleted +2025-06-04 20:48:33,209 - tests.orchestrator - INFO - Deleted pod: test-pod-715d3677 +2025-06-04 20:48:33,209 - tests.orchestrator - INFO - Deleting pod: test-pod-b778d0e5 from PVC: one2one-4e963a5b +2025-06-04 20:48:33,289 - tests.orchestrator - INFO - Waiting for pod test-pod-b778d0e5 to be deleted +2025-06-04 20:48:33,356 - tests.orchestrator - INFO - Pod test-pod-b778d0e5 has been deleted +2025-06-04 20:48:33,356 - tests.orchestrator - INFO - Deleted pod: test-pod-b778d0e5 +2025-06-04 20:48:33,356 - tests.orchestrator - INFO - Deleting PVC: test-pvc-a8738d91 +2025-06-04 20:48:33,430 - tests.orchestrator - INFO - Waiting for PVC test-pvc-a8738d91 to be deleted +2025-06-04 20:48:33,497 - tests.orchestrator - INFO - PVC test-pvc-a8738d91 has been deleted +2025-06-04 20:48:33,497 - tests.orchestrator - INFO - Deleted PVC: test-pvc-a8738d91 +2025-06-04 20:48:33,497 - tests.orchestrator - INFO - Deleting PVC: test-pvc-ba881b7a +2025-06-04 20:48:33,570 - tests.orchestrator - INFO - Waiting for PVC test-pvc-ba881b7a to be deleted +2025-06-04 20:48:33,637 - tests.orchestrator - INFO - PVC test-pvc-ba881b7a has been deleted +2025-06-04 20:48:33,637 - tests.orchestrator - INFO - Deleted PVC: test-pvc-ba881b7a +2025-06-04 20:48:33,637 - tests.orchestrator - INFO - Deleting PVC: one2one-43f0888a +2025-06-04 20:48:33,712 - tests.orchestrator - INFO - Waiting for PVC one2one-43f0888a to be deleted +2025-06-04 20:48:33,779 - tests.orchestrator - INFO - PVC one2one-43f0888a has been deleted +2025-06-04 20:48:33,780 - tests.orchestrator - INFO - Deleted PVC: one2one-43f0888a +2025-06-04 20:48:33,780 - tests.orchestrator - INFO - Deleting PVC: one2one-aac689b5 +2025-06-04 20:48:33,858 - tests.orchestrator - INFO - Waiting for PVC one2one-aac689b5 to be deleted +2025-06-04 20:48:33,925 - tests.orchestrator - INFO - PVC one2one-aac689b5 has been deleted +2025-06-04 20:48:33,925 - tests.orchestrator - INFO - Deleted PVC: one2one-aac689b5 +2025-06-04 20:48:33,925 - tests.orchestrator - INFO - Deleting PVC: one2one-116b8e13 +2025-06-04 20:48:34,000 - tests.orchestrator - INFO - Waiting for PVC one2one-116b8e13 to be deleted +2025-06-04 20:48:34,067 - tests.orchestrator - INFO - PVC one2one-116b8e13 has been deleted +2025-06-04 20:48:34,067 - tests.orchestrator - INFO - Deleted PVC: one2one-116b8e13 +2025-06-04 20:48:34,067 - tests.orchestrator - INFO - Deleting PVC: one2one-4e963a5b +2025-06-04 20:48:34,143 - tests.orchestrator - INFO - Waiting for PVC one2one-4e963a5b to be deleted +2025-06-04 20:48:34,209 - tests.orchestrator - INFO - PVC one2one-4e963a5b has been deleted +2025-06-04 20:48:34,210 - tests.orchestrator - INFO - Deleted PVC: one2one-4e963a5b +2025-06-04 20:48:34,210 - tests.orchestrator - INFO - Cleanup completed +2025-06-04 20:48:34,210 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-04 20:48:34,210 - tests.orchestrator - INFO - --- Operations --- +2025-06-04 20:48:34,210 - tests.orchestrator - INFO - create_pvc: 2 succeeded, 0 failed (100.0%) +2025-06-04 20:48:34,210 - tests.orchestrator - INFO - attach_pod: 5 succeeded, 0 failed (100.0%) +2025-06-04 20:48:34,210 - tests.orchestrator - INFO - delete_pod: 5 succeeded, 0 failed (100.0%) +2025-06-04 20:48:34,210 - tests.orchestrator - INFO - delete_pvc: 6 succeeded, 0 failed (100.0%) +2025-06-04 20:48:34,210 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 0 failed (0.0%) +2025-06-04 20:48:34,210 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 0 failed (0.0%) +2025-06-04 20:48:34,210 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-04 20:48:34,210 - tests.orchestrator - INFO - shared_volume_rw: No runs +2025-06-04 20:48:34,210 - tests.orchestrator - INFO - many_to_one: No runs +2025-06-04 20:48:34,210 - tests.orchestrator - INFO - one_to_one: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-04 20:48:34,210 - tests.orchestrator - INFO - concurrent_pvc: No runs +2025-06-04 20:48:34,210 - tests.orchestrator - INFO - ========================================= +2025-06-04 20:48:34,211 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250604_204834.json +2025-06-04 20:48:34,258 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250604_204834_20250604_204834.json +2025-06-04 20:48:34,258 - __main__ - INFO - All tests completed successfully diff --git a/test/e2e/stress-scale-tests/logs/efs_tests.log b/test/e2e/stress-scale-tests/logs/efs_tests.log new file mode 100644 index 000000000..9ec4f5fa2 --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/efs_tests.log @@ -0,0 +1,2869 @@ +2025-06-03 21:00:10,432 - __main__ - INFO - Starting EFS CSI tests with configuration from config/test_config.yaml +2025-06-03 21:00:10,433 - __main__ - INFO - Verifying AWS credentials +2025-06-03 21:00:10,434 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-03 21:00:10,434 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-03 21:00:10,435 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-03 21:00:10,435 - __main__ - INFO - Running orchestrator stress test suite +2025-06-03 21:00:10,435 - __main__ - INFO - Starting orchestrator for 300 seconds +2025-06-03 21:00:10,435 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-03 21:00:11,173 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-03 21:00:11,448 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-03 21:00:11,448 - __main__ - INFO - Test duration overridden to 300 seconds +2025-06-03 21:00:11,448 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) +2025-06-03 21:00:11,448 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-03 21:00:11,865 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-03 21:00:11,865 - tests.orchestrator - INFO - Selected operation: _attach_pod +2025-06-03 21:00:11,865 - tests.orchestrator - INFO - No PVCs available, skipping pod attachment +2025-06-03 21:00:12,865 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario +2025-06-03 21:00:12,865 - tests.orchestrator - INFO - Running specific scenario: _scenario_many_to_one +2025-06-03 21:00:12,866 - tests.orchestrator - INFO - Running scenario: Many pods to one PVC +2025-06-03 21:00:12,969 - tests.orchestrator - INFO - Waiting for PVC many2one-8ccf2b6f to be bound +2025-06-03 21:00:15,141 - tests.orchestrator - INFO - PVC many2one-8ccf2b6f is bound +2025-06-03 21:00:15,141 - tests.orchestrator - INFO - Creating 4 pods for the same PVC +2025-06-03 21:00:15,141 - tests.orchestrator - INFO - Creating pod test-pod-5c9bbadf on PVC many2one-8ccf2b6f +2025-06-03 21:00:15,239 - tests.orchestrator - INFO - Created pod: test-pod-5c9bbadf using PVC: many2one-8ccf2b6f +2025-06-03 21:00:15,239 - tests.orchestrator - INFO - Waiting for pod test-pod-5c9bbadf to be ready +2025-06-03 21:01:15,739 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-5c9bbadf to be ready after 60s +2025-06-03 21:01:15,740 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-5c9bbadf to be ready +2025-06-03 21:01:15,740 - tests.orchestrator - INFO - Creating pod test-pod-a795ee8f on PVC many2one-8ccf2b6f +2025-06-03 21:01:15,842 - tests.orchestrator - INFO - Created pod: test-pod-a795ee8f using PVC: many2one-8ccf2b6f +2025-06-03 21:01:15,842 - tests.orchestrator - INFO - Waiting for pod test-pod-a795ee8f to be ready +2025-06-03 21:02:16,279 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a795ee8f to be ready after 60s +2025-06-03 21:02:16,279 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a795ee8f to be ready +2025-06-03 21:02:16,279 - tests.orchestrator - INFO - Creating pod test-pod-bb687a78 on PVC many2one-8ccf2b6f +2025-06-03 21:02:16,379 - tests.orchestrator - INFO - Created pod: test-pod-bb687a78 using PVC: many2one-8ccf2b6f +2025-06-03 21:02:16,379 - tests.orchestrator - INFO - Waiting for pod test-pod-bb687a78 to be ready +2025-06-03 21:03:16,816 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-bb687a78 to be ready after 60s +2025-06-03 21:03:16,816 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-bb687a78 to be ready +2025-06-03 21:03:16,817 - tests.orchestrator - INFO - Creating pod test-pod-da23ad5a on PVC many2one-8ccf2b6f +2025-06-03 21:03:16,917 - tests.orchestrator - INFO - Created pod: test-pod-da23ad5a using PVC: many2one-8ccf2b6f +2025-06-03 21:03:16,917 - tests.orchestrator - INFO - Waiting for pod test-pod-da23ad5a to be ready +2025-06-03 21:04:17,450 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-da23ad5a to be ready after 60s +2025-06-03 21:04:17,450 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-da23ad5a to be ready +2025-06-03 21:04:17,450 - tests.orchestrator - WARNING - Failed to create enough pods for scenario, only created 0 +2025-06-03 21:04:18,450 - tests.orchestrator - INFO - Selected operation: _verify_readwrite +2025-06-03 21:04:18,451 - tests.orchestrator - INFO - Testing read/write between pods test-pod-a795ee8f and test-pod-5c9bbadf sharing PVC many2one-8ccf2b6f +2025-06-03 21:04:18,534 - tests.orchestrator - ERROR - Failed in read/write verification: (400) +Reason: Bad Request +HTTP response headers: HTTPHeaderDict({'Audit-Id': 'e5f54d7e-0b5d-44ba-9610-9f446b015a24', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Tue, 03 Jun 2025 21:04:18 GMT', 'Content-Length': '166'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-a795ee8f does not have a host assigned","reason":"BadRequest","code":400} + + +2025-06-03 21:04:19,534 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario +2025-06-03 21:04:19,534 - tests.orchestrator - INFO - Running specific scenario: _scenario_one_to_one +2025-06-03 21:04:19,534 - tests.orchestrator - INFO - Running scenario: One pod per PVC +2025-06-03 21:04:19,534 - tests.orchestrator - INFO - Creating 5 PVC-pod pairs +2025-06-03 21:04:19,631 - tests.orchestrator - INFO - Waiting for PVC one2one-c1de2935 to be bound +2025-06-03 21:04:21,797 - tests.orchestrator - INFO - PVC one2one-c1de2935 is bound +2025-06-03 21:04:21,797 - tests.orchestrator - INFO - Creating pod test-pod-c8cbcc77 on PVC one2one-c1de2935 +2025-06-03 21:04:21,894 - tests.orchestrator - INFO - Created pod: test-pod-c8cbcc77 using PVC: one2one-c1de2935 +2025-06-03 21:04:21,894 - tests.orchestrator - INFO - Waiting for pod test-pod-c8cbcc77 to be ready +2025-06-03 21:05:22,524 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-c8cbcc77 to be ready after 60s +2025-06-03 21:05:22,524 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-c8cbcc77 to be ready +2025-06-03 21:05:22,615 - tests.orchestrator - INFO - Waiting for PVC one2one-2f5447c5 to be bound +2025-06-03 21:05:24,783 - tests.orchestrator - INFO - PVC one2one-2f5447c5 is bound +2025-06-03 21:05:24,783 - tests.orchestrator - INFO - Creating pod test-pod-9c65d0a8 on PVC one2one-2f5447c5 +2025-06-03 21:05:24,879 - tests.orchestrator - INFO - Created pod: test-pod-9c65d0a8 using PVC: one2one-2f5447c5 +2025-06-03 21:05:24,879 - tests.orchestrator - INFO - Waiting for pod test-pod-9c65d0a8 to be ready +2025-06-03 21:06:25,353 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9c65d0a8 to be ready after 60s +2025-06-03 21:06:25,353 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9c65d0a8 to be ready +2025-06-03 21:06:25,454 - tests.orchestrator - INFO - Waiting for PVC one2one-65a70cda to be bound +2025-06-03 21:06:27,620 - tests.orchestrator - INFO - PVC one2one-65a70cda is bound +2025-06-03 21:06:27,621 - tests.orchestrator - INFO - Creating pod test-pod-396a98c7 on PVC one2one-65a70cda +2025-06-03 21:06:27,709 - tests.orchestrator - INFO - Created pod: test-pod-396a98c7 using PVC: one2one-65a70cda +2025-06-03 21:06:27,709 - tests.orchestrator - INFO - Waiting for pod test-pod-396a98c7 to be ready +2025-06-03 21:07:28,272 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-396a98c7 to be ready after 60s +2025-06-03 21:07:28,272 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-396a98c7 to be ready +2025-06-03 21:07:28,365 - tests.orchestrator - INFO - Waiting for PVC one2one-ac0c0034 to be bound +2025-06-03 21:07:30,531 - tests.orchestrator - INFO - PVC one2one-ac0c0034 is bound +2025-06-03 21:07:30,531 - tests.orchestrator - INFO - Creating pod test-pod-707ffe1e on PVC one2one-ac0c0034 +2025-06-03 21:07:30,620 - tests.orchestrator - INFO - Created pod: test-pod-707ffe1e using PVC: one2one-ac0c0034 +2025-06-03 21:07:30,620 - tests.orchestrator - INFO - Waiting for pod test-pod-707ffe1e to be ready +2025-06-03 21:08:31,077 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-707ffe1e to be ready after 60s +2025-06-03 21:08:31,077 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-707ffe1e to be ready +2025-06-03 21:08:31,174 - tests.orchestrator - INFO - Waiting for PVC one2one-6960322d to be bound +2025-06-03 21:08:33,340 - tests.orchestrator - INFO - PVC one2one-6960322d is bound +2025-06-03 21:08:33,340 - tests.orchestrator - INFO - Creating pod test-pod-db166953 on PVC one2one-6960322d +2025-06-03 21:08:33,433 - tests.orchestrator - INFO - Created pod: test-pod-db166953 using PVC: one2one-6960322d +2025-06-03 21:08:33,433 - tests.orchestrator - INFO - Waiting for pod test-pod-db166953 to be ready +2025-06-03 21:09:34,745 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-db166953 to be ready after 60s +2025-06-03 21:09:34,745 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-db166953 to be ready +2025-06-03 21:09:34,746 - tests.orchestrator - WARNING - Failed to create enough PVC-pod pairs, only created 0 +2025-06-03 21:09:35,746 - tests.orchestrator - INFO - Test completed in 564.30 seconds +2025-06-03 21:09:35,746 - tests.orchestrator - INFO - Cleaning up resources... +2025-06-03 21:09:35,746 - tests.orchestrator - INFO - Deleting pod: test-pod-5c9bbadf from PVC: many2one-8ccf2b6f +2025-06-03 21:09:35,842 - tests.orchestrator - INFO - Waiting for pod test-pod-5c9bbadf to be deleted +2025-06-03 21:09:35,926 - tests.orchestrator - INFO - Pod test-pod-5c9bbadf has been deleted +2025-06-03 21:09:35,926 - tests.orchestrator - INFO - Deleted pod: test-pod-5c9bbadf +2025-06-03 21:09:35,926 - tests.orchestrator - INFO - Deleting pod: test-pod-a795ee8f from PVC: many2one-8ccf2b6f +2025-06-03 21:09:36,025 - tests.orchestrator - INFO - Waiting for pod test-pod-a795ee8f to be deleted +2025-06-03 21:09:36,106 - tests.orchestrator - INFO - Pod test-pod-a795ee8f has been deleted +2025-06-03 21:09:36,106 - tests.orchestrator - INFO - Deleted pod: test-pod-a795ee8f +2025-06-03 21:09:36,106 - tests.orchestrator - INFO - Deleting pod: test-pod-bb687a78 from PVC: many2one-8ccf2b6f +2025-06-03 21:09:36,202 - tests.orchestrator - INFO - Waiting for pod test-pod-bb687a78 to be deleted +2025-06-03 21:09:36,284 - tests.orchestrator - INFO - Pod test-pod-bb687a78 has been deleted +2025-06-03 21:09:36,284 - tests.orchestrator - INFO - Deleted pod: test-pod-bb687a78 +2025-06-03 21:09:36,284 - tests.orchestrator - INFO - Deleting pod: test-pod-da23ad5a from PVC: many2one-8ccf2b6f +2025-06-03 21:09:36,386 - tests.orchestrator - INFO - Waiting for pod test-pod-da23ad5a to be deleted +2025-06-03 21:09:36,466 - tests.orchestrator - INFO - Pod test-pod-da23ad5a has been deleted +2025-06-03 21:09:36,466 - tests.orchestrator - INFO - Deleted pod: test-pod-da23ad5a +2025-06-03 21:09:36,466 - tests.orchestrator - INFO - Deleting pod: test-pod-c8cbcc77 from PVC: one2one-c1de2935 +2025-06-03 21:09:36,565 - tests.orchestrator - INFO - Waiting for pod test-pod-c8cbcc77 to be deleted +2025-06-03 21:09:36,651 - tests.orchestrator - INFO - Pod test-pod-c8cbcc77 has been deleted +2025-06-03 21:09:36,652 - tests.orchestrator - INFO - Deleted pod: test-pod-c8cbcc77 +2025-06-03 21:09:36,652 - tests.orchestrator - INFO - Deleting pod: test-pod-9c65d0a8 from PVC: one2one-2f5447c5 +2025-06-03 21:09:36,747 - tests.orchestrator - INFO - Waiting for pod test-pod-9c65d0a8 to be deleted +2025-06-03 21:09:36,832 - tests.orchestrator - INFO - Pod test-pod-9c65d0a8 has been deleted +2025-06-03 21:09:36,833 - tests.orchestrator - INFO - Deleted pod: test-pod-9c65d0a8 +2025-06-03 21:09:36,833 - tests.orchestrator - INFO - Deleting pod: test-pod-396a98c7 from PVC: one2one-65a70cda +2025-06-03 21:09:36,929 - tests.orchestrator - INFO - Waiting for pod test-pod-396a98c7 to be deleted +2025-06-03 21:09:37,009 - tests.orchestrator - INFO - Pod test-pod-396a98c7 has been deleted +2025-06-03 21:09:37,009 - tests.orchestrator - INFO - Deleted pod: test-pod-396a98c7 +2025-06-03 21:09:37,009 - tests.orchestrator - INFO - Deleting pod: test-pod-707ffe1e from PVC: one2one-ac0c0034 +2025-06-03 21:09:37,104 - tests.orchestrator - INFO - Waiting for pod test-pod-707ffe1e to be deleted +2025-06-03 21:09:37,187 - tests.orchestrator - INFO - Pod test-pod-707ffe1e has been deleted +2025-06-03 21:09:37,187 - tests.orchestrator - INFO - Deleted pod: test-pod-707ffe1e +2025-06-03 21:09:37,187 - tests.orchestrator - INFO - Deleting pod: test-pod-db166953 from PVC: one2one-6960322d +2025-06-03 21:09:37,284 - tests.orchestrator - INFO - Waiting for pod test-pod-db166953 to be deleted +2025-06-03 21:09:37,368 - tests.orchestrator - INFO - Pod test-pod-db166953 has been deleted +2025-06-03 21:09:37,368 - tests.orchestrator - INFO - Deleted pod: test-pod-db166953 +2025-06-03 21:09:37,368 - tests.orchestrator - INFO - Deleting PVC: many2one-8ccf2b6f +2025-06-03 21:09:37,456 - tests.orchestrator - INFO - Waiting for PVC many2one-8ccf2b6f to be deleted +2025-06-03 21:09:37,538 - tests.orchestrator - INFO - PVC many2one-8ccf2b6f has been deleted +2025-06-03 21:09:37,539 - tests.orchestrator - INFO - Deleted PVC: many2one-8ccf2b6f +2025-06-03 21:09:37,539 - tests.orchestrator - INFO - Deleting PVC: one2one-c1de2935 +2025-06-03 21:09:37,627 - tests.orchestrator - INFO - Waiting for PVC one2one-c1de2935 to be deleted +2025-06-03 21:09:37,711 - tests.orchestrator - INFO - PVC one2one-c1de2935 has been deleted +2025-06-03 21:09:37,711 - tests.orchestrator - INFO - Deleted PVC: one2one-c1de2935 +2025-06-03 21:09:37,711 - tests.orchestrator - INFO - Deleting PVC: one2one-2f5447c5 +2025-06-03 21:09:37,802 - tests.orchestrator - INFO - Waiting for PVC one2one-2f5447c5 to be deleted +2025-06-03 21:09:37,885 - tests.orchestrator - INFO - PVC one2one-2f5447c5 has been deleted +2025-06-03 21:09:37,886 - tests.orchestrator - INFO - Deleted PVC: one2one-2f5447c5 +2025-06-03 21:09:37,886 - tests.orchestrator - INFO - Deleting PVC: one2one-65a70cda +2025-06-03 21:09:37,973 - tests.orchestrator - INFO - Waiting for PVC one2one-65a70cda to be deleted +2025-06-03 21:09:38,059 - tests.orchestrator - INFO - PVC one2one-65a70cda has been deleted +2025-06-03 21:09:38,059 - tests.orchestrator - INFO - Deleted PVC: one2one-65a70cda +2025-06-03 21:09:38,059 - tests.orchestrator - INFO - Deleting PVC: one2one-ac0c0034 +2025-06-03 21:09:38,148 - tests.orchestrator - INFO - Waiting for PVC one2one-ac0c0034 to be deleted +2025-06-03 21:09:38,230 - tests.orchestrator - INFO - PVC one2one-ac0c0034 has been deleted +2025-06-03 21:09:38,231 - tests.orchestrator - INFO - Deleted PVC: one2one-ac0c0034 +2025-06-03 21:09:38,231 - tests.orchestrator - INFO - Deleting PVC: one2one-6960322d +2025-06-03 21:09:38,318 - tests.orchestrator - INFO - Waiting for PVC one2one-6960322d to be deleted +2025-06-03 21:09:38,401 - tests.orchestrator - INFO - PVC one2one-6960322d has been deleted +2025-06-03 21:09:38,401 - tests.orchestrator - INFO - Deleted PVC: one2one-6960322d +2025-06-03 21:09:38,401 - tests.orchestrator - INFO - Cleanup completed +2025-06-03 21:09:38,401 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-03 21:09:38,401 - tests.orchestrator - INFO - --- Operations --- +2025-06-03 21:09:38,401 - tests.orchestrator - INFO - create_pvc: 0 succeeded, 0 failed (0.0%) +2025-06-03 21:09:38,402 - tests.orchestrator - INFO - attach_pod: 9 succeeded, 0 failed (100.0%) +2025-06-03 21:09:38,402 - tests.orchestrator - INFO - delete_pod: 9 succeeded, 0 failed (100.0%) +2025-06-03 21:09:38,402 - tests.orchestrator - INFO - delete_pvc: 6 succeeded, 0 failed (100.0%) +2025-06-03 21:09:38,402 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) +2025-06-03 21:09:38,402 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) +2025-06-03 21:09:38,402 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-03 21:09:38,402 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-03 21:09:38,402 - tests.orchestrator - INFO - many_to_one: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-03 21:09:38,402 - tests.orchestrator - INFO - one_to_one: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-03 21:09:38,402 - tests.orchestrator - INFO - concurrent_pvc: No runs +2025-06-03 21:09:38,402 - tests.orchestrator - INFO - ========================================= +2025-06-03 21:09:38,402 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250603_210938.json +2025-06-03 21:09:38,449 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250603_210938_20250603_210938.json +2025-06-03 21:09:38,492 - __main__ - INFO - Summary report generated: reports/general/efs_tests_20250603_210938_20250603_210938_summary.txt +2025-06-03 21:09:38,492 - __main__ - INFO - All tests completed successfully +2025-06-04 21:12:54,295 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-04 21:12:54,295 - __main__ - INFO - Verifying AWS credentials +2025-06-04 21:12:54,296 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-04 21:12:54,297 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-04 21:12:54,297 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-04 21:12:54,298 - __main__ - INFO - Running orchestrator stress test suite +2025-06-04 21:12:54,298 - __main__ - INFO - Starting orchestrator for 300 seconds +2025-06-04 21:12:54,298 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-04 21:12:55,097 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-04 21:12:55,314 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-04 21:12:55,314 - __main__ - INFO - Test duration overridden to 300 seconds +2025-06-04 21:12:55,314 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) +2025-06-04 21:12:55,314 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-04 21:12:55,503 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-04 21:12:55,504 - tests.orchestrator - INFO - Selected operation: _delete_pod +2025-06-04 21:12:55,504 - tests.orchestrator - INFO - No pods to delete +2025-06-04 21:12:56,504 - tests.orchestrator - INFO - Selected operation: _verify_readwrite +2025-06-04 21:12:56,504 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test +2025-06-04 21:12:57,504 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario +2025-06-04 21:12:57,504 - tests.orchestrator - INFO - Running specific scenario: _scenario_one_to_one +2025-06-04 21:12:57,504 - tests.orchestrator - INFO - Running scenario: One pod per PVC +2025-06-04 21:12:57,505 - tests.orchestrator - INFO - Creating 4 PVC-pod pairs +2025-06-04 21:12:57,584 - tests.orchestrator - INFO - Waiting for PVC one2one-fa7d38c4 to be bound +2025-06-04 21:12:59,720 - tests.orchestrator - INFO - PVC one2one-fa7d38c4 is bound +2025-06-04 21:12:59,721 - tests.orchestrator - INFO - Creating pod test-pod-f3bd2110 on PVC one2one-fa7d38c4 +2025-06-04 21:12:59,822 - tests.orchestrator - INFO - Created pod: test-pod-f3bd2110 using PVC: one2one-fa7d38c4 +2025-06-04 21:12:59,822 - tests.orchestrator - INFO - Waiting for pod test-pod-f3bd2110 to be ready +2025-06-04 21:13:46,839 - tests.orchestrator - INFO - Test interrupted by user +2025-06-04 21:13:46,839 - tests.orchestrator - INFO - Test completed in 51.52 seconds +2025-06-04 21:13:46,839 - tests.orchestrator - INFO - Cleaning up resources... +2025-06-04 21:13:46,839 - tests.orchestrator - INFO - Deleting pod: test-pod-f3bd2110 from PVC: one2one-fa7d38c4 +2025-06-04 21:13:46,924 - tests.orchestrator - INFO - Waiting for pod test-pod-f3bd2110 to be deleted +2025-06-04 21:13:46,989 - tests.orchestrator - INFO - Pod test-pod-f3bd2110 has been deleted +2025-06-04 21:13:46,989 - tests.orchestrator - INFO - Deleted pod: test-pod-f3bd2110 +2025-06-04 21:13:46,989 - tests.orchestrator - INFO - Deleting PVC: one2one-fa7d38c4 +2025-06-04 21:13:47,063 - tests.orchestrator - INFO - Waiting for PVC one2one-fa7d38c4 to be deleted +2025-06-04 21:13:47,130 - tests.orchestrator - INFO - PVC one2one-fa7d38c4 has been deleted +2025-06-04 21:13:47,130 - tests.orchestrator - INFO - Deleted PVC: one2one-fa7d38c4 +2025-06-04 21:13:47,130 - tests.orchestrator - INFO - Cleanup completed +2025-06-04 21:13:47,131 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-04 21:13:47,131 - tests.orchestrator - INFO - --- Operations --- +2025-06-04 21:13:47,131 - tests.orchestrator - INFO - create_pvc: 0 succeeded, 0 failed (0.0%) +2025-06-04 21:13:47,131 - tests.orchestrator - INFO - attach_pod: 1 succeeded, 0 failed (100.0%) +2025-06-04 21:13:47,131 - tests.orchestrator - INFO - delete_pod: 1 succeeded, 0 failed (100.0%) +2025-06-04 21:13:47,131 - tests.orchestrator - INFO - delete_pvc: 1 succeeded, 0 failed (100.0%) +2025-06-04 21:13:47,131 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 0 failed (0.0%) +2025-06-04 21:13:47,131 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 0 failed (0.0%) +2025-06-04 21:13:47,131 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-04 21:13:47,131 - tests.orchestrator - INFO - shared_volume_rw: No runs +2025-06-04 21:13:47,131 - tests.orchestrator - INFO - many_to_one: No runs +2025-06-04 21:13:47,131 - tests.orchestrator - INFO - one_to_one: 0 succeeded, 0 failed out of 1 runs (0.0%) +2025-06-04 21:13:47,131 - tests.orchestrator - INFO - concurrent_pvc: No runs +2025-06-04 21:13:47,131 - tests.orchestrator - INFO - ========================================= +2025-06-04 21:13:47,132 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250604_211347.json +2025-06-04 21:13:47,178 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250604_211347_20250604_211347.json +2025-06-04 21:13:47,178 - __main__ - INFO - All tests completed successfully +2025-06-04 21:13:52,325 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-04 21:13:52,325 - __main__ - INFO - Verifying AWS credentials +2025-06-04 21:13:52,326 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-04 21:13:52,327 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-04 21:13:52,327 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-04 21:13:52,328 - __main__ - INFO - Running orchestrator stress test suite +2025-06-04 21:13:52,328 - __main__ - INFO - Starting orchestrator for 120 seconds +2025-06-04 21:13:52,328 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-04 21:13:53,143 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-04 21:13:53,342 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-04 21:13:53,342 - __main__ - INFO - Test duration overridden to 120 seconds +2025-06-04 21:13:53,342 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 10/s) +2025-06-04 21:13:53,342 - tests.orchestrator - INFO - Starting orchestrator test for 120 seconds +2025-06-04 21:13:53,535 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-04 21:13:53,536 - tests.orchestrator - INFO - Selected operation: _verify_readwrite +2025-06-04 21:13:53,536 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test +2025-06-04 21:13:54,536 - tests.orchestrator - INFO - Selected operation: _delete_pvc +2025-06-04 21:13:54,536 - tests.orchestrator - INFO - No PVCs to delete +2025-06-04 21:13:55,536 - tests.orchestrator - INFO - Selected operation: _verify_readwrite +2025-06-04 21:13:55,537 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test +2025-06-04 21:13:56,537 - tests.orchestrator - INFO - Selected operation: _delete_pod +2025-06-04 21:13:56,537 - tests.orchestrator - INFO - No pods to delete +2025-06-04 21:13:57,537 - tests.orchestrator - INFO - Selected operation: _delete_pod +2025-06-04 21:13:57,537 - tests.orchestrator - INFO - No pods to delete +2025-06-04 21:13:58,538 - tests.orchestrator - INFO - Selected operation: _verify_readwrite +2025-06-04 21:13:58,538 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test +2025-06-04 21:13:59,538 - tests.orchestrator - INFO - Selected operation: _delete_pod +2025-06-04 21:13:59,538 - tests.orchestrator - INFO - No pods to delete +2025-06-04 21:14:00,538 - tests.orchestrator - INFO - Selected operation: _attach_pod +2025-06-04 21:14:00,539 - tests.orchestrator - INFO - No PVCs available, skipping pod attachment +2025-06-04 21:14:01,539 - tests.orchestrator - INFO - Selected operation: _delete_pod +2025-06-04 21:14:01,540 - tests.orchestrator - INFO - No pods to delete +2025-06-04 21:14:02,540 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario +2025-06-04 21:14:02,541 - tests.orchestrator - INFO - Running specific scenario: _scenario_concurrent_pvc +2025-06-04 21:14:02,541 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations +2025-06-04 21:14:02,541 - tests.orchestrator - INFO - Creating 4 PVCs in quick succession +2025-06-04 21:14:02,617 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-68e38df4 +2025-06-04 21:14:02,617 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-68e38df4 to be bound +2025-06-04 21:14:04,753 - tests.orchestrator - INFO - PVC concurrent-pvc-68e38df4 is bound +2025-06-04 21:14:04,825 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-90dd2e0a +2025-06-04 21:14:04,825 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-90dd2e0a to be bound +2025-06-04 21:14:06,959 - tests.orchestrator - INFO - PVC concurrent-pvc-90dd2e0a is bound +2025-06-04 21:14:07,028 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-09b78e7f +2025-06-04 21:14:07,028 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-09b78e7f to be bound +2025-06-04 21:14:09,163 - tests.orchestrator - INFO - PVC concurrent-pvc-09b78e7f is bound +2025-06-04 21:14:09,232 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-51c7b76a +2025-06-04 21:14:09,232 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-51c7b76a to be bound +2025-06-04 21:14:11,368 - tests.orchestrator - INFO - PVC concurrent-pvc-51c7b76a is bound +2025-06-04 21:14:11,368 - tests.orchestrator - INFO - Successfully created 4 PVCs +2025-06-04 21:14:11,368 - tests.orchestrator - INFO - Creating pod test-pod-60af97a5 on PVC concurrent-pvc-90dd2e0a +2025-06-04 21:14:11,458 - tests.orchestrator - INFO - Created pod: test-pod-60af97a5 using PVC: concurrent-pvc-90dd2e0a +2025-06-04 21:14:11,458 - tests.orchestrator - INFO - Waiting for pod test-pod-60af97a5 to be ready +2025-06-04 21:14:56,603 - tests.orchestrator - INFO - Test interrupted by user +2025-06-04 21:14:56,603 - tests.orchestrator - INFO - Test completed in 63.26 seconds +2025-06-04 21:14:56,603 - tests.orchestrator - INFO - Cleaning up resources... +2025-06-04 21:14:56,603 - tests.orchestrator - INFO - Deleting pod: test-pod-60af97a5 from PVC: concurrent-pvc-90dd2e0a +2025-06-04 21:14:56,689 - tests.orchestrator - INFO - Waiting for pod test-pod-60af97a5 to be deleted +2025-06-04 21:14:56,754 - tests.orchestrator - INFO - Pod test-pod-60af97a5 has been deleted +2025-06-04 21:14:56,754 - tests.orchestrator - INFO - Deleted pod: test-pod-60af97a5 +2025-06-04 21:14:56,754 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-68e38df4 +2025-06-04 21:14:56,838 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-68e38df4 to be deleted +2025-06-04 21:14:56,902 - tests.orchestrator - INFO - PVC concurrent-pvc-68e38df4 has been deleted +2025-06-04 21:14:56,903 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-68e38df4 +2025-06-04 21:14:56,903 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-90dd2e0a +2025-06-04 21:14:56,977 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-90dd2e0a to be deleted +2025-06-04 21:14:57,041 - tests.orchestrator - INFO - PVC concurrent-pvc-90dd2e0a has been deleted +2025-06-04 21:14:57,041 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-90dd2e0a +2025-06-04 21:14:57,041 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-09b78e7f +2025-06-04 21:14:57,116 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-09b78e7f to be deleted +2025-06-04 21:14:57,181 - tests.orchestrator - INFO - PVC concurrent-pvc-09b78e7f has been deleted +2025-06-04 21:14:57,181 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-09b78e7f +2025-06-04 21:14:57,181 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-51c7b76a +2025-06-04 21:14:57,257 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-51c7b76a to be deleted +2025-06-04 21:14:57,322 - tests.orchestrator - INFO - PVC concurrent-pvc-51c7b76a has been deleted +2025-06-04 21:14:57,322 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-51c7b76a +2025-06-04 21:14:57,322 - tests.orchestrator - INFO - Cleanup completed +2025-06-04 21:14:57,322 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-04 21:14:57,322 - tests.orchestrator - INFO - --- Operations --- +2025-06-04 21:14:57,322 - tests.orchestrator - INFO - create_pvc: 4 succeeded, 0 failed (100.0%) +2025-06-04 21:14:57,322 - tests.orchestrator - INFO - attach_pod: 1 succeeded, 0 failed (100.0%) +2025-06-04 21:14:57,322 - tests.orchestrator - INFO - delete_pod: 1 succeeded, 0 failed (100.0%) +2025-06-04 21:14:57,322 - tests.orchestrator - INFO - delete_pvc: 4 succeeded, 0 failed (100.0%) +2025-06-04 21:14:57,322 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 0 failed (0.0%) +2025-06-04 21:14:57,322 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 0 failed (0.0%) +2025-06-04 21:14:57,322 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-04 21:14:57,323 - tests.orchestrator - INFO - shared_volume_rw: No runs +2025-06-04 21:14:57,323 - tests.orchestrator - INFO - many_to_one: No runs +2025-06-04 21:14:57,323 - tests.orchestrator - INFO - one_to_one: No runs +2025-06-04 21:14:57,323 - tests.orchestrator - INFO - concurrent_pvc: 0 succeeded, 0 failed out of 1 runs (0.0%) +2025-06-04 21:14:57,323 - tests.orchestrator - INFO - ========================================= +2025-06-04 21:14:57,323 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250604_211457.json +2025-06-04 21:14:57,371 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250604_211457_20250604_211457.json +2025-06-04 21:14:57,371 - __main__ - INFO - All tests completed successfully +2025-06-05 13:28:01,933 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-05 13:28:01,933 - __main__ - INFO - Verifying AWS credentials +2025-06-05 13:28:01,934 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 13:28:01,935 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 13:28:01,935 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 13:28:01,935 - __main__ - INFO - Running orchestrator stress test suite +2025-06-05 13:28:01,935 - __main__ - INFO - Starting orchestrator for 400 seconds +2025-06-05 13:28:01,935 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-05 13:28:02,670 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 13:28:02,885 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 13:28:02,886 - __main__ - INFO - Test duration overridden to 400 seconds +2025-06-05 13:28:02,886 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 20/s) +2025-06-05 13:28:02,886 - tests.orchestrator - INFO - Starting orchestrator test for 400 seconds +2025-06-05 13:28:03,078 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 13:28:03,079 - tests.orchestrator - INFO - Selected operation: _create_pvc +2025-06-05 13:28:03,079 - tests.orchestrator - INFO - Creating PVC: test-pvc-87f8dca2 +2025-06-05 13:28:03,159 - tests.orchestrator - INFO - Created PVC: test-pvc-87f8dca2 +2025-06-05 13:28:03,159 - tests.orchestrator - INFO - Waiting for PVC test-pvc-87f8dca2 to be bound +2025-06-05 13:28:05,290 - tests.orchestrator - INFO - PVC test-pvc-87f8dca2 is bound +2025-06-05 13:28:06,290 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario +2025-06-05 13:28:06,290 - tests.orchestrator - INFO - Running specific scenario: _scenario_concurrent_pvc +2025-06-05 13:28:06,290 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations +2025-06-05 13:28:06,290 - tests.orchestrator - INFO - Creating 6 PVCs in quick succession +2025-06-05 13:28:06,363 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-27248b37 +2025-06-05 13:28:06,363 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-27248b37 to be bound +2025-06-05 13:28:08,495 - tests.orchestrator - INFO - PVC concurrent-pvc-27248b37 is bound +2025-06-05 13:28:08,561 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-0be57a62 +2025-06-05 13:28:08,562 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-0be57a62 to be bound +2025-06-05 13:28:10,696 - tests.orchestrator - INFO - PVC concurrent-pvc-0be57a62 is bound +2025-06-05 13:28:10,764 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-d46ca8c1 +2025-06-05 13:28:10,764 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-d46ca8c1 to be bound +2025-06-05 13:28:12,902 - tests.orchestrator - INFO - PVC concurrent-pvc-d46ca8c1 is bound +2025-06-05 13:28:12,972 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-4b94a252 +2025-06-05 13:28:12,973 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-4b94a252 to be bound +2025-06-05 13:28:15,113 - tests.orchestrator - INFO - PVC concurrent-pvc-4b94a252 is bound +2025-06-05 13:28:15,180 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-737583a5 +2025-06-05 13:28:15,181 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-737583a5 to be bound +2025-06-05 13:28:17,317 - tests.orchestrator - INFO - PVC concurrent-pvc-737583a5 is bound +2025-06-05 13:28:17,385 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-b6ec6167 +2025-06-05 13:28:17,385 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-b6ec6167 to be bound +2025-06-05 13:28:19,520 - tests.orchestrator - INFO - PVC concurrent-pvc-b6ec6167 is bound +2025-06-05 13:28:19,520 - tests.orchestrator - INFO - Successfully created 6 PVCs +2025-06-05 13:28:19,520 - tests.orchestrator - INFO - Creating pod test-pod-05839583 on PVC concurrent-pvc-4b94a252 +2025-06-05 13:28:19,605 - tests.orchestrator - INFO - Created pod: test-pod-05839583 using PVC: concurrent-pvc-4b94a252 +2025-06-05 13:28:19,605 - tests.orchestrator - INFO - Waiting for pod test-pod-05839583 to be ready +2025-06-05 13:29:21,656 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-05839583 to be ready after 60s +2025-06-05 13:29:21,657 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-05839583 to be ready +2025-06-05 13:29:21,657 - tests.orchestrator - INFO - Creating pod test-pod-c11d7609 on PVC concurrent-pvc-b6ec6167 +2025-06-05 13:29:21,746 - tests.orchestrator - INFO - Created pod: test-pod-c11d7609 using PVC: concurrent-pvc-b6ec6167 +2025-06-05 13:29:21,746 - tests.orchestrator - INFO - Waiting for pod test-pod-c11d7609 to be ready +2025-06-05 13:30:23,813 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-c11d7609 to be ready after 60s +2025-06-05 13:30:23,813 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-c11d7609 to be ready +2025-06-05 13:30:23,813 - tests.orchestrator - INFO - Creating pod test-pod-f3958de4 on PVC concurrent-pvc-27248b37 +2025-06-05 13:30:23,897 - tests.orchestrator - INFO - Created pod: test-pod-f3958de4 using PVC: concurrent-pvc-27248b37 +2025-06-05 13:30:23,897 - tests.orchestrator - INFO - Waiting for pod test-pod-f3958de4 to be ready +2025-06-05 13:31:23,958 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-f3958de4 to be ready after 60s +2025-06-05 13:31:23,958 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-f3958de4 to be ready +2025-06-05 13:31:23,958 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-0be57a62 +2025-06-05 13:31:24,029 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-0be57a62 to be deleted +2025-06-05 13:31:24,096 - tests.orchestrator - INFO - PVC concurrent-pvc-0be57a62 has been deleted +2025-06-05 13:31:24,096 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-0be57a62 +2025-06-05 13:31:24,096 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-4b94a252 +2025-06-05 13:31:24,096 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-4b94a252 +2025-06-05 13:31:24,096 - tests.orchestrator - INFO - Deleting pod: test-pod-05839583 from PVC: concurrent-pvc-4b94a252 +2025-06-05 13:31:24,177 - tests.orchestrator - INFO - Waiting for pod test-pod-05839583 to be deleted +2025-06-05 13:31:24,241 - tests.orchestrator - INFO - Pod test-pod-05839583 has been deleted +2025-06-05 13:31:24,241 - tests.orchestrator - INFO - Deleted pod: test-pod-05839583 +2025-06-05 13:31:24,314 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-4b94a252 to be deleted +2025-06-05 13:31:24,380 - tests.orchestrator - INFO - PVC concurrent-pvc-4b94a252 has been deleted +2025-06-05 13:31:24,380 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-4b94a252 +2025-06-05 13:31:24,381 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-27248b37 +2025-06-05 13:31:24,381 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-27248b37 +2025-06-05 13:31:24,381 - tests.orchestrator - INFO - Deleting pod: test-pod-f3958de4 from PVC: concurrent-pvc-27248b37 +2025-06-05 13:31:24,464 - tests.orchestrator - INFO - Waiting for pod test-pod-f3958de4 to be deleted +2025-06-05 13:31:24,528 - tests.orchestrator - INFO - Pod test-pod-f3958de4 has been deleted +2025-06-05 13:31:24,528 - tests.orchestrator - INFO - Deleted pod: test-pod-f3958de4 +2025-06-05 13:31:24,601 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-27248b37 to be deleted +2025-06-05 13:31:24,668 - tests.orchestrator - INFO - PVC concurrent-pvc-27248b37 has been deleted +2025-06-05 13:31:24,668 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-27248b37 +2025-06-05 13:31:24,668 - tests.orchestrator - INFO - Rapid PVC scenario completed +2025-06-05 13:31:25,668 - tests.orchestrator - INFO - Selected operation: _attach_pod +2025-06-05 13:31:25,669 - tests.orchestrator - INFO - Creating pod test-pod-4386bacf on PVC concurrent-pvc-737583a5 +2025-06-05 13:31:25,750 - tests.orchestrator - INFO - Created pod: test-pod-4386bacf using PVC: concurrent-pvc-737583a5 +2025-06-05 13:31:25,750 - tests.orchestrator - INFO - Waiting for pod test-pod-4386bacf to be ready +2025-06-05 13:32:27,796 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-4386bacf to be ready after 60s +2025-06-05 13:32:27,796 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-4386bacf to be ready +2025-06-05 13:32:28,797 - tests.orchestrator - INFO - Selected operation: _attach_pod +2025-06-05 13:32:28,798 - tests.orchestrator - INFO - Creating pod test-pod-93094317 on PVC concurrent-pvc-b6ec6167 +2025-06-05 13:32:28,887 - tests.orchestrator - INFO - Created pod: test-pod-93094317 using PVC: concurrent-pvc-b6ec6167 +2025-06-05 13:32:28,887 - tests.orchestrator - INFO - Waiting for pod test-pod-93094317 to be ready +2025-06-05 13:33:30,941 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-93094317 to be ready after 60s +2025-06-05 13:33:30,941 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-93094317 to be ready +2025-06-05 13:33:31,941 - tests.orchestrator - INFO - Selected operation: _create_pvc +2025-06-05 13:33:31,942 - tests.orchestrator - INFO - Creating PVC: test-pvc-cd15165b +2025-06-05 13:33:32,022 - tests.orchestrator - INFO - Created PVC: test-pvc-cd15165b +2025-06-05 13:33:32,022 - tests.orchestrator - INFO - Waiting for PVC test-pvc-cd15165b to be bound +2025-06-05 13:33:34,157 - tests.orchestrator - INFO - PVC test-pvc-cd15165b is bound +2025-06-05 13:33:35,157 - tests.orchestrator - INFO - Selected operation: _create_pvc +2025-06-05 13:33:35,157 - tests.orchestrator - INFO - Creating PVC: test-pvc-67687a38 +2025-06-05 13:33:35,229 - tests.orchestrator - INFO - Created PVC: test-pvc-67687a38 +2025-06-05 13:33:35,229 - tests.orchestrator - INFO - Waiting for PVC test-pvc-67687a38 to be bound +2025-06-05 13:33:37,363 - tests.orchestrator - INFO - PVC test-pvc-67687a38 is bound +2025-06-05 13:33:38,364 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario +2025-06-05 13:33:38,364 - tests.orchestrator - INFO - Running specific scenario: _scenario_concurrent_pvc +2025-06-05 13:33:38,364 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations +2025-06-05 13:33:38,364 - tests.orchestrator - INFO - Creating 3 PVCs in quick succession +2025-06-05 13:33:38,431 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-a6fe87d2 +2025-06-05 13:33:38,431 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-a6fe87d2 to be bound +2025-06-05 13:33:40,563 - tests.orchestrator - INFO - PVC concurrent-pvc-a6fe87d2 is bound +2025-06-05 13:33:40,631 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-eddc8eb9 +2025-06-05 13:33:40,631 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-eddc8eb9 to be bound +2025-06-05 13:33:42,765 - tests.orchestrator - INFO - PVC concurrent-pvc-eddc8eb9 is bound +2025-06-05 13:33:42,835 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-2bd04bf8 +2025-06-05 13:33:42,836 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-2bd04bf8 to be bound +2025-06-05 13:33:44,970 - tests.orchestrator - INFO - PVC concurrent-pvc-2bd04bf8 is bound +2025-06-05 13:33:44,971 - tests.orchestrator - INFO - Successfully created 3 PVCs +2025-06-05 13:33:44,971 - tests.orchestrator - INFO - Creating pod test-pod-b5b431f0 on PVC concurrent-pvc-2bd04bf8 +2025-06-05 13:33:45,045 - tests.orchestrator - INFO - Created pod: test-pod-b5b431f0 using PVC: concurrent-pvc-2bd04bf8 +2025-06-05 13:33:45,045 - tests.orchestrator - INFO - Waiting for pod test-pod-b5b431f0 to be ready +2025-06-05 13:34:45,136 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b5b431f0 to be ready after 60s +2025-06-05 13:34:45,136 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b5b431f0 to be ready +2025-06-05 13:34:45,137 - tests.orchestrator - INFO - Creating pod test-pod-a900f8ea on PVC concurrent-pvc-a6fe87d2 +2025-06-05 13:34:45,234 - tests.orchestrator - INFO - Created pod: test-pod-a900f8ea using PVC: concurrent-pvc-a6fe87d2 +2025-06-05 13:34:45,234 - tests.orchestrator - INFO - Waiting for pod test-pod-a900f8ea to be ready +2025-06-05 13:35:45,421 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a900f8ea to be ready after 60s +2025-06-05 13:35:45,421 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a900f8ea to be ready +2025-06-05 13:35:45,421 - tests.orchestrator - INFO - Creating pod test-pod-71d9daf8 on PVC concurrent-pvc-eddc8eb9 +2025-06-05 13:35:45,506 - tests.orchestrator - INFO - Created pod: test-pod-71d9daf8 using PVC: concurrent-pvc-eddc8eb9 +2025-06-05 13:35:45,506 - tests.orchestrator - INFO - Waiting for pod test-pod-71d9daf8 to be ready +2025-06-05 13:36:47,551 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-71d9daf8 to be ready after 60s +2025-06-05 13:36:47,551 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-71d9daf8 to be ready +2025-06-05 13:36:47,551 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-2bd04bf8 +2025-06-05 13:36:47,551 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-2bd04bf8 +2025-06-05 13:36:47,551 - tests.orchestrator - INFO - Deleting pod: test-pod-b5b431f0 from PVC: concurrent-pvc-2bd04bf8 +2025-06-05 13:36:47,632 - tests.orchestrator - INFO - Waiting for pod test-pod-b5b431f0 to be deleted +2025-06-05 13:36:47,699 - tests.orchestrator - INFO - Pod test-pod-b5b431f0 has been deleted +2025-06-05 13:36:47,699 - tests.orchestrator - INFO - Deleted pod: test-pod-b5b431f0 +2025-06-05 13:36:47,771 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-2bd04bf8 to be deleted +2025-06-05 13:36:47,839 - tests.orchestrator - INFO - PVC concurrent-pvc-2bd04bf8 has been deleted +2025-06-05 13:36:47,839 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-2bd04bf8 +2025-06-05 13:36:47,839 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-eddc8eb9 +2025-06-05 13:36:47,839 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-eddc8eb9 +2025-06-05 13:36:47,839 - tests.orchestrator - INFO - Deleting pod: test-pod-71d9daf8 from PVC: concurrent-pvc-eddc8eb9 +2025-06-05 13:36:47,919 - tests.orchestrator - INFO - Waiting for pod test-pod-71d9daf8 to be deleted +2025-06-05 13:36:47,984 - tests.orchestrator - INFO - Pod test-pod-71d9daf8 has been deleted +2025-06-05 13:36:47,984 - tests.orchestrator - INFO - Deleted pod: test-pod-71d9daf8 +2025-06-05 13:36:48,056 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-eddc8eb9 to be deleted +2025-06-05 13:36:48,122 - tests.orchestrator - INFO - PVC concurrent-pvc-eddc8eb9 has been deleted +2025-06-05 13:36:48,122 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-eddc8eb9 +2025-06-05 13:36:48,122 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-a6fe87d2 +2025-06-05 13:36:48,122 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-a6fe87d2 +2025-06-05 13:36:48,122 - tests.orchestrator - INFO - Deleting pod: test-pod-a900f8ea from PVC: concurrent-pvc-a6fe87d2 +2025-06-05 13:36:48,206 - tests.orchestrator - INFO - Waiting for pod test-pod-a900f8ea to be deleted +2025-06-05 13:36:48,270 - tests.orchestrator - INFO - Pod test-pod-a900f8ea has been deleted +2025-06-05 13:36:48,271 - tests.orchestrator - INFO - Deleted pod: test-pod-a900f8ea +2025-06-05 13:36:48,345 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-a6fe87d2 to be deleted +2025-06-05 13:36:48,412 - tests.orchestrator - INFO - PVC concurrent-pvc-a6fe87d2 has been deleted +2025-06-05 13:36:48,412 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-a6fe87d2 +2025-06-05 13:36:48,412 - tests.orchestrator - INFO - Rapid PVC scenario completed +2025-06-05 13:36:49,412 - tests.orchestrator - INFO - Test completed in 526.53 seconds +2025-06-05 13:36:49,412 - tests.orchestrator - INFO - Cleaning up resources... +2025-06-05 13:36:49,412 - tests.orchestrator - INFO - Deleting pod: test-pod-4386bacf from PVC: concurrent-pvc-737583a5 +2025-06-05 13:36:49,492 - tests.orchestrator - INFO - Waiting for pod test-pod-4386bacf to be deleted +2025-06-05 13:36:49,559 - tests.orchestrator - INFO - Pod test-pod-4386bacf has been deleted +2025-06-05 13:36:49,560 - tests.orchestrator - INFO - Deleted pod: test-pod-4386bacf +2025-06-05 13:36:49,560 - tests.orchestrator - INFO - Deleting pod: test-pod-c11d7609 from PVC: concurrent-pvc-b6ec6167 +2025-06-05 13:36:49,640 - tests.orchestrator - INFO - Waiting for pod test-pod-c11d7609 to be deleted +2025-06-05 13:36:49,706 - tests.orchestrator - INFO - Pod test-pod-c11d7609 has been deleted +2025-06-05 13:36:49,706 - tests.orchestrator - INFO - Deleted pod: test-pod-c11d7609 +2025-06-05 13:36:49,706 - tests.orchestrator - INFO - Deleting pod: test-pod-93094317 from PVC: concurrent-pvc-b6ec6167 +2025-06-05 13:36:49,783 - tests.orchestrator - INFO - Waiting for pod test-pod-93094317 to be deleted +2025-06-05 13:36:49,850 - tests.orchestrator - INFO - Pod test-pod-93094317 has been deleted +2025-06-05 13:36:49,850 - tests.orchestrator - INFO - Deleted pod: test-pod-93094317 +2025-06-05 13:36:49,850 - tests.orchestrator - INFO - Deleting PVC: test-pvc-87f8dca2 +2025-06-05 13:36:49,922 - tests.orchestrator - INFO - Waiting for PVC test-pvc-87f8dca2 to be deleted +2025-06-05 13:36:49,988 - tests.orchestrator - INFO - PVC test-pvc-87f8dca2 has been deleted +2025-06-05 13:36:49,989 - tests.orchestrator - INFO - Deleted PVC: test-pvc-87f8dca2 +2025-06-05 13:36:49,989 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-d46ca8c1 +2025-06-05 13:36:50,060 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-d46ca8c1 to be deleted +2025-06-05 13:36:50,126 - tests.orchestrator - INFO - PVC concurrent-pvc-d46ca8c1 has been deleted +2025-06-05 13:36:50,127 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-d46ca8c1 +2025-06-05 13:36:50,127 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-737583a5 +2025-06-05 13:36:50,197 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-737583a5 to be deleted +2025-06-05 13:36:50,264 - tests.orchestrator - INFO - PVC concurrent-pvc-737583a5 has been deleted +2025-06-05 13:36:50,264 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-737583a5 +2025-06-05 13:36:50,264 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-b6ec6167 +2025-06-05 13:36:50,336 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-b6ec6167 to be deleted +2025-06-05 13:36:50,403 - tests.orchestrator - INFO - PVC concurrent-pvc-b6ec6167 has been deleted +2025-06-05 13:36:50,403 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-b6ec6167 +2025-06-05 13:36:50,403 - tests.orchestrator - INFO - Deleting PVC: test-pvc-cd15165b +2025-06-05 13:36:50,477 - tests.orchestrator - INFO - Waiting for PVC test-pvc-cd15165b to be deleted +2025-06-05 13:36:50,544 - tests.orchestrator - INFO - PVC test-pvc-cd15165b has been deleted +2025-06-05 13:36:50,545 - tests.orchestrator - INFO - Deleted PVC: test-pvc-cd15165b +2025-06-05 13:36:50,545 - tests.orchestrator - INFO - Deleting PVC: test-pvc-67687a38 +2025-06-05 13:36:50,617 - tests.orchestrator - INFO - Waiting for PVC test-pvc-67687a38 to be deleted +2025-06-05 13:36:50,683 - tests.orchestrator - INFO - PVC test-pvc-67687a38 has been deleted +2025-06-05 13:36:50,683 - tests.orchestrator - INFO - Deleted PVC: test-pvc-67687a38 +2025-06-05 13:36:50,683 - tests.orchestrator - INFO - Cleanup completed +2025-06-05 13:36:50,683 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-05 13:36:50,683 - tests.orchestrator - INFO - --- Operations --- +2025-06-05 13:36:50,683 - tests.orchestrator - INFO - create_pvc: 12 succeeded, 0 failed (100.0%) +2025-06-05 13:36:50,683 - tests.orchestrator - INFO - attach_pod: 8 succeeded, 0 failed (100.0%) +2025-06-05 13:36:50,683 - tests.orchestrator - INFO - delete_pod: 8 succeeded, 0 failed (100.0%) +2025-06-05 13:36:50,684 - tests.orchestrator - INFO - delete_pvc: 12 succeeded, 0 failed (100.0%) +2025-06-05 13:36:50,684 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 0 failed (0.0%) +2025-06-05 13:36:50,684 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 0 failed (0.0%) +2025-06-05 13:36:50,684 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-05 13:36:50,684 - tests.orchestrator - INFO - shared_volume_rw: No runs +2025-06-05 13:36:50,684 - tests.orchestrator - INFO - many_to_one: No runs +2025-06-05 13:36:50,684 - tests.orchestrator - INFO - one_to_one: No runs +2025-06-05 13:36:50,684 - tests.orchestrator - INFO - concurrent_pvc: 2 succeeded, 0 failed out of 2 runs (100.0%) +2025-06-05 13:36:50,684 - tests.orchestrator - INFO - ========================================= +2025-06-05 13:36:50,685 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250605_133650.json +2025-06-05 13:36:50,728 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250605_133650_20250605_133650.json +2025-06-05 13:36:50,728 - __main__ - INFO - All tests completed successfully +2025-06-05 13:50:01,756 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-05 13:50:01,756 - __main__ - INFO - Verifying AWS credentials +2025-06-05 13:50:01,757 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 13:50:01,758 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 13:50:01,758 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 13:50:01,758 - __main__ - INFO - Running orchestrator stress test suite +2025-06-05 13:50:01,758 - __main__ - INFO - Starting orchestrator for 300 seconds +2025-06-05 13:50:01,759 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-05 13:50:02,494 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 13:50:02,715 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 13:50:02,715 - __main__ - INFO - Test duration overridden to 300 seconds +2025-06-05 13:50:02,715 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) +2025-06-05 13:50:02,715 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-05 13:50:02,906 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 13:50:02,906 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 13:50:02,906 - tests.orchestrator - INFO - Creating PVC: test-pvc-040b70c6 +2025-06-05 13:50:02,982 - tests.orchestrator - INFO - Created PVC: test-pvc-040b70c6 +2025-06-05 13:50:02,982 - tests.orchestrator - INFO - Waiting for PVC test-pvc-040b70c6 to be bound +2025-06-05 13:50:05,116 - tests.orchestrator - INFO - PVC test-pvc-040b70c6 is bound +2025-06-05 13:50:05,116 - tests.orchestrator - INFO - Creating PVC: test-pvc-e5eb6b62 +2025-06-05 13:50:05,184 - tests.orchestrator - INFO - Created PVC: test-pvc-e5eb6b62 +2025-06-05 13:50:05,184 - tests.orchestrator - INFO - Waiting for PVC test-pvc-e5eb6b62 to be bound +2025-06-05 13:50:07,318 - tests.orchestrator - INFO - PVC test-pvc-e5eb6b62 is bound +2025-06-05 13:50:07,319 - tests.orchestrator - INFO - Creating pod test-pod-094962b3 on PVC test-pvc-e5eb6b62 +2025-06-05 13:50:07,399 - tests.orchestrator - INFO - Created pod: test-pod-094962b3 using PVC: test-pvc-e5eb6b62 +2025-06-05 13:50:07,400 - tests.orchestrator - INFO - Waiting for pod test-pod-094962b3 to be ready +2025-06-05 13:51:09,437 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-094962b3 to be ready after 60s +2025-06-05 13:51:09,437 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-094962b3 to be ready +2025-06-05 13:51:09,437 - tests.orchestrator - INFO - Creating pod test-pod-1b064942 on PVC test-pvc-040b70c6 +2025-06-05 13:51:09,521 - tests.orchestrator - INFO - Created pod: test-pod-1b064942 using PVC: test-pvc-040b70c6 +2025-06-05 13:51:09,521 - tests.orchestrator - INFO - Waiting for pod test-pod-1b064942 to be ready +2025-06-05 13:52:11,558 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1b064942 to be ready after 60s +2025-06-05 13:52:11,558 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1b064942 to be ready +2025-06-05 13:52:11,559 - tests.orchestrator - INFO - Creating pod test-pod-ab3159d4 on PVC test-pvc-e5eb6b62 +2025-06-05 13:52:11,645 - tests.orchestrator - INFO - Created pod: test-pod-ab3159d4 using PVC: test-pvc-e5eb6b62 +2025-06-05 13:52:11,646 - tests.orchestrator - INFO - Waiting for pod test-pod-ab3159d4 to be ready +2025-06-05 13:53:13,682 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-ab3159d4 to be ready after 60s +2025-06-05 13:53:13,682 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-ab3159d4 to be ready +2025-06-05 13:53:13,683 - tests.orchestrator - INFO - Testing read/write between pods test-pod-094962b3 and test-pod-ab3159d4 sharing PVC test-pvc-e5eb6b62 +2025-06-05 13:53:13,751 - tests.orchestrator - ERROR - Failed in read/write verification: (400) +Reason: Bad Request +HTTP response headers: HTTPHeaderDict({'Audit-Id': '7c3ae37d-dd7e-42d1-97d7-9f45f47124cc', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 13:53:13 GMT', 'Content-Length': '166'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-094962b3 does not have a host assigned","reason":"BadRequest","code":400} + + +2025-06-05 13:53:13,751 - tests.orchestrator - INFO - Running specific scenario: _scenario_one_to_one +2025-06-05 13:53:13,751 - tests.orchestrator - INFO - Running scenario: One pod per PVC +2025-06-05 13:53:13,751 - tests.orchestrator - INFO - Creating 3 PVC-pod pairs +2025-06-05 13:53:13,829 - tests.orchestrator - INFO - Waiting for PVC one2one-9268308a to be bound +2025-06-05 13:53:15,962 - tests.orchestrator - INFO - PVC one2one-9268308a is bound +2025-06-05 13:53:15,962 - tests.orchestrator - INFO - Creating pod test-pod-2d27ea53 on PVC one2one-9268308a +2025-06-05 13:53:16,038 - tests.orchestrator - INFO - Created pod: test-pod-2d27ea53 using PVC: one2one-9268308a +2025-06-05 13:53:16,038 - tests.orchestrator - INFO - Waiting for pod test-pod-2d27ea53 to be ready +2025-06-05 13:54:16,048 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-2d27ea53 to be ready after 60s +2025-06-05 13:54:16,048 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-2d27ea53 to be ready +2025-06-05 13:54:16,124 - tests.orchestrator - INFO - Waiting for PVC one2one-63559411 to be bound +2025-06-05 13:54:18,258 - tests.orchestrator - INFO - PVC one2one-63559411 is bound +2025-06-05 13:54:18,258 - tests.orchestrator - INFO - Creating pod test-pod-5c1f7339 on PVC one2one-63559411 +2025-06-05 13:54:18,330 - tests.orchestrator - INFO - Created pod: test-pod-5c1f7339 using PVC: one2one-63559411 +2025-06-05 13:54:18,330 - tests.orchestrator - INFO - Waiting for pod test-pod-5c1f7339 to be ready +2025-06-05 13:55:20,367 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-5c1f7339 to be ready after 60s +2025-06-05 13:55:20,367 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-5c1f7339 to be ready +2025-06-05 13:55:20,444 - tests.orchestrator - INFO - Waiting for PVC one2one-24717940 to be bound +2025-06-05 13:55:22,579 - tests.orchestrator - INFO - PVC one2one-24717940 is bound +2025-06-05 13:55:22,580 - tests.orchestrator - INFO - Creating pod test-pod-a84638d7 on PVC one2one-24717940 +2025-06-05 13:55:22,655 - tests.orchestrator - INFO - Created pod: test-pod-a84638d7 using PVC: one2one-24717940 +2025-06-05 13:55:22,655 - tests.orchestrator - INFO - Waiting for pod test-pod-a84638d7 to be ready +2025-06-05 13:56:24,701 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a84638d7 to be ready after 60s +2025-06-05 13:56:24,701 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a84638d7 to be ready +2025-06-05 13:56:24,701 - tests.orchestrator - WARNING - Failed to create enough PVC-pod pairs, only created 0 +2025-06-05 13:56:24,701 - tests.orchestrator - INFO - Deleting pod: test-pod-094962b3 from PVC: test-pvc-e5eb6b62 +2025-06-05 13:56:24,780 - tests.orchestrator - INFO - Waiting for pod test-pod-094962b3 to be deleted +2025-06-05 13:56:24,846 - tests.orchestrator - INFO - Pod test-pod-094962b3 has been deleted +2025-06-05 13:56:24,846 - tests.orchestrator - INFO - Deleted pod: test-pod-094962b3 +2025-06-05 13:56:24,846 - tests.orchestrator - INFO - Deleting PVC: one2one-63559411 +2025-06-05 13:56:24,846 - tests.orchestrator - INFO - Deleting 1 pods using PVC one2one-63559411 +2025-06-05 13:56:24,846 - tests.orchestrator - INFO - Deleting pod: test-pod-5c1f7339 from PVC: one2one-63559411 +2025-06-05 13:56:24,932 - tests.orchestrator - INFO - Waiting for pod test-pod-5c1f7339 to be deleted +2025-06-05 13:56:24,997 - tests.orchestrator - INFO - Pod test-pod-5c1f7339 has been deleted +2025-06-05 13:56:24,997 - tests.orchestrator - INFO - Deleted pod: test-pod-5c1f7339 +2025-06-05 13:56:25,070 - tests.orchestrator - INFO - Waiting for PVC one2one-63559411 to be deleted +2025-06-05 13:56:25,135 - tests.orchestrator - INFO - PVC one2one-63559411 has been deleted +2025-06-05 13:56:25,136 - tests.orchestrator - INFO - Deleted PVC: one2one-63559411 +2025-06-05 13:56:25,136 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations +2025-06-05 13:56:25,136 - tests.orchestrator - INFO - Test completed in 382.42 seconds +2025-06-05 13:56:25,136 - tests.orchestrator - INFO - Cleaning up resources... +2025-06-05 13:56:25,136 - tests.orchestrator - INFO - Deleting pod: test-pod-1b064942 from PVC: test-pvc-040b70c6 +2025-06-05 13:56:25,213 - tests.orchestrator - INFO - Waiting for pod test-pod-1b064942 to be deleted +2025-06-05 13:56:25,279 - tests.orchestrator - INFO - Pod test-pod-1b064942 has been deleted +2025-06-05 13:56:25,279 - tests.orchestrator - INFO - Deleted pod: test-pod-1b064942 +2025-06-05 13:56:25,279 - tests.orchestrator - INFO - Deleting pod: test-pod-ab3159d4 from PVC: test-pvc-e5eb6b62 +2025-06-05 13:56:25,363 - tests.orchestrator - INFO - Waiting for pod test-pod-ab3159d4 to be deleted +2025-06-05 13:56:25,429 - tests.orchestrator - INFO - Pod test-pod-ab3159d4 has been deleted +2025-06-05 13:56:25,429 - tests.orchestrator - INFO - Deleted pod: test-pod-ab3159d4 +2025-06-05 13:56:25,429 - tests.orchestrator - INFO - Deleting pod: test-pod-2d27ea53 from PVC: one2one-9268308a +2025-06-05 13:56:25,507 - tests.orchestrator - INFO - Waiting for pod test-pod-2d27ea53 to be deleted +2025-06-05 13:56:25,573 - tests.orchestrator - INFO - Pod test-pod-2d27ea53 has been deleted +2025-06-05 13:56:25,573 - tests.orchestrator - INFO - Deleted pod: test-pod-2d27ea53 +2025-06-05 13:56:25,574 - tests.orchestrator - INFO - Deleting pod: test-pod-a84638d7 from PVC: one2one-24717940 +2025-06-05 13:56:25,658 - tests.orchestrator - INFO - Waiting for pod test-pod-a84638d7 to be deleted +2025-06-05 13:56:25,725 - tests.orchestrator - INFO - Pod test-pod-a84638d7 has been deleted +2025-06-05 13:56:25,726 - tests.orchestrator - INFO - Deleted pod: test-pod-a84638d7 +2025-06-05 13:56:25,726 - tests.orchestrator - INFO - Deleting PVC: test-pvc-040b70c6 +2025-06-05 13:56:25,800 - tests.orchestrator - INFO - Waiting for PVC test-pvc-040b70c6 to be deleted +2025-06-05 13:56:25,866 - tests.orchestrator - INFO - PVC test-pvc-040b70c6 has been deleted +2025-06-05 13:56:25,866 - tests.orchestrator - INFO - Deleted PVC: test-pvc-040b70c6 +2025-06-05 13:56:25,866 - tests.orchestrator - INFO - Deleting PVC: test-pvc-e5eb6b62 +2025-06-05 13:56:25,940 - tests.orchestrator - INFO - Waiting for PVC test-pvc-e5eb6b62 to be deleted +2025-06-05 13:56:26,006 - tests.orchestrator - INFO - PVC test-pvc-e5eb6b62 has been deleted +2025-06-05 13:56:26,006 - tests.orchestrator - INFO - Deleted PVC: test-pvc-e5eb6b62 +2025-06-05 13:56:26,006 - tests.orchestrator - INFO - Deleting PVC: one2one-9268308a +2025-06-05 13:56:26,081 - tests.orchestrator - INFO - Waiting for PVC one2one-9268308a to be deleted +2025-06-05 13:56:26,146 - tests.orchestrator - INFO - PVC one2one-9268308a has been deleted +2025-06-05 13:56:26,146 - tests.orchestrator - INFO - Deleted PVC: one2one-9268308a +2025-06-05 13:56:26,146 - tests.orchestrator - INFO - Deleting PVC: one2one-24717940 +2025-06-05 13:56:26,221 - tests.orchestrator - INFO - Waiting for PVC one2one-24717940 to be deleted +2025-06-05 13:56:26,287 - tests.orchestrator - INFO - PVC one2one-24717940 has been deleted +2025-06-05 13:56:26,287 - tests.orchestrator - INFO - Deleted PVC: one2one-24717940 +2025-06-05 13:56:26,287 - tests.orchestrator - INFO - Cleanup completed +2025-06-05 13:56:26,287 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-05 13:56:26,287 - tests.orchestrator - INFO - --- Operations --- +2025-06-05 13:56:26,287 - tests.orchestrator - INFO - create_pvc: 2 succeeded, 0 failed (100.0%) +2025-06-05 13:56:26,287 - tests.orchestrator - INFO - attach_pod: 6 succeeded, 0 failed (100.0%) +2025-06-05 13:56:26,287 - tests.orchestrator - INFO - delete_pod: 6 succeeded, 0 failed (100.0%) +2025-06-05 13:56:26,287 - tests.orchestrator - INFO - delete_pvc: 5 succeeded, 0 failed (100.0%) +2025-06-05 13:56:26,287 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) +2025-06-05 13:56:26,287 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) +2025-06-05 13:56:26,287 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-05 13:56:26,287 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-05 13:56:26,287 - tests.orchestrator - INFO - many_to_one: No runs +2025-06-05 13:56:26,287 - tests.orchestrator - INFO - one_to_one: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-05 13:56:26,288 - tests.orchestrator - INFO - concurrent_pvc: No runs +2025-06-05 13:56:26,288 - tests.orchestrator - INFO - ========================================= +2025-06-05 13:56:26,288 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250605_135626.json +2025-06-05 13:56:26,330 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250605_135626_20250605_135626.json +2025-06-05 13:56:26,331 - __main__ - INFO - All tests completed successfully +2025-06-05 14:04:07,833 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-05 14:04:07,833 - __main__ - INFO - Verifying AWS credentials +2025-06-05 14:04:07,834 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 14:04:07,834 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 14:04:07,835 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 14:04:07,835 - __main__ - INFO - Running orchestrator stress test suite +2025-06-05 14:04:07,835 - __main__ - INFO - Starting orchestrator for 300 seconds +2025-06-05 14:04:07,835 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-05 14:04:08,567 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 14:04:08,780 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 14:04:08,780 - __main__ - INFO - Test duration overridden to 300 seconds +2025-06-05 14:04:08,780 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) +2025-06-05 14:04:08,780 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-05 14:04:08,972 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 14:04:08,972 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 14:04:08,972 - tests.orchestrator - INFO - Creating PVC: test-pvc-847a8a56 +2025-06-05 14:04:09,057 - tests.orchestrator - INFO - Created PVC: test-pvc-847a8a56 +2025-06-05 14:04:09,057 - tests.orchestrator - INFO - Waiting for PVC test-pvc-847a8a56 to be bound +2025-06-05 14:04:11,188 - tests.orchestrator - INFO - PVC test-pvc-847a8a56 is bound +2025-06-05 14:04:11,189 - tests.orchestrator - INFO - Creating PVC: test-pvc-9dee4752 +2025-06-05 14:04:11,257 - tests.orchestrator - INFO - Created PVC: test-pvc-9dee4752 +2025-06-05 14:04:11,258 - tests.orchestrator - INFO - Waiting for PVC test-pvc-9dee4752 to be bound +2025-06-05 14:04:13,391 - tests.orchestrator - INFO - PVC test-pvc-9dee4752 is bound +2025-06-05 14:04:13,391 - tests.orchestrator - INFO - Creating pod test-pod-d215a119 on PVC test-pvc-9dee4752 +2025-06-05 14:04:13,474 - tests.orchestrator - INFO - Created pod: test-pod-d215a119 using PVC: test-pvc-9dee4752 +2025-06-05 14:04:13,474 - tests.orchestrator - INFO - Waiting for pod test-pod-d215a119 to be ready +2025-06-05 14:05:15,521 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-d215a119 to be ready after 60s +2025-06-05 14:05:15,521 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-d215a119 to be ready +2025-06-05 14:05:15,522 - tests.orchestrator - INFO - Creating pod test-pod-54364ba6 on PVC test-pvc-9dee4752 +2025-06-05 14:05:15,608 - tests.orchestrator - INFO - Created pod: test-pod-54364ba6 using PVC: test-pvc-9dee4752 +2025-06-05 14:05:15,608 - tests.orchestrator - INFO - Waiting for pod test-pod-54364ba6 to be ready +2025-06-05 14:06:15,655 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-54364ba6 to be ready after 60s +2025-06-05 14:06:15,655 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-54364ba6 to be ready +2025-06-05 14:06:15,655 - tests.orchestrator - INFO - Creating pod test-pod-98a62ebf on PVC test-pvc-9dee4752 +2025-06-05 14:06:15,742 - tests.orchestrator - INFO - Created pod: test-pod-98a62ebf using PVC: test-pvc-9dee4752 +2025-06-05 14:06:15,742 - tests.orchestrator - INFO - Waiting for pod test-pod-98a62ebf to be ready +2025-06-05 14:07:15,768 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-98a62ebf to be ready after 60s +2025-06-05 14:07:15,769 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-98a62ebf to be ready +2025-06-05 14:07:15,769 - tests.orchestrator - INFO - Testing read/write between pods test-pod-d215a119 and test-pod-98a62ebf sharing PVC test-pvc-9dee4752 +2025-06-05 14:07:15,836 - tests.orchestrator - ERROR - Failed in read/write verification: (400) +Reason: Bad Request +HTTP response headers: HTTPHeaderDict({'Audit-Id': '0fd252b7-6182-4f97-9a5a-1503836973da', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 14:07:15 GMT', 'Content-Length': '166'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-d215a119 does not have a host assigned","reason":"BadRequest","code":400} + + +2025-06-05 14:07:15,836 - tests.orchestrator - INFO - Running specific scenario: _scenario_one_to_one +2025-06-05 14:07:15,836 - tests.orchestrator - INFO - Running scenario: One pod per PVC +2025-06-05 14:07:15,836 - tests.orchestrator - INFO - Creating 4 PVC-pod pairs +2025-06-05 14:07:15,913 - tests.orchestrator - INFO - Waiting for PVC one2one-71f9fc22 to be bound +2025-06-05 14:07:18,066 - tests.orchestrator - INFO - PVC one2one-71f9fc22 is bound +2025-06-05 14:07:18,066 - tests.orchestrator - INFO - Creating pod test-pod-78c97483 on PVC one2one-71f9fc22 +2025-06-05 14:07:18,197 - tests.orchestrator - INFO - Created pod: test-pod-78c97483 using PVC: one2one-71f9fc22 +2025-06-05 14:07:18,197 - tests.orchestrator - INFO - Waiting for pod test-pod-78c97483 to be ready +2025-06-05 14:08:18,229 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-78c97483 to be ready after 60s +2025-06-05 14:08:18,229 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-78c97483 to be ready +2025-06-05 14:08:18,310 - tests.orchestrator - INFO - Waiting for PVC one2one-d55f8841 to be bound +2025-06-05 14:08:20,442 - tests.orchestrator - INFO - PVC one2one-d55f8841 is bound +2025-06-05 14:08:20,442 - tests.orchestrator - INFO - Creating pod test-pod-a400277a on PVC one2one-d55f8841 +2025-06-05 14:08:20,518 - tests.orchestrator - INFO - Created pod: test-pod-a400277a using PVC: one2one-d55f8841 +2025-06-05 14:08:20,519 - tests.orchestrator - INFO - Waiting for pod test-pod-a400277a to be ready +2025-06-05 14:09:22,561 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a400277a to be ready after 60s +2025-06-05 14:09:22,561 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a400277a to be ready +2025-06-05 14:09:22,638 - tests.orchestrator - INFO - Waiting for PVC one2one-f3257be9 to be bound +2025-06-05 14:09:24,772 - tests.orchestrator - INFO - PVC one2one-f3257be9 is bound +2025-06-05 14:09:24,773 - tests.orchestrator - INFO - Creating pod test-pod-1bfbc8b3 on PVC one2one-f3257be9 +2025-06-05 14:09:24,850 - tests.orchestrator - INFO - Created pod: test-pod-1bfbc8b3 using PVC: one2one-f3257be9 +2025-06-05 14:09:24,850 - tests.orchestrator - INFO - Waiting for pod test-pod-1bfbc8b3 to be ready +2025-06-05 14:10:26,912 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1bfbc8b3 to be ready after 60s +2025-06-05 14:10:26,913 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1bfbc8b3 to be ready +2025-06-05 14:10:26,992 - tests.orchestrator - INFO - Waiting for PVC one2one-d2044d7d to be bound +2025-06-05 14:10:29,127 - tests.orchestrator - INFO - PVC one2one-d2044d7d is bound +2025-06-05 14:10:29,127 - tests.orchestrator - INFO - Creating pod test-pod-6edcdac8 on PVC one2one-d2044d7d +2025-06-05 14:10:29,203 - tests.orchestrator - INFO - Created pod: test-pod-6edcdac8 using PVC: one2one-d2044d7d +2025-06-05 14:10:29,203 - tests.orchestrator - INFO - Waiting for pod test-pod-6edcdac8 to be ready +2025-06-05 14:11:29,292 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-6edcdac8 to be ready after 60s +2025-06-05 14:11:29,293 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-6edcdac8 to be ready +2025-06-05 14:11:29,293 - tests.orchestrator - WARNING - Failed to create enough PVC-pod pairs, only created 0 +2025-06-05 14:11:29,293 - tests.orchestrator - INFO - Deleting pod: test-pod-54364ba6 from PVC: test-pvc-9dee4752 +2025-06-05 14:11:29,378 - tests.orchestrator - INFO - Waiting for pod test-pod-54364ba6 to be deleted +2025-06-05 14:11:29,443 - tests.orchestrator - INFO - Pod test-pod-54364ba6 has been deleted +2025-06-05 14:11:29,443 - tests.orchestrator - INFO - Deleted pod: test-pod-54364ba6 +2025-06-05 14:11:29,443 - tests.orchestrator - INFO - Deleting PVC: one2one-71f9fc22 +2025-06-05 14:11:29,443 - tests.orchestrator - INFO - Deleting 1 pods using PVC one2one-71f9fc22 +2025-06-05 14:11:29,443 - tests.orchestrator - INFO - Deleting pod: test-pod-78c97483 from PVC: one2one-71f9fc22 +2025-06-05 14:11:29,521 - tests.orchestrator - INFO - Waiting for pod test-pod-78c97483 to be deleted +2025-06-05 14:11:29,588 - tests.orchestrator - INFO - Pod test-pod-78c97483 has been deleted +2025-06-05 14:11:29,588 - tests.orchestrator - INFO - Deleted pod: test-pod-78c97483 +2025-06-05 14:11:29,666 - tests.orchestrator - INFO - Waiting for PVC one2one-71f9fc22 to be deleted +2025-06-05 14:11:29,733 - tests.orchestrator - INFO - PVC one2one-71f9fc22 has been deleted +2025-06-05 14:11:29,733 - tests.orchestrator - INFO - Deleted PVC: one2one-71f9fc22 +2025-06-05 14:11:29,733 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations +2025-06-05 14:11:29,733 - tests.orchestrator - INFO - Test completed in 440.95 seconds +2025-06-05 14:11:29,733 - tests.orchestrator - INFO - Cleaning up resources... +2025-06-05 14:11:29,733 - tests.orchestrator - INFO - Deleting pod: test-pod-d215a119 from PVC: test-pvc-9dee4752 +2025-06-05 14:11:29,832 - tests.orchestrator - INFO - Waiting for pod test-pod-d215a119 to be deleted +2025-06-05 14:11:29,897 - tests.orchestrator - INFO - Pod test-pod-d215a119 has been deleted +2025-06-05 14:11:29,897 - tests.orchestrator - INFO - Deleted pod: test-pod-d215a119 +2025-06-05 14:11:29,897 - tests.orchestrator - INFO - Deleting pod: test-pod-98a62ebf from PVC: test-pvc-9dee4752 +2025-06-05 14:11:29,977 - tests.orchestrator - INFO - Waiting for pod test-pod-98a62ebf to be deleted +2025-06-05 14:11:30,044 - tests.orchestrator - INFO - Pod test-pod-98a62ebf has been deleted +2025-06-05 14:11:30,044 - tests.orchestrator - INFO - Deleted pod: test-pod-98a62ebf +2025-06-05 14:11:30,044 - tests.orchestrator - INFO - Deleting pod: test-pod-a400277a from PVC: one2one-d55f8841 +2025-06-05 14:11:30,134 - tests.orchestrator - INFO - Waiting for pod test-pod-a400277a to be deleted +2025-06-05 14:11:30,199 - tests.orchestrator - INFO - Pod test-pod-a400277a has been deleted +2025-06-05 14:11:30,199 - tests.orchestrator - INFO - Deleted pod: test-pod-a400277a +2025-06-05 14:11:30,199 - tests.orchestrator - INFO - Deleting pod: test-pod-1bfbc8b3 from PVC: one2one-f3257be9 +2025-06-05 14:11:30,279 - tests.orchestrator - INFO - Waiting for pod test-pod-1bfbc8b3 to be deleted +2025-06-05 14:11:30,346 - tests.orchestrator - INFO - Pod test-pod-1bfbc8b3 has been deleted +2025-06-05 14:11:30,346 - tests.orchestrator - INFO - Deleted pod: test-pod-1bfbc8b3 +2025-06-05 14:11:30,346 - tests.orchestrator - INFO - Deleting pod: test-pod-6edcdac8 from PVC: one2one-d2044d7d +2025-06-05 14:11:30,432 - tests.orchestrator - INFO - Waiting for pod test-pod-6edcdac8 to be deleted +2025-06-05 14:11:30,500 - tests.orchestrator - INFO - Pod test-pod-6edcdac8 has been deleted +2025-06-05 14:11:30,500 - tests.orchestrator - INFO - Deleted pod: test-pod-6edcdac8 +2025-06-05 14:11:30,500 - tests.orchestrator - INFO - Deleting PVC: test-pvc-847a8a56 +2025-06-05 14:11:30,572 - tests.orchestrator - INFO - Waiting for PVC test-pvc-847a8a56 to be deleted +2025-06-05 14:11:30,642 - tests.orchestrator - INFO - PVC test-pvc-847a8a56 has been deleted +2025-06-05 14:11:30,642 - tests.orchestrator - INFO - Deleted PVC: test-pvc-847a8a56 +2025-06-05 14:11:30,642 - tests.orchestrator - INFO - Deleting PVC: test-pvc-9dee4752 +2025-06-05 14:11:30,715 - tests.orchestrator - INFO - Waiting for PVC test-pvc-9dee4752 to be deleted +2025-06-05 14:11:30,782 - tests.orchestrator - INFO - PVC test-pvc-9dee4752 has been deleted +2025-06-05 14:11:30,782 - tests.orchestrator - INFO - Deleted PVC: test-pvc-9dee4752 +2025-06-05 14:11:30,782 - tests.orchestrator - INFO - Deleting PVC: one2one-d55f8841 +2025-06-05 14:11:30,860 - tests.orchestrator - INFO - Waiting for PVC one2one-d55f8841 to be deleted +2025-06-05 14:11:30,927 - tests.orchestrator - INFO - PVC one2one-d55f8841 has been deleted +2025-06-05 14:11:30,927 - tests.orchestrator - INFO - Deleted PVC: one2one-d55f8841 +2025-06-05 14:11:30,928 - tests.orchestrator - INFO - Deleting PVC: one2one-f3257be9 +2025-06-05 14:11:30,999 - tests.orchestrator - INFO - Waiting for PVC one2one-f3257be9 to be deleted +2025-06-05 14:11:31,068 - tests.orchestrator - INFO - PVC one2one-f3257be9 has been deleted +2025-06-05 14:11:31,068 - tests.orchestrator - INFO - Deleted PVC: one2one-f3257be9 +2025-06-05 14:11:31,068 - tests.orchestrator - INFO - Deleting PVC: one2one-d2044d7d +2025-06-05 14:11:31,142 - tests.orchestrator - INFO - Waiting for PVC one2one-d2044d7d to be deleted +2025-06-05 14:11:31,208 - tests.orchestrator - INFO - PVC one2one-d2044d7d has been deleted +2025-06-05 14:11:31,208 - tests.orchestrator - INFO - Deleted PVC: one2one-d2044d7d +2025-06-05 14:11:31,208 - tests.orchestrator - INFO - Cleanup completed +2025-06-05 14:11:31,208 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-05 14:11:31,208 - tests.orchestrator - INFO - --- Operations --- +2025-06-05 14:11:31,209 - tests.orchestrator - INFO - create_pvc: 2 succeeded, 0 failed (100.0%) +2025-06-05 14:11:31,209 - tests.orchestrator - INFO - attach_pod: 7 succeeded, 0 failed (100.0%) +2025-06-05 14:11:31,209 - tests.orchestrator - INFO - delete_pod: 7 succeeded, 0 failed (100.0%) +2025-06-05 14:11:31,209 - tests.orchestrator - INFO - delete_pvc: 6 succeeded, 0 failed (100.0%) +2025-06-05 14:11:31,209 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) +2025-06-05 14:11:31,209 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) +2025-06-05 14:11:31,209 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-05 14:11:31,209 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-05 14:11:31,209 - tests.orchestrator - INFO - many_to_one: No runs +2025-06-05 14:11:31,209 - tests.orchestrator - INFO - one_to_one: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-05 14:11:31,209 - tests.orchestrator - INFO - concurrent_pvc: No runs +2025-06-05 14:11:31,209 - tests.orchestrator - INFO - ========================================= +2025-06-05 14:11:31,210 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250605_141131.json +2025-06-05 14:11:31,252 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250605_141131_20250605_141131.json +2025-06-05 14:11:31,252 - __main__ - INFO - All tests completed successfully +2025-06-05 14:13:20,869 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-05 14:13:20,869 - __main__ - INFO - Verifying AWS credentials +2025-06-05 14:13:20,870 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 14:13:20,871 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 14:13:20,871 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 14:13:20,871 - __main__ - INFO - Running orchestrator stress test suite +2025-06-05 14:13:20,871 - __main__ - INFO - Starting orchestrator for 60 seconds +2025-06-05 14:13:20,872 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-05 14:13:21,602 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 14:13:21,814 - tests.orchestrator - ERROR - Error checking namespace: (401) +Reason: Unauthorized +HTTP response headers: HTTPHeaderDict({'Audit-Id': 'eb0b1f42-f08c-4b18-b778-a83d8eb055f7', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 14:13:21 GMT', 'Content-Length': '129'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} + + +2025-06-05 14:13:21,814 - __main__ - ERROR - Error running tests: (401) +Reason: Unauthorized +HTTP response headers: HTTPHeaderDict({'Audit-Id': 'eb0b1f42-f08c-4b18-b778-a83d8eb055f7', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 14:13:21 GMT', 'Content-Length': '129'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} + +Traceback (most recent call last): + File "/local/home/ibhard/efs-csi-driver-fork/aws-efs-csi-driver/test/e2e/stress-scale-tests/run_tests.py", line 209, in main + orchestrator = EFSCSIOrchestrator(config_file=orchestrator_config) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/local/home/ibhard/efs-csi-driver-fork/aws-efs-csi-driver/test/e2e/stress-scale-tests/tests/orchestrator.py", line 86, in __init__ + self._ensure_namespace_exists() + File "/local/home/ibhard/efs-csi-driver-fork/aws-efs-csi-driver/test/e2e/stress-scale-tests/tests/orchestrator.py", line 92, in _ensure_namespace_exists + self.core_v1.read_namespace(name=self.namespace) + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api/core_v1_api.py", line 22992, in read_namespace + return self.read_namespace_with_http_info(name, **kwargs) # noqa: E501 + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api/core_v1_api.py", line 23071, in read_namespace_with_http_info + return self.api_client.call_api( + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api_client.py", line 348, in call_api + return self.__call_api(resource_path, method, + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api_client.py", line 180, in __call_api + response_data = self.request( + ^^^^^^^^^^^^^ + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api_client.py", line 373, in request + return self.rest_client.GET(url, + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/rest.py", line 244, in GET + return self.request("GET", url, + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/rest.py", line 238, in request + raise ApiException(http_resp=r) +kubernetes.client.exceptions.ApiException: (401) +Reason: Unauthorized +HTTP response headers: HTTPHeaderDict({'Audit-Id': 'eb0b1f42-f08c-4b18-b778-a83d8eb055f7', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 14:13:21 GMT', 'Content-Length': '129'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} + + +2025-06-05 14:14:34,079 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-05 14:14:34,079 - __main__ - INFO - Verifying AWS credentials +2025-06-05 14:14:34,080 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 14:14:34,080 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 14:14:34,081 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 14:14:34,081 - __main__ - INFO - Running orchestrator stress test suite +2025-06-05 14:14:34,081 - __main__ - INFO - Starting orchestrator for 60 seconds +2025-06-05 14:14:34,081 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-05 14:14:34,816 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 14:14:35,029 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 14:14:35,029 - __main__ - INFO - Test duration overridden to 60 seconds +2025-06-05 14:14:35,029 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 20/s) +2025-06-05 14:14:35,029 - tests.orchestrator - INFO - Starting orchestrator test for 60 seconds +2025-06-05 14:14:35,218 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 14:14:35,219 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 14:14:35,219 - tests.orchestrator - INFO - Creating PVC: test-pvc-0673a51e +2025-06-05 14:14:35,294 - tests.orchestrator - INFO - Created PVC: test-pvc-0673a51e +2025-06-05 14:14:35,295 - tests.orchestrator - INFO - Waiting for PVC test-pvc-0673a51e to be bound +2025-06-05 14:14:37,429 - tests.orchestrator - INFO - PVC test-pvc-0673a51e is bound +2025-06-05 14:14:37,429 - tests.orchestrator - INFO - Creating PVC: test-pvc-ad878159 +2025-06-05 14:14:37,497 - tests.orchestrator - INFO - Created PVC: test-pvc-ad878159 +2025-06-05 14:14:37,497 - tests.orchestrator - INFO - Waiting for PVC test-pvc-ad878159 to be bound +2025-06-05 14:14:39,631 - tests.orchestrator - INFO - PVC test-pvc-ad878159 is bound +2025-06-05 14:14:39,632 - tests.orchestrator - INFO - Creating pod test-pod-97d4c158 on PVC test-pvc-ad878159 +2025-06-05 14:14:39,709 - tests.orchestrator - INFO - Created pod: test-pod-97d4c158 using PVC: test-pvc-ad878159 +2025-06-05 14:14:39,710 - tests.orchestrator - INFO - Waiting for pod test-pod-97d4c158 to be ready +2025-06-05 14:15:39,755 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-97d4c158 to be ready after 60s +2025-06-05 14:15:39,755 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-97d4c158 to be ready +2025-06-05 14:15:39,756 - tests.orchestrator - INFO - Creating pod test-pod-27dfc967 on PVC test-pvc-ad878159 +2025-06-05 14:15:39,836 - tests.orchestrator - INFO - Created pod: test-pod-27dfc967 using PVC: test-pvc-ad878159 +2025-06-05 14:15:39,837 - tests.orchestrator - INFO - Waiting for pod test-pod-27dfc967 to be ready +2025-06-05 14:16:39,914 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-27dfc967 to be ready after 60s +2025-06-05 14:16:39,915 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-27dfc967 to be ready +2025-06-05 14:16:39,915 - tests.orchestrator - INFO - Creating pod test-pod-41d8fee0 on PVC test-pvc-0673a51e +2025-06-05 14:16:39,996 - tests.orchestrator - INFO - Created pod: test-pod-41d8fee0 using PVC: test-pvc-0673a51e +2025-06-05 14:16:39,996 - tests.orchestrator - INFO - Waiting for pod test-pod-41d8fee0 to be ready +2025-06-05 14:17:42,051 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-41d8fee0 to be ready after 60s +2025-06-05 14:17:42,051 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-41d8fee0 to be ready +2025-06-05 14:17:42,051 - tests.orchestrator - INFO - Testing read/write between pods test-pod-97d4c158 and test-pod-27dfc967 sharing PVC test-pvc-ad878159 +2025-06-05 14:17:42,118 - tests.orchestrator - ERROR - Failed in read/write verification: (400) +Reason: Bad Request +HTTP response headers: HTTPHeaderDict({'Audit-Id': '82f9299f-d78b-4a35-8447-4b9382646785', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 14:17:42 GMT', 'Content-Length': '166'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-97d4c158 does not have a host assigned","reason":"BadRequest","code":400} + + +2025-06-05 14:17:42,118 - tests.orchestrator - INFO - Running specific scenario: _scenario_concurrent_pvc +2025-06-05 14:17:42,118 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations +2025-06-05 14:17:42,118 - tests.orchestrator - INFO - Creating 5 PVCs in quick succession +2025-06-05 14:17:42,197 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-af92f77f +2025-06-05 14:17:42,197 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-af92f77f to be bound +2025-06-05 14:17:44,333 - tests.orchestrator - INFO - PVC concurrent-pvc-af92f77f is bound +2025-06-05 14:17:44,403 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-2a91a331 +2025-06-05 14:17:44,403 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-2a91a331 to be bound +2025-06-05 14:17:46,537 - tests.orchestrator - INFO - PVC concurrent-pvc-2a91a331 is bound +2025-06-05 14:17:46,606 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-b13c79e0 +2025-06-05 14:17:46,606 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-b13c79e0 to be bound +2025-06-05 14:17:48,740 - tests.orchestrator - INFO - PVC concurrent-pvc-b13c79e0 is bound +2025-06-05 14:17:48,810 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-2e52864c +2025-06-05 14:17:48,810 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-2e52864c to be bound +2025-06-05 14:17:50,943 - tests.orchestrator - INFO - PVC concurrent-pvc-2e52864c is bound +2025-06-05 14:17:51,013 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-83f53c18 +2025-06-05 14:17:51,014 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-83f53c18 to be bound +2025-06-05 14:17:53,147 - tests.orchestrator - INFO - PVC concurrent-pvc-83f53c18 is bound +2025-06-05 14:17:53,147 - tests.orchestrator - INFO - Successfully created 5 PVCs +2025-06-05 14:17:53,147 - tests.orchestrator - INFO - Creating pod test-pod-783e20b8 on PVC concurrent-pvc-2a91a331 +2025-06-05 14:17:53,221 - tests.orchestrator - INFO - Created pod: test-pod-783e20b8 using PVC: concurrent-pvc-2a91a331 +2025-06-05 14:17:53,221 - tests.orchestrator - INFO - Waiting for pod test-pod-783e20b8 to be ready +2025-06-05 14:18:53,279 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-783e20b8 to be ready after 60s +2025-06-05 14:18:53,279 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-783e20b8 to be ready +2025-06-05 14:18:53,279 - tests.orchestrator - INFO - Creating pod test-pod-206a8855 on PVC concurrent-pvc-b13c79e0 +2025-06-05 14:18:53,362 - tests.orchestrator - INFO - Created pod: test-pod-206a8855 using PVC: concurrent-pvc-b13c79e0 +2025-06-05 14:18:53,362 - tests.orchestrator - INFO - Waiting for pod test-pod-206a8855 to be ready +2025-06-05 14:19:55,415 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-206a8855 to be ready after 60s +2025-06-05 14:19:55,416 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-206a8855 to be ready +2025-06-05 14:19:55,416 - tests.orchestrator - INFO - Creating pod test-pod-7df5d0f2 on PVC concurrent-pvc-af92f77f +2025-06-05 14:19:55,499 - tests.orchestrator - INFO - Created pod: test-pod-7df5d0f2 using PVC: concurrent-pvc-af92f77f +2025-06-05 14:19:55,499 - tests.orchestrator - INFO - Waiting for pod test-pod-7df5d0f2 to be ready +2025-06-05 14:20:57,532 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-7df5d0f2 to be ready after 60s +2025-06-05 14:20:57,532 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-7df5d0f2 to be ready +2025-06-05 14:20:57,532 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-2e52864c +2025-06-05 14:20:57,607 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-2e52864c to be deleted +2025-06-05 14:20:57,673 - tests.orchestrator - INFO - PVC concurrent-pvc-2e52864c has been deleted +2025-06-05 14:20:57,674 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-2e52864c +2025-06-05 14:20:57,674 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-af92f77f +2025-06-05 14:20:57,675 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-af92f77f +2025-06-05 14:20:57,675 - tests.orchestrator - INFO - Deleting pod: test-pod-7df5d0f2 from PVC: concurrent-pvc-af92f77f +2025-06-05 14:20:57,761 - tests.orchestrator - INFO - Waiting for pod test-pod-7df5d0f2 to be deleted +2025-06-05 14:20:57,827 - tests.orchestrator - INFO - Pod test-pod-7df5d0f2 has been deleted +2025-06-05 14:20:57,827 - tests.orchestrator - INFO - Deleted pod: test-pod-7df5d0f2 +2025-06-05 14:20:57,901 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-af92f77f to be deleted +2025-06-05 14:20:57,969 - tests.orchestrator - INFO - PVC concurrent-pvc-af92f77f has been deleted +2025-06-05 14:20:57,969 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-af92f77f +2025-06-05 14:20:57,969 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-83f53c18 +2025-06-05 14:20:58,041 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-83f53c18 to be deleted +2025-06-05 14:20:58,108 - tests.orchestrator - INFO - PVC concurrent-pvc-83f53c18 has been deleted +2025-06-05 14:20:58,108 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-83f53c18 +2025-06-05 14:20:58,108 - tests.orchestrator - INFO - Rapid PVC scenario completed +2025-06-05 14:20:58,108 - tests.orchestrator - INFO - Deleting pod: test-pod-97d4c158 from PVC: test-pvc-ad878159 +2025-06-05 14:20:58,188 - tests.orchestrator - INFO - Waiting for pod test-pod-97d4c158 to be deleted +2025-06-05 14:20:58,254 - tests.orchestrator - INFO - Pod test-pod-97d4c158 has been deleted +2025-06-05 14:20:58,254 - tests.orchestrator - INFO - Deleted pod: test-pod-97d4c158 +2025-06-05 14:20:58,254 - tests.orchestrator - INFO - Deleting PVC: test-pvc-ad878159 +2025-06-05 14:20:58,255 - tests.orchestrator - INFO - Deleting 1 pods using PVC test-pvc-ad878159 +2025-06-05 14:20:58,255 - tests.orchestrator - INFO - Deleting pod: test-pod-27dfc967 from PVC: test-pvc-ad878159 +2025-06-05 14:20:58,341 - tests.orchestrator - INFO - Waiting for pod test-pod-27dfc967 to be deleted +2025-06-05 14:20:58,407 - tests.orchestrator - INFO - Pod test-pod-27dfc967 has been deleted +2025-06-05 14:20:58,407 - tests.orchestrator - INFO - Deleted pod: test-pod-27dfc967 +2025-06-05 14:20:58,479 - tests.orchestrator - INFO - Waiting for PVC test-pvc-ad878159 to be deleted +2025-06-05 14:20:58,546 - tests.orchestrator - INFO - PVC test-pvc-ad878159 has been deleted +2025-06-05 14:20:58,546 - tests.orchestrator - INFO - Deleted PVC: test-pvc-ad878159 +2025-06-05 14:20:58,546 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations +2025-06-05 14:20:58,546 - tests.orchestrator - INFO - Test completed in 383.52 seconds +2025-06-05 14:20:58,547 - tests.orchestrator - INFO - Cleaning up resources... +2025-06-05 14:20:58,547 - tests.orchestrator - INFO - Deleting pod: test-pod-41d8fee0 from PVC: test-pvc-0673a51e +2025-06-05 14:20:58,633 - tests.orchestrator - INFO - Waiting for pod test-pod-41d8fee0 to be deleted +2025-06-05 14:20:58,700 - tests.orchestrator - INFO - Pod test-pod-41d8fee0 has been deleted +2025-06-05 14:20:58,700 - tests.orchestrator - INFO - Deleted pod: test-pod-41d8fee0 +2025-06-05 14:20:58,700 - tests.orchestrator - INFO - Deleting pod: test-pod-783e20b8 from PVC: concurrent-pvc-2a91a331 +2025-06-05 14:20:58,781 - tests.orchestrator - INFO - Waiting for pod test-pod-783e20b8 to be deleted +2025-06-05 14:20:58,847 - tests.orchestrator - INFO - Pod test-pod-783e20b8 has been deleted +2025-06-05 14:20:58,847 - tests.orchestrator - INFO - Deleted pod: test-pod-783e20b8 +2025-06-05 14:20:58,847 - tests.orchestrator - INFO - Deleting pod: test-pod-206a8855 from PVC: concurrent-pvc-b13c79e0 +2025-06-05 14:20:58,925 - tests.orchestrator - INFO - Waiting for pod test-pod-206a8855 to be deleted +2025-06-05 14:20:58,991 - tests.orchestrator - INFO - Pod test-pod-206a8855 has been deleted +2025-06-05 14:20:58,992 - tests.orchestrator - INFO - Deleted pod: test-pod-206a8855 +2025-06-05 14:20:58,992 - tests.orchestrator - INFO - Deleting PVC: test-pvc-0673a51e +2025-06-05 14:20:59,063 - tests.orchestrator - INFO - Waiting for PVC test-pvc-0673a51e to be deleted +2025-06-05 14:20:59,130 - tests.orchestrator - INFO - PVC test-pvc-0673a51e has been deleted +2025-06-05 14:20:59,130 - tests.orchestrator - INFO - Deleted PVC: test-pvc-0673a51e +2025-06-05 14:20:59,130 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-2a91a331 +2025-06-05 14:20:59,204 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-2a91a331 to be deleted +2025-06-05 14:20:59,270 - tests.orchestrator - INFO - PVC concurrent-pvc-2a91a331 has been deleted +2025-06-05 14:20:59,270 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-2a91a331 +2025-06-05 14:20:59,270 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-b13c79e0 +2025-06-05 14:20:59,344 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-b13c79e0 to be deleted +2025-06-05 14:20:59,410 - tests.orchestrator - INFO - PVC concurrent-pvc-b13c79e0 has been deleted +2025-06-05 14:20:59,410 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-b13c79e0 +2025-06-05 14:20:59,410 - tests.orchestrator - INFO - Cleanup completed +2025-06-05 14:20:59,410 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-05 14:20:59,410 - tests.orchestrator - INFO - --- Operations --- +2025-06-05 14:20:59,410 - tests.orchestrator - INFO - create_pvc: 7 succeeded, 0 failed (100.0%) +2025-06-05 14:20:59,410 - tests.orchestrator - INFO - attach_pod: 6 succeeded, 0 failed (100.0%) +2025-06-05 14:20:59,410 - tests.orchestrator - INFO - delete_pod: 6 succeeded, 0 failed (100.0%) +2025-06-05 14:20:59,410 - tests.orchestrator - INFO - delete_pvc: 7 succeeded, 0 failed (100.0%) +2025-06-05 14:20:59,410 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) +2025-06-05 14:20:59,411 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) +2025-06-05 14:20:59,411 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-05 14:20:59,411 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-05 14:20:59,411 - tests.orchestrator - INFO - many_to_one: No runs +2025-06-05 14:20:59,411 - tests.orchestrator - INFO - one_to_one: No runs +2025-06-05 14:20:59,411 - tests.orchestrator - INFO - concurrent_pvc: 1 succeeded, 0 failed out of 1 runs (100.0%) +2025-06-05 14:20:59,411 - tests.orchestrator - INFO - ========================================= +2025-06-05 14:20:59,411 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250605_142059.json +2025-06-05 14:20:59,453 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250605_142059_20250605_142059.json +2025-06-05 14:20:59,453 - __main__ - INFO - All tests completed successfully +2025-06-05 14:39:25,645 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-05 14:39:25,645 - __main__ - INFO - Verifying AWS credentials +2025-06-05 14:39:25,646 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 14:39:25,647 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 14:39:25,647 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 14:39:25,647 - __main__ - INFO - Running orchestrator stress test suite +2025-06-05 14:39:25,647 - __main__ - INFO - Starting orchestrator for 300 seconds +2025-06-05 14:39:25,647 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-05 14:39:26,410 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 14:39:26,657 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 14:39:26,657 - __main__ - INFO - Test duration overridden to 300 seconds +2025-06-05 14:39:26,657 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) +2025-06-05 14:39:26,657 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-05 14:39:26,852 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 14:39:26,853 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 14:39:26,853 - tests.orchestrator - INFO - Creating PVC: test-pvc-e5985cb1 +2025-06-05 14:39:26,928 - tests.orchestrator - INFO - Created PVC: test-pvc-e5985cb1 +2025-06-05 14:39:26,928 - tests.orchestrator - INFO - Waiting for PVC test-pvc-e5985cb1 to be bound +2025-06-05 14:39:29,061 - tests.orchestrator - INFO - PVC test-pvc-e5985cb1 is bound +2025-06-05 14:39:29,061 - tests.orchestrator - INFO - Creating PVC: test-pvc-83426f91 +2025-06-05 14:39:29,130 - tests.orchestrator - INFO - Created PVC: test-pvc-83426f91 +2025-06-05 14:39:29,131 - tests.orchestrator - INFO - Waiting for PVC test-pvc-83426f91 to be bound +2025-06-05 14:39:31,262 - tests.orchestrator - INFO - PVC test-pvc-83426f91 is bound +2025-06-05 14:39:31,263 - tests.orchestrator - INFO - Creating pod test-pod-d86424ba on PVC test-pvc-83426f91 +2025-06-05 14:39:31,342 - tests.orchestrator - INFO - Created pod: test-pod-d86424ba using PVC: test-pvc-83426f91 +2025-06-05 14:39:31,342 - tests.orchestrator - INFO - Waiting for pod test-pod-d86424ba to be ready +2025-06-05 14:40:31,371 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-d86424ba to be ready after 60s +2025-06-05 14:40:31,371 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-d86424ba to be ready +2025-06-05 14:40:31,371 - tests.orchestrator - INFO - Creating pod test-pod-b7e9d0a0 on PVC test-pvc-e5985cb1 +2025-06-05 14:40:31,454 - tests.orchestrator - INFO - Created pod: test-pod-b7e9d0a0 using PVC: test-pvc-e5985cb1 +2025-06-05 14:40:31,455 - tests.orchestrator - INFO - Waiting for pod test-pod-b7e9d0a0 to be ready +2025-06-05 14:41:31,485 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b7e9d0a0 to be ready after 60s +2025-06-05 14:41:31,485 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b7e9d0a0 to be ready +2025-06-05 14:41:31,485 - tests.orchestrator - INFO - Creating pod test-pod-c947d819 on PVC test-pvc-e5985cb1 +2025-06-05 14:41:31,586 - tests.orchestrator - INFO - Created pod: test-pod-c947d819 using PVC: test-pvc-e5985cb1 +2025-06-05 14:41:31,586 - tests.orchestrator - INFO - Waiting for pod test-pod-c947d819 to be ready +2025-06-05 14:42:33,647 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-c947d819 to be ready after 60s +2025-06-05 14:42:33,647 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-c947d819 to be ready +2025-06-05 14:42:33,647 - tests.orchestrator - INFO - Testing read/write between pods test-pod-b7e9d0a0 and test-pod-c947d819 sharing PVC test-pvc-e5985cb1 +2025-06-05 14:42:33,714 - tests.orchestrator - ERROR - Failed in read/write verification: (400) +Reason: Bad Request +HTTP response headers: HTTPHeaderDict({'Audit-Id': '8b1165bc-48b9-433e-8654-68c6095da9bd', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 14:42:33 GMT', 'Content-Length': '166'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-b7e9d0a0 does not have a host assigned","reason":"BadRequest","code":400} + + +2025-06-05 14:42:33,714 - tests.orchestrator - INFO - Running specific scenario: _scenario_concurrent_pvc +2025-06-05 14:42:33,714 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations +2025-06-05 14:42:33,714 - tests.orchestrator - INFO - Creating 7 PVCs in quick succession +2025-06-05 14:42:33,792 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-f6a663b2 +2025-06-05 14:42:33,792 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-f6a663b2 to be bound +2025-06-05 14:42:35,924 - tests.orchestrator - INFO - PVC concurrent-pvc-f6a663b2 is bound +2025-06-05 14:42:35,993 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-903eb13a +2025-06-05 14:42:35,994 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-903eb13a to be bound +2025-06-05 14:42:38,128 - tests.orchestrator - INFO - PVC concurrent-pvc-903eb13a is bound +2025-06-05 14:42:38,196 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-ba7e5e4d +2025-06-05 14:42:38,196 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-ba7e5e4d to be bound +2025-06-05 14:42:40,331 - tests.orchestrator - INFO - PVC concurrent-pvc-ba7e5e4d is bound +2025-06-05 14:42:40,397 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-5b29e9cd +2025-06-05 14:42:40,397 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-5b29e9cd to be bound +2025-06-05 14:42:42,533 - tests.orchestrator - INFO - PVC concurrent-pvc-5b29e9cd is bound +2025-06-05 14:42:42,603 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-248d7584 +2025-06-05 14:42:42,603 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-248d7584 to be bound +2025-06-05 14:42:44,740 - tests.orchestrator - INFO - PVC concurrent-pvc-248d7584 is bound +2025-06-05 14:42:44,808 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-0aa045b0 +2025-06-05 14:42:44,808 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-0aa045b0 to be bound +2025-06-05 14:42:46,944 - tests.orchestrator - INFO - PVC concurrent-pvc-0aa045b0 is bound +2025-06-05 14:42:47,013 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-b0004a21 +2025-06-05 14:42:47,013 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-b0004a21 to be bound +2025-06-05 14:42:49,147 - tests.orchestrator - INFO - PVC concurrent-pvc-b0004a21 is bound +2025-06-05 14:42:49,148 - tests.orchestrator - INFO - Successfully created 7 PVCs +2025-06-05 14:42:49,148 - tests.orchestrator - INFO - Creating pod test-pod-edd264c5 on PVC concurrent-pvc-0aa045b0 +2025-06-05 14:42:49,220 - tests.orchestrator - INFO - Created pod: test-pod-edd264c5 using PVC: concurrent-pvc-0aa045b0 +2025-06-05 14:42:49,220 - tests.orchestrator - INFO - Waiting for pod test-pod-edd264c5 to be ready +2025-06-05 14:43:49,221 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-edd264c5 to be ready after 60s +2025-06-05 14:43:49,221 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-edd264c5 to be ready +2025-06-05 14:43:49,221 - tests.orchestrator - INFO - Creating pod test-pod-9f0f38ef on PVC concurrent-pvc-248d7584 +2025-06-05 14:43:49,306 - tests.orchestrator - INFO - Created pod: test-pod-9f0f38ef using PVC: concurrent-pvc-248d7584 +2025-06-05 14:43:49,306 - tests.orchestrator - INFO - Waiting for pod test-pod-9f0f38ef to be ready +2025-06-05 14:44:51,350 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9f0f38ef to be ready after 60s +2025-06-05 14:44:51,351 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9f0f38ef to be ready +2025-06-05 14:44:51,351 - tests.orchestrator - INFO - Creating pod test-pod-6bde4264 on PVC concurrent-pvc-903eb13a +2025-06-05 14:44:51,437 - tests.orchestrator - INFO - Created pod: test-pod-6bde4264 using PVC: concurrent-pvc-903eb13a +2025-06-05 14:44:51,437 - tests.orchestrator - INFO - Waiting for pod test-pod-6bde4264 to be ready +2025-06-05 14:51:32,599 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-05 14:51:32,599 - __main__ - INFO - Verifying AWS credentials +2025-06-05 14:51:32,600 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 14:51:32,601 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 14:51:32,601 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 14:51:32,601 - __main__ - INFO - Running orchestrator stress test suite +2025-06-05 14:51:32,601 - __main__ - INFO - Starting orchestrator for 300 seconds +2025-06-05 14:51:32,602 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-05 14:51:33,341 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 14:51:33,563 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 14:51:33,563 - __main__ - INFO - Test duration overridden to 300 seconds +2025-06-05 14:51:33,563 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) +2025-06-05 14:51:33,563 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-05 14:51:33,753 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 14:51:33,753 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 14:51:33,754 - tests.orchestrator - INFO - Creating PVC: test-pvc-49392d7c +2025-06-05 14:51:33,831 - tests.orchestrator - INFO - Created PVC: test-pvc-49392d7c +2025-06-05 14:51:33,831 - tests.orchestrator - INFO - Waiting for PVC test-pvc-49392d7c to be bound +2025-06-05 14:51:35,964 - tests.orchestrator - INFO - PVC test-pvc-49392d7c is bound +2025-06-05 14:51:35,966 - tests.orchestrator - INFO - Creating PVC: test-pvc-5e912838 +2025-06-05 14:51:36,038 - tests.orchestrator - INFO - Created PVC: test-pvc-5e912838 +2025-06-05 14:51:36,038 - tests.orchestrator - INFO - Waiting for PVC test-pvc-5e912838 to be bound +2025-06-05 14:51:38,171 - tests.orchestrator - INFO - PVC test-pvc-5e912838 is bound +2025-06-05 14:51:38,171 - tests.orchestrator - INFO - Creating pod test-pod-a16fe7e2 on PVC test-pvc-5e912838 +2025-06-05 14:51:38,249 - tests.orchestrator - INFO - Created pod: test-pod-a16fe7e2 using PVC: test-pvc-5e912838 +2025-06-05 14:51:38,250 - tests.orchestrator - INFO - Waiting for pod test-pod-a16fe7e2 to be ready +2025-06-05 14:52:40,287 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a16fe7e2 to be ready after 60s +2025-06-05 14:52:40,287 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a16fe7e2 to be ready +2025-06-05 14:52:40,288 - tests.orchestrator - INFO - Creating pod test-pod-6cc653e7 on PVC test-pvc-49392d7c +2025-06-05 14:52:40,370 - tests.orchestrator - INFO - Created pod: test-pod-6cc653e7 using PVC: test-pvc-49392d7c +2025-06-05 14:52:40,370 - tests.orchestrator - INFO - Waiting for pod test-pod-6cc653e7 to be ready +2025-06-05 14:53:42,411 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-6cc653e7 to be ready after 60s +2025-06-05 14:53:42,411 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-6cc653e7 to be ready +2025-06-05 14:53:42,411 - tests.orchestrator - INFO - Creating pod test-pod-807f64e5 on PVC test-pvc-49392d7c +2025-06-05 14:53:42,494 - tests.orchestrator - INFO - Created pod: test-pod-807f64e5 using PVC: test-pvc-49392d7c +2025-06-05 14:53:42,494 - tests.orchestrator - INFO - Waiting for pod test-pod-807f64e5 to be ready +2025-06-05 14:54:44,529 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-807f64e5 to be ready after 60s +2025-06-05 14:54:44,529 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-807f64e5 to be ready +2025-06-05 14:54:44,529 - tests.orchestrator - INFO - Testing read/write between pods test-pod-807f64e5 and test-pod-6cc653e7 sharing PVC test-pvc-49392d7c +2025-06-05 14:54:44,596 - tests.orchestrator - ERROR - Failed in read/write verification: (400) +Reason: Bad Request +HTTP response headers: HTTPHeaderDict({'Audit-Id': '12fe6240-5ebe-4564-8452-9306b897c940', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 14:54:44 GMT', 'Content-Length': '166'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-807f64e5 does not have a host assigned","reason":"BadRequest","code":400} + + +2025-06-05 14:54:44,596 - tests.orchestrator - INFO - Running specific scenario: _scenario_many_to_one +2025-06-05 14:54:44,597 - tests.orchestrator - INFO - Running scenario: Many pods to one PVC +2025-06-05 14:54:44,678 - tests.orchestrator - INFO - Waiting for PVC many2one-7a10cfd9 to be bound +2025-06-05 14:54:46,810 - tests.orchestrator - INFO - PVC many2one-7a10cfd9 is bound +2025-06-05 14:54:46,811 - tests.orchestrator - INFO - Creating 3 pods for the same PVC +2025-06-05 14:54:46,811 - tests.orchestrator - INFO - Creating pod test-pod-6d648a79 on PVC many2one-7a10cfd9 +2025-06-05 14:54:46,884 - tests.orchestrator - INFO - Created pod: test-pod-6d648a79 using PVC: many2one-7a10cfd9 +2025-06-05 14:54:46,884 - tests.orchestrator - INFO - Waiting for pod test-pod-6d648a79 to be ready +2025-06-05 14:55:48,914 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-6d648a79 to be ready after 60s +2025-06-05 14:55:48,915 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-6d648a79 to be ready +2025-06-05 14:55:48,915 - tests.orchestrator - INFO - Creating pod test-pod-264dcb36 on PVC many2one-7a10cfd9 +2025-06-05 14:55:48,997 - tests.orchestrator - INFO - Created pod: test-pod-264dcb36 using PVC: many2one-7a10cfd9 +2025-06-05 14:55:48,997 - tests.orchestrator - INFO - Waiting for pod test-pod-264dcb36 to be ready +2025-06-05 14:56:49,023 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-264dcb36 to be ready after 60s +2025-06-05 14:56:49,024 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-264dcb36 to be ready +2025-06-05 14:56:49,024 - tests.orchestrator - INFO - Creating pod test-pod-b6a62203 on PVC many2one-7a10cfd9 +2025-06-05 14:56:49,106 - tests.orchestrator - INFO - Created pod: test-pod-b6a62203 using PVC: many2one-7a10cfd9 +2025-06-05 14:56:49,106 - tests.orchestrator - INFO - Waiting for pod test-pod-b6a62203 to be ready +2025-06-05 14:57:49,129 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b6a62203 to be ready after 60s +2025-06-05 14:57:49,129 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b6a62203 to be ready +2025-06-05 14:57:49,129 - tests.orchestrator - WARNING - Failed to create enough pods for scenario, only created 0 +2025-06-05 14:57:49,130 - tests.orchestrator - INFO - Deleting pod: test-pod-6cc653e7 from PVC: test-pvc-49392d7c +2025-06-05 14:57:49,207 - tests.orchestrator - INFO - Waiting for pod test-pod-6cc653e7 to be deleted +2025-06-05 14:57:49,272 - tests.orchestrator - INFO - Pod test-pod-6cc653e7 has been deleted +2025-06-05 14:57:49,272 - tests.orchestrator - INFO - Deleted pod: test-pod-6cc653e7 +2025-06-05 14:57:49,272 - tests.orchestrator - INFO - Deleting PVC: many2one-7a10cfd9 +2025-06-05 14:57:49,272 - tests.orchestrator - INFO - Deleting 3 pods using PVC many2one-7a10cfd9 +2025-06-05 14:57:49,272 - tests.orchestrator - INFO - Deleting pod: test-pod-6d648a79 from PVC: many2one-7a10cfd9 +2025-06-05 14:57:49,348 - tests.orchestrator - INFO - Waiting for pod test-pod-6d648a79 to be deleted +2025-06-05 14:57:49,414 - tests.orchestrator - INFO - Pod test-pod-6d648a79 has been deleted +2025-06-05 14:57:49,415 - tests.orchestrator - INFO - Deleted pod: test-pod-6d648a79 +2025-06-05 14:57:49,415 - tests.orchestrator - INFO - Deleting pod: test-pod-264dcb36 from PVC: many2one-7a10cfd9 +2025-06-05 14:57:49,498 - tests.orchestrator - INFO - Waiting for pod test-pod-264dcb36 to be deleted +2025-06-05 14:57:49,563 - tests.orchestrator - INFO - Pod test-pod-264dcb36 has been deleted +2025-06-05 14:57:49,564 - tests.orchestrator - INFO - Deleted pod: test-pod-264dcb36 +2025-06-05 14:57:49,564 - tests.orchestrator - INFO - Deleting pod: test-pod-b6a62203 from PVC: many2one-7a10cfd9 +2025-06-05 14:57:49,643 - tests.orchestrator - INFO - Waiting for pod test-pod-b6a62203 to be deleted +2025-06-05 14:57:49,710 - tests.orchestrator - INFO - Pod test-pod-b6a62203 has been deleted +2025-06-05 14:57:49,710 - tests.orchestrator - INFO - Deleted pod: test-pod-b6a62203 +2025-06-05 14:57:49,782 - tests.orchestrator - INFO - Waiting for PVC many2one-7a10cfd9 to be deleted +2025-06-05 14:57:49,848 - tests.orchestrator - INFO - PVC many2one-7a10cfd9 has been deleted +2025-06-05 14:57:49,848 - tests.orchestrator - INFO - Deleted PVC: many2one-7a10cfd9 +2025-06-05 14:57:49,848 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations +2025-06-05 14:57:49,849 - tests.orchestrator - INFO - Test completed in 376.28 seconds +2025-06-05 14:57:49,849 - tests.orchestrator - INFO - Cleaning up resources... +2025-06-05 14:57:49,849 - tests.orchestrator - INFO - Deleting pod: test-pod-807f64e5 from PVC: test-pvc-49392d7c +2025-06-05 14:57:49,935 - tests.orchestrator - INFO - Waiting for pod test-pod-807f64e5 to be deleted +2025-06-05 14:57:50,000 - tests.orchestrator - INFO - Pod test-pod-807f64e5 has been deleted +2025-06-05 14:57:50,001 - tests.orchestrator - INFO - Deleted pod: test-pod-807f64e5 +2025-06-05 14:57:50,001 - tests.orchestrator - INFO - Deleting pod: test-pod-a16fe7e2 from PVC: test-pvc-5e912838 +2025-06-05 14:57:50,081 - tests.orchestrator - INFO - Waiting for pod test-pod-a16fe7e2 to be deleted +2025-06-05 14:57:50,147 - tests.orchestrator - INFO - Pod test-pod-a16fe7e2 has been deleted +2025-06-05 14:57:50,147 - tests.orchestrator - INFO - Deleted pod: test-pod-a16fe7e2 +2025-06-05 14:57:50,147 - tests.orchestrator - INFO - Deleting PVC: test-pvc-49392d7c +2025-06-05 14:57:50,221 - tests.orchestrator - INFO - Waiting for PVC test-pvc-49392d7c to be deleted +2025-06-05 14:57:50,287 - tests.orchestrator - INFO - PVC test-pvc-49392d7c has been deleted +2025-06-05 14:57:50,288 - tests.orchestrator - INFO - Deleted PVC: test-pvc-49392d7c +2025-06-05 14:57:50,288 - tests.orchestrator - INFO - Deleting PVC: test-pvc-5e912838 +2025-06-05 14:57:50,360 - tests.orchestrator - INFO - Waiting for PVC test-pvc-5e912838 to be deleted +2025-06-05 14:57:50,426 - tests.orchestrator - INFO - PVC test-pvc-5e912838 has been deleted +2025-06-05 14:57:50,426 - tests.orchestrator - INFO - Deleted PVC: test-pvc-5e912838 +2025-06-05 14:57:50,426 - tests.orchestrator - INFO - Cleanup completed +2025-06-05 14:57:50,426 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-05 14:57:50,426 - tests.orchestrator - INFO - --- Operations --- +2025-06-05 14:57:50,426 - tests.orchestrator - INFO - create_pvc: 2 succeeded, 0 failed (100.0%) +2025-06-05 14:57:50,426 - tests.orchestrator - INFO - attach_pod: 6 succeeded, 0 failed (100.0%) +2025-06-05 14:57:50,427 - tests.orchestrator - INFO - delete_pod: 6 succeeded, 0 failed (100.0%) +2025-06-05 14:57:50,427 - tests.orchestrator - INFO - delete_pvc: 3 succeeded, 0 failed (100.0%) +2025-06-05 14:57:50,427 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) +2025-06-05 14:57:50,427 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) +2025-06-05 14:57:50,427 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-05 14:57:50,427 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-05 14:57:50,427 - tests.orchestrator - INFO - many_to_one: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-05 14:57:50,427 - tests.orchestrator - INFO - one_to_one: No runs +2025-06-05 14:57:50,427 - tests.orchestrator - INFO - concurrent_pvc: No runs +2025-06-05 14:57:50,427 - tests.orchestrator - INFO - ========================================= +2025-06-05 14:57:50,428 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250605_145750.json +2025-06-05 14:57:50,470 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250605_145750_20250605_145750.json +2025-06-05 14:57:50,470 - __main__ - INFO - All tests completed successfully +2025-06-05 14:59:37,084 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-05 14:59:37,084 - __main__ - INFO - Verifying AWS credentials +2025-06-05 14:59:37,085 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 14:59:37,085 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 14:59:37,086 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 14:59:37,086 - __main__ - INFO - Running orchestrator stress test suite +2025-06-05 14:59:37,086 - __main__ - INFO - Starting orchestrator for 300 seconds +2025-06-05 14:59:37,086 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-05 14:59:37,825 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 14:59:38,037 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 14:59:38,038 - __main__ - INFO - Test duration overridden to 300 seconds +2025-06-05 14:59:38,038 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) +2025-06-05 14:59:38,038 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-05 14:59:38,227 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 14:59:38,227 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 14:59:38,227 - tests.orchestrator - INFO - Creating PVC: test-pvc-fe252ee6 +2025-06-05 14:59:38,302 - tests.orchestrator - INFO - Created PVC: test-pvc-fe252ee6 +2025-06-05 14:59:38,303 - tests.orchestrator - INFO - Waiting for PVC test-pvc-fe252ee6 to be bound +2025-06-05 14:59:40,435 - tests.orchestrator - INFO - PVC test-pvc-fe252ee6 is bound +2025-06-05 14:59:40,435 - tests.orchestrator - INFO - Creating PVC: test-pvc-4a7f89b4 +2025-06-05 14:59:40,502 - tests.orchestrator - INFO - Created PVC: test-pvc-4a7f89b4 +2025-06-05 14:59:40,502 - tests.orchestrator - INFO - Waiting for PVC test-pvc-4a7f89b4 to be bound +2025-06-05 14:59:42,634 - tests.orchestrator - INFO - PVC test-pvc-4a7f89b4 is bound +2025-06-05 14:59:42,635 - tests.orchestrator - INFO - Creating pod test-pod-5311b1cc on PVC test-pvc-4a7f89b4 +2025-06-05 14:59:42,712 - tests.orchestrator - INFO - Created pod: test-pod-5311b1cc using PVC: test-pvc-4a7f89b4 +2025-06-05 14:59:42,712 - tests.orchestrator - INFO - Waiting for pod test-pod-5311b1cc to be ready +2025-06-05 15:00:44,719 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-5311b1cc to be ready after 60s +2025-06-05 15:00:44,719 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-5311b1cc to be ready +2025-06-05 15:00:44,720 - tests.orchestrator - INFO - Creating pod test-pod-71d7f585 on PVC test-pvc-fe252ee6 +2025-06-05 15:00:44,803 - tests.orchestrator - INFO - Created pod: test-pod-71d7f585 using PVC: test-pvc-fe252ee6 +2025-06-05 15:00:44,803 - tests.orchestrator - INFO - Waiting for pod test-pod-71d7f585 to be ready +2025-06-05 15:01:46,811 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-71d7f585 to be ready after 60s +2025-06-05 15:01:46,811 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-71d7f585 to be ready +2025-06-05 15:01:46,812 - tests.orchestrator - INFO - Creating pod test-pod-cc8d2266 on PVC test-pvc-4a7f89b4 +2025-06-05 15:01:46,893 - tests.orchestrator - INFO - Created pod: test-pod-cc8d2266 using PVC: test-pvc-4a7f89b4 +2025-06-05 15:01:46,894 - tests.orchestrator - INFO - Waiting for pod test-pod-cc8d2266 to be ready +2025-06-05 15:02:48,905 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-cc8d2266 to be ready after 60s +2025-06-05 15:02:48,905 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-cc8d2266 to be ready +2025-06-05 15:02:48,905 - tests.orchestrator - INFO - Testing read/write between pods test-pod-5311b1cc and test-pod-cc8d2266 sharing PVC test-pvc-4a7f89b4 +2025-06-05 15:02:48,971 - tests.orchestrator - ERROR - Failed in read/write verification: (400) +Reason: Bad Request +HTTP response headers: HTTPHeaderDict({'Audit-Id': '6fd01cb2-cb35-43e3-8bf2-4cba5eccfd8d', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 15:02:48 GMT', 'Content-Length': '166'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-5311b1cc does not have a host assigned","reason":"BadRequest","code":400} + + +2025-06-05 15:02:48,971 - tests.orchestrator - INFO - ============================================================ +2025-06-05 15:02:48,971 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_concurrent_pvc +2025-06-05 15:02:48,971 - tests.orchestrator - INFO - ============================================================ +2025-06-05 15:02:48,972 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations +2025-06-05 15:02:48,972 - tests.orchestrator - INFO - Creating 3 PVCs in quick succession +2025-06-05 15:02:49,049 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-02666bec +2025-06-05 15:02:49,049 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-02666bec to be bound +2025-06-05 15:02:51,181 - tests.orchestrator - INFO - PVC concurrent-pvc-02666bec is bound +2025-06-05 15:02:51,248 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-6592afb4 +2025-06-05 15:02:51,249 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-6592afb4 to be bound +2025-06-05 15:02:53,380 - tests.orchestrator - INFO - PVC concurrent-pvc-6592afb4 is bound +2025-06-05 15:02:53,448 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-2386ec8a +2025-06-05 15:02:53,449 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-2386ec8a to be bound +2025-06-05 15:02:55,580 - tests.orchestrator - INFO - PVC concurrent-pvc-2386ec8a is bound +2025-06-05 15:02:55,580 - tests.orchestrator - INFO - Successfully created 3 PVCs +2025-06-05 15:02:55,580 - tests.orchestrator - INFO - Creating pod test-pod-27ffffcb on PVC concurrent-pvc-2386ec8a +2025-06-05 15:02:55,653 - tests.orchestrator - INFO - Created pod: test-pod-27ffffcb using PVC: concurrent-pvc-2386ec8a +2025-06-05 15:02:55,653 - tests.orchestrator - INFO - Waiting for pod test-pod-27ffffcb to be ready +2025-06-05 15:03:57,662 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-27ffffcb to be ready after 60s +2025-06-05 15:03:57,662 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-27ffffcb to be ready +2025-06-05 15:03:57,662 - tests.orchestrator - INFO - Creating pod test-pod-00a03fb8 on PVC concurrent-pvc-02666bec +2025-06-05 15:03:57,743 - tests.orchestrator - INFO - Created pod: test-pod-00a03fb8 using PVC: concurrent-pvc-02666bec +2025-06-05 15:03:57,744 - tests.orchestrator - INFO - Waiting for pod test-pod-00a03fb8 to be ready +2025-06-05 15:04:59,752 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-00a03fb8 to be ready after 60s +2025-06-05 15:04:59,753 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-00a03fb8 to be ready +2025-06-05 15:04:59,753 - tests.orchestrator - INFO - Creating pod test-pod-87658b4a on PVC concurrent-pvc-6592afb4 +2025-06-05 15:04:59,832 - tests.orchestrator - INFO - Created pod: test-pod-87658b4a using PVC: concurrent-pvc-6592afb4 +2025-06-05 15:04:59,832 - tests.orchestrator - INFO - Waiting for pod test-pod-87658b4a to be ready +2025-06-05 15:06:01,847 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-87658b4a to be ready after 60s +2025-06-05 15:06:01,847 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-87658b4a to be ready +2025-06-05 15:06:01,847 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-6592afb4 +2025-06-05 15:06:01,847 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-6592afb4 +2025-06-05 15:06:01,847 - tests.orchestrator - INFO - Deleting pod: test-pod-87658b4a from PVC: concurrent-pvc-6592afb4 +2025-06-05 15:06:01,968 - tests.orchestrator - INFO - Waiting for pod test-pod-87658b4a to be deleted +2025-06-05 15:06:02,040 - tests.orchestrator - INFO - Pod test-pod-87658b4a has been deleted +2025-06-05 15:06:02,041 - tests.orchestrator - INFO - Deleted pod: test-pod-87658b4a +2025-06-05 15:06:02,126 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-6592afb4 to be deleted +2025-06-05 15:06:02,196 - tests.orchestrator - INFO - PVC concurrent-pvc-6592afb4 has been deleted +2025-06-05 15:06:02,196 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-6592afb4 +2025-06-05 15:06:02,196 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-02666bec +2025-06-05 15:06:02,196 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-02666bec +2025-06-05 15:06:02,196 - tests.orchestrator - INFO - Deleting pod: test-pod-00a03fb8 from PVC: concurrent-pvc-02666bec +2025-06-05 15:06:02,295 - tests.orchestrator - INFO - Waiting for pod test-pod-00a03fb8 to be deleted +2025-06-05 15:06:02,359 - tests.orchestrator - INFO - Pod test-pod-00a03fb8 has been deleted +2025-06-05 15:06:02,360 - tests.orchestrator - INFO - Deleted pod: test-pod-00a03fb8 +2025-06-05 15:06:02,433 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-02666bec to be deleted +2025-06-05 15:06:02,497 - tests.orchestrator - INFO - PVC concurrent-pvc-02666bec has been deleted +2025-06-05 15:06:02,498 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-02666bec +2025-06-05 15:06:02,498 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-2386ec8a +2025-06-05 15:06:02,498 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-2386ec8a +2025-06-05 15:06:02,498 - tests.orchestrator - INFO - Deleting pod: test-pod-27ffffcb from PVC: concurrent-pvc-2386ec8a +2025-06-05 15:06:02,575 - tests.orchestrator - INFO - Waiting for pod test-pod-27ffffcb to be deleted +2025-06-05 15:06:02,640 - tests.orchestrator - INFO - Pod test-pod-27ffffcb has been deleted +2025-06-05 15:06:02,640 - tests.orchestrator - INFO - Deleted pod: test-pod-27ffffcb +2025-06-05 15:06:02,713 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-2386ec8a to be deleted +2025-06-05 15:06:02,777 - tests.orchestrator - INFO - PVC concurrent-pvc-2386ec8a has been deleted +2025-06-05 15:06:02,777 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-2386ec8a +2025-06-05 15:06:02,778 - tests.orchestrator - INFO - Rapid PVC scenario completed +2025-06-05 15:06:02,778 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_concurrent_pvc +2025-06-05 15:06:02,778 - tests.orchestrator - INFO - ------------------------------------------------------------ +2025-06-05 15:06:02,778 - tests.orchestrator - INFO - Deleting pod: test-pod-71d7f585 from PVC: test-pvc-fe252ee6 +2025-06-05 15:06:02,870 - tests.orchestrator - INFO - Waiting for pod test-pod-71d7f585 to be deleted +2025-06-05 15:06:02,935 - tests.orchestrator - INFO - Pod test-pod-71d7f585 has been deleted +2025-06-05 15:06:02,935 - tests.orchestrator - INFO - Deleted pod: test-pod-71d7f585 +2025-06-05 15:06:02,935 - tests.orchestrator - INFO - Deleting PVC: test-pvc-4a7f89b4 +2025-06-05 15:06:02,935 - tests.orchestrator - INFO - Deleting 2 pods using PVC test-pvc-4a7f89b4 +2025-06-05 15:06:02,935 - tests.orchestrator - INFO - Deleting pod: test-pod-5311b1cc from PVC: test-pvc-4a7f89b4 +2025-06-05 15:06:03,025 - tests.orchestrator - INFO - Waiting for pod test-pod-5311b1cc to be deleted +2025-06-05 15:06:03,090 - tests.orchestrator - INFO - Pod test-pod-5311b1cc has been deleted +2025-06-05 15:06:03,090 - tests.orchestrator - INFO - Deleted pod: test-pod-5311b1cc +2025-06-05 15:06:03,090 - tests.orchestrator - INFO - Deleting pod: test-pod-cc8d2266 from PVC: test-pvc-4a7f89b4 +2025-06-05 15:06:03,172 - tests.orchestrator - INFO - Waiting for pod test-pod-cc8d2266 to be deleted +2025-06-05 15:06:03,238 - tests.orchestrator - INFO - Pod test-pod-cc8d2266 has been deleted +2025-06-05 15:06:03,238 - tests.orchestrator - INFO - Deleted pod: test-pod-cc8d2266 +2025-06-05 15:06:03,311 - tests.orchestrator - INFO - Waiting for PVC test-pvc-4a7f89b4 to be deleted +2025-06-05 15:06:03,376 - tests.orchestrator - INFO - PVC test-pvc-4a7f89b4 has been deleted +2025-06-05 15:06:03,376 - tests.orchestrator - INFO - Deleted PVC: test-pvc-4a7f89b4 +2025-06-05 15:06:03,376 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations +2025-06-05 15:06:03,376 - tests.orchestrator - INFO - Test completed in 385.34 seconds +2025-06-05 15:06:03,376 - tests.orchestrator - INFO - Cleaning up resources... +2025-06-05 15:06:03,376 - tests.orchestrator - INFO - Deleting PVC: test-pvc-fe252ee6 +2025-06-05 15:06:03,447 - tests.orchestrator - INFO - Waiting for PVC test-pvc-fe252ee6 to be deleted +2025-06-05 15:06:03,512 - tests.orchestrator - INFO - PVC test-pvc-fe252ee6 has been deleted +2025-06-05 15:06:03,512 - tests.orchestrator - INFO - Deleted PVC: test-pvc-fe252ee6 +2025-06-05 15:06:03,512 - tests.orchestrator - INFO - Cleanup completed +2025-06-05 15:06:03,512 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-05 15:06:03,513 - tests.orchestrator - INFO - --- Operations --- +2025-06-05 15:06:03,513 - tests.orchestrator - INFO - create_pvc: 5 succeeded, 0 failed (100.0%) +2025-06-05 15:06:03,513 - tests.orchestrator - INFO - attach_pod: 6 succeeded, 0 failed (100.0%) +2025-06-05 15:06:03,513 - tests.orchestrator - INFO - delete_pod: 6 succeeded, 0 failed (100.0%) +2025-06-05 15:06:03,513 - tests.orchestrator - INFO - delete_pvc: 5 succeeded, 0 failed (100.0%) +2025-06-05 15:06:03,513 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) +2025-06-05 15:06:03,513 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) +2025-06-05 15:06:03,513 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-05 15:06:03,513 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-05 15:06:03,513 - tests.orchestrator - INFO - many_to_one: No runs +2025-06-05 15:06:03,513 - tests.orchestrator - INFO - one_to_one: No runs +2025-06-05 15:06:03,513 - tests.orchestrator - INFO - concurrent_pvc: 1 succeeded, 0 failed out of 1 runs (100.0%) +2025-06-05 15:06:03,513 - tests.orchestrator - INFO - ========================================= +2025-06-05 15:06:03,514 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250605_150603.json +2025-06-05 15:06:03,557 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250605_150603_20250605_150603.json +2025-06-05 15:06:03,557 - __main__ - INFO - All tests completed successfully +2025-06-05 15:06:19,120 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-05 15:06:19,121 - __main__ - INFO - Verifying AWS credentials +2025-06-05 15:06:19,122 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 15:06:19,122 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 15:06:19,122 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 15:06:19,123 - __main__ - INFO - Running orchestrator stress test suite +2025-06-05 15:06:19,123 - __main__ - INFO - Starting orchestrator for 300 seconds +2025-06-05 15:06:19,123 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-05 15:06:19,870 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 15:06:20,072 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 15:06:20,072 - __main__ - INFO - Test duration overridden to 300 seconds +2025-06-05 15:06:20,072 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) +2025-06-05 15:06:20,072 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-05 15:06:20,260 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 15:06:20,261 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 15:06:20,261 - tests.orchestrator - INFO - Creating PVC: test-pvc-7aa57f8c +2025-06-05 15:06:20,336 - tests.orchestrator - INFO - Created PVC: test-pvc-7aa57f8c +2025-06-05 15:06:20,336 - tests.orchestrator - INFO - Waiting for PVC test-pvc-7aa57f8c to be bound +2025-06-05 15:06:22,470 - tests.orchestrator - INFO - PVC test-pvc-7aa57f8c is bound +2025-06-05 15:06:22,471 - tests.orchestrator - INFO - Creating PVC: test-pvc-afe1c1f0 +2025-06-05 15:06:22,538 - tests.orchestrator - INFO - Created PVC: test-pvc-afe1c1f0 +2025-06-05 15:06:22,538 - tests.orchestrator - INFO - Waiting for PVC test-pvc-afe1c1f0 to be bound +2025-06-05 15:06:24,671 - tests.orchestrator - INFO - PVC test-pvc-afe1c1f0 is bound +2025-06-05 15:06:24,671 - tests.orchestrator - INFO - Creating pod test-pod-d8058d9f on PVC test-pvc-7aa57f8c +2025-06-05 15:06:24,755 - tests.orchestrator - INFO - Created pod: test-pod-d8058d9f using PVC: test-pvc-7aa57f8c +2025-06-05 15:06:24,755 - tests.orchestrator - INFO - Waiting for pod test-pod-d8058d9f to be ready +2025-06-05 15:07:26,793 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-d8058d9f to be ready after 60s +2025-06-05 15:07:26,793 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-d8058d9f to be ready +2025-06-05 15:07:26,793 - tests.orchestrator - INFO - Creating pod test-pod-fdb5f5f7 on PVC test-pvc-afe1c1f0 +2025-06-05 15:07:26,879 - tests.orchestrator - INFO - Created pod: test-pod-fdb5f5f7 using PVC: test-pvc-afe1c1f0 +2025-06-05 15:07:26,880 - tests.orchestrator - INFO - Waiting for pod test-pod-fdb5f5f7 to be ready +2025-06-05 15:08:28,914 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-fdb5f5f7 to be ready after 60s +2025-06-05 15:08:28,915 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-fdb5f5f7 to be ready +2025-06-05 15:08:28,915 - tests.orchestrator - INFO - Creating pod test-pod-415052fa on PVC test-pvc-7aa57f8c +2025-06-05 15:08:28,998 - tests.orchestrator - INFO - Created pod: test-pod-415052fa using PVC: test-pvc-7aa57f8c +2025-06-05 15:08:28,998 - tests.orchestrator - INFO - Waiting for pod test-pod-415052fa to be ready +2025-06-05 15:09:31,031 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-415052fa to be ready after 60s +2025-06-05 15:09:31,031 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-415052fa to be ready +2025-06-05 15:09:31,031 - tests.orchestrator - INFO - Testing read/write between pods test-pod-d8058d9f and test-pod-415052fa sharing PVC test-pvc-7aa57f8c +2025-06-05 15:09:31,098 - tests.orchestrator - ERROR - Failed in read/write verification: (400) +Reason: Bad Request +HTTP response headers: HTTPHeaderDict({'Audit-Id': '9f5a799b-0705-4213-aaf5-1218c7e9571a', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 15:09:31 GMT', 'Content-Length': '166'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-d8058d9f does not have a host assigned","reason":"BadRequest","code":400} + + +2025-06-05 15:09:31,098 - tests.orchestrator - INFO - ============================================================ +2025-06-05 15:09:31,098 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_concurrent_pvc +2025-06-05 15:09:31,098 - tests.orchestrator - INFO - ============================================================ +2025-06-05 15:09:31,098 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations +2025-06-05 15:09:31,099 - tests.orchestrator - INFO - Creating 5 PVCs in quick succession +2025-06-05 15:09:31,177 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-a396d6c2 +2025-06-05 15:09:31,178 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-a396d6c2 to be bound +2025-06-05 15:09:33,310 - tests.orchestrator - INFO - PVC concurrent-pvc-a396d6c2 is bound +2025-06-05 15:09:33,381 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-85edcaa9 +2025-06-05 15:09:33,381 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-85edcaa9 to be bound +2025-06-05 15:09:35,516 - tests.orchestrator - INFO - PVC concurrent-pvc-85edcaa9 is bound +2025-06-05 15:09:35,583 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-bb7b8fe6 +2025-06-05 15:09:35,583 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-bb7b8fe6 to be bound +2025-06-05 15:09:37,719 - tests.orchestrator - INFO - PVC concurrent-pvc-bb7b8fe6 is bound +2025-06-05 15:09:37,790 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-a3d1cb36 +2025-06-05 15:09:37,790 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-a3d1cb36 to be bound +2025-06-05 15:09:39,924 - tests.orchestrator - INFO - PVC concurrent-pvc-a3d1cb36 is bound +2025-06-05 15:09:39,993 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-e1cb5975 +2025-06-05 15:09:39,994 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-e1cb5975 to be bound +2025-06-05 15:09:42,273 - tests.orchestrator - INFO - PVC concurrent-pvc-e1cb5975 is bound +2025-06-05 15:09:42,273 - tests.orchestrator - INFO - Successfully created 5 PVCs +2025-06-05 15:09:42,274 - tests.orchestrator - INFO - Creating pod test-pod-1ea724b8 on PVC concurrent-pvc-a3d1cb36 +2025-06-05 15:09:42,449 - tests.orchestrator - INFO - Created pod: test-pod-1ea724b8 using PVC: concurrent-pvc-a3d1cb36 +2025-06-05 15:09:42,449 - tests.orchestrator - INFO - Waiting for pod test-pod-1ea724b8 to be ready +2025-06-05 15:10:42,631 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1ea724b8 to be ready after 60s +2025-06-05 15:10:42,631 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1ea724b8 to be ready +2025-06-05 15:10:42,631 - tests.orchestrator - INFO - Creating pod test-pod-7f110176 on PVC concurrent-pvc-e1cb5975 +2025-06-05 15:10:42,732 - tests.orchestrator - INFO - Created pod: test-pod-7f110176 using PVC: concurrent-pvc-e1cb5975 +2025-06-05 15:10:42,732 - tests.orchestrator - INFO - Waiting for pod test-pod-7f110176 to be ready +2025-06-05 15:11:42,757 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-7f110176 to be ready after 60s +2025-06-05 15:11:42,757 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-7f110176 to be ready +2025-06-05 15:11:42,757 - tests.orchestrator - INFO - Creating pod test-pod-f48c7103 on PVC concurrent-pvc-85edcaa9 +2025-06-05 15:11:42,854 - tests.orchestrator - INFO - Created pod: test-pod-f48c7103 using PVC: concurrent-pvc-85edcaa9 +2025-06-05 15:11:42,854 - tests.orchestrator - INFO - Waiting for pod test-pod-f48c7103 to be ready +2025-06-05 15:12:44,885 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-f48c7103 to be ready after 60s +2025-06-05 15:12:44,886 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-f48c7103 to be ready +2025-06-05 15:12:44,886 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-a3d1cb36 +2025-06-05 15:12:44,886 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-a3d1cb36 +2025-06-05 15:12:44,886 - tests.orchestrator - INFO - Deleting pod: test-pod-1ea724b8 from PVC: concurrent-pvc-a3d1cb36 +2025-06-05 15:12:44,967 - tests.orchestrator - INFO - Waiting for pod test-pod-1ea724b8 to be deleted +2025-06-05 15:12:45,035 - tests.orchestrator - INFO - Pod test-pod-1ea724b8 has been deleted +2025-06-05 15:12:45,035 - tests.orchestrator - INFO - Deleted pod: test-pod-1ea724b8 +2025-06-05 15:12:45,120 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-a3d1cb36 to be deleted +2025-06-05 15:12:45,197 - tests.orchestrator - INFO - PVC concurrent-pvc-a3d1cb36 has been deleted +2025-06-05 15:12:45,198 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-a3d1cb36 +2025-06-05 15:12:45,198 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-85edcaa9 +2025-06-05 15:12:45,198 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-85edcaa9 +2025-06-05 15:12:45,198 - tests.orchestrator - INFO - Deleting pod: test-pod-f48c7103 from PVC: concurrent-pvc-85edcaa9 +2025-06-05 15:12:45,285 - tests.orchestrator - INFO - Waiting for pod test-pod-f48c7103 to be deleted +2025-06-05 15:12:45,353 - tests.orchestrator - INFO - Pod test-pod-f48c7103 has been deleted +2025-06-05 15:12:45,353 - tests.orchestrator - INFO - Deleted pod: test-pod-f48c7103 +2025-06-05 15:12:45,426 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-85edcaa9 to be deleted +2025-06-05 15:12:45,492 - tests.orchestrator - INFO - PVC concurrent-pvc-85edcaa9 has been deleted +2025-06-05 15:12:45,493 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-85edcaa9 +2025-06-05 15:12:45,493 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-a396d6c2 +2025-06-05 15:12:45,566 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-a396d6c2 to be deleted +2025-06-05 15:12:45,632 - tests.orchestrator - INFO - PVC concurrent-pvc-a396d6c2 has been deleted +2025-06-05 15:12:45,632 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-a396d6c2 +2025-06-05 15:12:45,633 - tests.orchestrator - INFO - Rapid PVC scenario completed +2025-06-05 15:12:45,633 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_concurrent_pvc +2025-06-05 15:12:45,633 - tests.orchestrator - INFO - ------------------------------------------------------------ +2025-06-05 15:12:45,633 - tests.orchestrator - INFO - Deleting pod: test-pod-fdb5f5f7 from PVC: test-pvc-afe1c1f0 +2025-06-05 15:12:45,715 - tests.orchestrator - INFO - Waiting for pod test-pod-fdb5f5f7 to be deleted +2025-06-05 15:12:45,783 - tests.orchestrator - INFO - Pod test-pod-fdb5f5f7 has been deleted +2025-06-05 15:12:45,783 - tests.orchestrator - INFO - Deleted pod: test-pod-fdb5f5f7 +2025-06-05 15:12:45,783 - tests.orchestrator - INFO - Deleting PVC: test-pvc-7aa57f8c +2025-06-05 15:12:45,783 - tests.orchestrator - INFO - Deleting 2 pods using PVC test-pvc-7aa57f8c +2025-06-05 15:12:45,784 - tests.orchestrator - INFO - Deleting pod: test-pod-d8058d9f from PVC: test-pvc-7aa57f8c +2025-06-05 15:12:45,867 - tests.orchestrator - INFO - Waiting for pod test-pod-d8058d9f to be deleted +2025-06-05 15:12:45,934 - tests.orchestrator - INFO - Pod test-pod-d8058d9f has been deleted +2025-06-05 15:12:45,934 - tests.orchestrator - INFO - Deleted pod: test-pod-d8058d9f +2025-06-05 15:12:45,934 - tests.orchestrator - INFO - Deleting pod: test-pod-415052fa from PVC: test-pvc-7aa57f8c +2025-06-05 15:12:46,015 - tests.orchestrator - INFO - Waiting for pod test-pod-415052fa to be deleted +2025-06-05 15:12:46,079 - tests.orchestrator - INFO - Pod test-pod-415052fa has been deleted +2025-06-05 15:12:46,080 - tests.orchestrator - INFO - Deleted pod: test-pod-415052fa +2025-06-05 15:12:46,154 - tests.orchestrator - INFO - Waiting for PVC test-pvc-7aa57f8c to be deleted +2025-06-05 15:12:46,220 - tests.orchestrator - INFO - PVC test-pvc-7aa57f8c has been deleted +2025-06-05 15:12:46,220 - tests.orchestrator - INFO - Deleted PVC: test-pvc-7aa57f8c +2025-06-05 15:12:46,220 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations +2025-06-05 15:12:46,221 - tests.orchestrator - INFO - Test completed in 386.15 seconds +2025-06-05 15:12:46,221 - tests.orchestrator - INFO - Cleaning up resources... +2025-06-05 15:12:46,221 - tests.orchestrator - INFO - Deleting pod: test-pod-7f110176 from PVC: concurrent-pvc-e1cb5975 +2025-06-05 15:12:46,308 - tests.orchestrator - INFO - Waiting for pod test-pod-7f110176 to be deleted +2025-06-05 15:12:46,378 - tests.orchestrator - INFO - Pod test-pod-7f110176 has been deleted +2025-06-05 15:12:46,379 - tests.orchestrator - INFO - Deleted pod: test-pod-7f110176 +2025-06-05 15:12:46,379 - tests.orchestrator - INFO - Deleting PVC: test-pvc-afe1c1f0 +2025-06-05 15:12:46,452 - tests.orchestrator - INFO - Waiting for PVC test-pvc-afe1c1f0 to be deleted +2025-06-05 15:12:46,519 - tests.orchestrator - INFO - PVC test-pvc-afe1c1f0 has been deleted +2025-06-05 15:12:46,519 - tests.orchestrator - INFO - Deleted PVC: test-pvc-afe1c1f0 +2025-06-05 15:12:46,519 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-bb7b8fe6 +2025-06-05 15:12:46,593 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-bb7b8fe6 to be deleted +2025-06-05 15:12:46,660 - tests.orchestrator - INFO - PVC concurrent-pvc-bb7b8fe6 has been deleted +2025-06-05 15:12:46,660 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-bb7b8fe6 +2025-06-05 15:12:46,660 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-e1cb5975 +2025-06-05 15:12:46,735 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-e1cb5975 to be deleted +2025-06-05 15:12:46,805 - tests.orchestrator - INFO - PVC concurrent-pvc-e1cb5975 has been deleted +2025-06-05 15:12:46,805 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-e1cb5975 +2025-06-05 15:12:46,809 - tests.orchestrator - INFO - Cleanup completed +2025-06-05 15:12:46,809 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-05 15:12:46,809 - tests.orchestrator - INFO - --- Operations --- +2025-06-05 15:12:46,809 - tests.orchestrator - INFO - create_pvc: 7 succeeded, 0 failed (100.0%) +2025-06-05 15:12:46,809 - tests.orchestrator - INFO - attach_pod: 6 succeeded, 0 failed (100.0%) +2025-06-05 15:12:46,809 - tests.orchestrator - INFO - delete_pod: 6 succeeded, 0 failed (100.0%) +2025-06-05 15:12:46,809 - tests.orchestrator - INFO - delete_pvc: 7 succeeded, 0 failed (100.0%) +2025-06-05 15:12:46,809 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) +2025-06-05 15:12:46,809 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) +2025-06-05 15:12:46,809 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-05 15:12:46,809 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-05 15:12:46,809 - tests.orchestrator - INFO - many_to_one: No runs +2025-06-05 15:12:46,809 - tests.orchestrator - INFO - one_to_one: No runs +2025-06-05 15:12:46,809 - tests.orchestrator - INFO - concurrent_pvc: 1 succeeded, 0 failed out of 1 runs (100.0%) +2025-06-05 15:12:46,809 - tests.orchestrator - INFO - ========================================= +2025-06-05 15:12:46,810 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250605_151246.json +2025-06-05 15:12:46,852 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250605_151246_20250605_151246.json +2025-06-05 15:12:46,852 - __main__ - INFO - All tests completed successfully +2025-06-05 15:15:29,024 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-05 15:15:29,024 - __main__ - INFO - Verifying AWS credentials +2025-06-05 15:15:29,025 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 15:15:29,025 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 15:15:29,026 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 15:15:29,026 - __main__ - INFO - Running orchestrator stress test suite +2025-06-05 15:15:29,026 - __main__ - INFO - Starting orchestrator for 300 seconds +2025-06-05 15:15:29,026 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-05 15:15:29,738 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 15:15:29,948 - tests.orchestrator - ERROR - Error checking namespace: (401) +Reason: Unauthorized +HTTP response headers: HTTPHeaderDict({'Audit-Id': '95b9f34d-de93-4da4-99dd-fc17111bd0ce', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 15:15:29 GMT', 'Content-Length': '129'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} + + +2025-06-05 15:15:29,950 - __main__ - ERROR - Error running tests: (401) +Reason: Unauthorized +HTTP response headers: HTTPHeaderDict({'Audit-Id': '95b9f34d-de93-4da4-99dd-fc17111bd0ce', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 15:15:29 GMT', 'Content-Length': '129'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} + +Traceback (most recent call last): + File "/local/home/ibhard/efs-csi-driver-fork/aws-efs-csi-driver/test/e2e/stress-scale-tests/run_tests.py", line 209, in main + orchestrator = EFSCSIOrchestrator(config_file=orchestrator_config) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/local/home/ibhard/efs-csi-driver-fork/aws-efs-csi-driver/test/e2e/stress-scale-tests/tests/orchestrator.py", line 86, in __init__ + self._ensure_namespace_exists() + File "/local/home/ibhard/efs-csi-driver-fork/aws-efs-csi-driver/test/e2e/stress-scale-tests/tests/orchestrator.py", line 92, in _ensure_namespace_exists + self.core_v1.read_namespace(name=self.namespace) + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api/core_v1_api.py", line 22992, in read_namespace + return self.read_namespace_with_http_info(name, **kwargs) # noqa: E501 + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api/core_v1_api.py", line 23071, in read_namespace_with_http_info + return self.api_client.call_api( + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api_client.py", line 348, in call_api + return self.__call_api(resource_path, method, + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api_client.py", line 180, in __call_api + response_data = self.request( + ^^^^^^^^^^^^^ + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api_client.py", line 373, in request + return self.rest_client.GET(url, + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/rest.py", line 244, in GET + return self.request("GET", url, + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/rest.py", line 238, in request + raise ApiException(http_resp=r) +kubernetes.client.exceptions.ApiException: (401) +Reason: Unauthorized +HTTP response headers: HTTPHeaderDict({'Audit-Id': '95b9f34d-de93-4da4-99dd-fc17111bd0ce', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 15:15:29 GMT', 'Content-Length': '129'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} + + +2025-06-05 15:16:04,243 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-05 15:16:04,243 - __main__ - INFO - Verifying AWS credentials +2025-06-05 15:16:04,244 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 15:16:04,244 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 15:16:04,244 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 15:16:04,245 - __main__ - INFO - Running orchestrator stress test suite +2025-06-05 15:16:04,245 - __main__ - INFO - Starting orchestrator for 300 seconds +2025-06-05 15:16:04,245 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-05 15:16:04,962 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 15:16:05,167 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 15:16:05,168 - __main__ - INFO - Test duration overridden to 300 seconds +2025-06-05 15:16:05,168 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) +2025-06-05 15:16:05,168 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-05 15:16:05,359 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 15:16:05,359 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 15:16:05,359 - tests.orchestrator - INFO - Creating PVC: test-pvc-20f868e5 +2025-06-05 15:16:05,438 - tests.orchestrator - INFO - Created PVC: test-pvc-20f868e5 +2025-06-05 15:16:05,438 - tests.orchestrator - INFO - Waiting for PVC test-pvc-20f868e5 to be bound +2025-06-05 15:16:07,570 - tests.orchestrator - INFO - PVC test-pvc-20f868e5 is bound +2025-06-05 15:16:07,570 - tests.orchestrator - INFO - Creating PVC: test-pvc-48e3d2d9 +2025-06-05 15:16:07,639 - tests.orchestrator - INFO - Created PVC: test-pvc-48e3d2d9 +2025-06-05 15:16:07,639 - tests.orchestrator - INFO - Waiting for PVC test-pvc-48e3d2d9 to be bound +2025-06-05 15:16:09,773 - tests.orchestrator - INFO - PVC test-pvc-48e3d2d9 is bound +2025-06-05 15:16:09,773 - tests.orchestrator - INFO - Creating pod test-pod-03b0f7fd on PVC test-pvc-20f868e5 +2025-06-05 15:16:09,857 - tests.orchestrator - INFO - Created pod: test-pod-03b0f7fd using PVC: test-pvc-20f868e5 +2025-06-05 15:16:09,857 - tests.orchestrator - INFO - Waiting for pod test-pod-03b0f7fd to be ready +2025-06-05 15:16:09,926 - tests.orchestrator - INFO - Pod test-pod-03b0f7fd phase: Pending +2025-06-05 15:16:40,974 - tests.orchestrator - INFO - Pod test-pod-03b0f7fd taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 15:16:40,974 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-03b0f7fd ===== +2025-06-05 15:16:41,041 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:16:41,124 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:16:41,125 - tests.orchestrator - INFO - [2025-06-05 15:16:09+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:16:41,199 - tests.orchestrator - INFO - No logs available +2025-06-05 15:16:41,199 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:16:41,199 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-20f868e5'} +2025-06-05 15:16:41,199 - tests.orchestrator - INFO - - kube-api-access-f6t7l: {} +2025-06-05 15:16:41,199 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-03b0f7fd ===== +2025-06-05 15:17:10,102 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-03b0f7fd to be ready after 60s +2025-06-05 15:17:10,102 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-03b0f7fd ===== +2025-06-05 15:17:10,172 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:17:10,245 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:17:10,245 - tests.orchestrator - INFO - [2025-06-05 15:16:09+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:17:10,312 - tests.orchestrator - INFO - No logs available +2025-06-05 15:17:10,312 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:17:10,312 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-20f868e5'} +2025-06-05 15:17:10,312 - tests.orchestrator - INFO - - kube-api-access-f6t7l: {} +2025-06-05 15:17:10,312 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-03b0f7fd ===== +2025-06-05 15:17:10,313 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-03b0f7fd to be ready +2025-06-05 15:17:10,313 - tests.orchestrator - INFO - Creating pod test-pod-264015e0 on PVC test-pvc-20f868e5 +2025-06-05 15:17:10,399 - tests.orchestrator - INFO - Created pod: test-pod-264015e0 using PVC: test-pvc-20f868e5 +2025-06-05 15:17:10,399 - tests.orchestrator - INFO - Waiting for pod test-pod-264015e0 to be ready +2025-06-05 15:17:10,467 - tests.orchestrator - INFO - Pod test-pod-264015e0 phase: Pending +2025-06-05 15:17:41,811 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-05 15:17:41,811 - __main__ - INFO - Verifying AWS credentials +2025-06-05 15:17:41,812 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 15:17:41,812 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 15:17:41,813 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 15:17:41,813 - __main__ - INFO - Running orchestrator stress test suite +2025-06-05 15:17:41,813 - __main__ - INFO - Starting orchestrator for 300 seconds +2025-06-05 15:17:41,813 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-05 15:17:42,550 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 15:17:42,784 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 15:17:42,784 - __main__ - INFO - Test duration overridden to 300 seconds +2025-06-05 15:17:42,784 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) +2025-06-05 15:17:42,784 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-05 15:17:42,975 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 15:17:42,976 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 15:17:42,976 - tests.orchestrator - INFO - Creating PVC: test-pvc-b967555b +2025-06-05 15:17:43,061 - tests.orchestrator - INFO - Created PVC: test-pvc-b967555b +2025-06-05 15:17:43,061 - tests.orchestrator - INFO - Waiting for PVC test-pvc-b967555b to be bound +2025-06-05 15:17:45,204 - tests.orchestrator - INFO - PVC test-pvc-b967555b is bound +2025-06-05 15:17:45,205 - tests.orchestrator - INFO - Creating PVC: test-pvc-6282a942 +2025-06-05 15:17:45,276 - tests.orchestrator - INFO - Created PVC: test-pvc-6282a942 +2025-06-05 15:17:45,276 - tests.orchestrator - INFO - Waiting for PVC test-pvc-6282a942 to be bound +2025-06-05 15:17:47,412 - tests.orchestrator - INFO - PVC test-pvc-6282a942 is bound +2025-06-05 15:17:47,412 - tests.orchestrator - INFO - Creating pod test-pod-9ef8c6d5 on PVC test-pvc-b967555b +2025-06-05 15:17:47,490 - tests.orchestrator - INFO - Created pod: test-pod-9ef8c6d5 using PVC: test-pvc-b967555b +2025-06-05 15:17:47,491 - tests.orchestrator - INFO - Waiting for pod test-pod-9ef8c6d5 to be ready +2025-06-05 15:17:47,559 - tests.orchestrator - INFO - Pod test-pod-9ef8c6d5 phase: Pending +2025-06-05 15:37:50,109 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-05 15:37:50,109 - __main__ - INFO - Verifying AWS credentials +2025-06-05 15:37:50,110 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 15:37:50,111 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 15:37:50,111 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 15:37:50,111 - __main__ - INFO - Running orchestrator stress test suite +2025-06-05 15:37:50,111 - __main__ - INFO - Starting orchestrator for 300 seconds +2025-06-05 15:37:50,112 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-05 15:37:50,853 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 15:37:51,073 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 15:37:51,074 - __main__ - INFO - Test duration overridden to 300 seconds +2025-06-05 15:37:51,074 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) +2025-06-05 15:37:51,074 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-05 15:37:51,263 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 15:37:51,263 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 15:37:51,263 - tests.orchestrator - INFO - Creating PVC: test-pvc-011557a1 +2025-06-05 15:37:51,341 - tests.orchestrator - INFO - Created PVC: test-pvc-011557a1 +2025-06-05 15:37:51,341 - tests.orchestrator - INFO - Waiting for PVC test-pvc-011557a1 to be bound +2025-06-05 15:37:53,474 - tests.orchestrator - INFO - PVC test-pvc-011557a1 is bound +2025-06-05 15:37:53,477 - tests.orchestrator - INFO - Creating PVC: test-pvc-b52b04ee +2025-06-05 15:37:53,544 - tests.orchestrator - INFO - Created PVC: test-pvc-b52b04ee +2025-06-05 15:37:53,545 - tests.orchestrator - INFO - Waiting for PVC test-pvc-b52b04ee to be bound +2025-06-05 15:37:55,677 - tests.orchestrator - INFO - PVC test-pvc-b52b04ee is bound +2025-06-05 15:37:55,677 - tests.orchestrator - INFO - Creating pod test-pod-1f8e6665 on PVC test-pvc-b52b04ee +2025-06-05 15:37:55,757 - tests.orchestrator - INFO - Created pod: test-pod-1f8e6665 using PVC: test-pvc-b52b04ee +2025-06-05 15:37:55,757 - tests.orchestrator - INFO - Waiting for pod test-pod-1f8e6665 to be ready +2025-06-05 15:37:55,827 - tests.orchestrator - INFO - Pod test-pod-1f8e6665 phase: Pending +2025-06-05 15:38:26,850 - tests.orchestrator - INFO - Pod test-pod-1f8e6665 taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 15:38:26,850 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-1f8e6665 ===== +2025-06-05 15:38:26,917 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:38:26,987 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:38:26,987 - tests.orchestrator - INFO - [2025-06-05 15:37:55+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:38:27,053 - tests.orchestrator - INFO - No logs available +2025-06-05 15:38:27,053 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:38:27,054 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-b52b04ee'} +2025-06-05 15:38:27,054 - tests.orchestrator - INFO - - kube-api-access-z4xbc: {} +2025-06-05 15:38:27,054 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-1f8e6665 ===== +2025-06-05 15:38:55,926 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1f8e6665 to be ready after 60s +2025-06-05 15:38:55,927 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-1f8e6665 ===== +2025-06-05 15:38:55,995 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:38:56,064 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:38:56,064 - tests.orchestrator - INFO - [2025-06-05 15:37:55+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:38:56,129 - tests.orchestrator - INFO - No logs available +2025-06-05 15:38:56,129 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:38:56,129 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-b52b04ee'} +2025-06-05 15:38:56,129 - tests.orchestrator - INFO - - kube-api-access-z4xbc: {} +2025-06-05 15:38:56,129 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-1f8e6665 ===== +2025-06-05 15:38:56,130 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1f8e6665 to be ready +2025-06-05 15:38:56,130 - tests.orchestrator - INFO - Creating pod test-pod-43a71634 on PVC test-pvc-b52b04ee +2025-06-05 15:38:56,213 - tests.orchestrator - INFO - Created pod: test-pod-43a71634 using PVC: test-pvc-b52b04ee +2025-06-05 15:38:56,213 - tests.orchestrator - INFO - Waiting for pod test-pod-43a71634 to be ready +2025-06-05 15:38:56,283 - tests.orchestrator - INFO - Pod test-pod-43a71634 phase: Pending +2025-06-05 15:39:27,290 - tests.orchestrator - INFO - Pod test-pod-43a71634 taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 15:39:27,290 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-43a71634 ===== +2025-06-05 15:39:27,357 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:39:27,428 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:39:27,428 - tests.orchestrator - INFO - [2025-06-05 15:38:56+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:39:27,493 - tests.orchestrator - INFO - No logs available +2025-06-05 15:39:27,494 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:39:27,494 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-b52b04ee'} +2025-06-05 15:39:27,494 - tests.orchestrator - INFO - - kube-api-access-fgg2x: {} +2025-06-05 15:39:27,494 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-43a71634 ===== +2025-06-05 15:39:56,469 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-43a71634 to be ready after 60s +2025-06-05 15:39:56,469 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-43a71634 ===== +2025-06-05 15:39:56,538 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:39:56,609 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:39:56,609 - tests.orchestrator - INFO - [2025-06-05 15:38:56+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:39:56,674 - tests.orchestrator - INFO - No logs available +2025-06-05 15:39:56,674 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:39:56,674 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-b52b04ee'} +2025-06-05 15:39:56,674 - tests.orchestrator - INFO - - kube-api-access-fgg2x: {} +2025-06-05 15:39:56,675 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-43a71634 ===== +2025-06-05 15:39:56,675 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-43a71634 to be ready +2025-06-05 15:39:56,675 - tests.orchestrator - INFO - Creating pod test-pod-9785ae47 on PVC test-pvc-011557a1 +2025-06-05 15:39:56,757 - tests.orchestrator - INFO - Created pod: test-pod-9785ae47 using PVC: test-pvc-011557a1 +2025-06-05 15:39:56,757 - tests.orchestrator - INFO - Waiting for pod test-pod-9785ae47 to be ready +2025-06-05 15:39:56,823 - tests.orchestrator - INFO - Pod test-pod-9785ae47 phase: Pending +2025-06-05 15:40:27,832 - tests.orchestrator - INFO - Pod test-pod-9785ae47 taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 15:40:27,832 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-9785ae47 ===== +2025-06-05 15:40:27,899 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:40:27,970 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:40:27,970 - tests.orchestrator - INFO - [2025-06-05 15:39:56+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:40:28,037 - tests.orchestrator - INFO - No logs available +2025-06-05 15:40:28,037 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:40:28,037 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-011557a1'} +2025-06-05 15:40:28,037 - tests.orchestrator - INFO - - kube-api-access-p2phq: {} +2025-06-05 15:40:28,037 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-9785ae47 ===== +2025-06-05 15:40:56,911 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9785ae47 to be ready after 60s +2025-06-05 15:40:56,912 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-9785ae47 ===== +2025-06-05 15:40:56,980 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:40:57,050 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:40:57,050 - tests.orchestrator - INFO - [2025-06-05 15:39:56+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:40:57,118 - tests.orchestrator - INFO - No logs available +2025-06-05 15:40:57,118 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:40:57,118 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-011557a1'} +2025-06-05 15:40:57,118 - tests.orchestrator - INFO - - kube-api-access-p2phq: {} +2025-06-05 15:40:57,118 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-9785ae47 ===== +2025-06-05 15:40:57,118 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9785ae47 to be ready +2025-06-05 15:40:57,119 - tests.orchestrator - INFO - Testing read/write between pods test-pod-1f8e6665 and test-pod-43a71634 sharing PVC test-pvc-b52b04ee +2025-06-05 15:40:57,185 - tests.orchestrator - ERROR - Failed in read/write verification: (400) +Reason: Bad Request +HTTP response headers: HTTPHeaderDict({'Audit-Id': '4713df17-943b-4cf0-b43b-7454ef741357', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 15:40:57 GMT', 'Content-Length': '166'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-1f8e6665 does not have a host assigned","reason":"BadRequest","code":400} + + +2025-06-05 15:40:57,185 - tests.orchestrator - INFO - ============================================================ +2025-06-05 15:40:57,185 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_many_to_one +2025-06-05 15:40:57,185 - tests.orchestrator - INFO - ============================================================ +2025-06-05 15:40:57,185 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +2025-06-05 15:40:57,185 - tests.orchestrator - INFO - STARTING MANY-TO-ONE SCENARIO DIAGNOSTICS +2025-06-05 15:40:57,185 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +2025-06-05 15:40:57,185 - tests.orchestrator - INFO - [MANY2ONE] STEP 1: Creating dedicated PVC: many2one-21027765 +2025-06-05 15:40:57,185 - tests.orchestrator - INFO - [MANY2ONE] Creating PVC manifest with storageClassName: efs-sc +2025-06-05 15:40:57,263 - tests.orchestrator - INFO - [MANY2ONE] PVC many2one-21027765 created successfully +2025-06-05 15:40:57,264 - tests.orchestrator - INFO - [MANY2ONE] Waiting for PVC many2one-21027765 to be bound... +2025-06-05 15:40:57,264 - tests.orchestrator - INFO - Waiting for PVC many2one-21027765 to be bound +2025-06-05 15:40:59,396 - tests.orchestrator - INFO - PVC many2one-21027765 is bound +2025-06-05 15:40:59,463 - tests.orchestrator - INFO - [MANY2ONE] PVC status: phase=Bound, capacity={'storage': '1Gi'} +2025-06-05 15:40:59,463 - tests.orchestrator - INFO - [MANY2ONE] STEP 2: Creating 3 pods for the same PVC many2one-21027765 +2025-06-05 15:40:59,463 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 1/3 for PVC many2one-21027765 +2025-06-05 15:40:59,463 - tests.orchestrator - INFO - Creating pod test-pod-052e963e on PVC many2one-21027765 +2025-06-05 15:40:59,538 - tests.orchestrator - INFO - Created pod: test-pod-052e963e using PVC: many2one-21027765 +2025-06-05 15:40:59,538 - tests.orchestrator - INFO - Waiting for pod test-pod-052e963e to be ready +2025-06-05 15:40:59,604 - tests.orchestrator - INFO - Pod test-pod-052e963e phase: Pending +2025-06-05 15:41:30,643 - tests.orchestrator - INFO - Pod test-pod-052e963e taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 15:41:30,643 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-052e963e ===== +2025-06-05 15:41:30,710 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:41:30,779 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:41:30,780 - tests.orchestrator - INFO - [2025-06-05 15:40:59+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:41:30,846 - tests.orchestrator - INFO - No logs available +2025-06-05 15:41:30,846 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:41:30,846 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-21027765'} +2025-06-05 15:41:30,846 - tests.orchestrator - INFO - - kube-api-access-h4dzw: {} +2025-06-05 15:41:30,846 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-052e963e ===== +2025-06-05 15:41:59,722 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-052e963e to be ready after 60s +2025-06-05 15:41:59,722 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-052e963e ===== +2025-06-05 15:41:59,789 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:41:59,859 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:41:59,859 - tests.orchestrator - INFO - [2025-06-05 15:40:59+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:41:59,925 - tests.orchestrator - INFO - No logs available +2025-06-05 15:41:59,925 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:41:59,925 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-21027765'} +2025-06-05 15:41:59,925 - tests.orchestrator - INFO - - kube-api-access-h4dzw: {} +2025-06-05 15:41:59,925 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-052e963e ===== +2025-06-05 15:41:59,926 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-052e963e to be ready +2025-06-05 15:41:59,926 - tests.orchestrator - ERROR - [MANY2ONE] Failed to create pod 1/3 +2025-06-05 15:41:59,926 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 2/3 for PVC many2one-21027765 +2025-06-05 15:41:59,926 - tests.orchestrator - INFO - Creating pod test-pod-dda7e82b on PVC many2one-21027765 +2025-06-05 15:42:00,007 - tests.orchestrator - INFO - Created pod: test-pod-dda7e82b using PVC: many2one-21027765 +2025-06-05 15:42:00,008 - tests.orchestrator - INFO - Waiting for pod test-pod-dda7e82b to be ready +2025-06-05 15:42:00,075 - tests.orchestrator - INFO - Pod test-pod-dda7e82b phase: Pending +2025-06-05 15:42:31,084 - tests.orchestrator - INFO - Pod test-pod-dda7e82b taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 15:42:31,084 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-dda7e82b ===== +2025-06-05 15:42:31,150 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:42:31,219 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:42:31,219 - tests.orchestrator - INFO - [2025-06-05 15:41:59+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:42:31,285 - tests.orchestrator - INFO - No logs available +2025-06-05 15:42:31,285 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:42:31,285 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-21027765'} +2025-06-05 15:42:31,285 - tests.orchestrator - INFO - - kube-api-access-bktns: {} +2025-06-05 15:42:31,285 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-dda7e82b ===== +2025-06-05 15:43:00,160 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-dda7e82b to be ready after 60s +2025-06-05 15:43:00,160 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-dda7e82b ===== +2025-06-05 15:43:00,227 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:43:00,296 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:43:00,296 - tests.orchestrator - INFO - [2025-06-05 15:41:59+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:43:00,362 - tests.orchestrator - INFO - No logs available +2025-06-05 15:43:00,362 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:43:00,362 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-21027765'} +2025-06-05 15:43:00,362 - tests.orchestrator - INFO - - kube-api-access-bktns: {} +2025-06-05 15:43:00,363 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-dda7e82b ===== +2025-06-05 15:43:00,363 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-dda7e82b to be ready +2025-06-05 15:43:00,363 - tests.orchestrator - ERROR - [MANY2ONE] Failed to create pod 2/3 +2025-06-05 15:43:00,363 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 3/3 for PVC many2one-21027765 +2025-06-05 15:43:00,363 - tests.orchestrator - INFO - Creating pod test-pod-9b6c5221 on PVC many2one-21027765 +2025-06-05 15:43:00,448 - tests.orchestrator - INFO - Created pod: test-pod-9b6c5221 using PVC: many2one-21027765 +2025-06-05 15:43:00,448 - tests.orchestrator - INFO - Waiting for pod test-pod-9b6c5221 to be ready +2025-06-05 15:43:00,514 - tests.orchestrator - INFO - Pod test-pod-9b6c5221 phase: Pending +2025-06-05 15:43:31,552 - tests.orchestrator - INFO - Pod test-pod-9b6c5221 taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 15:43:31,552 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-9b6c5221 ===== +2025-06-05 15:43:31,684 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:43:31,759 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:43:31,759 - tests.orchestrator - INFO - [2025-06-05 15:43:00+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:43:31,829 - tests.orchestrator - INFO - No logs available +2025-06-05 15:43:31,830 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:43:31,830 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-21027765'} +2025-06-05 15:43:31,830 - tests.orchestrator - INFO - - kube-api-access-mh6kx: {} +2025-06-05 15:43:31,830 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-9b6c5221 ===== +2025-06-05 15:44:00,708 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9b6c5221 to be ready after 60s +2025-06-05 15:44:00,708 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-9b6c5221 ===== +2025-06-05 15:44:00,776 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:44:00,846 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:44:00,846 - tests.orchestrator - INFO - [2025-06-05 15:43:00+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:44:00,911 - tests.orchestrator - INFO - No logs available +2025-06-05 15:44:00,911 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:44:00,911 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-21027765'} +2025-06-05 15:44:00,911 - tests.orchestrator - INFO - - kube-api-access-mh6kx: {} +2025-06-05 15:44:00,911 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-9b6c5221 ===== +2025-06-05 15:44:00,912 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9b6c5221 to be ready +2025-06-05 15:44:00,912 - tests.orchestrator - ERROR - [MANY2ONE] Failed to create pod 3/3 +2025-06-05 15:44:00,912 - tests.orchestrator - INFO - [MANY2ONE] Created 0/3 pods successfully +2025-06-05 15:44:00,912 - tests.orchestrator - ERROR - [MANY2ONE] FAILED: Insufficient pods created (0), need at least 2 for read/write test +2025-06-05 15:44:00,912 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_many_to_one +2025-06-05 15:44:00,912 - tests.orchestrator - INFO - ------------------------------------------------------------ +2025-06-05 15:44:00,912 - tests.orchestrator - INFO - Deleting pod: test-pod-052e963e from PVC: many2one-21027765 +2025-06-05 15:44:00,990 - tests.orchestrator - INFO - Waiting for pod test-pod-052e963e to be deleted +2025-06-05 15:44:01,055 - tests.orchestrator - INFO - Pod test-pod-052e963e has been deleted +2025-06-05 15:44:01,055 - tests.orchestrator - INFO - Deleted pod: test-pod-052e963e +2025-06-05 15:44:01,056 - tests.orchestrator - INFO - Deleting PVC: test-pvc-b52b04ee +2025-06-05 15:44:01,056 - tests.orchestrator - INFO - Deleting 2 pods using PVC test-pvc-b52b04ee +2025-06-05 15:44:01,056 - tests.orchestrator - INFO - Deleting pod: test-pod-1f8e6665 from PVC: test-pvc-b52b04ee +2025-06-05 15:44:01,140 - tests.orchestrator - INFO - Waiting for pod test-pod-1f8e6665 to be deleted +2025-06-05 15:44:01,205 - tests.orchestrator - INFO - Pod test-pod-1f8e6665 has been deleted +2025-06-05 15:44:01,205 - tests.orchestrator - INFO - Deleted pod: test-pod-1f8e6665 +2025-06-05 15:44:01,205 - tests.orchestrator - INFO - Deleting pod: test-pod-43a71634 from PVC: test-pvc-b52b04ee +2025-06-05 15:44:01,287 - tests.orchestrator - INFO - Waiting for pod test-pod-43a71634 to be deleted +2025-06-05 15:44:01,352 - tests.orchestrator - INFO - Pod test-pod-43a71634 has been deleted +2025-06-05 15:44:01,352 - tests.orchestrator - INFO - Deleted pod: test-pod-43a71634 +2025-06-05 15:44:01,426 - tests.orchestrator - INFO - Waiting for PVC test-pvc-b52b04ee to be deleted +2025-06-05 15:44:01,527 - tests.orchestrator - INFO - PVC test-pvc-b52b04ee has been deleted +2025-06-05 15:44:01,527 - tests.orchestrator - INFO - Deleted PVC: test-pvc-b52b04ee +2025-06-05 15:44:01,527 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations +2025-06-05 15:44:01,527 - tests.orchestrator - INFO - Test completed in 370.45 seconds +2025-06-05 15:44:01,527 - tests.orchestrator - INFO - Cleaning up resources... +2025-06-05 15:44:01,527 - tests.orchestrator - INFO - Deleting pod: test-pod-9785ae47 from PVC: test-pvc-011557a1 +2025-06-05 15:44:01,678 - tests.orchestrator - INFO - Waiting for pod test-pod-9785ae47 to be deleted +2025-06-05 15:44:01,750 - tests.orchestrator - INFO - Pod test-pod-9785ae47 has been deleted +2025-06-05 15:44:01,751 - tests.orchestrator - INFO - Deleted pod: test-pod-9785ae47 +2025-06-05 15:44:01,751 - tests.orchestrator - INFO - Deleting pod: test-pod-dda7e82b from PVC: many2one-21027765 +2025-06-05 15:44:01,841 - tests.orchestrator - INFO - Waiting for pod test-pod-dda7e82b to be deleted +2025-06-05 15:44:01,912 - tests.orchestrator - INFO - Pod test-pod-dda7e82b has been deleted +2025-06-05 15:44:01,912 - tests.orchestrator - INFO - Deleted pod: test-pod-dda7e82b +2025-06-05 15:44:01,912 - tests.orchestrator - INFO - Deleting pod: test-pod-9b6c5221 from PVC: many2one-21027765 +2025-06-05 15:44:02,038 - tests.orchestrator - INFO - Waiting for pod test-pod-9b6c5221 to be deleted +2025-06-05 15:44:02,118 - tests.orchestrator - INFO - Pod test-pod-9b6c5221 has been deleted +2025-06-05 15:44:02,118 - tests.orchestrator - INFO - Deleted pod: test-pod-9b6c5221 +2025-06-05 15:44:02,118 - tests.orchestrator - INFO - Deleting PVC: test-pvc-011557a1 +2025-06-05 15:44:02,196 - tests.orchestrator - INFO - Waiting for PVC test-pvc-011557a1 to be deleted +2025-06-05 15:44:02,268 - tests.orchestrator - INFO - PVC test-pvc-011557a1 has been deleted +2025-06-05 15:44:02,268 - tests.orchestrator - INFO - Deleted PVC: test-pvc-011557a1 +2025-06-05 15:44:02,268 - tests.orchestrator - INFO - Deleting PVC: many2one-21027765 +2025-06-05 15:44:02,341 - tests.orchestrator - INFO - Waiting for PVC many2one-21027765 to be deleted +2025-06-05 15:44:02,406 - tests.orchestrator - INFO - PVC many2one-21027765 has been deleted +2025-06-05 15:44:02,406 - tests.orchestrator - INFO - Deleted PVC: many2one-21027765 +2025-06-05 15:44:02,406 - tests.orchestrator - INFO - Cleanup completed +2025-06-05 15:44:02,406 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-05 15:44:02,407 - tests.orchestrator - INFO - --- Operations --- +2025-06-05 15:44:02,407 - tests.orchestrator - INFO - create_pvc: 2 succeeded, 0 failed (100.0%) +2025-06-05 15:44:02,407 - tests.orchestrator - INFO - attach_pod: 6 succeeded, 0 failed (100.0%) +2025-06-05 15:44:02,407 - tests.orchestrator - INFO - delete_pod: 6 succeeded, 0 failed (100.0%) +2025-06-05 15:44:02,407 - tests.orchestrator - INFO - delete_pvc: 3 succeeded, 0 failed (100.0%) +2025-06-05 15:44:02,407 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) +2025-06-05 15:44:02,407 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) +2025-06-05 15:44:02,407 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-05 15:44:02,407 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-05 15:44:02,408 - tests.orchestrator - INFO - many_to_one: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-05 15:44:02,408 - tests.orchestrator - INFO - one_to_one: No runs +2025-06-05 15:44:02,408 - tests.orchestrator - INFO - concurrent_pvc: No runs +2025-06-05 15:44:02,408 - tests.orchestrator - INFO - ========================================= +2025-06-05 15:44:02,408 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250605_154402.json +2025-06-05 15:44:02,451 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250605_154402_20250605_154402.json +2025-06-05 15:44:02,451 - __main__ - INFO - All tests completed successfully +2025-06-05 15:45:45,975 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-05 15:45:45,975 - __main__ - INFO - Verifying AWS credentials +2025-06-05 15:45:45,976 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 15:45:45,976 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 15:45:45,977 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 15:45:45,977 - __main__ - INFO - Running orchestrator stress test suite +2025-06-05 15:45:45,977 - __main__ - INFO - Starting orchestrator for 300 seconds +2025-06-05 15:45:45,977 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-05 15:45:46,726 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 15:45:46,943 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 15:45:46,944 - __main__ - INFO - Test duration overridden to 300 seconds +2025-06-05 15:45:46,944 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) +2025-06-05 15:45:46,944 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-05 15:45:47,132 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 15:45:47,133 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 15:45:47,133 - tests.orchestrator - INFO - Creating PVC: test-pvc-7dbf87c6 +2025-06-05 15:45:47,210 - tests.orchestrator - INFO - Created PVC: test-pvc-7dbf87c6 +2025-06-05 15:45:47,210 - tests.orchestrator - INFO - Waiting for PVC test-pvc-7dbf87c6 to be bound +2025-06-05 15:45:49,343 - tests.orchestrator - INFO - PVC test-pvc-7dbf87c6 is bound +2025-06-05 15:45:49,343 - tests.orchestrator - INFO - Creating PVC: test-pvc-83f66d6c +2025-06-05 15:45:49,411 - tests.orchestrator - INFO - Created PVC: test-pvc-83f66d6c +2025-06-05 15:45:49,411 - tests.orchestrator - INFO - Waiting for PVC test-pvc-83f66d6c to be bound +2025-06-05 15:45:51,548 - tests.orchestrator - INFO - PVC test-pvc-83f66d6c is bound +2025-06-05 15:45:51,549 - tests.orchestrator - INFO - Creating pod test-pod-c3e945dc on PVC test-pvc-7dbf87c6 +2025-06-05 15:45:51,631 - tests.orchestrator - INFO - Created pod: test-pod-c3e945dc using PVC: test-pvc-7dbf87c6 +2025-06-05 15:45:51,631 - tests.orchestrator - INFO - Waiting for pod test-pod-c3e945dc to be ready +2025-06-05 15:45:51,697 - tests.orchestrator - INFO - Pod test-pod-c3e945dc phase: Pending +2025-06-05 15:46:22,714 - tests.orchestrator - INFO - Pod test-pod-c3e945dc taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 15:46:22,714 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-c3e945dc ===== +2025-06-05 15:46:22,781 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:46:22,850 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:46:22,850 - tests.orchestrator - INFO - [2025-06-05 15:45:51+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:46:22,916 - tests.orchestrator - INFO - No logs available +2025-06-05 15:46:22,917 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:46:22,917 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-7dbf87c6'} +2025-06-05 15:46:22,917 - tests.orchestrator - INFO - - kube-api-access-vmkzs: {} +2025-06-05 15:46:22,917 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-c3e945dc ===== +2025-06-05 15:46:51,793 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-c3e945dc to be ready after 60s +2025-06-05 15:46:51,793 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-c3e945dc ===== +2025-06-05 15:46:51,860 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:46:51,930 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:46:51,930 - tests.orchestrator - INFO - [2025-06-05 15:45:51+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:46:51,996 - tests.orchestrator - INFO - No logs available +2025-06-05 15:46:51,996 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:46:51,996 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-7dbf87c6'} +2025-06-05 15:46:51,996 - tests.orchestrator - INFO - - kube-api-access-vmkzs: {} +2025-06-05 15:46:51,996 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-c3e945dc ===== +2025-06-05 15:46:51,997 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-c3e945dc to be ready +2025-06-05 15:46:51,997 - tests.orchestrator - INFO - Creating pod test-pod-7abe409e on PVC test-pvc-83f66d6c +2025-06-05 15:46:52,079 - tests.orchestrator - INFO - Created pod: test-pod-7abe409e using PVC: test-pvc-83f66d6c +2025-06-05 15:46:52,079 - tests.orchestrator - INFO - Waiting for pod test-pod-7abe409e to be ready +2025-06-05 15:46:52,146 - tests.orchestrator - INFO - Pod test-pod-7abe409e phase: Pending +2025-06-05 15:47:23,154 - tests.orchestrator - INFO - Pod test-pod-7abe409e taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 15:47:23,154 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-7abe409e ===== +2025-06-05 15:47:23,220 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:47:23,289 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:47:23,290 - tests.orchestrator - INFO - [2025-06-05 15:46:52+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:47:23,355 - tests.orchestrator - INFO - No logs available +2025-06-05 15:47:23,355 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:47:23,355 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-83f66d6c'} +2025-06-05 15:47:23,355 - tests.orchestrator - INFO - - kube-api-access-swfv7: {} +2025-06-05 15:47:23,355 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-7abe409e ===== +2025-06-05 15:47:52,325 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-7abe409e to be ready after 60s +2025-06-05 15:47:52,325 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-7abe409e ===== +2025-06-05 15:47:52,392 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:47:52,462 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:47:52,462 - tests.orchestrator - INFO - [2025-06-05 15:46:52+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:47:52,528 - tests.orchestrator - INFO - No logs available +2025-06-05 15:47:52,528 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:47:52,528 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-83f66d6c'} +2025-06-05 15:47:52,528 - tests.orchestrator - INFO - - kube-api-access-swfv7: {} +2025-06-05 15:47:52,528 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-7abe409e ===== +2025-06-05 15:47:52,528 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-7abe409e to be ready +2025-06-05 15:47:52,528 - tests.orchestrator - INFO - Creating pod test-pod-a9fc1f04 on PVC test-pvc-7dbf87c6 +2025-06-05 15:47:52,611 - tests.orchestrator - INFO - Created pod: test-pod-a9fc1f04 using PVC: test-pvc-7dbf87c6 +2025-06-05 15:47:52,612 - tests.orchestrator - INFO - Waiting for pod test-pod-a9fc1f04 to be ready +2025-06-05 15:47:52,678 - tests.orchestrator - INFO - Pod test-pod-a9fc1f04 phase: Pending +2025-06-05 15:48:23,696 - tests.orchestrator - INFO - Pod test-pod-a9fc1f04 taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 15:48:23,696 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-a9fc1f04 ===== +2025-06-05 15:48:23,762 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:48:23,831 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:48:23,832 - tests.orchestrator - INFO - [2025-06-05 15:47:52+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:48:23,898 - tests.orchestrator - INFO - No logs available +2025-06-05 15:48:23,898 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:48:23,898 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-7dbf87c6'} +2025-06-05 15:48:23,898 - tests.orchestrator - INFO - - kube-api-access-bh5hw: {} +2025-06-05 15:48:23,898 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-a9fc1f04 ===== +2025-06-05 15:48:52,778 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a9fc1f04 to be ready after 60s +2025-06-05 15:48:52,778 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-a9fc1f04 ===== +2025-06-05 15:48:52,845 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:48:52,917 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:48:52,917 - tests.orchestrator - INFO - [2025-06-05 15:47:52+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:48:52,983 - tests.orchestrator - INFO - No logs available +2025-06-05 15:48:52,983 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:48:52,983 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-7dbf87c6'} +2025-06-05 15:48:52,983 - tests.orchestrator - INFO - - kube-api-access-bh5hw: {} +2025-06-05 15:48:52,984 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-a9fc1f04 ===== +2025-06-05 15:48:52,984 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a9fc1f04 to be ready +2025-06-05 15:48:52,984 - tests.orchestrator - INFO - Testing read/write between pods test-pod-c3e945dc and test-pod-a9fc1f04 sharing PVC test-pvc-7dbf87c6 +2025-06-05 15:48:53,049 - tests.orchestrator - ERROR - Failed in read/write verification: (400) +Reason: Bad Request +HTTP response headers: HTTPHeaderDict({'Audit-Id': '0f444cd6-376b-4233-8e31-012234fb64c8', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 15:48:53 GMT', 'Content-Length': '166'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-c3e945dc does not have a host assigned","reason":"BadRequest","code":400} + + +2025-06-05 15:48:53,049 - tests.orchestrator - INFO - ============================================================ +2025-06-05 15:48:53,049 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_concurrent_pvc +2025-06-05 15:48:53,049 - tests.orchestrator - INFO - ============================================================ +2025-06-05 15:48:53,049 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations +2025-06-05 15:48:53,049 - tests.orchestrator - INFO - Creating 3 PVCs in quick succession +2025-06-05 15:48:53,127 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-74639d37 +2025-06-05 15:48:53,127 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-74639d37 to be bound +2025-06-05 15:48:55,259 - tests.orchestrator - INFO - PVC concurrent-pvc-74639d37 is bound +2025-06-05 15:48:55,328 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-be273a95 +2025-06-05 15:48:55,328 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-be273a95 to be bound +2025-06-05 15:48:57,460 - tests.orchestrator - INFO - PVC concurrent-pvc-be273a95 is bound +2025-06-05 15:48:57,530 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-421903bd +2025-06-05 15:48:57,530 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-421903bd to be bound +2025-06-05 15:48:59,662 - tests.orchestrator - INFO - PVC concurrent-pvc-421903bd is bound +2025-06-05 15:48:59,662 - tests.orchestrator - INFO - Successfully created 3 PVCs +2025-06-05 15:48:59,662 - tests.orchestrator - INFO - Creating pod test-pod-d257ffb1 on PVC concurrent-pvc-be273a95 +2025-06-05 15:48:59,737 - tests.orchestrator - INFO - Created pod: test-pod-d257ffb1 using PVC: concurrent-pvc-be273a95 +2025-06-05 15:48:59,737 - tests.orchestrator - INFO - Waiting for pod test-pod-d257ffb1 to be ready +2025-06-05 15:48:59,805 - tests.orchestrator - INFO - Pod test-pod-d257ffb1 phase: Pending +2025-06-05 15:49:30,827 - tests.orchestrator - INFO - Pod test-pod-d257ffb1 taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 15:49:30,828 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-d257ffb1 ===== +2025-06-05 15:49:30,894 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:49:30,964 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:49:30,964 - tests.orchestrator - INFO - [2025-06-05 15:48:59+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:49:31,030 - tests.orchestrator - INFO - No logs available +2025-06-05 15:49:31,030 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:49:31,030 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'concurrent-pvc-be273a95'} +2025-06-05 15:49:31,031 - tests.orchestrator - INFO - - kube-api-access-tr4s4: {} +2025-06-05 15:49:31,031 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-d257ffb1 ===== +2025-06-05 15:49:59,906 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-d257ffb1 to be ready after 60s +2025-06-05 15:49:59,907 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-d257ffb1 ===== +2025-06-05 15:49:59,974 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:50:00,045 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:50:00,045 - tests.orchestrator - INFO - [2025-06-05 15:48:59+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:50:00,111 - tests.orchestrator - INFO - No logs available +2025-06-05 15:50:00,112 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:50:00,112 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'concurrent-pvc-be273a95'} +2025-06-05 15:50:00,112 - tests.orchestrator - INFO - - kube-api-access-tr4s4: {} +2025-06-05 15:50:00,112 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-d257ffb1 ===== +2025-06-05 15:50:00,112 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-d257ffb1 to be ready +2025-06-05 15:50:00,112 - tests.orchestrator - INFO - Creating pod test-pod-8c9ea815 on PVC concurrent-pvc-421903bd +2025-06-05 15:50:00,195 - tests.orchestrator - INFO - Created pod: test-pod-8c9ea815 using PVC: concurrent-pvc-421903bd +2025-06-05 15:50:00,196 - tests.orchestrator - INFO - Waiting for pod test-pod-8c9ea815 to be ready +2025-06-05 15:50:00,263 - tests.orchestrator - INFO - Pod test-pod-8c9ea815 phase: Pending +2025-06-05 15:50:31,275 - tests.orchestrator - INFO - Pod test-pod-8c9ea815 taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 15:50:31,275 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-8c9ea815 ===== +2025-06-05 15:50:31,341 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:50:31,411 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:50:31,411 - tests.orchestrator - INFO - [2025-06-05 15:50:00+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:50:31,526 - tests.orchestrator - INFO - No logs available +2025-06-05 15:50:31,526 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:50:31,526 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'concurrent-pvc-421903bd'} +2025-06-05 15:50:31,526 - tests.orchestrator - INFO - - kube-api-access-mk5vl: {} +2025-06-05 15:50:31,527 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-8c9ea815 ===== +2025-06-05 15:51:00,401 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-8c9ea815 to be ready after 60s +2025-06-05 15:51:00,401 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-8c9ea815 ===== +2025-06-05 15:51:00,469 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:51:00,539 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:51:00,539 - tests.orchestrator - INFO - [2025-06-05 15:50:00+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:51:00,604 - tests.orchestrator - INFO - No logs available +2025-06-05 15:51:00,604 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:51:00,604 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'concurrent-pvc-421903bd'} +2025-06-05 15:51:00,604 - tests.orchestrator - INFO - - kube-api-access-mk5vl: {} +2025-06-05 15:51:00,605 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-8c9ea815 ===== +2025-06-05 15:51:00,605 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-8c9ea815 to be ready +2025-06-05 15:51:00,605 - tests.orchestrator - INFO - Creating pod test-pod-3106ab46 on PVC concurrent-pvc-74639d37 +2025-06-05 15:51:00,689 - tests.orchestrator - INFO - Created pod: test-pod-3106ab46 using PVC: concurrent-pvc-74639d37 +2025-06-05 15:51:00,689 - tests.orchestrator - INFO - Waiting for pod test-pod-3106ab46 to be ready +2025-06-05 15:51:00,755 - tests.orchestrator - INFO - Pod test-pod-3106ab46 phase: Pending +2025-06-05 15:51:31,779 - tests.orchestrator - INFO - Pod test-pod-3106ab46 taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 15:51:31,779 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-3106ab46 ===== +2025-06-05 15:51:31,852 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:51:31,927 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:51:31,927 - tests.orchestrator - INFO - [2025-06-05 15:51:00+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:51:31,998 - tests.orchestrator - INFO - No logs available +2025-06-05 15:51:31,998 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:51:31,998 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'concurrent-pvc-74639d37'} +2025-06-05 15:51:31,998 - tests.orchestrator - INFO - - kube-api-access-97jdn: {} +2025-06-05 15:51:31,998 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-3106ab46 ===== +2025-06-05 15:52:00,875 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-3106ab46 to be ready after 60s +2025-06-05 15:52:00,876 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-3106ab46 ===== +2025-06-05 15:52:00,943 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:52:01,013 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:52:01,013 - tests.orchestrator - INFO - [2025-06-05 15:51:00+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:52:01,079 - tests.orchestrator - INFO - No logs available +2025-06-05 15:52:01,079 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:52:01,079 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'concurrent-pvc-74639d37'} +2025-06-05 15:52:01,080 - tests.orchestrator - INFO - - kube-api-access-97jdn: {} +2025-06-05 15:52:01,080 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-3106ab46 ===== +2025-06-05 15:52:01,080 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-3106ab46 to be ready +2025-06-05 15:52:01,080 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-be273a95 +2025-06-05 15:52:01,080 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-be273a95 +2025-06-05 15:52:01,080 - tests.orchestrator - INFO - Deleting pod: test-pod-d257ffb1 from PVC: concurrent-pvc-be273a95 +2025-06-05 15:52:01,164 - tests.orchestrator - INFO - Waiting for pod test-pod-d257ffb1 to be deleted +2025-06-05 15:52:01,229 - tests.orchestrator - INFO - Pod test-pod-d257ffb1 has been deleted +2025-06-05 15:52:01,230 - tests.orchestrator - INFO - Deleted pod: test-pod-d257ffb1 +2025-06-05 15:52:01,301 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-be273a95 to be deleted +2025-06-05 15:52:01,366 - tests.orchestrator - INFO - PVC concurrent-pvc-be273a95 has been deleted +2025-06-05 15:52:01,367 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-be273a95 +2025-06-05 15:52:01,367 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-421903bd +2025-06-05 15:52:01,367 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-421903bd +2025-06-05 15:52:01,367 - tests.orchestrator - INFO - Deleting pod: test-pod-8c9ea815 from PVC: concurrent-pvc-421903bd +2025-06-05 15:52:01,548 - tests.orchestrator - INFO - Waiting for pod test-pod-8c9ea815 to be deleted +2025-06-05 15:52:01,774 - tests.orchestrator - INFO - Pod test-pod-8c9ea815 has been deleted +2025-06-05 15:52:01,774 - tests.orchestrator - INFO - Deleted pod: test-pod-8c9ea815 +2025-06-05 15:52:01,852 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-421903bd to be deleted +2025-06-05 15:52:01,924 - tests.orchestrator - INFO - PVC concurrent-pvc-421903bd has been deleted +2025-06-05 15:52:01,925 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-421903bd +2025-06-05 15:52:01,925 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-74639d37 +2025-06-05 15:52:01,925 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-74639d37 +2025-06-05 15:52:01,925 - tests.orchestrator - INFO - Deleting pod: test-pod-3106ab46 from PVC: concurrent-pvc-74639d37 +2025-06-05 15:52:02,021 - tests.orchestrator - INFO - Waiting for pod test-pod-3106ab46 to be deleted +2025-06-05 15:52:02,092 - tests.orchestrator - INFO - Pod test-pod-3106ab46 has been deleted +2025-06-05 15:52:02,092 - tests.orchestrator - INFO - Deleted pod: test-pod-3106ab46 +2025-06-05 15:52:02,172 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-74639d37 to be deleted +2025-06-05 15:52:02,238 - tests.orchestrator - INFO - PVC concurrent-pvc-74639d37 has been deleted +2025-06-05 15:52:02,238 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-74639d37 +2025-06-05 15:52:02,238 - tests.orchestrator - INFO - Rapid PVC scenario completed +2025-06-05 15:52:02,238 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_concurrent_pvc +2025-06-05 15:52:02,238 - tests.orchestrator - INFO - ------------------------------------------------------------ +2025-06-05 15:52:02,238 - tests.orchestrator - INFO - Deleting pod: test-pod-7abe409e from PVC: test-pvc-83f66d6c +2025-06-05 15:52:02,322 - tests.orchestrator - INFO - Waiting for pod test-pod-7abe409e to be deleted +2025-06-05 15:52:02,390 - tests.orchestrator - INFO - Pod test-pod-7abe409e has been deleted +2025-06-05 15:52:02,390 - tests.orchestrator - INFO - Deleted pod: test-pod-7abe409e +2025-06-05 15:52:02,390 - tests.orchestrator - INFO - Deleting PVC: test-pvc-7dbf87c6 +2025-06-05 15:52:02,390 - tests.orchestrator - INFO - Deleting 2 pods using PVC test-pvc-7dbf87c6 +2025-06-05 15:52:02,390 - tests.orchestrator - INFO - Deleting pod: test-pod-c3e945dc from PVC: test-pvc-7dbf87c6 +2025-06-05 15:52:02,471 - tests.orchestrator - INFO - Waiting for pod test-pod-c3e945dc to be deleted +2025-06-05 15:52:02,537 - tests.orchestrator - INFO - Pod test-pod-c3e945dc has been deleted +2025-06-05 15:52:02,537 - tests.orchestrator - INFO - Deleted pod: test-pod-c3e945dc +2025-06-05 15:52:02,537 - tests.orchestrator - INFO - Deleting pod: test-pod-a9fc1f04 from PVC: test-pvc-7dbf87c6 +2025-06-05 15:52:02,625 - tests.orchestrator - INFO - Waiting for pod test-pod-a9fc1f04 to be deleted +2025-06-05 15:52:02,691 - tests.orchestrator - INFO - Pod test-pod-a9fc1f04 has been deleted +2025-06-05 15:52:02,691 - tests.orchestrator - INFO - Deleted pod: test-pod-a9fc1f04 +2025-06-05 15:52:02,763 - tests.orchestrator - INFO - Waiting for PVC test-pvc-7dbf87c6 to be deleted +2025-06-05 15:52:02,828 - tests.orchestrator - INFO - PVC test-pvc-7dbf87c6 has been deleted +2025-06-05 15:52:02,828 - tests.orchestrator - INFO - Deleted PVC: test-pvc-7dbf87c6 +2025-06-05 15:52:02,828 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations +2025-06-05 15:52:02,828 - tests.orchestrator - INFO - Test completed in 375.88 seconds +2025-06-05 15:52:02,828 - tests.orchestrator - INFO - Cleaning up resources... +2025-06-05 15:52:02,828 - tests.orchestrator - INFO - Deleting PVC: test-pvc-83f66d6c +2025-06-05 15:52:02,903 - tests.orchestrator - INFO - Waiting for PVC test-pvc-83f66d6c to be deleted +2025-06-05 15:52:02,968 - tests.orchestrator - INFO - PVC test-pvc-83f66d6c has been deleted +2025-06-05 15:52:02,968 - tests.orchestrator - INFO - Deleted PVC: test-pvc-83f66d6c +2025-06-05 15:52:02,968 - tests.orchestrator - INFO - Cleanup completed +2025-06-05 15:52:02,968 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-05 15:52:02,968 - tests.orchestrator - INFO - --- Operations --- +2025-06-05 15:52:02,968 - tests.orchestrator - INFO - create_pvc: 5 succeeded, 0 failed (100.0%) +2025-06-05 15:52:02,968 - tests.orchestrator - INFO - attach_pod: 6 succeeded, 0 failed (100.0%) +2025-06-05 15:52:02,968 - tests.orchestrator - INFO - delete_pod: 6 succeeded, 0 failed (100.0%) +2025-06-05 15:52:02,968 - tests.orchestrator - INFO - delete_pvc: 5 succeeded, 0 failed (100.0%) +2025-06-05 15:52:02,968 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) +2025-06-05 15:52:02,968 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) +2025-06-05 15:52:02,968 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-05 15:52:02,969 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-05 15:52:02,969 - tests.orchestrator - INFO - many_to_one: No runs +2025-06-05 15:52:02,969 - tests.orchestrator - INFO - one_to_one: No runs +2025-06-05 15:52:02,969 - tests.orchestrator - INFO - concurrent_pvc: 1 succeeded, 0 failed out of 1 runs (100.0%) +2025-06-05 15:52:02,969 - tests.orchestrator - INFO - ========================================= +2025-06-05 15:52:02,969 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250605_155202.json +2025-06-05 15:52:03,011 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250605_155202_20250605_155202.json +2025-06-05 15:52:03,012 - __main__ - INFO - All tests completed successfully +2025-06-05 15:59:48,062 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-05 15:59:48,062 - __main__ - INFO - Verifying AWS credentials +2025-06-05 15:59:48,063 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 15:59:48,064 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 15:59:48,064 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 15:59:48,064 - __main__ - INFO - Running orchestrator stress test suite +2025-06-05 15:59:48,065 - __main__ - INFO - Starting orchestrator for 300 seconds +2025-06-05 15:59:48,065 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-05 15:59:48,809 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 15:59:49,025 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 15:59:49,027 - __main__ - INFO - Test duration overridden to 300 seconds +2025-06-05 15:59:49,027 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) +2025-06-05 15:59:49,027 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-05 15:59:49,218 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 15:59:49,218 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 15:59:49,218 - tests.orchestrator - INFO - Creating PVC: test-pvc-47b1c666 +2025-06-05 15:59:49,295 - tests.orchestrator - INFO - Created PVC: test-pvc-47b1c666 +2025-06-05 15:59:49,296 - tests.orchestrator - INFO - Waiting for PVC test-pvc-47b1c666 to be bound +2025-06-05 15:59:51,432 - tests.orchestrator - INFO - PVC test-pvc-47b1c666 is bound +2025-06-05 15:59:51,433 - tests.orchestrator - INFO - Creating PVC: test-pvc-13de15dd +2025-06-05 15:59:51,506 - tests.orchestrator - INFO - Created PVC: test-pvc-13de15dd +2025-06-05 15:59:51,507 - tests.orchestrator - INFO - Waiting for PVC test-pvc-13de15dd to be bound +2025-06-05 15:59:53,652 - tests.orchestrator - INFO - PVC test-pvc-13de15dd is bound +2025-06-05 15:59:53,652 - tests.orchestrator - INFO - Creating pod test-pod-b12f5a09 on PVC test-pvc-47b1c666 +2025-06-05 15:59:53,736 - tests.orchestrator - INFO - Created pod: test-pod-b12f5a09 using PVC: test-pvc-47b1c666 +2025-06-05 15:59:53,736 - tests.orchestrator - INFO - Waiting for pod test-pod-b12f5a09 to be ready +2025-06-05 15:59:53,809 - tests.orchestrator - INFO - Pod test-pod-b12f5a09 phase: Pending +2025-06-05 16:00:24,845 - tests.orchestrator - INFO - Pod test-pod-b12f5a09 taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 16:00:24,846 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-b12f5a09 ===== +2025-06-05 16:00:24,914 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:00:24,986 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:00:24,987 - tests.orchestrator - INFO - [2025-06-05 15:59:53+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:00:25,052 - tests.orchestrator - INFO - No logs available +2025-06-05 16:00:25,053 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:00:25,053 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-47b1c666'} +2025-06-05 16:00:25,053 - tests.orchestrator - INFO - - kube-api-access-q49n5: {} +2025-06-05 16:00:25,053 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-b12f5a09 ===== +2025-06-05 16:00:54,038 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b12f5a09 to be ready after 60s +2025-06-05 16:00:54,038 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-b12f5a09 ===== +2025-06-05 16:00:54,106 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:00:54,179 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:00:54,179 - tests.orchestrator - INFO - [2025-06-05 15:59:53+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:00:54,244 - tests.orchestrator - INFO - No logs available +2025-06-05 16:00:54,245 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:00:54,245 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-47b1c666'} +2025-06-05 16:00:54,245 - tests.orchestrator - INFO - - kube-api-access-q49n5: {} +2025-06-05 16:00:54,245 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-b12f5a09 ===== +2025-06-05 16:00:54,245 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b12f5a09 to be ready +2025-06-05 16:00:54,245 - tests.orchestrator - INFO - Creating pod test-pod-397db057 on PVC test-pvc-13de15dd +2025-06-05 16:00:54,335 - tests.orchestrator - INFO - Created pod: test-pod-397db057 using PVC: test-pvc-13de15dd +2025-06-05 16:00:54,336 - tests.orchestrator - INFO - Waiting for pod test-pod-397db057 to be ready +2025-06-05 16:00:54,401 - tests.orchestrator - INFO - Pod test-pod-397db057 phase: Pending +2025-06-05 16:01:25,423 - tests.orchestrator - INFO - Pod test-pod-397db057 taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 16:01:25,423 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-397db057 ===== +2025-06-05 16:01:25,491 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:01:25,563 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:01:25,563 - tests.orchestrator - INFO - [2025-06-05 16:00:54+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:01:25,629 - tests.orchestrator - INFO - No logs available +2025-06-05 16:01:25,629 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:01:25,629 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-13de15dd'} +2025-06-05 16:01:25,630 - tests.orchestrator - INFO - - kube-api-access-q4p66: {} +2025-06-05 16:01:25,630 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-397db057 ===== +2025-06-05 16:01:54,513 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-397db057 to be ready after 60s +2025-06-05 16:01:54,513 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-397db057 ===== +2025-06-05 16:01:54,581 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:01:54,652 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:01:54,653 - tests.orchestrator - INFO - [2025-06-05 16:00:54+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:01:54,717 - tests.orchestrator - INFO - No logs available +2025-06-05 16:01:54,718 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:01:54,718 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-13de15dd'} +2025-06-05 16:01:54,718 - tests.orchestrator - INFO - - kube-api-access-q4p66: {} +2025-06-05 16:01:54,718 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-397db057 ===== +2025-06-05 16:01:54,718 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-397db057 to be ready +2025-06-05 16:01:54,718 - tests.orchestrator - INFO - Creating pod test-pod-b378ede9 on PVC test-pvc-13de15dd +2025-06-05 16:01:54,802 - tests.orchestrator - INFO - Created pod: test-pod-b378ede9 using PVC: test-pvc-13de15dd +2025-06-05 16:01:54,802 - tests.orchestrator - INFO - Waiting for pod test-pod-b378ede9 to be ready +2025-06-05 16:01:54,868 - tests.orchestrator - INFO - Pod test-pod-b378ede9 phase: Pending +2025-06-05 16:02:25,904 - tests.orchestrator - INFO - Pod test-pod-b378ede9 taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 16:02:25,904 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-b378ede9 ===== +2025-06-05 16:02:25,971 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:02:26,043 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:02:26,043 - tests.orchestrator - INFO - [2025-06-05 16:01:54+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:02:26,109 - tests.orchestrator - INFO - No logs available +2025-06-05 16:02:26,110 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:02:26,110 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-13de15dd'} +2025-06-05 16:02:26,110 - tests.orchestrator - INFO - - kube-api-access-ggwzs: {} +2025-06-05 16:02:26,110 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-b378ede9 ===== +2025-06-05 16:02:54,996 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b378ede9 to be ready after 60s +2025-06-05 16:02:54,996 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-b378ede9 ===== +2025-06-05 16:02:55,065 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:02:55,137 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:02:55,137 - tests.orchestrator - INFO - [2025-06-05 16:01:54+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:02:55,204 - tests.orchestrator - INFO - No logs available +2025-06-05 16:02:55,204 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:02:55,204 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-13de15dd'} +2025-06-05 16:02:55,204 - tests.orchestrator - INFO - - kube-api-access-ggwzs: {} +2025-06-05 16:02:55,204 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-b378ede9 ===== +2025-06-05 16:02:55,205 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b378ede9 to be ready +2025-06-05 16:02:55,205 - tests.orchestrator - INFO - Testing read/write between pods test-pod-397db057 and test-pod-b378ede9 sharing PVC test-pvc-13de15dd +2025-06-05 16:02:55,270 - tests.orchestrator - ERROR - Failed in read/write verification: (400) +Reason: Bad Request +HTTP response headers: HTTPHeaderDict({'Audit-Id': 'c1161df1-c0d5-4407-a027-5613955ae526', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 16:02:55 GMT', 'Content-Length': '166'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-397db057 does not have a host assigned","reason":"BadRequest","code":400} + + +2025-06-05 16:02:55,270 - tests.orchestrator - INFO - ============================================================ +2025-06-05 16:02:55,270 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_many_to_one +2025-06-05 16:02:55,270 - tests.orchestrator - INFO - ============================================================ +2025-06-05 16:02:55,270 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +2025-06-05 16:02:55,270 - tests.orchestrator - INFO - STARTING MANY-TO-ONE SCENARIO DIAGNOSTICS +2025-06-05 16:02:55,270 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +2025-06-05 16:02:55,271 - tests.orchestrator - INFO - [MANY2ONE] STEP 1: Creating dedicated PVC: many2one-68cd2a7e +2025-06-05 16:02:55,271 - tests.orchestrator - INFO - [MANY2ONE] Creating PVC manifest with storageClassName: efs-sc +2025-06-05 16:02:55,352 - tests.orchestrator - INFO - [MANY2ONE] PVC many2one-68cd2a7e created successfully +2025-06-05 16:02:55,352 - tests.orchestrator - INFO - [MANY2ONE] Waiting for PVC many2one-68cd2a7e to be bound... +2025-06-05 16:02:55,352 - tests.orchestrator - INFO - Waiting for PVC many2one-68cd2a7e to be bound +2025-06-05 16:02:57,487 - tests.orchestrator - INFO - PVC many2one-68cd2a7e is bound +2025-06-05 16:02:57,552 - tests.orchestrator - INFO - [MANY2ONE] PVC status: phase=Bound, capacity={'storage': '1Gi'} +2025-06-05 16:02:57,552 - tests.orchestrator - INFO - [MANY2ONE] STEP 2: Creating 4 pods for the same PVC many2one-68cd2a7e +2025-06-05 16:02:57,552 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 1/4 for PVC many2one-68cd2a7e +2025-06-05 16:02:57,552 - tests.orchestrator - INFO - Creating pod test-pod-4518fc8f on PVC many2one-68cd2a7e +2025-06-05 16:02:57,632 - tests.orchestrator - INFO - Created pod: test-pod-4518fc8f using PVC: many2one-68cd2a7e +2025-06-05 16:02:57,633 - tests.orchestrator - INFO - Waiting for pod test-pod-4518fc8f to be ready +2025-06-05 16:02:57,699 - tests.orchestrator - INFO - Pod test-pod-4518fc8f phase: Pending +2025-06-05 16:03:28,731 - tests.orchestrator - INFO - Pod test-pod-4518fc8f taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 16:03:28,732 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-4518fc8f ===== +2025-06-05 16:03:28,800 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:03:28,872 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:03:28,872 - tests.orchestrator - INFO - [2025-06-05 16:02:57+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:03:28,937 - tests.orchestrator - INFO - No logs available +2025-06-05 16:03:28,937 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:03:28,937 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-68cd2a7e'} +2025-06-05 16:03:28,937 - tests.orchestrator - INFO - - kube-api-access-v9z8m: {} +2025-06-05 16:03:28,937 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-4518fc8f ===== +2025-06-05 16:03:57,829 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-4518fc8f to be ready after 60s +2025-06-05 16:03:57,829 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-4518fc8f ===== +2025-06-05 16:03:57,897 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:03:57,967 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:03:57,968 - tests.orchestrator - INFO - [2025-06-05 16:02:57+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:03:58,032 - tests.orchestrator - INFO - No logs available +2025-06-05 16:03:58,032 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:03:58,032 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-68cd2a7e'} +2025-06-05 16:03:58,032 - tests.orchestrator - INFO - - kube-api-access-v9z8m: {} +2025-06-05 16:03:58,032 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-4518fc8f ===== +2025-06-05 16:03:58,033 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-4518fc8f to be ready +2025-06-05 16:03:58,033 - tests.orchestrator - ERROR - [MANY2ONE] Failed to create pod 1/4 +2025-06-05 16:03:58,033 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 2/4 for PVC many2one-68cd2a7e +2025-06-05 16:03:58,033 - tests.orchestrator - INFO - Creating pod test-pod-0c1359ff on PVC many2one-68cd2a7e +2025-06-05 16:03:58,117 - tests.orchestrator - INFO - Created pod: test-pod-0c1359ff using PVC: many2one-68cd2a7e +2025-06-05 16:03:58,117 - tests.orchestrator - INFO - Waiting for pod test-pod-0c1359ff to be ready +2025-06-05 16:03:58,183 - tests.orchestrator - INFO - Pod test-pod-0c1359ff phase: Pending +2025-06-05 16:04:29,215 - tests.orchestrator - INFO - Pod test-pod-0c1359ff taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 16:04:29,215 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-0c1359ff ===== +2025-06-05 16:04:29,283 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:04:29,355 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:04:29,356 - tests.orchestrator - INFO - [2025-06-05 16:03:58+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:04:29,422 - tests.orchestrator - INFO - No logs available +2025-06-05 16:04:29,422 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:04:29,422 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-68cd2a7e'} +2025-06-05 16:04:29,422 - tests.orchestrator - INFO - - kube-api-access-6jtzc: {} +2025-06-05 16:04:29,422 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-0c1359ff ===== +2025-06-05 16:04:58,441 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-0c1359ff to be ready after 60s +2025-06-05 16:04:58,441 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-0c1359ff ===== +2025-06-05 16:04:58,507 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:04:58,577 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:04:58,577 - tests.orchestrator - INFO - [2025-06-05 16:03:58+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:04:58,644 - tests.orchestrator - INFO - No logs available +2025-06-05 16:04:58,644 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:04:58,644 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-68cd2a7e'} +2025-06-05 16:04:58,644 - tests.orchestrator - INFO - - kube-api-access-6jtzc: {} +2025-06-05 16:04:58,644 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-0c1359ff ===== +2025-06-05 16:04:58,644 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-0c1359ff to be ready +2025-06-05 16:04:58,644 - tests.orchestrator - ERROR - [MANY2ONE] Failed to create pod 2/4 +2025-06-05 16:04:58,644 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 3/4 for PVC many2one-68cd2a7e +2025-06-05 16:04:58,645 - tests.orchestrator - INFO - Creating pod test-pod-1e078739 on PVC many2one-68cd2a7e +2025-06-05 16:04:58,732 - tests.orchestrator - INFO - Created pod: test-pod-1e078739 using PVC: many2one-68cd2a7e +2025-06-05 16:04:58,732 - tests.orchestrator - INFO - Waiting for pod test-pod-1e078739 to be ready +2025-06-05 16:04:58,800 - tests.orchestrator - INFO - Pod test-pod-1e078739 phase: Pending +2025-06-05 16:05:29,826 - tests.orchestrator - INFO - Pod test-pod-1e078739 taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 16:05:29,826 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-1e078739 ===== +2025-06-05 16:05:29,893 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:05:29,963 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:05:29,963 - tests.orchestrator - INFO - [2025-06-05 16:04:58+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:05:30,027 - tests.orchestrator - INFO - No logs available +2025-06-05 16:05:30,027 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:05:30,028 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-68cd2a7e'} +2025-06-05 16:05:30,028 - tests.orchestrator - INFO - - kube-api-access-rksgf: {} +2025-06-05 16:05:30,028 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-1e078739 ===== +2025-06-05 16:05:58,917 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1e078739 to be ready after 60s +2025-06-05 16:05:58,917 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-1e078739 ===== +2025-06-05 16:05:58,985 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:05:59,057 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:05:59,057 - tests.orchestrator - INFO - [2025-06-05 16:04:58+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:05:59,122 - tests.orchestrator - INFO - No logs available +2025-06-05 16:05:59,122 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:05:59,122 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-68cd2a7e'} +2025-06-05 16:05:59,122 - tests.orchestrator - INFO - - kube-api-access-rksgf: {} +2025-06-05 16:05:59,122 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-1e078739 ===== +2025-06-05 16:05:59,122 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1e078739 to be ready +2025-06-05 16:05:59,122 - tests.orchestrator - ERROR - [MANY2ONE] Failed to create pod 3/4 +2025-06-05 16:05:59,122 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 4/4 for PVC many2one-68cd2a7e +2025-06-05 16:05:59,122 - tests.orchestrator - INFO - Creating pod test-pod-a5d14df4 on PVC many2one-68cd2a7e +2025-06-05 16:05:59,208 - tests.orchestrator - INFO - Created pod: test-pod-a5d14df4 using PVC: many2one-68cd2a7e +2025-06-05 16:05:59,208 - tests.orchestrator - INFO - Waiting for pod test-pod-a5d14df4 to be ready +2025-06-05 16:05:59,273 - tests.orchestrator - INFO - Pod test-pod-a5d14df4 phase: Pending +2025-06-05 16:06:30,377 - tests.orchestrator - INFO - Pod test-pod-a5d14df4 taking longer than expected to become ready (31.2s). Collecting diagnostics... +2025-06-05 16:06:30,377 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-a5d14df4 ===== +2025-06-05 16:06:30,444 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:06:30,516 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:06:30,517 - tests.orchestrator - INFO - [2025-06-05 16:05:59+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:06:30,583 - tests.orchestrator - INFO - No logs available +2025-06-05 16:06:30,583 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:06:30,584 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-68cd2a7e'} +2025-06-05 16:06:30,584 - tests.orchestrator - INFO - - kube-api-access-dcxmb: {} +2025-06-05 16:06:30,584 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-a5d14df4 ===== +2025-06-05 16:06:59,487 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a5d14df4 to be ready after 60s +2025-06-05 16:06:59,487 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-a5d14df4 ===== +2025-06-05 16:06:59,555 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:06:59,627 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:06:59,628 - tests.orchestrator - INFO - [2025-06-05 16:05:59+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:06:59,694 - tests.orchestrator - INFO - No logs available +2025-06-05 16:06:59,694 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:06:59,694 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-68cd2a7e'} +2025-06-05 16:06:59,695 - tests.orchestrator - INFO - - kube-api-access-dcxmb: {} +2025-06-05 16:06:59,695 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-a5d14df4 ===== +2025-06-05 16:06:59,695 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a5d14df4 to be ready +2025-06-05 16:06:59,695 - tests.orchestrator - ERROR - [MANY2ONE] Failed to create pod 4/4 +2025-06-05 16:06:59,695 - tests.orchestrator - INFO - [MANY2ONE] Created 0/4 pods successfully +2025-06-05 16:06:59,695 - tests.orchestrator - ERROR - [MANY2ONE] FAILED: Insufficient pods created (0), need at least 2 for read/write test +2025-06-05 16:06:59,695 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_many_to_one +2025-06-05 16:06:59,695 - tests.orchestrator - INFO - ------------------------------------------------------------ +2025-06-05 16:06:59,695 - tests.orchestrator - INFO - Deleting pod: test-pod-1e078739 from PVC: many2one-68cd2a7e +2025-06-05 16:06:59,777 - tests.orchestrator - INFO - Waiting for pod test-pod-1e078739 to be deleted +2025-06-05 16:06:59,841 - tests.orchestrator - INFO - Pod test-pod-1e078739 has been deleted +2025-06-05 16:06:59,842 - tests.orchestrator - INFO - Deleted pod: test-pod-1e078739 +2025-06-05 16:06:59,842 - tests.orchestrator - INFO - Deleting PVC: test-pvc-13de15dd +2025-06-05 16:06:59,842 - tests.orchestrator - INFO - Deleting 2 pods using PVC test-pvc-13de15dd +2025-06-05 16:06:59,842 - tests.orchestrator - INFO - Deleting pod: test-pod-397db057 from PVC: test-pvc-13de15dd +2025-06-05 16:06:59,923 - tests.orchestrator - INFO - Waiting for pod test-pod-397db057 to be deleted +2025-06-05 16:06:59,989 - tests.orchestrator - INFO - Pod test-pod-397db057 has been deleted +2025-06-05 16:06:59,989 - tests.orchestrator - INFO - Deleted pod: test-pod-397db057 +2025-06-05 16:06:59,989 - tests.orchestrator - INFO - Deleting pod: test-pod-b378ede9 from PVC: test-pvc-13de15dd +2025-06-05 16:07:00,077 - tests.orchestrator - INFO - Waiting for pod test-pod-b378ede9 to be deleted +2025-06-05 16:07:00,142 - tests.orchestrator - INFO - Pod test-pod-b378ede9 has been deleted +2025-06-05 16:07:00,142 - tests.orchestrator - INFO - Deleted pod: test-pod-b378ede9 +2025-06-05 16:07:00,213 - tests.orchestrator - INFO - Waiting for PVC test-pvc-13de15dd to be deleted +2025-06-05 16:07:00,280 - tests.orchestrator - INFO - PVC test-pvc-13de15dd has been deleted +2025-06-05 16:07:00,280 - tests.orchestrator - INFO - Deleted PVC: test-pvc-13de15dd +2025-06-05 16:07:00,280 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations +2025-06-05 16:07:00,280 - tests.orchestrator - INFO - Test completed in 431.25 seconds +2025-06-05 16:07:00,280 - tests.orchestrator - INFO - Cleaning up resources... +2025-06-05 16:07:00,280 - tests.orchestrator - INFO - Deleting pod: test-pod-b12f5a09 from PVC: test-pvc-47b1c666 +2025-06-05 16:07:00,364 - tests.orchestrator - INFO - Waiting for pod test-pod-b12f5a09 to be deleted +2025-06-05 16:07:00,430 - tests.orchestrator - INFO - Pod test-pod-b12f5a09 has been deleted +2025-06-05 16:07:00,430 - tests.orchestrator - INFO - Deleted pod: test-pod-b12f5a09 +2025-06-05 16:07:00,431 - tests.orchestrator - INFO - Deleting pod: test-pod-4518fc8f from PVC: many2one-68cd2a7e +2025-06-05 16:07:00,512 - tests.orchestrator - INFO - Waiting for pod test-pod-4518fc8f to be deleted +2025-06-05 16:07:00,579 - tests.orchestrator - INFO - Pod test-pod-4518fc8f has been deleted +2025-06-05 16:07:00,579 - tests.orchestrator - INFO - Deleted pod: test-pod-4518fc8f +2025-06-05 16:07:00,579 - tests.orchestrator - INFO - Deleting pod: test-pod-0c1359ff from PVC: many2one-68cd2a7e +2025-06-05 16:07:00,659 - tests.orchestrator - INFO - Waiting for pod test-pod-0c1359ff to be deleted +2025-06-05 16:07:00,725 - tests.orchestrator - INFO - Pod test-pod-0c1359ff has been deleted +2025-06-05 16:07:00,726 - tests.orchestrator - INFO - Deleted pod: test-pod-0c1359ff +2025-06-05 16:07:00,726 - tests.orchestrator - INFO - Deleting pod: test-pod-a5d14df4 from PVC: many2one-68cd2a7e +2025-06-05 16:07:00,813 - tests.orchestrator - INFO - Waiting for pod test-pod-a5d14df4 to be deleted +2025-06-05 16:07:00,878 - tests.orchestrator - INFO - Pod test-pod-a5d14df4 has been deleted +2025-06-05 16:07:00,878 - tests.orchestrator - INFO - Deleted pod: test-pod-a5d14df4 +2025-06-05 16:07:00,878 - tests.orchestrator - INFO - Deleting PVC: test-pvc-47b1c666 +2025-06-05 16:07:00,951 - tests.orchestrator - INFO - Waiting for PVC test-pvc-47b1c666 to be deleted +2025-06-05 16:07:01,017 - tests.orchestrator - INFO - PVC test-pvc-47b1c666 has been deleted +2025-06-05 16:07:01,018 - tests.orchestrator - INFO - Deleted PVC: test-pvc-47b1c666 +2025-06-05 16:07:01,018 - tests.orchestrator - INFO - Deleting PVC: many2one-68cd2a7e +2025-06-05 16:07:01,090 - tests.orchestrator - INFO - Waiting for PVC many2one-68cd2a7e to be deleted +2025-06-05 16:07:01,157 - tests.orchestrator - INFO - PVC many2one-68cd2a7e has been deleted +2025-06-05 16:07:01,157 - tests.orchestrator - INFO - Deleted PVC: many2one-68cd2a7e +2025-06-05 16:07:01,157 - tests.orchestrator - INFO - Cleanup completed +2025-06-05 16:07:01,157 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-05 16:07:01,157 - tests.orchestrator - INFO - --- Operations --- +2025-06-05 16:07:01,157 - tests.orchestrator - INFO - create_pvc: 2 succeeded, 0 failed (100.0%) +2025-06-05 16:07:01,157 - tests.orchestrator - INFO - attach_pod: 7 succeeded, 0 failed (100.0%) +2025-06-05 16:07:01,157 - tests.orchestrator - INFO - delete_pod: 7 succeeded, 0 failed (100.0%) +2025-06-05 16:07:01,157 - tests.orchestrator - INFO - delete_pvc: 3 succeeded, 0 failed (100.0%) +2025-06-05 16:07:01,157 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) +2025-06-05 16:07:01,157 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) +2025-06-05 16:07:01,157 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-05 16:07:01,158 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-05 16:07:01,158 - tests.orchestrator - INFO - many_to_one: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-05 16:07:01,158 - tests.orchestrator - INFO - one_to_one: No runs +2025-06-05 16:07:01,158 - tests.orchestrator - INFO - concurrent_pvc: No runs +2025-06-05 16:07:01,158 - tests.orchestrator - INFO - ========================================= +2025-06-05 16:07:01,158 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250605_160701.json +2025-06-05 16:07:01,200 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250605_160701_20250605_160701.json +2025-06-05 16:07:01,200 - __main__ - INFO - All tests completed successfully +2025-06-05 16:08:43,289 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-05 16:08:43,289 - __main__ - INFO - Verifying AWS credentials +2025-06-05 16:08:43,290 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 16:08:43,291 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 16:08:43,291 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 16:08:43,292 - __main__ - INFO - Running orchestrator stress test suite +2025-06-05 16:08:43,292 - __main__ - INFO - Starting orchestrator for 300 seconds +2025-06-05 16:08:43,292 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-05 16:08:44,245 - __main__ - INFO - Test duration overridden to 300 seconds +2025-06-05 16:08:44,246 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) +2025-06-05 16:15:09,979 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250605_161509.json +2025-06-05 16:15:10,022 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250605_161509_20250605_161509.json +2025-06-05 16:15:10,022 - __main__ - INFO - All tests completed successfully +2025-06-05 16:21:33,305 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-05 16:21:33,305 - __main__ - INFO - Verifying AWS credentials +2025-06-05 16:21:33,306 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 16:21:33,307 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 16:21:33,307 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 16:21:33,308 - __main__ - INFO - Running orchestrator stress test suite +2025-06-05 16:21:33,308 - __main__ - INFO - Starting orchestrator for 300 seconds +2025-06-05 16:21:33,308 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-05 16:21:34,256 - __main__ - ERROR - Error running tests: (401) +Reason: Unauthorized +HTTP response headers: HTTPHeaderDict({'Audit-Id': 'ae5fa875-33f1-44a7-a754-80136c368fa1', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 16:21:34 GMT', 'Content-Length': '129'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} + +Traceback (most recent call last): + File "/local/home/ibhard/efs-csi-driver-fork/aws-efs-csi-driver/test/e2e/stress-scale-tests/run_tests.py", line 209, in main + orchestrator = EFSCSIOrchestrator(config_file=orchestrator_config) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/local/home/ibhard/efs-csi-driver-fork/aws-efs-csi-driver/test/e2e/stress-scale-tests/tests/orchestrator.py", line 92, in __init__ + self._ensure_namespace_exists() + File "/local/home/ibhard/efs-csi-driver-fork/aws-efs-csi-driver/test/e2e/stress-scale-tests/tests/orchestrator.py", line 98, in _ensure_namespace_exists + self.core_v1.read_namespace(name=self.namespace) + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api/core_v1_api.py", line 22992, in read_namespace + return self.read_namespace_with_http_info(name, **kwargs) # noqa: E501 + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api/core_v1_api.py", line 23071, in read_namespace_with_http_info + return self.api_client.call_api( + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api_client.py", line 348, in call_api + return self.__call_api(resource_path, method, + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api_client.py", line 180, in __call_api + response_data = self.request( + ^^^^^^^^^^^^^ + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api_client.py", line 373, in request + return self.rest_client.GET(url, + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/rest.py", line 244, in GET + return self.request("GET", url, + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/rest.py", line 238, in request + raise ApiException(http_resp=r) +kubernetes.client.exceptions.ApiException: (401) +Reason: Unauthorized +HTTP response headers: HTTPHeaderDict({'Audit-Id': 'ae5fa875-33f1-44a7-a754-80136c368fa1', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 16:21:34 GMT', 'Content-Length': '129'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} + + +2025-06-05 16:21:55,167 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-05 16:21:55,168 - __main__ - INFO - Verifying AWS credentials +2025-06-05 16:21:55,169 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 16:21:55,169 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 16:21:55,169 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 16:21:55,170 - __main__ - INFO - Running orchestrator stress test suite +2025-06-05 16:21:55,170 - __main__ - INFO - Starting orchestrator for 300 seconds +2025-06-05 16:21:55,170 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-05 16:21:56,123 - __main__ - INFO - Test duration overridden to 300 seconds +2025-06-05 16:21:56,123 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) +2025-06-05 16:23:20,187 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-05 16:23:20,187 - __main__ - INFO - Verifying AWS credentials +2025-06-05 16:23:20,188 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 16:23:20,189 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 16:23:20,189 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 16:23:20,190 - __main__ - INFO - Running orchestrator stress test suite +2025-06-05 16:23:20,190 - __main__ - INFO - Starting orchestrator for 300 seconds +2025-06-05 16:23:20,190 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-05 16:23:21,135 - __main__ - INFO - Test duration overridden to 300 seconds +2025-06-05 16:23:21,135 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) +2025-06-05 16:23:53,806 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-05 16:23:53,806 - __main__ - INFO - Verifying AWS credentials +2025-06-05 16:23:53,807 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 16:23:53,808 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 16:23:53,808 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 16:23:53,809 - __main__ - INFO - Running orchestrator stress test suite +2025-06-05 16:23:53,809 - __main__ - INFO - Starting orchestrator for 300 seconds +2025-06-05 16:23:53,809 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-05 16:23:54,744 - __main__ - INFO - Test duration overridden to 300 seconds +2025-06-05 16:23:54,744 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) +2025-06-05 16:30:10,897 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250605_163010.json +2025-06-05 16:30:10,941 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250605_163010_20250605_163010.json +2025-06-05 16:30:10,941 - __main__ - INFO - All tests completed successfully +2025-06-05 16:31:13,588 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-05 16:31:13,588 - __main__ - INFO - Verifying AWS credentials +2025-06-05 16:31:13,589 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 16:31:13,590 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 16:31:13,590 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 16:31:13,590 - __main__ - INFO - Running orchestrator stress test suite +2025-06-05 16:31:13,590 - __main__ - INFO - Starting orchestrator for 300 seconds +2025-06-05 16:31:13,590 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-05 16:31:14,546 - __main__ - INFO - Test duration overridden to 300 seconds +2025-06-05 16:31:14,546 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) +2025-06-05 16:33:39,799 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-05 16:33:39,799 - __main__ - INFO - Verifying AWS credentials +2025-06-05 16:33:39,800 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 16:33:39,800 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 16:33:39,801 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 16:33:39,801 - __main__ - INFO - Running orchestrator stress test suite +2025-06-05 16:33:39,801 - __main__ - INFO - Starting orchestrator for 300 seconds +2025-06-05 16:33:39,801 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-05 16:33:40,752 - __main__ - INFO - Test duration overridden to 300 seconds +2025-06-05 16:33:40,752 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) +2025-06-05 16:35:01,084 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-05 16:35:01,084 - __main__ - INFO - Verifying AWS credentials +2025-06-05 16:35:01,085 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 16:35:01,085 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 16:35:01,086 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 16:35:01,086 - __main__ - INFO - Running orchestrator stress test suite +2025-06-05 16:35:01,086 - __main__ - INFO - Starting orchestrator for 300 seconds +2025-06-05 16:35:01,086 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-05 16:35:02,054 - __main__ - INFO - Test duration overridden to 300 seconds +2025-06-05 16:35:02,054 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) +2025-06-05 16:40:17,452 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250605_164017.json +2025-06-05 16:40:17,511 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250605_164017_20250605_164017.json +2025-06-05 16:40:17,511 - __main__ - INFO - All tests completed successfully +2025-06-05 16:47:08,706 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-05 16:47:08,706 - __main__ - INFO - Verifying AWS credentials +2025-06-05 16:47:08,707 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 16:47:08,708 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 16:47:08,708 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 16:47:08,708 - __main__ - INFO - Running orchestrator stress test suite +2025-06-05 16:47:08,709 - __main__ - INFO - Starting orchestrator for 300 seconds +2025-06-05 16:47:08,709 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-05 16:47:09,670 - __main__ - INFO - Test duration overridden to 300 seconds +2025-06-05 16:47:09,670 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) +2025-06-05 16:49:33,984 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-05 16:49:33,984 - __main__ - INFO - Verifying AWS credentials +2025-06-05 16:49:33,985 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 16:49:33,985 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 16:49:33,986 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 16:49:33,986 - __main__ - INFO - Running orchestrator stress test suite +2025-06-05 16:49:33,986 - __main__ - INFO - Starting orchestrator for 300 seconds +2025-06-05 16:49:33,986 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-05 16:49:34,939 - __main__ - INFO - Test duration overridden to 300 seconds +2025-06-05 16:49:34,939 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) +2025-06-05 16:55:52,177 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250605_165552.json +2025-06-05 16:55:52,220 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250605_165552_20250605_165552.json +2025-06-05 16:55:52,220 - __main__ - INFO - All tests completed successfully +2025-06-05 17:03:06,202 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-05 17:03:06,202 - __main__ - INFO - Verifying AWS credentials +2025-06-05 17:03:06,203 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 17:03:06,204 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 17:03:06,204 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 17:03:06,204 - __main__ - INFO - Running orchestrator stress test suite +2025-06-05 17:03:06,204 - __main__ - INFO - Starting orchestrator for 300 seconds +2025-06-05 17:03:06,205 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-05 17:03:07,165 - __main__ - INFO - Test duration overridden to 300 seconds +2025-06-05 17:03:07,165 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) +2025-06-05 17:03:42,089 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-05 17:03:42,089 - __main__ - INFO - Verifying AWS credentials +2025-06-05 17:03:42,090 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 17:03:42,090 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 17:03:42,091 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 17:03:42,091 - __main__ - INFO - Running orchestrator stress test suite +2025-06-05 17:03:42,091 - __main__ - INFO - Starting orchestrator for 300 seconds +2025-06-05 17:03:42,091 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-05 17:03:43,043 - __main__ - INFO - Test duration overridden to 300 seconds +2025-06-05 17:03:43,043 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) +2025-06-05 17:04:04,822 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-05 17:04:04,822 - __main__ - INFO - Verifying AWS credentials +2025-06-05 17:04:04,823 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 17:04:04,824 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 17:04:04,824 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 17:04:04,825 - __main__ - INFO - Running orchestrator stress test suite +2025-06-05 17:04:04,825 - __main__ - INFO - Starting orchestrator for 300 seconds +2025-06-05 17:04:04,825 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-05 17:04:05,759 - __main__ - INFO - Test duration overridden to 300 seconds +2025-06-05 17:04:05,759 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) +2025-06-05 17:05:02,167 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-05 17:05:02,167 - __main__ - INFO - Verifying AWS credentials +2025-06-05 17:05:02,168 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 17:05:02,168 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 17:05:02,168 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 17:05:02,169 - __main__ - INFO - Running orchestrator stress test suite +2025-06-05 17:05:02,169 - __main__ - INFO - Starting orchestrator for 300 seconds +2025-06-05 17:05:02,169 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-05 17:05:03,121 - __main__ - INFO - Test duration overridden to 300 seconds +2025-06-05 17:05:03,121 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) +2025-06-05 17:05:25,486 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-05 17:05:25,486 - __main__ - INFO - Verifying AWS credentials +2025-06-05 17:05:25,487 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 17:05:25,488 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 17:05:25,488 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 17:05:25,489 - __main__ - INFO - Running orchestrator stress test suite +2025-06-05 17:05:25,489 - __main__ - INFO - Starting orchestrator for 300 seconds +2025-06-05 17:05:25,489 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-05 17:05:26,435 - __main__ - INFO - Test duration overridden to 300 seconds +2025-06-05 17:05:26,435 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) +2025-06-05 17:12:16,582 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250605_171216.json +2025-06-05 17:12:16,625 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250605_171216_20250605_171216.json +2025-06-05 17:12:16,625 - __main__ - INFO - All tests completed successfully +2025-06-05 18:22:15,497 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-05 18:22:15,497 - __main__ - INFO - Verifying AWS credentials +2025-06-05 18:22:15,498 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 18:22:15,498 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 18:22:15,499 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 18:22:15,499 - __main__ - INFO - Running orchestrator stress test suite +2025-06-05 18:22:15,499 - __main__ - INFO - Starting orchestrator for 300 seconds +2025-06-05 18:22:15,499 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-05 18:22:16,529 - __main__ - ERROR - Error running tests: (401) +Reason: Unauthorized +HTTP response headers: HTTPHeaderDict({'Audit-Id': '1a400704-0ba2-4e3d-a421-03adbfa89ebe', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 18:22:16 GMT', 'Content-Length': '129'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} + +Traceback (most recent call last): + File "/local/home/ibhard/efs-csi-driver-fork/aws-efs-csi-driver/test/e2e/stress-scale-tests/run_tests.py", line 209, in main + orchestrator = EFSCSIOrchestrator(config_file=orchestrator_config) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/local/home/ibhard/efs-csi-driver-fork/aws-efs-csi-driver/test/e2e/stress-scale-tests/tests/orchestrator.py", line 92, in __init__ + self._ensure_namespace_exists() + File "/local/home/ibhard/efs-csi-driver-fork/aws-efs-csi-driver/test/e2e/stress-scale-tests/tests/orchestrator.py", line 98, in _ensure_namespace_exists + self.core_v1.read_namespace(name=self.namespace) + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api/core_v1_api.py", line 22992, in read_namespace + return self.read_namespace_with_http_info(name, **kwargs) # noqa: E501 + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api/core_v1_api.py", line 23071, in read_namespace_with_http_info + return self.api_client.call_api( + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api_client.py", line 348, in call_api + return self.__call_api(resource_path, method, + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api_client.py", line 180, in __call_api + response_data = self.request( + ^^^^^^^^^^^^^ + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api_client.py", line 373, in request + return self.rest_client.GET(url, + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/rest.py", line 244, in GET + return self.request("GET", url, + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/rest.py", line 238, in request + raise ApiException(http_resp=r) +kubernetes.client.exceptions.ApiException: (401) +Reason: Unauthorized +HTTP response headers: HTTPHeaderDict({'Audit-Id': '1a400704-0ba2-4e3d-a421-03adbfa89ebe', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 18:22:16 GMT', 'Content-Length': '129'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} + + +2025-06-05 18:22:53,399 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-05 18:22:53,399 - __main__ - INFO - Verifying AWS credentials +2025-06-05 18:22:53,400 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 18:22:53,401 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 18:22:53,401 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 18:22:53,402 - __main__ - INFO - Running orchestrator stress test suite +2025-06-05 18:22:53,402 - __main__ - INFO - Starting orchestrator for 300 seconds +2025-06-05 18:22:53,402 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-05 18:22:54,350 - __main__ - INFO - Test duration overridden to 300 seconds +2025-06-05 18:22:54,350 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) +2025-06-05 18:29:32,601 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250605_182932.json +2025-06-05 18:29:32,643 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250605_182932_20250605_182932.json +2025-06-05 18:29:32,643 - __main__ - INFO - All tests completed successfully +2025-06-05 18:38:35,023 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-05 18:38:35,023 - __main__ - INFO - Verifying AWS credentials +2025-06-05 18:38:35,024 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 18:38:35,025 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 18:38:35,025 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 18:38:35,025 - __main__ - INFO - Running orchestrator stress test suite +2025-06-05 18:38:35,025 - __main__ - INFO - Starting orchestrator for 7200 seconds +2025-06-05 18:38:35,026 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-05 18:38:35,987 - __main__ - INFO - Test duration overridden to 7200 seconds +2025-06-05 18:38:35,987 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) +2025-06-05 19:12:14,835 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-05 19:12:14,835 - __main__ - INFO - Verifying AWS credentials +2025-06-05 19:12:14,836 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 19:12:14,836 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 19:12:14,837 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 19:12:14,837 - __main__ - INFO - Running orchestrator stress test suite +2025-06-05 19:12:14,837 - __main__ - INFO - Starting orchestrator for 300 seconds +2025-06-05 19:12:14,837 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-05 19:12:15,782 - __main__ - INFO - Test duration overridden to 300 seconds +2025-06-05 19:12:15,782 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) +2025-06-05 19:28:23,860 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-05 19:28:23,861 - __main__ - INFO - Verifying AWS credentials +2025-06-05 19:28:23,862 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 19:28:23,862 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 19:28:23,862 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 19:28:23,863 - __main__ - INFO - Running orchestrator stress test suite +2025-06-05 19:28:23,863 - __main__ - INFO - Starting orchestrator for 300 seconds +2025-06-05 19:28:23,863 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-05 19:28:24,815 - __main__ - ERROR - Error running tests: (401) +Reason: Unauthorized +HTTP response headers: HTTPHeaderDict({'Audit-Id': '0dff4cee-4147-4a36-b942-2640bcda6774', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 19:28:24 GMT', 'Content-Length': '129'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} + +Traceback (most recent call last): + File "/local/home/ibhard/efs-csi-driver-fork/aws-efs-csi-driver/test/e2e/stress-scale-tests/run_tests.py", line 209, in main + orchestrator = EFSCSIOrchestrator(config_file=orchestrator_config) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/local/home/ibhard/efs-csi-driver-fork/aws-efs-csi-driver/test/e2e/stress-scale-tests/tests/orchestrator.py", line 92, in __init__ + self._ensure_namespace_exists() + File "/local/home/ibhard/efs-csi-driver-fork/aws-efs-csi-driver/test/e2e/stress-scale-tests/tests/orchestrator.py", line 98, in _ensure_namespace_exists + self.core_v1.read_namespace(name=self.namespace) + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api/core_v1_api.py", line 22992, in read_namespace + return self.read_namespace_with_http_info(name, **kwargs) # noqa: E501 + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api/core_v1_api.py", line 23071, in read_namespace_with_http_info + return self.api_client.call_api( + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api_client.py", line 348, in call_api + return self.__call_api(resource_path, method, + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api_client.py", line 180, in __call_api + response_data = self.request( + ^^^^^^^^^^^^^ + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api_client.py", line 373, in request + return self.rest_client.GET(url, + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/rest.py", line 244, in GET + return self.request("GET", url, + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/rest.py", line 238, in request + raise ApiException(http_resp=r) +kubernetes.client.exceptions.ApiException: (401) +Reason: Unauthorized +HTTP response headers: HTTPHeaderDict({'Audit-Id': '0dff4cee-4147-4a36-b942-2640bcda6774', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 19:28:24 GMT', 'Content-Length': '129'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} + + +2025-06-05 19:28:50,833 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-05 19:28:50,833 - __main__ - INFO - Verifying AWS credentials +2025-06-05 19:28:50,834 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 19:28:50,835 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 19:28:50,835 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 19:28:50,836 - __main__ - INFO - Running orchestrator stress test suite +2025-06-05 19:28:50,836 - __main__ - INFO - Starting orchestrator for 300 seconds +2025-06-05 19:28:50,836 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-05 19:28:51,800 - __main__ - INFO - Test duration overridden to 300 seconds +2025-06-05 19:28:51,800 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) +2025-06-05 19:34:07,404 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250605_193407.json +2025-06-05 19:34:07,447 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250605_193407_20250605_193407.json +2025-06-05 19:34:07,447 - __main__ - INFO - All tests completed successfully +2025-06-05 19:52:24,900 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-05 19:52:24,900 - __main__ - INFO - Verifying AWS credentials +2025-06-05 19:52:24,901 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 19:52:24,902 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 19:52:24,902 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 19:52:24,902 - __main__ - INFO - Running orchestrator stress test suite +2025-06-05 19:52:24,903 - __main__ - INFO - Starting orchestrator for 300 seconds +2025-06-05 19:52:24,903 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-05 19:52:25,880 - __main__ - INFO - Test duration overridden to 300 seconds +2025-06-05 19:52:25,880 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) +2025-06-05 20:01:24,756 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250605_200124.json +2025-06-05 20:01:24,798 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250605_200124_20250605_200124.json +2025-06-05 20:01:24,798 - __main__ - INFO - All tests completed successfully +2025-06-05 20:06:45,182 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-05 20:06:45,182 - __main__ - INFO - Verifying AWS credentials +2025-06-05 20:06:45,183 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 20:06:45,184 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 20:06:45,184 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-05 20:06:45,184 - __main__ - INFO - Running orchestrator stress test suite +2025-06-05 20:06:45,184 - __main__ - INFO - Starting orchestrator for 300 seconds +2025-06-05 20:06:45,185 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-05 20:06:46,152 - __main__ - INFO - Test duration overridden to 300 seconds +2025-06-05 20:06:46,152 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) +2025-06-05 20:17:16,472 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250605_201716.json +2025-06-05 20:17:16,514 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250605_201716_20250605_201716.json +2025-06-05 20:17:16,515 - __main__ - INFO - All tests completed successfully diff --git a/test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749045783.log b/test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749045783.log new file mode 100644 index 000000000..b0f022fcd --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749045783.log @@ -0,0 +1,8 @@ +STDOUT: +2025-06-04 14:03:04 [▶] Setting credentials expiry window to 30 minutes +2025-06-04 14:03:05 [▶] role ARN for the current session is "arn:aws:sts::745939127895:assumed-role/Admin/ibhard-Isengard" + + +STDERR: +Error: resolving cluster version: invalid version, 1.25 is no longer supported, supported values: 1.26, 1.27, 1.28, 1.29, 1.30, 1.31, 1.32, 1.33 +see also: https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html diff --git a/test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749048222.log b/test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749048222.log new file mode 100644 index 000000000..952f62886 --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749048222.log @@ -0,0 +1,8 @@ +STDOUT: +2025-06-04 14:43:43 [▶] Setting credentials expiry window to 30 minutes +2025-06-04 14:43:43 [▶] role ARN for the current session is "arn:aws:sts::745939127895:assumed-role/Admin/ibhard-Isengard" + + +STDERR: +Error: resolving cluster version: invalid version, 1.25 is no longer supported, supported values: 1.26, 1.27, 1.28, 1.29, 1.30, 1.31, 1.32, 1.33 +see also: https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html diff --git a/test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749048235.log b/test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749048235.log new file mode 100644 index 000000000..1cf3eac2b --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749048235.log @@ -0,0 +1,17 @@ +STDOUT: +2025-06-04 14:43:56 [▶] Setting credentials expiry window to 30 minutes +2025-06-04 14:43:56 [▶] role ARN for the current session is "arn:aws:sts::745939127895:assumed-role/Admin/ibhard-Isengard" +2025-06-04 14:43:56 [ℹ] eksctl version 0.208.0 +2025-06-04 14:43:56 [ℹ] using region us-west-2 +2025-06-04 14:43:56 [!] Amazon EKS will no longer publish EKS-optimized Amazon Linux 2 (AL2) AMIs after November 26th, 2025. Additionally, Kubernetes version 1.32 is the last version for which Amazon EKS will release AL2 AMIs. From version 1.33 onwards, Amazon EKS will continue to release AL2023 and Bottlerocket based AMIs. The default AMI family when creating clusters and nodegroups in Eksctl will be changed to AL2023 in the future. +2025-06-04 14:43:56 [▶] determining availability zones +2025-06-04 14:43:57 [ℹ] setting availability zones to [us-west-2d us-west-2a us-west-2b] +2025-06-04 14:43:57 [▶] VPC CIDR (192.168.0.0/16) was divided into 8 subnets [192.168.0.0/19 192.168.32.0/19 192.168.64.0/19 192.168.96.0/19 192.168.128.0/19 192.168.160.0/19 192.168.192.0/19 192.168.224.0/19] +2025-06-04 14:43:57 [ℹ] subnets for us-west-2d - public:192.168.0.0/19 private:192.168.96.0/19 +2025-06-04 14:43:57 [ℹ] subnets for us-west-2a - public:192.168.32.0/19 private:192.168.128.0/19 +2025-06-04 14:43:57 [ℹ] subnets for us-west-2b - public:192.168.64.0/19 private:192.168.160.0/19 +2025-06-04 14:43:57 [▶] resolving AMI using SSM Parameter resolver for region us-west-2, instanceType t3.large and imageFamily AmazonLinux2 + + +STDERR: +Error: unable to determine AMI to use: error getting AMI from SSM Parameter Store: operation error SSM: GetParameter, https response error StatusCode: 400, RequestID: a443a5ad-af0e-4afb-b46f-a021532755a5, ParameterNotFound: . please verify that AMI Family is supported diff --git a/test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749048734.log b/test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749048734.log new file mode 100644 index 000000000..678235019 --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749048734.log @@ -0,0 +1,17 @@ +STDOUT: +2025-06-04 14:52:15 [▶] Setting credentials expiry window to 30 minutes +2025-06-04 14:52:15 [▶] role ARN for the current session is "arn:aws:sts::745939127895:assumed-role/Admin/ibhard-Isengard" +2025-06-04 14:52:16 [ℹ] eksctl version 0.208.0 +2025-06-04 14:52:16 [ℹ] using region us-west-2 +2025-06-04 14:52:16 [!] Amazon EKS will no longer publish EKS-optimized Amazon Linux 2 (AL2) AMIs after November 26th, 2025. Additionally, Kubernetes version 1.32 is the last version for which Amazon EKS will release AL2 AMIs. From version 1.33 onwards, Amazon EKS will continue to release AL2023 and Bottlerocket based AMIs. The default AMI family when creating clusters and nodegroups in Eksctl will be changed to AL2023 in the future. +2025-06-04 14:52:16 [▶] determining availability zones +2025-06-04 14:52:16 [ℹ] setting availability zones to [us-west-2a us-west-2c us-west-2d] +2025-06-04 14:52:16 [▶] VPC CIDR (192.168.0.0/16) was divided into 8 subnets [192.168.0.0/19 192.168.32.0/19 192.168.64.0/19 192.168.96.0/19 192.168.128.0/19 192.168.160.0/19 192.168.192.0/19 192.168.224.0/19] +2025-06-04 14:52:16 [ℹ] subnets for us-west-2a - public:192.168.0.0/19 private:192.168.96.0/19 +2025-06-04 14:52:16 [ℹ] subnets for us-west-2c - public:192.168.32.0/19 private:192.168.128.0/19 +2025-06-04 14:52:16 [ℹ] subnets for us-west-2d - public:192.168.64.0/19 private:192.168.160.0/19 +2025-06-04 14:52:16 [▶] resolving AMI using SSM Parameter resolver for region us-west-2, instanceType t3.large and imageFamily AmazonLinux2 + + +STDERR: +Error: unable to determine AMI to use: error getting AMI from SSM Parameter Store: operation error SSM: GetParameter, https response error StatusCode: 400, RequestID: 4650ea1c-e137-440e-835d-9214a82b7beb, ParameterNotFound: . please verify that AMI Family is supported diff --git a/test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749051325.log b/test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749051325.log new file mode 100644 index 000000000..5d9a498eb --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749051325.log @@ -0,0 +1,17 @@ +STDOUT: +2025-06-04 15:35:26 [▶] Setting credentials expiry window to 30 minutes +2025-06-04 15:35:26 [▶] role ARN for the current session is "arn:aws:sts::745939127895:assumed-role/Admin/ibhard-Isengard" +2025-06-04 15:35:26 [ℹ] eksctl version 0.208.0 +2025-06-04 15:35:26 [ℹ] using region us-west-2 +2025-06-04 15:35:26 [!] Amazon EKS will no longer publish EKS-optimized Amazon Linux 2 (AL2) AMIs after November 26th, 2025. Additionally, Kubernetes version 1.32 is the last version for which Amazon EKS will release AL2 AMIs. From version 1.33 onwards, Amazon EKS will continue to release AL2023 and Bottlerocket based AMIs. The default AMI family when creating clusters and nodegroups in Eksctl will be changed to AL2023 in the future. +2025-06-04 15:35:26 [▶] determining availability zones +2025-06-04 15:35:27 [ℹ] setting availability zones to [us-west-2b us-west-2a us-west-2c] +2025-06-04 15:35:27 [▶] VPC CIDR (192.168.0.0/16) was divided into 8 subnets [192.168.0.0/19 192.168.32.0/19 192.168.64.0/19 192.168.96.0/19 192.168.128.0/19 192.168.160.0/19 192.168.192.0/19 192.168.224.0/19] +2025-06-04 15:35:27 [ℹ] subnets for us-west-2b - public:192.168.0.0/19 private:192.168.96.0/19 +2025-06-04 15:35:27 [ℹ] subnets for us-west-2a - public:192.168.32.0/19 private:192.168.128.0/19 +2025-06-04 15:35:27 [ℹ] subnets for us-west-2c - public:192.168.64.0/19 private:192.168.160.0/19 +2025-06-04 15:35:27 [▶] resolving AMI using SSM Parameter resolver for region us-west-2, instanceType t3.large and imageFamily AmazonLinux2 + + +STDERR: +Error: unable to determine AMI to use: error getting AMI from SSM Parameter Store: operation error SSM: GetParameter, https response error StatusCode: 400, RequestID: 59f5b73c-2035-46bb-9074-986f681c13f8, ParameterNotFound: . please verify that AMI Family is supported diff --git a/test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749051374.log b/test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749051374.log new file mode 100644 index 000000000..24ee6f202 --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749051374.log @@ -0,0 +1,17 @@ +STDOUT: +2025-06-04 15:36:15 [▶] Setting credentials expiry window to 30 minutes +2025-06-04 15:36:15 [▶] role ARN for the current session is "arn:aws:sts::745939127895:assumed-role/Admin/ibhard-Isengard" +2025-06-04 15:36:15 [ℹ] eksctl version 0.208.0 +2025-06-04 15:36:15 [ℹ] using region us-west-2 +2025-06-04 15:36:15 [!] Amazon EKS will no longer publish EKS-optimized Amazon Linux 2 (AL2) AMIs after November 26th, 2025. Additionally, Kubernetes version 1.32 is the last version for which Amazon EKS will release AL2 AMIs. From version 1.33 onwards, Amazon EKS will continue to release AL2023 and Bottlerocket based AMIs. The default AMI family when creating clusters and nodegroups in Eksctl will be changed to AL2023 in the future. +2025-06-04 15:36:15 [▶] determining availability zones +2025-06-04 15:36:16 [ℹ] setting availability zones to [us-west-2b us-west-2d us-west-2a] +2025-06-04 15:36:16 [▶] VPC CIDR (192.168.0.0/16) was divided into 8 subnets [192.168.0.0/19 192.168.32.0/19 192.168.64.0/19 192.168.96.0/19 192.168.128.0/19 192.168.160.0/19 192.168.192.0/19 192.168.224.0/19] +2025-06-04 15:36:16 [ℹ] subnets for us-west-2b - public:192.168.0.0/19 private:192.168.96.0/19 +2025-06-04 15:36:16 [ℹ] subnets for us-west-2d - public:192.168.32.0/19 private:192.168.128.0/19 +2025-06-04 15:36:16 [ℹ] subnets for us-west-2a - public:192.168.64.0/19 private:192.168.160.0/19 +2025-06-04 15:36:16 [▶] resolving AMI using SSM Parameter resolver for region us-west-2, instanceType t3.large and imageFamily AmazonLinux2 + + +STDERR: +Error: unable to determine AMI to use: error getting AMI from SSM Parameter Store: operation error SSM: GetParameter, https response error StatusCode: 400, RequestID: 1189c451-cdbd-4c3b-9495-a627bfe2113d, ParameterNotFound: . please verify that AMI Family is supported diff --git a/test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749051392.log b/test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749051392.log new file mode 100644 index 000000000..4efc1291f --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749051392.log @@ -0,0 +1,254 @@ +STDOUT: +2025-06-04 15:36:32 [▶] Setting credentials expiry window to 30 minutes +2025-06-04 15:36:33 [▶] role ARN for the current session is "arn:aws:sts::745939127895:assumed-role/Admin/ibhard-Isengard" +2025-06-04 15:36:33 [ℹ] eksctl version 0.208.0 +2025-06-04 15:36:33 [ℹ] using region us-west-2 +2025-06-04 15:36:33 [!] Amazon EKS will no longer publish EKS-optimized Amazon Linux 2 (AL2) AMIs after November 26th, 2025. Additionally, Kubernetes version 1.32 is the last version for which Amazon EKS will release AL2 AMIs. From version 1.33 onwards, Amazon EKS will continue to release AL2023 and Bottlerocket based AMIs. The default AMI family when creating clusters and nodegroups in Eksctl will be changed to AL2023 in the future. +2025-06-04 15:36:33 [▶] determining availability zones +2025-06-04 15:36:33 [ℹ] setting availability zones to [us-west-2d us-west-2a us-west-2c] +2025-06-04 15:36:33 [▶] VPC CIDR (192.168.0.0/16) was divided into 8 subnets [192.168.0.0/19 192.168.32.0/19 192.168.64.0/19 192.168.96.0/19 192.168.128.0/19 192.168.160.0/19 192.168.192.0/19 192.168.224.0/19] +2025-06-04 15:36:33 [ℹ] subnets for us-west-2d - public:192.168.0.0/19 private:192.168.96.0/19 +2025-06-04 15:36:33 [ℹ] subnets for us-west-2a - public:192.168.32.0/19 private:192.168.128.0/19 +2025-06-04 15:36:33 [ℹ] subnets for us-west-2c - public:192.168.64.0/19 private:192.168.160.0/19 +2025-06-04 15:36:33 [▶] resolving AMI using SSM Parameter resolver for region us-west-2, instanceType t3.large and imageFamily AmazonLinux2 +2025-06-04 15:36:34 [ℹ] nodegroup "ng-1" will use "ami-0ff76e62eff72da2a" [AmazonLinux2/1.28] +2025-06-04 15:36:34 [ℹ] using Kubernetes version 1.28 +2025-06-04 15:36:34 [ℹ] creating EKS cluster "efs-csi-test-1749051392" in "us-west-2" region with un-managed nodes +2025-06-04 15:36:34 [▶] cfg.json = \ +{ + "kind": "ClusterConfig", + "apiVersion": "eksctl.io/v1alpha5", + "metadata": { + "name": "efs-csi-test-1749051392", + "region": "us-west-2", + "version": "1.28" + }, + "iam": { + "withOIDC": true, + "serviceAccounts": [ + { + "metadata": { + "name": "efs-csi-controller-sa", + "namespace": "kube-system" + }, + "attachPolicyARNs": [ + "arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy" + ], + "wellKnownPolicies": { + "imageBuilder": false, + "autoScaler": false, + "awsLoadBalancerController": false, + "externalDNS": false, + "certManager": false, + "ebsCSIController": false, + "efsCSIController": false + } + } + ], + "vpcResourceControllerPolicy": true + }, + "accessConfig": { + "authenticationMode": "API_AND_CONFIG_MAP" + }, + "vpc": { + "cidr": "192.168.0.0/16", + "subnets": { + "private": { + "us-west-2a": { + "az": "us-west-2a", + "cidr": "192.168.128.0/19" + }, + "us-west-2c": { + "az": "us-west-2c", + "cidr": "192.168.160.0/19" + }, + "us-west-2d": { + "az": "us-west-2d", + "cidr": "192.168.96.0/19" + } + }, + "public": { + "us-west-2a": { + "az": "us-west-2a", + "cidr": "192.168.32.0/19" + }, + "us-west-2c": { + "az": "us-west-2c", + "cidr": "192.168.64.0/19" + }, + "us-west-2d": { + "az": "us-west-2d", + "cidr": "192.168.0.0/19" + } + } + }, + "manageSharedNodeSecurityGroupRules": true, + "autoAllocateIPv6": false, + "nat": { + "gateway": "Single" + }, + "clusterEndpoints": { + "privateAccess": false, + "publicAccess": true + } + }, + "addonsConfig": {}, + "privateCluster": { + "enabled": false, + "skipEndpointCreation": false + }, + "nodeGroups": [ + { + "name": "ng-1", + "amiFamily": "AmazonLinux2", + "instanceType": "t3.large", + "desiredCapacity": 3, + "minSize": 2, + "maxSize": 5, + "volumeSize": 80, + "ssh": { + "allow": false + }, + "labels": { + "alpha.eksctl.io/cluster-name": "efs-csi-test-1749051392", + "alpha.eksctl.io/nodegroup-name": "ng-1", + "efs-issue": "true", + "role": "general" + }, + "privateNetworking": false, + "tags": { + "nodegroup-role": "general" + }, + "iam": { + "attachPolicy": { + "Statement": [ + { + "Action": [ + "elasticfilesystem:DescribeAccessPoints", + "elasticfilesystem:DescribeFileSystems", + "elasticfilesystem:DescribeMountTargets", + "ec2:DescribeAvailabilityZones" + ], + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "elasticfilesystem:CreateAccessPoint" + ], + "Condition": { + "StringLike": { + "aws:RequestTag/kubernetes.io/cluster/*": "owned" + } + }, + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "elasticfilesystem:DeleteAccessPoint" + ], + "Condition": { + "StringEquals": { + "aws:ResourceTag/kubernetes.io/cluster/*": "owned" + } + }, + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + }, + "withAddonPolicies": { + "imageBuilder": false, + "autoScaler": false, + "externalDNS": false, + "certManager": false, + "appMesh": null, + "appMeshPreview": null, + "ebs": false, + "fsx": false, + "efs": false, + "awsLoadBalancerController": false, + "albIngress": false, + "xRay": false, + "cloudWatch": false + } + }, + "ami": "ami-0ff76e62eff72da2a", + "securityGroups": { + "withShared": true, + "withLocal": true + }, + "volumeType": "gp3", + "volumeName": "/dev/xvda", + "volumeEncrypted": false, + "volumeIOPS": 3000, + "volumeThroughput": 125, + "disableIMDSv1": true, + "disablePodIMDS": false, + "instanceSelector": {}, + "containerRuntime": "containerd" + } + ], + "availabilityZones": [ + "us-west-2d", + "us-west-2a", + "us-west-2c" + ] +} + +2025-06-04 15:36:34 [ℹ] 1 nodegroup (ng-1) was included (based on the include/exclude rules) +2025-06-04 15:36:34 [ℹ] will create a CloudFormation stack for cluster itself and 1 nodegroup stack(s) +2025-06-04 15:36:34 [ℹ] if you encounter any issues, check CloudFormation console or try 'eksctl utils describe-stacks --region=us-west-2 --cluster=efs-csi-test-1749051392' +2025-06-04 15:36:34 [ℹ] Kubernetes API endpoint access will use default of {publicAccess=true, privateAccess=false} for cluster "efs-csi-test-1749051392" in "us-west-2" +2025-06-04 15:36:34 [ℹ] CloudWatch logging will not be enabled for cluster "efs-csi-test-1749051392" in "us-west-2" +2025-06-04 15:36:34 [ℹ] you can enable it with 'eksctl utils update-cluster-logging --enable-types={SPECIFY-YOUR-LOG-TYPES-HERE (e.g. all)} --region=us-west-2 --cluster=efs-csi-test-1749051392' +2025-06-04 15:36:34 [ℹ] default addons kube-proxy, coredns, metrics-server, vpc-cni were not specified, will install them as EKS addons +2025-06-04 15:36:34 [ℹ] +2 sequential tasks: { create cluster control plane "efs-csi-test-1749051392", + 2 sequential sub-tasks: { + 5 sequential sub-tasks: { + 1 task: { create addons }, + wait for control plane to become ready, + associate IAM OIDC provider, + 2 sequential sub-tasks: { + create IAM role for serviceaccount "kube-system/efs-csi-controller-sa", + create serviceaccount "kube-system/efs-csi-controller-sa", + }, + update VPC CNI to use IRSA if required, + }, + create nodegroup "ng-1", + } +} +2025-06-04 15:36:34 [▶] started task: create cluster control plane "efs-csi-test-1749051392" +2025-06-04 15:36:34 [ℹ] building cluster stack "eksctl-efs-csi-test-1749051392-cluster" +2025-06-04 15:36:34 [▶] CreateStackInput = &cloudformation.CreateStackInput{StackName:(*string)(0xc000f5db00), Capabilities:[]types.Capability{"CAPABILITY_IAM"}, ClientRequestToken:(*string)(nil), DisableRollback:(*bool)(0xc00063aed8), EnableTerminationProtection:(*bool)(nil), NotificationARNs:[]string(nil), OnFailure:"", Parameters:[]types.Parameter(nil), ResourceTypes:[]string(nil), RetainExceptOnCreate:(*bool)(nil), RoleARN:(*string)(nil), RollbackConfiguration:(*types.RollbackConfiguration)(nil), StackPolicyBody:(*string)(nil), StackPolicyURL:(*string)(nil), Tags:[]types.Tag{types.Tag{Key:(*string)(0xc000cdb8d0), Value:(*string)(0xc000cdb8e0), noSmithyDocumentSerde:document.NoSerde{}}, types.Tag{Key:(*string)(0xc000cdb8f0), Value:(*string)(0xc000cdb900), noSmithyDocumentSerde:document.NoSerde{}}, types.Tag{Key:(*string)(0xc000cdb910), Value:(*string)(0xc000cdb920), noSmithyDocumentSerde:document.NoSerde{}}, types.Tag{Key:(*string)(0xc000f718c0), Value:(*string)(0xc000f718d0), noSmithyDocumentSerde:document.NoSerde{}}}, TemplateBody:(*string)(0xc000f718e0), TemplateURL:(*string)(nil), TimeoutInMinutes:(*int32)(nil), noSmithyDocumentSerde:document.NoSerde{}} +2025-06-04 15:36:34 [ℹ] deploying stack "eksctl-efs-csi-test-1749051392-cluster" +2025-06-04 15:37:04 [ℹ] waiting for CloudFormation stack "eksctl-efs-csi-test-1749051392-cluster" +2025-06-04 15:37:05 [✖] unexpected status "ROLLBACK_IN_PROGRESS" while waiting for CloudFormation stack "eksctl-efs-csi-test-1749051392-cluster" +2025-06-04 15:37:05 [✖] unexpected status "ROLLBACK_IN_PROGRESS" while waiting for CloudFormation stack "eksctl-efs-csi-test-1749051392-cluster" +2025-06-04 15:37:05 [ℹ] fetching stack events in attempt to troubleshoot the root cause of the failure +2025-06-04 15:37:05 [▶] AWS::IAM::Role/ServiceRole: DELETE_COMPLETE +2025-06-04 15:37:05 [▶] AWS::EC2::EIP/NATIP: DELETE_COMPLETE +2025-06-04 15:37:05 [!] AWS::IAM::Role/ServiceRole: DELETE_IN_PROGRESS +2025-06-04 15:37:05 [!] AWS::EC2::EIP/NATIP: DELETE_IN_PROGRESS +2025-06-04 15:37:05 [▶] AWS::CloudFormation::Stack/eksctl-efs-csi-test-1749051392-cluster: ROLLBACK_IN_PROGRESS – "The following resource(s) failed to create: [InternetGateway, ServiceRole, NATIP, VPC]. Rollback requested by user." +2025-06-04 15:37:05 [✖] AWS::EC2::EIP/NATIP: CREATE_FAILED – "Resource creation cancelled" +2025-06-04 15:37:05 [✖] AWS::IAM::Role/ServiceRole: CREATE_FAILED – "Resource creation cancelled" +2025-06-04 15:37:05 [▶] AWS::EC2::EIP/NATIP: CREATE_IN_PROGRESS – "Resource creation Initiated" +2025-06-04 15:37:05 [✖] AWS::EC2::VPC/VPC: CREATE_FAILED – "Resource handler returned message: \"The maximum number of VPCs has been reached. (Service: Ec2, Status Code: 400, Request ID: 12f2bdae-04a6-4aea-8573-1101de3244a7) (SDK Attempt Count: 1)\" (RequestToken: 06fa18f0-e50e-789e-41a0-492077692cf3, HandlerErrorCode: GeneralServiceException)" +2025-06-04 15:37:05 [✖] AWS::EC2::InternetGateway/InternetGateway: CREATE_FAILED – "Resource handler returned message: \"The maximum number of internet gateways has been reached. (Service: Ec2, Status Code: 400, Request ID: e67da527-78fd-41bd-be9f-ce04d9ba2a26) (SDK Attempt Count: 1)\" (RequestToken: 6f816c80-7115-5bdf-5120-3a7099dbe90d, HandlerErrorCode: ServiceLimitExceeded)" +2025-06-04 15:37:05 [▶] AWS::EC2::InternetGateway/InternetGateway: CREATE_IN_PROGRESS +2025-06-04 15:37:05 [▶] AWS::IAM::Role/ServiceRole: CREATE_IN_PROGRESS +2025-06-04 15:37:05 [▶] AWS::EC2::EIP/NATIP: CREATE_IN_PROGRESS +2025-06-04 15:37:05 [▶] AWS::EC2::VPC/VPC: CREATE_IN_PROGRESS +2025-06-04 15:37:05 [▶] AWS::CloudFormation::Stack/eksctl-efs-csi-test-1749051392-cluster: CREATE_IN_PROGRESS – "User Initiated" +2025-06-04 15:37:05 [▶] failed task: create cluster control plane "efs-csi-test-1749051392" (will not run other sequential tasks) +2025-06-04 15:37:05 [!] 1 error(s) occurred and cluster hasn't been created properly, you may wish to check CloudFormation console +2025-06-04 15:37:05 [ℹ] to cleanup resources, run 'eksctl delete cluster --region=us-west-2 --name=efs-csi-test-1749051392' +2025-06-04 15:37:05 [✖] ResourceNotReady: failed waiting for successful resource state + + +STDERR: +Error: failed to create cluster "efs-csi-test-1749051392" diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250603_145210.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250603_145210.log new file mode 100644 index 000000000..662b37b48 --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250603_145210.log @@ -0,0 +1,147 @@ +2025-06-03 14:52:10,097 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-03 14:52:10,098 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-03 14:52:10,325 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-03 14:52:10,325 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario +2025-06-03 14:52:10,325 - tests.orchestrator - INFO - Running specific scenario: _scenario_many_to_one +2025-06-03 14:52:10,325 - tests.orchestrator - INFO - Running scenario: Many pods to one PVC +2025-06-03 14:52:10,528 - tests.orchestrator - INFO - Waiting for PVC many2one-020acd02 to be bound +2025-06-03 14:52:12,667 - tests.orchestrator - INFO - PVC many2one-020acd02 is bound +2025-06-03 14:52:12,668 - tests.orchestrator - INFO - Creating 3 pods for the same PVC +2025-06-03 14:52:12,669 - tests.orchestrator - INFO - Creating pod test-pod-ef805426 on PVC many2one-020acd02 +2025-06-03 14:52:12,771 - tests.orchestrator - INFO - Created pod: test-pod-ef805426 using PVC: many2one-020acd02 +2025-06-03 14:52:12,771 - tests.orchestrator - INFO - Waiting for pod test-pod-ef805426 to be ready +2025-06-03 14:52:25,251 - tests.orchestrator - INFO - Pod test-pod-ef805426 is ready +2025-06-03 14:52:25,252 - tests.orchestrator - INFO - Creating pod test-pod-a76bf50a on PVC many2one-020acd02 +2025-06-03 14:52:25,325 - tests.orchestrator - INFO - Created pod: test-pod-a76bf50a using PVC: many2one-020acd02 +2025-06-03 14:52:25,325 - tests.orchestrator - INFO - Waiting for pod test-pod-a76bf50a to be ready +2025-06-03 14:52:37,804 - tests.orchestrator - INFO - Pod test-pod-a76bf50a is ready +2025-06-03 14:52:37,804 - tests.orchestrator - INFO - Creating pod test-pod-6da904f7 on PVC many2one-020acd02 +2025-06-03 14:52:37,879 - tests.orchestrator - INFO - Created pod: test-pod-6da904f7 using PVC: many2one-020acd02 +2025-06-03 14:52:37,879 - tests.orchestrator - INFO - Waiting for pod test-pod-6da904f7 to be ready +2025-06-03 14:52:50,362 - tests.orchestrator - INFO - Pod test-pod-6da904f7 is ready +2025-06-03 14:52:50,434 - tests.orchestrator - ERROR - Error in many-to-one scenario: (400) +Reason: Bad Request +HTTP response headers: HTTPHeaderDict({'Audit-Id': '76d6f572-aa54-4107-b259-0d0f792714d5', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Tue, 03 Jun 2025 14:52:50 GMT', 'Content-Length': '139'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Upgrade request required","reason":"BadRequest","code":400} + + +2025-06-03 14:52:51,434 - tests.orchestrator - INFO - Selected operation: _attach_pod +2025-06-03 14:52:51,434 - tests.orchestrator - INFO - Creating pod test-pod-799aaf63 on PVC many2one-020acd02 +2025-06-03 14:52:51,531 - tests.orchestrator - INFO - Created pod: test-pod-799aaf63 using PVC: many2one-020acd02 +2025-06-03 14:52:51,531 - tests.orchestrator - INFO - Waiting for pod test-pod-799aaf63 to be ready +2025-06-03 14:52:57,803 - tests.orchestrator - INFO - Pod test-pod-799aaf63 is ready +2025-06-03 14:52:58,803 - tests.orchestrator - INFO - Selected operation: _verify_readwrite +2025-06-03 14:52:58,803 - tests.orchestrator - INFO - Testing read/write between pods test-pod-ef805426 and test-pod-a76bf50a sharing PVC many2one-020acd02 +2025-06-03 14:52:58,875 - tests.orchestrator - ERROR - Failed in read/write verification: (400) +Reason: Bad Request +HTTP response headers: HTTPHeaderDict({'Audit-Id': 'e1a62bf8-8f8a-41b5-851e-c5ce76b06ea8', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Tue, 03 Jun 2025 14:52:58 GMT', 'Content-Length': '139'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Upgrade request required","reason":"BadRequest","code":400} + + +2025-06-03 14:52:59,875 - tests.orchestrator - INFO - Selected operation: _delete_pod +2025-06-03 14:52:59,876 - tests.orchestrator - INFO - Deleting pod: test-pod-6da904f7 from PVC: many2one-020acd02 +2025-06-03 14:52:59,951 - tests.orchestrator - INFO - Waiting for pod test-pod-6da904f7 to be deleted +2025-06-03 14:53:32,578 - tests.orchestrator - INFO - Pod test-pod-6da904f7 has been deleted +2025-06-03 14:53:32,578 - tests.orchestrator - INFO - Deleted pod: test-pod-6da904f7 +2025-06-03 14:53:33,578 - tests.orchestrator - INFO - Selected operation: _create_pvc +2025-06-03 14:53:33,579 - tests.orchestrator - INFO - Creating PVC: test-pvc-4937168b +2025-06-03 14:53:33,660 - tests.orchestrator - INFO - Created PVC: test-pvc-4937168b +2025-06-03 14:53:33,661 - tests.orchestrator - INFO - Waiting for PVC test-pvc-4937168b to be bound +2025-06-03 14:53:35,792 - tests.orchestrator - INFO - PVC test-pvc-4937168b is bound +2025-06-03 14:53:36,793 - tests.orchestrator - INFO - Selected operation: _attach_pod +2025-06-03 14:53:36,793 - tests.orchestrator - INFO - Creating pod test-pod-dd8a0785 on PVC many2one-020acd02 +2025-06-03 14:53:36,868 - tests.orchestrator - INFO - Created pod: test-pod-dd8a0785 using PVC: many2one-020acd02 +2025-06-03 14:53:36,868 - tests.orchestrator - INFO - Waiting for pod test-pod-dd8a0785 to be ready +2025-06-03 14:53:49,354 - tests.orchestrator - INFO - Pod test-pod-dd8a0785 is ready +2025-06-03 14:53:50,355 - tests.orchestrator - INFO - Selected operation: _delete_pod +2025-06-03 14:53:50,355 - tests.orchestrator - INFO - Deleting pod: test-pod-799aaf63 from PVC: many2one-020acd02 +2025-06-03 14:53:50,431 - tests.orchestrator - INFO - Waiting for pod test-pod-799aaf63 to be deleted +2025-06-03 14:54:23,047 - tests.orchestrator - INFO - Pod test-pod-799aaf63 has been deleted +2025-06-03 14:54:23,047 - tests.orchestrator - INFO - Deleted pod: test-pod-799aaf63 +2025-06-03 14:54:24,047 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario +2025-06-03 14:54:24,048 - tests.orchestrator - INFO - Running specific scenario: _scenario_many_to_one +2025-06-03 14:54:24,048 - tests.orchestrator - INFO - Running scenario: Many pods to one PVC +2025-06-03 14:54:24,127 - tests.orchestrator - INFO - Waiting for PVC many2one-8b088f98 to be bound +2025-06-03 14:54:26,258 - tests.orchestrator - INFO - PVC many2one-8b088f98 is bound +2025-06-03 14:54:26,259 - tests.orchestrator - INFO - Creating 4 pods for the same PVC +2025-06-03 14:54:26,259 - tests.orchestrator - INFO - Creating pod test-pod-017db5e3 on PVC many2one-8b088f98 +2025-06-03 14:54:26,330 - tests.orchestrator - INFO - Created pod: test-pod-017db5e3 using PVC: many2one-8b088f98 +2025-06-03 14:54:26,330 - tests.orchestrator - INFO - Waiting for pod test-pod-017db5e3 to be ready +2025-06-03 14:54:32,600 - tests.orchestrator - INFO - Pod test-pod-017db5e3 is ready +2025-06-03 14:54:32,601 - tests.orchestrator - INFO - Creating pod test-pod-25f6ccfe on PVC many2one-8b088f98 +2025-06-03 14:54:32,676 - tests.orchestrator - INFO - Created pod: test-pod-25f6ccfe using PVC: many2one-8b088f98 +2025-06-03 14:54:32,676 - tests.orchestrator - INFO - Waiting for pod test-pod-25f6ccfe to be ready +2025-06-03 14:54:38,950 - tests.orchestrator - INFO - Pod test-pod-25f6ccfe is ready +2025-06-03 14:54:38,950 - tests.orchestrator - INFO - Creating pod test-pod-a3be1bc9 on PVC many2one-8b088f98 +2025-06-03 14:54:39,025 - tests.orchestrator - INFO - Created pod: test-pod-a3be1bc9 using PVC: many2one-8b088f98 +2025-06-03 14:54:39,025 - tests.orchestrator - INFO - Waiting for pod test-pod-a3be1bc9 to be ready +2025-06-03 14:54:51,508 - tests.orchestrator - INFO - Pod test-pod-a3be1bc9 is ready +2025-06-03 14:54:51,508 - tests.orchestrator - INFO - Creating pod test-pod-2da972ac on PVC many2one-8b088f98 +2025-06-03 14:54:51,601 - tests.orchestrator - INFO - Created pod: test-pod-2da972ac using PVC: many2one-8b088f98 +2025-06-03 14:54:51,602 - tests.orchestrator - INFO - Waiting for pod test-pod-2da972ac to be ready +2025-06-03 14:54:57,878 - tests.orchestrator - INFO - Pod test-pod-2da972ac is ready +2025-06-03 14:54:57,950 - tests.orchestrator - ERROR - Error in many-to-one scenario: (400) +Reason: Bad Request +HTTP response headers: HTTPHeaderDict({'Audit-Id': '458593e0-149a-4412-8f07-f85de3d6cf1d', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Tue, 03 Jun 2025 14:54:57 GMT', 'Content-Length': '139'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Upgrade request required","reason":"BadRequest","code":400} + + +2025-06-03 14:54:58,950 - tests.orchestrator - INFO - Selected operation: _delete_pvc +2025-06-03 14:54:58,950 - tests.orchestrator - INFO - Deleting PVC: many2one-8b088f98 +2025-06-03 14:54:58,950 - tests.orchestrator - INFO - Deleting 4 pods using PVC many2one-8b088f98 +2025-06-03 14:54:58,950 - tests.orchestrator - INFO - Deleting pod: test-pod-017db5e3 from PVC: many2one-8b088f98 +2025-06-03 14:54:59,025 - tests.orchestrator - INFO - Waiting for pod test-pod-017db5e3 to be deleted +2025-06-03 14:55:31,641 - tests.orchestrator - INFO - Pod test-pod-017db5e3 has been deleted +2025-06-03 14:55:31,641 - tests.orchestrator - INFO - Deleted pod: test-pod-017db5e3 +2025-06-03 14:55:31,641 - tests.orchestrator - INFO - Deleting pod: test-pod-25f6ccfe from PVC: many2one-8b088f98 +2025-06-03 14:55:31,722 - tests.orchestrator - INFO - Waiting for pod test-pod-25f6ccfe to be deleted +2025-06-03 14:56:04,344 - tests.orchestrator - INFO - Pod test-pod-25f6ccfe has been deleted +2025-06-03 14:56:04,344 - tests.orchestrator - INFO - Deleted pod: test-pod-25f6ccfe +2025-06-03 14:56:04,344 - tests.orchestrator - INFO - Deleting pod: test-pod-a3be1bc9 from PVC: many2one-8b088f98 +2025-06-03 14:56:04,420 - tests.orchestrator - INFO - Waiting for pod test-pod-a3be1bc9 to be deleted +2025-06-03 14:56:37,041 - tests.orchestrator - INFO - Pod test-pod-a3be1bc9 has been deleted +2025-06-03 14:56:37,042 - tests.orchestrator - INFO - Deleted pod: test-pod-a3be1bc9 +2025-06-03 14:56:37,042 - tests.orchestrator - INFO - Deleting pod: test-pod-2da972ac from PVC: many2one-8b088f98 +2025-06-03 14:56:37,119 - tests.orchestrator - INFO - Waiting for pod test-pod-2da972ac to be deleted +2025-06-03 14:57:09,741 - tests.orchestrator - INFO - Pod test-pod-2da972ac has been deleted +2025-06-03 14:57:09,741 - tests.orchestrator - INFO - Deleted pod: test-pod-2da972ac +2025-06-03 14:57:09,815 - tests.orchestrator - INFO - Waiting for PVC many2one-8b088f98 to be deleted +2025-06-03 14:57:09,880 - tests.orchestrator - INFO - PVC many2one-8b088f98 has been deleted +2025-06-03 14:57:09,880 - tests.orchestrator - INFO - Deleted PVC: many2one-8b088f98 +2025-06-03 14:57:10,880 - tests.orchestrator - INFO - Test completed in 300.78 seconds +2025-06-03 14:57:10,880 - tests.orchestrator - INFO - Cleaning up resources... +2025-06-03 14:57:10,880 - tests.orchestrator - INFO - Deleting pod: test-pod-ef805426 from PVC: many2one-020acd02 +2025-06-03 14:57:10,955 - tests.orchestrator - INFO - Waiting for pod test-pod-ef805426 to be deleted +2025-06-03 14:57:43,569 - tests.orchestrator - INFO - Pod test-pod-ef805426 has been deleted +2025-06-03 14:57:43,569 - tests.orchestrator - INFO - Deleted pod: test-pod-ef805426 +2025-06-03 14:57:43,569 - tests.orchestrator - INFO - Deleting pod: test-pod-a76bf50a from PVC: many2one-020acd02 +2025-06-03 14:57:43,647 - tests.orchestrator - INFO - Waiting for pod test-pod-a76bf50a to be deleted +2025-06-03 14:58:16,290 - tests.orchestrator - INFO - Pod test-pod-a76bf50a has been deleted +2025-06-03 14:58:16,290 - tests.orchestrator - INFO - Deleted pod: test-pod-a76bf50a +2025-06-03 14:58:16,290 - tests.orchestrator - INFO - Deleting pod: test-pod-dd8a0785 from PVC: many2one-020acd02 +2025-06-03 14:58:16,365 - tests.orchestrator - INFO - Waiting for pod test-pod-dd8a0785 to be deleted +2025-06-03 14:58:48,979 - tests.orchestrator - INFO - Pod test-pod-dd8a0785 has been deleted +2025-06-03 14:58:48,979 - tests.orchestrator - INFO - Deleted pod: test-pod-dd8a0785 +2025-06-03 14:58:48,979 - tests.orchestrator - INFO - Deleting PVC: many2one-020acd02 +2025-06-03 14:58:49,051 - tests.orchestrator - INFO - Waiting for PVC many2one-020acd02 to be deleted +2025-06-03 14:58:49,119 - tests.orchestrator - INFO - PVC many2one-020acd02 has been deleted +2025-06-03 14:58:49,119 - tests.orchestrator - INFO - Deleted PVC: many2one-020acd02 +2025-06-03 14:58:49,119 - tests.orchestrator - INFO - Deleting PVC: test-pvc-4937168b +2025-06-03 14:58:49,191 - tests.orchestrator - INFO - Waiting for PVC test-pvc-4937168b to be deleted +2025-06-03 14:58:49,257 - tests.orchestrator - INFO - PVC test-pvc-4937168b has been deleted +2025-06-03 14:58:49,257 - tests.orchestrator - INFO - Deleted PVC: test-pvc-4937168b +2025-06-03 14:58:49,258 - tests.orchestrator - INFO - Cleanup completed +2025-06-03 14:58:49,258 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-03 14:58:49,258 - tests.orchestrator - INFO - --- Operations --- +2025-06-03 14:58:49,258 - tests.orchestrator - INFO - create_pvc: 1 succeeded, 0 failed (100.0%) +2025-06-03 14:58:49,258 - tests.orchestrator - INFO - attach_pod: 9 succeeded, 0 failed (100.0%) +2025-06-03 14:58:49,258 - tests.orchestrator - INFO - delete_pod: 9 succeeded, 0 failed (100.0%) +2025-06-03 14:58:49,258 - tests.orchestrator - INFO - delete_pvc: 3 succeeded, 0 failed (100.0%) +2025-06-03 14:58:49,258 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) +2025-06-03 14:58:49,258 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) +2025-06-03 14:58:49,258 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-03 14:58:49,258 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-03 14:58:49,258 - tests.orchestrator - INFO - many_to_one: 0 succeeded, 2 failed out of 2 runs (0.0%) +2025-06-03 14:58:49,258 - tests.orchestrator - INFO - one_to_one: No runs +2025-06-03 14:58:49,258 - tests.orchestrator - INFO - concurrent_pvc: No runs +2025-06-03 14:58:49,258 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250603_154721.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250603_154721.log new file mode 100644 index 000000000..551c1f94b --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250603_154721.log @@ -0,0 +1,8 @@ +2025-06-03 15:47:21,097 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-03 15:47:21,097 - tests.orchestrator - INFO - Starting orchestrator test for 600 seconds +2025-06-03 15:47:21,297 - tests.orchestrator - ERROR - Error checking StorageClass: (401) +Reason: Unauthorized +HTTP response headers: HTTPHeaderDict({'Audit-Id': 'fb6b8162-e595-4821-8d1a-19fb10c247db', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Tue, 03 Jun 2025 15:47:21 GMT', 'Content-Length': '129'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} + + diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250603_154805.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250603_154805.log new file mode 100644 index 000000000..9b1f8b427 --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250603_154805.log @@ -0,0 +1,455 @@ +2025-06-03 15:48:05,458 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-03 15:48:05,458 - tests.orchestrator - INFO - Starting orchestrator test for 600 seconds +2025-06-03 15:48:05,664 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-03 15:48:05,664 - tests.orchestrator - INFO - Selected operation: _create_pvc +2025-06-03 15:48:05,665 - tests.orchestrator - INFO - Creating PVC: test-pvc-9e0f63a9 +2025-06-03 15:48:05,866 - tests.orchestrator - INFO - Created PVC: test-pvc-9e0f63a9 +2025-06-03 15:48:05,866 - tests.orchestrator - INFO - Waiting for PVC test-pvc-9e0f63a9 to be bound +2025-06-03 15:48:08,002 - tests.orchestrator - INFO - PVC test-pvc-9e0f63a9 is bound +2025-06-03 15:48:09,003 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario +2025-06-03 15:48:09,003 - tests.orchestrator - INFO - Running specific scenario: _scenario_one_to_one +2025-06-03 15:48:09,003 - tests.orchestrator - INFO - Running scenario: One pod per PVC +2025-06-03 15:48:09,003 - tests.orchestrator - INFO - Creating 4 PVC-pod pairs +2025-06-03 15:48:09,071 - tests.orchestrator - INFO - Waiting for PVC one2one-53c40f0a to be bound +2025-06-03 15:48:11,204 - tests.orchestrator - INFO - PVC one2one-53c40f0a is bound +2025-06-03 15:48:11,204 - tests.orchestrator - INFO - Creating pod test-pod-9dcfaf3b on PVC one2one-53c40f0a +2025-06-03 15:48:11,286 - tests.orchestrator - INFO - Created pod: test-pod-9dcfaf3b using PVC: one2one-53c40f0a +2025-06-03 15:48:11,286 - tests.orchestrator - INFO - Waiting for pod test-pod-9dcfaf3b to be ready +2025-06-03 15:48:17,562 - tests.orchestrator - INFO - Pod test-pod-9dcfaf3b is ready +2025-06-03 15:48:17,631 - tests.orchestrator - INFO - Waiting for PVC one2one-e698727d to be bound +2025-06-03 15:48:19,766 - tests.orchestrator - INFO - PVC one2one-e698727d is bound +2025-06-03 15:48:19,766 - tests.orchestrator - INFO - Creating pod test-pod-256bf78d on PVC one2one-e698727d +2025-06-03 15:48:19,842 - tests.orchestrator - INFO - Created pod: test-pod-256bf78d using PVC: one2one-e698727d +2025-06-03 15:48:19,842 - tests.orchestrator - INFO - Waiting for pod test-pod-256bf78d to be ready +2025-06-03 15:48:32,323 - tests.orchestrator - INFO - Pod test-pod-256bf78d is ready +2025-06-03 15:48:32,394 - tests.orchestrator - INFO - Waiting for PVC one2one-ee84a0d2 to be bound +2025-06-03 15:48:34,527 - tests.orchestrator - INFO - PVC one2one-ee84a0d2 is bound +2025-06-03 15:48:34,527 - tests.orchestrator - INFO - Creating pod test-pod-d95dbc73 on PVC one2one-ee84a0d2 +2025-06-03 15:48:34,600 - tests.orchestrator - INFO - Created pod: test-pod-d95dbc73 using PVC: one2one-ee84a0d2 +2025-06-03 15:48:34,600 - tests.orchestrator - INFO - Waiting for pod test-pod-d95dbc73 to be ready +2025-06-03 15:48:40,873 - tests.orchestrator - INFO - Pod test-pod-d95dbc73 is ready +2025-06-03 15:48:40,950 - tests.orchestrator - INFO - Waiting for PVC one2one-b38cd705 to be bound +2025-06-03 15:48:43,088 - tests.orchestrator - INFO - PVC one2one-b38cd705 is bound +2025-06-03 15:48:43,088 - tests.orchestrator - INFO - Creating pod test-pod-04cae7e2 on PVC one2one-b38cd705 +2025-06-03 15:48:43,173 - tests.orchestrator - INFO - Created pod: test-pod-04cae7e2 using PVC: one2one-b38cd705 +2025-06-03 15:48:43,173 - tests.orchestrator - INFO - Waiting for pod test-pod-04cae7e2 to be ready +2025-06-03 15:48:49,452 - tests.orchestrator - INFO - Pod test-pod-04cae7e2 is ready +2025-06-03 15:48:49,524 - tests.orchestrator - ERROR - Error in one-to-one scenario: (400) +Reason: Bad Request +HTTP response headers: HTTPHeaderDict({'Audit-Id': 'be1d7a01-f2a7-47e7-9b5f-595caf3d4262', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Tue, 03 Jun 2025 15:48:49 GMT', 'Content-Length': '139'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Upgrade request required","reason":"BadRequest","code":400} + + +2025-06-03 15:48:50,524 - tests.orchestrator - INFO - Selected operation: _delete_pod +2025-06-03 15:48:50,524 - tests.orchestrator - INFO - Deleting pod: test-pod-256bf78d from PVC: one2one-e698727d +2025-06-03 15:48:50,602 - tests.orchestrator - INFO - Waiting for pod test-pod-256bf78d to be deleted +2025-06-03 15:49:23,225 - tests.orchestrator - INFO - Pod test-pod-256bf78d has been deleted +2025-06-03 15:49:23,225 - tests.orchestrator - INFO - Deleted pod: test-pod-256bf78d +2025-06-03 15:49:24,225 - tests.orchestrator - INFO - Selected operation: _delete_pod +2025-06-03 15:49:24,226 - tests.orchestrator - INFO - Deleting pod: test-pod-04cae7e2 from PVC: one2one-b38cd705 +2025-06-03 15:49:24,300 - tests.orchestrator - INFO - Waiting for pod test-pod-04cae7e2 to be deleted +2025-06-03 15:49:57,000 - tests.orchestrator - INFO - Pod test-pod-04cae7e2 has been deleted +2025-06-03 15:49:57,000 - tests.orchestrator - INFO - Deleted pod: test-pod-04cae7e2 +2025-06-03 15:49:58,001 - tests.orchestrator - INFO - Selected operation: _attach_pod +2025-06-03 15:49:58,001 - tests.orchestrator - INFO - Creating pod test-pod-2a900eb4 on PVC one2one-e698727d +2025-06-03 15:49:58,086 - tests.orchestrator - INFO - Created pod: test-pod-2a900eb4 using PVC: one2one-e698727d +2025-06-03 15:49:58,086 - tests.orchestrator - INFO - Waiting for pod test-pod-2a900eb4 to be ready +2025-06-03 15:50:10,568 - tests.orchestrator - INFO - Pod test-pod-2a900eb4 is ready +2025-06-03 15:50:11,568 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario +2025-06-03 15:50:11,569 - tests.orchestrator - INFO - Running specific scenario: _scenario_one_to_one +2025-06-03 15:50:11,569 - tests.orchestrator - INFO - Running scenario: One pod per PVC +2025-06-03 15:50:11,569 - tests.orchestrator - INFO - Creating 5 PVC-pod pairs +2025-06-03 15:50:11,833 - tests.orchestrator - INFO - Waiting for PVC one2one-e7169041 to be bound +2025-06-03 15:50:14,174 - tests.orchestrator - INFO - PVC one2one-e7169041 is bound +2025-06-03 15:50:14,174 - tests.orchestrator - INFO - Creating pod test-pod-5aeb9067 on PVC one2one-e7169041 +2025-06-03 15:50:14,249 - tests.orchestrator - INFO - Created pod: test-pod-5aeb9067 using PVC: one2one-e7169041 +2025-06-03 15:50:14,250 - tests.orchestrator - INFO - Waiting for pod test-pod-5aeb9067 to be ready +2025-06-03 15:50:20,524 - tests.orchestrator - INFO - Pod test-pod-5aeb9067 is ready +2025-06-03 15:50:20,593 - tests.orchestrator - INFO - Waiting for PVC one2one-bee0c24d to be bound +2025-06-03 15:50:22,729 - tests.orchestrator - INFO - PVC one2one-bee0c24d is bound +2025-06-03 15:50:22,729 - tests.orchestrator - INFO - Creating pod test-pod-62c10933 on PVC one2one-bee0c24d +2025-06-03 15:50:22,802 - tests.orchestrator - INFO - Created pod: test-pod-62c10933 using PVC: one2one-bee0c24d +2025-06-03 15:50:22,802 - tests.orchestrator - INFO - Waiting for pod test-pod-62c10933 to be ready +2025-06-03 15:50:35,280 - tests.orchestrator - INFO - Pod test-pod-62c10933 is ready +2025-06-03 15:50:35,359 - tests.orchestrator - INFO - Waiting for PVC one2one-74ea88f9 to be bound +2025-06-03 15:50:37,491 - tests.orchestrator - INFO - PVC one2one-74ea88f9 is bound +2025-06-03 15:50:37,491 - tests.orchestrator - INFO - Creating pod test-pod-7f739327 on PVC one2one-74ea88f9 +2025-06-03 15:50:37,567 - tests.orchestrator - INFO - Created pod: test-pod-7f739327 using PVC: one2one-74ea88f9 +2025-06-03 15:50:37,567 - tests.orchestrator - INFO - Waiting for pod test-pod-7f739327 to be ready +2025-06-03 15:50:50,197 - tests.orchestrator - INFO - Pod test-pod-7f739327 is ready +2025-06-03 15:50:50,269 - tests.orchestrator - INFO - Waiting for PVC one2one-11548b80 to be bound +2025-06-03 15:50:52,400 - tests.orchestrator - INFO - PVC one2one-11548b80 is bound +2025-06-03 15:50:52,401 - tests.orchestrator - INFO - Creating pod test-pod-1d7db6ad on PVC one2one-11548b80 +2025-06-03 15:50:52,477 - tests.orchestrator - INFO - Created pod: test-pod-1d7db6ad using PVC: one2one-11548b80 +2025-06-03 15:50:52,477 - tests.orchestrator - INFO - Waiting for pod test-pod-1d7db6ad to be ready +2025-06-03 15:51:04,957 - tests.orchestrator - INFO - Pod test-pod-1d7db6ad is ready +2025-06-03 15:51:05,026 - tests.orchestrator - INFO - Waiting for PVC one2one-2b9e9546 to be bound +2025-06-03 15:51:07,162 - tests.orchestrator - INFO - PVC one2one-2b9e9546 is bound +2025-06-03 15:51:07,162 - tests.orchestrator - INFO - Creating pod test-pod-431935c6 on PVC one2one-2b9e9546 +2025-06-03 15:51:07,246 - tests.orchestrator - INFO - Created pod: test-pod-431935c6 using PVC: one2one-2b9e9546 +2025-06-03 15:51:07,246 - tests.orchestrator - INFO - Waiting for pod test-pod-431935c6 to be ready +2025-06-03 15:51:19,728 - tests.orchestrator - INFO - Pod test-pod-431935c6 is ready +2025-06-03 15:51:19,800 - tests.orchestrator - ERROR - Error in one-to-one scenario: (400) +Reason: Bad Request +HTTP response headers: HTTPHeaderDict({'Audit-Id': 'ae626c0b-b622-4551-b0e8-5ab032b19ba9', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Tue, 03 Jun 2025 15:51:19 GMT', 'Content-Length': '139'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Upgrade request required","reason":"BadRequest","code":400} + + +2025-06-03 15:51:20,800 - tests.orchestrator - INFO - Selected operation: _delete_pod +2025-06-03 15:51:20,800 - tests.orchestrator - INFO - Deleting pod: test-pod-5aeb9067 from PVC: one2one-e7169041 +2025-06-03 15:51:20,876 - tests.orchestrator - INFO - Waiting for pod test-pod-5aeb9067 to be deleted +2025-06-03 15:51:53,497 - tests.orchestrator - INFO - Pod test-pod-5aeb9067 has been deleted +2025-06-03 15:51:53,498 - tests.orchestrator - INFO - Deleted pod: test-pod-5aeb9067 +2025-06-03 15:51:54,498 - tests.orchestrator - INFO - Selected operation: _attach_pod +2025-06-03 15:51:54,498 - tests.orchestrator - INFO - Creating pod test-pod-8f39fd3b on PVC one2one-b38cd705 +2025-06-03 15:51:54,585 - tests.orchestrator - INFO - Created pod: test-pod-8f39fd3b using PVC: one2one-b38cd705 +2025-06-03 15:51:54,585 - tests.orchestrator - INFO - Waiting for pod test-pod-8f39fd3b to be ready +2025-06-03 15:52:07,068 - tests.orchestrator - INFO - Pod test-pod-8f39fd3b is ready +2025-06-03 15:52:08,069 - tests.orchestrator - INFO - Selected operation: _delete_pod +2025-06-03 15:52:08,069 - tests.orchestrator - INFO - Deleting pod: test-pod-d95dbc73 from PVC: one2one-ee84a0d2 +2025-06-03 15:52:08,147 - tests.orchestrator - INFO - Waiting for pod test-pod-d95dbc73 to be deleted +2025-06-03 15:52:40,769 - tests.orchestrator - INFO - Pod test-pod-d95dbc73 has been deleted +2025-06-03 15:52:40,769 - tests.orchestrator - INFO - Deleted pod: test-pod-d95dbc73 +2025-06-03 15:52:41,769 - tests.orchestrator - INFO - Selected operation: _create_pvc +2025-06-03 15:52:41,769 - tests.orchestrator - INFO - Creating PVC: test-pvc-c2fea111 +2025-06-03 15:52:41,945 - tests.orchestrator - INFO - Created PVC: test-pvc-c2fea111 +2025-06-03 15:52:41,945 - tests.orchestrator - INFO - Waiting for PVC test-pvc-c2fea111 to be bound +2025-06-03 15:52:44,085 - tests.orchestrator - INFO - PVC test-pvc-c2fea111 is bound +2025-06-03 15:52:45,086 - tests.orchestrator - INFO - Selected operation: _verify_readwrite +2025-06-03 15:52:45,086 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test +2025-06-03 15:52:46,086 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario +2025-06-03 15:52:46,086 - tests.orchestrator - INFO - Running specific scenario: _scenario_many_to_one +2025-06-03 15:52:46,086 - tests.orchestrator - INFO - Running scenario: Many pods to one PVC +2025-06-03 15:52:46,157 - tests.orchestrator - INFO - Waiting for PVC many2one-5c9be389 to be bound +2025-06-03 15:52:48,291 - tests.orchestrator - INFO - PVC many2one-5c9be389 is bound +2025-06-03 15:52:48,291 - tests.orchestrator - INFO - Creating 4 pods for the same PVC +2025-06-03 15:52:48,291 - tests.orchestrator - INFO - Creating pod test-pod-7eb2e7e7 on PVC many2one-5c9be389 +2025-06-03 15:52:48,366 - tests.orchestrator - INFO - Created pod: test-pod-7eb2e7e7 using PVC: many2one-5c9be389 +2025-06-03 15:52:48,366 - tests.orchestrator - INFO - Waiting for pod test-pod-7eb2e7e7 to be ready +2025-06-03 15:52:54,649 - tests.orchestrator - INFO - Pod test-pod-7eb2e7e7 is ready +2025-06-03 15:52:54,649 - tests.orchestrator - INFO - Creating pod test-pod-506e6bf9 on PVC many2one-5c9be389 +2025-06-03 15:52:54,725 - tests.orchestrator - INFO - Created pod: test-pod-506e6bf9 using PVC: many2one-5c9be389 +2025-06-03 15:52:54,725 - tests.orchestrator - INFO - Waiting for pod test-pod-506e6bf9 to be ready +2025-06-03 15:53:07,211 - tests.orchestrator - INFO - Pod test-pod-506e6bf9 is ready +2025-06-03 15:53:07,211 - tests.orchestrator - INFO - Creating pod test-pod-6b52ef99 on PVC many2one-5c9be389 +2025-06-03 15:53:07,283 - tests.orchestrator - INFO - Created pod: test-pod-6b52ef99 using PVC: many2one-5c9be389 +2025-06-03 15:53:07,284 - tests.orchestrator - INFO - Waiting for pod test-pod-6b52ef99 to be ready +2025-06-03 15:53:19,854 - tests.orchestrator - INFO - Pod test-pod-6b52ef99 is ready +2025-06-03 15:53:19,854 - tests.orchestrator - INFO - Creating pod test-pod-fc6a3c0c on PVC many2one-5c9be389 +2025-06-03 15:53:19,942 - tests.orchestrator - INFO - Created pod: test-pod-fc6a3c0c using PVC: many2one-5c9be389 +2025-06-03 15:53:19,942 - tests.orchestrator - INFO - Waiting for pod test-pod-fc6a3c0c to be ready +2025-06-03 15:53:32,426 - tests.orchestrator - INFO - Pod test-pod-fc6a3c0c is ready +2025-06-03 15:53:32,495 - tests.orchestrator - ERROR - Error in many-to-one scenario: (400) +Reason: Bad Request +HTTP response headers: HTTPHeaderDict({'Audit-Id': '9cd3b5fb-b62b-4ee0-b08b-2a3e761b6d8c', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Tue, 03 Jun 2025 15:53:32 GMT', 'Content-Length': '139'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Upgrade request required","reason":"BadRequest","code":400} + + +2025-06-03 15:53:33,496 - tests.orchestrator - INFO - Selected operation: _attach_pod +2025-06-03 15:53:33,496 - tests.orchestrator - INFO - Creating pod test-pod-4f87753e on PVC one2one-b38cd705 +2025-06-03 15:53:33,569 - tests.orchestrator - INFO - Created pod: test-pod-4f87753e using PVC: one2one-b38cd705 +2025-06-03 15:53:33,569 - tests.orchestrator - INFO - Waiting for pod test-pod-4f87753e to be ready +2025-06-03 15:53:39,842 - tests.orchestrator - INFO - Pod test-pod-4f87753e is ready +2025-06-03 15:53:40,842 - tests.orchestrator - INFO - Selected operation: _verify_readwrite +2025-06-03 15:53:40,842 - tests.orchestrator - INFO - Testing read/write between pods test-pod-8f39fd3b and test-pod-4f87753e sharing PVC one2one-b38cd705 +2025-06-03 15:53:40,913 - tests.orchestrator - ERROR - Failed in read/write verification: (400) +Reason: Bad Request +HTTP response headers: HTTPHeaderDict({'Audit-Id': '571b0988-ce3a-498b-9d74-580abe8cc33a', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Tue, 03 Jun 2025 15:53:40 GMT', 'Content-Length': '139'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Upgrade request required","reason":"BadRequest","code":400} + + +2025-06-03 15:53:41,914 - tests.orchestrator - INFO - Selected operation: _delete_pod +2025-06-03 15:53:41,914 - tests.orchestrator - INFO - Deleting pod: test-pod-7f739327 from PVC: one2one-74ea88f9 +2025-06-03 15:53:41,991 - tests.orchestrator - INFO - Waiting for pod test-pod-7f739327 to be deleted +2025-06-03 15:54:14,614 - tests.orchestrator - INFO - Pod test-pod-7f739327 has been deleted +2025-06-03 15:54:14,614 - tests.orchestrator - INFO - Deleted pod: test-pod-7f739327 +2025-06-03 15:54:15,615 - tests.orchestrator - INFO - Selected operation: _delete_pod +2025-06-03 15:54:15,615 - tests.orchestrator - INFO - Deleting pod: test-pod-4f87753e from PVC: one2one-b38cd705 +2025-06-03 15:54:15,691 - tests.orchestrator - INFO - Waiting for pod test-pod-4f87753e to be deleted +2025-06-03 15:54:48,304 - tests.orchestrator - INFO - Pod test-pod-4f87753e has been deleted +2025-06-03 15:54:48,304 - tests.orchestrator - INFO - Deleted pod: test-pod-4f87753e +2025-06-03 15:54:49,304 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario +2025-06-03 15:54:49,304 - tests.orchestrator - INFO - Running specific scenario: _scenario_concurrent_pvc +2025-06-03 15:54:49,304 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations +2025-06-03 15:54:49,304 - tests.orchestrator - INFO - Creating 7 PVCs in quick succession +2025-06-03 15:54:49,382 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-ba4fefcb +2025-06-03 15:54:49,382 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-ba4fefcb to be bound +2025-06-03 15:54:51,518 - tests.orchestrator - INFO - PVC concurrent-pvc-ba4fefcb is bound +2025-06-03 15:54:51,586 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-1f973741 +2025-06-03 15:54:51,587 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-1f973741 to be bound +2025-06-03 15:54:53,725 - tests.orchestrator - INFO - PVC concurrent-pvc-1f973741 is bound +2025-06-03 15:54:53,793 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-3559ffc8 +2025-06-03 15:54:53,793 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-3559ffc8 to be bound +2025-06-03 15:54:55,928 - tests.orchestrator - INFO - PVC concurrent-pvc-3559ffc8 is bound +2025-06-03 15:54:55,996 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-f3060a1d +2025-06-03 15:54:55,996 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-f3060a1d to be bound +2025-06-03 15:54:58,130 - tests.orchestrator - INFO - PVC concurrent-pvc-f3060a1d is bound +2025-06-03 15:54:58,198 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-e44ab328 +2025-06-03 15:54:58,198 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-e44ab328 to be bound +2025-06-03 15:55:00,332 - tests.orchestrator - INFO - PVC concurrent-pvc-e44ab328 is bound +2025-06-03 15:55:00,403 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-2d6e60cb +2025-06-03 15:55:00,403 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-2d6e60cb to be bound +2025-06-03 15:55:02,538 - tests.orchestrator - INFO - PVC concurrent-pvc-2d6e60cb is bound +2025-06-03 15:55:02,612 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-c27e9e10 +2025-06-03 15:55:02,612 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-c27e9e10 to be bound +2025-06-03 15:55:04,754 - tests.orchestrator - INFO - PVC concurrent-pvc-c27e9e10 is bound +2025-06-03 15:55:04,754 - tests.orchestrator - INFO - Successfully created 7 PVCs +2025-06-03 15:55:04,754 - tests.orchestrator - INFO - Creating pod test-pod-f3855837 on PVC concurrent-pvc-3559ffc8 +2025-06-03 15:55:04,834 - tests.orchestrator - INFO - Created pod: test-pod-f3855837 using PVC: concurrent-pvc-3559ffc8 +2025-06-03 15:55:04,834 - tests.orchestrator - INFO - Waiting for pod test-pod-f3855837 to be ready +2025-06-03 15:55:17,347 - tests.orchestrator - INFO - Pod test-pod-f3855837 is ready +2025-06-03 15:55:17,347 - tests.orchestrator - INFO - Creating pod test-pod-8e0cc6ec on PVC concurrent-pvc-e44ab328 +2025-06-03 15:55:17,423 - tests.orchestrator - INFO - Created pod: test-pod-8e0cc6ec using PVC: concurrent-pvc-e44ab328 +2025-06-03 15:55:17,423 - tests.orchestrator - INFO - Waiting for pod test-pod-8e0cc6ec to be ready +2025-06-03 15:55:23,703 - tests.orchestrator - INFO - Pod test-pod-8e0cc6ec is ready +2025-06-03 15:55:23,703 - tests.orchestrator - INFO - Creating pod test-pod-f7e46b0a on PVC concurrent-pvc-1f973741 +2025-06-03 15:55:23,790 - tests.orchestrator - INFO - Created pod: test-pod-f7e46b0a using PVC: concurrent-pvc-1f973741 +2025-06-03 15:55:23,790 - tests.orchestrator - INFO - Waiting for pod test-pod-f7e46b0a to be ready +2025-06-03 15:55:36,266 - tests.orchestrator - INFO - Pod test-pod-f7e46b0a is ready +2025-06-03 15:55:36,266 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-ba4fefcb +2025-06-03 15:55:36,339 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-ba4fefcb to be deleted +2025-06-03 15:55:36,406 - tests.orchestrator - INFO - PVC concurrent-pvc-ba4fefcb has been deleted +2025-06-03 15:55:36,406 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-ba4fefcb +2025-06-03 15:55:36,406 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-1f973741 +2025-06-03 15:55:36,406 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-1f973741 +2025-06-03 15:55:36,406 - tests.orchestrator - INFO - Deleting pod: test-pod-f7e46b0a from PVC: concurrent-pvc-1f973741 +2025-06-03 15:55:36,500 - tests.orchestrator - INFO - Waiting for pod test-pod-f7e46b0a to be deleted +2025-06-03 15:56:09,115 - tests.orchestrator - INFO - Pod test-pod-f7e46b0a has been deleted +2025-06-03 15:56:09,115 - tests.orchestrator - INFO - Deleted pod: test-pod-f7e46b0a +2025-06-03 15:56:09,188 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-1f973741 to be deleted +2025-06-03 15:56:09,254 - tests.orchestrator - INFO - PVC concurrent-pvc-1f973741 has been deleted +2025-06-03 15:56:09,255 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-1f973741 +2025-06-03 15:56:09,255 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-3559ffc8 +2025-06-03 15:56:09,255 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-3559ffc8 +2025-06-03 15:56:09,255 - tests.orchestrator - INFO - Deleting pod: test-pod-f3855837 from PVC: concurrent-pvc-3559ffc8 +2025-06-03 15:56:09,329 - tests.orchestrator - INFO - Waiting for pod test-pod-f3855837 to be deleted +2025-06-03 15:56:41,952 - tests.orchestrator - INFO - Pod test-pod-f3855837 has been deleted +2025-06-03 15:56:41,952 - tests.orchestrator - INFO - Deleted pod: test-pod-f3855837 +2025-06-03 15:56:42,031 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-3559ffc8 to be deleted +2025-06-03 15:56:42,102 - tests.orchestrator - INFO - PVC concurrent-pvc-3559ffc8 has been deleted +2025-06-03 15:56:42,102 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-3559ffc8 +2025-06-03 15:56:42,103 - tests.orchestrator - INFO - Rapid PVC scenario completed +2025-06-03 15:56:43,103 - tests.orchestrator - INFO - Selected operation: _delete_pod +2025-06-03 15:56:43,103 - tests.orchestrator - INFO - Deleting pod: test-pod-6b52ef99 from PVC: many2one-5c9be389 +2025-06-03 15:56:43,182 - tests.orchestrator - INFO - Waiting for pod test-pod-6b52ef99 to be deleted +2025-06-03 15:57:16,611 - tests.orchestrator - INFO - Pod test-pod-6b52ef99 has been deleted +2025-06-03 15:57:16,611 - tests.orchestrator - INFO - Deleted pod: test-pod-6b52ef99 +2025-06-03 15:57:17,611 - tests.orchestrator - INFO - Selected operation: _attach_pod +2025-06-03 15:57:17,611 - tests.orchestrator - INFO - Creating pod test-pod-a819397c on PVC concurrent-pvc-c27e9e10 +2025-06-03 15:57:17,702 - tests.orchestrator - INFO - Created pod: test-pod-a819397c using PVC: concurrent-pvc-c27e9e10 +2025-06-03 15:57:17,702 - tests.orchestrator - INFO - Waiting for pod test-pod-a819397c to be ready +2025-06-03 15:57:30,182 - tests.orchestrator - INFO - Pod test-pod-a819397c is ready +2025-06-03 15:57:31,182 - tests.orchestrator - INFO - Selected operation: _create_pvc +2025-06-03 15:57:31,182 - tests.orchestrator - INFO - Creating PVC: test-pvc-ee48e109 +2025-06-03 15:57:31,250 - tests.orchestrator - INFO - Created PVC: test-pvc-ee48e109 +2025-06-03 15:57:31,250 - tests.orchestrator - INFO - Waiting for PVC test-pvc-ee48e109 to be bound +2025-06-03 15:57:33,384 - tests.orchestrator - INFO - PVC test-pvc-ee48e109 is bound +2025-06-03 15:57:34,384 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario +2025-06-03 15:57:34,385 - tests.orchestrator - INFO - Running specific scenario: _scenario_concurrent_pvc +2025-06-03 15:57:34,385 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations +2025-06-03 15:57:34,385 - tests.orchestrator - INFO - Creating 7 PVCs in quick succession +2025-06-03 15:57:34,455 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-51a18d7b +2025-06-03 15:57:34,456 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-51a18d7b to be bound +2025-06-03 15:57:36,594 - tests.orchestrator - INFO - PVC concurrent-pvc-51a18d7b is bound +2025-06-03 15:57:36,666 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-c7fabc72 +2025-06-03 15:57:36,666 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-c7fabc72 to be bound +2025-06-03 15:57:38,803 - tests.orchestrator - INFO - PVC concurrent-pvc-c7fabc72 is bound +2025-06-03 15:57:38,877 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-07a19245 +2025-06-03 15:57:38,877 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-07a19245 to be bound +2025-06-03 15:57:41,013 - tests.orchestrator - INFO - PVC concurrent-pvc-07a19245 is bound +2025-06-03 15:57:41,080 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-5d6cd5f4 +2025-06-03 15:57:41,080 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-5d6cd5f4 to be bound +2025-06-03 15:57:43,222 - tests.orchestrator - INFO - PVC concurrent-pvc-5d6cd5f4 is bound +2025-06-03 15:57:43,291 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-9cdc5f04 +2025-06-03 15:57:43,291 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-9cdc5f04 to be bound +2025-06-03 15:57:45,427 - tests.orchestrator - INFO - PVC concurrent-pvc-9cdc5f04 is bound +2025-06-03 15:57:45,494 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-0112d2c9 +2025-06-03 15:57:45,494 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-0112d2c9 to be bound +2025-06-03 15:57:47,630 - tests.orchestrator - INFO - PVC concurrent-pvc-0112d2c9 is bound +2025-06-03 15:57:47,708 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-96aa4689 +2025-06-03 15:57:47,708 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-96aa4689 to be bound +2025-06-03 15:57:49,840 - tests.orchestrator - INFO - PVC concurrent-pvc-96aa4689 is bound +2025-06-03 15:57:49,840 - tests.orchestrator - INFO - Successfully created 7 PVCs +2025-06-03 15:57:49,840 - tests.orchestrator - INFO - Creating pod test-pod-cc67cce8 on PVC concurrent-pvc-07a19245 +2025-06-03 15:57:49,915 - tests.orchestrator - INFO - Created pod: test-pod-cc67cce8 using PVC: concurrent-pvc-07a19245 +2025-06-03 15:57:49,915 - tests.orchestrator - INFO - Waiting for pod test-pod-cc67cce8 to be ready +2025-06-03 15:58:02,392 - tests.orchestrator - INFO - Pod test-pod-cc67cce8 is ready +2025-06-03 15:58:02,393 - tests.orchestrator - INFO - Creating pod test-pod-39578545 on PVC concurrent-pvc-9cdc5f04 +2025-06-03 15:58:02,484 - tests.orchestrator - INFO - Created pod: test-pod-39578545 using PVC: concurrent-pvc-9cdc5f04 +2025-06-03 15:58:02,484 - tests.orchestrator - INFO - Waiting for pod test-pod-39578545 to be ready +2025-06-03 15:58:08,761 - tests.orchestrator - INFO - Pod test-pod-39578545 is ready +2025-06-03 15:58:08,761 - tests.orchestrator - INFO - Creating pod test-pod-8bd839df on PVC concurrent-pvc-96aa4689 +2025-06-03 15:58:08,836 - tests.orchestrator - INFO - Created pod: test-pod-8bd839df using PVC: concurrent-pvc-96aa4689 +2025-06-03 15:58:08,836 - tests.orchestrator - INFO - Waiting for pod test-pod-8bd839df to be ready +2025-06-03 15:58:21,318 - tests.orchestrator - INFO - Pod test-pod-8bd839df is ready +2025-06-03 15:58:21,318 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-07a19245 +2025-06-03 15:58:21,319 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-07a19245 +2025-06-03 15:58:21,319 - tests.orchestrator - INFO - Deleting pod: test-pod-cc67cce8 from PVC: concurrent-pvc-07a19245 +2025-06-03 15:58:21,396 - tests.orchestrator - INFO - Waiting for pod test-pod-cc67cce8 to be deleted +2025-06-03 15:58:54,172 - tests.orchestrator - INFO - Pod test-pod-cc67cce8 has been deleted +2025-06-03 15:58:54,172 - tests.orchestrator - INFO - Deleted pod: test-pod-cc67cce8 +2025-06-03 15:58:54,243 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-07a19245 to be deleted +2025-06-03 15:58:54,310 - tests.orchestrator - INFO - PVC concurrent-pvc-07a19245 has been deleted +2025-06-03 15:58:54,311 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-07a19245 +2025-06-03 15:58:54,311 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-5d6cd5f4 +2025-06-03 15:58:54,382 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-5d6cd5f4 to be deleted +2025-06-03 15:58:54,450 - tests.orchestrator - INFO - PVC concurrent-pvc-5d6cd5f4 has been deleted +2025-06-03 15:58:54,450 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-5d6cd5f4 +2025-06-03 15:58:54,450 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-51a18d7b +2025-06-03 15:58:54,523 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-51a18d7b to be deleted +2025-06-03 15:58:54,589 - tests.orchestrator - INFO - PVC concurrent-pvc-51a18d7b has been deleted +2025-06-03 15:58:54,590 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-51a18d7b +2025-06-03 15:58:54,590 - tests.orchestrator - INFO - Rapid PVC scenario completed +2025-06-03 15:58:55,590 - tests.orchestrator - INFO - Test completed in 650.13 seconds +2025-06-03 15:58:55,590 - tests.orchestrator - INFO - Cleaning up resources... +2025-06-03 15:58:55,590 - tests.orchestrator - INFO - Deleting pod: test-pod-9dcfaf3b from PVC: one2one-53c40f0a +2025-06-03 15:58:55,667 - tests.orchestrator - INFO - Waiting for pod test-pod-9dcfaf3b to be deleted +2025-06-03 15:59:28,324 - tests.orchestrator - INFO - Pod test-pod-9dcfaf3b has been deleted +2025-06-03 15:59:28,324 - tests.orchestrator - INFO - Deleted pod: test-pod-9dcfaf3b +2025-06-03 15:59:28,324 - tests.orchestrator - INFO - Deleting pod: test-pod-2a900eb4 from PVC: one2one-e698727d +2025-06-03 15:59:28,400 - tests.orchestrator - INFO - Waiting for pod test-pod-2a900eb4 to be deleted +2025-06-03 16:00:01,019 - tests.orchestrator - INFO - Pod test-pod-2a900eb4 has been deleted +2025-06-03 16:00:01,019 - tests.orchestrator - INFO - Deleted pod: test-pod-2a900eb4 +2025-06-03 16:00:01,019 - tests.orchestrator - INFO - Deleting pod: test-pod-8f39fd3b from PVC: one2one-b38cd705 +2025-06-03 16:00:01,100 - tests.orchestrator - INFO - Waiting for pod test-pod-8f39fd3b to be deleted +2025-06-03 16:00:33,739 - tests.orchestrator - INFO - Pod test-pod-8f39fd3b has been deleted +2025-06-03 16:00:33,739 - tests.orchestrator - INFO - Deleted pod: test-pod-8f39fd3b +2025-06-03 16:00:33,739 - tests.orchestrator - INFO - Deleting pod: test-pod-62c10933 from PVC: one2one-bee0c24d +2025-06-03 16:00:33,814 - tests.orchestrator - INFO - Waiting for pod test-pod-62c10933 to be deleted +2025-06-03 16:01:06,447 - tests.orchestrator - INFO - Pod test-pod-62c10933 has been deleted +2025-06-03 16:01:06,448 - tests.orchestrator - INFO - Deleted pod: test-pod-62c10933 +2025-06-03 16:01:06,448 - tests.orchestrator - INFO - Deleting pod: test-pod-1d7db6ad from PVC: one2one-11548b80 +2025-06-03 16:01:06,527 - tests.orchestrator - INFO - Waiting for pod test-pod-1d7db6ad to be deleted +2025-06-03 16:01:39,146 - tests.orchestrator - INFO - Pod test-pod-1d7db6ad has been deleted +2025-06-03 16:01:39,146 - tests.orchestrator - INFO - Deleted pod: test-pod-1d7db6ad +2025-06-03 16:01:39,146 - tests.orchestrator - INFO - Deleting pod: test-pod-431935c6 from PVC: one2one-2b9e9546 +2025-06-03 16:01:39,222 - tests.orchestrator - INFO - Waiting for pod test-pod-431935c6 to be deleted +2025-06-03 16:02:11,990 - tests.orchestrator - INFO - Pod test-pod-431935c6 has been deleted +2025-06-03 16:02:11,990 - tests.orchestrator - INFO - Deleted pod: test-pod-431935c6 +2025-06-03 16:02:11,990 - tests.orchestrator - INFO - Deleting pod: test-pod-7eb2e7e7 from PVC: many2one-5c9be389 +2025-06-03 16:02:12,112 - tests.orchestrator - INFO - Waiting for pod test-pod-7eb2e7e7 to be deleted +2025-06-03 16:02:44,728 - tests.orchestrator - INFO - Pod test-pod-7eb2e7e7 has been deleted +2025-06-03 16:02:44,728 - tests.orchestrator - INFO - Deleted pod: test-pod-7eb2e7e7 +2025-06-03 16:02:44,728 - tests.orchestrator - INFO - Deleting pod: test-pod-506e6bf9 from PVC: many2one-5c9be389 +2025-06-03 16:02:44,801 - tests.orchestrator - INFO - Waiting for pod test-pod-506e6bf9 to be deleted +2025-06-03 16:03:17,416 - tests.orchestrator - INFO - Pod test-pod-506e6bf9 has been deleted +2025-06-03 16:03:17,417 - tests.orchestrator - INFO - Deleted pod: test-pod-506e6bf9 +2025-06-03 16:03:17,417 - tests.orchestrator - INFO - Deleting pod: test-pod-fc6a3c0c from PVC: many2one-5c9be389 +2025-06-03 16:03:17,495 - tests.orchestrator - INFO - Waiting for pod test-pod-fc6a3c0c to be deleted +2025-06-03 16:03:50,115 - tests.orchestrator - INFO - Pod test-pod-fc6a3c0c has been deleted +2025-06-03 16:03:50,115 - tests.orchestrator - INFO - Deleted pod: test-pod-fc6a3c0c +2025-06-03 16:03:50,115 - tests.orchestrator - INFO - Deleting pod: test-pod-8e0cc6ec from PVC: concurrent-pvc-e44ab328 +2025-06-03 16:03:50,193 - tests.orchestrator - INFO - Waiting for pod test-pod-8e0cc6ec to be deleted +2025-06-03 16:04:22,834 - tests.orchestrator - INFO - Pod test-pod-8e0cc6ec has been deleted +2025-06-03 16:04:22,834 - tests.orchestrator - INFO - Deleted pod: test-pod-8e0cc6ec +2025-06-03 16:04:22,834 - tests.orchestrator - INFO - Deleting pod: test-pod-a819397c from PVC: concurrent-pvc-c27e9e10 +2025-06-03 16:04:22,909 - tests.orchestrator - INFO - Waiting for pod test-pod-a819397c to be deleted +2025-06-03 16:04:55,527 - tests.orchestrator - INFO - Pod test-pod-a819397c has been deleted +2025-06-03 16:04:55,527 - tests.orchestrator - INFO - Deleted pod: test-pod-a819397c +2025-06-03 16:04:55,527 - tests.orchestrator - INFO - Deleting pod: test-pod-39578545 from PVC: concurrent-pvc-9cdc5f04 +2025-06-03 16:04:55,601 - tests.orchestrator - INFO - Waiting for pod test-pod-39578545 to be deleted +2025-06-03 16:05:28,321 - tests.orchestrator - INFO - Pod test-pod-39578545 has been deleted +2025-06-03 16:05:28,321 - tests.orchestrator - INFO - Deleted pod: test-pod-39578545 +2025-06-03 16:05:28,322 - tests.orchestrator - INFO - Deleting pod: test-pod-8bd839df from PVC: concurrent-pvc-96aa4689 +2025-06-03 16:05:28,399 - tests.orchestrator - INFO - Waiting for pod test-pod-8bd839df to be deleted +2025-06-03 16:06:01,015 - tests.orchestrator - INFO - Pod test-pod-8bd839df has been deleted +2025-06-03 16:06:01,015 - tests.orchestrator - INFO - Deleted pod: test-pod-8bd839df +2025-06-03 16:06:01,016 - tests.orchestrator - INFO - Deleting PVC: test-pvc-9e0f63a9 +2025-06-03 16:06:01,091 - tests.orchestrator - INFO - Waiting for PVC test-pvc-9e0f63a9 to be deleted +2025-06-03 16:06:01,156 - tests.orchestrator - INFO - PVC test-pvc-9e0f63a9 has been deleted +2025-06-03 16:06:01,156 - tests.orchestrator - INFO - Deleted PVC: test-pvc-9e0f63a9 +2025-06-03 16:06:01,156 - tests.orchestrator - INFO - Deleting PVC: one2one-53c40f0a +2025-06-03 16:06:01,249 - tests.orchestrator - INFO - Waiting for PVC one2one-53c40f0a to be deleted +2025-06-03 16:06:01,313 - tests.orchestrator - INFO - PVC one2one-53c40f0a has been deleted +2025-06-03 16:06:01,314 - tests.orchestrator - INFO - Deleted PVC: one2one-53c40f0a +2025-06-03 16:06:01,315 - tests.orchestrator - INFO - Deleting PVC: one2one-e698727d +2025-06-03 16:06:01,401 - tests.orchestrator - INFO - Waiting for PVC one2one-e698727d to be deleted +2025-06-03 16:06:01,608 - tests.orchestrator - INFO - PVC one2one-e698727d has been deleted +2025-06-03 16:06:01,608 - tests.orchestrator - INFO - Deleted PVC: one2one-e698727d +2025-06-03 16:06:01,608 - tests.orchestrator - INFO - Deleting PVC: one2one-ee84a0d2 +2025-06-03 16:06:01,694 - tests.orchestrator - INFO - Waiting for PVC one2one-ee84a0d2 to be deleted +2025-06-03 16:06:01,786 - tests.orchestrator - INFO - PVC one2one-ee84a0d2 has been deleted +2025-06-03 16:06:01,786 - tests.orchestrator - INFO - Deleted PVC: one2one-ee84a0d2 +2025-06-03 16:06:01,786 - tests.orchestrator - INFO - Deleting PVC: one2one-b38cd705 +2025-06-03 16:06:01,891 - tests.orchestrator - INFO - Waiting for PVC one2one-b38cd705 to be deleted +2025-06-03 16:06:02,000 - tests.orchestrator - INFO - PVC one2one-b38cd705 has been deleted +2025-06-03 16:06:02,000 - tests.orchestrator - INFO - Deleted PVC: one2one-b38cd705 +2025-06-03 16:06:02,000 - tests.orchestrator - INFO - Deleting PVC: one2one-e7169041 +2025-06-03 16:06:02,121 - tests.orchestrator - INFO - Waiting for PVC one2one-e7169041 to be deleted +2025-06-03 16:06:02,191 - tests.orchestrator - INFO - PVC one2one-e7169041 has been deleted +2025-06-03 16:06:02,191 - tests.orchestrator - INFO - Deleted PVC: one2one-e7169041 +2025-06-03 16:06:02,191 - tests.orchestrator - INFO - Deleting PVC: one2one-bee0c24d +2025-06-03 16:06:02,270 - tests.orchestrator - INFO - Waiting for PVC one2one-bee0c24d to be deleted +2025-06-03 16:06:02,335 - tests.orchestrator - INFO - PVC one2one-bee0c24d has been deleted +2025-06-03 16:06:02,335 - tests.orchestrator - INFO - Deleted PVC: one2one-bee0c24d +2025-06-03 16:06:02,336 - tests.orchestrator - INFO - Deleting PVC: one2one-74ea88f9 +2025-06-03 16:06:02,412 - tests.orchestrator - INFO - Waiting for PVC one2one-74ea88f9 to be deleted +2025-06-03 16:06:02,477 - tests.orchestrator - INFO - PVC one2one-74ea88f9 has been deleted +2025-06-03 16:06:02,477 - tests.orchestrator - INFO - Deleted PVC: one2one-74ea88f9 +2025-06-03 16:06:02,477 - tests.orchestrator - INFO - Deleting PVC: one2one-11548b80 +2025-06-03 16:06:02,551 - tests.orchestrator - INFO - Waiting for PVC one2one-11548b80 to be deleted +2025-06-03 16:06:02,616 - tests.orchestrator - INFO - PVC one2one-11548b80 has been deleted +2025-06-03 16:06:02,616 - tests.orchestrator - INFO - Deleted PVC: one2one-11548b80 +2025-06-03 16:06:02,616 - tests.orchestrator - INFO - Deleting PVC: one2one-2b9e9546 +2025-06-03 16:06:02,694 - tests.orchestrator - INFO - Waiting for PVC one2one-2b9e9546 to be deleted +2025-06-03 16:06:02,759 - tests.orchestrator - INFO - PVC one2one-2b9e9546 has been deleted +2025-06-03 16:06:02,760 - tests.orchestrator - INFO - Deleted PVC: one2one-2b9e9546 +2025-06-03 16:06:02,760 - tests.orchestrator - INFO - Deleting PVC: test-pvc-c2fea111 +2025-06-03 16:06:02,838 - tests.orchestrator - INFO - Waiting for PVC test-pvc-c2fea111 to be deleted +2025-06-03 16:06:02,903 - tests.orchestrator - INFO - PVC test-pvc-c2fea111 has been deleted +2025-06-03 16:06:02,903 - tests.orchestrator - INFO - Deleted PVC: test-pvc-c2fea111 +2025-06-03 16:06:02,904 - tests.orchestrator - INFO - Deleting PVC: many2one-5c9be389 +2025-06-03 16:06:02,982 - tests.orchestrator - INFO - Waiting for PVC many2one-5c9be389 to be deleted +2025-06-03 16:06:03,061 - tests.orchestrator - INFO - PVC many2one-5c9be389 has been deleted +2025-06-03 16:06:03,061 - tests.orchestrator - INFO - Deleted PVC: many2one-5c9be389 +2025-06-03 16:06:03,061 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-f3060a1d +2025-06-03 16:06:03,144 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-f3060a1d to be deleted +2025-06-03 16:06:03,212 - tests.orchestrator - INFO - PVC concurrent-pvc-f3060a1d has been deleted +2025-06-03 16:06:03,212 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-f3060a1d +2025-06-03 16:06:03,212 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-e44ab328 +2025-06-03 16:06:03,288 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-e44ab328 to be deleted +2025-06-03 16:06:03,360 - tests.orchestrator - INFO - PVC concurrent-pvc-e44ab328 has been deleted +2025-06-03 16:06:03,360 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-e44ab328 +2025-06-03 16:06:03,360 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-2d6e60cb +2025-06-03 16:06:03,455 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-2d6e60cb to be deleted +2025-06-03 16:06:03,530 - tests.orchestrator - INFO - PVC concurrent-pvc-2d6e60cb has been deleted +2025-06-03 16:06:03,530 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-2d6e60cb +2025-06-03 16:06:03,530 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-c27e9e10 +2025-06-03 16:06:03,609 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-c27e9e10 to be deleted +2025-06-03 16:06:03,674 - tests.orchestrator - INFO - PVC concurrent-pvc-c27e9e10 has been deleted +2025-06-03 16:06:03,675 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-c27e9e10 +2025-06-03 16:06:03,675 - tests.orchestrator - INFO - Deleting PVC: test-pvc-ee48e109 +2025-06-03 16:06:03,750 - tests.orchestrator - INFO - Waiting for PVC test-pvc-ee48e109 to be deleted +2025-06-03 16:06:03,823 - tests.orchestrator - INFO - PVC test-pvc-ee48e109 has been deleted +2025-06-03 16:06:03,823 - tests.orchestrator - INFO - Deleted PVC: test-pvc-ee48e109 +2025-06-03 16:06:03,823 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-c7fabc72 +2025-06-03 16:06:03,898 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-c7fabc72 to be deleted +2025-06-03 16:06:03,997 - tests.orchestrator - INFO - PVC concurrent-pvc-c7fabc72 has been deleted +2025-06-03 16:06:03,998 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-c7fabc72 +2025-06-03 16:06:03,998 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-9cdc5f04 +2025-06-03 16:06:04,072 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-9cdc5f04 to be deleted +2025-06-03 16:06:04,139 - tests.orchestrator - INFO - PVC concurrent-pvc-9cdc5f04 has been deleted +2025-06-03 16:06:04,139 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-9cdc5f04 +2025-06-03 16:06:04,139 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-0112d2c9 +2025-06-03 16:06:04,229 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-0112d2c9 to be deleted +2025-06-03 16:06:04,297 - tests.orchestrator - INFO - PVC concurrent-pvc-0112d2c9 has been deleted +2025-06-03 16:06:04,297 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-0112d2c9 +2025-06-03 16:06:04,298 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-96aa4689 +2025-06-03 16:06:04,388 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-96aa4689 to be deleted +2025-06-03 16:06:04,456 - tests.orchestrator - INFO - PVC concurrent-pvc-96aa4689 has been deleted +2025-06-03 16:06:04,456 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-96aa4689 +2025-06-03 16:06:04,456 - tests.orchestrator - INFO - Cleanup completed +2025-06-03 16:06:04,456 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-03 16:06:04,456 - tests.orchestrator - INFO - --- Operations --- +2025-06-03 16:06:04,456 - tests.orchestrator - INFO - create_pvc: 17 succeeded, 0 failed (100.0%) +2025-06-03 16:06:04,456 - tests.orchestrator - INFO - attach_pod: 23 succeeded, 0 failed (100.0%) +2025-06-03 16:06:04,456 - tests.orchestrator - INFO - delete_pod: 23 succeeded, 0 failed (100.0%) +2025-06-03 16:06:04,456 - tests.orchestrator - INFO - delete_pvc: 27 succeeded, 0 failed (100.0%) +2025-06-03 16:06:04,456 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) +2025-06-03 16:06:04,456 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) +2025-06-03 16:06:04,456 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-03 16:06:04,457 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-03 16:06:04,457 - tests.orchestrator - INFO - many_to_one: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-03 16:06:04,457 - tests.orchestrator - INFO - one_to_one: 0 succeeded, 2 failed out of 2 runs (0.0%) +2025-06-03 16:06:04,457 - tests.orchestrator - INFO - concurrent_pvc: 2 succeeded, 0 failed out of 2 runs (100.0%) +2025-06-03 16:06:04,457 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250603_174810.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250603_174810.log new file mode 100644 index 000000000..6b0890219 --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250603_174810.log @@ -0,0 +1,8 @@ +2025-06-03 17:48:10,686 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-03 17:48:10,686 - tests.orchestrator - INFO - Starting orchestrator test for 50 seconds +2025-06-03 17:48:10,945 - tests.orchestrator - ERROR - Error checking StorageClass: (401) +Reason: Unauthorized +HTTP response headers: HTTPHeaderDict({'Audit-Id': '501ac0bb-a3bf-4a94-bd95-be7f3c4887ad', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Tue, 03 Jun 2025 17:48:10 GMT', 'Content-Length': '129'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} + + diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250603_175820.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250603_175820.log new file mode 100644 index 000000000..ebe1eb6bb --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250603_175820.log @@ -0,0 +1,77 @@ +2025-06-03 17:58:20,784 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-03 17:58:20,784 - tests.orchestrator - INFO - Starting orchestrator test for 50 seconds +2025-06-03 17:58:21,041 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-03 17:58:21,041 - tests.orchestrator - INFO - Selected operation: _create_pvc +2025-06-03 17:58:21,041 - tests.orchestrator - INFO - Creating PVC: test-pvc-595330e0 +2025-06-03 17:58:21,286 - tests.orchestrator - INFO - Created PVC: test-pvc-595330e0 +2025-06-03 17:58:21,287 - tests.orchestrator - INFO - Waiting for PVC test-pvc-595330e0 to be bound +2025-06-03 17:58:23,445 - tests.orchestrator - INFO - PVC test-pvc-595330e0 is bound +2025-06-03 17:58:24,446 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario +2025-06-03 17:58:24,446 - tests.orchestrator - INFO - Running specific scenario: _scenario_many_to_one +2025-06-03 17:58:24,446 - tests.orchestrator - INFO - Running scenario: Many pods to one PVC +2025-06-03 17:58:24,530 - tests.orchestrator - INFO - Waiting for PVC many2one-8a476379 to be bound +2025-06-03 17:58:26,691 - tests.orchestrator - INFO - PVC many2one-8a476379 is bound +2025-06-03 17:58:26,691 - tests.orchestrator - INFO - Creating 4 pods for the same PVC +2025-06-03 17:58:26,691 - tests.orchestrator - INFO - Creating pod test-pod-7ca8c26c on PVC many2one-8a476379 +2025-06-03 17:58:26,797 - tests.orchestrator - INFO - Created pod: test-pod-7ca8c26c using PVC: many2one-8a476379 +2025-06-03 17:58:26,797 - tests.orchestrator - INFO - Waiting for pod test-pod-7ca8c26c to be ready +2025-06-03 17:58:39,374 - tests.orchestrator - INFO - Pod test-pod-7ca8c26c is ready +2025-06-03 17:58:39,375 - tests.orchestrator - INFO - Creating pod test-pod-f9ae1d98 on PVC many2one-8a476379 +2025-06-03 17:58:39,463 - tests.orchestrator - INFO - Created pod: test-pod-f9ae1d98 using PVC: many2one-8a476379 +2025-06-03 17:58:39,463 - tests.orchestrator - INFO - Waiting for pod test-pod-f9ae1d98 to be ready +2025-06-03 17:58:52,156 - tests.orchestrator - INFO - Pod test-pod-f9ae1d98 is ready +2025-06-03 17:58:52,156 - tests.orchestrator - INFO - Creating pod test-pod-c048c330 on PVC many2one-8a476379 +2025-06-03 17:58:52,257 - tests.orchestrator - INFO - Created pod: test-pod-c048c330 using PVC: many2one-8a476379 +2025-06-03 17:58:52,257 - tests.orchestrator - INFO - Waiting for pod test-pod-c048c330 to be ready +2025-06-03 17:59:04,830 - tests.orchestrator - INFO - Pod test-pod-c048c330 is ready +2025-06-03 17:59:04,830 - tests.orchestrator - INFO - Creating pod test-pod-2414b40d on PVC many2one-8a476379 +2025-06-03 17:59:04,917 - tests.orchestrator - INFO - Created pod: test-pod-2414b40d using PVC: many2one-8a476379 +2025-06-03 17:59:04,917 - tests.orchestrator - INFO - Waiting for pod test-pod-2414b40d to be ready +2025-06-03 17:59:17,501 - tests.orchestrator - INFO - Pod test-pod-2414b40d is ready +2025-06-03 17:59:17,585 - tests.orchestrator - ERROR - Error in many-to-one scenario: (400) +Reason: Bad Request +HTTP response headers: HTTPHeaderDict({'Audit-Id': 'b1a66ba3-d997-4bf0-9a5c-eb8d855e80b5', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Tue, 03 Jun 2025 17:59:17 GMT', 'Content-Length': '139'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Upgrade request required","reason":"BadRequest","code":400} + + +2025-06-03 17:59:18,585 - tests.orchestrator - INFO - Test completed in 57.80 seconds +2025-06-03 17:59:18,586 - tests.orchestrator - INFO - Cleaning up resources... +2025-06-03 17:59:18,586 - tests.orchestrator - INFO - Deleting pod: test-pod-7ca8c26c from PVC: many2one-8a476379 +2025-06-03 17:59:18,675 - tests.orchestrator - INFO - Waiting for pod test-pod-7ca8c26c to be deleted +2025-06-03 17:59:51,422 - tests.orchestrator - INFO - Pod test-pod-7ca8c26c has been deleted +2025-06-03 17:59:51,423 - tests.orchestrator - INFO - Deleted pod: test-pod-7ca8c26c +2025-06-03 17:59:51,423 - tests.orchestrator - INFO - Deleting pod: test-pod-f9ae1d98 from PVC: many2one-8a476379 +2025-06-03 17:59:51,528 - tests.orchestrator - INFO - Waiting for pod test-pod-f9ae1d98 to be deleted +2025-06-03 18:00:24,370 - tests.orchestrator - INFO - Pod test-pod-f9ae1d98 has been deleted +2025-06-03 18:00:24,370 - tests.orchestrator - INFO - Deleted pod: test-pod-f9ae1d98 +2025-06-03 18:00:24,370 - tests.orchestrator - INFO - Deleting pod: test-pod-c048c330 from PVC: many2one-8a476379 +2025-06-03 18:00:24,459 - tests.orchestrator - INFO - Waiting for pod test-pod-c048c330 to be deleted +2025-06-03 18:00:57,202 - tests.orchestrator - INFO - Pod test-pod-c048c330 has been deleted +2025-06-03 18:00:57,203 - tests.orchestrator - INFO - Deleted pod: test-pod-c048c330 +2025-06-03 18:00:57,203 - tests.orchestrator - INFO - Deleting pod: test-pod-2414b40d from PVC: many2one-8a476379 +2025-06-03 18:00:57,295 - tests.orchestrator - INFO - Waiting for pod test-pod-2414b40d to be deleted +2025-06-03 18:01:30,035 - tests.orchestrator - INFO - Pod test-pod-2414b40d has been deleted +2025-06-03 18:01:30,035 - tests.orchestrator - INFO - Deleted pod: test-pod-2414b40d +2025-06-03 18:01:30,035 - tests.orchestrator - INFO - Deleting PVC: test-pvc-595330e0 +2025-06-03 18:01:30,127 - tests.orchestrator - INFO - Waiting for PVC test-pvc-595330e0 to be deleted +2025-06-03 18:01:30,207 - tests.orchestrator - INFO - PVC test-pvc-595330e0 has been deleted +2025-06-03 18:01:30,207 - tests.orchestrator - INFO - Deleted PVC: test-pvc-595330e0 +2025-06-03 18:01:30,207 - tests.orchestrator - INFO - Deleting PVC: many2one-8a476379 +2025-06-03 18:01:30,298 - tests.orchestrator - INFO - Waiting for PVC many2one-8a476379 to be deleted +2025-06-03 18:01:30,377 - tests.orchestrator - INFO - PVC many2one-8a476379 has been deleted +2025-06-03 18:01:30,377 - tests.orchestrator - INFO - Deleted PVC: many2one-8a476379 +2025-06-03 18:01:30,377 - tests.orchestrator - INFO - Cleanup completed +2025-06-03 18:01:30,377 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-03 18:01:30,377 - tests.orchestrator - INFO - --- Operations --- +2025-06-03 18:01:30,377 - tests.orchestrator - INFO - create_pvc: 1 succeeded, 0 failed (100.0%) +2025-06-03 18:01:30,377 - tests.orchestrator - INFO - attach_pod: 4 succeeded, 0 failed (100.0%) +2025-06-03 18:01:30,377 - tests.orchestrator - INFO - delete_pod: 4 succeeded, 0 failed (100.0%) +2025-06-03 18:01:30,377 - tests.orchestrator - INFO - delete_pvc: 2 succeeded, 0 failed (100.0%) +2025-06-03 18:01:30,377 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 0 failed (0.0%) +2025-06-03 18:01:30,377 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 0 failed (0.0%) +2025-06-03 18:01:30,377 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-03 18:01:30,377 - tests.orchestrator - INFO - shared_volume_rw: No runs +2025-06-03 18:01:30,378 - tests.orchestrator - INFO - many_to_one: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-03 18:01:30,378 - tests.orchestrator - INFO - one_to_one: No runs +2025-06-03 18:01:30,378 - tests.orchestrator - INFO - concurrent_pvc: No runs +2025-06-03 18:01:30,378 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250603_204418.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250603_204418.log new file mode 100644 index 000000000..55b365575 --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250603_204418.log @@ -0,0 +1,8 @@ +2025-06-03 20:44:18,852 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-03 20:44:18,853 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-03 20:44:19,110 - tests.orchestrator - ERROR - Error checking StorageClass: (401) +Reason: Unauthorized +HTTP response headers: HTTPHeaderDict({'Audit-Id': '75ffdf3f-c989-47e5-afa5-81c57f7481c8', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Tue, 03 Jun 2025 20:44:19 GMT', 'Content-Length': '129'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} + + diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250603_204433.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250603_204433.log new file mode 100644 index 000000000..6a692a266 --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250603_204433.log @@ -0,0 +1,94 @@ +2025-06-03 20:44:33,955 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-03 20:44:33,956 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-03 20:44:34,200 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-03 20:44:34,200 - tests.orchestrator - INFO - Selected operation: _attach_pod +2025-06-03 20:44:34,201 - tests.orchestrator - INFO - No PVCs available, skipping pod attachment +2025-06-03 20:44:35,201 - tests.orchestrator - INFO - Selected operation: _verify_readwrite +2025-06-03 20:44:35,201 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test +2025-06-03 20:44:36,201 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario +2025-06-03 20:44:36,201 - tests.orchestrator - INFO - Running specific scenario: _scenario_concurrent_pvc +2025-06-03 20:44:36,201 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations +2025-06-03 20:44:36,202 - tests.orchestrator - INFO - Creating 4 PVCs in quick succession +2025-06-03 20:44:36,496 - tests.orchestrator - ERROR - Failed to create PVC concurrent-pvc-de3117b2 concurrently: (404) +Reason: Not Found +HTTP response headers: HTTPHeaderDict({'Audit-Id': '6de3c195-b2f2-4b32-8ecf-97264e52a31c', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'X-Kubernetes-Pf-Flowschema-Uid': 'a278f63c-a854-47c8-82d3-d9efaaac2f24', 'X-Kubernetes-Pf-Prioritylevel-Uid': '6f082086-057e-4745-9def-b60972ad437b', 'Date': 'Tue, 03 Jun 2025 20:44:36 GMT', 'Content-Length': '210'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"namespaces \"efs-stress-test\" not found","reason":"NotFound","details":{"name":"efs-stress-test","kind":"namespaces"},"code":404} + + +2025-06-03 20:44:36,629 - tests.orchestrator - ERROR - Failed to create PVC concurrent-pvc-85b7f827 concurrently: (404) +Reason: Not Found +HTTP response headers: HTTPHeaderDict({'Audit-Id': 'c0386aa9-45b5-4f3c-a98b-030be2caf4ee', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'X-Kubernetes-Pf-Flowschema-Uid': 'a278f63c-a854-47c8-82d3-d9efaaac2f24', 'X-Kubernetes-Pf-Prioritylevel-Uid': '6f082086-057e-4745-9def-b60972ad437b', 'Date': 'Tue, 03 Jun 2025 20:44:36 GMT', 'Content-Length': '210'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"namespaces \"efs-stress-test\" not found","reason":"NotFound","details":{"name":"efs-stress-test","kind":"namespaces"},"code":404} + + +2025-06-03 20:44:36,761 - tests.orchestrator - ERROR - Failed to create PVC concurrent-pvc-0fd0fcb2 concurrently: (404) +Reason: Not Found +HTTP response headers: HTTPHeaderDict({'Audit-Id': '8b45ed55-a0d9-42cf-8f9e-49802dd09685', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'X-Kubernetes-Pf-Flowschema-Uid': 'a278f63c-a854-47c8-82d3-d9efaaac2f24', 'X-Kubernetes-Pf-Prioritylevel-Uid': '6f082086-057e-4745-9def-b60972ad437b', 'Date': 'Tue, 03 Jun 2025 20:44:36 GMT', 'Content-Length': '210'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"namespaces \"efs-stress-test\" not found","reason":"NotFound","details":{"name":"efs-stress-test","kind":"namespaces"},"code":404} + + +2025-06-03 20:44:36,894 - tests.orchestrator - ERROR - Failed to create PVC concurrent-pvc-70957dd1 concurrently: (404) +Reason: Not Found +HTTP response headers: HTTPHeaderDict({'Audit-Id': 'eb053268-dd23-487f-a943-fe937d3e637c', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'X-Kubernetes-Pf-Flowschema-Uid': 'a278f63c-a854-47c8-82d3-d9efaaac2f24', 'X-Kubernetes-Pf-Prioritylevel-Uid': '6f082086-057e-4745-9def-b60972ad437b', 'Date': 'Tue, 03 Jun 2025 20:44:36 GMT', 'Content-Length': '210'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"namespaces \"efs-stress-test\" not found","reason":"NotFound","details":{"name":"efs-stress-test","kind":"namespaces"},"code":404} + + +2025-06-03 20:44:36,894 - tests.orchestrator - WARNING - Failed to create enough PVCs, only created 0 +2025-06-03 20:44:37,895 - tests.orchestrator - INFO - Selected operation: _attach_pod +2025-06-03 20:44:37,895 - tests.orchestrator - INFO - No PVCs available, skipping pod attachment +2025-06-03 20:44:38,895 - tests.orchestrator - INFO - Selected operation: _delete_pvc +2025-06-03 20:44:38,895 - tests.orchestrator - INFO - No PVCs to delete +2025-06-03 20:44:39,895 - tests.orchestrator - INFO - Selected operation: _create_pvc +2025-06-03 20:44:39,896 - tests.orchestrator - INFO - Creating PVC: test-pvc-a59e84da +2025-06-03 20:44:40,030 - tests.orchestrator - ERROR - Failed to create PVC: (404) +Reason: Not Found +HTTP response headers: HTTPHeaderDict({'Audit-Id': '024a4209-6aa7-489a-8bef-e93376c42c84', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'X-Kubernetes-Pf-Flowschema-Uid': 'a278f63c-a854-47c8-82d3-d9efaaac2f24', 'X-Kubernetes-Pf-Prioritylevel-Uid': '6f082086-057e-4745-9def-b60972ad437b', 'Date': 'Tue, 03 Jun 2025 20:44:39 GMT', 'Content-Length': '210'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"namespaces \"efs-stress-test\" not found","reason":"NotFound","details":{"name":"efs-stress-test","kind":"namespaces"},"code":404} + + +2025-06-03 20:44:41,030 - tests.orchestrator - INFO - Selected operation: _delete_pvc +2025-06-03 20:44:41,030 - tests.orchestrator - INFO - No PVCs to delete +2025-06-03 20:44:42,030 - tests.orchestrator - INFO - Selected operation: _create_pvc +2025-06-03 20:44:42,031 - tests.orchestrator - INFO - Creating PVC: test-pvc-f2989ffe +2025-06-03 20:44:42,163 - tests.orchestrator - ERROR - Failed to create PVC: (404) +Reason: Not Found +HTTP response headers: HTTPHeaderDict({'Audit-Id': 'baeae3c9-acf2-4389-8ecd-6a83618aecd7', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'X-Kubernetes-Pf-Flowschema-Uid': 'a278f63c-a854-47c8-82d3-d9efaaac2f24', 'X-Kubernetes-Pf-Prioritylevel-Uid': '6f082086-057e-4745-9def-b60972ad437b', 'Date': 'Tue, 03 Jun 2025 20:44:42 GMT', 'Content-Length': '210'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"namespaces \"efs-stress-test\" not found","reason":"NotFound","details":{"name":"efs-stress-test","kind":"namespaces"},"code":404} + + +2025-06-03 20:44:43,164 - tests.orchestrator - INFO - Selected operation: _delete_pvc +2025-06-03 20:44:43,164 - tests.orchestrator - INFO - No PVCs to delete +2025-06-03 20:44:44,164 - tests.orchestrator - INFO - Selected operation: _create_pvc +2025-06-03 20:44:44,164 - tests.orchestrator - INFO - Creating PVC: test-pvc-53fee8c6 +2025-06-03 20:44:44,298 - tests.orchestrator - ERROR - Failed to create PVC: (404) +Reason: Not Found +HTTP response headers: HTTPHeaderDict({'Audit-Id': 'df53a974-5bea-4379-89b2-408d521a999c', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'X-Kubernetes-Pf-Flowschema-Uid': 'a278f63c-a854-47c8-82d3-d9efaaac2f24', 'X-Kubernetes-Pf-Prioritylevel-Uid': '6f082086-057e-4745-9def-b60972ad437b', 'Date': 'Tue, 03 Jun 2025 20:44:44 GMT', 'Content-Length': '210'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"namespaces \"efs-stress-test\" not found","reason":"NotFound","details":{"name":"efs-stress-test","kind":"namespaces"},"code":404} + + +2025-06-03 20:44:45,298 - tests.orchestrator - INFO - Selected operation: _delete_pod +2025-06-03 20:44:45,299 - tests.orchestrator - INFO - No pods to delete +2025-06-03 20:44:46,299 - tests.orchestrator - INFO - Selected operation: _delete_pod +2025-06-03 20:44:46,299 - tests.orchestrator - INFO - No pods to delete +2025-06-03 20:44:47,299 - tests.orchestrator - INFO - Selected operation: _delete_pod +2025-06-03 20:44:47,299 - tests.orchestrator - INFO - No pods to delete +2025-06-03 20:44:48,299 - tests.orchestrator - INFO - Selected operation: _delete_pod +2025-06-03 20:44:48,300 - tests.orchestrator - INFO - No pods to delete +2025-06-03 20:44:48,747 - tests.orchestrator - INFO - Test interrupted by user +2025-06-03 20:44:48,747 - tests.orchestrator - INFO - Test completed in 14.79 seconds +2025-06-03 20:44:48,747 - tests.orchestrator - INFO - Cleaning up resources... +2025-06-03 20:44:48,747 - tests.orchestrator - INFO - Cleanup completed +2025-06-03 20:44:48,747 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-03 20:44:48,747 - tests.orchestrator - INFO - --- Operations --- +2025-06-03 20:44:48,747 - tests.orchestrator - INFO - create_pvc: 0 succeeded, 7 failed (0.0%) +2025-06-03 20:44:48,747 - tests.orchestrator - INFO - attach_pod: 0 succeeded, 0 failed (0.0%) +2025-06-03 20:44:48,747 - tests.orchestrator - INFO - delete_pod: 0 succeeded, 0 failed (0.0%) +2025-06-03 20:44:48,747 - tests.orchestrator - INFO - delete_pvc: 0 succeeded, 0 failed (0.0%) +2025-06-03 20:44:48,747 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 0 failed (0.0%) +2025-06-03 20:44:48,747 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 0 failed (0.0%) +2025-06-03 20:44:48,748 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-03 20:44:48,748 - tests.orchestrator - INFO - shared_volume_rw: No runs +2025-06-03 20:44:48,748 - tests.orchestrator - INFO - many_to_one: No runs +2025-06-03 20:44:48,748 - tests.orchestrator - INFO - one_to_one: No runs +2025-06-03 20:44:48,748 - tests.orchestrator - INFO - concurrent_pvc: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-03 20:44:48,748 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250603_204821.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250603_204821.log new file mode 100644 index 000000000..e95a6a2ed --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250603_204821.log @@ -0,0 +1,175 @@ +2025-06-03 20:48:21,427 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-03 20:48:21,428 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-03 20:48:21,428 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-03 20:48:21,428 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-03 20:48:21,428 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-03 20:48:21,663 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-03 20:48:21,663 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-03 20:48:21,663 - tests.orchestrator - INFO - Selected operation: _attach_pod +2025-06-03 20:48:21,663 - tests.orchestrator - INFO - Selected operation: _attach_pod +2025-06-03 20:48:21,663 - tests.orchestrator - INFO - No PVCs available, skipping pod attachment +2025-06-03 20:48:21,663 - tests.orchestrator - INFO - No PVCs available, skipping pod attachment +2025-06-03 20:48:22,663 - tests.orchestrator - INFO - Selected operation: _create_pvc +2025-06-03 20:48:22,663 - tests.orchestrator - INFO - Selected operation: _create_pvc +2025-06-03 20:48:22,664 - tests.orchestrator - INFO - Creating PVC: test-pvc-b929a758 +2025-06-03 20:48:22,664 - tests.orchestrator - INFO - Creating PVC: test-pvc-b929a758 +2025-06-03 20:48:22,756 - tests.orchestrator - INFO - Created PVC: test-pvc-b929a758 +2025-06-03 20:48:22,756 - tests.orchestrator - INFO - Created PVC: test-pvc-b929a758 +2025-06-03 20:48:22,756 - tests.orchestrator - INFO - Waiting for PVC test-pvc-b929a758 to be bound +2025-06-03 20:48:22,756 - tests.orchestrator - INFO - Waiting for PVC test-pvc-b929a758 to be bound +2025-06-03 20:48:24,919 - tests.orchestrator - INFO - PVC test-pvc-b929a758 is bound +2025-06-03 20:48:24,919 - tests.orchestrator - INFO - PVC test-pvc-b929a758 is bound +2025-06-03 20:48:25,919 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario +2025-06-03 20:48:25,919 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario +2025-06-03 20:48:25,919 - tests.orchestrator - INFO - Running specific scenario: _scenario_many_to_one +2025-06-03 20:48:25,919 - tests.orchestrator - INFO - Running specific scenario: _scenario_many_to_one +2025-06-03 20:48:25,920 - tests.orchestrator - INFO - Running scenario: Many pods to one PVC +2025-06-03 20:48:25,920 - tests.orchestrator - INFO - Running scenario: Many pods to one PVC +2025-06-03 20:48:26,002 - tests.orchestrator - INFO - Waiting for PVC many2one-5e21f77a to be bound +2025-06-03 20:48:26,002 - tests.orchestrator - INFO - Waiting for PVC many2one-5e21f77a to be bound +2025-06-03 20:48:28,166 - tests.orchestrator - INFO - PVC many2one-5e21f77a is bound +2025-06-03 20:48:28,166 - tests.orchestrator - INFO - PVC many2one-5e21f77a is bound +2025-06-03 20:48:28,167 - tests.orchestrator - INFO - Creating 5 pods for the same PVC +2025-06-03 20:48:28,167 - tests.orchestrator - INFO - Creating 5 pods for the same PVC +2025-06-03 20:48:28,167 - tests.orchestrator - INFO - Creating pod test-pod-63513a27 on PVC many2one-5e21f77a +2025-06-03 20:48:28,167 - tests.orchestrator - INFO - Creating pod test-pod-63513a27 on PVC many2one-5e21f77a +2025-06-03 20:48:28,261 - tests.orchestrator - INFO - Created pod: test-pod-63513a27 using PVC: many2one-5e21f77a +2025-06-03 20:48:28,261 - tests.orchestrator - INFO - Created pod: test-pod-63513a27 using PVC: many2one-5e21f77a +2025-06-03 20:48:28,262 - tests.orchestrator - INFO - Waiting for pod test-pod-63513a27 to be ready +2025-06-03 20:48:28,262 - tests.orchestrator - INFO - Waiting for pod test-pod-63513a27 to be ready +2025-06-03 20:49:28,686 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-63513a27 to be ready after 60s +2025-06-03 20:49:28,686 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-63513a27 to be ready after 60s +2025-06-03 20:49:28,687 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-63513a27 to be ready +2025-06-03 20:49:28,687 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-63513a27 to be ready +2025-06-03 20:49:28,687 - tests.orchestrator - INFO - Creating pod test-pod-2c5c5642 on PVC many2one-5e21f77a +2025-06-03 20:49:28,687 - tests.orchestrator - INFO - Creating pod test-pod-2c5c5642 on PVC many2one-5e21f77a +2025-06-03 20:49:28,783 - tests.orchestrator - INFO - Created pod: test-pod-2c5c5642 using PVC: many2one-5e21f77a +2025-06-03 20:49:28,783 - tests.orchestrator - INFO - Created pod: test-pod-2c5c5642 using PVC: many2one-5e21f77a +2025-06-03 20:49:28,783 - tests.orchestrator - INFO - Waiting for pod test-pod-2c5c5642 to be ready +2025-06-03 20:49:28,783 - tests.orchestrator - INFO - Waiting for pod test-pod-2c5c5642 to be ready +2025-06-03 20:50:29,204 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-2c5c5642 to be ready after 60s +2025-06-03 20:50:29,204 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-2c5c5642 to be ready after 60s +2025-06-03 20:50:29,204 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-2c5c5642 to be ready +2025-06-03 20:50:29,204 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-2c5c5642 to be ready +2025-06-03 20:50:29,204 - tests.orchestrator - INFO - Creating pod test-pod-9d6a21f8 on PVC many2one-5e21f77a +2025-06-03 20:50:29,204 - tests.orchestrator - INFO - Creating pod test-pod-9d6a21f8 on PVC many2one-5e21f77a +2025-06-03 20:50:29,299 - tests.orchestrator - INFO - Created pod: test-pod-9d6a21f8 using PVC: many2one-5e21f77a +2025-06-03 20:50:29,299 - tests.orchestrator - INFO - Created pod: test-pod-9d6a21f8 using PVC: many2one-5e21f77a +2025-06-03 20:50:29,299 - tests.orchestrator - INFO - Waiting for pod test-pod-9d6a21f8 to be ready +2025-06-03 20:50:29,299 - tests.orchestrator - INFO - Waiting for pod test-pod-9d6a21f8 to be ready +2025-06-03 20:51:29,793 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9d6a21f8 to be ready after 60s +2025-06-03 20:51:29,793 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9d6a21f8 to be ready after 60s +2025-06-03 20:51:29,794 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9d6a21f8 to be ready +2025-06-03 20:51:29,794 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9d6a21f8 to be ready +2025-06-03 20:51:29,794 - tests.orchestrator - INFO - Creating pod test-pod-1bd90f06 on PVC many2one-5e21f77a +2025-06-03 20:51:29,794 - tests.orchestrator - INFO - Creating pod test-pod-1bd90f06 on PVC many2one-5e21f77a +2025-06-03 20:51:29,892 - tests.orchestrator - INFO - Created pod: test-pod-1bd90f06 using PVC: many2one-5e21f77a +2025-06-03 20:51:29,892 - tests.orchestrator - INFO - Created pod: test-pod-1bd90f06 using PVC: many2one-5e21f77a +2025-06-03 20:51:29,893 - tests.orchestrator - INFO - Waiting for pod test-pod-1bd90f06 to be ready +2025-06-03 20:51:29,893 - tests.orchestrator - INFO - Waiting for pod test-pod-1bd90f06 to be ready +2025-06-03 20:52:30,297 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1bd90f06 to be ready after 60s +2025-06-03 20:52:30,297 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1bd90f06 to be ready after 60s +2025-06-03 20:52:30,297 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1bd90f06 to be ready +2025-06-03 20:52:30,297 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1bd90f06 to be ready +2025-06-03 20:52:30,297 - tests.orchestrator - INFO - Creating pod test-pod-cb8a2172 on PVC many2one-5e21f77a +2025-06-03 20:52:30,297 - tests.orchestrator - INFO - Creating pod test-pod-cb8a2172 on PVC many2one-5e21f77a +2025-06-03 20:52:30,394 - tests.orchestrator - INFO - Created pod: test-pod-cb8a2172 using PVC: many2one-5e21f77a +2025-06-03 20:52:30,394 - tests.orchestrator - INFO - Created pod: test-pod-cb8a2172 using PVC: many2one-5e21f77a +2025-06-03 20:52:30,395 - tests.orchestrator - INFO - Waiting for pod test-pod-cb8a2172 to be ready +2025-06-03 20:52:30,395 - tests.orchestrator - INFO - Waiting for pod test-pod-cb8a2172 to be ready +2025-06-03 20:53:30,865 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-cb8a2172 to be ready after 60s +2025-06-03 20:53:30,865 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-cb8a2172 to be ready after 60s +2025-06-03 20:53:30,866 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-cb8a2172 to be ready +2025-06-03 20:53:30,866 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-cb8a2172 to be ready +2025-06-03 20:53:30,866 - tests.orchestrator - WARNING - Failed to create enough pods for scenario, only created 0 +2025-06-03 20:53:30,866 - tests.orchestrator - WARNING - Failed to create enough pods for scenario, only created 0 +2025-06-03 20:53:31,866 - tests.orchestrator - INFO - Test completed in 310.44 seconds +2025-06-03 20:53:31,866 - tests.orchestrator - INFO - Test completed in 310.44 seconds +2025-06-03 20:53:31,866 - tests.orchestrator - INFO - Cleaning up resources... +2025-06-03 20:53:31,866 - tests.orchestrator - INFO - Cleaning up resources... +2025-06-03 20:53:31,866 - tests.orchestrator - INFO - Deleting pod: test-pod-63513a27 from PVC: many2one-5e21f77a +2025-06-03 20:53:31,866 - tests.orchestrator - INFO - Deleting pod: test-pod-63513a27 from PVC: many2one-5e21f77a +2025-06-03 20:53:31,978 - tests.orchestrator - INFO - Waiting for pod test-pod-63513a27 to be deleted +2025-06-03 20:53:31,978 - tests.orchestrator - INFO - Waiting for pod test-pod-63513a27 to be deleted +2025-06-03 20:53:32,062 - tests.orchestrator - INFO - Pod test-pod-63513a27 has been deleted +2025-06-03 20:53:32,062 - tests.orchestrator - INFO - Pod test-pod-63513a27 has been deleted +2025-06-03 20:53:32,062 - tests.orchestrator - INFO - Deleted pod: test-pod-63513a27 +2025-06-03 20:53:32,062 - tests.orchestrator - INFO - Deleted pod: test-pod-63513a27 +2025-06-03 20:53:32,062 - tests.orchestrator - INFO - Deleting pod: test-pod-2c5c5642 from PVC: many2one-5e21f77a +2025-06-03 20:53:32,062 - tests.orchestrator - INFO - Deleting pod: test-pod-2c5c5642 from PVC: many2one-5e21f77a +2025-06-03 20:53:32,163 - tests.orchestrator - INFO - Waiting for pod test-pod-2c5c5642 to be deleted +2025-06-03 20:53:32,163 - tests.orchestrator - INFO - Waiting for pod test-pod-2c5c5642 to be deleted +2025-06-03 20:53:32,248 - tests.orchestrator - INFO - Pod test-pod-2c5c5642 has been deleted +2025-06-03 20:53:32,248 - tests.orchestrator - INFO - Pod test-pod-2c5c5642 has been deleted +2025-06-03 20:53:32,249 - tests.orchestrator - INFO - Deleted pod: test-pod-2c5c5642 +2025-06-03 20:53:32,249 - tests.orchestrator - INFO - Deleted pod: test-pod-2c5c5642 +2025-06-03 20:53:32,249 - tests.orchestrator - INFO - Deleting pod: test-pod-9d6a21f8 from PVC: many2one-5e21f77a +2025-06-03 20:53:32,249 - tests.orchestrator - INFO - Deleting pod: test-pod-9d6a21f8 from PVC: many2one-5e21f77a +2025-06-03 20:53:32,357 - tests.orchestrator - INFO - Waiting for pod test-pod-9d6a21f8 to be deleted +2025-06-03 20:53:32,357 - tests.orchestrator - INFO - Waiting for pod test-pod-9d6a21f8 to be deleted +2025-06-03 20:53:32,446 - tests.orchestrator - INFO - Pod test-pod-9d6a21f8 has been deleted +2025-06-03 20:53:32,446 - tests.orchestrator - INFO - Pod test-pod-9d6a21f8 has been deleted +2025-06-03 20:53:32,446 - tests.orchestrator - INFO - Deleted pod: test-pod-9d6a21f8 +2025-06-03 20:53:32,446 - tests.orchestrator - INFO - Deleted pod: test-pod-9d6a21f8 +2025-06-03 20:53:32,446 - tests.orchestrator - INFO - Deleting pod: test-pod-1bd90f06 from PVC: many2one-5e21f77a +2025-06-03 20:53:32,446 - tests.orchestrator - INFO - Deleting pod: test-pod-1bd90f06 from PVC: many2one-5e21f77a +2025-06-03 20:53:32,565 - tests.orchestrator - INFO - Waiting for pod test-pod-1bd90f06 to be deleted +2025-06-03 20:53:32,565 - tests.orchestrator - INFO - Waiting for pod test-pod-1bd90f06 to be deleted +2025-06-03 20:53:32,647 - tests.orchestrator - INFO - Pod test-pod-1bd90f06 has been deleted +2025-06-03 20:53:32,647 - tests.orchestrator - INFO - Pod test-pod-1bd90f06 has been deleted +2025-06-03 20:53:32,647 - tests.orchestrator - INFO - Deleted pod: test-pod-1bd90f06 +2025-06-03 20:53:32,647 - tests.orchestrator - INFO - Deleted pod: test-pod-1bd90f06 +2025-06-03 20:53:32,647 - tests.orchestrator - INFO - Deleting pod: test-pod-cb8a2172 from PVC: many2one-5e21f77a +2025-06-03 20:53:32,647 - tests.orchestrator - INFO - Deleting pod: test-pod-cb8a2172 from PVC: many2one-5e21f77a +2025-06-03 20:53:32,744 - tests.orchestrator - INFO - Waiting for pod test-pod-cb8a2172 to be deleted +2025-06-03 20:53:32,744 - tests.orchestrator - INFO - Waiting for pod test-pod-cb8a2172 to be deleted +2025-06-03 20:53:32,824 - tests.orchestrator - INFO - Pod test-pod-cb8a2172 has been deleted +2025-06-03 20:53:32,824 - tests.orchestrator - INFO - Pod test-pod-cb8a2172 has been deleted +2025-06-03 20:53:32,824 - tests.orchestrator - INFO - Deleted pod: test-pod-cb8a2172 +2025-06-03 20:53:32,824 - tests.orchestrator - INFO - Deleted pod: test-pod-cb8a2172 +2025-06-03 20:53:32,824 - tests.orchestrator - INFO - Deleting PVC: test-pvc-b929a758 +2025-06-03 20:53:32,824 - tests.orchestrator - INFO - Deleting PVC: test-pvc-b929a758 +2025-06-03 20:53:32,911 - tests.orchestrator - INFO - Waiting for PVC test-pvc-b929a758 to be deleted +2025-06-03 20:53:32,911 - tests.orchestrator - INFO - Waiting for PVC test-pvc-b929a758 to be deleted +2025-06-03 20:53:32,991 - tests.orchestrator - INFO - PVC test-pvc-b929a758 has been deleted +2025-06-03 20:53:32,991 - tests.orchestrator - INFO - PVC test-pvc-b929a758 has been deleted +2025-06-03 20:53:32,992 - tests.orchestrator - INFO - Deleted PVC: test-pvc-b929a758 +2025-06-03 20:53:32,992 - tests.orchestrator - INFO - Deleted PVC: test-pvc-b929a758 +2025-06-03 20:53:32,992 - tests.orchestrator - INFO - Deleting PVC: many2one-5e21f77a +2025-06-03 20:53:32,992 - tests.orchestrator - INFO - Deleting PVC: many2one-5e21f77a +2025-06-03 20:53:33,079 - tests.orchestrator - INFO - Waiting for PVC many2one-5e21f77a to be deleted +2025-06-03 20:53:33,079 - tests.orchestrator - INFO - Waiting for PVC many2one-5e21f77a to be deleted +2025-06-03 20:53:33,159 - tests.orchestrator - INFO - PVC many2one-5e21f77a has been deleted +2025-06-03 20:53:33,159 - tests.orchestrator - INFO - PVC many2one-5e21f77a has been deleted +2025-06-03 20:53:33,159 - tests.orchestrator - INFO - Deleted PVC: many2one-5e21f77a +2025-06-03 20:53:33,159 - tests.orchestrator - INFO - Deleted PVC: many2one-5e21f77a +2025-06-03 20:53:33,159 - tests.orchestrator - INFO - Cleanup completed +2025-06-03 20:53:33,159 - tests.orchestrator - INFO - Cleanup completed +2025-06-03 20:53:33,160 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-03 20:53:33,160 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-03 20:53:33,160 - tests.orchestrator - INFO - --- Operations --- +2025-06-03 20:53:33,160 - tests.orchestrator - INFO - --- Operations --- +2025-06-03 20:53:33,160 - tests.orchestrator - INFO - create_pvc: 1 succeeded, 0 failed (100.0%) +2025-06-03 20:53:33,160 - tests.orchestrator - INFO - create_pvc: 1 succeeded, 0 failed (100.0%) +2025-06-03 20:53:33,160 - tests.orchestrator - INFO - attach_pod: 5 succeeded, 0 failed (100.0%) +2025-06-03 20:53:33,160 - tests.orchestrator - INFO - attach_pod: 5 succeeded, 0 failed (100.0%) +2025-06-03 20:53:33,160 - tests.orchestrator - INFO - delete_pod: 5 succeeded, 0 failed (100.0%) +2025-06-03 20:53:33,160 - tests.orchestrator - INFO - delete_pod: 5 succeeded, 0 failed (100.0%) +2025-06-03 20:53:33,160 - tests.orchestrator - INFO - delete_pvc: 2 succeeded, 0 failed (100.0%) +2025-06-03 20:53:33,160 - tests.orchestrator - INFO - delete_pvc: 2 succeeded, 0 failed (100.0%) +2025-06-03 20:53:33,160 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 0 failed (0.0%) +2025-06-03 20:53:33,160 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 0 failed (0.0%) +2025-06-03 20:53:33,160 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 0 failed (0.0%) +2025-06-03 20:53:33,160 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 0 failed (0.0%) +2025-06-03 20:53:33,160 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-03 20:53:33,160 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-03 20:53:33,160 - tests.orchestrator - INFO - shared_volume_rw: No runs +2025-06-03 20:53:33,160 - tests.orchestrator - INFO - shared_volume_rw: No runs +2025-06-03 20:53:33,160 - tests.orchestrator - INFO - many_to_one: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-03 20:53:33,160 - tests.orchestrator - INFO - many_to_one: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-03 20:53:33,160 - tests.orchestrator - INFO - one_to_one: No runs +2025-06-03 20:53:33,160 - tests.orchestrator - INFO - one_to_one: No runs +2025-06-03 20:53:33,160 - tests.orchestrator - INFO - concurrent_pvc: No runs +2025-06-03 20:53:33,160 - tests.orchestrator - INFO - concurrent_pvc: No runs +2025-06-03 20:53:33,160 - tests.orchestrator - INFO - ========================================= +2025-06-03 20:53:33,160 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250603_210011.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250603_210011.log new file mode 100644 index 000000000..0bfa029ed --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250603_210011.log @@ -0,0 +1,158 @@ +2025-06-03 21:00:11,173 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-03 21:00:11,448 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-03 21:00:11,448 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-03 21:00:11,865 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-03 21:00:11,865 - tests.orchestrator - INFO - Selected operation: _attach_pod +2025-06-03 21:00:11,865 - tests.orchestrator - INFO - No PVCs available, skipping pod attachment +2025-06-03 21:00:12,865 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario +2025-06-03 21:00:12,865 - tests.orchestrator - INFO - Running specific scenario: _scenario_many_to_one +2025-06-03 21:00:12,866 - tests.orchestrator - INFO - Running scenario: Many pods to one PVC +2025-06-03 21:00:12,969 - tests.orchestrator - INFO - Waiting for PVC many2one-8ccf2b6f to be bound +2025-06-03 21:00:15,141 - tests.orchestrator - INFO - PVC many2one-8ccf2b6f is bound +2025-06-03 21:00:15,141 - tests.orchestrator - INFO - Creating 4 pods for the same PVC +2025-06-03 21:00:15,141 - tests.orchestrator - INFO - Creating pod test-pod-5c9bbadf on PVC many2one-8ccf2b6f +2025-06-03 21:00:15,239 - tests.orchestrator - INFO - Created pod: test-pod-5c9bbadf using PVC: many2one-8ccf2b6f +2025-06-03 21:00:15,239 - tests.orchestrator - INFO - Waiting for pod test-pod-5c9bbadf to be ready +2025-06-03 21:01:15,739 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-5c9bbadf to be ready after 60s +2025-06-03 21:01:15,740 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-5c9bbadf to be ready +2025-06-03 21:01:15,740 - tests.orchestrator - INFO - Creating pod test-pod-a795ee8f on PVC many2one-8ccf2b6f +2025-06-03 21:01:15,842 - tests.orchestrator - INFO - Created pod: test-pod-a795ee8f using PVC: many2one-8ccf2b6f +2025-06-03 21:01:15,842 - tests.orchestrator - INFO - Waiting for pod test-pod-a795ee8f to be ready +2025-06-03 21:02:16,279 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a795ee8f to be ready after 60s +2025-06-03 21:02:16,279 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a795ee8f to be ready +2025-06-03 21:02:16,279 - tests.orchestrator - INFO - Creating pod test-pod-bb687a78 on PVC many2one-8ccf2b6f +2025-06-03 21:02:16,379 - tests.orchestrator - INFO - Created pod: test-pod-bb687a78 using PVC: many2one-8ccf2b6f +2025-06-03 21:02:16,379 - tests.orchestrator - INFO - Waiting for pod test-pod-bb687a78 to be ready +2025-06-03 21:03:16,816 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-bb687a78 to be ready after 60s +2025-06-03 21:03:16,816 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-bb687a78 to be ready +2025-06-03 21:03:16,817 - tests.orchestrator - INFO - Creating pod test-pod-da23ad5a on PVC many2one-8ccf2b6f +2025-06-03 21:03:16,917 - tests.orchestrator - INFO - Created pod: test-pod-da23ad5a using PVC: many2one-8ccf2b6f +2025-06-03 21:03:16,917 - tests.orchestrator - INFO - Waiting for pod test-pod-da23ad5a to be ready +2025-06-03 21:04:17,450 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-da23ad5a to be ready after 60s +2025-06-03 21:04:17,450 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-da23ad5a to be ready +2025-06-03 21:04:17,450 - tests.orchestrator - WARNING - Failed to create enough pods for scenario, only created 0 +2025-06-03 21:04:18,450 - tests.orchestrator - INFO - Selected operation: _verify_readwrite +2025-06-03 21:04:18,451 - tests.orchestrator - INFO - Testing read/write between pods test-pod-a795ee8f and test-pod-5c9bbadf sharing PVC many2one-8ccf2b6f +2025-06-03 21:04:18,534 - tests.orchestrator - ERROR - Failed in read/write verification: (400) +Reason: Bad Request +HTTP response headers: HTTPHeaderDict({'Audit-Id': 'e5f54d7e-0b5d-44ba-9610-9f446b015a24', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Tue, 03 Jun 2025 21:04:18 GMT', 'Content-Length': '166'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-a795ee8f does not have a host assigned","reason":"BadRequest","code":400} + + +2025-06-03 21:04:19,534 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario +2025-06-03 21:04:19,534 - tests.orchestrator - INFO - Running specific scenario: _scenario_one_to_one +2025-06-03 21:04:19,534 - tests.orchestrator - INFO - Running scenario: One pod per PVC +2025-06-03 21:04:19,534 - tests.orchestrator - INFO - Creating 5 PVC-pod pairs +2025-06-03 21:04:19,631 - tests.orchestrator - INFO - Waiting for PVC one2one-c1de2935 to be bound +2025-06-03 21:04:21,797 - tests.orchestrator - INFO - PVC one2one-c1de2935 is bound +2025-06-03 21:04:21,797 - tests.orchestrator - INFO - Creating pod test-pod-c8cbcc77 on PVC one2one-c1de2935 +2025-06-03 21:04:21,894 - tests.orchestrator - INFO - Created pod: test-pod-c8cbcc77 using PVC: one2one-c1de2935 +2025-06-03 21:04:21,894 - tests.orchestrator - INFO - Waiting for pod test-pod-c8cbcc77 to be ready +2025-06-03 21:05:22,524 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-c8cbcc77 to be ready after 60s +2025-06-03 21:05:22,524 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-c8cbcc77 to be ready +2025-06-03 21:05:22,615 - tests.orchestrator - INFO - Waiting for PVC one2one-2f5447c5 to be bound +2025-06-03 21:05:24,783 - tests.orchestrator - INFO - PVC one2one-2f5447c5 is bound +2025-06-03 21:05:24,783 - tests.orchestrator - INFO - Creating pod test-pod-9c65d0a8 on PVC one2one-2f5447c5 +2025-06-03 21:05:24,879 - tests.orchestrator - INFO - Created pod: test-pod-9c65d0a8 using PVC: one2one-2f5447c5 +2025-06-03 21:05:24,879 - tests.orchestrator - INFO - Waiting for pod test-pod-9c65d0a8 to be ready +2025-06-03 21:06:25,353 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9c65d0a8 to be ready after 60s +2025-06-03 21:06:25,353 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9c65d0a8 to be ready +2025-06-03 21:06:25,454 - tests.orchestrator - INFO - Waiting for PVC one2one-65a70cda to be bound +2025-06-03 21:06:27,620 - tests.orchestrator - INFO - PVC one2one-65a70cda is bound +2025-06-03 21:06:27,621 - tests.orchestrator - INFO - Creating pod test-pod-396a98c7 on PVC one2one-65a70cda +2025-06-03 21:06:27,709 - tests.orchestrator - INFO - Created pod: test-pod-396a98c7 using PVC: one2one-65a70cda +2025-06-03 21:06:27,709 - tests.orchestrator - INFO - Waiting for pod test-pod-396a98c7 to be ready +2025-06-03 21:07:28,272 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-396a98c7 to be ready after 60s +2025-06-03 21:07:28,272 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-396a98c7 to be ready +2025-06-03 21:07:28,365 - tests.orchestrator - INFO - Waiting for PVC one2one-ac0c0034 to be bound +2025-06-03 21:07:30,531 - tests.orchestrator - INFO - PVC one2one-ac0c0034 is bound +2025-06-03 21:07:30,531 - tests.orchestrator - INFO - Creating pod test-pod-707ffe1e on PVC one2one-ac0c0034 +2025-06-03 21:07:30,620 - tests.orchestrator - INFO - Created pod: test-pod-707ffe1e using PVC: one2one-ac0c0034 +2025-06-03 21:07:30,620 - tests.orchestrator - INFO - Waiting for pod test-pod-707ffe1e to be ready +2025-06-03 21:08:31,077 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-707ffe1e to be ready after 60s +2025-06-03 21:08:31,077 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-707ffe1e to be ready +2025-06-03 21:08:31,174 - tests.orchestrator - INFO - Waiting for PVC one2one-6960322d to be bound +2025-06-03 21:08:33,340 - tests.orchestrator - INFO - PVC one2one-6960322d is bound +2025-06-03 21:08:33,340 - tests.orchestrator - INFO - Creating pod test-pod-db166953 on PVC one2one-6960322d +2025-06-03 21:08:33,433 - tests.orchestrator - INFO - Created pod: test-pod-db166953 using PVC: one2one-6960322d +2025-06-03 21:08:33,433 - tests.orchestrator - INFO - Waiting for pod test-pod-db166953 to be ready +2025-06-03 21:09:34,745 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-db166953 to be ready after 60s +2025-06-03 21:09:34,745 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-db166953 to be ready +2025-06-03 21:09:34,746 - tests.orchestrator - WARNING - Failed to create enough PVC-pod pairs, only created 0 +2025-06-03 21:09:35,746 - tests.orchestrator - INFO - Test completed in 564.30 seconds +2025-06-03 21:09:35,746 - tests.orchestrator - INFO - Cleaning up resources... +2025-06-03 21:09:35,746 - tests.orchestrator - INFO - Deleting pod: test-pod-5c9bbadf from PVC: many2one-8ccf2b6f +2025-06-03 21:09:35,842 - tests.orchestrator - INFO - Waiting for pod test-pod-5c9bbadf to be deleted +2025-06-03 21:09:35,926 - tests.orchestrator - INFO - Pod test-pod-5c9bbadf has been deleted +2025-06-03 21:09:35,926 - tests.orchestrator - INFO - Deleted pod: test-pod-5c9bbadf +2025-06-03 21:09:35,926 - tests.orchestrator - INFO - Deleting pod: test-pod-a795ee8f from PVC: many2one-8ccf2b6f +2025-06-03 21:09:36,025 - tests.orchestrator - INFO - Waiting for pod test-pod-a795ee8f to be deleted +2025-06-03 21:09:36,106 - tests.orchestrator - INFO - Pod test-pod-a795ee8f has been deleted +2025-06-03 21:09:36,106 - tests.orchestrator - INFO - Deleted pod: test-pod-a795ee8f +2025-06-03 21:09:36,106 - tests.orchestrator - INFO - Deleting pod: test-pod-bb687a78 from PVC: many2one-8ccf2b6f +2025-06-03 21:09:36,202 - tests.orchestrator - INFO - Waiting for pod test-pod-bb687a78 to be deleted +2025-06-03 21:09:36,284 - tests.orchestrator - INFO - Pod test-pod-bb687a78 has been deleted +2025-06-03 21:09:36,284 - tests.orchestrator - INFO - Deleted pod: test-pod-bb687a78 +2025-06-03 21:09:36,284 - tests.orchestrator - INFO - Deleting pod: test-pod-da23ad5a from PVC: many2one-8ccf2b6f +2025-06-03 21:09:36,386 - tests.orchestrator - INFO - Waiting for pod test-pod-da23ad5a to be deleted +2025-06-03 21:09:36,466 - tests.orchestrator - INFO - Pod test-pod-da23ad5a has been deleted +2025-06-03 21:09:36,466 - tests.orchestrator - INFO - Deleted pod: test-pod-da23ad5a +2025-06-03 21:09:36,466 - tests.orchestrator - INFO - Deleting pod: test-pod-c8cbcc77 from PVC: one2one-c1de2935 +2025-06-03 21:09:36,565 - tests.orchestrator - INFO - Waiting for pod test-pod-c8cbcc77 to be deleted +2025-06-03 21:09:36,651 - tests.orchestrator - INFO - Pod test-pod-c8cbcc77 has been deleted +2025-06-03 21:09:36,652 - tests.orchestrator - INFO - Deleted pod: test-pod-c8cbcc77 +2025-06-03 21:09:36,652 - tests.orchestrator - INFO - Deleting pod: test-pod-9c65d0a8 from PVC: one2one-2f5447c5 +2025-06-03 21:09:36,747 - tests.orchestrator - INFO - Waiting for pod test-pod-9c65d0a8 to be deleted +2025-06-03 21:09:36,832 - tests.orchestrator - INFO - Pod test-pod-9c65d0a8 has been deleted +2025-06-03 21:09:36,833 - tests.orchestrator - INFO - Deleted pod: test-pod-9c65d0a8 +2025-06-03 21:09:36,833 - tests.orchestrator - INFO - Deleting pod: test-pod-396a98c7 from PVC: one2one-65a70cda +2025-06-03 21:09:36,929 - tests.orchestrator - INFO - Waiting for pod test-pod-396a98c7 to be deleted +2025-06-03 21:09:37,009 - tests.orchestrator - INFO - Pod test-pod-396a98c7 has been deleted +2025-06-03 21:09:37,009 - tests.orchestrator - INFO - Deleted pod: test-pod-396a98c7 +2025-06-03 21:09:37,009 - tests.orchestrator - INFO - Deleting pod: test-pod-707ffe1e from PVC: one2one-ac0c0034 +2025-06-03 21:09:37,104 - tests.orchestrator - INFO - Waiting for pod test-pod-707ffe1e to be deleted +2025-06-03 21:09:37,187 - tests.orchestrator - INFO - Pod test-pod-707ffe1e has been deleted +2025-06-03 21:09:37,187 - tests.orchestrator - INFO - Deleted pod: test-pod-707ffe1e +2025-06-03 21:09:37,187 - tests.orchestrator - INFO - Deleting pod: test-pod-db166953 from PVC: one2one-6960322d +2025-06-03 21:09:37,284 - tests.orchestrator - INFO - Waiting for pod test-pod-db166953 to be deleted +2025-06-03 21:09:37,368 - tests.orchestrator - INFO - Pod test-pod-db166953 has been deleted +2025-06-03 21:09:37,368 - tests.orchestrator - INFO - Deleted pod: test-pod-db166953 +2025-06-03 21:09:37,368 - tests.orchestrator - INFO - Deleting PVC: many2one-8ccf2b6f +2025-06-03 21:09:37,456 - tests.orchestrator - INFO - Waiting for PVC many2one-8ccf2b6f to be deleted +2025-06-03 21:09:37,538 - tests.orchestrator - INFO - PVC many2one-8ccf2b6f has been deleted +2025-06-03 21:09:37,539 - tests.orchestrator - INFO - Deleted PVC: many2one-8ccf2b6f +2025-06-03 21:09:37,539 - tests.orchestrator - INFO - Deleting PVC: one2one-c1de2935 +2025-06-03 21:09:37,627 - tests.orchestrator - INFO - Waiting for PVC one2one-c1de2935 to be deleted +2025-06-03 21:09:37,711 - tests.orchestrator - INFO - PVC one2one-c1de2935 has been deleted +2025-06-03 21:09:37,711 - tests.orchestrator - INFO - Deleted PVC: one2one-c1de2935 +2025-06-03 21:09:37,711 - tests.orchestrator - INFO - Deleting PVC: one2one-2f5447c5 +2025-06-03 21:09:37,802 - tests.orchestrator - INFO - Waiting for PVC one2one-2f5447c5 to be deleted +2025-06-03 21:09:37,885 - tests.orchestrator - INFO - PVC one2one-2f5447c5 has been deleted +2025-06-03 21:09:37,886 - tests.orchestrator - INFO - Deleted PVC: one2one-2f5447c5 +2025-06-03 21:09:37,886 - tests.orchestrator - INFO - Deleting PVC: one2one-65a70cda +2025-06-03 21:09:37,973 - tests.orchestrator - INFO - Waiting for PVC one2one-65a70cda to be deleted +2025-06-03 21:09:38,059 - tests.orchestrator - INFO - PVC one2one-65a70cda has been deleted +2025-06-03 21:09:38,059 - tests.orchestrator - INFO - Deleted PVC: one2one-65a70cda +2025-06-03 21:09:38,059 - tests.orchestrator - INFO - Deleting PVC: one2one-ac0c0034 +2025-06-03 21:09:38,148 - tests.orchestrator - INFO - Waiting for PVC one2one-ac0c0034 to be deleted +2025-06-03 21:09:38,230 - tests.orchestrator - INFO - PVC one2one-ac0c0034 has been deleted +2025-06-03 21:09:38,231 - tests.orchestrator - INFO - Deleted PVC: one2one-ac0c0034 +2025-06-03 21:09:38,231 - tests.orchestrator - INFO - Deleting PVC: one2one-6960322d +2025-06-03 21:09:38,318 - tests.orchestrator - INFO - Waiting for PVC one2one-6960322d to be deleted +2025-06-03 21:09:38,401 - tests.orchestrator - INFO - PVC one2one-6960322d has been deleted +2025-06-03 21:09:38,401 - tests.orchestrator - INFO - Deleted PVC: one2one-6960322d +2025-06-03 21:09:38,401 - tests.orchestrator - INFO - Cleanup completed +2025-06-03 21:09:38,401 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-03 21:09:38,401 - tests.orchestrator - INFO - --- Operations --- +2025-06-03 21:09:38,401 - tests.orchestrator - INFO - create_pvc: 0 succeeded, 0 failed (0.0%) +2025-06-03 21:09:38,402 - tests.orchestrator - INFO - attach_pod: 9 succeeded, 0 failed (100.0%) +2025-06-03 21:09:38,402 - tests.orchestrator - INFO - delete_pod: 9 succeeded, 0 failed (100.0%) +2025-06-03 21:09:38,402 - tests.orchestrator - INFO - delete_pvc: 6 succeeded, 0 failed (100.0%) +2025-06-03 21:09:38,402 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) +2025-06-03 21:09:38,402 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) +2025-06-03 21:09:38,402 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-03 21:09:38,402 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-03 21:09:38,402 - tests.orchestrator - INFO - many_to_one: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-03 21:09:38,402 - tests.orchestrator - INFO - one_to_one: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-03 21:09:38,402 - tests.orchestrator - INFO - concurrent_pvc: No runs +2025-06-03 21:09:38,402 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250604_204301.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250604_204301.log new file mode 100644 index 000000000..75ab26ee1 --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250604_204301.log @@ -0,0 +1,125 @@ +2025-06-04 20:43:01,074 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-04 20:43:01,299 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-04 20:43:01,299 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-04 20:43:01,528 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-04 20:43:01,529 - tests.orchestrator - INFO - Selected operation: _verify_readwrite +2025-06-04 20:43:01,529 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test +2025-06-04 20:43:02,529 - tests.orchestrator - INFO - Selected operation: _attach_pod +2025-06-04 20:43:02,529 - tests.orchestrator - INFO - No PVCs available, skipping pod attachment +2025-06-04 20:43:03,529 - tests.orchestrator - INFO - Selected operation: _create_pvc +2025-06-04 20:43:03,530 - tests.orchestrator - INFO - Creating PVC: test-pvc-a8738d91 +2025-06-04 20:43:03,612 - tests.orchestrator - INFO - Created PVC: test-pvc-a8738d91 +2025-06-04 20:43:03,613 - tests.orchestrator - INFO - Waiting for PVC test-pvc-a8738d91 to be bound +2025-06-04 20:43:05,748 - tests.orchestrator - INFO - PVC test-pvc-a8738d91 is bound +2025-06-04 20:43:06,749 - tests.orchestrator - INFO - Selected operation: _create_pvc +2025-06-04 20:43:06,749 - tests.orchestrator - INFO - Creating PVC: test-pvc-ba881b7a +2025-06-04 20:43:06,823 - tests.orchestrator - INFO - Created PVC: test-pvc-ba881b7a +2025-06-04 20:43:06,824 - tests.orchestrator - INFO - Waiting for PVC test-pvc-ba881b7a to be bound +2025-06-04 20:43:08,959 - tests.orchestrator - INFO - PVC test-pvc-ba881b7a is bound +2025-06-04 20:43:09,959 - tests.orchestrator - INFO - Selected operation: _delete_pod +2025-06-04 20:43:09,959 - tests.orchestrator - INFO - No pods to delete +2025-06-04 20:43:10,959 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario +2025-06-04 20:43:10,960 - tests.orchestrator - INFO - Running specific scenario: _scenario_one_to_one +2025-06-04 20:43:10,960 - tests.orchestrator - INFO - Running scenario: One pod per PVC +2025-06-04 20:43:10,960 - tests.orchestrator - INFO - Creating 4 PVC-pod pairs +2025-06-04 20:43:11,032 - tests.orchestrator - INFO - Waiting for PVC one2one-43f0888a to be bound +2025-06-04 20:43:13,170 - tests.orchestrator - INFO - PVC one2one-43f0888a is bound +2025-06-04 20:43:13,170 - tests.orchestrator - INFO - Creating pod test-pod-f3b3a319 on PVC one2one-43f0888a +2025-06-04 20:43:13,279 - tests.orchestrator - INFO - Created pod: test-pod-f3b3a319 using PVC: one2one-43f0888a +2025-06-04 20:43:13,279 - tests.orchestrator - INFO - Waiting for pod test-pod-f3b3a319 to be ready +2025-06-04 20:44:15,334 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-f3b3a319 to be ready after 60s +2025-06-04 20:44:15,334 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-f3b3a319 to be ready +2025-06-04 20:44:15,411 - tests.orchestrator - INFO - Waiting for PVC one2one-aac689b5 to be bound +2025-06-04 20:44:17,545 - tests.orchestrator - INFO - PVC one2one-aac689b5 is bound +2025-06-04 20:44:17,545 - tests.orchestrator - INFO - Creating pod test-pod-80037404 on PVC one2one-aac689b5 +2025-06-04 20:44:17,620 - tests.orchestrator - INFO - Created pod: test-pod-80037404 using PVC: one2one-aac689b5 +2025-06-04 20:44:17,620 - tests.orchestrator - INFO - Waiting for pod test-pod-80037404 to be ready +2025-06-04 20:45:17,743 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-80037404 to be ready after 60s +2025-06-04 20:45:17,744 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-80037404 to be ready +2025-06-04 20:45:17,826 - tests.orchestrator - INFO - Waiting for PVC one2one-116b8e13 to be bound +2025-06-04 20:45:19,959 - tests.orchestrator - INFO - PVC one2one-116b8e13 is bound +2025-06-04 20:45:19,959 - tests.orchestrator - INFO - Creating pod test-pod-725477e8 on PVC one2one-116b8e13 +2025-06-04 20:45:20,032 - tests.orchestrator - INFO - Created pod: test-pod-725477e8 using PVC: one2one-116b8e13 +2025-06-04 20:45:20,032 - tests.orchestrator - INFO - Waiting for pod test-pod-725477e8 to be ready +2025-06-04 20:46:20,115 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-725477e8 to be ready after 60s +2025-06-04 20:46:20,116 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-725477e8 to be ready +2025-06-04 20:46:20,195 - tests.orchestrator - INFO - Waiting for PVC one2one-4e963a5b to be bound +2025-06-04 20:46:22,332 - tests.orchestrator - INFO - PVC one2one-4e963a5b is bound +2025-06-04 20:46:22,332 - tests.orchestrator - INFO - Creating pod test-pod-b778d0e5 on PVC one2one-4e963a5b +2025-06-04 20:46:22,407 - tests.orchestrator - INFO - Created pod: test-pod-b778d0e5 using PVC: one2one-4e963a5b +2025-06-04 20:46:22,408 - tests.orchestrator - INFO - Waiting for pod test-pod-b778d0e5 to be ready +2025-06-04 20:47:24,472 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b778d0e5 to be ready after 60s +2025-06-04 20:47:24,472 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b778d0e5 to be ready +2025-06-04 20:47:24,472 - tests.orchestrator - WARNING - Failed to create enough PVC-pod pairs, only created 0 +2025-06-04 20:47:25,473 - tests.orchestrator - INFO - Selected operation: _delete_pod +2025-06-04 20:47:25,473 - tests.orchestrator - INFO - Deleting pod: test-pod-725477e8 from PVC: one2one-116b8e13 +2025-06-04 20:47:25,551 - tests.orchestrator - INFO - Waiting for pod test-pod-725477e8 to be deleted +2025-06-04 20:47:25,616 - tests.orchestrator - INFO - Pod test-pod-725477e8 has been deleted +2025-06-04 20:47:25,616 - tests.orchestrator - INFO - Deleted pod: test-pod-725477e8 +2025-06-04 20:47:26,616 - tests.orchestrator - INFO - Selected operation: _delete_pod +2025-06-04 20:47:26,616 - tests.orchestrator - INFO - Deleting pod: test-pod-f3b3a319 from PVC: one2one-43f0888a +2025-06-04 20:47:26,699 - tests.orchestrator - INFO - Waiting for pod test-pod-f3b3a319 to be deleted +2025-06-04 20:47:26,766 - tests.orchestrator - INFO - Pod test-pod-f3b3a319 has been deleted +2025-06-04 20:47:26,766 - tests.orchestrator - INFO - Deleted pod: test-pod-f3b3a319 +2025-06-04 20:47:27,767 - tests.orchestrator - INFO - Selected operation: _delete_pod +2025-06-04 20:47:27,767 - tests.orchestrator - INFO - Deleting pod: test-pod-80037404 from PVC: one2one-aac689b5 +2025-06-04 20:47:27,853 - tests.orchestrator - INFO - Waiting for pod test-pod-80037404 to be deleted +2025-06-04 20:47:27,919 - tests.orchestrator - INFO - Pod test-pod-80037404 has been deleted +2025-06-04 20:47:27,919 - tests.orchestrator - INFO - Deleted pod: test-pod-80037404 +2025-06-04 20:47:28,919 - tests.orchestrator - INFO - Selected operation: _verify_readwrite +2025-06-04 20:47:28,920 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test +2025-06-04 20:47:29,920 - tests.orchestrator - INFO - Selected operation: _attach_pod +2025-06-04 20:47:29,920 - tests.orchestrator - INFO - Creating pod test-pod-715d3677 on PVC one2one-116b8e13 +2025-06-04 20:47:30,008 - tests.orchestrator - INFO - Created pod: test-pod-715d3677 using PVC: one2one-116b8e13 +2025-06-04 20:47:30,008 - tests.orchestrator - INFO - Waiting for pod test-pod-715d3677 to be ready +2025-06-04 20:48:32,062 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-715d3677 to be ready after 60s +2025-06-04 20:48:32,062 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-715d3677 to be ready +2025-06-04 20:48:33,062 - tests.orchestrator - INFO - Test completed in 331.76 seconds +2025-06-04 20:48:33,063 - tests.orchestrator - INFO - Cleaning up resources... +2025-06-04 20:48:33,063 - tests.orchestrator - INFO - Deleting pod: test-pod-715d3677 from PVC: one2one-116b8e13 +2025-06-04 20:48:33,144 - tests.orchestrator - INFO - Waiting for pod test-pod-715d3677 to be deleted +2025-06-04 20:48:33,208 - tests.orchestrator - INFO - Pod test-pod-715d3677 has been deleted +2025-06-04 20:48:33,209 - tests.orchestrator - INFO - Deleted pod: test-pod-715d3677 +2025-06-04 20:48:33,209 - tests.orchestrator - INFO - Deleting pod: test-pod-b778d0e5 from PVC: one2one-4e963a5b +2025-06-04 20:48:33,289 - tests.orchestrator - INFO - Waiting for pod test-pod-b778d0e5 to be deleted +2025-06-04 20:48:33,356 - tests.orchestrator - INFO - Pod test-pod-b778d0e5 has been deleted +2025-06-04 20:48:33,356 - tests.orchestrator - INFO - Deleted pod: test-pod-b778d0e5 +2025-06-04 20:48:33,356 - tests.orchestrator - INFO - Deleting PVC: test-pvc-a8738d91 +2025-06-04 20:48:33,430 - tests.orchestrator - INFO - Waiting for PVC test-pvc-a8738d91 to be deleted +2025-06-04 20:48:33,497 - tests.orchestrator - INFO - PVC test-pvc-a8738d91 has been deleted +2025-06-04 20:48:33,497 - tests.orchestrator - INFO - Deleted PVC: test-pvc-a8738d91 +2025-06-04 20:48:33,497 - tests.orchestrator - INFO - Deleting PVC: test-pvc-ba881b7a +2025-06-04 20:48:33,570 - tests.orchestrator - INFO - Waiting for PVC test-pvc-ba881b7a to be deleted +2025-06-04 20:48:33,637 - tests.orchestrator - INFO - PVC test-pvc-ba881b7a has been deleted +2025-06-04 20:48:33,637 - tests.orchestrator - INFO - Deleted PVC: test-pvc-ba881b7a +2025-06-04 20:48:33,637 - tests.orchestrator - INFO - Deleting PVC: one2one-43f0888a +2025-06-04 20:48:33,712 - tests.orchestrator - INFO - Waiting for PVC one2one-43f0888a to be deleted +2025-06-04 20:48:33,779 - tests.orchestrator - INFO - PVC one2one-43f0888a has been deleted +2025-06-04 20:48:33,780 - tests.orchestrator - INFO - Deleted PVC: one2one-43f0888a +2025-06-04 20:48:33,780 - tests.orchestrator - INFO - Deleting PVC: one2one-aac689b5 +2025-06-04 20:48:33,858 - tests.orchestrator - INFO - Waiting for PVC one2one-aac689b5 to be deleted +2025-06-04 20:48:33,925 - tests.orchestrator - INFO - PVC one2one-aac689b5 has been deleted +2025-06-04 20:48:33,925 - tests.orchestrator - INFO - Deleted PVC: one2one-aac689b5 +2025-06-04 20:48:33,925 - tests.orchestrator - INFO - Deleting PVC: one2one-116b8e13 +2025-06-04 20:48:34,000 - tests.orchestrator - INFO - Waiting for PVC one2one-116b8e13 to be deleted +2025-06-04 20:48:34,067 - tests.orchestrator - INFO - PVC one2one-116b8e13 has been deleted +2025-06-04 20:48:34,067 - tests.orchestrator - INFO - Deleted PVC: one2one-116b8e13 +2025-06-04 20:48:34,067 - tests.orchestrator - INFO - Deleting PVC: one2one-4e963a5b +2025-06-04 20:48:34,143 - tests.orchestrator - INFO - Waiting for PVC one2one-4e963a5b to be deleted +2025-06-04 20:48:34,209 - tests.orchestrator - INFO - PVC one2one-4e963a5b has been deleted +2025-06-04 20:48:34,210 - tests.orchestrator - INFO - Deleted PVC: one2one-4e963a5b +2025-06-04 20:48:34,210 - tests.orchestrator - INFO - Cleanup completed +2025-06-04 20:48:34,210 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-04 20:48:34,210 - tests.orchestrator - INFO - --- Operations --- +2025-06-04 20:48:34,210 - tests.orchestrator - INFO - create_pvc: 2 succeeded, 0 failed (100.0%) +2025-06-04 20:48:34,210 - tests.orchestrator - INFO - attach_pod: 5 succeeded, 0 failed (100.0%) +2025-06-04 20:48:34,210 - tests.orchestrator - INFO - delete_pod: 5 succeeded, 0 failed (100.0%) +2025-06-04 20:48:34,210 - tests.orchestrator - INFO - delete_pvc: 6 succeeded, 0 failed (100.0%) +2025-06-04 20:48:34,210 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 0 failed (0.0%) +2025-06-04 20:48:34,210 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 0 failed (0.0%) +2025-06-04 20:48:34,210 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-04 20:48:34,210 - tests.orchestrator - INFO - shared_volume_rw: No runs +2025-06-04 20:48:34,210 - tests.orchestrator - INFO - many_to_one: No runs +2025-06-04 20:48:34,210 - tests.orchestrator - INFO - one_to_one: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-04 20:48:34,210 - tests.orchestrator - INFO - concurrent_pvc: No runs +2025-06-04 20:48:34,210 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250604_211255.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250604_211255.log new file mode 100644 index 000000000..025920f41 --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250604_211255.log @@ -0,0 +1,43 @@ +2025-06-04 21:12:55,097 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-04 21:12:55,314 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-04 21:12:55,314 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-04 21:12:55,503 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-04 21:12:55,504 - tests.orchestrator - INFO - Selected operation: _delete_pod +2025-06-04 21:12:55,504 - tests.orchestrator - INFO - No pods to delete +2025-06-04 21:12:56,504 - tests.orchestrator - INFO - Selected operation: _verify_readwrite +2025-06-04 21:12:56,504 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test +2025-06-04 21:12:57,504 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario +2025-06-04 21:12:57,504 - tests.orchestrator - INFO - Running specific scenario: _scenario_one_to_one +2025-06-04 21:12:57,504 - tests.orchestrator - INFO - Running scenario: One pod per PVC +2025-06-04 21:12:57,505 - tests.orchestrator - INFO - Creating 4 PVC-pod pairs +2025-06-04 21:12:57,584 - tests.orchestrator - INFO - Waiting for PVC one2one-fa7d38c4 to be bound +2025-06-04 21:12:59,720 - tests.orchestrator - INFO - PVC one2one-fa7d38c4 is bound +2025-06-04 21:12:59,721 - tests.orchestrator - INFO - Creating pod test-pod-f3bd2110 on PVC one2one-fa7d38c4 +2025-06-04 21:12:59,822 - tests.orchestrator - INFO - Created pod: test-pod-f3bd2110 using PVC: one2one-fa7d38c4 +2025-06-04 21:12:59,822 - tests.orchestrator - INFO - Waiting for pod test-pod-f3bd2110 to be ready +2025-06-04 21:13:46,839 - tests.orchestrator - INFO - Test interrupted by user +2025-06-04 21:13:46,839 - tests.orchestrator - INFO - Test completed in 51.52 seconds +2025-06-04 21:13:46,839 - tests.orchestrator - INFO - Cleaning up resources... +2025-06-04 21:13:46,839 - tests.orchestrator - INFO - Deleting pod: test-pod-f3bd2110 from PVC: one2one-fa7d38c4 +2025-06-04 21:13:46,924 - tests.orchestrator - INFO - Waiting for pod test-pod-f3bd2110 to be deleted +2025-06-04 21:13:46,989 - tests.orchestrator - INFO - Pod test-pod-f3bd2110 has been deleted +2025-06-04 21:13:46,989 - tests.orchestrator - INFO - Deleted pod: test-pod-f3bd2110 +2025-06-04 21:13:46,989 - tests.orchestrator - INFO - Deleting PVC: one2one-fa7d38c4 +2025-06-04 21:13:47,063 - tests.orchestrator - INFO - Waiting for PVC one2one-fa7d38c4 to be deleted +2025-06-04 21:13:47,130 - tests.orchestrator - INFO - PVC one2one-fa7d38c4 has been deleted +2025-06-04 21:13:47,130 - tests.orchestrator - INFO - Deleted PVC: one2one-fa7d38c4 +2025-06-04 21:13:47,130 - tests.orchestrator - INFO - Cleanup completed +2025-06-04 21:13:47,131 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-04 21:13:47,131 - tests.orchestrator - INFO - --- Operations --- +2025-06-04 21:13:47,131 - tests.orchestrator - INFO - create_pvc: 0 succeeded, 0 failed (0.0%) +2025-06-04 21:13:47,131 - tests.orchestrator - INFO - attach_pod: 1 succeeded, 0 failed (100.0%) +2025-06-04 21:13:47,131 - tests.orchestrator - INFO - delete_pod: 1 succeeded, 0 failed (100.0%) +2025-06-04 21:13:47,131 - tests.orchestrator - INFO - delete_pvc: 1 succeeded, 0 failed (100.0%) +2025-06-04 21:13:47,131 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 0 failed (0.0%) +2025-06-04 21:13:47,131 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 0 failed (0.0%) +2025-06-04 21:13:47,131 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-04 21:13:47,131 - tests.orchestrator - INFO - shared_volume_rw: No runs +2025-06-04 21:13:47,131 - tests.orchestrator - INFO - many_to_one: No runs +2025-06-04 21:13:47,131 - tests.orchestrator - INFO - one_to_one: 0 succeeded, 0 failed out of 1 runs (0.0%) +2025-06-04 21:13:47,131 - tests.orchestrator - INFO - concurrent_pvc: No runs +2025-06-04 21:13:47,131 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250604_211353.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250604_211353.log new file mode 100644 index 000000000..55a38aabb --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250604_211353.log @@ -0,0 +1,80 @@ +2025-06-04 21:13:53,143 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-04 21:13:53,342 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-04 21:13:53,342 - tests.orchestrator - INFO - Starting orchestrator test for 120 seconds +2025-06-04 21:13:53,535 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-04 21:13:53,536 - tests.orchestrator - INFO - Selected operation: _verify_readwrite +2025-06-04 21:13:53,536 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test +2025-06-04 21:13:54,536 - tests.orchestrator - INFO - Selected operation: _delete_pvc +2025-06-04 21:13:54,536 - tests.orchestrator - INFO - No PVCs to delete +2025-06-04 21:13:55,536 - tests.orchestrator - INFO - Selected operation: _verify_readwrite +2025-06-04 21:13:55,537 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test +2025-06-04 21:13:56,537 - tests.orchestrator - INFO - Selected operation: _delete_pod +2025-06-04 21:13:56,537 - tests.orchestrator - INFO - No pods to delete +2025-06-04 21:13:57,537 - tests.orchestrator - INFO - Selected operation: _delete_pod +2025-06-04 21:13:57,537 - tests.orchestrator - INFO - No pods to delete +2025-06-04 21:13:58,538 - tests.orchestrator - INFO - Selected operation: _verify_readwrite +2025-06-04 21:13:58,538 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test +2025-06-04 21:13:59,538 - tests.orchestrator - INFO - Selected operation: _delete_pod +2025-06-04 21:13:59,538 - tests.orchestrator - INFO - No pods to delete +2025-06-04 21:14:00,538 - tests.orchestrator - INFO - Selected operation: _attach_pod +2025-06-04 21:14:00,539 - tests.orchestrator - INFO - No PVCs available, skipping pod attachment +2025-06-04 21:14:01,539 - tests.orchestrator - INFO - Selected operation: _delete_pod +2025-06-04 21:14:01,540 - tests.orchestrator - INFO - No pods to delete +2025-06-04 21:14:02,540 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario +2025-06-04 21:14:02,541 - tests.orchestrator - INFO - Running specific scenario: _scenario_concurrent_pvc +2025-06-04 21:14:02,541 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations +2025-06-04 21:14:02,541 - tests.orchestrator - INFO - Creating 4 PVCs in quick succession +2025-06-04 21:14:02,617 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-68e38df4 +2025-06-04 21:14:02,617 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-68e38df4 to be bound +2025-06-04 21:14:04,753 - tests.orchestrator - INFO - PVC concurrent-pvc-68e38df4 is bound +2025-06-04 21:14:04,825 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-90dd2e0a +2025-06-04 21:14:04,825 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-90dd2e0a to be bound +2025-06-04 21:14:06,959 - tests.orchestrator - INFO - PVC concurrent-pvc-90dd2e0a is bound +2025-06-04 21:14:07,028 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-09b78e7f +2025-06-04 21:14:07,028 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-09b78e7f to be bound +2025-06-04 21:14:09,163 - tests.orchestrator - INFO - PVC concurrent-pvc-09b78e7f is bound +2025-06-04 21:14:09,232 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-51c7b76a +2025-06-04 21:14:09,232 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-51c7b76a to be bound +2025-06-04 21:14:11,368 - tests.orchestrator - INFO - PVC concurrent-pvc-51c7b76a is bound +2025-06-04 21:14:11,368 - tests.orchestrator - INFO - Successfully created 4 PVCs +2025-06-04 21:14:11,368 - tests.orchestrator - INFO - Creating pod test-pod-60af97a5 on PVC concurrent-pvc-90dd2e0a +2025-06-04 21:14:11,458 - tests.orchestrator - INFO - Created pod: test-pod-60af97a5 using PVC: concurrent-pvc-90dd2e0a +2025-06-04 21:14:11,458 - tests.orchestrator - INFO - Waiting for pod test-pod-60af97a5 to be ready +2025-06-04 21:14:56,603 - tests.orchestrator - INFO - Test interrupted by user +2025-06-04 21:14:56,603 - tests.orchestrator - INFO - Test completed in 63.26 seconds +2025-06-04 21:14:56,603 - tests.orchestrator - INFO - Cleaning up resources... +2025-06-04 21:14:56,603 - tests.orchestrator - INFO - Deleting pod: test-pod-60af97a5 from PVC: concurrent-pvc-90dd2e0a +2025-06-04 21:14:56,689 - tests.orchestrator - INFO - Waiting for pod test-pod-60af97a5 to be deleted +2025-06-04 21:14:56,754 - tests.orchestrator - INFO - Pod test-pod-60af97a5 has been deleted +2025-06-04 21:14:56,754 - tests.orchestrator - INFO - Deleted pod: test-pod-60af97a5 +2025-06-04 21:14:56,754 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-68e38df4 +2025-06-04 21:14:56,838 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-68e38df4 to be deleted +2025-06-04 21:14:56,902 - tests.orchestrator - INFO - PVC concurrent-pvc-68e38df4 has been deleted +2025-06-04 21:14:56,903 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-68e38df4 +2025-06-04 21:14:56,903 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-90dd2e0a +2025-06-04 21:14:56,977 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-90dd2e0a to be deleted +2025-06-04 21:14:57,041 - tests.orchestrator - INFO - PVC concurrent-pvc-90dd2e0a has been deleted +2025-06-04 21:14:57,041 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-90dd2e0a +2025-06-04 21:14:57,041 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-09b78e7f +2025-06-04 21:14:57,116 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-09b78e7f to be deleted +2025-06-04 21:14:57,181 - tests.orchestrator - INFO - PVC concurrent-pvc-09b78e7f has been deleted +2025-06-04 21:14:57,181 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-09b78e7f +2025-06-04 21:14:57,181 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-51c7b76a +2025-06-04 21:14:57,257 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-51c7b76a to be deleted +2025-06-04 21:14:57,322 - tests.orchestrator - INFO - PVC concurrent-pvc-51c7b76a has been deleted +2025-06-04 21:14:57,322 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-51c7b76a +2025-06-04 21:14:57,322 - tests.orchestrator - INFO - Cleanup completed +2025-06-04 21:14:57,322 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-04 21:14:57,322 - tests.orchestrator - INFO - --- Operations --- +2025-06-04 21:14:57,322 - tests.orchestrator - INFO - create_pvc: 4 succeeded, 0 failed (100.0%) +2025-06-04 21:14:57,322 - tests.orchestrator - INFO - attach_pod: 1 succeeded, 0 failed (100.0%) +2025-06-04 21:14:57,322 - tests.orchestrator - INFO - delete_pod: 1 succeeded, 0 failed (100.0%) +2025-06-04 21:14:57,322 - tests.orchestrator - INFO - delete_pvc: 4 succeeded, 0 failed (100.0%) +2025-06-04 21:14:57,322 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 0 failed (0.0%) +2025-06-04 21:14:57,322 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 0 failed (0.0%) +2025-06-04 21:14:57,322 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-04 21:14:57,323 - tests.orchestrator - INFO - shared_volume_rw: No runs +2025-06-04 21:14:57,323 - tests.orchestrator - INFO - many_to_one: No runs +2025-06-04 21:14:57,323 - tests.orchestrator - INFO - one_to_one: No runs +2025-06-04 21:14:57,323 - tests.orchestrator - INFO - concurrent_pvc: 0 succeeded, 0 failed out of 1 runs (0.0%) +2025-06-04 21:14:57,323 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_132802.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_132802.log new file mode 100644 index 000000000..9cd4da221 --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_132802.log @@ -0,0 +1,202 @@ +2025-06-05 13:28:02,670 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 13:28:02,885 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 13:28:02,886 - tests.orchestrator - INFO - Starting orchestrator test for 400 seconds +2025-06-05 13:28:03,078 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 13:28:03,079 - tests.orchestrator - INFO - Selected operation: _create_pvc +2025-06-05 13:28:03,079 - tests.orchestrator - INFO - Creating PVC: test-pvc-87f8dca2 +2025-06-05 13:28:03,159 - tests.orchestrator - INFO - Created PVC: test-pvc-87f8dca2 +2025-06-05 13:28:03,159 - tests.orchestrator - INFO - Waiting for PVC test-pvc-87f8dca2 to be bound +2025-06-05 13:28:05,290 - tests.orchestrator - INFO - PVC test-pvc-87f8dca2 is bound +2025-06-05 13:28:06,290 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario +2025-06-05 13:28:06,290 - tests.orchestrator - INFO - Running specific scenario: _scenario_concurrent_pvc +2025-06-05 13:28:06,290 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations +2025-06-05 13:28:06,290 - tests.orchestrator - INFO - Creating 6 PVCs in quick succession +2025-06-05 13:28:06,363 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-27248b37 +2025-06-05 13:28:06,363 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-27248b37 to be bound +2025-06-05 13:28:08,495 - tests.orchestrator - INFO - PVC concurrent-pvc-27248b37 is bound +2025-06-05 13:28:08,561 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-0be57a62 +2025-06-05 13:28:08,562 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-0be57a62 to be bound +2025-06-05 13:28:10,696 - tests.orchestrator - INFO - PVC concurrent-pvc-0be57a62 is bound +2025-06-05 13:28:10,764 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-d46ca8c1 +2025-06-05 13:28:10,764 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-d46ca8c1 to be bound +2025-06-05 13:28:12,902 - tests.orchestrator - INFO - PVC concurrent-pvc-d46ca8c1 is bound +2025-06-05 13:28:12,972 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-4b94a252 +2025-06-05 13:28:12,973 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-4b94a252 to be bound +2025-06-05 13:28:15,113 - tests.orchestrator - INFO - PVC concurrent-pvc-4b94a252 is bound +2025-06-05 13:28:15,180 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-737583a5 +2025-06-05 13:28:15,181 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-737583a5 to be bound +2025-06-05 13:28:17,317 - tests.orchestrator - INFO - PVC concurrent-pvc-737583a5 is bound +2025-06-05 13:28:17,385 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-b6ec6167 +2025-06-05 13:28:17,385 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-b6ec6167 to be bound +2025-06-05 13:28:19,520 - tests.orchestrator - INFO - PVC concurrent-pvc-b6ec6167 is bound +2025-06-05 13:28:19,520 - tests.orchestrator - INFO - Successfully created 6 PVCs +2025-06-05 13:28:19,520 - tests.orchestrator - INFO - Creating pod test-pod-05839583 on PVC concurrent-pvc-4b94a252 +2025-06-05 13:28:19,605 - tests.orchestrator - INFO - Created pod: test-pod-05839583 using PVC: concurrent-pvc-4b94a252 +2025-06-05 13:28:19,605 - tests.orchestrator - INFO - Waiting for pod test-pod-05839583 to be ready +2025-06-05 13:29:21,656 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-05839583 to be ready after 60s +2025-06-05 13:29:21,657 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-05839583 to be ready +2025-06-05 13:29:21,657 - tests.orchestrator - INFO - Creating pod test-pod-c11d7609 on PVC concurrent-pvc-b6ec6167 +2025-06-05 13:29:21,746 - tests.orchestrator - INFO - Created pod: test-pod-c11d7609 using PVC: concurrent-pvc-b6ec6167 +2025-06-05 13:29:21,746 - tests.orchestrator - INFO - Waiting for pod test-pod-c11d7609 to be ready +2025-06-05 13:30:23,813 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-c11d7609 to be ready after 60s +2025-06-05 13:30:23,813 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-c11d7609 to be ready +2025-06-05 13:30:23,813 - tests.orchestrator - INFO - Creating pod test-pod-f3958de4 on PVC concurrent-pvc-27248b37 +2025-06-05 13:30:23,897 - tests.orchestrator - INFO - Created pod: test-pod-f3958de4 using PVC: concurrent-pvc-27248b37 +2025-06-05 13:30:23,897 - tests.orchestrator - INFO - Waiting for pod test-pod-f3958de4 to be ready +2025-06-05 13:31:23,958 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-f3958de4 to be ready after 60s +2025-06-05 13:31:23,958 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-f3958de4 to be ready +2025-06-05 13:31:23,958 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-0be57a62 +2025-06-05 13:31:24,029 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-0be57a62 to be deleted +2025-06-05 13:31:24,096 - tests.orchestrator - INFO - PVC concurrent-pvc-0be57a62 has been deleted +2025-06-05 13:31:24,096 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-0be57a62 +2025-06-05 13:31:24,096 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-4b94a252 +2025-06-05 13:31:24,096 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-4b94a252 +2025-06-05 13:31:24,096 - tests.orchestrator - INFO - Deleting pod: test-pod-05839583 from PVC: concurrent-pvc-4b94a252 +2025-06-05 13:31:24,177 - tests.orchestrator - INFO - Waiting for pod test-pod-05839583 to be deleted +2025-06-05 13:31:24,241 - tests.orchestrator - INFO - Pod test-pod-05839583 has been deleted +2025-06-05 13:31:24,241 - tests.orchestrator - INFO - Deleted pod: test-pod-05839583 +2025-06-05 13:31:24,314 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-4b94a252 to be deleted +2025-06-05 13:31:24,380 - tests.orchestrator - INFO - PVC concurrent-pvc-4b94a252 has been deleted +2025-06-05 13:31:24,380 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-4b94a252 +2025-06-05 13:31:24,381 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-27248b37 +2025-06-05 13:31:24,381 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-27248b37 +2025-06-05 13:31:24,381 - tests.orchestrator - INFO - Deleting pod: test-pod-f3958de4 from PVC: concurrent-pvc-27248b37 +2025-06-05 13:31:24,464 - tests.orchestrator - INFO - Waiting for pod test-pod-f3958de4 to be deleted +2025-06-05 13:31:24,528 - tests.orchestrator - INFO - Pod test-pod-f3958de4 has been deleted +2025-06-05 13:31:24,528 - tests.orchestrator - INFO - Deleted pod: test-pod-f3958de4 +2025-06-05 13:31:24,601 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-27248b37 to be deleted +2025-06-05 13:31:24,668 - tests.orchestrator - INFO - PVC concurrent-pvc-27248b37 has been deleted +2025-06-05 13:31:24,668 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-27248b37 +2025-06-05 13:31:24,668 - tests.orchestrator - INFO - Rapid PVC scenario completed +2025-06-05 13:31:25,668 - tests.orchestrator - INFO - Selected operation: _attach_pod +2025-06-05 13:31:25,669 - tests.orchestrator - INFO - Creating pod test-pod-4386bacf on PVC concurrent-pvc-737583a5 +2025-06-05 13:31:25,750 - tests.orchestrator - INFO - Created pod: test-pod-4386bacf using PVC: concurrent-pvc-737583a5 +2025-06-05 13:31:25,750 - tests.orchestrator - INFO - Waiting for pod test-pod-4386bacf to be ready +2025-06-05 13:32:27,796 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-4386bacf to be ready after 60s +2025-06-05 13:32:27,796 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-4386bacf to be ready +2025-06-05 13:32:28,797 - tests.orchestrator - INFO - Selected operation: _attach_pod +2025-06-05 13:32:28,798 - tests.orchestrator - INFO - Creating pod test-pod-93094317 on PVC concurrent-pvc-b6ec6167 +2025-06-05 13:32:28,887 - tests.orchestrator - INFO - Created pod: test-pod-93094317 using PVC: concurrent-pvc-b6ec6167 +2025-06-05 13:32:28,887 - tests.orchestrator - INFO - Waiting for pod test-pod-93094317 to be ready +2025-06-05 13:33:30,941 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-93094317 to be ready after 60s +2025-06-05 13:33:30,941 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-93094317 to be ready +2025-06-05 13:33:31,941 - tests.orchestrator - INFO - Selected operation: _create_pvc +2025-06-05 13:33:31,942 - tests.orchestrator - INFO - Creating PVC: test-pvc-cd15165b +2025-06-05 13:33:32,022 - tests.orchestrator - INFO - Created PVC: test-pvc-cd15165b +2025-06-05 13:33:32,022 - tests.orchestrator - INFO - Waiting for PVC test-pvc-cd15165b to be bound +2025-06-05 13:33:34,157 - tests.orchestrator - INFO - PVC test-pvc-cd15165b is bound +2025-06-05 13:33:35,157 - tests.orchestrator - INFO - Selected operation: _create_pvc +2025-06-05 13:33:35,157 - tests.orchestrator - INFO - Creating PVC: test-pvc-67687a38 +2025-06-05 13:33:35,229 - tests.orchestrator - INFO - Created PVC: test-pvc-67687a38 +2025-06-05 13:33:35,229 - tests.orchestrator - INFO - Waiting for PVC test-pvc-67687a38 to be bound +2025-06-05 13:33:37,363 - tests.orchestrator - INFO - PVC test-pvc-67687a38 is bound +2025-06-05 13:33:38,364 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario +2025-06-05 13:33:38,364 - tests.orchestrator - INFO - Running specific scenario: _scenario_concurrent_pvc +2025-06-05 13:33:38,364 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations +2025-06-05 13:33:38,364 - tests.orchestrator - INFO - Creating 3 PVCs in quick succession +2025-06-05 13:33:38,431 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-a6fe87d2 +2025-06-05 13:33:38,431 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-a6fe87d2 to be bound +2025-06-05 13:33:40,563 - tests.orchestrator - INFO - PVC concurrent-pvc-a6fe87d2 is bound +2025-06-05 13:33:40,631 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-eddc8eb9 +2025-06-05 13:33:40,631 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-eddc8eb9 to be bound +2025-06-05 13:33:42,765 - tests.orchestrator - INFO - PVC concurrent-pvc-eddc8eb9 is bound +2025-06-05 13:33:42,835 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-2bd04bf8 +2025-06-05 13:33:42,836 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-2bd04bf8 to be bound +2025-06-05 13:33:44,970 - tests.orchestrator - INFO - PVC concurrent-pvc-2bd04bf8 is bound +2025-06-05 13:33:44,971 - tests.orchestrator - INFO - Successfully created 3 PVCs +2025-06-05 13:33:44,971 - tests.orchestrator - INFO - Creating pod test-pod-b5b431f0 on PVC concurrent-pvc-2bd04bf8 +2025-06-05 13:33:45,045 - tests.orchestrator - INFO - Created pod: test-pod-b5b431f0 using PVC: concurrent-pvc-2bd04bf8 +2025-06-05 13:33:45,045 - tests.orchestrator - INFO - Waiting for pod test-pod-b5b431f0 to be ready +2025-06-05 13:34:45,136 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b5b431f0 to be ready after 60s +2025-06-05 13:34:45,136 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b5b431f0 to be ready +2025-06-05 13:34:45,137 - tests.orchestrator - INFO - Creating pod test-pod-a900f8ea on PVC concurrent-pvc-a6fe87d2 +2025-06-05 13:34:45,234 - tests.orchestrator - INFO - Created pod: test-pod-a900f8ea using PVC: concurrent-pvc-a6fe87d2 +2025-06-05 13:34:45,234 - tests.orchestrator - INFO - Waiting for pod test-pod-a900f8ea to be ready +2025-06-05 13:35:45,421 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a900f8ea to be ready after 60s +2025-06-05 13:35:45,421 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a900f8ea to be ready +2025-06-05 13:35:45,421 - tests.orchestrator - INFO - Creating pod test-pod-71d9daf8 on PVC concurrent-pvc-eddc8eb9 +2025-06-05 13:35:45,506 - tests.orchestrator - INFO - Created pod: test-pod-71d9daf8 using PVC: concurrent-pvc-eddc8eb9 +2025-06-05 13:35:45,506 - tests.orchestrator - INFO - Waiting for pod test-pod-71d9daf8 to be ready +2025-06-05 13:36:47,551 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-71d9daf8 to be ready after 60s +2025-06-05 13:36:47,551 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-71d9daf8 to be ready +2025-06-05 13:36:47,551 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-2bd04bf8 +2025-06-05 13:36:47,551 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-2bd04bf8 +2025-06-05 13:36:47,551 - tests.orchestrator - INFO - Deleting pod: test-pod-b5b431f0 from PVC: concurrent-pvc-2bd04bf8 +2025-06-05 13:36:47,632 - tests.orchestrator - INFO - Waiting for pod test-pod-b5b431f0 to be deleted +2025-06-05 13:36:47,699 - tests.orchestrator - INFO - Pod test-pod-b5b431f0 has been deleted +2025-06-05 13:36:47,699 - tests.orchestrator - INFO - Deleted pod: test-pod-b5b431f0 +2025-06-05 13:36:47,771 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-2bd04bf8 to be deleted +2025-06-05 13:36:47,839 - tests.orchestrator - INFO - PVC concurrent-pvc-2bd04bf8 has been deleted +2025-06-05 13:36:47,839 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-2bd04bf8 +2025-06-05 13:36:47,839 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-eddc8eb9 +2025-06-05 13:36:47,839 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-eddc8eb9 +2025-06-05 13:36:47,839 - tests.orchestrator - INFO - Deleting pod: test-pod-71d9daf8 from PVC: concurrent-pvc-eddc8eb9 +2025-06-05 13:36:47,919 - tests.orchestrator - INFO - Waiting for pod test-pod-71d9daf8 to be deleted +2025-06-05 13:36:47,984 - tests.orchestrator - INFO - Pod test-pod-71d9daf8 has been deleted +2025-06-05 13:36:47,984 - tests.orchestrator - INFO - Deleted pod: test-pod-71d9daf8 +2025-06-05 13:36:48,056 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-eddc8eb9 to be deleted +2025-06-05 13:36:48,122 - tests.orchestrator - INFO - PVC concurrent-pvc-eddc8eb9 has been deleted +2025-06-05 13:36:48,122 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-eddc8eb9 +2025-06-05 13:36:48,122 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-a6fe87d2 +2025-06-05 13:36:48,122 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-a6fe87d2 +2025-06-05 13:36:48,122 - tests.orchestrator - INFO - Deleting pod: test-pod-a900f8ea from PVC: concurrent-pvc-a6fe87d2 +2025-06-05 13:36:48,206 - tests.orchestrator - INFO - Waiting for pod test-pod-a900f8ea to be deleted +2025-06-05 13:36:48,270 - tests.orchestrator - INFO - Pod test-pod-a900f8ea has been deleted +2025-06-05 13:36:48,271 - tests.orchestrator - INFO - Deleted pod: test-pod-a900f8ea +2025-06-05 13:36:48,345 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-a6fe87d2 to be deleted +2025-06-05 13:36:48,412 - tests.orchestrator - INFO - PVC concurrent-pvc-a6fe87d2 has been deleted +2025-06-05 13:36:48,412 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-a6fe87d2 +2025-06-05 13:36:48,412 - tests.orchestrator - INFO - Rapid PVC scenario completed +2025-06-05 13:36:49,412 - tests.orchestrator - INFO - Test completed in 526.53 seconds +2025-06-05 13:36:49,412 - tests.orchestrator - INFO - Cleaning up resources... +2025-06-05 13:36:49,412 - tests.orchestrator - INFO - Deleting pod: test-pod-4386bacf from PVC: concurrent-pvc-737583a5 +2025-06-05 13:36:49,492 - tests.orchestrator - INFO - Waiting for pod test-pod-4386bacf to be deleted +2025-06-05 13:36:49,559 - tests.orchestrator - INFO - Pod test-pod-4386bacf has been deleted +2025-06-05 13:36:49,560 - tests.orchestrator - INFO - Deleted pod: test-pod-4386bacf +2025-06-05 13:36:49,560 - tests.orchestrator - INFO - Deleting pod: test-pod-c11d7609 from PVC: concurrent-pvc-b6ec6167 +2025-06-05 13:36:49,640 - tests.orchestrator - INFO - Waiting for pod test-pod-c11d7609 to be deleted +2025-06-05 13:36:49,706 - tests.orchestrator - INFO - Pod test-pod-c11d7609 has been deleted +2025-06-05 13:36:49,706 - tests.orchestrator - INFO - Deleted pod: test-pod-c11d7609 +2025-06-05 13:36:49,706 - tests.orchestrator - INFO - Deleting pod: test-pod-93094317 from PVC: concurrent-pvc-b6ec6167 +2025-06-05 13:36:49,783 - tests.orchestrator - INFO - Waiting for pod test-pod-93094317 to be deleted +2025-06-05 13:36:49,850 - tests.orchestrator - INFO - Pod test-pod-93094317 has been deleted +2025-06-05 13:36:49,850 - tests.orchestrator - INFO - Deleted pod: test-pod-93094317 +2025-06-05 13:36:49,850 - tests.orchestrator - INFO - Deleting PVC: test-pvc-87f8dca2 +2025-06-05 13:36:49,922 - tests.orchestrator - INFO - Waiting for PVC test-pvc-87f8dca2 to be deleted +2025-06-05 13:36:49,988 - tests.orchestrator - INFO - PVC test-pvc-87f8dca2 has been deleted +2025-06-05 13:36:49,989 - tests.orchestrator - INFO - Deleted PVC: test-pvc-87f8dca2 +2025-06-05 13:36:49,989 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-d46ca8c1 +2025-06-05 13:36:50,060 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-d46ca8c1 to be deleted +2025-06-05 13:36:50,126 - tests.orchestrator - INFO - PVC concurrent-pvc-d46ca8c1 has been deleted +2025-06-05 13:36:50,127 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-d46ca8c1 +2025-06-05 13:36:50,127 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-737583a5 +2025-06-05 13:36:50,197 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-737583a5 to be deleted +2025-06-05 13:36:50,264 - tests.orchestrator - INFO - PVC concurrent-pvc-737583a5 has been deleted +2025-06-05 13:36:50,264 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-737583a5 +2025-06-05 13:36:50,264 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-b6ec6167 +2025-06-05 13:36:50,336 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-b6ec6167 to be deleted +2025-06-05 13:36:50,403 - tests.orchestrator - INFO - PVC concurrent-pvc-b6ec6167 has been deleted +2025-06-05 13:36:50,403 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-b6ec6167 +2025-06-05 13:36:50,403 - tests.orchestrator - INFO - Deleting PVC: test-pvc-cd15165b +2025-06-05 13:36:50,477 - tests.orchestrator - INFO - Waiting for PVC test-pvc-cd15165b to be deleted +2025-06-05 13:36:50,544 - tests.orchestrator - INFO - PVC test-pvc-cd15165b has been deleted +2025-06-05 13:36:50,545 - tests.orchestrator - INFO - Deleted PVC: test-pvc-cd15165b +2025-06-05 13:36:50,545 - tests.orchestrator - INFO - Deleting PVC: test-pvc-67687a38 +2025-06-05 13:36:50,617 - tests.orchestrator - INFO - Waiting for PVC test-pvc-67687a38 to be deleted +2025-06-05 13:36:50,683 - tests.orchestrator - INFO - PVC test-pvc-67687a38 has been deleted +2025-06-05 13:36:50,683 - tests.orchestrator - INFO - Deleted PVC: test-pvc-67687a38 +2025-06-05 13:36:50,683 - tests.orchestrator - INFO - Cleanup completed +2025-06-05 13:36:50,683 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-05 13:36:50,683 - tests.orchestrator - INFO - --- Operations --- +2025-06-05 13:36:50,683 - tests.orchestrator - INFO - create_pvc: 12 succeeded, 0 failed (100.0%) +2025-06-05 13:36:50,683 - tests.orchestrator - INFO - attach_pod: 8 succeeded, 0 failed (100.0%) +2025-06-05 13:36:50,683 - tests.orchestrator - INFO - delete_pod: 8 succeeded, 0 failed (100.0%) +2025-06-05 13:36:50,684 - tests.orchestrator - INFO - delete_pvc: 12 succeeded, 0 failed (100.0%) +2025-06-05 13:36:50,684 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 0 failed (0.0%) +2025-06-05 13:36:50,684 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 0 failed (0.0%) +2025-06-05 13:36:50,684 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-05 13:36:50,684 - tests.orchestrator - INFO - shared_volume_rw: No runs +2025-06-05 13:36:50,684 - tests.orchestrator - INFO - many_to_one: No runs +2025-06-05 13:36:50,684 - tests.orchestrator - INFO - one_to_one: No runs +2025-06-05 13:36:50,684 - tests.orchestrator - INFO - concurrent_pvc: 2 succeeded, 0 failed out of 2 runs (100.0%) +2025-06-05 13:36:50,684 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_135002.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_135002.log new file mode 100644 index 000000000..570bd57de --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_135002.log @@ -0,0 +1,123 @@ +2025-06-05 13:50:02,494 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 13:50:02,715 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 13:50:02,715 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-05 13:50:02,906 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 13:50:02,906 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 13:50:02,906 - tests.orchestrator - INFO - Creating PVC: test-pvc-040b70c6 +2025-06-05 13:50:02,982 - tests.orchestrator - INFO - Created PVC: test-pvc-040b70c6 +2025-06-05 13:50:02,982 - tests.orchestrator - INFO - Waiting for PVC test-pvc-040b70c6 to be bound +2025-06-05 13:50:05,116 - tests.orchestrator - INFO - PVC test-pvc-040b70c6 is bound +2025-06-05 13:50:05,116 - tests.orchestrator - INFO - Creating PVC: test-pvc-e5eb6b62 +2025-06-05 13:50:05,184 - tests.orchestrator - INFO - Created PVC: test-pvc-e5eb6b62 +2025-06-05 13:50:05,184 - tests.orchestrator - INFO - Waiting for PVC test-pvc-e5eb6b62 to be bound +2025-06-05 13:50:07,318 - tests.orchestrator - INFO - PVC test-pvc-e5eb6b62 is bound +2025-06-05 13:50:07,319 - tests.orchestrator - INFO - Creating pod test-pod-094962b3 on PVC test-pvc-e5eb6b62 +2025-06-05 13:50:07,399 - tests.orchestrator - INFO - Created pod: test-pod-094962b3 using PVC: test-pvc-e5eb6b62 +2025-06-05 13:50:07,400 - tests.orchestrator - INFO - Waiting for pod test-pod-094962b3 to be ready +2025-06-05 13:51:09,437 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-094962b3 to be ready after 60s +2025-06-05 13:51:09,437 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-094962b3 to be ready +2025-06-05 13:51:09,437 - tests.orchestrator - INFO - Creating pod test-pod-1b064942 on PVC test-pvc-040b70c6 +2025-06-05 13:51:09,521 - tests.orchestrator - INFO - Created pod: test-pod-1b064942 using PVC: test-pvc-040b70c6 +2025-06-05 13:51:09,521 - tests.orchestrator - INFO - Waiting for pod test-pod-1b064942 to be ready +2025-06-05 13:52:11,558 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1b064942 to be ready after 60s +2025-06-05 13:52:11,558 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1b064942 to be ready +2025-06-05 13:52:11,559 - tests.orchestrator - INFO - Creating pod test-pod-ab3159d4 on PVC test-pvc-e5eb6b62 +2025-06-05 13:52:11,645 - tests.orchestrator - INFO - Created pod: test-pod-ab3159d4 using PVC: test-pvc-e5eb6b62 +2025-06-05 13:52:11,646 - tests.orchestrator - INFO - Waiting for pod test-pod-ab3159d4 to be ready +2025-06-05 13:53:13,682 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-ab3159d4 to be ready after 60s +2025-06-05 13:53:13,682 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-ab3159d4 to be ready +2025-06-05 13:53:13,683 - tests.orchestrator - INFO - Testing read/write between pods test-pod-094962b3 and test-pod-ab3159d4 sharing PVC test-pvc-e5eb6b62 +2025-06-05 13:53:13,751 - tests.orchestrator - ERROR - Failed in read/write verification: (400) +Reason: Bad Request +HTTP response headers: HTTPHeaderDict({'Audit-Id': '7c3ae37d-dd7e-42d1-97d7-9f45f47124cc', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 13:53:13 GMT', 'Content-Length': '166'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-094962b3 does not have a host assigned","reason":"BadRequest","code":400} + + +2025-06-05 13:53:13,751 - tests.orchestrator - INFO - Running specific scenario: _scenario_one_to_one +2025-06-05 13:53:13,751 - tests.orchestrator - INFO - Running scenario: One pod per PVC +2025-06-05 13:53:13,751 - tests.orchestrator - INFO - Creating 3 PVC-pod pairs +2025-06-05 13:53:13,829 - tests.orchestrator - INFO - Waiting for PVC one2one-9268308a to be bound +2025-06-05 13:53:15,962 - tests.orchestrator - INFO - PVC one2one-9268308a is bound +2025-06-05 13:53:15,962 - tests.orchestrator - INFO - Creating pod test-pod-2d27ea53 on PVC one2one-9268308a +2025-06-05 13:53:16,038 - tests.orchestrator - INFO - Created pod: test-pod-2d27ea53 using PVC: one2one-9268308a +2025-06-05 13:53:16,038 - tests.orchestrator - INFO - Waiting for pod test-pod-2d27ea53 to be ready +2025-06-05 13:54:16,048 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-2d27ea53 to be ready after 60s +2025-06-05 13:54:16,048 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-2d27ea53 to be ready +2025-06-05 13:54:16,124 - tests.orchestrator - INFO - Waiting for PVC one2one-63559411 to be bound +2025-06-05 13:54:18,258 - tests.orchestrator - INFO - PVC one2one-63559411 is bound +2025-06-05 13:54:18,258 - tests.orchestrator - INFO - Creating pod test-pod-5c1f7339 on PVC one2one-63559411 +2025-06-05 13:54:18,330 - tests.orchestrator - INFO - Created pod: test-pod-5c1f7339 using PVC: one2one-63559411 +2025-06-05 13:54:18,330 - tests.orchestrator - INFO - Waiting for pod test-pod-5c1f7339 to be ready +2025-06-05 13:55:20,367 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-5c1f7339 to be ready after 60s +2025-06-05 13:55:20,367 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-5c1f7339 to be ready +2025-06-05 13:55:20,444 - tests.orchestrator - INFO - Waiting for PVC one2one-24717940 to be bound +2025-06-05 13:55:22,579 - tests.orchestrator - INFO - PVC one2one-24717940 is bound +2025-06-05 13:55:22,580 - tests.orchestrator - INFO - Creating pod test-pod-a84638d7 on PVC one2one-24717940 +2025-06-05 13:55:22,655 - tests.orchestrator - INFO - Created pod: test-pod-a84638d7 using PVC: one2one-24717940 +2025-06-05 13:55:22,655 - tests.orchestrator - INFO - Waiting for pod test-pod-a84638d7 to be ready +2025-06-05 13:56:24,701 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a84638d7 to be ready after 60s +2025-06-05 13:56:24,701 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a84638d7 to be ready +2025-06-05 13:56:24,701 - tests.orchestrator - WARNING - Failed to create enough PVC-pod pairs, only created 0 +2025-06-05 13:56:24,701 - tests.orchestrator - INFO - Deleting pod: test-pod-094962b3 from PVC: test-pvc-e5eb6b62 +2025-06-05 13:56:24,780 - tests.orchestrator - INFO - Waiting for pod test-pod-094962b3 to be deleted +2025-06-05 13:56:24,846 - tests.orchestrator - INFO - Pod test-pod-094962b3 has been deleted +2025-06-05 13:56:24,846 - tests.orchestrator - INFO - Deleted pod: test-pod-094962b3 +2025-06-05 13:56:24,846 - tests.orchestrator - INFO - Deleting PVC: one2one-63559411 +2025-06-05 13:56:24,846 - tests.orchestrator - INFO - Deleting 1 pods using PVC one2one-63559411 +2025-06-05 13:56:24,846 - tests.orchestrator - INFO - Deleting pod: test-pod-5c1f7339 from PVC: one2one-63559411 +2025-06-05 13:56:24,932 - tests.orchestrator - INFO - Waiting for pod test-pod-5c1f7339 to be deleted +2025-06-05 13:56:24,997 - tests.orchestrator - INFO - Pod test-pod-5c1f7339 has been deleted +2025-06-05 13:56:24,997 - tests.orchestrator - INFO - Deleted pod: test-pod-5c1f7339 +2025-06-05 13:56:25,070 - tests.orchestrator - INFO - Waiting for PVC one2one-63559411 to be deleted +2025-06-05 13:56:25,135 - tests.orchestrator - INFO - PVC one2one-63559411 has been deleted +2025-06-05 13:56:25,136 - tests.orchestrator - INFO - Deleted PVC: one2one-63559411 +2025-06-05 13:56:25,136 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations +2025-06-05 13:56:25,136 - tests.orchestrator - INFO - Test completed in 382.42 seconds +2025-06-05 13:56:25,136 - tests.orchestrator - INFO - Cleaning up resources... +2025-06-05 13:56:25,136 - tests.orchestrator - INFO - Deleting pod: test-pod-1b064942 from PVC: test-pvc-040b70c6 +2025-06-05 13:56:25,213 - tests.orchestrator - INFO - Waiting for pod test-pod-1b064942 to be deleted +2025-06-05 13:56:25,279 - tests.orchestrator - INFO - Pod test-pod-1b064942 has been deleted +2025-06-05 13:56:25,279 - tests.orchestrator - INFO - Deleted pod: test-pod-1b064942 +2025-06-05 13:56:25,279 - tests.orchestrator - INFO - Deleting pod: test-pod-ab3159d4 from PVC: test-pvc-e5eb6b62 +2025-06-05 13:56:25,363 - tests.orchestrator - INFO - Waiting for pod test-pod-ab3159d4 to be deleted +2025-06-05 13:56:25,429 - tests.orchestrator - INFO - Pod test-pod-ab3159d4 has been deleted +2025-06-05 13:56:25,429 - tests.orchestrator - INFO - Deleted pod: test-pod-ab3159d4 +2025-06-05 13:56:25,429 - tests.orchestrator - INFO - Deleting pod: test-pod-2d27ea53 from PVC: one2one-9268308a +2025-06-05 13:56:25,507 - tests.orchestrator - INFO - Waiting for pod test-pod-2d27ea53 to be deleted +2025-06-05 13:56:25,573 - tests.orchestrator - INFO - Pod test-pod-2d27ea53 has been deleted +2025-06-05 13:56:25,573 - tests.orchestrator - INFO - Deleted pod: test-pod-2d27ea53 +2025-06-05 13:56:25,574 - tests.orchestrator - INFO - Deleting pod: test-pod-a84638d7 from PVC: one2one-24717940 +2025-06-05 13:56:25,658 - tests.orchestrator - INFO - Waiting for pod test-pod-a84638d7 to be deleted +2025-06-05 13:56:25,725 - tests.orchestrator - INFO - Pod test-pod-a84638d7 has been deleted +2025-06-05 13:56:25,726 - tests.orchestrator - INFO - Deleted pod: test-pod-a84638d7 +2025-06-05 13:56:25,726 - tests.orchestrator - INFO - Deleting PVC: test-pvc-040b70c6 +2025-06-05 13:56:25,800 - tests.orchestrator - INFO - Waiting for PVC test-pvc-040b70c6 to be deleted +2025-06-05 13:56:25,866 - tests.orchestrator - INFO - PVC test-pvc-040b70c6 has been deleted +2025-06-05 13:56:25,866 - tests.orchestrator - INFO - Deleted PVC: test-pvc-040b70c6 +2025-06-05 13:56:25,866 - tests.orchestrator - INFO - Deleting PVC: test-pvc-e5eb6b62 +2025-06-05 13:56:25,940 - tests.orchestrator - INFO - Waiting for PVC test-pvc-e5eb6b62 to be deleted +2025-06-05 13:56:26,006 - tests.orchestrator - INFO - PVC test-pvc-e5eb6b62 has been deleted +2025-06-05 13:56:26,006 - tests.orchestrator - INFO - Deleted PVC: test-pvc-e5eb6b62 +2025-06-05 13:56:26,006 - tests.orchestrator - INFO - Deleting PVC: one2one-9268308a +2025-06-05 13:56:26,081 - tests.orchestrator - INFO - Waiting for PVC one2one-9268308a to be deleted +2025-06-05 13:56:26,146 - tests.orchestrator - INFO - PVC one2one-9268308a has been deleted +2025-06-05 13:56:26,146 - tests.orchestrator - INFO - Deleted PVC: one2one-9268308a +2025-06-05 13:56:26,146 - tests.orchestrator - INFO - Deleting PVC: one2one-24717940 +2025-06-05 13:56:26,221 - tests.orchestrator - INFO - Waiting for PVC one2one-24717940 to be deleted +2025-06-05 13:56:26,287 - tests.orchestrator - INFO - PVC one2one-24717940 has been deleted +2025-06-05 13:56:26,287 - tests.orchestrator - INFO - Deleted PVC: one2one-24717940 +2025-06-05 13:56:26,287 - tests.orchestrator - INFO - Cleanup completed +2025-06-05 13:56:26,287 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-05 13:56:26,287 - tests.orchestrator - INFO - --- Operations --- +2025-06-05 13:56:26,287 - tests.orchestrator - INFO - create_pvc: 2 succeeded, 0 failed (100.0%) +2025-06-05 13:56:26,287 - tests.orchestrator - INFO - attach_pod: 6 succeeded, 0 failed (100.0%) +2025-06-05 13:56:26,287 - tests.orchestrator - INFO - delete_pod: 6 succeeded, 0 failed (100.0%) +2025-06-05 13:56:26,287 - tests.orchestrator - INFO - delete_pvc: 5 succeeded, 0 failed (100.0%) +2025-06-05 13:56:26,287 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) +2025-06-05 13:56:26,287 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) +2025-06-05 13:56:26,287 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-05 13:56:26,287 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-05 13:56:26,287 - tests.orchestrator - INFO - many_to_one: No runs +2025-06-05 13:56:26,287 - tests.orchestrator - INFO - one_to_one: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-05 13:56:26,288 - tests.orchestrator - INFO - concurrent_pvc: No runs +2025-06-05 13:56:26,288 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_140408.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_140408.log new file mode 100644 index 000000000..51b5d693f --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_140408.log @@ -0,0 +1,138 @@ +2025-06-05 14:04:08,567 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 14:04:08,780 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 14:04:08,780 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-05 14:04:08,972 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 14:04:08,972 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 14:04:08,972 - tests.orchestrator - INFO - Creating PVC: test-pvc-847a8a56 +2025-06-05 14:04:09,057 - tests.orchestrator - INFO - Created PVC: test-pvc-847a8a56 +2025-06-05 14:04:09,057 - tests.orchestrator - INFO - Waiting for PVC test-pvc-847a8a56 to be bound +2025-06-05 14:04:11,188 - tests.orchestrator - INFO - PVC test-pvc-847a8a56 is bound +2025-06-05 14:04:11,189 - tests.orchestrator - INFO - Creating PVC: test-pvc-9dee4752 +2025-06-05 14:04:11,257 - tests.orchestrator - INFO - Created PVC: test-pvc-9dee4752 +2025-06-05 14:04:11,258 - tests.orchestrator - INFO - Waiting for PVC test-pvc-9dee4752 to be bound +2025-06-05 14:04:13,391 - tests.orchestrator - INFO - PVC test-pvc-9dee4752 is bound +2025-06-05 14:04:13,391 - tests.orchestrator - INFO - Creating pod test-pod-d215a119 on PVC test-pvc-9dee4752 +2025-06-05 14:04:13,474 - tests.orchestrator - INFO - Created pod: test-pod-d215a119 using PVC: test-pvc-9dee4752 +2025-06-05 14:04:13,474 - tests.orchestrator - INFO - Waiting for pod test-pod-d215a119 to be ready +2025-06-05 14:05:15,521 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-d215a119 to be ready after 60s +2025-06-05 14:05:15,521 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-d215a119 to be ready +2025-06-05 14:05:15,522 - tests.orchestrator - INFO - Creating pod test-pod-54364ba6 on PVC test-pvc-9dee4752 +2025-06-05 14:05:15,608 - tests.orchestrator - INFO - Created pod: test-pod-54364ba6 using PVC: test-pvc-9dee4752 +2025-06-05 14:05:15,608 - tests.orchestrator - INFO - Waiting for pod test-pod-54364ba6 to be ready +2025-06-05 14:06:15,655 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-54364ba6 to be ready after 60s +2025-06-05 14:06:15,655 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-54364ba6 to be ready +2025-06-05 14:06:15,655 - tests.orchestrator - INFO - Creating pod test-pod-98a62ebf on PVC test-pvc-9dee4752 +2025-06-05 14:06:15,742 - tests.orchestrator - INFO - Created pod: test-pod-98a62ebf using PVC: test-pvc-9dee4752 +2025-06-05 14:06:15,742 - tests.orchestrator - INFO - Waiting for pod test-pod-98a62ebf to be ready +2025-06-05 14:07:15,768 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-98a62ebf to be ready after 60s +2025-06-05 14:07:15,769 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-98a62ebf to be ready +2025-06-05 14:07:15,769 - tests.orchestrator - INFO - Testing read/write between pods test-pod-d215a119 and test-pod-98a62ebf sharing PVC test-pvc-9dee4752 +2025-06-05 14:07:15,836 - tests.orchestrator - ERROR - Failed in read/write verification: (400) +Reason: Bad Request +HTTP response headers: HTTPHeaderDict({'Audit-Id': '0fd252b7-6182-4f97-9a5a-1503836973da', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 14:07:15 GMT', 'Content-Length': '166'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-d215a119 does not have a host assigned","reason":"BadRequest","code":400} + + +2025-06-05 14:07:15,836 - tests.orchestrator - INFO - Running specific scenario: _scenario_one_to_one +2025-06-05 14:07:15,836 - tests.orchestrator - INFO - Running scenario: One pod per PVC +2025-06-05 14:07:15,836 - tests.orchestrator - INFO - Creating 4 PVC-pod pairs +2025-06-05 14:07:15,913 - tests.orchestrator - INFO - Waiting for PVC one2one-71f9fc22 to be bound +2025-06-05 14:07:18,066 - tests.orchestrator - INFO - PVC one2one-71f9fc22 is bound +2025-06-05 14:07:18,066 - tests.orchestrator - INFO - Creating pod test-pod-78c97483 on PVC one2one-71f9fc22 +2025-06-05 14:07:18,197 - tests.orchestrator - INFO - Created pod: test-pod-78c97483 using PVC: one2one-71f9fc22 +2025-06-05 14:07:18,197 - tests.orchestrator - INFO - Waiting for pod test-pod-78c97483 to be ready +2025-06-05 14:08:18,229 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-78c97483 to be ready after 60s +2025-06-05 14:08:18,229 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-78c97483 to be ready +2025-06-05 14:08:18,310 - tests.orchestrator - INFO - Waiting for PVC one2one-d55f8841 to be bound +2025-06-05 14:08:20,442 - tests.orchestrator - INFO - PVC one2one-d55f8841 is bound +2025-06-05 14:08:20,442 - tests.orchestrator - INFO - Creating pod test-pod-a400277a on PVC one2one-d55f8841 +2025-06-05 14:08:20,518 - tests.orchestrator - INFO - Created pod: test-pod-a400277a using PVC: one2one-d55f8841 +2025-06-05 14:08:20,519 - tests.orchestrator - INFO - Waiting for pod test-pod-a400277a to be ready +2025-06-05 14:09:22,561 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a400277a to be ready after 60s +2025-06-05 14:09:22,561 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a400277a to be ready +2025-06-05 14:09:22,638 - tests.orchestrator - INFO - Waiting for PVC one2one-f3257be9 to be bound +2025-06-05 14:09:24,772 - tests.orchestrator - INFO - PVC one2one-f3257be9 is bound +2025-06-05 14:09:24,773 - tests.orchestrator - INFO - Creating pod test-pod-1bfbc8b3 on PVC one2one-f3257be9 +2025-06-05 14:09:24,850 - tests.orchestrator - INFO - Created pod: test-pod-1bfbc8b3 using PVC: one2one-f3257be9 +2025-06-05 14:09:24,850 - tests.orchestrator - INFO - Waiting for pod test-pod-1bfbc8b3 to be ready +2025-06-05 14:10:26,912 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1bfbc8b3 to be ready after 60s +2025-06-05 14:10:26,913 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1bfbc8b3 to be ready +2025-06-05 14:10:26,992 - tests.orchestrator - INFO - Waiting for PVC one2one-d2044d7d to be bound +2025-06-05 14:10:29,127 - tests.orchestrator - INFO - PVC one2one-d2044d7d is bound +2025-06-05 14:10:29,127 - tests.orchestrator - INFO - Creating pod test-pod-6edcdac8 on PVC one2one-d2044d7d +2025-06-05 14:10:29,203 - tests.orchestrator - INFO - Created pod: test-pod-6edcdac8 using PVC: one2one-d2044d7d +2025-06-05 14:10:29,203 - tests.orchestrator - INFO - Waiting for pod test-pod-6edcdac8 to be ready +2025-06-05 14:11:29,292 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-6edcdac8 to be ready after 60s +2025-06-05 14:11:29,293 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-6edcdac8 to be ready +2025-06-05 14:11:29,293 - tests.orchestrator - WARNING - Failed to create enough PVC-pod pairs, only created 0 +2025-06-05 14:11:29,293 - tests.orchestrator - INFO - Deleting pod: test-pod-54364ba6 from PVC: test-pvc-9dee4752 +2025-06-05 14:11:29,378 - tests.orchestrator - INFO - Waiting for pod test-pod-54364ba6 to be deleted +2025-06-05 14:11:29,443 - tests.orchestrator - INFO - Pod test-pod-54364ba6 has been deleted +2025-06-05 14:11:29,443 - tests.orchestrator - INFO - Deleted pod: test-pod-54364ba6 +2025-06-05 14:11:29,443 - tests.orchestrator - INFO - Deleting PVC: one2one-71f9fc22 +2025-06-05 14:11:29,443 - tests.orchestrator - INFO - Deleting 1 pods using PVC one2one-71f9fc22 +2025-06-05 14:11:29,443 - tests.orchestrator - INFO - Deleting pod: test-pod-78c97483 from PVC: one2one-71f9fc22 +2025-06-05 14:11:29,521 - tests.orchestrator - INFO - Waiting for pod test-pod-78c97483 to be deleted +2025-06-05 14:11:29,588 - tests.orchestrator - INFO - Pod test-pod-78c97483 has been deleted +2025-06-05 14:11:29,588 - tests.orchestrator - INFO - Deleted pod: test-pod-78c97483 +2025-06-05 14:11:29,666 - tests.orchestrator - INFO - Waiting for PVC one2one-71f9fc22 to be deleted +2025-06-05 14:11:29,733 - tests.orchestrator - INFO - PVC one2one-71f9fc22 has been deleted +2025-06-05 14:11:29,733 - tests.orchestrator - INFO - Deleted PVC: one2one-71f9fc22 +2025-06-05 14:11:29,733 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations +2025-06-05 14:11:29,733 - tests.orchestrator - INFO - Test completed in 440.95 seconds +2025-06-05 14:11:29,733 - tests.orchestrator - INFO - Cleaning up resources... +2025-06-05 14:11:29,733 - tests.orchestrator - INFO - Deleting pod: test-pod-d215a119 from PVC: test-pvc-9dee4752 +2025-06-05 14:11:29,832 - tests.orchestrator - INFO - Waiting for pod test-pod-d215a119 to be deleted +2025-06-05 14:11:29,897 - tests.orchestrator - INFO - Pod test-pod-d215a119 has been deleted +2025-06-05 14:11:29,897 - tests.orchestrator - INFO - Deleted pod: test-pod-d215a119 +2025-06-05 14:11:29,897 - tests.orchestrator - INFO - Deleting pod: test-pod-98a62ebf from PVC: test-pvc-9dee4752 +2025-06-05 14:11:29,977 - tests.orchestrator - INFO - Waiting for pod test-pod-98a62ebf to be deleted +2025-06-05 14:11:30,044 - tests.orchestrator - INFO - Pod test-pod-98a62ebf has been deleted +2025-06-05 14:11:30,044 - tests.orchestrator - INFO - Deleted pod: test-pod-98a62ebf +2025-06-05 14:11:30,044 - tests.orchestrator - INFO - Deleting pod: test-pod-a400277a from PVC: one2one-d55f8841 +2025-06-05 14:11:30,134 - tests.orchestrator - INFO - Waiting for pod test-pod-a400277a to be deleted +2025-06-05 14:11:30,199 - tests.orchestrator - INFO - Pod test-pod-a400277a has been deleted +2025-06-05 14:11:30,199 - tests.orchestrator - INFO - Deleted pod: test-pod-a400277a +2025-06-05 14:11:30,199 - tests.orchestrator - INFO - Deleting pod: test-pod-1bfbc8b3 from PVC: one2one-f3257be9 +2025-06-05 14:11:30,279 - tests.orchestrator - INFO - Waiting for pod test-pod-1bfbc8b3 to be deleted +2025-06-05 14:11:30,346 - tests.orchestrator - INFO - Pod test-pod-1bfbc8b3 has been deleted +2025-06-05 14:11:30,346 - tests.orchestrator - INFO - Deleted pod: test-pod-1bfbc8b3 +2025-06-05 14:11:30,346 - tests.orchestrator - INFO - Deleting pod: test-pod-6edcdac8 from PVC: one2one-d2044d7d +2025-06-05 14:11:30,432 - tests.orchestrator - INFO - Waiting for pod test-pod-6edcdac8 to be deleted +2025-06-05 14:11:30,500 - tests.orchestrator - INFO - Pod test-pod-6edcdac8 has been deleted +2025-06-05 14:11:30,500 - tests.orchestrator - INFO - Deleted pod: test-pod-6edcdac8 +2025-06-05 14:11:30,500 - tests.orchestrator - INFO - Deleting PVC: test-pvc-847a8a56 +2025-06-05 14:11:30,572 - tests.orchestrator - INFO - Waiting for PVC test-pvc-847a8a56 to be deleted +2025-06-05 14:11:30,642 - tests.orchestrator - INFO - PVC test-pvc-847a8a56 has been deleted +2025-06-05 14:11:30,642 - tests.orchestrator - INFO - Deleted PVC: test-pvc-847a8a56 +2025-06-05 14:11:30,642 - tests.orchestrator - INFO - Deleting PVC: test-pvc-9dee4752 +2025-06-05 14:11:30,715 - tests.orchestrator - INFO - Waiting for PVC test-pvc-9dee4752 to be deleted +2025-06-05 14:11:30,782 - tests.orchestrator - INFO - PVC test-pvc-9dee4752 has been deleted +2025-06-05 14:11:30,782 - tests.orchestrator - INFO - Deleted PVC: test-pvc-9dee4752 +2025-06-05 14:11:30,782 - tests.orchestrator - INFO - Deleting PVC: one2one-d55f8841 +2025-06-05 14:11:30,860 - tests.orchestrator - INFO - Waiting for PVC one2one-d55f8841 to be deleted +2025-06-05 14:11:30,927 - tests.orchestrator - INFO - PVC one2one-d55f8841 has been deleted +2025-06-05 14:11:30,927 - tests.orchestrator - INFO - Deleted PVC: one2one-d55f8841 +2025-06-05 14:11:30,928 - tests.orchestrator - INFO - Deleting PVC: one2one-f3257be9 +2025-06-05 14:11:30,999 - tests.orchestrator - INFO - Waiting for PVC one2one-f3257be9 to be deleted +2025-06-05 14:11:31,068 - tests.orchestrator - INFO - PVC one2one-f3257be9 has been deleted +2025-06-05 14:11:31,068 - tests.orchestrator - INFO - Deleted PVC: one2one-f3257be9 +2025-06-05 14:11:31,068 - tests.orchestrator - INFO - Deleting PVC: one2one-d2044d7d +2025-06-05 14:11:31,142 - tests.orchestrator - INFO - Waiting for PVC one2one-d2044d7d to be deleted +2025-06-05 14:11:31,208 - tests.orchestrator - INFO - PVC one2one-d2044d7d has been deleted +2025-06-05 14:11:31,208 - tests.orchestrator - INFO - Deleted PVC: one2one-d2044d7d +2025-06-05 14:11:31,208 - tests.orchestrator - INFO - Cleanup completed +2025-06-05 14:11:31,208 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-05 14:11:31,208 - tests.orchestrator - INFO - --- Operations --- +2025-06-05 14:11:31,209 - tests.orchestrator - INFO - create_pvc: 2 succeeded, 0 failed (100.0%) +2025-06-05 14:11:31,209 - tests.orchestrator - INFO - attach_pod: 7 succeeded, 0 failed (100.0%) +2025-06-05 14:11:31,209 - tests.orchestrator - INFO - delete_pod: 7 succeeded, 0 failed (100.0%) +2025-06-05 14:11:31,209 - tests.orchestrator - INFO - delete_pvc: 6 succeeded, 0 failed (100.0%) +2025-06-05 14:11:31,209 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) +2025-06-05 14:11:31,209 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) +2025-06-05 14:11:31,209 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-05 14:11:31,209 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-05 14:11:31,209 - tests.orchestrator - INFO - many_to_one: No runs +2025-06-05 14:11:31,209 - tests.orchestrator - INFO - one_to_one: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-05 14:11:31,209 - tests.orchestrator - INFO - concurrent_pvc: No runs +2025-06-05 14:11:31,209 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_141321.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_141321.log new file mode 100644 index 000000000..cf6449873 --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_141321.log @@ -0,0 +1,7 @@ +2025-06-05 14:13:21,602 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 14:13:21,814 - tests.orchestrator - ERROR - Error checking namespace: (401) +Reason: Unauthorized +HTTP response headers: HTTPHeaderDict({'Audit-Id': 'eb0b1f42-f08c-4b18-b778-a83d8eb055f7', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 14:13:21 GMT', 'Content-Length': '129'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} + + diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_141434.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_141434.log new file mode 100644 index 000000000..2964606c2 --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_141434.log @@ -0,0 +1,142 @@ +2025-06-05 14:14:34,816 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 14:14:35,029 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 14:14:35,029 - tests.orchestrator - INFO - Starting orchestrator test for 60 seconds +2025-06-05 14:14:35,218 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 14:14:35,219 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 14:14:35,219 - tests.orchestrator - INFO - Creating PVC: test-pvc-0673a51e +2025-06-05 14:14:35,294 - tests.orchestrator - INFO - Created PVC: test-pvc-0673a51e +2025-06-05 14:14:35,295 - tests.orchestrator - INFO - Waiting for PVC test-pvc-0673a51e to be bound +2025-06-05 14:14:37,429 - tests.orchestrator - INFO - PVC test-pvc-0673a51e is bound +2025-06-05 14:14:37,429 - tests.orchestrator - INFO - Creating PVC: test-pvc-ad878159 +2025-06-05 14:14:37,497 - tests.orchestrator - INFO - Created PVC: test-pvc-ad878159 +2025-06-05 14:14:37,497 - tests.orchestrator - INFO - Waiting for PVC test-pvc-ad878159 to be bound +2025-06-05 14:14:39,631 - tests.orchestrator - INFO - PVC test-pvc-ad878159 is bound +2025-06-05 14:14:39,632 - tests.orchestrator - INFO - Creating pod test-pod-97d4c158 on PVC test-pvc-ad878159 +2025-06-05 14:14:39,709 - tests.orchestrator - INFO - Created pod: test-pod-97d4c158 using PVC: test-pvc-ad878159 +2025-06-05 14:14:39,710 - tests.orchestrator - INFO - Waiting for pod test-pod-97d4c158 to be ready +2025-06-05 14:15:39,755 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-97d4c158 to be ready after 60s +2025-06-05 14:15:39,755 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-97d4c158 to be ready +2025-06-05 14:15:39,756 - tests.orchestrator - INFO - Creating pod test-pod-27dfc967 on PVC test-pvc-ad878159 +2025-06-05 14:15:39,836 - tests.orchestrator - INFO - Created pod: test-pod-27dfc967 using PVC: test-pvc-ad878159 +2025-06-05 14:15:39,837 - tests.orchestrator - INFO - Waiting for pod test-pod-27dfc967 to be ready +2025-06-05 14:16:39,914 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-27dfc967 to be ready after 60s +2025-06-05 14:16:39,915 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-27dfc967 to be ready +2025-06-05 14:16:39,915 - tests.orchestrator - INFO - Creating pod test-pod-41d8fee0 on PVC test-pvc-0673a51e +2025-06-05 14:16:39,996 - tests.orchestrator - INFO - Created pod: test-pod-41d8fee0 using PVC: test-pvc-0673a51e +2025-06-05 14:16:39,996 - tests.orchestrator - INFO - Waiting for pod test-pod-41d8fee0 to be ready +2025-06-05 14:17:42,051 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-41d8fee0 to be ready after 60s +2025-06-05 14:17:42,051 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-41d8fee0 to be ready +2025-06-05 14:17:42,051 - tests.orchestrator - INFO - Testing read/write between pods test-pod-97d4c158 and test-pod-27dfc967 sharing PVC test-pvc-ad878159 +2025-06-05 14:17:42,118 - tests.orchestrator - ERROR - Failed in read/write verification: (400) +Reason: Bad Request +HTTP response headers: HTTPHeaderDict({'Audit-Id': '82f9299f-d78b-4a35-8447-4b9382646785', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 14:17:42 GMT', 'Content-Length': '166'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-97d4c158 does not have a host assigned","reason":"BadRequest","code":400} + + +2025-06-05 14:17:42,118 - tests.orchestrator - INFO - Running specific scenario: _scenario_concurrent_pvc +2025-06-05 14:17:42,118 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations +2025-06-05 14:17:42,118 - tests.orchestrator - INFO - Creating 5 PVCs in quick succession +2025-06-05 14:17:42,197 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-af92f77f +2025-06-05 14:17:42,197 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-af92f77f to be bound +2025-06-05 14:17:44,333 - tests.orchestrator - INFO - PVC concurrent-pvc-af92f77f is bound +2025-06-05 14:17:44,403 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-2a91a331 +2025-06-05 14:17:44,403 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-2a91a331 to be bound +2025-06-05 14:17:46,537 - tests.orchestrator - INFO - PVC concurrent-pvc-2a91a331 is bound +2025-06-05 14:17:46,606 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-b13c79e0 +2025-06-05 14:17:46,606 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-b13c79e0 to be bound +2025-06-05 14:17:48,740 - tests.orchestrator - INFO - PVC concurrent-pvc-b13c79e0 is bound +2025-06-05 14:17:48,810 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-2e52864c +2025-06-05 14:17:48,810 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-2e52864c to be bound +2025-06-05 14:17:50,943 - tests.orchestrator - INFO - PVC concurrent-pvc-2e52864c is bound +2025-06-05 14:17:51,013 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-83f53c18 +2025-06-05 14:17:51,014 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-83f53c18 to be bound +2025-06-05 14:17:53,147 - tests.orchestrator - INFO - PVC concurrent-pvc-83f53c18 is bound +2025-06-05 14:17:53,147 - tests.orchestrator - INFO - Successfully created 5 PVCs +2025-06-05 14:17:53,147 - tests.orchestrator - INFO - Creating pod test-pod-783e20b8 on PVC concurrent-pvc-2a91a331 +2025-06-05 14:17:53,221 - tests.orchestrator - INFO - Created pod: test-pod-783e20b8 using PVC: concurrent-pvc-2a91a331 +2025-06-05 14:17:53,221 - tests.orchestrator - INFO - Waiting for pod test-pod-783e20b8 to be ready +2025-06-05 14:18:53,279 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-783e20b8 to be ready after 60s +2025-06-05 14:18:53,279 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-783e20b8 to be ready +2025-06-05 14:18:53,279 - tests.orchestrator - INFO - Creating pod test-pod-206a8855 on PVC concurrent-pvc-b13c79e0 +2025-06-05 14:18:53,362 - tests.orchestrator - INFO - Created pod: test-pod-206a8855 using PVC: concurrent-pvc-b13c79e0 +2025-06-05 14:18:53,362 - tests.orchestrator - INFO - Waiting for pod test-pod-206a8855 to be ready +2025-06-05 14:19:55,415 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-206a8855 to be ready after 60s +2025-06-05 14:19:55,416 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-206a8855 to be ready +2025-06-05 14:19:55,416 - tests.orchestrator - INFO - Creating pod test-pod-7df5d0f2 on PVC concurrent-pvc-af92f77f +2025-06-05 14:19:55,499 - tests.orchestrator - INFO - Created pod: test-pod-7df5d0f2 using PVC: concurrent-pvc-af92f77f +2025-06-05 14:19:55,499 - tests.orchestrator - INFO - Waiting for pod test-pod-7df5d0f2 to be ready +2025-06-05 14:20:57,532 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-7df5d0f2 to be ready after 60s +2025-06-05 14:20:57,532 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-7df5d0f2 to be ready +2025-06-05 14:20:57,532 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-2e52864c +2025-06-05 14:20:57,607 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-2e52864c to be deleted +2025-06-05 14:20:57,673 - tests.orchestrator - INFO - PVC concurrent-pvc-2e52864c has been deleted +2025-06-05 14:20:57,674 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-2e52864c +2025-06-05 14:20:57,674 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-af92f77f +2025-06-05 14:20:57,675 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-af92f77f +2025-06-05 14:20:57,675 - tests.orchestrator - INFO - Deleting pod: test-pod-7df5d0f2 from PVC: concurrent-pvc-af92f77f +2025-06-05 14:20:57,761 - tests.orchestrator - INFO - Waiting for pod test-pod-7df5d0f2 to be deleted +2025-06-05 14:20:57,827 - tests.orchestrator - INFO - Pod test-pod-7df5d0f2 has been deleted +2025-06-05 14:20:57,827 - tests.orchestrator - INFO - Deleted pod: test-pod-7df5d0f2 +2025-06-05 14:20:57,901 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-af92f77f to be deleted +2025-06-05 14:20:57,969 - tests.orchestrator - INFO - PVC concurrent-pvc-af92f77f has been deleted +2025-06-05 14:20:57,969 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-af92f77f +2025-06-05 14:20:57,969 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-83f53c18 +2025-06-05 14:20:58,041 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-83f53c18 to be deleted +2025-06-05 14:20:58,108 - tests.orchestrator - INFO - PVC concurrent-pvc-83f53c18 has been deleted +2025-06-05 14:20:58,108 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-83f53c18 +2025-06-05 14:20:58,108 - tests.orchestrator - INFO - Rapid PVC scenario completed +2025-06-05 14:20:58,108 - tests.orchestrator - INFO - Deleting pod: test-pod-97d4c158 from PVC: test-pvc-ad878159 +2025-06-05 14:20:58,188 - tests.orchestrator - INFO - Waiting for pod test-pod-97d4c158 to be deleted +2025-06-05 14:20:58,254 - tests.orchestrator - INFO - Pod test-pod-97d4c158 has been deleted +2025-06-05 14:20:58,254 - tests.orchestrator - INFO - Deleted pod: test-pod-97d4c158 +2025-06-05 14:20:58,254 - tests.orchestrator - INFO - Deleting PVC: test-pvc-ad878159 +2025-06-05 14:20:58,255 - tests.orchestrator - INFO - Deleting 1 pods using PVC test-pvc-ad878159 +2025-06-05 14:20:58,255 - tests.orchestrator - INFO - Deleting pod: test-pod-27dfc967 from PVC: test-pvc-ad878159 +2025-06-05 14:20:58,341 - tests.orchestrator - INFO - Waiting for pod test-pod-27dfc967 to be deleted +2025-06-05 14:20:58,407 - tests.orchestrator - INFO - Pod test-pod-27dfc967 has been deleted +2025-06-05 14:20:58,407 - tests.orchestrator - INFO - Deleted pod: test-pod-27dfc967 +2025-06-05 14:20:58,479 - tests.orchestrator - INFO - Waiting for PVC test-pvc-ad878159 to be deleted +2025-06-05 14:20:58,546 - tests.orchestrator - INFO - PVC test-pvc-ad878159 has been deleted +2025-06-05 14:20:58,546 - tests.orchestrator - INFO - Deleted PVC: test-pvc-ad878159 +2025-06-05 14:20:58,546 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations +2025-06-05 14:20:58,546 - tests.orchestrator - INFO - Test completed in 383.52 seconds +2025-06-05 14:20:58,547 - tests.orchestrator - INFO - Cleaning up resources... +2025-06-05 14:20:58,547 - tests.orchestrator - INFO - Deleting pod: test-pod-41d8fee0 from PVC: test-pvc-0673a51e +2025-06-05 14:20:58,633 - tests.orchestrator - INFO - Waiting for pod test-pod-41d8fee0 to be deleted +2025-06-05 14:20:58,700 - tests.orchestrator - INFO - Pod test-pod-41d8fee0 has been deleted +2025-06-05 14:20:58,700 - tests.orchestrator - INFO - Deleted pod: test-pod-41d8fee0 +2025-06-05 14:20:58,700 - tests.orchestrator - INFO - Deleting pod: test-pod-783e20b8 from PVC: concurrent-pvc-2a91a331 +2025-06-05 14:20:58,781 - tests.orchestrator - INFO - Waiting for pod test-pod-783e20b8 to be deleted +2025-06-05 14:20:58,847 - tests.orchestrator - INFO - Pod test-pod-783e20b8 has been deleted +2025-06-05 14:20:58,847 - tests.orchestrator - INFO - Deleted pod: test-pod-783e20b8 +2025-06-05 14:20:58,847 - tests.orchestrator - INFO - Deleting pod: test-pod-206a8855 from PVC: concurrent-pvc-b13c79e0 +2025-06-05 14:20:58,925 - tests.orchestrator - INFO - Waiting for pod test-pod-206a8855 to be deleted +2025-06-05 14:20:58,991 - tests.orchestrator - INFO - Pod test-pod-206a8855 has been deleted +2025-06-05 14:20:58,992 - tests.orchestrator - INFO - Deleted pod: test-pod-206a8855 +2025-06-05 14:20:58,992 - tests.orchestrator - INFO - Deleting PVC: test-pvc-0673a51e +2025-06-05 14:20:59,063 - tests.orchestrator - INFO - Waiting for PVC test-pvc-0673a51e to be deleted +2025-06-05 14:20:59,130 - tests.orchestrator - INFO - PVC test-pvc-0673a51e has been deleted +2025-06-05 14:20:59,130 - tests.orchestrator - INFO - Deleted PVC: test-pvc-0673a51e +2025-06-05 14:20:59,130 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-2a91a331 +2025-06-05 14:20:59,204 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-2a91a331 to be deleted +2025-06-05 14:20:59,270 - tests.orchestrator - INFO - PVC concurrent-pvc-2a91a331 has been deleted +2025-06-05 14:20:59,270 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-2a91a331 +2025-06-05 14:20:59,270 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-b13c79e0 +2025-06-05 14:20:59,344 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-b13c79e0 to be deleted +2025-06-05 14:20:59,410 - tests.orchestrator - INFO - PVC concurrent-pvc-b13c79e0 has been deleted +2025-06-05 14:20:59,410 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-b13c79e0 +2025-06-05 14:20:59,410 - tests.orchestrator - INFO - Cleanup completed +2025-06-05 14:20:59,410 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-05 14:20:59,410 - tests.orchestrator - INFO - --- Operations --- +2025-06-05 14:20:59,410 - tests.orchestrator - INFO - create_pvc: 7 succeeded, 0 failed (100.0%) +2025-06-05 14:20:59,410 - tests.orchestrator - INFO - attach_pod: 6 succeeded, 0 failed (100.0%) +2025-06-05 14:20:59,410 - tests.orchestrator - INFO - delete_pod: 6 succeeded, 0 failed (100.0%) +2025-06-05 14:20:59,410 - tests.orchestrator - INFO - delete_pvc: 7 succeeded, 0 failed (100.0%) +2025-06-05 14:20:59,410 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) +2025-06-05 14:20:59,411 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) +2025-06-05 14:20:59,411 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-05 14:20:59,411 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-05 14:20:59,411 - tests.orchestrator - INFO - many_to_one: No runs +2025-06-05 14:20:59,411 - tests.orchestrator - INFO - one_to_one: No runs +2025-06-05 14:20:59,411 - tests.orchestrator - INFO - concurrent_pvc: 1 succeeded, 0 failed out of 1 runs (100.0%) +2025-06-05 14:20:59,411 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_143926.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_143926.log new file mode 100644 index 000000000..35607442a --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_143926.log @@ -0,0 +1,73 @@ +2025-06-05 14:39:26,410 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 14:39:26,657 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 14:39:26,657 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-05 14:39:26,852 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 14:39:26,853 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 14:39:26,853 - tests.orchestrator - INFO - Creating PVC: test-pvc-e5985cb1 +2025-06-05 14:39:26,928 - tests.orchestrator - INFO - Created PVC: test-pvc-e5985cb1 +2025-06-05 14:39:26,928 - tests.orchestrator - INFO - Waiting for PVC test-pvc-e5985cb1 to be bound +2025-06-05 14:39:29,061 - tests.orchestrator - INFO - PVC test-pvc-e5985cb1 is bound +2025-06-05 14:39:29,061 - tests.orchestrator - INFO - Creating PVC: test-pvc-83426f91 +2025-06-05 14:39:29,130 - tests.orchestrator - INFO - Created PVC: test-pvc-83426f91 +2025-06-05 14:39:29,131 - tests.orchestrator - INFO - Waiting for PVC test-pvc-83426f91 to be bound +2025-06-05 14:39:31,262 - tests.orchestrator - INFO - PVC test-pvc-83426f91 is bound +2025-06-05 14:39:31,263 - tests.orchestrator - INFO - Creating pod test-pod-d86424ba on PVC test-pvc-83426f91 +2025-06-05 14:39:31,342 - tests.orchestrator - INFO - Created pod: test-pod-d86424ba using PVC: test-pvc-83426f91 +2025-06-05 14:39:31,342 - tests.orchestrator - INFO - Waiting for pod test-pod-d86424ba to be ready +2025-06-05 14:40:31,371 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-d86424ba to be ready after 60s +2025-06-05 14:40:31,371 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-d86424ba to be ready +2025-06-05 14:40:31,371 - tests.orchestrator - INFO - Creating pod test-pod-b7e9d0a0 on PVC test-pvc-e5985cb1 +2025-06-05 14:40:31,454 - tests.orchestrator - INFO - Created pod: test-pod-b7e9d0a0 using PVC: test-pvc-e5985cb1 +2025-06-05 14:40:31,455 - tests.orchestrator - INFO - Waiting for pod test-pod-b7e9d0a0 to be ready +2025-06-05 14:41:31,485 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b7e9d0a0 to be ready after 60s +2025-06-05 14:41:31,485 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b7e9d0a0 to be ready +2025-06-05 14:41:31,485 - tests.orchestrator - INFO - Creating pod test-pod-c947d819 on PVC test-pvc-e5985cb1 +2025-06-05 14:41:31,586 - tests.orchestrator - INFO - Created pod: test-pod-c947d819 using PVC: test-pvc-e5985cb1 +2025-06-05 14:41:31,586 - tests.orchestrator - INFO - Waiting for pod test-pod-c947d819 to be ready +2025-06-05 14:42:33,647 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-c947d819 to be ready after 60s +2025-06-05 14:42:33,647 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-c947d819 to be ready +2025-06-05 14:42:33,647 - tests.orchestrator - INFO - Testing read/write between pods test-pod-b7e9d0a0 and test-pod-c947d819 sharing PVC test-pvc-e5985cb1 +2025-06-05 14:42:33,714 - tests.orchestrator - ERROR - Failed in read/write verification: (400) +Reason: Bad Request +HTTP response headers: HTTPHeaderDict({'Audit-Id': '8b1165bc-48b9-433e-8654-68c6095da9bd', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 14:42:33 GMT', 'Content-Length': '166'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-b7e9d0a0 does not have a host assigned","reason":"BadRequest","code":400} + + +2025-06-05 14:42:33,714 - tests.orchestrator - INFO - Running specific scenario: _scenario_concurrent_pvc +2025-06-05 14:42:33,714 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations +2025-06-05 14:42:33,714 - tests.orchestrator - INFO - Creating 7 PVCs in quick succession +2025-06-05 14:42:33,792 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-f6a663b2 +2025-06-05 14:42:33,792 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-f6a663b2 to be bound +2025-06-05 14:42:35,924 - tests.orchestrator - INFO - PVC concurrent-pvc-f6a663b2 is bound +2025-06-05 14:42:35,993 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-903eb13a +2025-06-05 14:42:35,994 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-903eb13a to be bound +2025-06-05 14:42:38,128 - tests.orchestrator - INFO - PVC concurrent-pvc-903eb13a is bound +2025-06-05 14:42:38,196 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-ba7e5e4d +2025-06-05 14:42:38,196 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-ba7e5e4d to be bound +2025-06-05 14:42:40,331 - tests.orchestrator - INFO - PVC concurrent-pvc-ba7e5e4d is bound +2025-06-05 14:42:40,397 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-5b29e9cd +2025-06-05 14:42:40,397 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-5b29e9cd to be bound +2025-06-05 14:42:42,533 - tests.orchestrator - INFO - PVC concurrent-pvc-5b29e9cd is bound +2025-06-05 14:42:42,603 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-248d7584 +2025-06-05 14:42:42,603 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-248d7584 to be bound +2025-06-05 14:42:44,740 - tests.orchestrator - INFO - PVC concurrent-pvc-248d7584 is bound +2025-06-05 14:42:44,808 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-0aa045b0 +2025-06-05 14:42:44,808 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-0aa045b0 to be bound +2025-06-05 14:42:46,944 - tests.orchestrator - INFO - PVC concurrent-pvc-0aa045b0 is bound +2025-06-05 14:42:47,013 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-b0004a21 +2025-06-05 14:42:47,013 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-b0004a21 to be bound +2025-06-05 14:42:49,147 - tests.orchestrator - INFO - PVC concurrent-pvc-b0004a21 is bound +2025-06-05 14:42:49,148 - tests.orchestrator - INFO - Successfully created 7 PVCs +2025-06-05 14:42:49,148 - tests.orchestrator - INFO - Creating pod test-pod-edd264c5 on PVC concurrent-pvc-0aa045b0 +2025-06-05 14:42:49,220 - tests.orchestrator - INFO - Created pod: test-pod-edd264c5 using PVC: concurrent-pvc-0aa045b0 +2025-06-05 14:42:49,220 - tests.orchestrator - INFO - Waiting for pod test-pod-edd264c5 to be ready +2025-06-05 14:43:49,221 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-edd264c5 to be ready after 60s +2025-06-05 14:43:49,221 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-edd264c5 to be ready +2025-06-05 14:43:49,221 - tests.orchestrator - INFO - Creating pod test-pod-9f0f38ef on PVC concurrent-pvc-248d7584 +2025-06-05 14:43:49,306 - tests.orchestrator - INFO - Created pod: test-pod-9f0f38ef using PVC: concurrent-pvc-248d7584 +2025-06-05 14:43:49,306 - tests.orchestrator - INFO - Waiting for pod test-pod-9f0f38ef to be ready +2025-06-05 14:44:51,350 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9f0f38ef to be ready after 60s +2025-06-05 14:44:51,351 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9f0f38ef to be ready +2025-06-05 14:44:51,351 - tests.orchestrator - INFO - Creating pod test-pod-6bde4264 on PVC concurrent-pvc-903eb13a +2025-06-05 14:44:51,437 - tests.orchestrator - INFO - Created pod: test-pod-6bde4264 using PVC: concurrent-pvc-903eb13a +2025-06-05 14:44:51,437 - tests.orchestrator - INFO - Waiting for pod test-pod-6bde4264 to be ready diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_145133.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_145133.log new file mode 100644 index 000000000..a89fdcdab --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_145133.log @@ -0,0 +1,111 @@ +2025-06-05 14:51:33,341 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 14:51:33,563 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 14:51:33,563 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-05 14:51:33,753 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 14:51:33,753 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 14:51:33,754 - tests.orchestrator - INFO - Creating PVC: test-pvc-49392d7c +2025-06-05 14:51:33,831 - tests.orchestrator - INFO - Created PVC: test-pvc-49392d7c +2025-06-05 14:51:33,831 - tests.orchestrator - INFO - Waiting for PVC test-pvc-49392d7c to be bound +2025-06-05 14:51:35,964 - tests.orchestrator - INFO - PVC test-pvc-49392d7c is bound +2025-06-05 14:51:35,966 - tests.orchestrator - INFO - Creating PVC: test-pvc-5e912838 +2025-06-05 14:51:36,038 - tests.orchestrator - INFO - Created PVC: test-pvc-5e912838 +2025-06-05 14:51:36,038 - tests.orchestrator - INFO - Waiting for PVC test-pvc-5e912838 to be bound +2025-06-05 14:51:38,171 - tests.orchestrator - INFO - PVC test-pvc-5e912838 is bound +2025-06-05 14:51:38,171 - tests.orchestrator - INFO - Creating pod test-pod-a16fe7e2 on PVC test-pvc-5e912838 +2025-06-05 14:51:38,249 - tests.orchestrator - INFO - Created pod: test-pod-a16fe7e2 using PVC: test-pvc-5e912838 +2025-06-05 14:51:38,250 - tests.orchestrator - INFO - Waiting for pod test-pod-a16fe7e2 to be ready +2025-06-05 14:52:40,287 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a16fe7e2 to be ready after 60s +2025-06-05 14:52:40,287 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a16fe7e2 to be ready +2025-06-05 14:52:40,288 - tests.orchestrator - INFO - Creating pod test-pod-6cc653e7 on PVC test-pvc-49392d7c +2025-06-05 14:52:40,370 - tests.orchestrator - INFO - Created pod: test-pod-6cc653e7 using PVC: test-pvc-49392d7c +2025-06-05 14:52:40,370 - tests.orchestrator - INFO - Waiting for pod test-pod-6cc653e7 to be ready +2025-06-05 14:53:42,411 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-6cc653e7 to be ready after 60s +2025-06-05 14:53:42,411 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-6cc653e7 to be ready +2025-06-05 14:53:42,411 - tests.orchestrator - INFO - Creating pod test-pod-807f64e5 on PVC test-pvc-49392d7c +2025-06-05 14:53:42,494 - tests.orchestrator - INFO - Created pod: test-pod-807f64e5 using PVC: test-pvc-49392d7c +2025-06-05 14:53:42,494 - tests.orchestrator - INFO - Waiting for pod test-pod-807f64e5 to be ready +2025-06-05 14:54:44,529 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-807f64e5 to be ready after 60s +2025-06-05 14:54:44,529 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-807f64e5 to be ready +2025-06-05 14:54:44,529 - tests.orchestrator - INFO - Testing read/write between pods test-pod-807f64e5 and test-pod-6cc653e7 sharing PVC test-pvc-49392d7c +2025-06-05 14:54:44,596 - tests.orchestrator - ERROR - Failed in read/write verification: (400) +Reason: Bad Request +HTTP response headers: HTTPHeaderDict({'Audit-Id': '12fe6240-5ebe-4564-8452-9306b897c940', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 14:54:44 GMT', 'Content-Length': '166'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-807f64e5 does not have a host assigned","reason":"BadRequest","code":400} + + +2025-06-05 14:54:44,596 - tests.orchestrator - INFO - Running specific scenario: _scenario_many_to_one +2025-06-05 14:54:44,597 - tests.orchestrator - INFO - Running scenario: Many pods to one PVC +2025-06-05 14:54:44,678 - tests.orchestrator - INFO - Waiting for PVC many2one-7a10cfd9 to be bound +2025-06-05 14:54:46,810 - tests.orchestrator - INFO - PVC many2one-7a10cfd9 is bound +2025-06-05 14:54:46,811 - tests.orchestrator - INFO - Creating 3 pods for the same PVC +2025-06-05 14:54:46,811 - tests.orchestrator - INFO - Creating pod test-pod-6d648a79 on PVC many2one-7a10cfd9 +2025-06-05 14:54:46,884 - tests.orchestrator - INFO - Created pod: test-pod-6d648a79 using PVC: many2one-7a10cfd9 +2025-06-05 14:54:46,884 - tests.orchestrator - INFO - Waiting for pod test-pod-6d648a79 to be ready +2025-06-05 14:55:48,914 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-6d648a79 to be ready after 60s +2025-06-05 14:55:48,915 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-6d648a79 to be ready +2025-06-05 14:55:48,915 - tests.orchestrator - INFO - Creating pod test-pod-264dcb36 on PVC many2one-7a10cfd9 +2025-06-05 14:55:48,997 - tests.orchestrator - INFO - Created pod: test-pod-264dcb36 using PVC: many2one-7a10cfd9 +2025-06-05 14:55:48,997 - tests.orchestrator - INFO - Waiting for pod test-pod-264dcb36 to be ready +2025-06-05 14:56:49,023 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-264dcb36 to be ready after 60s +2025-06-05 14:56:49,024 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-264dcb36 to be ready +2025-06-05 14:56:49,024 - tests.orchestrator - INFO - Creating pod test-pod-b6a62203 on PVC many2one-7a10cfd9 +2025-06-05 14:56:49,106 - tests.orchestrator - INFO - Created pod: test-pod-b6a62203 using PVC: many2one-7a10cfd9 +2025-06-05 14:56:49,106 - tests.orchestrator - INFO - Waiting for pod test-pod-b6a62203 to be ready +2025-06-05 14:57:49,129 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b6a62203 to be ready after 60s +2025-06-05 14:57:49,129 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b6a62203 to be ready +2025-06-05 14:57:49,129 - tests.orchestrator - WARNING - Failed to create enough pods for scenario, only created 0 +2025-06-05 14:57:49,130 - tests.orchestrator - INFO - Deleting pod: test-pod-6cc653e7 from PVC: test-pvc-49392d7c +2025-06-05 14:57:49,207 - tests.orchestrator - INFO - Waiting for pod test-pod-6cc653e7 to be deleted +2025-06-05 14:57:49,272 - tests.orchestrator - INFO - Pod test-pod-6cc653e7 has been deleted +2025-06-05 14:57:49,272 - tests.orchestrator - INFO - Deleted pod: test-pod-6cc653e7 +2025-06-05 14:57:49,272 - tests.orchestrator - INFO - Deleting PVC: many2one-7a10cfd9 +2025-06-05 14:57:49,272 - tests.orchestrator - INFO - Deleting 3 pods using PVC many2one-7a10cfd9 +2025-06-05 14:57:49,272 - tests.orchestrator - INFO - Deleting pod: test-pod-6d648a79 from PVC: many2one-7a10cfd9 +2025-06-05 14:57:49,348 - tests.orchestrator - INFO - Waiting for pod test-pod-6d648a79 to be deleted +2025-06-05 14:57:49,414 - tests.orchestrator - INFO - Pod test-pod-6d648a79 has been deleted +2025-06-05 14:57:49,415 - tests.orchestrator - INFO - Deleted pod: test-pod-6d648a79 +2025-06-05 14:57:49,415 - tests.orchestrator - INFO - Deleting pod: test-pod-264dcb36 from PVC: many2one-7a10cfd9 +2025-06-05 14:57:49,498 - tests.orchestrator - INFO - Waiting for pod test-pod-264dcb36 to be deleted +2025-06-05 14:57:49,563 - tests.orchestrator - INFO - Pod test-pod-264dcb36 has been deleted +2025-06-05 14:57:49,564 - tests.orchestrator - INFO - Deleted pod: test-pod-264dcb36 +2025-06-05 14:57:49,564 - tests.orchestrator - INFO - Deleting pod: test-pod-b6a62203 from PVC: many2one-7a10cfd9 +2025-06-05 14:57:49,643 - tests.orchestrator - INFO - Waiting for pod test-pod-b6a62203 to be deleted +2025-06-05 14:57:49,710 - tests.orchestrator - INFO - Pod test-pod-b6a62203 has been deleted +2025-06-05 14:57:49,710 - tests.orchestrator - INFO - Deleted pod: test-pod-b6a62203 +2025-06-05 14:57:49,782 - tests.orchestrator - INFO - Waiting for PVC many2one-7a10cfd9 to be deleted +2025-06-05 14:57:49,848 - tests.orchestrator - INFO - PVC many2one-7a10cfd9 has been deleted +2025-06-05 14:57:49,848 - tests.orchestrator - INFO - Deleted PVC: many2one-7a10cfd9 +2025-06-05 14:57:49,848 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations +2025-06-05 14:57:49,849 - tests.orchestrator - INFO - Test completed in 376.28 seconds +2025-06-05 14:57:49,849 - tests.orchestrator - INFO - Cleaning up resources... +2025-06-05 14:57:49,849 - tests.orchestrator - INFO - Deleting pod: test-pod-807f64e5 from PVC: test-pvc-49392d7c +2025-06-05 14:57:49,935 - tests.orchestrator - INFO - Waiting for pod test-pod-807f64e5 to be deleted +2025-06-05 14:57:50,000 - tests.orchestrator - INFO - Pod test-pod-807f64e5 has been deleted +2025-06-05 14:57:50,001 - tests.orchestrator - INFO - Deleted pod: test-pod-807f64e5 +2025-06-05 14:57:50,001 - tests.orchestrator - INFO - Deleting pod: test-pod-a16fe7e2 from PVC: test-pvc-5e912838 +2025-06-05 14:57:50,081 - tests.orchestrator - INFO - Waiting for pod test-pod-a16fe7e2 to be deleted +2025-06-05 14:57:50,147 - tests.orchestrator - INFO - Pod test-pod-a16fe7e2 has been deleted +2025-06-05 14:57:50,147 - tests.orchestrator - INFO - Deleted pod: test-pod-a16fe7e2 +2025-06-05 14:57:50,147 - tests.orchestrator - INFO - Deleting PVC: test-pvc-49392d7c +2025-06-05 14:57:50,221 - tests.orchestrator - INFO - Waiting for PVC test-pvc-49392d7c to be deleted +2025-06-05 14:57:50,287 - tests.orchestrator - INFO - PVC test-pvc-49392d7c has been deleted +2025-06-05 14:57:50,288 - tests.orchestrator - INFO - Deleted PVC: test-pvc-49392d7c +2025-06-05 14:57:50,288 - tests.orchestrator - INFO - Deleting PVC: test-pvc-5e912838 +2025-06-05 14:57:50,360 - tests.orchestrator - INFO - Waiting for PVC test-pvc-5e912838 to be deleted +2025-06-05 14:57:50,426 - tests.orchestrator - INFO - PVC test-pvc-5e912838 has been deleted +2025-06-05 14:57:50,426 - tests.orchestrator - INFO - Deleted PVC: test-pvc-5e912838 +2025-06-05 14:57:50,426 - tests.orchestrator - INFO - Cleanup completed +2025-06-05 14:57:50,426 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-05 14:57:50,426 - tests.orchestrator - INFO - --- Operations --- +2025-06-05 14:57:50,426 - tests.orchestrator - INFO - create_pvc: 2 succeeded, 0 failed (100.0%) +2025-06-05 14:57:50,426 - tests.orchestrator - INFO - attach_pod: 6 succeeded, 0 failed (100.0%) +2025-06-05 14:57:50,427 - tests.orchestrator - INFO - delete_pod: 6 succeeded, 0 failed (100.0%) +2025-06-05 14:57:50,427 - tests.orchestrator - INFO - delete_pvc: 3 succeeded, 0 failed (100.0%) +2025-06-05 14:57:50,427 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) +2025-06-05 14:57:50,427 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) +2025-06-05 14:57:50,427 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-05 14:57:50,427 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-05 14:57:50,427 - tests.orchestrator - INFO - many_to_one: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-05 14:57:50,427 - tests.orchestrator - INFO - one_to_one: No runs +2025-06-05 14:57:50,427 - tests.orchestrator - INFO - concurrent_pvc: No runs +2025-06-05 14:57:50,427 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_145937.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_145937.log new file mode 100644 index 000000000..5711d3fb8 --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_145937.log @@ -0,0 +1,134 @@ +2025-06-05 14:59:37,825 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 14:59:38,037 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 14:59:38,038 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-05 14:59:38,227 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 14:59:38,227 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 14:59:38,227 - tests.orchestrator - INFO - Creating PVC: test-pvc-fe252ee6 +2025-06-05 14:59:38,302 - tests.orchestrator - INFO - Created PVC: test-pvc-fe252ee6 +2025-06-05 14:59:38,303 - tests.orchestrator - INFO - Waiting for PVC test-pvc-fe252ee6 to be bound +2025-06-05 14:59:40,435 - tests.orchestrator - INFO - PVC test-pvc-fe252ee6 is bound +2025-06-05 14:59:40,435 - tests.orchestrator - INFO - Creating PVC: test-pvc-4a7f89b4 +2025-06-05 14:59:40,502 - tests.orchestrator - INFO - Created PVC: test-pvc-4a7f89b4 +2025-06-05 14:59:40,502 - tests.orchestrator - INFO - Waiting for PVC test-pvc-4a7f89b4 to be bound +2025-06-05 14:59:42,634 - tests.orchestrator - INFO - PVC test-pvc-4a7f89b4 is bound +2025-06-05 14:59:42,635 - tests.orchestrator - INFO - Creating pod test-pod-5311b1cc on PVC test-pvc-4a7f89b4 +2025-06-05 14:59:42,712 - tests.orchestrator - INFO - Created pod: test-pod-5311b1cc using PVC: test-pvc-4a7f89b4 +2025-06-05 14:59:42,712 - tests.orchestrator - INFO - Waiting for pod test-pod-5311b1cc to be ready +2025-06-05 15:00:44,719 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-5311b1cc to be ready after 60s +2025-06-05 15:00:44,719 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-5311b1cc to be ready +2025-06-05 15:00:44,720 - tests.orchestrator - INFO - Creating pod test-pod-71d7f585 on PVC test-pvc-fe252ee6 +2025-06-05 15:00:44,803 - tests.orchestrator - INFO - Created pod: test-pod-71d7f585 using PVC: test-pvc-fe252ee6 +2025-06-05 15:00:44,803 - tests.orchestrator - INFO - Waiting for pod test-pod-71d7f585 to be ready +2025-06-05 15:01:46,811 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-71d7f585 to be ready after 60s +2025-06-05 15:01:46,811 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-71d7f585 to be ready +2025-06-05 15:01:46,812 - tests.orchestrator - INFO - Creating pod test-pod-cc8d2266 on PVC test-pvc-4a7f89b4 +2025-06-05 15:01:46,893 - tests.orchestrator - INFO - Created pod: test-pod-cc8d2266 using PVC: test-pvc-4a7f89b4 +2025-06-05 15:01:46,894 - tests.orchestrator - INFO - Waiting for pod test-pod-cc8d2266 to be ready +2025-06-05 15:02:48,905 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-cc8d2266 to be ready after 60s +2025-06-05 15:02:48,905 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-cc8d2266 to be ready +2025-06-05 15:02:48,905 - tests.orchestrator - INFO - Testing read/write between pods test-pod-5311b1cc and test-pod-cc8d2266 sharing PVC test-pvc-4a7f89b4 +2025-06-05 15:02:48,971 - tests.orchestrator - ERROR - Failed in read/write verification: (400) +Reason: Bad Request +HTTP response headers: HTTPHeaderDict({'Audit-Id': '6fd01cb2-cb35-43e3-8bf2-4cba5eccfd8d', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 15:02:48 GMT', 'Content-Length': '166'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-5311b1cc does not have a host assigned","reason":"BadRequest","code":400} + + +2025-06-05 15:02:48,971 - tests.orchestrator - INFO - ============================================================ +2025-06-05 15:02:48,971 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_concurrent_pvc +2025-06-05 15:02:48,971 - tests.orchestrator - INFO - ============================================================ +2025-06-05 15:02:48,972 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations +2025-06-05 15:02:48,972 - tests.orchestrator - INFO - Creating 3 PVCs in quick succession +2025-06-05 15:02:49,049 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-02666bec +2025-06-05 15:02:49,049 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-02666bec to be bound +2025-06-05 15:02:51,181 - tests.orchestrator - INFO - PVC concurrent-pvc-02666bec is bound +2025-06-05 15:02:51,248 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-6592afb4 +2025-06-05 15:02:51,249 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-6592afb4 to be bound +2025-06-05 15:02:53,380 - tests.orchestrator - INFO - PVC concurrent-pvc-6592afb4 is bound +2025-06-05 15:02:53,448 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-2386ec8a +2025-06-05 15:02:53,449 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-2386ec8a to be bound +2025-06-05 15:02:55,580 - tests.orchestrator - INFO - PVC concurrent-pvc-2386ec8a is bound +2025-06-05 15:02:55,580 - tests.orchestrator - INFO - Successfully created 3 PVCs +2025-06-05 15:02:55,580 - tests.orchestrator - INFO - Creating pod test-pod-27ffffcb on PVC concurrent-pvc-2386ec8a +2025-06-05 15:02:55,653 - tests.orchestrator - INFO - Created pod: test-pod-27ffffcb using PVC: concurrent-pvc-2386ec8a +2025-06-05 15:02:55,653 - tests.orchestrator - INFO - Waiting for pod test-pod-27ffffcb to be ready +2025-06-05 15:03:57,662 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-27ffffcb to be ready after 60s +2025-06-05 15:03:57,662 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-27ffffcb to be ready +2025-06-05 15:03:57,662 - tests.orchestrator - INFO - Creating pod test-pod-00a03fb8 on PVC concurrent-pvc-02666bec +2025-06-05 15:03:57,743 - tests.orchestrator - INFO - Created pod: test-pod-00a03fb8 using PVC: concurrent-pvc-02666bec +2025-06-05 15:03:57,744 - tests.orchestrator - INFO - Waiting for pod test-pod-00a03fb8 to be ready +2025-06-05 15:04:59,752 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-00a03fb8 to be ready after 60s +2025-06-05 15:04:59,753 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-00a03fb8 to be ready +2025-06-05 15:04:59,753 - tests.orchestrator - INFO - Creating pod test-pod-87658b4a on PVC concurrent-pvc-6592afb4 +2025-06-05 15:04:59,832 - tests.orchestrator - INFO - Created pod: test-pod-87658b4a using PVC: concurrent-pvc-6592afb4 +2025-06-05 15:04:59,832 - tests.orchestrator - INFO - Waiting for pod test-pod-87658b4a to be ready +2025-06-05 15:06:01,847 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-87658b4a to be ready after 60s +2025-06-05 15:06:01,847 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-87658b4a to be ready +2025-06-05 15:06:01,847 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-6592afb4 +2025-06-05 15:06:01,847 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-6592afb4 +2025-06-05 15:06:01,847 - tests.orchestrator - INFO - Deleting pod: test-pod-87658b4a from PVC: concurrent-pvc-6592afb4 +2025-06-05 15:06:01,968 - tests.orchestrator - INFO - Waiting for pod test-pod-87658b4a to be deleted +2025-06-05 15:06:02,040 - tests.orchestrator - INFO - Pod test-pod-87658b4a has been deleted +2025-06-05 15:06:02,041 - tests.orchestrator - INFO - Deleted pod: test-pod-87658b4a +2025-06-05 15:06:02,126 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-6592afb4 to be deleted +2025-06-05 15:06:02,196 - tests.orchestrator - INFO - PVC concurrent-pvc-6592afb4 has been deleted +2025-06-05 15:06:02,196 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-6592afb4 +2025-06-05 15:06:02,196 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-02666bec +2025-06-05 15:06:02,196 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-02666bec +2025-06-05 15:06:02,196 - tests.orchestrator - INFO - Deleting pod: test-pod-00a03fb8 from PVC: concurrent-pvc-02666bec +2025-06-05 15:06:02,295 - tests.orchestrator - INFO - Waiting for pod test-pod-00a03fb8 to be deleted +2025-06-05 15:06:02,359 - tests.orchestrator - INFO - Pod test-pod-00a03fb8 has been deleted +2025-06-05 15:06:02,360 - tests.orchestrator - INFO - Deleted pod: test-pod-00a03fb8 +2025-06-05 15:06:02,433 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-02666bec to be deleted +2025-06-05 15:06:02,497 - tests.orchestrator - INFO - PVC concurrent-pvc-02666bec has been deleted +2025-06-05 15:06:02,498 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-02666bec +2025-06-05 15:06:02,498 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-2386ec8a +2025-06-05 15:06:02,498 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-2386ec8a +2025-06-05 15:06:02,498 - tests.orchestrator - INFO - Deleting pod: test-pod-27ffffcb from PVC: concurrent-pvc-2386ec8a +2025-06-05 15:06:02,575 - tests.orchestrator - INFO - Waiting for pod test-pod-27ffffcb to be deleted +2025-06-05 15:06:02,640 - tests.orchestrator - INFO - Pod test-pod-27ffffcb has been deleted +2025-06-05 15:06:02,640 - tests.orchestrator - INFO - Deleted pod: test-pod-27ffffcb +2025-06-05 15:06:02,713 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-2386ec8a to be deleted +2025-06-05 15:06:02,777 - tests.orchestrator - INFO - PVC concurrent-pvc-2386ec8a has been deleted +2025-06-05 15:06:02,777 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-2386ec8a +2025-06-05 15:06:02,778 - tests.orchestrator - INFO - Rapid PVC scenario completed +2025-06-05 15:06:02,778 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_concurrent_pvc +2025-06-05 15:06:02,778 - tests.orchestrator - INFO - ------------------------------------------------------------ +2025-06-05 15:06:02,778 - tests.orchestrator - INFO - Deleting pod: test-pod-71d7f585 from PVC: test-pvc-fe252ee6 +2025-06-05 15:06:02,870 - tests.orchestrator - INFO - Waiting for pod test-pod-71d7f585 to be deleted +2025-06-05 15:06:02,935 - tests.orchestrator - INFO - Pod test-pod-71d7f585 has been deleted +2025-06-05 15:06:02,935 - tests.orchestrator - INFO - Deleted pod: test-pod-71d7f585 +2025-06-05 15:06:02,935 - tests.orchestrator - INFO - Deleting PVC: test-pvc-4a7f89b4 +2025-06-05 15:06:02,935 - tests.orchestrator - INFO - Deleting 2 pods using PVC test-pvc-4a7f89b4 +2025-06-05 15:06:02,935 - tests.orchestrator - INFO - Deleting pod: test-pod-5311b1cc from PVC: test-pvc-4a7f89b4 +2025-06-05 15:06:03,025 - tests.orchestrator - INFO - Waiting for pod test-pod-5311b1cc to be deleted +2025-06-05 15:06:03,090 - tests.orchestrator - INFO - Pod test-pod-5311b1cc has been deleted +2025-06-05 15:06:03,090 - tests.orchestrator - INFO - Deleted pod: test-pod-5311b1cc +2025-06-05 15:06:03,090 - tests.orchestrator - INFO - Deleting pod: test-pod-cc8d2266 from PVC: test-pvc-4a7f89b4 +2025-06-05 15:06:03,172 - tests.orchestrator - INFO - Waiting for pod test-pod-cc8d2266 to be deleted +2025-06-05 15:06:03,238 - tests.orchestrator - INFO - Pod test-pod-cc8d2266 has been deleted +2025-06-05 15:06:03,238 - tests.orchestrator - INFO - Deleted pod: test-pod-cc8d2266 +2025-06-05 15:06:03,311 - tests.orchestrator - INFO - Waiting for PVC test-pvc-4a7f89b4 to be deleted +2025-06-05 15:06:03,376 - tests.orchestrator - INFO - PVC test-pvc-4a7f89b4 has been deleted +2025-06-05 15:06:03,376 - tests.orchestrator - INFO - Deleted PVC: test-pvc-4a7f89b4 +2025-06-05 15:06:03,376 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations +2025-06-05 15:06:03,376 - tests.orchestrator - INFO - Test completed in 385.34 seconds +2025-06-05 15:06:03,376 - tests.orchestrator - INFO - Cleaning up resources... +2025-06-05 15:06:03,376 - tests.orchestrator - INFO - Deleting PVC: test-pvc-fe252ee6 +2025-06-05 15:06:03,447 - tests.orchestrator - INFO - Waiting for PVC test-pvc-fe252ee6 to be deleted +2025-06-05 15:06:03,512 - tests.orchestrator - INFO - PVC test-pvc-fe252ee6 has been deleted +2025-06-05 15:06:03,512 - tests.orchestrator - INFO - Deleted PVC: test-pvc-fe252ee6 +2025-06-05 15:06:03,512 - tests.orchestrator - INFO - Cleanup completed +2025-06-05 15:06:03,512 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-05 15:06:03,513 - tests.orchestrator - INFO - --- Operations --- +2025-06-05 15:06:03,513 - tests.orchestrator - INFO - create_pvc: 5 succeeded, 0 failed (100.0%) +2025-06-05 15:06:03,513 - tests.orchestrator - INFO - attach_pod: 6 succeeded, 0 failed (100.0%) +2025-06-05 15:06:03,513 - tests.orchestrator - INFO - delete_pod: 6 succeeded, 0 failed (100.0%) +2025-06-05 15:06:03,513 - tests.orchestrator - INFO - delete_pvc: 5 succeeded, 0 failed (100.0%) +2025-06-05 15:06:03,513 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) +2025-06-05 15:06:03,513 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) +2025-06-05 15:06:03,513 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-05 15:06:03,513 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-05 15:06:03,513 - tests.orchestrator - INFO - many_to_one: No runs +2025-06-05 15:06:03,513 - tests.orchestrator - INFO - one_to_one: No runs +2025-06-05 15:06:03,513 - tests.orchestrator - INFO - concurrent_pvc: 1 succeeded, 0 failed out of 1 runs (100.0%) +2025-06-05 15:06:03,513 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_150619.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_150619.log new file mode 100644 index 000000000..78223d473 --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_150619.log @@ -0,0 +1,147 @@ +2025-06-05 15:06:19,870 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 15:06:20,072 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 15:06:20,072 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-05 15:06:20,260 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 15:06:20,261 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 15:06:20,261 - tests.orchestrator - INFO - Creating PVC: test-pvc-7aa57f8c +2025-06-05 15:06:20,336 - tests.orchestrator - INFO - Created PVC: test-pvc-7aa57f8c +2025-06-05 15:06:20,336 - tests.orchestrator - INFO - Waiting for PVC test-pvc-7aa57f8c to be bound +2025-06-05 15:06:22,470 - tests.orchestrator - INFO - PVC test-pvc-7aa57f8c is bound +2025-06-05 15:06:22,471 - tests.orchestrator - INFO - Creating PVC: test-pvc-afe1c1f0 +2025-06-05 15:06:22,538 - tests.orchestrator - INFO - Created PVC: test-pvc-afe1c1f0 +2025-06-05 15:06:22,538 - tests.orchestrator - INFO - Waiting for PVC test-pvc-afe1c1f0 to be bound +2025-06-05 15:06:24,671 - tests.orchestrator - INFO - PVC test-pvc-afe1c1f0 is bound +2025-06-05 15:06:24,671 - tests.orchestrator - INFO - Creating pod test-pod-d8058d9f on PVC test-pvc-7aa57f8c +2025-06-05 15:06:24,755 - tests.orchestrator - INFO - Created pod: test-pod-d8058d9f using PVC: test-pvc-7aa57f8c +2025-06-05 15:06:24,755 - tests.orchestrator - INFO - Waiting for pod test-pod-d8058d9f to be ready +2025-06-05 15:07:26,793 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-d8058d9f to be ready after 60s +2025-06-05 15:07:26,793 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-d8058d9f to be ready +2025-06-05 15:07:26,793 - tests.orchestrator - INFO - Creating pod test-pod-fdb5f5f7 on PVC test-pvc-afe1c1f0 +2025-06-05 15:07:26,879 - tests.orchestrator - INFO - Created pod: test-pod-fdb5f5f7 using PVC: test-pvc-afe1c1f0 +2025-06-05 15:07:26,880 - tests.orchestrator - INFO - Waiting for pod test-pod-fdb5f5f7 to be ready +2025-06-05 15:08:28,914 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-fdb5f5f7 to be ready after 60s +2025-06-05 15:08:28,915 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-fdb5f5f7 to be ready +2025-06-05 15:08:28,915 - tests.orchestrator - INFO - Creating pod test-pod-415052fa on PVC test-pvc-7aa57f8c +2025-06-05 15:08:28,998 - tests.orchestrator - INFO - Created pod: test-pod-415052fa using PVC: test-pvc-7aa57f8c +2025-06-05 15:08:28,998 - tests.orchestrator - INFO - Waiting for pod test-pod-415052fa to be ready +2025-06-05 15:09:31,031 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-415052fa to be ready after 60s +2025-06-05 15:09:31,031 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-415052fa to be ready +2025-06-05 15:09:31,031 - tests.orchestrator - INFO - Testing read/write between pods test-pod-d8058d9f and test-pod-415052fa sharing PVC test-pvc-7aa57f8c +2025-06-05 15:09:31,098 - tests.orchestrator - ERROR - Failed in read/write verification: (400) +Reason: Bad Request +HTTP response headers: HTTPHeaderDict({'Audit-Id': '9f5a799b-0705-4213-aaf5-1218c7e9571a', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 15:09:31 GMT', 'Content-Length': '166'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-d8058d9f does not have a host assigned","reason":"BadRequest","code":400} + + +2025-06-05 15:09:31,098 - tests.orchestrator - INFO - ============================================================ +2025-06-05 15:09:31,098 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_concurrent_pvc +2025-06-05 15:09:31,098 - tests.orchestrator - INFO - ============================================================ +2025-06-05 15:09:31,098 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations +2025-06-05 15:09:31,099 - tests.orchestrator - INFO - Creating 5 PVCs in quick succession +2025-06-05 15:09:31,177 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-a396d6c2 +2025-06-05 15:09:31,178 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-a396d6c2 to be bound +2025-06-05 15:09:33,310 - tests.orchestrator - INFO - PVC concurrent-pvc-a396d6c2 is bound +2025-06-05 15:09:33,381 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-85edcaa9 +2025-06-05 15:09:33,381 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-85edcaa9 to be bound +2025-06-05 15:09:35,516 - tests.orchestrator - INFO - PVC concurrent-pvc-85edcaa9 is bound +2025-06-05 15:09:35,583 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-bb7b8fe6 +2025-06-05 15:09:35,583 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-bb7b8fe6 to be bound +2025-06-05 15:09:37,719 - tests.orchestrator - INFO - PVC concurrent-pvc-bb7b8fe6 is bound +2025-06-05 15:09:37,790 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-a3d1cb36 +2025-06-05 15:09:37,790 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-a3d1cb36 to be bound +2025-06-05 15:09:39,924 - tests.orchestrator - INFO - PVC concurrent-pvc-a3d1cb36 is bound +2025-06-05 15:09:39,993 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-e1cb5975 +2025-06-05 15:09:39,994 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-e1cb5975 to be bound +2025-06-05 15:09:42,273 - tests.orchestrator - INFO - PVC concurrent-pvc-e1cb5975 is bound +2025-06-05 15:09:42,273 - tests.orchestrator - INFO - Successfully created 5 PVCs +2025-06-05 15:09:42,274 - tests.orchestrator - INFO - Creating pod test-pod-1ea724b8 on PVC concurrent-pvc-a3d1cb36 +2025-06-05 15:09:42,449 - tests.orchestrator - INFO - Created pod: test-pod-1ea724b8 using PVC: concurrent-pvc-a3d1cb36 +2025-06-05 15:09:42,449 - tests.orchestrator - INFO - Waiting for pod test-pod-1ea724b8 to be ready +2025-06-05 15:10:42,631 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1ea724b8 to be ready after 60s +2025-06-05 15:10:42,631 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1ea724b8 to be ready +2025-06-05 15:10:42,631 - tests.orchestrator - INFO - Creating pod test-pod-7f110176 on PVC concurrent-pvc-e1cb5975 +2025-06-05 15:10:42,732 - tests.orchestrator - INFO - Created pod: test-pod-7f110176 using PVC: concurrent-pvc-e1cb5975 +2025-06-05 15:10:42,732 - tests.orchestrator - INFO - Waiting for pod test-pod-7f110176 to be ready +2025-06-05 15:11:42,757 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-7f110176 to be ready after 60s +2025-06-05 15:11:42,757 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-7f110176 to be ready +2025-06-05 15:11:42,757 - tests.orchestrator - INFO - Creating pod test-pod-f48c7103 on PVC concurrent-pvc-85edcaa9 +2025-06-05 15:11:42,854 - tests.orchestrator - INFO - Created pod: test-pod-f48c7103 using PVC: concurrent-pvc-85edcaa9 +2025-06-05 15:11:42,854 - tests.orchestrator - INFO - Waiting for pod test-pod-f48c7103 to be ready +2025-06-05 15:12:44,885 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-f48c7103 to be ready after 60s +2025-06-05 15:12:44,886 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-f48c7103 to be ready +2025-06-05 15:12:44,886 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-a3d1cb36 +2025-06-05 15:12:44,886 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-a3d1cb36 +2025-06-05 15:12:44,886 - tests.orchestrator - INFO - Deleting pod: test-pod-1ea724b8 from PVC: concurrent-pvc-a3d1cb36 +2025-06-05 15:12:44,967 - tests.orchestrator - INFO - Waiting for pod test-pod-1ea724b8 to be deleted +2025-06-05 15:12:45,035 - tests.orchestrator - INFO - Pod test-pod-1ea724b8 has been deleted +2025-06-05 15:12:45,035 - tests.orchestrator - INFO - Deleted pod: test-pod-1ea724b8 +2025-06-05 15:12:45,120 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-a3d1cb36 to be deleted +2025-06-05 15:12:45,197 - tests.orchestrator - INFO - PVC concurrent-pvc-a3d1cb36 has been deleted +2025-06-05 15:12:45,198 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-a3d1cb36 +2025-06-05 15:12:45,198 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-85edcaa9 +2025-06-05 15:12:45,198 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-85edcaa9 +2025-06-05 15:12:45,198 - tests.orchestrator - INFO - Deleting pod: test-pod-f48c7103 from PVC: concurrent-pvc-85edcaa9 +2025-06-05 15:12:45,285 - tests.orchestrator - INFO - Waiting for pod test-pod-f48c7103 to be deleted +2025-06-05 15:12:45,353 - tests.orchestrator - INFO - Pod test-pod-f48c7103 has been deleted +2025-06-05 15:12:45,353 - tests.orchestrator - INFO - Deleted pod: test-pod-f48c7103 +2025-06-05 15:12:45,426 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-85edcaa9 to be deleted +2025-06-05 15:12:45,492 - tests.orchestrator - INFO - PVC concurrent-pvc-85edcaa9 has been deleted +2025-06-05 15:12:45,493 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-85edcaa9 +2025-06-05 15:12:45,493 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-a396d6c2 +2025-06-05 15:12:45,566 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-a396d6c2 to be deleted +2025-06-05 15:12:45,632 - tests.orchestrator - INFO - PVC concurrent-pvc-a396d6c2 has been deleted +2025-06-05 15:12:45,632 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-a396d6c2 +2025-06-05 15:12:45,633 - tests.orchestrator - INFO - Rapid PVC scenario completed +2025-06-05 15:12:45,633 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_concurrent_pvc +2025-06-05 15:12:45,633 - tests.orchestrator - INFO - ------------------------------------------------------------ +2025-06-05 15:12:45,633 - tests.orchestrator - INFO - Deleting pod: test-pod-fdb5f5f7 from PVC: test-pvc-afe1c1f0 +2025-06-05 15:12:45,715 - tests.orchestrator - INFO - Waiting for pod test-pod-fdb5f5f7 to be deleted +2025-06-05 15:12:45,783 - tests.orchestrator - INFO - Pod test-pod-fdb5f5f7 has been deleted +2025-06-05 15:12:45,783 - tests.orchestrator - INFO - Deleted pod: test-pod-fdb5f5f7 +2025-06-05 15:12:45,783 - tests.orchestrator - INFO - Deleting PVC: test-pvc-7aa57f8c +2025-06-05 15:12:45,783 - tests.orchestrator - INFO - Deleting 2 pods using PVC test-pvc-7aa57f8c +2025-06-05 15:12:45,784 - tests.orchestrator - INFO - Deleting pod: test-pod-d8058d9f from PVC: test-pvc-7aa57f8c +2025-06-05 15:12:45,867 - tests.orchestrator - INFO - Waiting for pod test-pod-d8058d9f to be deleted +2025-06-05 15:12:45,934 - tests.orchestrator - INFO - Pod test-pod-d8058d9f has been deleted +2025-06-05 15:12:45,934 - tests.orchestrator - INFO - Deleted pod: test-pod-d8058d9f +2025-06-05 15:12:45,934 - tests.orchestrator - INFO - Deleting pod: test-pod-415052fa from PVC: test-pvc-7aa57f8c +2025-06-05 15:12:46,015 - tests.orchestrator - INFO - Waiting for pod test-pod-415052fa to be deleted +2025-06-05 15:12:46,079 - tests.orchestrator - INFO - Pod test-pod-415052fa has been deleted +2025-06-05 15:12:46,080 - tests.orchestrator - INFO - Deleted pod: test-pod-415052fa +2025-06-05 15:12:46,154 - tests.orchestrator - INFO - Waiting for PVC test-pvc-7aa57f8c to be deleted +2025-06-05 15:12:46,220 - tests.orchestrator - INFO - PVC test-pvc-7aa57f8c has been deleted +2025-06-05 15:12:46,220 - tests.orchestrator - INFO - Deleted PVC: test-pvc-7aa57f8c +2025-06-05 15:12:46,220 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations +2025-06-05 15:12:46,221 - tests.orchestrator - INFO - Test completed in 386.15 seconds +2025-06-05 15:12:46,221 - tests.orchestrator - INFO - Cleaning up resources... +2025-06-05 15:12:46,221 - tests.orchestrator - INFO - Deleting pod: test-pod-7f110176 from PVC: concurrent-pvc-e1cb5975 +2025-06-05 15:12:46,308 - tests.orchestrator - INFO - Waiting for pod test-pod-7f110176 to be deleted +2025-06-05 15:12:46,378 - tests.orchestrator - INFO - Pod test-pod-7f110176 has been deleted +2025-06-05 15:12:46,379 - tests.orchestrator - INFO - Deleted pod: test-pod-7f110176 +2025-06-05 15:12:46,379 - tests.orchestrator - INFO - Deleting PVC: test-pvc-afe1c1f0 +2025-06-05 15:12:46,452 - tests.orchestrator - INFO - Waiting for PVC test-pvc-afe1c1f0 to be deleted +2025-06-05 15:12:46,519 - tests.orchestrator - INFO - PVC test-pvc-afe1c1f0 has been deleted +2025-06-05 15:12:46,519 - tests.orchestrator - INFO - Deleted PVC: test-pvc-afe1c1f0 +2025-06-05 15:12:46,519 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-bb7b8fe6 +2025-06-05 15:12:46,593 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-bb7b8fe6 to be deleted +2025-06-05 15:12:46,660 - tests.orchestrator - INFO - PVC concurrent-pvc-bb7b8fe6 has been deleted +2025-06-05 15:12:46,660 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-bb7b8fe6 +2025-06-05 15:12:46,660 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-e1cb5975 +2025-06-05 15:12:46,735 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-e1cb5975 to be deleted +2025-06-05 15:12:46,805 - tests.orchestrator - INFO - PVC concurrent-pvc-e1cb5975 has been deleted +2025-06-05 15:12:46,805 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-e1cb5975 +2025-06-05 15:12:46,809 - tests.orchestrator - INFO - Cleanup completed +2025-06-05 15:12:46,809 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-05 15:12:46,809 - tests.orchestrator - INFO - --- Operations --- +2025-06-05 15:12:46,809 - tests.orchestrator - INFO - create_pvc: 7 succeeded, 0 failed (100.0%) +2025-06-05 15:12:46,809 - tests.orchestrator - INFO - attach_pod: 6 succeeded, 0 failed (100.0%) +2025-06-05 15:12:46,809 - tests.orchestrator - INFO - delete_pod: 6 succeeded, 0 failed (100.0%) +2025-06-05 15:12:46,809 - tests.orchestrator - INFO - delete_pvc: 7 succeeded, 0 failed (100.0%) +2025-06-05 15:12:46,809 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) +2025-06-05 15:12:46,809 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) +2025-06-05 15:12:46,809 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-05 15:12:46,809 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-05 15:12:46,809 - tests.orchestrator - INFO - many_to_one: No runs +2025-06-05 15:12:46,809 - tests.orchestrator - INFO - one_to_one: No runs +2025-06-05 15:12:46,809 - tests.orchestrator - INFO - concurrent_pvc: 1 succeeded, 0 failed out of 1 runs (100.0%) +2025-06-05 15:12:46,809 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_151529.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_151529.log new file mode 100644 index 000000000..fb9e26ae4 --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_151529.log @@ -0,0 +1,7 @@ +2025-06-05 15:15:29,738 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 15:15:29,948 - tests.orchestrator - ERROR - Error checking namespace: (401) +Reason: Unauthorized +HTTP response headers: HTTPHeaderDict({'Audit-Id': '95b9f34d-de93-4da4-99dd-fc17111bd0ce', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 15:15:29 GMT', 'Content-Length': '129'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} + + diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_151604.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_151604.log new file mode 100644 index 000000000..ead869f56 --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_151604.log @@ -0,0 +1,42 @@ +2025-06-05 15:16:04,962 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 15:16:05,167 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 15:16:05,168 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-05 15:16:05,359 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 15:16:05,359 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 15:16:05,359 - tests.orchestrator - INFO - Creating PVC: test-pvc-20f868e5 +2025-06-05 15:16:05,438 - tests.orchestrator - INFO - Created PVC: test-pvc-20f868e5 +2025-06-05 15:16:05,438 - tests.orchestrator - INFO - Waiting for PVC test-pvc-20f868e5 to be bound +2025-06-05 15:16:07,570 - tests.orchestrator - INFO - PVC test-pvc-20f868e5 is bound +2025-06-05 15:16:07,570 - tests.orchestrator - INFO - Creating PVC: test-pvc-48e3d2d9 +2025-06-05 15:16:07,639 - tests.orchestrator - INFO - Created PVC: test-pvc-48e3d2d9 +2025-06-05 15:16:07,639 - tests.orchestrator - INFO - Waiting for PVC test-pvc-48e3d2d9 to be bound +2025-06-05 15:16:09,773 - tests.orchestrator - INFO - PVC test-pvc-48e3d2d9 is bound +2025-06-05 15:16:09,773 - tests.orchestrator - INFO - Creating pod test-pod-03b0f7fd on PVC test-pvc-20f868e5 +2025-06-05 15:16:09,857 - tests.orchestrator - INFO - Created pod: test-pod-03b0f7fd using PVC: test-pvc-20f868e5 +2025-06-05 15:16:09,857 - tests.orchestrator - INFO - Waiting for pod test-pod-03b0f7fd to be ready +2025-06-05 15:16:09,926 - tests.orchestrator - INFO - Pod test-pod-03b0f7fd phase: Pending +2025-06-05 15:16:40,974 - tests.orchestrator - INFO - Pod test-pod-03b0f7fd taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 15:16:40,974 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-03b0f7fd ===== +2025-06-05 15:16:41,041 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:16:41,124 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:16:41,125 - tests.orchestrator - INFO - [2025-06-05 15:16:09+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:16:41,199 - tests.orchestrator - INFO - No logs available +2025-06-05 15:16:41,199 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:16:41,199 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-20f868e5'} +2025-06-05 15:16:41,199 - tests.orchestrator - INFO - - kube-api-access-f6t7l: {} +2025-06-05 15:16:41,199 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-03b0f7fd ===== +2025-06-05 15:17:10,102 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-03b0f7fd to be ready after 60s +2025-06-05 15:17:10,102 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-03b0f7fd ===== +2025-06-05 15:17:10,172 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:17:10,245 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:17:10,245 - tests.orchestrator - INFO - [2025-06-05 15:16:09+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:17:10,312 - tests.orchestrator - INFO - No logs available +2025-06-05 15:17:10,312 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:17:10,312 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-20f868e5'} +2025-06-05 15:17:10,312 - tests.orchestrator - INFO - - kube-api-access-f6t7l: {} +2025-06-05 15:17:10,312 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-03b0f7fd ===== +2025-06-05 15:17:10,313 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-03b0f7fd to be ready +2025-06-05 15:17:10,313 - tests.orchestrator - INFO - Creating pod test-pod-264015e0 on PVC test-pvc-20f868e5 +2025-06-05 15:17:10,399 - tests.orchestrator - INFO - Created pod: test-pod-264015e0 using PVC: test-pvc-20f868e5 +2025-06-05 15:17:10,399 - tests.orchestrator - INFO - Waiting for pod test-pod-264015e0 to be ready +2025-06-05 15:17:10,467 - tests.orchestrator - INFO - Pod test-pod-264015e0 phase: Pending diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_151742.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_151742.log new file mode 100644 index 000000000..596cad530 --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_151742.log @@ -0,0 +1,17 @@ +2025-06-05 15:17:42,550 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 15:17:42,784 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 15:17:42,784 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-05 15:17:42,975 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 15:17:42,976 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 15:17:42,976 - tests.orchestrator - INFO - Creating PVC: test-pvc-b967555b +2025-06-05 15:17:43,061 - tests.orchestrator - INFO - Created PVC: test-pvc-b967555b +2025-06-05 15:17:43,061 - tests.orchestrator - INFO - Waiting for PVC test-pvc-b967555b to be bound +2025-06-05 15:17:45,204 - tests.orchestrator - INFO - PVC test-pvc-b967555b is bound +2025-06-05 15:17:45,205 - tests.orchestrator - INFO - Creating PVC: test-pvc-6282a942 +2025-06-05 15:17:45,276 - tests.orchestrator - INFO - Created PVC: test-pvc-6282a942 +2025-06-05 15:17:45,276 - tests.orchestrator - INFO - Waiting for PVC test-pvc-6282a942 to be bound +2025-06-05 15:17:47,412 - tests.orchestrator - INFO - PVC test-pvc-6282a942 is bound +2025-06-05 15:17:47,412 - tests.orchestrator - INFO - Creating pod test-pod-9ef8c6d5 on PVC test-pvc-b967555b +2025-06-05 15:17:47,490 - tests.orchestrator - INFO - Created pod: test-pod-9ef8c6d5 using PVC: test-pvc-b967555b +2025-06-05 15:17:47,491 - tests.orchestrator - INFO - Waiting for pod test-pod-9ef8c6d5 to be ready +2025-06-05 15:17:47,559 - tests.orchestrator - INFO - Pod test-pod-9ef8c6d5 phase: Pending diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_153750.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_153750.log new file mode 100644 index 000000000..4fba39f91 --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_153750.log @@ -0,0 +1,249 @@ +2025-06-05 15:37:50,853 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 15:37:51,073 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 15:37:51,074 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-05 15:37:51,263 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 15:37:51,263 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 15:37:51,263 - tests.orchestrator - INFO - Creating PVC: test-pvc-011557a1 +2025-06-05 15:37:51,341 - tests.orchestrator - INFO - Created PVC: test-pvc-011557a1 +2025-06-05 15:37:51,341 - tests.orchestrator - INFO - Waiting for PVC test-pvc-011557a1 to be bound +2025-06-05 15:37:53,474 - tests.orchestrator - INFO - PVC test-pvc-011557a1 is bound +2025-06-05 15:37:53,477 - tests.orchestrator - INFO - Creating PVC: test-pvc-b52b04ee +2025-06-05 15:37:53,544 - tests.orchestrator - INFO - Created PVC: test-pvc-b52b04ee +2025-06-05 15:37:53,545 - tests.orchestrator - INFO - Waiting for PVC test-pvc-b52b04ee to be bound +2025-06-05 15:37:55,677 - tests.orchestrator - INFO - PVC test-pvc-b52b04ee is bound +2025-06-05 15:37:55,677 - tests.orchestrator - INFO - Creating pod test-pod-1f8e6665 on PVC test-pvc-b52b04ee +2025-06-05 15:37:55,757 - tests.orchestrator - INFO - Created pod: test-pod-1f8e6665 using PVC: test-pvc-b52b04ee +2025-06-05 15:37:55,757 - tests.orchestrator - INFO - Waiting for pod test-pod-1f8e6665 to be ready +2025-06-05 15:37:55,827 - tests.orchestrator - INFO - Pod test-pod-1f8e6665 phase: Pending +2025-06-05 15:38:26,850 - tests.orchestrator - INFO - Pod test-pod-1f8e6665 taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 15:38:26,850 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-1f8e6665 ===== +2025-06-05 15:38:26,917 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:38:26,987 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:38:26,987 - tests.orchestrator - INFO - [2025-06-05 15:37:55+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:38:27,053 - tests.orchestrator - INFO - No logs available +2025-06-05 15:38:27,053 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:38:27,054 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-b52b04ee'} +2025-06-05 15:38:27,054 - tests.orchestrator - INFO - - kube-api-access-z4xbc: {} +2025-06-05 15:38:27,054 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-1f8e6665 ===== +2025-06-05 15:38:55,926 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1f8e6665 to be ready after 60s +2025-06-05 15:38:55,927 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-1f8e6665 ===== +2025-06-05 15:38:55,995 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:38:56,064 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:38:56,064 - tests.orchestrator - INFO - [2025-06-05 15:37:55+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:38:56,129 - tests.orchestrator - INFO - No logs available +2025-06-05 15:38:56,129 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:38:56,129 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-b52b04ee'} +2025-06-05 15:38:56,129 - tests.orchestrator - INFO - - kube-api-access-z4xbc: {} +2025-06-05 15:38:56,129 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-1f8e6665 ===== +2025-06-05 15:38:56,130 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1f8e6665 to be ready +2025-06-05 15:38:56,130 - tests.orchestrator - INFO - Creating pod test-pod-43a71634 on PVC test-pvc-b52b04ee +2025-06-05 15:38:56,213 - tests.orchestrator - INFO - Created pod: test-pod-43a71634 using PVC: test-pvc-b52b04ee +2025-06-05 15:38:56,213 - tests.orchestrator - INFO - Waiting for pod test-pod-43a71634 to be ready +2025-06-05 15:38:56,283 - tests.orchestrator - INFO - Pod test-pod-43a71634 phase: Pending +2025-06-05 15:39:27,290 - tests.orchestrator - INFO - Pod test-pod-43a71634 taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 15:39:27,290 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-43a71634 ===== +2025-06-05 15:39:27,357 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:39:27,428 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:39:27,428 - tests.orchestrator - INFO - [2025-06-05 15:38:56+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:39:27,493 - tests.orchestrator - INFO - No logs available +2025-06-05 15:39:27,494 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:39:27,494 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-b52b04ee'} +2025-06-05 15:39:27,494 - tests.orchestrator - INFO - - kube-api-access-fgg2x: {} +2025-06-05 15:39:27,494 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-43a71634 ===== +2025-06-05 15:39:56,469 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-43a71634 to be ready after 60s +2025-06-05 15:39:56,469 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-43a71634 ===== +2025-06-05 15:39:56,538 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:39:56,609 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:39:56,609 - tests.orchestrator - INFO - [2025-06-05 15:38:56+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:39:56,674 - tests.orchestrator - INFO - No logs available +2025-06-05 15:39:56,674 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:39:56,674 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-b52b04ee'} +2025-06-05 15:39:56,674 - tests.orchestrator - INFO - - kube-api-access-fgg2x: {} +2025-06-05 15:39:56,675 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-43a71634 ===== +2025-06-05 15:39:56,675 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-43a71634 to be ready +2025-06-05 15:39:56,675 - tests.orchestrator - INFO - Creating pod test-pod-9785ae47 on PVC test-pvc-011557a1 +2025-06-05 15:39:56,757 - tests.orchestrator - INFO - Created pod: test-pod-9785ae47 using PVC: test-pvc-011557a1 +2025-06-05 15:39:56,757 - tests.orchestrator - INFO - Waiting for pod test-pod-9785ae47 to be ready +2025-06-05 15:39:56,823 - tests.orchestrator - INFO - Pod test-pod-9785ae47 phase: Pending +2025-06-05 15:40:27,832 - tests.orchestrator - INFO - Pod test-pod-9785ae47 taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 15:40:27,832 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-9785ae47 ===== +2025-06-05 15:40:27,899 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:40:27,970 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:40:27,970 - tests.orchestrator - INFO - [2025-06-05 15:39:56+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:40:28,037 - tests.orchestrator - INFO - No logs available +2025-06-05 15:40:28,037 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:40:28,037 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-011557a1'} +2025-06-05 15:40:28,037 - tests.orchestrator - INFO - - kube-api-access-p2phq: {} +2025-06-05 15:40:28,037 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-9785ae47 ===== +2025-06-05 15:40:56,911 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9785ae47 to be ready after 60s +2025-06-05 15:40:56,912 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-9785ae47 ===== +2025-06-05 15:40:56,980 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:40:57,050 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:40:57,050 - tests.orchestrator - INFO - [2025-06-05 15:39:56+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:40:57,118 - tests.orchestrator - INFO - No logs available +2025-06-05 15:40:57,118 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:40:57,118 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-011557a1'} +2025-06-05 15:40:57,118 - tests.orchestrator - INFO - - kube-api-access-p2phq: {} +2025-06-05 15:40:57,118 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-9785ae47 ===== +2025-06-05 15:40:57,118 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9785ae47 to be ready +2025-06-05 15:40:57,119 - tests.orchestrator - INFO - Testing read/write between pods test-pod-1f8e6665 and test-pod-43a71634 sharing PVC test-pvc-b52b04ee +2025-06-05 15:40:57,185 - tests.orchestrator - ERROR - Failed in read/write verification: (400) +Reason: Bad Request +HTTP response headers: HTTPHeaderDict({'Audit-Id': '4713df17-943b-4cf0-b43b-7454ef741357', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 15:40:57 GMT', 'Content-Length': '166'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-1f8e6665 does not have a host assigned","reason":"BadRequest","code":400} + + +2025-06-05 15:40:57,185 - tests.orchestrator - INFO - ============================================================ +2025-06-05 15:40:57,185 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_many_to_one +2025-06-05 15:40:57,185 - tests.orchestrator - INFO - ============================================================ +2025-06-05 15:40:57,185 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +2025-06-05 15:40:57,185 - tests.orchestrator - INFO - STARTING MANY-TO-ONE SCENARIO DIAGNOSTICS +2025-06-05 15:40:57,185 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +2025-06-05 15:40:57,185 - tests.orchestrator - INFO - [MANY2ONE] STEP 1: Creating dedicated PVC: many2one-21027765 +2025-06-05 15:40:57,185 - tests.orchestrator - INFO - [MANY2ONE] Creating PVC manifest with storageClassName: efs-sc +2025-06-05 15:40:57,263 - tests.orchestrator - INFO - [MANY2ONE] PVC many2one-21027765 created successfully +2025-06-05 15:40:57,264 - tests.orchestrator - INFO - [MANY2ONE] Waiting for PVC many2one-21027765 to be bound... +2025-06-05 15:40:57,264 - tests.orchestrator - INFO - Waiting for PVC many2one-21027765 to be bound +2025-06-05 15:40:59,396 - tests.orchestrator - INFO - PVC many2one-21027765 is bound +2025-06-05 15:40:59,463 - tests.orchestrator - INFO - [MANY2ONE] PVC status: phase=Bound, capacity={'storage': '1Gi'} +2025-06-05 15:40:59,463 - tests.orchestrator - INFO - [MANY2ONE] STEP 2: Creating 3 pods for the same PVC many2one-21027765 +2025-06-05 15:40:59,463 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 1/3 for PVC many2one-21027765 +2025-06-05 15:40:59,463 - tests.orchestrator - INFO - Creating pod test-pod-052e963e on PVC many2one-21027765 +2025-06-05 15:40:59,538 - tests.orchestrator - INFO - Created pod: test-pod-052e963e using PVC: many2one-21027765 +2025-06-05 15:40:59,538 - tests.orchestrator - INFO - Waiting for pod test-pod-052e963e to be ready +2025-06-05 15:40:59,604 - tests.orchestrator - INFO - Pod test-pod-052e963e phase: Pending +2025-06-05 15:41:30,643 - tests.orchestrator - INFO - Pod test-pod-052e963e taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 15:41:30,643 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-052e963e ===== +2025-06-05 15:41:30,710 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:41:30,779 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:41:30,780 - tests.orchestrator - INFO - [2025-06-05 15:40:59+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:41:30,846 - tests.orchestrator - INFO - No logs available +2025-06-05 15:41:30,846 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:41:30,846 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-21027765'} +2025-06-05 15:41:30,846 - tests.orchestrator - INFO - - kube-api-access-h4dzw: {} +2025-06-05 15:41:30,846 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-052e963e ===== +2025-06-05 15:41:59,722 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-052e963e to be ready after 60s +2025-06-05 15:41:59,722 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-052e963e ===== +2025-06-05 15:41:59,789 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:41:59,859 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:41:59,859 - tests.orchestrator - INFO - [2025-06-05 15:40:59+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:41:59,925 - tests.orchestrator - INFO - No logs available +2025-06-05 15:41:59,925 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:41:59,925 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-21027765'} +2025-06-05 15:41:59,925 - tests.orchestrator - INFO - - kube-api-access-h4dzw: {} +2025-06-05 15:41:59,925 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-052e963e ===== +2025-06-05 15:41:59,926 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-052e963e to be ready +2025-06-05 15:41:59,926 - tests.orchestrator - ERROR - [MANY2ONE] Failed to create pod 1/3 +2025-06-05 15:41:59,926 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 2/3 for PVC many2one-21027765 +2025-06-05 15:41:59,926 - tests.orchestrator - INFO - Creating pod test-pod-dda7e82b on PVC many2one-21027765 +2025-06-05 15:42:00,007 - tests.orchestrator - INFO - Created pod: test-pod-dda7e82b using PVC: many2one-21027765 +2025-06-05 15:42:00,008 - tests.orchestrator - INFO - Waiting for pod test-pod-dda7e82b to be ready +2025-06-05 15:42:00,075 - tests.orchestrator - INFO - Pod test-pod-dda7e82b phase: Pending +2025-06-05 15:42:31,084 - tests.orchestrator - INFO - Pod test-pod-dda7e82b taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 15:42:31,084 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-dda7e82b ===== +2025-06-05 15:42:31,150 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:42:31,219 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:42:31,219 - tests.orchestrator - INFO - [2025-06-05 15:41:59+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:42:31,285 - tests.orchestrator - INFO - No logs available +2025-06-05 15:42:31,285 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:42:31,285 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-21027765'} +2025-06-05 15:42:31,285 - tests.orchestrator - INFO - - kube-api-access-bktns: {} +2025-06-05 15:42:31,285 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-dda7e82b ===== +2025-06-05 15:43:00,160 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-dda7e82b to be ready after 60s +2025-06-05 15:43:00,160 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-dda7e82b ===== +2025-06-05 15:43:00,227 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:43:00,296 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:43:00,296 - tests.orchestrator - INFO - [2025-06-05 15:41:59+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:43:00,362 - tests.orchestrator - INFO - No logs available +2025-06-05 15:43:00,362 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:43:00,362 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-21027765'} +2025-06-05 15:43:00,362 - tests.orchestrator - INFO - - kube-api-access-bktns: {} +2025-06-05 15:43:00,363 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-dda7e82b ===== +2025-06-05 15:43:00,363 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-dda7e82b to be ready +2025-06-05 15:43:00,363 - tests.orchestrator - ERROR - [MANY2ONE] Failed to create pod 2/3 +2025-06-05 15:43:00,363 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 3/3 for PVC many2one-21027765 +2025-06-05 15:43:00,363 - tests.orchestrator - INFO - Creating pod test-pod-9b6c5221 on PVC many2one-21027765 +2025-06-05 15:43:00,448 - tests.orchestrator - INFO - Created pod: test-pod-9b6c5221 using PVC: many2one-21027765 +2025-06-05 15:43:00,448 - tests.orchestrator - INFO - Waiting for pod test-pod-9b6c5221 to be ready +2025-06-05 15:43:00,514 - tests.orchestrator - INFO - Pod test-pod-9b6c5221 phase: Pending +2025-06-05 15:43:31,552 - tests.orchestrator - INFO - Pod test-pod-9b6c5221 taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 15:43:31,552 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-9b6c5221 ===== +2025-06-05 15:43:31,684 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:43:31,759 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:43:31,759 - tests.orchestrator - INFO - [2025-06-05 15:43:00+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:43:31,829 - tests.orchestrator - INFO - No logs available +2025-06-05 15:43:31,830 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:43:31,830 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-21027765'} +2025-06-05 15:43:31,830 - tests.orchestrator - INFO - - kube-api-access-mh6kx: {} +2025-06-05 15:43:31,830 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-9b6c5221 ===== +2025-06-05 15:44:00,708 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9b6c5221 to be ready after 60s +2025-06-05 15:44:00,708 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-9b6c5221 ===== +2025-06-05 15:44:00,776 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:44:00,846 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:44:00,846 - tests.orchestrator - INFO - [2025-06-05 15:43:00+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:44:00,911 - tests.orchestrator - INFO - No logs available +2025-06-05 15:44:00,911 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:44:00,911 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-21027765'} +2025-06-05 15:44:00,911 - tests.orchestrator - INFO - - kube-api-access-mh6kx: {} +2025-06-05 15:44:00,911 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-9b6c5221 ===== +2025-06-05 15:44:00,912 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9b6c5221 to be ready +2025-06-05 15:44:00,912 - tests.orchestrator - ERROR - [MANY2ONE] Failed to create pod 3/3 +2025-06-05 15:44:00,912 - tests.orchestrator - INFO - [MANY2ONE] Created 0/3 pods successfully +2025-06-05 15:44:00,912 - tests.orchestrator - ERROR - [MANY2ONE] FAILED: Insufficient pods created (0), need at least 2 for read/write test +2025-06-05 15:44:00,912 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_many_to_one +2025-06-05 15:44:00,912 - tests.orchestrator - INFO - ------------------------------------------------------------ +2025-06-05 15:44:00,912 - tests.orchestrator - INFO - Deleting pod: test-pod-052e963e from PVC: many2one-21027765 +2025-06-05 15:44:00,990 - tests.orchestrator - INFO - Waiting for pod test-pod-052e963e to be deleted +2025-06-05 15:44:01,055 - tests.orchestrator - INFO - Pod test-pod-052e963e has been deleted +2025-06-05 15:44:01,055 - tests.orchestrator - INFO - Deleted pod: test-pod-052e963e +2025-06-05 15:44:01,056 - tests.orchestrator - INFO - Deleting PVC: test-pvc-b52b04ee +2025-06-05 15:44:01,056 - tests.orchestrator - INFO - Deleting 2 pods using PVC test-pvc-b52b04ee +2025-06-05 15:44:01,056 - tests.orchestrator - INFO - Deleting pod: test-pod-1f8e6665 from PVC: test-pvc-b52b04ee +2025-06-05 15:44:01,140 - tests.orchestrator - INFO - Waiting for pod test-pod-1f8e6665 to be deleted +2025-06-05 15:44:01,205 - tests.orchestrator - INFO - Pod test-pod-1f8e6665 has been deleted +2025-06-05 15:44:01,205 - tests.orchestrator - INFO - Deleted pod: test-pod-1f8e6665 +2025-06-05 15:44:01,205 - tests.orchestrator - INFO - Deleting pod: test-pod-43a71634 from PVC: test-pvc-b52b04ee +2025-06-05 15:44:01,287 - tests.orchestrator - INFO - Waiting for pod test-pod-43a71634 to be deleted +2025-06-05 15:44:01,352 - tests.orchestrator - INFO - Pod test-pod-43a71634 has been deleted +2025-06-05 15:44:01,352 - tests.orchestrator - INFO - Deleted pod: test-pod-43a71634 +2025-06-05 15:44:01,426 - tests.orchestrator - INFO - Waiting for PVC test-pvc-b52b04ee to be deleted +2025-06-05 15:44:01,527 - tests.orchestrator - INFO - PVC test-pvc-b52b04ee has been deleted +2025-06-05 15:44:01,527 - tests.orchestrator - INFO - Deleted PVC: test-pvc-b52b04ee +2025-06-05 15:44:01,527 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations +2025-06-05 15:44:01,527 - tests.orchestrator - INFO - Test completed in 370.45 seconds +2025-06-05 15:44:01,527 - tests.orchestrator - INFO - Cleaning up resources... +2025-06-05 15:44:01,527 - tests.orchestrator - INFO - Deleting pod: test-pod-9785ae47 from PVC: test-pvc-011557a1 +2025-06-05 15:44:01,678 - tests.orchestrator - INFO - Waiting for pod test-pod-9785ae47 to be deleted +2025-06-05 15:44:01,750 - tests.orchestrator - INFO - Pod test-pod-9785ae47 has been deleted +2025-06-05 15:44:01,751 - tests.orchestrator - INFO - Deleted pod: test-pod-9785ae47 +2025-06-05 15:44:01,751 - tests.orchestrator - INFO - Deleting pod: test-pod-dda7e82b from PVC: many2one-21027765 +2025-06-05 15:44:01,841 - tests.orchestrator - INFO - Waiting for pod test-pod-dda7e82b to be deleted +2025-06-05 15:44:01,912 - tests.orchestrator - INFO - Pod test-pod-dda7e82b has been deleted +2025-06-05 15:44:01,912 - tests.orchestrator - INFO - Deleted pod: test-pod-dda7e82b +2025-06-05 15:44:01,912 - tests.orchestrator - INFO - Deleting pod: test-pod-9b6c5221 from PVC: many2one-21027765 +2025-06-05 15:44:02,038 - tests.orchestrator - INFO - Waiting for pod test-pod-9b6c5221 to be deleted +2025-06-05 15:44:02,118 - tests.orchestrator - INFO - Pod test-pod-9b6c5221 has been deleted +2025-06-05 15:44:02,118 - tests.orchestrator - INFO - Deleted pod: test-pod-9b6c5221 +2025-06-05 15:44:02,118 - tests.orchestrator - INFO - Deleting PVC: test-pvc-011557a1 +2025-06-05 15:44:02,196 - tests.orchestrator - INFO - Waiting for PVC test-pvc-011557a1 to be deleted +2025-06-05 15:44:02,268 - tests.orchestrator - INFO - PVC test-pvc-011557a1 has been deleted +2025-06-05 15:44:02,268 - tests.orchestrator - INFO - Deleted PVC: test-pvc-011557a1 +2025-06-05 15:44:02,268 - tests.orchestrator - INFO - Deleting PVC: many2one-21027765 +2025-06-05 15:44:02,341 - tests.orchestrator - INFO - Waiting for PVC many2one-21027765 to be deleted +2025-06-05 15:44:02,406 - tests.orchestrator - INFO - PVC many2one-21027765 has been deleted +2025-06-05 15:44:02,406 - tests.orchestrator - INFO - Deleted PVC: many2one-21027765 +2025-06-05 15:44:02,406 - tests.orchestrator - INFO - Cleanup completed +2025-06-05 15:44:02,406 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-05 15:44:02,407 - tests.orchestrator - INFO - --- Operations --- +2025-06-05 15:44:02,407 - tests.orchestrator - INFO - create_pvc: 2 succeeded, 0 failed (100.0%) +2025-06-05 15:44:02,407 - tests.orchestrator - INFO - attach_pod: 6 succeeded, 0 failed (100.0%) +2025-06-05 15:44:02,407 - tests.orchestrator - INFO - delete_pod: 6 succeeded, 0 failed (100.0%) +2025-06-05 15:44:02,407 - tests.orchestrator - INFO - delete_pvc: 3 succeeded, 0 failed (100.0%) +2025-06-05 15:44:02,407 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) +2025-06-05 15:44:02,407 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) +2025-06-05 15:44:02,407 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-05 15:44:02,407 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-05 15:44:02,408 - tests.orchestrator - INFO - many_to_one: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-05 15:44:02,408 - tests.orchestrator - INFO - one_to_one: No runs +2025-06-05 15:44:02,408 - tests.orchestrator - INFO - concurrent_pvc: No runs +2025-06-05 15:44:02,408 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_154546.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_154546.log new file mode 100644 index 000000000..6863cb59c --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_154546.log @@ -0,0 +1,254 @@ +2025-06-05 15:45:46,726 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 15:45:46,943 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 15:45:46,944 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-05 15:45:47,132 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 15:45:47,133 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 15:45:47,133 - tests.orchestrator - INFO - Creating PVC: test-pvc-7dbf87c6 +2025-06-05 15:45:47,210 - tests.orchestrator - INFO - Created PVC: test-pvc-7dbf87c6 +2025-06-05 15:45:47,210 - tests.orchestrator - INFO - Waiting for PVC test-pvc-7dbf87c6 to be bound +2025-06-05 15:45:49,343 - tests.orchestrator - INFO - PVC test-pvc-7dbf87c6 is bound +2025-06-05 15:45:49,343 - tests.orchestrator - INFO - Creating PVC: test-pvc-83f66d6c +2025-06-05 15:45:49,411 - tests.orchestrator - INFO - Created PVC: test-pvc-83f66d6c +2025-06-05 15:45:49,411 - tests.orchestrator - INFO - Waiting for PVC test-pvc-83f66d6c to be bound +2025-06-05 15:45:51,548 - tests.orchestrator - INFO - PVC test-pvc-83f66d6c is bound +2025-06-05 15:45:51,549 - tests.orchestrator - INFO - Creating pod test-pod-c3e945dc on PVC test-pvc-7dbf87c6 +2025-06-05 15:45:51,631 - tests.orchestrator - INFO - Created pod: test-pod-c3e945dc using PVC: test-pvc-7dbf87c6 +2025-06-05 15:45:51,631 - tests.orchestrator - INFO - Waiting for pod test-pod-c3e945dc to be ready +2025-06-05 15:45:51,697 - tests.orchestrator - INFO - Pod test-pod-c3e945dc phase: Pending +2025-06-05 15:46:22,714 - tests.orchestrator - INFO - Pod test-pod-c3e945dc taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 15:46:22,714 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-c3e945dc ===== +2025-06-05 15:46:22,781 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:46:22,850 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:46:22,850 - tests.orchestrator - INFO - [2025-06-05 15:45:51+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:46:22,916 - tests.orchestrator - INFO - No logs available +2025-06-05 15:46:22,917 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:46:22,917 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-7dbf87c6'} +2025-06-05 15:46:22,917 - tests.orchestrator - INFO - - kube-api-access-vmkzs: {} +2025-06-05 15:46:22,917 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-c3e945dc ===== +2025-06-05 15:46:51,793 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-c3e945dc to be ready after 60s +2025-06-05 15:46:51,793 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-c3e945dc ===== +2025-06-05 15:46:51,860 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:46:51,930 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:46:51,930 - tests.orchestrator - INFO - [2025-06-05 15:45:51+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:46:51,996 - tests.orchestrator - INFO - No logs available +2025-06-05 15:46:51,996 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:46:51,996 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-7dbf87c6'} +2025-06-05 15:46:51,996 - tests.orchestrator - INFO - - kube-api-access-vmkzs: {} +2025-06-05 15:46:51,996 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-c3e945dc ===== +2025-06-05 15:46:51,997 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-c3e945dc to be ready +2025-06-05 15:46:51,997 - tests.orchestrator - INFO - Creating pod test-pod-7abe409e on PVC test-pvc-83f66d6c +2025-06-05 15:46:52,079 - tests.orchestrator - INFO - Created pod: test-pod-7abe409e using PVC: test-pvc-83f66d6c +2025-06-05 15:46:52,079 - tests.orchestrator - INFO - Waiting for pod test-pod-7abe409e to be ready +2025-06-05 15:46:52,146 - tests.orchestrator - INFO - Pod test-pod-7abe409e phase: Pending +2025-06-05 15:47:23,154 - tests.orchestrator - INFO - Pod test-pod-7abe409e taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 15:47:23,154 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-7abe409e ===== +2025-06-05 15:47:23,220 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:47:23,289 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:47:23,290 - tests.orchestrator - INFO - [2025-06-05 15:46:52+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:47:23,355 - tests.orchestrator - INFO - No logs available +2025-06-05 15:47:23,355 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:47:23,355 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-83f66d6c'} +2025-06-05 15:47:23,355 - tests.orchestrator - INFO - - kube-api-access-swfv7: {} +2025-06-05 15:47:23,355 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-7abe409e ===== +2025-06-05 15:47:52,325 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-7abe409e to be ready after 60s +2025-06-05 15:47:52,325 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-7abe409e ===== +2025-06-05 15:47:52,392 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:47:52,462 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:47:52,462 - tests.orchestrator - INFO - [2025-06-05 15:46:52+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:47:52,528 - tests.orchestrator - INFO - No logs available +2025-06-05 15:47:52,528 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:47:52,528 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-83f66d6c'} +2025-06-05 15:47:52,528 - tests.orchestrator - INFO - - kube-api-access-swfv7: {} +2025-06-05 15:47:52,528 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-7abe409e ===== +2025-06-05 15:47:52,528 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-7abe409e to be ready +2025-06-05 15:47:52,528 - tests.orchestrator - INFO - Creating pod test-pod-a9fc1f04 on PVC test-pvc-7dbf87c6 +2025-06-05 15:47:52,611 - tests.orchestrator - INFO - Created pod: test-pod-a9fc1f04 using PVC: test-pvc-7dbf87c6 +2025-06-05 15:47:52,612 - tests.orchestrator - INFO - Waiting for pod test-pod-a9fc1f04 to be ready +2025-06-05 15:47:52,678 - tests.orchestrator - INFO - Pod test-pod-a9fc1f04 phase: Pending +2025-06-05 15:48:23,696 - tests.orchestrator - INFO - Pod test-pod-a9fc1f04 taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 15:48:23,696 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-a9fc1f04 ===== +2025-06-05 15:48:23,762 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:48:23,831 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:48:23,832 - tests.orchestrator - INFO - [2025-06-05 15:47:52+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:48:23,898 - tests.orchestrator - INFO - No logs available +2025-06-05 15:48:23,898 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:48:23,898 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-7dbf87c6'} +2025-06-05 15:48:23,898 - tests.orchestrator - INFO - - kube-api-access-bh5hw: {} +2025-06-05 15:48:23,898 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-a9fc1f04 ===== +2025-06-05 15:48:52,778 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a9fc1f04 to be ready after 60s +2025-06-05 15:48:52,778 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-a9fc1f04 ===== +2025-06-05 15:48:52,845 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:48:52,917 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:48:52,917 - tests.orchestrator - INFO - [2025-06-05 15:47:52+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:48:52,983 - tests.orchestrator - INFO - No logs available +2025-06-05 15:48:52,983 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:48:52,983 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-7dbf87c6'} +2025-06-05 15:48:52,983 - tests.orchestrator - INFO - - kube-api-access-bh5hw: {} +2025-06-05 15:48:52,984 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-a9fc1f04 ===== +2025-06-05 15:48:52,984 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a9fc1f04 to be ready +2025-06-05 15:48:52,984 - tests.orchestrator - INFO - Testing read/write between pods test-pod-c3e945dc and test-pod-a9fc1f04 sharing PVC test-pvc-7dbf87c6 +2025-06-05 15:48:53,049 - tests.orchestrator - ERROR - Failed in read/write verification: (400) +Reason: Bad Request +HTTP response headers: HTTPHeaderDict({'Audit-Id': '0f444cd6-376b-4233-8e31-012234fb64c8', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 15:48:53 GMT', 'Content-Length': '166'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-c3e945dc does not have a host assigned","reason":"BadRequest","code":400} + + +2025-06-05 15:48:53,049 - tests.orchestrator - INFO - ============================================================ +2025-06-05 15:48:53,049 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_concurrent_pvc +2025-06-05 15:48:53,049 - tests.orchestrator - INFO - ============================================================ +2025-06-05 15:48:53,049 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations +2025-06-05 15:48:53,049 - tests.orchestrator - INFO - Creating 3 PVCs in quick succession +2025-06-05 15:48:53,127 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-74639d37 +2025-06-05 15:48:53,127 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-74639d37 to be bound +2025-06-05 15:48:55,259 - tests.orchestrator - INFO - PVC concurrent-pvc-74639d37 is bound +2025-06-05 15:48:55,328 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-be273a95 +2025-06-05 15:48:55,328 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-be273a95 to be bound +2025-06-05 15:48:57,460 - tests.orchestrator - INFO - PVC concurrent-pvc-be273a95 is bound +2025-06-05 15:48:57,530 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-421903bd +2025-06-05 15:48:57,530 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-421903bd to be bound +2025-06-05 15:48:59,662 - tests.orchestrator - INFO - PVC concurrent-pvc-421903bd is bound +2025-06-05 15:48:59,662 - tests.orchestrator - INFO - Successfully created 3 PVCs +2025-06-05 15:48:59,662 - tests.orchestrator - INFO - Creating pod test-pod-d257ffb1 on PVC concurrent-pvc-be273a95 +2025-06-05 15:48:59,737 - tests.orchestrator - INFO - Created pod: test-pod-d257ffb1 using PVC: concurrent-pvc-be273a95 +2025-06-05 15:48:59,737 - tests.orchestrator - INFO - Waiting for pod test-pod-d257ffb1 to be ready +2025-06-05 15:48:59,805 - tests.orchestrator - INFO - Pod test-pod-d257ffb1 phase: Pending +2025-06-05 15:49:30,827 - tests.orchestrator - INFO - Pod test-pod-d257ffb1 taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 15:49:30,828 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-d257ffb1 ===== +2025-06-05 15:49:30,894 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:49:30,964 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:49:30,964 - tests.orchestrator - INFO - [2025-06-05 15:48:59+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:49:31,030 - tests.orchestrator - INFO - No logs available +2025-06-05 15:49:31,030 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:49:31,030 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'concurrent-pvc-be273a95'} +2025-06-05 15:49:31,031 - tests.orchestrator - INFO - - kube-api-access-tr4s4: {} +2025-06-05 15:49:31,031 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-d257ffb1 ===== +2025-06-05 15:49:59,906 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-d257ffb1 to be ready after 60s +2025-06-05 15:49:59,907 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-d257ffb1 ===== +2025-06-05 15:49:59,974 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:50:00,045 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:50:00,045 - tests.orchestrator - INFO - [2025-06-05 15:48:59+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:50:00,111 - tests.orchestrator - INFO - No logs available +2025-06-05 15:50:00,112 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:50:00,112 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'concurrent-pvc-be273a95'} +2025-06-05 15:50:00,112 - tests.orchestrator - INFO - - kube-api-access-tr4s4: {} +2025-06-05 15:50:00,112 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-d257ffb1 ===== +2025-06-05 15:50:00,112 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-d257ffb1 to be ready +2025-06-05 15:50:00,112 - tests.orchestrator - INFO - Creating pod test-pod-8c9ea815 on PVC concurrent-pvc-421903bd +2025-06-05 15:50:00,195 - tests.orchestrator - INFO - Created pod: test-pod-8c9ea815 using PVC: concurrent-pvc-421903bd +2025-06-05 15:50:00,196 - tests.orchestrator - INFO - Waiting for pod test-pod-8c9ea815 to be ready +2025-06-05 15:50:00,263 - tests.orchestrator - INFO - Pod test-pod-8c9ea815 phase: Pending +2025-06-05 15:50:31,275 - tests.orchestrator - INFO - Pod test-pod-8c9ea815 taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 15:50:31,275 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-8c9ea815 ===== +2025-06-05 15:50:31,341 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:50:31,411 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:50:31,411 - tests.orchestrator - INFO - [2025-06-05 15:50:00+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:50:31,526 - tests.orchestrator - INFO - No logs available +2025-06-05 15:50:31,526 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:50:31,526 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'concurrent-pvc-421903bd'} +2025-06-05 15:50:31,526 - tests.orchestrator - INFO - - kube-api-access-mk5vl: {} +2025-06-05 15:50:31,527 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-8c9ea815 ===== +2025-06-05 15:51:00,401 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-8c9ea815 to be ready after 60s +2025-06-05 15:51:00,401 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-8c9ea815 ===== +2025-06-05 15:51:00,469 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:51:00,539 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:51:00,539 - tests.orchestrator - INFO - [2025-06-05 15:50:00+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:51:00,604 - tests.orchestrator - INFO - No logs available +2025-06-05 15:51:00,604 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:51:00,604 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'concurrent-pvc-421903bd'} +2025-06-05 15:51:00,604 - tests.orchestrator - INFO - - kube-api-access-mk5vl: {} +2025-06-05 15:51:00,605 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-8c9ea815 ===== +2025-06-05 15:51:00,605 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-8c9ea815 to be ready +2025-06-05 15:51:00,605 - tests.orchestrator - INFO - Creating pod test-pod-3106ab46 on PVC concurrent-pvc-74639d37 +2025-06-05 15:51:00,689 - tests.orchestrator - INFO - Created pod: test-pod-3106ab46 using PVC: concurrent-pvc-74639d37 +2025-06-05 15:51:00,689 - tests.orchestrator - INFO - Waiting for pod test-pod-3106ab46 to be ready +2025-06-05 15:51:00,755 - tests.orchestrator - INFO - Pod test-pod-3106ab46 phase: Pending +2025-06-05 15:51:31,779 - tests.orchestrator - INFO - Pod test-pod-3106ab46 taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 15:51:31,779 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-3106ab46 ===== +2025-06-05 15:51:31,852 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:51:31,927 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:51:31,927 - tests.orchestrator - INFO - [2025-06-05 15:51:00+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:51:31,998 - tests.orchestrator - INFO - No logs available +2025-06-05 15:51:31,998 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:51:31,998 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'concurrent-pvc-74639d37'} +2025-06-05 15:51:31,998 - tests.orchestrator - INFO - - kube-api-access-97jdn: {} +2025-06-05 15:51:31,998 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-3106ab46 ===== +2025-06-05 15:52:00,875 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-3106ab46 to be ready after 60s +2025-06-05 15:52:00,876 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-3106ab46 ===== +2025-06-05 15:52:00,943 - tests.orchestrator - INFO - No container statuses available +2025-06-05 15:52:01,013 - tests.orchestrator - INFO - Pod events: +2025-06-05 15:52:01,013 - tests.orchestrator - INFO - [2025-06-05 15:51:00+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 15:52:01,079 - tests.orchestrator - INFO - No logs available +2025-06-05 15:52:01,079 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 15:52:01,079 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'concurrent-pvc-74639d37'} +2025-06-05 15:52:01,080 - tests.orchestrator - INFO - - kube-api-access-97jdn: {} +2025-06-05 15:52:01,080 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-3106ab46 ===== +2025-06-05 15:52:01,080 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-3106ab46 to be ready +2025-06-05 15:52:01,080 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-be273a95 +2025-06-05 15:52:01,080 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-be273a95 +2025-06-05 15:52:01,080 - tests.orchestrator - INFO - Deleting pod: test-pod-d257ffb1 from PVC: concurrent-pvc-be273a95 +2025-06-05 15:52:01,164 - tests.orchestrator - INFO - Waiting for pod test-pod-d257ffb1 to be deleted +2025-06-05 15:52:01,229 - tests.orchestrator - INFO - Pod test-pod-d257ffb1 has been deleted +2025-06-05 15:52:01,230 - tests.orchestrator - INFO - Deleted pod: test-pod-d257ffb1 +2025-06-05 15:52:01,301 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-be273a95 to be deleted +2025-06-05 15:52:01,366 - tests.orchestrator - INFO - PVC concurrent-pvc-be273a95 has been deleted +2025-06-05 15:52:01,367 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-be273a95 +2025-06-05 15:52:01,367 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-421903bd +2025-06-05 15:52:01,367 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-421903bd +2025-06-05 15:52:01,367 - tests.orchestrator - INFO - Deleting pod: test-pod-8c9ea815 from PVC: concurrent-pvc-421903bd +2025-06-05 15:52:01,548 - tests.orchestrator - INFO - Waiting for pod test-pod-8c9ea815 to be deleted +2025-06-05 15:52:01,774 - tests.orchestrator - INFO - Pod test-pod-8c9ea815 has been deleted +2025-06-05 15:52:01,774 - tests.orchestrator - INFO - Deleted pod: test-pod-8c9ea815 +2025-06-05 15:52:01,852 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-421903bd to be deleted +2025-06-05 15:52:01,924 - tests.orchestrator - INFO - PVC concurrent-pvc-421903bd has been deleted +2025-06-05 15:52:01,925 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-421903bd +2025-06-05 15:52:01,925 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-74639d37 +2025-06-05 15:52:01,925 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-74639d37 +2025-06-05 15:52:01,925 - tests.orchestrator - INFO - Deleting pod: test-pod-3106ab46 from PVC: concurrent-pvc-74639d37 +2025-06-05 15:52:02,021 - tests.orchestrator - INFO - Waiting for pod test-pod-3106ab46 to be deleted +2025-06-05 15:52:02,092 - tests.orchestrator - INFO - Pod test-pod-3106ab46 has been deleted +2025-06-05 15:52:02,092 - tests.orchestrator - INFO - Deleted pod: test-pod-3106ab46 +2025-06-05 15:52:02,172 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-74639d37 to be deleted +2025-06-05 15:52:02,238 - tests.orchestrator - INFO - PVC concurrent-pvc-74639d37 has been deleted +2025-06-05 15:52:02,238 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-74639d37 +2025-06-05 15:52:02,238 - tests.orchestrator - INFO - Rapid PVC scenario completed +2025-06-05 15:52:02,238 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_concurrent_pvc +2025-06-05 15:52:02,238 - tests.orchestrator - INFO - ------------------------------------------------------------ +2025-06-05 15:52:02,238 - tests.orchestrator - INFO - Deleting pod: test-pod-7abe409e from PVC: test-pvc-83f66d6c +2025-06-05 15:52:02,322 - tests.orchestrator - INFO - Waiting for pod test-pod-7abe409e to be deleted +2025-06-05 15:52:02,390 - tests.orchestrator - INFO - Pod test-pod-7abe409e has been deleted +2025-06-05 15:52:02,390 - tests.orchestrator - INFO - Deleted pod: test-pod-7abe409e +2025-06-05 15:52:02,390 - tests.orchestrator - INFO - Deleting PVC: test-pvc-7dbf87c6 +2025-06-05 15:52:02,390 - tests.orchestrator - INFO - Deleting 2 pods using PVC test-pvc-7dbf87c6 +2025-06-05 15:52:02,390 - tests.orchestrator - INFO - Deleting pod: test-pod-c3e945dc from PVC: test-pvc-7dbf87c6 +2025-06-05 15:52:02,471 - tests.orchestrator - INFO - Waiting for pod test-pod-c3e945dc to be deleted +2025-06-05 15:52:02,537 - tests.orchestrator - INFO - Pod test-pod-c3e945dc has been deleted +2025-06-05 15:52:02,537 - tests.orchestrator - INFO - Deleted pod: test-pod-c3e945dc +2025-06-05 15:52:02,537 - tests.orchestrator - INFO - Deleting pod: test-pod-a9fc1f04 from PVC: test-pvc-7dbf87c6 +2025-06-05 15:52:02,625 - tests.orchestrator - INFO - Waiting for pod test-pod-a9fc1f04 to be deleted +2025-06-05 15:52:02,691 - tests.orchestrator - INFO - Pod test-pod-a9fc1f04 has been deleted +2025-06-05 15:52:02,691 - tests.orchestrator - INFO - Deleted pod: test-pod-a9fc1f04 +2025-06-05 15:52:02,763 - tests.orchestrator - INFO - Waiting for PVC test-pvc-7dbf87c6 to be deleted +2025-06-05 15:52:02,828 - tests.orchestrator - INFO - PVC test-pvc-7dbf87c6 has been deleted +2025-06-05 15:52:02,828 - tests.orchestrator - INFO - Deleted PVC: test-pvc-7dbf87c6 +2025-06-05 15:52:02,828 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations +2025-06-05 15:52:02,828 - tests.orchestrator - INFO - Test completed in 375.88 seconds +2025-06-05 15:52:02,828 - tests.orchestrator - INFO - Cleaning up resources... +2025-06-05 15:52:02,828 - tests.orchestrator - INFO - Deleting PVC: test-pvc-83f66d6c +2025-06-05 15:52:02,903 - tests.orchestrator - INFO - Waiting for PVC test-pvc-83f66d6c to be deleted +2025-06-05 15:52:02,968 - tests.orchestrator - INFO - PVC test-pvc-83f66d6c has been deleted +2025-06-05 15:52:02,968 - tests.orchestrator - INFO - Deleted PVC: test-pvc-83f66d6c +2025-06-05 15:52:02,968 - tests.orchestrator - INFO - Cleanup completed +2025-06-05 15:52:02,968 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-05 15:52:02,968 - tests.orchestrator - INFO - --- Operations --- +2025-06-05 15:52:02,968 - tests.orchestrator - INFO - create_pvc: 5 succeeded, 0 failed (100.0%) +2025-06-05 15:52:02,968 - tests.orchestrator - INFO - attach_pod: 6 succeeded, 0 failed (100.0%) +2025-06-05 15:52:02,968 - tests.orchestrator - INFO - delete_pod: 6 succeeded, 0 failed (100.0%) +2025-06-05 15:52:02,968 - tests.orchestrator - INFO - delete_pvc: 5 succeeded, 0 failed (100.0%) +2025-06-05 15:52:02,968 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) +2025-06-05 15:52:02,968 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) +2025-06-05 15:52:02,968 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-05 15:52:02,969 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-05 15:52:02,969 - tests.orchestrator - INFO - many_to_one: No runs +2025-06-05 15:52:02,969 - tests.orchestrator - INFO - one_to_one: No runs +2025-06-05 15:52:02,969 - tests.orchestrator - INFO - concurrent_pvc: 1 succeeded, 0 failed out of 1 runs (100.0%) +2025-06-05 15:52:02,969 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_155948.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_155948.log new file mode 100644 index 000000000..eb30e4717 --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_155948.log @@ -0,0 +1,280 @@ +2025-06-05 15:59:48,809 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 15:59:49,025 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 15:59:49,027 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-05 15:59:49,218 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 15:59:49,218 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 15:59:49,218 - tests.orchestrator - INFO - Creating PVC: test-pvc-47b1c666 +2025-06-05 15:59:49,295 - tests.orchestrator - INFO - Created PVC: test-pvc-47b1c666 +2025-06-05 15:59:49,296 - tests.orchestrator - INFO - Waiting for PVC test-pvc-47b1c666 to be bound +2025-06-05 15:59:51,432 - tests.orchestrator - INFO - PVC test-pvc-47b1c666 is bound +2025-06-05 15:59:51,433 - tests.orchestrator - INFO - Creating PVC: test-pvc-13de15dd +2025-06-05 15:59:51,506 - tests.orchestrator - INFO - Created PVC: test-pvc-13de15dd +2025-06-05 15:59:51,507 - tests.orchestrator - INFO - Waiting for PVC test-pvc-13de15dd to be bound +2025-06-05 15:59:53,652 - tests.orchestrator - INFO - PVC test-pvc-13de15dd is bound +2025-06-05 15:59:53,652 - tests.orchestrator - INFO - Creating pod test-pod-b12f5a09 on PVC test-pvc-47b1c666 +2025-06-05 15:59:53,736 - tests.orchestrator - INFO - Created pod: test-pod-b12f5a09 using PVC: test-pvc-47b1c666 +2025-06-05 15:59:53,736 - tests.orchestrator - INFO - Waiting for pod test-pod-b12f5a09 to be ready +2025-06-05 15:59:53,809 - tests.orchestrator - INFO - Pod test-pod-b12f5a09 phase: Pending +2025-06-05 16:00:24,845 - tests.orchestrator - INFO - Pod test-pod-b12f5a09 taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 16:00:24,846 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-b12f5a09 ===== +2025-06-05 16:00:24,914 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:00:24,986 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:00:24,987 - tests.orchestrator - INFO - [2025-06-05 15:59:53+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:00:25,052 - tests.orchestrator - INFO - No logs available +2025-06-05 16:00:25,053 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:00:25,053 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-47b1c666'} +2025-06-05 16:00:25,053 - tests.orchestrator - INFO - - kube-api-access-q49n5: {} +2025-06-05 16:00:25,053 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-b12f5a09 ===== +2025-06-05 16:00:54,038 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b12f5a09 to be ready after 60s +2025-06-05 16:00:54,038 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-b12f5a09 ===== +2025-06-05 16:00:54,106 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:00:54,179 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:00:54,179 - tests.orchestrator - INFO - [2025-06-05 15:59:53+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:00:54,244 - tests.orchestrator - INFO - No logs available +2025-06-05 16:00:54,245 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:00:54,245 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-47b1c666'} +2025-06-05 16:00:54,245 - tests.orchestrator - INFO - - kube-api-access-q49n5: {} +2025-06-05 16:00:54,245 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-b12f5a09 ===== +2025-06-05 16:00:54,245 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b12f5a09 to be ready +2025-06-05 16:00:54,245 - tests.orchestrator - INFO - Creating pod test-pod-397db057 on PVC test-pvc-13de15dd +2025-06-05 16:00:54,335 - tests.orchestrator - INFO - Created pod: test-pod-397db057 using PVC: test-pvc-13de15dd +2025-06-05 16:00:54,336 - tests.orchestrator - INFO - Waiting for pod test-pod-397db057 to be ready +2025-06-05 16:00:54,401 - tests.orchestrator - INFO - Pod test-pod-397db057 phase: Pending +2025-06-05 16:01:25,423 - tests.orchestrator - INFO - Pod test-pod-397db057 taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 16:01:25,423 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-397db057 ===== +2025-06-05 16:01:25,491 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:01:25,563 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:01:25,563 - tests.orchestrator - INFO - [2025-06-05 16:00:54+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:01:25,629 - tests.orchestrator - INFO - No logs available +2025-06-05 16:01:25,629 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:01:25,629 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-13de15dd'} +2025-06-05 16:01:25,630 - tests.orchestrator - INFO - - kube-api-access-q4p66: {} +2025-06-05 16:01:25,630 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-397db057 ===== +2025-06-05 16:01:54,513 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-397db057 to be ready after 60s +2025-06-05 16:01:54,513 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-397db057 ===== +2025-06-05 16:01:54,581 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:01:54,652 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:01:54,653 - tests.orchestrator - INFO - [2025-06-05 16:00:54+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:01:54,717 - tests.orchestrator - INFO - No logs available +2025-06-05 16:01:54,718 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:01:54,718 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-13de15dd'} +2025-06-05 16:01:54,718 - tests.orchestrator - INFO - - kube-api-access-q4p66: {} +2025-06-05 16:01:54,718 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-397db057 ===== +2025-06-05 16:01:54,718 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-397db057 to be ready +2025-06-05 16:01:54,718 - tests.orchestrator - INFO - Creating pod test-pod-b378ede9 on PVC test-pvc-13de15dd +2025-06-05 16:01:54,802 - tests.orchestrator - INFO - Created pod: test-pod-b378ede9 using PVC: test-pvc-13de15dd +2025-06-05 16:01:54,802 - tests.orchestrator - INFO - Waiting for pod test-pod-b378ede9 to be ready +2025-06-05 16:01:54,868 - tests.orchestrator - INFO - Pod test-pod-b378ede9 phase: Pending +2025-06-05 16:02:25,904 - tests.orchestrator - INFO - Pod test-pod-b378ede9 taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 16:02:25,904 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-b378ede9 ===== +2025-06-05 16:02:25,971 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:02:26,043 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:02:26,043 - tests.orchestrator - INFO - [2025-06-05 16:01:54+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:02:26,109 - tests.orchestrator - INFO - No logs available +2025-06-05 16:02:26,110 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:02:26,110 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-13de15dd'} +2025-06-05 16:02:26,110 - tests.orchestrator - INFO - - kube-api-access-ggwzs: {} +2025-06-05 16:02:26,110 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-b378ede9 ===== +2025-06-05 16:02:54,996 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b378ede9 to be ready after 60s +2025-06-05 16:02:54,996 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-b378ede9 ===== +2025-06-05 16:02:55,065 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:02:55,137 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:02:55,137 - tests.orchestrator - INFO - [2025-06-05 16:01:54+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:02:55,204 - tests.orchestrator - INFO - No logs available +2025-06-05 16:02:55,204 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:02:55,204 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-13de15dd'} +2025-06-05 16:02:55,204 - tests.orchestrator - INFO - - kube-api-access-ggwzs: {} +2025-06-05 16:02:55,204 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-b378ede9 ===== +2025-06-05 16:02:55,205 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b378ede9 to be ready +2025-06-05 16:02:55,205 - tests.orchestrator - INFO - Testing read/write between pods test-pod-397db057 and test-pod-b378ede9 sharing PVC test-pvc-13de15dd +2025-06-05 16:02:55,270 - tests.orchestrator - ERROR - Failed in read/write verification: (400) +Reason: Bad Request +HTTP response headers: HTTPHeaderDict({'Audit-Id': 'c1161df1-c0d5-4407-a027-5613955ae526', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 16:02:55 GMT', 'Content-Length': '166'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-397db057 does not have a host assigned","reason":"BadRequest","code":400} + + +2025-06-05 16:02:55,270 - tests.orchestrator - INFO - ============================================================ +2025-06-05 16:02:55,270 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_many_to_one +2025-06-05 16:02:55,270 - tests.orchestrator - INFO - ============================================================ +2025-06-05 16:02:55,270 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +2025-06-05 16:02:55,270 - tests.orchestrator - INFO - STARTING MANY-TO-ONE SCENARIO DIAGNOSTICS +2025-06-05 16:02:55,270 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +2025-06-05 16:02:55,271 - tests.orchestrator - INFO - [MANY2ONE] STEP 1: Creating dedicated PVC: many2one-68cd2a7e +2025-06-05 16:02:55,271 - tests.orchestrator - INFO - [MANY2ONE] Creating PVC manifest with storageClassName: efs-sc +2025-06-05 16:02:55,352 - tests.orchestrator - INFO - [MANY2ONE] PVC many2one-68cd2a7e created successfully +2025-06-05 16:02:55,352 - tests.orchestrator - INFO - [MANY2ONE] Waiting for PVC many2one-68cd2a7e to be bound... +2025-06-05 16:02:55,352 - tests.orchestrator - INFO - Waiting for PVC many2one-68cd2a7e to be bound +2025-06-05 16:02:57,487 - tests.orchestrator - INFO - PVC many2one-68cd2a7e is bound +2025-06-05 16:02:57,552 - tests.orchestrator - INFO - [MANY2ONE] PVC status: phase=Bound, capacity={'storage': '1Gi'} +2025-06-05 16:02:57,552 - tests.orchestrator - INFO - [MANY2ONE] STEP 2: Creating 4 pods for the same PVC many2one-68cd2a7e +2025-06-05 16:02:57,552 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 1/4 for PVC many2one-68cd2a7e +2025-06-05 16:02:57,552 - tests.orchestrator - INFO - Creating pod test-pod-4518fc8f on PVC many2one-68cd2a7e +2025-06-05 16:02:57,632 - tests.orchestrator - INFO - Created pod: test-pod-4518fc8f using PVC: many2one-68cd2a7e +2025-06-05 16:02:57,633 - tests.orchestrator - INFO - Waiting for pod test-pod-4518fc8f to be ready +2025-06-05 16:02:57,699 - tests.orchestrator - INFO - Pod test-pod-4518fc8f phase: Pending +2025-06-05 16:03:28,731 - tests.orchestrator - INFO - Pod test-pod-4518fc8f taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 16:03:28,732 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-4518fc8f ===== +2025-06-05 16:03:28,800 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:03:28,872 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:03:28,872 - tests.orchestrator - INFO - [2025-06-05 16:02:57+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:03:28,937 - tests.orchestrator - INFO - No logs available +2025-06-05 16:03:28,937 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:03:28,937 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-68cd2a7e'} +2025-06-05 16:03:28,937 - tests.orchestrator - INFO - - kube-api-access-v9z8m: {} +2025-06-05 16:03:28,937 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-4518fc8f ===== +2025-06-05 16:03:57,829 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-4518fc8f to be ready after 60s +2025-06-05 16:03:57,829 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-4518fc8f ===== +2025-06-05 16:03:57,897 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:03:57,967 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:03:57,968 - tests.orchestrator - INFO - [2025-06-05 16:02:57+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:03:58,032 - tests.orchestrator - INFO - No logs available +2025-06-05 16:03:58,032 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:03:58,032 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-68cd2a7e'} +2025-06-05 16:03:58,032 - tests.orchestrator - INFO - - kube-api-access-v9z8m: {} +2025-06-05 16:03:58,032 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-4518fc8f ===== +2025-06-05 16:03:58,033 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-4518fc8f to be ready +2025-06-05 16:03:58,033 - tests.orchestrator - ERROR - [MANY2ONE] Failed to create pod 1/4 +2025-06-05 16:03:58,033 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 2/4 for PVC many2one-68cd2a7e +2025-06-05 16:03:58,033 - tests.orchestrator - INFO - Creating pod test-pod-0c1359ff on PVC many2one-68cd2a7e +2025-06-05 16:03:58,117 - tests.orchestrator - INFO - Created pod: test-pod-0c1359ff using PVC: many2one-68cd2a7e +2025-06-05 16:03:58,117 - tests.orchestrator - INFO - Waiting for pod test-pod-0c1359ff to be ready +2025-06-05 16:03:58,183 - tests.orchestrator - INFO - Pod test-pod-0c1359ff phase: Pending +2025-06-05 16:04:29,215 - tests.orchestrator - INFO - Pod test-pod-0c1359ff taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 16:04:29,215 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-0c1359ff ===== +2025-06-05 16:04:29,283 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:04:29,355 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:04:29,356 - tests.orchestrator - INFO - [2025-06-05 16:03:58+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:04:29,422 - tests.orchestrator - INFO - No logs available +2025-06-05 16:04:29,422 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:04:29,422 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-68cd2a7e'} +2025-06-05 16:04:29,422 - tests.orchestrator - INFO - - kube-api-access-6jtzc: {} +2025-06-05 16:04:29,422 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-0c1359ff ===== +2025-06-05 16:04:58,441 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-0c1359ff to be ready after 60s +2025-06-05 16:04:58,441 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-0c1359ff ===== +2025-06-05 16:04:58,507 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:04:58,577 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:04:58,577 - tests.orchestrator - INFO - [2025-06-05 16:03:58+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:04:58,644 - tests.orchestrator - INFO - No logs available +2025-06-05 16:04:58,644 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:04:58,644 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-68cd2a7e'} +2025-06-05 16:04:58,644 - tests.orchestrator - INFO - - kube-api-access-6jtzc: {} +2025-06-05 16:04:58,644 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-0c1359ff ===== +2025-06-05 16:04:58,644 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-0c1359ff to be ready +2025-06-05 16:04:58,644 - tests.orchestrator - ERROR - [MANY2ONE] Failed to create pod 2/4 +2025-06-05 16:04:58,644 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 3/4 for PVC many2one-68cd2a7e +2025-06-05 16:04:58,645 - tests.orchestrator - INFO - Creating pod test-pod-1e078739 on PVC many2one-68cd2a7e +2025-06-05 16:04:58,732 - tests.orchestrator - INFO - Created pod: test-pod-1e078739 using PVC: many2one-68cd2a7e +2025-06-05 16:04:58,732 - tests.orchestrator - INFO - Waiting for pod test-pod-1e078739 to be ready +2025-06-05 16:04:58,800 - tests.orchestrator - INFO - Pod test-pod-1e078739 phase: Pending +2025-06-05 16:05:29,826 - tests.orchestrator - INFO - Pod test-pod-1e078739 taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 16:05:29,826 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-1e078739 ===== +2025-06-05 16:05:29,893 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:05:29,963 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:05:29,963 - tests.orchestrator - INFO - [2025-06-05 16:04:58+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:05:30,027 - tests.orchestrator - INFO - No logs available +2025-06-05 16:05:30,027 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:05:30,028 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-68cd2a7e'} +2025-06-05 16:05:30,028 - tests.orchestrator - INFO - - kube-api-access-rksgf: {} +2025-06-05 16:05:30,028 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-1e078739 ===== +2025-06-05 16:05:58,917 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1e078739 to be ready after 60s +2025-06-05 16:05:58,917 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-1e078739 ===== +2025-06-05 16:05:58,985 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:05:59,057 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:05:59,057 - tests.orchestrator - INFO - [2025-06-05 16:04:58+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:05:59,122 - tests.orchestrator - INFO - No logs available +2025-06-05 16:05:59,122 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:05:59,122 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-68cd2a7e'} +2025-06-05 16:05:59,122 - tests.orchestrator - INFO - - kube-api-access-rksgf: {} +2025-06-05 16:05:59,122 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-1e078739 ===== +2025-06-05 16:05:59,122 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1e078739 to be ready +2025-06-05 16:05:59,122 - tests.orchestrator - ERROR - [MANY2ONE] Failed to create pod 3/4 +2025-06-05 16:05:59,122 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 4/4 for PVC many2one-68cd2a7e +2025-06-05 16:05:59,122 - tests.orchestrator - INFO - Creating pod test-pod-a5d14df4 on PVC many2one-68cd2a7e +2025-06-05 16:05:59,208 - tests.orchestrator - INFO - Created pod: test-pod-a5d14df4 using PVC: many2one-68cd2a7e +2025-06-05 16:05:59,208 - tests.orchestrator - INFO - Waiting for pod test-pod-a5d14df4 to be ready +2025-06-05 16:05:59,273 - tests.orchestrator - INFO - Pod test-pod-a5d14df4 phase: Pending +2025-06-05 16:06:30,377 - tests.orchestrator - INFO - Pod test-pod-a5d14df4 taking longer than expected to become ready (31.2s). Collecting diagnostics... +2025-06-05 16:06:30,377 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-a5d14df4 ===== +2025-06-05 16:06:30,444 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:06:30,516 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:06:30,517 - tests.orchestrator - INFO - [2025-06-05 16:05:59+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:06:30,583 - tests.orchestrator - INFO - No logs available +2025-06-05 16:06:30,583 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:06:30,584 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-68cd2a7e'} +2025-06-05 16:06:30,584 - tests.orchestrator - INFO - - kube-api-access-dcxmb: {} +2025-06-05 16:06:30,584 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-a5d14df4 ===== +2025-06-05 16:06:59,487 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a5d14df4 to be ready after 60s +2025-06-05 16:06:59,487 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-a5d14df4 ===== +2025-06-05 16:06:59,555 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:06:59,627 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:06:59,628 - tests.orchestrator - INFO - [2025-06-05 16:05:59+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:06:59,694 - tests.orchestrator - INFO - No logs available +2025-06-05 16:06:59,694 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:06:59,694 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-68cd2a7e'} +2025-06-05 16:06:59,695 - tests.orchestrator - INFO - - kube-api-access-dcxmb: {} +2025-06-05 16:06:59,695 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-a5d14df4 ===== +2025-06-05 16:06:59,695 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a5d14df4 to be ready +2025-06-05 16:06:59,695 - tests.orchestrator - ERROR - [MANY2ONE] Failed to create pod 4/4 +2025-06-05 16:06:59,695 - tests.orchestrator - INFO - [MANY2ONE] Created 0/4 pods successfully +2025-06-05 16:06:59,695 - tests.orchestrator - ERROR - [MANY2ONE] FAILED: Insufficient pods created (0), need at least 2 for read/write test +2025-06-05 16:06:59,695 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_many_to_one +2025-06-05 16:06:59,695 - tests.orchestrator - INFO - ------------------------------------------------------------ +2025-06-05 16:06:59,695 - tests.orchestrator - INFO - Deleting pod: test-pod-1e078739 from PVC: many2one-68cd2a7e +2025-06-05 16:06:59,777 - tests.orchestrator - INFO - Waiting for pod test-pod-1e078739 to be deleted +2025-06-05 16:06:59,841 - tests.orchestrator - INFO - Pod test-pod-1e078739 has been deleted +2025-06-05 16:06:59,842 - tests.orchestrator - INFO - Deleted pod: test-pod-1e078739 +2025-06-05 16:06:59,842 - tests.orchestrator - INFO - Deleting PVC: test-pvc-13de15dd +2025-06-05 16:06:59,842 - tests.orchestrator - INFO - Deleting 2 pods using PVC test-pvc-13de15dd +2025-06-05 16:06:59,842 - tests.orchestrator - INFO - Deleting pod: test-pod-397db057 from PVC: test-pvc-13de15dd +2025-06-05 16:06:59,923 - tests.orchestrator - INFO - Waiting for pod test-pod-397db057 to be deleted +2025-06-05 16:06:59,989 - tests.orchestrator - INFO - Pod test-pod-397db057 has been deleted +2025-06-05 16:06:59,989 - tests.orchestrator - INFO - Deleted pod: test-pod-397db057 +2025-06-05 16:06:59,989 - tests.orchestrator - INFO - Deleting pod: test-pod-b378ede9 from PVC: test-pvc-13de15dd +2025-06-05 16:07:00,077 - tests.orchestrator - INFO - Waiting for pod test-pod-b378ede9 to be deleted +2025-06-05 16:07:00,142 - tests.orchestrator - INFO - Pod test-pod-b378ede9 has been deleted +2025-06-05 16:07:00,142 - tests.orchestrator - INFO - Deleted pod: test-pod-b378ede9 +2025-06-05 16:07:00,213 - tests.orchestrator - INFO - Waiting for PVC test-pvc-13de15dd to be deleted +2025-06-05 16:07:00,280 - tests.orchestrator - INFO - PVC test-pvc-13de15dd has been deleted +2025-06-05 16:07:00,280 - tests.orchestrator - INFO - Deleted PVC: test-pvc-13de15dd +2025-06-05 16:07:00,280 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations +2025-06-05 16:07:00,280 - tests.orchestrator - INFO - Test completed in 431.25 seconds +2025-06-05 16:07:00,280 - tests.orchestrator - INFO - Cleaning up resources... +2025-06-05 16:07:00,280 - tests.orchestrator - INFO - Deleting pod: test-pod-b12f5a09 from PVC: test-pvc-47b1c666 +2025-06-05 16:07:00,364 - tests.orchestrator - INFO - Waiting for pod test-pod-b12f5a09 to be deleted +2025-06-05 16:07:00,430 - tests.orchestrator - INFO - Pod test-pod-b12f5a09 has been deleted +2025-06-05 16:07:00,430 - tests.orchestrator - INFO - Deleted pod: test-pod-b12f5a09 +2025-06-05 16:07:00,431 - tests.orchestrator - INFO - Deleting pod: test-pod-4518fc8f from PVC: many2one-68cd2a7e +2025-06-05 16:07:00,512 - tests.orchestrator - INFO - Waiting for pod test-pod-4518fc8f to be deleted +2025-06-05 16:07:00,579 - tests.orchestrator - INFO - Pod test-pod-4518fc8f has been deleted +2025-06-05 16:07:00,579 - tests.orchestrator - INFO - Deleted pod: test-pod-4518fc8f +2025-06-05 16:07:00,579 - tests.orchestrator - INFO - Deleting pod: test-pod-0c1359ff from PVC: many2one-68cd2a7e +2025-06-05 16:07:00,659 - tests.orchestrator - INFO - Waiting for pod test-pod-0c1359ff to be deleted +2025-06-05 16:07:00,725 - tests.orchestrator - INFO - Pod test-pod-0c1359ff has been deleted +2025-06-05 16:07:00,726 - tests.orchestrator - INFO - Deleted pod: test-pod-0c1359ff +2025-06-05 16:07:00,726 - tests.orchestrator - INFO - Deleting pod: test-pod-a5d14df4 from PVC: many2one-68cd2a7e +2025-06-05 16:07:00,813 - tests.orchestrator - INFO - Waiting for pod test-pod-a5d14df4 to be deleted +2025-06-05 16:07:00,878 - tests.orchestrator - INFO - Pod test-pod-a5d14df4 has been deleted +2025-06-05 16:07:00,878 - tests.orchestrator - INFO - Deleted pod: test-pod-a5d14df4 +2025-06-05 16:07:00,878 - tests.orchestrator - INFO - Deleting PVC: test-pvc-47b1c666 +2025-06-05 16:07:00,951 - tests.orchestrator - INFO - Waiting for PVC test-pvc-47b1c666 to be deleted +2025-06-05 16:07:01,017 - tests.orchestrator - INFO - PVC test-pvc-47b1c666 has been deleted +2025-06-05 16:07:01,018 - tests.orchestrator - INFO - Deleted PVC: test-pvc-47b1c666 +2025-06-05 16:07:01,018 - tests.orchestrator - INFO - Deleting PVC: many2one-68cd2a7e +2025-06-05 16:07:01,090 - tests.orchestrator - INFO - Waiting for PVC many2one-68cd2a7e to be deleted +2025-06-05 16:07:01,157 - tests.orchestrator - INFO - PVC many2one-68cd2a7e has been deleted +2025-06-05 16:07:01,157 - tests.orchestrator - INFO - Deleted PVC: many2one-68cd2a7e +2025-06-05 16:07:01,157 - tests.orchestrator - INFO - Cleanup completed +2025-06-05 16:07:01,157 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-05 16:07:01,157 - tests.orchestrator - INFO - --- Operations --- +2025-06-05 16:07:01,157 - tests.orchestrator - INFO - create_pvc: 2 succeeded, 0 failed (100.0%) +2025-06-05 16:07:01,157 - tests.orchestrator - INFO - attach_pod: 7 succeeded, 0 failed (100.0%) +2025-06-05 16:07:01,157 - tests.orchestrator - INFO - delete_pod: 7 succeeded, 0 failed (100.0%) +2025-06-05 16:07:01,157 - tests.orchestrator - INFO - delete_pvc: 3 succeeded, 0 failed (100.0%) +2025-06-05 16:07:01,157 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) +2025-06-05 16:07:01,157 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) +2025-06-05 16:07:01,157 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-05 16:07:01,158 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-05 16:07:01,158 - tests.orchestrator - INFO - many_to_one: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-05 16:07:01,158 - tests.orchestrator - INFO - one_to_one: No runs +2025-06-05 16:07:01,158 - tests.orchestrator - INFO - concurrent_pvc: No runs +2025-06-05 16:07:01,158 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_160844.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_160844.log new file mode 100644 index 000000000..2336bab83 --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_160844.log @@ -0,0 +1,281 @@ +2025-06-05 16:08:44,038 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 16:08:44,245 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 16:08:44,246 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-05 16:08:44,436 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 16:08:44,436 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 16:08:44,436 - tests.orchestrator - INFO - Creating PVC: test-pvc-6fb3132b +2025-06-05 16:08:44,509 - tests.orchestrator - INFO - Created PVC: test-pvc-6fb3132b +2025-06-05 16:08:44,510 - tests.orchestrator - INFO - Waiting for PVC test-pvc-6fb3132b to be bound +2025-06-05 16:08:46,643 - tests.orchestrator - INFO - PVC test-pvc-6fb3132b is bound +2025-06-05 16:08:46,643 - tests.orchestrator - INFO - Creating PVC: test-pvc-989dfd43 +2025-06-05 16:08:46,714 - tests.orchestrator - INFO - Created PVC: test-pvc-989dfd43 +2025-06-05 16:08:46,714 - tests.orchestrator - INFO - Waiting for PVC test-pvc-989dfd43 to be bound +2025-06-05 16:08:48,850 - tests.orchestrator - INFO - PVC test-pvc-989dfd43 is bound +2025-06-05 16:08:48,851 - tests.orchestrator - INFO - Creating pod test-pod-f073851f on PVC test-pvc-6fb3132b +2025-06-05 16:08:48,934 - tests.orchestrator - INFO - Created pod: test-pod-f073851f using PVC: test-pvc-6fb3132b +2025-06-05 16:08:48,934 - tests.orchestrator - INFO - Waiting for pod test-pod-f073851f to be ready +2025-06-05 16:08:49,001 - tests.orchestrator - INFO - Pod test-pod-f073851f phase: Pending +2025-06-05 16:09:20,151 - tests.orchestrator - INFO - Pod test-pod-f073851f taking longer than expected to become ready (31.2s). Collecting diagnostics... +2025-06-05 16:09:20,151 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-f073851f ===== +2025-06-05 16:09:20,217 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:09:20,287 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:09:20,287 - tests.orchestrator - INFO - [2025-06-05 16:08:48+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:09:20,354 - tests.orchestrator - INFO - No logs available +2025-06-05 16:09:20,354 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:09:20,354 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-6fb3132b'} +2025-06-05 16:09:20,354 - tests.orchestrator - INFO - - kube-api-access-l7zvz: {} +2025-06-05 16:09:20,355 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-f073851f ===== +2025-06-05 16:09:49,251 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-f073851f to be ready after 60s +2025-06-05 16:09:49,251 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-f073851f ===== +2025-06-05 16:09:49,317 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:09:49,389 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:09:49,390 - tests.orchestrator - INFO - [2025-06-05 16:08:48+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:09:49,456 - tests.orchestrator - INFO - No logs available +2025-06-05 16:09:49,456 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:09:49,456 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-6fb3132b'} +2025-06-05 16:09:49,456 - tests.orchestrator - INFO - - kube-api-access-l7zvz: {} +2025-06-05 16:09:49,457 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-f073851f ===== +2025-06-05 16:09:49,457 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-f073851f to be ready +2025-06-05 16:09:49,457 - tests.orchestrator - INFO - Creating pod test-pod-6b63ccc1 on PVC test-pvc-6fb3132b +2025-06-05 16:09:49,543 - tests.orchestrator - INFO - Created pod: test-pod-6b63ccc1 using PVC: test-pvc-6fb3132b +2025-06-05 16:09:49,543 - tests.orchestrator - INFO - Waiting for pod test-pod-6b63ccc1 to be ready +2025-06-05 16:09:49,612 - tests.orchestrator - INFO - Pod test-pod-6b63ccc1 phase: Pending +2025-06-05 16:10:20,641 - tests.orchestrator - INFO - Pod test-pod-6b63ccc1 taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 16:10:20,641 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-6b63ccc1 ===== +2025-06-05 16:10:20,710 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:10:20,782 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:10:20,782 - tests.orchestrator - INFO - [2025-06-05 16:09:49+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:10:20,849 - tests.orchestrator - INFO - No logs available +2025-06-05 16:10:20,849 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:10:20,849 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-6fb3132b'} +2025-06-05 16:10:20,849 - tests.orchestrator - INFO - - kube-api-access-8pzqd: {} +2025-06-05 16:10:20,849 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-6b63ccc1 ===== +2025-06-05 16:10:49,843 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-6b63ccc1 to be ready after 60s +2025-06-05 16:10:49,843 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-6b63ccc1 ===== +2025-06-05 16:10:49,909 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:10:49,979 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:10:49,979 - tests.orchestrator - INFO - [2025-06-05 16:09:49+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:10:50,046 - tests.orchestrator - INFO - No logs available +2025-06-05 16:10:50,046 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:10:50,046 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-6fb3132b'} +2025-06-05 16:10:50,046 - tests.orchestrator - INFO - - kube-api-access-8pzqd: {} +2025-06-05 16:10:50,046 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-6b63ccc1 ===== +2025-06-05 16:10:50,046 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-6b63ccc1 to be ready +2025-06-05 16:10:50,046 - tests.orchestrator - INFO - Creating pod test-pod-91cf9293 on PVC test-pvc-989dfd43 +2025-06-05 16:10:50,129 - tests.orchestrator - INFO - Created pod: test-pod-91cf9293 using PVC: test-pvc-989dfd43 +2025-06-05 16:10:50,129 - tests.orchestrator - INFO - Waiting for pod test-pod-91cf9293 to be ready +2025-06-05 16:10:50,197 - tests.orchestrator - INFO - Pod test-pod-91cf9293 phase: Pending +2025-06-05 16:11:21,219 - tests.orchestrator - INFO - Pod test-pod-91cf9293 taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 16:11:21,219 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-91cf9293 ===== +2025-06-05 16:11:21,287 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:11:21,355 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:11:21,356 - tests.orchestrator - INFO - [2025-06-05 16:10:50+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:11:21,422 - tests.orchestrator - INFO - No logs available +2025-06-05 16:11:21,422 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:11:21,422 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-989dfd43'} +2025-06-05 16:11:21,423 - tests.orchestrator - INFO - - kube-api-access-z69bk: {} +2025-06-05 16:11:21,423 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-91cf9293 ===== +2025-06-05 16:11:50,358 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-91cf9293 to be ready after 60s +2025-06-05 16:11:50,358 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-91cf9293 ===== +2025-06-05 16:11:50,425 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:11:50,494 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:11:50,495 - tests.orchestrator - INFO - [2025-06-05 16:10:50+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:11:50,561 - tests.orchestrator - INFO - No logs available +2025-06-05 16:11:50,561 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:11:50,561 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-989dfd43'} +2025-06-05 16:11:50,561 - tests.orchestrator - INFO - - kube-api-access-z69bk: {} +2025-06-05 16:11:50,561 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-91cf9293 ===== +2025-06-05 16:11:50,561 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-91cf9293 to be ready +2025-06-05 16:11:50,562 - tests.orchestrator - INFO - Testing read/write between pods test-pod-6b63ccc1 and test-pod-f073851f sharing PVC test-pvc-6fb3132b +2025-06-05 16:11:50,628 - tests.orchestrator - ERROR - Failed in read/write verification: (400) +Reason: Bad Request +HTTP response headers: HTTPHeaderDict({'Audit-Id': 'ced94a61-feed-4098-84b1-2d400a53a883', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 16:11:50 GMT', 'Content-Length': '166'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-6b63ccc1 does not have a host assigned","reason":"BadRequest","code":400} + + +2025-06-05 16:11:50,629 - tests.orchestrator - INFO - ============================================================ +2025-06-05 16:11:50,629 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_concurrent_pvc +2025-06-05 16:11:50,629 - tests.orchestrator - INFO - ============================================================ +2025-06-05 16:11:50,629 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations +2025-06-05 16:11:50,629 - tests.orchestrator - INFO - Creating 7 PVCs in quick succession +2025-06-05 16:11:50,709 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-7711078c +2025-06-05 16:11:50,710 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-7711078c to be bound +2025-06-05 16:11:52,845 - tests.orchestrator - INFO - PVC concurrent-pvc-7711078c is bound +2025-06-05 16:11:52,913 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-4da2d71f +2025-06-05 16:11:52,913 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-4da2d71f to be bound +2025-06-05 16:11:55,058 - tests.orchestrator - INFO - PVC concurrent-pvc-4da2d71f is bound +2025-06-05 16:11:55,126 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-58107a6b +2025-06-05 16:11:55,126 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-58107a6b to be bound +2025-06-05 16:11:57,266 - tests.orchestrator - INFO - PVC concurrent-pvc-58107a6b is bound +2025-06-05 16:11:57,333 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-0bbb89d4 +2025-06-05 16:11:57,333 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-0bbb89d4 to be bound +2025-06-05 16:11:59,469 - tests.orchestrator - INFO - PVC concurrent-pvc-0bbb89d4 is bound +2025-06-05 16:11:59,537 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-aad17783 +2025-06-05 16:11:59,537 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-aad17783 to be bound +2025-06-05 16:12:01,702 - tests.orchestrator - INFO - PVC concurrent-pvc-aad17783 is bound +2025-06-05 16:12:01,782 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-792a8a18 +2025-06-05 16:12:01,782 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-792a8a18 to be bound +2025-06-05 16:12:03,930 - tests.orchestrator - INFO - PVC concurrent-pvc-792a8a18 is bound +2025-06-05 16:12:03,999 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-95d0108f +2025-06-05 16:12:03,999 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-95d0108f to be bound +2025-06-05 16:12:06,134 - tests.orchestrator - INFO - PVC concurrent-pvc-95d0108f is bound +2025-06-05 16:12:06,134 - tests.orchestrator - INFO - Successfully created 7 PVCs +2025-06-05 16:12:06,134 - tests.orchestrator - INFO - Creating pod test-pod-2497c925 on PVC concurrent-pvc-58107a6b +2025-06-05 16:12:06,207 - tests.orchestrator - INFO - Created pod: test-pod-2497c925 using PVC: concurrent-pvc-58107a6b +2025-06-05 16:12:06,207 - tests.orchestrator - INFO - Waiting for pod test-pod-2497c925 to be ready +2025-06-05 16:12:06,275 - tests.orchestrator - INFO - Pod test-pod-2497c925 phase: Pending +2025-06-05 16:12:37,298 - tests.orchestrator - INFO - Pod test-pod-2497c925 taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 16:12:37,298 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-2497c925 ===== +2025-06-05 16:12:37,365 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:12:37,437 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:12:37,437 - tests.orchestrator - INFO - [2025-06-05 16:12:06+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:12:37,504 - tests.orchestrator - INFO - No logs available +2025-06-05 16:12:37,504 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:12:37,504 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'concurrent-pvc-58107a6b'} +2025-06-05 16:12:37,504 - tests.orchestrator - INFO - - kube-api-access-bfxtj: {} +2025-06-05 16:12:37,504 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-2497c925 ===== +2025-06-05 16:13:06,471 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-2497c925 to be ready after 60s +2025-06-05 16:13:06,471 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-2497c925 ===== +2025-06-05 16:13:06,541 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:13:06,616 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:13:06,616 - tests.orchestrator - INFO - [2025-06-05 16:12:06+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:13:06,684 - tests.orchestrator - INFO - No logs available +2025-06-05 16:13:06,684 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:13:06,685 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'concurrent-pvc-58107a6b'} +2025-06-05 16:13:06,685 - tests.orchestrator - INFO - - kube-api-access-bfxtj: {} +2025-06-05 16:13:06,685 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-2497c925 ===== +2025-06-05 16:13:06,685 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-2497c925 to be ready +2025-06-05 16:13:06,685 - tests.orchestrator - INFO - Creating pod test-pod-3bfa61d5 on PVC concurrent-pvc-4da2d71f +2025-06-05 16:13:06,776 - tests.orchestrator - INFO - Created pod: test-pod-3bfa61d5 using PVC: concurrent-pvc-4da2d71f +2025-06-05 16:13:06,777 - tests.orchestrator - INFO - Waiting for pod test-pod-3bfa61d5 to be ready +2025-06-05 16:13:06,847 - tests.orchestrator - INFO - Pod test-pod-3bfa61d5 phase: Pending +2025-06-05 16:13:37,879 - tests.orchestrator - INFO - Pod test-pod-3bfa61d5 taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 16:13:37,879 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-3bfa61d5 ===== +2025-06-05 16:13:37,945 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:13:38,018 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:13:38,018 - tests.orchestrator - INFO - [2025-06-05 16:13:06+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:13:38,085 - tests.orchestrator - INFO - No logs available +2025-06-05 16:13:38,085 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:13:38,085 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'concurrent-pvc-4da2d71f'} +2025-06-05 16:13:38,086 - tests.orchestrator - INFO - - kube-api-access-8b465: {} +2025-06-05 16:13:38,086 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-3bfa61d5 ===== +2025-06-05 16:14:06,978 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-3bfa61d5 to be ready after 60s +2025-06-05 16:14:06,978 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-3bfa61d5 ===== +2025-06-05 16:14:07,046 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:14:07,118 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:14:07,119 - tests.orchestrator - INFO - [2025-06-05 16:13:06+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:14:07,186 - tests.orchestrator - INFO - No logs available +2025-06-05 16:14:07,186 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:14:07,186 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'concurrent-pvc-4da2d71f'} +2025-06-05 16:14:07,186 - tests.orchestrator - INFO - - kube-api-access-8b465: {} +2025-06-05 16:14:07,186 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-3bfa61d5 ===== +2025-06-05 16:14:07,186 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-3bfa61d5 to be ready +2025-06-05 16:14:07,186 - tests.orchestrator - INFO - Creating pod test-pod-d5f1e1e8 on PVC concurrent-pvc-792a8a18 +2025-06-05 16:14:07,273 - tests.orchestrator - INFO - Created pod: test-pod-d5f1e1e8 using PVC: concurrent-pvc-792a8a18 +2025-06-05 16:14:07,273 - tests.orchestrator - INFO - Waiting for pod test-pod-d5f1e1e8 to be ready +2025-06-05 16:14:07,342 - tests.orchestrator - INFO - Pod test-pod-d5f1e1e8 phase: Pending +2025-06-05 16:14:38,491 - tests.orchestrator - INFO - Pod test-pod-d5f1e1e8 taking longer than expected to become ready (31.2s). Collecting diagnostics... +2025-06-05 16:14:38,491 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-d5f1e1e8 ===== +2025-06-05 16:14:38,557 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:14:38,629 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:14:38,629 - tests.orchestrator - INFO - [2025-06-05 16:14:07+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:14:38,696 - tests.orchestrator - INFO - No logs available +2025-06-05 16:14:38,696 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:14:38,696 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'concurrent-pvc-792a8a18'} +2025-06-05 16:14:38,696 - tests.orchestrator - INFO - - kube-api-access-7h2vf: {} +2025-06-05 16:14:38,696 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-d5f1e1e8 ===== +2025-06-05 16:15:07,592 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-d5f1e1e8 to be ready after 60s +2025-06-05 16:15:07,593 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-d5f1e1e8 ===== +2025-06-05 16:15:07,662 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:15:07,735 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:15:07,735 - tests.orchestrator - INFO - [2025-06-05 16:14:07+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:15:07,803 - tests.orchestrator - INFO - No logs available +2025-06-05 16:15:07,803 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:15:07,804 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'concurrent-pvc-792a8a18'} +2025-06-05 16:15:07,804 - tests.orchestrator - INFO - - kube-api-access-7h2vf: {} +2025-06-05 16:15:07,804 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-d5f1e1e8 ===== +2025-06-05 16:15:07,804 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-d5f1e1e8 to be ready +2025-06-05 16:15:07,804 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-58107a6b +2025-06-05 16:15:07,804 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-58107a6b +2025-06-05 16:15:07,804 - tests.orchestrator - INFO - Deleting pod: test-pod-2497c925 from PVC: concurrent-pvc-58107a6b +2025-06-05 16:15:07,888 - tests.orchestrator - INFO - Waiting for pod test-pod-2497c925 to be deleted +2025-06-05 16:15:07,953 - tests.orchestrator - INFO - Pod test-pod-2497c925 has been deleted +2025-06-05 16:15:07,953 - tests.orchestrator - INFO - Deleted pod: test-pod-2497c925 +2025-06-05 16:15:08,028 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-58107a6b to be deleted +2025-06-05 16:15:08,107 - tests.orchestrator - INFO - PVC concurrent-pvc-58107a6b has been deleted +2025-06-05 16:15:08,108 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-58107a6b +2025-06-05 16:15:08,108 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-4da2d71f +2025-06-05 16:15:08,108 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-4da2d71f +2025-06-05 16:15:08,108 - tests.orchestrator - INFO - Deleting pod: test-pod-3bfa61d5 from PVC: concurrent-pvc-4da2d71f +2025-06-05 16:15:08,193 - tests.orchestrator - INFO - Waiting for pod test-pod-3bfa61d5 to be deleted +2025-06-05 16:15:08,258 - tests.orchestrator - INFO - Pod test-pod-3bfa61d5 has been deleted +2025-06-05 16:15:08,258 - tests.orchestrator - INFO - Deleted pod: test-pod-3bfa61d5 +2025-06-05 16:15:08,332 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-4da2d71f to be deleted +2025-06-05 16:15:08,397 - tests.orchestrator - INFO - PVC concurrent-pvc-4da2d71f has been deleted +2025-06-05 16:15:08,397 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-4da2d71f +2025-06-05 16:15:08,397 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-792a8a18 +2025-06-05 16:15:08,397 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-792a8a18 +2025-06-05 16:15:08,397 - tests.orchestrator - INFO - Deleting pod: test-pod-d5f1e1e8 from PVC: concurrent-pvc-792a8a18 +2025-06-05 16:15:08,484 - tests.orchestrator - INFO - Waiting for pod test-pod-d5f1e1e8 to be deleted +2025-06-05 16:15:08,549 - tests.orchestrator - INFO - Pod test-pod-d5f1e1e8 has been deleted +2025-06-05 16:15:08,549 - tests.orchestrator - INFO - Deleted pod: test-pod-d5f1e1e8 +2025-06-05 16:15:08,623 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-792a8a18 to be deleted +2025-06-05 16:15:08,689 - tests.orchestrator - INFO - PVC concurrent-pvc-792a8a18 has been deleted +2025-06-05 16:15:08,689 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-792a8a18 +2025-06-05 16:15:08,689 - tests.orchestrator - INFO - Rapid PVC scenario completed +2025-06-05 16:15:08,689 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_concurrent_pvc +2025-06-05 16:15:08,689 - tests.orchestrator - INFO - ------------------------------------------------------------ +2025-06-05 16:15:08,689 - tests.orchestrator - INFO - Deleting pod: test-pod-f073851f from PVC: test-pvc-6fb3132b +2025-06-05 16:15:08,770 - tests.orchestrator - INFO - Waiting for pod test-pod-f073851f to be deleted +2025-06-05 16:15:08,837 - tests.orchestrator - INFO - Pod test-pod-f073851f has been deleted +2025-06-05 16:15:08,837 - tests.orchestrator - INFO - Deleted pod: test-pod-f073851f +2025-06-05 16:15:08,837 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-aad17783 +2025-06-05 16:15:08,912 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-aad17783 to be deleted +2025-06-05 16:15:08,977 - tests.orchestrator - INFO - PVC concurrent-pvc-aad17783 has been deleted +2025-06-05 16:15:08,977 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-aad17783 +2025-06-05 16:15:08,977 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations +2025-06-05 16:15:08,977 - tests.orchestrator - INFO - Test completed in 384.73 seconds +2025-06-05 16:15:08,977 - tests.orchestrator - INFO - Cleaning up resources... +2025-06-05 16:15:08,977 - tests.orchestrator - INFO - Deleting pod: test-pod-6b63ccc1 from PVC: test-pvc-6fb3132b +2025-06-05 16:15:09,060 - tests.orchestrator - INFO - Waiting for pod test-pod-6b63ccc1 to be deleted +2025-06-05 16:15:09,127 - tests.orchestrator - INFO - Pod test-pod-6b63ccc1 has been deleted +2025-06-05 16:15:09,127 - tests.orchestrator - INFO - Deleted pod: test-pod-6b63ccc1 +2025-06-05 16:15:09,127 - tests.orchestrator - INFO - Deleting pod: test-pod-91cf9293 from PVC: test-pvc-989dfd43 +2025-06-05 16:15:09,214 - tests.orchestrator - INFO - Waiting for pod test-pod-91cf9293 to be deleted +2025-06-05 16:15:09,280 - tests.orchestrator - INFO - Pod test-pod-91cf9293 has been deleted +2025-06-05 16:15:09,280 - tests.orchestrator - INFO - Deleted pod: test-pod-91cf9293 +2025-06-05 16:15:09,280 - tests.orchestrator - INFO - Deleting PVC: test-pvc-6fb3132b +2025-06-05 16:15:09,355 - tests.orchestrator - INFO - Waiting for PVC test-pvc-6fb3132b to be deleted +2025-06-05 16:15:09,420 - tests.orchestrator - INFO - PVC test-pvc-6fb3132b has been deleted +2025-06-05 16:15:09,420 - tests.orchestrator - INFO - Deleted PVC: test-pvc-6fb3132b +2025-06-05 16:15:09,420 - tests.orchestrator - INFO - Deleting PVC: test-pvc-989dfd43 +2025-06-05 16:15:09,496 - tests.orchestrator - INFO - Waiting for PVC test-pvc-989dfd43 to be deleted +2025-06-05 16:15:09,561 - tests.orchestrator - INFO - PVC test-pvc-989dfd43 has been deleted +2025-06-05 16:15:09,561 - tests.orchestrator - INFO - Deleted PVC: test-pvc-989dfd43 +2025-06-05 16:15:09,561 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-7711078c +2025-06-05 16:15:09,635 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-7711078c to be deleted +2025-06-05 16:15:09,700 - tests.orchestrator - INFO - PVC concurrent-pvc-7711078c has been deleted +2025-06-05 16:15:09,701 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-7711078c +2025-06-05 16:15:09,701 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-0bbb89d4 +2025-06-05 16:15:09,775 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-0bbb89d4 to be deleted +2025-06-05 16:15:09,840 - tests.orchestrator - INFO - PVC concurrent-pvc-0bbb89d4 has been deleted +2025-06-05 16:15:09,840 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-0bbb89d4 +2025-06-05 16:15:09,840 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-95d0108f +2025-06-05 16:15:09,913 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-95d0108f to be deleted +2025-06-05 16:15:09,978 - tests.orchestrator - INFO - PVC concurrent-pvc-95d0108f has been deleted +2025-06-05 16:15:09,978 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-95d0108f +2025-06-05 16:15:09,978 - tests.orchestrator - INFO - Cleanup completed +2025-06-05 16:15:09,978 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-05 16:15:09,978 - tests.orchestrator - INFO - --- Operations --- +2025-06-05 16:15:09,978 - tests.orchestrator - INFO - create_pvc: 9 succeeded, 0 failed (100.0%) +2025-06-05 16:15:09,979 - tests.orchestrator - INFO - attach_pod: 6 succeeded, 0 failed (100.0%) +2025-06-05 16:15:09,979 - tests.orchestrator - INFO - delete_pod: 6 succeeded, 0 failed (100.0%) +2025-06-05 16:15:09,979 - tests.orchestrator - INFO - delete_pvc: 9 succeeded, 0 failed (100.0%) +2025-06-05 16:15:09,979 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) +2025-06-05 16:15:09,979 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) +2025-06-05 16:15:09,979 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-05 16:15:09,979 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-05 16:15:09,979 - tests.orchestrator - INFO - many_to_one: No runs +2025-06-05 16:15:09,979 - tests.orchestrator - INFO - one_to_one: No runs +2025-06-05 16:15:09,979 - tests.orchestrator - INFO - concurrent_pvc: 1 succeeded, 0 failed out of 1 runs (100.0%) +2025-06-05 16:15:09,979 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_162134.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_162134.log new file mode 100644 index 000000000..02712b5c3 --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_162134.log @@ -0,0 +1,7 @@ +2025-06-05 16:21:34,044 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 16:21:34,256 - tests.orchestrator - ERROR - Error checking namespace: (401) +Reason: Unauthorized +HTTP response headers: HTTPHeaderDict({'Audit-Id': 'ae5fa875-33f1-44a7-a754-80136c368fa1', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 16:21:34 GMT', 'Content-Length': '129'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} + + diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_162155.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_162155.log new file mode 100644 index 000000000..e7d55845b --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_162155.log @@ -0,0 +1,52 @@ +2025-06-05 16:21:55,920 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 16:21:56,123 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 16:21:56,123 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-05 16:21:56,313 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 16:21:56,313 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 16:21:56,313 - tests.orchestrator - INFO - Creating PVC: test-pvc-a253186e +2025-06-05 16:21:56,391 - tests.orchestrator - INFO - Created PVC: test-pvc-a253186e +2025-06-05 16:21:56,391 - tests.orchestrator - INFO - Waiting for PVC test-pvc-a253186e to be bound +2025-06-05 16:21:58,525 - tests.orchestrator - INFO - PVC test-pvc-a253186e is bound +2025-06-05 16:21:58,526 - tests.orchestrator - INFO - Creating PVC: test-pvc-98b7ae8a +2025-06-05 16:21:58,593 - tests.orchestrator - INFO - Created PVC: test-pvc-98b7ae8a +2025-06-05 16:21:58,594 - tests.orchestrator - INFO - Waiting for PVC test-pvc-98b7ae8a to be bound +2025-06-05 16:22:00,726 - tests.orchestrator - INFO - PVC test-pvc-98b7ae8a is bound +2025-06-05 16:22:00,726 - tests.orchestrator - INFO - Creating pod test-pod-955db556 on PVC test-pvc-a253186e +2025-06-05 16:22:00,807 - tests.orchestrator - INFO - Created pod: test-pod-955db556 using PVC: test-pvc-a253186e +2025-06-05 16:22:00,807 - tests.orchestrator - INFO - Waiting for pod test-pod-955db556 to be ready +2025-06-05 16:22:00,874 - tests.orchestrator - INFO - Pod test-pod-955db556 phase: Pending +2025-06-05 16:22:31,890 - tests.orchestrator - INFO - Pod test-pod-955db556 taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 16:22:31,890 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-955db556 ===== +2025-06-05 16:22:31,957 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:22:32,031 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:22:32,031 - tests.orchestrator - INFO - [2025-06-05 16:22:00+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:22:32,103 - tests.orchestrator - INFO - No logs available +2025-06-05 16:22:32,103 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:22:32,104 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-a253186e'} +2025-06-05 16:22:32,104 - tests.orchestrator - INFO - - kube-api-access-tlm4p: {} +2025-06-05 16:22:32,104 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-955db556 ===== +2025-06-05 16:23:00,974 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-955db556 to be ready after 60s +2025-06-05 16:23:00,974 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-955db556 ===== +2025-06-05 16:23:01,041 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:23:01,110 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:23:01,110 - tests.orchestrator - INFO - [2025-06-05 16:22:00+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:23:01,176 - tests.orchestrator - INFO - No logs available +2025-06-05 16:23:01,176 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:23:01,176 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-a253186e'} +2025-06-05 16:23:01,176 - tests.orchestrator - INFO - - kube-api-access-tlm4p: {} +2025-06-05 16:23:01,176 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-955db556 ===== +2025-06-05 16:23:01,176 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-955db556 to be ready +2025-06-05 16:23:01,177 - tests.orchestrator - INFO - Creating pod test-pod-28a31e9a on PVC test-pvc-98b7ae8a +2025-06-05 16:23:01,259 - tests.orchestrator - INFO - Created pod: test-pod-28a31e9a using PVC: test-pvc-98b7ae8a +2025-06-05 16:23:01,259 - tests.orchestrator - INFO - Waiting for pod test-pod-28a31e9a to be ready +2025-06-05 16:23:01,326 - tests.orchestrator - INFO - Pod test-pod-28a31e9a phase: Pending +2025-06-05 16:23:32,335 - tests.orchestrator - INFO - Pod test-pod-28a31e9a taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 16:23:32,335 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-28a31e9a ===== +2025-06-05 16:23:32,410 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:23:32,482 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:23:32,483 - tests.orchestrator - INFO - [2025-06-05 16:23:01+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:23:32,550 - tests.orchestrator - INFO - No logs available +2025-06-05 16:23:32,550 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:23:32,550 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-98b7ae8a'} +2025-06-05 16:23:32,550 - tests.orchestrator - INFO - - kube-api-access-gfrq6: {} +2025-06-05 16:23:32,550 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-28a31e9a ===== diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_162320.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_162320.log new file mode 100644 index 000000000..e7442a34e --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_162320.log @@ -0,0 +1,27 @@ +2025-06-05 16:23:20,930 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 16:23:21,135 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 16:23:21,135 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-05 16:23:21,326 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 16:23:21,326 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 16:23:21,326 - tests.orchestrator - INFO - Creating PVC: test-pvc-3f880237 +2025-06-05 16:23:21,404 - tests.orchestrator - INFO - Created PVC: test-pvc-3f880237 +2025-06-05 16:23:21,404 - tests.orchestrator - INFO - Waiting for PVC test-pvc-3f880237 to be bound +2025-06-05 16:23:23,540 - tests.orchestrator - INFO - PVC test-pvc-3f880237 is bound +2025-06-05 16:23:23,540 - tests.orchestrator - INFO - Creating PVC: test-pvc-e3594f67 +2025-06-05 16:23:23,611 - tests.orchestrator - INFO - Created PVC: test-pvc-e3594f67 +2025-06-05 16:23:23,611 - tests.orchestrator - INFO - Waiting for PVC test-pvc-e3594f67 to be bound +2025-06-05 16:23:25,746 - tests.orchestrator - INFO - PVC test-pvc-e3594f67 is bound +2025-06-05 16:23:25,747 - tests.orchestrator - INFO - Creating pod test-pod-0fcf8a5b on PVC test-pvc-3f880237 +2025-06-05 16:23:25,829 - tests.orchestrator - INFO - Created pod: test-pod-0fcf8a5b using PVC: test-pvc-3f880237 +2025-06-05 16:23:25,829 - tests.orchestrator - INFO - Waiting for pod test-pod-0fcf8a5b to be ready +2025-06-05 16:23:25,895 - tests.orchestrator - INFO - Pod test-pod-0fcf8a5b phase: Pending +2025-06-05 16:23:56,949 - tests.orchestrator - INFO - Pod test-pod-0fcf8a5b taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 16:23:56,950 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-0fcf8a5b ===== +2025-06-05 16:23:57,018 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:23:57,090 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:23:57,090 - tests.orchestrator - INFO - [2025-06-05 16:23:25+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:23:57,157 - tests.orchestrator - INFO - No logs available +2025-06-05 16:23:57,157 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:23:57,157 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-3f880237'} +2025-06-05 16:23:57,157 - tests.orchestrator - INFO - - kube-api-access-pj9g4: {} +2025-06-05 16:23:57,157 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-0fcf8a5b ===== diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_162354.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_162354.log new file mode 100644 index 000000000..9c680a51e --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_162354.log @@ -0,0 +1,247 @@ +2025-06-05 16:23:54,549 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 16:23:54,744 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 16:23:54,744 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-05 16:23:54,939 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 16:23:54,939 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 16:23:54,939 - tests.orchestrator - INFO - Creating PVC: test-pvc-d9e565cc +2025-06-05 16:23:55,018 - tests.orchestrator - INFO - Created PVC: test-pvc-d9e565cc +2025-06-05 16:23:55,018 - tests.orchestrator - INFO - Waiting for PVC test-pvc-d9e565cc to be bound +2025-06-05 16:23:57,153 - tests.orchestrator - INFO - PVC test-pvc-d9e565cc is bound +2025-06-05 16:23:57,153 - tests.orchestrator - INFO - Creating PVC: test-pvc-7efe733c +2025-06-05 16:23:57,224 - tests.orchestrator - INFO - Created PVC: test-pvc-7efe733c +2025-06-05 16:23:57,224 - tests.orchestrator - INFO - Waiting for PVC test-pvc-7efe733c to be bound +2025-06-05 16:23:59,358 - tests.orchestrator - INFO - PVC test-pvc-7efe733c is bound +2025-06-05 16:23:59,359 - tests.orchestrator - INFO - Creating pod test-pod-dcc5bed9 on PVC test-pvc-7efe733c +2025-06-05 16:23:59,435 - tests.orchestrator - INFO - Created pod: test-pod-dcc5bed9 using PVC: test-pvc-7efe733c +2025-06-05 16:23:59,435 - tests.orchestrator - INFO - Waiting for pod test-pod-dcc5bed9 to be ready +2025-06-05 16:23:59,501 - tests.orchestrator - INFO - Pod test-pod-dcc5bed9 phase: Pending +2025-06-05 16:24:30,649 - tests.orchestrator - INFO - Pod test-pod-dcc5bed9 taking longer than expected to become ready (31.2s). Collecting diagnostics... +2025-06-05 16:24:30,649 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-dcc5bed9 ===== +2025-06-05 16:24:30,717 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:24:30,789 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:24:30,790 - tests.orchestrator - INFO - [2025-06-05 16:23:59+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:24:30,854 - tests.orchestrator - INFO - No logs available +2025-06-05 16:24:30,854 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:24:30,854 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-7efe733c'} +2025-06-05 16:24:30,854 - tests.orchestrator - INFO - - kube-api-access-b8bk8: {} +2025-06-05 16:24:30,854 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-dcc5bed9 ===== +2025-06-05 16:24:59,749 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-dcc5bed9 to be ready after 60s +2025-06-05 16:24:59,749 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-dcc5bed9 ===== +2025-06-05 16:24:59,815 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:24:59,888 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:24:59,888 - tests.orchestrator - INFO - [2025-06-05 16:23:59+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:24:59,954 - tests.orchestrator - INFO - No logs available +2025-06-05 16:24:59,954 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:24:59,955 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-7efe733c'} +2025-06-05 16:24:59,955 - tests.orchestrator - INFO - - kube-api-access-b8bk8: {} +2025-06-05 16:24:59,955 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-dcc5bed9 ===== +2025-06-05 16:24:59,955 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-dcc5bed9 to be ready +2025-06-05 16:24:59,955 - tests.orchestrator - INFO - Creating pod test-pod-9af93280 on PVC test-pvc-7efe733c +2025-06-05 16:25:00,043 - tests.orchestrator - INFO - Created pod: test-pod-9af93280 using PVC: test-pvc-7efe733c +2025-06-05 16:25:00,043 - tests.orchestrator - INFO - Waiting for pod test-pod-9af93280 to be ready +2025-06-05 16:25:00,113 - tests.orchestrator - INFO - Pod test-pod-9af93280 phase: Pending +2025-06-05 16:25:31,135 - tests.orchestrator - INFO - Pod test-pod-9af93280 taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 16:25:31,135 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-9af93280 ===== +2025-06-05 16:25:31,201 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:25:31,273 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:25:31,273 - tests.orchestrator - INFO - [2025-06-05 16:25:00+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:25:31,339 - tests.orchestrator - INFO - No logs available +2025-06-05 16:25:31,340 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:25:31,340 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-7efe733c'} +2025-06-05 16:25:31,340 - tests.orchestrator - INFO - - kube-api-access-6qzmx: {} +2025-06-05 16:25:31,340 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-9af93280 ===== +2025-06-05 16:26:00,299 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9af93280 to be ready after 60s +2025-06-05 16:26:00,299 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-9af93280 ===== +2025-06-05 16:26:00,365 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:26:00,436 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:26:00,436 - tests.orchestrator - INFO - [2025-06-05 16:25:00+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:26:00,504 - tests.orchestrator - INFO - No logs available +2025-06-05 16:26:00,504 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:26:00,504 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-7efe733c'} +2025-06-05 16:26:00,504 - tests.orchestrator - INFO - - kube-api-access-6qzmx: {} +2025-06-05 16:26:00,504 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-9af93280 ===== +2025-06-05 16:26:00,504 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9af93280 to be ready +2025-06-05 16:26:00,504 - tests.orchestrator - INFO - Creating pod test-pod-5feeb276 on PVC test-pvc-7efe733c +2025-06-05 16:26:00,589 - tests.orchestrator - INFO - Created pod: test-pod-5feeb276 using PVC: test-pvc-7efe733c +2025-06-05 16:26:00,590 - tests.orchestrator - INFO - Waiting for pod test-pod-5feeb276 to be ready +2025-06-05 16:26:00,657 - tests.orchestrator - INFO - Pod test-pod-5feeb276 phase: Pending +2025-06-05 16:26:31,674 - tests.orchestrator - INFO - Pod test-pod-5feeb276 taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 16:26:31,674 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-5feeb276 ===== +2025-06-05 16:26:31,743 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:26:31,818 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:26:31,818 - tests.orchestrator - INFO - [2025-06-05 16:26:00+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:26:31,884 - tests.orchestrator - INFO - No logs available +2025-06-05 16:26:31,884 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:26:31,884 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-7efe733c'} +2025-06-05 16:26:31,884 - tests.orchestrator - INFO - - kube-api-access-78f5j: {} +2025-06-05 16:26:31,884 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-5feeb276 ===== +2025-06-05 16:27:00,780 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-5feeb276 to be ready after 60s +2025-06-05 16:27:00,781 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-5feeb276 ===== +2025-06-05 16:27:00,848 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:27:00,917 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:27:00,917 - tests.orchestrator - INFO - [2025-06-05 16:26:00+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:27:00,982 - tests.orchestrator - INFO - No logs available +2025-06-05 16:27:00,982 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:27:00,982 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-7efe733c'} +2025-06-05 16:27:00,982 - tests.orchestrator - INFO - - kube-api-access-78f5j: {} +2025-06-05 16:27:00,982 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-5feeb276 ===== +2025-06-05 16:27:00,982 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-5feeb276 to be ready +2025-06-05 16:27:00,982 - tests.orchestrator - INFO - Testing read/write between pods test-pod-5feeb276 and test-pod-dcc5bed9 sharing PVC test-pvc-7efe733c +2025-06-05 16:27:01,050 - tests.orchestrator - ERROR - Failed in read/write verification: (400) +Reason: Bad Request +HTTP response headers: HTTPHeaderDict({'Audit-Id': '957ef347-88e9-4e31-8db6-db8deac109f5', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 16:27:01 GMT', 'Content-Length': '166'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-5feeb276 does not have a host assigned","reason":"BadRequest","code":400} + + +2025-06-05 16:27:01,050 - tests.orchestrator - INFO - ============================================================ +2025-06-05 16:27:01,051 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_one_to_one +2025-06-05 16:27:01,051 - tests.orchestrator - INFO - ============================================================ +2025-06-05 16:27:01,051 - tests.orchestrator - INFO - Running scenario: One pod per PVC +2025-06-05 16:27:01,051 - tests.orchestrator - INFO - Creating 3 PVC-pod pairs +2025-06-05 16:27:01,134 - tests.orchestrator - INFO - Waiting for PVC one2one-afd352b2 to be bound +2025-06-05 16:27:03,267 - tests.orchestrator - INFO - PVC one2one-afd352b2 is bound +2025-06-05 16:27:03,267 - tests.orchestrator - INFO - Creating pod test-pod-95808816 on PVC one2one-afd352b2 +2025-06-05 16:27:03,344 - tests.orchestrator - INFO - Created pod: test-pod-95808816 using PVC: one2one-afd352b2 +2025-06-05 16:27:03,344 - tests.orchestrator - INFO - Waiting for pod test-pod-95808816 to be ready +2025-06-05 16:27:03,412 - tests.orchestrator - INFO - Pod test-pod-95808816 phase: Pending +2025-06-05 16:27:34,490 - tests.orchestrator - INFO - Pod test-pod-95808816 taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 16:27:34,490 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-95808816 ===== +2025-06-05 16:27:34,558 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:27:34,627 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:27:34,627 - tests.orchestrator - INFO - [2025-06-05 16:27:03+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:27:34,692 - tests.orchestrator - INFO - No logs available +2025-06-05 16:27:34,692 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:27:34,692 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'one2one-afd352b2'} +2025-06-05 16:27:34,692 - tests.orchestrator - INFO - - kube-api-access-rmslv: {} +2025-06-05 16:27:34,692 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-95808816 ===== +2025-06-05 16:28:03,588 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-95808816 to be ready after 60s +2025-06-05 16:28:03,589 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-95808816 ===== +2025-06-05 16:28:03,656 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:28:03,727 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:28:03,727 - tests.orchestrator - INFO - [2025-06-05 16:27:03+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:28:03,792 - tests.orchestrator - INFO - No logs available +2025-06-05 16:28:03,792 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:28:03,792 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'one2one-afd352b2'} +2025-06-05 16:28:03,792 - tests.orchestrator - INFO - - kube-api-access-rmslv: {} +2025-06-05 16:28:03,792 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-95808816 ===== +2025-06-05 16:28:03,792 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-95808816 to be ready +2025-06-05 16:28:03,870 - tests.orchestrator - INFO - Waiting for PVC one2one-cc87f2c5 to be bound +2025-06-05 16:28:06,005 - tests.orchestrator - INFO - PVC one2one-cc87f2c5 is bound +2025-06-05 16:28:06,005 - tests.orchestrator - INFO - Creating pod test-pod-b29bd9ef on PVC one2one-cc87f2c5 +2025-06-05 16:28:06,081 - tests.orchestrator - INFO - Created pod: test-pod-b29bd9ef using PVC: one2one-cc87f2c5 +2025-06-05 16:28:06,081 - tests.orchestrator - INFO - Waiting for pod test-pod-b29bd9ef to be ready +2025-06-05 16:28:06,149 - tests.orchestrator - INFO - Pod test-pod-b29bd9ef phase: Pending +2025-06-05 16:28:37,171 - tests.orchestrator - INFO - Pod test-pod-b29bd9ef taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 16:28:37,171 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-b29bd9ef ===== +2025-06-05 16:28:37,238 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:28:37,308 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:28:37,308 - tests.orchestrator - INFO - [2025-06-05 16:28:06+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:28:37,373 - tests.orchestrator - INFO - No logs available +2025-06-05 16:28:37,373 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:28:37,373 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'one2one-cc87f2c5'} +2025-06-05 16:28:37,373 - tests.orchestrator - INFO - - kube-api-access-wlwbk: {} +2025-06-05 16:28:37,373 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-b29bd9ef ===== +2025-06-05 16:29:06,383 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b29bd9ef to be ready after 60s +2025-06-05 16:29:06,383 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-b29bd9ef ===== +2025-06-05 16:29:06,451 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:29:06,520 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:29:06,521 - tests.orchestrator - INFO - [2025-06-05 16:28:06+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:29:06,587 - tests.orchestrator - INFO - No logs available +2025-06-05 16:29:06,587 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:29:06,587 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'one2one-cc87f2c5'} +2025-06-05 16:29:06,587 - tests.orchestrator - INFO - - kube-api-access-wlwbk: {} +2025-06-05 16:29:06,587 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-b29bd9ef ===== +2025-06-05 16:29:06,587 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b29bd9ef to be ready +2025-06-05 16:29:06,670 - tests.orchestrator - INFO - Waiting for PVC one2one-010b3efe to be bound +2025-06-05 16:29:08,808 - tests.orchestrator - INFO - PVC one2one-010b3efe is bound +2025-06-05 16:29:08,808 - tests.orchestrator - INFO - Creating pod test-pod-1fbeeb48 on PVC one2one-010b3efe +2025-06-05 16:29:08,885 - tests.orchestrator - INFO - Created pod: test-pod-1fbeeb48 using PVC: one2one-010b3efe +2025-06-05 16:29:08,885 - tests.orchestrator - INFO - Waiting for pod test-pod-1fbeeb48 to be ready +2025-06-05 16:29:08,950 - tests.orchestrator - INFO - Pod test-pod-1fbeeb48 phase: Pending +2025-06-05 16:29:39,968 - tests.orchestrator - INFO - Pod test-pod-1fbeeb48 taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 16:29:39,968 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-1fbeeb48 ===== +2025-06-05 16:29:40,037 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:29:40,110 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:29:40,111 - tests.orchestrator - INFO - [2025-06-05 16:29:08+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:29:40,178 - tests.orchestrator - INFO - No logs available +2025-06-05 16:29:40,178 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:29:40,179 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'one2one-010b3efe'} +2025-06-05 16:29:40,179 - tests.orchestrator - INFO - - kube-api-access-q67hr: {} +2025-06-05 16:29:40,179 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-1fbeeb48 ===== +2025-06-05 16:30:09,074 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1fbeeb48 to be ready after 60s +2025-06-05 16:30:09,074 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-1fbeeb48 ===== +2025-06-05 16:30:09,143 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:30:09,215 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:30:09,215 - tests.orchestrator - INFO - [2025-06-05 16:29:08+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:30:09,280 - tests.orchestrator - INFO - No logs available +2025-06-05 16:30:09,280 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:30:09,280 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'one2one-010b3efe'} +2025-06-05 16:30:09,280 - tests.orchestrator - INFO - - kube-api-access-q67hr: {} +2025-06-05 16:30:09,280 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-1fbeeb48 ===== +2025-06-05 16:30:09,280 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1fbeeb48 to be ready +2025-06-05 16:30:09,280 - tests.orchestrator - WARNING - Failed to create enough PVC-pod pairs, only created 0 +2025-06-05 16:30:09,280 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_one_to_one +2025-06-05 16:30:09,280 - tests.orchestrator - INFO - ------------------------------------------------------------ +2025-06-05 16:30:09,280 - tests.orchestrator - INFO - Deleting pod: test-pod-b29bd9ef from PVC: one2one-cc87f2c5 +2025-06-05 16:30:09,361 - tests.orchestrator - INFO - Waiting for pod test-pod-b29bd9ef to be deleted +2025-06-05 16:30:09,427 - tests.orchestrator - INFO - Pod test-pod-b29bd9ef has been deleted +2025-06-05 16:30:09,428 - tests.orchestrator - INFO - Deleted pod: test-pod-b29bd9ef +2025-06-05 16:30:09,428 - tests.orchestrator - INFO - Deleting PVC: one2one-afd352b2 +2025-06-05 16:30:09,428 - tests.orchestrator - INFO - Deleting 1 pods using PVC one2one-afd352b2 +2025-06-05 16:30:09,428 - tests.orchestrator - INFO - Deleting pod: test-pod-95808816 from PVC: one2one-afd352b2 +2025-06-05 16:30:09,509 - tests.orchestrator - INFO - Waiting for pod test-pod-95808816 to be deleted +2025-06-05 16:30:09,575 - tests.orchestrator - INFO - Pod test-pod-95808816 has been deleted +2025-06-05 16:30:09,576 - tests.orchestrator - INFO - Deleted pod: test-pod-95808816 +2025-06-05 16:30:09,650 - tests.orchestrator - INFO - Waiting for PVC one2one-afd352b2 to be deleted +2025-06-05 16:30:09,715 - tests.orchestrator - INFO - PVC one2one-afd352b2 has been deleted +2025-06-05 16:30:09,715 - tests.orchestrator - INFO - Deleted PVC: one2one-afd352b2 +2025-06-05 16:30:09,716 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations +2025-06-05 16:30:09,716 - tests.orchestrator - INFO - Test completed in 374.97 seconds +2025-06-05 16:30:09,716 - tests.orchestrator - INFO - Cleaning up resources... +2025-06-05 16:30:09,717 - tests.orchestrator - INFO - Deleting pod: test-pod-dcc5bed9 from PVC: test-pvc-7efe733c +2025-06-05 16:30:09,800 - tests.orchestrator - INFO - Waiting for pod test-pod-dcc5bed9 to be deleted +2025-06-05 16:30:09,868 - tests.orchestrator - INFO - Pod test-pod-dcc5bed9 has been deleted +2025-06-05 16:30:09,868 - tests.orchestrator - INFO - Deleted pod: test-pod-dcc5bed9 +2025-06-05 16:30:09,868 - tests.orchestrator - INFO - Deleting pod: test-pod-9af93280 from PVC: test-pvc-7efe733c +2025-06-05 16:30:09,955 - tests.orchestrator - INFO - Waiting for pod test-pod-9af93280 to be deleted +2025-06-05 16:30:10,021 - tests.orchestrator - INFO - Pod test-pod-9af93280 has been deleted +2025-06-05 16:30:10,022 - tests.orchestrator - INFO - Deleted pod: test-pod-9af93280 +2025-06-05 16:30:10,022 - tests.orchestrator - INFO - Deleting pod: test-pod-5feeb276 from PVC: test-pvc-7efe733c +2025-06-05 16:30:10,106 - tests.orchestrator - INFO - Waiting for pod test-pod-5feeb276 to be deleted +2025-06-05 16:30:10,173 - tests.orchestrator - INFO - Pod test-pod-5feeb276 has been deleted +2025-06-05 16:30:10,173 - tests.orchestrator - INFO - Deleted pod: test-pod-5feeb276 +2025-06-05 16:30:10,173 - tests.orchestrator - INFO - Deleting pod: test-pod-1fbeeb48 from PVC: one2one-010b3efe +2025-06-05 16:30:10,260 - tests.orchestrator - INFO - Waiting for pod test-pod-1fbeeb48 to be deleted +2025-06-05 16:30:10,325 - tests.orchestrator - INFO - Pod test-pod-1fbeeb48 has been deleted +2025-06-05 16:30:10,325 - tests.orchestrator - INFO - Deleted pod: test-pod-1fbeeb48 +2025-06-05 16:30:10,326 - tests.orchestrator - INFO - Deleting PVC: test-pvc-d9e565cc +2025-06-05 16:30:10,400 - tests.orchestrator - INFO - Waiting for PVC test-pvc-d9e565cc to be deleted +2025-06-05 16:30:10,464 - tests.orchestrator - INFO - PVC test-pvc-d9e565cc has been deleted +2025-06-05 16:30:10,464 - tests.orchestrator - INFO - Deleted PVC: test-pvc-d9e565cc +2025-06-05 16:30:10,464 - tests.orchestrator - INFO - Deleting PVC: test-pvc-7efe733c +2025-06-05 16:30:10,538 - tests.orchestrator - INFO - Waiting for PVC test-pvc-7efe733c to be deleted +2025-06-05 16:30:10,602 - tests.orchestrator - INFO - PVC test-pvc-7efe733c has been deleted +2025-06-05 16:30:10,602 - tests.orchestrator - INFO - Deleted PVC: test-pvc-7efe733c +2025-06-05 16:30:10,603 - tests.orchestrator - INFO - Deleting PVC: one2one-cc87f2c5 +2025-06-05 16:30:10,693 - tests.orchestrator - INFO - Waiting for PVC one2one-cc87f2c5 to be deleted +2025-06-05 16:30:10,757 - tests.orchestrator - INFO - PVC one2one-cc87f2c5 has been deleted +2025-06-05 16:30:10,757 - tests.orchestrator - INFO - Deleted PVC: one2one-cc87f2c5 +2025-06-05 16:30:10,757 - tests.orchestrator - INFO - Deleting PVC: one2one-010b3efe +2025-06-05 16:30:10,831 - tests.orchestrator - INFO - Waiting for PVC one2one-010b3efe to be deleted +2025-06-05 16:30:10,896 - tests.orchestrator - INFO - PVC one2one-010b3efe has been deleted +2025-06-05 16:30:10,896 - tests.orchestrator - INFO - Deleted PVC: one2one-010b3efe +2025-06-05 16:30:10,896 - tests.orchestrator - INFO - Cleanup completed +2025-06-05 16:30:10,896 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-05 16:30:10,896 - tests.orchestrator - INFO - --- Operations --- +2025-06-05 16:30:10,896 - tests.orchestrator - INFO - create_pvc: 2 succeeded, 0 failed (100.0%) +2025-06-05 16:30:10,896 - tests.orchestrator - INFO - attach_pod: 6 succeeded, 0 failed (100.0%) +2025-06-05 16:30:10,896 - tests.orchestrator - INFO - delete_pod: 6 succeeded, 0 failed (100.0%) +2025-06-05 16:30:10,897 - tests.orchestrator - INFO - delete_pvc: 5 succeeded, 0 failed (100.0%) +2025-06-05 16:30:10,897 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) +2025-06-05 16:30:10,897 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) +2025-06-05 16:30:10,897 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-05 16:30:10,897 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-05 16:30:10,897 - tests.orchestrator - INFO - many_to_one: No runs +2025-06-05 16:30:10,897 - tests.orchestrator - INFO - one_to_one: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-05 16:30:10,897 - tests.orchestrator - INFO - concurrent_pvc: No runs +2025-06-05 16:30:10,897 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_163114.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_163114.log new file mode 100644 index 000000000..1377e47c0 --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_163114.log @@ -0,0 +1,67 @@ +2025-06-05 16:31:14,327 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 16:31:14,546 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 16:31:14,546 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-05 16:31:14,735 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 16:31:14,735 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 16:31:14,736 - tests.orchestrator - INFO - Creating PVC: test-pvc-dbe6630f +2025-06-05 16:31:14,812 - tests.orchestrator - INFO - Created PVC: test-pvc-dbe6630f +2025-06-05 16:31:14,812 - tests.orchestrator - INFO - Waiting for PVC test-pvc-dbe6630f to be bound +2025-06-05 16:31:16,944 - tests.orchestrator - INFO - PVC test-pvc-dbe6630f is bound +2025-06-05 16:31:16,944 - tests.orchestrator - INFO - Creating PVC: test-pvc-2381ce5e +2025-06-05 16:31:17,012 - tests.orchestrator - INFO - Created PVC: test-pvc-2381ce5e +2025-06-05 16:31:17,012 - tests.orchestrator - INFO - Waiting for PVC test-pvc-2381ce5e to be bound +2025-06-05 16:31:19,144 - tests.orchestrator - INFO - PVC test-pvc-2381ce5e is bound +2025-06-05 16:31:19,144 - tests.orchestrator - INFO - Creating pod test-pod-4022a58d on PVC test-pvc-2381ce5e +2025-06-05 16:31:19,223 - tests.orchestrator - INFO - Created pod: test-pod-4022a58d using PVC: test-pvc-2381ce5e +2025-06-05 16:31:19,223 - tests.orchestrator - INFO - Waiting for pod test-pod-4022a58d to be ready +2025-06-05 16:31:19,289 - tests.orchestrator - INFO - Pod test-pod-4022a58d phase: Pending +2025-06-05 16:31:50,320 - tests.orchestrator - INFO - Pod test-pod-4022a58d taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 16:31:50,320 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-4022a58d ===== +2025-06-05 16:31:50,387 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:31:50,457 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:31:50,457 - tests.orchestrator - INFO - [2025-06-05 16:31:19+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:31:50,524 - tests.orchestrator - INFO - No logs available +2025-06-05 16:31:50,525 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:31:50,525 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-2381ce5e'} +2025-06-05 16:31:50,525 - tests.orchestrator - INFO - - kube-api-access-xrgfl: {} +2025-06-05 16:31:50,525 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-4022a58d ===== +2025-06-05 16:32:19,393 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-4022a58d to be ready after 60s +2025-06-05 16:32:19,393 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-4022a58d ===== +2025-06-05 16:32:19,461 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:32:19,530 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:32:19,530 - tests.orchestrator - INFO - [2025-06-05 16:31:19+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:32:19,595 - tests.orchestrator - INFO - No logs available +2025-06-05 16:32:19,595 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:32:19,595 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-2381ce5e'} +2025-06-05 16:32:19,595 - tests.orchestrator - INFO - - kube-api-access-xrgfl: {} +2025-06-05 16:32:19,595 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-4022a58d ===== +2025-06-05 16:32:19,595 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-4022a58d to be ready +2025-06-05 16:32:19,595 - tests.orchestrator - INFO - Creating pod test-pod-9de3bebb on PVC test-pvc-dbe6630f +2025-06-05 16:32:19,678 - tests.orchestrator - INFO - Created pod: test-pod-9de3bebb using PVC: test-pvc-dbe6630f +2025-06-05 16:32:19,678 - tests.orchestrator - INFO - Waiting for pod test-pod-9de3bebb to be ready +2025-06-05 16:32:19,745 - tests.orchestrator - INFO - Pod test-pod-9de3bebb phase: Pending +2025-06-05 16:32:50,757 - tests.orchestrator - INFO - Pod test-pod-9de3bebb taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 16:32:50,757 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-9de3bebb ===== +2025-06-05 16:32:50,823 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:32:50,895 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:32:50,895 - tests.orchestrator - INFO - [2025-06-05 16:32:19+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:32:50,960 - tests.orchestrator - INFO - No logs available +2025-06-05 16:32:50,960 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:32:50,960 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-dbe6630f'} +2025-06-05 16:32:50,960 - tests.orchestrator - INFO - - kube-api-access-vfnnb: {} +2025-06-05 16:32:50,960 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-9de3bebb ===== +2025-06-05 16:33:19,831 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9de3bebb to be ready after 60s +2025-06-05 16:33:19,831 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-9de3bebb ===== +2025-06-05 16:33:19,897 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:33:19,966 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:33:19,966 - tests.orchestrator - INFO - [2025-06-05 16:32:19+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:33:20,031 - tests.orchestrator - INFO - No logs available +2025-06-05 16:33:20,031 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:33:20,031 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-dbe6630f'} +2025-06-05 16:33:20,031 - tests.orchestrator - INFO - - kube-api-access-vfnnb: {} +2025-06-05 16:33:20,031 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-9de3bebb ===== +2025-06-05 16:33:20,031 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9de3bebb to be ready +2025-06-05 16:33:20,032 - tests.orchestrator - INFO - Creating pod test-pod-4eee03ff on PVC test-pvc-2381ce5e +2025-06-05 16:33:20,115 - tests.orchestrator - INFO - Created pod: test-pod-4eee03ff using PVC: test-pvc-2381ce5e +2025-06-05 16:33:20,115 - tests.orchestrator - INFO - Waiting for pod test-pod-4eee03ff to be ready +2025-06-05 16:33:20,181 - tests.orchestrator - INFO - Pod test-pod-4eee03ff phase: Pending diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_163340.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_163340.log new file mode 100644 index 000000000..ef9c60d94 --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_163340.log @@ -0,0 +1,42 @@ +2025-06-05 16:33:40,546 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 16:33:40,752 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 16:33:40,752 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-05 16:33:40,943 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 16:33:40,944 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 16:33:40,944 - tests.orchestrator - INFO - Creating PVC: test-pvc-567589be +2025-06-05 16:33:41,017 - tests.orchestrator - INFO - Created PVC: test-pvc-567589be +2025-06-05 16:33:41,017 - tests.orchestrator - INFO - Waiting for PVC test-pvc-567589be to be bound +2025-06-05 16:33:43,150 - tests.orchestrator - INFO - PVC test-pvc-567589be is bound +2025-06-05 16:33:43,150 - tests.orchestrator - INFO - Creating PVC: test-pvc-4348540c +2025-06-05 16:33:43,218 - tests.orchestrator - INFO - Created PVC: test-pvc-4348540c +2025-06-05 16:33:43,218 - tests.orchestrator - INFO - Waiting for PVC test-pvc-4348540c to be bound +2025-06-05 16:33:45,350 - tests.orchestrator - INFO - PVC test-pvc-4348540c is bound +2025-06-05 16:33:45,350 - tests.orchestrator - INFO - Creating pod test-pod-0dadc981 on PVC test-pvc-567589be +2025-06-05 16:33:45,423 - tests.orchestrator - INFO - Created pod: test-pod-0dadc981 using PVC: test-pvc-567589be +2025-06-05 16:33:45,424 - tests.orchestrator - INFO - Waiting for pod test-pod-0dadc981 to be ready +2025-06-05 16:33:45,490 - tests.orchestrator - INFO - Pod test-pod-0dadc981 phase: Pending +2025-06-05 16:34:16,508 - tests.orchestrator - INFO - Pod test-pod-0dadc981 taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 16:34:16,508 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-0dadc981 ===== +2025-06-05 16:34:16,574 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:34:16,646 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:34:16,646 - tests.orchestrator - INFO - [2025-06-05 16:33:45+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:34:16,711 - tests.orchestrator - INFO - No logs available +2025-06-05 16:34:16,712 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:34:16,712 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-567589be'} +2025-06-05 16:34:16,712 - tests.orchestrator - INFO - - kube-api-access-jhmdz: {} +2025-06-05 16:34:16,712 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-0dadc981 ===== +2025-06-05 16:34:45,586 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-0dadc981 to be ready after 60s +2025-06-05 16:34:45,586 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-0dadc981 ===== +2025-06-05 16:34:45,653 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:34:45,725 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:34:45,725 - tests.orchestrator - INFO - [2025-06-05 16:33:45+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:34:45,791 - tests.orchestrator - INFO - No logs available +2025-06-05 16:34:45,791 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:34:45,791 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-567589be'} +2025-06-05 16:34:45,791 - tests.orchestrator - INFO - - kube-api-access-jhmdz: {} +2025-06-05 16:34:45,791 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-0dadc981 ===== +2025-06-05 16:34:45,791 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-0dadc981 to be ready +2025-06-05 16:34:45,791 - tests.orchestrator - INFO - Creating pod test-pod-74abb322 on PVC test-pvc-4348540c +2025-06-05 16:34:45,873 - tests.orchestrator - INFO - Created pod: test-pod-74abb322 using PVC: test-pvc-4348540c +2025-06-05 16:34:45,873 - tests.orchestrator - INFO - Waiting for pod test-pod-74abb322 to be ready +2025-06-05 16:34:45,939 - tests.orchestrator - INFO - Pod test-pod-74abb322 phase: Pending diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_163501.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_163501.log new file mode 100644 index 000000000..7c87c876e --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_163501.log @@ -0,0 +1,205 @@ +2025-06-05 16:35:01,819 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 16:35:02,054 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 16:35:02,054 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-05 16:35:02,250 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 16:35:02,250 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 16:35:02,250 - tests.orchestrator - INFO - Creating PVC: test-pvc-1f8e9142 +2025-06-05 16:35:02,318 - tests.orchestrator - INFO - Created PVC: test-pvc-1f8e9142 +2025-06-05 16:35:02,318 - tests.orchestrator - INFO - Waiting for PVC test-pvc-1f8e9142 to be bound +2025-06-05 16:35:04,451 - tests.orchestrator - INFO - PVC test-pvc-1f8e9142 is bound +2025-06-05 16:35:04,451 - tests.orchestrator - INFO - Creating PVC: test-pvc-6d74da5a +2025-06-05 16:35:04,519 - tests.orchestrator - INFO - Created PVC: test-pvc-6d74da5a +2025-06-05 16:35:04,520 - tests.orchestrator - INFO - Waiting for PVC test-pvc-6d74da5a to be bound +2025-06-05 16:35:06,652 - tests.orchestrator - INFO - PVC test-pvc-6d74da5a is bound +2025-06-05 16:35:06,652 - tests.orchestrator - INFO - Creating pod test-pod-dff251c2 on PVC test-pvc-6d74da5a +2025-06-05 16:35:06,728 - tests.orchestrator - INFO - Created pod: test-pod-dff251c2 using PVC: test-pvc-6d74da5a +2025-06-05 16:35:06,728 - tests.orchestrator - INFO - Waiting for pod test-pod-dff251c2 to be ready +2025-06-05 16:35:06,794 - tests.orchestrator - INFO - Pod test-pod-dff251c2 phase: Pending +2025-06-05 16:35:37,873 - tests.orchestrator - INFO - Pod test-pod-dff251c2 taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 16:35:37,873 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-dff251c2 ===== +2025-06-05 16:35:37,941 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:35:38,013 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:35:38,013 - tests.orchestrator - INFO - [2025-06-05 16:35:06+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:35:38,078 - tests.orchestrator - INFO - No logs available +2025-06-05 16:35:38,078 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:35:38,078 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-6d74da5a'} +2025-06-05 16:35:38,079 - tests.orchestrator - INFO - - kube-api-access-4d7rh: {} +2025-06-05 16:35:38,079 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-dff251c2 ===== +2025-06-05 16:36:06,966 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-dff251c2 to be ready after 60s +2025-06-05 16:36:06,966 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-dff251c2 ===== +2025-06-05 16:36:07,034 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:36:07,104 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:36:07,104 - tests.orchestrator - INFO - [2025-06-05 16:35:06+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:36:07,170 - tests.orchestrator - INFO - No logs available +2025-06-05 16:36:07,170 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:36:07,170 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-6d74da5a'} +2025-06-05 16:36:07,170 - tests.orchestrator - INFO - - kube-api-access-4d7rh: {} +2025-06-05 16:36:07,170 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-dff251c2 ===== +2025-06-05 16:36:07,170 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-dff251c2 to be ready +2025-06-05 16:36:07,170 - tests.orchestrator - INFO - Creating pod test-pod-9ee2d595 on PVC test-pvc-6d74da5a +2025-06-05 16:36:07,253 - tests.orchestrator - INFO - Created pod: test-pod-9ee2d595 using PVC: test-pvc-6d74da5a +2025-06-05 16:36:07,253 - tests.orchestrator - INFO - Waiting for pod test-pod-9ee2d595 to be ready +2025-06-05 16:36:07,321 - tests.orchestrator - INFO - Pod test-pod-9ee2d595 phase: Pending +2025-06-05 16:36:38,341 - tests.orchestrator - INFO - Pod test-pod-9ee2d595 taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 16:36:38,341 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-9ee2d595 ===== +2025-06-05 16:36:38,408 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:36:38,478 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:36:38,478 - tests.orchestrator - INFO - [2025-06-05 16:36:07+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:36:38,544 - tests.orchestrator - INFO - No logs available +2025-06-05 16:36:38,544 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:36:38,544 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-6d74da5a'} +2025-06-05 16:36:38,544 - tests.orchestrator - INFO - - kube-api-access-vll4f: {} +2025-06-05 16:36:38,544 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-9ee2d595 ===== +2025-06-05 16:37:07,418 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9ee2d595 to be ready after 60s +2025-06-05 16:37:07,418 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-9ee2d595 ===== +2025-06-05 16:37:07,485 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:37:07,558 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:37:07,558 - tests.orchestrator - INFO - [2025-06-05 16:36:07+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:37:07,624 - tests.orchestrator - INFO - No logs available +2025-06-05 16:37:07,624 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:37:07,624 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-6d74da5a'} +2025-06-05 16:37:07,624 - tests.orchestrator - INFO - - kube-api-access-vll4f: {} +2025-06-05 16:37:07,624 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-9ee2d595 ===== +2025-06-05 16:37:07,625 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9ee2d595 to be ready +2025-06-05 16:37:07,625 - tests.orchestrator - INFO - Creating pod test-pod-38614e6b on PVC test-pvc-1f8e9142 +2025-06-05 16:37:07,706 - tests.orchestrator - INFO - Created pod: test-pod-38614e6b using PVC: test-pvc-1f8e9142 +2025-06-05 16:37:07,707 - tests.orchestrator - INFO - Waiting for pod test-pod-38614e6b to be ready +2025-06-05 16:37:07,773 - tests.orchestrator - INFO - Pod test-pod-38614e6b phase: Pending +2025-06-05 16:37:38,780 - tests.orchestrator - INFO - Pod test-pod-38614e6b taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 16:37:38,781 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-38614e6b ===== +2025-06-05 16:37:38,848 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:37:38,919 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:37:38,919 - tests.orchestrator - INFO - [2025-06-05 16:37:07+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:37:38,984 - tests.orchestrator - INFO - No logs available +2025-06-05 16:37:38,984 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:37:38,984 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-1f8e9142'} +2025-06-05 16:37:38,984 - tests.orchestrator - INFO - - kube-api-access-c44px: {} +2025-06-05 16:37:38,984 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-38614e6b ===== +2025-06-05 16:38:07,879 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-38614e6b to be ready after 60s +2025-06-05 16:38:07,879 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-38614e6b ===== +2025-06-05 16:38:07,947 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:38:08,019 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:38:08,019 - tests.orchestrator - INFO - [2025-06-05 16:37:07+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:38:08,084 - tests.orchestrator - INFO - No logs available +2025-06-05 16:38:08,084 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:38:08,084 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-1f8e9142'} +2025-06-05 16:38:08,084 - tests.orchestrator - INFO - - kube-api-access-c44px: {} +2025-06-05 16:38:08,084 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-38614e6b ===== +2025-06-05 16:38:08,084 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-38614e6b to be ready +2025-06-05 16:38:08,084 - tests.orchestrator - INFO - Testing read/write between pods test-pod-9ee2d595 and test-pod-dff251c2 sharing PVC test-pvc-6d74da5a +2025-06-05 16:38:08,150 - tests.orchestrator - ERROR - Failed in read/write verification: (400) +Reason: Bad Request +HTTP response headers: HTTPHeaderDict({'Audit-Id': '2b5c9bbe-dcea-4a5d-82b0-b2b0e2833174', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 16:38:08 GMT', 'Content-Length': '166'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-9ee2d595 does not have a host assigned","reason":"BadRequest","code":400} + + +2025-06-05 16:38:08,150 - tests.orchestrator - INFO - Deleting pod: test-pod-dff251c2 from PVC: test-pvc-6d74da5a +2025-06-05 16:38:08,235 - tests.orchestrator - INFO - Waiting for pod test-pod-dff251c2 to be deleted +2025-06-05 16:38:08,300 - tests.orchestrator - INFO - Pod test-pod-dff251c2 has been deleted +2025-06-05 16:38:08,300 - tests.orchestrator - INFO - Deleted pod: test-pod-dff251c2 +2025-06-05 16:38:08,300 - tests.orchestrator - INFO - Deleting PVC: test-pvc-6d74da5a +2025-06-05 16:38:08,300 - tests.orchestrator - INFO - Deleting 1 pods using PVC test-pvc-6d74da5a +2025-06-05 16:38:08,300 - tests.orchestrator - INFO - Deleting pod: test-pod-9ee2d595 from PVC: test-pvc-6d74da5a +2025-06-05 16:38:08,378 - tests.orchestrator - INFO - Waiting for pod test-pod-9ee2d595 to be deleted +2025-06-05 16:38:08,444 - tests.orchestrator - INFO - Pod test-pod-9ee2d595 has been deleted +2025-06-05 16:38:08,444 - tests.orchestrator - INFO - Deleted pod: test-pod-9ee2d595 +2025-06-05 16:38:08,520 - tests.orchestrator - INFO - Waiting for PVC test-pvc-6d74da5a to be deleted +2025-06-05 16:38:08,585 - tests.orchestrator - INFO - PVC test-pvc-6d74da5a has been deleted +2025-06-05 16:38:08,585 - tests.orchestrator - INFO - Deleted PVC: test-pvc-6d74da5a +2025-06-05 16:38:08,585 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations +2025-06-05 16:38:08,585 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 1 times) +2025-06-05 16:38:08,585 - tests.orchestrator - INFO - Creating PVC: test-pvc-cf863d95 +2025-06-05 16:38:08,662 - tests.orchestrator - INFO - Created PVC: test-pvc-cf863d95 +2025-06-05 16:38:08,662 - tests.orchestrator - INFO - Waiting for PVC test-pvc-cf863d95 to be bound +2025-06-05 16:38:10,795 - tests.orchestrator - INFO - PVC test-pvc-cf863d95 is bound +2025-06-05 16:38:11,795 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 1 times) +2025-06-05 16:38:11,795 - tests.orchestrator - INFO - Deleting pod: test-pod-38614e6b from PVC: test-pvc-1f8e9142 +2025-06-05 16:38:11,879 - tests.orchestrator - INFO - Waiting for pod test-pod-38614e6b to be deleted +2025-06-05 16:38:11,944 - tests.orchestrator - INFO - Pod test-pod-38614e6b has been deleted +2025-06-05 16:38:11,944 - tests.orchestrator - INFO - Deleted pod: test-pod-38614e6b +2025-06-05 16:38:12,944 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 1 times) +2025-06-05 16:38:12,944 - tests.orchestrator - INFO - Deleting PVC: test-pvc-1f8e9142 +2025-06-05 16:38:13,016 - tests.orchestrator - INFO - Waiting for PVC test-pvc-1f8e9142 to be deleted +2025-06-05 16:38:13,082 - tests.orchestrator - INFO - PVC test-pvc-1f8e9142 has been deleted +2025-06-05 16:38:13,082 - tests.orchestrator - INFO - Deleted PVC: test-pvc-1f8e9142 +2025-06-05 16:38:14,082 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 1 times) +2025-06-05 16:38:14,083 - tests.orchestrator - INFO - Creating pod test-pod-3d503390 on PVC test-pvc-cf863d95 +2025-06-05 16:38:14,156 - tests.orchestrator - INFO - Created pod: test-pod-3d503390 using PVC: test-pvc-cf863d95 +2025-06-05 16:38:14,156 - tests.orchestrator - INFO - Waiting for pod test-pod-3d503390 to be ready +2025-06-05 16:38:14,224 - tests.orchestrator - INFO - Pod test-pod-3d503390 phase: Pending +2025-06-05 16:38:45,253 - tests.orchestrator - INFO - Pod test-pod-3d503390 taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 16:38:45,254 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-3d503390 ===== +2025-06-05 16:38:45,320 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:38:45,413 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:38:45,413 - tests.orchestrator - INFO - [2025-06-05 16:38:14+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:38:45,481 - tests.orchestrator - INFO - No logs available +2025-06-05 16:38:45,481 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:38:45,481 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-cf863d95'} +2025-06-05 16:38:45,481 - tests.orchestrator - INFO - - kube-api-access-j6x6f: {} +2025-06-05 16:38:45,481 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-3d503390 ===== +2025-06-05 16:39:14,367 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-3d503390 to be ready after 60s +2025-06-05 16:39:14,368 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-3d503390 ===== +2025-06-05 16:39:14,435 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:39:14,506 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:39:14,506 - tests.orchestrator - INFO - [2025-06-05 16:38:14+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:39:14,572 - tests.orchestrator - INFO - No logs available +2025-06-05 16:39:14,572 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:39:14,572 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-cf863d95'} +2025-06-05 16:39:14,572 - tests.orchestrator - INFO - - kube-api-access-j6x6f: {} +2025-06-05 16:39:14,572 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-3d503390 ===== +2025-06-05 16:39:14,572 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-3d503390 to be ready +2025-06-05 16:39:15,572 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 2 times) +2025-06-05 16:39:15,573 - tests.orchestrator - INFO - Creating pod test-pod-bf6753dc on PVC test-pvc-cf863d95 +2025-06-05 16:39:15,656 - tests.orchestrator - INFO - Created pod: test-pod-bf6753dc using PVC: test-pvc-cf863d95 +2025-06-05 16:39:15,656 - tests.orchestrator - INFO - Waiting for pod test-pod-bf6753dc to be ready +2025-06-05 16:39:15,723 - tests.orchestrator - INFO - Pod test-pod-bf6753dc phase: Pending +2025-06-05 16:39:46,730 - tests.orchestrator - INFO - Pod test-pod-bf6753dc taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 16:39:46,731 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-bf6753dc ===== +2025-06-05 16:39:46,797 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:39:46,867 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:39:46,867 - tests.orchestrator - INFO - [2025-06-05 16:39:15+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:39:46,932 - tests.orchestrator - INFO - No logs available +2025-06-05 16:39:46,933 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:39:46,933 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-cf863d95'} +2025-06-05 16:39:46,933 - tests.orchestrator - INFO - - kube-api-access-2ptll: {} +2025-06-05 16:39:46,933 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-bf6753dc ===== +2025-06-05 16:40:15,809 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-bf6753dc to be ready after 60s +2025-06-05 16:40:15,809 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-bf6753dc ===== +2025-06-05 16:40:15,877 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:40:15,949 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:40:15,949 - tests.orchestrator - INFO - [2025-06-05 16:39:15+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:40:16,015 - tests.orchestrator - INFO - No logs available +2025-06-05 16:40:16,015 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:40:16,015 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-cf863d95'} +2025-06-05 16:40:16,015 - tests.orchestrator - INFO - - kube-api-access-2ptll: {} +2025-06-05 16:40:16,015 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-bf6753dc ===== +2025-06-05 16:40:16,015 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-bf6753dc to be ready +2025-06-05 16:40:17,016 - tests.orchestrator - INFO - Test completed in 314.96 seconds +2025-06-05 16:40:17,016 - tests.orchestrator - INFO - Cleaning up resources... +2025-06-05 16:40:17,016 - tests.orchestrator - INFO - Deleting pod: test-pod-3d503390 from PVC: test-pvc-cf863d95 +2025-06-05 16:40:17,103 - tests.orchestrator - INFO - Waiting for pod test-pod-3d503390 to be deleted +2025-06-05 16:40:17,168 - tests.orchestrator - INFO - Pod test-pod-3d503390 has been deleted +2025-06-05 16:40:17,168 - tests.orchestrator - INFO - Deleted pod: test-pod-3d503390 +2025-06-05 16:40:17,168 - tests.orchestrator - INFO - Deleting pod: test-pod-bf6753dc from PVC: test-pvc-cf863d95 +2025-06-05 16:40:17,247 - tests.orchestrator - INFO - Waiting for pod test-pod-bf6753dc to be deleted +2025-06-05 16:40:17,312 - tests.orchestrator - INFO - Pod test-pod-bf6753dc has been deleted +2025-06-05 16:40:17,313 - tests.orchestrator - INFO - Deleted pod: test-pod-bf6753dc +2025-06-05 16:40:17,313 - tests.orchestrator - INFO - Deleting PVC: test-pvc-cf863d95 +2025-06-05 16:40:17,385 - tests.orchestrator - INFO - Waiting for PVC test-pvc-cf863d95 to be deleted +2025-06-05 16:40:17,450 - tests.orchestrator - INFO - PVC test-pvc-cf863d95 has been deleted +2025-06-05 16:40:17,450 - tests.orchestrator - INFO - Deleted PVC: test-pvc-cf863d95 +2025-06-05 16:40:17,450 - tests.orchestrator - INFO - Cleanup completed +2025-06-05 16:40:17,450 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-05 16:40:17,451 - tests.orchestrator - INFO - --- Operations --- +2025-06-05 16:40:17,451 - tests.orchestrator - INFO - create_pvc: 3 succeeded, 0 failed (100.0%) +2025-06-05 16:40:17,451 - tests.orchestrator - INFO - attach_pod: 5 succeeded, 0 failed (100.0%) +2025-06-05 16:40:17,451 - tests.orchestrator - INFO - delete_pod: 5 succeeded, 0 failed (100.0%) +2025-06-05 16:40:17,451 - tests.orchestrator - INFO - delete_pvc: 3 succeeded, 0 failed (100.0%) +2025-06-05 16:40:17,451 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) +2025-06-05 16:40:17,451 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) +2025-06-05 16:40:17,451 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-05 16:40:17,451 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-05 16:40:17,451 - tests.orchestrator - INFO - many_to_one: No runs +2025-06-05 16:40:17,451 - tests.orchestrator - INFO - one_to_one: No runs +2025-06-05 16:40:17,451 - tests.orchestrator - INFO - concurrent_pvc: No runs +2025-06-05 16:40:17,451 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_164709.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_164709.log new file mode 100644 index 000000000..793f7d128 --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_164709.log @@ -0,0 +1,67 @@ +2025-06-05 16:47:09,450 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 16:47:09,670 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 16:47:09,670 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-05 16:47:09,860 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 16:47:09,860 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 16:47:09,860 - tests.orchestrator - INFO - Creating PVC: test-pvc-2686f57c +2025-06-05 16:47:09,941 - tests.orchestrator - INFO - Created PVC: test-pvc-2686f57c +2025-06-05 16:47:09,941 - tests.orchestrator - INFO - Waiting for PVC test-pvc-2686f57c to be bound +2025-06-05 16:47:12,088 - tests.orchestrator - INFO - PVC test-pvc-2686f57c is bound +2025-06-05 16:47:12,088 - tests.orchestrator - INFO - Creating PVC: test-pvc-269e9543 +2025-06-05 16:47:12,161 - tests.orchestrator - INFO - Created PVC: test-pvc-269e9543 +2025-06-05 16:47:12,161 - tests.orchestrator - INFO - Waiting for PVC test-pvc-269e9543 to be bound +2025-06-05 16:47:14,297 - tests.orchestrator - INFO - PVC test-pvc-269e9543 is bound +2025-06-05 16:47:14,297 - tests.orchestrator - INFO - Creating pod test-pod-60374c6e on PVC test-pvc-269e9543 +2025-06-05 16:47:14,379 - tests.orchestrator - INFO - Created pod: test-pod-60374c6e using PVC: test-pvc-269e9543 +2025-06-05 16:47:14,379 - tests.orchestrator - INFO - Waiting for pod test-pod-60374c6e to be ready +2025-06-05 16:47:14,447 - tests.orchestrator - INFO - Pod test-pod-60374c6e phase: Pending +2025-06-05 16:47:45,472 - tests.orchestrator - INFO - Pod test-pod-60374c6e taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 16:47:45,472 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-60374c6e ===== +2025-06-05 16:47:45,538 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:47:45,612 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:47:45,613 - tests.orchestrator - INFO - [2025-06-05 16:47:14+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:47:45,680 - tests.orchestrator - INFO - No logs available +2025-06-05 16:47:45,680 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:47:45,680 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-269e9543'} +2025-06-05 16:47:45,680 - tests.orchestrator - INFO - - kube-api-access-cnr5h: {} +2025-06-05 16:47:45,680 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-60374c6e ===== +2025-06-05 16:48:14,576 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-60374c6e to be ready after 60s +2025-06-05 16:48:14,576 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-60374c6e ===== +2025-06-05 16:48:14,643 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:48:14,718 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:48:14,718 - tests.orchestrator - INFO - [2025-06-05 16:47:14+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:48:14,785 - tests.orchestrator - INFO - No logs available +2025-06-05 16:48:14,785 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:48:14,786 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-269e9543'} +2025-06-05 16:48:14,786 - tests.orchestrator - INFO - - kube-api-access-cnr5h: {} +2025-06-05 16:48:14,786 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-60374c6e ===== +2025-06-05 16:48:14,786 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-60374c6e to be ready +2025-06-05 16:48:14,786 - tests.orchestrator - INFO - Creating pod test-pod-2f20e8f1 on PVC test-pvc-2686f57c +2025-06-05 16:48:14,872 - tests.orchestrator - INFO - Created pod: test-pod-2f20e8f1 using PVC: test-pvc-2686f57c +2025-06-05 16:48:14,872 - tests.orchestrator - INFO - Waiting for pod test-pod-2f20e8f1 to be ready +2025-06-05 16:48:14,941 - tests.orchestrator - INFO - Pod test-pod-2f20e8f1 phase: Pending +2025-06-05 16:48:46,015 - tests.orchestrator - INFO - Pod test-pod-2f20e8f1 taking longer than expected to become ready (31.1s). Collecting diagnostics... +2025-06-05 16:48:46,015 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-2f20e8f1 ===== +2025-06-05 16:48:46,081 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:48:46,155 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:48:46,155 - tests.orchestrator - INFO - [2025-06-05 16:48:14+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:48:46,222 - tests.orchestrator - INFO - No logs available +2025-06-05 16:48:46,222 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:48:46,222 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-2686f57c'} +2025-06-05 16:48:46,222 - tests.orchestrator - INFO - - kube-api-access-nqmbn: {} +2025-06-05 16:48:46,222 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-2f20e8f1 ===== +2025-06-05 16:49:15,114 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-2f20e8f1 to be ready after 60s +2025-06-05 16:49:15,114 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-2f20e8f1 ===== +2025-06-05 16:49:15,181 - tests.orchestrator - INFO - No container statuses available +2025-06-05 16:49:15,256 - tests.orchestrator - INFO - Pod events: +2025-06-05 16:49:15,256 - tests.orchestrator - INFO - [2025-06-05 16:48:14+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. +2025-06-05 16:49:15,323 - tests.orchestrator - INFO - No logs available +2025-06-05 16:49:15,323 - tests.orchestrator - INFO - Pod volumes: +2025-06-05 16:49:15,324 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-2686f57c'} +2025-06-05 16:49:15,324 - tests.orchestrator - INFO - - kube-api-access-nqmbn: {} +2025-06-05 16:49:15,324 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-2f20e8f1 ===== +2025-06-05 16:49:15,324 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-2f20e8f1 to be ready +2025-06-05 16:49:15,324 - tests.orchestrator - INFO - Creating pod test-pod-988a488e on PVC test-pvc-269e9543 +2025-06-05 16:49:15,409 - tests.orchestrator - INFO - Created pod: test-pod-988a488e using PVC: test-pvc-269e9543 +2025-06-05 16:49:15,409 - tests.orchestrator - INFO - Waiting for pod test-pod-988a488e to be ready +2025-06-05 16:49:15,477 - tests.orchestrator - INFO - Pod test-pod-988a488e phase: Pending diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_164934.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_164934.log new file mode 100644 index 000000000..1301857d1 --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_164934.log @@ -0,0 +1,246 @@ +2025-06-05 16:49:34,727 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 16:49:34,939 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 16:49:34,939 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-05 16:49:35,128 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 16:49:35,128 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 16:49:35,128 - tests.orchestrator - INFO - Creating PVC: test-pvc-410242e8 +2025-06-05 16:49:35,205 - tests.orchestrator - INFO - Created PVC: test-pvc-410242e8 +2025-06-05 16:49:35,205 - tests.orchestrator - INFO - Waiting for PVC test-pvc-410242e8 to be bound +2025-06-05 16:49:37,342 - tests.orchestrator - INFO - PVC test-pvc-410242e8 is bound +2025-06-05 16:49:37,342 - tests.orchestrator - INFO - Creating PVC: test-pvc-7b3bb2da +2025-06-05 16:49:37,410 - tests.orchestrator - INFO - Created PVC: test-pvc-7b3bb2da +2025-06-05 16:49:37,410 - tests.orchestrator - INFO - Waiting for PVC test-pvc-7b3bb2da to be bound +2025-06-05 16:49:39,543 - tests.orchestrator - INFO - PVC test-pvc-7b3bb2da is bound +2025-06-05 16:49:39,544 - tests.orchestrator - INFO - Creating pod test-pod-005090d9 on PVC test-pvc-7b3bb2da +2025-06-05 16:49:39,624 - tests.orchestrator - INFO - Created pod: test-pod-005090d9 using PVC: test-pvc-7b3bb2da +2025-06-05 16:49:39,625 - tests.orchestrator - INFO - Waiting for pod test-pod-005090d9 to be ready +2025-06-05 16:49:39,693 - tests.orchestrator - INFO - Pod test-pod-005090d9 phase: Pending +2025-06-05 16:49:43,827 - tests.orchestrator - INFO - Pod test-pod-005090d9 phase: Running +2025-06-05 16:49:43,827 - tests.orchestrator - INFO - Pod test-pod-005090d9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:49:45,895 - tests.orchestrator - INFO - Pod test-pod-005090d9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:49:47,962 - tests.orchestrator - INFO - Pod test-pod-005090d9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:49:50,030 - tests.orchestrator - INFO - Pod test-pod-005090d9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:49:52,098 - tests.orchestrator - INFO - Pod test-pod-005090d9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:49:54,164 - tests.orchestrator - INFO - Pod test-pod-005090d9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:49:56,232 - tests.orchestrator - INFO - Pod test-pod-005090d9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:49:58,300 - tests.orchestrator - INFO - Pod test-pod-005090d9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:50:00,367 - tests.orchestrator - INFO - Pod test-pod-005090d9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:50:02,435 - tests.orchestrator - INFO - Pod test-pod-005090d9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 16:50:02,435 - tests.orchestrator - INFO - Pod test-pod-005090d9 is ready +2025-06-05 16:50:02,435 - tests.orchestrator - INFO - Creating pod test-pod-4d7d2fab on PVC test-pvc-7b3bb2da +2025-06-05 16:50:02,511 - tests.orchestrator - INFO - Created pod: test-pod-4d7d2fab using PVC: test-pvc-7b3bb2da +2025-06-05 16:50:02,511 - tests.orchestrator - INFO - Waiting for pod test-pod-4d7d2fab to be ready +2025-06-05 16:50:02,579 - tests.orchestrator - INFO - Pod test-pod-4d7d2fab phase: Pending +2025-06-05 16:50:04,645 - tests.orchestrator - INFO - Pod test-pod-4d7d2fab phase: Running +2025-06-05 16:50:04,645 - tests.orchestrator - INFO - Pod test-pod-4d7d2fab conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:50:06,713 - tests.orchestrator - INFO - Pod test-pod-4d7d2fab conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:50:08,781 - tests.orchestrator - INFO - Pod test-pod-4d7d2fab conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:50:10,848 - tests.orchestrator - INFO - Pod test-pod-4d7d2fab conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:50:12,915 - tests.orchestrator - INFO - Pod test-pod-4d7d2fab conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:50:14,984 - tests.orchestrator - INFO - Pod test-pod-4d7d2fab conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:50:17,051 - tests.orchestrator - INFO - Pod test-pod-4d7d2fab conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:50:19,118 - tests.orchestrator - INFO - Pod test-pod-4d7d2fab conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:50:21,187 - tests.orchestrator - INFO - Pod test-pod-4d7d2fab conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:50:23,254 - tests.orchestrator - INFO - Pod test-pod-4d7d2fab conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:50:25,321 - tests.orchestrator - INFO - Pod test-pod-4d7d2fab conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 16:50:25,321 - tests.orchestrator - INFO - Pod test-pod-4d7d2fab is ready +2025-06-05 16:50:25,321 - tests.orchestrator - INFO - Creating pod test-pod-5ae56b7a on PVC test-pvc-7b3bb2da +2025-06-05 16:50:25,404 - tests.orchestrator - INFO - Created pod: test-pod-5ae56b7a using PVC: test-pvc-7b3bb2da +2025-06-05 16:50:25,404 - tests.orchestrator - INFO - Waiting for pod test-pod-5ae56b7a to be ready +2025-06-05 16:50:25,472 - tests.orchestrator - INFO - Pod test-pod-5ae56b7a phase: Pending +2025-06-05 16:50:27,539 - tests.orchestrator - INFO - Pod test-pod-5ae56b7a phase: Running +2025-06-05 16:50:27,539 - tests.orchestrator - INFO - Pod test-pod-5ae56b7a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:50:29,606 - tests.orchestrator - INFO - Pod test-pod-5ae56b7a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:50:31,799 - tests.orchestrator - INFO - Pod test-pod-5ae56b7a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:50:33,866 - tests.orchestrator - INFO - Pod test-pod-5ae56b7a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:50:35,933 - tests.orchestrator - INFO - Pod test-pod-5ae56b7a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:50:38,001 - tests.orchestrator - INFO - Pod test-pod-5ae56b7a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:50:40,068 - tests.orchestrator - INFO - Pod test-pod-5ae56b7a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:50:42,136 - tests.orchestrator - INFO - Pod test-pod-5ae56b7a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:50:44,203 - tests.orchestrator - INFO - Pod test-pod-5ae56b7a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:50:46,270 - tests.orchestrator - INFO - Pod test-pod-5ae56b7a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 16:50:46,270 - tests.orchestrator - INFO - Pod test-pod-5ae56b7a is ready +2025-06-05 16:50:46,270 - tests.orchestrator - INFO - Testing read/write between pods test-pod-005090d9 and test-pod-4d7d2fab sharing PVC test-pvc-7b3bb2da +2025-06-05 16:50:46,341 - tests.orchestrator - ERROR - Failed in read/write verification: (400) +Reason: Bad Request +HTTP response headers: HTTPHeaderDict({'Audit-Id': '76467c30-4547-40d8-b95c-8757f3c9c795', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 16:50:46 GMT', 'Content-Length': '139'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Upgrade request required","reason":"BadRequest","code":400} + + +2025-06-05 16:50:46,341 - tests.orchestrator - INFO - Deleting pod: test-pod-005090d9 from PVC: test-pvc-7b3bb2da +2025-06-05 16:50:46,416 - tests.orchestrator - INFO - Waiting for pod test-pod-005090d9 to be deleted +2025-06-05 16:51:19,031 - tests.orchestrator - INFO - Pod test-pod-005090d9 has been deleted +2025-06-05 16:51:19,031 - tests.orchestrator - INFO - Deleted pod: test-pod-005090d9 +2025-06-05 16:51:19,031 - tests.orchestrator - INFO - Deleting PVC: test-pvc-7b3bb2da +2025-06-05 16:51:19,031 - tests.orchestrator - INFO - Deleting 2 pods using PVC test-pvc-7b3bb2da +2025-06-05 16:51:19,031 - tests.orchestrator - INFO - Deleting pod: test-pod-4d7d2fab from PVC: test-pvc-7b3bb2da +2025-06-05 16:51:19,108 - tests.orchestrator - INFO - Waiting for pod test-pod-4d7d2fab to be deleted +2025-06-05 16:51:51,715 - tests.orchestrator - INFO - Pod test-pod-4d7d2fab has been deleted +2025-06-05 16:51:51,715 - tests.orchestrator - INFO - Deleted pod: test-pod-4d7d2fab +2025-06-05 16:51:51,715 - tests.orchestrator - INFO - Deleting pod: test-pod-5ae56b7a from PVC: test-pvc-7b3bb2da +2025-06-05 16:51:51,790 - tests.orchestrator - INFO - Waiting for pod test-pod-5ae56b7a to be deleted +2025-06-05 16:52:24,395 - tests.orchestrator - INFO - Pod test-pod-5ae56b7a has been deleted +2025-06-05 16:52:24,395 - tests.orchestrator - INFO - Deleted pod: test-pod-5ae56b7a +2025-06-05 16:52:24,466 - tests.orchestrator - INFO - Waiting for PVC test-pvc-7b3bb2da to be deleted +2025-06-05 16:52:24,531 - tests.orchestrator - INFO - PVC test-pvc-7b3bb2da has been deleted +2025-06-05 16:52:24,531 - tests.orchestrator - INFO - Deleted PVC: test-pvc-7b3bb2da +2025-06-05 16:52:24,531 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations +2025-06-05 16:52:24,531 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 1 times) +2025-06-05 16:52:24,531 - tests.orchestrator - INFO - Creating PVC: test-pvc-614c6d80 +2025-06-05 16:52:24,607 - tests.orchestrator - INFO - Created PVC: test-pvc-614c6d80 +2025-06-05 16:52:24,608 - tests.orchestrator - INFO - Waiting for PVC test-pvc-614c6d80 to be bound +2025-06-05 16:52:26,740 - tests.orchestrator - INFO - PVC test-pvc-614c6d80 is bound +2025-06-05 16:52:27,740 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 2 times) +2025-06-05 16:52:27,740 - tests.orchestrator - INFO - Creating PVC: test-pvc-18e6fc84 +2025-06-05 16:52:27,807 - tests.orchestrator - INFO - Created PVC: test-pvc-18e6fc84 +2025-06-05 16:52:27,807 - tests.orchestrator - INFO - Waiting for PVC test-pvc-18e6fc84 to be bound +2025-06-05 16:52:29,940 - tests.orchestrator - INFO - PVC test-pvc-18e6fc84 is bound +2025-06-05 16:52:30,940 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 1 times) +2025-06-05 16:52:30,940 - tests.orchestrator - INFO - Creating pod test-pod-367335c6 on PVC test-pvc-410242e8 +2025-06-05 16:52:31,020 - tests.orchestrator - INFO - Created pod: test-pod-367335c6 using PVC: test-pvc-410242e8 +2025-06-05 16:52:31,021 - tests.orchestrator - INFO - Waiting for pod test-pod-367335c6 to be ready +2025-06-05 16:52:31,088 - tests.orchestrator - INFO - Pod test-pod-367335c6 phase: Pending +2025-06-05 16:52:35,223 - tests.orchestrator - INFO - Pod test-pod-367335c6 phase: Running +2025-06-05 16:52:35,224 - tests.orchestrator - INFO - Pod test-pod-367335c6 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:52:37,292 - tests.orchestrator - INFO - Pod test-pod-367335c6 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:52:39,358 - tests.orchestrator - INFO - Pod test-pod-367335c6 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:52:41,427 - tests.orchestrator - INFO - Pod test-pod-367335c6 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:52:43,493 - tests.orchestrator - INFO - Pod test-pod-367335c6 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:52:45,560 - tests.orchestrator - INFO - Pod test-pod-367335c6 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:52:47,628 - tests.orchestrator - INFO - Pod test-pod-367335c6 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:52:49,696 - tests.orchestrator - INFO - Pod test-pod-367335c6 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:52:51,763 - tests.orchestrator - INFO - Pod test-pod-367335c6 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:52:53,830 - tests.orchestrator - INFO - Pod test-pod-367335c6 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 16:52:53,830 - tests.orchestrator - INFO - Pod test-pod-367335c6 is ready +2025-06-05 16:52:54,831 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 1 times) +2025-06-05 16:52:54,831 - tests.orchestrator - INFO - Deleting PVC: test-pvc-410242e8 +2025-06-05 16:52:54,831 - tests.orchestrator - INFO - Deleting 1 pods using PVC test-pvc-410242e8 +2025-06-05 16:52:54,831 - tests.orchestrator - INFO - Deleting pod: test-pod-367335c6 from PVC: test-pvc-410242e8 +2025-06-05 16:52:54,907 - tests.orchestrator - INFO - Waiting for pod test-pod-367335c6 to be deleted +2025-06-05 16:53:27,511 - tests.orchestrator - INFO - Pod test-pod-367335c6 has been deleted +2025-06-05 16:53:27,511 - tests.orchestrator - INFO - Deleted pod: test-pod-367335c6 +2025-06-05 16:53:27,583 - tests.orchestrator - INFO - Waiting for PVC test-pvc-410242e8 to be deleted +2025-06-05 16:53:27,649 - tests.orchestrator - INFO - PVC test-pvc-410242e8 has been deleted +2025-06-05 16:53:27,649 - tests.orchestrator - INFO - Deleted PVC: test-pvc-410242e8 +2025-06-05 16:53:28,649 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 3 times) +2025-06-05 16:53:28,649 - tests.orchestrator - INFO - Creating PVC: test-pvc-be750096 +2025-06-05 16:53:28,727 - tests.orchestrator - INFO - Created PVC: test-pvc-be750096 +2025-06-05 16:53:28,727 - tests.orchestrator - INFO - Waiting for PVC test-pvc-be750096 to be bound +2025-06-05 16:53:30,858 - tests.orchestrator - INFO - PVC test-pvc-be750096 is bound +2025-06-05 16:53:31,858 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 4 times) +2025-06-05 16:53:31,858 - tests.orchestrator - INFO - Creating PVC: test-pvc-6cb60179 +2025-06-05 16:53:31,939 - tests.orchestrator - INFO - Created PVC: test-pvc-6cb60179 +2025-06-05 16:53:31,939 - tests.orchestrator - INFO - Waiting for PVC test-pvc-6cb60179 to be bound +2025-06-05 16:53:34,074 - tests.orchestrator - INFO - PVC test-pvc-6cb60179 is bound +2025-06-05 16:53:35,075 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 1 times) +2025-06-05 16:53:35,075 - tests.orchestrator - INFO - No pods to delete +2025-06-05 16:53:36,075 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 2 times) +2025-06-05 16:53:36,075 - tests.orchestrator - INFO - No pods to delete +2025-06-05 16:53:37,075 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 2 times) +2025-06-05 16:53:37,075 - tests.orchestrator - INFO - Deleting PVC: test-pvc-be750096 +2025-06-05 16:53:37,148 - tests.orchestrator - INFO - Waiting for PVC test-pvc-be750096 to be deleted +2025-06-05 16:53:37,212 - tests.orchestrator - INFO - PVC test-pvc-be750096 has been deleted +2025-06-05 16:53:37,212 - tests.orchestrator - INFO - Deleted PVC: test-pvc-be750096 +2025-06-05 16:53:38,212 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 5 times) +2025-06-05 16:53:38,212 - tests.orchestrator - INFO - Creating PVC: test-pvc-ee8508e1 +2025-06-05 16:53:38,282 - tests.orchestrator - INFO - Created PVC: test-pvc-ee8508e1 +2025-06-05 16:53:38,282 - tests.orchestrator - INFO - Waiting for PVC test-pvc-ee8508e1 to be bound +2025-06-05 16:53:40,414 - tests.orchestrator - INFO - PVC test-pvc-ee8508e1 is bound +2025-06-05 16:53:41,414 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 1 times) +2025-06-05 16:53:41,414 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test +2025-06-05 16:53:42,416 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 2 times) +2025-06-05 16:53:42,416 - tests.orchestrator - INFO - Creating pod test-pod-6b2c2055 on PVC test-pvc-18e6fc84 +2025-06-05 16:53:42,490 - tests.orchestrator - INFO - Created pod: test-pod-6b2c2055 using PVC: test-pvc-18e6fc84 +2025-06-05 16:53:42,490 - tests.orchestrator - INFO - Waiting for pod test-pod-6b2c2055 to be ready +2025-06-05 16:53:42,557 - tests.orchestrator - INFO - Pod test-pod-6b2c2055 phase: Pending +2025-06-05 16:53:44,625 - tests.orchestrator - INFO - Pod test-pod-6b2c2055 phase: Running +2025-06-05 16:53:44,625 - tests.orchestrator - INFO - Pod test-pod-6b2c2055 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:53:46,692 - tests.orchestrator - INFO - Pod test-pod-6b2c2055 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:53:48,760 - tests.orchestrator - INFO - Pod test-pod-6b2c2055 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:53:50,827 - tests.orchestrator - INFO - Pod test-pod-6b2c2055 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:53:52,895 - tests.orchestrator - INFO - Pod test-pod-6b2c2055 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:53:54,963 - tests.orchestrator - INFO - Pod test-pod-6b2c2055 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:53:57,031 - tests.orchestrator - INFO - Pod test-pod-6b2c2055 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:53:59,098 - tests.orchestrator - INFO - Pod test-pod-6b2c2055 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:54:01,166 - tests.orchestrator - INFO - Pod test-pod-6b2c2055 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:54:03,233 - tests.orchestrator - INFO - Pod test-pod-6b2c2055 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:54:05,301 - tests.orchestrator - INFO - Pod test-pod-6b2c2055 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 16:54:05,301 - tests.orchestrator - INFO - Pod test-pod-6b2c2055 is ready +2025-06-05 16:54:06,301 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 6 times) +2025-06-05 16:54:06,301 - tests.orchestrator - INFO - Creating PVC: test-pvc-9c79d5b3 +2025-06-05 16:54:06,378 - tests.orchestrator - INFO - Created PVC: test-pvc-9c79d5b3 +2025-06-05 16:54:06,379 - tests.orchestrator - INFO - Waiting for PVC test-pvc-9c79d5b3 to be bound +2025-06-05 16:54:08,513 - tests.orchestrator - INFO - PVC test-pvc-9c79d5b3 is bound +2025-06-05 16:54:09,514 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 3 times) +2025-06-05 16:54:09,514 - tests.orchestrator - INFO - Deleting PVC: test-pvc-ee8508e1 +2025-06-05 16:54:09,586 - tests.orchestrator - INFO - Waiting for PVC test-pvc-ee8508e1 to be deleted +2025-06-05 16:54:09,651 - tests.orchestrator - INFO - PVC test-pvc-ee8508e1 has been deleted +2025-06-05 16:54:09,651 - tests.orchestrator - INFO - Deleted PVC: test-pvc-ee8508e1 +2025-06-05 16:54:10,651 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 3 times) +2025-06-05 16:54:10,651 - tests.orchestrator - INFO - Creating pod test-pod-71e98828 on PVC test-pvc-9c79d5b3 +2025-06-05 16:54:10,724 - tests.orchestrator - INFO - Created pod: test-pod-71e98828 using PVC: test-pvc-9c79d5b3 +2025-06-05 16:54:10,725 - tests.orchestrator - INFO - Waiting for pod test-pod-71e98828 to be ready +2025-06-05 16:54:10,792 - tests.orchestrator - INFO - Pod test-pod-71e98828 phase: Pending +2025-06-05 16:54:12,859 - tests.orchestrator - INFO - Pod test-pod-71e98828 phase: Running +2025-06-05 16:54:12,859 - tests.orchestrator - INFO - Pod test-pod-71e98828 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:54:14,928 - tests.orchestrator - INFO - Pod test-pod-71e98828 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:54:16,996 - tests.orchestrator - INFO - Pod test-pod-71e98828 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:54:19,062 - tests.orchestrator - INFO - Pod test-pod-71e98828 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:54:21,130 - tests.orchestrator - INFO - Pod test-pod-71e98828 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:54:23,198 - tests.orchestrator - INFO - Pod test-pod-71e98828 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:54:25,265 - tests.orchestrator - INFO - Pod test-pod-71e98828 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:54:27,332 - tests.orchestrator - INFO - Pod test-pod-71e98828 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:54:29,400 - tests.orchestrator - INFO - Pod test-pod-71e98828 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:54:31,568 - tests.orchestrator - INFO - Pod test-pod-71e98828 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 16:54:33,635 - tests.orchestrator - INFO - Pod test-pod-71e98828 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 16:54:33,636 - tests.orchestrator - INFO - Pod test-pod-71e98828 is ready +2025-06-05 16:54:34,636 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 3 times) +2025-06-05 16:54:34,636 - tests.orchestrator - INFO - Deleting pod: test-pod-71e98828 from PVC: test-pvc-9c79d5b3 +2025-06-05 16:54:34,710 - tests.orchestrator - INFO - Waiting for pod test-pod-71e98828 to be deleted +2025-06-05 16:55:07,317 - tests.orchestrator - INFO - Pod test-pod-71e98828 has been deleted +2025-06-05 16:55:07,317 - tests.orchestrator - INFO - Deleted pod: test-pod-71e98828 +2025-06-05 16:55:08,318 - tests.orchestrator - INFO - Test completed in 333.38 seconds +2025-06-05 16:55:08,318 - tests.orchestrator - INFO - ===== STARTING COMPREHENSIVE CLEANUP ===== +2025-06-05 16:55:08,318 - tests.orchestrator - INFO - Deleting 1 pods (force=False)... +2025-06-05 16:55:08,318 - tests.orchestrator - INFO - Deleting pod: test-pod-6b2c2055 from PVC: test-pvc-18e6fc84 +2025-06-05 16:55:08,393 - tests.orchestrator - INFO - Waiting for pod test-pod-6b2c2055 to be deleted +2025-06-05 16:55:41,002 - tests.orchestrator - INFO - Pod test-pod-6b2c2055 has been deleted +2025-06-05 16:55:41,002 - tests.orchestrator - INFO - Deleted pod: test-pod-6b2c2055 +2025-06-05 16:55:46,003 - tests.orchestrator - INFO - Deleting 4 PVCs (force=False)... +2025-06-05 16:55:46,003 - tests.orchestrator - INFO - Deleting PVC: test-pvc-614c6d80 +2025-06-05 16:55:46,074 - tests.orchestrator - INFO - Waiting for PVC test-pvc-614c6d80 to be deleted +2025-06-05 16:55:46,139 - tests.orchestrator - INFO - PVC test-pvc-614c6d80 has been deleted +2025-06-05 16:55:46,139 - tests.orchestrator - INFO - Deleted PVC: test-pvc-614c6d80 +2025-06-05 16:55:46,139 - tests.orchestrator - INFO - Deleting PVC: test-pvc-18e6fc84 +2025-06-05 16:55:46,210 - tests.orchestrator - INFO - Waiting for PVC test-pvc-18e6fc84 to be deleted +2025-06-05 16:55:46,276 - tests.orchestrator - INFO - PVC test-pvc-18e6fc84 has been deleted +2025-06-05 16:55:46,276 - tests.orchestrator - INFO - Deleted PVC: test-pvc-18e6fc84 +2025-06-05 16:55:46,277 - tests.orchestrator - INFO - Deleting PVC: test-pvc-6cb60179 +2025-06-05 16:55:46,347 - tests.orchestrator - INFO - Waiting for PVC test-pvc-6cb60179 to be deleted +2025-06-05 16:55:46,413 - tests.orchestrator - INFO - PVC test-pvc-6cb60179 has been deleted +2025-06-05 16:55:46,413 - tests.orchestrator - INFO - Deleted PVC: test-pvc-6cb60179 +2025-06-05 16:55:46,413 - tests.orchestrator - INFO - Deleting PVC: test-pvc-9c79d5b3 +2025-06-05 16:55:46,488 - tests.orchestrator - INFO - Waiting for PVC test-pvc-9c79d5b3 to be deleted +2025-06-05 16:55:46,567 - tests.orchestrator - INFO - PVC test-pvc-9c79d5b3 has been deleted +2025-06-05 16:55:46,568 - tests.orchestrator - INFO - Deleted PVC: test-pvc-9c79d5b3 +2025-06-05 16:55:46,843 - tests.orchestrator - WARNING - First cleanup pass incomplete. Remaining resources: ['pod/test-pod-0dadc981', 'pod/test-pod-0fcf8a5b', 'pod/test-pod-28a31e9a', 'pod/test-pod-2f20e8f1', 'pod/test-pod-4022a58d', 'pod/test-pod-4eee03ff', 'pod/test-pod-60374c6e', 'pod/test-pod-74abb322', 'pod/test-pod-955db556', 'pod/test-pod-988a488e', 'pod/test-pod-9de3bebb', 'pvc/concurrent-pvc-0aa045b0', 'pvc/concurrent-pvc-248d7584', 'pvc/concurrent-pvc-5b29e9cd', 'pvc/concurrent-pvc-903eb13a', 'pvc/concurrent-pvc-b0004a21', 'pvc/concurrent-pvc-ba7e5e4d', 'pvc/concurrent-pvc-f6a663b2', 'pvc/test-pvc-20f868e5', 'pvc/test-pvc-2381ce5e', 'pvc/test-pvc-2686f57c', 'pvc/test-pvc-269e9543', 'pvc/test-pvc-3f880237', 'pvc/test-pvc-4348540c', 'pvc/test-pvc-48e3d2d9', 'pvc/test-pvc-567589be', 'pvc/test-pvc-6282a942', 'pvc/test-pvc-83426f91', 'pvc/test-pvc-98b7ae8a', 'pvc/test-pvc-a253186e', 'pvc/test-pvc-b967555b', 'pvc/test-pvc-dbe6630f', 'pvc/test-pvc-e3594f67', 'pvc/test-pvc-e5985cb1'] +2025-06-05 16:55:46,843 - tests.orchestrator - INFO - Attempting force deletion of remaining resources... +2025-06-05 16:55:46,843 - tests.orchestrator - INFO - Deleting 0 pods (force=True)... +2025-06-05 16:55:51,843 - tests.orchestrator - INFO - Deleting 0 PVCs (force=True)... +2025-06-05 16:55:52,175 - tests.orchestrator - ERROR - Cleanup incomplete. Remaining resources after force deletion: ['pod/test-pod-0dadc981', 'pod/test-pod-0fcf8a5b', 'pod/test-pod-28a31e9a', 'pod/test-pod-2f20e8f1', 'pod/test-pod-4022a58d', 'pod/test-pod-4eee03ff', 'pod/test-pod-60374c6e', 'pod/test-pod-74abb322', 'pod/test-pod-955db556', 'pod/test-pod-988a488e', 'pod/test-pod-9de3bebb', 'pvc/concurrent-pvc-0aa045b0', 'pvc/concurrent-pvc-248d7584', 'pvc/concurrent-pvc-5b29e9cd', 'pvc/concurrent-pvc-903eb13a', 'pvc/concurrent-pvc-b0004a21', 'pvc/concurrent-pvc-ba7e5e4d', 'pvc/concurrent-pvc-f6a663b2', 'pvc/test-pvc-20f868e5', 'pvc/test-pvc-2381ce5e', 'pvc/test-pvc-2686f57c', 'pvc/test-pvc-269e9543', 'pvc/test-pvc-3f880237', 'pvc/test-pvc-4348540c', 'pvc/test-pvc-48e3d2d9', 'pvc/test-pvc-567589be', 'pvc/test-pvc-6282a942', 'pvc/test-pvc-83426f91', 'pvc/test-pvc-98b7ae8a', 'pvc/test-pvc-a253186e', 'pvc/test-pvc-b967555b', 'pvc/test-pvc-dbe6630f', 'pvc/test-pvc-e3594f67', 'pvc/test-pvc-e5985cb1'] +2025-06-05 16:55:52,175 - tests.orchestrator - INFO - Cleanup completed successfully in 43.86 seconds +2025-06-05 16:55:52,175 - tests.orchestrator - INFO - ===== CLEANUP PROCESS FINISHED ===== +2025-06-05 16:55:52,176 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-05 16:55:52,176 - tests.orchestrator - INFO - --- Operations --- +2025-06-05 16:55:52,176 - tests.orchestrator - INFO - create_pvc: 8 succeeded, 0 failed (100.0%) +2025-06-05 16:55:52,176 - tests.orchestrator - INFO - attach_pod: 6 succeeded, 0 failed (100.0%) +2025-06-05 16:55:52,176 - tests.orchestrator - INFO - delete_pod: 6 succeeded, 0 failed (100.0%) +2025-06-05 16:55:52,176 - tests.orchestrator - INFO - delete_pvc: 8 succeeded, 0 failed (100.0%) +2025-06-05 16:55:52,176 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) +2025-06-05 16:55:52,176 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) +2025-06-05 16:55:52,176 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-05 16:55:52,176 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-05 16:55:52,176 - tests.orchestrator - INFO - many_to_one: No runs +2025-06-05 16:55:52,176 - tests.orchestrator - INFO - one_to_one: No runs +2025-06-05 16:55:52,176 - tests.orchestrator - INFO - concurrent_pvc: No runs +2025-06-05 16:55:52,176 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_170306.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_170306.log new file mode 100644 index 000000000..aea06a813 --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_170306.log @@ -0,0 +1,17 @@ +2025-06-05 17:03:06,949 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 17:03:07,165 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 17:03:07,165 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-05 17:03:07,357 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 17:03:07,357 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 17:03:07,357 - tests.orchestrator - INFO - Creating PVC: test-pvc-63ada5ce +2025-06-05 17:03:07,436 - tests.orchestrator - INFO - Created PVC: test-pvc-63ada5ce +2025-06-05 17:03:07,436 - tests.orchestrator - INFO - Waiting for PVC test-pvc-63ada5ce to be bound +2025-06-05 17:03:09,571 - tests.orchestrator - INFO - PVC test-pvc-63ada5ce is bound +2025-06-05 17:03:09,571 - tests.orchestrator - INFO - Creating PVC: test-pvc-601432ce +2025-06-05 17:03:09,638 - tests.orchestrator - INFO - Created PVC: test-pvc-601432ce +2025-06-05 17:03:09,638 - tests.orchestrator - INFO - Waiting for PVC test-pvc-601432ce to be bound +2025-06-05 17:03:11,896 - tests.orchestrator - INFO - PVC test-pvc-601432ce is bound +2025-06-05 17:03:11,896 - tests.orchestrator - INFO - Creating pod test-pod-564344d0 on PVC test-pvc-63ada5ce +2025-06-05 17:03:12,343 - tests.orchestrator - INFO - Created pod: test-pod-564344d0 using PVC: test-pvc-63ada5ce +2025-06-05 17:03:12,343 - tests.orchestrator - INFO - Waiting for pod test-pod-564344d0 to be ready +2025-06-05 17:03:12,413 - tests.orchestrator - INFO - Pod test-pod-564344d0 phase: Pending diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_170342.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_170342.log new file mode 100644 index 000000000..525986131 --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_170342.log @@ -0,0 +1,17 @@ +2025-06-05 17:03:42,827 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 17:03:43,043 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 17:03:43,044 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-05 17:03:43,233 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 17:03:43,233 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 17:03:43,233 - tests.orchestrator - INFO - Creating PVC: test-pvc-455abf8b +2025-06-05 17:03:43,309 - tests.orchestrator - INFO - Created PVC: test-pvc-455abf8b +2025-06-05 17:03:43,309 - tests.orchestrator - INFO - Waiting for PVC test-pvc-455abf8b to be bound +2025-06-05 17:03:45,443 - tests.orchestrator - INFO - PVC test-pvc-455abf8b is bound +2025-06-05 17:03:45,443 - tests.orchestrator - INFO - Creating PVC: test-pvc-51dbaa96 +2025-06-05 17:03:45,510 - tests.orchestrator - INFO - Created PVC: test-pvc-51dbaa96 +2025-06-05 17:03:45,511 - tests.orchestrator - INFO - Waiting for PVC test-pvc-51dbaa96 to be bound +2025-06-05 17:03:47,644 - tests.orchestrator - INFO - PVC test-pvc-51dbaa96 is bound +2025-06-05 17:03:47,644 - tests.orchestrator - INFO - Creating pod test-pod-e1337987 on PVC test-pvc-455abf8b +2025-06-05 17:03:47,724 - tests.orchestrator - INFO - Created pod: test-pod-e1337987 using PVC: test-pvc-455abf8b +2025-06-05 17:03:47,724 - tests.orchestrator - INFO - Waiting for pod test-pod-e1337987 to be ready +2025-06-05 17:03:47,791 - tests.orchestrator - INFO - Pod test-pod-e1337987 phase: Pending diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_170405.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_170405.log new file mode 100644 index 000000000..2e9f3bea5 --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_170405.log @@ -0,0 +1,39 @@ +2025-06-05 17:04:05,563 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 17:04:05,759 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 17:04:05,759 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-05 17:04:05,948 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 17:04:05,948 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 17:04:05,948 - tests.orchestrator - INFO - Creating PVC: test-pvc-0b5af805 +2025-06-05 17:04:06,017 - tests.orchestrator - INFO - Created PVC: test-pvc-0b5af805 +2025-06-05 17:04:06,017 - tests.orchestrator - INFO - Waiting for PVC test-pvc-0b5af805 to be bound +2025-06-05 17:04:08,153 - tests.orchestrator - INFO - PVC test-pvc-0b5af805 is bound +2025-06-05 17:04:08,153 - tests.orchestrator - INFO - Creating PVC: test-pvc-d3ee5325 +2025-06-05 17:04:08,222 - tests.orchestrator - INFO - Created PVC: test-pvc-d3ee5325 +2025-06-05 17:04:08,222 - tests.orchestrator - INFO - Waiting for PVC test-pvc-d3ee5325 to be bound +2025-06-05 17:04:10,358 - tests.orchestrator - INFO - PVC test-pvc-d3ee5325 is bound +2025-06-05 17:04:10,358 - tests.orchestrator - INFO - Creating pod test-pod-3cc2d48d on PVC test-pvc-0b5af805 +2025-06-05 17:04:10,433 - tests.orchestrator - INFO - Created pod: test-pod-3cc2d48d using PVC: test-pvc-0b5af805 +2025-06-05 17:04:10,433 - tests.orchestrator - INFO - Waiting for pod test-pod-3cc2d48d to be ready +2025-06-05 17:04:10,500 - tests.orchestrator - INFO - Pod test-pod-3cc2d48d phase: Pending +2025-06-05 17:04:12,570 - tests.orchestrator - INFO - Pod test-pod-3cc2d48d phase: Running +2025-06-05 17:04:12,570 - tests.orchestrator - INFO - Pod test-pod-3cc2d48d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:04:14,639 - tests.orchestrator - INFO - Pod test-pod-3cc2d48d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:04:16,707 - tests.orchestrator - INFO - Pod test-pod-3cc2d48d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:04:18,775 - tests.orchestrator - INFO - Pod test-pod-3cc2d48d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:04:20,845 - tests.orchestrator - INFO - Pod test-pod-3cc2d48d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:04:22,913 - tests.orchestrator - INFO - Pod test-pod-3cc2d48d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:04:24,982 - tests.orchestrator - INFO - Pod test-pod-3cc2d48d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:04:27,051 - tests.orchestrator - INFO - Pod test-pod-3cc2d48d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 17:04:27,051 - tests.orchestrator - INFO - Pod test-pod-3cc2d48d is ready +2025-06-05 17:04:27,051 - tests.orchestrator - INFO - Creating pod test-pod-a9935180 on PVC test-pvc-0b5af805 +2025-06-05 17:04:27,135 - tests.orchestrator - INFO - Created pod: test-pod-a9935180 using PVC: test-pvc-0b5af805 +2025-06-05 17:04:27,135 - tests.orchestrator - INFO - Waiting for pod test-pod-a9935180 to be ready +2025-06-05 17:04:27,202 - tests.orchestrator - INFO - Pod test-pod-a9935180 phase: Pending +2025-06-05 17:04:29,271 - tests.orchestrator - INFO - Pod test-pod-a9935180 phase: Running +2025-06-05 17:04:29,271 - tests.orchestrator - INFO - Pod test-pod-a9935180 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:04:31,340 - tests.orchestrator - INFO - Pod test-pod-a9935180 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:04:33,409 - tests.orchestrator - INFO - Pod test-pod-a9935180 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:04:35,477 - tests.orchestrator - INFO - Pod test-pod-a9935180 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:04:37,547 - tests.orchestrator - INFO - Pod test-pod-a9935180 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:04:39,615 - tests.orchestrator - INFO - Pod test-pod-a9935180 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:04:41,683 - tests.orchestrator - INFO - Pod test-pod-a9935180 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_170502.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_170502.log new file mode 100644 index 000000000..b4d515d9c --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_170502.log @@ -0,0 +1,12 @@ +2025-06-05 17:05:02,906 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 17:05:03,121 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 17:05:03,121 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-05 17:05:03,312 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 17:05:03,312 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 17:05:03,313 - tests.orchestrator - INFO - Creating PVC: test-pvc-3691f2a0 +2025-06-05 17:05:03,396 - tests.orchestrator - INFO - Created PVC: test-pvc-3691f2a0 +2025-06-05 17:05:03,396 - tests.orchestrator - INFO - Waiting for PVC test-pvc-3691f2a0 to be bound +2025-06-05 17:05:05,534 - tests.orchestrator - INFO - PVC test-pvc-3691f2a0 is bound +2025-06-05 17:05:05,534 - tests.orchestrator - INFO - Creating PVC: test-pvc-64bf8174 +2025-06-05 17:05:05,608 - tests.orchestrator - INFO - Created PVC: test-pvc-64bf8174 +2025-06-05 17:05:05,608 - tests.orchestrator - INFO - Waiting for PVC test-pvc-64bf8174 to be bound diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_170526.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_170526.log new file mode 100644 index 000000000..af01d9c7c --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_170526.log @@ -0,0 +1,234 @@ +2025-06-05 17:05:26,238 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 17:05:26,435 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 17:05:26,436 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-05 17:05:26,638 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 17:05:26,638 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 17:05:26,638 - tests.orchestrator - INFO - Creating PVC: test-pvc-9dae599f +2025-06-05 17:05:26,711 - tests.orchestrator - INFO - Created PVC: test-pvc-9dae599f +2025-06-05 17:05:26,712 - tests.orchestrator - INFO - Waiting for PVC test-pvc-9dae599f to be bound +2025-06-05 17:05:28,848 - tests.orchestrator - INFO - PVC test-pvc-9dae599f is bound +2025-06-05 17:05:28,849 - tests.orchestrator - INFO - Creating PVC: test-pvc-759dbae5 +2025-06-05 17:05:28,916 - tests.orchestrator - INFO - Created PVC: test-pvc-759dbae5 +2025-06-05 17:05:28,916 - tests.orchestrator - INFO - Waiting for PVC test-pvc-759dbae5 to be bound +2025-06-05 17:05:31,051 - tests.orchestrator - INFO - PVC test-pvc-759dbae5 is bound +2025-06-05 17:05:31,052 - tests.orchestrator - INFO - Creating pod test-pod-60858a96 on PVC test-pvc-9dae599f +2025-06-05 17:05:31,134 - tests.orchestrator - INFO - Created pod: test-pod-60858a96 using PVC: test-pvc-9dae599f +2025-06-05 17:05:31,134 - tests.orchestrator - INFO - Waiting for pod test-pod-60858a96 to be ready +2025-06-05 17:05:31,203 - tests.orchestrator - INFO - Pod test-pod-60858a96 phase: Pending +2025-06-05 17:05:35,349 - tests.orchestrator - INFO - Pod test-pod-60858a96 phase: Running +2025-06-05 17:05:35,349 - tests.orchestrator - INFO - Pod test-pod-60858a96 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:05:37,418 - tests.orchestrator - INFO - Pod test-pod-60858a96 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:05:39,486 - tests.orchestrator - INFO - Pod test-pod-60858a96 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:05:41,723 - tests.orchestrator - INFO - Pod test-pod-60858a96 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:05:43,792 - tests.orchestrator - INFO - Pod test-pod-60858a96 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:05:45,860 - tests.orchestrator - INFO - Pod test-pod-60858a96 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:05:47,929 - tests.orchestrator - INFO - Pod test-pod-60858a96 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 17:05:47,929 - tests.orchestrator - INFO - Pod test-pod-60858a96 is ready +2025-06-05 17:05:47,929 - tests.orchestrator - INFO - Creating pod test-pod-e1e1abfd on PVC test-pvc-759dbae5 +2025-06-05 17:05:48,053 - tests.orchestrator - INFO - Created pod: test-pod-e1e1abfd using PVC: test-pvc-759dbae5 +2025-06-05 17:05:48,053 - tests.orchestrator - INFO - Waiting for pod test-pod-e1e1abfd to be ready +2025-06-05 17:05:48,139 - tests.orchestrator - INFO - Pod test-pod-e1e1abfd phase: Pending +2025-06-05 17:05:52,277 - tests.orchestrator - INFO - Pod test-pod-e1e1abfd phase: Running +2025-06-05 17:05:52,278 - tests.orchestrator - INFO - Pod test-pod-e1e1abfd conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:05:54,345 - tests.orchestrator - INFO - Pod test-pod-e1e1abfd conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:05:56,414 - tests.orchestrator - INFO - Pod test-pod-e1e1abfd conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:05:58,484 - tests.orchestrator - INFO - Pod test-pod-e1e1abfd conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:06:00,553 - tests.orchestrator - INFO - Pod test-pod-e1e1abfd conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:06:02,623 - tests.orchestrator - INFO - Pod test-pod-e1e1abfd conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:06:04,693 - tests.orchestrator - INFO - Pod test-pod-e1e1abfd conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:06:06,766 - tests.orchestrator - INFO - Pod test-pod-e1e1abfd conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:06:08,838 - tests.orchestrator - INFO - Pod test-pod-e1e1abfd conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:06:10,908 - tests.orchestrator - INFO - Pod test-pod-e1e1abfd conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 17:06:10,908 - tests.orchestrator - INFO - Pod test-pod-e1e1abfd is ready +2025-06-05 17:06:10,908 - tests.orchestrator - INFO - Creating pod test-pod-177eb3e7 on PVC test-pvc-9dae599f +2025-06-05 17:06:10,984 - tests.orchestrator - INFO - Created pod: test-pod-177eb3e7 using PVC: test-pvc-9dae599f +2025-06-05 17:06:10,984 - tests.orchestrator - INFO - Waiting for pod test-pod-177eb3e7 to be ready +2025-06-05 17:06:11,053 - tests.orchestrator - INFO - Pod test-pod-177eb3e7 phase: Pending +2025-06-05 17:06:13,125 - tests.orchestrator - INFO - Pod test-pod-177eb3e7 phase: Running +2025-06-05 17:06:13,125 - tests.orchestrator - INFO - Pod test-pod-177eb3e7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:06:15,195 - tests.orchestrator - INFO - Pod test-pod-177eb3e7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:06:17,263 - tests.orchestrator - INFO - Pod test-pod-177eb3e7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:06:19,332 - tests.orchestrator - INFO - Pod test-pod-177eb3e7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:06:21,402 - tests.orchestrator - INFO - Pod test-pod-177eb3e7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:06:23,470 - tests.orchestrator - INFO - Pod test-pod-177eb3e7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:06:25,540 - tests.orchestrator - INFO - Pod test-pod-177eb3e7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:06:27,607 - tests.orchestrator - INFO - Pod test-pod-177eb3e7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:06:29,677 - tests.orchestrator - INFO - Pod test-pod-177eb3e7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:06:31,747 - tests.orchestrator - INFO - Pod test-pod-177eb3e7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:06:33,819 - tests.orchestrator - INFO - Pod test-pod-177eb3e7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 17:06:33,819 - tests.orchestrator - INFO - Pod test-pod-177eb3e7 is ready +2025-06-05 17:06:33,819 - tests.orchestrator - INFO - Testing read/write between pods test-pod-177eb3e7 and test-pod-60858a96 sharing PVC test-pvc-9dae599f +2025-06-05 17:06:33,887 - tests.orchestrator - ERROR - Failed in read/write verification: (400) +Reason: Bad Request +HTTP response headers: HTTPHeaderDict({'Audit-Id': 'b5f76f0a-dcce-47f0-8633-e17708ea40ac', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 17:06:33 GMT', 'Content-Length': '139'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Upgrade request required","reason":"BadRequest","code":400} + + +2025-06-05 17:06:33,887 - tests.orchestrator - INFO - Deleting pod: test-pod-60858a96 from PVC: test-pvc-9dae599f +2025-06-05 17:06:33,966 - tests.orchestrator - INFO - Waiting for pod test-pod-60858a96 to be deleted +2025-06-05 17:07:06,621 - tests.orchestrator - INFO - Pod test-pod-60858a96 has been deleted +2025-06-05 17:07:06,621 - tests.orchestrator - INFO - Deleted pod: test-pod-60858a96 +2025-06-05 17:07:06,621 - tests.orchestrator - INFO - Deleting PVC: test-pvc-9dae599f +2025-06-05 17:07:06,621 - tests.orchestrator - INFO - Deleting 1 pods using PVC test-pvc-9dae599f +2025-06-05 17:07:06,621 - tests.orchestrator - INFO - Deleting pod: test-pod-177eb3e7 from PVC: test-pvc-9dae599f +2025-06-05 17:07:06,699 - tests.orchestrator - INFO - Waiting for pod test-pod-177eb3e7 to be deleted +2025-06-05 17:07:39,317 - tests.orchestrator - INFO - Pod test-pod-177eb3e7 has been deleted +2025-06-05 17:07:39,317 - tests.orchestrator - INFO - Deleted pod: test-pod-177eb3e7 +2025-06-05 17:07:39,393 - tests.orchestrator - INFO - Waiting for PVC test-pvc-9dae599f to be deleted +2025-06-05 17:07:39,461 - tests.orchestrator - INFO - PVC test-pvc-9dae599f has been deleted +2025-06-05 17:07:39,461 - tests.orchestrator - INFO - Deleted PVC: test-pvc-9dae599f +2025-06-05 17:07:39,461 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations +2025-06-05 17:07:39,461 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 1 times) +2025-06-05 17:07:39,461 - tests.orchestrator - INFO - Creating pod test-pod-3ce2f282 on PVC test-pvc-759dbae5 +2025-06-05 17:07:39,585 - tests.orchestrator - INFO - Created pod: test-pod-3ce2f282 using PVC: test-pvc-759dbae5 +2025-06-05 17:07:39,585 - tests.orchestrator - INFO - Waiting for pod test-pod-3ce2f282 to be ready +2025-06-05 17:07:39,655 - tests.orchestrator - INFO - Pod test-pod-3ce2f282 phase: Pending +2025-06-05 17:07:41,879 - tests.orchestrator - INFO - Pod test-pod-3ce2f282 phase: Running +2025-06-05 17:07:41,879 - tests.orchestrator - INFO - Pod test-pod-3ce2f282 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:07:43,948 - tests.orchestrator - INFO - Pod test-pod-3ce2f282 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:07:46,017 - tests.orchestrator - INFO - Pod test-pod-3ce2f282 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:07:48,085 - tests.orchestrator - INFO - Pod test-pod-3ce2f282 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:07:50,155 - tests.orchestrator - INFO - Pod test-pod-3ce2f282 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:07:52,225 - tests.orchestrator - INFO - Pod test-pod-3ce2f282 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:07:54,292 - tests.orchestrator - INFO - Pod test-pod-3ce2f282 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:07:56,363 - tests.orchestrator - INFO - Pod test-pod-3ce2f282 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:07:58,434 - tests.orchestrator - INFO - Pod test-pod-3ce2f282 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:08:00,502 - tests.orchestrator - INFO - Pod test-pod-3ce2f282 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:08:02,573 - tests.orchestrator - INFO - Pod test-pod-3ce2f282 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 17:08:02,573 - tests.orchestrator - INFO - Pod test-pod-3ce2f282 is ready +2025-06-05 17:08:03,574 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 1 times) +2025-06-05 17:08:03,574 - tests.orchestrator - INFO - Deleting pod: test-pod-3ce2f282 from PVC: test-pvc-759dbae5 +2025-06-05 17:08:03,651 - tests.orchestrator - INFO - Waiting for pod test-pod-3ce2f282 to be deleted +2025-06-05 17:08:36,384 - tests.orchestrator - INFO - Pod test-pod-3ce2f282 has been deleted +2025-06-05 17:08:36,384 - tests.orchestrator - INFO - Deleted pod: test-pod-3ce2f282 +2025-06-05 17:08:37,384 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 1 times) +2025-06-05 17:08:37,384 - tests.orchestrator - INFO - Creating PVC: test-pvc-e7dcdff0 +2025-06-05 17:08:37,462 - tests.orchestrator - INFO - Created PVC: test-pvc-e7dcdff0 +2025-06-05 17:08:37,463 - tests.orchestrator - INFO - Waiting for PVC test-pvc-e7dcdff0 to be bound +2025-06-05 17:08:39,598 - tests.orchestrator - INFO - PVC test-pvc-e7dcdff0 is bound +2025-06-05 17:08:40,599 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 2 times) +2025-06-05 17:08:40,599 - tests.orchestrator - INFO - Creating PVC: test-pvc-1511fcaf +2025-06-05 17:08:40,670 - tests.orchestrator - INFO - Created PVC: test-pvc-1511fcaf +2025-06-05 17:08:40,670 - tests.orchestrator - INFO - Waiting for PVC test-pvc-1511fcaf to be bound +2025-06-05 17:08:42,808 - tests.orchestrator - INFO - PVC test-pvc-1511fcaf is bound +2025-06-05 17:08:43,808 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 3 times) +2025-06-05 17:08:43,809 - tests.orchestrator - INFO - Creating PVC: test-pvc-47e77b85 +2025-06-05 17:08:43,878 - tests.orchestrator - INFO - Created PVC: test-pvc-47e77b85 +2025-06-05 17:08:43,879 - tests.orchestrator - INFO - Waiting for PVC test-pvc-47e77b85 to be bound +2025-06-05 17:08:46,014 - tests.orchestrator - INFO - PVC test-pvc-47e77b85 is bound +2025-06-05 17:08:47,015 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 1 times) +2025-06-05 17:08:47,015 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test +2025-06-05 17:08:48,015 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 2 times) +2025-06-05 17:08:48,015 - tests.orchestrator - INFO - Creating pod test-pod-94de82f9 on PVC test-pvc-1511fcaf +2025-06-05 17:08:48,137 - tests.orchestrator - INFO - Created pod: test-pod-94de82f9 using PVC: test-pvc-1511fcaf +2025-06-05 17:08:48,137 - tests.orchestrator - INFO - Waiting for pod test-pod-94de82f9 to be ready +2025-06-05 17:08:48,224 - tests.orchestrator - INFO - Pod test-pod-94de82f9 phase: Pending +2025-06-05 17:08:52,367 - tests.orchestrator - INFO - Pod test-pod-94de82f9 phase: Running +2025-06-05 17:08:52,367 - tests.orchestrator - INFO - Pod test-pod-94de82f9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:08:54,435 - tests.orchestrator - INFO - Pod test-pod-94de82f9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:08:56,504 - tests.orchestrator - INFO - Pod test-pod-94de82f9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:08:58,574 - tests.orchestrator - INFO - Pod test-pod-94de82f9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:09:00,641 - tests.orchestrator - INFO - Pod test-pod-94de82f9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:09:02,713 - tests.orchestrator - INFO - Pod test-pod-94de82f9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:09:04,782 - tests.orchestrator - INFO - Pod test-pod-94de82f9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 17:09:04,782 - tests.orchestrator - INFO - Pod test-pod-94de82f9 is ready +2025-06-05 17:09:05,782 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 3 times) +2025-06-05 17:09:05,782 - tests.orchestrator - INFO - Creating pod test-pod-42c169cc on PVC test-pvc-1511fcaf +2025-06-05 17:09:05,861 - tests.orchestrator - INFO - Created pod: test-pod-42c169cc using PVC: test-pvc-1511fcaf +2025-06-05 17:09:05,861 - tests.orchestrator - INFO - Waiting for pod test-pod-42c169cc to be ready +2025-06-05 17:09:05,928 - tests.orchestrator - INFO - Pod test-pod-42c169cc phase: Pending +2025-06-05 17:09:07,998 - tests.orchestrator - INFO - Pod test-pod-42c169cc phase: Running +2025-06-05 17:09:07,998 - tests.orchestrator - INFO - Pod test-pod-42c169cc conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:09:10,071 - tests.orchestrator - INFO - Pod test-pod-42c169cc conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:09:12,149 - tests.orchestrator - INFO - Pod test-pod-42c169cc conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:09:14,219 - tests.orchestrator - INFO - Pod test-pod-42c169cc conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:09:16,290 - tests.orchestrator - INFO - Pod test-pod-42c169cc conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:09:18,358 - tests.orchestrator - INFO - Pod test-pod-42c169cc conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:09:20,429 - tests.orchestrator - INFO - Pod test-pod-42c169cc conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:09:22,498 - tests.orchestrator - INFO - Pod test-pod-42c169cc conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:09:24,566 - tests.orchestrator - INFO - Pod test-pod-42c169cc conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:09:26,636 - tests.orchestrator - INFO - Pod test-pod-42c169cc conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 17:09:26,637 - tests.orchestrator - INFO - Pod test-pod-42c169cc is ready +2025-06-05 17:09:27,637 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 4 times) +2025-06-05 17:09:27,637 - tests.orchestrator - INFO - Creating PVC: test-pvc-5d63ba99 +2025-06-05 17:09:27,713 - tests.orchestrator - INFO - Created PVC: test-pvc-5d63ba99 +2025-06-05 17:09:27,714 - tests.orchestrator - INFO - Waiting for PVC test-pvc-5d63ba99 to be bound +2025-06-05 17:09:29,848 - tests.orchestrator - INFO - PVC test-pvc-5d63ba99 is bound +2025-06-05 17:09:30,848 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 2 times) +2025-06-05 17:09:30,848 - tests.orchestrator - INFO - Deleting pod: test-pod-e1e1abfd from PVC: test-pvc-759dbae5 +2025-06-05 17:09:30,923 - tests.orchestrator - INFO - Waiting for pod test-pod-e1e1abfd to be deleted +2025-06-05 17:10:03,546 - tests.orchestrator - INFO - Pod test-pod-e1e1abfd has been deleted +2025-06-05 17:10:03,546 - tests.orchestrator - INFO - Deleted pod: test-pod-e1e1abfd +2025-06-05 17:10:04,546 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 4 times) +2025-06-05 17:10:04,546 - tests.orchestrator - INFO - Creating pod test-pod-21da98c0 on PVC test-pvc-47e77b85 +2025-06-05 17:10:04,641 - tests.orchestrator - INFO - Created pod: test-pod-21da98c0 using PVC: test-pvc-47e77b85 +2025-06-05 17:10:04,641 - tests.orchestrator - INFO - Waiting for pod test-pod-21da98c0 to be ready +2025-06-05 17:10:04,710 - tests.orchestrator - INFO - Pod test-pod-21da98c0 phase: Pending +2025-06-05 17:10:06,780 - tests.orchestrator - INFO - Pod test-pod-21da98c0 phase: Running +2025-06-05 17:10:06,780 - tests.orchestrator - INFO - Pod test-pod-21da98c0 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:10:08,850 - tests.orchestrator - INFO - Pod test-pod-21da98c0 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:10:10,920 - tests.orchestrator - INFO - Pod test-pod-21da98c0 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:10:12,990 - tests.orchestrator - INFO - Pod test-pod-21da98c0 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:10:15,059 - tests.orchestrator - INFO - Pod test-pod-21da98c0 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:10:17,130 - tests.orchestrator - INFO - Pod test-pod-21da98c0 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:10:19,198 - tests.orchestrator - INFO - Pod test-pod-21da98c0 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 17:10:21,265 - tests.orchestrator - INFO - Pod test-pod-21da98c0 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 17:10:21,265 - tests.orchestrator - INFO - Pod test-pod-21da98c0 is ready +2025-06-05 17:10:22,266 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 3 times) +2025-06-05 17:10:22,266 - tests.orchestrator - INFO - Deleting pod: test-pod-42c169cc from PVC: test-pvc-1511fcaf +2025-06-05 17:10:22,353 - tests.orchestrator - INFO - Waiting for pod test-pod-42c169cc to be deleted +2025-06-05 17:10:54,992 - tests.orchestrator - INFO - Pod test-pod-42c169cc has been deleted +2025-06-05 17:10:54,992 - tests.orchestrator - INFO - Deleted pod: test-pod-42c169cc +2025-06-05 17:10:55,992 - tests.orchestrator - INFO - Test completed in 329.56 seconds +2025-06-05 17:10:55,993 - tests.orchestrator - INFO - ===== STARTING COMPREHENSIVE CLEANUP ===== +2025-06-05 17:10:55,993 - tests.orchestrator - INFO - Deleting 2 pods (force=False)... +2025-06-05 17:10:55,993 - tests.orchestrator - INFO - Deleting pod: test-pod-94de82f9 from PVC: test-pvc-1511fcaf +2025-06-05 17:10:56,071 - tests.orchestrator - INFO - Waiting for pod test-pod-94de82f9 to be deleted +2025-06-05 17:11:28,699 - tests.orchestrator - INFO - Pod test-pod-94de82f9 has been deleted +2025-06-05 17:11:28,699 - tests.orchestrator - INFO - Deleted pod: test-pod-94de82f9 +2025-06-05 17:11:28,699 - tests.orchestrator - INFO - Deleting pod: test-pod-21da98c0 from PVC: test-pvc-47e77b85 +2025-06-05 17:11:28,774 - tests.orchestrator - INFO - Waiting for pod test-pod-21da98c0 to be deleted +2025-06-05 17:12:01,396 - tests.orchestrator - INFO - Pod test-pod-21da98c0 has been deleted +2025-06-05 17:12:01,396 - tests.orchestrator - INFO - Deleted pod: test-pod-21da98c0 +2025-06-05 17:12:06,396 - tests.orchestrator - INFO - Deleting 5 PVCs (force=False)... +2025-06-05 17:12:06,396 - tests.orchestrator - INFO - Deleting PVC: test-pvc-759dbae5 +2025-06-05 17:12:06,474 - tests.orchestrator - INFO - Waiting for PVC test-pvc-759dbae5 to be deleted +2025-06-05 17:12:10,610 - tests.orchestrator - INFO - PVC test-pvc-759dbae5 has been deleted +2025-06-05 17:12:10,610 - tests.orchestrator - INFO - Deleted PVC: test-pvc-759dbae5 +2025-06-05 17:12:10,610 - tests.orchestrator - INFO - Deleting PVC: test-pvc-e7dcdff0 +2025-06-05 17:12:10,682 - tests.orchestrator - INFO - Waiting for PVC test-pvc-e7dcdff0 to be deleted +2025-06-05 17:12:10,753 - tests.orchestrator - INFO - PVC test-pvc-e7dcdff0 has been deleted +2025-06-05 17:12:10,753 - tests.orchestrator - INFO - Deleted PVC: test-pvc-e7dcdff0 +2025-06-05 17:12:10,753 - tests.orchestrator - INFO - Deleting PVC: test-pvc-1511fcaf +2025-06-05 17:12:10,826 - tests.orchestrator - INFO - Waiting for PVC test-pvc-1511fcaf to be deleted +2025-06-05 17:12:10,894 - tests.orchestrator - INFO - PVC test-pvc-1511fcaf has been deleted +2025-06-05 17:12:10,894 - tests.orchestrator - INFO - Deleted PVC: test-pvc-1511fcaf +2025-06-05 17:12:10,894 - tests.orchestrator - INFO - Deleting PVC: test-pvc-47e77b85 +2025-06-05 17:12:10,967 - tests.orchestrator - INFO - Waiting for PVC test-pvc-47e77b85 to be deleted +2025-06-05 17:12:11,032 - tests.orchestrator - INFO - PVC test-pvc-47e77b85 has been deleted +2025-06-05 17:12:11,032 - tests.orchestrator - INFO - Deleted PVC: test-pvc-47e77b85 +2025-06-05 17:12:11,032 - tests.orchestrator - INFO - Deleting PVC: test-pvc-5d63ba99 +2025-06-05 17:12:11,107 - tests.orchestrator - INFO - Waiting for PVC test-pvc-5d63ba99 to be deleted +2025-06-05 17:12:11,173 - tests.orchestrator - INFO - PVC test-pvc-5d63ba99 has been deleted +2025-06-05 17:12:11,173 - tests.orchestrator - INFO - Deleted PVC: test-pvc-5d63ba99 +2025-06-05 17:12:11,377 - tests.orchestrator - WARNING - First cleanup pass incomplete. Remaining resources: ['pod/test-pod-3cc2d48d', 'pod/test-pod-564344d0', 'pod/test-pod-a9935180', 'pod/test-pod-e1337987', 'pvc/test-pvc-0b5af805', 'pvc/test-pvc-3691f2a0', 'pvc/test-pvc-455abf8b', 'pvc/test-pvc-51dbaa96', 'pvc/test-pvc-601432ce', 'pvc/test-pvc-63ada5ce', 'pvc/test-pvc-64bf8174', 'pvc/test-pvc-d3ee5325'] +2025-06-05 17:12:11,377 - tests.orchestrator - INFO - Attempting force deletion of remaining resources... +2025-06-05 17:12:11,377 - tests.orchestrator - INFO - Deleting 0 pods (force=True)... +2025-06-05 17:12:16,377 - tests.orchestrator - INFO - Deleting 0 PVCs (force=True)... +2025-06-05 17:12:16,581 - tests.orchestrator - ERROR - Cleanup incomplete. Remaining resources after force deletion: ['pod/test-pod-3cc2d48d', 'pod/test-pod-564344d0', 'pod/test-pod-a9935180', 'pod/test-pod-e1337987', 'pvc/test-pvc-0b5af805', 'pvc/test-pvc-3691f2a0', 'pvc/test-pvc-455abf8b', 'pvc/test-pvc-51dbaa96', 'pvc/test-pvc-601432ce', 'pvc/test-pvc-63ada5ce', 'pvc/test-pvc-64bf8174', 'pvc/test-pvc-d3ee5325'] +2025-06-05 17:12:16,581 - tests.orchestrator - INFO - Cleanup completed successfully in 80.59 seconds +2025-06-05 17:12:16,582 - tests.orchestrator - INFO - ===== CLEANUP PROCESS FINISHED ===== +2025-06-05 17:12:16,582 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-05 17:12:16,582 - tests.orchestrator - INFO - --- Operations --- +2025-06-05 17:12:16,582 - tests.orchestrator - INFO - create_pvc: 6 succeeded, 0 failed (100.0%) +2025-06-05 17:12:16,582 - tests.orchestrator - INFO - attach_pod: 7 succeeded, 0 failed (100.0%) +2025-06-05 17:12:16,582 - tests.orchestrator - INFO - delete_pod: 7 succeeded, 0 failed (100.0%) +2025-06-05 17:12:16,582 - tests.orchestrator - INFO - delete_pvc: 6 succeeded, 0 failed (100.0%) +2025-06-05 17:12:16,582 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) +2025-06-05 17:12:16,582 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) +2025-06-05 17:12:16,582 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-05 17:12:16,582 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-05 17:12:16,582 - tests.orchestrator - INFO - many_to_one: No runs +2025-06-05 17:12:16,582 - tests.orchestrator - INFO - one_to_one: No runs +2025-06-05 17:12:16,582 - tests.orchestrator - INFO - concurrent_pvc: No runs +2025-06-05 17:12:16,582 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_182216.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_182216.log new file mode 100644 index 000000000..79d31d520 --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_182216.log @@ -0,0 +1,7 @@ +2025-06-05 18:22:16,318 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 18:22:16,528 - tests.orchestrator - ERROR - Error checking namespace: (401) +Reason: Unauthorized +HTTP response headers: HTTPHeaderDict({'Audit-Id': '1a400704-0ba2-4e3d-a421-03adbfa89ebe', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 18:22:16 GMT', 'Content-Length': '129'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} + + diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_182254.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_182254.log new file mode 100644 index 000000000..e9292aaa6 --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_182254.log @@ -0,0 +1,207 @@ +2025-06-05 18:22:54,149 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 18:22:54,350 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 18:22:54,351 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-05 18:22:54,543 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 18:22:54,543 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 18:22:54,543 - tests.orchestrator - INFO - Creating PVC: test-pvc-7a09d3a9 +2025-06-05 18:22:54,620 - tests.orchestrator - INFO - Created PVC: test-pvc-7a09d3a9 +2025-06-05 18:22:54,620 - tests.orchestrator - INFO - Waiting for PVC test-pvc-7a09d3a9 to be bound +2025-06-05 18:22:56,752 - tests.orchestrator - INFO - PVC test-pvc-7a09d3a9 is bound +2025-06-05 18:22:56,752 - tests.orchestrator - INFO - Creating PVC: test-pvc-af47e7e8 +2025-06-05 18:22:56,820 - tests.orchestrator - INFO - Created PVC: test-pvc-af47e7e8 +2025-06-05 18:22:56,820 - tests.orchestrator - INFO - Waiting for PVC test-pvc-af47e7e8 to be bound +2025-06-05 18:22:58,953 - tests.orchestrator - INFO - PVC test-pvc-af47e7e8 is bound +2025-06-05 18:22:58,953 - tests.orchestrator - INFO - Creating pod test-pod-09840b6a on PVC test-pvc-7a09d3a9 +2025-06-05 18:22:59,033 - tests.orchestrator - INFO - Created pod: test-pod-09840b6a using PVC: test-pvc-7a09d3a9 +2025-06-05 18:22:59,033 - tests.orchestrator - INFO - Waiting for pod test-pod-09840b6a to be ready +2025-06-05 18:22:59,100 - tests.orchestrator - INFO - Pod test-pod-09840b6a phase: Pending +2025-06-05 18:23:03,237 - tests.orchestrator - INFO - Pod test-pod-09840b6a phase: Running +2025-06-05 18:23:03,237 - tests.orchestrator - INFO - Pod test-pod-09840b6a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:23:05,303 - tests.orchestrator - INFO - Pod test-pod-09840b6a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:23:07,371 - tests.orchestrator - INFO - Pod test-pod-09840b6a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:23:09,439 - tests.orchestrator - INFO - Pod test-pod-09840b6a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:23:11,506 - tests.orchestrator - INFO - Pod test-pod-09840b6a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:23:13,575 - tests.orchestrator - INFO - Pod test-pod-09840b6a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:23:15,642 - tests.orchestrator - INFO - Pod test-pod-09840b6a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:23:17,710 - tests.orchestrator - INFO - Pod test-pod-09840b6a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:23:19,778 - tests.orchestrator - INFO - Pod test-pod-09840b6a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 18:23:19,778 - tests.orchestrator - INFO - Pod test-pod-09840b6a is ready +2025-06-05 18:23:19,778 - tests.orchestrator - INFO - Creating pod test-pod-606ca02c on PVC test-pvc-af47e7e8 +2025-06-05 18:23:19,851 - tests.orchestrator - INFO - Created pod: test-pod-606ca02c using PVC: test-pvc-af47e7e8 +2025-06-05 18:23:19,851 - tests.orchestrator - INFO - Waiting for pod test-pod-606ca02c to be ready +2025-06-05 18:23:19,918 - tests.orchestrator - INFO - Pod test-pod-606ca02c phase: Pending +2025-06-05 18:23:24,056 - tests.orchestrator - INFO - Pod test-pod-606ca02c phase: Running +2025-06-05 18:23:24,056 - tests.orchestrator - INFO - Pod test-pod-606ca02c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:23:26,123 - tests.orchestrator - INFO - Pod test-pod-606ca02c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:23:28,191 - tests.orchestrator - INFO - Pod test-pod-606ca02c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:23:30,260 - tests.orchestrator - INFO - Pod test-pod-606ca02c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:23:32,329 - tests.orchestrator - INFO - Pod test-pod-606ca02c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:23:34,397 - tests.orchestrator - INFO - Pod test-pod-606ca02c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:23:36,470 - tests.orchestrator - INFO - Pod test-pod-606ca02c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:23:38,538 - tests.orchestrator - INFO - Pod test-pod-606ca02c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:23:40,606 - tests.orchestrator - INFO - Pod test-pod-606ca02c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:23:42,674 - tests.orchestrator - INFO - Pod test-pod-606ca02c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 18:23:42,674 - tests.orchestrator - INFO - Pod test-pod-606ca02c is ready +2025-06-05 18:23:42,674 - tests.orchestrator - INFO - Creating pod test-pod-6e1e895f on PVC test-pvc-af47e7e8 +2025-06-05 18:23:42,758 - tests.orchestrator - INFO - Created pod: test-pod-6e1e895f using PVC: test-pvc-af47e7e8 +2025-06-05 18:23:42,758 - tests.orchestrator - INFO - Waiting for pod test-pod-6e1e895f to be ready +2025-06-05 18:23:42,825 - tests.orchestrator - INFO - Pod test-pod-6e1e895f phase: Pending +2025-06-05 18:23:44,892 - tests.orchestrator - INFO - Pod test-pod-6e1e895f phase: Running +2025-06-05 18:23:44,892 - tests.orchestrator - INFO - Pod test-pod-6e1e895f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:23:46,961 - tests.orchestrator - INFO - Pod test-pod-6e1e895f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:23:49,028 - tests.orchestrator - INFO - Pod test-pod-6e1e895f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:23:51,095 - tests.orchestrator - INFO - Pod test-pod-6e1e895f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:23:53,163 - tests.orchestrator - INFO - Pod test-pod-6e1e895f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:23:55,230 - tests.orchestrator - INFO - Pod test-pod-6e1e895f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:23:57,298 - tests.orchestrator - INFO - Pod test-pod-6e1e895f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:23:59,366 - tests.orchestrator - INFO - Pod test-pod-6e1e895f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:24:01,442 - tests.orchestrator - INFO - Pod test-pod-6e1e895f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:24:03,509 - tests.orchestrator - INFO - Pod test-pod-6e1e895f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 18:24:03,509 - tests.orchestrator - INFO - Pod test-pod-6e1e895f is ready +2025-06-05 18:24:03,510 - tests.orchestrator - INFO - Testing read/write between pods test-pod-6e1e895f and test-pod-606ca02c sharing PVC test-pvc-af47e7e8 +2025-06-05 18:24:03,580 - tests.orchestrator - ERROR - Failed in read/write verification: (400) +Reason: Bad Request +HTTP response headers: HTTPHeaderDict({'Audit-Id': 'ebf31bfa-8019-42fb-ae12-2d291ecd43e4', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 18:24:03 GMT', 'Content-Length': '139'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Upgrade request required","reason":"BadRequest","code":400} + + +2025-06-05 18:24:03,580 - tests.orchestrator - INFO - Deleting pod: test-pod-606ca02c from PVC: test-pvc-af47e7e8 +2025-06-05 18:24:03,653 - tests.orchestrator - INFO - Waiting for pod test-pod-606ca02c to be deleted +2025-06-05 18:24:36,256 - tests.orchestrator - INFO - Pod test-pod-606ca02c has been deleted +2025-06-05 18:24:36,256 - tests.orchestrator - INFO - Deleted pod: test-pod-606ca02c +2025-06-05 18:24:36,256 - tests.orchestrator - INFO - Deleting PVC: test-pvc-af47e7e8 +2025-06-05 18:24:36,256 - tests.orchestrator - INFO - Deleting 1 pods using PVC test-pvc-af47e7e8 +2025-06-05 18:24:36,256 - tests.orchestrator - INFO - Deleting pod: test-pod-6e1e895f from PVC: test-pvc-af47e7e8 +2025-06-05 18:24:36,332 - tests.orchestrator - INFO - Waiting for pod test-pod-6e1e895f to be deleted +2025-06-05 18:25:08,936 - tests.orchestrator - INFO - Pod test-pod-6e1e895f has been deleted +2025-06-05 18:25:08,936 - tests.orchestrator - INFO - Deleted pod: test-pod-6e1e895f +2025-06-05 18:25:09,008 - tests.orchestrator - INFO - Waiting for PVC test-pvc-af47e7e8 to be deleted +2025-06-05 18:25:09,073 - tests.orchestrator - INFO - PVC test-pvc-af47e7e8 has been deleted +2025-06-05 18:25:09,074 - tests.orchestrator - INFO - Deleted PVC: test-pvc-af47e7e8 +2025-06-05 18:25:09,074 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations +2025-06-05 18:25:09,074 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 1 times) +2025-06-05 18:25:09,074 - tests.orchestrator - INFO - Creating pod test-pod-d40af63e on PVC test-pvc-7a09d3a9 +2025-06-05 18:25:09,153 - tests.orchestrator - INFO - Created pod: test-pod-d40af63e using PVC: test-pvc-7a09d3a9 +2025-06-05 18:25:09,153 - tests.orchestrator - INFO - Waiting for pod test-pod-d40af63e to be ready +2025-06-05 18:25:09,220 - tests.orchestrator - INFO - Pod test-pod-d40af63e phase: Pending +2025-06-05 18:25:13,353 - tests.orchestrator - INFO - Pod test-pod-d40af63e phase: Running +2025-06-05 18:25:13,353 - tests.orchestrator - INFO - Pod test-pod-d40af63e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:25:15,421 - tests.orchestrator - INFO - Pod test-pod-d40af63e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:25:17,489 - tests.orchestrator - INFO - Pod test-pod-d40af63e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:25:19,556 - tests.orchestrator - INFO - Pod test-pod-d40af63e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:25:21,625 - tests.orchestrator - INFO - Pod test-pod-d40af63e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:25:23,693 - tests.orchestrator - INFO - Pod test-pod-d40af63e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:25:25,761 - tests.orchestrator - INFO - Pod test-pod-d40af63e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 18:25:25,762 - tests.orchestrator - INFO - Pod test-pod-d40af63e is ready +2025-06-05 18:25:26,762 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 2 times) +2025-06-05 18:25:26,762 - tests.orchestrator - INFO - Creating pod test-pod-bf04a30a on PVC test-pvc-7a09d3a9 +2025-06-05 18:25:26,834 - tests.orchestrator - INFO - Created pod: test-pod-bf04a30a using PVC: test-pvc-7a09d3a9 +2025-06-05 18:25:26,834 - tests.orchestrator - INFO - Waiting for pod test-pod-bf04a30a to be ready +2025-06-05 18:25:26,901 - tests.orchestrator - INFO - Pod test-pod-bf04a30a phase: Pending +2025-06-05 18:25:28,970 - tests.orchestrator - INFO - Pod test-pod-bf04a30a phase: Running +2025-06-05 18:25:28,970 - tests.orchestrator - INFO - Pod test-pod-bf04a30a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:25:31,038 - tests.orchestrator - INFO - Pod test-pod-bf04a30a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:25:33,105 - tests.orchestrator - INFO - Pod test-pod-bf04a30a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:25:35,173 - tests.orchestrator - INFO - Pod test-pod-bf04a30a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:25:37,240 - tests.orchestrator - INFO - Pod test-pod-bf04a30a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:25:39,308 - tests.orchestrator - INFO - Pod test-pod-bf04a30a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:25:41,376 - tests.orchestrator - INFO - Pod test-pod-bf04a30a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:25:43,443 - tests.orchestrator - INFO - Pod test-pod-bf04a30a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:25:45,511 - tests.orchestrator - INFO - Pod test-pod-bf04a30a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:25:47,579 - tests.orchestrator - INFO - Pod test-pod-bf04a30a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 18:25:47,579 - tests.orchestrator - INFO - Pod test-pod-bf04a30a is ready +2025-06-05 18:25:48,579 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 1 times) +2025-06-05 18:25:48,579 - tests.orchestrator - INFO - Creating PVC: test-pvc-5faad9c0 +2025-06-05 18:25:48,657 - tests.orchestrator - INFO - Created PVC: test-pvc-5faad9c0 +2025-06-05 18:25:48,657 - tests.orchestrator - INFO - Waiting for PVC test-pvc-5faad9c0 to be bound +2025-06-05 18:25:50,789 - tests.orchestrator - INFO - PVC test-pvc-5faad9c0 is bound +2025-06-05 18:25:51,789 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 1 times) +2025-06-05 18:25:51,789 - tests.orchestrator - INFO - Deleting PVC: test-pvc-7a09d3a9 +2025-06-05 18:25:51,789 - tests.orchestrator - INFO - Deleting 3 pods using PVC test-pvc-7a09d3a9 +2025-06-05 18:25:51,789 - tests.orchestrator - INFO - Deleting pod: test-pod-09840b6a from PVC: test-pvc-7a09d3a9 +2025-06-05 18:25:51,863 - tests.orchestrator - INFO - Waiting for pod test-pod-09840b6a to be deleted +2025-06-05 18:26:24,469 - tests.orchestrator - INFO - Pod test-pod-09840b6a has been deleted +2025-06-05 18:26:24,469 - tests.orchestrator - INFO - Deleted pod: test-pod-09840b6a +2025-06-05 18:26:24,469 - tests.orchestrator - INFO - Deleting pod: test-pod-d40af63e from PVC: test-pvc-7a09d3a9 +2025-06-05 18:26:24,543 - tests.orchestrator - INFO - Waiting for pod test-pod-d40af63e to be deleted +2025-06-05 18:26:57,150 - tests.orchestrator - INFO - Pod test-pod-d40af63e has been deleted +2025-06-05 18:26:57,150 - tests.orchestrator - INFO - Deleted pod: test-pod-d40af63e +2025-06-05 18:26:57,151 - tests.orchestrator - INFO - Deleting pod: test-pod-bf04a30a from PVC: test-pvc-7a09d3a9 +2025-06-05 18:26:57,225 - tests.orchestrator - INFO - Waiting for pod test-pod-bf04a30a to be deleted +2025-06-05 18:27:29,830 - tests.orchestrator - INFO - Pod test-pod-bf04a30a has been deleted +2025-06-05 18:27:29,830 - tests.orchestrator - INFO - Deleted pod: test-pod-bf04a30a +2025-06-05 18:27:29,900 - tests.orchestrator - INFO - Waiting for PVC test-pvc-7a09d3a9 to be deleted +2025-06-05 18:27:29,967 - tests.orchestrator - INFO - PVC test-pvc-7a09d3a9 has been deleted +2025-06-05 18:27:29,968 - tests.orchestrator - INFO - Deleted PVC: test-pvc-7a09d3a9 +2025-06-05 18:27:30,968 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 3 times) +2025-06-05 18:27:30,968 - tests.orchestrator - INFO - Creating pod test-pod-8c05e159 on PVC test-pvc-5faad9c0 +2025-06-05 18:27:31,056 - tests.orchestrator - INFO - Created pod: test-pod-8c05e159 using PVC: test-pvc-5faad9c0 +2025-06-05 18:27:31,056 - tests.orchestrator - INFO - Waiting for pod test-pod-8c05e159 to be ready +2025-06-05 18:27:31,123 - tests.orchestrator - INFO - Pod test-pod-8c05e159 phase: Pending +2025-06-05 18:27:33,192 - tests.orchestrator - INFO - Pod test-pod-8c05e159 phase: Running +2025-06-05 18:27:33,192 - tests.orchestrator - INFO - Pod test-pod-8c05e159 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:27:35,259 - tests.orchestrator - INFO - Pod test-pod-8c05e159 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:27:37,326 - tests.orchestrator - INFO - Pod test-pod-8c05e159 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:27:39,394 - tests.orchestrator - INFO - Pod test-pod-8c05e159 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:27:41,463 - tests.orchestrator - INFO - Pod test-pod-8c05e159 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:27:43,531 - tests.orchestrator - INFO - Pod test-pod-8c05e159 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:27:45,599 - tests.orchestrator - INFO - Pod test-pod-8c05e159 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:27:47,667 - tests.orchestrator - INFO - Pod test-pod-8c05e159 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:27:49,734 - tests.orchestrator - INFO - Pod test-pod-8c05e159 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:27:51,802 - tests.orchestrator - INFO - Pod test-pod-8c05e159 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 18:27:51,803 - tests.orchestrator - INFO - Pod test-pod-8c05e159 is ready +2025-06-05 18:27:52,803 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 4 times) +2025-06-05 18:27:52,803 - tests.orchestrator - INFO - Creating pod test-pod-7e1c33ef on PVC test-pvc-5faad9c0 +2025-06-05 18:27:52,877 - tests.orchestrator - INFO - Created pod: test-pod-7e1c33ef using PVC: test-pvc-5faad9c0 +2025-06-05 18:27:52,877 - tests.orchestrator - INFO - Waiting for pod test-pod-7e1c33ef to be ready +2025-06-05 18:27:52,943 - tests.orchestrator - INFO - Pod test-pod-7e1c33ef phase: Pending +2025-06-05 18:27:55,011 - tests.orchestrator - INFO - Pod test-pod-7e1c33ef phase: Running +2025-06-05 18:27:55,011 - tests.orchestrator - INFO - Pod test-pod-7e1c33ef conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:27:57,079 - tests.orchestrator - INFO - Pod test-pod-7e1c33ef conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:27:59,146 - tests.orchestrator - INFO - Pod test-pod-7e1c33ef conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:28:01,216 - tests.orchestrator - INFO - Pod test-pod-7e1c33ef conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:28:03,284 - tests.orchestrator - INFO - Pod test-pod-7e1c33ef conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:28:05,352 - tests.orchestrator - INFO - Pod test-pod-7e1c33ef conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:28:07,421 - tests.orchestrator - INFO - Pod test-pod-7e1c33ef conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:28:09,487 - tests.orchestrator - INFO - Pod test-pod-7e1c33ef conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:28:11,563 - tests.orchestrator - INFO - Pod test-pod-7e1c33ef conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:28:13,632 - tests.orchestrator - INFO - Pod test-pod-7e1c33ef conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:28:15,700 - tests.orchestrator - INFO - Pod test-pod-7e1c33ef conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 18:28:15,700 - tests.orchestrator - INFO - Pod test-pod-7e1c33ef is ready +2025-06-05 18:28:16,700 - tests.orchestrator - INFO - Test completed in 322.35 seconds +2025-06-05 18:28:16,701 - tests.orchestrator - INFO - ===== STARTING COMPREHENSIVE CLEANUP ===== +2025-06-05 18:28:16,701 - tests.orchestrator - INFO - Deleting 2 pods (force=False)... +2025-06-05 18:28:16,701 - tests.orchestrator - INFO - Deleting pod: test-pod-8c05e159 from PVC: test-pvc-5faad9c0 +2025-06-05 18:28:16,774 - tests.orchestrator - INFO - Waiting for pod test-pod-8c05e159 to be deleted +2025-06-05 18:28:49,382 - tests.orchestrator - INFO - Pod test-pod-8c05e159 has been deleted +2025-06-05 18:28:49,382 - tests.orchestrator - INFO - Deleted pod: test-pod-8c05e159 +2025-06-05 18:28:49,382 - tests.orchestrator - INFO - Deleting pod: test-pod-7e1c33ef from PVC: test-pvc-5faad9c0 +2025-06-05 18:28:49,455 - tests.orchestrator - INFO - Waiting for pod test-pod-7e1c33ef to be deleted +2025-06-05 18:29:22,059 - tests.orchestrator - INFO - Pod test-pod-7e1c33ef has been deleted +2025-06-05 18:29:22,059 - tests.orchestrator - INFO - Deleted pod: test-pod-7e1c33ef +2025-06-05 18:29:27,059 - tests.orchestrator - INFO - Deleting 1 PVCs (force=False)... +2025-06-05 18:29:27,059 - tests.orchestrator - INFO - Deleting PVC: test-pvc-5faad9c0 +2025-06-05 18:29:27,131 - tests.orchestrator - INFO - Waiting for PVC test-pvc-5faad9c0 to be deleted +2025-06-05 18:29:27,195 - tests.orchestrator - INFO - PVC test-pvc-5faad9c0 has been deleted +2025-06-05 18:29:27,195 - tests.orchestrator - INFO - Deleted PVC: test-pvc-5faad9c0 +2025-06-05 18:29:27,397 - tests.orchestrator - WARNING - First cleanup pass incomplete. Remaining resources: ['pod/test-pod-3cc2d48d', 'pod/test-pod-564344d0', 'pod/test-pod-a9935180', 'pod/test-pod-e1337987', 'pvc/test-pvc-0b5af805', 'pvc/test-pvc-3691f2a0', 'pvc/test-pvc-455abf8b', 'pvc/test-pvc-51dbaa96', 'pvc/test-pvc-601432ce', 'pvc/test-pvc-63ada5ce', 'pvc/test-pvc-64bf8174', 'pvc/test-pvc-d3ee5325'] +2025-06-05 18:29:27,397 - tests.orchestrator - INFO - Attempting force deletion of remaining resources... +2025-06-05 18:29:27,397 - tests.orchestrator - INFO - Deleting 0 pods (force=True)... +2025-06-05 18:29:32,397 - tests.orchestrator - INFO - Deleting 0 PVCs (force=True)... +2025-06-05 18:29:32,600 - tests.orchestrator - ERROR - Cleanup incomplete. Remaining resources after force deletion: ['pod/test-pod-3cc2d48d', 'pod/test-pod-564344d0', 'pod/test-pod-a9935180', 'pod/test-pod-e1337987', 'pvc/test-pvc-0b5af805', 'pvc/test-pvc-3691f2a0', 'pvc/test-pvc-455abf8b', 'pvc/test-pvc-51dbaa96', 'pvc/test-pvc-601432ce', 'pvc/test-pvc-63ada5ce', 'pvc/test-pvc-64bf8174', 'pvc/test-pvc-d3ee5325'] +2025-06-05 18:29:32,600 - tests.orchestrator - INFO - Cleanup completed successfully in 75.90 seconds +2025-06-05 18:29:32,600 - tests.orchestrator - INFO - ===== CLEANUP PROCESS FINISHED ===== +2025-06-05 18:29:32,600 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-05 18:29:32,600 - tests.orchestrator - INFO - --- Operations --- +2025-06-05 18:29:32,600 - tests.orchestrator - INFO - create_pvc: 3 succeeded, 0 failed (100.0%) +2025-06-05 18:29:32,600 - tests.orchestrator - INFO - attach_pod: 7 succeeded, 0 failed (100.0%) +2025-06-05 18:29:32,600 - tests.orchestrator - INFO - delete_pod: 7 succeeded, 0 failed (100.0%) +2025-06-05 18:29:32,600 - tests.orchestrator - INFO - delete_pvc: 3 succeeded, 0 failed (100.0%) +2025-06-05 18:29:32,601 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) +2025-06-05 18:29:32,601 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) +2025-06-05 18:29:32,601 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-05 18:29:32,601 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) +2025-06-05 18:29:32,601 - tests.orchestrator - INFO - many_to_one: No runs +2025-06-05 18:29:32,601 - tests.orchestrator - INFO - one_to_one: No runs +2025-06-05 18:29:32,601 - tests.orchestrator - INFO - concurrent_pvc: No runs +2025-06-05 18:29:32,601 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_183835.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_183835.log new file mode 100644 index 000000000..034a984a2 --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_183835.log @@ -0,0 +1,983 @@ +2025-06-05 18:38:35,776 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 18:38:35,987 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 18:38:35,987 - tests.orchestrator - INFO - Starting orchestrator test for 7200 seconds +2025-06-05 18:38:36,176 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 18:38:36,176 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 18:38:36,176 - tests.orchestrator - INFO - Creating PVC: test-pvc-21c303a0 +2025-06-05 18:38:36,254 - tests.orchestrator - INFO - Created PVC: test-pvc-21c303a0 +2025-06-05 18:38:36,254 - tests.orchestrator - INFO - Waiting for PVC test-pvc-21c303a0 to be bound +2025-06-05 18:38:38,387 - tests.orchestrator - INFO - PVC test-pvc-21c303a0 is bound +2025-06-05 18:38:38,387 - tests.orchestrator - INFO - Creating PVC: test-pvc-7f1f1d08 +2025-06-05 18:38:38,456 - tests.orchestrator - INFO - Created PVC: test-pvc-7f1f1d08 +2025-06-05 18:38:38,456 - tests.orchestrator - INFO - Waiting for PVC test-pvc-7f1f1d08 to be bound +2025-06-05 18:38:40,589 - tests.orchestrator - INFO - PVC test-pvc-7f1f1d08 is bound +2025-06-05 18:38:40,589 - tests.orchestrator - INFO - Creating pod test-pod-b63b288d on PVC test-pvc-21c303a0 +2025-06-05 18:38:40,671 - tests.orchestrator - INFO - Created pod: test-pod-b63b288d using PVC: test-pvc-21c303a0 +2025-06-05 18:38:40,671 - tests.orchestrator - INFO - Waiting for pod test-pod-b63b288d to be ready +2025-06-05 18:38:40,739 - tests.orchestrator - INFO - Pod test-pod-b63b288d phase: Pending +2025-06-05 18:38:44,876 - tests.orchestrator - INFO - Pod test-pod-b63b288d phase: Running +2025-06-05 18:38:44,876 - tests.orchestrator - INFO - Pod test-pod-b63b288d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:38:46,944 - tests.orchestrator - INFO - Pod test-pod-b63b288d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:38:49,012 - tests.orchestrator - INFO - Pod test-pod-b63b288d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:38:51,080 - tests.orchestrator - INFO - Pod test-pod-b63b288d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:38:53,150 - tests.orchestrator - INFO - Pod test-pod-b63b288d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:38:55,221 - tests.orchestrator - INFO - Pod test-pod-b63b288d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:38:57,289 - tests.orchestrator - INFO - Pod test-pod-b63b288d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:38:59,357 - tests.orchestrator - INFO - Pod test-pod-b63b288d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:39:01,428 - tests.orchestrator - INFO - Pod test-pod-b63b288d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 18:39:01,428 - tests.orchestrator - INFO - Pod test-pod-b63b288d is ready +2025-06-05 18:39:01,428 - tests.orchestrator - INFO - Creating pod test-pod-4deff76d on PVC test-pvc-21c303a0 +2025-06-05 18:39:01,570 - tests.orchestrator - INFO - Created pod: test-pod-4deff76d using PVC: test-pvc-21c303a0 +2025-06-05 18:39:01,570 - tests.orchestrator - INFO - Waiting for pod test-pod-4deff76d to be ready +2025-06-05 18:39:01,694 - tests.orchestrator - INFO - Pod test-pod-4deff76d phase: Pending +2025-06-05 18:39:03,763 - tests.orchestrator - INFO - Pod test-pod-4deff76d phase: Running +2025-06-05 18:39:03,763 - tests.orchestrator - INFO - Pod test-pod-4deff76d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:39:05,831 - tests.orchestrator - INFO - Pod test-pod-4deff76d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:39:07,901 - tests.orchestrator - INFO - Pod test-pod-4deff76d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:39:09,970 - tests.orchestrator - INFO - Pod test-pod-4deff76d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:39:12,038 - tests.orchestrator - INFO - Pod test-pod-4deff76d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:39:14,107 - tests.orchestrator - INFO - Pod test-pod-4deff76d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:39:16,176 - tests.orchestrator - INFO - Pod test-pod-4deff76d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:39:18,244 - tests.orchestrator - INFO - Pod test-pod-4deff76d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 18:39:18,244 - tests.orchestrator - INFO - Pod test-pod-4deff76d is ready +2025-06-05 18:39:18,245 - tests.orchestrator - INFO - Creating pod test-pod-674c680f on PVC test-pvc-21c303a0 +2025-06-05 18:39:18,327 - tests.orchestrator - INFO - Created pod: test-pod-674c680f using PVC: test-pvc-21c303a0 +2025-06-05 18:39:18,327 - tests.orchestrator - INFO - Waiting for pod test-pod-674c680f to be ready +2025-06-05 18:39:18,395 - tests.orchestrator - INFO - Pod test-pod-674c680f phase: Pending +2025-06-05 18:39:20,464 - tests.orchestrator - INFO - Pod test-pod-674c680f phase: Running +2025-06-05 18:39:20,464 - tests.orchestrator - INFO - Pod test-pod-674c680f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:39:22,532 - tests.orchestrator - INFO - Pod test-pod-674c680f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:39:24,601 - tests.orchestrator - INFO - Pod test-pod-674c680f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:39:26,669 - tests.orchestrator - INFO - Pod test-pod-674c680f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:39:28,737 - tests.orchestrator - INFO - Pod test-pod-674c680f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:39:30,806 - tests.orchestrator - INFO - Pod test-pod-674c680f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:39:32,874 - tests.orchestrator - INFO - Pod test-pod-674c680f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:39:34,942 - tests.orchestrator - INFO - Pod test-pod-674c680f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:39:37,011 - tests.orchestrator - INFO - Pod test-pod-674c680f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:39:39,080 - tests.orchestrator - INFO - Pod test-pod-674c680f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 18:39:39,080 - tests.orchestrator - INFO - Pod test-pod-674c680f is ready +2025-06-05 18:39:39,081 - tests.orchestrator - INFO - Testing read/write between pods test-pod-4deff76d and test-pod-674c680f sharing PVC test-pvc-21c303a0 +2025-06-05 18:39:39,150 - tests.orchestrator - ERROR - Failed in read/write verification: (400) +Reason: Bad Request +HTTP response headers: HTTPHeaderDict({'Audit-Id': '6945d221-5c2e-4401-a73f-a03b2205822c', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 18:39:39 GMT', 'Content-Length': '139'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Upgrade request required","reason":"BadRequest","code":400} + + +2025-06-05 18:39:39,150 - tests.orchestrator - INFO - Deleting pod: test-pod-b63b288d from PVC: test-pvc-21c303a0 +2025-06-05 18:39:39,226 - tests.orchestrator - INFO - Waiting for pod test-pod-b63b288d to be deleted +2025-06-05 18:40:11,843 - tests.orchestrator - INFO - Pod test-pod-b63b288d has been deleted +2025-06-05 18:40:11,844 - tests.orchestrator - INFO - Deleted pod: test-pod-b63b288d +2025-06-05 18:40:11,844 - tests.orchestrator - INFO - Deleting PVC: test-pvc-7f1f1d08 +2025-06-05 18:40:11,916 - tests.orchestrator - INFO - Waiting for PVC test-pvc-7f1f1d08 to be deleted +2025-06-05 18:40:16,049 - tests.orchestrator - INFO - PVC test-pvc-7f1f1d08 has been deleted +2025-06-05 18:40:16,050 - tests.orchestrator - INFO - Deleted PVC: test-pvc-7f1f1d08 +2025-06-05 18:40:16,050 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations +2025-06-05 18:40:16,050 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 1 times) +2025-06-05 18:40:16,050 - tests.orchestrator - INFO - Creating pod test-pod-ebc67d81 on PVC test-pvc-21c303a0 +2025-06-05 18:40:16,133 - tests.orchestrator - INFO - Created pod: test-pod-ebc67d81 using PVC: test-pvc-21c303a0 +2025-06-05 18:40:16,133 - tests.orchestrator - INFO - Waiting for pod test-pod-ebc67d81 to be ready +2025-06-05 18:40:16,200 - tests.orchestrator - INFO - Pod test-pod-ebc67d81 phase: Pending +2025-06-05 18:40:20,337 - tests.orchestrator - INFO - Pod test-pod-ebc67d81 phase: Running +2025-06-05 18:40:20,337 - tests.orchestrator - INFO - Pod test-pod-ebc67d81 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:40:22,404 - tests.orchestrator - INFO - Pod test-pod-ebc67d81 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:40:24,473 - tests.orchestrator - INFO - Pod test-pod-ebc67d81 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:40:26,548 - tests.orchestrator - INFO - Pod test-pod-ebc67d81 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:40:28,616 - tests.orchestrator - INFO - Pod test-pod-ebc67d81 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:40:30,684 - tests.orchestrator - INFO - Pod test-pod-ebc67d81 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:40:32,753 - tests.orchestrator - INFO - Pod test-pod-ebc67d81 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:40:34,822 - tests.orchestrator - INFO - Pod test-pod-ebc67d81 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:40:36,891 - tests.orchestrator - INFO - Pod test-pod-ebc67d81 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 18:40:36,891 - tests.orchestrator - INFO - Pod test-pod-ebc67d81 is ready +2025-06-05 18:40:37,891 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 1 times) +2025-06-05 18:40:37,891 - tests.orchestrator - INFO - Deleting pod: test-pod-ebc67d81 from PVC: test-pvc-21c303a0 +2025-06-05 18:40:37,968 - tests.orchestrator - INFO - Waiting for pod test-pod-ebc67d81 to be deleted +2025-06-05 18:41:10,581 - tests.orchestrator - INFO - Pod test-pod-ebc67d81 has been deleted +2025-06-05 18:41:10,581 - tests.orchestrator - INFO - Deleted pod: test-pod-ebc67d81 +2025-06-05 18:41:11,581 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 1 times) +2025-06-05 18:41:11,582 - tests.orchestrator - INFO - Creating PVC: test-pvc-9c78a2d7 +2025-06-05 18:41:11,661 - tests.orchestrator - INFO - Created PVC: test-pvc-9c78a2d7 +2025-06-05 18:41:11,661 - tests.orchestrator - INFO - Waiting for PVC test-pvc-9c78a2d7 to be bound +2025-06-05 18:41:13,794 - tests.orchestrator - INFO - PVC test-pvc-9c78a2d7 is bound +2025-06-05 18:41:14,795 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 2 times) +2025-06-05 18:41:14,795 - tests.orchestrator - INFO - Deleting pod: test-pod-4deff76d from PVC: test-pvc-21c303a0 +2025-06-05 18:41:14,871 - tests.orchestrator - INFO - Waiting for pod test-pod-4deff76d to be deleted +2025-06-05 18:41:47,485 - tests.orchestrator - INFO - Pod test-pod-4deff76d has been deleted +2025-06-05 18:41:47,485 - tests.orchestrator - INFO - Deleted pod: test-pod-4deff76d +2025-06-05 18:41:48,485 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 1 times) +2025-06-05 18:41:48,485 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test +2025-06-05 18:41:49,485 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 3 times) +2025-06-05 18:41:49,485 - tests.orchestrator - INFO - Deleting pod: test-pod-674c680f from PVC: test-pvc-21c303a0 +2025-06-05 18:41:49,560 - tests.orchestrator - INFO - Waiting for pod test-pod-674c680f to be deleted +2025-06-05 18:42:22,181 - tests.orchestrator - INFO - Pod test-pod-674c680f has been deleted +2025-06-05 18:42:22,181 - tests.orchestrator - INFO - Deleted pod: test-pod-674c680f +2025-06-05 18:42:23,181 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 1 times) +2025-06-05 18:42:23,181 - tests.orchestrator - INFO - Deleting PVC: test-pvc-21c303a0 +2025-06-05 18:42:23,255 - tests.orchestrator - INFO - Waiting for PVC test-pvc-21c303a0 to be deleted +2025-06-05 18:42:23,320 - tests.orchestrator - INFO - PVC test-pvc-21c303a0 has been deleted +2025-06-05 18:42:23,320 - tests.orchestrator - INFO - Deleted PVC: test-pvc-21c303a0 +2025-06-05 18:42:24,320 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 2 times) +2025-06-05 18:42:24,320 - tests.orchestrator - INFO - Creating PVC: test-pvc-f072232d +2025-06-05 18:42:24,396 - tests.orchestrator - INFO - Created PVC: test-pvc-f072232d +2025-06-05 18:42:24,396 - tests.orchestrator - INFO - Waiting for PVC test-pvc-f072232d to be bound +2025-06-05 18:42:26,534 - tests.orchestrator - INFO - PVC test-pvc-f072232d is bound +2025-06-05 18:42:27,534 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 3 times) +2025-06-05 18:42:27,534 - tests.orchestrator - INFO - Creating PVC: test-pvc-5da89758 +2025-06-05 18:42:27,603 - tests.orchestrator - INFO - Created PVC: test-pvc-5da89758 +2025-06-05 18:42:27,603 - tests.orchestrator - INFO - Waiting for PVC test-pvc-5da89758 to be bound +2025-06-05 18:42:29,737 - tests.orchestrator - INFO - PVC test-pvc-5da89758 is bound +2025-06-05 18:42:30,737 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 4 times) +2025-06-05 18:42:30,737 - tests.orchestrator - INFO - Creating PVC: test-pvc-24af11bc +2025-06-05 18:42:30,806 - tests.orchestrator - INFO - Created PVC: test-pvc-24af11bc +2025-06-05 18:42:30,806 - tests.orchestrator - INFO - Waiting for PVC test-pvc-24af11bc to be bound +2025-06-05 18:42:32,939 - tests.orchestrator - INFO - PVC test-pvc-24af11bc is bound +2025-06-05 18:42:33,939 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 4 times) +2025-06-05 18:42:33,939 - tests.orchestrator - INFO - No pods to delete +2025-06-05 18:42:34,939 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 5 times) +2025-06-05 18:42:34,939 - tests.orchestrator - INFO - Creating PVC: test-pvc-29a67853 +2025-06-05 18:42:35,009 - tests.orchestrator - INFO - Created PVC: test-pvc-29a67853 +2025-06-05 18:42:35,010 - tests.orchestrator - INFO - Waiting for PVC test-pvc-29a67853 to be bound +2025-06-05 18:42:37,142 - tests.orchestrator - INFO - PVC test-pvc-29a67853 is bound +2025-06-05 18:42:38,142 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 5 times) +2025-06-05 18:42:38,142 - tests.orchestrator - INFO - No pods to delete +2025-06-05 18:42:39,142 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 6 times) +2025-06-05 18:42:39,143 - tests.orchestrator - INFO - Creating PVC: test-pvc-0b86173e +2025-06-05 18:42:39,212 - tests.orchestrator - INFO - Created PVC: test-pvc-0b86173e +2025-06-05 18:42:39,212 - tests.orchestrator - INFO - Waiting for PVC test-pvc-0b86173e to be bound +2025-06-05 18:42:41,345 - tests.orchestrator - INFO - PVC test-pvc-0b86173e is bound +2025-06-05 18:42:42,345 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 6 times) +2025-06-05 18:42:42,346 - tests.orchestrator - INFO - No pods to delete +2025-06-05 18:42:43,346 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 2 times) +2025-06-05 18:42:43,346 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test +2025-06-05 18:42:44,346 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 3 times) +2025-06-05 18:42:44,346 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test +2025-06-05 18:42:45,346 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 7 times) +2025-06-05 18:42:45,347 - tests.orchestrator - INFO - Creating PVC: test-pvc-41d09b77 +2025-06-05 18:42:45,416 - tests.orchestrator - INFO - Created PVC: test-pvc-41d09b77 +2025-06-05 18:42:45,416 - tests.orchestrator - INFO - Waiting for PVC test-pvc-41d09b77 to be bound +2025-06-05 18:42:47,550 - tests.orchestrator - INFO - PVC test-pvc-41d09b77 is bound +2025-06-05 18:42:48,550 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 8 times) +2025-06-05 18:42:48,551 - tests.orchestrator - INFO - Creating PVC: test-pvc-f1842f61 +2025-06-05 18:42:48,618 - tests.orchestrator - INFO - Created PVC: test-pvc-f1842f61 +2025-06-05 18:42:48,619 - tests.orchestrator - INFO - Waiting for PVC test-pvc-f1842f61 to be bound +2025-06-05 18:42:50,752 - tests.orchestrator - INFO - PVC test-pvc-f1842f61 is bound +2025-06-05 18:42:51,753 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 7 times) +2025-06-05 18:42:51,753 - tests.orchestrator - INFO - No pods to delete +2025-06-05 18:42:52,753 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 8 times) +2025-06-05 18:42:52,753 - tests.orchestrator - INFO - No pods to delete +2025-06-05 18:42:53,753 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 2 times) +2025-06-05 18:42:53,754 - tests.orchestrator - INFO - Creating pod test-pod-e254da33 on PVC test-pvc-9c78a2d7 +2025-06-05 18:42:53,834 - tests.orchestrator - INFO - Created pod: test-pod-e254da33 using PVC: test-pvc-9c78a2d7 +2025-06-05 18:42:53,834 - tests.orchestrator - INFO - Waiting for pod test-pod-e254da33 to be ready +2025-06-05 18:42:53,902 - tests.orchestrator - INFO - Pod test-pod-e254da33 phase: Pending +2025-06-05 18:42:55,971 - tests.orchestrator - INFO - Pod test-pod-e254da33 phase: Running +2025-06-05 18:42:55,971 - tests.orchestrator - INFO - Pod test-pod-e254da33 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:42:58,039 - tests.orchestrator - INFO - Pod test-pod-e254da33 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:43:00,107 - tests.orchestrator - INFO - Pod test-pod-e254da33 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:43:02,185 - tests.orchestrator - INFO - Pod test-pod-e254da33 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:43:04,253 - tests.orchestrator - INFO - Pod test-pod-e254da33 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:43:06,322 - tests.orchestrator - INFO - Pod test-pod-e254da33 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:43:08,391 - tests.orchestrator - INFO - Pod test-pod-e254da33 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:43:10,459 - tests.orchestrator - INFO - Pod test-pod-e254da33 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:43:12,527 - tests.orchestrator - INFO - Pod test-pod-e254da33 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:43:14,596 - tests.orchestrator - INFO - Pod test-pod-e254da33 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 18:43:14,596 - tests.orchestrator - INFO - Pod test-pod-e254da33 is ready +2025-06-05 18:43:15,596 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 9 times) +2025-06-05 18:43:15,596 - tests.orchestrator - INFO - Creating PVC: test-pvc-ded65224 +2025-06-05 18:43:15,673 - tests.orchestrator - INFO - Created PVC: test-pvc-ded65224 +2025-06-05 18:43:15,674 - tests.orchestrator - INFO - Waiting for PVC test-pvc-ded65224 to be bound +2025-06-05 18:43:17,807 - tests.orchestrator - INFO - PVC test-pvc-ded65224 is bound +2025-06-05 18:43:18,807 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 2 times) +2025-06-05 18:43:18,807 - tests.orchestrator - INFO - Deleting PVC: test-pvc-f072232d +2025-06-05 18:43:18,881 - tests.orchestrator - INFO - Waiting for PVC test-pvc-f072232d to be deleted +2025-06-05 18:43:18,946 - tests.orchestrator - INFO - PVC test-pvc-f072232d has been deleted +2025-06-05 18:43:18,946 - tests.orchestrator - INFO - Deleted PVC: test-pvc-f072232d +2025-06-05 18:43:19,946 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 9 times) +2025-06-05 18:43:19,947 - tests.orchestrator - INFO - Deleting pod: test-pod-e254da33 from PVC: test-pvc-9c78a2d7 +2025-06-05 18:43:20,021 - tests.orchestrator - INFO - Waiting for pod test-pod-e254da33 to be deleted +2025-06-05 18:43:52,636 - tests.orchestrator - INFO - Pod test-pod-e254da33 has been deleted +2025-06-05 18:43:52,637 - tests.orchestrator - INFO - Deleted pod: test-pod-e254da33 +2025-06-05 18:43:53,637 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 3 times) +2025-06-05 18:43:53,637 - tests.orchestrator - INFO - Deleting PVC: test-pvc-ded65224 +2025-06-05 18:43:53,712 - tests.orchestrator - INFO - Waiting for PVC test-pvc-ded65224 to be deleted +2025-06-05 18:43:53,778 - tests.orchestrator - INFO - PVC test-pvc-ded65224 has been deleted +2025-06-05 18:43:53,778 - tests.orchestrator - INFO - Deleted PVC: test-pvc-ded65224 +2025-06-05 18:43:54,778 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 4 times) +2025-06-05 18:43:54,778 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test +2025-06-05 18:43:55,778 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 10 times) +2025-06-05 18:43:55,778 - tests.orchestrator - INFO - No pods to delete +2025-06-05 18:43:56,779 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 3 times) +2025-06-05 18:43:56,779 - tests.orchestrator - INFO - Creating pod test-pod-c065ef92 on PVC test-pvc-29a67853 +2025-06-05 18:43:56,863 - tests.orchestrator - INFO - Created pod: test-pod-c065ef92 using PVC: test-pvc-29a67853 +2025-06-05 18:43:56,863 - tests.orchestrator - INFO - Waiting for pod test-pod-c065ef92 to be ready +2025-06-05 18:43:56,931 - tests.orchestrator - INFO - Pod test-pod-c065ef92 phase: Pending +2025-06-05 18:43:58,999 - tests.orchestrator - INFO - Pod test-pod-c065ef92 phase: Running +2025-06-05 18:43:58,999 - tests.orchestrator - INFO - Pod test-pod-c065ef92 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:44:01,068 - tests.orchestrator - INFO - Pod test-pod-c065ef92 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:44:03,137 - tests.orchestrator - INFO - Pod test-pod-c065ef92 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:44:05,205 - tests.orchestrator - INFO - Pod test-pod-c065ef92 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:44:07,273 - tests.orchestrator - INFO - Pod test-pod-c065ef92 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:44:09,342 - tests.orchestrator - INFO - Pod test-pod-c065ef92 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:44:11,410 - tests.orchestrator - INFO - Pod test-pod-c065ef92 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:44:13,478 - tests.orchestrator - INFO - Pod test-pod-c065ef92 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:44:15,546 - tests.orchestrator - INFO - Pod test-pod-c065ef92 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:44:17,614 - tests.orchestrator - INFO - Pod test-pod-c065ef92 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 18:44:17,614 - tests.orchestrator - INFO - Pod test-pod-c065ef92 is ready +2025-06-05 18:44:18,614 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 4 times) +2025-06-05 18:44:18,614 - tests.orchestrator - INFO - Deleting PVC: test-pvc-5da89758 +2025-06-05 18:44:18,686 - tests.orchestrator - INFO - Waiting for PVC test-pvc-5da89758 to be deleted +2025-06-05 18:44:18,753 - tests.orchestrator - INFO - PVC test-pvc-5da89758 has been deleted +2025-06-05 18:44:18,753 - tests.orchestrator - INFO - Deleted PVC: test-pvc-5da89758 +2025-06-05 18:44:19,753 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 5 times) +2025-06-05 18:44:19,753 - tests.orchestrator - INFO - Deleting PVC: test-pvc-24af11bc +2025-06-05 18:44:19,824 - tests.orchestrator - INFO - Waiting for PVC test-pvc-24af11bc to be deleted +2025-06-05 18:44:19,890 - tests.orchestrator - INFO - PVC test-pvc-24af11bc has been deleted +2025-06-05 18:44:19,890 - tests.orchestrator - INFO - Deleted PVC: test-pvc-24af11bc +2025-06-05 18:44:20,890 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 4 times) +2025-06-05 18:44:20,891 - tests.orchestrator - INFO - Creating pod test-pod-b81d1ccb on PVC test-pvc-29a67853 +2025-06-05 18:44:20,965 - tests.orchestrator - INFO - Created pod: test-pod-b81d1ccb using PVC: test-pvc-29a67853 +2025-06-05 18:44:20,965 - tests.orchestrator - INFO - Waiting for pod test-pod-b81d1ccb to be ready +2025-06-05 18:44:21,033 - tests.orchestrator - INFO - Pod test-pod-b81d1ccb phase: Pending +2025-06-05 18:44:23,101 - tests.orchestrator - INFO - Pod test-pod-b81d1ccb phase: Running +2025-06-05 18:44:23,101 - tests.orchestrator - INFO - Pod test-pod-b81d1ccb conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:44:25,170 - tests.orchestrator - INFO - Pod test-pod-b81d1ccb conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:44:27,240 - tests.orchestrator - INFO - Pod test-pod-b81d1ccb conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:44:29,307 - tests.orchestrator - INFO - Pod test-pod-b81d1ccb conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:44:31,375 - tests.orchestrator - INFO - Pod test-pod-b81d1ccb conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:44:33,444 - tests.orchestrator - INFO - Pod test-pod-b81d1ccb conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:44:35,512 - tests.orchestrator - INFO - Pod test-pod-b81d1ccb conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:44:37,580 - tests.orchestrator - INFO - Pod test-pod-b81d1ccb conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:44:39,649 - tests.orchestrator - INFO - Pod test-pod-b81d1ccb conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:44:41,717 - tests.orchestrator - INFO - Pod test-pod-b81d1ccb conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:44:43,785 - tests.orchestrator - INFO - Pod test-pod-b81d1ccb conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 18:44:43,785 - tests.orchestrator - INFO - Pod test-pod-b81d1ccb is ready +2025-06-05 18:44:44,785 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 10 times) +2025-06-05 18:44:44,785 - tests.orchestrator - INFO - Creating PVC: test-pvc-60b59274 +2025-06-05 18:44:44,862 - tests.orchestrator - INFO - Created PVC: test-pvc-60b59274 +2025-06-05 18:44:44,862 - tests.orchestrator - INFO - Waiting for PVC test-pvc-60b59274 to be bound +2025-06-05 18:44:46,995 - tests.orchestrator - INFO - PVC test-pvc-60b59274 is bound +2025-06-05 18:44:47,995 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 5 times) +2025-06-05 18:44:47,996 - tests.orchestrator - INFO - Testing read/write between pods test-pod-b81d1ccb and test-pod-c065ef92 sharing PVC test-pvc-29a67853 +2025-06-05 18:44:48,066 - tests.orchestrator - ERROR - Failed in read/write verification: (400) +Reason: Bad Request +HTTP response headers: HTTPHeaderDict({'Audit-Id': 'a677ce6d-d92f-45e9-89ad-e0bdd2abd606', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 18:44:48 GMT', 'Content-Length': '139'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Upgrade request required","reason":"BadRequest","code":400} + + +2025-06-05 18:44:49,066 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 11 times) +2025-06-05 18:44:49,066 - tests.orchestrator - INFO - Deleting pod: test-pod-c065ef92 from PVC: test-pvc-29a67853 +2025-06-05 18:44:49,141 - tests.orchestrator - INFO - Waiting for pod test-pod-c065ef92 to be deleted +2025-06-05 18:45:21,752 - tests.orchestrator - INFO - Pod test-pod-c065ef92 has been deleted +2025-06-05 18:45:21,752 - tests.orchestrator - INFO - Deleted pod: test-pod-c065ef92 +2025-06-05 18:45:22,753 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 6 times) +2025-06-05 18:45:22,754 - tests.orchestrator - INFO - Deleting PVC: test-pvc-0b86173e +2025-06-05 18:45:22,826 - tests.orchestrator - INFO - Waiting for PVC test-pvc-0b86173e to be deleted +2025-06-05 18:45:22,893 - tests.orchestrator - INFO - PVC test-pvc-0b86173e has been deleted +2025-06-05 18:45:22,893 - tests.orchestrator - INFO - Deleted PVC: test-pvc-0b86173e +2025-06-05 18:45:23,893 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 5 times) +2025-06-05 18:45:23,893 - tests.orchestrator - INFO - Creating pod test-pod-077370c2 on PVC test-pvc-41d09b77 +2025-06-05 18:45:23,976 - tests.orchestrator - INFO - Created pod: test-pod-077370c2 using PVC: test-pvc-41d09b77 +2025-06-05 18:45:23,976 - tests.orchestrator - INFO - Waiting for pod test-pod-077370c2 to be ready +2025-06-05 18:45:24,044 - tests.orchestrator - INFO - Pod test-pod-077370c2 phase: Pending +2025-06-05 18:45:26,113 - tests.orchestrator - INFO - Pod test-pod-077370c2 phase: Running +2025-06-05 18:45:26,113 - tests.orchestrator - INFO - Pod test-pod-077370c2 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:45:28,181 - tests.orchestrator - INFO - Pod test-pod-077370c2 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:45:30,250 - tests.orchestrator - INFO - Pod test-pod-077370c2 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:45:32,319 - tests.orchestrator - INFO - Pod test-pod-077370c2 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:45:34,387 - tests.orchestrator - INFO - Pod test-pod-077370c2 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:45:36,454 - tests.orchestrator - INFO - Pod test-pod-077370c2 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:45:38,523 - tests.orchestrator - INFO - Pod test-pod-077370c2 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:45:40,591 - tests.orchestrator - INFO - Pod test-pod-077370c2 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:45:42,666 - tests.orchestrator - INFO - Pod test-pod-077370c2 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:45:44,734 - tests.orchestrator - INFO - Pod test-pod-077370c2 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 18:45:44,734 - tests.orchestrator - INFO - Pod test-pod-077370c2 is ready +2025-06-05 18:45:45,735 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 6 times) +2025-06-05 18:45:45,735 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test +2025-06-05 18:45:46,735 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 12 times) +2025-06-05 18:45:46,735 - tests.orchestrator - INFO - Deleting pod: test-pod-b81d1ccb from PVC: test-pvc-29a67853 +2025-06-05 18:45:46,810 - tests.orchestrator - INFO - Waiting for pod test-pod-b81d1ccb to be deleted +2025-06-05 18:46:19,436 - tests.orchestrator - INFO - Pod test-pod-b81d1ccb has been deleted +2025-06-05 18:46:19,436 - tests.orchestrator - INFO - Deleted pod: test-pod-b81d1ccb +2025-06-05 18:46:20,437 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 7 times) +2025-06-05 18:46:20,437 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test +2025-06-05 18:46:21,437 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 11 times) +2025-06-05 18:46:21,437 - tests.orchestrator - INFO - Creating PVC: test-pvc-33a8ceec +2025-06-05 18:46:21,523 - tests.orchestrator - INFO - Created PVC: test-pvc-33a8ceec +2025-06-05 18:46:21,523 - tests.orchestrator - INFO - Waiting for PVC test-pvc-33a8ceec to be bound +2025-06-05 18:46:23,658 - tests.orchestrator - INFO - PVC test-pvc-33a8ceec is bound +2025-06-05 18:46:24,658 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 6 times) +2025-06-05 18:46:24,658 - tests.orchestrator - INFO - Creating pod test-pod-f1a6e117 on PVC test-pvc-9c78a2d7 +2025-06-05 18:46:24,733 - tests.orchestrator - INFO - Created pod: test-pod-f1a6e117 using PVC: test-pvc-9c78a2d7 +2025-06-05 18:46:24,734 - tests.orchestrator - INFO - Waiting for pod test-pod-f1a6e117 to be ready +2025-06-05 18:46:24,801 - tests.orchestrator - INFO - Pod test-pod-f1a6e117 phase: Pending +2025-06-05 18:46:28,938 - tests.orchestrator - INFO - Pod test-pod-f1a6e117 phase: Running +2025-06-05 18:46:28,938 - tests.orchestrator - INFO - Pod test-pod-f1a6e117 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:46:31,006 - tests.orchestrator - INFO - Pod test-pod-f1a6e117 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:46:33,074 - tests.orchestrator - INFO - Pod test-pod-f1a6e117 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:46:35,143 - tests.orchestrator - INFO - Pod test-pod-f1a6e117 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:46:37,211 - tests.orchestrator - INFO - Pod test-pod-f1a6e117 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:46:39,279 - tests.orchestrator - INFO - Pod test-pod-f1a6e117 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:46:41,348 - tests.orchestrator - INFO - Pod test-pod-f1a6e117 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:46:43,417 - tests.orchestrator - INFO - Pod test-pod-f1a6e117 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:46:45,485 - tests.orchestrator - INFO - Pod test-pod-f1a6e117 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 18:46:45,485 - tests.orchestrator - INFO - Pod test-pod-f1a6e117 is ready +2025-06-05 18:46:46,485 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 7 times) +2025-06-05 18:46:46,486 - tests.orchestrator - INFO - Creating pod test-pod-ba1a0549 on PVC test-pvc-41d09b77 +2025-06-05 18:46:46,563 - tests.orchestrator - INFO - Created pod: test-pod-ba1a0549 using PVC: test-pvc-41d09b77 +2025-06-05 18:46:46,563 - tests.orchestrator - INFO - Waiting for pod test-pod-ba1a0549 to be ready +2025-06-05 18:46:46,632 - tests.orchestrator - INFO - Pod test-pod-ba1a0549 phase: Pending +2025-06-05 18:46:48,701 - tests.orchestrator - INFO - Pod test-pod-ba1a0549 phase: Running +2025-06-05 18:46:48,701 - tests.orchestrator - INFO - Pod test-pod-ba1a0549 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:46:50,769 - tests.orchestrator - INFO - Pod test-pod-ba1a0549 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:46:52,838 - tests.orchestrator - INFO - Pod test-pod-ba1a0549 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:46:54,906 - tests.orchestrator - INFO - Pod test-pod-ba1a0549 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:46:56,974 - tests.orchestrator - INFO - Pod test-pod-ba1a0549 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:46:59,043 - tests.orchestrator - INFO - Pod test-pod-ba1a0549 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:47:01,111 - tests.orchestrator - INFO - Pod test-pod-ba1a0549 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:47:03,180 - tests.orchestrator - INFO - Pod test-pod-ba1a0549 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 18:47:03,180 - tests.orchestrator - INFO - Pod test-pod-ba1a0549 is ready +2025-06-05 18:47:04,180 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 8 times) +2025-06-05 18:47:04,180 - tests.orchestrator - INFO - Testing read/write between pods test-pod-077370c2 and test-pod-ba1a0549 sharing PVC test-pvc-41d09b77 +2025-06-05 18:47:04,252 - tests.orchestrator - ERROR - Failed in read/write verification: (400) +Reason: Bad Request +HTTP response headers: HTTPHeaderDict({'Audit-Id': 'f65371f5-232a-415d-a016-435e1f074adf', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 18:47:04 GMT', 'Content-Length': '139'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Upgrade request required","reason":"BadRequest","code":400} + + +2025-06-05 18:47:05,253 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 13 times) +2025-06-05 18:47:05,253 - tests.orchestrator - INFO - Deleting pod: test-pod-ba1a0549 from PVC: test-pvc-41d09b77 +2025-06-05 18:47:05,328 - tests.orchestrator - INFO - Waiting for pod test-pod-ba1a0549 to be deleted +2025-06-05 18:47:38,669 - tests.orchestrator - INFO - Pod test-pod-ba1a0549 has been deleted +2025-06-05 18:47:38,669 - tests.orchestrator - INFO - Deleted pod: test-pod-ba1a0549 +2025-06-05 18:47:39,669 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 14 times) +2025-06-05 18:47:39,670 - tests.orchestrator - INFO - Deleting pod: test-pod-077370c2 from PVC: test-pvc-41d09b77 +2025-06-05 18:47:39,746 - tests.orchestrator - INFO - Waiting for pod test-pod-077370c2 to be deleted +2025-06-05 18:48:12,360 - tests.orchestrator - INFO - Pod test-pod-077370c2 has been deleted +2025-06-05 18:48:12,360 - tests.orchestrator - INFO - Deleted pod: test-pod-077370c2 +2025-06-05 18:48:13,360 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 15 times) +2025-06-05 18:48:13,360 - tests.orchestrator - INFO - Deleting pod: test-pod-f1a6e117 from PVC: test-pvc-9c78a2d7 +2025-06-05 18:48:13,437 - tests.orchestrator - INFO - Waiting for pod test-pod-f1a6e117 to be deleted +2025-06-05 18:48:46,049 - tests.orchestrator - INFO - Pod test-pod-f1a6e117 has been deleted +2025-06-05 18:48:46,049 - tests.orchestrator - INFO - Deleted pod: test-pod-f1a6e117 +2025-06-05 18:48:47,049 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 9 times) +2025-06-05 18:48:47,050 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test +2025-06-05 18:48:48,050 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 7 times) +2025-06-05 18:48:48,050 - tests.orchestrator - INFO - Deleting PVC: test-pvc-9c78a2d7 +2025-06-05 18:48:48,123 - tests.orchestrator - INFO - Waiting for PVC test-pvc-9c78a2d7 to be deleted +2025-06-05 18:48:48,188 - tests.orchestrator - INFO - PVC test-pvc-9c78a2d7 has been deleted +2025-06-05 18:48:48,189 - tests.orchestrator - INFO - Deleted PVC: test-pvc-9c78a2d7 +2025-06-05 18:48:49,189 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 8 times) +2025-06-05 18:48:49,189 - tests.orchestrator - INFO - Creating pod test-pod-f445578c on PVC test-pvc-29a67853 +2025-06-05 18:48:49,275 - tests.orchestrator - INFO - Created pod: test-pod-f445578c using PVC: test-pvc-29a67853 +2025-06-05 18:48:49,275 - tests.orchestrator - INFO - Waiting for pod test-pod-f445578c to be ready +2025-06-05 18:48:49,343 - tests.orchestrator - INFO - Pod test-pod-f445578c phase: Pending +2025-06-05 18:48:51,411 - tests.orchestrator - INFO - Pod test-pod-f445578c phase: Running +2025-06-05 18:48:51,412 - tests.orchestrator - INFO - Pod test-pod-f445578c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:48:53,479 - tests.orchestrator - INFO - Pod test-pod-f445578c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:48:55,548 - tests.orchestrator - INFO - Pod test-pod-f445578c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:48:57,617 - tests.orchestrator - INFO - Pod test-pod-f445578c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:48:59,684 - tests.orchestrator - INFO - Pod test-pod-f445578c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:49:01,806 - tests.orchestrator - INFO - Pod test-pod-f445578c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:49:03,875 - tests.orchestrator - INFO - Pod test-pod-f445578c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:49:05,943 - tests.orchestrator - INFO - Pod test-pod-f445578c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:49:08,011 - tests.orchestrator - INFO - Pod test-pod-f445578c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:49:10,080 - tests.orchestrator - INFO - Pod test-pod-f445578c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 18:49:10,080 - tests.orchestrator - INFO - Pod test-pod-f445578c is ready +2025-06-05 18:49:11,080 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 12 times) +2025-06-05 18:49:11,080 - tests.orchestrator - INFO - Creating PVC: test-pvc-6e8e6345 +2025-06-05 18:49:11,150 - tests.orchestrator - INFO - Created PVC: test-pvc-6e8e6345 +2025-06-05 18:49:11,150 - tests.orchestrator - INFO - Waiting for PVC test-pvc-6e8e6345 to be bound +2025-06-05 18:49:13,283 - tests.orchestrator - INFO - PVC test-pvc-6e8e6345 is bound +2025-06-05 18:49:14,283 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 9 times) +2025-06-05 18:49:14,283 - tests.orchestrator - INFO - Creating pod test-pod-28044324 on PVC test-pvc-41d09b77 +2025-06-05 18:49:14,357 - tests.orchestrator - INFO - Created pod: test-pod-28044324 using PVC: test-pvc-41d09b77 +2025-06-05 18:49:14,358 - tests.orchestrator - INFO - Waiting for pod test-pod-28044324 to be ready +2025-06-05 18:49:14,425 - tests.orchestrator - INFO - Pod test-pod-28044324 phase: Pending +2025-06-05 18:49:16,500 - tests.orchestrator - INFO - Pod test-pod-28044324 phase: Running +2025-06-05 18:49:16,500 - tests.orchestrator - INFO - Pod test-pod-28044324 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:49:18,569 - tests.orchestrator - INFO - Pod test-pod-28044324 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:49:20,636 - tests.orchestrator - INFO - Pod test-pod-28044324 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:49:22,705 - tests.orchestrator - INFO - Pod test-pod-28044324 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:49:24,775 - tests.orchestrator - INFO - Pod test-pod-28044324 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:49:26,842 - tests.orchestrator - INFO - Pod test-pod-28044324 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:49:28,911 - tests.orchestrator - INFO - Pod test-pod-28044324 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:49:30,980 - tests.orchestrator - INFO - Pod test-pod-28044324 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:49:33,064 - tests.orchestrator - INFO - Pod test-pod-28044324 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:49:35,131 - tests.orchestrator - INFO - Pod test-pod-28044324 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 18:49:35,132 - tests.orchestrator - INFO - Pod test-pod-28044324 is ready +2025-06-05 18:49:36,132 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 13 times) +2025-06-05 18:49:36,132 - tests.orchestrator - INFO - Creating PVC: test-pvc-b84d5df8 +2025-06-05 18:49:36,208 - tests.orchestrator - INFO - Created PVC: test-pvc-b84d5df8 +2025-06-05 18:49:36,208 - tests.orchestrator - INFO - Waiting for PVC test-pvc-b84d5df8 to be bound +2025-06-05 18:49:38,342 - tests.orchestrator - INFO - PVC test-pvc-b84d5df8 is bound +2025-06-05 18:49:39,342 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 10 times) +2025-06-05 18:49:39,342 - tests.orchestrator - INFO - Creating pod test-pod-5a3a12f1 on PVC test-pvc-41d09b77 +2025-06-05 18:49:39,418 - tests.orchestrator - INFO - Created pod: test-pod-5a3a12f1 using PVC: test-pvc-41d09b77 +2025-06-05 18:49:39,418 - tests.orchestrator - INFO - Waiting for pod test-pod-5a3a12f1 to be ready +2025-06-05 18:49:39,485 - tests.orchestrator - INFO - Pod test-pod-5a3a12f1 phase: Pending +2025-06-05 18:49:41,571 - tests.orchestrator - INFO - Pod test-pod-5a3a12f1 phase: Running +2025-06-05 18:49:41,571 - tests.orchestrator - INFO - Pod test-pod-5a3a12f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:49:43,642 - tests.orchestrator - INFO - Pod test-pod-5a3a12f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:49:45,710 - tests.orchestrator - INFO - Pod test-pod-5a3a12f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:49:47,777 - tests.orchestrator - INFO - Pod test-pod-5a3a12f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:49:49,847 - tests.orchestrator - INFO - Pod test-pod-5a3a12f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:49:51,914 - tests.orchestrator - INFO - Pod test-pod-5a3a12f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:49:53,982 - tests.orchestrator - INFO - Pod test-pod-5a3a12f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:49:56,051 - tests.orchestrator - INFO - Pod test-pod-5a3a12f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 18:49:56,052 - tests.orchestrator - INFO - Pod test-pod-5a3a12f1 is ready +2025-06-05 18:49:57,052 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 14 times) +2025-06-05 18:49:57,052 - tests.orchestrator - INFO - Creating PVC: test-pvc-634bfb7f +2025-06-05 18:49:57,120 - tests.orchestrator - INFO - Created PVC: test-pvc-634bfb7f +2025-06-05 18:49:57,121 - tests.orchestrator - INFO - Waiting for PVC test-pvc-634bfb7f to be bound +2025-06-05 18:49:59,254 - tests.orchestrator - INFO - PVC test-pvc-634bfb7f is bound +2025-06-05 18:50:00,254 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 10 times) +2025-06-05 18:50:00,254 - tests.orchestrator - INFO - Testing read/write between pods test-pod-5a3a12f1 and test-pod-28044324 sharing PVC test-pvc-41d09b77 +2025-06-05 18:50:00,324 - tests.orchestrator - ERROR - Failed in read/write verification: (400) +Reason: Bad Request +HTTP response headers: HTTPHeaderDict({'Audit-Id': '72eb5033-42c4-4730-9e3d-d9327f160c1c', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 18:50:00 GMT', 'Content-Length': '139'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Upgrade request required","reason":"BadRequest","code":400} + + +2025-06-05 18:50:01,324 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 8 times) +2025-06-05 18:50:01,324 - tests.orchestrator - INFO - Deleting PVC: test-pvc-33a8ceec +2025-06-05 18:50:01,401 - tests.orchestrator - INFO - Waiting for PVC test-pvc-33a8ceec to be deleted +2025-06-05 18:50:01,530 - tests.orchestrator - INFO - PVC test-pvc-33a8ceec has been deleted +2025-06-05 18:50:01,530 - tests.orchestrator - INFO - Deleted PVC: test-pvc-33a8ceec +2025-06-05 18:50:02,530 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 11 times) +2025-06-05 18:50:02,530 - tests.orchestrator - INFO - Creating pod test-pod-ace60ac9 on PVC test-pvc-634bfb7f +2025-06-05 18:50:02,605 - tests.orchestrator - INFO - Created pod: test-pod-ace60ac9 using PVC: test-pvc-634bfb7f +2025-06-05 18:50:02,605 - tests.orchestrator - INFO - Waiting for pod test-pod-ace60ac9 to be ready +2025-06-05 18:50:02,673 - tests.orchestrator - INFO - Pod test-pod-ace60ac9 phase: Pending +2025-06-05 18:50:04,742 - tests.orchestrator - INFO - Pod test-pod-ace60ac9 phase: Running +2025-06-05 18:50:04,742 - tests.orchestrator - INFO - Pod test-pod-ace60ac9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:50:06,809 - tests.orchestrator - INFO - Pod test-pod-ace60ac9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:50:08,877 - tests.orchestrator - INFO - Pod test-pod-ace60ac9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:50:10,946 - tests.orchestrator - INFO - Pod test-pod-ace60ac9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:50:13,014 - tests.orchestrator - INFO - Pod test-pod-ace60ac9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:50:15,082 - tests.orchestrator - INFO - Pod test-pod-ace60ac9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:50:17,151 - tests.orchestrator - INFO - Pod test-pod-ace60ac9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:50:19,220 - tests.orchestrator - INFO - Pod test-pod-ace60ac9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 18:50:19,220 - tests.orchestrator - INFO - Pod test-pod-ace60ac9 is ready +2025-06-05 18:50:20,220 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 9 times) +2025-06-05 18:50:20,220 - tests.orchestrator - INFO - Deleting PVC: test-pvc-634bfb7f +2025-06-05 18:50:20,221 - tests.orchestrator - INFO - Deleting 1 pods using PVC test-pvc-634bfb7f +2025-06-05 18:50:20,221 - tests.orchestrator - INFO - Deleting pod: test-pod-ace60ac9 from PVC: test-pvc-634bfb7f +2025-06-05 18:50:20,296 - tests.orchestrator - INFO - Waiting for pod test-pod-ace60ac9 to be deleted +2025-06-05 18:50:52,910 - tests.orchestrator - INFO - Pod test-pod-ace60ac9 has been deleted +2025-06-05 18:50:52,910 - tests.orchestrator - INFO - Deleted pod: test-pod-ace60ac9 +2025-06-05 18:50:52,983 - tests.orchestrator - INFO - Waiting for PVC test-pvc-634bfb7f to be deleted +2025-06-05 18:50:53,052 - tests.orchestrator - INFO - PVC test-pvc-634bfb7f has been deleted +2025-06-05 18:50:53,052 - tests.orchestrator - INFO - Deleted PVC: test-pvc-634bfb7f +2025-06-05 18:50:54,052 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 16 times) +2025-06-05 18:50:54,053 - tests.orchestrator - INFO - Deleting pod: test-pod-f445578c from PVC: test-pvc-29a67853 +2025-06-05 18:50:54,129 - tests.orchestrator - INFO - Waiting for pod test-pod-f445578c to be deleted +2025-06-05 18:51:26,740 - tests.orchestrator - INFO - Pod test-pod-f445578c has been deleted +2025-06-05 18:51:26,741 - tests.orchestrator - INFO - Deleted pod: test-pod-f445578c +2025-06-05 18:51:27,741 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 17 times) +2025-06-05 18:51:27,741 - tests.orchestrator - INFO - Deleting pod: test-pod-5a3a12f1 from PVC: test-pvc-41d09b77 +2025-06-05 18:51:27,817 - tests.orchestrator - INFO - Waiting for pod test-pod-5a3a12f1 to be deleted +2025-06-05 18:52:00,433 - tests.orchestrator - INFO - Pod test-pod-5a3a12f1 has been deleted +2025-06-05 18:52:00,433 - tests.orchestrator - INFO - Deleted pod: test-pod-5a3a12f1 +2025-06-05 18:52:01,433 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 12 times) +2025-06-05 18:52:01,434 - tests.orchestrator - INFO - Creating pod test-pod-66a16102 on PVC test-pvc-29a67853 +2025-06-05 18:52:01,788 - tests.orchestrator - INFO - Created pod: test-pod-66a16102 using PVC: test-pvc-29a67853 +2025-06-05 18:52:01,788 - tests.orchestrator - INFO - Waiting for pod test-pod-66a16102 to be ready +2025-06-05 18:52:01,860 - tests.orchestrator - INFO - Pod test-pod-66a16102 phase: Pending +2025-06-05 18:52:03,929 - tests.orchestrator - INFO - Pod test-pod-66a16102 phase: Running +2025-06-05 18:52:03,929 - tests.orchestrator - INFO - Pod test-pod-66a16102 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:52:05,997 - tests.orchestrator - INFO - Pod test-pod-66a16102 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:52:08,065 - tests.orchestrator - INFO - Pod test-pod-66a16102 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:52:10,135 - tests.orchestrator - INFO - Pod test-pod-66a16102 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:52:12,202 - tests.orchestrator - INFO - Pod test-pod-66a16102 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:52:14,271 - tests.orchestrator - INFO - Pod test-pod-66a16102 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:52:16,339 - tests.orchestrator - INFO - Pod test-pod-66a16102 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:52:18,407 - tests.orchestrator - INFO - Pod test-pod-66a16102 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:52:20,476 - tests.orchestrator - INFO - Pod test-pod-66a16102 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:52:22,544 - tests.orchestrator - INFO - Pod test-pod-66a16102 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 18:52:22,544 - tests.orchestrator - INFO - Pod test-pod-66a16102 is ready +2025-06-05 18:52:23,544 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 13 times) +2025-06-05 18:52:23,545 - tests.orchestrator - INFO - Creating pod test-pod-075367b4 on PVC test-pvc-41d09b77 +2025-06-05 18:52:23,620 - tests.orchestrator - INFO - Created pod: test-pod-075367b4 using PVC: test-pvc-41d09b77 +2025-06-05 18:52:23,620 - tests.orchestrator - INFO - Waiting for pod test-pod-075367b4 to be ready +2025-06-05 18:52:23,687 - tests.orchestrator - INFO - Pod test-pod-075367b4 phase: Pending +2025-06-05 18:52:25,757 - tests.orchestrator - INFO - Pod test-pod-075367b4 phase: Running +2025-06-05 18:52:25,757 - tests.orchestrator - INFO - Pod test-pod-075367b4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:52:27,825 - tests.orchestrator - INFO - Pod test-pod-075367b4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:52:29,893 - tests.orchestrator - INFO - Pod test-pod-075367b4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:52:31,966 - tests.orchestrator - INFO - Pod test-pod-075367b4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:52:34,035 - tests.orchestrator - INFO - Pod test-pod-075367b4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:52:36,103 - tests.orchestrator - INFO - Pod test-pod-075367b4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:52:38,173 - tests.orchestrator - INFO - Pod test-pod-075367b4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:52:40,242 - tests.orchestrator - INFO - Pod test-pod-075367b4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 18:52:40,242 - tests.orchestrator - INFO - Pod test-pod-075367b4 is ready +2025-06-05 18:52:41,243 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 14 times) +2025-06-05 18:52:41,243 - tests.orchestrator - INFO - Creating pod test-pod-dc5857c7 on PVC test-pvc-29a67853 +2025-06-05 18:52:41,326 - tests.orchestrator - INFO - Created pod: test-pod-dc5857c7 using PVC: test-pvc-29a67853 +2025-06-05 18:52:41,326 - tests.orchestrator - INFO - Waiting for pod test-pod-dc5857c7 to be ready +2025-06-05 18:52:41,394 - tests.orchestrator - INFO - Pod test-pod-dc5857c7 phase: Pending +2025-06-05 18:52:43,462 - tests.orchestrator - INFO - Pod test-pod-dc5857c7 phase: Running +2025-06-05 18:52:43,463 - tests.orchestrator - INFO - Pod test-pod-dc5857c7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:52:45,530 - tests.orchestrator - INFO - Pod test-pod-dc5857c7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:52:47,598 - tests.orchestrator - INFO - Pod test-pod-dc5857c7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:52:49,667 - tests.orchestrator - INFO - Pod test-pod-dc5857c7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:52:51,735 - tests.orchestrator - INFO - Pod test-pod-dc5857c7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:52:53,803 - tests.orchestrator - INFO - Pod test-pod-dc5857c7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:52:55,872 - tests.orchestrator - INFO - Pod test-pod-dc5857c7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:52:57,940 - tests.orchestrator - INFO - Pod test-pod-dc5857c7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:53:00,009 - tests.orchestrator - INFO - Pod test-pod-dc5857c7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:53:02,085 - tests.orchestrator - INFO - Pod test-pod-dc5857c7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 18:53:02,086 - tests.orchestrator - INFO - Pod test-pod-dc5857c7 is ready +2025-06-05 18:53:03,086 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 18 times) +2025-06-05 18:53:03,086 - tests.orchestrator - INFO - Deleting pod: test-pod-66a16102 from PVC: test-pvc-29a67853 +2025-06-05 18:53:03,163 - tests.orchestrator - INFO - Waiting for pod test-pod-66a16102 to be deleted +2025-06-05 18:53:35,844 - tests.orchestrator - INFO - Pod test-pod-66a16102 has been deleted +2025-06-05 18:53:35,844 - tests.orchestrator - INFO - Deleted pod: test-pod-66a16102 +2025-06-05 18:53:36,844 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 15 times) +2025-06-05 18:53:36,844 - tests.orchestrator - INFO - Creating pod test-pod-ab6b7029 on PVC test-pvc-41d09b77 +2025-06-05 18:53:36,927 - tests.orchestrator - INFO - Created pod: test-pod-ab6b7029 using PVC: test-pvc-41d09b77 +2025-06-05 18:53:36,927 - tests.orchestrator - INFO - Waiting for pod test-pod-ab6b7029 to be ready +2025-06-05 18:53:36,994 - tests.orchestrator - INFO - Pod test-pod-ab6b7029 phase: Pending +2025-06-05 18:53:39,063 - tests.orchestrator - INFO - Pod test-pod-ab6b7029 phase: Running +2025-06-05 18:53:39,064 - tests.orchestrator - INFO - Pod test-pod-ab6b7029 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:53:41,131 - tests.orchestrator - INFO - Pod test-pod-ab6b7029 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:53:43,199 - tests.orchestrator - INFO - Pod test-pod-ab6b7029 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:53:45,268 - tests.orchestrator - INFO - Pod test-pod-ab6b7029 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:53:47,336 - tests.orchestrator - INFO - Pod test-pod-ab6b7029 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:53:49,404 - tests.orchestrator - INFO - Pod test-pod-ab6b7029 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:53:51,473 - tests.orchestrator - INFO - Pod test-pod-ab6b7029 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:53:53,541 - tests.orchestrator - INFO - Pod test-pod-ab6b7029 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:53:55,609 - tests.orchestrator - INFO - Pod test-pod-ab6b7029 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:53:57,678 - tests.orchestrator - INFO - Pod test-pod-ab6b7029 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:53:59,746 - tests.orchestrator - INFO - Pod test-pod-ab6b7029 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 18:53:59,746 - tests.orchestrator - INFO - Pod test-pod-ab6b7029 is ready +2025-06-05 18:54:00,746 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 19 times) +2025-06-05 18:54:00,746 - tests.orchestrator - INFO - Deleting pod: test-pod-075367b4 from PVC: test-pvc-41d09b77 +2025-06-05 18:54:00,824 - tests.orchestrator - INFO - Waiting for pod test-pod-075367b4 to be deleted +2025-06-05 18:54:33,435 - tests.orchestrator - INFO - Pod test-pod-075367b4 has been deleted +2025-06-05 18:54:33,436 - tests.orchestrator - INFO - Deleted pod: test-pod-075367b4 +2025-06-05 18:54:34,436 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 20 times) +2025-06-05 18:54:34,436 - tests.orchestrator - INFO - Deleting pod: test-pod-28044324 from PVC: test-pvc-41d09b77 +2025-06-05 18:54:34,511 - tests.orchestrator - INFO - Waiting for pod test-pod-28044324 to be deleted +2025-06-05 18:55:07,139 - tests.orchestrator - INFO - Pod test-pod-28044324 has been deleted +2025-06-05 18:55:07,139 - tests.orchestrator - INFO - Deleted pod: test-pod-28044324 +2025-06-05 18:55:08,139 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 15 times) +2025-06-05 18:55:08,140 - tests.orchestrator - INFO - Creating PVC: test-pvc-0bf22cd9 +2025-06-05 18:55:08,219 - tests.orchestrator - INFO - Created PVC: test-pvc-0bf22cd9 +2025-06-05 18:55:08,219 - tests.orchestrator - INFO - Waiting for PVC test-pvc-0bf22cd9 to be bound +2025-06-05 18:55:10,352 - tests.orchestrator - INFO - PVC test-pvc-0bf22cd9 is bound +2025-06-05 18:55:11,352 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 10 times) +2025-06-05 18:55:11,352 - tests.orchestrator - INFO - Deleting PVC: test-pvc-f1842f61 +2025-06-05 18:55:11,424 - tests.orchestrator - INFO - Waiting for PVC test-pvc-f1842f61 to be deleted +2025-06-05 18:55:11,492 - tests.orchestrator - INFO - PVC test-pvc-f1842f61 has been deleted +2025-06-05 18:55:11,492 - tests.orchestrator - INFO - Deleted PVC: test-pvc-f1842f61 +2025-06-05 18:55:12,492 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 16 times) +2025-06-05 18:55:12,492 - tests.orchestrator - INFO - Creating pod test-pod-670c9b02 on PVC test-pvc-60b59274 +2025-06-05 18:55:12,572 - tests.orchestrator - INFO - Created pod: test-pod-670c9b02 using PVC: test-pvc-60b59274 +2025-06-05 18:55:12,572 - tests.orchestrator - INFO - Waiting for pod test-pod-670c9b02 to be ready +2025-06-05 18:55:12,639 - tests.orchestrator - INFO - Pod test-pod-670c9b02 phase: Pending +2025-06-05 18:55:16,774 - tests.orchestrator - INFO - Pod test-pod-670c9b02 phase: Running +2025-06-05 18:55:16,774 - tests.orchestrator - INFO - Pod test-pod-670c9b02 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:55:18,842 - tests.orchestrator - INFO - Pod test-pod-670c9b02 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:55:20,911 - tests.orchestrator - INFO - Pod test-pod-670c9b02 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:55:22,979 - tests.orchestrator - INFO - Pod test-pod-670c9b02 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:55:25,047 - tests.orchestrator - INFO - Pod test-pod-670c9b02 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:55:27,116 - tests.orchestrator - INFO - Pod test-pod-670c9b02 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:55:29,184 - tests.orchestrator - INFO - Pod test-pod-670c9b02 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:55:31,252 - tests.orchestrator - INFO - Pod test-pod-670c9b02 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:55:33,321 - tests.orchestrator - INFO - Pod test-pod-670c9b02 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:55:35,390 - tests.orchestrator - INFO - Pod test-pod-670c9b02 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 18:55:35,390 - tests.orchestrator - INFO - Pod test-pod-670c9b02 is ready +2025-06-05 18:55:36,391 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 17 times) +2025-06-05 18:55:36,391 - tests.orchestrator - INFO - Creating pod test-pod-392b4b23 on PVC test-pvc-b84d5df8 +2025-06-05 18:55:36,465 - tests.orchestrator - INFO - Created pod: test-pod-392b4b23 using PVC: test-pvc-b84d5df8 +2025-06-05 18:55:36,465 - tests.orchestrator - INFO - Waiting for pod test-pod-392b4b23 to be ready +2025-06-05 18:55:36,535 - tests.orchestrator - INFO - Pod test-pod-392b4b23 phase: Pending +2025-06-05 18:55:38,603 - tests.orchestrator - INFO - Pod test-pod-392b4b23 phase: Running +2025-06-05 18:55:38,603 - tests.orchestrator - INFO - Pod test-pod-392b4b23 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:55:40,672 - tests.orchestrator - INFO - Pod test-pod-392b4b23 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:55:42,740 - tests.orchestrator - INFO - Pod test-pod-392b4b23 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:55:44,809 - tests.orchestrator - INFO - Pod test-pod-392b4b23 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:55:46,878 - tests.orchestrator - INFO - Pod test-pod-392b4b23 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:55:48,946 - tests.orchestrator - INFO - Pod test-pod-392b4b23 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:55:51,014 - tests.orchestrator - INFO - Pod test-pod-392b4b23 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:55:53,084 - tests.orchestrator - INFO - Pod test-pod-392b4b23 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 18:55:53,084 - tests.orchestrator - INFO - Pod test-pod-392b4b23 is ready +2025-06-05 18:55:54,085 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 16 times) +2025-06-05 18:55:54,085 - tests.orchestrator - INFO - Creating PVC: test-pvc-54d1de91 +2025-06-05 18:55:54,160 - tests.orchestrator - INFO - Created PVC: test-pvc-54d1de91 +2025-06-05 18:55:54,160 - tests.orchestrator - INFO - Waiting for PVC test-pvc-54d1de91 to be bound +2025-06-05 18:55:56,294 - tests.orchestrator - INFO - PVC test-pvc-54d1de91 is bound +2025-06-05 18:55:57,295 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 17 times) +2025-06-05 18:55:57,295 - tests.orchestrator - INFO - Creating PVC: test-pvc-4184bff1 +2025-06-05 18:55:57,364 - tests.orchestrator - INFO - Created PVC: test-pvc-4184bff1 +2025-06-05 18:55:57,364 - tests.orchestrator - INFO - Waiting for PVC test-pvc-4184bff1 to be bound +2025-06-05 18:55:59,498 - tests.orchestrator - INFO - PVC test-pvc-4184bff1 is bound +2025-06-05 18:56:00,498 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 21 times) +2025-06-05 18:56:00,498 - tests.orchestrator - INFO - Deleting pod: test-pod-392b4b23 from PVC: test-pvc-b84d5df8 +2025-06-05 18:56:00,575 - tests.orchestrator - INFO - Waiting for pod test-pod-392b4b23 to be deleted +2025-06-05 18:56:33,187 - tests.orchestrator - INFO - Pod test-pod-392b4b23 has been deleted +2025-06-05 18:56:33,187 - tests.orchestrator - INFO - Deleted pod: test-pod-392b4b23 +2025-06-05 18:56:34,187 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 18 times) +2025-06-05 18:56:34,187 - tests.orchestrator - INFO - Creating pod test-pod-01b55f76 on PVC test-pvc-41d09b77 +2025-06-05 18:56:34,271 - tests.orchestrator - INFO - Created pod: test-pod-01b55f76 using PVC: test-pvc-41d09b77 +2025-06-05 18:56:34,271 - tests.orchestrator - INFO - Waiting for pod test-pod-01b55f76 to be ready +2025-06-05 18:56:34,339 - tests.orchestrator - INFO - Pod test-pod-01b55f76 phase: Pending +2025-06-05 18:56:36,408 - tests.orchestrator - INFO - Pod test-pod-01b55f76 phase: Running +2025-06-05 18:56:36,408 - tests.orchestrator - INFO - Pod test-pod-01b55f76 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:56:39,222 - tests.orchestrator - INFO - Pod test-pod-01b55f76 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:56:41,289 - tests.orchestrator - INFO - Pod test-pod-01b55f76 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:56:43,358 - tests.orchestrator - INFO - Pod test-pod-01b55f76 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:56:45,426 - tests.orchestrator - INFO - Pod test-pod-01b55f76 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:56:47,494 - tests.orchestrator - INFO - Pod test-pod-01b55f76 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:56:49,563 - tests.orchestrator - INFO - Pod test-pod-01b55f76 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:56:51,631 - tests.orchestrator - INFO - Pod test-pod-01b55f76 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 18:56:51,632 - tests.orchestrator - INFO - Pod test-pod-01b55f76 is ready +2025-06-05 18:56:52,632 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 19 times) +2025-06-05 18:56:52,632 - tests.orchestrator - INFO - Creating pod test-pod-3ab2a061 on PVC test-pvc-4184bff1 +2025-06-05 18:56:52,706 - tests.orchestrator - INFO - Created pod: test-pod-3ab2a061 using PVC: test-pvc-4184bff1 +2025-06-05 18:56:52,706 - tests.orchestrator - INFO - Waiting for pod test-pod-3ab2a061 to be ready +2025-06-05 18:56:52,774 - tests.orchestrator - INFO - Pod test-pod-3ab2a061 phase: Pending +2025-06-05 18:56:54,842 - tests.orchestrator - INFO - Pod test-pod-3ab2a061 phase: Running +2025-06-05 18:56:54,842 - tests.orchestrator - INFO - Pod test-pod-3ab2a061 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:56:56,911 - tests.orchestrator - INFO - Pod test-pod-3ab2a061 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:56:58,978 - tests.orchestrator - INFO - Pod test-pod-3ab2a061 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:57:01,047 - tests.orchestrator - INFO - Pod test-pod-3ab2a061 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:57:03,116 - tests.orchestrator - INFO - Pod test-pod-3ab2a061 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:57:05,183 - tests.orchestrator - INFO - Pod test-pod-3ab2a061 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:57:07,252 - tests.orchestrator - INFO - Pod test-pod-3ab2a061 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:57:09,320 - tests.orchestrator - INFO - Pod test-pod-3ab2a061 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:57:11,389 - tests.orchestrator - INFO - Pod test-pod-3ab2a061 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:57:13,457 - tests.orchestrator - INFO - Pod test-pod-3ab2a061 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 18:57:13,457 - tests.orchestrator - INFO - Pod test-pod-3ab2a061 is ready +2025-06-05 18:57:14,458 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 11 times) +2025-06-05 18:57:14,458 - tests.orchestrator - INFO - Testing read/write between pods test-pod-ab6b7029 and test-pod-01b55f76 sharing PVC test-pvc-41d09b77 +2025-06-05 18:57:14,528 - tests.orchestrator - ERROR - Failed in read/write verification: (400) +Reason: Bad Request +HTTP response headers: HTTPHeaderDict({'Audit-Id': '7c8180b4-3675-4d2d-a309-c082223d6092', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 18:57:14 GMT', 'Content-Length': '139'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Upgrade request required","reason":"BadRequest","code":400} + + +2025-06-05 18:57:15,529 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 20 times) +2025-06-05 18:57:15,529 - tests.orchestrator - INFO - Creating pod test-pod-afdd5b8d on PVC test-pvc-29a67853 +2025-06-05 18:57:15,611 - tests.orchestrator - INFO - Created pod: test-pod-afdd5b8d using PVC: test-pvc-29a67853 +2025-06-05 18:57:15,612 - tests.orchestrator - INFO - Waiting for pod test-pod-afdd5b8d to be ready +2025-06-05 18:57:15,680 - tests.orchestrator - INFO - Pod test-pod-afdd5b8d phase: Pending +2025-06-05 18:57:17,749 - tests.orchestrator - INFO - Pod test-pod-afdd5b8d phase: Running +2025-06-05 18:57:17,749 - tests.orchestrator - INFO - Pod test-pod-afdd5b8d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:57:19,817 - tests.orchestrator - INFO - Pod test-pod-afdd5b8d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:57:21,885 - tests.orchestrator - INFO - Pod test-pod-afdd5b8d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:57:23,954 - tests.orchestrator - INFO - Pod test-pod-afdd5b8d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:57:26,022 - tests.orchestrator - INFO - Pod test-pod-afdd5b8d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:57:28,091 - tests.orchestrator - INFO - Pod test-pod-afdd5b8d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:57:30,160 - tests.orchestrator - INFO - Pod test-pod-afdd5b8d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:57:32,235 - tests.orchestrator - INFO - Pod test-pod-afdd5b8d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 18:57:32,235 - tests.orchestrator - INFO - Pod test-pod-afdd5b8d is ready +2025-06-05 18:57:33,235 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 22 times) +2025-06-05 18:57:33,236 - tests.orchestrator - INFO - Deleting pod: test-pod-3ab2a061 from PVC: test-pvc-4184bff1 +2025-06-05 18:57:33,312 - tests.orchestrator - INFO - Waiting for pod test-pod-3ab2a061 to be deleted +2025-06-05 18:58:05,947 - tests.orchestrator - INFO - Pod test-pod-3ab2a061 has been deleted +2025-06-05 18:58:05,947 - tests.orchestrator - INFO - Deleted pod: test-pod-3ab2a061 +2025-06-05 18:58:06,947 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 23 times) +2025-06-05 18:58:06,947 - tests.orchestrator - INFO - Deleting pod: test-pod-670c9b02 from PVC: test-pvc-60b59274 +2025-06-05 18:58:07,022 - tests.orchestrator - INFO - Waiting for pod test-pod-670c9b02 to be deleted +2025-06-05 18:58:39,738 - tests.orchestrator - INFO - Pod test-pod-670c9b02 has been deleted +2025-06-05 18:58:39,738 - tests.orchestrator - INFO - Deleted pod: test-pod-670c9b02 +2025-06-05 18:58:40,738 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 18 times) +2025-06-05 18:58:40,738 - tests.orchestrator - INFO - Creating PVC: test-pvc-fc8a37f7 +2025-06-05 18:58:40,817 - tests.orchestrator - INFO - Created PVC: test-pvc-fc8a37f7 +2025-06-05 18:58:40,817 - tests.orchestrator - INFO - Waiting for PVC test-pvc-fc8a37f7 to be bound +2025-06-05 18:58:42,953 - tests.orchestrator - INFO - PVC test-pvc-fc8a37f7 is bound +2025-06-05 18:58:43,953 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 12 times) +2025-06-05 18:58:43,953 - tests.orchestrator - INFO - Testing read/write between pods test-pod-afdd5b8d and test-pod-dc5857c7 sharing PVC test-pvc-29a67853 +2025-06-05 18:58:44,024 - tests.orchestrator - ERROR - Failed in read/write verification: (400) +Reason: Bad Request +HTTP response headers: HTTPHeaderDict({'Audit-Id': 'b3898348-0d5a-437c-b7e8-9d9ee4242002', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 18:58:43 GMT', 'Content-Length': '139'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Upgrade request required","reason":"BadRequest","code":400} + + +2025-06-05 18:58:45,024 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 24 times) +2025-06-05 18:58:45,024 - tests.orchestrator - INFO - Deleting pod: test-pod-ab6b7029 from PVC: test-pvc-41d09b77 +2025-06-05 18:58:45,100 - tests.orchestrator - INFO - Waiting for pod test-pod-ab6b7029 to be deleted +2025-06-05 18:59:17,791 - tests.orchestrator - INFO - Pod test-pod-ab6b7029 has been deleted +2025-06-05 18:59:17,792 - tests.orchestrator - INFO - Deleted pod: test-pod-ab6b7029 +2025-06-05 18:59:18,792 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 21 times) +2025-06-05 18:59:18,792 - tests.orchestrator - INFO - Creating pod test-pod-9b9f8243 on PVC test-pvc-0bf22cd9 +2025-06-05 18:59:18,884 - tests.orchestrator - INFO - Created pod: test-pod-9b9f8243 using PVC: test-pvc-0bf22cd9 +2025-06-05 18:59:18,884 - tests.orchestrator - INFO - Waiting for pod test-pod-9b9f8243 to be ready +2025-06-05 18:59:18,954 - tests.orchestrator - INFO - Pod test-pod-9b9f8243 phase: Pending +2025-06-05 18:59:23,090 - tests.orchestrator - INFO - Pod test-pod-9b9f8243 phase: Running +2025-06-05 18:59:23,090 - tests.orchestrator - INFO - Pod test-pod-9b9f8243 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:59:25,159 - tests.orchestrator - INFO - Pod test-pod-9b9f8243 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:59:27,226 - tests.orchestrator - INFO - Pod test-pod-9b9f8243 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:59:29,295 - tests.orchestrator - INFO - Pod test-pod-9b9f8243 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:59:31,364 - tests.orchestrator - INFO - Pod test-pod-9b9f8243 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:59:33,432 - tests.orchestrator - INFO - Pod test-pod-9b9f8243 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:59:35,500 - tests.orchestrator - INFO - Pod test-pod-9b9f8243 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:59:37,569 - tests.orchestrator - INFO - Pod test-pod-9b9f8243 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:59:39,637 - tests.orchestrator - INFO - Pod test-pod-9b9f8243 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 18:59:41,704 - tests.orchestrator - INFO - Pod test-pod-9b9f8243 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 18:59:41,705 - tests.orchestrator - INFO - Pod test-pod-9b9f8243 is ready +2025-06-05 18:59:42,705 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 25 times) +2025-06-05 18:59:42,705 - tests.orchestrator - INFO - Deleting pod: test-pod-9b9f8243 from PVC: test-pvc-0bf22cd9 +2025-06-05 18:59:42,781 - tests.orchestrator - INFO - Waiting for pod test-pod-9b9f8243 to be deleted +2025-06-05 19:00:15,395 - tests.orchestrator - INFO - Pod test-pod-9b9f8243 has been deleted +2025-06-05 19:00:15,395 - tests.orchestrator - INFO - Deleted pod: test-pod-9b9f8243 +2025-06-05 19:00:16,397 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 26 times) +2025-06-05 19:00:16,397 - tests.orchestrator - INFO - Deleting pod: test-pod-afdd5b8d from PVC: test-pvc-29a67853 +2025-06-05 19:00:16,479 - tests.orchestrator - INFO - Waiting for pod test-pod-afdd5b8d to be deleted +2025-06-05 19:00:49,098 - tests.orchestrator - INFO - Pod test-pod-afdd5b8d has been deleted +2025-06-05 19:00:49,098 - tests.orchestrator - INFO - Deleted pod: test-pod-afdd5b8d +2025-06-05 19:00:50,098 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 13 times) +2025-06-05 19:00:50,098 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test +2025-06-05 19:00:51,098 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 14 times) +2025-06-05 19:00:51,098 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test +2025-06-05 19:00:52,099 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 19 times) +2025-06-05 19:00:52,099 - tests.orchestrator - INFO - Creating PVC: test-pvc-7ae6d5e7 +2025-06-05 19:00:52,178 - tests.orchestrator - INFO - Created PVC: test-pvc-7ae6d5e7 +2025-06-05 19:00:52,178 - tests.orchestrator - INFO - Waiting for PVC test-pvc-7ae6d5e7 to be bound +2025-06-05 19:00:54,312 - tests.orchestrator - INFO - PVC test-pvc-7ae6d5e7 is bound +2025-06-05 19:00:55,312 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 15 times) +2025-06-05 19:00:55,312 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test +2025-06-05 19:00:56,313 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 27 times) +2025-06-05 19:00:56,313 - tests.orchestrator - INFO - Deleting pod: test-pod-01b55f76 from PVC: test-pvc-41d09b77 +2025-06-05 19:00:56,387 - tests.orchestrator - INFO - Waiting for pod test-pod-01b55f76 to be deleted +2025-06-05 19:01:28,999 - tests.orchestrator - INFO - Pod test-pod-01b55f76 has been deleted +2025-06-05 19:01:28,999 - tests.orchestrator - INFO - Deleted pod: test-pod-01b55f76 +2025-06-05 19:01:29,999 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 16 times) +2025-06-05 19:01:29,999 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test +2025-06-05 19:01:30,999 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 20 times) +2025-06-05 19:01:30,999 - tests.orchestrator - INFO - Creating PVC: test-pvc-c196bb79 +2025-06-05 19:01:31,077 - tests.orchestrator - INFO - Created PVC: test-pvc-c196bb79 +2025-06-05 19:01:31,077 - tests.orchestrator - INFO - Waiting for PVC test-pvc-c196bb79 to be bound +2025-06-05 19:01:33,210 - tests.orchestrator - INFO - PVC test-pvc-c196bb79 is bound +2025-06-05 19:01:34,210 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 21 times) +2025-06-05 19:01:34,210 - tests.orchestrator - INFO - Creating PVC: test-pvc-d0c53501 +2025-06-05 19:01:34,279 - tests.orchestrator - INFO - Created PVC: test-pvc-d0c53501 +2025-06-05 19:01:34,279 - tests.orchestrator - INFO - Waiting for PVC test-pvc-d0c53501 to be bound +2025-06-05 19:01:36,412 - tests.orchestrator - INFO - PVC test-pvc-d0c53501 is bound +2025-06-05 19:01:37,412 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 28 times) +2025-06-05 19:01:37,412 - tests.orchestrator - INFO - Deleting pod: test-pod-dc5857c7 from PVC: test-pvc-29a67853 +2025-06-05 19:01:37,488 - tests.orchestrator - INFO - Waiting for pod test-pod-dc5857c7 to be deleted +2025-06-05 19:02:10,113 - tests.orchestrator - INFO - Pod test-pod-dc5857c7 has been deleted +2025-06-05 19:02:10,113 - tests.orchestrator - INFO - Deleted pod: test-pod-dc5857c7 +2025-06-05 19:02:11,113 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 22 times) +2025-06-05 19:02:11,113 - tests.orchestrator - INFO - Creating PVC: test-pvc-9bad69f7 +2025-06-05 19:02:11,192 - tests.orchestrator - INFO - Created PVC: test-pvc-9bad69f7 +2025-06-05 19:02:11,192 - tests.orchestrator - INFO - Waiting for PVC test-pvc-9bad69f7 to be bound +2025-06-05 19:02:13,324 - tests.orchestrator - INFO - PVC test-pvc-9bad69f7 is bound +2025-06-05 19:02:14,324 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 23 times) +2025-06-05 19:02:14,324 - tests.orchestrator - INFO - Creating PVC: test-pvc-3d16a983 +2025-06-05 19:02:14,394 - tests.orchestrator - INFO - Created PVC: test-pvc-3d16a983 +2025-06-05 19:02:14,394 - tests.orchestrator - INFO - Waiting for PVC test-pvc-3d16a983 to be bound +2025-06-05 19:02:16,527 - tests.orchestrator - INFO - PVC test-pvc-3d16a983 is bound +2025-06-05 19:02:17,528 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 11 times) +2025-06-05 19:02:17,528 - tests.orchestrator - INFO - Deleting PVC: test-pvc-54d1de91 +2025-06-05 19:02:17,603 - tests.orchestrator - INFO - Waiting for PVC test-pvc-54d1de91 to be deleted +2025-06-05 19:02:21,735 - tests.orchestrator - INFO - PVC test-pvc-54d1de91 has been deleted +2025-06-05 19:02:21,735 - tests.orchestrator - INFO - Deleted PVC: test-pvc-54d1de91 +2025-06-05 19:02:22,736 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 22 times) +2025-06-05 19:02:22,736 - tests.orchestrator - INFO - Creating pod test-pod-b11af190 on PVC test-pvc-d0c53501 +2025-06-05 19:02:22,817 - tests.orchestrator - INFO - Created pod: test-pod-b11af190 using PVC: test-pvc-d0c53501 +2025-06-05 19:02:22,818 - tests.orchestrator - INFO - Waiting for pod test-pod-b11af190 to be ready +2025-06-05 19:02:22,885 - tests.orchestrator - INFO - Pod test-pod-b11af190 phase: Pending +2025-06-05 19:02:24,955 - tests.orchestrator - INFO - Pod test-pod-b11af190 phase: Running +2025-06-05 19:02:24,955 - tests.orchestrator - INFO - Pod test-pod-b11af190 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:02:27,024 - tests.orchestrator - INFO - Pod test-pod-b11af190 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:02:29,093 - tests.orchestrator - INFO - Pod test-pod-b11af190 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:02:31,161 - tests.orchestrator - INFO - Pod test-pod-b11af190 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:02:33,230 - tests.orchestrator - INFO - Pod test-pod-b11af190 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:02:35,299 - tests.orchestrator - INFO - Pod test-pod-b11af190 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:02:37,368 - tests.orchestrator - INFO - Pod test-pod-b11af190 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:02:39,437 - tests.orchestrator - INFO - Pod test-pod-b11af190 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:02:41,507 - tests.orchestrator - INFO - Pod test-pod-b11af190 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:02:43,575 - tests.orchestrator - INFO - Pod test-pod-b11af190 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:02:45,644 - tests.orchestrator - INFO - Pod test-pod-b11af190 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 19:02:45,644 - tests.orchestrator - INFO - Pod test-pod-b11af190 is ready +2025-06-05 19:02:46,646 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 29 times) +2025-06-05 19:02:46,646 - tests.orchestrator - INFO - Deleting pod: test-pod-b11af190 from PVC: test-pvc-d0c53501 +2025-06-05 19:02:46,725 - tests.orchestrator - INFO - Waiting for pod test-pod-b11af190 to be deleted +2025-06-05 19:03:19,340 - tests.orchestrator - INFO - Pod test-pod-b11af190 has been deleted +2025-06-05 19:03:19,341 - tests.orchestrator - INFO - Deleted pod: test-pod-b11af190 +2025-06-05 19:03:20,341 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 23 times) +2025-06-05 19:03:20,341 - tests.orchestrator - INFO - Creating pod test-pod-02a62d88 on PVC test-pvc-9bad69f7 +2025-06-05 19:03:20,425 - tests.orchestrator - INFO - Created pod: test-pod-02a62d88 using PVC: test-pvc-9bad69f7 +2025-06-05 19:03:20,425 - tests.orchestrator - INFO - Waiting for pod test-pod-02a62d88 to be ready +2025-06-05 19:03:20,492 - tests.orchestrator - INFO - Pod test-pod-02a62d88 phase: Pending +2025-06-05 19:03:22,560 - tests.orchestrator - INFO - Pod test-pod-02a62d88 phase: Running +2025-06-05 19:03:22,560 - tests.orchestrator - INFO - Pod test-pod-02a62d88 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:03:24,629 - tests.orchestrator - INFO - Pod test-pod-02a62d88 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:03:26,697 - tests.orchestrator - INFO - Pod test-pod-02a62d88 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:03:28,765 - tests.orchestrator - INFO - Pod test-pod-02a62d88 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:03:30,834 - tests.orchestrator - INFO - Pod test-pod-02a62d88 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:03:32,902 - tests.orchestrator - INFO - Pod test-pod-02a62d88 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:03:34,970 - tests.orchestrator - INFO - Pod test-pod-02a62d88 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:03:37,041 - tests.orchestrator - INFO - Pod test-pod-02a62d88 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 19:03:37,041 - tests.orchestrator - INFO - Pod test-pod-02a62d88 is ready +2025-06-05 19:03:38,042 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 30 times) +2025-06-05 19:03:38,042 - tests.orchestrator - INFO - Deleting pod: test-pod-02a62d88 from PVC: test-pvc-9bad69f7 +2025-06-05 19:03:38,117 - tests.orchestrator - INFO - Waiting for pod test-pod-02a62d88 to be deleted +2025-06-05 19:04:10,749 - tests.orchestrator - INFO - Pod test-pod-02a62d88 has been deleted +2025-06-05 19:04:10,749 - tests.orchestrator - INFO - Deleted pod: test-pod-02a62d88 +2025-06-05 19:04:11,749 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 12 times) +2025-06-05 19:04:11,749 - tests.orchestrator - INFO - Deleting PVC: test-pvc-fc8a37f7 +2025-06-05 19:04:11,822 - tests.orchestrator - INFO - Waiting for PVC test-pvc-fc8a37f7 to be deleted +2025-06-05 19:04:15,956 - tests.orchestrator - INFO - PVC test-pvc-fc8a37f7 has been deleted +2025-06-05 19:04:15,956 - tests.orchestrator - INFO - Deleted PVC: test-pvc-fc8a37f7 +2025-06-05 19:04:16,957 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 17 times) +2025-06-05 19:04:16,957 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test +2025-06-05 19:04:17,957 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 31 times) +2025-06-05 19:04:17,957 - tests.orchestrator - INFO - No pods to delete +2025-06-05 19:04:18,958 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 18 times) +2025-06-05 19:04:18,959 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test +2025-06-05 19:04:19,959 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 24 times) +2025-06-05 19:04:19,959 - tests.orchestrator - INFO - Creating pod test-pod-fbf9cd30 on PVC test-pvc-29a67853 +2025-06-05 19:04:20,042 - tests.orchestrator - INFO - Created pod: test-pod-fbf9cd30 using PVC: test-pvc-29a67853 +2025-06-05 19:04:20,042 - tests.orchestrator - INFO - Waiting for pod test-pod-fbf9cd30 to be ready +2025-06-05 19:04:20,109 - tests.orchestrator - INFO - Pod test-pod-fbf9cd30 phase: Pending +2025-06-05 19:04:22,177 - tests.orchestrator - INFO - Pod test-pod-fbf9cd30 phase: Running +2025-06-05 19:04:22,177 - tests.orchestrator - INFO - Pod test-pod-fbf9cd30 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:04:24,246 - tests.orchestrator - INFO - Pod test-pod-fbf9cd30 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:04:26,314 - tests.orchestrator - INFO - Pod test-pod-fbf9cd30 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:04:28,382 - tests.orchestrator - INFO - Pod test-pod-fbf9cd30 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:04:30,451 - tests.orchestrator - INFO - Pod test-pod-fbf9cd30 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:04:32,518 - tests.orchestrator - INFO - Pod test-pod-fbf9cd30 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:04:34,586 - tests.orchestrator - INFO - Pod test-pod-fbf9cd30 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:04:36,656 - tests.orchestrator - INFO - Pod test-pod-fbf9cd30 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:04:38,723 - tests.orchestrator - INFO - Pod test-pod-fbf9cd30 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:04:40,791 - tests.orchestrator - INFO - Pod test-pod-fbf9cd30 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 19:04:40,791 - tests.orchestrator - INFO - Pod test-pod-fbf9cd30 is ready +2025-06-05 19:04:41,792 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 24 times) +2025-06-05 19:04:41,792 - tests.orchestrator - INFO - Creating PVC: test-pvc-c1c2eedb +2025-06-05 19:04:41,862 - tests.orchestrator - INFO - Created PVC: test-pvc-c1c2eedb +2025-06-05 19:04:41,863 - tests.orchestrator - INFO - Waiting for PVC test-pvc-c1c2eedb to be bound +2025-06-05 19:04:43,996 - tests.orchestrator - INFO - PVC test-pvc-c1c2eedb is bound +2025-06-05 19:04:44,997 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 32 times) +2025-06-05 19:04:44,997 - tests.orchestrator - INFO - Deleting pod: test-pod-fbf9cd30 from PVC: test-pvc-29a67853 +2025-06-05 19:04:45,072 - tests.orchestrator - INFO - Waiting for pod test-pod-fbf9cd30 to be deleted +2025-06-05 19:05:17,689 - tests.orchestrator - INFO - Pod test-pod-fbf9cd30 has been deleted +2025-06-05 19:05:17,689 - tests.orchestrator - INFO - Deleted pod: test-pod-fbf9cd30 +2025-06-05 19:05:18,689 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 25 times) +2025-06-05 19:05:18,689 - tests.orchestrator - INFO - Creating PVC: test-pvc-2ddd2ed2 +2025-06-05 19:05:18,768 - tests.orchestrator - INFO - Created PVC: test-pvc-2ddd2ed2 +2025-06-05 19:05:18,768 - tests.orchestrator - INFO - Waiting for PVC test-pvc-2ddd2ed2 to be bound +2025-06-05 19:05:20,901 - tests.orchestrator - INFO - PVC test-pvc-2ddd2ed2 is bound +2025-06-05 19:05:21,902 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 19 times) +2025-06-05 19:05:21,902 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test +2025-06-05 19:05:22,902 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 25 times) +2025-06-05 19:05:22,902 - tests.orchestrator - INFO - Creating pod test-pod-7012ed49 on PVC test-pvc-0bf22cd9 +2025-06-05 19:05:22,978 - tests.orchestrator - INFO - Created pod: test-pod-7012ed49 using PVC: test-pvc-0bf22cd9 +2025-06-05 19:05:22,978 - tests.orchestrator - INFO - Waiting for pod test-pod-7012ed49 to be ready +2025-06-05 19:05:23,047 - tests.orchestrator - INFO - Pod test-pod-7012ed49 phase: Pending +2025-06-05 19:05:27,183 - tests.orchestrator - INFO - Pod test-pod-7012ed49 phase: Running +2025-06-05 19:05:27,183 - tests.orchestrator - INFO - Pod test-pod-7012ed49 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:05:29,252 - tests.orchestrator - INFO - Pod test-pod-7012ed49 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:05:31,319 - tests.orchestrator - INFO - Pod test-pod-7012ed49 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:05:33,388 - tests.orchestrator - INFO - Pod test-pod-7012ed49 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:05:35,456 - tests.orchestrator - INFO - Pod test-pod-7012ed49 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:05:37,524 - tests.orchestrator - INFO - Pod test-pod-7012ed49 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:05:40,341 - tests.orchestrator - INFO - Pod test-pod-7012ed49 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:05:42,410 - tests.orchestrator - INFO - Pod test-pod-7012ed49 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:05:44,479 - tests.orchestrator - INFO - Pod test-pod-7012ed49 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 19:05:44,479 - tests.orchestrator - INFO - Pod test-pod-7012ed49 is ready +2025-06-05 19:05:45,479 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 20 times) +2025-06-05 19:05:45,479 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test +2025-06-05 19:05:46,480 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 33 times) +2025-06-05 19:05:46,480 - tests.orchestrator - INFO - Deleting pod: test-pod-7012ed49 from PVC: test-pvc-0bf22cd9 +2025-06-05 19:05:46,561 - tests.orchestrator - INFO - Waiting for pod test-pod-7012ed49 to be deleted +2025-06-05 19:06:19,176 - tests.orchestrator - INFO - Pod test-pod-7012ed49 has been deleted +2025-06-05 19:06:19,176 - tests.orchestrator - INFO - Deleted pod: test-pod-7012ed49 +2025-06-05 19:06:20,176 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 13 times) +2025-06-05 19:06:20,177 - tests.orchestrator - INFO - Deleting PVC: test-pvc-9bad69f7 +2025-06-05 19:06:20,249 - tests.orchestrator - INFO - Waiting for PVC test-pvc-9bad69f7 to be deleted +2025-06-05 19:06:20,316 - tests.orchestrator - INFO - PVC test-pvc-9bad69f7 has been deleted +2025-06-05 19:06:20,316 - tests.orchestrator - INFO - Deleted PVC: test-pvc-9bad69f7 +2025-06-05 19:06:21,316 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 34 times) +2025-06-05 19:06:21,317 - tests.orchestrator - INFO - No pods to delete +2025-06-05 19:06:22,317 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 26 times) +2025-06-05 19:06:22,317 - tests.orchestrator - INFO - Creating pod test-pod-d45ef83e on PVC test-pvc-6e8e6345 +2025-06-05 19:06:22,400 - tests.orchestrator - INFO - Created pod: test-pod-d45ef83e using PVC: test-pvc-6e8e6345 +2025-06-05 19:06:22,400 - tests.orchestrator - INFO - Waiting for pod test-pod-d45ef83e to be ready +2025-06-05 19:06:22,468 - tests.orchestrator - INFO - Pod test-pod-d45ef83e phase: Pending +2025-06-05 19:06:24,537 - tests.orchestrator - INFO - Pod test-pod-d45ef83e phase: Running +2025-06-05 19:06:24,538 - tests.orchestrator - INFO - Pod test-pod-d45ef83e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:06:26,611 - tests.orchestrator - INFO - Pod test-pod-d45ef83e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:06:28,680 - tests.orchestrator - INFO - Pod test-pod-d45ef83e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:06:30,748 - tests.orchestrator - INFO - Pod test-pod-d45ef83e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:06:32,816 - tests.orchestrator - INFO - Pod test-pod-d45ef83e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:06:34,885 - tests.orchestrator - INFO - Pod test-pod-d45ef83e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:06:36,953 - tests.orchestrator - INFO - Pod test-pod-d45ef83e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:06:39,022 - tests.orchestrator - INFO - Pod test-pod-d45ef83e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:06:41,091 - tests.orchestrator - INFO - Pod test-pod-d45ef83e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:06:43,158 - tests.orchestrator - INFO - Pod test-pod-d45ef83e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:06:45,226 - tests.orchestrator - INFO - Pod test-pod-d45ef83e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 19:06:45,226 - tests.orchestrator - INFO - Pod test-pod-d45ef83e is ready +2025-06-05 19:06:46,226 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 26 times) +2025-06-05 19:06:46,228 - tests.orchestrator - INFO - Creating PVC: test-pvc-1bb798a3 +2025-06-05 19:06:46,297 - tests.orchestrator - INFO - Created PVC: test-pvc-1bb798a3 +2025-06-05 19:06:46,297 - tests.orchestrator - INFO - Waiting for PVC test-pvc-1bb798a3 to be bound +2025-06-05 19:06:48,430 - tests.orchestrator - INFO - PVC test-pvc-1bb798a3 is bound +2025-06-05 19:06:49,430 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 27 times) +2025-06-05 19:06:49,431 - tests.orchestrator - INFO - Creating PVC: test-pvc-36370699 +2025-06-05 19:06:49,499 - tests.orchestrator - INFO - Created PVC: test-pvc-36370699 +2025-06-05 19:06:49,499 - tests.orchestrator - INFO - Waiting for PVC test-pvc-36370699 to be bound +2025-06-05 19:06:51,633 - tests.orchestrator - INFO - PVC test-pvc-36370699 is bound +2025-06-05 19:06:52,633 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 35 times) +2025-06-05 19:06:52,633 - tests.orchestrator - INFO - Deleting pod: test-pod-d45ef83e from PVC: test-pvc-6e8e6345 +2025-06-05 19:06:52,708 - tests.orchestrator - INFO - Waiting for pod test-pod-d45ef83e to be deleted +2025-06-05 19:07:25,324 - tests.orchestrator - INFO - Pod test-pod-d45ef83e has been deleted +2025-06-05 19:07:25,324 - tests.orchestrator - INFO - Deleted pod: test-pod-d45ef83e +2025-06-05 19:07:26,324 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 36 times) +2025-06-05 19:07:26,324 - tests.orchestrator - INFO - No pods to delete +2025-06-05 19:07:27,324 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 28 times) +2025-06-05 19:07:27,325 - tests.orchestrator - INFO - Creating PVC: test-pvc-faea1890 +2025-06-05 19:07:27,403 - tests.orchestrator - INFO - Created PVC: test-pvc-faea1890 +2025-06-05 19:07:27,403 - tests.orchestrator - INFO - Waiting for PVC test-pvc-faea1890 to be bound +2025-06-05 19:07:29,537 - tests.orchestrator - INFO - PVC test-pvc-faea1890 is bound +2025-06-05 19:07:30,537 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 21 times) +2025-06-05 19:07:30,538 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test +2025-06-05 19:07:31,538 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 37 times) +2025-06-05 19:07:31,538 - tests.orchestrator - INFO - No pods to delete +2025-06-05 19:07:32,538 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 22 times) +2025-06-05 19:07:32,538 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test +2025-06-05 19:07:33,539 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 29 times) +2025-06-05 19:07:33,539 - tests.orchestrator - INFO - Creating PVC: test-pvc-c6976eae +2025-06-05 19:07:33,607 - tests.orchestrator - INFO - Created PVC: test-pvc-c6976eae +2025-06-05 19:07:33,608 - tests.orchestrator - INFO - Waiting for PVC test-pvc-c6976eae to be bound +2025-06-05 19:07:35,740 - tests.orchestrator - INFO - PVC test-pvc-c6976eae is bound +2025-06-05 19:07:36,740 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 23 times) +2025-06-05 19:07:36,741 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test +2025-06-05 19:07:37,741 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 27 times) +2025-06-05 19:07:37,741 - tests.orchestrator - INFO - Creating pod test-pod-bb3a758b on PVC test-pvc-faea1890 +2025-06-05 19:07:37,833 - tests.orchestrator - INFO - Created pod: test-pod-bb3a758b using PVC: test-pvc-faea1890 +2025-06-05 19:07:37,833 - tests.orchestrator - INFO - Waiting for pod test-pod-bb3a758b to be ready +2025-06-05 19:07:37,904 - tests.orchestrator - INFO - Pod test-pod-bb3a758b phase: Pending +2025-06-05 19:07:42,041 - tests.orchestrator - INFO - Pod test-pod-bb3a758b phase: Running +2025-06-05 19:07:42,041 - tests.orchestrator - INFO - Pod test-pod-bb3a758b conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:07:44,110 - tests.orchestrator - INFO - Pod test-pod-bb3a758b conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:07:46,178 - tests.orchestrator - INFO - Pod test-pod-bb3a758b conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:07:48,247 - tests.orchestrator - INFO - Pod test-pod-bb3a758b conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:07:50,316 - tests.orchestrator - INFO - Pod test-pod-bb3a758b conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:07:52,383 - tests.orchestrator - INFO - Pod test-pod-bb3a758b conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:07:54,451 - tests.orchestrator - INFO - Pod test-pod-bb3a758b conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:07:56,524 - tests.orchestrator - INFO - Pod test-pod-bb3a758b conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:07:58,591 - tests.orchestrator - INFO - Pod test-pod-bb3a758b conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:08:00,658 - tests.orchestrator - INFO - Pod test-pod-bb3a758b conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 19:08:00,659 - tests.orchestrator - INFO - Pod test-pod-bb3a758b is ready +2025-06-05 19:08:01,659 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 24 times) +2025-06-05 19:08:01,659 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test +2025-06-05 19:08:02,659 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 38 times) +2025-06-05 19:08:02,659 - tests.orchestrator - INFO - Deleting pod: test-pod-bb3a758b from PVC: test-pvc-faea1890 +2025-06-05 19:08:02,736 - tests.orchestrator - INFO - Waiting for pod test-pod-bb3a758b to be deleted +2025-06-05 19:08:35,351 - tests.orchestrator - INFO - Pod test-pod-bb3a758b has been deleted +2025-06-05 19:08:35,352 - tests.orchestrator - INFO - Deleted pod: test-pod-bb3a758b +2025-06-05 19:08:36,352 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 39 times) +2025-06-05 19:08:36,352 - tests.orchestrator - INFO - No pods to delete +2025-06-05 19:08:37,352 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 14 times) +2025-06-05 19:08:37,352 - tests.orchestrator - INFO - Deleting PVC: test-pvc-2ddd2ed2 +2025-06-05 19:08:37,424 - tests.orchestrator - INFO - Waiting for PVC test-pvc-2ddd2ed2 to be deleted +2025-06-05 19:08:37,490 - tests.orchestrator - INFO - PVC test-pvc-2ddd2ed2 has been deleted +2025-06-05 19:08:37,490 - tests.orchestrator - INFO - Deleted PVC: test-pvc-2ddd2ed2 +2025-06-05 19:08:38,490 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 30 times) +2025-06-05 19:08:38,491 - tests.orchestrator - INFO - Creating PVC: test-pvc-fe0fe649 +2025-06-05 19:08:38,567 - tests.orchestrator - INFO - Created PVC: test-pvc-fe0fe649 +2025-06-05 19:08:38,567 - tests.orchestrator - INFO - Waiting for PVC test-pvc-fe0fe649 to be bound +2025-06-05 19:08:40,700 - tests.orchestrator - INFO - PVC test-pvc-fe0fe649 is bound diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_191215.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_191215.log new file mode 100644 index 000000000..40bba3461 --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_191215.log @@ -0,0 +1,158 @@ +2025-06-05 19:12:15,577 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 19:12:15,781 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 19:12:15,782 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-05 19:12:15,974 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 19:12:15,974 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 19:12:15,974 - tests.orchestrator - INFO - Creating PVC: test-pvc-0bb0d20c +2025-06-05 19:12:16,053 - tests.orchestrator - INFO - Created PVC: test-pvc-0bb0d20c +2025-06-05 19:12:16,053 - tests.orchestrator - INFO - Waiting for PVC test-pvc-0bb0d20c to be bound +2025-06-05 19:12:18,183 - tests.orchestrator - INFO - PVC test-pvc-0bb0d20c is bound +2025-06-05 19:12:18,183 - tests.orchestrator - INFO - Creating PVC: test-pvc-5076c293 +2025-06-05 19:12:18,253 - tests.orchestrator - INFO - Created PVC: test-pvc-5076c293 +2025-06-05 19:12:18,253 - tests.orchestrator - INFO - Waiting for PVC test-pvc-5076c293 to be bound +2025-06-05 19:12:20,383 - tests.orchestrator - INFO - PVC test-pvc-5076c293 is bound +2025-06-05 19:12:20,383 - tests.orchestrator - INFO - Creating pod test-pod-fd8d328f on PVC test-pvc-5076c293 +2025-06-05 19:12:20,469 - tests.orchestrator - INFO - Created pod: test-pod-fd8d328f using PVC: test-pvc-5076c293 +2025-06-05 19:12:20,469 - tests.orchestrator - INFO - Waiting for pod test-pod-fd8d328f to be ready +2025-06-05 19:12:20,535 - tests.orchestrator - INFO - Pod test-pod-fd8d328f phase: Pending +2025-06-05 19:12:22,606 - tests.orchestrator - INFO - Pod test-pod-fd8d328f phase: Running +2025-06-05 19:12:22,606 - tests.orchestrator - INFO - Pod test-pod-fd8d328f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:12:24,675 - tests.orchestrator - INFO - Pod test-pod-fd8d328f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:12:26,743 - tests.orchestrator - INFO - Pod test-pod-fd8d328f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:12:28,812 - tests.orchestrator - INFO - Pod test-pod-fd8d328f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:12:30,880 - tests.orchestrator - INFO - Pod test-pod-fd8d328f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:12:32,948 - tests.orchestrator - INFO - Pod test-pod-fd8d328f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:12:35,017 - tests.orchestrator - INFO - Pod test-pod-fd8d328f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:12:37,085 - tests.orchestrator - INFO - Pod test-pod-fd8d328f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 19:12:37,085 - tests.orchestrator - INFO - Pod test-pod-fd8d328f is ready +2025-06-05 19:12:37,085 - tests.orchestrator - INFO - Creating pod test-pod-a2dd05ca on PVC test-pvc-0bb0d20c +2025-06-05 19:12:37,162 - tests.orchestrator - INFO - Created pod: test-pod-a2dd05ca using PVC: test-pvc-0bb0d20c +2025-06-05 19:12:37,162 - tests.orchestrator - INFO - Waiting for pod test-pod-a2dd05ca to be ready +2025-06-05 19:12:37,229 - tests.orchestrator - INFO - Pod test-pod-a2dd05ca phase: Pending +2025-06-05 19:12:39,298 - tests.orchestrator - INFO - Pod test-pod-a2dd05ca phase: Running +2025-06-05 19:12:39,298 - tests.orchestrator - INFO - Pod test-pod-a2dd05ca conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:12:41,367 - tests.orchestrator - INFO - Pod test-pod-a2dd05ca conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:12:43,436 - tests.orchestrator - INFO - Pod test-pod-a2dd05ca conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:12:45,505 - tests.orchestrator - INFO - Pod test-pod-a2dd05ca conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:12:47,574 - tests.orchestrator - INFO - Pod test-pod-a2dd05ca conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:12:49,643 - tests.orchestrator - INFO - Pod test-pod-a2dd05ca conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:12:51,713 - tests.orchestrator - INFO - Pod test-pod-a2dd05ca conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:12:53,781 - tests.orchestrator - INFO - Pod test-pod-a2dd05ca conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 19:12:53,782 - tests.orchestrator - INFO - Pod test-pod-a2dd05ca is ready +2025-06-05 19:12:53,782 - tests.orchestrator - INFO - Creating pod test-pod-284d58ca on PVC test-pvc-5076c293 +2025-06-05 19:12:53,869 - tests.orchestrator - INFO - Created pod: test-pod-284d58ca using PVC: test-pvc-5076c293 +2025-06-05 19:12:53,869 - tests.orchestrator - INFO - Waiting for pod test-pod-284d58ca to be ready +2025-06-05 19:12:53,938 - tests.orchestrator - INFO - Pod test-pod-284d58ca phase: Pending +2025-06-05 19:12:58,073 - tests.orchestrator - INFO - Pod test-pod-284d58ca phase: Running +2025-06-05 19:12:58,073 - tests.orchestrator - INFO - Pod test-pod-284d58ca conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:13:00,143 - tests.orchestrator - INFO - Pod test-pod-284d58ca conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:13:02,214 - tests.orchestrator - INFO - Pod test-pod-284d58ca conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:13:04,282 - tests.orchestrator - INFO - Pod test-pod-284d58ca conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:13:06,351 - tests.orchestrator - INFO - Pod test-pod-284d58ca conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:13:08,422 - tests.orchestrator - INFO - Pod test-pod-284d58ca conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:13:10,488 - tests.orchestrator - INFO - Pod test-pod-284d58ca conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:13:12,559 - tests.orchestrator - INFO - Pod test-pod-284d58ca conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:13:14,628 - tests.orchestrator - INFO - Pod test-pod-284d58ca conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:13:16,697 - tests.orchestrator - INFO - Pod test-pod-284d58ca conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 19:13:16,697 - tests.orchestrator - INFO - Pod test-pod-284d58ca is ready +2025-06-05 19:13:16,697 - tests.orchestrator - INFO - Testing read/write between pods test-pod-284d58ca and test-pod-fd8d328f sharing PVC test-pvc-5076c293 +2025-06-05 19:13:16,697 - tests.orchestrator - INFO - Executing write command: kubectl exec -n efs-stress-test test-pod-284d58ca -- /bin/sh -c 'echo "Test content: e2446e14-a753-4aa8-80af-e120861c76a8" > /data/test-421a110b.txt' +2025-06-05 19:13:17,954 - tests.orchestrator - INFO - Pod test-pod-284d58ca wrote to /data/test-421a110b.txt +2025-06-05 19:13:19,954 - tests.orchestrator - INFO - Executing read command: kubectl exec -n efs-stress-test test-pod-fd8d328f -- cat /data/test-421a110b.txt +2025-06-05 19:13:21,152 - tests.orchestrator - INFO - Read result: 'Test content: e2446e14-a753-4aa8-80af-e120861c76a8' +2025-06-05 19:13:21,152 - tests.orchestrator - INFO - Pod test-pod-fd8d328f successfully read content written by test-pod-284d58ca +2025-06-05 19:13:21,153 - tests.orchestrator - INFO - Deleting pod: test-pod-284d58ca from PVC: test-pvc-5076c293 +2025-06-05 19:13:21,229 - tests.orchestrator - INFO - Waiting for pod test-pod-284d58ca to be deleted +2025-06-05 19:13:53,872 - tests.orchestrator - INFO - Pod test-pod-284d58ca has been deleted +2025-06-05 19:13:53,872 - tests.orchestrator - INFO - Deleted pod: test-pod-284d58ca +2025-06-05 19:13:53,872 - tests.orchestrator - INFO - Deleting PVC: test-pvc-5076c293 +2025-06-05 19:13:53,872 - tests.orchestrator - INFO - Deleting 1 pods using PVC test-pvc-5076c293 +2025-06-05 19:13:53,872 - tests.orchestrator - INFO - Deleting pod: test-pod-fd8d328f from PVC: test-pvc-5076c293 +2025-06-05 19:13:53,950 - tests.orchestrator - INFO - Waiting for pod test-pod-fd8d328f to be deleted +2025-06-05 19:14:26,581 - tests.orchestrator - INFO - Pod test-pod-fd8d328f has been deleted +2025-06-05 19:14:26,581 - tests.orchestrator - INFO - Deleted pod: test-pod-fd8d328f +2025-06-05 19:14:26,658 - tests.orchestrator - INFO - Waiting for PVC test-pvc-5076c293 to be deleted +2025-06-05 19:14:30,797 - tests.orchestrator - INFO - PVC test-pvc-5076c293 has been deleted +2025-06-05 19:14:30,797 - tests.orchestrator - INFO - Deleted PVC: test-pvc-5076c293 +2025-06-05 19:14:30,797 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations +2025-06-05 19:14:30,797 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 1 times) +2025-06-05 19:14:30,797 - tests.orchestrator - INFO - Deleting pod: test-pod-a2dd05ca from PVC: test-pvc-0bb0d20c +2025-06-05 19:14:30,876 - tests.orchestrator - INFO - Waiting for pod test-pod-a2dd05ca to be deleted +2025-06-05 19:15:03,503 - tests.orchestrator - INFO - Pod test-pod-a2dd05ca has been deleted +2025-06-05 19:15:03,503 - tests.orchestrator - INFO - Deleted pod: test-pod-a2dd05ca +2025-06-05 19:15:04,503 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 1 times) +2025-06-05 19:15:04,504 - tests.orchestrator - INFO - Creating PVC: test-pvc-2127e71a +2025-06-05 19:15:04,583 - tests.orchestrator - INFO - Created PVC: test-pvc-2127e71a +2025-06-05 19:15:04,583 - tests.orchestrator - INFO - Waiting for PVC test-pvc-2127e71a to be bound +2025-06-05 19:15:06,716 - tests.orchestrator - INFO - PVC test-pvc-2127e71a is bound +2025-06-05 19:15:07,716 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 2 times) +2025-06-05 19:15:07,716 - tests.orchestrator - INFO - No pods to delete +2025-06-05 19:15:08,716 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 2 times) +2025-06-05 19:15:08,717 - tests.orchestrator - INFO - Creating PVC: test-pvc-a0df5547 +2025-06-05 19:15:08,786 - tests.orchestrator - INFO - Created PVC: test-pvc-a0df5547 +2025-06-05 19:15:08,786 - tests.orchestrator - INFO - Waiting for PVC test-pvc-a0df5547 to be bound +2025-06-05 19:15:10,934 - tests.orchestrator - INFO - PVC test-pvc-a0df5547 is bound +2025-06-05 19:15:11,934 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 1 times) +2025-06-05 19:15:11,935 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test +2025-06-05 19:15:12,935 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 3 times) +2025-06-05 19:15:12,935 - tests.orchestrator - INFO - Creating PVC: test-pvc-ecb12f74 +2025-06-05 19:15:13,010 - tests.orchestrator - INFO - Created PVC: test-pvc-ecb12f74 +2025-06-05 19:15:13,010 - tests.orchestrator - INFO - Waiting for PVC test-pvc-ecb12f74 to be bound +2025-06-05 19:15:15,146 - tests.orchestrator - INFO - PVC test-pvc-ecb12f74 is bound +2025-06-05 19:15:16,146 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 4 times) +2025-06-05 19:15:16,146 - tests.orchestrator - INFO - Creating PVC: test-pvc-31fbd501 +2025-06-05 19:15:16,214 - tests.orchestrator - INFO - Created PVC: test-pvc-31fbd501 +2025-06-05 19:15:16,214 - tests.orchestrator - INFO - Waiting for PVC test-pvc-31fbd501 to be bound +2025-06-05 19:15:18,349 - tests.orchestrator - INFO - PVC test-pvc-31fbd501 is bound +2025-06-05 19:15:19,349 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 5 times) +2025-06-05 19:15:19,349 - tests.orchestrator - INFO - Creating PVC: test-pvc-89991955 +2025-06-05 19:15:19,416 - tests.orchestrator - INFO - Created PVC: test-pvc-89991955 +2025-06-05 19:15:19,416 - tests.orchestrator - INFO - Waiting for PVC test-pvc-89991955 to be bound +2025-06-05 19:15:21,555 - tests.orchestrator - INFO - PVC test-pvc-89991955 is bound +2025-06-05 19:15:22,555 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 2 times) +2025-06-05 19:15:22,556 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test +2025-06-05 19:15:23,556 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 1 times) +2025-06-05 19:15:23,556 - tests.orchestrator - INFO - Deleting PVC: test-pvc-89991955 +2025-06-05 19:15:23,631 - tests.orchestrator - INFO - Waiting for PVC test-pvc-89991955 to be deleted +2025-06-05 19:15:23,698 - tests.orchestrator - INFO - PVC test-pvc-89991955 has been deleted +2025-06-05 19:15:23,698 - tests.orchestrator - INFO - Deleted PVC: test-pvc-89991955 +2025-06-05 19:15:24,699 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 3 times) +2025-06-05 19:15:24,699 - tests.orchestrator - INFO - No pods to delete +2025-06-05 19:15:25,699 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 6 times) +2025-06-05 19:15:25,699 - tests.orchestrator - INFO - Creating PVC: test-pvc-2cf5b7a2 +2025-06-05 19:15:25,768 - tests.orchestrator - INFO - Created PVC: test-pvc-2cf5b7a2 +2025-06-05 19:15:25,768 - tests.orchestrator - INFO - Waiting for PVC test-pvc-2cf5b7a2 to be bound +2025-06-05 19:15:27,904 - tests.orchestrator - INFO - PVC test-pvc-2cf5b7a2 is bound +2025-06-05 19:15:28,904 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 7 times) +2025-06-05 19:15:28,904 - tests.orchestrator - INFO - Creating PVC: test-pvc-528b4f5c +2025-06-05 19:15:28,973 - tests.orchestrator - INFO - Created PVC: test-pvc-528b4f5c +2025-06-05 19:15:28,973 - tests.orchestrator - INFO - Waiting for PVC test-pvc-528b4f5c to be bound +2025-06-05 19:15:31,115 - tests.orchestrator - INFO - PVC test-pvc-528b4f5c is bound +2025-06-05 19:15:32,115 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 1 times) +2025-06-05 19:15:32,115 - tests.orchestrator - INFO - Creating pod test-pod-0d0fddfe on PVC test-pvc-2cf5b7a2 +2025-06-05 19:15:32,198 - tests.orchestrator - INFO - Created pod: test-pod-0d0fddfe using PVC: test-pvc-2cf5b7a2 +2025-06-05 19:15:32,199 - tests.orchestrator - INFO - Waiting for pod test-pod-0d0fddfe to be ready +2025-06-05 19:15:32,267 - tests.orchestrator - INFO - Pod test-pod-0d0fddfe phase: Pending +2025-06-05 19:15:36,403 - tests.orchestrator - INFO - Pod test-pod-0d0fddfe phase: Running +2025-06-05 19:15:36,403 - tests.orchestrator - INFO - Pod test-pod-0d0fddfe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:15:38,472 - tests.orchestrator - INFO - Pod test-pod-0d0fddfe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:15:40,540 - tests.orchestrator - INFO - Pod test-pod-0d0fddfe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:15:42,613 - tests.orchestrator - INFO - Pod test-pod-0d0fddfe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:15:44,682 - tests.orchestrator - INFO - Pod test-pod-0d0fddfe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:15:46,750 - tests.orchestrator - INFO - Pod test-pod-0d0fddfe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:15:48,818 - tests.orchestrator - INFO - Pod test-pod-0d0fddfe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:15:50,886 - tests.orchestrator - INFO - Pod test-pod-0d0fddfe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:15:52,955 - tests.orchestrator - INFO - Pod test-pod-0d0fddfe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 19:15:52,955 - tests.orchestrator - INFO - Pod test-pod-0d0fddfe is ready +2025-06-05 19:15:53,955 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 2 times) +2025-06-05 19:15:53,955 - tests.orchestrator - INFO - Deleting PVC: test-pvc-a0df5547 +2025-06-05 19:15:54,028 - tests.orchestrator - INFO - Waiting for PVC test-pvc-a0df5547 to be deleted +2025-06-05 19:15:54,099 - tests.orchestrator - INFO - PVC test-pvc-a0df5547 has been deleted +2025-06-05 19:15:54,099 - tests.orchestrator - INFO - Deleted PVC: test-pvc-a0df5547 +2025-06-05 19:15:55,100 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 2 times) +2025-06-05 19:15:55,100 - tests.orchestrator - INFO - Creating pod test-pod-158cdfa4 on PVC test-pvc-31fbd501 +2025-06-05 19:15:55,181 - tests.orchestrator - INFO - Created pod: test-pod-158cdfa4 using PVC: test-pvc-31fbd501 +2025-06-05 19:15:55,181 - tests.orchestrator - INFO - Waiting for pod test-pod-158cdfa4 to be ready +2025-06-05 19:15:55,248 - tests.orchestrator - INFO - Pod test-pod-158cdfa4 phase: Pending +2025-06-05 19:15:59,387 - tests.orchestrator - INFO - Pod test-pod-158cdfa4 phase: Running +2025-06-05 19:15:59,387 - tests.orchestrator - INFO - Pod test-pod-158cdfa4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_192824.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_192824.log new file mode 100644 index 000000000..834b6116e --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_192824.log @@ -0,0 +1,7 @@ +2025-06-05 19:28:24,606 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 19:28:24,815 - tests.orchestrator - ERROR - Error checking namespace: (401) +Reason: Unauthorized +HTTP response headers: HTTPHeaderDict({'Audit-Id': '0dff4cee-4147-4a36-b942-2640bcda6774', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 19:28:24 GMT', 'Content-Length': '129'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} + + diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_192851.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_192851.log new file mode 100644 index 000000000..164f8db03 --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_192851.log @@ -0,0 +1,207 @@ +2025-06-05 19:28:51,597 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 19:28:51,800 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 19:28:51,800 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-05 19:28:51,990 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 19:28:51,990 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 19:28:51,991 - tests.orchestrator - INFO - Creating PVC: test-pvc-4046c78c +2025-06-05 19:28:52,067 - tests.orchestrator - INFO - Created PVC: test-pvc-4046c78c +2025-06-05 19:28:52,067 - tests.orchestrator - INFO - Waiting for PVC test-pvc-4046c78c to be bound +2025-06-05 19:28:54,199 - tests.orchestrator - INFO - PVC test-pvc-4046c78c is bound +2025-06-05 19:28:54,200 - tests.orchestrator - INFO - Creating PVC: test-pvc-d086aade +2025-06-05 19:28:54,267 - tests.orchestrator - INFO - Created PVC: test-pvc-d086aade +2025-06-05 19:28:54,267 - tests.orchestrator - INFO - Waiting for PVC test-pvc-d086aade to be bound +2025-06-05 19:28:56,399 - tests.orchestrator - INFO - PVC test-pvc-d086aade is bound +2025-06-05 19:28:56,399 - tests.orchestrator - INFO - Creating pod test-pod-7c93ffdb on PVC test-pvc-d086aade +2025-06-05 19:28:56,484 - tests.orchestrator - INFO - Created pod: test-pod-7c93ffdb using PVC: test-pvc-d086aade +2025-06-05 19:28:56,484 - tests.orchestrator - INFO - Waiting for pod test-pod-7c93ffdb to be ready +2025-06-05 19:28:56,552 - tests.orchestrator - INFO - Pod test-pod-7c93ffdb phase: Pending +2025-06-05 19:29:00,688 - tests.orchestrator - INFO - Pod test-pod-7c93ffdb phase: Running +2025-06-05 19:29:00,688 - tests.orchestrator - INFO - Pod test-pod-7c93ffdb conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:29:02,755 - tests.orchestrator - INFO - Pod test-pod-7c93ffdb conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:29:04,823 - tests.orchestrator - INFO - Pod test-pod-7c93ffdb conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:29:06,896 - tests.orchestrator - INFO - Pod test-pod-7c93ffdb conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:29:08,964 - tests.orchestrator - INFO - Pod test-pod-7c93ffdb conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:29:11,031 - tests.orchestrator - INFO - Pod test-pod-7c93ffdb conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:29:13,099 - tests.orchestrator - INFO - Pod test-pod-7c93ffdb conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:29:15,166 - tests.orchestrator - INFO - Pod test-pod-7c93ffdb conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:29:17,234 - tests.orchestrator - INFO - Pod test-pod-7c93ffdb conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:29:19,302 - tests.orchestrator - INFO - Pod test-pod-7c93ffdb conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 19:29:19,302 - tests.orchestrator - INFO - Pod test-pod-7c93ffdb is ready +2025-06-05 19:29:19,302 - tests.orchestrator - INFO - Creating pod test-pod-7340c349 on PVC test-pvc-d086aade +2025-06-05 19:29:19,376 - tests.orchestrator - INFO - Created pod: test-pod-7340c349 using PVC: test-pvc-d086aade +2025-06-05 19:29:19,376 - tests.orchestrator - INFO - Waiting for pod test-pod-7340c349 to be ready +2025-06-05 19:29:19,443 - tests.orchestrator - INFO - Pod test-pod-7340c349 phase: Pending +2025-06-05 19:29:21,512 - tests.orchestrator - INFO - Pod test-pod-7340c349 phase: Running +2025-06-05 19:29:21,512 - tests.orchestrator - INFO - Pod test-pod-7340c349 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:29:23,580 - tests.orchestrator - INFO - Pod test-pod-7340c349 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:29:25,647 - tests.orchestrator - INFO - Pod test-pod-7340c349 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:29:27,715 - tests.orchestrator - INFO - Pod test-pod-7340c349 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:29:29,783 - tests.orchestrator - INFO - Pod test-pod-7340c349 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:29:31,854 - tests.orchestrator - INFO - Pod test-pod-7340c349 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:29:33,922 - tests.orchestrator - INFO - Pod test-pod-7340c349 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:29:35,989 - tests.orchestrator - INFO - Pod test-pod-7340c349 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:29:38,057 - tests.orchestrator - INFO - Pod test-pod-7340c349 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:29:40,125 - tests.orchestrator - INFO - Pod test-pod-7340c349 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 19:29:40,125 - tests.orchestrator - INFO - Pod test-pod-7340c349 is ready +2025-06-05 19:29:40,125 - tests.orchestrator - INFO - Creating pod test-pod-b4873308 on PVC test-pvc-4046c78c +2025-06-05 19:29:40,208 - tests.orchestrator - INFO - Created pod: test-pod-b4873308 using PVC: test-pvc-4046c78c +2025-06-05 19:29:40,208 - tests.orchestrator - INFO - Waiting for pod test-pod-b4873308 to be ready +2025-06-05 19:29:40,275 - tests.orchestrator - INFO - Pod test-pod-b4873308 phase: Pending +2025-06-05 19:29:42,343 - tests.orchestrator - INFO - Pod test-pod-b4873308 phase: Running +2025-06-05 19:29:42,343 - tests.orchestrator - INFO - Pod test-pod-b4873308 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:29:44,411 - tests.orchestrator - INFO - Pod test-pod-b4873308 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:29:46,478 - tests.orchestrator - INFO - Pod test-pod-b4873308 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:29:48,546 - tests.orchestrator - INFO - Pod test-pod-b4873308 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:29:50,614 - tests.orchestrator - INFO - Pod test-pod-b4873308 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:29:52,681 - tests.orchestrator - INFO - Pod test-pod-b4873308 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:29:54,749 - tests.orchestrator - INFO - Pod test-pod-b4873308 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:29:56,817 - tests.orchestrator - INFO - Pod test-pod-b4873308 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 19:29:56,817 - tests.orchestrator - INFO - Pod test-pod-b4873308 is ready +2025-06-05 19:29:56,817 - tests.orchestrator - INFO - Testing read/write between pods test-pod-7340c349 and test-pod-7c93ffdb sharing PVC test-pvc-d086aade +2025-06-05 19:29:56,817 - tests.orchestrator - INFO - Executing write command: kubectl exec -n efs-stress-test test-pod-7340c349 -- /bin/sh -c 'echo "Test content: 0a7dd7bd-4b4d-493f-9b70-64073a2558bf" > /data/test-7c27f5ae.txt' +2025-06-05 19:29:58,091 - tests.orchestrator - INFO - Pod test-pod-7340c349 wrote to /data/test-7c27f5ae.txt +2025-06-05 19:30:00,091 - tests.orchestrator - INFO - Executing read command: kubectl exec -n efs-stress-test test-pod-7c93ffdb -- cat /data/test-7c27f5ae.txt +2025-06-05 19:30:01,307 - tests.orchestrator - INFO - Read result: 'Test content: 0a7dd7bd-4b4d-493f-9b70-64073a2558bf' +2025-06-05 19:30:01,307 - tests.orchestrator - INFO - Pod test-pod-7c93ffdb successfully read content written by test-pod-7340c349 +2025-06-05 19:30:01,307 - tests.orchestrator - INFO - Deleting pod: test-pod-b4873308 from PVC: test-pvc-4046c78c +2025-06-05 19:30:01,397 - tests.orchestrator - INFO - Waiting for pod test-pod-b4873308 to be deleted +2025-06-05 19:30:34,119 - tests.orchestrator - INFO - Pod test-pod-b4873308 has been deleted +2025-06-05 19:30:34,119 - tests.orchestrator - INFO - Deleted pod: test-pod-b4873308 +2025-06-05 19:30:34,119 - tests.orchestrator - INFO - Deleting PVC: test-pvc-4046c78c +2025-06-05 19:30:34,189 - tests.orchestrator - INFO - Waiting for PVC test-pvc-4046c78c to be deleted +2025-06-05 19:30:34,254 - tests.orchestrator - INFO - PVC test-pvc-4046c78c has been deleted +2025-06-05 19:30:34,255 - tests.orchestrator - INFO - Deleted PVC: test-pvc-4046c78c +2025-06-05 19:30:34,255 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations +2025-06-05 19:30:34,255 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 1 times) +2025-06-05 19:30:34,255 - tests.orchestrator - INFO - Creating PVC: test-pvc-2da07839 +2025-06-05 19:30:34,332 - tests.orchestrator - INFO - Created PVC: test-pvc-2da07839 +2025-06-05 19:30:34,332 - tests.orchestrator - INFO - Waiting for PVC test-pvc-2da07839 to be bound +2025-06-05 19:30:36,474 - tests.orchestrator - INFO - PVC test-pvc-2da07839 is bound +2025-06-05 19:30:37,474 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 1 times) +2025-06-05 19:30:37,474 - tests.orchestrator - INFO - Creating pod test-pod-ada6dc83 on PVC test-pvc-2da07839 +2025-06-05 19:30:37,547 - tests.orchestrator - INFO - Created pod: test-pod-ada6dc83 using PVC: test-pvc-2da07839 +2025-06-05 19:30:37,547 - tests.orchestrator - INFO - Waiting for pod test-pod-ada6dc83 to be ready +2025-06-05 19:30:37,614 - tests.orchestrator - INFO - Pod test-pod-ada6dc83 phase: Pending +2025-06-05 19:30:41,747 - tests.orchestrator - INFO - Pod test-pod-ada6dc83 phase: Running +2025-06-05 19:30:41,747 - tests.orchestrator - INFO - Pod test-pod-ada6dc83 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:30:43,815 - tests.orchestrator - INFO - Pod test-pod-ada6dc83 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:30:45,883 - tests.orchestrator - INFO - Pod test-pod-ada6dc83 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:30:47,950 - tests.orchestrator - INFO - Pod test-pod-ada6dc83 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:30:50,019 - tests.orchestrator - INFO - Pod test-pod-ada6dc83 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:30:52,088 - tests.orchestrator - INFO - Pod test-pod-ada6dc83 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:30:54,154 - tests.orchestrator - INFO - Pod test-pod-ada6dc83 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:30:56,222 - tests.orchestrator - INFO - Pod test-pod-ada6dc83 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:30:58,290 - tests.orchestrator - INFO - Pod test-pod-ada6dc83 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:31:00,358 - tests.orchestrator - INFO - Pod test-pod-ada6dc83 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 19:31:00,358 - tests.orchestrator - INFO - Pod test-pod-ada6dc83 is ready +2025-06-05 19:31:01,359 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 1 times) +2025-06-05 19:31:01,359 - tests.orchestrator - INFO - Deleting pod: test-pod-7340c349 from PVC: test-pvc-d086aade +2025-06-05 19:31:01,460 - tests.orchestrator - INFO - Waiting for pod test-pod-7340c349 to be deleted +2025-06-05 19:31:34,130 - tests.orchestrator - INFO - Pod test-pod-7340c349 has been deleted +2025-06-05 19:31:34,130 - tests.orchestrator - INFO - Deleted pod: test-pod-7340c349 +2025-06-05 19:31:35,130 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 2 times) +2025-06-05 19:31:35,131 - tests.orchestrator - INFO - Deleting pod: test-pod-7c93ffdb from PVC: test-pvc-d086aade +2025-06-05 19:31:35,204 - tests.orchestrator - INFO - Waiting for pod test-pod-7c93ffdb to be deleted +2025-06-05 19:32:07,811 - tests.orchestrator - INFO - Pod test-pod-7c93ffdb has been deleted +2025-06-05 19:32:07,811 - tests.orchestrator - INFO - Deleted pod: test-pod-7c93ffdb +2025-06-05 19:32:08,811 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 2 times) +2025-06-05 19:32:08,812 - tests.orchestrator - INFO - Creating pod test-pod-98d8b402 on PVC test-pvc-d086aade +2025-06-05 19:32:08,902 - tests.orchestrator - INFO - Created pod: test-pod-98d8b402 using PVC: test-pvc-d086aade +2025-06-05 19:32:08,902 - tests.orchestrator - INFO - Waiting for pod test-pod-98d8b402 to be ready +2025-06-05 19:32:08,972 - tests.orchestrator - INFO - Pod test-pod-98d8b402 phase: Pending +2025-06-05 19:32:13,114 - tests.orchestrator - INFO - Pod test-pod-98d8b402 phase: Running +2025-06-05 19:32:13,115 - tests.orchestrator - INFO - Pod test-pod-98d8b402 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:32:15,182 - tests.orchestrator - INFO - Pod test-pod-98d8b402 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:32:17,250 - tests.orchestrator - INFO - Pod test-pod-98d8b402 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:32:19,317 - tests.orchestrator - INFO - Pod test-pod-98d8b402 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:32:21,384 - tests.orchestrator - INFO - Pod test-pod-98d8b402 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:32:23,453 - tests.orchestrator - INFO - Pod test-pod-98d8b402 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:32:25,520 - tests.orchestrator - INFO - Pod test-pod-98d8b402 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:32:27,588 - tests.orchestrator - INFO - Pod test-pod-98d8b402 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:32:29,656 - tests.orchestrator - INFO - Pod test-pod-98d8b402 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:32:31,766 - tests.orchestrator - INFO - Pod test-pod-98d8b402 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 19:32:31,766 - tests.orchestrator - INFO - Pod test-pod-98d8b402 is ready +2025-06-05 19:32:32,766 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 2 times) +2025-06-05 19:32:32,766 - tests.orchestrator - INFO - Creating PVC: test-pvc-4a306e31 +2025-06-05 19:32:32,835 - tests.orchestrator - INFO - Created PVC: test-pvc-4a306e31 +2025-06-05 19:32:32,835 - tests.orchestrator - INFO - Waiting for PVC test-pvc-4a306e31 to be bound +2025-06-05 19:32:34,969 - tests.orchestrator - INFO - PVC test-pvc-4a306e31 is bound +2025-06-05 19:32:35,969 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 3 times) +2025-06-05 19:32:35,969 - tests.orchestrator - INFO - Deleting pod: test-pod-98d8b402 from PVC: test-pvc-d086aade +2025-06-05 19:32:36,045 - tests.orchestrator - INFO - Waiting for pod test-pod-98d8b402 to be deleted +2025-06-05 19:33:08,653 - tests.orchestrator - INFO - Pod test-pod-98d8b402 has been deleted +2025-06-05 19:33:08,653 - tests.orchestrator - INFO - Deleted pod: test-pod-98d8b402 +2025-06-05 19:33:09,653 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 3 times) +2025-06-05 19:33:09,653 - tests.orchestrator - INFO - Creating PVC: test-pvc-1d05425d +2025-06-05 19:33:09,730 - tests.orchestrator - INFO - Created PVC: test-pvc-1d05425d +2025-06-05 19:33:09,730 - tests.orchestrator - INFO - Waiting for PVC test-pvc-1d05425d to be bound +2025-06-05 19:33:11,862 - tests.orchestrator - INFO - PVC test-pvc-1d05425d is bound +2025-06-05 19:33:12,863 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 4 times) +2025-06-05 19:33:12,863 - tests.orchestrator - INFO - Creating PVC: test-pvc-16969258 +2025-06-05 19:33:12,931 - tests.orchestrator - INFO - Created PVC: test-pvc-16969258 +2025-06-05 19:33:12,931 - tests.orchestrator - INFO - Waiting for PVC test-pvc-16969258 to be bound +2025-06-05 19:33:15,064 - tests.orchestrator - INFO - PVC test-pvc-16969258 is bound +2025-06-05 19:33:16,064 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 1 times) +2025-06-05 19:33:16,064 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test +2025-06-05 19:33:17,065 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 1 times) +2025-06-05 19:33:17,065 - tests.orchestrator - INFO - Deleting PVC: test-pvc-d086aade +2025-06-05 19:33:17,139 - tests.orchestrator - INFO - Waiting for PVC test-pvc-d086aade to be deleted +2025-06-05 19:33:17,204 - tests.orchestrator - INFO - PVC test-pvc-d086aade has been deleted +2025-06-05 19:33:17,204 - tests.orchestrator - INFO - Deleted PVC: test-pvc-d086aade +2025-06-05 19:33:18,204 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 5 times) +2025-06-05 19:33:18,204 - tests.orchestrator - INFO - Creating PVC: test-pvc-8e9317d0 +2025-06-05 19:33:18,272 - tests.orchestrator - INFO - Created PVC: test-pvc-8e9317d0 +2025-06-05 19:33:18,272 - tests.orchestrator - INFO - Waiting for PVC test-pvc-8e9317d0 to be bound +2025-06-05 19:33:20,404 - tests.orchestrator - INFO - PVC test-pvc-8e9317d0 is bound +2025-06-05 19:33:21,405 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 2 times) +2025-06-05 19:33:21,405 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test +2025-06-05 19:33:22,405 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 4 times) +2025-06-05 19:33:22,405 - tests.orchestrator - INFO - Deleting pod: test-pod-ada6dc83 from PVC: test-pvc-2da07839 +2025-06-05 19:33:22,481 - tests.orchestrator - INFO - Waiting for pod test-pod-ada6dc83 to be deleted +2025-06-05 19:33:55,087 - tests.orchestrator - INFO - Pod test-pod-ada6dc83 has been deleted +2025-06-05 19:33:55,088 - tests.orchestrator - INFO - Deleted pod: test-pod-ada6dc83 +2025-06-05 19:33:56,088 - tests.orchestrator - INFO - Test completed in 304.29 seconds +2025-06-05 19:33:56,088 - tests.orchestrator - INFO - ===== STARTING COMPREHENSIVE CLEANUP ===== +2025-06-05 19:33:56,088 - tests.orchestrator - INFO - Deleting 0 pods (force=False)... +2025-06-05 19:34:01,088 - tests.orchestrator - INFO - Deleting 5 PVCs (force=False)... +2025-06-05 19:34:01,088 - tests.orchestrator - INFO - Deleting PVC: test-pvc-2da07839 +2025-06-05 19:34:01,160 - tests.orchestrator - INFO - Waiting for PVC test-pvc-2da07839 to be deleted +2025-06-05 19:34:01,226 - tests.orchestrator - INFO - PVC test-pvc-2da07839 has been deleted +2025-06-05 19:34:01,226 - tests.orchestrator - INFO - Deleted PVC: test-pvc-2da07839 +2025-06-05 19:34:01,226 - tests.orchestrator - INFO - Deleting PVC: test-pvc-4a306e31 +2025-06-05 19:34:01,298 - tests.orchestrator - INFO - Waiting for PVC test-pvc-4a306e31 to be deleted +2025-06-05 19:34:01,364 - tests.orchestrator - INFO - PVC test-pvc-4a306e31 has been deleted +2025-06-05 19:34:01,364 - tests.orchestrator - INFO - Deleted PVC: test-pvc-4a306e31 +2025-06-05 19:34:01,364 - tests.orchestrator - INFO - Deleting PVC: test-pvc-1d05425d +2025-06-05 19:34:01,514 - tests.orchestrator - INFO - Waiting for PVC test-pvc-1d05425d to be deleted +2025-06-05 19:34:01,667 - tests.orchestrator - INFO - PVC test-pvc-1d05425d has been deleted +2025-06-05 19:34:01,667 - tests.orchestrator - INFO - Deleted PVC: test-pvc-1d05425d +2025-06-05 19:34:01,667 - tests.orchestrator - INFO - Deleting PVC: test-pvc-16969258 +2025-06-05 19:34:01,774 - tests.orchestrator - INFO - Waiting for PVC test-pvc-16969258 to be deleted +2025-06-05 19:34:01,847 - tests.orchestrator - INFO - PVC test-pvc-16969258 has been deleted +2025-06-05 19:34:01,847 - tests.orchestrator - INFO - Deleted PVC: test-pvc-16969258 +2025-06-05 19:34:01,847 - tests.orchestrator - INFO - Deleting PVC: test-pvc-8e9317d0 +2025-06-05 19:34:01,925 - tests.orchestrator - INFO - Waiting for PVC test-pvc-8e9317d0 to be deleted +2025-06-05 19:34:01,996 - tests.orchestrator - INFO - PVC test-pvc-8e9317d0 has been deleted +2025-06-05 19:34:01,996 - tests.orchestrator - INFO - Deleted PVC: test-pvc-8e9317d0 +2025-06-05 19:34:02,206 - tests.orchestrator - WARNING - First cleanup pass incomplete. Remaining resources: ['pod/test-pod-0d0fddfe', 'pod/test-pod-158cdfa4', 'pvc/test-pvc-0bb0d20c', 'pvc/test-pvc-2127e71a', 'pvc/test-pvc-2cf5b7a2', 'pvc/test-pvc-31fbd501', 'pvc/test-pvc-528b4f5c', 'pvc/test-pvc-ecb12f74'] +2025-06-05 19:34:02,206 - tests.orchestrator - INFO - Attempting force deletion of remaining resources... +2025-06-05 19:34:02,206 - tests.orchestrator - INFO - Deleting 0 pods (force=True)... +2025-06-05 19:34:07,206 - tests.orchestrator - INFO - Deleting 0 PVCs (force=True)... +2025-06-05 19:34:07,403 - tests.orchestrator - ERROR - Cleanup incomplete. Remaining resources after force deletion: ['pod/test-pod-0d0fddfe', 'pod/test-pod-158cdfa4', 'pvc/test-pvc-0bb0d20c', 'pvc/test-pvc-2127e71a', 'pvc/test-pvc-2cf5b7a2', 'pvc/test-pvc-31fbd501', 'pvc/test-pvc-528b4f5c', 'pvc/test-pvc-ecb12f74'] +2025-06-05 19:34:07,403 - tests.orchestrator - INFO - Cleanup completed successfully in 11.32 seconds +2025-06-05 19:34:07,404 - tests.orchestrator - INFO - ===== CLEANUP PROCESS FINISHED ===== +2025-06-05 19:34:07,404 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-05 19:34:07,404 - tests.orchestrator - INFO - --- Operations --- +2025-06-05 19:34:07,404 - tests.orchestrator - INFO - create_pvc: 7 succeeded, 0 failed (100.0%) +2025-06-05 19:34:07,404 - tests.orchestrator - INFO - attach_pod: 5 succeeded, 0 failed (100.0%) +2025-06-05 19:34:07,404 - tests.orchestrator - INFO - delete_pod: 5 succeeded, 0 failed (100.0%) +2025-06-05 19:34:07,404 - tests.orchestrator - INFO - delete_pvc: 7 succeeded, 0 failed (100.0%) +2025-06-05 19:34:07,404 - tests.orchestrator - INFO - verify_read_write: Writes 1 succeeded, 0 failed (100.0%) +2025-06-05 19:34:07,404 - tests.orchestrator - INFO - verify_read_write: Reads 1 succeeded, 0 failed (100.0%) +2025-06-05 19:34:07,404 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-05 19:34:07,404 - tests.orchestrator - INFO - shared_volume_rw: 1 succeeded, 0 failed out of 1 runs (100.0%) +2025-06-05 19:34:07,404 - tests.orchestrator - INFO - many_to_one: No runs +2025-06-05 19:34:07,404 - tests.orchestrator - INFO - one_to_one: No runs +2025-06-05 19:34:07,404 - tests.orchestrator - INFO - concurrent_pvc: No runs +2025-06-05 19:34:07,404 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_195225.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_195225.log new file mode 100644 index 000000000..cf0578c8f --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_195225.log @@ -0,0 +1,335 @@ +2025-06-05 19:52:25,645 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 19:52:25,880 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 19:52:25,881 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-05 19:52:26,070 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 19:52:26,071 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 19:52:26,071 - tests.orchestrator - INFO - Creating PVC: test-pvc-b5f8638e +2025-06-05 19:52:26,146 - tests.orchestrator - INFO - Created PVC: test-pvc-b5f8638e +2025-06-05 19:52:26,146 - tests.orchestrator - INFO - Waiting for PVC test-pvc-b5f8638e to be bound +2025-06-05 19:52:28,278 - tests.orchestrator - INFO - PVC test-pvc-b5f8638e is bound +2025-06-05 19:52:28,278 - tests.orchestrator - INFO - Creating PVC: test-pvc-81ab07bf +2025-06-05 19:52:28,347 - tests.orchestrator - INFO - Created PVC: test-pvc-81ab07bf +2025-06-05 19:52:28,347 - tests.orchestrator - INFO - Waiting for PVC test-pvc-81ab07bf to be bound +2025-06-05 19:52:30,483 - tests.orchestrator - INFO - PVC test-pvc-81ab07bf is bound +2025-06-05 19:52:30,483 - tests.orchestrator - INFO - Creating pod test-pod-a16d9846 on PVC test-pvc-81ab07bf +2025-06-05 19:52:30,563 - tests.orchestrator - INFO - Created pod: test-pod-a16d9846 using PVC: test-pvc-81ab07bf +2025-06-05 19:52:30,563 - tests.orchestrator - INFO - Waiting for pod test-pod-a16d9846 to be ready +2025-06-05 19:52:30,632 - tests.orchestrator - INFO - Pod test-pod-a16d9846 phase: Pending +2025-06-05 19:52:32,699 - tests.orchestrator - INFO - Pod test-pod-a16d9846 phase: Running +2025-06-05 19:52:32,700 - tests.orchestrator - INFO - Pod test-pod-a16d9846 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:52:34,768 - tests.orchestrator - INFO - Pod test-pod-a16d9846 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:52:36,837 - tests.orchestrator - INFO - Pod test-pod-a16d9846 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:52:38,905 - tests.orchestrator - INFO - Pod test-pod-a16d9846 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:52:40,973 - tests.orchestrator - INFO - Pod test-pod-a16d9846 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:52:43,042 - tests.orchestrator - INFO - Pod test-pod-a16d9846 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:52:45,109 - tests.orchestrator - INFO - Pod test-pod-a16d9846 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:52:47,178 - tests.orchestrator - INFO - Pod test-pod-a16d9846 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 19:52:47,178 - tests.orchestrator - INFO - Pod test-pod-a16d9846 is ready +2025-06-05 19:52:47,178 - tests.orchestrator - INFO - Creating pod test-pod-28ec4f7d on PVC test-pvc-b5f8638e +2025-06-05 19:52:47,253 - tests.orchestrator - INFO - Created pod: test-pod-28ec4f7d using PVC: test-pvc-b5f8638e +2025-06-05 19:52:47,253 - tests.orchestrator - INFO - Waiting for pod test-pod-28ec4f7d to be ready +2025-06-05 19:52:47,321 - tests.orchestrator - INFO - Pod test-pod-28ec4f7d phase: Pending +2025-06-05 19:52:51,458 - tests.orchestrator - INFO - Pod test-pod-28ec4f7d phase: Running +2025-06-05 19:52:51,458 - tests.orchestrator - INFO - Pod test-pod-28ec4f7d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:52:53,527 - tests.orchestrator - INFO - Pod test-pod-28ec4f7d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:52:55,594 - tests.orchestrator - INFO - Pod test-pod-28ec4f7d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:52:57,662 - tests.orchestrator - INFO - Pod test-pod-28ec4f7d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:52:59,731 - tests.orchestrator - INFO - Pod test-pod-28ec4f7d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:53:01,804 - tests.orchestrator - INFO - Pod test-pod-28ec4f7d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:53:03,871 - tests.orchestrator - INFO - Pod test-pod-28ec4f7d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 19:53:03,872 - tests.orchestrator - INFO - Pod test-pod-28ec4f7d is ready +2025-06-05 19:53:03,872 - tests.orchestrator - INFO - Creating pod test-pod-044934ff on PVC test-pvc-b5f8638e +2025-06-05 19:53:03,956 - tests.orchestrator - INFO - Created pod: test-pod-044934ff using PVC: test-pvc-b5f8638e +2025-06-05 19:53:03,956 - tests.orchestrator - INFO - Waiting for pod test-pod-044934ff to be ready +2025-06-05 19:53:04,024 - tests.orchestrator - INFO - Pod test-pod-044934ff phase: Pending +2025-06-05 19:53:08,160 - tests.orchestrator - INFO - Pod test-pod-044934ff phase: Running +2025-06-05 19:53:08,161 - tests.orchestrator - INFO - Pod test-pod-044934ff conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:53:10,230 - tests.orchestrator - INFO - Pod test-pod-044934ff conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:53:12,298 - tests.orchestrator - INFO - Pod test-pod-044934ff conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:53:14,366 - tests.orchestrator - INFO - Pod test-pod-044934ff conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:53:16,438 - tests.orchestrator - INFO - Pod test-pod-044934ff conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:53:18,506 - tests.orchestrator - INFO - Pod test-pod-044934ff conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:53:20,574 - tests.orchestrator - INFO - Pod test-pod-044934ff conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:53:22,644 - tests.orchestrator - INFO - Pod test-pod-044934ff conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:53:24,712 - tests.orchestrator - INFO - Pod test-pod-044934ff conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:53:26,780 - tests.orchestrator - INFO - Pod test-pod-044934ff conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 19:53:26,781 - tests.orchestrator - INFO - Pod test-pod-044934ff is ready +2025-06-05 19:53:26,781 - tests.orchestrator - INFO - Testing read/write between pods test-pod-044934ff and test-pod-28ec4f7d sharing PVC test-pvc-b5f8638e +2025-06-05 19:53:26,781 - tests.orchestrator - INFO - Executing write command: kubectl exec -n efs-stress-test test-pod-044934ff -- /bin/sh -c 'echo "Test content: 90b0f8ff-b5eb-4887-86da-4de1b6273ee1" > /data/test-309c7d73.txt' +2025-06-05 19:53:28,021 - tests.orchestrator - INFO - Pod test-pod-044934ff wrote to /data/test-309c7d73.txt +2025-06-05 19:53:30,021 - tests.orchestrator - INFO - Executing read command: kubectl exec -n efs-stress-test test-pod-28ec4f7d -- cat /data/test-309c7d73.txt +2025-06-05 19:53:31,221 - tests.orchestrator - INFO - Read result: 'Test content: 90b0f8ff-b5eb-4887-86da-4de1b6273ee1' +2025-06-05 19:53:31,221 - tests.orchestrator - INFO - Pod test-pod-28ec4f7d successfully read content written by test-pod-044934ff +2025-06-05 19:53:31,221 - tests.orchestrator - INFO - Deleting pod: test-pod-a16d9846 from PVC: test-pvc-81ab07bf +2025-06-05 19:53:31,296 - tests.orchestrator - INFO - Waiting for pod test-pod-a16d9846 to be deleted +2025-06-05 19:54:03,907 - tests.orchestrator - INFO - Pod test-pod-a16d9846 has been deleted +2025-06-05 19:54:03,907 - tests.orchestrator - INFO - Deleted pod: test-pod-a16d9846 +2025-06-05 19:54:03,907 - tests.orchestrator - INFO - Deleting PVC: test-pvc-81ab07bf +2025-06-05 19:54:03,979 - tests.orchestrator - INFO - Waiting for PVC test-pvc-81ab07bf to be deleted +2025-06-05 19:54:04,044 - tests.orchestrator - INFO - PVC test-pvc-81ab07bf has been deleted +2025-06-05 19:54:04,044 - tests.orchestrator - INFO - Deleted PVC: test-pvc-81ab07bf +2025-06-05 19:54:04,044 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations +2025-06-05 19:54:04,044 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 1 times) +2025-06-05 19:54:04,045 - tests.orchestrator - INFO - Testing read/write between pods test-pod-044934ff and test-pod-28ec4f7d sharing PVC test-pvc-b5f8638e +2025-06-05 19:54:04,045 - tests.orchestrator - INFO - Executing write command: kubectl exec -n efs-stress-test test-pod-044934ff -- /bin/sh -c 'echo "Test content: eafc327a-6e13-4907-8a5f-8caa39cd7559" > /data/test-cc6083d7.txt' +2025-06-05 19:54:05,250 - tests.orchestrator - INFO - Pod test-pod-044934ff wrote to /data/test-cc6083d7.txt +2025-06-05 19:54:07,250 - tests.orchestrator - INFO - Executing read command: kubectl exec -n efs-stress-test test-pod-28ec4f7d -- cat /data/test-cc6083d7.txt +2025-06-05 19:54:08,454 - tests.orchestrator - INFO - Read result: 'Test content: eafc327a-6e13-4907-8a5f-8caa39cd7559' +2025-06-05 19:54:08,454 - tests.orchestrator - INFO - Pod test-pod-28ec4f7d successfully read content written by test-pod-044934ff +2025-06-05 19:54:09,454 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario (selected 1 times) +2025-06-05 19:54:09,454 - tests.orchestrator - INFO - ============================================================ +2025-06-05 19:54:09,454 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_concurrent_pvc +2025-06-05 19:54:09,454 - tests.orchestrator - INFO - ============================================================ +2025-06-05 19:54:09,454 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations +2025-06-05 19:54:09,454 - tests.orchestrator - INFO - Creating 5 PVCs in quick succession +2025-06-05 19:54:09,531 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-1bfddf93 +2025-06-05 19:54:09,531 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-1bfddf93 to be bound +2025-06-05 19:54:11,664 - tests.orchestrator - INFO - PVC concurrent-pvc-1bfddf93 is bound +2025-06-05 19:54:11,732 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-0edb68b1 +2025-06-05 19:54:11,732 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-0edb68b1 to be bound +2025-06-05 19:54:13,866 - tests.orchestrator - INFO - PVC concurrent-pvc-0edb68b1 is bound +2025-06-05 19:54:13,933 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-25754791 +2025-06-05 19:54:13,933 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-25754791 to be bound +2025-06-05 19:54:16,067 - tests.orchestrator - INFO - PVC concurrent-pvc-25754791 is bound +2025-06-05 19:54:16,135 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-e4c09a13 +2025-06-05 19:54:16,135 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-e4c09a13 to be bound +2025-06-05 19:54:18,270 - tests.orchestrator - INFO - PVC concurrent-pvc-e4c09a13 is bound +2025-06-05 19:54:18,339 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-3f3c40ea +2025-06-05 19:54:18,339 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-3f3c40ea to be bound +2025-06-05 19:54:20,472 - tests.orchestrator - INFO - PVC concurrent-pvc-3f3c40ea is bound +2025-06-05 19:54:20,472 - tests.orchestrator - INFO - Successfully created 5 PVCs +2025-06-05 19:54:20,472 - tests.orchestrator - INFO - Creating pod test-pod-a0c335d2 on PVC concurrent-pvc-0edb68b1 +2025-06-05 19:54:20,547 - tests.orchestrator - INFO - Created pod: test-pod-a0c335d2 using PVC: concurrent-pvc-0edb68b1 +2025-06-05 19:54:20,547 - tests.orchestrator - INFO - Waiting for pod test-pod-a0c335d2 to be ready +2025-06-05 19:54:20,614 - tests.orchestrator - INFO - Pod test-pod-a0c335d2 phase: Pending +2025-06-05 19:54:22,683 - tests.orchestrator - INFO - Pod test-pod-a0c335d2 phase: Running +2025-06-05 19:54:22,683 - tests.orchestrator - INFO - Pod test-pod-a0c335d2 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:54:24,752 - tests.orchestrator - INFO - Pod test-pod-a0c335d2 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:54:26,819 - tests.orchestrator - INFO - Pod test-pod-a0c335d2 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:54:28,888 - tests.orchestrator - INFO - Pod test-pod-a0c335d2 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:54:30,956 - tests.orchestrator - INFO - Pod test-pod-a0c335d2 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:54:33,024 - tests.orchestrator - INFO - Pod test-pod-a0c335d2 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:54:35,092 - tests.orchestrator - INFO - Pod test-pod-a0c335d2 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:54:37,161 - tests.orchestrator - INFO - Pod test-pod-a0c335d2 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:54:39,228 - tests.orchestrator - INFO - Pod test-pod-a0c335d2 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:54:41,296 - tests.orchestrator - INFO - Pod test-pod-a0c335d2 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:54:43,365 - tests.orchestrator - INFO - Pod test-pod-a0c335d2 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 19:54:43,365 - tests.orchestrator - INFO - Pod test-pod-a0c335d2 is ready +2025-06-05 19:54:43,365 - tests.orchestrator - INFO - Creating pod test-pod-a24dabfe on PVC concurrent-pvc-3f3c40ea +2025-06-05 19:54:43,449 - tests.orchestrator - INFO - Created pod: test-pod-a24dabfe using PVC: concurrent-pvc-3f3c40ea +2025-06-05 19:54:43,449 - tests.orchestrator - INFO - Waiting for pod test-pod-a24dabfe to be ready +2025-06-05 19:54:43,516 - tests.orchestrator - INFO - Pod test-pod-a24dabfe phase: Pending +2025-06-05 19:54:47,656 - tests.orchestrator - INFO - Pod test-pod-a24dabfe phase: Running +2025-06-05 19:54:47,656 - tests.orchestrator - INFO - Pod test-pod-a24dabfe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:54:49,725 - tests.orchestrator - INFO - Pod test-pod-a24dabfe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:54:51,793 - tests.orchestrator - INFO - Pod test-pod-a24dabfe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:54:53,862 - tests.orchestrator - INFO - Pod test-pod-a24dabfe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:54:55,930 - tests.orchestrator - INFO - Pod test-pod-a24dabfe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:54:57,998 - tests.orchestrator - INFO - Pod test-pod-a24dabfe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:55:00,068 - tests.orchestrator - INFO - Pod test-pod-a24dabfe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:55:02,150 - tests.orchestrator - INFO - Pod test-pod-a24dabfe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:55:04,218 - tests.orchestrator - INFO - Pod test-pod-a24dabfe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:55:06,287 - tests.orchestrator - INFO - Pod test-pod-a24dabfe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 19:55:06,287 - tests.orchestrator - INFO - Pod test-pod-a24dabfe is ready +2025-06-05 19:55:06,287 - tests.orchestrator - INFO - Creating pod test-pod-cbfe1bf0 on PVC concurrent-pvc-25754791 +2025-06-05 19:55:06,364 - tests.orchestrator - INFO - Created pod: test-pod-cbfe1bf0 using PVC: concurrent-pvc-25754791 +2025-06-05 19:55:06,364 - tests.orchestrator - INFO - Waiting for pod test-pod-cbfe1bf0 to be ready +2025-06-05 19:55:06,431 - tests.orchestrator - INFO - Pod test-pod-cbfe1bf0 phase: Pending +2025-06-05 19:55:10,568 - tests.orchestrator - INFO - Pod test-pod-cbfe1bf0 phase: Running +2025-06-05 19:55:10,568 - tests.orchestrator - INFO - Pod test-pod-cbfe1bf0 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:55:12,636 - tests.orchestrator - INFO - Pod test-pod-cbfe1bf0 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:55:14,704 - tests.orchestrator - INFO - Pod test-pod-cbfe1bf0 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:55:16,773 - tests.orchestrator - INFO - Pod test-pod-cbfe1bf0 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:55:18,840 - tests.orchestrator - INFO - Pod test-pod-cbfe1bf0 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:55:20,909 - tests.orchestrator - INFO - Pod test-pod-cbfe1bf0 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:55:22,978 - tests.orchestrator - INFO - Pod test-pod-cbfe1bf0 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:55:25,045 - tests.orchestrator - INFO - Pod test-pod-cbfe1bf0 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:55:27,114 - tests.orchestrator - INFO - Pod test-pod-cbfe1bf0 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 19:55:27,114 - tests.orchestrator - INFO - Pod test-pod-cbfe1bf0 is ready +2025-06-05 19:55:27,114 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-1bfddf93 +2025-06-05 19:55:27,186 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-1bfddf93 to be deleted +2025-06-05 19:55:27,252 - tests.orchestrator - INFO - PVC concurrent-pvc-1bfddf93 has been deleted +2025-06-05 19:55:27,252 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-1bfddf93 +2025-06-05 19:55:27,253 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-25754791 +2025-06-05 19:55:27,253 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-25754791 +2025-06-05 19:55:27,253 - tests.orchestrator - INFO - Deleting pod: test-pod-cbfe1bf0 from PVC: concurrent-pvc-25754791 +2025-06-05 19:55:27,328 - tests.orchestrator - INFO - Waiting for pod test-pod-cbfe1bf0 to be deleted +2025-06-05 19:55:59,981 - tests.orchestrator - INFO - Pod test-pod-cbfe1bf0 has been deleted +2025-06-05 19:55:59,981 - tests.orchestrator - INFO - Deleted pod: test-pod-cbfe1bf0 +2025-06-05 19:56:00,058 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-25754791 to be deleted +2025-06-05 19:56:00,122 - tests.orchestrator - INFO - PVC concurrent-pvc-25754791 has been deleted +2025-06-05 19:56:00,123 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-25754791 +2025-06-05 19:56:00,123 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-3f3c40ea +2025-06-05 19:56:00,123 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-3f3c40ea +2025-06-05 19:56:00,123 - tests.orchestrator - INFO - Deleting pod: test-pod-a24dabfe from PVC: concurrent-pvc-3f3c40ea +2025-06-05 19:56:00,197 - tests.orchestrator - INFO - Waiting for pod test-pod-a24dabfe to be deleted +2025-06-05 19:56:32,809 - tests.orchestrator - INFO - Pod test-pod-a24dabfe has been deleted +2025-06-05 19:56:32,809 - tests.orchestrator - INFO - Deleted pod: test-pod-a24dabfe +2025-06-05 19:56:32,881 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-3f3c40ea to be deleted +2025-06-05 19:56:32,947 - tests.orchestrator - INFO - PVC concurrent-pvc-3f3c40ea has been deleted +2025-06-05 19:56:32,947 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-3f3c40ea +2025-06-05 19:56:32,947 - tests.orchestrator - INFO - Rapid PVC scenario completed +2025-06-05 19:56:32,947 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_concurrent_pvc +2025-06-05 19:56:32,947 - tests.orchestrator - INFO - ------------------------------------------------------------ +2025-06-05 19:56:33,947 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario (selected 2 times) +2025-06-05 19:56:33,947 - tests.orchestrator - INFO - ============================================================ +2025-06-05 19:56:33,947 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_concurrent_pvc +2025-06-05 19:56:33,947 - tests.orchestrator - INFO - ============================================================ +2025-06-05 19:56:33,948 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations +2025-06-05 19:56:33,948 - tests.orchestrator - INFO - Creating 6 PVCs in quick succession +2025-06-05 19:56:34,025 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-cbac8cde +2025-06-05 19:56:34,025 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-cbac8cde to be bound +2025-06-05 19:56:36,158 - tests.orchestrator - INFO - PVC concurrent-pvc-cbac8cde is bound +2025-06-05 19:56:36,227 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-12483e38 +2025-06-05 19:56:36,227 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-12483e38 to be bound +2025-06-05 19:56:38,360 - tests.orchestrator - INFO - PVC concurrent-pvc-12483e38 is bound +2025-06-05 19:56:38,428 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-ab9c791a +2025-06-05 19:56:38,428 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-ab9c791a to be bound +2025-06-05 19:56:40,563 - tests.orchestrator - INFO - PVC concurrent-pvc-ab9c791a is bound +2025-06-05 19:56:40,630 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-ac5d027e +2025-06-05 19:56:40,630 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-ac5d027e to be bound +2025-06-05 19:56:42,763 - tests.orchestrator - INFO - PVC concurrent-pvc-ac5d027e is bound +2025-06-05 19:56:42,832 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-ee19f53f +2025-06-05 19:56:42,832 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-ee19f53f to be bound +2025-06-05 19:56:44,965 - tests.orchestrator - INFO - PVC concurrent-pvc-ee19f53f is bound +2025-06-05 19:56:45,033 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-f87f399d +2025-06-05 19:56:45,033 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-f87f399d to be bound +2025-06-05 19:56:47,168 - tests.orchestrator - INFO - PVC concurrent-pvc-f87f399d is bound +2025-06-05 19:56:47,168 - tests.orchestrator - INFO - Successfully created 6 PVCs +2025-06-05 19:56:47,168 - tests.orchestrator - INFO - Creating pod test-pod-025216b5 on PVC concurrent-pvc-ab9c791a +2025-06-05 19:56:47,249 - tests.orchestrator - INFO - Created pod: test-pod-025216b5 using PVC: concurrent-pvc-ab9c791a +2025-06-05 19:56:47,249 - tests.orchestrator - INFO - Waiting for pod test-pod-025216b5 to be ready +2025-06-05 19:56:47,317 - tests.orchestrator - INFO - Pod test-pod-025216b5 phase: Pending +2025-06-05 19:56:49,385 - tests.orchestrator - INFO - Pod test-pod-025216b5 phase: Running +2025-06-05 19:56:49,385 - tests.orchestrator - INFO - Pod test-pod-025216b5 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:56:51,460 - tests.orchestrator - INFO - Pod test-pod-025216b5 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:56:53,529 - tests.orchestrator - INFO - Pod test-pod-025216b5 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:56:55,598 - tests.orchestrator - INFO - Pod test-pod-025216b5 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:56:57,666 - tests.orchestrator - INFO - Pod test-pod-025216b5 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:56:59,735 - tests.orchestrator - INFO - Pod test-pod-025216b5 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:57:01,809 - tests.orchestrator - INFO - Pod test-pod-025216b5 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:57:03,877 - tests.orchestrator - INFO - Pod test-pod-025216b5 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:57:05,946 - tests.orchestrator - INFO - Pod test-pod-025216b5 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:57:08,014 - tests.orchestrator - INFO - Pod test-pod-025216b5 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 19:57:08,014 - tests.orchestrator - INFO - Pod test-pod-025216b5 is ready +2025-06-05 19:57:08,014 - tests.orchestrator - INFO - Creating pod test-pod-96bcc503 on PVC concurrent-pvc-12483e38 +2025-06-05 19:57:08,097 - tests.orchestrator - INFO - Created pod: test-pod-96bcc503 using PVC: concurrent-pvc-12483e38 +2025-06-05 19:57:08,097 - tests.orchestrator - INFO - Waiting for pod test-pod-96bcc503 to be ready +2025-06-05 19:57:08,165 - tests.orchestrator - INFO - Pod test-pod-96bcc503 phase: Pending +2025-06-05 19:57:12,300 - tests.orchestrator - INFO - Pod test-pod-96bcc503 phase: Running +2025-06-05 19:57:12,300 - tests.orchestrator - INFO - Pod test-pod-96bcc503 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:57:14,370 - tests.orchestrator - INFO - Pod test-pod-96bcc503 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:57:16,442 - tests.orchestrator - INFO - Pod test-pod-96bcc503 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:57:18,510 - tests.orchestrator - INFO - Pod test-pod-96bcc503 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:57:20,578 - tests.orchestrator - INFO - Pod test-pod-96bcc503 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:57:22,646 - tests.orchestrator - INFO - Pod test-pod-96bcc503 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:57:24,715 - tests.orchestrator - INFO - Pod test-pod-96bcc503 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:57:26,783 - tests.orchestrator - INFO - Pod test-pod-96bcc503 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:57:28,852 - tests.orchestrator - INFO - Pod test-pod-96bcc503 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 19:57:28,852 - tests.orchestrator - INFO - Pod test-pod-96bcc503 is ready +2025-06-05 19:57:28,854 - tests.orchestrator - INFO - Creating pod test-pod-1bba039d on PVC concurrent-pvc-ee19f53f +2025-06-05 19:57:28,927 - tests.orchestrator - INFO - Created pod: test-pod-1bba039d using PVC: concurrent-pvc-ee19f53f +2025-06-05 19:57:28,927 - tests.orchestrator - INFO - Waiting for pod test-pod-1bba039d to be ready +2025-06-05 19:57:28,995 - tests.orchestrator - INFO - Pod test-pod-1bba039d phase: Pending +2025-06-05 19:57:33,132 - tests.orchestrator - INFO - Pod test-pod-1bba039d phase: Running +2025-06-05 19:57:33,132 - tests.orchestrator - INFO - Pod test-pod-1bba039d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:57:35,200 - tests.orchestrator - INFO - Pod test-pod-1bba039d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:57:37,270 - tests.orchestrator - INFO - Pod test-pod-1bba039d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:57:39,339 - tests.orchestrator - INFO - Pod test-pod-1bba039d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:57:41,409 - tests.orchestrator - INFO - Pod test-pod-1bba039d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:57:43,476 - tests.orchestrator - INFO - Pod test-pod-1bba039d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:57:45,545 - tests.orchestrator - INFO - Pod test-pod-1bba039d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:57:47,614 - tests.orchestrator - INFO - Pod test-pod-1bba039d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:57:49,682 - tests.orchestrator - INFO - Pod test-pod-1bba039d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 19:57:51,749 - tests.orchestrator - INFO - Pod test-pod-1bba039d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 19:57:51,749 - tests.orchestrator - INFO - Pod test-pod-1bba039d is ready +2025-06-05 19:57:51,749 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-cbac8cde +2025-06-05 19:57:51,821 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-cbac8cde to be deleted +2025-06-05 19:57:51,887 - tests.orchestrator - INFO - PVC concurrent-pvc-cbac8cde has been deleted +2025-06-05 19:57:51,887 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-cbac8cde +2025-06-05 19:57:51,887 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-12483e38 +2025-06-05 19:57:51,888 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-12483e38 +2025-06-05 19:57:51,888 - tests.orchestrator - INFO - Deleting pod: test-pod-96bcc503 from PVC: concurrent-pvc-12483e38 +2025-06-05 19:57:51,966 - tests.orchestrator - INFO - Waiting for pod test-pod-96bcc503 to be deleted +2025-06-05 19:58:24,579 - tests.orchestrator - INFO - Pod test-pod-96bcc503 has been deleted +2025-06-05 19:58:24,579 - tests.orchestrator - INFO - Deleted pod: test-pod-96bcc503 +2025-06-05 19:58:24,652 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-12483e38 to be deleted +2025-06-05 19:58:24,718 - tests.orchestrator - INFO - PVC concurrent-pvc-12483e38 has been deleted +2025-06-05 19:58:24,718 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-12483e38 +2025-06-05 19:58:24,718 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-ee19f53f +2025-06-05 19:58:24,718 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-ee19f53f +2025-06-05 19:58:24,718 - tests.orchestrator - INFO - Deleting pod: test-pod-1bba039d from PVC: concurrent-pvc-ee19f53f +2025-06-05 19:58:24,793 - tests.orchestrator - INFO - Waiting for pod test-pod-1bba039d to be deleted +2025-06-05 19:58:57,410 - tests.orchestrator - INFO - Pod test-pod-1bba039d has been deleted +2025-06-05 19:58:57,410 - tests.orchestrator - INFO - Deleted pod: test-pod-1bba039d +2025-06-05 19:58:57,483 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-ee19f53f to be deleted +2025-06-05 19:59:01,743 - tests.orchestrator - INFO - PVC concurrent-pvc-ee19f53f has been deleted +2025-06-05 19:59:01,743 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-ee19f53f +2025-06-05 19:59:01,743 - tests.orchestrator - INFO - Rapid PVC scenario completed +2025-06-05 19:59:01,743 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_concurrent_pvc +2025-06-05 19:59:01,743 - tests.orchestrator - INFO - ------------------------------------------------------------ +2025-06-05 19:59:02,743 - tests.orchestrator - INFO - Test completed in 396.86 seconds +2025-06-05 19:59:02,743 - tests.orchestrator - INFO - ===== STARTING COMPREHENSIVE CLEANUP ===== +2025-06-05 19:59:02,744 - tests.orchestrator - INFO - Deleting 4 pods (force=False)... +2025-06-05 19:59:02,744 - tests.orchestrator - INFO - Deleting pod: test-pod-28ec4f7d from PVC: test-pvc-b5f8638e +2025-06-05 19:59:02,820 - tests.orchestrator - INFO - Waiting for pod test-pod-28ec4f7d to be deleted +2025-06-05 19:59:35,451 - tests.orchestrator - INFO - Pod test-pod-28ec4f7d has been deleted +2025-06-05 19:59:35,451 - tests.orchestrator - INFO - Deleted pod: test-pod-28ec4f7d +2025-06-05 19:59:35,452 - tests.orchestrator - INFO - Deleting pod: test-pod-044934ff from PVC: test-pvc-b5f8638e +2025-06-05 19:59:35,528 - tests.orchestrator - INFO - Waiting for pod test-pod-044934ff to be deleted +2025-06-05 20:00:08,148 - tests.orchestrator - INFO - Pod test-pod-044934ff has been deleted +2025-06-05 20:00:08,148 - tests.orchestrator - INFO - Deleted pod: test-pod-044934ff +2025-06-05 20:00:08,148 - tests.orchestrator - INFO - Deleting pod: test-pod-a0c335d2 from PVC: concurrent-pvc-0edb68b1 +2025-06-05 20:00:08,225 - tests.orchestrator - INFO - Waiting for pod test-pod-a0c335d2 to be deleted +2025-06-05 20:00:40,838 - tests.orchestrator - INFO - Pod test-pod-a0c335d2 has been deleted +2025-06-05 20:00:40,838 - tests.orchestrator - INFO - Deleted pod: test-pod-a0c335d2 +2025-06-05 20:00:40,838 - tests.orchestrator - INFO - Deleting pod: test-pod-025216b5 from PVC: concurrent-pvc-ab9c791a +2025-06-05 20:00:40,913 - tests.orchestrator - INFO - Waiting for pod test-pod-025216b5 to be deleted +2025-06-05 20:01:13,524 - tests.orchestrator - INFO - Pod test-pod-025216b5 has been deleted +2025-06-05 20:01:13,524 - tests.orchestrator - INFO - Deleted pod: test-pod-025216b5 +2025-06-05 20:01:18,524 - tests.orchestrator - INFO - Deleting 6 PVCs (force=False)... +2025-06-05 20:01:18,524 - tests.orchestrator - INFO - Deleting PVC: test-pvc-b5f8638e +2025-06-05 20:01:18,598 - tests.orchestrator - INFO - Waiting for PVC test-pvc-b5f8638e to be deleted +2025-06-05 20:01:18,663 - tests.orchestrator - INFO - PVC test-pvc-b5f8638e has been deleted +2025-06-05 20:01:18,663 - tests.orchestrator - INFO - Deleted PVC: test-pvc-b5f8638e +2025-06-05 20:01:18,663 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-0edb68b1 +2025-06-05 20:01:18,736 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-0edb68b1 to be deleted +2025-06-05 20:01:18,801 - tests.orchestrator - INFO - PVC concurrent-pvc-0edb68b1 has been deleted +2025-06-05 20:01:18,801 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-0edb68b1 +2025-06-05 20:01:18,801 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-e4c09a13 +2025-06-05 20:01:18,875 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-e4c09a13 to be deleted +2025-06-05 20:01:18,940 - tests.orchestrator - INFO - PVC concurrent-pvc-e4c09a13 has been deleted +2025-06-05 20:01:18,941 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-e4c09a13 +2025-06-05 20:01:18,942 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-ab9c791a +2025-06-05 20:01:19,016 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-ab9c791a to be deleted +2025-06-05 20:01:19,081 - tests.orchestrator - INFO - PVC concurrent-pvc-ab9c791a has been deleted +2025-06-05 20:01:19,082 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-ab9c791a +2025-06-05 20:01:19,082 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-ac5d027e +2025-06-05 20:01:19,155 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-ac5d027e to be deleted +2025-06-05 20:01:19,219 - tests.orchestrator - INFO - PVC concurrent-pvc-ac5d027e has been deleted +2025-06-05 20:01:19,220 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-ac5d027e +2025-06-05 20:01:19,220 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-f87f399d +2025-06-05 20:01:19,292 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-f87f399d to be deleted +2025-06-05 20:01:19,357 - tests.orchestrator - INFO - PVC concurrent-pvc-f87f399d has been deleted +2025-06-05 20:01:19,357 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-f87f399d +2025-06-05 20:01:19,556 - tests.orchestrator - WARNING - First cleanup pass incomplete. Remaining resources: ['pod/test-pod-0d0fddfe', 'pod/test-pod-158cdfa4', 'pvc/test-pvc-0bb0d20c', 'pvc/test-pvc-2127e71a', 'pvc/test-pvc-2cf5b7a2', 'pvc/test-pvc-31fbd501', 'pvc/test-pvc-528b4f5c', 'pvc/test-pvc-ecb12f74'] +2025-06-05 20:01:19,557 - tests.orchestrator - INFO - Attempting force deletion of remaining resources... +2025-06-05 20:01:19,557 - tests.orchestrator - INFO - Deleting 0 pods (force=True)... +2025-06-05 20:01:24,557 - tests.orchestrator - INFO - Deleting 0 PVCs (force=True)... +2025-06-05 20:01:24,754 - tests.orchestrator - ERROR - Cleanup incomplete. Remaining resources after force deletion: ['pod/test-pod-0d0fddfe', 'pod/test-pod-158cdfa4', 'pvc/test-pvc-0bb0d20c', 'pvc/test-pvc-2127e71a', 'pvc/test-pvc-2cf5b7a2', 'pvc/test-pvc-31fbd501', 'pvc/test-pvc-528b4f5c', 'pvc/test-pvc-ecb12f74'] +2025-06-05 20:01:24,755 - tests.orchestrator - INFO - Cleanup completed successfully in 142.01 seconds +2025-06-05 20:01:24,755 - tests.orchestrator - INFO - ===== CLEANUP PROCESS FINISHED ===== +2025-06-05 20:01:24,755 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-05 20:01:24,755 - tests.orchestrator - INFO - --- Operations --- +2025-06-05 20:01:24,755 - tests.orchestrator - INFO - create_pvc: 13 succeeded, 0 failed (100.0%) +2025-06-05 20:01:24,755 - tests.orchestrator - INFO - attach_pod: 9 succeeded, 0 failed (100.0%) +2025-06-05 20:01:24,755 - tests.orchestrator - INFO - delete_pod: 9 succeeded, 0 failed (100.0%) +2025-06-05 20:01:24,755 - tests.orchestrator - INFO - delete_pvc: 13 succeeded, 0 failed (100.0%) +2025-06-05 20:01:24,755 - tests.orchestrator - INFO - verify_read_write: Writes 2 succeeded, 0 failed (100.0%) +2025-06-05 20:01:24,755 - tests.orchestrator - INFO - verify_read_write: Reads 2 succeeded, 0 failed (100.0%) +2025-06-05 20:01:24,755 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-05 20:01:24,755 - tests.orchestrator - INFO - shared_volume_rw: 2 succeeded, 0 failed out of 2 runs (100.0%) +2025-06-05 20:01:24,755 - tests.orchestrator - INFO - many_to_one: No runs +2025-06-05 20:01:24,755 - tests.orchestrator - INFO - one_to_one: No runs +2025-06-05 20:01:24,755 - tests.orchestrator - INFO - concurrent_pvc: 2 succeeded, 0 failed out of 2 runs (100.0%) +2025-06-05 20:01:24,755 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_200645.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_200645.log new file mode 100644 index 000000000..1aefe7d2e --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_200645.log @@ -0,0 +1,348 @@ +2025-06-05 20:06:45,933 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 20:06:46,152 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 20:06:46,152 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-05 20:06:46,343 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 20:06:46,343 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 20:06:46,343 - tests.orchestrator - INFO - Creating PVC: test-pvc-779d01ad +2025-06-05 20:06:46,420 - tests.orchestrator - INFO - Created PVC: test-pvc-779d01ad +2025-06-05 20:06:46,420 - tests.orchestrator - INFO - Waiting for PVC test-pvc-779d01ad to be bound +2025-06-05 20:06:48,554 - tests.orchestrator - INFO - PVC test-pvc-779d01ad is bound +2025-06-05 20:06:48,554 - tests.orchestrator - INFO - Creating PVC: test-pvc-69cf206a +2025-06-05 20:06:48,623 - tests.orchestrator - INFO - Created PVC: test-pvc-69cf206a +2025-06-05 20:06:48,624 - tests.orchestrator - INFO - Waiting for PVC test-pvc-69cf206a to be bound +2025-06-05 20:06:50,757 - tests.orchestrator - INFO - PVC test-pvc-69cf206a is bound +2025-06-05 20:06:50,757 - tests.orchestrator - INFO - Creating pod test-pod-88b786b1 on PVC test-pvc-69cf206a +2025-06-05 20:06:50,838 - tests.orchestrator - INFO - Created pod: test-pod-88b786b1 using PVC: test-pvc-69cf206a +2025-06-05 20:06:50,838 - tests.orchestrator - INFO - Waiting for pod test-pod-88b786b1 to be ready +2025-06-05 20:06:50,907 - tests.orchestrator - INFO - Pod test-pod-88b786b1 phase: Pending +2025-06-05 20:06:55,042 - tests.orchestrator - INFO - Pod test-pod-88b786b1 phase: Running +2025-06-05 20:06:55,043 - tests.orchestrator - INFO - Pod test-pod-88b786b1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:06:57,112 - tests.orchestrator - INFO - Pod test-pod-88b786b1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:06:59,189 - tests.orchestrator - INFO - Pod test-pod-88b786b1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:07:01,258 - tests.orchestrator - INFO - Pod test-pod-88b786b1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:07:03,327 - tests.orchestrator - INFO - Pod test-pod-88b786b1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:07:05,395 - tests.orchestrator - INFO - Pod test-pod-88b786b1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:07:07,466 - tests.orchestrator - INFO - Pod test-pod-88b786b1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:07:09,535 - tests.orchestrator - INFO - Pod test-pod-88b786b1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:07:11,603 - tests.orchestrator - INFO - Pod test-pod-88b786b1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 20:07:11,603 - tests.orchestrator - INFO - Pod test-pod-88b786b1 is ready +2025-06-05 20:07:11,603 - tests.orchestrator - INFO - Creating pod test-pod-da70686e on PVC test-pvc-779d01ad +2025-06-05 20:07:11,677 - tests.orchestrator - INFO - Created pod: test-pod-da70686e using PVC: test-pvc-779d01ad +2025-06-05 20:07:11,677 - tests.orchestrator - INFO - Waiting for pod test-pod-da70686e to be ready +2025-06-05 20:07:11,745 - tests.orchestrator - INFO - Pod test-pod-da70686e phase: Pending +2025-06-05 20:07:15,881 - tests.orchestrator - INFO - Pod test-pod-da70686e phase: Running +2025-06-05 20:07:15,881 - tests.orchestrator - INFO - Pod test-pod-da70686e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:07:17,949 - tests.orchestrator - INFO - Pod test-pod-da70686e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:07:20,018 - tests.orchestrator - INFO - Pod test-pod-da70686e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:07:22,087 - tests.orchestrator - INFO - Pod test-pod-da70686e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:07:24,155 - tests.orchestrator - INFO - Pod test-pod-da70686e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:07:26,224 - tests.orchestrator - INFO - Pod test-pod-da70686e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:07:28,293 - tests.orchestrator - INFO - Pod test-pod-da70686e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:07:30,361 - tests.orchestrator - INFO - Pod test-pod-da70686e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:07:32,429 - tests.orchestrator - INFO - Pod test-pod-da70686e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:07:34,498 - tests.orchestrator - INFO - Pod test-pod-da70686e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 20:07:34,498 - tests.orchestrator - INFO - Pod test-pod-da70686e is ready +2025-06-05 20:07:34,498 - tests.orchestrator - INFO - Creating pod test-pod-a0822d95 on PVC test-pvc-779d01ad +2025-06-05 20:07:34,580 - tests.orchestrator - INFO - Created pod: test-pod-a0822d95 using PVC: test-pvc-779d01ad +2025-06-05 20:07:34,580 - tests.orchestrator - INFO - Waiting for pod test-pod-a0822d95 to be ready +2025-06-05 20:07:34,649 - tests.orchestrator - INFO - Pod test-pod-a0822d95 phase: Pending +2025-06-05 20:07:36,718 - tests.orchestrator - INFO - Pod test-pod-a0822d95 phase: Running +2025-06-05 20:07:36,718 - tests.orchestrator - INFO - Pod test-pod-a0822d95 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:07:38,787 - tests.orchestrator - INFO - Pod test-pod-a0822d95 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:07:40,855 - tests.orchestrator - INFO - Pod test-pod-a0822d95 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:07:42,923 - tests.orchestrator - INFO - Pod test-pod-a0822d95 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:07:44,992 - tests.orchestrator - INFO - Pod test-pod-a0822d95 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:07:47,060 - tests.orchestrator - INFO - Pod test-pod-a0822d95 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:07:49,131 - tests.orchestrator - INFO - Pod test-pod-a0822d95 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:07:51,201 - tests.orchestrator - INFO - Pod test-pod-a0822d95 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:07:53,268 - tests.orchestrator - INFO - Pod test-pod-a0822d95 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:07:55,337 - tests.orchestrator - INFO - Pod test-pod-a0822d95 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:07:57,405 - tests.orchestrator - INFO - Pod test-pod-a0822d95 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 20:07:57,405 - tests.orchestrator - INFO - Pod test-pod-a0822d95 is ready +2025-06-05 20:07:57,406 - tests.orchestrator - INFO - Testing read/write between pods test-pod-da70686e and test-pod-a0822d95 sharing PVC test-pvc-779d01ad +2025-06-05 20:07:57,406 - tests.orchestrator - INFO - Executing write command: kubectl exec -n efs-stress-test test-pod-da70686e -- /bin/sh -c 'echo "Test content: e5a1b64d-ce63-4074-93b0-af11de5623c7" > /data/test-85c4aefc.txt' +2025-06-05 20:07:58,672 - tests.orchestrator - INFO - Pod test-pod-da70686e wrote to /data/test-85c4aefc.txt +2025-06-05 20:08:00,672 - tests.orchestrator - INFO - Executing read command: kubectl exec -n efs-stress-test test-pod-a0822d95 -- cat /data/test-85c4aefc.txt +2025-06-05 20:08:01,892 - tests.orchestrator - INFO - Read result: 'Test content: e5a1b64d-ce63-4074-93b0-af11de5623c7' +2025-06-05 20:08:01,892 - tests.orchestrator - INFO - Pod test-pod-a0822d95 successfully read content written by test-pod-da70686e +2025-06-05 20:08:01,892 - tests.orchestrator - INFO - Deleting pod: test-pod-88b786b1 from PVC: test-pvc-69cf206a +2025-06-05 20:08:01,981 - tests.orchestrator - INFO - Waiting for pod test-pod-88b786b1 to be deleted +2025-06-05 20:08:34,617 - tests.orchestrator - INFO - Pod test-pod-88b786b1 has been deleted +2025-06-05 20:08:34,617 - tests.orchestrator - INFO - Deleted pod: test-pod-88b786b1 +2025-06-05 20:08:34,617 - tests.orchestrator - INFO - Deleting PVC: test-pvc-779d01ad +2025-06-05 20:08:34,617 - tests.orchestrator - INFO - Deleting 2 pods using PVC test-pvc-779d01ad +2025-06-05 20:08:34,617 - tests.orchestrator - INFO - Deleting pod: test-pod-da70686e from PVC: test-pvc-779d01ad +2025-06-05 20:08:34,693 - tests.orchestrator - INFO - Waiting for pod test-pod-da70686e to be deleted +2025-06-05 20:09:07,311 - tests.orchestrator - INFO - Pod test-pod-da70686e has been deleted +2025-06-05 20:09:07,311 - tests.orchestrator - INFO - Deleted pod: test-pod-da70686e +2025-06-05 20:09:07,311 - tests.orchestrator - INFO - Deleting pod: test-pod-a0822d95 from PVC: test-pvc-779d01ad +2025-06-05 20:09:07,387 - tests.orchestrator - INFO - Waiting for pod test-pod-a0822d95 to be deleted +2025-06-05 20:09:39,999 - tests.orchestrator - INFO - Pod test-pod-a0822d95 has been deleted +2025-06-05 20:09:39,999 - tests.orchestrator - INFO - Deleted pod: test-pod-a0822d95 +2025-06-05 20:09:40,073 - tests.orchestrator - INFO - Waiting for PVC test-pvc-779d01ad to be deleted +2025-06-05 20:09:44,206 - tests.orchestrator - INFO - PVC test-pvc-779d01ad has been deleted +2025-06-05 20:09:44,206 - tests.orchestrator - INFO - Deleted PVC: test-pvc-779d01ad +2025-06-05 20:09:44,206 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations +2025-06-05 20:09:44,206 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario (selected 1 times) +2025-06-05 20:09:44,206 - tests.orchestrator - INFO - ============================================================ +2025-06-05 20:09:44,206 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_many_to_one +2025-06-05 20:09:44,206 - tests.orchestrator - INFO - ============================================================ +2025-06-05 20:09:44,206 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +2025-06-05 20:09:44,206 - tests.orchestrator - INFO - STARTING MANY-TO-ONE SCENARIO DIAGNOSTICS +2025-06-05 20:09:44,206 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +2025-06-05 20:09:44,206 - tests.orchestrator - INFO - [MANY2ONE] STEP 1: Creating dedicated PVC: many2one-45c123ce +2025-06-05 20:09:44,206 - tests.orchestrator - INFO - [MANY2ONE] Creating PVC manifest with storageClassName: efs-sc +2025-06-05 20:09:44,285 - tests.orchestrator - INFO - [MANY2ONE] PVC many2one-45c123ce created successfully +2025-06-05 20:09:44,285 - tests.orchestrator - INFO - [MANY2ONE] Waiting for PVC many2one-45c123ce to be bound... +2025-06-05 20:09:44,285 - tests.orchestrator - INFO - Waiting for PVC many2one-45c123ce to be bound +2025-06-05 20:09:46,420 - tests.orchestrator - INFO - PVC many2one-45c123ce is bound +2025-06-05 20:09:46,489 - tests.orchestrator - INFO - [MANY2ONE] PVC status: phase=Bound, capacity={'storage': '1Gi'} +2025-06-05 20:09:46,489 - tests.orchestrator - INFO - [MANY2ONE] STEP 2: Creating 4 pods for the same PVC many2one-45c123ce +2025-06-05 20:09:46,489 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 1/4 for PVC many2one-45c123ce +2025-06-05 20:09:46,489 - tests.orchestrator - INFO - Creating pod test-pod-d7eb09ac on PVC many2one-45c123ce +2025-06-05 20:09:46,575 - tests.orchestrator - INFO - Created pod: test-pod-d7eb09ac using PVC: many2one-45c123ce +2025-06-05 20:09:46,576 - tests.orchestrator - INFO - Waiting for pod test-pod-d7eb09ac to be ready +2025-06-05 20:09:46,644 - tests.orchestrator - INFO - Pod test-pod-d7eb09ac phase: Pending +2025-06-05 20:09:48,712 - tests.orchestrator - INFO - Pod test-pod-d7eb09ac phase: Running +2025-06-05 20:09:48,712 - tests.orchestrator - INFO - Pod test-pod-d7eb09ac conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:09:50,781 - tests.orchestrator - INFO - Pod test-pod-d7eb09ac conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:09:52,850 - tests.orchestrator - INFO - Pod test-pod-d7eb09ac conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:09:54,918 - tests.orchestrator - INFO - Pod test-pod-d7eb09ac conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:09:56,986 - tests.orchestrator - INFO - Pod test-pod-d7eb09ac conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:09:59,055 - tests.orchestrator - INFO - Pod test-pod-d7eb09ac conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:10:01,123 - tests.orchestrator - INFO - Pod test-pod-d7eb09ac conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:10:03,193 - tests.orchestrator - INFO - Pod test-pod-d7eb09ac conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 20:10:03,193 - tests.orchestrator - INFO - Pod test-pod-d7eb09ac is ready +2025-06-05 20:10:03,193 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-d7eb09ac +2025-06-05 20:10:03,193 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 2/4 for PVC many2one-45c123ce +2025-06-05 20:10:03,193 - tests.orchestrator - INFO - Creating pod test-pod-440ae01d on PVC many2one-45c123ce +2025-06-05 20:10:03,268 - tests.orchestrator - INFO - Created pod: test-pod-440ae01d using PVC: many2one-45c123ce +2025-06-05 20:10:03,268 - tests.orchestrator - INFO - Waiting for pod test-pod-440ae01d to be ready +2025-06-05 20:10:03,337 - tests.orchestrator - INFO - Pod test-pod-440ae01d phase: Pending +2025-06-05 20:10:07,474 - tests.orchestrator - INFO - Pod test-pod-440ae01d phase: Running +2025-06-05 20:10:07,474 - tests.orchestrator - INFO - Pod test-pod-440ae01d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:10:09,543 - tests.orchestrator - INFO - Pod test-pod-440ae01d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:10:11,611 - tests.orchestrator - INFO - Pod test-pod-440ae01d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:10:13,679 - tests.orchestrator - INFO - Pod test-pod-440ae01d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:10:15,747 - tests.orchestrator - INFO - Pod test-pod-440ae01d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:10:17,816 - tests.orchestrator - INFO - Pod test-pod-440ae01d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:10:19,885 - tests.orchestrator - INFO - Pod test-pod-440ae01d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 20:10:19,885 - tests.orchestrator - INFO - Pod test-pod-440ae01d is ready +2025-06-05 20:10:19,885 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-440ae01d +2025-06-05 20:10:19,885 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 3/4 for PVC many2one-45c123ce +2025-06-05 20:10:19,885 - tests.orchestrator - INFO - Creating pod test-pod-d651a57e on PVC many2one-45c123ce +2025-06-05 20:10:19,968 - tests.orchestrator - INFO - Created pod: test-pod-d651a57e using PVC: many2one-45c123ce +2025-06-05 20:10:19,968 - tests.orchestrator - INFO - Waiting for pod test-pod-d651a57e to be ready +2025-06-05 20:10:20,036 - tests.orchestrator - INFO - Pod test-pod-d651a57e phase: Pending +2025-06-05 20:10:22,105 - tests.orchestrator - INFO - Pod test-pod-d651a57e phase: Running +2025-06-05 20:10:22,105 - tests.orchestrator - INFO - Pod test-pod-d651a57e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:10:24,174 - tests.orchestrator - INFO - Pod test-pod-d651a57e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:10:26,242 - tests.orchestrator - INFO - Pod test-pod-d651a57e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:10:28,310 - tests.orchestrator - INFO - Pod test-pod-d651a57e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:10:30,379 - tests.orchestrator - INFO - Pod test-pod-d651a57e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:10:32,447 - tests.orchestrator - INFO - Pod test-pod-d651a57e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:10:34,515 - tests.orchestrator - INFO - Pod test-pod-d651a57e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:10:36,584 - tests.orchestrator - INFO - Pod test-pod-d651a57e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:10:38,652 - tests.orchestrator - INFO - Pod test-pod-d651a57e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:10:40,720 - tests.orchestrator - INFO - Pod test-pod-d651a57e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 20:10:40,720 - tests.orchestrator - INFO - Pod test-pod-d651a57e is ready +2025-06-05 20:10:40,720 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-d651a57e +2025-06-05 20:10:40,721 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 4/4 for PVC many2one-45c123ce +2025-06-05 20:10:40,721 - tests.orchestrator - INFO - Creating pod test-pod-3929c90d on PVC many2one-45c123ce +2025-06-05 20:10:40,796 - tests.orchestrator - INFO - Created pod: test-pod-3929c90d using PVC: many2one-45c123ce +2025-06-05 20:10:40,796 - tests.orchestrator - INFO - Waiting for pod test-pod-3929c90d to be ready +2025-06-05 20:10:40,864 - tests.orchestrator - INFO - Pod test-pod-3929c90d phase: Pending +2025-06-05 20:10:45,001 - tests.orchestrator - INFO - Pod test-pod-3929c90d phase: Running +2025-06-05 20:10:45,001 - tests.orchestrator - INFO - Pod test-pod-3929c90d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:10:47,070 - tests.orchestrator - INFO - Pod test-pod-3929c90d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:10:49,137 - tests.orchestrator - INFO - Pod test-pod-3929c90d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:10:51,206 - tests.orchestrator - INFO - Pod test-pod-3929c90d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:10:53,275 - tests.orchestrator - INFO - Pod test-pod-3929c90d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:10:55,343 - tests.orchestrator - INFO - Pod test-pod-3929c90d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:10:57,412 - tests.orchestrator - INFO - Pod test-pod-3929c90d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:10:59,481 - tests.orchestrator - INFO - Pod test-pod-3929c90d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:11:01,576 - tests.orchestrator - INFO - Pod test-pod-3929c90d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 20:11:01,576 - tests.orchestrator - INFO - Pod test-pod-3929c90d is ready +2025-06-05 20:11:01,576 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-3929c90d +2025-06-05 20:11:01,576 - tests.orchestrator - INFO - [MANY2ONE] Created 4/4 pods successfully +2025-06-05 20:11:01,576 - tests.orchestrator - INFO - [MANY2ONE] STEP 3: Testing read/write operations +2025-06-05 20:11:01,576 - tests.orchestrator - INFO - [MANY2ONE] Writer pod: test-pod-440ae01d, Reader pod: test-pod-d651a57e +2025-06-05 20:11:01,576 - tests.orchestrator - INFO - [MANY2ONE] Test file: /data/many2one-c2ee41a1.txt +2025-06-05 20:11:01,576 - tests.orchestrator - INFO - [MANY2ONE] Test content: Many2One test content: 036ddf73-684a-4b98-b06f-4292a885f843 +2025-06-05 20:11:01,576 - tests.orchestrator - INFO - [MANY2ONE] Executing write command: kubectl exec -n efs-stress-test test-pod-440ae01d -- /bin/sh -c 'echo "Many2One test content: 036ddf73-684a-4b98-b06f-4292a885f843" > /data/many2one-c2ee41a1.txt' +2025-06-05 20:11:03,979 - tests.orchestrator - INFO - [MANY2ONE] Writer pod ls check: '-rw-r--r-- 1 50006 50006 60 Jun 5 20:11 /data/many2one-c2ee41a1.txt' +2025-06-05 20:11:03,979 - tests.orchestrator - INFO - [MANY2ONE] Sleeping for 5 seconds to ensure filesystem sync +2025-06-05 20:11:08,979 - tests.orchestrator - INFO - [MANY2ONE] Executing read command: kubectl exec -n efs-stress-test test-pod-d651a57e -- cat /data/many2one-c2ee41a1.txt +2025-06-05 20:11:10,142 - tests.orchestrator - INFO - [MANY2ONE] Read command result: 'Many2One test content: 036ddf73-684a-4b98-b06f-4292a885f843' +2025-06-05 20:11:10,142 - tests.orchestrator - INFO - [MANY2ONE] SUCCESS: Many-to-one scenario successful with 4 pods +2025-06-05 20:11:10,142 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +2025-06-05 20:11:10,142 - tests.orchestrator - INFO - COMPLETED MANY-TO-ONE SCENARIO DIAGNOSTICS +2025-06-05 20:11:10,142 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +2025-06-05 20:11:10,142 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_many_to_one +2025-06-05 20:11:10,142 - tests.orchestrator - INFO - ------------------------------------------------------------ +2025-06-05 20:11:11,143 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 1 times) +2025-06-05 20:11:11,143 - tests.orchestrator - INFO - Creating pod test-pod-3f02ab55 on PVC many2one-45c123ce +2025-06-05 20:11:11,230 - tests.orchestrator - INFO - Created pod: test-pod-3f02ab55 using PVC: many2one-45c123ce +2025-06-05 20:11:11,230 - tests.orchestrator - INFO - Waiting for pod test-pod-3f02ab55 to be ready +2025-06-05 20:11:11,298 - tests.orchestrator - INFO - Pod test-pod-3f02ab55 phase: Pending +2025-06-05 20:11:15,434 - tests.orchestrator - INFO - Pod test-pod-3f02ab55 phase: Running +2025-06-05 20:11:15,434 - tests.orchestrator - INFO - Pod test-pod-3f02ab55 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:11:17,502 - tests.orchestrator - INFO - Pod test-pod-3f02ab55 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:11:19,571 - tests.orchestrator - INFO - Pod test-pod-3f02ab55 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:11:21,639 - tests.orchestrator - INFO - Pod test-pod-3f02ab55 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:11:23,708 - tests.orchestrator - INFO - Pod test-pod-3f02ab55 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:11:25,777 - tests.orchestrator - INFO - Pod test-pod-3f02ab55 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:11:27,844 - tests.orchestrator - INFO - Pod test-pod-3f02ab55 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:11:29,913 - tests.orchestrator - INFO - Pod test-pod-3f02ab55 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:11:31,984 - tests.orchestrator - INFO - Pod test-pod-3f02ab55 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 20:11:31,984 - tests.orchestrator - INFO - Pod test-pod-3f02ab55 is ready +2025-06-05 20:11:32,984 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario (selected 2 times) +2025-06-05 20:11:32,985 - tests.orchestrator - INFO - ============================================================ +2025-06-05 20:11:32,985 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_many_to_one +2025-06-05 20:11:32,985 - tests.orchestrator - INFO - ============================================================ +2025-06-05 20:11:32,985 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +2025-06-05 20:11:32,985 - tests.orchestrator - INFO - STARTING MANY-TO-ONE SCENARIO DIAGNOSTICS +2025-06-05 20:11:32,985 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +2025-06-05 20:11:32,985 - tests.orchestrator - INFO - [MANY2ONE] STEP 1: Creating dedicated PVC: many2one-89ac31d4 +2025-06-05 20:11:32,985 - tests.orchestrator - INFO - [MANY2ONE] Creating PVC manifest with storageClassName: efs-sc +2025-06-05 20:11:33,054 - tests.orchestrator - INFO - [MANY2ONE] PVC many2one-89ac31d4 created successfully +2025-06-05 20:11:33,054 - tests.orchestrator - INFO - [MANY2ONE] Waiting for PVC many2one-89ac31d4 to be bound... +2025-06-05 20:11:33,054 - tests.orchestrator - INFO - Waiting for PVC many2one-89ac31d4 to be bound +2025-06-05 20:11:35,189 - tests.orchestrator - INFO - PVC many2one-89ac31d4 is bound +2025-06-05 20:11:35,256 - tests.orchestrator - INFO - [MANY2ONE] PVC status: phase=Bound, capacity={'storage': '1Gi'} +2025-06-05 20:11:35,256 - tests.orchestrator - INFO - [MANY2ONE] STEP 2: Creating 3 pods for the same PVC many2one-89ac31d4 +2025-06-05 20:11:35,256 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 1/3 for PVC many2one-89ac31d4 +2025-06-05 20:11:35,256 - tests.orchestrator - INFO - Creating pod test-pod-09cac2b7 on PVC many2one-89ac31d4 +2025-06-05 20:11:35,328 - tests.orchestrator - INFO - Created pod: test-pod-09cac2b7 using PVC: many2one-89ac31d4 +2025-06-05 20:11:35,328 - tests.orchestrator - INFO - Waiting for pod test-pod-09cac2b7 to be ready +2025-06-05 20:11:35,395 - tests.orchestrator - INFO - Pod test-pod-09cac2b7 phase: Pending +2025-06-05 20:11:37,464 - tests.orchestrator - INFO - Pod test-pod-09cac2b7 phase: Running +2025-06-05 20:11:37,464 - tests.orchestrator - INFO - Pod test-pod-09cac2b7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:11:39,533 - tests.orchestrator - INFO - Pod test-pod-09cac2b7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:11:41,600 - tests.orchestrator - INFO - Pod test-pod-09cac2b7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:11:43,668 - tests.orchestrator - INFO - Pod test-pod-09cac2b7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:11:45,737 - tests.orchestrator - INFO - Pod test-pod-09cac2b7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:11:47,805 - tests.orchestrator - INFO - Pod test-pod-09cac2b7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:11:49,873 - tests.orchestrator - INFO - Pod test-pod-09cac2b7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:11:51,942 - tests.orchestrator - INFO - Pod test-pod-09cac2b7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:11:54,009 - tests.orchestrator - INFO - Pod test-pod-09cac2b7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:11:56,078 - tests.orchestrator - INFO - Pod test-pod-09cac2b7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 20:11:56,078 - tests.orchestrator - INFO - Pod test-pod-09cac2b7 is ready +2025-06-05 20:11:56,078 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-09cac2b7 +2025-06-05 20:11:56,078 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 2/3 for PVC many2one-89ac31d4 +2025-06-05 20:11:56,078 - tests.orchestrator - INFO - Creating pod test-pod-fc15ec0f on PVC many2one-89ac31d4 +2025-06-05 20:11:56,163 - tests.orchestrator - INFO - Created pod: test-pod-fc15ec0f using PVC: many2one-89ac31d4 +2025-06-05 20:11:56,163 - tests.orchestrator - INFO - Waiting for pod test-pod-fc15ec0f to be ready +2025-06-05 20:11:56,231 - tests.orchestrator - INFO - Pod test-pod-fc15ec0f phase: Pending +2025-06-05 20:12:00,366 - tests.orchestrator - INFO - Pod test-pod-fc15ec0f phase: Running +2025-06-05 20:12:00,366 - tests.orchestrator - INFO - Pod test-pod-fc15ec0f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:12:02,435 - tests.orchestrator - INFO - Pod test-pod-fc15ec0f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:12:04,503 - tests.orchestrator - INFO - Pod test-pod-fc15ec0f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:12:06,575 - tests.orchestrator - INFO - Pod test-pod-fc15ec0f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:12:08,643 - tests.orchestrator - INFO - Pod test-pod-fc15ec0f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:12:10,712 - tests.orchestrator - INFO - Pod test-pod-fc15ec0f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:12:12,780 - tests.orchestrator - INFO - Pod test-pod-fc15ec0f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 20:12:12,780 - tests.orchestrator - INFO - Pod test-pod-fc15ec0f is ready +2025-06-05 20:12:12,780 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-fc15ec0f +2025-06-05 20:12:12,780 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 3/3 for PVC many2one-89ac31d4 +2025-06-05 20:12:12,780 - tests.orchestrator - INFO - Creating pod test-pod-5a86841b on PVC many2one-89ac31d4 +2025-06-05 20:12:12,855 - tests.orchestrator - INFO - Created pod: test-pod-5a86841b using PVC: many2one-89ac31d4 +2025-06-05 20:12:12,855 - tests.orchestrator - INFO - Waiting for pod test-pod-5a86841b to be ready +2025-06-05 20:12:12,923 - tests.orchestrator - INFO - Pod test-pod-5a86841b phase: Pending +2025-06-05 20:12:17,061 - tests.orchestrator - INFO - Pod test-pod-5a86841b phase: Running +2025-06-05 20:12:17,061 - tests.orchestrator - INFO - Pod test-pod-5a86841b conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:12:19,128 - tests.orchestrator - INFO - Pod test-pod-5a86841b conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:12:21,196 - tests.orchestrator - INFO - Pod test-pod-5a86841b conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:12:23,265 - tests.orchestrator - INFO - Pod test-pod-5a86841b conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:12:25,333 - tests.orchestrator - INFO - Pod test-pod-5a86841b conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:12:27,401 - tests.orchestrator - INFO - Pod test-pod-5a86841b conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:12:29,470 - tests.orchestrator - INFO - Pod test-pod-5a86841b conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:12:31,563 - tests.orchestrator - INFO - Pod test-pod-5a86841b conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 20:12:33,631 - tests.orchestrator - INFO - Pod test-pod-5a86841b conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 20:12:33,631 - tests.orchestrator - INFO - Pod test-pod-5a86841b is ready +2025-06-05 20:12:33,631 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-5a86841b +2025-06-05 20:12:33,631 - tests.orchestrator - INFO - [MANY2ONE] Created 3/3 pods successfully +2025-06-05 20:12:33,631 - tests.orchestrator - INFO - [MANY2ONE] STEP 3: Testing read/write operations +2025-06-05 20:12:33,631 - tests.orchestrator - INFO - [MANY2ONE] Writer pod: test-pod-09cac2b7, Reader pod: test-pod-5a86841b +2025-06-05 20:12:33,631 - tests.orchestrator - INFO - [MANY2ONE] Test file: /data/many2one-9e3adf41.txt +2025-06-05 20:12:33,631 - tests.orchestrator - INFO - [MANY2ONE] Test content: Many2One test content: eeb4c284-9658-4254-984e-6a8c4fda8c94 +2025-06-05 20:12:33,632 - tests.orchestrator - INFO - [MANY2ONE] Executing write command: kubectl exec -n efs-stress-test test-pod-09cac2b7 -- /bin/sh -c 'echo "Many2One test content: eeb4c284-9658-4254-984e-6a8c4fda8c94" > /data/many2one-9e3adf41.txt' +2025-06-05 20:12:36,048 - tests.orchestrator - INFO - [MANY2ONE] Writer pod ls check: '-rw-r--r-- 1 50014 50014 60 Jun 5 20:12 /data/many2one-9e3adf41.txt' +2025-06-05 20:12:36,048 - tests.orchestrator - INFO - [MANY2ONE] Sleeping for 5 seconds to ensure filesystem sync +2025-06-05 20:12:41,048 - tests.orchestrator - INFO - [MANY2ONE] Executing read command: kubectl exec -n efs-stress-test test-pod-5a86841b -- cat /data/many2one-9e3adf41.txt +2025-06-05 20:12:42,254 - tests.orchestrator - INFO - [MANY2ONE] Read command result: 'Many2One test content: eeb4c284-9658-4254-984e-6a8c4fda8c94' +2025-06-05 20:12:42,254 - tests.orchestrator - INFO - [MANY2ONE] SUCCESS: Many-to-one scenario successful with 3 pods +2025-06-05 20:12:42,254 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +2025-06-05 20:12:42,254 - tests.orchestrator - INFO - COMPLETED MANY-TO-ONE SCENARIO DIAGNOSTICS +2025-06-05 20:12:42,254 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +2025-06-05 20:12:42,254 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_many_to_one +2025-06-05 20:12:42,254 - tests.orchestrator - INFO - ------------------------------------------------------------ +2025-06-05 20:12:43,255 - tests.orchestrator - INFO - Test completed in 357.10 seconds +2025-06-05 20:12:43,255 - tests.orchestrator - INFO - ===== STARTING COMPREHENSIVE CLEANUP ===== +2025-06-05 20:12:43,255 - tests.orchestrator - INFO - Deleting 8 pods (force=False)... +2025-06-05 20:12:43,255 - tests.orchestrator - INFO - Deleting pod: test-pod-d7eb09ac from PVC: many2one-45c123ce +2025-06-05 20:12:43,331 - tests.orchestrator - INFO - Waiting for pod test-pod-d7eb09ac to be deleted +2025-06-05 20:13:15,956 - tests.orchestrator - INFO - Pod test-pod-d7eb09ac has been deleted +2025-06-05 20:13:15,956 - tests.orchestrator - INFO - Deleted pod: test-pod-d7eb09ac +2025-06-05 20:13:15,956 - tests.orchestrator - INFO - Deleting pod: test-pod-440ae01d from PVC: many2one-45c123ce +2025-06-05 20:13:16,032 - tests.orchestrator - INFO - Waiting for pod test-pod-440ae01d to be deleted +2025-06-05 20:13:48,650 - tests.orchestrator - INFO - Pod test-pod-440ae01d has been deleted +2025-06-05 20:13:48,650 - tests.orchestrator - INFO - Deleted pod: test-pod-440ae01d +2025-06-05 20:13:48,650 - tests.orchestrator - INFO - Deleting pod: test-pod-d651a57e from PVC: many2one-45c123ce +2025-06-05 20:13:48,727 - tests.orchestrator - INFO - Waiting for pod test-pod-d651a57e to be deleted +2025-06-05 20:14:21,356 - tests.orchestrator - INFO - Pod test-pod-d651a57e has been deleted +2025-06-05 20:14:21,356 - tests.orchestrator - INFO - Deleted pod: test-pod-d651a57e +2025-06-05 20:14:21,356 - tests.orchestrator - INFO - Deleting pod: test-pod-3929c90d from PVC: many2one-45c123ce +2025-06-05 20:14:21,432 - tests.orchestrator - INFO - Waiting for pod test-pod-3929c90d to be deleted +2025-06-05 20:14:54,051 - tests.orchestrator - INFO - Pod test-pod-3929c90d has been deleted +2025-06-05 20:14:54,051 - tests.orchestrator - INFO - Deleted pod: test-pod-3929c90d +2025-06-05 20:14:54,051 - tests.orchestrator - INFO - Deleting pod: test-pod-3f02ab55 from PVC: many2one-45c123ce +2025-06-05 20:14:54,127 - tests.orchestrator - INFO - Waiting for pod test-pod-3f02ab55 to be deleted +2025-06-05 20:15:26,738 - tests.orchestrator - INFO - Pod test-pod-3f02ab55 has been deleted +2025-06-05 20:15:26,739 - tests.orchestrator - INFO - Deleted pod: test-pod-3f02ab55 +2025-06-05 20:15:26,739 - tests.orchestrator - INFO - Deleting pod: test-pod-09cac2b7 from PVC: many2one-89ac31d4 +2025-06-05 20:15:26,813 - tests.orchestrator - INFO - Waiting for pod test-pod-09cac2b7 to be deleted +2025-06-05 20:16:00,160 - tests.orchestrator - INFO - Pod test-pod-09cac2b7 has been deleted +2025-06-05 20:16:00,161 - tests.orchestrator - INFO - Deleted pod: test-pod-09cac2b7 +2025-06-05 20:16:00,161 - tests.orchestrator - INFO - Deleting pod: test-pod-fc15ec0f from PVC: many2one-89ac31d4 +2025-06-05 20:16:00,237 - tests.orchestrator - INFO - Waiting for pod test-pod-fc15ec0f to be deleted +2025-06-05 20:16:32,849 - tests.orchestrator - INFO - Pod test-pod-fc15ec0f has been deleted +2025-06-05 20:16:32,849 - tests.orchestrator - INFO - Deleted pod: test-pod-fc15ec0f +2025-06-05 20:16:32,849 - tests.orchestrator - INFO - Deleting pod: test-pod-5a86841b from PVC: many2one-89ac31d4 +2025-06-05 20:16:32,925 - tests.orchestrator - INFO - Waiting for pod test-pod-5a86841b to be deleted +2025-06-05 20:17:05,648 - tests.orchestrator - INFO - Pod test-pod-5a86841b has been deleted +2025-06-05 20:17:05,648 - tests.orchestrator - INFO - Deleted pod: test-pod-5a86841b +2025-06-05 20:17:10,649 - tests.orchestrator - INFO - Deleting 3 PVCs (force=False)... +2025-06-05 20:17:10,649 - tests.orchestrator - INFO - Deleting PVC: test-pvc-69cf206a +2025-06-05 20:17:10,724 - tests.orchestrator - INFO - Waiting for PVC test-pvc-69cf206a to be deleted +2025-06-05 20:17:10,792 - tests.orchestrator - INFO - PVC test-pvc-69cf206a has been deleted +2025-06-05 20:17:10,792 - tests.orchestrator - INFO - Deleted PVC: test-pvc-69cf206a +2025-06-05 20:17:10,792 - tests.orchestrator - INFO - Deleting PVC: many2one-45c123ce +2025-06-05 20:17:10,865 - tests.orchestrator - INFO - Waiting for PVC many2one-45c123ce to be deleted +2025-06-05 20:17:10,933 - tests.orchestrator - INFO - PVC many2one-45c123ce has been deleted +2025-06-05 20:17:10,933 - tests.orchestrator - INFO - Deleted PVC: many2one-45c123ce +2025-06-05 20:17:10,934 - tests.orchestrator - INFO - Deleting PVC: many2one-89ac31d4 +2025-06-05 20:17:11,006 - tests.orchestrator - INFO - Waiting for PVC many2one-89ac31d4 to be deleted +2025-06-05 20:17:11,071 - tests.orchestrator - INFO - PVC many2one-89ac31d4 has been deleted +2025-06-05 20:17:11,071 - tests.orchestrator - INFO - Deleted PVC: many2one-89ac31d4 +2025-06-05 20:17:11,272 - tests.orchestrator - WARNING - First cleanup pass incomplete. Remaining resources: ['pod/test-pod-0d0fddfe', 'pod/test-pod-158cdfa4', 'pvc/test-pvc-0bb0d20c', 'pvc/test-pvc-2127e71a', 'pvc/test-pvc-2cf5b7a2', 'pvc/test-pvc-31fbd501', 'pvc/test-pvc-528b4f5c', 'pvc/test-pvc-ecb12f74'] +2025-06-05 20:17:11,272 - tests.orchestrator - INFO - Attempting force deletion of remaining resources... +2025-06-05 20:17:11,272 - tests.orchestrator - INFO - Deleting 0 pods (force=True)... +2025-06-05 20:17:16,272 - tests.orchestrator - INFO - Deleting 0 PVCs (force=True)... +2025-06-05 20:17:16,471 - tests.orchestrator - ERROR - Cleanup incomplete. Remaining resources after force deletion: ['pod/test-pod-0d0fddfe', 'pod/test-pod-158cdfa4', 'pvc/test-pvc-0bb0d20c', 'pvc/test-pvc-2127e71a', 'pvc/test-pvc-2cf5b7a2', 'pvc/test-pvc-31fbd501', 'pvc/test-pvc-528b4f5c', 'pvc/test-pvc-ecb12f74'] +2025-06-05 20:17:16,471 - tests.orchestrator - INFO - Cleanup completed successfully in 273.22 seconds +2025-06-05 20:17:16,471 - tests.orchestrator - INFO - ===== CLEANUP PROCESS FINISHED ===== +2025-06-05 20:17:16,471 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-05 20:17:16,471 - tests.orchestrator - INFO - --- Operations --- +2025-06-05 20:17:16,472 - tests.orchestrator - INFO - create_pvc: 2 succeeded, 0 failed (100.0%) +2025-06-05 20:17:16,472 - tests.orchestrator - INFO - attach_pod: 11 succeeded, 0 failed (100.0%) +2025-06-05 20:17:16,472 - tests.orchestrator - INFO - delete_pod: 11 succeeded, 0 failed (100.0%) +2025-06-05 20:17:16,472 - tests.orchestrator - INFO - delete_pvc: 4 succeeded, 0 failed (100.0%) +2025-06-05 20:17:16,472 - tests.orchestrator - INFO - verify_read_write: Writes 1 succeeded, 0 failed (100.0%) +2025-06-05 20:17:16,472 - tests.orchestrator - INFO - verify_read_write: Reads 1 succeeded, 0 failed (100.0%) +2025-06-05 20:17:16,472 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-05 20:17:16,472 - tests.orchestrator - INFO - shared_volume_rw: 1 succeeded, 0 failed out of 1 runs (100.0%) +2025-06-05 20:17:16,472 - tests.orchestrator - INFO - many_to_one: 2 succeeded, 0 failed out of 2 runs (100.0%) +2025-06-05 20:17:16,472 - tests.orchestrator - INFO - one_to_one: No runs +2025-06-05 20:17:16,472 - tests.orchestrator - INFO - concurrent_pvc: No runs +2025-06-05 20:17:16,472 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/reports/chaos/efs_chaos_20250603_135616.json b/test/e2e/stress-scale-tests/reports/chaos/efs_chaos_20250603_135616.json new file mode 100644 index 000000000..66bfed3bb --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/chaos/efs_chaos_20250603_135616.json @@ -0,0 +1,29 @@ +{ + "api_flood": { + "test": "rapid_api_flood", + "pvcs_requested": 50, + "pvcs_created": 50, + "success_rate": 100.0, + "duration": 0.46927380561828613, + "access_points_used": true + }, + "race_conditions": { + "test": "race_conditions", + "iterations": 30, + "successes": 17, + "failures": 13, + "exceptions": 0, + "duration": 6.168970346450806 + }, + "resource_exhaustion": { + "test": "resource_exhaustion", + "pods_requested": 75, + "pods_created": 75, + "pod_success_rate": 100.0, + "access_points_requested": 25, + "access_points_created": 0, + "ap_success_rate": 0.0, + "duration": 2.9329886436462402 + }, + "error": "type object 'Thread' has no attribute '_wait'" +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_145849_20250603_145849.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_145849_20250603_145849.json new file mode 100644 index 000000000..7d530574e --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_145849_20250603_145849.json @@ -0,0 +1,104 @@ +{ + "test_name": "efs_tests_20250603_145849", + "test_type": "general", + "timestamp": "20250603_145849", + "system_info": { + "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", + "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", + "python_version": "3.12.10", + "cpu_count": 8, + "memory_total_gb": 30.88, + "timestamp": "2025-06-03T14:58:49.261345", + "kubernetes_version": "Unknown", + "aws_region": "us-west-1", + "efs_filesystem_id": "fs-02b74eafb14930828" + }, + "results": { + "orchestrator": { + "test_duration": 1748962729.2581227, + "operations": { + "create_pvc": { + "success": 1, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 9, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 9, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 3, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 1, + "read_success": 0, + "read_fail": 1, + "write_success_rate": 0.0, + "read_success_rate": 0.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "many_to_one": { + "runs": 2, + "success": 0, + "fail": 2, + "success_rate": 0.0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + } + } + }, + "metrics": { + "operations": {}, + "system": {}, + "kubernetes": {}, + "csi": {}, + "controller": { + "request_latency": {}, + "operation_counts": {}, + "success_rates": {}, + "volume_attach_timing": {} + }, + "node": { + "mount_timing": {}, + "mount_errors": {}, + "resource_utilization": {} + }, + "efs": { + "access_point_timing": {}, + "mount_completion_timing": {}, + "api_throttling_incidents": [] + }, + "k8s_events": { + "volume_events": [], + "binding_times": {}, + "pod_startup_delays": {} + } + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_145849_20250603_145849_summary.txt b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_145849_20250603_145849_summary.txt new file mode 100644 index 000000000..a3794720e --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_145849_20250603_145849_summary.txt @@ -0,0 +1,61 @@ +================================================================================ +EFS CSI DRIVER TEST REPORT: EFS_TESTS_20250603_145849 +================================================================================ + +SYSTEM INFORMATION +-------------------------------------------------------------------------------- +Hostname: dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com +Platform: Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26 +Python Version: 3.12.10 +CPU Count: 8 +Memory Total: 30.88 GB +Kubernetes Version: Unknown +AWS Region: us-west-1 +EFS Filesystem ID: fs-02b74eafb14930828 +Test Timestamp: 2025-06-03T14:58:49.307018 + +TEST RESULTS +-------------------------------------------------------------------------------- + +ORCHESTRATOR +---------------------------------------- +test_duration: 1748962729.2581227 +operations: + create_pvc: {'success': 1, 'fail': 0, 'success_rate': 100.0} + attach_pod: {'success': 9, 'fail': 0, 'success_rate': 100.0} + delete_pod: {'success': 9, 'fail': 0, 'success_rate': 100.0} + delete_pvc: {'success': 3, 'fail': 0, 'success_rate': 100.0} + verify_read_write: {'write_success': 0, 'write_fail': 1, 'read_success': 0, 'read_fail': 1, 'write_success_rate': 0.0, 'read_success_rate': 0.0} +scenarios: + shared_volume_rw: {'runs': 1, 'success': 0, 'fail': 1, 'success_rate': 0.0} + many_to_one: {'runs': 2, 'success': 0, 'fail': 2, 'success_rate': 0.0} + one_to_one: {'runs': 0, 'success': 0, 'fail': 0, 'success_rate': 0} + concurrent_pvc: {'runs': 0, 'success': 0, 'fail': 0, 'success_rate': 0} + +METRICS +---------------------------------------- +operations: +system: +kubernetes: +csi: +controller: + request_latency: {} + operation_counts: defaultdict(, {}) + success_rates: defaultdict(. at 0x7f77f5c3ede0>, {}) + volume_attach_timing: {} +node: + mount_timing: {} + mount_errors: defaultdict(, {}) + resource_utilization: {} +efs: + access_point_timing: {} + mount_completion_timing: {} + api_throttling_incidents: [] +k8s_events: + volume_events: [] + binding_times: {} + pod_startup_delays: {} + +================================================================================ +END OF REPORT: 2025-06-03T14:58:49.351704 +================================================================================ diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_160604_20250603_160604.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_160604_20250603_160604.json new file mode 100644 index 000000000..de426e9b7 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_160604_20250603_160604.json @@ -0,0 +1,104 @@ +{ + "test_name": "efs_tests_20250603_160604", + "test_type": "general", + "timestamp": "20250603_160604", + "system_info": { + "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", + "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", + "python_version": "3.12.10", + "cpu_count": 8, + "memory_total_gb": 30.88, + "timestamp": "2025-06-03T16:06:04.459863", + "kubernetes_version": "Unknown", + "aws_region": "us-west-1", + "efs_filesystem_id": "fs-02b74eafb14930828" + }, + "results": { + "orchestrator": { + "test_duration": 1748966764.4565506, + "operations": { + "create_pvc": { + "success": 17, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 23, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 23, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 27, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 1, + "read_success": 0, + "read_fail": 1, + "write_success_rate": 0.0, + "read_success_rate": 0.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "many_to_one": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "one_to_one": { + "runs": 2, + "success": 0, + "fail": 2, + "success_rate": 0.0 + }, + "concurrent_pvc": { + "runs": 2, + "success": 2, + "fail": 0, + "success_rate": 100.0 + } + } + }, + "metrics": { + "operations": {}, + "system": {}, + "kubernetes": {}, + "csi": {}, + "controller": { + "request_latency": {}, + "operation_counts": {}, + "success_rates": {}, + "volume_attach_timing": {} + }, + "node": { + "mount_timing": {}, + "mount_errors": {}, + "resource_utilization": {} + }, + "efs": { + "access_point_timing": {}, + "mount_completion_timing": {}, + "api_throttling_incidents": [] + }, + "k8s_events": { + "volume_events": [], + "binding_times": {}, + "pod_startup_delays": {} + } + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_160604_20250603_160604_summary.txt b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_160604_20250603_160604_summary.txt new file mode 100644 index 000000000..29bc48b33 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_160604_20250603_160604_summary.txt @@ -0,0 +1,61 @@ +================================================================================ +EFS CSI DRIVER TEST REPORT: EFS_TESTS_20250603_160604 +================================================================================ + +SYSTEM INFORMATION +-------------------------------------------------------------------------------- +Hostname: dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com +Platform: Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26 +Python Version: 3.12.10 +CPU Count: 8 +Memory Total: 30.88 GB +Kubernetes Version: Unknown +AWS Region: us-west-1 +EFS Filesystem ID: fs-02b74eafb14930828 +Test Timestamp: 2025-06-03T16:06:04.505249 + +TEST RESULTS +-------------------------------------------------------------------------------- + +ORCHESTRATOR +---------------------------------------- +test_duration: 1748966764.4565506 +operations: + create_pvc: {'success': 17, 'fail': 0, 'success_rate': 100.0} + attach_pod: {'success': 23, 'fail': 0, 'success_rate': 100.0} + delete_pod: {'success': 23, 'fail': 0, 'success_rate': 100.0} + delete_pvc: {'success': 27, 'fail': 0, 'success_rate': 100.0} + verify_read_write: {'write_success': 0, 'write_fail': 1, 'read_success': 0, 'read_fail': 1, 'write_success_rate': 0.0, 'read_success_rate': 0.0} +scenarios: + shared_volume_rw: {'runs': 1, 'success': 0, 'fail': 1, 'success_rate': 0.0} + many_to_one: {'runs': 1, 'success': 0, 'fail': 1, 'success_rate': 0.0} + one_to_one: {'runs': 2, 'success': 0, 'fail': 2, 'success_rate': 0.0} + concurrent_pvc: {'runs': 2, 'success': 2, 'fail': 0, 'success_rate': 100.0} + +METRICS +---------------------------------------- +operations: +system: +kubernetes: +csi: +controller: + request_latency: {} + operation_counts: defaultdict(, {}) + success_rates: defaultdict(. at 0x7fa9e4df2de0>, {}) + volume_attach_timing: {} +node: + mount_timing: {} + mount_errors: defaultdict(, {}) + resource_utilization: {} +efs: + access_point_timing: {} + mount_completion_timing: {} + api_throttling_incidents: [] +k8s_events: + volume_events: [] + binding_times: {} + pod_startup_delays: {} + +================================================================================ +END OF REPORT: 2025-06-03T16:06:04.550169 +================================================================================ diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_180130_20250603_180130.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_180130_20250603_180130.json new file mode 100644 index 000000000..6c58d6804 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_180130_20250603_180130.json @@ -0,0 +1,104 @@ +{ + "test_name": "efs_tests_20250603_180130", + "test_type": "general", + "timestamp": "20250603_180130", + "system_info": { + "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", + "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", + "python_version": "3.12.10", + "cpu_count": 8, + "memory_total_gb": 30.88, + "timestamp": "2025-06-03T18:01:30.380488", + "kubernetes_version": "Unknown", + "aws_region": "us-west-1", + "efs_filesystem_id": "fs-02b74eafb14930828" + }, + "results": { + "orchestrator": { + "test_duration": 1748973690.377472, + "operations": { + "create_pvc": { + "success": 1, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 4, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 4, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 2, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 0, + "read_success": 0, + "read_fail": 0, + "write_success_rate": 0, + "read_success_rate": 0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "many_to_one": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + } + } + }, + "metrics": { + "operations": {}, + "system": {}, + "kubernetes": {}, + "csi": {}, + "controller": { + "request_latency": {}, + "operation_counts": {}, + "success_rates": {}, + "volume_attach_timing": {} + }, + "node": { + "mount_timing": {}, + "mount_errors": {}, + "resource_utilization": {} + }, + "efs": { + "access_point_timing": {}, + "mount_completion_timing": {}, + "api_throttling_incidents": [] + }, + "k8s_events": { + "volume_events": [], + "binding_times": {}, + "pod_startup_delays": {} + } + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_180130_20250603_180130_summary.txt b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_180130_20250603_180130_summary.txt new file mode 100644 index 000000000..7cfca2169 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_180130_20250603_180130_summary.txt @@ -0,0 +1,61 @@ +================================================================================ +EFS CSI DRIVER TEST REPORT: EFS_TESTS_20250603_180130 +================================================================================ + +SYSTEM INFORMATION +-------------------------------------------------------------------------------- +Hostname: dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com +Platform: Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26 +Python Version: 3.12.10 +CPU Count: 8 +Memory Total: 30.88 GB +Kubernetes Version: Unknown +AWS Region: us-west-1 +EFS Filesystem ID: fs-02b74eafb14930828 +Test Timestamp: 2025-06-03T18:01:30.426564 + +TEST RESULTS +-------------------------------------------------------------------------------- + +ORCHESTRATOR +---------------------------------------- +test_duration: 1748973690.377472 +operations: + create_pvc: {'success': 1, 'fail': 0, 'success_rate': 100.0} + attach_pod: {'success': 4, 'fail': 0, 'success_rate': 100.0} + delete_pod: {'success': 4, 'fail': 0, 'success_rate': 100.0} + delete_pvc: {'success': 2, 'fail': 0, 'success_rate': 100.0} + verify_read_write: {'write_success': 0, 'write_fail': 0, 'read_success': 0, 'read_fail': 0, 'write_success_rate': 0, 'read_success_rate': 0} +scenarios: + shared_volume_rw: {'runs': 0, 'success': 0, 'fail': 0, 'success_rate': 0} + many_to_one: {'runs': 1, 'success': 0, 'fail': 1, 'success_rate': 0.0} + one_to_one: {'runs': 0, 'success': 0, 'fail': 0, 'success_rate': 0} + concurrent_pvc: {'runs': 0, 'success': 0, 'fail': 0, 'success_rate': 0} + +METRICS +---------------------------------------- +operations: +system: +kubernetes: +csi: +controller: + request_latency: {} + operation_counts: defaultdict(, {}) + success_rates: defaultdict(. at 0x7f892f362d40>, {}) + volume_attach_timing: {} +node: + mount_timing: {} + mount_errors: defaultdict(, {}) + resource_utilization: {} +efs: + access_point_timing: {} + mount_completion_timing: {} + api_throttling_incidents: [] +k8s_events: + volume_events: [] + binding_times: {} + pod_startup_delays: {} + +================================================================================ +END OF REPORT: 2025-06-03T18:01:30.469590 +================================================================================ diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_204448_20250603_204448.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_204448_20250603_204448.json new file mode 100644 index 000000000..acc51b261 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_204448_20250603_204448.json @@ -0,0 +1,104 @@ +{ + "test_name": "efs_tests_20250603_204448", + "test_type": "general", + "timestamp": "20250603_204448", + "system_info": { + "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", + "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", + "python_version": "3.12.10", + "cpu_count": 8, + "memory_total_gb": 30.88, + "timestamp": "2025-06-03T20:44:48.750764", + "kubernetes_version": "Unknown", + "aws_region": "us-west-1", + "efs_filesystem_id": "fs-02b74eafb14930828" + }, + "results": { + "orchestrator": { + "test_duration": 1748983488.7475276, + "operations": { + "create_pvc": { + "success": 0, + "fail": 7, + "success_rate": 0.0 + }, + "attach_pod": { + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "delete_pod": { + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "delete_pvc": { + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 0, + "read_success": 0, + "read_fail": 0, + "write_success_rate": 0, + "read_success_rate": 0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "many_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + } + } + }, + "metrics": { + "operations": {}, + "system": {}, + "kubernetes": {}, + "csi": {}, + "controller": { + "request_latency": {}, + "operation_counts": {}, + "success_rates": {}, + "volume_attach_timing": {} + }, + "node": { + "mount_timing": {}, + "mount_errors": {}, + "resource_utilization": {} + }, + "efs": { + "access_point_timing": {}, + "mount_completion_timing": {}, + "api_throttling_incidents": [] + }, + "k8s_events": { + "volume_events": [], + "binding_times": {}, + "pod_startup_delays": {} + } + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_204448_20250603_204448_summary.txt b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_204448_20250603_204448_summary.txt new file mode 100644 index 000000000..bec1dc247 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_204448_20250603_204448_summary.txt @@ -0,0 +1,61 @@ +================================================================================ +EFS CSI DRIVER TEST REPORT: EFS_TESTS_20250603_204448 +================================================================================ + +SYSTEM INFORMATION +-------------------------------------------------------------------------------- +Hostname: dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com +Platform: Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26 +Python Version: 3.12.10 +CPU Count: 8 +Memory Total: 30.88 GB +Kubernetes Version: Unknown +AWS Region: us-west-1 +EFS Filesystem ID: fs-02b74eafb14930828 +Test Timestamp: 2025-06-03T20:44:48.794425 + +TEST RESULTS +-------------------------------------------------------------------------------- + +ORCHESTRATOR +---------------------------------------- +test_duration: 1748983488.7475276 +operations: + create_pvc: {'success': 0, 'fail': 7, 'success_rate': 0.0} + attach_pod: {'success': 0, 'fail': 0, 'success_rate': 0} + delete_pod: {'success': 0, 'fail': 0, 'success_rate': 0} + delete_pvc: {'success': 0, 'fail': 0, 'success_rate': 0} + verify_read_write: {'write_success': 0, 'write_fail': 0, 'read_success': 0, 'read_fail': 0, 'write_success_rate': 0, 'read_success_rate': 0} +scenarios: + shared_volume_rw: {'runs': 0, 'success': 0, 'fail': 0, 'success_rate': 0} + many_to_one: {'runs': 0, 'success': 0, 'fail': 0, 'success_rate': 0} + one_to_one: {'runs': 0, 'success': 0, 'fail': 0, 'success_rate': 0} + concurrent_pvc: {'runs': 1, 'success': 0, 'fail': 1, 'success_rate': 0.0} + +METRICS +---------------------------------------- +operations: +system: +kubernetes: +csi: +controller: + request_latency: {} + operation_counts: defaultdict(, {}) + success_rates: defaultdict(. at 0x7f9ccfd9b6a0>, {}) + volume_attach_timing: {} +node: + mount_timing: {} + mount_errors: defaultdict(, {}) + resource_utilization: {} +efs: + access_point_timing: {} + mount_completion_timing: {} + api_throttling_incidents: [] +k8s_events: + volume_events: [] + binding_times: {} + pod_startup_delays: {} + +================================================================================ +END OF REPORT: 2025-06-03T20:44:48.840194 +================================================================================ diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_205333_20250603_205333.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_205333_20250603_205333.json new file mode 100644 index 000000000..8c9fc25e7 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_205333_20250603_205333.json @@ -0,0 +1,104 @@ +{ + "test_name": "efs_tests_20250603_205333", + "test_type": "general", + "timestamp": "20250603_205333", + "system_info": { + "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", + "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", + "python_version": "3.12.10", + "cpu_count": 8, + "memory_total_gb": 30.88, + "timestamp": "2025-06-03T20:53:33.163444", + "kubernetes_version": "Unknown", + "aws_region": "us-west-1", + "efs_filesystem_id": "fs-02b74eafb14930828" + }, + "results": { + "orchestrator": { + "test_duration": 1748984013.159983, + "operations": { + "create_pvc": { + "success": 1, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 5, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 5, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 2, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 0, + "read_success": 0, + "read_fail": 0, + "write_success_rate": 0, + "read_success_rate": 0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "many_to_one": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + } + } + }, + "metrics": { + "operations": {}, + "system": {}, + "kubernetes": {}, + "csi": {}, + "controller": { + "request_latency": {}, + "operation_counts": {}, + "success_rates": {}, + "volume_attach_timing": {} + }, + "node": { + "mount_timing": {}, + "mount_errors": {}, + "resource_utilization": {} + }, + "efs": { + "access_point_timing": {}, + "mount_completion_timing": {}, + "api_throttling_incidents": [] + }, + "k8s_events": { + "volume_events": [], + "binding_times": {}, + "pod_startup_delays": {} + } + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_205333_20250603_205333_summary.txt b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_205333_20250603_205333_summary.txt new file mode 100644 index 000000000..17b8c4bb2 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_205333_20250603_205333_summary.txt @@ -0,0 +1,61 @@ +================================================================================ +EFS CSI DRIVER TEST REPORT: EFS_TESTS_20250603_205333 +================================================================================ + +SYSTEM INFORMATION +-------------------------------------------------------------------------------- +Hostname: dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com +Platform: Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26 +Python Version: 3.12.10 +CPU Count: 8 +Memory Total: 30.88 GB +Kubernetes Version: Unknown +AWS Region: us-west-1 +EFS Filesystem ID: fs-02b74eafb14930828 +Test Timestamp: 2025-06-03T20:53:33.207801 + +TEST RESULTS +-------------------------------------------------------------------------------- + +ORCHESTRATOR +---------------------------------------- +test_duration: 1748984013.159983 +operations: + create_pvc: {'success': 1, 'fail': 0, 'success_rate': 100.0} + attach_pod: {'success': 5, 'fail': 0, 'success_rate': 100.0} + delete_pod: {'success': 5, 'fail': 0, 'success_rate': 100.0} + delete_pvc: {'success': 2, 'fail': 0, 'success_rate': 100.0} + verify_read_write: {'write_success': 0, 'write_fail': 0, 'read_success': 0, 'read_fail': 0, 'write_success_rate': 0, 'read_success_rate': 0} +scenarios: + shared_volume_rw: {'runs': 0, 'success': 0, 'fail': 0, 'success_rate': 0} + many_to_one: {'runs': 1, 'success': 0, 'fail': 1, 'success_rate': 0.0} + one_to_one: {'runs': 0, 'success': 0, 'fail': 0, 'success_rate': 0} + concurrent_pvc: {'runs': 0, 'success': 0, 'fail': 0, 'success_rate': 0} + +METRICS +---------------------------------------- +operations: +system: +kubernetes: +csi: +controller: + request_latency: {} + operation_counts: defaultdict(, {}) + success_rates: defaultdict(. at 0x7fa4a85236a0>, {}) + volume_attach_timing: {} +node: + mount_timing: {} + mount_errors: defaultdict(, {}) + resource_utilization: {} +efs: + access_point_timing: {} + mount_completion_timing: {} + api_throttling_incidents: [] +k8s_events: + volume_events: [] + binding_times: {} + pod_startup_delays: {} + +================================================================================ +END OF REPORT: 2025-06-03T20:53:33.262686 +================================================================================ diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_210938_20250603_210938.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_210938_20250603_210938.json new file mode 100644 index 000000000..e60fce0c4 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_210938_20250603_210938.json @@ -0,0 +1,104 @@ +{ + "test_name": "efs_tests_20250603_210938", + "test_type": "general", + "timestamp": "20250603_210938", + "system_info": { + "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", + "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", + "python_version": "3.12.10", + "cpu_count": 8, + "memory_total_gb": 30.88, + "timestamp": "2025-06-03T21:09:38.404991", + "kubernetes_version": "Unknown", + "aws_region": "us-west-1", + "efs_filesystem_id": "fs-02b74eafb14930828" + }, + "results": { + "orchestrator": { + "test_duration": 1748984978.40179, + "operations": { + "create_pvc": { + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "attach_pod": { + "success": 9, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 9, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 1, + "read_success": 0, + "read_fail": 1, + "write_success_rate": 0.0, + "read_success_rate": 0.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "many_to_one": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "one_to_one": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "concurrent_pvc": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + } + } + }, + "metrics": { + "operations": {}, + "system": {}, + "kubernetes": {}, + "csi": {}, + "controller": { + "request_latency": {}, + "operation_counts": {}, + "success_rates": {}, + "volume_attach_timing": {} + }, + "node": { + "mount_timing": {}, + "mount_errors": {}, + "resource_utilization": {} + }, + "efs": { + "access_point_timing": {}, + "mount_completion_timing": {}, + "api_throttling_incidents": [] + }, + "k8s_events": { + "volume_events": [], + "binding_times": {}, + "pod_startup_delays": {} + } + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_210938_20250603_210938_summary.txt b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_210938_20250603_210938_summary.txt new file mode 100644 index 000000000..6cd5590c3 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_210938_20250603_210938_summary.txt @@ -0,0 +1,61 @@ +================================================================================ +EFS CSI DRIVER TEST REPORT: EFS_TESTS_20250603_210938 +================================================================================ + +SYSTEM INFORMATION +-------------------------------------------------------------------------------- +Hostname: dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com +Platform: Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26 +Python Version: 3.12.10 +CPU Count: 8 +Memory Total: 30.88 GB +Kubernetes Version: Unknown +AWS Region: us-west-1 +EFS Filesystem ID: fs-02b74eafb14930828 +Test Timestamp: 2025-06-03T21:09:38.449707 + +TEST RESULTS +-------------------------------------------------------------------------------- + +ORCHESTRATOR +---------------------------------------- +test_duration: 1748984978.40179 +operations: + create_pvc: {'success': 0, 'fail': 0, 'success_rate': 0} + attach_pod: {'success': 9, 'fail': 0, 'success_rate': 100.0} + delete_pod: {'success': 9, 'fail': 0, 'success_rate': 100.0} + delete_pvc: {'success': 6, 'fail': 0, 'success_rate': 100.0} + verify_read_write: {'write_success': 0, 'write_fail': 1, 'read_success': 0, 'read_fail': 1, 'write_success_rate': 0.0, 'read_success_rate': 0.0} +scenarios: + shared_volume_rw: {'runs': 1, 'success': 0, 'fail': 1, 'success_rate': 0.0} + many_to_one: {'runs': 1, 'success': 0, 'fail': 1, 'success_rate': 0.0} + one_to_one: {'runs': 1, 'success': 0, 'fail': 1, 'success_rate': 0.0} + concurrent_pvc: {'runs': 0, 'success': 0, 'fail': 0, 'success_rate': 0} + +METRICS +---------------------------------------- +operations: +system: +kubernetes: +csi: +controller: + request_latency: {} + operation_counts: defaultdict(, {}) + success_rates: defaultdict(. at 0x7f6d73f9d120>, {}) + volume_attach_timing: {} +node: + mount_timing: {} + mount_errors: defaultdict(, {}) + resource_utilization: {} +efs: + access_point_timing: {} + mount_completion_timing: {} + api_throttling_incidents: [] +k8s_events: + volume_events: [] + binding_times: {} + pod_startup_delays: {} + +================================================================================ +END OF REPORT: 2025-06-03T21:09:38.492151 +================================================================================ diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250604_204834_20250604_204834.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250604_204834_20250604_204834.json new file mode 100644 index 000000000..57194f1c9 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250604_204834_20250604_204834.json @@ -0,0 +1,104 @@ +{ + "test_name": "efs_tests_20250604_204834", + "test_type": "general", + "timestamp": "20250604_204834", + "system_info": { + "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", + "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", + "python_version": "3.12.10", + "cpu_count": 8, + "memory_total_gb": 30.88, + "timestamp": "2025-06-04T20:48:34.213634", + "kubernetes_version": "Unknown", + "aws_region": "us-west-1", + "efs_filesystem_id": "fs-02b74eafb14930828" + }, + "results": { + "orchestrator": { + "test_duration": 1749070114.2102087, + "operations": { + "create_pvc": { + "success": 2, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 5, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 5, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 0, + "read_success": 0, + "read_fail": 0, + "write_success_rate": 0, + "read_success_rate": 0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "many_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "one_to_one": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "concurrent_pvc": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + } + } + }, + "metrics": { + "operations": {}, + "system": {}, + "kubernetes": {}, + "csi": {}, + "controller": { + "request_latency": {}, + "operation_counts": {}, + "success_rates": {}, + "volume_attach_timing": {} + }, + "node": { + "mount_timing": {}, + "mount_errors": {}, + "resource_utilization": {} + }, + "efs": { + "access_point_timing": {}, + "mount_completion_timing": {}, + "api_throttling_incidents": [] + }, + "k8s_events": { + "volume_events": [], + "binding_times": {}, + "pod_startup_delays": {} + } + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250604_211347_20250604_211347.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250604_211347_20250604_211347.json new file mode 100644 index 000000000..ef6d374f1 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250604_211347_20250604_211347.json @@ -0,0 +1,104 @@ +{ + "test_name": "efs_tests_20250604_211347", + "test_type": "general", + "timestamp": "20250604_211347", + "system_info": { + "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", + "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", + "python_version": "3.12.10", + "cpu_count": 8, + "memory_total_gb": 30.88, + "timestamp": "2025-06-04T21:13:47.134029", + "kubernetes_version": "Unknown", + "aws_region": "us-west-1", + "efs_filesystem_id": "fs-02b74eafb14930828" + }, + "results": { + "orchestrator": { + "test_duration": 1749071627.1310592, + "operations": { + "create_pvc": { + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "attach_pod": { + "success": 1, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 1, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 1, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 0, + "read_success": 0, + "read_fail": 0, + "write_success_rate": 0, + "read_success_rate": 0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "many_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "one_to_one": { + "runs": 1, + "success": 0, + "fail": 0, + "success_rate": 0.0 + }, + "concurrent_pvc": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + } + } + }, + "metrics": { + "operations": {}, + "system": {}, + "kubernetes": {}, + "csi": {}, + "controller": { + "request_latency": {}, + "operation_counts": {}, + "success_rates": {}, + "volume_attach_timing": {} + }, + "node": { + "mount_timing": {}, + "mount_errors": {}, + "resource_utilization": {} + }, + "efs": { + "access_point_timing": {}, + "mount_completion_timing": {}, + "api_throttling_incidents": [] + }, + "k8s_events": { + "volume_events": [], + "binding_times": {}, + "pod_startup_delays": {} + } + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250604_211457_20250604_211457.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250604_211457_20250604_211457.json new file mode 100644 index 000000000..7ac9cea43 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250604_211457_20250604_211457.json @@ -0,0 +1,104 @@ +{ + "test_name": "efs_tests_20250604_211457", + "test_type": "general", + "timestamp": "20250604_211457", + "system_info": { + "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", + "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", + "python_version": "3.12.10", + "cpu_count": 8, + "memory_total_gb": 30.88, + "timestamp": "2025-06-04T21:14:57.325771", + "kubernetes_version": "Unknown", + "aws_region": "us-west-1", + "efs_filesystem_id": "fs-02b74eafb14930828" + }, + "results": { + "orchestrator": { + "test_duration": 1749071697.3223903, + "operations": { + "create_pvc": { + "success": 4, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 1, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 1, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 4, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 0, + "read_success": 0, + "read_fail": 0, + "write_success_rate": 0, + "read_success_rate": 0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "many_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 1, + "success": 0, + "fail": 0, + "success_rate": 0.0 + } + } + }, + "metrics": { + "operations": {}, + "system": {}, + "kubernetes": {}, + "csi": {}, + "controller": { + "request_latency": {}, + "operation_counts": {}, + "success_rates": {}, + "volume_attach_timing": {} + }, + "node": { + "mount_timing": {}, + "mount_errors": {}, + "resource_utilization": {} + }, + "efs": { + "access_point_timing": {}, + "mount_completion_timing": {}, + "api_throttling_incidents": [] + }, + "k8s_events": { + "volume_events": [], + "binding_times": {}, + "pod_startup_delays": {} + } + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_133650_20250605_133650.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_133650_20250605_133650.json new file mode 100644 index 000000000..e8751550c --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_133650_20250605_133650.json @@ -0,0 +1,104 @@ +{ + "test_name": "efs_tests_20250605_133650", + "test_type": "general", + "timestamp": "20250605_133650", + "system_info": { + "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", + "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", + "python_version": "3.12.10", + "cpu_count": 8, + "memory_total_gb": 30.88, + "timestamp": "2025-06-05T13:36:50.687414", + "kubernetes_version": "Unknown", + "aws_region": "Unknown", + "efs_filesystem_id": "Unknown" + }, + "results": { + "orchestrator": { + "test_duration": 1749130610.6835227, + "operations": { + "create_pvc": { + "success": 12, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 8, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 8, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 12, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 0, + "read_success": 0, + "read_fail": 0, + "write_success_rate": 0, + "read_success_rate": 0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "many_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 2, + "success": 2, + "fail": 0, + "success_rate": 100.0 + } + } + }, + "metrics": { + "operations": {}, + "system": {}, + "kubernetes": {}, + "csi": {}, + "controller": { + "request_latency": {}, + "operation_counts": {}, + "success_rates": {}, + "volume_attach_timing": {} + }, + "node": { + "mount_timing": {}, + "mount_errors": {}, + "resource_utilization": {} + }, + "efs": { + "access_point_timing": {}, + "mount_completion_timing": {}, + "api_throttling_incidents": [] + }, + "k8s_events": { + "volume_events": [], + "binding_times": {}, + "pod_startup_delays": {} + } + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_135626_20250605_135626.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_135626_20250605_135626.json new file mode 100644 index 000000000..5e51eaa30 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_135626_20250605_135626.json @@ -0,0 +1,104 @@ +{ + "test_name": "efs_tests_20250605_135626", + "test_type": "general", + "timestamp": "20250605_135626", + "system_info": { + "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", + "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", + "python_version": "3.12.10", + "cpu_count": 8, + "memory_total_gb": 30.88, + "timestamp": "2025-06-05T13:56:26.290855", + "kubernetes_version": "Unknown", + "aws_region": "Unknown", + "efs_filesystem_id": "Unknown" + }, + "results": { + "orchestrator": { + "test_duration": 1749131786.2873476, + "operations": { + "create_pvc": { + "success": 2, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 5, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 1, + "read_success": 0, + "read_fail": 1, + "write_success_rate": 0.0, + "read_success_rate": 0.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "many_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "one_to_one": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "concurrent_pvc": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + } + } + }, + "metrics": { + "operations": {}, + "system": {}, + "kubernetes": {}, + "csi": {}, + "controller": { + "request_latency": {}, + "operation_counts": {}, + "success_rates": {}, + "volume_attach_timing": {} + }, + "node": { + "mount_timing": {}, + "mount_errors": {}, + "resource_utilization": {} + }, + "efs": { + "access_point_timing": {}, + "mount_completion_timing": {}, + "api_throttling_incidents": [] + }, + "k8s_events": { + "volume_events": [], + "binding_times": {}, + "pod_startup_delays": {} + } + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_141131_20250605_141131.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_141131_20250605_141131.json new file mode 100644 index 000000000..d2243619b --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_141131_20250605_141131.json @@ -0,0 +1,104 @@ +{ + "test_name": "efs_tests_20250605_141131", + "test_type": "general", + "timestamp": "20250605_141131", + "system_info": { + "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", + "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", + "python_version": "3.12.10", + "cpu_count": 8, + "memory_total_gb": 30.88, + "timestamp": "2025-06-05T14:11:31.212354", + "kubernetes_version": "Unknown", + "aws_region": "Unknown", + "efs_filesystem_id": "Unknown" + }, + "results": { + "orchestrator": { + "test_duration": 1749132691.2088823, + "operations": { + "create_pvc": { + "success": 2, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 7, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 7, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 1, + "read_success": 0, + "read_fail": 1, + "write_success_rate": 0.0, + "read_success_rate": 0.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "many_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "one_to_one": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "concurrent_pvc": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + } + } + }, + "metrics": { + "operations": {}, + "system": {}, + "kubernetes": {}, + "csi": {}, + "controller": { + "request_latency": {}, + "operation_counts": {}, + "success_rates": {}, + "volume_attach_timing": {} + }, + "node": { + "mount_timing": {}, + "mount_errors": {}, + "resource_utilization": {} + }, + "efs": { + "access_point_timing": {}, + "mount_completion_timing": {}, + "api_throttling_incidents": [] + }, + "k8s_events": { + "volume_events": [], + "binding_times": {}, + "pod_startup_delays": {} + } + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_142059_20250605_142059.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_142059_20250605_142059.json new file mode 100644 index 000000000..9aebf3995 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_142059_20250605_142059.json @@ -0,0 +1,104 @@ +{ + "test_name": "efs_tests_20250605_142059", + "test_type": "general", + "timestamp": "20250605_142059", + "system_info": { + "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", + "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", + "python_version": "3.12.10", + "cpu_count": 8, + "memory_total_gb": 30.88, + "timestamp": "2025-06-05T14:20:59.414086", + "kubernetes_version": "Unknown", + "aws_region": "Unknown", + "efs_filesystem_id": "Unknown" + }, + "results": { + "orchestrator": { + "test_duration": 1749133259.4106185, + "operations": { + "create_pvc": { + "success": 7, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 7, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 1, + "read_success": 0, + "read_fail": 1, + "write_success_rate": 0.0, + "read_success_rate": 0.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "many_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 1, + "success": 1, + "fail": 0, + "success_rate": 100.0 + } + } + }, + "metrics": { + "operations": {}, + "system": {}, + "kubernetes": {}, + "csi": {}, + "controller": { + "request_latency": {}, + "operation_counts": {}, + "success_rates": {}, + "volume_attach_timing": {} + }, + "node": { + "mount_timing": {}, + "mount_errors": {}, + "resource_utilization": {} + }, + "efs": { + "access_point_timing": {}, + "mount_completion_timing": {}, + "api_throttling_incidents": [] + }, + "k8s_events": { + "volume_events": [], + "binding_times": {}, + "pod_startup_delays": {} + } + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_145750_20250605_145750.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_145750_20250605_145750.json new file mode 100644 index 000000000..ec26fc187 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_145750_20250605_145750.json @@ -0,0 +1,104 @@ +{ + "test_name": "efs_tests_20250605_145750", + "test_type": "general", + "timestamp": "20250605_145750", + "system_info": { + "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", + "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", + "python_version": "3.12.10", + "cpu_count": 8, + "memory_total_gb": 30.88, + "timestamp": "2025-06-05T14:57:50.430436", + "kubernetes_version": "Unknown", + "aws_region": "Unknown", + "efs_filesystem_id": "Unknown" + }, + "results": { + "orchestrator": { + "test_duration": 1749135470.4266138, + "operations": { + "create_pvc": { + "success": 2, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 3, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 1, + "read_success": 0, + "read_fail": 1, + "write_success_rate": 0.0, + "read_success_rate": 0.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "many_to_one": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + } + } + }, + "metrics": { + "operations": {}, + "system": {}, + "kubernetes": {}, + "csi": {}, + "controller": { + "request_latency": {}, + "operation_counts": {}, + "success_rates": {}, + "volume_attach_timing": {} + }, + "node": { + "mount_timing": {}, + "mount_errors": {}, + "resource_utilization": {} + }, + "efs": { + "access_point_timing": {}, + "mount_completion_timing": {}, + "api_throttling_incidents": [] + }, + "k8s_events": { + "volume_events": [], + "binding_times": {}, + "pod_startup_delays": {} + } + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_150603_20250605_150603.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_150603_20250605_150603.json new file mode 100644 index 000000000..072a7ceb4 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_150603_20250605_150603.json @@ -0,0 +1,104 @@ +{ + "test_name": "efs_tests_20250605_150603", + "test_type": "general", + "timestamp": "20250605_150603", + "system_info": { + "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", + "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", + "python_version": "3.12.10", + "cpu_count": 8, + "memory_total_gb": 30.88, + "timestamp": "2025-06-05T15:06:03.516189", + "kubernetes_version": "Unknown", + "aws_region": "Unknown", + "efs_filesystem_id": "Unknown" + }, + "results": { + "orchestrator": { + "test_duration": 1749135963.512927, + "operations": { + "create_pvc": { + "success": 5, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 5, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 1, + "read_success": 0, + "read_fail": 1, + "write_success_rate": 0.0, + "read_success_rate": 0.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "many_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 1, + "success": 1, + "fail": 0, + "success_rate": 100.0 + } + } + }, + "metrics": { + "operations": {}, + "system": {}, + "kubernetes": {}, + "csi": {}, + "controller": { + "request_latency": {}, + "operation_counts": {}, + "success_rates": {}, + "volume_attach_timing": {} + }, + "node": { + "mount_timing": {}, + "mount_errors": {}, + "resource_utilization": {} + }, + "efs": { + "access_point_timing": {}, + "mount_completion_timing": {}, + "api_throttling_incidents": [] + }, + "k8s_events": { + "volume_events": [], + "binding_times": {}, + "pod_startup_delays": {} + } + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_151246_20250605_151246.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_151246_20250605_151246.json new file mode 100644 index 000000000..dd620e847 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_151246_20250605_151246.json @@ -0,0 +1,104 @@ +{ + "test_name": "efs_tests_20250605_151246", + "test_type": "general", + "timestamp": "20250605_151246", + "system_info": { + "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", + "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", + "python_version": "3.12.10", + "cpu_count": 8, + "memory_total_gb": 30.88, + "timestamp": "2025-06-05T15:12:46.812571", + "kubernetes_version": "Unknown", + "aws_region": "Unknown", + "efs_filesystem_id": "Unknown" + }, + "results": { + "orchestrator": { + "test_duration": 1749136366.8090806, + "operations": { + "create_pvc": { + "success": 7, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 7, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 1, + "read_success": 0, + "read_fail": 1, + "write_success_rate": 0.0, + "read_success_rate": 0.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "many_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 1, + "success": 1, + "fail": 0, + "success_rate": 100.0 + } + } + }, + "metrics": { + "operations": {}, + "system": {}, + "kubernetes": {}, + "csi": {}, + "controller": { + "request_latency": {}, + "operation_counts": {}, + "success_rates": {}, + "volume_attach_timing": {} + }, + "node": { + "mount_timing": {}, + "mount_errors": {}, + "resource_utilization": {} + }, + "efs": { + "access_point_timing": {}, + "mount_completion_timing": {}, + "api_throttling_incidents": [] + }, + "k8s_events": { + "volume_events": [], + "binding_times": {}, + "pod_startup_delays": {} + } + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_154402_20250605_154402.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_154402_20250605_154402.json new file mode 100644 index 000000000..74c5aca15 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_154402_20250605_154402.json @@ -0,0 +1,104 @@ +{ + "test_name": "efs_tests_20250605_154402", + "test_type": "general", + "timestamp": "20250605_154402", + "system_info": { + "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", + "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", + "python_version": "3.12.10", + "cpu_count": 8, + "memory_total_gb": 30.88, + "timestamp": "2025-06-05T15:44:02.410888", + "kubernetes_version": "Unknown", + "aws_region": "Unknown", + "efs_filesystem_id": "Unknown" + }, + "results": { + "orchestrator": { + "test_duration": 1749138242.406942, + "operations": { + "create_pvc": { + "success": 2, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 3, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 1, + "read_success": 0, + "read_fail": 1, + "write_success_rate": 0.0, + "read_success_rate": 0.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "many_to_one": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + } + } + }, + "metrics": { + "operations": {}, + "system": {}, + "kubernetes": {}, + "csi": {}, + "controller": { + "request_latency": {}, + "operation_counts": {}, + "success_rates": {}, + "volume_attach_timing": {} + }, + "node": { + "mount_timing": {}, + "mount_errors": {}, + "resource_utilization": {} + }, + "efs": { + "access_point_timing": {}, + "mount_completion_timing": {}, + "api_throttling_incidents": [] + }, + "k8s_events": { + "volume_events": [], + "binding_times": {}, + "pod_startup_delays": {} + } + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_155202_20250605_155202.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_155202_20250605_155202.json new file mode 100644 index 000000000..bfceb1589 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_155202_20250605_155202.json @@ -0,0 +1,104 @@ +{ + "test_name": "efs_tests_20250605_155202", + "test_type": "general", + "timestamp": "20250605_155202", + "system_info": { + "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", + "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", + "python_version": "3.12.10", + "cpu_count": 8, + "memory_total_gb": 30.88, + "timestamp": "2025-06-05T15:52:02.971763", + "kubernetes_version": "Unknown", + "aws_region": "Unknown", + "efs_filesystem_id": "Unknown" + }, + "results": { + "orchestrator": { + "test_duration": 1749138722.968492, + "operations": { + "create_pvc": { + "success": 5, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 5, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 1, + "read_success": 0, + "read_fail": 1, + "write_success_rate": 0.0, + "read_success_rate": 0.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "many_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 1, + "success": 1, + "fail": 0, + "success_rate": 100.0 + } + } + }, + "metrics": { + "operations": {}, + "system": {}, + "kubernetes": {}, + "csi": {}, + "controller": { + "request_latency": {}, + "operation_counts": {}, + "success_rates": {}, + "volume_attach_timing": {} + }, + "node": { + "mount_timing": {}, + "mount_errors": {}, + "resource_utilization": {} + }, + "efs": { + "access_point_timing": {}, + "mount_completion_timing": {}, + "api_throttling_incidents": [] + }, + "k8s_events": { + "volume_events": [], + "binding_times": {}, + "pod_startup_delays": {} + } + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_160701_20250605_160701.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_160701_20250605_160701.json new file mode 100644 index 000000000..6504484d1 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_160701_20250605_160701.json @@ -0,0 +1,104 @@ +{ + "test_name": "efs_tests_20250605_160701", + "test_type": "general", + "timestamp": "20250605_160701", + "system_info": { + "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", + "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", + "python_version": "3.12.10", + "cpu_count": 8, + "memory_total_gb": 30.88, + "timestamp": "2025-06-05T16:07:01.160981", + "kubernetes_version": "Unknown", + "aws_region": "Unknown", + "efs_filesystem_id": "Unknown" + }, + "results": { + "orchestrator": { + "test_duration": 1749139621.1574326, + "operations": { + "create_pvc": { + "success": 2, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 7, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 7, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 3, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 1, + "read_success": 0, + "read_fail": 1, + "write_success_rate": 0.0, + "read_success_rate": 0.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "many_to_one": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + } + } + }, + "metrics": { + "operations": {}, + "system": {}, + "kubernetes": {}, + "csi": {}, + "controller": { + "request_latency": {}, + "operation_counts": {}, + "success_rates": {}, + "volume_attach_timing": {} + }, + "node": { + "mount_timing": {}, + "mount_errors": {}, + "resource_utilization": {} + }, + "efs": { + "access_point_timing": {}, + "mount_completion_timing": {}, + "api_throttling_incidents": [] + }, + "k8s_events": { + "volume_events": [], + "binding_times": {}, + "pod_startup_delays": {} + } + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_161509_20250605_161509.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_161509_20250605_161509.json new file mode 100644 index 000000000..b41af0ee6 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_161509_20250605_161509.json @@ -0,0 +1,104 @@ +{ + "test_name": "efs_tests_20250605_161509", + "test_type": "general", + "timestamp": "20250605_161509", + "system_info": { + "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", + "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", + "python_version": "3.12.10", + "cpu_count": 8, + "memory_total_gb": 30.88, + "timestamp": "2025-06-05T16:15:09.982171", + "kubernetes_version": "Unknown", + "aws_region": "Unknown", + "efs_filesystem_id": "Unknown" + }, + "results": { + "orchestrator": { + "test_duration": 1749140109.9788902, + "operations": { + "create_pvc": { + "success": 9, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 9, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 1, + "read_success": 0, + "read_fail": 1, + "write_success_rate": 0.0, + "read_success_rate": 0.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "many_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 1, + "success": 1, + "fail": 0, + "success_rate": 100.0 + } + } + }, + "metrics": { + "operations": {}, + "system": {}, + "kubernetes": {}, + "csi": {}, + "controller": { + "request_latency": {}, + "operation_counts": {}, + "success_rates": {}, + "volume_attach_timing": {} + }, + "node": { + "mount_timing": {}, + "mount_errors": {}, + "resource_utilization": {} + }, + "efs": { + "access_point_timing": {}, + "mount_completion_timing": {}, + "api_throttling_incidents": [] + }, + "k8s_events": { + "volume_events": [], + "binding_times": {}, + "pod_startup_delays": {} + } + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_163010_20250605_163010.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_163010_20250605_163010.json new file mode 100644 index 000000000..2ab04a1d2 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_163010_20250605_163010.json @@ -0,0 +1,104 @@ +{ + "test_name": "efs_tests_20250605_163010", + "test_type": "general", + "timestamp": "20250605_163010", + "system_info": { + "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", + "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", + "python_version": "3.12.10", + "cpu_count": 8, + "memory_total_gb": 30.88, + "timestamp": "2025-06-05T16:30:10.899998", + "kubernetes_version": "Unknown", + "aws_region": "Unknown", + "efs_filesystem_id": "Unknown" + }, + "results": { + "orchestrator": { + "test_duration": 1749141010.8966942, + "operations": { + "create_pvc": { + "success": 2, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 5, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 1, + "read_success": 0, + "read_fail": 1, + "write_success_rate": 0.0, + "read_success_rate": 0.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "many_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "one_to_one": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "concurrent_pvc": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + } + } + }, + "metrics": { + "operations": {}, + "system": {}, + "kubernetes": {}, + "csi": {}, + "controller": { + "request_latency": {}, + "operation_counts": {}, + "success_rates": {}, + "volume_attach_timing": {} + }, + "node": { + "mount_timing": {}, + "mount_errors": {}, + "resource_utilization": {} + }, + "efs": { + "access_point_timing": {}, + "mount_completion_timing": {}, + "api_throttling_incidents": [] + }, + "k8s_events": { + "volume_events": [], + "binding_times": {}, + "pod_startup_delays": {} + } + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_164017_20250605_164017.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_164017_20250605_164017.json new file mode 100644 index 000000000..8dfa7eb5e --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_164017_20250605_164017.json @@ -0,0 +1,104 @@ +{ + "test_name": "efs_tests_20250605_164017", + "test_type": "general", + "timestamp": "20250605_164017", + "system_info": { + "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", + "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", + "python_version": "3.12.10", + "cpu_count": 8, + "memory_total_gb": 30.88, + "timestamp": "2025-06-05T16:40:17.454819", + "kubernetes_version": "Unknown", + "aws_region": "Unknown", + "efs_filesystem_id": "Unknown" + }, + "results": { + "orchestrator": { + "test_duration": 1749141617.450957, + "operations": { + "create_pvc": { + "success": 3, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 5, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 5, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 3, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 1, + "read_success": 0, + "read_fail": 1, + "write_success_rate": 0.0, + "read_success_rate": 0.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "many_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + } + } + }, + "metrics": { + "operations": {}, + "system": {}, + "kubernetes": {}, + "csi": {}, + "controller": { + "request_latency": {}, + "operation_counts": {}, + "success_rates": {}, + "volume_attach_timing": {} + }, + "node": { + "mount_timing": {}, + "mount_errors": {}, + "resource_utilization": {} + }, + "efs": { + "access_point_timing": {}, + "mount_completion_timing": {}, + "api_throttling_incidents": [] + }, + "k8s_events": { + "volume_events": [], + "binding_times": {}, + "pod_startup_delays": {} + } + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_165552_20250605_165552.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_165552_20250605_165552.json new file mode 100644 index 000000000..826a5bd2b --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_165552_20250605_165552.json @@ -0,0 +1,104 @@ +{ + "test_name": "efs_tests_20250605_165552", + "test_type": "general", + "timestamp": "20250605_165552", + "system_info": { + "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", + "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", + "python_version": "3.12.10", + "cpu_count": 8, + "memory_total_gb": 30.88, + "timestamp": "2025-06-05T16:55:52.179504", + "kubernetes_version": "Unknown", + "aws_region": "Unknown", + "efs_filesystem_id": "Unknown" + }, + "results": { + "orchestrator": { + "test_duration": 1749142552.17605, + "operations": { + "create_pvc": { + "success": 8, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 8, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 1, + "read_success": 0, + "read_fail": 1, + "write_success_rate": 0.0, + "read_success_rate": 0.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "many_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + } + } + }, + "metrics": { + "operations": {}, + "system": {}, + "kubernetes": {}, + "csi": {}, + "controller": { + "request_latency": {}, + "operation_counts": {}, + "success_rates": {}, + "volume_attach_timing": {} + }, + "node": { + "mount_timing": {}, + "mount_errors": {}, + "resource_utilization": {} + }, + "efs": { + "access_point_timing": {}, + "mount_completion_timing": {}, + "api_throttling_incidents": [] + }, + "k8s_events": { + "volume_events": [], + "binding_times": {}, + "pod_startup_delays": {} + } + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_171216_20250605_171216.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_171216_20250605_171216.json new file mode 100644 index 000000000..a629ca01a --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_171216_20250605_171216.json @@ -0,0 +1,104 @@ +{ + "test_name": "efs_tests_20250605_171216", + "test_type": "general", + "timestamp": "20250605_171216", + "system_info": { + "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", + "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", + "python_version": "3.12.10", + "cpu_count": 8, + "memory_total_gb": 30.88, + "timestamp": "2025-06-05T17:12:16.584971", + "kubernetes_version": "Unknown", + "aws_region": "Unknown", + "efs_filesystem_id": "Unknown" + }, + "results": { + "orchestrator": { + "test_duration": 1749143536.5820618, + "operations": { + "create_pvc": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 7, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 7, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 1, + "read_success": 0, + "read_fail": 1, + "write_success_rate": 0.0, + "read_success_rate": 0.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "many_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + } + } + }, + "metrics": { + "operations": {}, + "system": {}, + "kubernetes": {}, + "csi": {}, + "controller": { + "request_latency": {}, + "operation_counts": {}, + "success_rates": {}, + "volume_attach_timing": {} + }, + "node": { + "mount_timing": {}, + "mount_errors": {}, + "resource_utilization": {} + }, + "efs": { + "access_point_timing": {}, + "mount_completion_timing": {}, + "api_throttling_incidents": [] + }, + "k8s_events": { + "volume_events": [], + "binding_times": {}, + "pod_startup_delays": {} + } + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_182932_20250605_182932.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_182932_20250605_182932.json new file mode 100644 index 000000000..fcfbb965f --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_182932_20250605_182932.json @@ -0,0 +1,104 @@ +{ + "test_name": "efs_tests_20250605_182932", + "test_type": "general", + "timestamp": "20250605_182932", + "system_info": { + "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", + "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", + "python_version": "3.12.10", + "cpu_count": 8, + "memory_total_gb": 30.88, + "timestamp": "2025-06-05T18:29:32.603793", + "kubernetes_version": "Unknown", + "aws_region": "Unknown", + "efs_filesystem_id": "Unknown" + }, + "results": { + "orchestrator": { + "test_duration": 1749148172.6007035, + "operations": { + "create_pvc": { + "success": 3, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 7, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 7, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 3, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 1, + "read_success": 0, + "read_fail": 1, + "write_success_rate": 0.0, + "read_success_rate": 0.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "many_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + } + } + }, + "metrics": { + "operations": {}, + "system": {}, + "kubernetes": {}, + "csi": {}, + "controller": { + "request_latency": {}, + "operation_counts": {}, + "success_rates": {}, + "volume_attach_timing": {} + }, + "node": { + "mount_timing": {}, + "mount_errors": {}, + "resource_utilization": {} + }, + "efs": { + "access_point_timing": {}, + "mount_completion_timing": {}, + "api_throttling_incidents": [] + }, + "k8s_events": { + "volume_events": [], + "binding_times": {}, + "pod_startup_delays": {} + } + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_193407_20250605_193407.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_193407_20250605_193407.json new file mode 100644 index 000000000..331016cd9 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_193407_20250605_193407.json @@ -0,0 +1,104 @@ +{ + "test_name": "efs_tests_20250605_193407", + "test_type": "general", + "timestamp": "20250605_193407", + "system_info": { + "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", + "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", + "python_version": "3.12.10", + "cpu_count": 8, + "memory_total_gb": 30.88, + "timestamp": "2025-06-05T19:34:07.407073", + "kubernetes_version": "Unknown", + "aws_region": "Unknown", + "efs_filesystem_id": "Unknown" + }, + "results": { + "orchestrator": { + "test_duration": 1749152047.404046, + "operations": { + "create_pvc": { + "success": 7, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 5, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 5, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 7, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 1, + "write_fail": 0, + "read_success": 1, + "read_fail": 0, + "write_success_rate": 100.0, + "read_success_rate": 100.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 1, + "success": 1, + "fail": 0, + "success_rate": 100.0 + }, + "many_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + } + } + }, + "metrics": { + "operations": {}, + "system": {}, + "kubernetes": {}, + "csi": {}, + "controller": { + "request_latency": {}, + "operation_counts": {}, + "success_rates": {}, + "volume_attach_timing": {} + }, + "node": { + "mount_timing": {}, + "mount_errors": {}, + "resource_utilization": {} + }, + "efs": { + "access_point_timing": {}, + "mount_completion_timing": {}, + "api_throttling_incidents": [] + }, + "k8s_events": { + "volume_events": [], + "binding_times": {}, + "pod_startup_delays": {} + } + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_200124_20250605_200124.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_200124_20250605_200124.json new file mode 100644 index 000000000..dfa7a6bda --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_200124_20250605_200124.json @@ -0,0 +1,104 @@ +{ + "test_name": "efs_tests_20250605_200124", + "test_type": "general", + "timestamp": "20250605_200124", + "system_info": { + "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", + "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", + "python_version": "3.12.10", + "cpu_count": 8, + "memory_total_gb": 30.88, + "timestamp": "2025-06-05T20:01:24.758278", + "kubernetes_version": "Unknown", + "aws_region": "Unknown", + "efs_filesystem_id": "Unknown" + }, + "results": { + "orchestrator": { + "test_duration": 1749153684.7552128, + "operations": { + "create_pvc": { + "success": 13, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 9, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 9, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 13, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 2, + "write_fail": 0, + "read_success": 2, + "read_fail": 0, + "write_success_rate": 100.0, + "read_success_rate": 100.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 2, + "success": 2, + "fail": 0, + "success_rate": 100.0 + }, + "many_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 2, + "success": 2, + "fail": 0, + "success_rate": 100.0 + } + } + }, + "metrics": { + "operations": {}, + "system": {}, + "kubernetes": {}, + "csi": {}, + "controller": { + "request_latency": {}, + "operation_counts": {}, + "success_rates": {}, + "volume_attach_timing": {} + }, + "node": { + "mount_timing": {}, + "mount_errors": {}, + "resource_utilization": {} + }, + "efs": { + "access_point_timing": {}, + "mount_completion_timing": {}, + "api_throttling_incidents": [] + }, + "k8s_events": { + "volume_events": [], + "binding_times": {}, + "pod_startup_delays": {} + } + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_201716_20250605_201716.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_201716_20250605_201716.json new file mode 100644 index 000000000..1b0bbef9f --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_201716_20250605_201716.json @@ -0,0 +1,104 @@ +{ + "test_name": "efs_tests_20250605_201716", + "test_type": "general", + "timestamp": "20250605_201716", + "system_info": { + "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", + "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", + "python_version": "3.12.10", + "cpu_count": 8, + "memory_total_gb": 30.88, + "timestamp": "2025-06-05T20:17:16.474900", + "kubernetes_version": "Unknown", + "aws_region": "Unknown", + "efs_filesystem_id": "Unknown" + }, + "results": { + "orchestrator": { + "test_duration": 1749154636.4718866, + "operations": { + "create_pvc": { + "success": 2, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 11, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 11, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 4, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 1, + "write_fail": 0, + "read_success": 1, + "read_fail": 0, + "write_success_rate": 100.0, + "read_success_rate": 100.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 1, + "success": 1, + "fail": 0, + "success_rate": 100.0 + }, + "many_to_one": { + "runs": 2, + "success": 2, + "fail": 0, + "success_rate": 100.0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + } + } + }, + "metrics": { + "operations": {}, + "system": {}, + "kubernetes": {}, + "csi": {}, + "controller": { + "request_latency": {}, + "operation_counts": {}, + "success_rates": {}, + "volume_attach_timing": {} + }, + "node": { + "mount_timing": {}, + "mount_errors": {}, + "resource_utilization": {} + }, + "efs": { + "access_point_timing": {}, + "mount_completion_timing": {}, + "api_throttling_incidents": [] + }, + "k8s_events": { + "volume_events": [], + "binding_times": {}, + "pod_startup_delays": {} + } + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_145849.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_145849.json new file mode 100644 index 000000000..d77f05172 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_145849.json @@ -0,0 +1,59 @@ +{ + "test_duration": 1748962729.2581227, + "operations": { + "create_pvc": { + "success": 1, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 9, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 9, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 3, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 1, + "read_success": 0, + "read_fail": 1, + "write_success_rate": 0.0, + "read_success_rate": 0.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "many_to_one": { + "runs": 2, + "success": 0, + "fail": 2, + "success_rate": 0.0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_160604.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_160604.json new file mode 100644 index 000000000..bfafde25a --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_160604.json @@ -0,0 +1,59 @@ +{ + "test_duration": 1748966764.4565506, + "operations": { + "create_pvc": { + "success": 17, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 23, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 23, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 27, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 1, + "read_success": 0, + "read_fail": 1, + "write_success_rate": 0.0, + "read_success_rate": 0.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "many_to_one": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "one_to_one": { + "runs": 2, + "success": 0, + "fail": 2, + "success_rate": 0.0 + }, + "concurrent_pvc": { + "runs": 2, + "success": 2, + "fail": 0, + "success_rate": 100.0 + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_180130.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_180130.json new file mode 100644 index 000000000..7d0710919 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_180130.json @@ -0,0 +1,59 @@ +{ + "test_duration": 1748973690.377472, + "operations": { + "create_pvc": { + "success": 1, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 4, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 4, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 2, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 0, + "read_success": 0, + "read_fail": 0, + "write_success_rate": 0, + "read_success_rate": 0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "many_to_one": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_204448.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_204448.json new file mode 100644 index 000000000..2b13f3158 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_204448.json @@ -0,0 +1,59 @@ +{ + "test_duration": 1748983488.7475276, + "operations": { + "create_pvc": { + "success": 0, + "fail": 7, + "success_rate": 0.0 + }, + "attach_pod": { + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "delete_pod": { + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "delete_pvc": { + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 0, + "read_success": 0, + "read_fail": 0, + "write_success_rate": 0, + "read_success_rate": 0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "many_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_205333.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_205333.json new file mode 100644 index 000000000..b20e31af6 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_205333.json @@ -0,0 +1,59 @@ +{ + "test_duration": 1748984013.159983, + "operations": { + "create_pvc": { + "success": 1, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 5, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 5, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 2, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 0, + "read_success": 0, + "read_fail": 0, + "write_success_rate": 0, + "read_success_rate": 0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "many_to_one": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_210938.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_210938.json new file mode 100644 index 000000000..77933bb92 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_210938.json @@ -0,0 +1,59 @@ +{ + "test_duration": 1748984978.40179, + "operations": { + "create_pvc": { + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "attach_pod": { + "success": 9, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 9, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 1, + "read_success": 0, + "read_fail": 1, + "write_success_rate": 0.0, + "read_success_rate": 0.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "many_to_one": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "one_to_one": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "concurrent_pvc": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250604_204834.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250604_204834.json new file mode 100644 index 000000000..69e4dfcad --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250604_204834.json @@ -0,0 +1,59 @@ +{ + "test_duration": 1749070114.2102087, + "operations": { + "create_pvc": { + "success": 2, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 5, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 5, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 0, + "read_success": 0, + "read_fail": 0, + "write_success_rate": 0, + "read_success_rate": 0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "many_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "one_to_one": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "concurrent_pvc": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250604_211347.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250604_211347.json new file mode 100644 index 000000000..c44b60ca6 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250604_211347.json @@ -0,0 +1,59 @@ +{ + "test_duration": 1749071627.1310592, + "operations": { + "create_pvc": { + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "attach_pod": { + "success": 1, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 1, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 1, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 0, + "read_success": 0, + "read_fail": 0, + "write_success_rate": 0, + "read_success_rate": 0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "many_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "one_to_one": { + "runs": 1, + "success": 0, + "fail": 0, + "success_rate": 0.0 + }, + "concurrent_pvc": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250604_211457.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250604_211457.json new file mode 100644 index 000000000..793073ebd --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250604_211457.json @@ -0,0 +1,59 @@ +{ + "test_duration": 1749071697.3223903, + "operations": { + "create_pvc": { + "success": 4, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 1, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 1, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 4, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 0, + "read_success": 0, + "read_fail": 0, + "write_success_rate": 0, + "read_success_rate": 0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "many_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 1, + "success": 0, + "fail": 0, + "success_rate": 0.0 + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_133650.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_133650.json new file mode 100644 index 000000000..86ea6b2eb --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_133650.json @@ -0,0 +1,59 @@ +{ + "test_duration": 1749130610.6835227, + "operations": { + "create_pvc": { + "success": 12, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 8, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 8, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 12, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 0, + "read_success": 0, + "read_fail": 0, + "write_success_rate": 0, + "read_success_rate": 0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "many_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 2, + "success": 2, + "fail": 0, + "success_rate": 100.0 + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_135626.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_135626.json new file mode 100644 index 000000000..a05a9878f --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_135626.json @@ -0,0 +1,59 @@ +{ + "test_duration": 1749131786.2873476, + "operations": { + "create_pvc": { + "success": 2, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 5, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 1, + "read_success": 0, + "read_fail": 1, + "write_success_rate": 0.0, + "read_success_rate": 0.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "many_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "one_to_one": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "concurrent_pvc": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_141131.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_141131.json new file mode 100644 index 000000000..8fd7476b9 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_141131.json @@ -0,0 +1,59 @@ +{ + "test_duration": 1749132691.2088823, + "operations": { + "create_pvc": { + "success": 2, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 7, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 7, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 1, + "read_success": 0, + "read_fail": 1, + "write_success_rate": 0.0, + "read_success_rate": 0.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "many_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "one_to_one": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "concurrent_pvc": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_142059.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_142059.json new file mode 100644 index 000000000..fda6ab914 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_142059.json @@ -0,0 +1,59 @@ +{ + "test_duration": 1749133259.4106185, + "operations": { + "create_pvc": { + "success": 7, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 7, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 1, + "read_success": 0, + "read_fail": 1, + "write_success_rate": 0.0, + "read_success_rate": 0.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "many_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 1, + "success": 1, + "fail": 0, + "success_rate": 100.0 + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_145750.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_145750.json new file mode 100644 index 000000000..889fb64be --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_145750.json @@ -0,0 +1,59 @@ +{ + "test_duration": 1749135470.4266138, + "operations": { + "create_pvc": { + "success": 2, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 3, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 1, + "read_success": 0, + "read_fail": 1, + "write_success_rate": 0.0, + "read_success_rate": 0.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "many_to_one": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_150603.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_150603.json new file mode 100644 index 000000000..96d858b04 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_150603.json @@ -0,0 +1,59 @@ +{ + "test_duration": 1749135963.512927, + "operations": { + "create_pvc": { + "success": 5, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 5, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 1, + "read_success": 0, + "read_fail": 1, + "write_success_rate": 0.0, + "read_success_rate": 0.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "many_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 1, + "success": 1, + "fail": 0, + "success_rate": 100.0 + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_151246.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_151246.json new file mode 100644 index 000000000..dcafe6de2 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_151246.json @@ -0,0 +1,59 @@ +{ + "test_duration": 1749136366.8090806, + "operations": { + "create_pvc": { + "success": 7, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 7, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 1, + "read_success": 0, + "read_fail": 1, + "write_success_rate": 0.0, + "read_success_rate": 0.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "many_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 1, + "success": 1, + "fail": 0, + "success_rate": 100.0 + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_154402.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_154402.json new file mode 100644 index 000000000..4b6eff392 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_154402.json @@ -0,0 +1,59 @@ +{ + "test_duration": 1749138242.406942, + "operations": { + "create_pvc": { + "success": 2, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 3, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 1, + "read_success": 0, + "read_fail": 1, + "write_success_rate": 0.0, + "read_success_rate": 0.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "many_to_one": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_155202.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_155202.json new file mode 100644 index 000000000..8952e04d1 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_155202.json @@ -0,0 +1,59 @@ +{ + "test_duration": 1749138722.968492, + "operations": { + "create_pvc": { + "success": 5, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 5, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 1, + "read_success": 0, + "read_fail": 1, + "write_success_rate": 0.0, + "read_success_rate": 0.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "many_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 1, + "success": 1, + "fail": 0, + "success_rate": 100.0 + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_160701.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_160701.json new file mode 100644 index 000000000..a440b0e36 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_160701.json @@ -0,0 +1,59 @@ +{ + "test_duration": 1749139621.1574326, + "operations": { + "create_pvc": { + "success": 2, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 7, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 7, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 3, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 1, + "read_success": 0, + "read_fail": 1, + "write_success_rate": 0.0, + "read_success_rate": 0.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "many_to_one": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_161509.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_161509.json new file mode 100644 index 000000000..6e139bc85 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_161509.json @@ -0,0 +1,59 @@ +{ + "test_duration": 1749140109.9788902, + "operations": { + "create_pvc": { + "success": 9, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 9, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 1, + "read_success": 0, + "read_fail": 1, + "write_success_rate": 0.0, + "read_success_rate": 0.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "many_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 1, + "success": 1, + "fail": 0, + "success_rate": 100.0 + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_163010.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_163010.json new file mode 100644 index 000000000..29617dfab --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_163010.json @@ -0,0 +1,59 @@ +{ + "test_duration": 1749141010.8966942, + "operations": { + "create_pvc": { + "success": 2, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 5, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 1, + "read_success": 0, + "read_fail": 1, + "write_success_rate": 0.0, + "read_success_rate": 0.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "many_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "one_to_one": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "concurrent_pvc": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_164017.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_164017.json new file mode 100644 index 000000000..8f2fe7194 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_164017.json @@ -0,0 +1,59 @@ +{ + "test_duration": 1749141617.450957, + "operations": { + "create_pvc": { + "success": 3, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 5, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 5, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 3, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 1, + "read_success": 0, + "read_fail": 1, + "write_success_rate": 0.0, + "read_success_rate": 0.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "many_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_165552.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_165552.json new file mode 100644 index 000000000..3e2159340 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_165552.json @@ -0,0 +1,59 @@ +{ + "test_duration": 1749142552.17605, + "operations": { + "create_pvc": { + "success": 8, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 8, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 1, + "read_success": 0, + "read_fail": 1, + "write_success_rate": 0.0, + "read_success_rate": 0.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "many_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_171216.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_171216.json new file mode 100644 index 000000000..a0fa1ac4b --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_171216.json @@ -0,0 +1,59 @@ +{ + "test_duration": 1749143536.5820618, + "operations": { + "create_pvc": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 7, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 7, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 1, + "read_success": 0, + "read_fail": 1, + "write_success_rate": 0.0, + "read_success_rate": 0.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "many_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_182932.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_182932.json new file mode 100644 index 000000000..a02f395fe --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_182932.json @@ -0,0 +1,59 @@ +{ + "test_duration": 1749148172.6007035, + "operations": { + "create_pvc": { + "success": 3, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 7, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 7, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 3, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 0, + "write_fail": 1, + "read_success": 0, + "read_fail": 1, + "write_success_rate": 0.0, + "read_success_rate": 0.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 1, + "success": 0, + "fail": 1, + "success_rate": 0.0 + }, + "many_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_193407.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_193407.json new file mode 100644 index 000000000..606ae9787 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_193407.json @@ -0,0 +1,59 @@ +{ + "test_duration": 1749152047.404046, + "operations": { + "create_pvc": { + "success": 7, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 5, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 5, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 7, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 1, + "write_fail": 0, + "read_success": 1, + "read_fail": 0, + "write_success_rate": 100.0, + "read_success_rate": 100.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 1, + "success": 1, + "fail": 0, + "success_rate": 100.0 + }, + "many_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_200124.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_200124.json new file mode 100644 index 000000000..5080be634 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_200124.json @@ -0,0 +1,59 @@ +{ + "test_duration": 1749153684.7552128, + "operations": { + "create_pvc": { + "success": 13, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 9, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 9, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 13, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 2, + "write_fail": 0, + "read_success": 2, + "read_fail": 0, + "write_success_rate": 100.0, + "read_success_rate": 100.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 2, + "success": 2, + "fail": 0, + "success_rate": 100.0 + }, + "many_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 2, + "success": 2, + "fail": 0, + "success_rate": 100.0 + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_201716.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_201716.json new file mode 100644 index 000000000..a62039169 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_201716.json @@ -0,0 +1,59 @@ +{ + "test_duration": 1749154636.4718866, + "operations": { + "create_pvc": { + "success": 2, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 11, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 11, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 4, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 1, + "write_fail": 0, + "read_success": 1, + "read_fail": 0, + "write_success_rate": 100.0, + "read_success_rate": 100.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 1, + "success": 1, + "fail": 0, + "success_rate": 100.0 + }, + "many_to_one": { + "runs": 2, + "success": 2, + "fail": 0, + "success_rate": 100.0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/scalability/efs_tests_20250602_195427_20250602_195427.json b/test/e2e/stress-scale-tests/reports/scalability/efs_tests_20250602_195427_20250602_195427.json new file mode 100644 index 000000000..90e53bbd7 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/scalability/efs_tests_20250602_195427_20250602_195427.json @@ -0,0 +1,100 @@ +{ + "test_name": "efs_tests_20250602_195427", + "test_type": "scalability", + "timestamp": "20250602_195427", + "system_info": { + "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", + "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", + "python_version": "3.12.10", + "cpu_count": 8, + "memory_total_gb": 30.88, + "timestamp": "2025-06-02T19:54:27.467966", + "kubernetes_version": "Unknown", + "aws_region": "us-west-1", + "efs_filesystem_id": "fs-02b74eafb14930828" + }, + "results": { + "scalability": { + "pod_scaling": { + "5": { + "duration": 5.151206731796265, + "success": true, + "pods_ready": 5 + }, + "10": { + "duration": 5.1346681118011475, + "success": true, + "pods_ready": 10 + }, + "15": { + "duration": 10.208761215209961, + "success": true, + "pods_ready": 15 + }, + "20": { + "duration": 10.21699595451355, + "success": true, + "pods_ready": 20 + }, + "25": { + "duration": 25.416666746139526, + "success": true, + "pods_ready": 25 + }, + "30": { + "duration": 10.209507942199707, + "success": true, + "pods_ready": 30 + }, + "35": { + "duration": 30.496685028076172, + "success": true, + "pods_ready": 35 + }, + "40": { + "duration": 15.27869462966919, + "success": true, + "pods_ready": 40 + }, + "45": { + "duration": 304.97394347190857, + "success": false, + "pods_ready": 44 + }, + "50": { + "duration": 304.2392809391022, + "success": false, + "pods_ready": 44 + } + }, + "volume_scaling": { + "1": { + "duration": 5.21432638168335, + "success": true, + "volumes_ready": 1 + }, + "6": { + "duration": 16.051532745361328, + "success": true, + "volumes_ready": 6 + }, + "11": { + "duration": 11.683096647262573, + "success": true, + "volumes_ready": 11 + }, + "16": { + "duration": 12.391531705856323, + "success": true, + "volumes_ready": 16 + } + } + }, + "metrics": { + "operations": {}, + "system": {}, + "kubernetes": {}, + "csi": {} + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/scalability/efs_tests_20250602_195427_20250602_195427_summary.txt b/test/e2e/stress-scale-tests/reports/scalability/efs_tests_20250602_195427_20250602_195427_summary.txt new file mode 100644 index 000000000..5ea1adb78 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/scalability/efs_tests_20250602_195427_20250602_195427_summary.txt @@ -0,0 +1,22 @@ +================================================================================ +EFS CSI DRIVER TEST REPORT: EFS_TESTS_20250602_195427 +================================================================================ + +SYSTEM INFORMATION +-------------------------------------------------------------------------------- +Hostname: dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com +Platform: Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26 +Python Version: 3.12.10 +CPU Count: 8 +Memory Total: 30.88 GB +Kubernetes Version: Unknown +AWS Region: us-west-1 +EFS Filesystem ID: fs-02b74eafb14930828 +Test Timestamp: 2025-06-02T19:54:27.512219 + +TEST RESULTS +-------------------------------------------------------------------------------- + +================================================================================ +END OF REPORT: 2025-06-02T19:54:27.556124 +================================================================================ diff --git a/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_183718_20250602_183718.json b/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_183718_20250602_183718.json new file mode 100644 index 000000000..23db83d18 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_183718_20250602_183718.json @@ -0,0 +1,58 @@ +{ + "test_name": "efs_tests_20250602_183718", + "test_type": "stress", + "timestamp": "20250602_183718", + "system_info": { + "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", + "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", + "python_version": "3.12.10", + "cpu_count": 8, + "memory_total_gb": 30.88, + "timestamp": "2025-06-02T18:37:18.745829", + "kubernetes_version": "Unknown", + "aws_region": "us-west-1", + "efs_filesystem_id": "fs-02b74eafb14930828" + }, + "results": { + "stress": { + "single_pvc": { + "operation": "concurrent_mount_single_pvc", + "duration": 4.789013147354126, + "pods": 20, + "pods_created": 20 + }, + "multi_pvc": { + "operation": "concurrent_mount_multiple_pvc", + "duration": 8.21653962135315, + "pvcs_created": 10, + "pods_created": 20, + "total_pods": 20 + }, + "sequential_write": { + "status": "running", + "duration": 184.95399236679077, + "pod": "sequential-write-test-pod", + "pvc": "io-test-pvc-sequential-write" + }, + "random_write": { + "status": "running", + "duration": 185.02785754203796, + "pod": "random-write-test-pod", + "pvc": "io-test-pvc-random-write" + }, + "mixed_io": { + "status": "running", + "duration": 180.6199607849121, + "pod": "mixed-io-test-pod", + "pvc": "io-test-pvc-mixed-io" + }, + "total_duration": 552.7662425041199 + }, + "metrics": { + "operations": {}, + "system": {}, + "kubernetes": {}, + "csi": {} + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_183718_20250602_183718_summary.txt b/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_183718_20250602_183718_summary.txt new file mode 100644 index 000000000..6cecb22e9 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_183718_20250602_183718_summary.txt @@ -0,0 +1,40 @@ +================================================================================ +EFS CSI DRIVER TEST REPORT: EFS_TESTS_20250602_183718 +================================================================================ + +SYSTEM INFORMATION +-------------------------------------------------------------------------------- +Hostname: dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com +Platform: Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26 +Python Version: 3.12.10 +CPU Count: 8 +Memory Total: 30.88 GB +Kubernetes Version: Unknown +AWS Region: us-west-1 +EFS Filesystem ID: fs-02b74eafb14930828 +Test Timestamp: 2025-06-02T18:37:18.790768 + +TEST RESULTS +-------------------------------------------------------------------------------- + +STRESS RESULTS +---------------------------------------- + +Sequential Write Test: + Status: running + Duration: 184.95399236679077 seconds + +Random Write Test: + Status: running + Duration: 185.02785754203796 seconds + +Mixed I/O Test: + Status: running + Duration: 180.6199607849121 seconds + +METRICS RESULTS +---------------------------------------- + +================================================================================ +END OF REPORT: 2025-06-02T18:37:18.834481 +================================================================================ diff --git a/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_201648_20250602_201648.json b/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_201648_20250602_201648.json new file mode 100644 index 000000000..4260f4bfd --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_201648_20250602_201648.json @@ -0,0 +1,58 @@ +{ + "test_name": "efs_tests_20250602_201648", + "test_type": "stress", + "timestamp": "20250602_201648", + "system_info": { + "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", + "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", + "python_version": "3.12.10", + "cpu_count": 8, + "memory_total_gb": 30.88, + "timestamp": "2025-06-02T20:16:48.627010", + "kubernetes_version": "Unknown", + "aws_region": "us-west-1", + "efs_filesystem_id": "fs-02b74eafb14930828" + }, + "results": { + "concurrent_api": { + "single_pvc": { + "operation": "concurrent_mount_single_pvc", + "duration": 4.969545602798462, + "pods": 20, + "pods_created": 20 + }, + "multi_pvc": { + "operation": "concurrent_mount_multiple_pvc", + "duration": 7.825149059295654, + "pvcs_created": 10, + "pods_created": 20, + "total_pods": 20 + }, + "sequential_write": { + "status": "running", + "duration": 185.06655764579773, + "pod": "sequential-write-test-pod", + "pvc": "io-test-pvc-sequential-write" + }, + "random_write": { + "status": "running", + "duration": 185.08776235580444, + "pod": "random-write-test-pod", + "pvc": "io-test-pvc-random-write" + }, + "mixed_io": { + "status": "running", + "duration": 180.72687149047852, + "pod": "mixed-io-test-pod", + "pvc": "io-test-pvc-mixed-io" + }, + "total_duration": 552.9746143817902 + }, + "metrics": { + "operations": {}, + "system": {}, + "kubernetes": {}, + "csi": {} + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_201648_20250602_201648_summary.txt b/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_201648_20250602_201648_summary.txt new file mode 100644 index 000000000..835064c5e --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_201648_20250602_201648_summary.txt @@ -0,0 +1,40 @@ +================================================================================ +EFS CSI DRIVER TEST REPORT: EFS_TESTS_20250602_201648 +================================================================================ + +SYSTEM INFORMATION +-------------------------------------------------------------------------------- +Hostname: dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com +Platform: Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26 +Python Version: 3.12.10 +CPU Count: 8 +Memory Total: 30.88 GB +Kubernetes Version: Unknown +AWS Region: us-west-1 +EFS Filesystem ID: fs-02b74eafb14930828 +Test Timestamp: 2025-06-02T20:16:48.670978 + +TEST RESULTS +-------------------------------------------------------------------------------- + +CONCURRENT_API RESULTS +---------------------------------------- + +Sequential Write Test: + Status: running + Duration: 185.06655764579773 seconds + +Random Write Test: + Status: running + Duration: 185.08776235580444 seconds + +Mixed I/O Test: + Status: running + Duration: 180.72687149047852 seconds + +METRICS RESULTS +---------------------------------------- + +================================================================================ +END OF REPORT: 2025-06-02T20:16:48.713866 +================================================================================ diff --git a/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_210512_20250602_210512.json b/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_210512_20250602_210512.json new file mode 100644 index 000000000..3c97bcf97 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_210512_20250602_210512.json @@ -0,0 +1,79 @@ +{ + "test_name": "efs_tests_20250602_210512", + "test_type": "stress", + "timestamp": "20250602_210512", + "system_info": { + "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", + "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", + "python_version": "3.12.10", + "cpu_count": 8, + "memory_total_gb": 30.88, + "timestamp": "2025-06-02T21:05:12.574471", + "kubernetes_version": "Unknown", + "aws_region": "us-west-1", + "efs_filesystem_id": "fs-02b74eafb14930828" + }, + "results": { + "concurrent_api": { + "single_pvc": { + "operation": "concurrent_mount_single_pvc", + "duration": 5.6173036098480225, + "pods": 20, + "pods_created": 20 + }, + "multi_pvc": { + "operation": "concurrent_mount_multiple_pvc", + "duration": 8.005990028381348, + "pvcs_created": 10, + "pods_created": 20, + "total_pods": 20 + }, + "sequential_write": { + "status": "running", + "duration": 184.9669418334961, + "pod": "sequential-write-test-pod", + "pvc": "io-test-pvc-sequential-write" + }, + "random_write": { + "status": "running", + "duration": 185.1048445701599, + "pod": "random-write-test-pod", + "pvc": "io-test-pvc-random-write" + }, + "mixed_io": { + "status": "running", + "duration": 180.83833074569702, + "pod": "mixed-io-test-pod", + "pvc": "io-test-pvc-mixed-io" + }, + "total_duration": 552.9123649597168 + }, + "metrics": { + "operations": {}, + "system": {}, + "kubernetes": {}, + "csi": {}, + "controller": { + "request_latency": {}, + "operation_counts": {}, + "success_rates": {}, + "volume_attach_timing": {} + }, + "node": { + "mount_timing": {}, + "mount_errors": {}, + "resource_utilization": {} + }, + "efs": { + "access_point_timing": {}, + "mount_completion_timing": {}, + "api_throttling_incidents": [] + }, + "k8s_events": { + "volume_events": [], + "binding_times": {}, + "pod_startup_delays": {} + } + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_210512_20250602_210512_summary.txt b/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_210512_20250602_210512_summary.txt new file mode 100644 index 000000000..95f882868 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_210512_20250602_210512_summary.txt @@ -0,0 +1,40 @@ +================================================================================ +EFS CSI DRIVER TEST REPORT: EFS_TESTS_20250602_210512 +================================================================================ + +SYSTEM INFORMATION +-------------------------------------------------------------------------------- +Hostname: dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com +Platform: Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26 +Python Version: 3.12.10 +CPU Count: 8 +Memory Total: 30.88 GB +Kubernetes Version: Unknown +AWS Region: us-west-1 +EFS Filesystem ID: fs-02b74eafb14930828 +Test Timestamp: 2025-06-02T21:05:12.619174 + +TEST RESULTS +-------------------------------------------------------------------------------- + +CONCURRENT_API RESULTS +---------------------------------------- + +Sequential Write Test: + Status: running + Duration: 184.9669418334961 seconds + +Random Write Test: + Status: running + Duration: 185.1048445701599 seconds + +Mixed I/O Test: + Status: running + Duration: 180.83833074569702 seconds + +METRICS RESULTS +---------------------------------------- + +================================================================================ +END OF REPORT: 2025-06-02T21:05:12.663439 +================================================================================ diff --git a/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250603_135616_20250603_135616.json b/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250603_135616_20250603_135616.json new file mode 100644 index 000000000..474ba6725 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250603_135616_20250603_135616.json @@ -0,0 +1,216 @@ +{ + "test_name": "efs_tests_20250603_135616", + "test_type": "stress", + "timestamp": "20250603_135616", + "system_info": { + "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", + "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", + "python_version": "3.12.10", + "cpu_count": 8, + "memory_total_gb": 30.88, + "timestamp": "2025-06-03T13:56:16.918247", + "kubernetes_version": "Unknown", + "aws_region": "us-west-1", + "efs_filesystem_id": "fs-02b74eafb14930828" + }, + "results": { + "stress": { + "single_pvc": { + "operation": "concurrent_mount_single_pvc", + "duration": 4.935153245925903, + "pods": 20, + "pods_created": 20 + }, + "multi_pvc": { + "operation": "concurrent_mount_multiple_pvc", + "duration": 7.949950933456421, + "pvcs_created": 10, + "pods_created": 20, + "total_pods": 20 + }, + "sequential_write": { + "status": "running", + "duration": 184.93946361541748, + "pod": "sequential-write-test-pod", + "pvc": "io-test-pvc-sequential-write" + }, + "random_write": { + "status": "running", + "duration": 184.98985695838928, + "pod": "random-write-test-pod", + "pvc": "io-test-pvc-random-write" + }, + "mixed_io": { + "status": "running", + "duration": 180.7335159778595, + "pod": "mixed-io-test-pod", + "pvc": "io-test-pvc-mixed-io" + }, + "total_duration": 552.7618699073792 + }, + "scalability": { + "pod_scaling": { + "5": { + "duration": 5.142866611480713, + "success": true, + "pods_ready": 5 + }, + "10": { + "duration": 5.132429361343384, + "success": true, + "pods_ready": 10 + }, + "15": { + "duration": 10.2158362865448, + "success": true, + "pods_ready": 15 + }, + "20": { + "duration": 10.207310199737549, + "success": true, + "pods_ready": 20 + }, + "25": { + "duration": 25.424506664276123, + "success": true, + "pods_ready": 25 + }, + "30": { + "duration": 10.213911771774292, + "success": true, + "pods_ready": 30 + }, + "35": { + "duration": 25.42242169380188, + "success": true, + "pods_ready": 35 + }, + "40": { + "duration": 15.276111841201782, + "success": true, + "pods_ready": 40 + }, + "45": { + "duration": 305.03443932533264, + "success": false, + "pods_ready": 44 + }, + "50": { + "duration": 304.205361366272, + "success": false, + "pods_ready": 44 + } + }, + "volume_scaling": { + "1": { + "duration": 5.215110540390015, + "success": true, + "volumes_ready": 1 + }, + "6": { + "duration": 10.974458932876587, + "success": true, + "volumes_ready": 6 + }, + "11": { + "duration": 11.669871091842651, + "success": true, + "volumes_ready": 11 + }, + "16": { + "duration": 12.36742639541626, + "success": true, + "volumes_ready": 16 + } + } + }, + "access_points": { + "access_point_scaling": { + "2": { + "duration": 5.353680372238159, + "success": true, + "access_points_ready": 2 + }, + "7": { + "duration": 6.05517315864563, + "success": true, + "access_points_ready": 7 + }, + "12": { + "duration": 6.7946085929870605, + "success": true, + "access_points_ready": 12 + }, + "17": { + "duration": 7.489370346069336, + "success": true, + "access_points_ready": 17 + } + } + }, + "statefulset": { + "statefulset_scaling": { + "replicas": 20, + "duration": 10.138212442398071, + "success": true, + "pods_ready": 20 + } + }, + "chaos": { + "api_flood": { + "test": "rapid_api_flood", + "pvcs_requested": 50, + "pvcs_created": 50, + "success_rate": 100.0, + "duration": 0.46927380561828613, + "access_points_used": true + }, + "race_conditions": { + "test": "race_conditions", + "iterations": 30, + "successes": 17, + "failures": 13, + "exceptions": 0, + "duration": 6.168970346450806 + }, + "resource_exhaustion": { + "test": "resource_exhaustion", + "pods_requested": 75, + "pods_created": 75, + "pod_success_rate": 100.0, + "access_points_requested": 25, + "access_points_created": 0, + "ap_success_rate": 0.0, + "duration": 2.9329886436462402 + }, + "error": "type object 'Thread' has no attribute '_wait'" + }, + "metrics": { + "operations": {}, + "system": {}, + "kubernetes": {}, + "csi": {}, + "controller": { + "request_latency": {}, + "operation_counts": {}, + "success_rates": {}, + "volume_attach_timing": {} + }, + "node": { + "mount_timing": {}, + "mount_errors": {}, + "resource_utilization": {} + }, + "efs": { + "access_point_timing": {}, + "mount_completion_timing": {}, + "api_throttling_incidents": [] + }, + "k8s_events": { + "volume_events": [], + "binding_times": {}, + "pod_startup_delays": {} + } + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250603_135616_20250603_135616_summary.txt b/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250603_135616_20250603_135616_summary.txt new file mode 100644 index 000000000..23b550ff2 --- /dev/null +++ b/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250603_135616_20250603_135616_summary.txt @@ -0,0 +1,52 @@ +================================================================================ +EFS CSI DRIVER TEST REPORT: EFS_TESTS_20250603_135616 +================================================================================ + +SYSTEM INFORMATION +-------------------------------------------------------------------------------- +Hostname: dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com +Platform: Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26 +Python Version: 3.12.10 +CPU Count: 8 +Memory Total: 30.88 GB +Kubernetes Version: Unknown +AWS Region: us-west-1 +EFS Filesystem ID: fs-02b74eafb14930828 +Test Timestamp: 2025-06-03T13:56:16.964029 + +TEST RESULTS +-------------------------------------------------------------------------------- + +STRESS RESULTS +---------------------------------------- + +Sequential Write Test: + Status: running + Duration: 184.93946361541748 seconds + +Random Write Test: + Status: running + Duration: 184.98985695838928 seconds + +Mixed I/O Test: + Status: running + Duration: 180.7335159778595 seconds + +SCALABILITY RESULTS +---------------------------------------- + +ACCESS_POINTS RESULTS +---------------------------------------- + +STATEFULSET RESULTS +---------------------------------------- + +CHAOS RESULTS +---------------------------------------- + +METRICS RESULTS +---------------------------------------- + +================================================================================ +END OF REPORT: 2025-06-03T13:56:17.008291 +================================================================================ diff --git a/test/e2e/stress-scale-tests/requirements.txt b/test/e2e/stress-scale-tests/requirements.txt new file mode 100644 index 000000000..315306b91 --- /dev/null +++ b/test/e2e/stress-scale-tests/requirements.txt @@ -0,0 +1,6 @@ +kubernetes +pytest +pyyaml +prometheus_client +pandas +psutil diff --git a/test/e2e/stress-scale-tests/run_tests.py b/test/e2e/stress-scale-tests/run_tests.py new file mode 100644 index 000000000..8b51b9d2e --- /dev/null +++ b/test/e2e/stress-scale-tests/run_tests.py @@ -0,0 +1,327 @@ +#!/usr/bin/env python3 +import os +import sys +import yaml +import logging +import argparse +from datetime import datetime +from kubernetes import client, config +# Import test frameworks +from tests.orchestrator import EFSCSIOrchestrator +from utils.report_generator import ReportGenerator +from utils.metrics_collector import MetricsCollector +# Commented out to remove dependency on cluster setup +# from cluster_setup import ClusterSetup + +def setup_logging(config): + """Setup logging based on configuration + + Args: + config: Configuration dictionary + """ + log_config = config.get('logging', {}) + log_level = getattr(logging, log_config.get('level', 'INFO')) + log_file = log_config.get('file', 'logs/efs_tests.log') + + # Create logs directory if it doesn't exist + os.makedirs(os.path.dirname(log_file), exist_ok=True) + + logging.basicConfig( + level=log_level, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + handlers=[ + logging.FileHandler(log_file), + logging.StreamHandler() + ] + ) + + return logging.getLogger(__name__) + +def parse_args(): + """Parse command line arguments + + Returns: + Parsed arguments + """ + parser = argparse.ArgumentParser(description='Run EFS CSI tests') + parser.add_argument( + '--config', + default='config/orchestrator_config.yaml', + help='Path to configuration file' + ) + parser.add_argument( + '--test-suite', + choices=['orchestrator', 'chaos', 'all'], + default='orchestrator', + help='Test suite to run' + ) + parser.add_argument( + '--duration', + type=int, + default=300, + help='Duration in seconds for test execution' + ) + parser.add_argument( + '--rate', + type=int, + default=5, + help='Operations per second for tests' + ) + parser.add_argument( + '--dry-run', + action='store_true', + help='Print what would be done without executing tests' + ) + + # Cluster setup options - kept for compatibility but functionality is disabled + parser.add_argument( + '--setup-cluster', + action='store_true', + help='Set up EKS cluster and EFS CSI driver before running tests (DISABLED)' + ) + parser.add_argument( + '--kubernetes-version', + help='Kubernetes version to use for cluster (DISABLED)' + ) + parser.add_argument( + '--driver-version', + help='EFS CSI Driver version to install (DISABLED)' + ) + parser.add_argument( + '--skip-cleanup', + action='store_true', + help='Skip cleanup of cluster resources after tests (DISABLED)' + ) + + return parser.parse_args() + +def check_credentials(): + """Check if credentials are valid by making a simple API call""" + try: + # Attempt to get a list of namespaces - a simple, harmless API call + api = client.CoreV1Api() + api.list_namespace(_request_timeout=10) + return True + except Exception as e: + error_str = str(e) + if "401" in error_str or "Unauthorized" in error_str: + return False + # For other types of errors, we assume credentials are valid but other issues exist + return True + +def print_credential_renewal_instructions(): + """Print instructions for renewing AWS credentials""" + print("\n" + "="*80) + print(f"{'AWS CREDENTIALS EXPIRED OR INVALID':^80}") + print("="*80) + print("\nYour AWS credentials have expired or are invalid.") + print("\nTo renew your credentials, run:") + print("\n ada credentials update --provider isengard --role=Admin --once --account YOUR_ACCOUNT_ID") + print("\nOr for temporary credentials:") + print("\n aws sts get-session-token") + print("\nAfter renewing credentials, try running the tests again.") + print("="*80 + "\n") + +def main(): + """Main entry point""" + args = parse_args() + + # Load configuration + try: + with open(args.config, 'r') as f: + config = yaml.safe_load(f) + except Exception as e: + print(f"Error loading configuration: {e}") + sys.exit(1) + + # Setup logging + logger = setup_logging(config) + logger.info(f"Starting EFS CSI tests with configuration from {args.config}") + + # Verify credentials before proceeding + logger.info("Verifying AWS credentials") + if not check_credentials(): + logger.error("AWS credentials are expired or invalid") + print_credential_renewal_instructions() + sys.exit(1) + + # Set up cluster if requested - DISABLED + if args.setup_cluster: + logger.warning("Cluster setup functionality is disabled - skipping setup") + logger.warning("Please make sure you have a working cluster with EFS CSI driver installed") + + # The following code has been commented out + """ + # Update config with command-line overrides + if args.kubernetes_version: + config['cluster']['kubernetes_version'] = args.kubernetes_version + logger.info(f"Using Kubernetes version from command line: {args.kubernetes_version}") + + if args.driver_version: + config['driver']['version'] = args.driver_version + logger.info(f"Using EFS CSI Driver version from command line: {args.driver_version}") + + # Write updated config back to file + with open(args.config, 'w') as f: + yaml.dump(config, f) + + # Initialize and run cluster setup + try: + cluster_setup = ClusterSetup(config_file=args.config) + setup_result = cluster_setup.setup() + logger.info(f"Cluster setup completed: {setup_result}") + except Exception as e: + logger.error(f"Cluster setup failed: {e}") + sys.exit(1) + """ + + # Initialize report generator + report_dir = config.get('reporting', {}).get('output_dir', 'reports') + report_generator = ReportGenerator(output_dir=report_dir) + + # Initialize metrics collector + metrics_collector = MetricsCollector() + + # Run tests based on test suite + results = {} + + try: + if args.test_suite in ['orchestrator', 'all']: + logger.info("Running orchestrator stress test suite") + + if args.dry_run: + logger.info("DRY RUN MODE: Would run orchestrator with randomized operations") + orchestrator_results = { + "status": "would_run", + "description": "Would run the orchestrator with randomized operations" + } + else: + logger.info(f"Starting orchestrator for {args.duration} seconds") + + # Create orchestrator instance - use dedicated orchestrator config file if it exists, + # otherwise fall back to the main config + orchestrator_config = 'config/orchestrator_config.yaml' + if not os.path.exists(orchestrator_config): + logger.warning(f"Orchestrator config file {orchestrator_config} not found, falling back to {args.config}") + orchestrator_config = args.config + + logger.info(f"Using orchestrator config: {orchestrator_config}") + orchestrator = EFSCSIOrchestrator(config_file=orchestrator_config) + + # Override default test parameters if specified + if args.duration: + orchestrator.test_duration = args.duration + logger.info(f"Test duration overridden to {args.duration} seconds") + + if args.rate: # Use rate as operation interval (inverse relationship) + orchestrator.operation_interval = max(1, int(1 / args.rate)) + logger.info(f"Operation interval overridden to {orchestrator.operation_interval} seconds (from rate {args.rate}/s)") + + # Run the orchestrator test + orchestrator_results = orchestrator.run_test() + + # Generate orchestrator test specific report + orchestrator_report_dir = os.path.join(report_dir, 'orchestrator') + os.makedirs(orchestrator_report_dir, exist_ok=True) + + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + orchestrator_report_path = os.path.join( + orchestrator_report_dir, + f"efs_orchestrator_{timestamp}.json" + ) + + with open(orchestrator_report_path, 'w') as f: + import json + json.dump(orchestrator_results, f, indent=2) + + logger.info(f"Orchestrator report generated: {orchestrator_report_path}") + + results['orchestrator'] = orchestrator_results + + if args.test_suite in ['chaos', 'all']: + logger.info("Running chaos tests") + + if args.dry_run: + logger.info("DRY RUN MODE: Would run chaos tests to stress the CSI driver") + chaos_results = { + 'api_flood': {'status': 'would_run'}, + 'race_conditions': {'status': 'would_run'}, + 'resource_exhaustion': {'status': 'would_run'}, + 'mixed_chaos': {'status': 'would_run'} + } + else: + # Import here to avoid loading it unnecessarily for other tests + from tests.chaos_test import ChaosTester + + logger.warning("CAUTION: Running chaos tests that may cause system instability") + chaos_tester = ChaosTester(args.config) + chaos_results = chaos_tester.run_all_tests() + + # Generate chaos test specific reports + chaos_report_dir = os.path.join(report_dir, 'chaos') + os.makedirs(chaos_report_dir, exist_ok=True) + + # Write a dedicated report for chaos tests + chaos_timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + chaos_report_path = os.path.join(chaos_report_dir, f"efs_chaos_{chaos_timestamp}.json") + with open(chaos_report_path, 'w') as f: + import json + json.dump(chaos_results, f, indent=2) + logger.info(f"Chaos test report generated: {chaos_report_path}") + + results['chaos'] = chaos_results + + # Collect CSI metrics + metrics_collector.collect_csi_metrics(config) + results['metrics'] = metrics_collector.get_all_metrics() + + # Generate reports + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + report_formats = config.get('reporting', {}).get('formats', ['json']) + + if 'json' in report_formats: + json_path = report_generator.generate_json_report(results, f"efs_tests_{timestamp}") + logger.info(f"JSON report generated: {json_path}") + + if 'summary' in report_formats: + summary_path = report_generator.generate_summary_report(results, f"efs_tests_{timestamp}") + logger.info(f"Summary report generated: {summary_path}") + + logger.info("All tests completed successfully") + + # Clean up resources if cluster was set up and --skip-cleanup wasn't specified - DISABLED + if args.setup_cluster and not args.skip_cleanup: + logger.warning("Cluster cleanup functionality is disabled") + """ + logger.info("Cleaning up cluster resources") + try: + cluster_setup = ClusterSetup(config_file=args.config) + cluster_setup.cleanup() + logger.info("Cleanup completed successfully") + except Exception as e: + logger.error(f"Error during cleanup: {e}") + # Don't exit with failure, as the tests themselves succeeded + """ + elif args.setup_cluster and args.skip_cleanup: + logger.warning("Cluster cleanup functionality is disabled") + + except Exception as e: + logger.error(f"Error running tests: {e}", exc_info=True) + + # Even if tests failed, try to clean up if requested - DISABLED + if args.setup_cluster and not args.skip_cleanup: + logger.warning("Cluster cleanup functionality is disabled") + """ + try: + logger.info("Attempting cleanup after test failure") + cluster_setup = ClusterSetup(config_file=args.config) + cluster_setup.cleanup() + logger.info("Cleanup completed") + except Exception as cleanup_error: + logger.error(f"Cleanup after failure error: {cleanup_error}") + """ + + sys.exit(1) + +if __name__ == "__main__": + main() diff --git a/test/e2e/stress-scale-tests/tests/__init__.py b/test/e2e/stress-scale-tests/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/e2e/stress-scale-tests/tests/__pycache__/__init__.cpython-312.pyc b/test/e2e/stress-scale-tests/tests/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..649fc6e4c3571841f822be955d8b0327656a813b GIT binary patch literal 200 zcmX@j%ge<81V3NdW`O9&AOanHW&w&!XQ*V*Wb|9fP{ah}eFmxdm9C$YpPZPZpOK%N zs-KyZkywupIBb3iz=>Pl3HA%pK6qgjEsy$%s>_Z DCone& literal 0 HcmV?d00001 diff --git a/test/e2e/stress-scale-tests/tests/__pycache__/base_test.cpython-312.pyc b/test/e2e/stress-scale-tests/tests/__pycache__/base_test.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..74cbed3a4fc852d4af197fa5f7ef7411cd2c0fa8 GIT binary patch literal 8668 zcmcgyeQ;A(cE9gS-~N*PA%9`{3tQNhFyKHOhr}Ozm=A}PsUgy zr!=LrW``YMx?7W$cFA9X37yjaoM~saB)bVao#{w18L2OxDKpI<&3^{76BeekJ?B0> zNj74knan=mzI*RG_ug~QJ?D4NIpW`oiYx@4e<|5L@xPUX{5LlAhtE{j{uL^VL?j{; zB4gT{8DnUh9b@6mhPWwyjGq$51V;OA7&EX$GE5fhljYvfTbgU>HIci)f6QtvhS0Hg z{afxWSw3d8)|izg`-mu$Mb-Fw% zQ|_x?D?mjxOaHFD3sBkCQ(#M{5}7OfC~>o~E&HUXYh0F;3D*;XP+V{vF28G9js)PW z+`MWGL}bZ#s#`T3gHxKonnj65WH?5w8$oUgRNO`xHC!zXMf{@g1PIutDTRz8RLz4h z%bVT((?QiTfb(c|+o(2^)-w0QT(o+ftRd@jhOiZkSaftX8DriP@c;f|^Co2vR$Ra>&EZMn+5;@p&Sx|2@# zva@5Q#+9n+O4f8O*LYX*XzrPXQp+k4%oe#ErVC`%1sbRdROP+|Wm_a>5+{>cwS#Y& zC(gwA8;ow0pmJG-oXnpsLoKp#=I_Ygv%0YZy-}ZOvVii5T#iySiDuCfXM&7qJ)ZlW10nCkrTQA$V&c@{V~SE))bg z{=`l~{3@<$~wu_JE|x@LpX3A8nU9+xD`V0sh&uow!4 zk6&k19)VLeW2sF015&JVND2XHi7x4MP>BK&&e7}o7SH_KpMjpl9DC>_fe9&aBJr0FPdq^LGPqCI+3H7HSkbO!c1>zBhI zH`T2%@jR=VbJGa)2muLE%`;-vp%PeB6xew%CTaU-B-J*3D&U*)hlArNX|zd~_Q+^W zswKk&P~}5XuD1@Tlp;9yGi23dHV-gqnGQm8lX}vc#1t=}MM#-6t$ET2gx9^tscQxBukNS4t}vi!T)?8h6}0dUNkm z)6(lpgNcS$@09lc{^6>C#E&0%NNF_=w6ECg7Y>~}l&Gb7)$_uYXV+sYmz zx&4u{6~yv>Kk08Lw<_!dedJbq{lJUdr^1$j9o(lo1g!U&2RCz{33Y>w+-Hpf)B}jR z`hzDyQCPe9pNhhOt2q>=_?E?Cz>UFqF3uFj;~>#dvEX-3%sOxLd^GDsOqz8Db8kR5 zo}wGK9^EMRaPM(>*kg*h26gP2j))retOp%=3(PFHV?jZ|I1@e*j?9L$xPs#mS8%*s z4|F4LL9HlYED=$D0Y0l7T%d9nP6$v<+yXTXfP#$DQ&W(8kIll7>R-T5kMqT0j(VO7Io{8Ietft&1d)%H}?&Scfj zCBwg&m#Ype?9Zy2=i2FOlS}MU_fm7BdiNdMD^%5tF#eH(7x?@33i5Mvea13Dy1K;^r)?hC3$Q5yHpgraUn-V1}kAvO8vc6~n8;`2>K#F>x;b(_z3 za>h@6W@s6GJ~0x&ZbYbH03dV%M2pvx+=s;=7I^DXUxEru#w(o`7%nKULUES-C*OQm zD7(j+1j`yn4E9y5t(HUWKHDtVGks87J;2aTOQ!Q43pH(WoHqdfvX8FqYiM!n-E{C9 z*t5(S53V^G6TlrOV+IT_#*Cr?O4FRtZB|VK;DH?l?29#H4SaFHNa}#@**oeQ936HI zQ6JAmy|#e99IlV%Nlo-LUHWNmQtrCXyTS{&NV<<4CqA5&J}_qigVQM)0+A3Ks~rvp zqd|WNU|lmP&FOWG!)=DPVWsYH#uKIogJ%a$HH}0R)!PL%bq|##O6E>cjC0 zZrJ70tN^qn2ut~Vu`*q19=+=gpi|sXoFR8jwv@@4G&wKV-Z8bVRJl@B9m%SW<*Kd) z{=6-1vR$yAv;MS5^E~qfiYP+`8iUFr$pt%cM$CGeJ!GDZW(73K1L=k*p7k>GMJclQ zN5?!69otO0iNegX40($%@Mn(Id@nZ_t2zV`1U+(>CX5*wR3F?3Mjipjf(K^_T#7mW zR7f=|{&C5NBBsHIo0Cx#@+cH4Gp-<~03@1KyatU}c~*9L=kllUMM2~{L%u00yV!AA zxl`19_2f#;reAnI@LVffuIX49I)Csxjuf|~ikg!}&5sm3%AeMDbN?*#8Uy(rI?9yF zu>~rNPx0S3C{g7o>VURoDSRp@N5WI!qm8&3wZbQbl^F=Y*7*?tBJ|90;mr14hia@O zyFE><-i1yD(*uZ%#9WjzH6=|=Uzu98^UvoST^7ps-$F0t8#*>VB7@h$JOei(Gnv(c z9F@r4$cr7KS^b#I#v6IGdA|5uIr!WtQu<6HFB_gh@dDhj(Rnt`7K$50;Y2%V!rPD> zFd`N~yn=BIDHumsaaW>^mPJpYDW+dbXjyuN7t5{)^nNc8am2YB_B`PE!%}eMC?O&# zpJ(4==gM<6ttammuZV5hLp8#M7&)qi&(sIkq5?(6$Vwi90<;9#jkTgIYwf zYYxJUDPK@jxt&|LBHqnFhpZZ8={VRj*l{U@J*R>gAAm!FA%$wgfunTbu&5S`L0JmG z{^lTRnNnzK2K2fy~+pyS<1)i!}p7DIv8z--!5IqO#+E|r5Hy;NAC}qs{9caZi z<;=E_wI2f-K&c@nOfcV6th~k~7&f09lOY%6nF8gZ-j))k(@~Q>ETysrTmFiBd24?&#H9 zFIf|HFQrQRlBInQ>PczC<5dn;f280Qc&Y70@=5)mg}YVR-#RFOGcG`5og0pTKq2Wr zKxRKS{@C;kSmFSdxM#uAJRj!^U0g%t-4lfT#F$HsEEvZba9m7rW+sQ~hhFN3)95(S}j{GC!39d6Ekdnho zm0t3Y?qfZ0aZmfBaG!=%4#M+{Lw!bzNX&i5rLc5*THYP&%3fdrZBxGNems3NgRkuN zWXb;m)>vzV;^F_0hi6El^2n{eCF8}}tIZ#Ee%P66+MaCMey3{tt-jx_;br-kO|49< zj7ker40n}zJe26qV)KWf_(^A}Fy%AG*%Hd7A@@}rOv$|xYDY%yD^x3oMG zkB6$Ik9d@h+eS6|f>w(JSzjMY3{ap!QB-$O9;J8-+~m5ZI1Ixo*9 z?d}ERs-ei-Lld8#q{FjRop5-T9fJ!)>8jer*w11YxfMs{GAcC`UpbDfvc`%(Vc?gc zUsyk|CN}R};(lrS#FpAMoZK~>*m)q)a`2AxP}-4n;l8%AfZzUW9%Z?xY0Ba8?ox_FJKEUJ_fB2w`)HS7Q zJCn7YsoEElwJ+W$OlEJk4{V9H-lgro?E9oI_0pl_ONSD@hi|`;IP&^!CDA&XavuHC zc@!K#q<{R)MyHf9_hj>J-kZ!dB~iUSW!v$kZ3hl9_dI@3_s3m1sjvJ<`72O_zwX&< zBDXC4jv){GX*~;#PaAvox027C{f>QB_E#M&G=Al&-zx~ewtDvx;n%$eAo`pL(DS*# zVck@}&%}M+>fFb1UvL7{zp&QtZxX&}_wKJ3ZdVy#hTHW5^xSUZu-;lf+`|2abq+Uh zziAMl4*7ssm6j5rzhGv5VD(@s1WaZh@G0-dVi=19SRBOyGavHnSYWgvBm8bm86L(y zERa^&1O=uHyViJKsJw6EgwFdGPH4SvsSsXX^E3%PT0#&9xQkUbqR3vT%9uz|E%@87 z_T##S1c>3P5r3NjfL5)?{7Nt|NdLcp$`6n1eN8n(j`$G$=|z z$)rOknqw)GMpVX$=p>D)W3^V(H07i{8QGad$+SIqA~ir@HBnoqr_O25p_arGP1^MP z@56go2rZs0Jw1Jt?z`_U|K0EZ-T(gg;s<4ARu0c!|HYOQzyH%5_h0EldGuM!y>1Q1 zUEp}`2*+!9?WpF6hP|~%wCt@rqGNCU5j}ewju_b6c*MxwrXwc2b))7n%Mr_%^@vqN zbo8UPG5ZnwSlN*>_H7t-j5&`uHJp|Un?`&}CUS4Mw&cDP8*jW~`ammxW#hIfez|wp z{Q=5A&k>K78{~NNS2^CI#5+>STffR3sp4%gtHW;I{#qx;3H2=Z-XmV*$UD*}$Imz6 zUj0GVL7}hVCbg4Ve_7~RBxv1!A`}&Uhr?nlGJf25Aj(gShJDAPf^X-pL%!{Y_V{)P zkvGDE)q40uMD&fx{=?DnSST_s`ogDULU=6f6Y&y7UkqeKpLjAn961&l4vmgZ`uMOI zIX)ideX*$TSY({{9Tmc%*C?%%(a3mAv_{5#u@m8<%P3o|bve0*XwBrxjP z9Ee@yBf_7h7!txh5DSmTBGGY5)*3o~oN|P$`$9-`EHvpG4ox6}I4Q=$V+bwALPsN` zk=SIn)%xXugF}0U1~>W+=Mfc8L?=dhUvwO$I2!f|6XS^G3;C$DxBIoN5gM~ zN5`21`yNBdL&xEbR1M5CV?>jVg<`{nokRY*H-q;DF3e%na5Q2tTB>>NS2Y+dI`lUK zuZL;m4KPi-5vG~<@+SCOsFxnG@-^t4nj<#WJMDZeZ^u^|U&oi>?cnQq2i{J;fp_BV z;v0Dv-sOBb;#Tk#Fx|Wxrib^ytmG?UR`FFZt0yh~ri_jCQR<>$A-;0^Xb3%xdgw6g zy3_{LotAf^;q0Jer8L6S;s3oi*i;eA4mOTUV5(4t**b1Y8>3L{8BwMe#R2bxRvAVK zE~tj&we+V@(Qq-75@p1!*m7@WKq8z?aUHQMw%l6@hm=%$c`;nMkcurYhDCA9i{Vk+ za__ttYF=VBihrJOnc|k*S`GISr@`OZ>C?KGxKkRxE?&21 zJQ9n9MkDdC?^Gmqf(8>t2WE!Ra7H5x`JEYk^kjHEqn`|ojbVh42fj2Tn#ZreRovZ>-MP4`K9R5XjC}QT`$y%fQK#Vm5RL zR=5dNzZuh*Joqxkpcsx#ob>B5dNDkDEMt=s3(C1XCQ_n4pYK7WXEcgguIEH_EZh@0 zdLkt7J>g?wU|5UF>E^z2Z`Mz+Y zC$Af6{MvZcA!>(Z)9est%F1APT%15#D=j-r^TK6~Pzy35k@gAHPX+4C@rr^Zo3CJt|03KKv>p)3v@4V7Ro#YDtbmJOwq$@uBbBx#yj<5 z_Kc__T5;hxUZ>107)9<@E~d^MBMbuNLXxf2jM2skZ%%0vnh{4~So-4i3^Igy18?L3 zcOn|z%v(^B67vXerCQ@{%(OowOd0caFw^<2o_D=#n9@F^X57p3lh9r%&z1KvHB*rb z`4Y#wr!;SBCfzxk>~czTip$P4o_OO<);$=m$W8?^=LnyUV30j9(ts3)(UbJVIpb{w zfjVU04qs@Lc=jZK4#oQHm(c;@<=HOR{aU}+9f|hj%{D9vp&g}-J3*owGeI{-e>Zr1 zoTV5#nbE-`Q$}-%j75SGKBHUT+l#qtbb@$X#t6g%Ami6(TqlL-8xav7U~vFv;f$Rk z9%3M751(=H5rN@^$!EjDm?(51WxpX~4V{d<5Edw2Mh~#XXUv$}LVPF|%GgwrNXdFK zB%t(2RHT|X8s#VB4ck#|ncXwY5-p_X*Rj!@(S$QLaX2^@8jl>K8IlNN{*}kK z4|w@;zuwr-1QZfEor;!ooC*DCS~uI?A=#)C+*#f_AR&U74PhOdtcJK z^``ylAKh|Rrkp;>=}S2Sk~46Zb0V74_0H+HPtUCRg|}|EtV`FgOgDGTd6T}rbnDvN zU2D^wT_4&m+pg}EIycTgEp_fouk1;;tayLbd#mOmQcM55S8CaI$6{!$x?{67Rh+Xe zdAah+3(oV-lxwZzT06fg=~}z!+I7oSb-{MtmaN~Da&4Aeo9{PpuBIO^>5zj5Viij9 zmDOvx-)-3L)O}Pw(6imD`$MY%9|f}(%{6S++6xO7N17bw8OcXR@TEtYdP|`mpc`E+ z_f78Cw1+sqK3+Q*9uEuXBfgMtVm$KtMA-N23)_8c99bFP3Y>gnI56}XO<nz&FD^qPYccP$eW}y6~W_Kl0n1_GD?jFZMw{V4@lf=g>i9P$lQw-jl3>F96*0uljR5I2z^`L)$!JM^;5cpR_Nw! ze}^7p=gZ_CGo|Mp34KJ9(D6>jD_onAA@B)8|k@r#7v5C>kWazyaR_9cI$5s!gU%{6WmPvYw0;4PkQ6 zSg;a~P6$Z;vR>!~DPc7kBrX8J#bP~fW3Y8BIx!y0cFcHt5&P|k4!>kh5Q>1>1WXUlcLs$@^g5SiL zid@g-)HFf4XhFGY>@)!>tR`cV=eHmgHlu@=umw~_LT}(e^e`@zu?4f4Ly&lN!OWHO zc#fjpf8c^w_SBM2+fsGMWUQ&U zW3kmcruW~e=UmNc->NG+KHPhG?+5!+U7Mw@&GQH6L&>f$C07k38@HXa-S%vne;H9X zRh+ZW9Fbg|a}APf{gRn;S6}El-<7J^EmiDJ*S4lBo0klndynQGr*k`|x8FB&Wo}}B zGhMT>*<(p>VA0;4c2`aB1t;|0<+K(r@yGpd?@!kCUiDlxUOhTrKCesGY$nd>uKw|o z0YM&!UKH>v1N{S=xQ{kjw)u4*>+80)={{~Vz+SK3m$&NfF2ww)t z%Uag)gqL7fym{8~59bdEUXDlOL^y(=L%aOO>>!Z67*k+F>j?&d@Crx>uTt_dqLTB# zdNMUYUN6p4U*3yz9s**BwEm%9Y_Zf3l0v6Q*|$mdZRyGybS?C&vhAAtoUY9AkX}`- z^r@Oh^(imwQ#B9n5#s#Hx_)lnGO$ATk*02-S@%)10rsI`J-n2Mgea^>^)XhL(4Eoo z+T)rjodt_4UQcZZCP&L3EPak^!IG@JjkmvxFnk&B;GOSkc^6;K-W9OB-!;O=3U;7H zyh;|XimwKf^nyw1AF|k?927l#O+tUAR_U)`Vrs5SE(Gg}`)ihosSCm^6U$L4)nn^f zk8LQy#59bFVGPkbug5mV+qNrAj6lnTs7ThbGF?p9=#xY7=flG%qP~v9U`OC+JmkM2 zfuwL9n|!BEU{?3Vgo*GbA0PFxfNh;vdW8MHty_IPvIZ;%GR0(zlL+ z0#fcnwPkeAMtRmj;|`L31R#7NVKc3ZLZc^PiJN+kM#g)@6LD=|IBun-nmqZ(4UD>9 z9%C5`V`$IP%&mWF?SV)}zq+?~jAXN8;jySNnb8fOoX8kf_w|!sAM+E2C=15X$XFyM zV$9J}E@Oy{L6StWr7M}Vo(}303SY({B(?J2u6F4Cf zE6FDFLcE1UL=A>AAik|)`ssA_c(N)wJ-DRr$}-OeVxBK1>$@SsX1qV|s4-Bz{4xm&B3 zR&vdO+x1x2fs6Ll-Z2=O;bHJqgThnm@2syUe%ws_cxjhL<*I9?inTeex~>FMJv}>; z^sZjC_dS%UdaqjNTjrme-=3`b(xQE!h^c;Uz&Fszebi~$R;BxBb=|gd-N)qy*#Ey} zs?_0#sR}$B2|F0mADRGS`so_UT$fbGX7inkSlrSt?5SXV7ZyL^N&Yt3#EeEiQ^3wX) zyjU3ZJZT0b5DInwW2isAq_!{83Xd%5uE*r1<*_BTKdhu@KQE^3cf>okkA_3zKI{!a zz~WQo8$KD^K*?ZI)(nW(vVeqQm3<^NFk>Bl4O@!I9HcxyFu27=U);TuWCqY;z&eho z6V5utzvuyP3+O!`q28*YgG8OgT%jyy2N1RrinqhQ4sg*X+$AxQj5 z03$Ewex7XVN-xu~EYd?{C1J$N=t;vMV_}gfc{iq;+upB#uRhh>CpGt_SM*@TTC)i}xTY2h)|#eE zwu`ppiuLolZ`;3RPxbGS`u8N)?@hMtyXoD3$H{qHmg@6!GAOwRlLuc;x(65BuY8fL z3`yRhQdxP17SB(bm;3eDzI?~}w)I@!*Bw8)AK>cS?{XTH_?EYkN}TN2IKSrGo4>U= zwQ0Y!X@7F#f$Im8&pvluOtv3Nc@N+49>&fBh5zxVB{?Z+Fm`1p9sfNu91?mOCnhQWaL zyG>ere7E1SYm?!7nx0)x8NSzRM5ymQWgwSL6#aW!ErTm{-#2*&+jZY>H;_GG+1+dS z{!=}>R~fFi8$sdvDg#`u_Y&#rYb<*jb=Nn0_tfbAsKx+$fq;SJ(@enddk{T69!aqU zxc1jnUaldz_mkqOyjaJ9nPRGJ96P6~Od1+12KX9tS{@0F!u^1@wgPRLkY))dqvC^nZA88-3Cl=-Re>9q%AIwDl#pWDU#irB@lo7Fe6DyT( zTfz>Zmi=8L?RgYw;Y?)r#>+xJ%&b;rl4dd-0UO4=Z{XQIzGI`&C{O$hER=CmvSW=u zNrE5f_C>}Z+YgP0qZ1<3>=YuZWdM?qjp`S950iXkqi<+p>}XiVmLi zq0_-r(3k1OXc+(Ds6yiWduFL%pv+ln)wTJ`sjq7*q$N zkYzy*o3{re`Z^&53bR5p-9b_o^s5Bpt%tHQEniXP$m%F;PbY3*rO7y1kRU2C$m;L1 zCS%Jh%g|>$+Av}SkJJ`KbdDS($>>I*%2kHYc?5q6 zLH%Vi!Z5JTA^Zx(E??Gs8GE)!v?nZP%wa`q(sXPh2Gt)iqX`OUSjw^kLm7nc`0}>BaB5DU+KG$T0(x_4){u`0p%|OZh3aC&qSW>lNzJ0#>njz`gwP+qB zl_>k$_LR9zGPfn$pIS7pzg=B3ZHLO0!#B6@W?BFI?sS!Rdhk|d?aaDgdF7lYUDuSV z>yzsG7V6fXGpAkE7aZpuGp8@Tb@8pV$D8uBNuIXZ6Up9zdpfPJ;x9RExf@WWs^z|e ztLvCO^~2hpbH;R4{e`jfW8a9L(`DRMX^$`E>5x1fbGi@hm+h&}4N~WZo1Tq#Ok7>> z=Wfoi;eI3Mt()1rP~ADVPO9#`x<;ygYTB;Cs(|DUB)gwpaDQpqd@GB-s`{?>&u_ol zea&>;o_zkrr1z+xl?9x zS(iAI#rk;wc}SM^KM)Drexw`d+3D1Myru)@cb$fvUQ>bWLfs@f_C&Iazb}r9%>w!} zysMki<#FCT-g-{+D*6|8xAS;!DV;23)-I`&g^h^pF1aD5a@&#|mK*V)wu+vFfwvsf z^45f&R>@av3J(SzcpNoSA}M+jhGQDuzKoU$_<%O>Yk0Nyl@ ztW;cbZ@z-*Cpsv-(zQC?4nI$6zq}d<(c+m!Zg3fQ{&Qfhry#C3F$nKQhQ+Eau5j5*G6^!v$!`L8Y84AsC!M zP_sqsOl8tRq#v3Ljv?+2vf;Qw>{K{Bo(mxo#2AZ=LM9+@Bl$XenGXFpw%zhTPfH%? z$uDv~=CS#5*&^gK%#^XSRU^=rJa)D$Sr7syZ9WD)4`SEC2KdFD!=$z!pm?&D7*m8g z95-!eJC6ZGHUfDahc5OT2rushnourmqz5AZ9auUi^A||E{uxat0`)-QUy;F5X(6n| z^gXELlvu9)SYaTSX+F5z8KJVLF<87A5X@-AxaA;&HmG21qkWSct+`5IxyYr>u!p~< z2w4g5QQm%ojBR9WCj*lvHVtJg3!I%X9%iss##G12)4=eUaF#-UogDNdI5s1z0(1}r zE)WYnM8ZSs4^}M#)?J*iiHujtc%KXg$gHwb%)fy{reb-=%9vzq2gntXAy=7<&4O~~ zGi7WDgtMPaDkNhtl4vpP#f(A#R?u@w0c)TjI#B>x8NyRA<8w)X_V}Zi*{d{ ze6sc>qqA(ECXLwk^Y)Z$hveFku4zs;b^suj?<4_Ic?GsP%PUhZpXBn*dKXwdm;}lvzKsf5GFQdrtCn<9*9hdtv|i z{mIsy*Fx70C7bswdiMV9UoSa{&;yb9)^}<)4H~uIH#U&;r*%L};vXjXDS(@*faf*_ z&65gvNGFKb6)oWPNVDiExoxHj9W)#SIr(5y75hVA0;UG2i7hB%V^an&Z~&>|2N1@n z0GKIk8$o>1fQ>i+lj0{I#+>cZ6Eh?+Q7M{$s0vM%Lvu_O$MBXbRt3yK5M@FBt);>8 zt7};PZQ1S&@@HVJ`*h6yP&emMCK)Cv(Z% zmHosbC#Z|^_FOIIiEhkF-m@a7GA43zbiI*z@_U8fXniPgRejw_ZNucNjR6sL5Z4+GK5d1Z6Ng07YMy_HRniZOV*bY zrKx<^TtXD~QC1XI75~JlFee%4Wi@4S-s1$0ixS!^0DDXjMw=h(RB3@YNhPFOmdnCu z4`rD=jFBrX56hR>ERR@_JOM9ts6#=wtx_84DIm+48X9QItaXO)^N(Cj1r|KSu^h)P#RVMjsjNWc)T6 zIm}ir3(+i;Yu2j@MSNK|6S73|GvV75`wST&7#XYFVC0u4XhU@fG~|RUWV}VjzaZl) zWc(2s|DKFXFhmkDD5FPx%Lva)?W*qg<@cPn?F9d6u!V*0hlToSLV8Ou3XtBG`N;gs z*R9Dz;pD+%$z!pkXJXO(MiJ!OxM=-SSiX|rUd!z2+17J!P47-u zRHZ6fq>7f={<*^o6@Am&GtLTC@OIPHxn$ugRspJ2bxvDBJZ1JuX75bx?DLD}RiNyx zowna9Ynt13v#jT8zbeh!@WaY>i1Qjc<}|ZnvTju=VO~J;1m;f9*A~k1R{R}gdA*S3 z)i=!~7V5g@CZ)Rm`887A)}+~ctF~e0_&2+f{!LQtrukv1c3|3BB;eb=;NCH9R<>8G zHqXC!ZFqj@dUMjVf6;tE75uGE_HDbiCb@d>+UrTro<;Ls)dvThuCGaMeC9elo?A2@ zTn+(O0*tDHKwodVW5ccX&bjK}d?nq~O%Sl$LLxy6M1t!$M{lyMM-~ZwTs_b>*rxl# z%^fhm-)0yL6p93CNkQD}9*05lkB!m5rD%)0=z%Ncs_20$X(Sgta8(bZ!Z(2{Kn~l7sxS-Nho%oOw&+o}4+$Z&CwIUd zm}xy?njRz6Q}GGgNGyRr5{8jxg?jFtQ2Do#7ofKd`RMJDrP9BKb)ax9)}l}xX;W;u z_ee+1uGkXB9o##XVeN5lSo@<2pjWIspGMz6q@;87?qaXsOBrEqZf(-ACtNpWvWoxG53}aM;B5;NUU>;pIqA$ zf=xIcLXYL_A%5&2c5IzO`5~d0cvcxrj}2W(HKBy&t8m((n#8)pxVC}%`6)Uy*QKP$ z6z9tFYO0hF+nOj_Rv#s6P8H$W^M&RR$(Bu(y;%m4>^EdvP9$5rnT;?{Wih-r{-;(2 zeGsgGnDaD_DX226>@^|S&W8Myj#2w)2Rni*S8`!~l6`GuUlo_eP^!u&i4C(dRyA9c zy}a$<$82q9H^9}{@lF-Smo6HiC??DmE~35SW?z;Pm7=fX{-9EKws@rq}ZePJI7b;Hn*ljVy48Zn0Aod-%* zJeDUgicxy=oHA zLwK!(Q|M>viU{N{DKfQ+Cy3_=V9R=RG9!!7Rd_AR;$gOc_zhGWHsV>1aQD!OXat8Z z+4_PlE#~1Ze2c;uf~P`}SVqsz-LQ3rsydikVz3p5-zm$63G>Ta_10YNVOJjcqyCY% zZ20GtfHrzkpiMyGQ~EOFfM}4V#g-FHYtX5*541L@c|;N{DMLB?mX{N?)P;g2dvkon zvIRxSaKBDViVwlC3zigH=TFa%TsI}3efj!S@)bVm2``$D6|E>XFPgUi+m@B5?5&c$ z702E3ES-|Qleu(B_AX{wCD~Uk*^E`zY3q`QtE^6Ys?s&J>DoFR`}bL=^^4|uNUtGo z-Ee-xH#W_BCAS|i91Ds(^#DkptLkPp{jjp*oaR<#-Gx`rzdE~dp|bm&R$Zu&hQOS# z(A0O+wT3hTI`3QQD2o5Q{~N1j%{MAKKW$!_YF;fhuTC|umzvjqG`P_J%149quO|;3 zP91tlI`q=bLofe+`L#i*|CQviNV0kTLi5OZ$2tAE6Q4A;oZGde)mMHAI+Kl;$}X0@ z>x7KkTYq8l{Ny)&X7cFjr)=RHme06T)Lc>!tTF7@VIW9V8Prv`xdv9G87CJWG z^lnuI}($<-3>zCU4=j)`lo$1=fRBg9Z+dY?%YBzrhC5auW`d+EN_iB|?-@oA9 zaBH~}P7}2_?vz<+C1bT%|N3r#+wiL9N6;cA9Z4>Jp;Z;kFd^_6=&t>5X>llfFR zT)t~|?G5O^>#Zhx?|=iZKkhV;8L;f@HU05ZJ^NOfe$Z}&%MVr=;PQiB9og3y_H8M= zo`+x?TU5~$@wn@GaK{W+>WoDGU0S#cMk_QT98oTMc(!L=N*tckUVNCYB(GE9gFYWV zBAA`UN6)nGPizi0iTEVDZ?+0^OUm9O*?X><7wsFEETUdA*U!AXV5W@^b2r{;XT`Mj z5%*jCKi-_1N;N0by_2~m1eZ|WE8W0{iyBz?AJm5bb6FeyDYaooJ{wVZZaHa3yt;)mDNz)93h(j81Rl8YW?88W4b5mI1JsXI6jTk+0&%hZ+S z0=wr)6-4BMVR@q-%BM46Z2afERU5CPyq4Tu92#9lmM$Y9)Ia?)X>+G%@&Q%c970wg4+%*@roZm9*Nj0yNn%7-D zCN*zaaBs~{ARUsUW6pfFbHTCcAEM9E#chx7bHeFI-hr4;@(sbz~XyN7kgtZ;p4%8aTcXlxb+WqaM0?$l#)d<>6u$JERP++{pST z2uGU3V^Q2PdmJCxOTXm3hcj>}WVrtcF7eia$-4CN%8Gs~(zs>=44~cCB-xr~jW=u^ zS9X1P;PQd_wbv@=1B+d|Z#&(2qwiK-^QA2px6FA`{w+J4DcYtZ<&3`9)i=Khov;|Sf_7~0m-?ZYV2qKL+*6(Pc}r1f^56ZdXD|{xCo{X~1;esqlYj*+s@68i+ut zon0$TOKR$k*-b_CHEQh7(CWB<*O&+8a(#`umpr{M2LXQuzrwYo0s@vV!0V}N5iWlO zpu1A_VZ-HydHsC&>eK>ykYiVFa;}%`^)oLn+EDX%vSaHNM_yPb3#(xlcBwljkUg%(wY)Mxm?NtGiR#4hE_N?Z-V~QpfX+>t z$W1w=07+v7-Pw~sO0q2{Y4p3}?t{4Mna-8w)(T?08jehQSmf~;+n)U{s6rAohnFlO zv8ZQ3K5K*{?j=3y5N^)Go{fCD`Zg6j1D)?Ds$~0kSob#3g+5O8Bz+O zyzCZQFlS#m(SY~?;$uKmc%Wye9NG*ln!A_uPRokhtsSY>%~I>;YkkQ>&nH_q zFSNceJ$P;();X9kS9YiT+a&+C>qnFRZ43VArU#RrcFDd1Ufq4Ez+Ne^H~IXRlcVFw zz}|%bZUBMOnPexWvlS~-?K`CQ9oL7G!BDb&$3pv2kUb#TTX0#2t9lyeAuXJ%hHkG~ zp`L1d>gwxPhm$oM7wwyFHDP1;n{Ob_;7xn|ZD-ZA9Uq!Fk=k{TT@KTfs-Ron$aDdz5T!2K+g)5#MF1pC!8qEo%i@7cnd=5YrWCh7>TLaF^vv24$>E z!&)Gt3I9a_##gyN(XYLyw;HPMdUb~OyH<~3%iS)mVa;7LjCjm=PVuXZ)8 z$7E}}q59sCrpsWx_ljn(2Co;jn_(t2d+>4hMXkx;zFWZzJ8bnXL(kndqoL=XtJ+Y_ zm?6u=Fw$2a><-Jb=213lKKe4l{EK?0Xcjc6bX1Fchtw6LqIbqPTB&d*?AV%r1J^0C zxlk_?cxY&QjB?5zx`hgd5FQu9l^3I&uXutO?z|WtzVZoT;GQeqd!?qJtR?3A53)JO z;;S}Z|7=LWF+0fm>7FFtC@yRAL6sYK8{&$jcq67z^`3wEe(d}y78?yVTwpjXMA*%V zadUvKexgghHE6$h6J6v)T29&R1oFuqZDZGmV=#RgdCBNcghx-tjRAIK7lh_^lR@MAV;+!_cd zB6Q(zCtY>%3%Mjk+?e$z_Gi4EZ~sub`+r*gv-*Q%8e_$ z;yur!J@Z8^>?8q;UGn@S-L3^`KJ>K&sv+D(Y(@uw@J$S798id3U4iuy`tbuE2k^3P z86D&}>^@>T#fej-=nh5}PsqC{R`7x-Gqw=Rkx*#5@w_rs)+Uv;C09SUSa$HtuC(5o z(pO9R>KSXYYiGJ{^OBA;c<*x>!`9oXm-)uZmvK-nxpJrKJE+NdTdwG?9!%HmP(63$ zdD^bDUKP@H1FGltJWuOg*!!8gUlP#yA9a^lK^gLxrK0lhp3W#RIac}ps|C?Io614v41 zRu!})L=P>O+TiEU-LdUB~Mwkrl@0>kMuBZHTv z^I>{|T&_Sas};(*cj;X6<~Z8qv_`~+dcLAG@?m>KKG+YtJf^nRBOinxi=NW?$SIKU zWu-&NIaOeI9w689G9WKqW3DHtv6aYaLs2&IiF^=FDSF^qqFr%Ie70|+<|?S>@_KM7 z=BxgY4j8U>q`4>&d@7>`N;}VS98==aK|$)#BgjK%==jQfC1>1B@M!A3EOi6UV?LUC z72STrR~KK^c(z5)ojs)ICcRL`Xi-Xn1<@*~W#uTgD?Yh5UlZ$4zMv?doiPg4Gq4P* zoie>?nyi!Gd5fU>xa$D!T*Rh(Oqi5q?D17wwrs(j@bX2EDj%1B9!S2*k7+`PK7@J89yR})>#66wMoX%!jI|m zJ7hS>?SGN&f5V8^S*cKLT_dY;9F)&_Q5iBCYeus%u7ylO(qS zgHAL3q!0KW&hN49B)>;yR_iHlKa7m&R7jvDBtxp~+IyyyECX;V8A8aHAIPQqm}1h> zEu+JIkn;MCtqgyGJYFH=RWkk-1{9p%zzav`z}Ru_?GNa~LLc(Yjdo@QEP?Ken9;o! zp3E5D2#rpJ*01gR*?mdZ zmPP&6Tlg6I*`cKSrxx|WCB5HJ{%Lvr%;8HfUwnC?+&^th(}Iwgoa>zpZ*Q2f-LSRJ z#@?TNZ}RHS8yy?ct^QQ&R;hJsx~lfV$oY|Uz3-4X+Pe?XmC46p%kT9-&f5Uqlt{QH%u1mMCO11Bj+IOXE8ZI?l zY{(W4BvuR(i4{~RTB~ArJ$s*kr*F}`Cf(8f{?vO@zx@`}4W{>;OVGlV?qzP7d1+?f z?CVLdf6)%z+LfusUa7J7hmC#bt><*-p3nYhz|7HE_pD*|;GALhRIK7Q}{f+BU~^1;Lwt`OakJ_C@oKTTU14LmSHFMey$p z>IXM*-``}}-J`p%Z{EF9cYUP+c3PNHF9`(8TdDmnc*=X-kpi_GS@%D^~ zejS32Ia=u#e1vfb@I>|_5%Nz!Q~>Q~ulMNAUhp6*#nb92Q;oqjDt2doq6Z7^>M>zRxNCrJ0RpQ8|Tlm#{pX`w0)OX}H>flP`SJ$~CGCYUHp z3waa2%9uCH3+=5!FFf#%6Mq>7E0jAU2k`#ZF~OGpwnn^7q_);x;ka4l`OS0{R@`1Gse&L zub4H}cQ`WdCR{E})7`dSP5Bb9;q>6eq26_ATeVqF~_q*quJ3sS!-5j33{hyDW`TTy4`z!j;9&6rn zr_01~*Eo?AO;PTI`Zk>~v2XJUGkaT3SlHWo!ph#Z6E^m?pRlvHYI5QWvjjhuvQFtS?x$vd=MB5Gf+h+AEpI6Z03h!{m z+h_wlCo0X{eol0}#EDKV--#-b7hNxzPWVOlOWcX-aE0hO+r@EG9qM8l>s50aU4&P^ zoe$6`o49Gyv?<{In8J?)Oo~$s#l!K)csO8I?4!|0cp{E>Y~obpG-7>+!*MAxDi6h? z(eP+ICM5zLT_JgtWsrqFp`$ArJ{OKogg`+39uLcM=rq1S(8eMa`>B{T9*U2yuAKUH z=Sz5B z!O2NE9~Od_!uVrYT39q{uv=EY5F^ptQXJ#zDpab2;R$&PbhH$s(U2@F7C9VO_)#g0 zr7Q*~&yC84ub3zYEQ(bQM^7mpwQ|8zk!V;koswymVGZ1q2BNXiP;}r-Y&<*=Ie8`| zi38zNa^I*N=@X^Mxv#dW+1#dJfJS3J{feveH1FQIC-G3l7gYh zNRXoXC#Mx>Fc?9v2ZM>GQp>Yni*7_6=_UIXUZXOTBQjq4|yeSNrEH zmKu6yhhIMUsf8o|$=K%<(&5QAe?=t@2;xNw-;Mm|K3`D*CS5X3T5!OjUul=1ZDy=-cH3d#19`GtYBk)r{#y z({xqAr-qz2o#*l#!JlXzVGE6I5BbG8L~Bh5Ux;98-?P&)kVmmaT50b~w699ksm68+ zp(qxmIGxY+@pmu*5*7Ky+kbYa+#iVz7}h^4LTW=B6XhV$kM-A&HP=5H8&9wjLz9XH z5sH^quskit!{fn-s91Jv+z1X6okD=5*ro7k>{J1(;+vFW=OQvbu&*7CiD8AOjK^5U zCq<=9j7V%-pME+ljmuIuN)Ff*H`eL1VTtl7*0YfbQE`F;g@6=7ibp4jlH8La39Uz= zGWEpCm^hti8dCX?P_jpk1l&>$N>M7QV~Y5gJsLZG8f2}J39uW+LEYgCqv1)m@hUv{ z=!o{I*yVUAJ|(OC*RL|a)${@ui4dA%3M(FYG&mlbh@2w6l;?U9?IuzkoJ4nN4pfTS zZbEwbT{suH6>FvIu^jKaeDu=MWxhYd_b(hs^Zi-=v73D5<%5?Frt9{6z(4*sH_NM+ z%Y{t2uw34kDewD)D@V3+-{lLJF3fHD?XTZjzb&_}JJ;GdUzZj(=i0X3>fM^_>ivP| zEziP8rfb*Y(7J(^7fGvr@-ZR9`E< zTE6Vtn(=L2>`D8!W_|l^`f9Fuu6okzb}#!L&G;U@+r;@=ezjsj4erS_4&U6gmHT1S zP`TyZiot;)x8=uf8$MRS8)74jef=Il4-hsO#B1E^<{9&^!^n~VF*hs)7=wkbGc(Z7 zKc(1O0TyAI7g!oG+@eRcCe5#LqV079BTSl)abSfy1S-YaO0h7o!bfH-Ny}JeQ6cyg z?Pqsj<_Lz`RF_EQV6|~3E$A)lDhA}5vAk%Rc7Xvg-vk&bg%lW&=oUR9pEQHXdQY2x zhs%L-Dzch^TcR%Jr1ggX@3HrZYZpCYHMRG3!*-SL`vxr``m|%l z@_=5ec~Gwv_f?=)V>v=dc>38P6~LVaXqgbey22L{zC1J`2`&Iw5hOSwUc&UGW6- zr6N&+Hu53;NS8kX_8U?_uE5D(!v1`byH(wh3a6e+SM|+0SM0oN9r5@>mkurSPh|Kf za@9?_hPD+O=N&ZN;bRt(0+% zEjfQruD&Hzm#*u+B@ConHs;!UZ~2>3_46axO;#-C=E~a+dv)1um#5A>%da%JU3GeU z_Gb9KXiToUmD)3uZ;x}W_Fzk!>rGuqjiu{0W%+xYwPHh(dot~w-x{nOY~bE)Xxh8c^6rlAy*-xqdTj9j*gjY>T*v*S z&NX7U{G_#U#ANv|CL8>twEJieadNgBonv5H-;FFdCWBvN53qns+OC@!Oa*5>Z5G|& zq2Q;Y_qti*riibAjKGI?75@9L+gTd$FyyXb>1xF~@Y?#%>|Xqy&9_AcyB zf!4%Em8l3JVKOG($8m-6*wh4$qCz|-#Lt9@PZlxZnAj)nMI98&(=otToPz^-R&0-k zPKKj$qKt8CoE_t#$OM7fP;?TWgk#`jWMV)*lQ8#X8?AExHJ{sEr>CJC2sE`fmcA-vqu;pKsykdiU-@jcHro-<3|P$kIeMPFT{5X zax@&C6gF+7d??gScwS|J2^-6%RoBIed^!|Aqgc0ZJseT2n>KD7Ct2EfcswReE0)p8 zDaE#F^G=G}%;KaGYPmfc8IQ!}fL+}oY>{yY07y(Q4u&Qw){t}>$9T2S!_=R$^c3ZD zF&<1CfmVm=0-rW)JRcJ~#Af2w`w$zAN={aq0NKYQ6HKDDU&6kJy(K+v8eabGyDqs9fP{TB=qya!r9-b#19HfMxgB+_u@8 z5Mf(a1qz$D+AHPce*bU4$LZAc zqe1^(H}{^~wRflGy{g8&TP*KwvBAIU@IZ13#;)5LyIvEtmSChLJ%#imFKKT@b-A%OgMa0{IiG*o*P8LQF8jJNzOMPYtPk*{7Ep_UllspB zCv^;*)Ze>9llq&3&4aDnyRELhI1+a^?sZ$lYhj&+oT5nE;XrX=eU1T zQ)tnot}ZdDL9#G`+14_J%HBVFCn=PU4y%MfI4Wp!bL|bIkm64$|F_{3&tArh_1UZP z;(w&cE1gGuRsm%z0vPB0Dm>S?f>vrQ&ruPqKj4)*rxi^J@If5Z2|%!Cj5gS!Cusny z(`M*U9y`V`69ji)ToNEmY+MlY8WbGF84y%Y<%m2T7P`hG(P%_g5l7%OBmczZCyqdD zX;-r=4thtRkhGG%QL!;p7O<)(LObwFa9jq^3SFQ#PNC7zv`oz=ETEnNy+M@}jc;f= zvA*;v(3s}u$S=3SxyXHlgIL+svbl4qb4$Mdg_?|S$E@w9lfUe~V*&@p|;3R$6o z#MVoXi4+ed6bBkj#A6dx>mizd-*n9cCE0;Lf?Nv2TldV=gfJQnhbE>b1prQ<#RybA z(Po=yyOSUMMqD*QzFLR?D-O_r**4bmJa z8r}*LQLvh5$rBq&4{>q24C*C62j{Q3zrje?HeQ>)I{nJmQeVi__RaFSvdWu{EpI&X z>Lai1o;{Q+Z=6g1qP#~Jrf$#rcCa0%Gvn*b`U1I@p7}GGmhD&g5BZ7}FIQgmw+xeh zbFg)=&9q7vq_4^alxif zU38tUR@L zg%9`*D;BPPBiQg~Cb~_Dq?bT&6%*BE-;)?P3>V_nr?j$;fju*mU|L?cY2D1$9SwZZ zQ)(@WhSNM$ZS-fqZUHma2dEUJ)mHF~HEGSSU|T^sFFNA7id}Rj&7v!TNkmLutDy$6 zXY7VtZbL3l(p)mV15>9I?B#EGHQF=IhpT(ib`}fZib-eUod_sonNb)_C_7KvDTty z;xp;iQm_{I*Qlj;?G}J8?SMs3(w=l^vkYVTP_tQ`5t8|e>1Fpr=CCH9>YnjDV4YSc zJz~vkR6TIsA5}W{^MLjAS4GH9*+Ml;LBMoHP*6$F%I; zGD+XU>-awDEQJ#&lCF^Rd*ob&lV}|(QC&#nCh0sbQuo}m^lv_e+UC3I9)A?=R}I5< z32#|oPNx|SxeI+H5i<(-3pj`cDCN*#OYxwdj)5h`0z9i&0EVRBMlP5a01Qb<^4cNn zjEsgAOEf&8?y55gmI%d2beLwFtn^#tqrFt|h9|@V^?+BEsl7o_RvJ<%MP67NQ7st4 zc2W-Jb1}di1jbf0FRJCy_Tkg0!ExkPg>w#o>q$8*vS!J&*U%)>9~Fw%N>dYL9HqfI zZEO?0CAa)FQvY`lmpvF=?E5~L`ds#0@}z5aEIJnw?=-z1fB#ha_?OcDFK3-!$pNVI zm-uC8d&b$G?%0)e?p}dNZ(XjTF;`Q2+hMJ7&${pOTusBZ=dV8h$`@zdxvHj=GgH+y z>x8vWb={o#`cTR`Z=HYPEoVm9w6t#Xl7Gvr>!!bU&T{=&%0F*?vvH}ZZ^_?3>w>XR zZR2uHZ>FYq;bgj|cd6!)Sr4d`S6z-=ihTR*+<~RC_K$eqmBuXJgm7QY<(W$}b7E>_ z?#tQoo*VMQ*5xh3nJvRh10(O7elhUmM|I6_l)YM(dSSkOscztkE!Wtx+}M+8?3o`~ zYTSIq`OB)toWFT4x#aI%FlGFkvVPL~%w1UWcg+uG{C!z}SFWOZxnlhuC^H?=yR`dR+n3aZAO5l z!?r&F(7A?VjReEe36p4^u|9y!Tas4D>#V=V#$XVJhz=v@1Joi~uGcLyHf)N<`e72& zQwjhk4X_br0QBbp0JJ~c1~}tLI`RO}X@GLP0m^v+0L#_{fG$I>azifP8UWBOR?J|_ zlJ=s7C4gW7%9S*vs1&^#D0n-6NCh>I+-Ml3YPGM7u2`j|hEcXVX&0-Yz{7@#I_R*k zkY235QKO}w;UBIalTLz#0HfOr7OP2KqXGbP4<^Kgx>SWl%& zFBqtoGCwi&+6;i0R0XP~pqw}9(Rz2LY@)052qgHJzhYH!q$8!G|Ql4V*7)`E3w)Y6OAto4NKsoFan*nlLehpqGB<)2eJlt5I znDRq_+@AaM6!_XqMbd}kK!w=KK<@g)|8i?kp$O!X@R#XVNxDSPQBSCQ{?V~5Rh87A zQHDZy3D2DgL&Xhauo%>$q_a^Y89-;UP>4y;4)xjqWrpD*%dXhQVSyA$_)B4B7$6Z` zF6grp?%_PJ+$AI`H2B#oJj0;*i_-6+$C-Jz2Co!1L(h!DeefANcO5=Mbj|?qE1)2K zhn#;-&hL|R4bBniU%*W??h8Sg&Q$uU)JKqECXN8q6M`K=l?+cZ{jD@dpRdBnqiyL& z6u`>)_wXg!3_!ckNL;!wKyB$Y%5$BZ*U9+nVb=XiPj#-o zX}La-sSn)P@Pq!h`WGt}hnKnsm+JS_`s)Tzl4;wr zFtWIFscraG=dHHRIp<&glK}lgrXtjTz_cdpe`K-qowmiMch02!N3+hSwF&gFuz&t5 zOO=m)7V5_hwtKZWRN}G@yKR!cebY~ubYLDsOe}I2Gdy5oEGc0AnC3eZ0Dx)z39U5} z1{@ER(|r^&EDQ$0IfLUdaS9whsPMty3sWI>zYSlGU{H*W27{77IZ3Rj@*2iNNW?7_ zcnD)SjFJ3+BK{+sw>WlGVM|zYQz$t#_%9E^d5ODiHrt$coPOK-PkQ~f$3E%pvDJNY z!n6}^+~l>@f6@%+PP4_jsvrp0 zPZX#~UyUuFBj>I~Po+OvFijEu3x2YfS;N;+8M7bx{ zD7RKa`sg`_n)?C0g0mXy6(u~s&is~oYB8Fa2IHt2+4*>;Luhgf|N65{n2)q79(-H_atzYo=v z*swngD@+poO5w?v1R*Y6Kw_d?Cewum5NK9Tcsri&81E2+9S1rNcN{xT7mV4BsbK7^ zLRXrM62OG5e@w;|Bhz^XIg!>*R^AiwBT8*iQh5qik-|Z7kR& zQWXL-i+o}>iX%3unrYfbnawKk`T~Ge!K)9_V>Xvy1i*Sg`U|A`T>_#Xz#+qnIJbA0RIHx6ELr;dMM?Z&si=Gu;{JFY!?_0g1jKDHQH{Nnqz zbnlU+s-qVVedO}aM(2jIE+I8YMjNh6t}C5$n^K?qz}bnbqV86xgpK*r{Uh&0-uYts z>Er3%&o5Oyd-2ds1BIN^b=h&rk@oG$Iv+3faix3i`MHCsc-q&Sb@ts_ANYRXn|-_H4Yg8)dBmq^WXp4n_pYl zwK%oZIg;M6?;V)he=h5L=2lfpYDcE3`$htQsyDOiX!diT&kj7B^H*Qne|3L)9pt_J zx$aF1y_xREm%B$Y-6Mb2z5ms+IY+8GS6Bar<5fqhJ#{`6nU@wi7HSvGX<_#Vb&o>n z-E+VM1X@waieEqX+}x9?XVP_DFlxA!uk!Hk9KPMiwQc^?Lw(l8**r7`>B^2&H1(B* zZHwy`%hJC6S?htEr*>{#+S8G?c6_=GX)`Y2-luMO((uT{x8LolKfqbudvti+0k7qK z&UV14f#P}#dtw>h6I2R_!d8+;JVa^5INqcUsY#a z$EvoMTy67bmwZdWFzJ}#rkoD63i-u`!gl(Al#RGX>?`EdeAoilA*KXCLrfE5{94Mw z8!^qVI|?OFw-~hhrc&Dd8-kX4#vFku8_bSN)<$dvz!-c$OV(?&Fk{SPBr_;_O1GzU zTW+*zaEIAySj2WxO|DV94%BY%18TQnRqX&?il1Mnc4Fs^E^Q7$X<-Az4tz%ax*wu` zJ%6wD>owHR4k4mN?4$Y_tRG;(HYODJ0cL(4o-=6YfK3Xor75Mh5c>cD@f?!))q3GEP%(fE3Yb^yX`L|5Bz#x5t))E+`Z-dzRvs<>g&@vdI z|E48xncfobGL-w&jLT4Nj}}vSi(BJ;+E>9+hI#^XZkskb^wS(7H{*KI^>?Y6N}*s< z-}W1~^E{{jB*69~iB1wvYsgdmS&ZVJi6JaZSK&Rlxj8CD*v0jPr;q&}Mjr_ZVclwh z=P`N;Drw4k32RSPGRu!e(Gp7(_T!lK`0+R4djq z;pk+-+1IB^M5O;jL|W)_61#~-In@}=ESDf0#3Xx=PWm5|*xlEs889&ENTQAbVlB*I zAcDfJ)F4AzG()*Z(Mrf|HprAUHDQilEd2yg<0G8~v_7}kD|fGSKEU$ygw zoN%FP(sUo%O(qNUV=8p9E3xru3APfr>IE~B^RcO@D4YzlOIz$#8ZLL?w->^rQ|yL> zIy$(U6~Ye~#U|7y`*AuDhM##}c;@p*gu_RNM+5{U+|MyuD2*;Wv5czS3a`O^vRh$d zcU8J>l8E=BGyf}fCL6v+%ok=Di%u^#f3Lk3`AnMgQkj@g+VAa)IKRl?BywT zY5CL?*#!IYr?31O2S*(3m!$L%Odqs3Kf##ouQWADRyHT6Ns}U}YKJEE$~7jSrUTN( zGlnyodb}LP56PU01!@&1rW}5eLWVdVdLi7vo`^q9pOV7kvT_iyEQ~1$*l$yYi-6_JLmnyz?X8(ZRzf( zU_WcCyUUquhfKHhXm`4IB-i+;9=E3u#oySnXvsBh)8lp&;+)@=7Y1{UkLYo`3vp!& zu3X~|J?4=@jA#B(uJN!QbCixOeC4udUBib8n$`lRD`$@;YaUDa zzu)|3b6VJlJ8#2(diakIFFyW$Z~D0}E{3 zX4(c<6N2f_ii2z4Yr4Z(+so3tfO~|!Pnc%+r~RE7zDuXKCF9wm(;Lq6Bly9P<@HZw z)<029PXN8PL88|NdM%*WvxmrS$nc%$fDPLhPA|s)^y`28^`(xZ`1zw*{z~*CO(C>B z>)fgJbIpcSGBuI*Z_7HjmkPU8?pyI=q`vd)D__WXnwVL3TZV7@fbYOy^38v9w}RuF zaC_WVp=I~?XPpBW2V8qxJKtRo$5-pwVC0pq-TK30ixvOwS^NM?^<(c_XZfu2@Q0v6 z^JJ~kf3@O4!M{T5*G_X!-hmbKllBqsQ$FsezUD(MmY?>U4z1^Y=5@mVv+A}(4VItn zv?Ao^4K{@Qyv0KP^(}`@Fb0Go0jn+;g!S;8PWsNeXk&Z{L8& zNw+CHO%5G`6*v8;m^1kQ-3eab221HMj;e~T12rv zd1T*F#f8225c_o<=|2-~CIlirp>i?e3yKwzy14Wn1-?X13k5nKRdpeIBnS8g{&DhE zI4c&D$#logncN?8_~&8&ydQF9A9CgJ|2fz67hK1mb6YV2CjUjpGFO%1s&ej%%e|L+ zbDr)0$7V5^u`1k+v$e~vhK#FWZs(G#{i6L->p7dreVZfq&ZMW^wBeIOru`<dCa#s*_|+qk>cLREldA<-paHySZL zF)=^ri;iuWnG0~V`$hbhS~cVcINiF3N#^YjW^D5*MEG!a5o#V z*Zt>SX6XxA#_rqwaaT53S(z{2d-?KZ=F9g|zt`)v1U&!yfA!7$ivfc8pXej`C=!`( zW+1alFa$$-i3#aVPLTLpHX(zz%uCJ6C*&l-P>kHGm{m?FXH^rbS@ndPBxJ-zf>Hd0 zV3e%#b4tQFp~0mpC{?qXRH=4>v}>LskgIl5RJy!w))#zA!nd6g)n}an_68dWiki#a zfk9j+D!u;cX_j+IXr)w^`1r;GnN@-%5U;=|IRV@;vY(I>@&(GS5S2sM5V}weLQK27 z&Om^k@^kc1>owXx$8ye~+wY@;01|K|_{Ny^kfQHzE`aP-i7?^8#Rn6HEQNwaDwt~N zN%EBe_<15fO-zFnA5x3*U{dOYN0|Yb$TC!z{D}Bi_BoOXJS6dI_!kw7d{L=+Md+|1 z1U*|+g;inM-E#!7OmemizSI)!%#=}tWgo!^WA5Z>?leT0^5kVgO8GF!sD7dTJRy-q zbq4laz1D=)9^La;Jo(R;Wl5(n36UEEqfPR_&Ax<(ymYT}4kW6FnjAfcn*3l%5-zM^3K&Cj7J&qmX{;KPj>IMDHp*>6^th)Ixm8Cr;%NLDQO;S4PH}@wm z;3Eeta|G`f1sR+aFh^5V-I?VcTbf2P$26s2$HN7=AUyR+*z}w0dv#b3XnFRtXvol% zTq}2T*l=(x!-jA^({c(B!HUp|`s_KHjTP$h)3YOEhiiHsl4(mrE>j=8lL39mehajZ zCwMrHc!2K{=#BP0$akYUrU!Dp64W?RO9ac4)NKS;yO@{J^ITeN4Cfu}`E3IB;JhE^ zEz3T5U&`Gl?-TD*w~71Y_r`8I1bgJ4jJ@uK*vSumn*)1GQZk;lWG?;obfoji97yoT zFO^_BUaj?*myA^=oR@1yWIFJw*qt#dx$G2a2k88mN^1?%^@=)!ts>^wLzzpK9LobM z(o3{6*=Cvi;AXZA?I%MQM*Kc^Fkyaz^$C{ftkdUW=~*_&xm^MJzB@QW2i>!*e?BOE zu)Y}t!O)D`IqmZYf&eC}Tz-yq+-nDmSsMsCxu65J5%cE!j3a?cRI>NLunvUE#}RQN^%R&UtU}kxZ2P zoIq5A(!iY4#fFMT0bQT3SMuLI1p$p?&@!Jt$WI*AhR7_Y4UF8C62eAVa0Zt*B<^r2~OjkcM@*eN+B#<%orlfu5Ve=Li^+ zX_f)T<#cmD9a1F)Mzyq0iDW?3jL*AVEX%M=s1lf&o%6Cm5N!aJ1_D#_UhhJpiKx8b zbbDDwRDl!&&S^HJzv;W{^WXQ;Eb2-~Lr-As+8` zocCCIl4X4~!(s`7w)uk&3}l!3!=WWzRI@$?&jAh@(o&fwC3oC$yA`GlK2QckIqEv6 zLM`QJcu|KqA9P482A&y5N~IYGFo>ITbqHDo!|x0RIj$JD4KB=qFhB)SnOKsetpoH@ zq`a(8)SQ3U#m=FLz#-Fst3nCduDJ%3ppBO^K_V)hb92xgkhPa(=Qwn>6t!%!aX{1! z%(+vz?1iE{z>I{01F(fH`Dbh!ivt+oWnMz))T~h7q2u;Ck0!cb2q! zO2TYi(mu&A`pRrwb+5R8c6WVPFt;pSj2D>KN_GqA#|HCi{z^VyeJb*LWQafi8vnY3 zf9-9)${96G?#T-a$H=&~I%Yj4SdT@zw-+0o|%Y04O=J;Pt{KZ7< z*m>dDdA{ocPhX50FMU-~xfb{&^dPixDq7P0*i!jP-h({eb}VW+9=A1oHvH*u%+?dN z^{kzHTwe3ZhKFcOtS#*RN0h@?@lWD-^wco1b`#e|v`a_;;_*@ni4s zjocr~1A7!v*Sn{do6eK*s`^+}zfje`b&0>_|^WE(p z-rT-xzPU`r>Dtep`@}>h@9xO z!8@k-X&>+B_^ALN2=erN)Oasm-LgFVq}aULyV4sgt`&-F*MG26+>wRn>h^v9^@;6S z{>~db{btno*54UR<0i}M#L7g>R416~HY_`)_Q%KjBCcP1e(Bk^M~~lFrz93TcIXa% z!o56%a9&xtvfj0!UJq@SMtUMg`RWT%mH%9ee+`EiqJJAR1PI8${Qf37SyFr3d~FS|G57iNtD-pO^~3D zh!-ro(_3UDz}F4NEa!GC=f2ycQ0;g7O2SeNkT9(hL&Miz-X7TQ+yHMQzHKEW~qKU0r(eYJX zo3}QHBa>T&TNGb$E^0U*FE*}ruXL{ketH`5aCPPC`i+go4J+UII$wJ$YP=nc88|0~+53K7s}7uMLB;(OTj^7oWb?Mf{i2 zvzqH9v27cuf!FU;N5_hY-*w1Q{=~TgczxNW8KtQ&`z)hn6mL>OL%vJ_B|J@`e641z zfa04hV_HhkDj+WuX~y~~q0BPYLkXQqz##M}P)R=p`5m(222Jf$4wT<0qjq~q_=plF z$VbZ*kOzmZ`xe*<;CEF!@Mh$2h*}4%zRbLrg}lz;cxT?}McXTUsT>Z*?{YY}%h0Vy z#5a%<+%QTs$pmfD0koI>g!nso)4tqEDoVJqWUUJ_sV8|7{N^|mWqE`(l1G@KenL!u zmlRSZ@>Wl%7$u}?27J^Lngz98E9!@?jlyQma=sg|1%*aX3Wnp<6pMC6>^7d0>I3wA z02$2z_MLgCa)Z&EawsPZc`%lc;~E+7U|=mA^f1G=BYBwL#{32o-d;E>Cwl_r$tO&P zWy!NYLxw;i;B`LKiyY24{Vd4rRH$zpnG@LZcha2G$M|RIq`gbK!3RFW($lah`%vXH zOE=ECyM_JT6Bs>mW87ey<`j!KgbI>qQ3(1d&ucR$>?cVymT|>*JSumVfQao1c zKGeRiT|V{)bt%@5+|K+T!j8X++>@jyC2l~Pco>pN33-tMH#~(7_lt7$X5^4lWE>(< zSF#>(kjnuQJIEQ;#~IGQAkt&}cyb(AA-I4nk*tSNOHPaAu!GBzWHk5yd?qaOAfr0# zVYEzM5*nPFxnXpSJ_UoE|H=@k+oGDuX9}>()DTuf9|}KKgaA8JgwtZ28c{k;E^2)L z3OF%gPC8yd#mtnXsAw7eVjg1wmn=9-80*Ir##V+LfJ}Mfs{)r*;xwDuG*GL;)NZ_h z8qK8Kpo_X-K~gMXohRi!O_o8QYLadNrWQ#!`L(ELY`|w-7N7N)rv{uh;?#~)iQ^+U zpUrW85FHQj!yG*r%=t0NG%?L#1^gqKmWMK?75>_ocBX^roR)!;zl%A>9DgXACYcjV zH*;)S<|dguribZePCk?~rkU_4|m;g#0%$)<2a=62pH}H&k^VKnfo@cl=eeD^Vc_d2=@X6ScD-z71RQT^O zeb3ZkP6n<=*Y|0+Xbu;K&9ENH83(KvC9fpsx5I_7YBgLI-t)g#G*_JWrv0`M_Fp7{%5c6y=$79CqRo94tr$Tfo&_zi7(fBA4F0 zj`ikN*mO`Iz-4@U(ey(TjGJD{9S9j@(m4528gJ#Mk)aPS(YOq5PQQekx|edp=H@0z zUOX5 z2$-H)WCf$=`n->J`tZ2|xwD=>Qw)p_(ct{@22xE@nE?)Y#3ejZicmO^Q?i%70V(aD z!dF}9#`Gb@ai4Ps*`~w^Ao(@xewSsyduBg?4!nF*?rF!IlXK33FCTop@R@Lt)3_A8 z+ME;|K*t9}{s4m1$8|P1&-t>rJ?91wz2E2L77}p}Bmi*tL3APATak2#WOGR8_G9;7 z^V~gGb4ZiO`yN-Z6#5!gtIkZTL zx+(C&CySmUdpvZFbf(q3$;Ow62!<8YH^TYVS2eiOGO$XwDo(%Mb)FTixnD+(yLS_K* zI;Z;pJ%lp=aeX_+bmB19Qhx6P@dnIjs7`8l8IFSJ@S5`D(oQd&{UEj?fmPg}LChe| zPtuH^#;$0(y$yZ%SW%ZaxM6u55a->=r)QAtE=zQ-OXtjgqTC8mxXO(VsWzm^bUPhV z+A&|Dh76H{X5v$AN~(BBZEh^nlDi3+wjW2mzSlPuSn#<{(jhhK5f*Cai+DanprSfC zbwmSp|KqO0*%dT9q&U&m+u0_Py*PNnp`|x;Ng^_WCR}>j|KDg+3&vD41QA8#e88G# zbJx)Qk`}fmdfyU!H;B7e>T(8OP}L-%8iA-{2A&roD#?imMA??JZ-+=PRGf@;;l)o{ zxkTB$c99zOGj^FsK`bPsL6yy@5@J5+L*p3k z$~8AHD%v}`QC){rr{>Ti1D6n?YwlThFkn|nJyp18AtuQAP_N}qSnx%i)Rhs$I(%Wx znV%cw{F4xi1mh7zw*bTS?L2y>RhNUrxAW#T7B|wNQOV|Tq z1+47g#KHF?FLDsw9nI`42uACibH9c#`hBAOt{X(BhUkzJsUd`<=2%z8(hQ*^nnUMu z;04{F6XlbBW+7CN5qm};VnHG%>qxkro;JZ=DXQkBfGB4`O;xaEqEKBrb_RYZsVGz~ zt1?kQCX0 zr-4X;F<`eyzDc~CU@HltotYMnm>&Q+2f-?USTc(2s-d$|OcgG8LhySsfXs!raUO5b zcwXX*$)Xl7x42YP;N>57Nfb+lS?7W&C+c979I(W=r$FdZkQT?Pu_TaIa6>3Lhm!Ls zLB0lb83Nlw2rARk%gjsORGkB60SbVkK@C~6ovG=%c|aBTmoTV5BKFLeC{6y8iuzbZ zi%`+B5!f8ut>|958ZWI|_wjYd_>$vGm*Un&zSO=^%9nI5U4n>;zWBq-?_XYP#xX9_ z=ECOPEfa6K7}Z^RtTV-Qw4kHcjh~f&TE5{Hs=FhNQQaU!?+m4UdCN{gOWaZyv)BcT zeWNwfzH8}QR>Z5CVpVNIRoiCeR?%+Nxn*74V%ty%mgZ%}o{}{6Ju#n(+>Bh@>fS!T zeS*I}!yDbt6f%>C1hl5B_zBL0``*r$ZW*HUvcq9 z29aSgnJPr43XvHm5uIUjPlt%>l|Vcn5id|7;u=ich=_Z33L8^B=Z~CHv+r0otpzf?eX2J6Ci+!%0sv^UBrF9h?!|a%(Nk9&SPfIV`f?qGcAaj zD#T0`hyb-JR71K~gqSVX0JBEKtVxTQ)gfm0%ZZX|zV-9nbVCyA^mi+8%Fch8Ffp=*H0w>7$|YS)Qie`mDCh zX5J&pYD@NdiR$Jj@(ycS{m0G zKD_e&m9W(3Gu*J*`g1Lcj z?1^;lnoloXgy{fR1xg>3epI$J95+=iUHA$v64+PlKRddv6^f2*yuMr1wI{17tce3$ z(}Sk<1;N@Lvz`~M=i?Q1@tUJRwd?}<4MCNemj|K$ma^5*N+?#+Ae1z0^z4*$eN|So z-uYSYr@egZV6^PqcJY_izp=)Krgny=;*CeY&_2?}8qW)j=eL|fXSMDkEs^1{vM_a!*_2^VYBOE;&J31j8orpIdi+7v^ zR+{c0R+{S9&!M5{+9>0@M)`&@{^o6=;dZ?J)Q0kLW6KwLkMcIz$WY`Yf8kBO>#b;` zBi`7&ry?3Bkn&k8KCwQq#!Br%shw{=E0mrE4OvqrLbdC`jY}K7{IQ|!+Aoj%&5_vf zgfKk858dJG-iTV>1U380XjvUz8D4wm=Og&heT#qX_V!hN;tp?l zBdU9IPpQ(MBT@J^ZzYefY}t5s!^4;NL=C-h2wFo(ckQ-dXowm51VdlEv^rkd0Aeca z2R$h)+RJ~nwqt@}EZz1xnP?1gYE5O4N@uawP323OK zMp{_xAtTu|`Tu|>9k2*Z zz76~lwx=Io(%&G6ZNu4`F*EVI&H>q|jrjdB63Tvmd_W6je@mc`za>p5Z?>X*M?a+h zt-bqt1@YyP?(uwruR$=pO*2+Z@y%sp`IMkm0=$r~fD)mYLV2_1x|M>x=emj7DNq9Z zj!6L}J5~ziD>UPI)J}cbxRTl>6;QIPR6xmY9;)5V*Npd3yCr4g-PCReuI*N!l0H-$ zB{VlqQBh6fjT2O~okaN)3P@dG(WV|0!9{NEZ_dJzc#$3+TjHs`h zSd?W19+4H7^`rtOnZIDlg*C%{4dD?KozHq8^hq)b#9=ZJ5SB5DQ}7M&d0D{l$tW3B z67r>?BK7A9nStXqH+2nA*JM%G{4-Hc{SM}Z)boIPUKaH{ME%vIAExjxvE(lqSsgg& z_26vJ&kS*d4lB&I(ticI&5?RJ@@^P}j!Qx|rH3sI6r_;2 z%V^yiRky`!>SHyXLQUu9)vXKL{ldA4-I_aa4J%(zR~^!{DXMOcSJuQTTZPKjO>N}D zRTcz=#9gioducr+l(sMBKTdSX47bU)i}{|B?J?eXEvg=k+d2@hw4koW_r9cS z9XVIeAA+d=xmQnEX;POq0DagCDASUahiWW{_%~0aS;p z7S^3YQ>?H?D6HXYk8R%L`$vTnW4nddmy}NmOiTLz*GIyd8j0tZkn|{g0g~X*io*Rn zl>8MW2L+Aiu7n{WRwHzc-=|Kw0Gm)rblQU*eyJpDO2#?Jw?A$!kdj< z%UwbF5tLj-$u*RqHJ-z({a>P71SS6-CBH(+C`!gqf>w8KJekZwCQt_nx~BO0H*!j0 z{aQOtDe}MeD9aTkU!PJb%wHEl@(oHP45+a^4_}=VW&Qwn6TU_5-T6uI_Ci=KAYJ^| zOv3L?@Mg@Rtp)qyr1b*dk;EJv6`o?~jD$kA(j3 oiHiSDbO=PpzMLV+lBHYw@XLR)Wd90vLr$8PFZ=_6-k8(>1A*9V-2eap literal 0 HcmV?d00001 diff --git a/test/e2e/stress-scale-tests/tests/__pycache__/mount_tests.cpython-312.pyc b/test/e2e/stress-scale-tests/tests/__pycache__/mount_tests.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a6c2b84c4687f9153e535e54f6b38b86fdd190c9 GIT binary patch literal 7627 zcmcIpTWl0pnyzbiRo~ohx9>I-7z0JzHaL(#!U6^|7zhqovl;L}dYe9FyG&o4s%nFK zx@V$Qq-mr`G1`^zD2&FtFYDPvT1&et@dz`cQKWt8mYuk#!fKH=FYJ>Ej5N!>?EjzY z8^jTL*^+(g)VciU_W$4a|EK@m@Aok9{3&!~`rmpO=D+Zx`iPaxlWEA@W(Y&r1T*2h z*$I|@a}ylAxdfjSCIprtJP{IN(ly~qx+mO8X+mNdjyc5;@gs(~G}mXmBXdHgrEVyd zG`UvlnP*k`5jGi9S=$v)=vvBp#KO1A+tTrvsa?=a%a+d`G*8ev+m%R9O=(8l5$mK} z<>N^QWNtGWgL8#>vJ)^n!hgh02=kmO+VZ({CS{?->_L>R#1k>oR3_7glEj*bIjhC> zNjnGT z=`3j?LDEb@cLdTx!lV^?w2^lD?tpyf9XGW0kggoI?WTQuNben zpC&423#nLAgL8Fc2XN~n@pQ_H=_$>yU9rS0WU}s|t9ok4oX&EQc-Es$nvuD5B9qjz zA{|ZTi~ww|Ez^TMAG4-ycV*@_moN zEyKv^i~%}m%b>G*atyYty5v@vuBn?ASk8NNOHN>mN2sP?M4sAKnzq{+!nAuW%}DC0 zn5Cyvr;J!!JKwls*d0%7R(}u0rc<_XO-~V9N@`XN&(#*tO3+#}v-HGWP@*L!opnjG zbMYu1C|#P-gzNKwZbhBrHC<4k(C69AP$C_VC5EQcNo`2KIvq2}5T0w?)FZ^u=QJY% zmU(R`cHN9TUp|C7GNcV_L#72gV@6DnqeZCE(mHd9szH>N49?EmEsmB)aVAwQ&-T*2 zz5ucYH>pOLG!Y&@WFC4$g`*FHV;elz9w?mJ@G!nm;Yhi)z3`(A7t`Lecwym6!Bdt( z>r!V)>RjBpCT)X??yco;=fjSka@UT>BHtO<@G-rSa&LcGQJ41@x9%E|%mwl0vecVAiEQyv(6?05IKZG@R!k#e63J^H&~9=ise zkG`(QUU_@li>HR$&n z#&{iG#sN&^YT|~BiF^1F^GD(MQ{k}aahhR^&Cz==)VzS+65$ZN0mGRIkqD5wkXa_& z#09A|?^b2Iif9*iC@8Gclc2#Q}fLGFBD^%-E2jfgM&lDnnGIIX30&0{n zDbj)YA^r&s3SH+7jM?>D*_Pa>8c443p=r8i&lF zLA>zf_&O^9Z}Hi`7^et|JQY(60Ahe;l5rNI=qbgT)|4qQm=sn{Y0B=To=E7X183?K z6@ToLV`E_0E~mTg#G{6b7Gbau&LqeUN;}d=qTD9k>-(oLib(4Gylfm4dGq#0Qe^6VFZ0!htWP zPO2(#9I*PJKG8kj4ZD4tk(nH`DWk$}BLjcVReg-S04+B;7s7Xf$eG$nR0m=hBIJb6 z#p;>mMGMEK&x};f)!)RGV?Skn#nsP@xN|I#?r`HwA7fE(3tR7CjQx36gDhWNMm0xd zltnx_7f5XM4ZJzlLM)|E&Q%=?`~p7CU1Eqo@A{c*yz;)zUT1Fbmze9U8pv)r;qZHm zGCY#Fp;;LNR7EqvwaHFW^4>L$bOZiP#ZC}7cPTq}DnFFzqdR^&`p)=;v5|9clu`O~zmXK%g?Bia853WD8f*WIpS2=1l06fye0z&{c4OSy~y><^jjE zEpHhHSOl%bnM0ZKjC(_|MGY%d-WY@JGTy=DT}Z}MiQ*Lqq^_iCw>pw3-qG<15`=-r zZ3}v8GHuIm-H2&^8g;Oh$kW47my^0-V`~;@dZrDD|R%wP#z-BcQ#iIGZ z<5-Y4S{WajXSZ%@KlKLZp||-@kbKB&2qF12`>=cKdiSfP?pMpf4*1&;*w!e!!SkJ= z4L8%X{|V#n2^8K2!f0u~J#cGaalRB%*F#53p`%}@#fiyc=;&%_s&L|gFZ9XTn`ajX zmToM~6g&2<`G(6|cCARImP19~p+A4q)bTCD!Eo5_?VE2Gy9bwVEYIBK?(VzWSL{5p z<~v&6(!V_Y`<8p}{;sXmGWw-&^nW)*?D5P*SL<&^nn%LS{jhwznZMtw9{2ITb+M5D ztxtqB-e{OL6aR1M2jK_n7*?K>Uk0(vOe0i4Z+r-c23`kWmT54)%?z2(anBoa?z<-3 z58z#ggUw7>Kq*Ab@z0wu7^y4I|BRn^IqwD|b!P`pRE_lfdnYLWRE!inMg>(&(I*u% z6UQKAGLy&#D}E1}X#_m3h;bU0n(Z2|HA&1@9DAaG$TyVlSPKqhpi+&2J|hW%r${Fe z1$o9_LIJe{;{xXX3KO)UDnp1=(G665V*(Nr_kT0Zo{PUkv*+TI*x!Hz9Ie*=C2i@) z#g<4x+Th%B$2Y##b)Qo5DNEsh?D_kiWxcfR;N9J8zN*?5yY{bKTsc$hIJ)LLR&MDo zyz}P=j>dL;zrsFfX{RdM@yu+8VSjOWL}2a<^2jiM-`_nF;qOO8$j57}4s8&{9rgi* zg*cCyfr1ww!lJ@CYUi*^p0%(SeKzBu&nk6!jtDRH3YbmVN2E&5SJ?`b96Su?nb}M| zT~sj}*b2DHjWY5A#H(Th%hdSiBy%MK^)4b&UgGj@SbuYMl=?g8&hfxal@;g2oa-=D zLfe*ln`*AQa+djcNML^0F6Fo^oI_rE$qMD1L_D7g#QUoTKZ^KrQckd1tK&mFj%WYE zQ=QdG<`X&2O)+uq3PkvMIVTf8J=?&w-Jq=O$G)6=i~G1qfEIbV#>Pz;UB@|=mR@Jn zVD=PsL!CPWh?oKBO^0ETuuL>;w8ISEEGP}WsN;vyDBN+qY_DVSG@`Ma>wYbG=x}ji z+8z`Eo;nzBw&?|QaKIt#U<^^5mJTdHSO+d1PV3q8n35)uY+s#{8r%ZB5SUG97P!XX zdOqU^hbNzHgUQ33U~#(}R?#qj)$V?A&e8ge9il!5hXEd{T=UUwsp{m~0#1?gs#zgT zqWa*zD4m+rr;Ov!lVX^01#{1Xmn~AK*!T%nor1*X6I#l4&BYQBnX4X3=0*}LJT&Nz zqDM@j&GuMnE0&=910FMVm`t11jUUt2l^7HKXV##x4kDoQI1C>!GO31jX1CXOv=*ui zKHmm`&{P0>01ca4Z+GA7Uc6Fjex*2YwA6g;UQelctl)a;V%oZHU%GYaZ+}woK!_Ob zFGvrh=5?v3BtbCx>&w5qysVab_peN^Nk<<@fls_Qy~VaY%LkYD6vGGBq}QNNThHQ| zQtK`VTSMW38zhF>7qVY9Z!e6LJJscF%PqzBNMW?x*}LAUmY`(!%IMwLYUi=SsfCf7 zXCC^3>%J`|-6v2q+M1Mr(YszR zcWf_*w>;?Cx_s)N$9^-mHgI@Vxl|fBy!636_TKLM-Y@1?k6rwS=HkHNl5%OaXX3Fe zVz4U8p8xq)Vzy1NCZgN#_WMsZ1%A&7kZumxmjRCcKL8tGQ=qWp91c<8a#rQIJ=na4AH+UJZ-b z2)!)MN>#qhdL7=T<7Bsu zo$&!}1>;5I=tMvSTe-V64GXFq3iQGbMXS_DV_c^(J40E}xQIghG<1L{+2PJPC^PXo zzRHG;oESvDYut$Jj_+f|##VoaGQapfUffb@KL1@_Z0%lrqZHm*kSkoYpfAabKP~PV zErw68NoO90+n}nsZDIba=54@x?cM9`yGrf5mf6+zJ%!OfHno?7o$EoR6jYWv)`EMU zBRww@+76^AH#y|`%I7CmzMok69oK*Yv?2RiODGZn+ha&4Snf}z>8hqdcxvq z_S*rE*zs+LB6^;jWCyF4)7VAzI#zZJ^0pg)ZUnTkJ=Zcy%YMyxzh?YjGlBnLdjFFdE-}MT bg`=!zL3+x->*;YW%d(zD^-m1G=}P|(tJIa& literal 0 HcmV?d00001 diff --git a/test/e2e/stress-scale-tests/tests/__pycache__/orchestrator.cpython-312.pyc b/test/e2e/stress-scale-tests/tests/__pycache__/orchestrator.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dbdb46556fa9eda1c705312e099a72c676a5c476 GIT binary patch literal 75796 zcmd?S33OXmdM1dCBmjaWNP_$JxbLJWQ5&_HBB@=}Mq8U@8sY;bP~7|gN+Jx(l}z;< zI#G|QBppdpt`?oD8Pjde6g^3&Se?v-PF2N9Dye!f;R1trrj=CXobhzd3|)42s^rYc zeE)rT3&2O2v^}T$%z)Bc@ABW}-TU9~U+$06((DHOLci?1`bW7&!_VkO{#jxdpL`t` zcMZIOHx3&vNN3{(BfB?UFtM}wf|;Ey7cA_YbRh|6({S=g%7v5>>jkTk;+cnSBen}R zqrqf2Xy7ehH}FYso1|PW*!kqI8!n`J9DK^PI)g#5nhc)Qw__w24U@)6W4-kwy6>$w zMUw}Hy`E7&&SRs4-YfOSsFip7J$~+LXV+NmrqmwV^ zod}!BXW}jkcUid0#yJ~j7tSu6bNC#*&E<1(%;PKgJUq+i^Kn$LAN^{xKo2S$oF~jtQI}@4TZ|I0WVpn&SoUb&tS} zje8KoJ2vX;9WZO^$ck&a|Kw&0RgvE)zY7|9que}O4O3>n5^G5A8gfvagXXtmF@uJF zB_wa6KRFen!H+zo-w@@S@Kc-HpDMo^a>!TeSq_Jsl=L()oN7qE8`hM{CNq6cHpWXWEG+`78&Q+TWVUOrD*)}Xyn z&R;zTEppj0fY|io_2COo5|V3V4&d?nZVRB|wR zXp8(*JqMFy3Rxf1Yu~MEX!$Cb92ZNcXX>`h=e1m#rd(70?Q(8IJLD_%%%}5?hZ%A$ zpiim&x9ZE7xG1# z(8^e|M%u-Ei6(Tloc0xrpZzuRotN&E5sELx9j1Tjhl#k9FMHjz?lbp#GU6*E8jDxT zm%ncQoU4l0F(&ml$ocAzGec+P_v)FijO!`q)i?5$+?&C|m?{lz2A}bU$!IulFye2< zeEZd#=JSRd#`>y2%i&S4-|HUs20Wbqs)rlX%<|j~um371@j4UIEFLuqy-^Fz^-+|=IT?=H< zlU8jJ`U7dz=c`Am`Tpud)yJ#P1S~BG1GX(z9jS=zczW+-w`07oA8Zai|7bHkPWO-b-NXIz%a3T9 z6^JPW51Hbd7#Khqqn1Ipcep+|Y9A0hU?M4Y)b95C-2+$q$H(}no%alT=(+Y9VH{wq zyn~bdHw3TW6HSfXAq~FX9JL4&qdtM_h?*L6 z^8&GIQ72vYUz@n>>6bc@z$(?&H70nTYwH~MMpFiG*?+w)YU>;y_r;#MaY@e{XTWKq zD@c#+KIx9$n6G&JLMl}%F*Q*eZjYhkd4#C7zn}V2e}6O??aZEBb&vAH9>Et)8W{Gt zg=h*wqnZ)chiI&u+Qx;karYIpLezF(Oc=pyq?LBYk6Il$B*l+95L|s}ck{gb#x%w{ z#x)P`MK-Ai&`D#jge0QRJUVs*x%q`b`UIkB^!fD5Jy6J@a^%yYkPgHSMJ?3Da753F zW-3iG#-ko5u>MFvx5Ve>LyZsM;}ECWpr%#F1VbnMnwV|^5rBBudW>cG$Z+It?nDX=J@BWR4J{V&7M}&6F!e-AJXwO(^DUN z7yn(ij89TqQ(x@w4-~Clp|r>^Gl)tK9q(&~Pm>MlSu>6IeBrdp*_S>nEPc>&zhy3S zsjzXn`}Wa)G#k>ZKD8Qb84+t)$XXV*mVcO663MF#<<&0b)lXY)J67Ena*@L3P+{{@ zVJqHetRx!>E8j18c9wth+8fvAYeF?UBQ<+MHG3AWJ=(cc^DM%ogNVG+NN#N?xAviR zDYp&JQXg9#ucf}48nG6Kti=&)MaWtawpRVzRrLD7*}gX~zH#x*{x|yP&o7#ms=Ff9 z2Se2dAEk(=&MsA-TXLPBwk>O;-#Z_+);{K{r!ACZ*jj>QBChI?t6Hpie#v!d+6F2e z&ewLmy6d%lukO3IbN2F5YJDWNIh5Kgw(MJUg;NiFSXvz^-4ZI@vQ)Zt`oyxMDB`FK zIqK#L!j3J=hb@vGe{HFISnL`RM_v+Jg{30j^s!|}e#B87a#YWH!;Vey zneWCEWL{eF;Nty@bGf0CO_7rJP)Ymzg++$pn4YvsgP&nclz`r1ehBk^ z;;+ge%uE@B#vz@hJa1w6&^bPdPv%qfsqh+k>+9wzqA@XP+z(~1ei$@8w0)s^uD4VwItf4C5j1|o@J*9S&#!}F&?PR%Ejo3W-0H$CXRXsX`Z9hF_j9$@e3z#2sG+VY-`N$ zmk`1CcQ|~_uwu!!t>AIVG zCpF^aLQYPs+&1t3?yYxjiQ5h=y2bKCA2<&`mIIWBoaJKG-i4PI2E>Z}VQ1H}Gw0J3 zL*^0VXDbCD<%y5bu3zuW=*%*_n^n@a&-`xd{^YI>^Y=TFa1%Fh(ICJEu46#{?rvb% z;*D#5VkYe;rok3^G#KcJdPghC0m(n+6-5p)6?9 zjj2Hd2Pvk`K(|KBWED$fF|$;9~^8W5KQ9JsT%_$(Vs4Krcypb9h?!1SlfhOTz~q!_X?`3CK4C_ z%f6=tLc%XDeVqw0?!C?=756?NZMxNCKzOI7*38|8*KDuZ2k;FIm@c6gvlg{Y*@HGc zG@MWSZPRU+#W0mRg)la)B;hGvd(arbw~Q9jVSKKk2MK@SR}i|Piq*y7(}D_qJ>~E# z7?vvSK(D(Tqdkq4ARQ%e1e1bk)TgdtCFwKX&I+dMm1!y?n5s8A1T*+F`FrCt14t*B z8qC;D=k@o{tY8M1kZg)cl;GO+O%1+mG6d7rHZ&SWjTUX4)*wp44|yxdocc;_(e;hL zpW7nF+kdf@x6idSY}ndP>*@ zcX^T%03eNx3{P^vUpzoT!Fnoqqfbc!xQ=(2CBov#@DS(qao|b8wDL;OK-n3;TaXgc z!nG_S@D=Wf6?p;YA@oSes{e+^d*!O%Cva%8kLcqRXb=);{P~C`WucO;Xwrx}HK?M5 zGv)wT?Dvg(2E2pbfqofwVxZ12LLaRaT4naNEzt^ z*$W))8XFlWl#}OTu%afW&-2oR2N9bHf<|d4D0K!&OEn6}npcDSKBDz%z;%XIvi3~} z7f9vmq{_$hKnh1NoUcBR-$yOP5MNkEaQRIhC5aER*F!T`V=8KjdT&4FRJ&&M!R zVH*WY?bl$!sr?#IIJIAc3adAOg){qCwOMYzZlgub0q=M;88EA7lov>_7EP9FG@3Gj zr3mo9sLeA9v>2GbCz|9N_ISn_Oc~8Q>Y2Pe22}4b!Q@nFQJb>-5w1~E)_#InM<>Rk z&i*T&QI;A!sQ6QmN0}~??!_XtR z?vcAOfXoq=! zJhbaLu&GB^#e%b8>$wke3hti2bN;t4PTOOMO$-*qU3T8x#ygF7Tko`r6|*7Xkd>RxxR-}wlyV{fy4<|! z)W69snLe=W$h)^KHXm_Om*a>$TtLgenbLOT$)f#lb+bSZeVpC}^#OsSSp$S3XWQ?Cdgzi@LsNe`{h+ zm~Xqc|H0AwN9Qu{_b%8)5LMK*l3Z$Q{4gu`ZvCD5-)@|>hq4;x`XbHyLe2Z$&)WC% zlFA1!+<#%Nb1t}4vU|o3l(=Ebe8*4g_X4<`>AZ96o^7^mIj>x--o9WARd+1qbu3pj zMk?Av747#@mfO1{?T13`haTmLr_U_4_lalEh1$=F`E_%7q5QVx{Ne|x_fsSJb)oz^ zv2ovG%EIKMRI!g2J;R~C5%J8ZI65KLUk~Ts`0M=g*HiyGzhouN(6H~*ELzFi@7V9P zM~XIwiZ+LxTRttJ-nY68OIiCqZ8GE)-Syt_{`R%mBca@u>4R}dgILx!U%OB_U$Ssj zEIAl<973(8WxjU&)#JCR@m%?l?7C2P-BNag=xF%WCtXHEakuf8pVk?i<-af(Z9C%= zE7|^2$Bqv(^6AwXqw=mKF#a^Bw!?^jA$)CoJY4J54823_s4Y(sd!%yU+aNjwE`rF9}b6 zVlm_TCn=qUCoAJI3m_E|X0aE;MVxMwVF;^-#GlN+fm734Z#<6~s2J~b)+>651hlVc zBE^qn$Glmiuk;k1C_M}UI#`xMtN^}gV(7XwFH(sC|A&(w}2;_pV{LcZ7l1(X`Wmn9q{%{y60D zPm)Ov>KgzTLF_J2U}y+PlJ2Ov?V$H9b2L>V_xKp^0kj-@i3EK)J=|poqf~?kz1R2P()~;Cr#2{)72)qd+`|q?IkO(;l)mnN5V#-s zMle#=8Y*j@H_c~?Wn0A3b}@VFv{e!E64!BW?_Awnp;*u!wr*XqRN6?WS5~>4-@05< zIeShlZh2g_MXYRJuG{oDzhd_Me3DqT4Iq65f%FAgAKOxk)2ENE6x006LD<&qy^`~j+*6)s_AZeHgh%PAOWhB$~^?AnVnp6 zw9TIiIkw>(SI~P?bLZwDVAvjZ?1)YGjiKzuxkC$=#q7p#cBdjAs0=wP#hSefHy4H; zImD`GKX5*`Vm7$;gBeIRI17GZFxi+41C&d7USD#wKvwG5g0qsR#0hMhe`$U|EbWLm zc7+_fK3hpbTuFTJr>3s7o=n3JGi?Wo%|9&MaUjq9XL(7u9?%Z_1g&EdHhl8O(BxQ4 zeBh($RT~Qt=r=^#9|^zs=~aL5#Zwwu5`Nd@iF~~|DQb!|kq3>xmL4P_L_(@`d)+9M zJ$;(jk0&z4*|!xJhpU2Pq7D9GRuzbNRe0rl1Av6O=Fc!q-i>D!O9ei+_*Us1j0n z#DDJfBv=bn@Rdllimztp8eG@Dp2A|*F(#~@#ctpm`6j-ZZ+YFqx2o4yO#CJ=Wo=-} zlK9O*^K~dTLD;c{F*lp(IHbc|1kF5qC-)I3UbnNjsd&>)v6M8n#uy_G4-gChCIwrrD<+CrI4Gfyj5DeWShi8l+ z)MB4f;%7EgVjZxY?+hmEFoMaqk{Kh2vxX7eANWnXjNEiGElbV~4v?`cj5So=9v=Pb|c!1@gG7ZUsW-ppr__eOz52 zwO-9J5aVjc_-0`iO&E~)LgM&{mVgxqE|JSc?YNI=bGjQaGukW5TQ7S@TYXpUm0Y*i zePwhEc0dC_RS8FAmOp@RhZ*oaHfr|_Tpi;oP9n{!x~m9DY*#(U@IcCJV!R?21Wn`_ zfszFtqP}Acdn@UAaL}>@GCL+rR@jH3PCV@9r02mPCneu=`t*s@9bA`tbac$WW?azQ zCroO%RZ{)%jzag4Bi+at(hNmYug7O6zSoOFac$gYZVT7WZQTVO7uM=ho-um4D({*p zRK_?zrQ9dEni{zglxJ#7D6t2<$|b|Vxy|ezk9lZK+tGm_hnWPefTis>u2QG* z3lkotXGoE@Y?6XYD?>B-e!xm=K86^l#+X^0ygRfr~eM@Wj2<{ri()zblh9LYRMiG7eyOw9uUJ7p!+O(2PNIXO`qtE!W3 z|5Z}M9>KB)3gp6{;p-RPqaonn0LHk4PRecG+sb>cw~nGia`@g@5S=R2h7){9?IU7^ zKO&%^J}GMP-1H1YvttsUZeZ+_XJX>2G_3QyW4wIpq@DzI=Dw>~@Te*@za(c}@&2u6pfWi!1>YTLYn-U;7E=Fkdfdi?b%tYU>9;+gR0 z^upp6uK1u^&sOWlsT27mWiU1(oD{|`dxU>O0c~vENh@!;B6HNWsb;YsC?3Uvy@j;j zg;7Yjd@bIwR*Xl|*B(voA)^IoaQBY&pf)D_9&Z!YjP;hN`I-m1&5}f7Oo##>7{qWF z&Gf?*LNdaLl|4|Zlf%R)dicOqnovkxPU$Ov6{NC3L0wJBClJ>w{5>5WQt8qt$$n|( zLgo&-l_Y9og7803FoF#v!6Bgtv55k!ehg#~v^gL@qcIzEMF;m0F>1nhaL@-55Jd$N z*oV0mkiZhJFOV>Vos=q*sU4-OVmkZ_9AK7vb<8{95q_VZFgSpiY~c^+@NenxhjgG7 zy(DH!_S^(Q!%LV$PHY8FGZM3E0p>8qYw(3t&;lkxiRkZ9iVCcNzD2Nt@JBdAtw=+X zC;bWCrn8xDVjKcg!3*J;gI>o24kQw%VGBPAbGU^E`j~?>*Ato01u*!|U^<3yDc#_7 zy*Bw9lgdh1f*B5n9Y>b4N$lcqMpDZ|spYeMk?I|x>K*T=?nnrGChX{og>{LkMfYyb z`M)#y)?}z`2Q=XCPTiS;R{Hrj&M)P)P9MUWT*f%=<%cpVmQpKc{gK+;q1xT=rzYqc z`ezT#c8Ja0i+z!v3!$D1B6Pkl4vHmL!j7xRZ(|V1p1J%v`+VKPj)exXpeJlSuwr?} zWXpeCQ1YPYe$j)n`(<;*kc|~|OdnZEE-tl9_pa<^AW6yN;;P4G^~=?pKTa~2<$j!$ zT#~htWhif3E^mCZ^o`Q_l!daT^25tj&2xSNQOa9IZqwtcP0RJ05TT)P#bB;40x(kE z@C$>X>5vJ)N;S!++P5t?w=8dJTW)FnIK$dl2!-vsmgNSDR9_02AKkR!rndCsR9j8q z$JNDExhsa^%Dk1`hVtge#kF(!NNRK8$4N=$c#*{Ap)I++%T-Ow70t`do1oTRor`?A zXN=ls`ME2p_Nu&(4;xzn?c{VYXeVGaI7=X8%4k@27TeB;McU`EDYGX+gw3qoa(OpxVJnzNo-j*Lb?(W@q==#>Q3LW>@z z6ikLvTgm{G5~0$DITp%@Nx_ss(;d@os|C-dG=`!5mQVTcvY#tA=NfskgcJNY@{ zAk}c1DWyU1Gy2S0sFR-hVi0AJ*Q7wDDo|f~RO%=u@-i9(#O0eBtYwmf5qfFpDC6?U z3{gs<^{qAy;5BBUinmT#dD|43gReI|hWsRGJw-ou&a!?Rla$xwhS`S6_B1S`aN8FKcuMAswAq^ z5&b|?XgW{$@yc}OTnj5p!H`m2x!tVK1Lhoo+l0C;HAc?#g64W{JV2|l|+xg$15CIe;|;Jvsp zMi{N4rM%>X(;T`U*--V(A(AGE3hl@#Hxwejr7o`rw|J zIyh)`5wDHKD9{6+@DG%Z%AylvF8l{NkY<~}SY?9pqB&ABaWcLbza7od2=Gv#7@g2k z!@>?)Zj9|Gc3i`Br%Fus|3yz3=bgkFJ!*9i4-<#%i<$xIN&NUQR$jP8Zoj19zoNr1 z4$K0M`b+F5@!zlDjXwV!D*+OkCH}h|O$+`z-{6p#X0fbwIj>;)&~kb8n+0zayjk`} z*-y(i-$|P>&zxI!=HJb@lkv{YCGMGbZqA!$D`$PbS2%wEzYvlHU3iY?G7auq8= zF=jMV-X3;tUCt{8&s|aXX2}~Rb2pdDcZ3|J(}(U{T~5n^i1GaW^CV`R+cAH7sbohe zO=a!NAgr6Mn!PzYByQTzxGRphtL5zcIE5xjv^c+u#BXB3;js0{ilxR@@V6PR$F+5$ zvue4z4MslkbitVC=KPTOOrKcJq;<)Lx+W#4!uUlTjEl@Ekr~PAoy(P5S8Rqf@RCW8 zH98re*Br`jp1T^(-mc`K6$qw_RcFJ_bEpKQ>&(B`{$S_*owMg)P8!N+p%}J1wtIDu zAV!=GA!ozfjAiZ}q3!1PHt#yL+5Bj860YMsDA7U@D?Y;~L5a=OWi7viX!7#}hN|wij<8an4R{-$ z3H_Kpke;|9NqVn6N{pS>ED>qjwIQTyNyI1&;_Z4catdJPp?SpgV{}-qAlNIJ`867D=UNP4JWXIcQy}!SGVej_Et!q640bK^HfrI` zNGFx0^JN)N3&dxR4Kkj3wCga0uzewRCtXWBiLdp06j;t8{Wp37(Ku$4Qq*ZNq?o4v zCTQG4`dxaAC$n|ooeHa$nZS7Jm0ho|XR-dJWjwPJw5xis2z(k1t_{(k$8P4Z`p;#e zHBC;3Nrl~QhGYec75RQiDfImhY6O= zr4N~I3PXn}J0ILajH7!B464IJ#niKyw~d{z?wmRIerhQhWi#&?Oq*)*VKxl2&%bpZ zj9>FJOZA;%-F|WZY4OZO@$~az$)&L4g%69lSr@U7b2~!i?LRHvI%8$*4Do}r`Exry z08D2#lx$l`uKkLbG^Put!=#Z$*q7qcLK#&t22HX7pUn?D8H4>T}g(b8K+EsWwKB;Pk19~Ou@_(19s5F&_0+IPno9> zK4|`b4dGL?${20X@zKzx7d|86=bMZnY##Hj>XrLu&FiuR{jcVXQQ0#+tuz^9)zcs2&3qDReDTTOG*8)}JC=dAWxuDTMA|3}4_YrmXr*Zu zE+(xw+JhFv*O-Yz@jVsZYwc4|cgLi!g8(8{8cQLa&>=n}Xv5L@P1_WOUoQ=rK?l(- zGpWGgQuUnn1s!}ApM`finrr$``K*W8Z^x*ZBBn|Y+jAA5R(TaA%S~XN)?11^RuB19 z16u6XPv|}S4RNsx%}J|``P4e5_9FcW7^e_I134>_$sw`1 z0i+$*kQx!z{YvUk!|L>ad;gFbWUn{>%3x-~cc4E$>Cs*l%p7W3O+$Qoe6_z-4kPz* zNRVi8sMmhdsIx}Bjux&F>%WLrXA7~+TbaQ9%~LqvF#0#iB}cn_b3^S?lh`i*f38Qx zw^gm%sa?=M#~f}l#kE#u4X?2}2afK);qi=0%Q*72Nz1x;h0?yOP$MPPEF4L3l;CJi z4oXf}eI)mg)N`OxH44p9Pt1&-v`K4y9LrVp6`-tjd&m5cYdp(TZb;K;z(36SCN2Z` zCl{k0A4C!`LUKbvZjd{wXd-c)Cl8Ydzbc@85dUp#E1*cmlM z-f(aFg(@VYuD*p&C{-sCLjzG&OL}yWynrB8uP%_U=1YY}WiUNFR7Uz~x|jTXc5t-< zP33_aG%hn;lN3T@QDBNc>SuK0f;pTZEpve;J4tp^8B@q4Le`2Cu(ogq$IuM$lh!3s z0e5Vqm`2huNud*DIpue#1Qfo5i>9#(>yLH0KxIr$1X&nm6>CQ3n!t;KBQZ#f3MB2U zeSMyBn2Q2|6sjb#R;tw5$&$<`NtSH*ekHM0)XoYI zGc)+;HIBor0E$eGfN{$-s+pHnI3+;GCZ%a=D4~Gzv`Mu#Fv1J@bPwx4_?^RT)WNP| z)d_#TP)3u4Q-s;dk}35p$*6t}?*dhNc4>)DA?oq9_`;AF0QSf-Y~A_Y!|xpa?umC! zgxh<t(UMXF4^Md+u*D3KNvY`ofN9 zVZR2MR9W?dk^3WaN0&->gser=)|vVbQ`4u(^sV##)DkI7Zt>kqcP`EDT*_&lwttvY zM0amZE>#|TYjUo9p>whDk52qv=iFqd^4L<&@oD?x+@jaJ9~`-Vfi#YvIVk?!}2m7akoG&w0eb zQPDFd7L13jFRfS(8*RJ)#!&*Wp{P;ugSSvlF~ox}i2c{Z7ly^*8!HCv1hU_R(@kR+ zow`ijbSmgEO&^Z0jwK7AY@RLqX9l(|}6a%`D@DdgCJb6gJh{If@9cZ;q27HtbJOV)#^dPF_) z>XDh3ZXZWk3LECCzSHtni&(#NA#IZx8?}cn~ZF-9vKrd4<%QCpQNOnyqyC#y|6v}Q|%5ItNj!XRBtvP<~x%oD+ zWNX*~gvtO_G=C_gl29P{2fPQ3w8pu^OU`Wzkjw4DnO-O^0%zOj?6W~}%X#qvFP`^^ z<%4186(jU_kaHi-6kC0>bIG}9F*D?Z|G+)uSy0LFz>fyXOk$9Af)f@!Vzc?0`6MO&lHE5topHi37n@#hX z-_3s~f8H*Z?jkRa8KvJI{MOZ&)RRgFmA$69=a!u9@DSnL3AyK*#YQcn!}Pw|D;70F zv2P9rl+*@}T{u&&I&UJ5ZK7k_XUyY*Z57pBi=B(GR3)zt|9R+I$p%qReBBra|8?&1 zyySnIjKiNc+K-o5|8$!L#~)N2G#sxm{HU^X3r>Gkk#VNb{8w%Jcb#lAi){|vh7A`NfAg2fUn7i~OsY=~TvpiN({R7#R_(E9 zy5(dSRcfGkH(VUk$sz8W4DVr5#4Ax$mN&3_&zJZGvd{D!>*?z2>E_OK_4Ib0K70Z! zl&kB+@sr1T5}xjE{t~}lY$K4PIOQURa@oH>Q;e7NZ+4*bsERrDlJPs^kKmg==0-l6 zCSOSm4t0IvH6$+M*Krr9*W)tC$&qXhr_gbSSdDKZ4p{WQ`9#osU@l$n_-U{raY)jhzVqcp?BYwORUKA6HM1waq-X<)fEvfMPHEYPdKvnt^lmIymV z37a-hswM)owWA33Hq%p`x2j*=?U6Oo*V963PArqQ?r#4Uj40M$KP`UJcx6A;3$#*K zty(t~dLj8u*D#X`U2Ej1M}3=KI>>SJm;VmGiW!67!i1{r>zjM1ou};Tj)6m)<=Rxw zYQ0XSeqKljk{vN)84@Hr;5Ru{5hOblL9#>cIZSk^W~E%|&Td=fSd1-J!m1a#v9<+M z(K_3MY4P9a8ZE9YGt^Z19ZWd+#dwIF%(Orgm+#=$Ttr4Ed@jdC5t)p6H`%6LYn zj2rHW8k&yEG|9sJgdVUl>iWK7>i*kA>gt~h{DWPV<}rCVfp?M^o%v^xvy4q|cJd zx-52ArLzi!?Vy8P!9$-wc@{X?_!a*G^=JAzPxl?}J;)vJ>^YNtRR?WDg*cGDV_? zR~U^xq%_{51NmSHWM~qi_<4qZ28lGT<|B0fJmgQTLw<0E;50q%KCKBQJ1UsX3aO3bs=GhZSg>xu3_73*? zC-(%ZHDSBPCWf(d8*FN^O)GV4C8j`tneZ1l1d25gSUYXjv=cm@vA#=MDB3>G1@bk~ zl=??l=!&!kY>IE&K$|9PW}D_h7Beui76&PG_gsOE34NnXm>o?zD%L zA!7I1CS&6q`B>|S>BR&JG|8~H3Qf#jT1W%gLOLBX=s?<8lHyL4CZeiK!;~vjWtu)A z6A=QLq>q2UNkEp} zXtac+5e14<)L$g??roZ6ln z*@+8sSFpE-)QoWj)zqdGk{kk3DYUhr_K?_|BnOHg5tvSto;H$pd`nbJiu;P?ax$$S zc`p}gftY?0eFx~YyLGDO*->e~V|gPOC^pfwkyz)5-JlBgK5r(fuT&**l8REiwi7e1 z6240VUMd|9)8XrMI6{X7I$Xmc+Cqhhv!U0wzUr68U%^X}lGq>|waO!~WJdozO7vgQ z`xJ6m3D@Y+BwSz*?OXJQ>1q83x*`Q8p^y$mbRcajp_mRY)7xn}l+Z0HS78gBll0(^ z=r)lP~L!T+y0g7Iv@9-JBZ|cN`Z>dczLdW(0aK z8oD~jd)VR$v4knHW#_HwWr}ot#HthG$#ahaq>NNS%1ANop?g!<=5%gw$+>j_`bHf% zFFSKrl2g)78JA1SA3T5m`AEsRP|3OFg7RhTRXuut6b{x)&l^86m`k%}?C`+_l_o{) zNvwQku`tqkI@EbO(s?e_c}{%hym%h9Hx}_+4|%RfJgN{xGNT-j3hxpE1ViUx}5w z7CIN)-|Je~x@cUiTP%FEQ+)Q4*!O~1-XC_pxLi~gDQXNsZ|=}i5v&sbI=hJM_m18< z8gW&HTvcN2Iq|%k)S{{f!Y+QwjqZr>}oBv?T{N(TNrOmd>7Y{9Vh{yZIk{81c_s=u3W_JGOty$yy z8Rbejq-HxEKJ%P-_Qj<$Zt?VG)ZBpaF!jF5Bdq3*nx5W{hN7x(cfWb$jU(SW7O83v zRkhEbo_C8?JH*PJ3obE#FEjv4+W(;MyBFTM@cYltL#3)~`WQ6w3b_X*_eLN9zjyB?Rez@GKiTerlPy$kL| z`$AAWdRZJ86EBa8<1eFrri=%unu5lIbUJ8)r#pk878^)`pW0`UjQ5efRgtrvlnosJs;_gymT?X;68DLTC4Pdf)058+SqR?ZUz_ z@euU=2B^aqcxIB9OKNFbq-9s(+R2qRAvyLTv5ki{A5!uE-{C3lxLZ8o5xKz`I8JPNv-geO<$N0M=pSn1mHV&Ar;>8?f$A+NeRb~4aX5X! zMrWE)gqy3Sip3qUx($Op8k*nfWuUI`4c3pc6mDZ|1Y587@xbrXujgCi6 z;&a1d?TF~AT5jA*8(A;s*R7pxQwOqb$`xI8tMe5p%fZmm!N}3!(9z+gqod-HF>!2M ztbQr%5~!+_I;>f8?OZq&a_z+#yJ&olwi0trnJ{NV>-=gt7>$}#4fLL)P>^w?up)i{;e?^>IL(Z+AeVAR0oi5l2c^?{dqR%-xf*#3 zk%*%$_iPg}b)2TBa@nL0Q17aJaJ?#aU4kCQs9aQaET{l!e{PueYX z+_B$`N1@%v(qGIq{QF|ti|OWnUtRj5&HUJ$j2Dk>NqF)&-AdQF83Tpp$JL$OCkan}2F*}hm)PJb`DZmd_$u?ytCDaXr#A+fvZO9L z44&>w=#A;2pwbReW<|bZ(du$NU7F4(@yUw*$Qs^MbW!*erom&?N}r%RV!~d?v@~Hq zRgVMOBR|8;VdibBZymk8!QPI=)RQffHdsur%tHHoVY>?pwy4N}W*D}-fT3(Q(+qP# zGc1QTnE)!jUL7o5BWn1vQTD@PQhONQC;s$wn^b3)Mg!C6(lIm)T6QBTP^g$8t0@O9 z+MRNlb4K;0ay3-0Tq+kZMTV(3U#M-9+y66isZ@d}7b=zT9Fq)p0U;qrcOmSFOr zM%Hor?=1$+&eO^pzKCg~>|a9`p+{}8UOMZNL`eO;M8CgJ+WAu6KAb(3!k1webyK%t z#(3#>Vonn*Q^{XNALKcvsZ!4ByJCHP8YQ#PUg6LL)9r6)xhiIeTg@b_HB5J+7P=Fu zYt*Tpbh9pI$_sg`Rx>G>GE}cLy>byu#pW($c-NpjQZDqL@?{U}6%7i<=a!@)Ci5pF ze5#}Zsr1NKdOB(H$h09%mL^t~X14cvtyZ54X7U*@ zL#|(=UKG7BnIagmOKG*RzR?;>QDdsjZJLco&2GF(k%(ZQ;FL{m9=+)2mw*jxvkB!x9=e6dj0r_l-o(>u0v(K%$ z-?<(on`1qgc0Jc)#wD84ju496 zP;y4b|Py{2ra z;UWaNk}y{lyGRWav$JNYP<=1~ktD6D*Na6^up*Hm;SWU$q^m(s(tQS#m1`t6^@)bp za&KT%v}mL&!WLB6fxsq>=mvEnnU9mSkrBWwmgFQPcL5O9Dut9KAcmGtJNYD@rS*vEBTe3FI8$;GMoPQ2$F2YCJX5CA!I+$f6 zVCt)>(kATpcFvxkJvw(wve=JuixpyCYdEh*!QE;@u3E8y`BR2{dENQ2>jGdu#3|LxJGh2T%aIf<>ZYhY31O#0OUj!=0FJ1JA z7Y4-e$)gYP)7MBdB7?0^aVn%FJ2N~xFTMpLJM66kW=;DmUmjjw$8;&a%26dSG>rJ7d&Fw;LM?fEKiE3&Wk56h!+0jm<~R4g0i=*y8!4>X45Y9scjg%2*~->P#nw>8)=0&!P{ppLiaj$&>P>fN9UKE+ZSL_zdP*QPG;kZ zfxB37DC|5O7glRx{>zmzLw-dhuRWC4{tJU8vrwrRNwRZB?0a53gG~Fy7u{n2WwAyQ z2E_zLQlR~dZSU=Pm-c(9IUaWPE|*r$9QtNi zSVN7~ESJA0D!w>D9RXF`wYiBx6@9phwK4CZgWupl%Vez zrr_qsuB|x!#GY}gI_W0`ov9}qumN*29*GSOJQBB6p5l^11r#*ICE-b^+DzAVw$oe8 zp|%~TTg>mbB;h&^SR^;6v@HAN53%U^|0b}g4=KVUrnI*WT-r>Tz6x09o3u9$cEW+T zOr$(zp&ecTT|x<%!HLW@9!G*Wp@;^hLhIqe#eg#_83(Y4f5Oj(LuGR%kgDFZ_7zd(t&?>BT z^q^F2t>>WD*Gq-%bE&W?RM>-d4He~T>&RDXAHgymRIC96=+$lJ+u-f!_c@?O)>`ct zQ$IJc{b?$svKQvp6QHPoy`MwP8oi1w(Nk}wvL!msnkBj`@E?_> zGh3ny?r|^kQ61YXl(r$odWSiNhO;Oo@ER+1+H(co8wRe)3ny(Hb@h(r8g?bcy3IEx z?Jxty&=DnVtl=eR+0tu50G^sI6q{p8(yAsTtq$P;4c@-gh0pM$l0~3q^%DNHQgk&L zp%IOdu#W$T27aLn9VAddOJ`l;o@78}a$rIbpd8u^6|0y^v!qSh_>_gXPjCGl$kEk% zg&ra0A1GEymk|l2{JgZ#D^V7aY75I@v0BUC&}uDO*e|KNk{7Oiv?DpwiWOE;i(}ep zv|x*xN4%r(&E}@okTlo6M%DKlbfA@5y+c}>Nor^0)a!TgG-{)inHTe@Bc@iXy8MzH zyLF%iqzz;c)`zV!p|^(RmE^-bnobYZdug5KK+?+k3>VT8Eni)tspX*D0*y~yro{*T z2U@B%LwOADHRAQg#DdPSb$@KBCg#2!KI?6IrmcDrs+fS=9bUT?^w#)J$>kB#Z__M)<1AI zkgxS>$h1qiZ<}Y2Epcu0ZA-;lr;mSFQpvX6*}YJ*RB~{-_h-codQls(FpSq{f-9#z zF)mbuI+deo?);L2SaHYhuw(nLe$i<(yn-UX;hA(z=D_xej3(F* ztUwR*da-s%b2n5{JvKFu(UH%bJZU5%UU}q)$%P6;fU#Ph;YsXiSD*Gdn%~DLqn&iCu2#dOGYlquJGqv`BuvE3GTj z@cm3%cbWP7g}b^7%|9qi!gbt6Udo`_0!Hty!H-IpetH64MLccB0Hht7Ke@W3{MqTL(Tm0Qppz~C7)T|4_ z9IRLOV%3cOC5H4S9NUm^F5n3V8*Xcie8mnV&!+j-w^RbLI>RV@W=4nraLu=%xH(^R8 z&Rl;EXq~J$1ruGjTUyp<(z20n9J`m;;XW-pkZLeo>qf8J5lr%JTcbpJV~tcISJ2Mq ze8a-$Zj4qwgCRV6{e@=NgkNmGX6*c-*ZO*+2Wl^0OOd8#fGsBe`0Tf1qoZ-1qq26K z6ihRw+y82fn$yExirFY#Lc34%rCKVHhJt6QG{(0vSUr*;vfmIFyU?75Dl|8-OR$6& zfexkW)gE8KzMZNK&@Z)ooKhkfKl2nJLA3L^UaRWWLs23IC8u(GSxQaAK9&WpD*PW= zI4k05s9s$5FtND`!Xz??Bvglx9S-b)BKalz=CxVHCr+&iHPagSKTwgq2ZT2C63OGc zRkCk}TlAC|-V-(Mi6)(`^xATOne7?umOLPLZqNJo@7ZmHE}m2yKyoQzl7pG zv8x*%2Z|(+huO*!Ayg@{p||i6LI<*AM&?pgy9dcYIBN9OyCh>lm6IiTe6)~)#aT1b zcE-$dT(X1oB9yQd)g85~b;%GQxZ$51_e3p2W8P88G}6^SJa&Z)3;TJsjL{_Cb9v&5 zK#N%63LUP}fw9ojxQd}el3aeCUj0iPqV{2!RQF3}lbLFAl*1JdFIb483|2-$rldBG zrqkBL%B?SIQ?7;ADdk}td^Bwlqb~m>hW-D<(?E?L!%oe?5HKnj%kczp>bp^(uNi=$ zOXhuiZNm zO0SvATT0)wVrok}XoMj7?(m)ANba6c?w&a7wY^tjc`JP)C_>-aU4Mq0$hW7S~drpYeC&R8&%jH#X7QRs!DQ^pvx6K?O zu3ekhMx`>i350_uf0aT-*>T zZVMH+&7c473-7!jws*rJwKsC$xzK^<#GbR_=5yiV^UGy*WR)|Yw^X)cdGoF}GJrcZ z>_Z6(N+by_ld87QTjv4`C5!&|Cf}VDPxUVyd{I2$7V9sE^9TO6sA2AvNNZ21wdYap z`>iLIi@66!?;nkMruWYun(tURFE$?x7av-#q%G{U#Sv?Fhq8Aol|!P>ISVaW=DmyN z_w4W5BhQ=+J#$jrcS<~UR;)P}cAZDyh^sl|qWzFv+n0+tfL29i59;sNf1`2E9xB{C zld_y$c<=1&OZP9npIsMf+pi?A*1fnDa#FY@9}@@0Ayj_J*h{kIniI^D=cFkikGdw& zRSn9a6rsp~;p9Aj&OTqeQ28ef-)o3;90_$C5qBOHk6schU^LbLsgpphnE&}ty+%Vu zc3g)FqeBfF6*HM4Yz`on#VoZlOSp&6oBMJAsMcCoH2?CSooh*N~& z5|Y@-wqUO6k#s^nTd`4MPkf|>`L3(md92v*!|Ki|oZho{mf`fMVSgGoWi4 z&X(Q`^ItWU;`rl?Bs__O$G|g5@Yw&3&zj&d%zAi~_|sL=)mftR}uqu_p-VKyAb^Yn;amn$m3pz*FNKc2G_L&g= zzuo|?j`gR#MysT4AcZQmM&+wjtpbyu(Y3bICXOlL$EU07BZvZGFgJ7mr8jQXCPc?A zYD;I-3R)gIm2oR6m^4(kI;IfZKuv3<>$<4{Cpm=?jJDQZot6FwmW5Sk!|K5JRyf`c|B%Db0}XzA z`(Fct)aPcpSC@cqV7SM(FrGC|Irt{NIq0}zW-g%hzMg}tQ&{Wk38wKad@KH19&UO& z#>26_WLGDlKjO3{?j_4_p2}bpX8adI;g&=SH}G9-T`i-S%m=E~B3BR4s(S4Dtlvin zK1lfSdBK$U`5ZV>{#44X6yS2x(v^xNeW{$dr{q-iIXTsBs7&o^<#hWGzi>W>6Dkh3 z{I<37naoejr+#~2!Y;!XF=UD9Z~+{F9+%$@pJ*__0yLxoC#r=N+Zz+xD~|i**B=t5Hk;ClU@Tioq*b1%(N*G)7}z1*hA5CU555z zDRI5R4-i`raKJV~7wMW-z6{`!D398y+53l4<-S0fTKnh{KCX_yG;Z@Ij=iq$@czR; z|MNcwOdJ=;1U)Q>THYAMm_P{yLMu6_e3BuQuOpfVJazlXztzWOBL z&#CB1u%iO-=%T76zh_ai-#-~BI8HrB?%2RsIcyv{KH&%H+x%k_16O1I56S&pzhs$K zQ^T=MN^8%OH$}-oqpHYI8_3o?R}yRNl3e#mmR}xihWK*et`6Fd0Cn*VBIi=+tLf?O z*8g%UVxX>y5hr}RP~WT8=pZoUHk&@p1msqQY@=C9pN_$~9v|yB@GwWvcr+#UsYKJ@ zsExiFd=98Ns#bh0(Ucen9Zi-#x@b!5i-W5=>0`t8pwh?I@AjkfDc@PtrhHt{oMChh zEsd-{3;#szm(BoQk^vx{k#W{rRJVB1Je}eZ_={}EhrNE*ktHXsKd01E;N%qkK?Ome z{y;(|VS)}f=&%)sXc;PG6d!3nyqs!DLRI%uD@v-0PNu3TfzgC93*&ScqXd$rk1?7g z6@sv2f#8a$g@}nJOCLZqJ@z3;9WY)EK_`MVFo``f`40T_ObPtBQ2gR-{V*S;h=6jWRTfzlf65{P&-1*+V zclSlQE`_=-MY{M<7ccJjkfoRKG6?cX0aEwI6+2k zi^Jk+V9)0xeJ_UkUKHUPj?9U?5&oqR|B^T$i1oe?^8Fv14HYfun{h6xs`rLn`&R7f z1VZN(-3{IeM)F|kwM&(dESI({mzA&B4EZ2BDL-q*@-gHj`3g?`|Fn1QF>zh#eeOJ9 z1_ow;;T2v39SQe~QMZk)csE#E z$wW;Xw!7_8ZPns#oUmJU_g;fD<7ujOrT)>Cs>+X4jhxl0zwexThnevOLlsqBsaN)y zd(S!$67jew+Urd{*S+N;KB+BZ@N)v27jS(`yi zVck=-)GfdT6+vge{n}fv1#AU(Yz5&BZyM-HSaekd^9$cUbL~tZzj7hJa>5fVUOUnC zd1n6A3)0#h3z)NRC(kokCGd9Fvecd$du=!-5>0p+6oKA zZ4z_{#k=o1Iv=Fz3f4b>xt8*27s$4P$@)9C@&|fn%2DCIa}&7~tC$*@J}j--KGAs} z8j!|oje(qYIj5a$_+S*5d@VWXEQbkL5Fj=D%PoZWz}ybuvCf>EG0_SezNk02cE_w^ zHcl#T#~Z+i79JM9{w^A3HqDes#Z3#2X1t$h(C^lnGc!kKFTg3~U2FH3U+jTDFXjgf z--2p;$CVX4uBxj&VDGqN?_hqwb_!p8QKXc|A7FI(2<65dyN8u;M|rQH%PNGUF$`z; zZ;%_Vs_wkj9S%5-$c`ga1*N_sGVA>9Y?6}`980*aZWZI`X>R32rX_}&B8 zc%!}cuuk{smZnDh{H(6Ub^wu9&COW{Gj+dowroe{ytBFJkX84Yz5o}W6dV+0?zzHMnhm5-3=N|$O&5LZqRBuwABzy&p%XW2v(aY zr_MwK3V35yv` z#-csE38ba`(Nf?r(2Ehqdm73ZuW4D(%hGVC_(-J{wi(P6C1lis&5D&iWkPJE3AL3g zpYYZew<_E=|6q6#Ov_Pq*`{DExIv0HTqG9uqTF9G@I^cbxleq8SHqDmiF8R=5Qceq zlH!E4i#2-xIc~Mp@9o-;SoESu`0X?OeGH<)mJnE$68J@3fIAsP0%U6LT0gib#a+4>06Efb1;y|Un38Tv%PeSd6S-|GXisP+y%W$Cf#z; z_P4mycG(;SxKM`9hT1;z(6AE7#fuVi+8m3rxI-ulYtmjeg)<)owX(f-QHCHN1$E3t z0p?<8;}x%@(Yp46SXDb=H;S}1?U?G1QyBOcd!*bG3+7&?TTj!_t%D>D-MYfT0V0tg z7BklQdCsD&!THs$xvIG}lB+9V@0RV|Ou4>Sc=DLEdbeziE%~~i=I6C#7=F5?vMt5% zab-Tz+bJgGXwT25@MixH&(EKJz4J2^XGSe`yw*my*T`n7)j(TKY#xn^oT)}>J5_6j zO2G?@IlR2SdF3=ht!0UvuOoPhh7!zc`%`>SbVTtfo)wiY!Bak6;#snzDzgi(osC5b z<0(;a+R-HNl(5=gi$`cF(iu-7MWq(6WPoUKDIld5b$p}~Bi|Ibfpy$;(KJdP_O#xZ zRy)ZQi!~Bnptb^@!di>qDQQ|jN-|R6KuQ)QW=W31?yi`hL<1?Q4<18?V3Z+KGi^kQ ze~b6TagnU;{bCPI`6no0oTnFOU7n`^h}0Nu8BwGe29jhhslq5kPGm(bd$ zwtKX978iH8*shLvhUkmoM#P9%Py}}z=SDxFj74swr#iHK<0E?t#*!4P9Ww5D76=eV ztL&l5h$D@msg-af1q7(cky0XYf4k(2T~QnOYLwb3r`~Fx*S@hZl1|7AGU2Xjwi7l51H0)LJuW?T|on{VM=> zHKDHFc(%s4=rwySUj3LJHvS1qi~vmb$WOc1c0D{2Ui)>^xY3(3Ccq@lJSKQkNJNFt z^HhTR03@g8rz!akmY*KZPgn9Y6ljpvm}=O<13Z|yxRO}Km}apt&5o41&lN0DUu&NV zkk)8v`wboB?Hp$Jp2L45s7YBmzLt0xhJE-~hf6_!sf;^Cw4DSt}kxCKHYQDN4P{m+O3$14HXCRE4aOh5fevQ@+YR-#}&WtN0x{ zi;!HsB47m2n)>{4xLMIV?2pIKAsit??S>6*L&%N*SQiF-m_308*aNIW$a)IEj(sfH z-A->P**hRI(81_lC|`ZpdO1N9^-^*YNhmSgK))zX;sXsS5x9ninMG8{$~5iW%2hI2 zW4ij`p}~Q^QBn-E>%IIkP0?$f9H2;LrxCUb6s7RYW}ZhxihbzXp8@SZfi9wl8A3kY zRG<#wovO&8B8z;hNPl=DK><>l1VUA8}^!5l)4KGr`}-|Q_bbzVjt zWrr$c@sU3E+^K=nP=#|pG;B=}O2ifP_XDS(m-fXlkufx&%|v)K!s!0a5ED7hn24g4 zdAIwW?#b@y=jR=D@Yk7|^KQjE6_asO=K0jB%kdA4#TMp8rTeY!fVEP#R!+A|*2)EI zV^|-+jP5T^ot`=@)p!tG=&rQ`2K4Cxn^U$q1GWm;R`E#Zv}6Qrc2Z7Vt$zD;tO3^4 zz*vX8cxFn@s^FSU!J@L86*nrTt?=a;Tvrt=UH>rNxIN>s&bTJ?VPZl->O>-RhmJLq z19IAWnC;AT%NsiScX_y$$=DBNRW{mj~#?Ab!8u=TFJEoR{& zxww|4sEyT2|H7h6@d#f%sK=u|@ex#idP`He?pAqT>vqH6Tbj#T8w?*em~aurl4z=D zEa^JZ)v=_->1~AmB-o+Fa@38IaX4vVe(nrp_0JfTf+-qB05cmWsKd5SfVbCNX!ABarmpH1EJ^R7`puHpgw;I3X*tVk*PO6mim>A z`-hB!#Ld+xS(~PgS2l(2b@*RI#faLIf#Y5(BCAHf$W5=#5*n5)g>hrVzIC z$}ax9$ZFy4f8|dZL8a>Q-ks)eQ{z}WDSS2{N<-(xKEU$$ybfUFd66)31mjYiT@Rf+ zk8^N3C_z~KP~7*Ia^zJwUNyxFeZH-ut!v-T18vTh&bFqm7xr@8;;(;%;qkzOI($Zt zIQuA!_W5DwFo_2P=fJa>tGP;`7taFLNPUNi0>Z1hX#^2y!AOx0L|-2OVXEq7=oCsm zRR_ks`~BP4L!y^wbf73`q8mo9>1utIJTGC+L_Zz`p67kQa1soStgW3CY#I1KNIyP* zDmQlylSsI}qNkf$#K6HXI<-4Q?PhQ_lQRCwFk$QOHQem-yZCsk1H1S6Zi<=f^z7`~ zxv!(Gm5;RB5}{opWmCxB10W$D?<^QK8~0EagK32a4tBT7UjR28CQi=dqrQzZx0&tp z0rSRd6XFekHo}&LY-*ljZ630#8R~dItfDujdX)pNPI+4qonEK@*s_wC2;VYPUHyCu zIoKBU>}%P#vs;8zgP_m7kUJ{Xhxrc40YBn$K&00~iWvvn?c;UH*wdH&#_bl|LX_JZVkb;15xm@;0t4w2@hx?PfM_g#Bu*fla26WV4^ z&-Kk7C%@K(M;GkJ08onpdgQg-wYGi<0fA#FcE@5VYNlVAhU1`y+5M6=FJRpzTX!)) z#cqfbW{swv_)775f81K!R;RmNXQAyx!RaY6d{SugRO&t{F|iFs1D#jqwI@bzGZH&z z0*9Xq9QsS-`c}V+$&t!xtir&O-~kS)F_JMYI#xxk6|QSl@j_)4Fnigd#B+k+647FK z)r=AqV6i49)Mzk^G1C}mgR3D$uwc%69WVz8vsR3Xa=@KPjK1W`dZX@2Gu+9B*O^+{ zAFoBFmx)bka}mFq3{^iu3tV&oB7Qsqwa*J6oG`~@C6`klRfBju&8>c;8|>RHh!Br;PmH*=-k+whMaK0Z1Q7ayA^ydmLZa|RMrX*Z@;e?G z+(6ATm&5>=zcy2R<(o;DXUt9NCQN3)5l-lQowvEn{BnRnGv`MJ`mx|z+=bEWgoHBH3d%gjd<3bw%aPSs77O0LZdjv68$ zd)=2`q%kz09^|V58aN+i0itK0pA%+%QpxTGM`vss0Y}XpM-6!rO{x9ri(H06)Pn?d z@yw%)GBodODm-vhOGWJq_Ku)Ei&or|uF2OXU&5+CVBaX)H&Qc6)lYmI@WyYsOIu5f zKW%WgZPb5UVrg3+|MB|GZKd(IOHDYxz0u-nFx}p=*|XX7$z~JIqgK+i24;-y&)McK zn3}*QJ7mWmS`)E0Q}YTR>K8Y4F&&Y1Hs-+83Vw|3Q{g3zwKABZVWmD4Vy4w{BYa0J z`3zxZv`CKjGkh4QvJiOHGA;20d)W##Zl#oJd1mv5!_&dqxaloleWwofqJm=yowhV}tsl@ycKF-zdfr*^hfw z{aBXjhn^s|CwGFsy1jqI*$Z1Jtn;us>peHRxRxWAPFPXG_Y;}Hv84ibG0zYCt-Zs; zTWP13mYiYhFIb564~DJ3#Gml#G8%{nrV+6MBt}A;oB~sC!I!{G$rz&e>FBFR(A2CdK zJYRBI!mB0DqnOPMEtEJHBB+Ir_KuV#6NmoMT1eaRHreKcTU=a!{ffBDcGLoQab#S! zJcu(}fZT@tO5;0?fwXEljV5I>UI`Xd0LJIo1k8jZ`En~{`mEe{{onCV7=rd}CE(De zDZ@wB53GT8_42xUX>Eg4xaF?B5n%tUq6bz>WW()pTDxi$W6(JW-L7ZNQh*}f?ESP1 zP6YRsvmRTf(uRS%X)hrld%&SU_*y>d{Gc;XzEv*YDwS;mKI5)q$HRE)FBuCJto6S9 zqCi=(;IR;yIU;%r2c+k>2do{kl>yR0p&D#OAZnK^FIhY8+p^F+BD7G+C%#&|iZ`2_ zEtR@kB~8z@7Qq*FDW#RnU21K9*j?(!MV2;);p4LWHk;wL&4lwP20$MxV*pq1W>$(9 z#FVMXyr>;MyaEa3UKV$D*%AgShZ4kmVvgnNsS~o-czp>6u3pnKDc~hmPXVj90t(`u zNdYghdJ0&*6;KfWObU33)lsx(_Rs9vR z9@U!g&mQMZyiO6cHN+d$|2aLyvL)27W$8B5C)5&?tYK=BualcdtyXMoB$Jk*gqK*3 z5?*3CO2PhWPQiTKhQ;%cRoUt7p-N&Q z6)d7;4J9s0ijlbELP^YEx<@GQ3wJsb!H8T>!?@>(PQL6TiEiIo=Ln70YP-r8P*6o9@k|pl{APQ zl<{p!_E1I~Uov;%Je1kf*E`sEeh|;DJ{cbfDj9M{-c@4RMimRC_Y6Z7L1GJyGw#90 zR~&_e2r(ZDNdJP#P^A@cy+qoAGm-zcBDC>L#;xgZyL@ccJBZ*)%Q$u3v|%dTC) z!s45`H*%-W%7u+H2js$5IJOLwdgM|M+)j?&8k6>)l-v8}OT9sNMZn!AyW3{HpAOy{ zl=hvFJ-zeplMj>POHv<}<>e$l*dge!XWwhd_Vk%CB0La^#LR0lKC30?)(iBK7Xq-7L7qlxaMU+q*nyZ+1 z9YQ&}eBN~s?ck3?rAI2;BInXj<+g^O zZJS&`qgk*k{A`P5)TX3eN9J8e(bk&OhikK+OMciSG^8e6j(gxIca#Bhk!&uStXVLZ z-nXS+P7*8e%|hNs7gCKNVKCvmbAm|={|)+}YCrdLL72_1w|o^z+z@vMj#Ph}=t zZeVGS=3~lcXc|~7dfD{LM^~(%`I?GF59}{!V5^@ouv$~GhemiS+QSILAL_-oE?zw= z+sda7$hJ+Fjlo3QMAh5%Y=j-}Ccl$>bzHVrOby70Vgy{u_StIL=8?>vr-z!99Biof zBVB5!wf4EZf+|$3PE8R}OYpLnp|$8yRX5lZ`!hvlqaQb5*Tn$%-$sZFAUpQZqLP8| zptSL_r2>>5@Wbh4;x5kN$I@$L0$2j@5sBZ8^o`f(#g;w#qPi?_&pE-H1OQ@kI5(MU zvRo2)V~N4_lJwZnCKK8edYIV6=mAP2mXcMror<;@1X5|+K~%D~O;5Ld<)c+a6-A%a3F&y}UNi!#|bk9?mAi}YUc ztYRFgzvm(iov!q+YMgQw`yI`7=lP5Fk`U$W#f-w_C)$`JT@3?xa2d+KR&k>P^d?m8 z&Q@^%;-PatK|uB?wO9#%YDGnbvm29qFT*)Hamj^0CLw->cfzK9zm?Bl{oa0Wxl<$= zQh`Ze%Ws|dBF#hkVz>Bnnx|;K75{;5 zjD5;Xf;q+!Sy&$j*k(oQM>hd%1n$V9=H~ znJL@Wg>@gy*+%=1!Q0CjX<%4-<@Ja72!$r{ zkkYJgqn`o6Lq8rpi=$n%Q~#ejUCJ)~{cX>~VpraF{OUoexc*LFgVg7pNC-A|OZ$$< zjYqDwORm~Gxpj!Ej0oPU1}03!&gs)LLvl@*w7Gk3OgaYNCd2ZvSEQo|+D&1n$Axz4 zvqR7P%8Ut}bk(V6uEC3#Yw#4f27jv6j_HFlFUp%crA>S0j!8#)rITmnBj=>UL(Y5vrj$)XHzSXLCOz|0b4cGq_fF`~($6`Fb0~5GfJt0k)HfOM)1z;upJqcV z{S>ts=nZ)cQD~4!BE>BgHr};wh3A)w4gpJ>=)b4dyXH2=(A|1JIe-G(cRj#9VT_If z0j?bFJv)d!d3;4(T^_d{{{`{i=p)j5y4?9<9eq9tJ%o8aKiJ>X6SDU7yaF9Exq`!0 zd{2*esIR9-JcM#$HznVtWSo+Vl)O&KB}#suk~b(RqYty4lKqtIrsO~9#%VhHEuGC! zLY6Dy7@d8K5>m;Ew1Y0vI#MJtR3wo}qz_ZPLkS7h{9~o9caah+{`+nsc~f`K*!bAk zDVWwhI;KxIWjv}pE||(5UDQ{Y8XvzX9MhY!9*-GY1XK3oUHWGHJnGP&5=@Pc_8PKH zYaZ2%>yb9cjSHrlM;8s%riMo^3FRn!OlT8K>5sbfuL>q?l#7PvOzDs73Xn7lIDJ%C zioOmBP%jFP4+(ju>POph(0D_WfP9bMWU6|UY%;YAkCQV^9)UjtQ4apNc+h?wgfQLX z&+v%vvRH_Nx^dboquiLoYgn}PV3s6QJLcf=6F)SoXcY)*f)>L@2yDz`{O~3qI-=6w zlI5)t<&#!W9+n)4MHo$o#@?MO^o{Ey0(a}RRC-XLhE-}R<#10Qj&=wos$R=JMTCvE zR{ofwCrl4YX!!DAH8t{46Du=*KtW4VNz9=>6@aRGcdFmM2M|jN!6J?VNCCNkao;Wm zvWsN>k!U-&5K8p+`})KIX3pR@S5)xt-Jhv^_asV0e((M?@;^w{*1Uj|tShi@L!U~> zNDean2^AFpIx;E3pTz_X^^yRQ3cwOo$kU;C9|_d}z5U{a-a&s}SOh_7o)=NgpIA}B zqTdep_Vu$){W-hOpX?XU!7Rk*4BsLG&-;k5-veP>YnT)=_ll=sL*nZXB{zwu&!5F$ z?WIdGWbO5OdvKTKg$(GKS)2I!DbvR$18!eI??j~6#dgO+W|c;fwB2k1!xr1W;Gsfs ztS3TL)(@+}Oj+*UnB4s=_l3pWq^VJ9`!SY#JeZgg zNX(TJbLSKDFSTOp%uzD!`f=G0%H|xu%>P;bLfWBAo!CV)TmNAEjqzZy`{wZ*$1htK z%tgWYM3z-r`BB9O6_iCGMNBCHW43I}zH7{3(^AL`PJjpP=|S+wZ_;29h*hkhgw~28 zA&JEnB%y?t!*7&ZsT=7Q%|rY}&`>e%5;Jj!%#EDaY!m+-MetAOYexd#PlA8}6_Wo+ zm&pF{6U(o3#s93!`!D06Y{4Tu)UlsO7dEB~xeqFJM(bt&9Ygj#qxqX2fMJZ!vB&st DGi%J3 literal 0 HcmV?d00001 diff --git a/test/e2e/stress-scale-tests/tests/__pycache__/run_stress_tests.cpython-312.pyc b/test/e2e/stress-scale-tests/tests/__pycache__/run_stress_tests.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6ba2710b710b01a2ee10f64359818b64c9b183b5 GIT binary patch literal 11283 zcmcgyYit`=cAgQ3CxqdmY<}B}=mWTFZ7~Wj6%P8A+7+(9R4a zYnE~qG(hAwh2>@eDaZzqgGKGY2I`^<)IU>nf4J=*QlcSZq5x`}Kj|-T5+JSHA3gWZ zgOp5JMjLb_-nox+&pC7NoqN7H=l-+PX(!l{$C-pKyU;{ zMu`#iPL7bcTryIE@1_wGzEdL<+)JY7n0drZsywMyS_!mbAD2xxsb$5kr@9iqJRoZG%+^D z3y>>6nTUszg22b6vx#I}y2OhTR67UHUkFJOFT}M9&skm)B4P1lA{ynxQbGuqsKRh+ zf)xF}|1%JJ%hVz{^rVqGdUQa>fL_ zY%fxzOa;6gdiw;U=koVdNdaa430!#!J*N-)43(n(kNO)jXUxE>nEvJhtzND_m8YCG z`6;;t=65tQmK1mv&3Pm~#!3lxG!o_6P@H4KiMW`E^77%9_K+AxOT;g-0ru^d_DLZT z=0(xZ{SwM8?NR<+J{k{c?JeywXbX)2wd~*6^(Ca|{Ip`#v``$UK!*X)A<7F1E%MSC zBtfyAN(eE$@)QOtjgo+th++?MoZiPdED3xlrWO4bg$C9^^kj+|YpXD*5FFQJxNvFI zVDHwR-5@B{D-0SDtu_{4&l6qIL^u@f8c)RduE@3Vkid2EqhcT|Mgp7=d6yRg(C>Oz zXi5xhtnZTGo$BKE@m(TlEQ$dU`0xQ#E8;e>OGw6pYGV*rcTP?#PI&y}WKbI~p$1fO z!%zGal3C(ktnRhy9jnzHnd*-FhgPZ&%rQB-VwJAX(DjRamTq42va8;XjJG4bYjnjs zHs`|4J2UjorOqt9ca3(f(%ua1UEH0e+kUVRz^rOlrfSzpRbcjfjw)ZJd>P8OcsWZo zt<^NF*7!3u{&dIXm6|JQsuH)e8H!Es+?S>HuTl0@sxm`W&L^Hyt>1%!MA&$Fpn&EM zZx0qo7YbkJB3ncLbtZ`jF+)nm=8%G&m@_>3YuCY)vWCPcXXM=~K2rAN1C3x(t0_ z^9OzKxP#`Uo5-NZm5-5J#Tfap@`1KS;Td7v{Z;agi^(_}Lq<0s@KLbP9INgDEZzg; zreP^0NNC6IJ2lLn93E7OHxMaSPM8kD_Hjzy*L(3Qd-3u)_U!rIz9a0LiDZ;xBk_n7 z2}L6^&w?S?Z0HqJC>nik7eNMWNR#h|b2nt4FO8JJ9Pn=0_wF;E|zRFRpeFz_Ts$!}9_Bo&E|CKT%!52GvziVd*~O4Bgb zXbV&6wYGvGyY0B;NLLJG>A{@Cef!w0V~dxU9gR!ez4+buV`jOvca15# zJ#=d*U2`$Z4ClIzqz6Yc&0}-@^Jg;5&h;1jIF?&a8htKhnajD(!|8#sOw%~@8Okv1 zH@eVMS-L;xu9|a!Qe`bmm+rlN_wAL^y|bs+t&R_FzISu6a@o3Lsrg>#-Ofj6mK#qP z+8)c&$8+^*aMv%8QS;ELYUpF6td2_s<806w+`nT|@ z^plzC|K4Wv|7py0=QdJ`n`s*|m|(UYg=QPdj8 z;Ol-R2vA_iH$TA^d`n-6Z-MdKJQCZCae4kp|KFfR&o(+$6zH@av>+#nnEJwR$M}OS zMpj)3+rf&;4Z0Ko3!7lY_G@H2iB%gUmcwG+bQu!ODf2eKN?kqMNvz%=v2v3*C>jzi zDGNx%Zx}!)WD7Q3V+2?8ImvkeJk@RjPj#{n3@ba>b)MCX;RaMB`(J=gTagNNAoY~I zyBG#208bm-A%YM3Jhwy=0f0~PmLLThH)x2Aq{xc=uae+w0Z zt$N^Q{FvzRgf`GT4|=vA0l6OLQH4^*Qx%Apy?XNtn#omq(?%8iCb?ih1NiA>Z7j8t zI+lS*LUzC$BE1pX6yRHs@G}@h3A-RKAa|P(K#3J4osg(737*SS1&n(z zKR4REVOX_cCK*Jh&!(^!nEI{4J|tj25@3xaFc4DgkkMh31Na&R{4hqzsj|{2MWiW4 zS1haID=bdqDGH~dxgUOF1$>PZfUBQnsiQ9gS8rtLlSRN4z1(_YBT(u-kv<>FbX=S3 zPgk^Nn6}*Bqfd@yT1Nmloy#yyxxk?>lwn;zyfoul*CbWKObvrDI}S$6vk%KofpAlH53$<0j1JM-2>SH{z((^W6K z+YP$jtfvntawOAw1?cPlw!a7n8(3~Ux)Fp``c^9gnaaRQ zW!IcNN4sykZn@^=r*xx^Du1!;@Ear4lVuL&YIm;I9?aApT&X=YH)<;V3@z%1V z#ptm=%N)qnHLlj}&(!T-sq3CQlWX0ljiuh<&1FZk(P2-P*_-n{4%{plRQll@a^ZO2+W3bZyCXl>r8HF%Ya*jJ7z zu+_yW3fxRv@Ni*0+@Oc%@B{Zoab6ynRpi9nbivqf>`hD{lf3ED*Htg^OYndG0@mk@ zWyZRNq|%fHxRw>_B-QH-Q&!0c#OU)PoK0D`k$PB@O3&C*G*^z1boCbUD^fPD@)L@y zD%9Y^#x1C;Q#5E>ld@r*3q1WF{){5xm0R|*ZhposC=2lXho4b|?Zq{@Qy-g&Mm?9m zbKaB*B-N(OpAerGeyf>M=FyU2qKVi`h~!iWNn9mJ_?ss`6mOWW5>w>nmJhqtdVihV z+Rw+4?}rEX%49;2a6m$2!GVr}3Uaz2_krwex!MwI;ess#EoWPXFMR=_EdE9W{27Vs z3IksDU`&ll1!Z&Rgb0o#a_U>OcR>Wp5{Ywwv(Bk>VloyBfu~HSI;9(u+@yQdda>u8 z9mbA%_7Xq=c335~%Z;yXbJ^{(9d`kXeuR}vpv>Sh^0tUjV?m`OtU}r$1ea8OT{6S6 zy;}YV``pyIZ&bDcUIVubC)(u_78pU-a(#q2ZHN#*GmhE#%LKF^E2em23L+>zkz%vR(wnx_`rcii`zpHf{=S5dXlbEjwF=$)fW)IIlI_oJg<+8+PZ?-=&g7(r}v&;DZem# zX03onj<(&l-m<1W2eWifvBG@E;?>2WB`NKJ2uwhunBTo9eRlKXo9P{$_qnuZf0jP5 z-sryn@KWFXvip7admc5W+3u$` z2mfdzn)W?oh|;nJ>mBQ2G5XQ_dlqk7SocC@6amLI!;He+L-|7eFy8ZsjCME(77?|{wp54FA4 zfhyAzn|Ywz`lQM{&|nQiqj4eFfWgS`UIC17(**8nb#Fq-nNZa(>iGb3l#v2Xu{PNjoOaVkHq@QkTA|&Gs zO*4Gk&)5ohpzr%l7uPJI4|IH`Z2G7nuCU@-^e0Eqha+ZeH*M3c>V1lwA~dkmc1~^p zPlXY~Q_l))a906Zad5mC79x`<8YVjes#5~N2=EP{H7OwcDm0@6kSHO%hLU3_IgS!U zBG+C5dNwgy$QvE@0CeIvDc10KA_8G7h#2ytAwX9OHO@yTWjYX0BSLbuddEi+f@;2V zIuwh_4R91_9C4BmK2lpW1cQN_zeJcsTKo!W!2}?hiOh8XK@SKNVbd+w_C^GJZfF`Y z65#-mj{QX09tglGhX4W!av9`Abg)7@2mwXec_uNY9$7&^Mz#e4aN6QpLgc|0h8P7U zA<4^CeeuvW2nlL!`f(RO1RO(fvLz8$^@H9JG7Z3>$UPT?NL*s4#zT7)!R^n-c*U6~)}!_r~RZg&I6}>b&q4)CgBm@-}L-!lFSaQ9$Mc zpLtOz1TymgRCUOqAffO&N=~B$0Yk;scO%T>H%Kw_=&=+NfMdU1v0zmM4YR_E3aaOW z9LSHSyVzhU!Au4*ly<67O|3~dg+%Zu8AEc&;&7FUg(7j`7EpWvKd~9cca~UlR<1gm zGfser7qiac*;6@mRHiyZRWCZfBgj4H$@Lvg>5fy#4^#64QDPn-k(b7y(b&_w`*L+h zpP6tgX+E}Ybay?nqx-J>TcP-&hx%f9d>v&)*&Wo1?RXYm@~+zTcD}kcjX$pljz{=ZdHE zam7>W*cw$j-@FjG6Ik+Qsa@-36|+NYrIicbJKm4#W=}t(>dd8vLk;;DkyH-~;il`4 zXB@}Ze7o-VuK4zX7mKN0Wp)4rUYyA?fdaDg8OQlGZ_5(5;@vfOI!`8NnGS&c{=i?` zzOX%TJetU~pO}L%adU=lf%?{ikFI}tV5MasLpRPb^HJR&SatBxuhS=oGd-8mm)}Ys zyqdlmN>^OV(qUcAs@A2{QaoMJouv;Jt6O(_{^J`*#dk25_|ceKOuG8O?}s0k{_T}V z1L?}wo={Kevuke8e?2n;i3)!IaQI}yfR}jUwGFyWPa2yB?WTXUn<1~z!5~^!!Jxky zzbmk&r$u23Eyj0H@+T-ks8D^kY}Y~}?1%6ktA_y``S`r3#{X%<7mRSBQVrW%RP5Bh z-opCvP644pVFo40X@jHk`F9O3bEF%3SA#)?g0FJv3j&9f)sHv&Xe_}cqx>=9Bj}EN zLt+e)XC{&)zqb&i{VM|g-Cq$7`2U)y|B7%y`ZclpYoh6E;=p=c!_w5f)ZNtggo|wZ zj%_8&zq>#lf%FF1OtyY^hU_LwpT$XnV&>#!lQ%~(vnM|ocyHj=r9Z#&l&XM*Ox0p_ F{s%RGAKw4~ literal 0 HcmV?d00001 diff --git a/test/e2e/stress-scale-tests/tests/__pycache__/test_access_point_scaling.cpython-312.pyc b/test/e2e/stress-scale-tests/tests/__pycache__/test_access_point_scaling.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8c7a90ccfbac2a7d54a3e44b37699af5d49c2894 GIT binary patch literal 16665 zcmcJ0X>c1?o?kb*ao@y2fF#5gZ%E>;gOX+06ffN)Te3Z-4=jvsiJ(A&>INkd3wl+K zKhTLLCTFKsblJ0}vy(NwcGYroR5+ES*5}x>o2sBqY5?^p6;;Jw^l2%HQ{$CCB>#Um z8fa3WkI8IHeErV74cm3Z5W&)~60H7x=pUb92Wxnh7eSK=XsN0EkCD9G$7x{c{#s23jQ?b{SK-J3sftI z12R=BR|!;YTwt_YGmJu%wX=>QZ5`_zCs`NNc5hL;f+f`2tmjRd_5R#g8tb(~G;!^M z+$W6rP2%S|P1$9dm?EdhdV;HJAo%LxYPp^t!Wynxt+%nwv+&J_tDmc$B9-@L@-lIS zdX~6MhAZ#j_)mp(k|CCebK@y^C&$ml&xc9Lb&^Z*@tDw;OeDBiD#^#R(=L2X;{FKa zUMDyL_tLPAqge7)Lj5+wG^BN!w4SBl+Z%943>#TJgeF)y!>i=58Qw;CTUZn1T3IuM zHr4_mtSoDV633J+?399MIUzMLc&0ZN;{@S!GCrO<6N@I|F`%AR z%^kI=vC>@xwms&T)U9sVav7(K!5FTqVA;yR#KL z7tgHPJC~lg6%4(-?Tu}VvLlRmXa^0(=5&ESp!SPN!G}kV0>jxE^Edk#9HLgxmW+X_6cBH87mC6PmWc?r(z~-xf=brDa;t z35kLN$$l=L;Djk5#f?SctVHeW=z#N@n1l>oGVt8_cyc^UORfn%c_}VH0yg`}B+E%Q zY}zWDru()TE#Q zls5(hQk#K;;*lYeNDj_zEHV}ykDtSXjvJU^7(M`R-UmUtee0847Lst$g};H|DzRp8 znFq7l1 zeIVdOU#Y1j-VPmZr2boP)nSJEn9)Og0|?;S;NShz5L`zHC|=f(QFz$mMGV&{)|Q`g zS2kSlh4yecQDHhA>`QRbab|LYd5Hsz<`^y-8(|QCAK0T48aTd#SHr0=B~<|0LK8q; z0PN%>51fWXr{ZIrq!$t#H}M|qC4ozv6R;!Yqe?afe_^wYB-wP+)?Zg!uM^q})eu}I zZkqr+5rh|;aQMpe{b|GHMQPZ_;*~O5 z2KZU}`_tXX_h0){8dh@48C0s{F1@vv>NK69zhy1;u$KBi(puu3xas-+w13cA>c2nT za%&0Lgr{}WQxXyYaSXtS6j}AricM7^Ia-D-2S$X(OMRBS_LM4{{eg5Sq;u_?KMIxK z{ZXjAsZt3W(=A01Dk3X$Ii3PI9^shtz($WV6Fhe=enm2}+_~svA_Z7$M&3l2h{m}0 zbTae7_rW@*-7+m;91z+klI#IyW7g@X`wlRt!VVq}mE=hP#}g@1Vu8_SC4C$i6oC&z z1(~qqW4Q#d3!Cd163O%DfuEq`<8Xr@Q1NXTv?Jx_hbSX!0-O%;@t2Z`$uW>%64CgW z6fQ%2i>h^z%h7l$0-Z%BF2#fhu2Do;nTSG#4IB7JuzM@csZb`-Fe^c#aBegcAkHUZ;(5H88jpGkrfAX#HHe7eeeb~+Jyr=T|!MTGuPq*mlUXEuy-7B6WGyVC> z>g(yBrDv(UXWK&kySv`mm8$<=6bg$`*zED0;^S12`TP|=w3>?UBZ+N%notoVC zE^&KTzNvjpPXr%XC#Yc6x`_zZzHNQOnr+&*O#Qv>m$qEbQL*P}cHgmV!|@w|6L+0N zpl+>3JCGsKGn9SuxvXbs#q<0R7|AIyaH`x$enkGDp%7+!&eSfN+Lx?1Ob5SSZzXE# za@D)U>Rq|&{bKe0dj#nVNr9R*3f|v*wV6vJ&F-5v7ZjS^->lb^YkTwiw0$5X;nTj- zOAS)QN5yw z$}V0$r3$P;mSw$~evNX!CT+hUNt&nWm+2{s8oM^3ft< zty5>y_0L3k6cZ0HXMq!EndD@OL6F=)+_91PVA#xqLYz>pfM!|Z(_R5vps#*{2PF)@ zA6sq&d%gqm!g?8bl^er|asiA*0v%J7Pem}vcR?)yw_yo%LDQs#bQQ(FRSqoIAcCa7 zgaC9qzTov|=bruZ=VmOo92Kv}UyJ{n3-iZBN7G{EmyV7#oyE@NUEb@qIa|)vFS`2k z!BBo%Gr+8S0Kv^&0XXaSX9F#Z-bF*U?Xj%?@l{vvEofqyv&?t?%)agKK zEpKqXf5E$8094Ob^{jgL->R-%*Aw1)5E!BA0|BMurC#!I4e@bJsIQ6oc>lJ(?bIjR z^$;(QG7OyPe^Hcq?}wnwL+o=IJybvOx;~~m58}@Yu;Zr{Ho6STl+#HmB1uL+r@K~5 z6VnE0Z4g|kqOP_amQCq3<%U#Iz{nU>6a(}#YH>h=Y2~m)W0=v@gfF-G*#iCE(~QR~ zqtFIJ&6G6;n&g;nT7MZd%c&v*U5;%oXM6%2lv&^@XkXAn+tQC93MrVCNFWRe<+;wS z!R#>1F75B#&|2ZgnC7&nnHh;*;x-`G*|ZHiRnaN!SGrZuaufEYJHH#-3L`G|>1^H% zb+!`J@yB4e{2&HJ-j;s~k`k5R#>)X)*r;IINo?~p2G3w{8iOYwknCg8RBQyMBQvA| zS}0_2Q+di0kdZ$4Ej**sSPUldQi_~oG&|D9*aGHcH=k(-qb07%)I zuBAtoTCOGsUuD#{*&D(x)W-+wVxG&q#lkGW`_5Wzqby|yy?sB*oka4m1 z!<`?7zDQ;rFRq#*fQ$cp&qr+k6Y}eI+e7e3+G}oBRs+*#uX!M#D6zDoua)?yD%96P zf7C-`c+dkmAG=y1{G>&XVQZ-0PJL=4A>5!8LIV)qRriPC-jS6;kl6gNq^y3{VK`0d z2vB{OVW3!wrBzmGTDN5_y`~nmWi111EYhOm2zW#}t-@&0WmSJdOM4!P8sp`ecI=NcyOJTYz5 z^j`!{&??PB4<^=~F`>3#+6;3mf^lgc8M6wtB#|+e)A49)q}*!Da@vE8#-^Pyu@-sm zx-?ugYy~oTptCxjF=Y%;0xTpl%8+wbQxkHgEg3^bhb@&_5XW4rW~DqC3#@`SV};PC zU5oD;9ZRQS|CQ!@?WZ&W>u1xnD|4x+SH-kFg*s#T$rLT(u+FWDEq>ZGZD#$2)po#Y zm(gKm9I%>&T@CAPE4$tf%?R6RSi5Ow#s>X>c4r#t;#S)9WE^be2LY8btZHS(3F`;n zd)j#|L=)d{h8a74muXkVm2qZdwj8Jg-=Z>bm4m|pj2A}1;3#CqqiWM2AX&~F3uCML zpjsUpm|*K@*l7hy<}E2TK1@AhE<=eJZGN~WTT@y?&|57-kL6D`-awn4(#f87Yj#~I z;KbT@{~5aI(2gG3SC_3M2`EiKZCY>?^tO6@4k5h^M9^DIg?y%Nc$p%S(9O zMud2pyMvoqvIEZ$or(w?SkXbn<1C0BYDRn)$fS~4kwKyp0$8}CS0WI(gR&|Xa1%1M zwDpW)TSoRF?O-;HGi~leqZUpk?cG?UFih!c0Y$Rt6pCl5%P66N4WEZ5U@c2mY%r&^ zg6gT2$HAvveaSIcm=uU@^1K*{Y)`lBVkV->%As)Bd-^B?gg7P<9|LABmP{tl>;UrN zLv)jD>z)4ubirnUtO{6cOzfmG2+1f+%7%2wqg_`4EICnjN+X!M9~x`AxmRwfd^7ob z;X>d?g3+kskB>8H6I1vMVV`0Pq|Spo0qS)nr}71giz4ay=s2hjW#%gsDcB5QWfYa3 zc8rn%oENxpR$=4zBQ-NO4(dZ(70Fs`3%WgzMwU#ng}EVX&ktd+NoGBcdB~qhb|4=q zjxAiF@O(6uqg=^HF~B92jG&)|?F{<}Mu7WBGUK3S;aMW%l1WWTW@V>BHOZbzrlN_6 zoB<{?GpHez7BVvlZ3=10Dt`@d?3e^GJC6O?@La%h%kr~;Tz`SP)VyTnr4+;uQr24{ z-Hj)+tZz`;mW^J8u7rPvT?$4jx9@uOTy-|oF1kCiokODg$czEp8vM-c)7OXRh8Jv$ z(G_3Ej3pnan>V~|dBd{s*i!FGVDF4QUr{sLbbZI%j%=uHiCU@HJ!8CG;hiz&!H#Ep z4J>;NqNyR<_{gg1AoyDZLi0~A92Y|!OHIo`v8(UHsMt01g+F`xIq~T8xyl!2hHly1 zv&R>V^HYnCY{kA+TMx9dSIi!rKR^F$w&}p~qs#5tqinWEVip3ct{p%Sri+69qpPkk%+lwd zC+7y{hec2GV&{ry*OGhbndOUr_uPN1&2O(;c;X%3;sKG_GkY}OxN9*jHtt{U6&oM@ zaEI7%V)mrWN$*;GcJcVq#pTelGdq;bR!yvWFW&M7t{)Cn3b2#si4rnrKTGJWa!8}an#N3ITcdzK( zn};TKJAtBa-|q;@=a*^UuP9UdnCLxLs(qX+RbTTGc6ZL!E!w)XyN7;bJF?cNBLXjy zYlHBrQN){0v9dGUb!4UT=#1@_$)0uXTedB|{NazYj#I0qAKiBPXB=OzHxhwuxylx? zvL#o!N37ff+Gcm~rZ)hxg**7odd0UDKM)$=2z*{2I@(NsVG13y(7$dD9WxsL!9ZeY z2_4&E{D&|J;RcH>sw`xAV*dg7vW8 zIJzMD-lAClTlBQPLB#bh}m9^_GzH9*qytAWANXk% zY@bidrcdw~i;txnH#v*7D@3nk z){mEYupeU#;8J5fmJJFLJvj+pa(WDROVkK=MdkseaDnu9n=BOOvpU5`V;45^MluO> zl%wFoNOY$r*bn!^4I6H8OLsPP1TknFHWL0)j1V4-9)J_fE`E1DislIIrZwEeo zGQWu1L1oz#V{;O6()+gN%!+(%IiC0?G!}jhA`IG{9p?Rco9p!xubs%*9v5wo=Y7F^ zRXtq1_FlAS*d1`S1C!K0@0~Yfw>_S%=v}oPF19!slZPUISPiw(P%o?MEe93Fm#=Vb1;& za3q8$@SYwyW<`;fu~sE{RU|IZDIzj z)Mr*b#6RyqDYh*!L2kn4CdI%w&Y>-pevrPxZiV5u$G4(xZ+!v-?c z2SF9C&k*6NbVCuGLF6fD2hhU>Atrt<4q`CaAb1o#Bzd>q|m zqlpQKq>b$_#mC!)k+iNgmOjqKMv_c39w!84;ei~0dy?6J?xI}k$bd2H8M8>$WBtHf2 z@e`AhzO$GULCAp-vs ztZ0^fWDup*G5A2S@bF!72iXchdP{Mk;1*EG2~r3g8^_|~^3^c}SjfS36i>2e6r~CO zZ=tDjAA;VNfPN69C0`koO|oHHgNajE&xYcwih2SKSqO9r$f51x+nn=@AeUslfM&wq zLVbiZ87qSMVC{l;-hpyRD@qDXcoFcVBOe;g)?B#F>;jSNuAXZ0uMt#ZB^>wW)_i>! zavG}uzna^WM19p=tGUj9*W+O-)(B6nXKe=&3f~UYFZ>ulrp*r+6oLYMjRz=n?Y_IO z1~A9G9WiHZ5Kg)G^~`IToU>DOcIN%Hd3Ojv#?=KF8g@F6tIIWSQ0m>$Ry*FljG=daTd@0U=)Ca zm5fh~Ul<3q+@S2VvBABf3fph;oCE)Q*<-!~rYq?uz^#MZ%mAUHOZBLjhPcN1_#;Se za--k{$V<0=3p-J=Gpd@gLwMr;JG9$iSp2G-%Y@Y)%ER@Wm`mjRkBa_B^TF+k%S7cN zI0=;%vlN^-Z{_s^a|d#sdeKwAF!iOU1FVL5UjS7^IbWmbYs`iZiM~VmYG#ch{Ew|8 zkMa7k*N)9zymoRuRK|1&`Vg1z>vd}x3t{MHCHjonLk|SBbu8^Y+(mrU9DwlSu25eE z^@*E=5WKhVp!EP0q9gcnN~VbXB_a{Y5{Zl@*~tV4aa$yEaWa}feiBkvfLuXP+`9Q7 zw3C0Cz#}34F$_TBB6yi;MeQMf6$51Fc!WFoOT0(Ou%Lh-n(%lG&;}+v55cQMUhlfA zv+5h}Q-s02hOxCOh~3py=m*I{CPweIP0CJ{fJ0CYT6^I=1mhNS+g7% zsP&+2QOi)yX+y0CKkF+(il#*wS1wJ9B1makR1~LFjA&XE0U}kjjcH1eb&{VH3R3c; z{bdC(Jt+HadP>$B`28<3MrAYrY9DbDe^3QX5S`uz@UekG*`QJv(mh)LU6800ffTu# zqLcCR|hfInxm^lx_Xrk-SSl{kg?)xzB>5osfW^_*9+_v13Om& ztyc$EY2PhhW1%V%vScm&tMovgw&duLNQdT=t8^Q%5$4(jLymb|WFB8(4#Vy5G+Z58 zGZ8k=)l>4ZL3kJgE;ABIJ1mH$%#(`vc0eL65_teD91NO?IEH@(=ECDOH7FU+D5e#D z8Ioz50aJ*qyiZeYUFRfHNShf(RvLgO_6|jR04s(!$hT~R157w1878K1(RnL&)`P*% zAdrmcb`8xzcAXL=3;r0C9|tRvpnPHc;8NveDxMJ90LAdP%3x9`{8(A}ou?TLGbd!< zZ5}TG{s;y#PT|LrKe4g_29cH-2O;Ao|CgAyz)#c0VegH{IEiNCu@wCN2Ui6Wz-IFa zxlK_uAbb7sZy?=|FnAP$0M;=bR1o+O{~Jic4k3g-2m#kg^0$Qjw}e&w@AxfY`QJq2 z{~~t7SF}{mROZZ8qPc3mXT{ub)$n_IKu`Ma5*Xe)W$}@owKjsb&7{Aif{3jHMJK*=6On!UIyPIaWOEPWv0we^whaAjX3@kyh;Ur&9q!`)9%pAs1Ve-~BO{r~^~ literal 0 HcmV?d00001 diff --git a/test/e2e/stress-scale-tests/tests/__pycache__/test_basic_operations.cpython-312-pytest-8.3.5.pyc b/test/e2e/stress-scale-tests/tests/__pycache__/test_basic_operations.cpython-312-pytest-8.3.5.pyc new file mode 100644 index 0000000000000000000000000000000000000000..857fb2da566c60080a3459f933d5ab70a4147d91 GIT binary patch literal 10279 zcmeHNTaXi1dhS-MrBaBg7@Ul|1FZ{C&>p|NMO}|M~y#{71j@cz6oZX8P5+Z`vv9-?3mOdo}YY51A{J zL`igtnx@IqmS*BMno8TJS=xBxn0Ami&S@uka?>0jr&y{pNF%7vB7qp}b?O9Od$r zxb7m?y~6d}3wM;l1I6&b-SDP!%mw$oYVJe#w;o_#_KZGOaS1p(#IZc(O zsYSMz(rRj-#7)zXnkjT2rL-^61v*LW+S&$^{p5Bq4fGjMNQqsde@b7NHJ%If0%a)Y z|G_uUG=4L}<=0Q9O8^iKS)T zg;dSO;&S5Gu&Prg@2U7VsQVF@f^)2gBsnE(aP&F{c(No#fd>Xc5Dwas=b}a$DV%3x^=U~6tRXH`Q({gpgDXrKU z!PpS6E9bIOzW15C#$du`>fZwSfU4LzSNOi*zxdYqw@SirQ5Y_Vy36go6_)ahtWu1} zTk+R(OcaF)i(`@|EUN*^<+;e8=P!+Z==pB7oAQTCKC$Q%OTM9^Z|EUqbA`)3|Haei zPhZ{o7e6ceL#qts@BY5Rz>Du!TmMJ=kJVx5`I*lBLF#(vgonBv?4014+gtn-EOUot zA%DlyIcaC^Z1zv=W$x@{A>Vx8a{qh#wr2XFtJNfd3#{MsfADT((EMN6^GEJ4B4v>^ zr9>S0w%U$~&1$k@Z5U-83_)o_vK~nXkg14+Yz3tYYtY0g-AF_v8-PS;r5_$DI#kh6 zRG%h;N60Q@qE%fTc#5tz@C~7yUjRXMb+}r}&s0}$Jx8Ou{wHemcdM=^G>&$){9f=^ z7VMYj$EpZjIn%iZ)Ng+e^$(r(LmQ|96EU7DgF%{CEz|9 z?CA^p3KI&@=wHwXH6#nPEZKmg1|}>vi-X9pUet2Q8pxDiX-1$)wK$ux6IY=Ld&t(F{M$(;p9!Fh5MMJ;g!PQ9%K%~naYTMbEa6u4Ph zB5h`EpI5z`#A&FrBqdRC8#A~=aNvivzSgopi6nOKC=JO0K38(2L7XBRY z$1ORvUbBJZDKNjGB=5(J$qgt&i1_mBaA%7I;)zF&O^R_vCa|5@4qeEdibrczHN&E_ z6WRHUbd!E;8y~+(=YP<@DYv*SqVn%oGvc_2xiPZ}GS#Xr5p~$`uKdz)ZZ4+E{E|3s zuy2p3I*(s87;f5iCbzf+$rzGtNVey_06AqvBOllSEueD8R6bDOc~Q$|Pl|a?RAm6z zk{Ssp2-tNW2v#%;%Ia=oaB>vBst2PBv4j={41&ez3=UEE$Qj9MV1SBYm?LQoq-YKb zs^d_4@Y^gItj_PtEh4Ik#uU9J3K%>URpgu!&!%%RMb-s)9i5p^q_jjv)g1(-t9n~h zoy*RrB*Gd!k;=}*Qsn!(D~e|Xl%RX}lX-z7*8xSzDli-vNjj!Q<=1?2s|zqxFTY^@UgK3$GXj zT}5x#)zf#qaDE^hB~C1I;`O!_ZgaV<;}hW{q0}~3Y#Y0^=Wg4?IS=rJ+pqli@{dcQ zBgN2>a?AR1xEqjG_=of(iV5@Q_zFk)+YRir%#AK1f*L6L1|WZ9BdHj;v9ahIEphn& zE98I*qg10sZuDHF<0qp16lnkvcW6AUa zq)p9VSbUb>1V2V45&JYGi$lE)91U-=n0)EU%0?eqH);<2>-r#A{s26^XtgwRuTy_E0A9*rwJA7~h#MOccoz@6 z>1(KlB7D8h>s;7mM|{+5T~#NNaHztZd0BVtYRBaU5i(r?5)(=d}W%g3g}nJ(jzI1 zky&Y7!BfReUTuddzwVnm%{uE@sA-hhTfj|Dz)LRpurkD7 z29_9WpFk`Z&s30z-KvEQjXI>J2H-hSaxO7HWiHt+TrhH0Tqe?%ZzCuz4TuDA!3hG? z5*ZOA8RW!=2E#h`U{q|3l*8($EoDF?7Oh|-kZNE!~wZ(|*_;*I1b}803L2A@R!=CC`0{li5ub?#q5rtD${Y9!Y(53h8oKQ; z)>3DUF7jk04TSio$oT^#Zz4H_EVPtQZ(8hdPZITyX1}j~%+se8zvum&7ea zam#Y|=yG6eMc8^zXjyLUyMFYu=})Ii{X2{OJD2-*EeCe52;+5#b9*tg9fX#9Ub!`Q zYkIkR9|F(U5O;dd9W=%=vLb9KukZe(<66fh_MSgk#hZjmRza>L*jaZAEw$Vo@bwLs8gJ68gRr`o6EQ@bsNhGNLJMbP;v-lCj$z-x zl-W-v;55yF?3fhOVh|wD$#LSdZdM(MeI_d{!X2g^`fb?n2ssX}4c^y=x+eS%&HAw! z88Zo5q+^-HERGNt4a^QPI?8KMci55CvcFg@a!Y z?C34!zQOygUFEi(>)Srt_35tVk$uJ9{r|YJ*!$)`j~9F21Kyr+g|T-99yqy>_kqXP z#-9@^BIR$rSU6uO`3@9)2jEayZs~;W?LUaN%FkCQ#>Fc~;KOT_51|NAz5-;{Dj%vI zD4XJN?7nIFM>ipMiQcG3kKu%+7di@mul?&FXTX&ga(pf; z_H6=9kR#%rJ>m%J1i5n@)=q1?L~@NHzJk0NInSfk8futl2`hjPveYi(jE=;#xd?02 zu!%HkGhBqGK?Nk;9#c-Jx?r?Ei~vg2UF6CJ^_4`qOkp9o;jG^6)a@g<%t6O(j1KmT zAxTwKm<9uLorWC9@Z-FMny746YEVyfW=+-eIEZ+@dS<1oz^7gWQdQ6Ob?V7eokQi$ z{<0Xk`u_d4j&j?v3d?kcD?HURRPG;qz%m12P_zDZ4_X8?E zI~;z%vYqgP?Fc^TY(>?5iKy;tHPs!3z=ox|V~v`NnrbEt6A;bq04Q94-=Az^^h`Jm zVQi+E0fYnK3m&;88uZrod9^2Sgatr?A#RC51Kc!3-sI-LK;r>Wr7wuJmQ6zB&BEhw z-D%;!{_L7D17DJ%kS21jP@0UeW+s{y%i@F z?)opv4pFxQpvC^srFYN&tmK2}+QyP^u;?4SWxwm&Rt~mZZM(d;6pR#uky3D^7#z8^ z{cdn)xixg<;N^qA{*&Lo)Lm%-A(a4#Fih!ZfC z#;|e11jGlIBnyZ;byuxQcUJ2ZJg({vqlto+SLwwX9)H`UWB`Av#v$B}wd>|JJnX}K z;LqLQr%%=2yOAF`5i>}JseFub5IY$MOm4R^y?d0uf;x=EsGEVn%`r`Xjg#?n&%aP^ d`2RK4|F6`xuc^TzHTbRlO@vI>{|&*^-A4cb literal 0 HcmV?d00001 diff --git a/test/e2e/stress-scale-tests/tests/__pycache__/test_scalability.cpython-312.pyc b/test/e2e/stress-scale-tests/tests/__pycache__/test_scalability.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..28de1b2c7f9998fdfcf53a6718f46ae1e65f5250 GIT binary patch literal 21309 zcmcJ1dsH0fm1kAIyXhD73mO`Vm!Lrn0wDiC5XP zv-ehY^-C=ePR^9XS6_XP`#tKr_q*Tk-s)diEG7c3O55HuhmI1&|H235vZNvp)<}Z5 zOwa^PMu=hgP7agOvuaoccU43^t{K*hYlpQY=Bp#Ras9BKBvixzL2KS1XzjZyncA>{ z*1bUt8yOR=KifzU>=h%FVvO&m8XGQxCo^q$7uv*&B$%SHaygzID&_YRFHIA~i$HzY zI&72b8X4<%3EK2Mb%q9{+Zh{Etn`9C#@PM^sy!*HCBqJhY7tWcRLzWI%%9y4IZj(> zYnrx-woQ<<9ZDBJp>zpND5Yt~cQv&0d-_aY=l5!e`Lj}+F2?p<;(MyBybBs)hMXa5 z3C2}Nu;s&UsT_tL#;uggt@Av*w_*1S?io_PUm!0K7uC-b7f8S73+(=oUnS~7kuWn6 zh5O{hX!x|B6zxZuC>suOeUp(06N*lM*M4p+pmp4LKWpzW|8$YCR`hbI$lfUt-*LTIK<5L)OW2(5H6ZH7`dx`ejC z-A-E}wRpzncZem=GFoq>aY$%HT*p|FRkKeI*%r4mGTyL z5VK?yOG($5Mu}zaaZ;JxI5Cng32PnxDO5-zs#j`^8I)M&uFMORD^gO&%t|bCSIR+4 zX?j^T?3t2EEUSi5Nz1C?RMIl{tQzUQM9Y-?tXxG35OHG6r{rhuN=cHKA!>;>f+H`e zNaA_mTauWcORY2Y&?Aq$uv(+=wnpT5%S>=QB9g4%CTb?9m{ z)Jkrdnh}dQCOSPeB6TNL`S@yY$;!t?$NdomuM?kJiusb}<>PCXwv}(Lm-*hQyi&PL zu9h{;_01ppjhe8wKhzOMTf$H!7^>C`)$8txgnNtN-m>a$oKw#iJ@#Byl_=XPlxW614!S0PRW*%_WC0at1I+`iybT=kw^kv+&+ zbnz_t8aY#(iODHha=j#0c|cms6bMEt#Q=S{o+;*H7?#^39WG5S1ae+85gcc}_zt{= zSaE9Ux187$c0Bbu?|ajD`#TC~z(R@!xD zjxp>w$JRm1eywN%&W0yw(E$7#q=V6*Sd=CSEt#f*EQ|{p<#3FzPSP_& zeglhwP%L2?5GXlpE{hZsb1}qBMS(acnnAD}P##4c7Y#hzZ=ZET5Nj;J!Ubgymgas4!Ry3^&Ti}{lV;mnLvIcx%mKk1 zSn1@=f!pSuFV}6ZgpCqxRKgYzY=QfP6^q*JZ(V%z;^Ou{{np)@9m$HOWOc)GDNnU0 zYdh|4?MOB@zi+y3TG=5q?!M6~G#*MewI*xouQgq5T7F)r*>%Gp)b!pnYHQv1ii)aA z7K%1XiDKtv>m_T#zFn|yzfsQHwccLi_G@BgeGw)& zK}}CluQDKv8Hx#p&QK_5ADJ>j_Y2$rhWu)=1cWP;2O$e$d2*T!F`TH0hQ}FE%S9Mw z>N>0-j){zN*oyorMYAk*VTqlYq+?YD-f4D$W~j)a7W+DJ*8s8-W$u!B%@9~OnBOwJ zXR4%d@WD}7|CzC~JAurQpJY4X&UQnPJs?|@fvs%v z!o)~wOf?0@lvidO(m-`2XIQrl0d_AWh7=_c;snmM^vZb+j^%e1_#ik{g_Zyu111P! zRvjnf>dkHf=VAQw6&W>&R&Ty;u8RExVV+id zdh)L9ZE{u<*L=mSX<^p1zw@kVH(&R$St~Ral$3eb=3Dq$vsU0(nKNs;xb6wF_9w8` z^e}7suQ6-h&6kdosDi-i+>y09b(^ohsd+A-%x}Jhym@{TS0`{mRo;t($IQVAgtH!$ zHLFr)tiVBLtVGGp+(B8>pyIPY&i|oubp1W)n0`+>YV6Paj6%h4h&86AZh8-)E`+17 z>zrYz({L=9pr%-6G<;Fi)68gaIueCl#fZ8WHx&#qG1q=3!bCwCqG)C+GC4C2r|~Z8 zx=PZy*!?gYF>AWW)FjlU>njXne6t!Vc%s93Q%1M+( z?Gzh^bAJVm%`=5-N!k(Higi%pv-sXbM}8~NDbe*f8BA_?tSstqpb}0*N6t@1rpMuE z7YT;PMSp?%PblghxeyFTM__v&nK~chV30Tf8k9UhZV0gm$3K#q@}kTPaH1*mAZnm@ zoT$b=YLE&vRy9Z-~0!`6pdo5@(5^j5yJ^~%Adsn^B%^glw`V*C{Yn81Frlg~CsVPyjL#Wx2sOb`Fx^6kT)}3X1McYbKqWyr- zejw3)NN7LAw;kraN7kH2lg`r1M=l*nIQIz7JxO0f!naNEZAi}?cu$9*PP!-H@{PG?o9fqcV4{mV#3!V_*znp z?@}5st;ubCKyVE3&%D4p238#}KDl-1;tKbp_y_UCu0dhf;LXFUyH0MRJBI}4p+em` zOlJ3oFj^9ZTES4ebojPm+m{db5O#O2{!AK+MYDZF4fo$Z^!^UncT%+>+-t>vh5HBVVt2u$uU|y4MA_Pda-m4r+-{UA->2{cMkKuu^@~M`Bp*JE&6soFp;S z`VQ8se_lsI7)mR`KsgOyU8>r% zF*7!Zdm6SGbITRw22y2OT3Xm$NVx4}kVPz@i+UCgUNkj19i>pfE44E{PPb`mNOd+v ztKV;CcVkcSeX&O{7{DOC*|CH0D5@jO1lxrf$V-`R&99es4)$p*q{rhQ^g#r`Fvvzn z@R5BM%b&n(%XlyvIs<)^_8tyV+MNAj2ohrcLixOw{}(UJ znbxf(Z-w6s|C_UmhXm`EWzQ|^whfihQh`D{ekq>t92Pu>`R7J>&*4?iD+`+BwvHc} z{)1_i8u`Gq?6`6E=JC&7{P~NCqc00bU%s{X<-c;Qm=-KNHFC#OCv7k7E8Qyr{u|Hm z-s5Y|XOrbs>vq?o`TNJ0eXI4m__|KMbBK38y=FhQZg*ZbT{11UeaG^!n6No=dbMiZ z_OO&NT6uf#hs_`QZcg&nm)8sJpAN!O?b~AT1Sl zJmnx_EftSAR4i7S`U1pHy}rIi%}+Zu81^_IzI7`xM34zNNrl|_?EG!|62TG#Q5nwEh%V*ljMj@hU6h2PU z8f+8pTB!#*w3xt?9-rP#^%k&|x zjjQ4sg(~1CX+bxZR=YE$711=S18T*YbR`DVG<4B>W`*9YeslEVI@~&D5i@ltv@-X& zRuQ?g#<(G_j~i*r+i>)bYfy2E>lakqOxzgP;Ks9{V!5mtvBr%sBQ|MX?B7e%A-~qa zI6Z1p+%#62>6a1%C;_(Z*`j#Sm{&>5+{Y?1aV1vh*aH@;66N?lYt9;j1L#sp&CGqQ zCKFd;Fji;W4DVZ`%ox~Ip8iT8ph2mdxzirHG_J)}^IloHN7lHt@Yvt7rMBftm;+nf zR&Wj)Gt`wB?M)q7eQ_IX%oJ?QcBpS(AlX098hv@RS*a0EquMaeitJS$x6^Lg=OutE z$d}GRKZ>&`RK|#7 z#9&UUA*Vp!A*Ut^vO>P%C3G!ar*J5YA+6BBTG^!`qTACY3(f>xudE4x8*V&dBzUID zyP))En=cLH{!N5-R`mUq!x zGqPhrJ>Ma<1Q2ZgHFh9B1?*bsy>QlpQBAE=Xn^`zHfzbM)D06IWjRkUZBVn?D-AB;?sU^?t4tb_xSM%?8mt zCF;595LhVy%)lnVY!ReqVznoGpBXwhG|)vUEkwfOpzzaxi-Z6gp@LB=W|0Sr>NN~J zW};Z;+%$}gq1k7!qx$KIvlCzih-rtwDo4c}px1L)hYAG&r~>Z_K%UADuqk1N(liHM&`?x>dZdDwX)E;%6XYf*x+pX-Vl1W} zYV8$Ez|9~U3{NmD82iwC2WB0uL{rqKjutTwlIc^*DJKq~s)qDfdG0uI7GN@i;Ry7A zMTYqT^c*5qiJikYC@mmdW?&Ybl8#O|oJR(VW1ufI=dYD5h(#kQ6C$h{7ENZNO+IFT z8${|2YuVt$X+}P=fPG4~IA)fHbkGrhrfxCuuCXEQXEYN9h!jHrjo!Dy5f^-u~U z>;x2KCo!19U>xg#vjxY2FNTCuvIj98g{)|S1u>EyyWb@r#866!2ACoVl@#>=tO9Jx zi3VU?X;`8yGa*tL_Ia$Ll?Ki-NdB~xfD06QiZ^xylW|j&*E=~Sn!)!3*P}dD$FYVn zJwQ*cLbKd0SnFU)D|THjzf{iq zT7}|mD?Xw48*@7Fv2fPS8Iou`EENo;i=}IZn!6>=IXxu3WquQq>jXpH(o1WG?cmo@ zR=GI3^rBGNzS4iAOX%poSuAuMy4lYke_1#(k|=#;ZgAaPyl{BQuozo*@g=Qm=55eI z^{$nRD`R|R@7&W#vpr$170k6uPp_Q1adFK&2vh;dEW1+n-SW9Z00AbP0l^tqp1g69 z|K_v2GqCDBF?V1cj;1r0X8z(2mWBmS>)ap^^;RazT7|OK729grQ*(zP#aEv!r;?S8 z_w<@->)gP?pkVghw-SIi4lg&Z`r6?b%9nQEICP`u!&&~sOZ>Ah^Dm#_-QhL+*oH<| z+?}kT-Z^sRNTQ-$sAylU=vX+Ar0TB~eZOc?ldNsH)_t`*QM*g1-Ib{A7HYe1^nZA8 zwRULHuwLq0(k{{0#;=YiwsZezaCqzzVXeUoci&p zpS=9zmv7Fj?m6{Wz00wMBYf@iqNf&{)qS5^?pbZ<CI&M{UB-=WE#XzO2V>t8y!Uguxlzf$(2$`2~}ZBKou`fzum@3_!+oZt5> zOyb%IjADvsV;i7}k^58+2;vp8PX$jS9~1Q}vnQ=LP_I zZ$I@S9}FdeVIde!1kbGo&#gBG_-zAnfyCe`VenL9a6}j!;RjyfUkUN_Xo8LibRuCzi1P1*!c+e%>7^VFPq-C*JEEz9xci5r8w z^}w2;|E|qFXZ`ZwH;B?|04a*QC7fi(N^s>k@7#{NMX#qHY+ljnY)n zJyOwKH^>!^_jV_oJ%Y1GD!S(p7Y5$`xTkmL&>q#_ceFg+p!&3?^647g&qxi1+P*SK z`X5zp40qN;_=^gR|6)6a|6uAZf!jasF+N?T{Y+A4jKW_1D%uL@ST4Z#Co$-x{UK9z#Hlm=plh0v^ECBAXS$K zdXO(Y1&dLgflFw8Tn);lAzQgTjj%`>aEYhgdf`yvhuT`&C?9Vn7^5h^*U)a(kqFS} zLa<3ziTP#?0WEG4>X3S55qg-KZxR5I zf;}`UwBcCSsv+p2SyNmO?SKtiqKjHY-UYyl`8|t584g08xC!0_jDOZN?}MY^rof@N zK>~9C4vCxMl4~4Lu>xSDimPS726{nZFC-g50W5k9UwYmJeY1gETYA*col+}fl6h>Y*<@%Z^hG5AU9!>4zM`qU;HPmN<-jR%PQL_ z#6?Srimb%S1_)HRS%8CxSoL#~(Ks`d$8>HM^(ntYQnw`yn~h@aX$;T`z;Y0XIyk0) z?k6kLMh5gsRKLN4Pf@r*ixV9=N_MJ4cBF9_13aE&j0o&eeC)#Dk09_{GO8^5$B+Pa zX;5BMgAGtwWy#*CG6lXbVsH}xLBn4A$ zD6NQYd6FWUTkh|WZY#WlKRk&FLQ3JNfz@YlfYmB7!30UbYFn1h0Z4sTkc%5cRWUp>Hu=(6?2DwRZ$OZJ3I+Tv*2i69=s9a9nH5L`;yM8r8dD? zcj+*w9KJ1}ZGd8tr}CnLyJfjqDBqE+-U?c79csCU1vB-)3RM;i8!BCKS)LWWKF4-0 z(SA@;ptNk{R!XN#+j3V*vK5~&NmPuH8gh{qK5Z*oS1Lh$w+^lu4yH7p zTB5Wv;b{~+jS0^V!LtLkpt3v8QdD}%e*3WGtFj+)b+B+gY4i2(*8IKOH&CJZbhmGy zO#ib|62l7LK#%@s-6VuLo`GPrk2m%e#Rmcs!kZ}k0ktWGo5=B+sMQhK3(UmREnlGJXgVx7Yxd0`=bcR{g zYpVHk@;w96!G2=ylVAxdh+tVpMgv&$Tptyyp@I=fWTkx0x5{>5#7NKz9R|w|i~C>9 zA)_pzGfW6vx=$l9ij6?dkf;I|bQgGlr<{dku@AoJ;c?k}$?^;!Unf~`YC0O6m|-iR zl4wXFS{$Hc^2Z*cx@{l~_*qd<0?p9foC9o6&XhdchXC;tjcL!V>lzjf1T3hRviTVN zf2Wubcw_Cy6wvJmzexsF8gPi&Zc~8Z6mo5ncO7YWK;v-!wjp6L{uIoU<+}9*YJQEp zub>Z~1ZDt;0XRw8>s$lrxGBiqOSsn+{Bk6{&7BVsz|b;)>iEj1_>$f=^S*SAeF8kBYxK!AHcTL$66OZM+_0=!GY3%2 zRwbRy$+D^?=c4s4)yh|GOV(|@>#STVTRs4u>KkfRWyw9g&Skx4EGjk4nc?7PcfJ*W zGoG*=5NrpM?y6+54@956pL{^5?WPSvZ8UxPu=X*J9Tf~G;}0*v3(d>>mv^p|tn{vI z=iR&3?4AGo<%Sl@J>t+J_-$}pFCsoF^7WOdKWe1ERZC_+bW2eQ7*gXAFGD3BNa1m*!7UBC_#qSj*wjFd)Lnw|@04h7PqcDp9;pJMN8K0Bz}Fg7l$~a}DSDD(LT4tax<>d8gYi>) z_foC+MMf)t%q`K2(QZoeqNjFXQJ`ePZ%<^6nR0U!jpGPh9t%d#h?<=pN5i6~ zZQHi->+pHvI5R%U&cL@=Q`4fht-TZT+NC`92iSj|1a))Qbt$jL@HoJX_+i*M@B}#b z1=-U8ZuaBh2??4X zV8NRcBvyEmJ}!g4?0ZmEmX3b|F+jypB8#p9%sxryF!mc*&WsA+6Bzpn6!>b!o}}xM zvEPS+$k-Ckm8EzNp}0U2loiUq+=1NZ8|EusxJzwK)^EM1Rc~01E9L_jXkv zBUej|+)t)CxlOROWjMJ#!^rX{1SrUq3y-T3!>vFho8jVWWI1F#U?E*P248I<^8p(c zWtb0Q$bJ&*(GKYjOh>yUHZ;?2i48px8b{82keU z$Oh~`LIBIS3FH4B<6xRnx+)sdb{+$`8c#()Bs|UmB~VMOc$@<=6XVxr?QqBl>>aG* z$dt%11!{e`+oiU<^YHitXI)8O$?}JO8kyr)P+n$^x}=j*Hl&KWyUyyR1Irytdtg1o zQZ^A~)k`~;>y`rWT9V~viFv)7u*>?2W1lx`*ZV0A36#+7;d-;yGz~>vPCV9N?OOKApF<# ze+DO}Q(wmeI&UpM!3TN_Pri9q>f_^I0#03E1s6K|A*Zmtz%6MsH$zh4$ywdz);oOD zL=^lg$C06_#Ng{E_##PhJObQWvsvF{c69S)0TG5X6QJy6>2?jk$0?Ztn6^D`DC8ss zIld>11&XF$|=1uTmEdx>kK^pp5Emu zK($Xv-&#=$?Xemqmbs7BW#URqL7!*zY24Tt^JXNTJ1bT3bo{R>a z^L5X&_&X#0;61_6qW*;JX3{SSMZYuSxGj3~Wg;rTvIB=C4B1)etfYK^hX<5JHw4)* z9}SLAiAHqQhT`K>vS)=S3L5!Y*&dIs-0+nz_=^W;B+Z58k=7q#@HqscTP`deLqL*B zP8{fOW0H;uvX242{Vc#2v0lGX7RB#i7CP(5qW3t&#G;gzgWVFXsfUz5h-i}Kx|F9& z9VZZJl4Ui7@&W$eK~k&`c$*`xWoY^@QChzO!B!``7FMq+j}{3DUxvS5!`!{ zW!1^b22dY7dqHA*N*2^0v7MgFU6;BNj#|M{yEJplvF$E=CoX*`)GO5VE*!q&sk!U# zNci^({=KXIrxva2_5N$mUw!_sUsyCHODoc>ejXxP&D}5CX9;M}% zFJ8Ks@HE`=G$gn9ubsJiCb4Db+LoP*{p(W!%kFCNmuFR;Y-5_SH0k>=b>+| zz2I{Ic5U}J9wHEt-feox{Ox89W@bOARjw}|78USY_ucVSpx?SrMms;~GxZz6xNC;+ z6XSLa_mo2TsnOT(SAW__LYRYgYT#8$dlstVzYp5^q%WZh$)bxS2VaxWvXfqV*V*h2 zla>kUv`o;%TKe!5o##g%I%QAQ1ttUmCq1?oTSKVg6x4tp1aT-E}EP+Gx})fmiA zGsQ)Ch}9L+Ve;Ld{tj8zD93!{fl~vfpAtTA1vSpbGICwE9ZD~=i|S%KpI_l z98OpeM%N<_ci?aD+)e!0*ITCkX(;0+?Fwck?}Y3m-S30?69#sGEG72i{pXaQxD z6ES+fQLS~~H>tEO_YFA@Ij z)4CqiY=PhyY1GOZozw%IoAi}d>#VTNgIbr)DD-PEii(r1(kp4R*#?39UwgtqOIM+d zIs@&)Z!l#U6=*fBR+JBDu_#Rg!N9yr=>?Qat7e6iE2SK?lr}PC)o|vKg~zNK=^kX& zDE-PB79}^DR)n(XrF$B!R?1}`n?R+iRnjwe_;oknFuXtwB*yCHcy^d+VXPrLU5+cc zwC-&ciZwKN*ZbEr zu60*^swfgN@uvN2ngdCVDWUNR8sB1MP1B6qJs1^R1W(JVXIs)sCA=Miw`0}26HpPG z?)3rCGR%(GhkjEJ-QkS^`In>KE${1Af84FzuhFBh90y0!xwNSDm!F?CD(s7P|)O-xCaC-C;ImQ7GF z&8WdXzP!39f2pXOn!(p3uYuh_R5anQlCcxuU(Ly@0C#uMGaUsF;TDkF`1^UFMy7s1 zPhJJmg2Fe)KEU9QF_0F;0M<2P3)0SjzRWmC(F8w}Gyw~BBE*OqIvk3E6Ek{NzypBn z>>nV3I0oC1fEqf;{stfSVt~gc(ahy>xitz+%T8 z@c0~DI9y_?*Teh^AwEF!^cg-po(P{4!smE4&cF68n6WXk z7kw4?sSe;ROoF!}ylm9~-pU8jfzfqPbx%#ywcpbb#qLGp(w3!C-raQD-h9tO)U<6F z361wl02ItFux-P5&Ar&|WBl4sd$ zY71tW0%oQLQXv&ndkLAOc0D^=wM^|ikn7gY?n4VZE79YvWoze!eX@b6VaUVo|DV>S zW#kN#ZCj_$^*`tI`Okm(&iPlH%}l}b`~UgYi9`=Y{Wm6Zk1-p0P_ChwI20r6PPX87)ZsN_H z;bc2S2{%npi#L6!6jS3CNLe}KhqRJGQ@mxeCR?a_G#llt9F?p24)w04h>n%Bt<%cRIliV9^k1^C zvO-^6`8!(9wPxH_&b>~{ibBicQC|2ynVpAstjYh0mYSt!=_ZPIHB&;(xSQ150AU61 z&dsv8%~4qOardk4Sz39YrcYC6n4{Eb+E;lGzt@;gBkMzvFdvP>I~JV?ANSF+^8g1NY23i+ zAT+{C>EEKqO`HKzX3hwqg|l%c$g^^0NZDpBKD%7?5--L_Mh_i|2jl$2Oym$BKNJc^ z!qMZQESr(h=^?Ia>D;GGQh|I;*cR}Y zqe4@0L$1YSZlmQg;GO#jP|uRfp0ws-g?FwV^pvNkYT+!@%*9kKOu0N&3wJKB@K&|R zk0oA}D_51;a(PK=(wmDH-np7IHA^*7y_86w*3i^ZiiW=n=dw)2j3qWGT;z|#q6L(A z&unliBGZD;A!}pPd{oww43ij~-~*9ZkP~nnWG6-gCuffF*>(bce%Uk}6Zn^V2dBfb zAq3IDsb1MMI6W<9^MVk?JnJD~)Zp=KRkNt1phVCi8N-yq({e?CkBT!uCs!jB35p`@ z111y^WqLwH{$|<7r~Q!_u$livY>M}XkDUk#oS&Z%yF+5Qn-jvPc%geDCYZAO?gqVB)p2h4!oJ@zl)dkvo-#So#`=`8 ze#zLd?5RzAT2r3ZMNj)V=7QykbX9%2sw-91wOG{y<@V)#%@v?yO&c3h#)dDAO~0!E z(vq$Bv50K;gPI{f^NC+K)Nd%-q^PfORUaHAo76-Bvq5t0GnwCbg%XbbNF~c>%bKWo ze)~@1=3n!a+L$0W!*g_!o+Ns+xBt7vN<>%ewi1rovb|>j&iaQ zwsVjR#)GmYPZD}EPX`5<7j!D(9FN7g*)g9{Z~+;)T;M^z6zzIN!kGCpA$~dz#6{T( za%Lo#l=ZkpXGBFlxkEr8zE*B9f(MDyJnY+0U@905PvAaB24-1SsD!sr0YSpQ{!T6_ zk&vwTQwYvdD|)ACl+IWk7st+zrLEm5YxhFGWbOXay7SSpqay8KQw}!m=uSDh?^AZH z>Tq5>bN-Y-mf%ts4bfsU{p z0`+tpaUK*}>~u7cosn<-8Lg?h5lPu0P)+JfV}r00vWjM+g$y#2m%o9q=QJNNDtil; zjy#;UAQXXf@>WZ(9K^DUBZz{*v95V0%hqH-679rw`qUNroYjNk$*Y@EN~V-@0mZ|h z5Wg^nOmk{Nf8irN5MOe4`$RFM}>g+u(>N@G5gfL~dz!yWp zG)_dZ0A4I5__38Ya5@-{2Ve*1YeVILLW>hnVain{C@X_m%kN|pB)Zn;@h2t1X6Qlu z2m+M922)wa>b$uB{Qk6cTgtjEQ_+&CZd%b%w*H3{W3xXrQnqrbe8-Y?=Zcv!+0w?2 zl(9qV+_7ZbS=_oWW$i1p-lT55?MrL_qlZ5X^2VD=cO(h7`j#DYiEGL%dn1X}# zb@h@=0&K0M<|K$U;dQbdQNXQvCiC07P>=+l6)e7NZMDnAF-aOQ0c6Fys~-fanl!GFF1T$8R@-T-TSNwwlmCAc}iJ~lO=>uGj)j;I_I=W?PLj8kJF<) z3N@Unz73>rqq=X-G^cyIInC*xu3l1)YsQ($n*Cc+C;p18TX}ANh4&;X{521(nkDzO z=L|{1BpNbno;}n>V@PwH3Ta*kS91=KfTVE}O{F!@Wc8X1NS*>ccA++DysWuUuce~8 z5V)Zs&7AQxHD{VLCQTy4+2(Z7Y1LP=$MX(t(v%w;aF9d&9aZ1v4{%Vl<(sXxfcP4W z?I;y($O&*&uTM@j4oZU-%XH0nRz?_q4ONT)B1wbV-%jU%DAPhnnf*?_etxy$ULzOOI6J^MBJ{#Xb3KgSLo)%al3{o~HltoS6^qPF@h>H;>K+qq?7^3ZMRr1nO%-*ojJnYG zSfcYtP(XKLfPD!}IF60Y#97oGB?oUW9=y*XY{Rh=MO8~2gmu)EFoN~^F+gEi&{P82 zLyA6Xi-41*)^pS7vw#Z}jR%xjqE{cDl67J5lSILXEz!A>nFt^CnT3ITh5^k{n28KP zMo@?cFe&t5RqOPrkeJhhvK>;2rX=-bEtbj*gt8IbkN_q{luY1^WFN&R8HK#Wwy#l$ z_zBljzXzbP_z?(Fz#A&6t{lC5^hd9rGcViAFNV*D|KjA;eJOkEjSaW$8&@FvdG-ufW=)SUY(41u+sdD3jdqFR4-79&XU2;CR3@yx;%~yMW zYLmNYCF2Vjhx^e(Kjp4YyP8w3=CrFfbeeZPMu&1iG-Ymaez2oEkD;jfEW2UL?{hil$rkjqWnvO`vPD)Kj7Mmhh_hh#1 z_{YTGCl=d||17b<+|IZCS2unBV^ou7y{A?p<(5yC$T@<4d(CmTTGf_PxDt{*8AJJX)@AN!M>o)o)DK ze=Al0t^1S)#?k4&y8ljX!-@_Pj~}|In)Al~&04yqSj5&s(}7{zW;)zi`!$ zZqa?wr@{DU8sl5)A^y*-4#*YR{&_U^&w)#b7XzKIeK?wcBN_}5mf;u@Ly!db*qSG) zf`6a}dKYV+$vlc$C;;p?tMEbf=P}b#c6-6J0qT|lg3iO%4bdxCy&iyjYVSh zbU1#3phk+E5@jdW3LXnb!tvQa9K2`Q1rmk;@Mwo;X~;%_pN@n>K~dNZ-|5~fV6K61 z|2D)Et!h<}>k}!lfF4bwOTe+=D4Q^{A`d_@C;GI)QD`U|xfvmdaExpaXF}kdi-7CI zI1#+{AU6wc1unhp3gq#a{ICQBz-5ObG68B<$Pf`P*OYcB2u<0>N4Y?u0m^#03}EOP zkldt300-bRC=wd`CqoquV(c{xaB~4jpuo69T#Lw!YMCon;_QpunQZ+;-}-`hy*6(E zb@5*y05`m>;!4fs8p-QVm2H%IM^j~c&*?L+s;eC-SIapg0=3|guqh)uU%6!LAnA+N z^8iRRr;N=~%fOOx2cW{9>YOCclzYUU0Hi2c{!Qh(3{%OEA{PN-0$1@%Ikzq(7oxYGmFx$WjAJWCs;VJV z$!0dxWvc7a)f-dQFw()r>LI|HoaPluXEG}Yg4#Vm-pAecpbNq))E@^Iuj|d{_XWKk z{9HwcKXXVhfsxKyyufOqM!K<}^ud;j0{i)2VObiV1B1&TC3KV!D%HBMSC- z-bv;3NzD@nt%ff#TfPcf$)T}!i7a0xxCLRfpzQF9)t?kvfHPJE*f-BCm z#YX>HIZwp4P%!&ca*jL=@sX9%6(Ljctu zSF4!;c1+g5@q*z{7N#M_uXZdJVVHD%8do;wJ3 zE#OIbDh^nF_{3 zC-w)A@o@a72bG6!UG#W3dM05duYq7>IvnL;NH)+@@gT&7gSoK@`+$C;!Vjkeei04{ zc|U36KM|jbz!1m;O;e~y4hG{VWbM|?2g0(pcjLw>pr@bWr((h^96e6Y$hzLXek|)# z%0RQeF$4Mn2Nj+Q!#s5gHC=cLjyZ)G^ebBxb$tNoi?S(?`rSj5A38AS2Q127K!&Hn zQKDJ*3V`>)5lcA69m*BU#&vplOc?;H4Z0MrQ@Q!6D3HfgyW*A~U^v!>}xt zAkoiwl7l9La^^}7jm7{~6#f)Pp~DkR0uhqDqev`@#EjqsO-F!7Ww$a7Xc(S)DVr$@ zN3iBASks2bI&c>uLe%i6Y&@7v3uL%1P|h$=_$pK2Y8{M$K{D&J6Q@pi9*Gd&Z4buK z?OES}tIC=bY9BDmsl?`Bjssp@z|W%>E};}WH(qSc~dHXW`y#E z*b9n>_8mwjy4N=hO4vU?fF|N|pg|F{w3`O$k!`{Z33Wa zD@$8@Qq~^HziY|5d&Le8N5<{VRMkrjy;5c0U2jvy+bA_}mTI?Tnp$qmq^h?_Ra;jL z+6w!s$y95WjFl@MgUOTchoGRsPFle@mvb7jn8Ap}v2slIUz)wU|2UR?EuUt5>M)YhNr+I+XR^+wggh}1p+owru68uUM-4jlitX}q-$3u zXbNEC9p`sQ8+<7X0K&Udmcg`TBxM=76_bvFp88A~ekA9P2NZ*b5@RwydKlC^0RwAbxeLdB%gzd^a`VCq3&WD9f62N1 ze;tbiVYO#2g#H7ygYVrd|mHHJpEvG67|9R*;m0P;0q zZ8b)h6UBHIMSL^R$r%Gv%&NyIR0oOuIIxT$^tiZ@YG9YU+Q3Ql#s~tLg4tsqS6r?y*$&*zN8^(u;?r zu9u{j4ofe8M>-sk0#V5mTXIh0DOlOXedqUG5-yBA^p*hbR#?he@#vvt4NP~RqHO~x zvON9&?%WP1UABS8;s%)B!q#C8^>JGVgr92Khr5|iyJ!eQY9|qe4G}OmA@avLiN$A3 z#N<|sE}(}`DZ!B4#0+sa^5+Gmj($s>2IY=}XPNxQ(I))EXvMMd6FeK8nK}l?_^}CP zi?9ejAXZXj%NI5d*&zTB z>-Hxvwa})DJ1#dKfmS>g5fylI>vrlF-oYy7<4PJrxaV;13}gfLG>$L3Yz&Z19tgY&ge!x)rvK5T2R6}f4Q?47OvzZvZ-_Gbf_stqz z<9*`>UD<=aMqS1IXX$F4`F{T~T4#RnEaTQyJm@m%DwH0vj}ZLw4=ytP9-L8hY`&lr zj+(3Kttj=#qe4qd9EPe|t>#m+k*FTf0 ze`c|MHyGs({n@=BTdc0LV`RUeESLhXJrGFP%_z^<7nR8l$h29e2NG5mauh8O<_0=$ z2@A^}Qm|*pL4bD@YwgJb(~yUFZ-VLRn~-TSVQ1NcD576Pg9Zv;$3hdU*e3&Q!k0Bv zW}=5d6(qb zz2qFkBFp^f`}?l%yD@vSWwCj;)HEo0hL)VePqw-#O3lN!dL_@ul5-S`?enMJPh3wd zblsvC+lHi8@T&HF=^XhlC^F5r&R0sFjxU{^t7TMU?}~}iRzCtsZ><2c1}B7I`1W5k z-3&;5ZK9n{Nr;NjJDu<=2wxRv^}RkugaeA-#Ihh{#5LOt_;nQ;gy literal 0 HcmV?d00001 diff --git a/test/e2e/stress-scale-tests/tests/__pycache__/test_stress_operations.cpython-312-pytest-8.3.5.pyc b/test/e2e/stress-scale-tests/tests/__pycache__/test_stress_operations.cpython-312-pytest-8.3.5.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c44fc824ffc4d7253f5740321d4f0f018a7a650f GIT binary patch literal 20282 zcmch932+{bc1KNt? zZ0JN;QCcB~6nq6imRZDMD;|{^Rk#+H7VDuD98(gJ18Xs9)lPdKvRK557QC)HRBu zI66)Z(BxM$tc_}DYFIbG(8?SAfS$ZD3>e6-ali<_nz(7$JYXKS3|NM(16ER28@CPH z2khit7k3Oh2b?sep$<|U^9sf3-_$7m4!AhOE7X9Sui%X5*He@b*HFCs&HSJSJn-b@ zOmD*2@S-W+Gt{cwm!@j@Uh0LXDe5$|J>VY*kh*5x{~e06d{#kq*l(wnO~s?Y@7w2 z?3@)+2WNxS$=M-wac<5D@7?2$P=)M2!Hdbh1ILdi1zr@N9p!~cGL{$-qxo-0My`ic z3;#cuCA%(Jc*oJ4MqU3c)C8TxQsf%asdvTSG_8KsG!-tCgdGY0)K+LJX;5nnsk>dx zfM0cipqxd`9Ho=_^Xzrsd1{F>ZC;aB2fPGKyfs6<@HP2=c$WyNZX(CD;u^w z2_HGE+_#NKhT}3Vgj}*NG0Km~I+9@$BWL+=JQ3jp{Bp7rZ^P%u&hYtq0)8;r)RPeS zlb!oUW3nL%x8Vz&vUxvJgOX`HgZG$gJq{Z`axP!kEGkd%M!@eO8-^7ckxlzXN5y=L z5xB*=-Y`ERjzQ~ct!O+ViXn}xjmAZpJ}V*(vFxM0?eRo35^o<&4D;==GlLO!nF8WEO-1hS^s*;zkb2r4A0g(!Mbd4gB0Ab5Nv~I`<-IR6*cb~Em`9l$++gWvHmw+ z=qO|Bd?ex?{AN{;QTvXO>2Vp#Q*a7C^Z`1{xOMT(x(pQV9?8L>}w534*uq^|^49YKL2iz9ed zKSbnYF$eF;@d6ktQ{*qm-&dKZe|cN@v}~SCn)!rz8cODAfO#4}!#oSzqd1aR4WE6U zSN@NVN>-_D4VB?JrP6xbP){Z8h0NECQ`Q*J`k^Vq^e!7|7TG?yJoC@~D`N#-6Hp-si&jJT! z`ABq-MQ#;h9$6G7MVQwkLoA`(ct!q8V}L(mk$CGxA(rHoY?lxj;S$58S+?QWC7x@| zSCkDLe>O4}PrgCRrjf`nFOEi{{4&0Pl#RIRK|dsaN7(8xyM^ARqRjL1410e~q@TZc@K%G%KcCt~+3 zds)DZDVuhl<45>Qqr&dgwom57(B@!*YuibE5R zMW#DvwBNVBZq2ft65F|0)%jM}Zy%tP`^wrz`VEcH0=GgFBp-bnkWK&puLTkSKvO~p zl8{!|fcJ=%VPP&rmo1FY4%wf1N$~3m9hh`N@)5GkkVA06k6aOt0W2tN3@!|xL}p-r z4U4iiGBPfEIX=!O`4vTkW^9ct8h$_7%wLM~qXad|RzL=Q>Z7a|lab_@D6f00BVG`p zZ`jk}3yJvHFo2ABBsMIE%5=C&QC%`N%*%`z=lRi)QPzok{H$yojU$M*9r1627 z3b2z0mo#;@%DeumYw4@$tbd#2-}aMG#=mXBe|SoltE#)%eWQD-=Z?Q}0rw~*xBb0K z8bh!tSIyo$eEo2?dbd=)JJ-1W`=?(&oo(DIHEzu{Z(3rgnmvC{X=`ei3{-W~Qh*9n zD;+!|`Jc(vu3OSlfy49zN)vET>6Y{qH1>NtIwn;gE7#F+n)Gy!^t28p_3!{qnQX6_ zUo}r{{HFcmhn*DLl&$NK>N>J@yQI2Z_bCmGzmC0m>H4MFP5=6(9NYN!T8eG|-I5kw z{O)1x?=Szbw7ik>RAwt0q>6@YMWedI!iuNL+}Cpa;iB z&K*f`yllmC$4R-TIoZy|1ftlFALWH%F~rE`$Y|^&a;gLwBN{}LCS;FH4{I03i)mU3uy{i|hh}R*xOfBghrUP`&>UgdH)&8ut zRkF6ub!Durx2-!rzT@y_9jxSFvyN8D(R!b$lIhOY0Bin%nOgboIV_7IU3@%Ch96T%K$8tM;sO zljPhqUzKrg$~yZbXWzph<*Wn3$4(xJxP~uR*HJfX_ItEH-e=nH(EiN9z8J@Ukc<`- zpm(5MT~RS8N4lyaG=b`cMQOP-El;RJ1)yFm%V{|6E!~@W76ZM%487~qdXB-*{-iTV z>yTZ*tl2`%CsYLh?n6xhOc7TmK6PEg1?+PmiQkf3z@l_u)H8izOFp%9@%KB_cXdSu z1Y>zeM_qXjV5u3+qfyO-<{}lcq}Cn*@&>B}8)1pjf=!%d5i78O%2@@tC`hLPn*L}5 zY?+XTphVe$&?7OH42wM2Q~+J*6K95iAIOGdV5DSMUjXnWqA5nfKni=1-Qi(n zRS*k?$-1PPSLGIul|2RRiT?|d%hdnexP8HM@(1f?H~sjzpB#U``Sz~QO&z~>`s(Sd zw`sxKG`DWjl<}PWWnj(ZQ+J%+YnH2)Z`mgI<${$r1J?sNr{|jWsx|BEkenU4%K98z z4}hm~?LSbuO8XSEWTrg9>A=-J*@_maq9t3=AystDJ+)A=Gv}|KK6Uk_tiM(Aw`TpD zB>$$l@ellaKMd65g0<6^u7ioCT?)2mgIlEFmdy744}$%70yQ(bnNu@|W@Gcr{NP)T zY*)Y3)t~J;DRrI9YKX9}!Y4omO&J(zH`0C-TXRGAdnyX=Np1yuMThk`hwB^>efX3zB26~X!2Mowl z3xGuJq(^e}ydTRrdKMg~K1C;oB+sF8ojilume&W`mZ$dPhZZVOs~Cr~{wnxF0N?DcBi*;v!-X5Mwtn0jk^w=nN+rXgLX zz@R=%7*_u$Ff5KHih+itT1+Rvpi9()GOR3JP-}pZ(FzJdrp4(=O$L~v4w_hT4HXIhVPGhyM(_)pEJQPAMxuDD*Zs5O8+|0!Y5aXt(@C*>$HoVG#+*c~I65W_$*#U&-iQoMlQ?~!v^ z*#y@L+L#54UrY$+K{@M}t5$XvF6pYir!YvCj6~tEpjyJjl03)8M%YJ2HjfR#)`kl$ zTf%vz4F)IRrbIJ>oa802n(0#m~2HR zUl@jw&-=QC7?w4YRR|*}7Lk1NfKvHUENg4<8=@mI|8?`9uyk3s;|3|?h;IA6RbPQV z#dSmm+;w@cRb8!`enE1zWH#)PTzh{~ExGz9^o}rV}oRDSTr`>@m5ZEP7lpC|H8XvvM*QRo$SS| zE#Ir3;b(`Wrmp$sg|&UZ@E(|=b9MFEx-O}%E9YyP`O-`@<7v)$svj^~);(oiGEy); zK)Tb1Z#zRv7Rq;!e(0p!4Rhw(t{wB2?o`6KZlKd=+w5cNCXU z?f0DvfsyU<=RI2b9)hU!t@&FsK?SXAmSk6!uiz zI6mY0&dE$g!)_M_XQwUskLfWfhNdavUT_ z{?kD>g=+{m#EPp#8W&_WNOc_w9#zhOOg#aTQo}K64N6Zy0Mt-Hb{88=7v0Dub`ljM zl(g0Yts!Et6zi&YfCQiqkkbt6{?KB69-uIkmFQ``+Ac`LRvmTaB$0^aW?x3i7hR@+ zGzt=)Gp7wOhGn5k~h2*;!OAfMJ?Ce<{WgUt>QAomUgfU2Dt12c@wB;ok z_%2e*1=odWE8)vx7)S$43gcLrZk20RGTLxygQ>bBsuV_%i|iHTZi(;`($G$S$V4Os zkyQeQDhV%PGJ(mLF+oZY{sksPx)_VnMff`Aehm`@{W3j7s7baF0t6GOA}bJZsL26&F5lFJ=^@D<`h}}`a^Kxx zXm-tPMaJJc`BcvB%evP}?zJ;r3+@e*JrM8VtD4?A($eA*#Ib;lIHf2ot0yw44TzH#G!2VdYh%KnVZS5=uhDf+hv@06UubE_041 zlvfVn1GN<~IH5P)%J#wXSl|-HcO2LyZx!$?!L8-EgN`!-a?VrcaS9yhYY06RUE(Zp ziL*ciV7LQ}66GIR@(yrBg2Ec;Gu6ZaE<3xco#QXG0}f1pu@4@=Ev>}vcF(wizSeL)uF3eHSsq9JWBE5;WM)B40yWFtB(q0 zoyTS%25T6m%n|x}T)Fmz2|*RtDooZY)wUW5l`sDebS2`jfodvPKv5OVlSY6}CR^4R zkc@%pz@l+2fD^}-xfkZ1U83k-+6&@sW|!pc$ap#-V&RS$~J* z@0bhBcm3qK1^=c~)PYCu4mS{_nblNqWwJy}PK!WKZ9W{+xobPaHBr>M@X+xMUy_73RYgKAUj8*p>Aj=v`kw%(j#-= ze0b72CSvCANfwl45PmQZ&XO#M%kr&pR+W^6>hdMZS;z4d7p5`ghFu76D`0up?^_-{8OwgXbz;#mA_JcOEib-)cdNF=jrxm zOGN~#5*VV*5+s)A)n*lq6mIqj=fv*96TU`87!?R`;Goe0d`8JZV$MLVB<~MdRw3VZ zH+v*z8Hxm(N>T*6tL^_!CKvHX9o*5fHC0Rp+ZyOm4+BvG#Frga4e%j zo)4k5fG1mhNLA%$jCK_)%CX?7Vk2k4?aZcJh2GG~1#PgxpTpJw#};~*Wg|DH#KtP} z3=PDx>G)VQ%JUr031N5^BAA@PWDt^+E$`oAiLNE!HU{sP344a&4vz|FA?G91AyVEx zRL9gs!tvn%Qkrg7*o5yJ*pWmMyl-P8+y)lbW;im=4n{73VCMM|R*4MaAdoGQ43k3r zvJtHu*p=|_paJ18A(2f5EKBATvI~3h9b(jc)Dv;FFINH;=Q<8m_!cHnOmKd}%b4uL z1cge`5S|y*0*Ya>h6+$7;~+L~qB+@jAu)BZdsCaGs)$woq@$`V?SU0=w z14k!#*)oAma|h?PXLcTWfAIZ*jPKN<^Lemw&-*3+zKnC~u5t`20qjgq%PB z^G;>`3^&^~+dTL5;)%~^PKUFn&q}AyW(L2sctV(uzCHBTPui;rt#{Tmen0qn@Gq*+xIS=wAY0Qa)wJGY zbnWhkl+INFD+Ihx=)&uKs51dh9#rCk<0($yk-EZJZgBYCGo+ z&L5UGKP}Zht-h#U2X{uv7`*H7O@gZKe=IhMvDK0#Zk-*R9hej5*UVRBe7hE%yGtUL z?tt5}cP21n&NTODDxY2S9L+U#{y6#e_*>&Y`s4Qpq&?3`Ri|!yPTc|jCmHHB+g01| zmMqZrBXI+4gP+!I?b}HG%(uS{e%{+yb8x5j=Z<~GgWI%!?Wf`4uWP(`zm0+WWv~b- zkieqPt`TrdUL(-|Gid|{&{QMuK>f%|3J z#Q8HtZ9@1y7XJn$k82WcMKe&QpVuN*j|%|Fl?<1bg~Mc&_!_ia&>))1X%HHl=dNeX z)Sj8p?BV$}Z#GT_jOfisdn>MGIyX4I0X&x+lK>T!aLE4v-tC*%Op4Ad6tA(82B;&S=S z4jzH9HW#RFOGgJ%oJunvA#hGT135*)frozrK^DnQ z_)j=>Bs&O6)?#=LBaX&nNyP?m8DD-GlY${YKtKRNfcxPG9Y7F8O<+usDrGCdD_|pt zf~l~CMp0SIV~c$7j$I*AvS{I0La{uSvy?3_7eeAh%UFRp*G?LLf1so1U0S)VL zr8i3m}kLOU;F^@1-)dRK1>Tw zMD@NED9jT$G|bbxVxB=T9DKTYf>|e2nW`fEvuOS&x@rF40&xS#V8B$vXv7qok17P+ zO+5ybM>!~G+uGV>XYq|9n*|i~!oP(CRJ+1`s`Yre7D}xDL>mV91BW(1DM=F>-XNWE!gug21()+L57c`VvP-3^G7(=f8x}P2hErEx0b~gH;)NK44Q1Wfgv|I76H`Ltx{@jj;X% zX+!4Fxhu$7}Yvlb? zndiQ^usf33btYqvE*iPKw7>(f8Ft%z&7$$(9We9a36jn$w)_}2EE-#LzRIkxRr0mM z7-sAcpg9N09vH$*tyI}DS0`2OLoKuowNUe<1x8Tc1`3%S8YTnSKHu_o`&;erug{!# zVWImAneE|>{fmpn$g%-!ntNf<*z@=RLW^K<_E%;7&62-4=MGF)O74bC<2K3Ng+T!r z9*|kzEqQmq5CSbQgn$_Yp^>U<$!mvX2wUgU*=+}$q&8=2|`T_-%jJ{4Ba^hmV%QSa7rVF&!!t%&U`}&vx2fKud;yD z0_QLaVuHXnPu`1Obf1I&PuAdh>!XE3Rwv1EEI2bh^iYF3PJQA1Xi`|#0+`2;@v3(*A9(Eyaj zlMi5c$$46Ee|o0vciHw#T}Q5A!(C79OmOxaKV8KkJEyG0Cz-wcbtPm z1_&QG%MA_e8S4#j)xonZt${Ljk_Q%R&Ur%zJHVCdd~64x1x{Tg#TFVJgO4_h%n%(* zsT~&w(KnV9#`qm9mry)l4eL>94zas;vx=GuCdRhpr6f4R@Hig31&faGLIx%vYmY)~ z27Z#19r;u%IH!{l(3u5UKXwN6*s+wM{Y-46T^vklTB9j5p7c=;1*RC%I&1{;MNK5m z=x8K47-EzhCN>O_?*tRex(L{6Wvf#2h!P=YisC-Tji1y%x;xHQ0JJt+|@U92=V5jJvM7{w}*A*SO&xqiyoRE^74OvzS=- z5@o8ZfB@B+(A_}&%qeKB&3BJsYTyM^U2)Iv!ky{Eedo*XyAT~a1ssVSM1{WvVhj`0 zb~r4X!{DFf#vr&ZEL+3jFO5awXq12_3oNN9DuFmc4UArR3KK+6B=Ss=i~kZIzlzCs zF+r(GG@YWD6#fV1Atfpr(Xa6ioq5V~qM|Ecf=p7x(1=&49OJyFu`mq}w3Obpgm?EV zTNwY+01eOgG%g0{N_mC1_nVwd?fr%T6TH8pc^Ph??B%-m%35F(OAUo7AR%KwT4p5VlUx*QW|zn)K-3Am;F6U>Xum48rvOWaa{)Ox_PaK;FPp-6yi-Vaq&2@21N&7eVjVRRNJ)GFgS1ZwEO6Miq!(!GW`@xh7!i0=a3(!_z zG2f>6g#fE#&nW|`+0DxOa6W#|L`YC^;SkQU0(%oyLi{%0hW0{w*+u}dx`s-Wsvh&8 z!CojLn_->uuv;LGn#aJmFgBw8#Tcw83crPVuOrKT781Y%M#{Se&N}D(L2$v400r;b zT-}HF9KeuQ-3F0Ih?2v*T z3&D+(hjQ)PCl5|toW5}NrHpUgjF|DXNDxw5(sXPtxMLB z#}92JSW##4-m$u-j?Ao^ZF;>uvu5Mmx*yliM}E{aAIP}(8X_;#jiE}} zcxEgHVd?Nk1FU;T$HRrEV0d&KzkM4BJRZm*G14aRM7@+vg)BU8r{tNTG8PX<0NE(F z0&+lMFC?-V|CEw20s-Qp;z`uuE9BX-yGU$pXUCH8caKCR!rMQVjK#&ayqg!or}KYK zjQj;9g(h%U3LjyDs-{9BUD&9J3`LQUOxDt&Xh4%E$XUsH4QgkxcC=|9TY;qM}`m|>pY zdb9g__sq~-<3d$ergA%MANzjL*6^i$`ohiB_0(+JeB(mh-c0SjpSUvip4-OWdj_hu M70%`8DhYJ>f9_6}v;Y7A literal 0 HcmV?d00001 diff --git a/test/e2e/stress-scale-tests/tests/__pycache__/test_stress_operations.cpython-312.pyc b/test/e2e/stress-scale-tests/tests/__pycache__/test_stress_operations.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..244096b4ba86f728de5a80a4d79ef0c770c077f9 GIT binary patch literal 48424 zcmeFad3ambl_!Xu00@8}0FvMi?gIBs5=DuUC~99QS(L5CmKll0XuDFW zF1kiFrjkmFt|()=Hq&kQ7v`&;DW)pbq3tB5opj|r5eiT!^?d3n_sn>DdOph5lpT4d zYkucGJm3)mwK(aSf96TN@8P|>FXx{7JLjJJbF0;?!gcZIZRaxHQ>p%ue#nXdX>pHj2mno}CwHN)Bw-6`FO{*->iaLS-2Kic865#uQ%d)5t`M$D(o zBk8BoM=Yl-?45qtI$}FzW6$d0j8hpbKKm&YL!N{N5!RmQ^gtI)=0TI z<={-;RGo58sJ-S-=vB8@8#VL~5Bf&OaUUBU7(5%*U+x?4Kj&3P9mme`zCP~A*x2x{ zE581V<6}I$>*#kh?Fc{C@ALbkX@~ko`_B4!__rVOjq`*3{;sj%VP8M_R!5Co-?(pl zaKzUy5U{L17q;5s-vTqlbGMdgR0)RPqZoi$+4(a<7sk`A*1}1xXaZCf9dku zA&dN!xXb>KQanCI2z$b({GKj+ICsmi#UTydN9I4YcjH8izrRVyI zzLDXmn)hZ#bz>KNqfs4uVezjD2{d zuvxvnQU686E{EzL?(_S-ny9va*dJ97_-TZBJomRZ43G8q4L6({8}T&^o;laYa}B-$ ze_g+Su#SrCX9bsB#6>;47iErh@%jrB(R9CW{NjaP7Jr~@jj>R#@MJ^E{%devR()*E z6dW~kJt1q;{8P7c3tumKt!PfYoLfEBb#?#eT9vK&oayz>qoLwnq zS1xC-o6=rQznxPU&RHkstXs~h#&65*+~RO4j9^ZC zRx_zFsnFNpE%_RXkJQ8$PWO&p9?O&3pf=TL->Eu%UZwg2Lt;En83xoKX~v*>XPgYm zvG~GnV@2bA1k>gRXRf_nCO2qg0DWJUZSq39)M5i+0 z-x~E5rvQnRR@Ik2s9&SLei!wXrl_wp)R%3I`pSJJ?Q!cTsJR3RakX(({boFV_{XZ> z{6uNxr^inrseXTt>Q~Vh9EsK)FO*{+Dw3ZPcg~scIbkVNjmPQykR#zsei}+a)5}%- zwn<%3Hw^9RiVE%^Ja8_svs2f$}s2mLz@E@o04RKKE0IdjO2aAF9lE~);d`nno4v`7cu zJ1C159#0~v!X@xpq+vytyuRnU$gB-Pc-qP^R_{7)2`Q(ql*WP#T;`uS& z!zsLZ?rYjT-S=J6m*GLb9NypV`G2c^`3A~nBOI@mr#9qU;h{i z5raCa_%@0rQCUZx>1y{pP&at%Nv;;WL$w6F$NL6ZHDOG6z_N{9G^m+vVhUqdueR=;s_Y$;l9C< zs5eD2Yk1YsQZlOd5Bq!I7nBI) z8yB=cw*1f%Zrdle?GrZcUn*UCYN<;oJrXK->YsnTqNgxlVp(C&4rh483{NR)8xiZu82QS^WX9@1wxu@p3gzUzUy@`c1HGg@p{Qvt*-1q+r^sKAf_Bd7V zT05P)S~c%^H2Cped1ubPcGdeGozA`anxEP=`0>-$!aa8V&oW#0r0YM>8{q4MbUogD zVAs;Kvv99l`$1#jUZ-}+smJqDe&N22`lYtkea-q0YYm9u!)86+e7I3d&+Uc#w`o7h zDcs+#{it1!=T#(_78^9re0BuH{QBDLJ^4wxIQ2UUCTb!RtXmxZ7$Wc_=}HA`fbkrp zsWcT+&uP!E!{9!`={P-S;M9X^E{!u{`H=$n;7qjs;LOZTU&D`ueYG;zc0zJ@SU3eW@7Px{v6++V7u6G)U>{>o(mlbFx~TmEKXz%*j~|%g4~=oYsD%Rd zuz>rxsEr%s8SFQ4#K({L`99?DEq&D7cVX}uBIjd3QqB*Ka%_T$rpIGLBIXNyJa`(U z=Ld(SJ2S>jbbF0_9%6_(2!mXWbFdkR-;1yOK63Vx!$28VKYWiT!y;O~n0)lW0iMy{ zJJL5gI6&x!#5H-?6u{GJDNwf-6!YQPSh|oLlfTN6K37)V4;})qLOlmU(`?xNg%z zgSc*gq`DzeTJdJ}8`X0s#nO!nMzOT>u1Q~(eK$S5*fE{H;#Ot4uG_BJ!uA%?-m;J{ z*jvK(U7~&0y?m9u_{$Y7rF!3=h3tN#pjh>K;f@OJk2>>qc(m_%^my)9GNS}Du(AGM z*jOh(;#Ez*=B%n;bGia7ttJ6+1~r^o1}CwIwg)vs%9s!lU8xrI*ECnFbgD@$MwS*} z$oq&_mss4Vq5(3jH%Sly!8u7+>d`zzm{ih5pw2tGw_~&lsec2h8-fN-PhB7;tCLS0HjTj*r^cx**#%mMg)?V*XiO;M11Eq&B=z&jq`aBH2^o$JkPjh+N_rC2A3M^o~e%LTq#| ztCc|Yn#AJ)zvmF2|9^(_vg$uHZ(4Re^Mk6nmLDBo>{+V5)$#On&-Lf7Jr~ZZT+XVT zubMImu4n$yT{5*7Y|HiZYw6#%PIX3d^Imtq=8o81*Dcp9VSA%!Zv=-I@szEksq#wy zNu|rPP3u?8Dp&4|``Wg!vqp5*gq@9|vvGd!vU5u$r(ov9wU@#&4vl!lv$>Sj;O?w=c6&@Y_(VMe&ETWsqN zw>=}aJtJ&9E)<;zxlba7yIDy6E*R3%Uw^7sWgS=hiFf$D%&vCbbvK;;*s#M(mCA@e z`u*8QXwa|DRy+h{9V5aLR0FemOjd&puL2-!aT3F6A3yYCp@DIU2UUS|J$?#cr*tyO z#h`uRp=fKINXY4c`BAWAMD$V-XOKjyx3r)pnD!ggo9XdGKUTfvBuXJaJ$?!g)*B`( zqW4J`mo77J{N`1?z!KDaN#RJNL;Z&{OLok*KV1B)D=wuT;aQUQzHIH^~}tbhDI zh3Uf()UTcpz^R;3Yef?AEDZ#LQ#r);2%Ab*?s8|{(QXKp(rMS9A%#Yt&9#JFH zzoq;&FlL5$-K2}NF*H3}(Vk}NDQmaWM4DWG6KTJt+__h^SH@cH^veTi|r_9S`_pfEmA?-s=dR4ODjm{}}-o zg?NC1NpO;f5VXsKgyl(5by=?EU)KcKPfH@TpS*rhDlbvmFfJgG4qC{5^#2c+fAR_Wu#sb6T)iP4$n$ry|ax*|Ill->40j zwuq%IH=QjjnzXEL!l(pK(|p-q)&8J1+}tHLcL_~9g`!;{_wFYGsS5HGc_5HVIQp#M z+_UU_P63krCYg8yAhp7vGPlZ}AI>NfGsf5edp;|JYk#D0U8J-IkXzOP^*xm~EBl^Nm6azHY?%N0JpAvx<=VB9p|aP0 zanJhzprn@TuVudLuMYm;V7PUc*t$z-*}ZsXDRW6H6deq^58ZSh!itT&e))0o>QQZ~ zzir7BTSIx5(YJYCF9VpcPyikwlep-ge z7r{-8m;D;!WfP0!R132GK_1O5P@aVE}uR>P&ARdW{3%Gqvc@RY&XxlDX_a8RAX-G#rb z8wTblJE#Hlmc#tHx!fCCF7Jj89HxFKQ%)&y=kkMwcM4=ab~5cNm98+kbmLArg;dLK zxn3rXT+yWI@%#pj0W4pFrt_ei{C#LO=!a79)Il|u|4wnDG~hE*EVzfkeBisnxHPIr8#t>;tBFnc}c~e_ziMW_4sL_R~qDAhg`2i zk9_<(^WMOUELOg1MLVYAqiPbVD115+Eg(P1y<=62tFPYw4eFy|Rm(IcwM;d*Xp$qb z3*7szN%L1qtCsbGx}bS=ohnNEhI)_}df(Du8 zcH&wRy;siJM0Ik`ysd%i<751JAMXK}OiUede-Z-hfxO=nLtQ+5#99VS&&G6CJ%NJL z0NDF@S=KCt0YJ_0tf&Scy7tHzN9uG?xQiMtvqWO@?5JV5?~HHQf8R!2Z5>dp@xH-P zLdyGwFW}*RTEm&a(FXsy`>tF@6FngBLDF^-axf^sQO?X5>kY z0th)uJ2)~p?)Mrbk+6Pn1P~gjHjV&A=c2kk{;WT0ky1WHh4V*Ean>|yViMkNu-#zA!k(^+<@~ zTRQ$M0`r8ZNumWENrBlSBZflq!}u1ko%VDw=>SLF!Sl%#3cywy7wNO%v4`iM8NV*OE z=Y0JTe=y|;kF>T4m^}oKavrpBG%bddDnd|hK zrX>mN)Qq9&&ickLkU$66Xf!J(8Cfm==MN!7)QN;*;3w&|FwFodnHMQS(rAgA;sSne zI$O{10L-z}p>+(!DFLVu@A$4n(i9%`k6KLIPU^CF!3!++jzl=qO!wluxC7 zY_v|BZj`?6ea-vbnwdhOa>ruZVzpp96f$3uiOygvBa;CIi@ zoSv&+a4(nzM_0(Q6P1xy^m^`Vx!=v7+8@czo!axUBm27dn)ll^vu4pzJ$L-3qjg1N z%`A=Nm4x%wi+Sr8+Lk7Sy!Fd@C#Sn2IeFIu*8<_3Dlw;OZqv=2R>F%-@05Pu`0nUK3bvSsJe389Y*_njB}&WHP65b?i>k0()n z8;iL8*Y?l6@JEO4<*1;Xp)B39qZPUdR>!LPshoGun-{7D$F7iNccgLC;ySV7z|?^m zlV~Z8INe{|%TT#XXn|CU1>?-VKRfg0;2VQ~dVcn_(7b2KxEK(g>Jg4TE%ZDiVmeHPpg%z60oK}9Xx zC5LIB2l@b;_vLG85mcu7Fkg>1A9}R(TxQx|r~R;Y-g$)shQUY8{iJ(-JA^3dTV|0QqW33P+$lXJ8@qNJEzt1V4x3TNe`FxHOrK zz``oNV3JuoY{D=kmQEx$3+fe8KzIY>Z5T=c&&x4P8qcb*bV!#|45leqn6#jtCWc@f zo|hK{fYZAWw;3`zV$w8|>ErPw5z3Z=84acbdN&7+1G;z2ax9aUpeg0z!;&a-&@z+) zB$sm!altMf7Bk2X(nNfgQ~z;=(;*me&`U1}}i`!gS&XlL)4TW?4UGF$Nl5^fMq4OqqxD60n(RSY|MNekc$~+a!qli)Mmf z>A%SHq|?SC@?%~@=wTmepz&T51B#n{S4c>RPtx8G|6sru%});5s|bhsRo%C*(nuZS z>WF8~kg+b3lBb!q8skFzByLgg9&z@g zgqoT*`K)efMS}%4t;K~rGU$InqHHhO%c?t>S=aNg<4KjG0T`;p>%&M9)N3!x~4vJacDf1oZ)LCA!gpK8*v0SLw z5i)jRVz*^oJvmeLabf9f)9lcE^{)y$rcHP9N?xyht@gY1(`k`{qG=P&|HU&C@03Bj zTP@b_TpU`ie(G1bN2fa@B~9}s^G>07{q$b=%JpEj4i|3~i?>FKD(8&&c3ddzxTn|V zW>0sn017P%yUImZ`Rr36S5?H7b$!pZJ>S{4;#B4Js6TV7^2!h;-tNA(`#XD~lbBbA zNHeo%3^NmR)nak${0m}nn^3e#*tG8#WlLv&Ub|Ez9Dh#8K7Gsn{M{N=(bmuPDqGG< zvno4Jl9M-xSq(ztq2;XZDf4Zk6`C*e`{%a^+c?4IyJZ~sd_BBRoxJZSs{XEf$Ch12 z?N5#RUDyIdBTc?&4MH>1XWxYLq=M)miBw~;^*QhOuJuNQ$Y-^{q#u7c|Hvfy!SZ*F3EYy1hC{e6dJx-XrJ7 zaIi>WW9Vn}f-&1v*my5bjA^Wu*WBn8ANW`vgYNigF#=hF&S0vD%M#X$&h?~5u?Lkwike9s+rp$Bit`Y1255fgH1BBT_Ev7S zfUYhEx@w2w z^L8TVfP)S1WXU-PucHH(p(dCx!O=WH8M2ehFkx2q{Zt!>Qo*7aBioo5G!;CW#>AkN zUnzgGUK9zo74?TPXC}e+)jyZ+i7}T1$gN)`{(Stl&57w_C8f(jfGgFuoP!B-at^%i zK&b>&c={MWg88`*(=+iR<6udo={T^OJ-F0gCsBb-XvzD=IMU^A_w1zY7T^k)7AbKy zlCblK@W>>dfueTL@jjlEvD%Zd3X-A5>;7)uyEwWQb0KQtnl1v;o0Yfyxx%OYybQOmgWff z<-JxmV`)s452U+w;mX3$l4hxCsZ4Nmhb)I9B^8v%Us;N1VQ;Fu?!;{Pf+&<72xV}{~QtVE#RhDpPVh6#goPs|_Q zGzIkrZ{|r!y%ikWC&=cSG!79!XC?aq8pqyu1Bs>nkqiJY(z_;dn#rLxm4e<8KNwSi z>>3xu!nR>IBbG6ktWMylct2{J|ay-*&j}tK#5G%5l60sUMA>4 zqf8Ers!H#h&%N>qr_2Fn2@GF>KAP_O>l4Hcq2K}pP{iqe}Z2)C^4y`b$2WOo2vid62SFuaMf?Wzm zdZ(y0a{`B2g?ptdYmgy@g>u3Pf5<&}3^}D$^~`o2DeAeEp@vpcyyI*PJJe0muB8+x z84$y?IA>2tVXB(8C}Kj}$SI*-H|SJX52Cgn-g~SllTA?%dPO~$v6nQc;n2COA2xg*Rn$RyDlbPcc za+qM;5X7W&yh?sMv`&6X+~wFOUE?+K+o5XtDRGzmaSptv3i zG&#W>kuEkCT>^} z-zS)ERe_S&?lQ^&JSDVoAQ^xm0ZR7;@|Ujg7jR1PB{rauH=<-bPkyu5wy=bHg(MS^ zL&3_#Fl0IqF(6HToI+hBXM&vno}5eM5XIwPf^***+nVNKN*IBXP6|%t>4R00n2v}C zd*R|jl|Z4S^}$_)O$Dh~F>NIL1WR9hNESj7TtQNTL|Ce%Kz>3XL+U~CT0=W~y`>W0 zPw*c9D{^Sr#z)9u+FLBcu;AGIq(wf1Tb?Z8Ero z{~z(>%~ObV?%;)_7cXzj`=7}3KaxX)mH{achM+Je z36ApMhgM0cFKO*mwuU5V)kxgD7R>V)fq7b6DUMjIukLwe&mZo? z?nFn;XBL$spS)OK-Tlh$KO`BDBOh|3=8kaF4zX!RsA*@&by{rN`9{rb$?SV*7yGEc(##fm1+wDA%9Qa_jWrQWlsP>N00Hhp{ZY|c&Ry4#+z z*@MfTrs;ib+eOp7cRpX(a#+Ya5^_ED_paPK`JP$#Yum#4En|3*-%_p{B1s|@d# z=-@6@q#dr)E!FAZex%u1eYjcmcl9>7zic)gacY0L*>=RH6>NGuuM(Bg+@i(bXSCOw zEG3e4#JJ#03-${ildqI^60!9^AxLAE*+9aQFD%S9f#ug@X-c3}(gmQ9@{@FdeOm)U z#1}FjKzlWkF0g6}J^HJgNoger!7NxjQVw7jSiD>Sikb3nfPw@Qu4eJl2ppu4*oB8D z1(RnCl<)D4d!$|E*b_I5$(JL!Zl6asUPPQlz%@7u{ntxA?l9XgUr$nq4#wa>7nRmi zQQN?vZy1&_Al@NkyG%@TU+bv@UMTS%P!wT`EmB&vjou__V`e2=W@0`hzJPdzs10UE zB-Ko?U?Ug!1-w*D-Na+a5aKt;zd=fZx1%>i&Fn`nrNL&;HNeD-UgVE$o|gg#s@9tP zl04p|iS93uEKPJyXh&#Frp8Ed%lzK?cA;?VRCfepsNl7NaPDR?cQc`f#dQ#4G8Zuu5Cq`*r57adJ?{m56PFjEdjo2!ZV$i3O7_1(g3al+<44g?u(D;*z^x|6+fYzi8 zmrB+TvIR7K(x`|hJ!njYlcJ_9_>!VT_+m|L;|Ll#8#L&sm*7Il6zJF0jf`BSuH6)f zDq*^c+BGF|IUYw;gKQN3B^DzS4u1)vb8dp;K`=kfm!m_YG~2WQ*!6_Nq176MZ}R znzROi!!m7zd`0eZ+18dkhO6c?jl=@jVJZBsjw||Jy#5H~!vvPP@1#hJD`sd=)&uE~ zr5P!1#Z)z|86ACsxJy{v!AFjT()d`&UL*E`3A zrq`@SXqo zzBw=9m6L>GdmfBSkw-gME$4@~z<23`zR{Lid?|nM3s-YP8_eKpx8T#NIP2nZzN>sc zRGA`<`ncb>kxB|5sgP^_JCE--6{1bKrW)cJSl$!;wDC8K7xXawQ6)}Q9xgG;JWr5= z)FZEH^%w(vY^JiNMBpcAj%x`RP)lA$E7oDGK8mtyxz;t)GJv|Tzmb|F zX_V7W+_?>jyiMz_?t9D;tL3&(o1)ZPC?~jg(4B4H)pA?u8@CN_w@bblV^4xzb}`1@ zte{&?NmDpRS{TI^&%~(M$YwW-Bza8KMbMmzTjAPRZ>d*=m#F_HC6dmyKW2L}4wKt_1D5CH zy6;hPr;q44au`Sp%988G+Zos$!;?Mx8V&;t=7A+cIji4C%C>ZliG&%)w&h7be52>c z2j}4i`_7Jz0Y~olL$CG&)Sm=7#)9%Lu+V{m9#U;5{5OI9vMr`q;CZ?O`3gKcz7NpT zH-PPqo6C>Szl}Su&t9E>;mIxl0XK12k0OB-Ol4Q z7|EVWz`PlYPTHRcdndH43uwC=ItTAjcFh!OcUsqqxR4GNF_$+soUxaq{+3Vt#2BLsWO_pEt$K=(3zRlJKl}>KOyFV^-X{X48Ny-%D+Qp z&yreB(Nn3-Bn{DW)&~ZVC$82r2nki+gy&q}B_9k<`$lCWIv_KRGXw!(<1!6q{~*;sUAx{gJEzMsuk0 zk_=Sike0Jdz99)3vZ;tn?odKcW#mw0^X24Jl2b(v134J{p(Qie-IOko!GvC@H#4#G zRgwn*QXXRc$dF|x4pPe!Teb3Ir?h~8iW9-Tp zyf*ml^RxTK?E0xak&Nt_oaGD;v$@y`^QPw)QwoI;k#=8(H3Qc*LvW3KB@pPktyWl%qRdA9Br`$Fv^x74xJAe{V~(05K492UMdB8-d*d1E2hg-A*L^!_`} z?Cb5<+QZH=(OEY8^_$LSMGSkFa)hUjEm?)9W1Cq4TSv0o*H2wL70#*PP zSXhT%T;!NG-?OP)`7r)M{X%UW-iwF=MnM!d~o?{?7(xsa|QdwT!O*F{$iQgCA* z)m&95r}^JpUYPjtOFw)GjTPE>O6WZo?j03-N5j3BZuVY^>^$_r;dc**cRnlbd^WuE zjJWg6jCppm=*DKI?V`JrItz-u@2?L3;BdJ0fY^E<+H&>#{A=Ok1LE<4@bOXc_-Od}rJKhuA(mmy z6^iV*dVc+kZIIQ$e%6+-yW^(2 zgJd3ATWNPSZO~SO$S_i}bMdLgE+KDk*tPGbYae;S7t-^VV%ja)Hru|KxrnHChTXev zx_7atcEi_B^#l6ldTSA*Y;|+z=1$G?3ndFqA-f}F-j;E#KN|n> z#1AL_=IcwR#BImL{1dmZJ6x`v>(*}pKTb#7_x;}m^?$c+M}3c5^^<~aM_W`s&EJ6M zpSe4?9V=BW<#xJp`>?sDr$hBo;huH)akItLlcl}6#nxljE}ISTylmIw&2pBOp53ON zcI|SBt!KSSoXj`Z8nRVJdP%+#Hm6|yM|)NccubW_Mnragf_gIT zKB#`j@OBL5gA69F8p~lDQxXUimlLI2ro=ZYGsrRppp$9?IGI;YY{h0F<5HnS*|>CC za&ig-@+oa!5sXWmATFqys#2;p>Vj!7BOmkCqw;13+G$qD%0NEj#>39IS+#g!`a=Fe zuva(;11lA5v*pKdZVq3Kq%cS zYXj%I)G!M3iJoAPnXVy2HsfO#u@zaeS3`01l4*GQAf>~yP0TD74OJf5uuTe&Jt;F3 z=N3ImPM&2?ivIx41EIx6oL~0z4G&)OVG)zc%ubT+924cEj`&J!s*0yCofQOQ``1uq zf%3SHRKiEB@+mYegs`UhAysh=Ie$scM{uGzM@TmGNSmGF@%OT(lvKtb6)<* zzfa-5LC&9(^D}ZNLB(MxvYvl-FKe0Pyawuzp{Oa2!mLm2{}_6%|KA}A2|b_P zW2+IiBw52ow`g?FxI@O$6{EpgyP`D`gSnE86G_53jbct?#NuNAoe@{DP|_keTgg;o z(`PEZ+dO5tXJ?#f>--7mW!ScbjN9dtM4GNz<4P{oqOm%XT^z|Qg!s=?gH22(GjPOABnwb&NQ9E~9bZi#3bc>F|VMmYXfT<3jaCUInF@(%O^}-d-s1Y-uF0gP`$f#M) z*ew`$e})37{(quJwUD^}s}j8p)a^Xd30_Z3@T!o>z#mIg;wU==OgU^i7;Gh7LR_h3?M&@g@z25`V4l}@5Ld^^38 zj8zd)lfEgly(YEcHzQ~?7Mp^?@Ev&1ZzX3NoEX92JIS|_PMkmUBy_?dje9A7Qfdz| zM$2f5(l0namH+2xpE!M}Ohq4b)~zdA1EUXM;=I*l$1{ z7C;>igdK-LBZMP;Lced>F|amu;K3oTK^^`8UvLm~j3r7Spg7ScjIauPAfPhcXA?>J;% zV5XeXNp2o?3?N~mT!X|+k%2Kf+gCz-*W{$uA?B#Moxyp5Uq}Sz5FIJV;AHCTtNAF& z$3C2+fwKucL`&9Sg!=o&*Jh*+(J_dzz$w|NhlgIBzlV|}qkKN1M2-sM43Jb|gqgEK zK7tlW(Ew~>oKAucc`bY+Q6Ie@jOzH~@;v-@gpqh26T{-j#0-FAhAJjV7bqaXg22NF zrA}seNLe}#r%5oHO`>G1Xvk}zlBS%sIUU*GpA!Z9M?@E=V5L*Wk8|>7*1x{_wav5s z<@}ntPBFiJIj3RD@*wAWh2H@zT)~yK&$>M>_<-6?&6-%*fFP7s_{WF5q88NomCu+kcy7I z9V-_}P(fuWgu+tTBSL7g9F6ateuDhMn$~KGtz>jLh^21A>@t_ZEPL2*Xkl?%S#c)q74`3zUGhVN3YOpbpIAByHz zTSYGBPSfZ`O$Wi{vJs@I6MhO3p+MT~W^80%SF4a{w1ZgT9q5qlGhy=9G8Ic!hLs11pVfyiE4ZF?nu6UG*Dk^td+Y@EMS&O)%x% zj=#;^7OsQadR7BnuWj6Ru9F$`-obTo+s~4L?_y|b?c{dhT)f@f9&Rr}?Bn*c`vE*3 zypaZ*G~2jCtQU8)u!p%LvEFO>O5^0It5_j}9i?XjN`mR0UOMa3^{G^pT z3XP?&GJboK^YwTs2CYmd!FC>f5!yW>8_~2>NxHzPDMs{UMld6l0r8YtCC6C*+WD1A z^)!`=dqzn`x#Le%GMIelPTU|~li3!U$bEvGpTus&lfOZ}Pd$EGpuGwWO72;V0{!EM z=AKiQFtHysbqw5Tc|`MbseKI9CN)2P^51%VztZTRv@3FA=bn$JI--1^YWFSI8~1x0 znn?8z<|fttT)y)wwbj=iKZR6#Z&%NnioTg@6>(UmkC+qViuk zo>u8BwaRS_y`NpsRQ@V7u#ZuT2U1ql^ds6t&g0czJ|c&mUonRqg2muUNBqS49UPd|>b_fM=YmiZjD$IYZz$ z-o-R+*oIOl8Ni~|s(=d9-LSiL4*Oe4ai^b@bNcuR+NeVIx_oq+6CIuy56gdonhc~P zjI8s7{lHX$fMsy3jvnZ!raJ!rB5wxiMIfU+fpRfZV^{H&|KG?V#Wnsvk#n4!KOyIT zC+9zt^M8=@19ED}*#sw$liJi-43%#uFIjOzIB zds_t=KAvzSP|3j#&_D;1oCJLQ7+M!U$hJRKj3g^L`7HCR1Z2a2y579LJY&AZrWL8T)T>V{kY>O;6t<=iB7`5jo!>=UwFzW{;lsX(*eMa%1Fk$YB5IH6@Etn`LUwL!nB)JLA|zQme}kOY z$(beR4LEP5Nd)Q7=mnFX{RKU}2`6e{t$*>t5q|7U%y{8j^zmPjL#lQBzlO7l)&Vo~ zNHz^J+dT>`Q1<>J`8Kq((EBfD>$+z(miAL z1tq1n5(*PnbR#O*l|oq%^fR>z;9pbBiW4qa%tY!X@0jxLr!i6w3FzNY%0Hq21u1Ql zF^T5ilBa)5&Ovexk;4R>d+BK(IjpQuW>HCqA3=ruK{!$KGh~c_rOYqV_jk!jC<(4E zt5p8?$jgu6#1#0TuSsN=dSBGKYNk}!L3^J9eS;j5_(|G(tmO|;Ub@L)GSm;~D^-!y zDvwgwbM?Yp1nJ(fEX0fw1Zs~VLn-mb6uy56V&u;=FjYOr+6EqJJG$|PLpso5#yb1# zeC~Yu!g|5h88Yrz(Y>TG+3w_4%nss%`rnlPmG=kUzpR-r6mqsr?Zu|anf%$^?=*jZ z(_5R~-aJ<)9-SDZLJ;!aU%#QkI@9}7&@6HjJua_8cS#r;C|;gJ1E zTvZ14CM_zfD{ScyEgf-Nan`L6_gZcEWXCan^*1x>aR5hF$DQmg3#Y>!-C{@g(o4eE z`j$J+2wVCg)Dc_*clDYqoX}xVTJKs7S*?(8=R2;Y ztvF!lU}p4YW-Vo>@lJNf!bo`QVR0*DOT9v0|MFH&={n9~TPTSwFi3#JGSVn96HClRi4EP%To-Xygxy}z?G>uGESz0-@1EA* zE-sy2|K{d5HqZH&%Ub3;#j*{{#T%!AQky7(bc*0!o+_tADBCbUF@Iht*|D76CD^-= z4f)WMJH=Izih3McQk;EPZ@`%+j@4(LL_Ae@${SaobYfGLR6|{|YHk~BfL0Wtc&+Or zb@h?P=DXH3Z{CXQ;U}R~6wpa11vm+1%R^5>*{806ZPBbovMqXVK>d)88?iXU7O!aW zCi+e_>pM_)O+F4K*}`D^LYsUvNWw^hbW{os6Is!eDvnBt6jn$_M&^@6?ELKME;OS% zKgE$LkW+m>_pRKw^XIJd&n)a+XkUC*D0(X7K6w>?r;ga3C; z`sTdY87Jrtxy5T7r1D_ACzt%c415>}AMOsm?lX5P>!pYzu(CMx#!$F& zgIKxYfmA>apytT*Z-ts|p`7iDJs+HU_f&Y-nVY-L#C32x#q!R@60v;ejOliM>FlPt zv!VPAKiass>4RkvY`2!1=7hhQH z7aaRTmIJWfe>)e(RlbX{rb;Gf=})z*exli3h+iLh zIvaZORKKuyHsbbo>Mkd4qAL%d#6}aj?S(x#+M7A+dmP$jive$z9eTW3&e76yp6S>+ z?Q+@XV`bVjJcyJTnv2e^=dOTGL)ahH8yD|}q=4PkJKP1V@ch>eW|mBT!xHgPI9%wj$&BpJ)p zrwFNkWJm(o*R3u-T^V0h7(0Ez&+q*2`_U^N+#guep$YbssIgc1V4wOA}E z{lyLCO`3v6zLsr5cFDDyxTjPhFmjf6ta6T48U430Xi2fXF=#@LFRmFo?HftDAZl4F zc1Q9S=c|8$LwVBW2$Y-X6FRX8g^+#?mr4H;Ie%c|V?KmTiTE<8G@Lzo%OAk7Om;@( zm~>wSB)a9?07xuY6`PZCzjkccoCl)<6t6PACiINU8nx;q^dNDlPiE{0pRB+h;*^0% z$@cKr-aY9Y?X~RUNs1hjp)JX1F|?{AV|k=5#S`Yki^xBNHW(TZ*c^wkaQ;sm-dknA zE@p)E38G(%1VJXE@Bpih$rdZC^P%apnWs%MHvo+ynde300 zK>P!PuD0VtpqR|jDJp_3#{7x>Z#X@NaB78>IHHMsFJ*u&$o_$T-i8x358_x6ENHRG zQCfL1ozp$^ovos1!6qL7$l|<1pI(L&btsU^*t#Zai|KpGzpawyMC>PjnsUI_SZuY` zgJ}HJ4LylV($BE7)sjM!Hn4836;ra${|+;VzZpv=G8dCLU}x@-nAtvM023n}jdLe% z{%jB4m3kvN`7uq@>~(XMLQ|Jmy>s!bkkyS-Zc1%}rRW~E_f#l+mCp{$Jug;*zZSA~ z!&8xMsvCX^!F$AG^2}Dw<%y-O^YF73ehO_4hb?8WC>F9*Fki7R%vTXCe%`lSyk&ZC%9nqiQ@o-v=I&G9 zDe%00@U?^Cf~{h~)<{KNq`C#lwT0WjF&7qL^9>FdI>^-SN;iiJwtQ+-xjjh%kBS9H z69IeZY=PvEw+gnb)TxTP)!3Cmb<+~Ew4y{V_e|QiJ3gOO<20XFPJVIklv?HSVrK`k z1)dS73M?4s1B;Hu&czm?bWgZouUN2`c5dMNm-n(C;rC|T@BPnTpcLr5pkegI?{!t} zwP}B%$JyiBpX64<{i&{I_cq;68#MIXQnRO0_p_oLJpXK)X-`?&2gb}jMQI=8>hb1- zB0aq+)6#RLX>WSk2Mw8f4QWd%1KzMR$FbhBzc**~aYQsm8Mkx^qvXksBLb(P zFxiipD$Xqt=K}$eVx~&)Eh#xAg~1Lw4IBr+(IGPy2>5XxIGqK~R8uk`+;>1VWi88J zPLw6-g0(PhLRp2)NM_kfDrK8giiMbKsu?$!Lb%UKhFUTeWd{d_nJA466DbXmU|uuQ z0Zdn4knV|j$m(4zeLntv$R>3 zp=gYdC!jRyIfl^$M0l7!G5HD+Tn2CXiIx-5W#pD=<2}Y1P?!4uk%;TRM_8Nz zWiXj`s3R8ps|Q~>7`E&XEjuDvxmZ+#K3KbGm|JZi5E##{>bX{sR7Ra{!~gzQ_D{cX z^$?b>`Na`eO(eH?)-_|h<7p6zv6r*}k9RdQ6k_oSGbgGyMufiQ|Fy7_a(V~5?MUUt1 z{`lZRJrK*z2JIXku*A{6jPs!DKv zeO=NFgGP>)1|er*S~WBT)0svN1|nEI3_;GxxwtHZ$mVj`-Hqp5rjcXj@`9wjlh49J z_X>ly=!Oouu*HEb4|H?lJLI!!Bo3?Nxetwsn8|17;^ZAqHB?U;&M2cQ@ z)Nt_vq;ozTdYu?6?^8Ss0KSGNX_K~8d_SOAHhARWqeAo#fpnSiSR4?kuwJXd_>&b$ z(;ZXW?${5Zv**~R_5dWZIHtBnV32h;=Kh?5nPbzZao(TN5jN&xC*(|J$Vf~94V~OO z?t(~OIq-m7LOu)Y2y^my=UQnX6_nla)J7_52`vDGrme^UU|3i=r<*%F`w|kvxx@xl zBhDqRj?~rzi>t^67FSglDf1#k1sz&k1z3oFl;z=6VjSO7QQ&a^iz{{li!1crapSZf z#8VGELJurZr7v(I6U{AmfyH6OX4KJmtTUOJio91ncmh0`SJU0?bw`c8Yzu#HZ`9n2 z4ei`T?5pUFT6%k5xY##LmJjhO9Y=}#{g7Cn!-03D)EG~b!z>#79zFd5IZNdHM{@ob za_*7yPvqFB$(bnt+A+m5ICGk$mu`> z^o+XT>8xf_1sbKWkM(ve_@pKQ#1I{lmy!Tuw229Jhz3H^l~{%bHN7;;unRN{|KwP} zpb#{~u2dHMZxGLf$mpGo>t}i@$;m}Q8{u{5zA3!QjRtzWd}dy9MM@Zn5h%BT=hM6?$sLk z{EBsV^*^(-S>ioAksq#x@+?_3gN9Ux{3X?i3$P$r_fFjI8b(xS-1iRoDRGxeIJ7JA zPJZGVnFEp^>;NKm zH|au67bLyH&mo$5I{IJ^hGFyPaIes(hp_)7BtjW4(AA>}GC%j1V1(TDW?Eoi3i#f^-t;Y8VkZkn1HN5t^HE z>OEARR9O5>Ws4AtO&>E*z3xH(oRw)?ACQ*BTspXHaO3EesYa`lmD9!CtTp^(v#&YK zb%U$Vy`jgNA@!_K^I(30dT7DNX9X;9&6Bzpbtr$P^psfQq9)qp@F?LHh(Hn)dYyu% zdB+KaWEW1ikHx)uA)U&{D!N!L(M3gmjz5^6q&8Du{h8cmtG`T`W1w(nU>&JpO2?(g zu+OAZQ!d01;;4Ce@a(y9ScvN5m~PG|vyF3n8CyJmv?44ByPe(-XA@ZNl(tHp$ zmi+qRH()um1NPfs$bnUYmUhNjC8eL(xhDLx@D?@n`p9?{9g}fkj9HF|n#V4VU${8l z3)>OLu%I$BbfkCee85_9vSOry>#f*Zaj2pvYVi&DW45mRy#amw5Nw(8zf0x%AskwY z(Q&FUg~mMkFJkG*PxJV=joHkUtODtWp)$s-(!si(8lHB>RA=EyTi-koFv%wR%-$Mlg^)=ZcaTGX zS4ssahNtJySCW|UG{E^ha)|%riIL=qPVe$hx(jH_vQV3I^IRKqhpt&MvRsLvRa3OQ2NHtN#MVZYCo#uE{eMtTn~UtpB~obi@N-f zuJCqNzGN6{IjidB-QVoKom~*lt`f7Wmb1P1efYM!NIH*V*ju>sysy}S36^*s?>jYzM$hi5la@hNtN~7~y%rhHjH@vn@$g7#_5Nw-5#&)dg z=?vf!(O5EDC*W)qV;75tz3|SRMtFG-Y4cUviZ0uf^)VeF(FXHlokzt@Jwn?t;n+#R zaVlhaHezvvEjZ1jHDuXz&TXFBA8FV$wP*VB%%y8D3E5S%e!*2ET56SX56%~cEZgEy z|6%t^6NkPkt;CBv8r_XY?bT!CBX6|*%HOsec zVSANmubRtPwzs{!@3u4R`o?P;uWz}w1-9FQ3;Px}FZzVKgUilCFCRoBPYus>g-o8= zPO2EwE2inn>-E>_!;VJL(YWkrp6^`fyJg%?4PJr_n5xwR^chgf=?a$Snl~3hqf3dDp zjoZI#(I4ta`#V+P5smim3On--cW8gvp~nwFZ+gnC$N4%>>Gh&sk7v=Ue<~+U%+o(r zmgaRu4HqVe5=xT-(dwxA{6(kCB6`S=k`o8HqQ)%} z^tP4%m-vAHDnBiPAVE|AKV!VjzX^QELjB;Sp0k+1Z4*&oF literal 0 HcmV?d00001 diff --git a/test/e2e/stress-scale-tests/tests/orchestrator.py b/test/e2e/stress-scale-tests/tests/orchestrator.py new file mode 100644 index 000000000..ebaf5dbc5 --- /dev/null +++ b/test/e2e/stress-scale-tests/tests/orchestrator.py @@ -0,0 +1,1663 @@ +#!/usr/bin/env python3 + +import random +import time +import yaml +import logging +import uuid +import os +from kubernetes import client, config +from datetime import datetime + +class EFSCSIOrchestrator: + """Orchestrator for testing EFS CSI driver operations""" + + def __init__(self, config_file='config/orchestrator_config.yaml', namespace=None): + """Initialize the orchestrator with configuration""" + # Load configuration + with open(config_file, 'r') as f: + self.config = yaml.safe_load(f) + + # Initialize Kubernetes client + config.load_kube_config() + self.core_v1 = client.CoreV1Api() + self.apps_v1 = client.AppsV1Api() + self.storage_v1 = client.StorageV1Api() + + # Set namespace from config or use default + self.namespace = namespace or self.config['test'].get('namespace', 'default') + + # Set up logging based on configuration + self.logger = logging.getLogger(__name__) + # Remove any existing handlers to prevent duplicates + self.logger.handlers.clear() + + log_level = getattr(logging, self.config.get('logging', {}).get('level', 'INFO')) + self.logger.setLevel(log_level) + # Prevent propagation to root logger to avoid duplicate logs + self.logger.propagate = False + + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + + # Console handler if enabled + if self.config.get('logging', {}).get('console_enabled', True): + console_handler = logging.StreamHandler() + console_handler.setFormatter(formatter) + self.logger.addHandler(console_handler) + + # File handler if enabled + if self.config.get('logging', {}).get('file_enabled', True): + os.makedirs('logs', exist_ok=True) + file_handler = logging.FileHandler(f'logs/orchestrator_{datetime.now().strftime("%Y%m%d_%H%M%S")}.log') + file_handler.setFormatter(formatter) + self.logger.addHandler(file_handler) + + # Test parameters + self.test_duration = self.config['test'].get('duration', 3600) # seconds + self.operation_interval = self.config['test'].get('operation_interval', 3) # seconds + + # Resource limits + resource_limits = self.config.get('resource_limits', {}) + self.max_pvcs = resource_limits.get('max_pvcs', 100) + self.max_pods_per_pvc = resource_limits.get('max_pods_per_pvc', 50) + self.total_max_pods = resource_limits.get('total_max_pods', 30) + + # Resource tracking + self.pvcs = [] # List of PVC names + self.pods = {} # Maps pvc_name -> list of pod_names + self.current_pod_count = 0 + + # Test status tracking + self.results = { + 'create_pvc': {'success': 0, 'fail': 0}, + 'attach_pod': {'success': 0, 'fail': 0}, + 'delete_pod': {'success': 0, 'fail': 0}, + 'delete_pvc': {'success': 0, 'fail': 0}, + 'verify_write': {'success': 0, 'fail': 0}, + 'verify_read': {'success': 0, 'fail': 0} + } + + # Initialize test scenarios + self.scenarios = { + 'shared_volume_rw': {'runs': 0, 'success': 0, 'fail': 0}, + 'many_to_one': {'runs': 0, 'success': 0, 'fail': 0}, + 'one_to_one': {'runs': 0, 'success': 0, 'fail': 0}, + 'concurrent_pvc': {'runs': 0, 'success': 0, 'fail': 0} + } + + self.logger.info("EFS CSI Orchestrator initialized") + + # Create namespace if it doesn't exist + self._ensure_namespace_exists() + + def _ensure_namespace_exists(self): + """Create the namespace if it doesn't exist already""" + try: + # Check if namespace exists + self.core_v1.read_namespace(name=self.namespace) + self.logger.info(f"Namespace '{self.namespace}' already exists") + except client.exceptions.ApiException as e: + if e.status == 404: + # Create namespace if it doesn't exist + namespace_manifest = { + "apiVersion": "v1", + "kind": "Namespace", + "metadata": { + "name": self.namespace + } + } + + self.core_v1.create_namespace(body=namespace_manifest) + self.logger.info(f"Created namespace '{self.namespace}'") + else: + self.logger.error(f"Error checking namespace: {e}") + raise + + def run_test(self): + """ + Run the orchestrator test by randomly selecting operations + until the test duration is reached + """ + self.logger.info(f"Starting orchestrator test for {self.test_duration} seconds") + start_time = time.time() + + # Ensure storage class exists + self._ensure_storage_class() + + # Get operation weights from config or use defaults + weights = self.config.get('operation_weights', {}) + operations = [ + (self._create_pvc, weights.get('create_pvc', 25)), + (self._attach_pod, weights.get('attach_pod', 25)), + (self._delete_pod, weights.get('delete_pod', 20)), + (self._delete_pvc, weights.get('delete_pvc', 15)), + (self._verify_readwrite, weights.get('verify_readwrite', 15)), + (self._run_specific_scenario, weights.get('run_specific_scenario', 20)) + ] + + # Extract operations and weights into separate lists for cleaner selection + operation_funcs, weights = zip(*operations) + + # Pre-calculate cumulative weights + cumulative_weights = [] + current_sum = 0 + for weight in weights: + current_sum += weight + cumulative_weights.append(current_sum) + total_weight = cumulative_weights[-1] + + # First, ensure we run all operations at least once in a controlled sequence + self.logger.info("Running each operation type once to ensure coverage") + + # First create some PVCs + self._create_pvc() + self._create_pvc() + + # Then attach pods to those PVCs to create pairs + self._attach_pod() + self._attach_pod() + self._attach_pod() # One more to ensure we have multiple pods per PVC + + # Now we can run verify_readwrite which requires at least 2 pods on same PVC + self._verify_readwrite() + + # Run a specific scenario + # self._run_specific_scenario() + + # Delete a pod + self._delete_pod() + + # Delete a PVC + self._delete_pvc() + + self.logger.info("Completed initial operation sequence, continuing with randomized operations") + + # Track operation selections to verify distribution + operation_counts = {op.__name__: 0 for op, _ in operations} + + # Run operations randomly until test_duration is reached + try: + while time.time() - start_time < self.test_duration: + # Select a random operation based on pre-calculated cumulative weights + random_val = random.uniform(0, total_weight) + + # Find the operation corresponding to the random value + for i, (operation, _) in enumerate(operations): + if random_val <= cumulative_weights[i]: + op_name = operation.__name__ + operation_counts[op_name] = operation_counts.get(op_name, 0) + 1 + self.logger.info(f"Selected operation: {op_name} (selected {operation_counts[op_name]} times)") + operation() + break + + # Wait between operations to avoid overwhelming the system + time.sleep(self.operation_interval) + + except KeyboardInterrupt: + self.logger.info("Test interrupted by user") + except Exception as e: + self.logger.error(f"Unexpected error during test: {e}", exc_info=True) + finally: + # Get test duration + elapsed = time.time() - start_time + self.logger.info(f"Test completed in {elapsed:.2f} seconds") + + # Clean up resources + self._cleanup() + + # Return test results + return self._generate_report() + + def _create_pvc(self): + """Create a PVC with access point""" + # Check if we've reached the maximum PVC count + if len(self.pvcs) >= self.max_pvcs: + self.logger.info("Maximum PVC count reached, skipping creation") + return + + pvc_name = f"test-pvc-{uuid.uuid4().hex[:8]}" + self.logger.info(f"Creating PVC: {pvc_name}") + + try: + # Create PVC manifest + pvc_manifest = { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": {"name": pvc_name}, + "spec": { + "accessModes": ["ReadWriteMany"], + "storageClassName": "efs-sc", + "resources": { + "requests": {"storage": "1Gi"} + } + } + } + + # Create PVC + self.core_v1.create_namespaced_persistent_volume_claim( + namespace=self.namespace, + body=pvc_manifest + ) + + # Track the PVC + self.pvcs.append(pvc_name) + self.pods[pvc_name] = [] + + # Update results + self.results['create_pvc']['success'] += 1 + self.logger.info(f"Created PVC: {pvc_name}") + + # Wait for PVC to be bound + if not self._wait_for_pvc_bound(pvc_name, timeout=30): + self.logger.warning(f"Timeout waiting for PVC {pvc_name} to be bound") + + except Exception as e: + self.results['create_pvc']['fail'] += 1 + self.logger.error(f"Failed to create PVC: {e}") + + def _attach_pod(self, pvc_name=None): + """ + Attach a pod to a PVC + If pvc_name is provided, attach to that specific PVC + Otherwise, select a random PVC + """ + # If no PVCs exist, skip + if not self.pvcs: + self.logger.info("No PVCs available, skipping pod attachment") + return None + + # If we've reached max total pod count, skip + if self.current_pod_count >= self.total_max_pods: + self.logger.info("Maximum total pod count reached, skipping attachment") + return None + + # Select PVC (either specified or random) + if pvc_name is None or pvc_name not in self.pvcs: + pvc_name = random.choice(self.pvcs) + + # Check if this PVC has reached max pods + if len(self.pods[pvc_name]) >= self.max_pods_per_pvc: + self.logger.info(f"PVC {pvc_name} has reached max pods ({self.max_pods_per_pvc}), skipping") + return None + + pod_name = f"test-pod-{uuid.uuid4().hex[:8]}" + self.logger.info(f"Creating pod {pod_name} on PVC {pvc_name}") + + try: + # Get pod configuration from config + pod_config = self.config.get('pod_config', {}) + + # Create better startup script with diagnostic capabilities + startup_script = """ +#!/bin/sh +# Diagnostic information collection +echo "Pod $(hostname) starting up" +echo "Attempting to access /data directory" +ls -la /data || echo "ERROR: Cannot access /data directory" + +# Try to create readiness file with retries +for i in 1 2 3 4 5; do + echo "Attempt $i to create readiness file" + if touch /data/pod-ready; then + echo "Successfully created /data/pod-ready" + break + else + echo "Failed to create readiness file on attempt $i" + if [ $i -eq 5 ]; then + echo "All attempts failed, creating alternative readiness file" + # Create alternative readiness file that doesn't depend on volume mount + mkdir -p /tmp/ready && touch /tmp/ready/pod-ready + fi + sleep 2 + fi +done + +# Stay alive +while true; do + sleep 30 +done +""" + # Create pod manifest with enhanced readiness check and debugging + pod_manifest = { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "name": pod_name, + "labels": { + "app": "efs-test", + "component": "stress-test" + } + }, + "spec": { + "containers": [{ + "name": "test-container", + "image": pod_config.get('image', 'alpine:latest'), + "command": ["/bin/sh", "-c"], + "args": [startup_script], + "volumeMounts": [{ + "name": "efs-volume", + "mountPath": "/data" + }], + "readinessProbe": { + "exec": { + "command": ["/bin/sh", "-c", "cat /data/pod-ready 2>/dev/null || cat /tmp/ready/pod-ready 2>/dev/null"] + }, + "initialDelaySeconds": 15, + "periodSeconds": 5, + "failureThreshold": 6, + "timeoutSeconds": 5 + }, + "resources": { + "requests": { + "cpu": "100m", + "memory": "64Mi" + }, + "limits": { + "cpu": "200m", + "memory": "128Mi" + } + } + }], + "volumes": [{ + "name": "efs-volume", + "persistentVolumeClaim": { + "claimName": pvc_name + } + }], + "tolerations": [{ + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "effect": "NoExecute", + "tolerationSeconds": 300 + }, { + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "effect": "NoExecute", + "tolerationSeconds": 300 + }] + } + } + + # Add any additional tolerations from config + if 'tolerations' in pod_config: + pod_manifest['spec']['tolerations'].extend(pod_config['tolerations']) + + # Add node selector only if explicitly defined in config + # Otherwise, let Kubernetes schedule freely + if 'node_selector' in pod_config: + pod_manifest['spec']['nodeSelector'] = pod_config['node_selector'] + + # Create pod in Kubernetes + self.core_v1.create_namespaced_pod( + namespace=self.namespace, + body=pod_manifest + ) + + # Track the pod + self.pods[pvc_name].append(pod_name) + self.current_pod_count += 1 + + # Update results + self.results['attach_pod']['success'] += 1 + self.logger.info(f"Created pod: {pod_name} using PVC: {pvc_name}") + + # Wait for pod to be ready + if not self._wait_for_pod_ready(pod_name, timeout=60): + self.logger.warning(f"Timeout waiting for pod {pod_name} to be ready") + return None + + return pod_name + + except Exception as e: + self.results['attach_pod']['fail'] += 1 + self.logger.error(f"Failed to create pod: {e}") + return None + + def _delete_pod(self, pod_name=None, pvc_name=None, force=False): + """ + Delete a pod + If pod_name and pvc_name are provided, delete that specific pod + Otherwise, select a random pod + + Args: + pod_name: Name of pod to delete + pvc_name: Name of PVC associated with pod + force: If True, use force deletion with grace period 0 + """ + # Find all pods if not specified + if pod_name is None or pvc_name is None: + all_pods = [] + for pvc, pod_list in self.pods.items(): + all_pods.extend([(pvc, pod) for pod in pod_list]) + + if not all_pods: + self.logger.info("No pods to delete") + return False + + # Pick a random pod + pvc_name, pod_name = random.choice(all_pods) + elif pod_name not in self.pods.get(pvc_name, []): + self.logger.warning(f"Pod {pod_name} not found in PVC {pvc_name}") + return False + + self.logger.info(f"Deleting pod: {pod_name} from PVC: {pvc_name}") + + try: + # Set up delete options + if force: + # Force delete with grace period 0 seconds + grace_period_seconds = 0 + propagation_policy = 'Background' + self.logger.info(f"Force deleting pod {pod_name} with grace period 0") + else: + # Normal delete with default grace period (usually 30s) + grace_period_seconds = None + propagation_policy = 'Foreground' + + # Delete the pod + self.core_v1.delete_namespaced_pod( + name=pod_name, + namespace=self.namespace, + grace_period_seconds=grace_period_seconds, + propagation_policy=propagation_policy + ) + + # Wait for the pod to be deleted + if not self._wait_for_pod_deleted(pod_name): + self.logger.warning(f"Timeout waiting for pod {pod_name} to be deleted") + return False + + # Remove from tracking + if pod_name in self.pods[pvc_name]: + self.pods[pvc_name].remove(pod_name) + self.current_pod_count -= 1 + + # Update results + self.results['delete_pod']['success'] += 1 + self.logger.info(f"Deleted pod: {pod_name}") + return True + + except Exception as e: + self.results['delete_pod']['fail'] += 1 + self.logger.error(f"Failed to delete pod {pod_name}: {e}") + return False + + def _delete_pvc(self, pvc_name=None, force=False): + """ + Delete a PVC + If pvc_name is provided, delete that specific PVC + Otherwise, select a random PVC + + Args: + pvc_name: Name of PVC to delete + force: If True, use force deletion with grace period 0 + """ + if not self.pvcs: + self.logger.info("No PVCs to delete") + return False + + # Pick a PVC if not specified + if pvc_name is None or pvc_name not in self.pvcs: + pvc_name = random.choice(self.pvcs) + + self.logger.info(f"Deleting PVC: {pvc_name}") + + # Ensure no pods are using it + if self.pods[pvc_name]: + # Delete all pods using this PVC + self.logger.info(f"Deleting {len(self.pods[pvc_name])} pods using PVC {pvc_name}") + for pod_name in list(self.pods[pvc_name]): # Create a copy of the list to avoid modification during iteration + self._delete_pod(pod_name, pvc_name) + + try: + # Set up delete options + if force: + # Force delete with grace period 0 seconds + grace_period_seconds = 0 + propagation_policy = 'Background' + self.logger.info(f"Force deleting PVC {pvc_name} with grace period 0") + else: + # Normal delete with default grace period + grace_period_seconds = None + propagation_policy = 'Foreground' + + # Delete the PVC + self.core_v1.delete_namespaced_persistent_volume_claim( + name=pvc_name, + namespace=self.namespace, + grace_period_seconds=grace_period_seconds, + propagation_policy=propagation_policy + ) + + # Wait for the PVC to be deleted + if not self._wait_for_pvc_deleted(pvc_name): + self.logger.warning(f"Timeout waiting for PVC {pvc_name} to be deleted") + return False + + # Remove from tracking + if pvc_name in self.pvcs: + self.pvcs.remove(pvc_name) + if pvc_name in self.pods: + del self.pods[pvc_name] + + # Update results + self.results['delete_pvc']['success'] += 1 + self.logger.info(f"Deleted PVC: {pvc_name}") + return True + + except Exception as e: + self.results['delete_pvc']['fail'] += 1 + self.logger.error(f"Failed to delete PVC {pvc_name}: {e}") + return False + + def _verify_readwrite(self): + """ + Verify read/write operations between pods sharing a PVC + This tests that pods sharing the same volume can see each other's writes + + Note: Using kubectl subprocesses instead of Kubernetes API to avoid + WebSocket upgrade errors + """ + # Find PVCs that have multiple pods + shared_pvcs = [(pvc, pods) for pvc, pods in self.pods.items() if len(pods) >= 2] + + if not shared_pvcs: + self.logger.info("No shared PVCs with multiple pods for read/write test") + return + + # Pick a random shared PVC + pvc_name, pod_names = random.choice(shared_pvcs) + if len(pod_names) < 2: + return + + # Pick two distinct pods + writer_pod = random.choice(pod_names) + reader_pod = random.choice([p for p in pod_names if p != writer_pod]) + + test_file = f"test-{uuid.uuid4().hex[:8]}.txt" + test_content = f"Test content: {uuid.uuid4()}" + + self.logger.info(f"Testing read/write between pods {writer_pod} and {reader_pod} sharing PVC {pvc_name}") + + try: + import subprocess + + # Write with first pod using kubectl + write_cmd = f"kubectl exec -n {self.namespace} {writer_pod} -- /bin/sh -c 'echo \"{test_content}\" > /data/{test_file}'" + self.logger.info(f"Executing write command: {write_cmd}") + + write_process = subprocess.run( + write_cmd, + shell=True, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + + # Update results + self.results['verify_write']['success'] += 1 + self.logger.info(f"Pod {writer_pod} wrote to /data/{test_file}") + + # Sleep briefly to ensure the write completes and is visible + time.sleep(2) + + # Read with second pod using kubectl + read_cmd = f"kubectl exec -n {self.namespace} {reader_pod} -- cat /data/{test_file}" + self.logger.info(f"Executing read command: {read_cmd}") + + read_process = subprocess.run( + read_cmd, + shell=True, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + + resp = read_process.stdout.strip() + self.logger.info(f"Read result: '{resp}'") + + # Verify content + if test_content in resp: + self.results['verify_read']['success'] += 1 + self.logger.info(f"Pod {reader_pod} successfully read content written by {writer_pod}") + + # Update scenario tracking + self.scenarios['shared_volume_rw']['runs'] += 1 + self.scenarios['shared_volume_rw']['success'] += 1 + else: + self.results['verify_read']['fail'] += 1 + self.scenarios['shared_volume_rw']['runs'] += 1 + self.scenarios['shared_volume_rw']['fail'] += 1 + self.logger.error(f"Pod {reader_pod} failed to read content written by {writer_pod}. Got: '{resp}'") + + except subprocess.CalledProcessError as e: + self.logger.error(f"Command execution failed: {e}") + self.logger.error(f"Command stderr: {e.stderr}") + if 'write_process' not in locals(): + self.results['verify_write']['fail'] += 1 + else: + self.results['verify_read']['fail'] += 1 + + # Update scenario tracking + self.scenarios['shared_volume_rw']['runs'] += 1 + self.scenarios['shared_volume_rw']['fail'] += 1 + except Exception as e: + self.logger.error(f"Failed in read/write verification: {e}") + if 'write_process' not in locals(): + self.results['verify_write']['fail'] += 1 + else: + self.results['verify_read']['fail'] += 1 + + # Update scenario tracking + self.scenarios['shared_volume_rw']['runs'] += 1 + self.scenarios['shared_volume_rw']['fail'] += 1 + + def _run_specific_scenario(self): + """ + Run a specific test scenario + Randomly select from the required scenarios + """ + scenarios = [ + self._scenario_many_to_one + #self._scenario_one_to_one, + #self._scenario_concurrent_pvc + ] + + # Pick a random scenario + scenario = random.choice(scenarios) + scenario_name = scenario.__name__ + + # Enhanced logging - make it very clear which scenario was selected + self.logger.info("=" * 60) + self.logger.info(f"SELECTED SCENARIO: {scenario_name}") + self.logger.info("=" * 60) + + # Execute the scenario + scenario() + + # Log when scenario completes + self.logger.info(f"COMPLETED SCENARIO: {scenario_name}") + self.logger.info("-" * 60) + + def _scenario_many_to_one(self): + """ + Test many pods mounting a single PVC + 1. Create one PVC + 2. Create multiple pods that all mount the same PVC + 3. Verify pods can read/write successfully using kubectl subprocess + 4. Clean up + """ + self.logger.info("+" * 80) + self.logger.info("STARTING MANY-TO-ONE SCENARIO DIAGNOSTICS") + self.logger.info("+" * 80) + self.scenarios['many_to_one']['runs'] += 1 + + # Create PVC + pvc_name = f"many2one-{uuid.uuid4().hex[:8]}" + self.logger.info(f"[MANY2ONE] STEP 1: Creating dedicated PVC: {pvc_name}") + + try: + # Create PVC manifest + pvc_manifest = { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": {"name": pvc_name}, + "spec": { + "accessModes": ["ReadWriteMany"], + "storageClassName": "efs-sc", + "resources": { + "requests": {"storage": "1Gi"} + } + } + } + + # Create PVC + self.logger.info(f"[MANY2ONE] Creating PVC manifest with storageClassName: {pvc_manifest['spec']['storageClassName']}") + self.core_v1.create_namespaced_persistent_volume_claim( + namespace=self.namespace, + body=pvc_manifest + ) + + # Track the PVC + self.pvcs.append(pvc_name) + self.pods[pvc_name] = [] + self.logger.info(f"[MANY2ONE] PVC {pvc_name} created successfully") + + # Wait for PVC to be bound + self.logger.info(f"[MANY2ONE] Waiting for PVC {pvc_name} to be bound...") + if not self._wait_for_pvc_bound(pvc_name, timeout=30): + self.logger.error(f"[MANY2ONE] FAILED: Timeout waiting for PVC {pvc_name} to be bound") + self.scenarios['many_to_one']['fail'] += 1 + return + + # Get PVC status to verify it's bound correctly + try: + pvc_status = self.core_v1.read_namespaced_persistent_volume_claim_status( + name=pvc_name, + namespace=self.namespace + ) + self.logger.info(f"[MANY2ONE] PVC status: phase={pvc_status.status.phase}, capacity={pvc_status.status.capacity}") + except Exception as e: + self.logger.warning(f"[MANY2ONE] Could not get PVC status: {e}") + + # Create multiple pods (3-5) + num_pods = random.randint(3, 5) + self.logger.info(f"[MANY2ONE] STEP 2: Creating {num_pods} pods for the same PVC {pvc_name}") + + pod_names = [] + for i in range(num_pods): + self.logger.info(f"[MANY2ONE] Creating pod {i+1}/{num_pods} for PVC {pvc_name}") + pod_name = self._attach_pod(pvc_name) + if pod_name: + self.logger.info(f"[MANY2ONE] Successfully created and attached pod {pod_name}") + pod_names.append(pod_name) + else: + self.logger.error(f"[MANY2ONE] Failed to create pod {i+1}/{num_pods}") + + self.logger.info(f"[MANY2ONE] Created {len(pod_names)}/{num_pods} pods successfully") + + if len(pod_names) < 2: + self.logger.error(f"[MANY2ONE] FAILED: Insufficient pods created ({len(pod_names)}), need at least 2 for read/write test") + self.scenarios['many_to_one']['fail'] += 1 + return + + # Test read/write between two pods + test_file = f"many2one-{uuid.uuid4().hex[:8]}.txt" + test_content = f"Many2One test content: {uuid.uuid4()}" + + # Select two random pods + writer_pod = random.choice(pod_names) + reader_pod = random.choice([p for p in pod_names if p != writer_pod]) + + self.logger.info(f"[MANY2ONE] STEP 3: Testing read/write operations") + self.logger.info(f"[MANY2ONE] Writer pod: {writer_pod}, Reader pod: {reader_pod}") + self.logger.info(f"[MANY2ONE] Test file: /data/{test_file}") + self.logger.info(f"[MANY2ONE] Test content: {test_content}") + + import subprocess + + try: + # Write with first pod using kubectl subprocess + write_cmd = f"kubectl exec -n {self.namespace} {writer_pod} -- /bin/sh -c 'echo \"{test_content}\" > /data/{test_file}'" + self.logger.info(f"[MANY2ONE] Executing write command: {write_cmd}") + + write_process = subprocess.run( + write_cmd, + shell=True, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + + # Verify the file exists in writer pod + ls_cmd = f"kubectl exec -n {self.namespace} {writer_pod} -- ls -la /data/{test_file}" + ls_process = subprocess.run( + ls_cmd, + shell=True, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + self.logger.info(f"[MANY2ONE] Writer pod ls check: '{ls_process.stdout.strip()}'") + + # Sleep to ensure filesystem sync + self.logger.info(f"[MANY2ONE] Sleeping for 5 seconds to ensure filesystem sync") + time.sleep(5) + + # Read with second pod using kubectl subprocess + read_cmd = f"kubectl exec -n {self.namespace} {reader_pod} -- cat /data/{test_file}" + self.logger.info(f"[MANY2ONE] Executing read command: {read_cmd}") + + read_process = subprocess.run( + read_cmd, + shell=True, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + + read_result = read_process.stdout.strip() + self.logger.info(f"[MANY2ONE] Read command result: '{read_result}'") + + # Verify content + if test_content in read_result: + self.logger.info(f"[MANY2ONE] SUCCESS: Many-to-one scenario successful with {len(pod_names)} pods") + self.scenarios['many_to_one']['success'] += 1 + else: + self.logger.error(f"[MANY2ONE] FAILED: Pods cannot share data - Expected '{test_content}', got '{read_result}'") + + # Check mount info using kubectl subprocess + mount_cmd = f"kubectl exec -n {self.namespace} {writer_pod} -- mount | grep /data" + mount_process = subprocess.run( + mount_cmd, + shell=True, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + self.logger.info(f"[MANY2ONE] Writer pod mount info: '{mount_process.stdout.strip()}'") + + reader_mount_cmd = f"kubectl exec -n {self.namespace} {reader_pod} -- mount | grep /data" + reader_mount_process = subprocess.run( + reader_mount_cmd, + shell=True, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + self.logger.info(f"[MANY2ONE] Reader pod mount info: '{reader_mount_process.stdout.strip()}'") + + self.scenarios['many_to_one']['fail'] += 1 + + except subprocess.CalledProcessError as e: + self.logger.error(f"[MANY2ONE] Command execution failed: {e}") + self.logger.error(f"[MANY2ONE] Command stderr: {e.stderr}") + self.scenarios['many_to_one']['fail'] += 1 + except Exception as e: + self.logger.error(f"[MANY2ONE] FAILED: Error during read/write test: {e}") + self.scenarios['many_to_one']['fail'] += 1 + + except Exception as e: + self.logger.error(f"[MANY2ONE] FAILED: Unhandled error in many-to-one scenario: {e}") + self.scenarios['many_to_one']['fail'] += 1 + + self.logger.info("+" * 80) + self.logger.info("COMPLETED MANY-TO-ONE SCENARIO DIAGNOSTICS") + self.logger.info("+" * 80) + + # Clean up (will be managed by the orchestrator's cleanup) + + def _scenario_one_to_one(self): + """ + Test one pod per PVC scenario + 1. Create multiple PVCs + 2. Create one pod per PVC + 3. Verify each pod can write to its own volume using kubectl subprocess + 4. Clean up + """ + self.logger.info("Running scenario: One pod per PVC") + self.scenarios['one_to_one']['runs'] += 1 + + num_pairs = random.randint(3, 5) + self.logger.info(f"Creating {num_pairs} PVC-pod pairs") + + pairs = [] + + try: + # Create multiple PVC-pod pairs + for i in range(num_pairs): + # Create PVC + pvc_name = f"one2one-{uuid.uuid4().hex[:8]}" + + # Create PVC manifest + pvc_manifest = { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": {"name": pvc_name}, + "spec": { + "accessModes": ["ReadWriteMany"], + "storageClassName": "efs-sc", + "resources": { + "requests": {"storage": "1Gi"} + } + } + } + + # Create PVC + self.core_v1.create_namespaced_persistent_volume_claim( + namespace=self.namespace, + body=pvc_manifest + ) + + # Track the PVC + self.pvcs.append(pvc_name) + self.pods[pvc_name] = [] + + # Wait for PVC to be bound + if not self._wait_for_pvc_bound(pvc_name, timeout=30): + self.logger.warning(f"Timeout waiting for PVC {pvc_name} to be bound") + continue + + # Create a pod for this PVC + pod_name = self._attach_pod(pvc_name) + if pod_name: + pairs.append((pvc_name, pod_name)) + + if len(pairs) < 2: + self.logger.warning(f"Failed to create enough PVC-pod pairs, only created {len(pairs)}") + self.scenarios['one_to_one']['fail'] += 1 + return + + # Test that each pod can write to its own volume using kubectl subprocess + import subprocess + success = True + + for pvc_name, pod_name in pairs: + test_file = f"one2one-{uuid.uuid4().hex[:8]}.txt" + test_content = f"One2One test content for {pvc_name}: {uuid.uuid4()}" + + try: + # Write to file using kubectl + write_cmd = f"kubectl exec -n {self.namespace} {pod_name} -- /bin/sh -c 'echo \"{test_content}\" > /data/{test_file}'" + self.logger.info(f"[ONE2ONE] Executing write command for pod {pod_name}: {write_cmd}") + + write_process = subprocess.run( + write_cmd, + shell=True, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + + # Read the file back to verify using kubectl + read_cmd = f"kubectl exec -n {self.namespace} {pod_name} -- cat /data/{test_file}" + self.logger.info(f"[ONE2ONE] Executing read command for pod {pod_name}: {read_cmd}") + + read_process = subprocess.run( + read_cmd, + shell=True, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + + read_result = read_process.stdout.strip() + self.logger.info(f"[ONE2ONE] Pod {pod_name} read result: '{read_result}'") + + if test_content not in read_result: + self.logger.error(f"[ONE2ONE] Pod {pod_name} failed to read its own write. Expected '{test_content}', got '{read_result}'") + success = False + break + else: + self.logger.info(f"[ONE2ONE] Pod {pod_name} successfully wrote and read from its own volume") + + except subprocess.CalledProcessError as e: + self.logger.error(f"[ONE2ONE] Command execution failed for pod {pod_name}: {e}") + self.logger.error(f"[ONE2ONE] Command stderr: {e.stderr}") + success = False + break + except Exception as e: + self.logger.error(f"[ONE2ONE] Error in one-to-one scenario for pod {pod_name}: {e}") + success = False + break + + if success: + self.logger.info(f"[ONE2ONE] One-to-one scenario successful with {len(pairs)} PVC-pod pairs") + self.scenarios['one_to_one']['success'] += 1 + else: + self.logger.error("[ONE2ONE] One-to-one scenario failed") + self.scenarios['one_to_one']['fail'] += 1 + + except Exception as e: + self.logger.error(f"[ONE2ONE] Unhandled error in one-to-one scenario: {e}") + self.scenarios['one_to_one']['fail'] += 1 + + def _scenario_concurrent_pvc(self): + """ + Test rapid PVC creation and deletion + 1. Create multiple PVCs in quick succession + 2. Create pods for some of them + 3. Delete some PVCs in quick succession + 4. Verify operations complete successfully + """ + self.logger.info("Running scenario: Rapid PVC operations") + self.scenarios['concurrent_pvc']['runs'] += 1 + + # Number of PVCs to create + num_pvcs = random.randint(3, 7) + self.logger.info(f"Creating {num_pvcs} PVCs in quick succession") + + pvc_names = [f"concurrent-pvc-{uuid.uuid4().hex[:8]}" for _ in range(num_pvcs)] + created_pvcs = [] + + try: + # Create multiple PVCs in quick succession + for pvc_name in pvc_names: + success = self._create_pvc_for_concurrent(pvc_name) + if success: + created_pvcs.append(pvc_name) + + if len(created_pvcs) < 2: + self.logger.warning(f"Failed to create enough PVCs, only created {len(created_pvcs)}") + self.scenarios['concurrent_pvc']['fail'] += 1 + return + + self.logger.info(f"Successfully created {len(created_pvcs)} PVCs") + + # Create pods for some of the PVCs + num_pods = min(len(created_pvcs), 3) + pod_pvcs = random.sample(created_pvcs, num_pods) + + for pvc_name in pod_pvcs: + self._attach_pod(pvc_name) + + # Delete some PVCs in quick succession + num_to_delete = min(len(created_pvcs), 3) + pvcs_to_delete = random.sample(created_pvcs, num_to_delete) + + for pvc_name in pvcs_to_delete: + self._delete_pvc(pvc_name) + + self.logger.info(f"Rapid PVC scenario completed") + self.scenarios['concurrent_pvc']['success'] += 1 + + except Exception as e: + self.logger.error(f"Error in rapid PVC scenario: {e}") + self.scenarios['concurrent_pvc']['fail'] += 1 + + def _create_pvc_for_concurrent(self, pvc_name): + """ + Helper method for creating PVCs in concurrent scenario + Returns True if successful, False otherwise + """ + try: + # Create PVC manifest + pvc_manifest = { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": {"name": pvc_name}, + "spec": { + "accessModes": ["ReadWriteMany"], + "storageClassName": "efs-sc", + "resources": { + "requests": {"storage": "1Gi"} + } + } + } + + # Create PVC + self.core_v1.create_namespaced_persistent_volume_claim( + namespace=self.namespace, + body=pvc_manifest + ) + + # Track the PVC + self.pvcs.append(pvc_name) + self.pods[pvc_name] = [] + + # Update results + self.results['create_pvc']['success'] += 1 + self.logger.info(f"Created PVC: {pvc_name}") + + # Wait for PVC to be bound + if not self._wait_for_pvc_bound(pvc_name, timeout=30): + self.logger.warning(f"Timeout waiting for PVC {pvc_name} to be bound") + return False + + return True + + except Exception as e: + self.results['create_pvc']['fail'] += 1 + self.logger.error(f"Failed to create PVC {pvc_name} concurrently: {e}") + return False + + def _wait_for_pod_ready(self, pod_name, timeout=60): + """ + Wait for pod to be ready + Returns True if ready within timeout, False otherwise + """ + start_time = time.time() + self.logger.info(f"Waiting for pod {pod_name} to be ready") + + # For diagnostics + last_phase = None + diagnostic_logged = False + + while time.time() - start_time < timeout: + try: + pod = self.core_v1.read_namespaced_pod_status( + name=pod_name, + namespace=self.namespace + ) + + # Log phase transitions for better diagnostics + if pod.status.phase != last_phase: + self.logger.info(f"Pod {pod_name} phase: {pod.status.phase}") + last_phase = pod.status.phase + + # Check if pod is ready + if pod.status.phase == "Running": + if pod.status.conditions: + ready_condition = None + all_conditions = [] + + for condition in pod.status.conditions: + all_conditions.append(f"{condition.type}={condition.status}") + if condition.type == "Ready": + ready_condition = condition + + # Log all conditions for diagnostics + self.logger.info(f"Pod {pod_name} conditions: {', '.join(all_conditions)}") + + if ready_condition and ready_condition.status == "True": + self.logger.info(f"Pod {pod_name} is ready") + return True + + # Check for failure states + if pod.status.phase in ["Failed", "Unknown"]: + self.logger.error(f"Pod {pod_name} entered {pod.status.phase} state") + self._log_pod_diagnostics(pod_name) + return False + + # Generate diagnostics only once during waiting if we're halfway through the timeout + elapsed = time.time() - start_time + if elapsed > timeout / 2 and not diagnostic_logged: + self.logger.info(f"Pod {pod_name} taking longer than expected to become ready ({elapsed:.1f}s). Collecting diagnostics...") + self._log_pod_diagnostics(pod_name) + diagnostic_logged = True + + # Still waiting + self.logger.debug(f"Pod {pod_name} is in {pod.status.phase} state, waiting...") + + except client.exceptions.ApiException as e: + if e.status == 404: + self.logger.warning(f"Pod {pod_name} not found") + return False + self.logger.warning(f"Error checking pod status: {e}") + + time.sleep(2) + + self.logger.warning(f"Timeout waiting for pod {pod_name} to be ready after {timeout}s") + # Collect detailed diagnostics on timeout + self._log_pod_diagnostics(pod_name) + return False + + def _log_pod_diagnostics(self, pod_name): + """ + Collect and log detailed pod diagnostics + This helps diagnose why a pod isn't becoming ready + """ + try: + # Get detailed pod information + self.logger.info(f"===== DIAGNOSTICS FOR POD {pod_name} =====") + + # 1. Get full pod details + pod = self.core_v1.read_namespaced_pod( + name=pod_name, + namespace=self.namespace + ) + + # 2. Check container statuses + if pod.status.container_statuses: + for container in pod.status.container_statuses: + self.logger.info(f"Container {container.name} status:") + self.logger.info(f" - Ready: {container.ready}") + self.logger.info(f" - Started: {container.started}") + self.logger.info(f" - Restart Count: {container.restart_count}") + + if container.state.waiting: + self.logger.info(f" - Waiting: reason={container.state.waiting.reason}, message={container.state.waiting.message}") + elif container.state.running: + self.logger.info(f" - Running: started at {container.state.running.started_at}") + elif container.state.terminated: + self.logger.info(f" - Terminated: reason={container.state.terminated.reason}, exit_code={container.state.terminated.exit_code}") + else: + self.logger.info("No container statuses available") + + # 3. Check pod events + try: + field_selector = f"involvedObject.name={pod_name}" + events = self.core_v1.list_namespaced_event( + namespace=self.namespace, + field_selector=field_selector + ) + + if events.items: + self.logger.info(f"Pod events:") + for event in events.items: + self.logger.info(f" [{event.last_timestamp}] {event.type}/{event.reason}: {event.message}") + else: + self.logger.info("No events found for pod") + except Exception as e: + self.logger.warning(f"Error retrieving pod events: {e}") + + # 4. Get pod logs if container is running + try: + logs = self.core_v1.read_namespaced_pod_log( + name=pod_name, + namespace=self.namespace, + container="test-container", + tail_lines=20 + ) + if logs: + self.logger.info(f"Container logs (last 20 lines):") + for line in logs.splitlines()[-20:]: + self.logger.info(f" {line}") + else: + self.logger.info("No logs available") + except Exception as e: + self.logger.warning(f"Error retrieving pod logs: {e}") + + # 5. Check volume issues + if pod.spec.volumes: + self.logger.info(f"Pod volumes:") + for volume in pod.spec.volumes: + volume_details = {} + if hasattr(volume, 'persistent_volume_claim') and volume.persistent_volume_claim: + volume_details["type"] = "PVC" + volume_details["claim_name"] = volume.persistent_volume_claim.claim_name + elif hasattr(volume, 'host_path') and volume.host_path: + volume_details["type"] = "HostPath" + volume_details["path"] = volume.host_path.path + + self.logger.info(f" - {volume.name}: {volume_details}") + + # 6. Try to run diagnostic command in pod if it's running + if pod.status.phase == "Running": + try: + # Check mount points + mount_cmd = "mount | grep /data" + exec_command = ['/bin/sh', '-c', mount_cmd] + + resp = self.core_v1.connect_get_namespaced_pod_exec( + pod_name, + self.namespace, + command=exec_command, + stdin=False, + stdout=True, + stderr=True, + tty=False + ) + self.logger.info(f"Mount diagnostic output: {resp}") + + # Check if we can write to the volume + touch_cmd = "touch /data/test_write && echo 'Write test successful'" + exec_command = ['/bin/sh', '-c', touch_cmd] + + resp = self.core_v1.connect_get_namespaced_pod_exec( + pod_name, + self.namespace, + command=exec_command, + stdin=False, + stdout=True, + stderr=True, + tty=False + ) + self.logger.info(f"Write test output: {resp}") + + except Exception as e: + self.logger.warning(f"Cannot execute diagnostic commands in pod: {e}") + + self.logger.info(f"===== END DIAGNOSTICS FOR POD {pod_name} =====") + + except Exception as e: + self.logger.error(f"Error collecting pod diagnostics: {e}") + + def _wait_for_pod_deleted(self, pod_name, timeout=60): + """ + Wait for pod to be deleted + Returns True if deleted within timeout, False otherwise + """ + start_time = time.time() + self.logger.info(f"Waiting for pod {pod_name} to be deleted") + + while time.time() - start_time < timeout: + try: + self.core_v1.read_namespaced_pod_status( + name=pod_name, + namespace=self.namespace + ) + # Pod still exists, wait + time.sleep(2) + + except client.exceptions.ApiException as e: + if e.status == 404: + self.logger.info(f"Pod {pod_name} has been deleted") + return True + self.logger.warning(f"Error checking pod deletion status: {e}") + + time.sleep(2) + + self.logger.warning(f"Timeout waiting for pod {pod_name} to be deleted after {timeout}s") + return False + + def _wait_for_pvc_bound(self, pvc_name, timeout=60): + """ + Wait for PVC to be bound + Returns True if bound within timeout, False otherwise + """ + start_time = time.time() + self.logger.info(f"Waiting for PVC {pvc_name} to be bound") + + while time.time() - start_time < timeout: + try: + pvc = self.core_v1.read_namespaced_persistent_volume_claim( + name=pvc_name, + namespace=self.namespace + ) + + if pvc.status.phase == "Bound": + self.logger.info(f"PVC {pvc_name} is bound") + return True + + # Still waiting + self.logger.debug(f"PVC {pvc_name} is in {pvc.status.phase} state, waiting...") + + except client.exceptions.ApiException as e: + if e.status == 404: + self.logger.warning(f"PVC {pvc_name} not found") + return False + self.logger.warning(f"Error checking PVC status: {e}") + + time.sleep(2) + + self.logger.warning(f"Timeout waiting for PVC {pvc_name} to be bound after {timeout}s") + return False + + def _wait_for_pvc_deleted(self, pvc_name, timeout=60): + """ + Wait for PVC to be deleted + Returns True if deleted within timeout, False otherwise + """ + start_time = time.time() + self.logger.info(f"Waiting for PVC {pvc_name} to be deleted") + + while time.time() - start_time < timeout: + try: + self.core_v1.read_namespaced_persistent_volume_claim( + name=pvc_name, + namespace=self.namespace + ) + # PVC still exists, wait + time.sleep(2) + + except client.exceptions.ApiException as e: + if e.status == 404: + self.logger.info(f"PVC {pvc_name} has been deleted") + return True + self.logger.warning(f"Error checking PVC deletion status: {e}") + + time.sleep(2) + + self.logger.warning(f"Timeout waiting for PVC {pvc_name} to be deleted after {timeout}s") + return False + + def _ensure_storage_class(self): + """Ensure EFS StorageClass exists""" + sc_config = self.config.get('storage_class', {}) + sc_name = sc_config.get('name', 'efs-sc') + + try: + # Check if storage class already exists + self.storage_v1.read_storage_class(name=sc_name) + self.logger.info(f"StorageClass '{sc_name}' already exists") + + except client.exceptions.ApiException as e: + if e.status == 404: + # Create storage class + sc_manifest = { + "apiVersion": "storage.k8s.io/v1", + "kind": "StorageClass", + "metadata": {"name": sc_name}, + "provisioner": "efs.csi.aws.com", + "parameters": sc_config.get('parameters', { + "provisioningMode": "efs-ap", + "fileSystemId": "fs-XXXX", # This should be replaced with actual filesystem ID + "directoryPerms": "700" + }) + } + + # Add mount options if defined + if 'mount_options' in sc_config: + sc_manifest["mountOptions"] = sc_config['mount_options'] + + # Add reclaim policy if defined + if 'reclaim_policy' in sc_config: + sc_manifest["reclaimPolicy"] = sc_config['reclaim_policy'] + + # Add volume binding mode if defined + if 'volume_binding_mode' in sc_config: + sc_manifest["volumeBindingMode"] = sc_config['volume_binding_mode'] + + self.storage_v1.create_storage_class(body=sc_manifest) + self.logger.info(f"Created StorageClass '{sc_name}'") + + else: + self.logger.error(f"Error checking StorageClass: {e}") + raise + + def _cleanup(self): + """Clean up all resources created during test with robust error handling""" + self.logger.info("===== STARTING COMPREHENSIVE CLEANUP =====") + cleanup_start_time = time.time() + cleanup_timeout = 180 # 3 minutes timeout for entire cleanup + + # Track cleanup failures for reporting + cleanup_failures = [] + + # Force delete flag - initially false, but will be set to true for retry attempts + force_delete = False + + try: + # First attempt - normal deletion + self._perform_cleanup(force_delete, cleanup_failures) + + # Check if there are any remaining resources that weren't cleaned up + remaining_resources = self._get_remaining_resources() + + # If resources remain, try force delete + if remaining_resources: + self.logger.warning(f"First cleanup pass incomplete. Remaining resources: {remaining_resources}") + self.logger.info("Attempting force deletion of remaining resources...") + force_delete = True + self._perform_cleanup(force_delete, cleanup_failures) + + # Check again for any remaining resources + remaining_resources = self._get_remaining_resources() + if remaining_resources: + self.logger.error(f"Cleanup incomplete. Remaining resources after force deletion: {remaining_resources}") + + elapsed = time.time() - cleanup_start_time + if cleanup_failures: + self.logger.warning(f"Cleanup completed in {elapsed:.2f} seconds with {len(cleanup_failures)} failures") + self.logger.warning(f"Failed deletions: {cleanup_failures}") + else: + self.logger.info(f"Cleanup completed successfully in {elapsed:.2f} seconds") + + except Exception as e: + self.logger.error(f"Error during cleanup: {e}", exc_info=True) + + finally: + self.logger.info("===== CLEANUP PROCESS FINISHED =====") + + def _perform_cleanup(self, force=False, failures=None): + """Perform the actual cleanup with optional force deletion""" + if failures is None: + failures = [] + + # Delete all pods first + self.logger.info(f"Deleting {self.current_pod_count} pods (force={force})...") + for pvc_name, pod_list in list(self.pods.items()): + for pod_name in list(pod_list): + try: + success = self._delete_pod(pod_name, pvc_name, force=force) + if not success: + failures.append(f"pod/{pod_name}") + except Exception as e: + self.logger.error(f"Error deleting pod {pod_name}: {e}") + failures.append(f"pod/{pod_name}") + + # Wait briefly to allow pod termination before PVC deletion + time.sleep(5) + + # Then delete all PVCs + self.logger.info(f"Deleting {len(self.pvcs)} PVCs (force={force})...") + for pvc_name in list(self.pvcs): + try: + success = self._delete_pvc(pvc_name, force=force) + if not success: + failures.append(f"pvc/{pvc_name}") + except Exception as e: + self.logger.error(f"Error deleting PVC {pvc_name}: {e}") + failures.append(f"pvc/{pvc_name}") + + def _get_remaining_resources(self): + """Get a list of any resources that weren't cleaned up""" + remaining = [] + + # Check for remaining pods with our test labels + try: + pods = self.core_v1.list_namespaced_pod( + namespace=self.namespace, + label_selector="app=efs-test" + ) + for pod in pods.items: + remaining.append(f"pod/{pod.metadata.name}") + except Exception as e: + self.logger.error(f"Error checking for remaining pods: {e}") + + # Check for remaining PVCs created by our tests + try: + pvcs = self.core_v1.list_namespaced_persistent_volume_claim( + namespace=self.namespace + ) + for pvc in pvcs.items: + # Only include PVCs that match our naming pattern + if pvc.metadata.name.startswith(("test-pvc-", "many2one-", "one2one-", "concurrent-pvc-")): + remaining.append(f"pvc/{pvc.metadata.name}") + except Exception as e: + self.logger.error(f"Error checking for remaining PVCs: {e}") + + return remaining + + def _generate_report(self): + """Generate test report""" + report = { + "test_duration": time.time(), + "operations": { + "create_pvc": { + "success": self.results['create_pvc']['success'], + "fail": self.results['create_pvc']['fail'], + "success_rate": self._calculate_success_rate(self.results['create_pvc']), + }, + "attach_pod": { + "success": self.results['attach_pod']['success'], + "fail": self.results['attach_pod']['fail'], + "success_rate": self._calculate_success_rate(self.results['attach_pod']), + }, + "delete_pod": { + "success": self.results['delete_pod']['success'], + "fail": self.results['delete_pod']['fail'], + "success_rate": self._calculate_success_rate(self.results['delete_pod']), + }, + "delete_pvc": { + "success": self.results['delete_pvc']['success'], + "fail": self.results['delete_pvc']['fail'], + "success_rate": self._calculate_success_rate(self.results['delete_pvc']), + }, + "verify_read_write": { + "write_success": self.results['verify_write']['success'], + "write_fail": self.results['verify_write']['fail'], + "read_success": self.results['verify_read']['success'], + "read_fail": self.results['verify_read']['fail'], + "write_success_rate": self._calculate_success_rate(self.results['verify_write']), + "read_success_rate": self._calculate_success_rate(self.results['verify_read']), + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": self.scenarios['shared_volume_rw']['runs'], + "success": self.scenarios['shared_volume_rw']['success'], + "fail": self.scenarios['shared_volume_rw']['fail'], + "success_rate": self._calculate_scenario_success_rate('shared_volume_rw') + }, + "many_to_one": { + "runs": self.scenarios['many_to_one']['runs'], + "success": self.scenarios['many_to_one']['success'], + "fail": self.scenarios['many_to_one']['fail'], + "success_rate": self._calculate_scenario_success_rate('many_to_one') + }, + "one_to_one": { + "runs": self.scenarios['one_to_one']['runs'], + "success": self.scenarios['one_to_one']['success'], + "fail": self.scenarios['one_to_one']['fail'], + "success_rate": self._calculate_scenario_success_rate('one_to_one') + }, + "concurrent_pvc": { + "runs": self.scenarios['concurrent_pvc']['runs'], + "success": self.scenarios['concurrent_pvc']['success'], + "fail": self.scenarios['concurrent_pvc']['fail'], + "success_rate": self._calculate_scenario_success_rate('concurrent_pvc') + } + } + } + + # Print report summary + self._print_report_summary(report) + + return report + + def _calculate_success_rate(self, result): + """Calculate success rate as percentage""" + total = result['success'] + result['fail'] + if total == 0: + return 0 + return (result['success'] / total) * 100 + + def _calculate_scenario_success_rate(self, scenario_name): + """Calculate scenario success rate as percentage""" + runs = self.scenarios[scenario_name]['runs'] + if runs == 0: + return 0 + return (self.scenarios[scenario_name]['success'] / runs) * 100 + + def _print_report_summary(self, report): + """Print a summary of the test report""" + self.logger.info("===== EFS CSI Driver Test Summary =====") + + # Operations summary + self.logger.info("--- Operations ---") + for op_name, op_data in report['operations'].items(): + if 'success_rate' in op_data: # Regular operations + self.logger.info(f"{op_name}: {op_data['success']} succeeded, {op_data['fail']} failed ({op_data['success_rate']:.1f}%)") + else: # Read/write operations with separate metrics + write_rate = op_data['write_success_rate'] if 'write_success_rate' in op_data else 0 + read_rate = op_data['read_success_rate'] if 'read_success_rate' in op_data else 0 + self.logger.info(f"{op_name}: Writes {op_data['write_success']} succeeded, {op_data['write_fail']} failed ({write_rate:.1f}%)") + self.logger.info(f"{op_name}: Reads {op_data['read_success']} succeeded, {op_data['read_fail']} failed ({read_rate:.1f}%)") + + # Scenarios summary + self.logger.info("--- Scenarios ---") + for scenario_name, scenario_data in report['scenarios'].items(): + if scenario_data['runs'] > 0: + self.logger.info(f"{scenario_name}: {scenario_data['success']} succeeded, {scenario_data['fail']} failed out of {scenario_data['runs']} runs ({scenario_data['success_rate']:.1f}%)") + else: + self.logger.info(f"{scenario_name}: No runs") + + self.logger.info("=========================================") + +# Main function to run the orchestrator +def main(): + """Main entry point""" + # Setup argument parsing + import argparse + parser = argparse.ArgumentParser(description='EFS CSI Driver Orchestrator') + parser.add_argument('--config', default='config/test_config.yaml', help='Path to config file') + parser.add_argument('--duration', type=int, help='Test duration in seconds') + parser.add_argument('--interval', type=int, help='Operation interval in seconds') + parser.add_argument('--namespace', default='default', help='Kubernetes namespace to use') + args = parser.parse_args() + + # Initialize and run orchestrator + orchestrator = EFSCSIOrchestrator(config_file=args.config, namespace=args.namespace) + + # Override default test parameters if specified + if args.duration: + orchestrator.test_duration = args.duration + if args.interval: + orchestrator.operation_interval = args.interval + + # Run the test + orchestrator.run_test() + +if __name__ == "__main__": + main() diff --git a/test/e2e/stress-scale-tests/utils/__init__.py b/test/e2e/stress-scale-tests/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/e2e/stress-scale-tests/utils/__pycache__/__init__.cpython-312.pyc b/test/e2e/stress-scale-tests/utils/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fb113996f76cbcc85af4ac5eb7fe86b0f6012f3e GIT binary patch literal 200 zcmX@j%ge<81V3NdW`O9&AOanHW&w&!XQ*V*Wb|9fP{ah}eFmxdm9C$YpPZPZpOK%N zs-KyZkywupIBb3iz=>Pl3HA%pK6q-5!pP83g5+AQuPc3YdAk4>SR5ok0=!9ZB~m0IUe;k9G-@Cu}-K(FLl(;B(+Df+{OKzp8UtvLiY+B}C95QDqp5o~^HKP8~ zBQz;nMl9slI$|ZiwhsjyOr#7I%%gN8B`Jp>|U| z^A^S1FI&{nMm$q=$niNAhC>#GT!318G#(SilL~V(l8hb;(TbZF zjzlKoNj?@$M%9kgRO^+x_m@yUO9>Rt8|F;Id@XNLFxyJr_7*+D;4DV$yd6>pU&TA1 zj)iwD{5p9j)Vla;-UTH$?}m~GN**Yc@Fh?xEFO z(f}WTw1TgIv~tQCs#SdZgrpdYO8tp=T!D2Q{f*x49 zthLR(a+{q&7FJ3jO_P zrJl?ECad+lK4%!GM6a8&ha9Q;J>#)tEE12Ua{C+A51{~oa98~Bch3U}Tml;Yvz)ga zq(atV5jPOf(IHJnqk<$Ujw6v+d{PwNr^Hfh1Ao+)6W#GdG!pMVmKYPdV~3ALM7~=% zB6UWkSSK&WP6%S>k%V}>J91L$EUfRwv+fqw3*Ayug#J4v=tthe{Qfu~tbbm8PVmFc#K!U2z}ZBIOEjpn5?>v{tMzVWm8p7*C3bTcZ9} z@r);U!6>=m)Qy4*yVNG=h$oI7g$XLoqeAjY$P1#vhQs3#*x9gR$G3v`KCRd!A$~-{ z`^kwu92kJqQ%=>ahyZvYr1v0sle%rE>KbNkr@Yx9H*34W)@8AHgRRzoZ?hD)`X<+Y zrGJTgW#tWsu7@G-g*fLuDOQp7MU-p{jzJ%2v6u^$NRGxo+0>7jA-r!%>4$!F-V| z5i0lhkU2|D)34D|9k3NTX;8OmYBawEz~Q4)1KyH?(!11qmKjSDX(o9_b#|v$!0%-( z4|K_T-eTVCw51oI;GIFqave-t^f75_8fk_t6+eg=#;Mo}RGu@?%f)Grk?9kUNpDr1_42xB^kC^W}D}qy}`C;y@6Q|!JPusF&N|s>lilE7#1(=@1Ba-#!?G zNC6YSFO*#y$zZU?K(I%=;|i-wXMo10JEqOYj-vsZJ5&ES2=a>wk`uW`6q1wTI0`g` z#vABrWR@d7LAM_6DFfU03~&jN+y|o-UJ#>jfQ(UeeFn46$!PGUE7$p# zE^a)L**N~KCSRB#E?5Aw&PpJ$%r@(bX$BBO5!F+ zLKbp)#4V7QkTmIcrUd)V-I(CA8&mN9acurQNZ`JdQ=YOp+Xe4Auk3EkdVKHfeS2@l z(DA>*|U1S zNA`4KKSo8b?0Mvll`dI#+X}bKxND?k!G&A2rP<~i?4!55{&(JZ`;ECt*}G8p191bq%g1AJ zC$>NwR_u|93B+<+4vobD@shq$S;X#xlEGAXA45f1~Uw~Fp1QNjK(g($7 zPx%*i3V1=yxte$DvQElb{_-ct0$Y%>nR`%pLiLGK+g_NV;K8Mc_b^wKEL68^36Op*$j!sO5feA&?{7^b3j3vY=qwM8l(($k; zMELOGDO7`eq~fF)OA2buaLB3HCnQvG#3x~Il+qJ1&^ID+kpHnOtwk)M>Qw@$MGk#5 zS6;{Nygo$^JF-tsps6Ip=H1X8A$j%9ORpWi3l( zZL{{9<<*(;kX#;GDsP{4fDYoToIQ5>cp6yI-mJF*^5bdGn)wr#UcdOdT27=rEenoJ zTaVn2Z{=6Dh#kUtC@5L~miNO~9d83(Tn}$o=T&Q2IGg6>wR|;MtA)^P_h@K2 zgwCBns&|i~T7pqo(L4DjC$lMqBh;?zfo|+hP8xhvAN(&>V>!?cw^#udeH$OPsWnR^bSD* zN+v-`$!S2tO1UNq>U18ODGp&gayTyVyA)rZC2JxfCZ*VEK=IfaNK)?OlZS;)m7%99 zL0a6Vn=*5aQ0zqGfiw|`3QB1_a#)Cm!E~iszPaH9H@NE{hfB}3;lL;-tdQQ1nB9ZM!Bdb(9Z?6HKD>`Ae@>GX5d zM}WsWP$GtwpI?Kr!eH;=RP!z{=nA08CSXp)MsIB25iv0avjbaB%F@H7ng-E|sYAiO zngIrv0pV)3P4D4CRf-iXeIm|CarT4p|G9O26EVdfkAd;k*g76ep}fL?P%$PEg_8*C z3Jd*2Kn942o3Xu1nmjxqCWu|y2?kM+N_3S}VfKT{Ij-3D@7X`7Fj8C)CKOgP@=A&u z1fb-k1W3ROU=$KwONvNz6yE{W3_SQ+R3N4%&>{p8EW03gD7KT4SWO9}sFhAe&@8<+r>Q@1)*NrE9wuElb`F z8SiG)kkl5_uy+^E>NGe7m#0%3BSqE{vQTS)k9oGCQ2DT9c{jlB>EhRU7204YNb{ z>{LzgwC%Q~q3k94SAjL31)4t#G=iYj7|H~@FvYm z{m-PIi>9mjY-8xc8|U88er`Lc#_oHRt+Dd-ZV<348cz?Led5d$bFY4L-)%cn@fG@J zQ~Rafi@oXf{Z|LBZn{>Te&j&<^!}YpN*&VxoH1mU*xq(c5 zhg{#WFuC~5;;w~5a(yo>wxT{0SR)74+z7O019fMgJoDs2+36=g4Yc3c3~Tw_7adf6 zbEa;iT(|KqMZ>7s>iP@rb8c1EOLrYeAACN2;Dz)HBk7l3PLI5j?l_dG4$IZy-`}<( z`0u|crK(p!Q`nF!Z0z1MdozJnInbJ3yYCu*?YZ>9XVcF;pMLg*blZ!Wz{vH$2sQ-Z z--9sgtAU1acm=kW^y-~g%dT3}97vd8MHxhRfca9I1oZjP(^tB?m-^SbwUB;f?cs1T)ApjFZ1zc2J3qnsQHApLjDt*bN5!}laf`t zH!`1e*#Z8OjSSXowPL>4In-&r#sr7jtk>EY$eSBH{{yCTUt7y6VA)73i=GEHc*el& z2R#(E#-eIJF%vy?@ngn9lu;8U_XH)DL$4-p)Ly48yp@PIs%`zdyM?4qO@U{K=4#&m zf_iCmVQyv2G2EOv#%PkW!vK#W?xbZl7sVD$OiU5IPP4ib!=b9Ip%e;4P9<*@z%Fv; zL*0p#YFS3$OyPbET}iJ(@=)f)irGE4$}7)ypXr`oBbT>j$~)!q&ZY8hqQiO0P7kP8 z3aye{bB5a>a~l>1Wv*v-_vszI0FHa2VVA3c*irBd#gkjjY4+Jr9n3(^k^MuwF#;{;p@Kae{BRSrh0<|d$ zHflg0fpRHHK(9-)x#asonOoS zSH{Fh8`YtnZ^OxVbrEN?zzccS&2rdc(T~YGOwj5jqH-z*AQ^^hL~QZRSThJo$gYYf zrN-jIxM*VCR~v5!C}Il&QAk3Q9nMR?chyPUX^@Z%*RHnDsnD{kw-V^jp-*W6k~gU@ ze8}dvp4s}1$L4EgfBP&8T-sARAIPlQBCi4&;t6@xUfI1jyJs(&GMZ!$$N-PZo=uCd z%bs0XG-x!+Y-5^RC$sApx6AC~8Fr7%?zz@>gFOgcp>IqDSTD017KfME{;RLb>=69k z0db(^OwGd9S-a8Cycf-;L7ClowQ`9ay4EMN`);s9%WbH5VacU^4l-w{hq|<2k5~cS z$)#1VV;a$7%y5@O*UzrVtEj3!v_g zZAEh|Z28dC5u>EUDol_%8W_6<3OXfp8mklEKtS|!<|yGZ3Rt1yq#A}}5K$A!!xZkn zpmS*o62RQ@V$8ke4xo0>Dc5%{jLY@gWcRl0W80AMHGupuH!gcR7h%MqIGuZd`mpX1D*+3+kG$0Dj4{<%2%iv+agw3%E0VRm&w1r19h;d;po1W#uB7 z0$__D6~d4yieE!nhd{(0#W9PX2a8IXwoF41ERisX2loE}oe<%LuksAOc8 zQ%nOF3Q>3&rssJ>!0fdD+5B-h&2bSqT6jAOohCgHB!ZrgKh#phpgv&Z{qRG48R9<3 zA!A1*COn-`VckV}rU0&p^afO>B%Jpr ziTV_bCL-RS;rBK6i^gc3{d$W=Uy%vdshCgzu8))ISOFnSp&Jk{R%FKgdy!OC%WQSJ z2543HqW3zxb2$w%&i8L%F%QX%A3zIwAZ83vy!X?TaL?74HzJ&UsN#ysu5#g+4~!-c z#w#aIM2ShTkR0uQuHV3gKa4&raa0j~K*Qp<5PcZcdx{KDVf8`tkOu`P97z-5Dpl-J znP6`5sn!(`rx>smappzn(@X`p<@V2Q%+$5Zb?xbnLAh?1?B10f*oCyEUiQ?d8#c+F z%^$F`r|;Gi2Xfd&B>W+n-S)w;>+J63GP}X%;xI0-Ia8yJ%Ao!T)oK|zjituu8i#3QmUAXU)FAqrnUb137D;j^Bk1oZBjjm`cyc7#7%U}{ z1dOSNQ!VmZkt0TZj1zzv8j}#~V5fA}^JkD(N(wqbCWOQ%j|r)SIO(bi)gn>?~%c!D;#meOc>)c$^X2ik<=Ll{sRxG4(W?d6{|32}+_* zp${7M`l5%o5-|#6)r+2FnO0wnHX(F2fZcPM(G76tL52WnBa zDS99z&93hx_+GyazE?YVJ`I00cs|>R!Kl8dArwc!BUk+J;A_W_=UkuQiL}o!uQOBd z)eHXSc_JODKi55xs(RWfLOgsD9ASIGeIt&8tw-Y2Aa&!G7@6gF*6>{Qk`YU#@ zTS@5C)b6{&kk1!V8WqsIK(L zxl*S4vDE2n-w~J)7E`!>BrKmozd%?#UguD;o2r7#RBVzfHeIbrS8Q6U*gL!HR!!rD z`t$XtnXJD$<8PDwZ5e;3?C(skJ9OQD2t6!Moq1~hDLJq)6L?GxJa%(U=+cW9U;NHX zS#C{+dsOBg&2T+n8ZIubZoSlTu_N2mc4^ziZP`^Vm+CIo-6?TwtNab+sHnQbP_-?; z`y!xuWQK}8GLK(fceN?a?aFv|%bwlDBQpdZ8N3vK(pS>&qdxTY?SP+;e9i&d`cX}= zzt{RvF9Z3g4Pf*Kc|9SGNNa&nt`Hxp>v9+ZK5fC^5;SSR=Uukc4^`P{|8t!nsc_oY zCJ!zdJ&?>Zal%wEGA<9YJRF$>F9W*-lNpzc92drkgG8T`Nj$XJ0LxJ5&f)iIg1)+9 z^=+(nXu~Q_qi=H+3*bHlou|f(v&Lmy z;4>!+eCA}IaU{{PL7o-b7!kL6+Zh((-z?w=W55vxF~f-I_`8ziH0_vXdG>ssv~cR8 zP5p$)nfl58h$so%eh7fU0NY6^=X1rF7{uXR{E`a^5I#m``IUig@GICbhMM$n2O=lU z`o@<=A4cW@9Iz8OTV9`|(K5B=P}@t9&QxFLNVw+}E%!+0Rw%3uy)3P5`|3-FW?or4 z6WW1=m*7p8FxcMtN_$A5T?*Z#`k8pxNQL?wkqU#4NKBKYFc2y{39+~kTQd=duowou z-;7OYUtmcXg{4?T_=su@KVMduqhexmLSc@?6Op9qopP$)DNp{+D5dI3bT2OuVQ?CN zG^W_#OGiQc5U1i+TM(Wg{u9)QKZJyQ)~Ox?q^kR&n)DWiXd3Ml^?prUSSx`h0C}ZP zU;wz%jTd&D+p)0WMqPK-SN-ci<=LS#Lvzp1^GkvD*+In{$OfA(96xt_Vb#0ir`_3_ z;Dx$#b@NZZ8@yiAxv>7U{bqH;oS*|NRd=3dZux`rs?=e{cY z+B3fOvTuDh*qm)x4c`oGEdLEtqSrjNlg+^gh{$Xw~ zfm$2sqacA=6P*ZZ&FF|}%q_s0XhBeG5=qO`=UMn3%aS^bp%IaITn|t)xe&(9cz`>F zJ!!aUG=tD{6tToVg+y^-oR_L7C{~EQQd~J-sbWKmDttC|L{cRSts#f4S7Cx<tul z4JIv^5D~fqa`_*+dNHm6CS%|(Sd$CCsP8I*_$x%&1JDhiY$;XOc%l1Tceb)NQyG#g zL)qG(8kYrFn(;Tu{w7khPOe=C!mJval>Z7F-zjtYT(>Ew(?zf+L=T7PT_GAj40CAz zDnY19Tni&Au5frP!B58V>s?Pc{3^u#qY?xqM>xzUqT#UEg-hLvNgpP=G1-L4TbRsZ zatf1oF!?$r-^2tIYD#<;lfTB~dzdU@@_kJH4U;q`h$3W~ga(L+-6U~AM#31ox2T`n zw%)a^Vb-TDBt{%(ht*?MnvHM8N~#vRNq`tIWvKT~yYZGai3?{=_|-V2m5 z+wZn`8TZ}QEb}CNH&DVHr0)jZjQd_~1v5n7?Lx4-0VlJQzFXPM1n)gbdl_}h$P7Xy zs%o?!r__dtgvnJ*kh+R^m&Bi9g4|ieO9WqIs6m-vmzKyU*XoK2R|%iPtGm8k{24UD z+bv-*G(>9A^ks`N9;`x)i@1=acss`=lkrr+A&pRRD-g?|!)Rww!2hF;<& literal 0 HcmV?d00001 diff --git a/test/e2e/stress-scale-tests/utils/__pycache__/report_generator.cpython-312.pyc b/test/e2e/stress-scale-tests/utils/__pycache__/report_generator.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c7b1337e55e171b83692dacd49b6c4a0c50aba95 GIT binary patch literal 18806 zcmeHvdr%wMdS|!ZFF=3*fp|0zVT-pP_yPEVjd&RZ25gV7=gaI+#!ZwPyxtrYV zGFc~APEs*WGG)!yY(}n`)X1r9hW(FwZ)Mim`%kwe2Hn1Ft*iEKva^4A?5UlMx3+4( z(=P-_e$39@)YhgwbJTs#cfS74*QdY7@0{l6Sy=`Qj&J|_v8llh4ErzWLON99$qN!r z?qMWG;xpK|@QaV*{Iz0S!T&19mHe-2T*d#Y$JOwwn9nqH+w|jl1=fR+ z${%5*>SKj)mvO@)?ofY;E?f?rQw@6pQxigyP>OWei!r#qhj}q10o)yjJ7<%MAK`Fk z;0X6sL8&3tP-;mHlsZxirJmG5X(07b8c73`CejFH7HNVqdojzA!{v;4=lxWm$2;ex zJOMuy+$WxT3DO(z_-4E$5%AIhg5v3D!Z%0U^iV$k0!>W%saa2eo{;23QX>s2;r|!? zP~5{pcnEXDnQ#P93x9<0k8$B_S}j(k{E#?_f1;4|gky_Jhbl<-&G`a8&x|kV6@?+L zOHvyM@sLkCsq1u`e1%&53xWBCfSdGDZNw=L4de4s-U%4;A`$QtG@M9$X}RCnLj|Xu z_fX!s0L|&Vw|#WL?Z4r2D7l<#z{0)7lR`t|)U!8$t3$!5Xz$DQ0-BuZM3;MxQCd)dL&GWjG^d(}E}UxG@0+8Lnd7G=g*4?* zQYQG}^s?Ag7F^S4-hv<$ngm=9%pI8F@|zN@*fnRi6BHp9U`1i-e4U3CTJK%Bd*R;Y zyO$$18&!wcsza;#c)_V|m!u%D0mp?&gBK z=vCOF(P0Yy$1ZQcOU?S`y!^~T51c^rl$fOavwRiUkA;~44OWj@n1G4*=JQ?b*+z8v zCIUYHoCnP3gnte^#hh>My5uT&?(z(M{X`#=MFfd}kKC2ZrvsjVcXDBdhKY&ZBWUG} zDV3bXGcf_?&^-@xev2e4wU*|L^L|q9o_*6lvoPzGuZ=11=Y%|o;E^I{b^!i=aU6;> z7>*nocBKF&08Gf~R7z+^2Z>43=}+)`IF8*Zf=7b?rScXIjzsD9`CSij@FR01eajb^ z0zaW>YT=YOyo;iH5Z*$42eRO|=c(ht!@J%;@XErob;~buQwQ%;=;KXt$AcgYT?)m2 z!~XGy7-Q~YTTVyw!}<40@0PB3H*#y>`r8+1e7|ig!#Ry$tA1UBrwF(aRRx7ZLDj$y zhu=sYF;xq7-+8Pjp)*|qg)mlY=c9q>yDPa5O753LoEyb;j|RS__CPbI6&&M?L(M5Y zbBmk`DV|gN0^V7gGx%uV9L%e^2`{H2VRAcEg26QLdKTwBWk-zEH$5{8UK*(n&28bS zS=?zJO|a%o9w1T){V5dSfGwD*GBUs#Tax)iGOIcgVzUk;^Q)3sB`Y&*RvpuHe64|P zazX26mYFqo#Y`J5ooq{IqUChl+?B}cO6J?1X)xQKjgkhoq~VK_ruX`nPlpRObBb5& zA6nLPYLoeUqfRz|-!E)?lLhtB1-9VeFKl(mf~ts@EvRRjPpq}F&ArgNSzu+YoiXQU z{g3+pdLVA?TrcSSg{}HoK4xovR*0EJ!=3sbAVbI-?rTs&EFia~iv=XZ8jy|dG*an~ zBMnuWXhEiuBZX-`%-_yTrb4cdKei%^Y2*QS91{IB@=!Yt5)T^TjhDmJK%PV}EuTp> z7&Z-7W9K1rM_zQtA+?dpKi4ebdMr>Z5lo}0zT>zKX~n<616HYe8ol~Ej!)E*bDFN$ z(FB*IMzUN>ijZQuayNw}Q3xls$m4@d=&j-pQcoHr?yVR&M~BohT_>Gw{f3nCy;LF9 zbmLZwbVeF|n8Yt_yNT3E?I2GUk|zVF?ALJ0*~ZC?IFY&NpTr-BP+}CT#!|G4EZL4D zfZk#Lm{inYlnxR=ousLY*qwW~u-nRu*e$#o+lSHkEeHxOfOD%yWi3f1V7CG=^ z<9YG2UF4;axH?FW+4pl%+5o4rMh|E;-T5d4pfu`8O7CYj_^_vwnAD-W^x3TuNxV>Y;eu}l9TD#8LUCi*=c*V$O zaV5kCY;oO2aT8nI6fbVsC~jkm+v3F?Uu9{F49mu67PY=@vwW|>{mG#Vo9)hx_5rqi z;Ay3Q2uSH4le$- zw!B-b{9J>h(xC3PXg|+ULjC6!zAVb`Zc={Uh(kG%nxM%k-yg&C-i9K;-F69+UU2koOA?hdM( zrfFmiMFa)OK_P`x?Wni{1*a89#^JZ9R;WO}$~Q*>dgyYPC;$&IY6#Kkz(!9ZJ5QnP zl0pxQLNOzS9xY`-g$EVyKmlM5+!j)j1#pZ8-V{^&P@@+W{JnW8P(;h=kaH4zFgoMG zAaHKT`%?&prO`a)ZF8!DmbBnTcLM*1$afp)bW~G_=g*)318unf(3By3PE8dx!5HzSnWLBVvg5#`9a3b7* zuZ)^J!?NaUGnSnf-n()-kwrv0H%;ar_r2E_o?P)n@pV((H%ctK5@a!3mvg{w%I7a> z^ydDwr1;b^As{m0;X(2lWJ4c8y=ja$Q;v60t8EYe6Dh!5T{^-TgG3OxB=jt9B8SEVE z8*&+f!;Rnb5j6CQXhs`B9YVu342z%!RG{QOXddRV4dT3jZ9p|j_k+gM!{><82;zX6 zqo^i(kViU*F@$tLWgDtg5k032LCYB#x=7)30uh)xFN_i+JT3u!P|6j|N6JBIya@I5 zbwNuHYB7(Y1(Y!fK!Iu$7zVRM^^Kkz7#lTE)7SU z+X_@E1n|5`jHyPAPHom;CyQxaOD#II27$VaEX38@b?43fVtT&8Bw@Mckg!uQM(a*-VSif;kjh7XeWjPSprmGDzd z83^A4_A~;@zV|qJfXP3aK2CmGSh7*r%oaAs3->J>f0dOhhr~}!SwHT1uP1ynJhgHy zJeV+5C50NMaDVLRXD1$=h#%-;_B+=`7)x)$)F-y`^kjG_VXA@nIV&6SU@UbBQ++DI zHiqG82<6Aw$|DSMbTyB$bRzBMosd}0MJ_S>0OvW))O4*m8B2e{bVh7r3hScn%;7U^{Qy%p_!veuoG_i0I?hDe zqan7sgQ+^U3QZ>yrcSw^BU;ZK>SOEr8ONE&a6?zZG$c2Ips{v#&k3gX}&dXihq*CYHUx-UAjZf-~Q z{wKMOe0>W5KC#@`z`D5;)z3c3t>4M#DdJntn(L#T(NMzNA&V5gwoTaTQ~MDdLi-w^Mj@tid2G9S%T*($i?S@R}uE1{&D*CG$aB z8pNbV;4GA+A%ub>qA z;O}5l!}}Se&j<8O$FIJb;Xans``r0Vh0PUY=vTlac7>_*WRW=Yj%Mg%O-T*Xk2l@S zGW!M2&VShiR;0qcOsR@{P8I&+i<_gz5Y6p^wmW!uIYX-k+8gb)bnqtJqH zu*5Jt)J2S*?i}cI^+>U^5d6K|`=${x^)9gDwDbbMt)?3mr50dkC@cg~$T)EZ*hzyH z$vouZOYJDyg4jj`H3X_Pyo97@6oRw`x{ctpqvuYac8-p6n(ofN0cY1s6G3!|tRR+G zHtnB$sV8VywSw(1hF~^93-|^#nZB64X#XK*evN zHfev7)ZzD_l1C@EkXv!n3AF%C!MdGMs1##wZ9A0W;>U>9G9QROHCk7Co*1i>0Fd^- z*UuESCQOIKiO-Q;YRp|K0vbQM4ecjoxA^4ks)}l451a zzb|^&w>HJp4?U)sijjn6RB9=WPQ~cx)zvnpqB~*fk*dpMhG+<4#fswz%ZX%BMWphN zLP=}s1H*kon!O+Oe$X3ruX^G&C&Po=6&GYG+7gy_Ne^$U zIHq!c>{zUMEsv?_N?4q))Z92zeQDi#8EI~Co93=EH3wr|jP2;Ux$T()Gwu2Ic?D+g z!@qsrfY}FdeqR5iD6j9B?sH3Cze2YSbw>df3f^9f^I;;O!D)h-t?(?YxN=Y2p{ayr zytF$&(XPyNON0^^Fw=@uvz~zqa{72`xS0|}s&?-O4J)XFBy4tp{h)u7fulMl&`^;o z-eJQY*K`;e?0HpB%fTn;M?A=s@Q{~pd$1;l!4f@ebAis0J4#j3%}@=X(BIcwPtWPqde`$~QQT?HAYc0VuK%aeeOPhlcQVLX6jJT9 z2|)d)5VTqy#lZ$K?+A-LwuwD;WwSnGywbc6sTXf=$=RfKQ@ z1XQ*esDFVvz&qaq93o|}1Uvo#qC(*xzf_Ml znIQ-iwsLvj6~i$O_i-9wY@88^Dbh)^HuKs zsO*EXk1AMG*>Yd_rWCg1t@J$@x<3>zZDeds(P^gWXu{McHEzX%zk)!f_QRU>oIS~W zd$f?vZx%xfrlLJz=}1=YVfMDMmF?kvIpm1;f9iVZT0OFMG2Sr5)DJVmBTU6;!ZIdN zlt!mNoqIU9I{vsI-ZaEC0>wB}aVcTB45*dW9yPE<`^8|5scubJ4{bX3$J$nBVH?yJ zj=}qVE1fGhlcmIi{`>t=TkLG?-AAJk(plGP|3lqR>KhxoKylI$e1d2;c#pQ(MiZB8rTXbO82-;Vd*aL%KI9%Ux1Vj7? z$|HsG{5|pr4kj$E$!bTmj;(GD4@mNrLUjx$;GqvHjC$Nfg^YV} zde6Kld%7)Z)GeS6)@Sf`sS1$pR1J@j?uFuq*i&O=>3ZY0&$BUmFHURV5kQ{q#I|SXke3S?ddqf$oSWfQ`gI!Q zpFn1xW^3=Zg%#Llo0cZkK*CZ@S{m3sE9JyXnhZ3C6dIJ*rZXo-J`z`@=F}M|O(FGL zFTOy|{KktfTF3~t%b_7PXz_(A$siHr(;-dzr59upGxW+}Kpj>I>PDzB`pR<@eW;E0 zGKisXI;b0c7QPvvY@GyP zZ&}KqVGFmWwhX2z6|WAOQqgK^8Q7+nM);O+J9I1n#X4a-ube1AZV*6b5akI}6tR#k zB##W!DEyH?VE*e0>Nb);BLKX6!$dnWj1{6`Bq9I+!|J^V03S=3j_(fopCZF}h%G*x zY8b)hJve^SVEtR{ za5-8REsc#Zwj&AiQTYCG#q)<9k`YCi@NKr~#F~<6>|(l5+`Dc$4@rQ9NR2&H&7K`I zHS7*OQ5J?m(HCEX_TWR{*I{px$d0uFyqErh3Oc>;QhtQ9Q$ z^5ad#`6^V{olEI^@shx&7JMIhNgh7ThL3meJju{Uz)4a()t!F;ltkmu1{;MSPuy(u zT1$ms5&d`;e%vq5vqc<8GR4}DRh8%Nx%sH;?)?zHKu4^!nA+4xezge1YiVJ zdp~OUpy4l??)HaIuM}(+RYvTeSQ14|(%Q>JGi4^qBgxk#--0oRFxiEvy zO>XzQ@QI$-qIJ7T|AgC3p>KJFFE;pZH$3Rlj|v1-^XqXuHp7RkymjWE!$XWWD1yyK zF%yLf1Z&c#p!gB?3sw6Il~&#HJkO{ud0tYXE`ELruTy8ZcO zyk5QU`8ZyvKK#7JqCWhhAxB*&ut!WCMrs;zH7qY z2I)ohXHmh6^;hW3jEZb1I9-RJkK@!|!zFr!=>{m^3o;!4C1(63rvD|D!~d5Bm;VW~ V{t|_)CB+l literal 0 HcmV?d00001 diff --git a/test/e2e/stress-scale-tests/utils/metrics_collector.py b/test/e2e/stress-scale-tests/utils/metrics_collector.py new file mode 100644 index 000000000..743e05f69 --- /dev/null +++ b/test/e2e/stress-scale-tests/utils/metrics_collector.py @@ -0,0 +1,420 @@ +import time +import psutil +import logging +import requests +import re +from datetime import datetime, timezone +from kubernetes import client, watch +from collections import defaultdict + +class MetricsCollector: + """Collect and store metrics during test execution""" + + def __init__(self): + """Initialize metrics collector""" + # Basic metrics structure + self.operations = {} + self.system_metrics = {} + self.k8s_metrics = {} + self.csi_metrics = {} + + # Controller metrics + self.controller_metrics = { + "request_latency": {}, + "operation_counts": defaultdict(int), + "success_rates": defaultdict(lambda: {"success": 0, "failure": 0}), + "volume_attach_timing": {} + } + + # Node-level metrics + self.node_metrics = { + "mount_timing": {}, + "mount_errors": defaultdict(int), + "resource_utilization": {} + } + + # EFS-specific metrics + self.efs_metrics = { + "access_point_timing": {}, + "mount_completion_timing": {}, + "api_throttling_incidents": [] + } + + # Kubernetes events + self.k8s_events = { + "volume_events": [], + "binding_times": {}, + "pod_startup_delays": {} + } + + self.logger = logging.getLogger(__name__) + + def start_operation(self, name=None): + """Start timing an operation + + Args: + name: Name of the operation (optional) + """ + op_id = name or f"op_{len(self.operations) + 1}" + self.operations[op_id] = { + "start_time": time.time(), + "samples": [] + } + self._collect_system_metrics(op_id) + return op_id + + def end_operation(self, op_id): + """End timing an operation and return duration + + Args: + op_id: Operation ID or name + + Returns: + Duration in seconds + """ + if op_id not in self.operations: + self.logger.warning(f"Operation {op_id} not found") + return 0 + + self.operations[op_id]["end_time"] = time.time() + self.operations[op_id]["duration"] = ( + self.operations[op_id]["end_time"] - + self.operations[op_id]["start_time"] + ) + self._collect_system_metrics(op_id, end=True) + + return self.operations[op_id]["duration"] + + def add_sample(self, op_id, metrics): + """Add a sample to an operation + + Args: + op_id: Operation ID or name + metrics: Dictionary of metrics to add + """ + if op_id not in self.operations: + self.logger.warning(f"Operation {op_id} not found") + return + + sample = { + "timestamp": time.time(), + "metrics": metrics + } + self.operations[op_id]["samples"].append(sample) + + def _collect_system_metrics(self, op_id, end=False): + """Collect system metrics + + Args: + op_id: Operation ID or name + end: Whether this is the end of an operation + """ + prefix = "end_" if end else "start_" + + # Collect CPU, memory, disk I/O metrics + cpu_percent = psutil.cpu_percent(interval=0.1) + memory = psutil.virtual_memory() + disk_io = psutil.disk_io_counters() + + metrics = { + f"{prefix}cpu_percent": cpu_percent, + f"{prefix}memory_percent": memory.percent, + f"{prefix}disk_read_bytes": disk_io.read_bytes, + f"{prefix}disk_write_bytes": disk_io.write_bytes + } + + if op_id not in self.system_metrics: + self.system_metrics[op_id] = {} + + self.system_metrics[op_id].update(metrics) + + def collect_csi_metrics(self, config=None): + """Collect CSI driver metrics + + Args: + config: Configuration dictionary with metrics settings + """ + if not config: + return + + if not config.get('metrics_collection', {}).get('enabled', False): + return + + ports = config.get('metrics_collection', {}).get('controller_ports', [8080, 8081]) + + # Get EFS CSI controller pod + try: + kube_client = client.CoreV1Api() + pods = kube_client.list_namespaced_pod( + namespace="kube-system", + label_selector="app=efs-csi-controller" + ) + + if not pods.items: + self.logger.warning("No EFS CSI controller pods found") + return + + controller_pod = pods.items[0] + pod_name = controller_pod.metadata.name + + # Port-forward to the controller pod + for port in ports: + try: + # Use kubectl port-forward in a subprocess + import subprocess + import threading + import time + + # Start port-forwarding in a separate process + process = subprocess.Popen( + ["kubectl", "port-forward", pod_name, f"{port}:{port}", "-n", "kube-system"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + + # Give it time to establish the connection + time.sleep(2) + + # Collect metrics + try: + response = requests.get(f"http://localhost:{port}/metrics", timeout=5) + if response.status_code == 200: + self.csi_metrics[f"port_{port}"] = response.text + except requests.RequestException as e: + self.logger.warning(f"Failed to collect metrics from port {port}: {e}") + + # Terminate the port-forwarding process + process.terminate() + process.wait(timeout=5) + + except Exception as e: + self.logger.warning(f"Error collecting metrics from port {port}: {e}") + + except Exception as e: + self.logger.warning(f"Error collecting CSI metrics: {e}") + + def get_operation_metrics(self, op_id): + """Get metrics for an operation + + Args: + op_id: Operation ID or name + + Returns: + Dictionary of metrics + """ + if op_id not in self.operations: + self.logger.warning(f"Operation {op_id} not found") + return {} + + metrics = self.operations[op_id].copy() + + # Add system metrics if available + if op_id in self.system_metrics: + metrics["system"] = self.system_metrics[op_id] + + return metrics + + def track_controller_request(self, operation_type, start_time, success=True): + """Track a controller request + + Args: + operation_type: Type of operation (e.g., 'create_volume', 'delete_volume') + start_time: Start time of the operation + success: Whether the operation succeeded + """ + duration = time.time() - start_time + + # Record request latency + if operation_type not in self.controller_metrics["request_latency"]: + self.controller_metrics["request_latency"][operation_type] = [] + self.controller_metrics["request_latency"][operation_type].append(duration) + + # Record operation count + self.controller_metrics["operation_counts"][operation_type] += 1 + + # Record success/failure + status = "success" if success else "failure" + self.controller_metrics["success_rates"][operation_type][status] += 1 + + def track_volume_attachment(self, volume_id, start_time): + """Track volume attachment time + + Args: + volume_id: ID of the volume + start_time: Start time of the attachment + """ + duration = time.time() - start_time + self.controller_metrics["volume_attach_timing"][volume_id] = duration + + def track_mount_operation(self, node_name, pod_name, start_time, success=True): + """Track mount operation time + + Args: + node_name: Name of the node + pod_name: Name of the pod + start_time: Start time of the mount operation + success: Whether the operation succeeded + """ + duration = time.time() - start_time + + if node_name not in self.node_metrics["mount_timing"]: + self.node_metrics["mount_timing"][node_name] = {} + + self.node_metrics["mount_timing"][node_name][pod_name] = duration + + if not success: + self.node_metrics["mount_errors"][node_name] += 1 + + def track_node_resources(self, node_name, cpu_percent, memory_percent): + """Track node resource utilization + + Args: + node_name: Name of the node + cpu_percent: CPU utilization percentage + memory_percent: Memory utilization percentage + """ + if node_name not in self.node_metrics["resource_utilization"]: + self.node_metrics["resource_utilization"][node_name] = [] + + self.node_metrics["resource_utilization"][node_name].append({ + "timestamp": time.time(), + "cpu_percent": cpu_percent, + "memory_percent": memory_percent + }) + + def track_access_point_creation(self, access_point_id, start_time): + """Track access point creation time + + Args: + access_point_id: ID of the access point + start_time: Start time of the creation + """ + duration = time.time() - start_time + self.efs_metrics["access_point_timing"][access_point_id] = duration + + def track_mount_completion(self, pod_name, pvc_name, start_time): + """Track mount completion time + + Args: + pod_name: Name of the pod + pvc_name: Name of the PVC + start_time: Start time of the mount operation + """ + duration = time.time() - start_time + + if pod_name not in self.efs_metrics["mount_completion_timing"]: + self.efs_metrics["mount_completion_timing"][pod_name] = {} + + self.efs_metrics["mount_completion_timing"][pod_name][pvc_name] = duration + + def record_api_throttling(self, operation_type, error_message): + """Record API throttling incident + + Args: + operation_type: Type of operation that was throttled + error_message: Error message from the API + """ + self.efs_metrics["api_throttling_incidents"].append({ + "timestamp": time.time(), + "operation_type": operation_type, + "error_message": error_message + }) + + def collect_volume_events(self, namespace="default"): + """Collect volume-related Kubernetes events + + Args: + namespace: Kubernetes namespace to collect events from + """ + try: + kube_client = client.CoreV1Api() + events = kube_client.list_namespaced_event(namespace=namespace) + + for event in events.items: + if event.involved_object.kind in ["PersistentVolume", "PersistentVolumeClaim"]: + self.k8s_events["volume_events"].append({ + "timestamp": time.time(), + "name": event.involved_object.name, + "kind": event.involved_object.kind, + "reason": event.reason, + "message": event.message, + "count": event.count + }) + except Exception as e: + self.logger.warning(f"Error collecting volume events: {e}") + + def track_pv_pvc_binding(self, pvc_name, pv_name, bind_time): + """Track PV-PVC binding time + + Args: + pvc_name: Name of the PVC + pv_name: Name of the PV + bind_time: Time taken for binding in seconds + """ + self.k8s_events["binding_times"][f"{pvc_name}-{pv_name}"] = bind_time + + def track_pod_startup_delay(self, pod_name, create_time, ready_time): + """Track pod startup delay + + Args: + pod_name: Name of the pod + create_time: Time when the pod was created + ready_time: Time when the pod became ready + """ + delay = ready_time - create_time + self.k8s_events["pod_startup_delays"][pod_name] = delay + + def parse_prometheus_metrics(self, metrics_text): + """Parse Prometheus metrics from the CSI driver + + Args: + metrics_text: Raw Prometheus metrics text + + Returns: + Dictionary of parsed metrics + """ + parsed_metrics = {} + + if not metrics_text: + return parsed_metrics + + # Simple regex pattern to extract metrics + pattern = r'^([a-zA-Z_:][a-zA-Z0-9_:]*)\s*({[^}]*})?\s*([0-9.eE+-]+)' + + for line in metrics_text.split('\n'): + line = line.strip() + if not line or line.startswith('#'): + continue + + match = re.match(pattern, line) + if match: + metric_name = match.group(1) + labels = match.group(2) or "" + value = float(match.group(3)) + + if metric_name not in parsed_metrics: + parsed_metrics[metric_name] = [] + + parsed_metrics[metric_name].append({ + "labels": labels, + "value": value + }) + + return parsed_metrics + + def get_all_metrics(self): + """Get all collected metrics + + Returns: + Dictionary of all metrics + """ + return { + "operations": self.operations, + "system": self.system_metrics, + "kubernetes": self.k8s_metrics, + "csi": self.csi_metrics, + "controller": self.controller_metrics, + "node": self.node_metrics, + "efs": self.efs_metrics, + "k8s_events": self.k8s_events + } diff --git a/test/e2e/stress-scale-tests/utils/report_generator.py b/test/e2e/stress-scale-tests/utils/report_generator.py new file mode 100644 index 000000000..a7435575a --- /dev/null +++ b/test/e2e/stress-scale-tests/utils/report_generator.py @@ -0,0 +1,352 @@ +import json +import os +import datetime +import platform +import psutil +import yaml +from pathlib import Path +import socket +import subprocess + +class ReportGenerator: + """Generate detailed test reports in various formats""" + + def __init__(self, output_dir="reports"): + """Initialize report generator + + Args: + output_dir: Base directory to store reports + """ + self.base_output_dir = output_dir + Path(output_dir).mkdir(parents=True, exist_ok=True) + + def _get_output_dir(self, test_type): + """Get the output directory for a specific test type + + Args: + test_type: Type of test (e.g., 'stress', 'scalability') + + Returns: + Path to the output directory + """ + output_dir = os.path.join(self.base_output_dir, test_type) + Path(output_dir).mkdir(parents=True, exist_ok=True) + return output_dir + + def _determine_test_type(self, test_results): + """Determine the test type from the results structure + + Args: + test_results: Dictionary containing test results + + Returns: + Test type string + """ + # Check for specific test keys + if "statefulset_scaling" in test_results: + return "statefulset" + elif "access_point_scaling" in test_results: + return "access_points" + elif "pod_scaling" in test_results or "volume_scaling" in test_results: + return "scalability" + elif any(key.endswith("_io") for key in test_results.keys()): + return "stress" + + # Check nested dictionaries + for key, value in test_results.items(): + if isinstance(value, dict): + if "statefulset_scaling" in value: + return "statefulset" + elif "access_point_scaling" in value: + return "access_points" + elif "pod_scaling" in value or "volume_scaling" in value: + return "scalability" + elif any(k.endswith("_io") for k in value.keys()): + return "stress" + + return "general" + + def _collect_system_info(self): + """Collect system information for the report + + Returns: + Dictionary with system information + """ + system_info = { + "hostname": socket.gethostname(), + "platform": platform.platform(), + "python_version": platform.python_version(), + "cpu_count": psutil.cpu_count(), + "memory_total_gb": round(psutil.virtual_memory().total / (1024**3), 2), + "timestamp": datetime.datetime.now().isoformat() + } + + # Try to get Kubernetes cluster info + try: + kubectl_version = subprocess.check_output(["kubectl", "version", "--short"], + stderr=subprocess.STDOUT).decode('utf-8') + system_info["kubernetes_version"] = kubectl_version.strip() + except (subprocess.SubprocessError, FileNotFoundError): + system_info["kubernetes_version"] = "Unknown" + + # Try to get AWS region + try: + with open('config/test_config.yaml', 'r') as f: + config = yaml.safe_load(f) + system_info["aws_region"] = config.get('cluster', {}).get('region', 'Unknown') + system_info["efs_filesystem_id"] = config.get('efs', {}).get('filesystem_id', 'Unknown') + except Exception: + system_info["aws_region"] = "Unknown" + system_info["efs_filesystem_id"] = "Unknown" + + return system_info + + def generate_json_report(self, test_results, test_name): + """Generate detailed JSON report + + Args: + test_results: Dictionary containing test results + test_name: Name of the test + + Returns: + Path to the generated report + """ + timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + + # Determine test type from results structure + test_type = self._determine_test_type(test_results) + + output_dir = self._get_output_dir(test_type) + filename = f"{test_name}_{timestamp}.json" + filepath = os.path.join(output_dir, filename) + + # Add metadata and system info + report = { + "test_name": test_name, + "test_type": test_type, + "timestamp": timestamp, + "system_info": self._collect_system_info(), + "results": test_results + } + + with open(filepath, 'w') as f: + json.dump(report, f, indent=2) + + return filepath + + def generate_summary_report(self, test_results, test_name): + """Generate a detailed human-readable summary report + + Args: + test_results: Dictionary containing test results + test_name: Name of the test + + Returns: + Path to the generated report + """ + timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + + # Determine test type from results structure + test_type = self._determine_test_type(test_results) + + output_dir = self._get_output_dir(test_type) + filename = f"{test_name}_{timestamp}_summary.txt" + filepath = os.path.join(output_dir, filename) + + system_info = self._collect_system_info() + + with open(filepath, 'w') as f: + # Write header + f.write(f"{'='*80}\n") + f.write(f"EFS CSI DRIVER TEST REPORT: {test_name.upper()}\n") + f.write(f"{'='*80}\n\n") + + # Write system information + f.write("SYSTEM INFORMATION\n") + f.write(f"{'-'*80}\n") + f.write(f"Hostname: {system_info['hostname']}\n") + f.write(f"Platform: {system_info['platform']}\n") + f.write(f"Python Version: {system_info['python_version']}\n") + f.write(f"CPU Count: {system_info['cpu_count']}\n") + f.write(f"Memory Total: {system_info['memory_total_gb']} GB\n") + f.write(f"Kubernetes Version: {system_info['kubernetes_version']}\n") + f.write(f"AWS Region: {system_info['aws_region']}\n") + f.write(f"EFS Filesystem ID: {system_info['efs_filesystem_id']}\n") + f.write(f"Test Timestamp: {system_info['timestamp']}\n\n") + + # Write test results + f.write("TEST RESULTS\n") + f.write(f"{'-'*80}\n") + + # Process different test types + if test_type == "scalability": + self._write_scalability_results(f, test_results) + elif test_type == "stress": + self._write_stress_results(f, test_results) + elif test_type == "access_points": + self._write_access_point_results(f, test_results) + elif test_type == "statefulset": + self._write_statefulset_results(f, test_results) + else: + self._write_generic_results(f, test_results) + + # Write footer + f.write(f"\n{'='*80}\n") + f.write(f"END OF REPORT: {datetime.datetime.now().isoformat()}\n") + f.write(f"{'='*80}\n") + + return filepath + + def _write_scalability_results(self, file, results): + """Write scalability test results to the report file""" + if "pod_scaling" in results: + file.write("\nPOD SCALING TEST RESULTS\n") + file.write(f"{'-'*40}\n") + + pod_results = results["pod_scaling"] + if isinstance(pod_results, dict): + # Sort by scale (number of pods) + for scale in sorted([int(k) for k in pod_results.keys()]): + data = pod_results[scale] + success = data.get('success', False) + duration = data.get('duration', 0) + pods_ready = data.get('pods_ready', 0) + + file.write(f"Scale: {scale} pods\n") + file.write(f" Status: {'SUCCESS' if success else 'FAILED'}\n") + file.write(f" Duration: {duration:.2f} seconds\n") + file.write(f" Pods Ready: {pods_ready}\n") + if duration > 0: + file.write(f" Scale Rate: {scale/duration:.2f} pods/second\n\n") + else: + file.write(f"Error: {pod_results}\n\n") + + if "volume_scaling" in results: + file.write("\nVOLUME SCALING TEST RESULTS\n") + file.write(f"{'-'*40}\n") + + volume_results = results["volume_scaling"] + if isinstance(volume_results, dict): + # Sort by scale (number of volumes) + for scale in sorted([int(k) for k in volume_results.keys()]): + data = volume_results[scale] + success = data.get('success', False) + duration = data.get('duration', 0) + volumes_ready = data.get('volumes_ready', 0) + + file.write(f"Scale: {scale} volumes\n") + file.write(f" Status: {'SUCCESS' if success else 'FAILED'}\n") + file.write(f" Duration: {duration:.2f} seconds\n") + file.write(f" Volumes Ready: {volumes_ready}\n") + if duration > 0: + file.write(f" Scale Rate: {scale/duration:.2f} volumes/second\n\n") + else: + file.write(f"Error: {volume_results}\n\n") + + def _write_stress_results(self, file, results): + """Write stress test results to the report file""" + for test_name, result in results.items(): + file.write(f"\n{test_name.upper()} RESULTS\n") + file.write(f"{'-'*40}\n") + + if isinstance(result, dict): + if "sequential_write" in result: + file.write("\nSequential Write Test:\n") + seq_result = result["sequential_write"] + file.write(f" Status: {seq_result.get('status', 'unknown')}\n") + file.write(f" Duration: {seq_result.get('duration', 'N/A')} seconds\n") + + # Add detailed metrics if available + if "pod_metrics" in seq_result: + file.write(" Pod Metrics:\n") + for i, metric in enumerate(seq_result["pod_metrics"]): + file.write(f" Sample {i+1}: Phase={metric.get('phase', 'unknown')}\n") + + if "random_write" in result: + file.write("\nRandom Write Test:\n") + rand_result = result["random_write"] + file.write(f" Status: {rand_result.get('status', 'unknown')}\n") + file.write(f" Duration: {rand_result.get('duration', 'N/A')} seconds\n") + + # Add detailed metrics if available + if "pod_metrics" in rand_result: + file.write(" Pod Metrics:\n") + for i, metric in enumerate(rand_result["pod_metrics"]): + file.write(f" Sample {i+1}: Phase={metric.get('phase', 'unknown')}\n") + + if "mixed_io" in result: + file.write("\nMixed I/O Test:\n") + mixed_result = result["mixed_io"] + file.write(f" Status: {mixed_result.get('status', 'unknown')}\n") + file.write(f" Duration: {mixed_result.get('duration', 'N/A')} seconds\n") + + # Add detailed metrics if available + if "pod_metrics" in mixed_result: + file.write(" Pod Metrics:\n") + for i, metric in enumerate(mixed_result["pod_metrics"]): + file.write(f" Sample {i+1}: Phase={metric.get('phase', 'unknown')}\n") + else: + file.write(f"Error: {result}\n") + + def _write_access_point_results(self, file, results): + """Write access point test results to the report file""" + if "access_point_scaling" in results: + file.write("\nACCESS POINT SCALING TEST RESULTS\n") + file.write(f"{'-'*40}\n") + + ap_results = results["access_point_scaling"] + if isinstance(ap_results, dict): + # Sort by scale (number of access points) + for scale in sorted([int(k) for k in ap_results.keys()]): + data = ap_results[scale] + success = data.get('success', False) + duration = data.get('duration', 0) + aps_ready = data.get('access_points_ready', 0) + + file.write(f"Scale: {scale} access points\n") + file.write(f" Status: {'SUCCESS' if success else 'FAILED'}\n") + file.write(f" Duration: {duration:.2f} seconds\n") + file.write(f" Access Points Ready: {aps_ready}\n") + if duration > 0: + file.write(f" Scale Rate: {scale/duration:.2f} access points/second\n\n") + else: + file.write(f"Error: {ap_results}\n\n") + + def _write_statefulset_results(self, file, results): + """Write StatefulSet test results to the report file""" + if "statefulset_scaling" in results: + file.write("\nSTATEFULSET SCALING TEST RESULTS\n") + file.write(f"{'-'*40}\n") + + sts_result = results["statefulset_scaling"] + if isinstance(sts_result, dict): + replicas = sts_result.get('replicas', 0) + success = sts_result.get('success', False) + duration = sts_result.get('duration', 0) + pods_ready = sts_result.get('pods_ready', 0) + + file.write(f"Replicas: {replicas}\n") + file.write(f"Status: {'SUCCESS' if success else 'FAILED'}\n") + file.write(f"Duration: {duration:.2f} seconds\n") + file.write(f"Pods Ready: {pods_ready}\n") + if duration > 0: + file.write(f"Scale Rate: {replicas/duration:.2f} pods/second\n\n") + else: + file.write(f"Error: {sts_result}\n\n") + + def _write_generic_results(self, file, results): + """Write generic test results to the report file""" + for test_name, result in results.items(): + file.write(f"\n{test_name.upper()}\n") + file.write(f"{'-'*40}\n") + + if isinstance(result, dict): + for key, value in result.items(): + if isinstance(value, dict): + file.write(f"{key}:\n") + for sub_key, sub_value in value.items(): + file.write(f" {sub_key}: {sub_value}\n") + else: + file.write(f"{key}: {value}\n") + else: + file.write(f"{result}\n") \ No newline at end of file From 2a86c48e77d352e27c5cd48b59d33211223124c8 Mon Sep 17 00:00:00 2001 From: Ishaan Bhardwaj Date: Thu, 5 Jun 2025 21:15:11 +0000 Subject: [PATCH 02/13] Add stress and scalability test framework for EFS CSI Driver --- .../cluster-config-code/cluster_setup.py | 756 ----- .../efs-csi-test-1749045745.yaml | 51 - .../efs-csi-test-1749045783.yaml | 51 - .../efs-csi-test-1749048222.yaml | 51 - .../efs-csi-test-1749048235.yaml | 51 - .../efs-csi-test-1749048734.yaml | 51 - .../efs-csi-test-1749051234.yaml | 51 - .../efs-csi-test-1749051325.yaml | 51 - .../efs-csi-test-1749051374.yaml | 51 - .../efs-csi-test-1749051392.yaml | 51 - .../logs/cleanup_20250605_190904.log | 129 - .../stress-scale-tests/logs/cluster_setup.log | 383 --- .../e2e/stress-scale-tests/logs/efs_tests.log | 2869 ----------------- ...creation-error-efs-csi-test-1749045783.log | 8 - ...creation-error-efs-csi-test-1749048222.log | 8 - ...creation-error-efs-csi-test-1749048235.log | 17 - ...creation-error-efs-csi-test-1749048734.log | 17 - ...creation-error-efs-csi-test-1749051325.log | 17 - ...creation-error-efs-csi-test-1749051374.log | 17 - ...creation-error-efs-csi-test-1749051392.log | 254 -- .../logs/orchestrator_20250603_145210.log | 147 - .../logs/orchestrator_20250603_154721.log | 8 - .../logs/orchestrator_20250603_154805.log | 455 --- .../logs/orchestrator_20250603_174810.log | 8 - .../logs/orchestrator_20250603_175820.log | 77 - .../logs/orchestrator_20250603_204418.log | 8 - .../logs/orchestrator_20250603_204433.log | 94 - .../logs/orchestrator_20250603_204821.log | 175 - .../logs/orchestrator_20250603_210011.log | 158 - .../logs/orchestrator_20250604_204301.log | 125 - .../logs/orchestrator_20250604_211255.log | 43 - .../logs/orchestrator_20250604_211353.log | 80 - .../logs/orchestrator_20250605_132802.log | 202 -- .../logs/orchestrator_20250605_135002.log | 123 - .../logs/orchestrator_20250605_140408.log | 138 - .../logs/orchestrator_20250605_141321.log | 7 - .../logs/orchestrator_20250605_141434.log | 142 - .../logs/orchestrator_20250605_143926.log | 73 - .../logs/orchestrator_20250605_145133.log | 111 - .../logs/orchestrator_20250605_145937.log | 134 - .../logs/orchestrator_20250605_150619.log | 147 - .../logs/orchestrator_20250605_151529.log | 7 - .../logs/orchestrator_20250605_151604.log | 42 - .../logs/orchestrator_20250605_151742.log | 17 - .../logs/orchestrator_20250605_153750.log | 249 -- .../logs/orchestrator_20250605_154546.log | 254 -- .../logs/orchestrator_20250605_155948.log | 280 -- .../logs/orchestrator_20250605_160844.log | 281 -- .../logs/orchestrator_20250605_162134.log | 7 - .../logs/orchestrator_20250605_162155.log | 52 - .../logs/orchestrator_20250605_162320.log | 27 - .../logs/orchestrator_20250605_162354.log | 247 -- .../logs/orchestrator_20250605_163114.log | 67 - .../logs/orchestrator_20250605_163340.log | 42 - .../logs/orchestrator_20250605_163501.log | 205 -- .../logs/orchestrator_20250605_164709.log | 67 - .../logs/orchestrator_20250605_164934.log | 246 -- .../logs/orchestrator_20250605_170306.log | 17 - .../logs/orchestrator_20250605_170342.log | 17 - .../logs/orchestrator_20250605_170405.log | 39 - .../logs/orchestrator_20250605_170502.log | 12 - .../logs/orchestrator_20250605_170526.log | 234 -- .../logs/orchestrator_20250605_182216.log | 7 - .../logs/orchestrator_20250605_182254.log | 207 -- .../logs/orchestrator_20250605_183835.log | 983 ------ .../logs/orchestrator_20250605_191215.log | 158 - .../logs/orchestrator_20250605_192824.log | 7 - .../logs/orchestrator_20250605_192851.log | 207 -- .../logs/orchestrator_20250605_195225.log | 335 -- .../logs/orchestrator_20250605_200645.log | 348 -- .../logs/orchestrator_20250605_210820.log | 7 + .../logs/orchestrator_20250605_210842.log | 70 + .../chaos/efs_chaos_20250603_135616.json | 29 - ...tests_20250603_145849_20250603_145849.json | 104 - ...0250603_145849_20250603_145849_summary.txt | 61 - ...tests_20250603_160604_20250603_160604.json | 104 - ...0250603_160604_20250603_160604_summary.txt | 61 - ...tests_20250603_180130_20250603_180130.json | 104 - ...0250603_180130_20250603_180130_summary.txt | 61 - ...tests_20250603_204448_20250603_204448.json | 104 - ...0250603_204448_20250603_204448_summary.txt | 61 - ...tests_20250603_205333_20250603_205333.json | 104 - ...0250603_205333_20250603_205333_summary.txt | 61 - ...tests_20250603_210938_20250603_210938.json | 104 - ...0250603_210938_20250603_210938_summary.txt | 61 - ...tests_20250604_204834_20250604_204834.json | 104 - ...tests_20250604_211347_20250604_211347.json | 104 - ...tests_20250604_211457_20250604_211457.json | 104 - ...tests_20250605_133650_20250605_133650.json | 104 - ...tests_20250605_135626_20250605_135626.json | 104 - ...tests_20250605_141131_20250605_141131.json | 104 - ...tests_20250605_142059_20250605_142059.json | 104 - ...tests_20250605_145750_20250605_145750.json | 104 - ...tests_20250605_150603_20250605_150603.json | 104 - ...tests_20250605_151246_20250605_151246.json | 104 - ...tests_20250605_154402_20250605_154402.json | 104 - ...tests_20250605_155202_20250605_155202.json | 104 - ...tests_20250605_160701_20250605_160701.json | 104 - ...tests_20250605_161509_20250605_161509.json | 104 - ...tests_20250605_163010_20250605_163010.json | 104 - ...tests_20250605_164017_20250605_164017.json | 104 - ...tests_20250605_165552_20250605_165552.json | 104 - ...tests_20250605_171216_20250605_171216.json | 104 - ...tests_20250605_182932_20250605_182932.json | 104 - ...tests_20250605_193407_20250605_193407.json | 104 - ...tests_20250605_200124_20250605_200124.json | 104 - .../efs_orchestrator_20250603_145849.json | 59 - .../efs_orchestrator_20250603_160604.json | 59 - .../efs_orchestrator_20250603_180130.json | 59 - .../efs_orchestrator_20250603_204448.json | 59 - .../efs_orchestrator_20250603_205333.json | 59 - .../efs_orchestrator_20250603_210938.json | 59 - .../efs_orchestrator_20250604_204834.json | 59 - .../efs_orchestrator_20250604_211347.json | 59 - .../efs_orchestrator_20250604_211457.json | 59 - .../efs_orchestrator_20250605_133650.json | 59 - .../efs_orchestrator_20250605_135626.json | 59 - .../efs_orchestrator_20250605_141131.json | 59 - .../efs_orchestrator_20250605_142059.json | 59 - .../efs_orchestrator_20250605_145750.json | 59 - .../efs_orchestrator_20250605_150603.json | 59 - .../efs_orchestrator_20250605_151246.json | 59 - .../efs_orchestrator_20250605_154402.json | 59 - .../efs_orchestrator_20250605_155202.json | 59 - .../efs_orchestrator_20250605_160701.json | 59 - .../efs_orchestrator_20250605_161509.json | 59 - .../efs_orchestrator_20250605_163010.json | 59 - .../efs_orchestrator_20250605_164017.json | 59 - .../efs_orchestrator_20250605_165552.json | 59 - .../efs_orchestrator_20250605_171216.json | 59 - .../efs_orchestrator_20250605_182932.json | 59 - .../efs_orchestrator_20250605_193407.json | 59 - .../efs_orchestrator_20250605_200124.json | 59 - ...tests_20250602_195427_20250602_195427.json | 100 - ...0250602_195427_20250602_195427_summary.txt | 22 - ...tests_20250602_183718_20250602_183718.json | 58 - ...0250602_183718_20250602_183718_summary.txt | 40 - ...tests_20250602_201648_20250602_201648.json | 58 - ...0250602_201648_20250602_201648_summary.txt | 40 - ...tests_20250602_210512_20250602_210512.json | 79 - ...0250602_210512_20250602_210512_summary.txt | 40 - ...tests_20250603_135616_20250603_135616.json | 216 -- ...0250603_135616_20250603_135616_summary.txt | 52 - test/e2e/stress-scale-tests/run_tests.py | 67 - 144 files changed, 77 insertions(+), 17618 deletions(-) delete mode 100644 test/e2e/stress-scale-tests/cluster-config-code/cluster_setup.py delete mode 100644 test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749045745.yaml delete mode 100644 test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749045783.yaml delete mode 100644 test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749048222.yaml delete mode 100644 test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749048235.yaml delete mode 100644 test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749048734.yaml delete mode 100644 test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749051234.yaml delete mode 100644 test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749051325.yaml delete mode 100644 test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749051374.yaml delete mode 100644 test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749051392.yaml delete mode 100644 test/e2e/stress-scale-tests/logs/cleanup_20250605_190904.log delete mode 100644 test/e2e/stress-scale-tests/logs/cluster_setup.log delete mode 100644 test/e2e/stress-scale-tests/logs/efs_tests.log delete mode 100644 test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749045783.log delete mode 100644 test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749048222.log delete mode 100644 test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749048235.log delete mode 100644 test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749048734.log delete mode 100644 test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749051325.log delete mode 100644 test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749051374.log delete mode 100644 test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749051392.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250603_145210.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250603_154721.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250603_154805.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250603_174810.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250603_175820.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250603_204418.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250603_204433.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250603_204821.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250603_210011.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250604_204301.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250604_211255.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250604_211353.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_132802.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_135002.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_140408.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_141321.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_141434.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_143926.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_145133.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_145937.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_150619.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_151529.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_151604.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_151742.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_153750.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_154546.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_155948.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_160844.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_162134.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_162155.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_162320.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_162354.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_163114.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_163340.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_163501.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_164709.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_164934.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_170306.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_170342.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_170405.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_170502.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_170526.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_182216.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_182254.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_183835.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_191215.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_192824.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_192851.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_195225.log delete mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_200645.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_210820.log create mode 100644 test/e2e/stress-scale-tests/logs/orchestrator_20250605_210842.log delete mode 100644 test/e2e/stress-scale-tests/reports/chaos/efs_chaos_20250603_135616.json delete mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_145849_20250603_145849.json delete mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_145849_20250603_145849_summary.txt delete mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_160604_20250603_160604.json delete mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_160604_20250603_160604_summary.txt delete mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_180130_20250603_180130.json delete mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_180130_20250603_180130_summary.txt delete mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_204448_20250603_204448.json delete mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_204448_20250603_204448_summary.txt delete mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_205333_20250603_205333.json delete mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_205333_20250603_205333_summary.txt delete mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_210938_20250603_210938.json delete mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_210938_20250603_210938_summary.txt delete mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250604_204834_20250604_204834.json delete mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250604_211347_20250604_211347.json delete mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250604_211457_20250604_211457.json delete mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_133650_20250605_133650.json delete mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_135626_20250605_135626.json delete mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_141131_20250605_141131.json delete mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_142059_20250605_142059.json delete mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_145750_20250605_145750.json delete mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_150603_20250605_150603.json delete mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_151246_20250605_151246.json delete mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_154402_20250605_154402.json delete mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_155202_20250605_155202.json delete mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_160701_20250605_160701.json delete mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_161509_20250605_161509.json delete mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_163010_20250605_163010.json delete mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_164017_20250605_164017.json delete mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_165552_20250605_165552.json delete mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_171216_20250605_171216.json delete mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_182932_20250605_182932.json delete mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_193407_20250605_193407.json delete mode 100644 test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_200124_20250605_200124.json delete mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_145849.json delete mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_160604.json delete mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_180130.json delete mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_204448.json delete mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_205333.json delete mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_210938.json delete mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250604_204834.json delete mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250604_211347.json delete mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250604_211457.json delete mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_133650.json delete mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_135626.json delete mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_141131.json delete mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_142059.json delete mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_145750.json delete mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_150603.json delete mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_151246.json delete mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_154402.json delete mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_155202.json delete mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_160701.json delete mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_161509.json delete mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_163010.json delete mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_164017.json delete mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_165552.json delete mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_171216.json delete mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_182932.json delete mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_193407.json delete mode 100644 test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_200124.json delete mode 100644 test/e2e/stress-scale-tests/reports/scalability/efs_tests_20250602_195427_20250602_195427.json delete mode 100644 test/e2e/stress-scale-tests/reports/scalability/efs_tests_20250602_195427_20250602_195427_summary.txt delete mode 100644 test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_183718_20250602_183718.json delete mode 100644 test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_183718_20250602_183718_summary.txt delete mode 100644 test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_201648_20250602_201648.json delete mode 100644 test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_201648_20250602_201648_summary.txt delete mode 100644 test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_210512_20250602_210512.json delete mode 100644 test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_210512_20250602_210512_summary.txt delete mode 100644 test/e2e/stress-scale-tests/reports/stress/efs_tests_20250603_135616_20250603_135616.json delete mode 100644 test/e2e/stress-scale-tests/reports/stress/efs_tests_20250603_135616_20250603_135616_summary.txt diff --git a/test/e2e/stress-scale-tests/cluster-config-code/cluster_setup.py b/test/e2e/stress-scale-tests/cluster-config-code/cluster_setup.py deleted file mode 100644 index b41437803..000000000 --- a/test/e2e/stress-scale-tests/cluster-config-code/cluster_setup.py +++ /dev/null @@ -1,756 +0,0 @@ -#!/usr/bin/env python3 -""" -EFS CSI Driver Cluster Setup Script - -This script automates the provisioning of an EKS cluster and installation of the EFS CSI Driver: -1. Creates an EKS cluster with the specified Kubernetes version -2. Creates an EFS filesystem in the same VPC (or uses an existing one) -3. Installs the EFS CSI Driver with the specified version -4. Sets up the storage class and other necessary resources - -This can be used standalone or imported by the test orchestrator. -""" - -import os -import sys -import time -import json -import yaml -import logging -import subprocess -import argparse -from pathlib import Path -from datetime import datetime - -# Set up logging -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', - handlers=[ - logging.FileHandler("logs/cluster_setup.log"), - logging.StreamHandler(sys.stdout) - ] -) -logger = logging.getLogger("cluster_setup") - -class ClusterSetup: - """Class to automate EKS cluster creation and EFS CSI driver installation""" - - def __init__(self, config_file='config/orchestrator_config.yaml'): - """Initialize setup parameters from configuration file""" - # Load configuration - with open(config_file, 'r') as f: - self.config = yaml.safe_load(f) - - # Extract cluster configuration - cluster_config = self.config.get('cluster', {}) - self.create_cluster = cluster_config.get('create', True) - self.kubernetes_version = cluster_config.get('kubernetes_version', '1.26') - self.region = cluster_config.get('region', 'us-west-2') - self.node_type = cluster_config.get('node_type', 't3.large') - self.node_count = cluster_config.get('node_count', 3) - self.cluster_name = cluster_config.get('name', f"efs-csi-test-{int(time.time())}") - - # Extract driver configuration - driver_config = self.config.get('driver', {}) - self.csi_version = driver_config.get('version', '2.4.8') - self.install_method = driver_config.get('install_method', 'helm') - self.create_filesystem = driver_config.get('create_filesystem', True) - self.filesystem_id = driver_config.get('filesystem_id', None) - - # Track resources - self.cluster_config_path = None - self.existing_cluster = not self.create_cluster - - def setup(self): - """Run the complete setup process""" - # Create directories for logs and configs - os.makedirs("logs", exist_ok=True) - - if self.create_cluster: - self.create_cluster_config() - self.provision_cluster() - else: - logger.info("Using existing cluster as specified in config") - self.verify_cluster_connection() - - if self.create_filesystem: - self.create_efs_filesystem() - else: - if not self.filesystem_id: - logger.error("No filesystem ID provided but create_filesystem=false") - raise ValueError("When create_filesystem is false, you must specify a filesystem_id") - logger.info(f"Using existing filesystem: {self.filesystem_id}") - - self.install_efs_csi_driver() - self.create_storage_class() - - logger.info("Cluster setup completed successfully!") - return { - "cluster_name": self.cluster_name, - "region": self.region, - "kubernetes_version": self.kubernetes_version, - "filesystem_id": self.filesystem_id, - "csi_version": self.csi_version - } - - def create_cluster_config(self): - """Create eksctl cluster configuration""" - config_dir = "cluster-config" - os.makedirs(config_dir, exist_ok=True) - - # Generate cluster configuration for eksctl - cluster_config = { - "apiVersion": "eksctl.io/v1alpha5", - "kind": "ClusterConfig", - "metadata": { - "name": self.cluster_name, - "region": self.region, - "version": self.kubernetes_version - }, - "iam": { - "withOIDC": True, - "serviceAccounts": [ - { - "metadata": { - "name": "efs-csi-controller-sa", - "namespace": "kube-system" - }, - "attachPolicyARNs": [ - "arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy" - ] - } - ] - }, - "nodeGroups": [ - { - "name": "ng-1", - "instanceType": self.node_type, - "desiredCapacity": self.node_count, - "minSize": max(1, self.node_count - 1), - "maxSize": self.node_count + 2, - "volumeSize": 80, - "labels": { - "role": "general", - "efs-issue": "true" - }, - "tags": { - "nodegroup-role": "general" - }, - "iam": { - "attachPolicy": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "elasticfilesystem:DescribeAccessPoints", - "elasticfilesystem:DescribeFileSystems", - "elasticfilesystem:DescribeMountTargets", - "ec2:DescribeAvailabilityZones" - ], - "Resource": "*" - }, - { - "Effect": "Allow", - "Action": [ - "elasticfilesystem:CreateAccessPoint" - ], - "Resource": "*", - "Condition": { - "StringLike": { - "aws:RequestTag/kubernetes.io/cluster/*": "owned" - } - } - }, - { - "Effect": "Allow", - "Action": [ - "elasticfilesystem:DeleteAccessPoint" - ], - "Resource": "*", - "Condition": { - "StringEquals": { - "aws:ResourceTag/kubernetes.io/cluster/*": "owned" - } - } - } - ] - } - } - } - ] - } - - # Write config to file - config_path = os.path.join(config_dir, f"{self.cluster_name}.yaml") - with open(config_path, 'w') as f: - yaml.dump(cluster_config, f) - - logger.info(f"Created cluster config at {config_path}") - self.cluster_config_path = config_path - - return config_path - - def provision_cluster(self): - """Create EKS cluster using eksctl""" - if not self.cluster_config_path: - self.create_cluster_config() - - # Verify AWS credentials before attempting to create cluster - logger.info("Verifying AWS credentials...") - aws_check = self._run_command(["aws", "sts", "get-caller-identity"]) - - if aws_check['returncode'] != 0: - logger.error("AWS credentials verification failed. Please check your AWS configuration.") - logger.error(f"Error: {aws_check['stderr']}") - sys.exit(1) - else: - ident = json.loads(aws_check['stdout']) - logger.info(f"Using AWS account: {ident.get('Account', 'unknown')} with user: {ident.get('UserId', 'unknown')}") - - # Verify eksctl installation - eksctl_check = self._run_command(["eksctl", "version"]) - if eksctl_check['returncode'] != 0: - logger.error("eksctl not found or not working properly. Please install eksctl.") - logger.error(f"Error: {eksctl_check['stderr']}") - sys.exit(1) - else: - logger.info(f"eksctl version: {eksctl_check['stdout'].strip()}") - - logger.info(f"Creating EKS cluster {self.cluster_name} in {self.region}...") - logger.info("This may take 15-20 minutes to complete...") - - # Use verbose flag for more detailed output - result = self._run_command( - ["eksctl", "create", "cluster", "-f", self.cluster_config_path, "--verbose", "4"] - ) - - if result['returncode'] != 0: - logger.error("Failed to create EKS cluster") - logger.error(f"Error details: {result['stderr']}") - logger.error("Common causes of failure:") - logger.error(" 1. Insufficient IAM permissions") - logger.error(" 2. Service quotas reached (check your AWS console)") - logger.error(" 3. Network connectivity issues") - logger.error(" 4. Region availability zone limitations") - - # Save the detailed output to a log file for debugging - error_log = f"logs/eks-cluster-creation-error-{self.cluster_name}.log" - with open(error_log, 'w') as f: - f.write(f"STDOUT:\n{result['stdout']}\n\nSTDERR:\n{result['stderr']}") - logger.error(f"Detailed error log saved to: {error_log}") - - sys.exit(1) - - logger.info("EKS cluster created successfully") - - # Update kubeconfig - self._run_command([ - "aws", "eks", "update-kubeconfig", - "--name", self.cluster_name, - "--region", self.region - ]) - - # Verify cluster is accessible - self.verify_cluster_connection() - - def verify_cluster_connection(self): - """Verify that we can connect to the Kubernetes cluster""" - nodes_check = self._run_command(["kubectl", "get", "nodes"]) - if nodes_check['returncode'] == 0: - logger.info("Cluster verified - nodes accessible:") - logger.info(nodes_check['stdout']) - else: - logger.error("Could not verify cluster nodes. There might be connectivity issues.") - logger.error(f"Error: {nodes_check['stderr']}") - sys.exit(1) - - def create_efs_filesystem(self): - """Create EFS filesystem in the same VPC as the cluster""" - logger.info("Creating EFS filesystem...") - - # Get VPC ID from cluster - logger.info(f"Getting VPC ID for cluster {self.cluster_name} in region {self.region}") - result = self._run_command([ - "aws", "eks", "describe-cluster", - "--name", self.cluster_name, - "--region", self.region, - "--query", "cluster.resourcesVpcConfig.vpcId", - "--output", "text" - ]) - - if result['returncode'] != 0: - logger.error("Failed to get VPC ID") - sys.exit(1) - - vpc_id = result['stdout'].strip() - logger.info(f"Cluster VPC: {vpc_id}") - - # Create security group for EFS - sg_result = self._run_command([ - "aws", "ec2", "create-security-group", - "--group-name", f"efs-test-sg-{self.cluster_name}", - "--description", "EFS security group for CSI driver testing", - "--vpc-id", vpc_id, - "--output", "json" - ]) - - if sg_result['returncode'] != 0: - logger.error("Failed to create security group") - sys.exit(1) - - sg_data = json.loads(sg_result['stdout']) - sg_id = sg_data['GroupId'] - - # Add inbound rule for NFS - self._run_command([ - "aws", "ec2", "authorize-security-group-ingress", - "--group-id", sg_id, - "--protocol", "tcp", - "--port", "2049", - "--cidr", "0.0.0.0/0" - ]) - - # Create EFS filesystem - efs_result = self._run_command([ - "aws", "efs", "create-file-system", - "--performance-mode", "generalPurpose", - "--throughput-mode", "bursting", - "--tags", f"Key=Name,Value=efs-csi-test-{self.cluster_name}", - "--region", self.region, - "--output", "json" - ]) - - if efs_result['returncode'] != 0: - logger.error("Failed to create EFS filesystem") - sys.exit(1) - - efs_data = json.loads(efs_result['stdout']) - filesystem_id = efs_data['FileSystemId'] - self.filesystem_id = filesystem_id - logger.info(f"Created EFS filesystem: {filesystem_id}") - - # Wait for filesystem creation to complete - logger.info("Waiting for EFS filesystem to be available...") - time.sleep(10) # Initial wait - - max_retries = 12 - retry_count = 0 - while retry_count < max_retries: - status_result = self._run_command([ - "aws", "efs", "describe-file-systems", - "--file-system-id", filesystem_id, - "--query", "FileSystems[0].LifeCycleState", - "--output", "text" - ]) - - status = status_result['stdout'].strip() - logger.info(f"EFS status: {status}") - - if status == "available": - break - - retry_count += 1 - time.sleep(10) - - if retry_count >= max_retries: - logger.error("EFS filesystem failed to become available in time") - sys.exit(1) - - # Get subnet IDs - subnet_result = self._run_command([ - "aws", "ec2", "describe-subnets", - "--filters", f"Name=vpc-id,Values={vpc_id}", - "--query", "Subnets[*].SubnetId", - "--output", "json" - ]) - - subnets = json.loads(subnet_result['stdout']) - - # Create mount targets in each subnet - for subnet_id in subnets: - logger.info(f"Creating mount target in subnet {subnet_id}") - self._run_command([ - "aws", "efs", "create-mount-target", - "--file-system-id", filesystem_id, - "--subnet-id", subnet_id, - "--security-groups", sg_id - ]) - - # Wait for mount targets to be available - logger.info("Waiting for mount targets to be available...") - time.sleep(20) # Initial wait - - max_retries = 12 - retry_count = 0 - while retry_count < max_retries: - targets_result = self._run_command([ - "aws", "efs", "describe-mount-targets", - "--file-system-id", filesystem_id, - "--output", "json" - ]) - - targets = json.loads(targets_result['stdout'])['MountTargets'] - - # Check if all targets are available - all_available = True - for target in targets: - if target['LifeCycleState'] != 'available': - all_available = False - break - - if all_available: - logger.info(f"All {len(targets)} mount targets are available") - break - - retry_count += 1 - logger.info(f"Waiting for mount targets... (retry {retry_count}/{max_retries})") - time.sleep(10) - - if retry_count >= max_retries: - logger.warning("Some mount targets may not be available yet, but continuing...") - - # Update config file with filesystem ID - self.update_config_with_filesystem() - return filesystem_id - - def update_config_with_filesystem(self): - """Update the configuration file with filesystem ID""" - if not self.filesystem_id: - logger.warning("No filesystem ID to update in config") - return - - # Update the original config file - with open('config/orchestrator_config.yaml', 'r') as f: - config_data = yaml.safe_load(f) - - # Update the driver section - if 'driver' in config_data: - config_data['driver']['filesystem_id'] = self.filesystem_id - - # Update the storage_class section - if 'storage_class' in config_data and 'parameters' in config_data['storage_class']: - config_data['storage_class']['parameters']['fileSystemId'] = self.filesystem_id - - # Write updated config - with open('config/orchestrator_config.yaml', 'w') as f: - yaml.dump(config_data, f) - - logger.info(f"Updated config file with filesystem ID: {self.filesystem_id}") - - def install_efs_csi_driver(self): - """Install EFS CSI driver on the cluster""" - logger.info(f"Installing EFS CSI Driver {self.csi_version}...") - - if self.install_method == "helm": - self._install_via_helm() - elif self.install_method == "yaml": - self._install_via_yaml() - elif self.install_method == "custom": - self._install_custom() - else: - logger.error(f"Unknown installation method: {self.install_method}") - sys.exit(1) - - logger.info("Verifying EFS CSI Driver installation...") - self._verify_driver_installation() - - def _install_via_helm(self): - """Install the EFS CSI Driver using Helm""" - # Install driver using Helm - self._run_command([ - "helm", "repo", "add", "aws-efs-csi-driver", - "https://kubernetes-sigs.github.io/aws-efs-csi-driver/" - ]) - - self._run_command(["helm", "repo", "update"]) - - result = self._run_command([ - "helm", "upgrade", "--install", "aws-efs-csi-driver", - "aws-efs-csi-driver/aws-efs-csi-driver", - "--namespace", "kube-system", - "--version", self.csi_version, - "--set", "controller.serviceAccount.create=false", - "--set", "controller.serviceAccount.name=efs-csi-controller-sa" - ]) - - if result['returncode'] != 0: - logger.error("Failed to install EFS CSI driver via Helm") - sys.exit(1) - - logger.info("EFS CSI driver installed successfully via Helm") - - def _install_via_yaml(self): - """Install the EFS CSI Driver using kubectl and YAML manifests""" - # Download the driver YAML for the specified version - yaml_url = f"https://raw.githubusercontent.com/kubernetes-sigs/aws-efs-csi-driver/v{self.csi_version}/deploy/kubernetes/overlays/stable/ecr/manifest.yaml" - - # Download the manifest - result = self._run_command([ - "curl", "-sSL", yaml_url, "-o", "efs-csi-driver.yaml" - ]) - - if result['returncode'] != 0: - logger.error(f"Failed to download YAML manifest from {yaml_url}") - sys.exit(1) - - # Apply the manifest - apply_result = self._run_command([ - "kubectl", "apply", "-f", "efs-csi-driver.yaml" - ]) - - if apply_result['returncode'] != 0: - logger.error("Failed to apply EFS CSI driver YAML") - sys.exit(1) - - logger.info("EFS CSI driver installed successfully via YAML") - - def _install_custom(self): - """Install the EFS CSI Driver using custom method specified in config""" - logger.info("Using custom installation method - this must be implemented by user") - # This is a placeholder for custom installation logic - # Users would implement this in their own subclass or modify this method - - def _verify_driver_installation(self): - """Verify that the EFS CSI Driver is installed and running""" - pod_check = self._run_command([ - "kubectl", "get", "pods", "-n", "kube-system", - "-l", "app=efs-csi-controller", - "--no-headers" - ]) - - if pod_check['returncode'] != 0 or not pod_check['stdout'].strip(): - logger.error("Could not find EFS CSI controller pods") - sys.exit(1) - - # Count running pods - running_pods = 0 - for line in pod_check['stdout'].strip().split('\n'): - if line and "Running" in line: - running_pods += 1 - - if running_pods == 0: - logger.error("No EFS CSI controller pods are running") - sys.exit(1) - - logger.info(f"Found {running_pods} running EFS CSI controller pods") - - # Check CSI driver registration - csi_check = self._run_command([ - "kubectl", "get", "csidrivers.storage.k8s.io", - "efs.csi.aws.com", "--no-headers" - ]) - - if csi_check['returncode'] != 0: - logger.warning("CSI driver not registered as a custom resource") - else: - logger.info("CSI driver registered successfully") - - def create_storage_class(self): - """Create the storage class for the EFS CSI Driver""" - if not self.filesystem_id: - logger.error("Cannot create storage class without filesystem ID") - sys.exit(1) - - logger.info(f"Creating StorageClass with filesystem ID: {self.filesystem_id}") - - # Get storage class configuration from config - sc_config = self.config.get('storage_class', {}) - sc_name = sc_config.get('name', 'efs-sc') - sc_parameters = sc_config.get('parameters', {}) - - # Ensure filesystem ID is set - sc_parameters['fileSystemId'] = self.filesystem_id - - storage_class = { - "apiVersion": "storage.k8s.io/v1", - "kind": "StorageClass", - "metadata": {"name": sc_name}, - "provisioner": "efs.csi.aws.com", - "parameters": sc_parameters - } - - # Add mount options if specified - mount_options = sc_config.get('mount_options') - if mount_options: - storage_class["mountOptions"] = mount_options - - # Add reclaim policy if specified - reclaim_policy = sc_config.get('reclaim_policy') - if reclaim_policy: - storage_class["reclaimPolicy"] = reclaim_policy - - # Add volume binding mode if specified - volume_binding_mode = sc_config.get('volume_binding_mode') - if volume_binding_mode: - storage_class["volumeBindingMode"] = volume_binding_mode - - # Write storage class to file - sc_path = "storage-class.yaml" - with open(sc_path, 'w') as f: - yaml.dump(storage_class, f) - - # Apply storage class - result = self._run_command(["kubectl", "apply", "-f", sc_path]) - - if result['returncode'] != 0: - logger.error("Failed to create storage class") - sys.exit(1) - - logger.info(f"Storage class '{sc_name}' created successfully") - - # Verify storage class - verify_result = self._run_command([ - "kubectl", "get", "storageclass", sc_name - ]) - - if verify_result['returncode'] == 0: - logger.info(f"Verified storage class '{sc_name}' exists") - else: - logger.warning(f"Could not verify storage class '{sc_name}'") - - def cleanup(self, delete_cluster=True): - """Clean up resources - delete EFS filesystem and optionally the EKS cluster""" - logger.info("Starting cleanup process...") - - if self.filesystem_id and self.create_filesystem: - logger.info(f"Deleting EFS filesystem {self.filesystem_id}...") - - # First delete mount targets - try: - targets_result = self._run_command([ - "aws", "efs", "describe-mount-targets", - "--file-system-id", self.filesystem_id, - "--output", "json" - ]) - - if targets_result['returncode'] == 0: - targets = json.loads(targets_result['stdout']).get('MountTargets', []) - - for target in targets: - target_id = target['MountTargetId'] - logger.info(f"Deleting mount target {target_id}") - self._run_command([ - "aws", "efs", "delete-mount-target", - "--mount-target-id", target_id - ]) - - # Wait for mount targets to be deleted - logger.info("Waiting for mount targets to be deleted...") - time.sleep(30) - except Exception as e: - logger.warning(f"Error while deleting mount targets: {e}") - - # Now delete the filesystem - try: - self._run_command([ - "aws", "efs", "delete-file-system", - "--file-system-id", self.filesystem_id - ]) - logger.info(f"EFS filesystem {self.filesystem_id} deletion initiated") - except Exception as e: - logger.warning(f"Error while deleting filesystem: {e}") - - if delete_cluster and self.cluster_name and self.create_cluster: - logger.info(f"Deleting EKS cluster {self.cluster_name}...") - - try: - self._run_command([ - "eksctl", "delete", "cluster", - "--name", self.cluster_name, - "--region", self.region - ]) - logger.info(f"EKS cluster {self.cluster_name} deletion initiated") - except Exception as e: - logger.warning(f"Error while deleting cluster: {e}") - - logger.info("Cleanup process completed") - - def _run_command(self, cmd): - """Run a command and return stdout, stderr, and return code""" - logger.info(f"Running: {' '.join(cmd)}") - - process = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True - ) - - stdout, stderr = process.communicate() - - if process.returncode != 0: - logger.warning(f"Command returned non-zero exit code: {process.returncode}") - logger.warning(f"stderr: {stderr}") - - return { - "stdout": stdout, - "stderr": stderr, - "returncode": process.returncode - } - -def main(): - """Main function when run as a script""" - parser = argparse.ArgumentParser(description='EFS CSI Driver Cluster Setup') - - parser.add_argument( - '--config', - default='config/orchestrator_config.yaml', - help='Path to configuration file' - ) - - parser.add_argument( - '--no-cleanup', - action='store_true', - help='Do not clean up resources on failure' - ) - - parser.add_argument( - '--verify-only', - action='store_true', - help='Only verify existing cluster and exit' - ) - - parser.add_argument( - '--debug', - action='store_true', - help='Enable debug logging' - ) - - args = parser.parse_args() - - # Set debug logging if requested - if args.debug: - logger.setLevel(logging.DEBUG) - for handler in logger.handlers: - handler.setLevel(logging.DEBUG) - - try: - # Initialize setup - setup = ClusterSetup(config_file=args.config) - - if args.verify_only: - setup.verify_cluster_connection() - logger.info("Cluster verification completed successfully") - return 0 - - # Run setup - result = setup.setup() - logger.info(f"Setup completed successfully: {result}") - return 0 - - except Exception as e: - logger.exception(f"Error during setup: {e}") - - if not args.no_cleanup: - try: - logger.info("Attempting cleanup after failure...") - setup.cleanup() - except Exception as cleanup_error: - logger.error(f"Cleanup error: {cleanup_error}") - - return 1 - -if __name__ == "__main__": - sys.exit(main()) diff --git a/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749045745.yaml b/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749045745.yaml deleted file mode 100644 index 49d66df21..000000000 --- a/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749045745.yaml +++ /dev/null @@ -1,51 +0,0 @@ -apiVersion: eksctl.io/v1alpha5 -iam: - serviceAccounts: - - attachPolicyARNs: - - arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy - metadata: - name: efs-csi-controller-sa - namespace: kube-system - withOIDC: true -kind: ClusterConfig -metadata: - name: efs-csi-test-1749045745 - region: us-west-2 - version: '1.25' -nodeGroups: -- desiredCapacity: 3 - iam: - attachPolicy: - Statement: - - Action: - - elasticfilesystem:DescribeAccessPoints - - elasticfilesystem:DescribeFileSystems - - elasticfilesystem:DescribeMountTargets - - ec2:DescribeAvailabilityZones - Effect: Allow - Resource: '*' - - Action: - - elasticfilesystem:CreateAccessPoint - Condition: - StringLike: - aws:RequestTag/kubernetes.io/cluster/*: owned - Effect: Allow - Resource: '*' - - Action: - - elasticfilesystem:DeleteAccessPoint - Condition: - StringEquals: - aws:ResourceTag/kubernetes.io/cluster/*: owned - Effect: Allow - Resource: '*' - Version: '2012-10-17' - instanceType: t3.large - labels: - efs-issue: 'true' - role: general - maxSize: 5 - minSize: 2 - name: ng-1 - tags: - nodegroup-role: general - volumeSize: 80 diff --git a/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749045783.yaml b/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749045783.yaml deleted file mode 100644 index 6da348e66..000000000 --- a/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749045783.yaml +++ /dev/null @@ -1,51 +0,0 @@ -apiVersion: eksctl.io/v1alpha5 -iam: - serviceAccounts: - - attachPolicyARNs: - - arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy - metadata: - name: efs-csi-controller-sa - namespace: kube-system - withOIDC: true -kind: ClusterConfig -metadata: - name: efs-csi-test-1749045783 - region: us-west-2 - version: '1.25' -nodeGroups: -- desiredCapacity: 3 - iam: - attachPolicy: - Statement: - - Action: - - elasticfilesystem:DescribeAccessPoints - - elasticfilesystem:DescribeFileSystems - - elasticfilesystem:DescribeMountTargets - - ec2:DescribeAvailabilityZones - Effect: Allow - Resource: '*' - - Action: - - elasticfilesystem:CreateAccessPoint - Condition: - StringLike: - aws:RequestTag/kubernetes.io/cluster/*: owned - Effect: Allow - Resource: '*' - - Action: - - elasticfilesystem:DeleteAccessPoint - Condition: - StringEquals: - aws:ResourceTag/kubernetes.io/cluster/*: owned - Effect: Allow - Resource: '*' - Version: '2012-10-17' - instanceType: t3.large - labels: - efs-issue: 'true' - role: general - maxSize: 5 - minSize: 2 - name: ng-1 - tags: - nodegroup-role: general - volumeSize: 80 diff --git a/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749048222.yaml b/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749048222.yaml deleted file mode 100644 index b4f93b86e..000000000 --- a/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749048222.yaml +++ /dev/null @@ -1,51 +0,0 @@ -apiVersion: eksctl.io/v1alpha5 -iam: - serviceAccounts: - - attachPolicyARNs: - - arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy - metadata: - name: efs-csi-controller-sa - namespace: kube-system - withOIDC: true -kind: ClusterConfig -metadata: - name: efs-csi-test-1749048222 - region: us-west-2 - version: '1.25' -nodeGroups: -- desiredCapacity: 3 - iam: - attachPolicy: - Statement: - - Action: - - elasticfilesystem:DescribeAccessPoints - - elasticfilesystem:DescribeFileSystems - - elasticfilesystem:DescribeMountTargets - - ec2:DescribeAvailabilityZones - Effect: Allow - Resource: '*' - - Action: - - elasticfilesystem:CreateAccessPoint - Condition: - StringLike: - aws:RequestTag/kubernetes.io/cluster/*: owned - Effect: Allow - Resource: '*' - - Action: - - elasticfilesystem:DeleteAccessPoint - Condition: - StringEquals: - aws:ResourceTag/kubernetes.io/cluster/*: owned - Effect: Allow - Resource: '*' - Version: '2012-10-17' - instanceType: t3.large - labels: - efs-issue: 'true' - role: general - maxSize: 5 - minSize: 2 - name: ng-1 - tags: - nodegroup-role: general - volumeSize: 80 diff --git a/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749048235.yaml b/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749048235.yaml deleted file mode 100644 index 3defffefe..000000000 --- a/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749048235.yaml +++ /dev/null @@ -1,51 +0,0 @@ -apiVersion: eksctl.io/v1alpha5 -iam: - serviceAccounts: - - attachPolicyARNs: - - arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy - metadata: - name: efs-csi-controller-sa - namespace: kube-system - withOIDC: true -kind: ClusterConfig -metadata: - name: efs-csi-test-1749048235 - region: us-west-2 - version: '1.33' -nodeGroups: -- desiredCapacity: 3 - iam: - attachPolicy: - Statement: - - Action: - - elasticfilesystem:DescribeAccessPoints - - elasticfilesystem:DescribeFileSystems - - elasticfilesystem:DescribeMountTargets - - ec2:DescribeAvailabilityZones - Effect: Allow - Resource: '*' - - Action: - - elasticfilesystem:CreateAccessPoint - Condition: - StringLike: - aws:RequestTag/kubernetes.io/cluster/*: owned - Effect: Allow - Resource: '*' - - Action: - - elasticfilesystem:DeleteAccessPoint - Condition: - StringEquals: - aws:ResourceTag/kubernetes.io/cluster/*: owned - Effect: Allow - Resource: '*' - Version: '2012-10-17' - instanceType: t3.large - labels: - efs-issue: 'true' - role: general - maxSize: 5 - minSize: 2 - name: ng-1 - tags: - nodegroup-role: general - volumeSize: 80 diff --git a/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749048734.yaml b/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749048734.yaml deleted file mode 100644 index dfcfaea6b..000000000 --- a/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749048734.yaml +++ /dev/null @@ -1,51 +0,0 @@ -apiVersion: eksctl.io/v1alpha5 -iam: - serviceAccounts: - - attachPolicyARNs: - - arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy - metadata: - name: efs-csi-controller-sa - namespace: kube-system - withOIDC: true -kind: ClusterConfig -metadata: - name: efs-csi-test-1749048734 - region: us-west-2 - version: '1.33' -nodeGroups: -- desiredCapacity: 3 - iam: - attachPolicy: - Statement: - - Action: - - elasticfilesystem:DescribeAccessPoints - - elasticfilesystem:DescribeFileSystems - - elasticfilesystem:DescribeMountTargets - - ec2:DescribeAvailabilityZones - Effect: Allow - Resource: '*' - - Action: - - elasticfilesystem:CreateAccessPoint - Condition: - StringLike: - aws:RequestTag/kubernetes.io/cluster/*: owned - Effect: Allow - Resource: '*' - - Action: - - elasticfilesystem:DeleteAccessPoint - Condition: - StringEquals: - aws:ResourceTag/kubernetes.io/cluster/*: owned - Effect: Allow - Resource: '*' - Version: '2012-10-17' - instanceType: t3.large - labels: - efs-issue: 'true' - role: general - maxSize: 5 - minSize: 2 - name: ng-1 - tags: - nodegroup-role: general - volumeSize: 80 diff --git a/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749051234.yaml b/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749051234.yaml deleted file mode 100644 index dfb706581..000000000 --- a/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749051234.yaml +++ /dev/null @@ -1,51 +0,0 @@ -apiVersion: eksctl.io/v1alpha5 -iam: - serviceAccounts: - - attachPolicyARNs: - - arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy - metadata: - name: efs-csi-controller-sa - namespace: kube-system - withOIDC: true -kind: ClusterConfig -metadata: - name: efs-csi-test-1749051234 - region: us-west-2 - version: '1.33' -nodeGroups: -- desiredCapacity: 3 - iam: - attachPolicy: - Statement: - - Action: - - elasticfilesystem:DescribeAccessPoints - - elasticfilesystem:DescribeFileSystems - - elasticfilesystem:DescribeMountTargets - - ec2:DescribeAvailabilityZones - Effect: Allow - Resource: '*' - - Action: - - elasticfilesystem:CreateAccessPoint - Condition: - StringLike: - aws:RequestTag/kubernetes.io/cluster/*: owned - Effect: Allow - Resource: '*' - - Action: - - elasticfilesystem:DeleteAccessPoint - Condition: - StringEquals: - aws:ResourceTag/kubernetes.io/cluster/*: owned - Effect: Allow - Resource: '*' - Version: '2012-10-17' - instanceType: t3.large - labels: - efs-issue: 'true' - role: general - maxSize: 5 - minSize: 2 - name: ng-1 - tags: - nodegroup-role: general - volumeSize: 80 diff --git a/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749051325.yaml b/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749051325.yaml deleted file mode 100644 index e185004f3..000000000 --- a/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749051325.yaml +++ /dev/null @@ -1,51 +0,0 @@ -apiVersion: eksctl.io/v1alpha5 -iam: - serviceAccounts: - - attachPolicyARNs: - - arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy - metadata: - name: efs-csi-controller-sa - namespace: kube-system - withOIDC: true -kind: ClusterConfig -metadata: - name: efs-csi-test-1749051325 - region: us-west-2 - version: '1.33' -nodeGroups: -- desiredCapacity: 3 - iam: - attachPolicy: - Statement: - - Action: - - elasticfilesystem:DescribeAccessPoints - - elasticfilesystem:DescribeFileSystems - - elasticfilesystem:DescribeMountTargets - - ec2:DescribeAvailabilityZones - Effect: Allow - Resource: '*' - - Action: - - elasticfilesystem:CreateAccessPoint - Condition: - StringLike: - aws:RequestTag/kubernetes.io/cluster/*: owned - Effect: Allow - Resource: '*' - - Action: - - elasticfilesystem:DeleteAccessPoint - Condition: - StringEquals: - aws:ResourceTag/kubernetes.io/cluster/*: owned - Effect: Allow - Resource: '*' - Version: '2012-10-17' - instanceType: t3.large - labels: - efs-issue: 'true' - role: general - maxSize: 5 - minSize: 2 - name: ng-1 - tags: - nodegroup-role: general - volumeSize: 80 diff --git a/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749051374.yaml b/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749051374.yaml deleted file mode 100644 index c1d32de98..000000000 --- a/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749051374.yaml +++ /dev/null @@ -1,51 +0,0 @@ -apiVersion: eksctl.io/v1alpha5 -iam: - serviceAccounts: - - attachPolicyARNs: - - arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy - metadata: - name: efs-csi-controller-sa - namespace: kube-system - withOIDC: true -kind: ClusterConfig -metadata: - name: efs-csi-test-1749051374 - region: us-west-2 - version: '1.33' -nodeGroups: -- desiredCapacity: 3 - iam: - attachPolicy: - Statement: - - Action: - - elasticfilesystem:DescribeAccessPoints - - elasticfilesystem:DescribeFileSystems - - elasticfilesystem:DescribeMountTargets - - ec2:DescribeAvailabilityZones - Effect: Allow - Resource: '*' - - Action: - - elasticfilesystem:CreateAccessPoint - Condition: - StringLike: - aws:RequestTag/kubernetes.io/cluster/*: owned - Effect: Allow - Resource: '*' - - Action: - - elasticfilesystem:DeleteAccessPoint - Condition: - StringEquals: - aws:ResourceTag/kubernetes.io/cluster/*: owned - Effect: Allow - Resource: '*' - Version: '2012-10-17' - instanceType: t3.large - labels: - efs-issue: 'true' - role: general - maxSize: 5 - minSize: 2 - name: ng-1 - tags: - nodegroup-role: general - volumeSize: 80 diff --git a/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749051392.yaml b/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749051392.yaml deleted file mode 100644 index e4fc85ea9..000000000 --- a/test/e2e/stress-scale-tests/cluster-config-code/efs-csi-test-1749051392.yaml +++ /dev/null @@ -1,51 +0,0 @@ -apiVersion: eksctl.io/v1alpha5 -iam: - serviceAccounts: - - attachPolicyARNs: - - arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy - metadata: - name: efs-csi-controller-sa - namespace: kube-system - withOIDC: true -kind: ClusterConfig -metadata: - name: efs-csi-test-1749051392 - region: us-west-2 - version: '1.28' -nodeGroups: -- desiredCapacity: 3 - iam: - attachPolicy: - Statement: - - Action: - - elasticfilesystem:DescribeAccessPoints - - elasticfilesystem:DescribeFileSystems - - elasticfilesystem:DescribeMountTargets - - ec2:DescribeAvailabilityZones - Effect: Allow - Resource: '*' - - Action: - - elasticfilesystem:CreateAccessPoint - Condition: - StringLike: - aws:RequestTag/kubernetes.io/cluster/*: owned - Effect: Allow - Resource: '*' - - Action: - - elasticfilesystem:DeleteAccessPoint - Condition: - StringEquals: - aws:ResourceTag/kubernetes.io/cluster/*: owned - Effect: Allow - Resource: '*' - Version: '2012-10-17' - instanceType: t3.large - labels: - efs-issue: 'true' - role: general - maxSize: 5 - minSize: 2 - name: ng-1 - tags: - nodegroup-role: general - volumeSize: 80 diff --git a/test/e2e/stress-scale-tests/logs/cleanup_20250605_190904.log b/test/e2e/stress-scale-tests/logs/cleanup_20250605_190904.log deleted file mode 100644 index 1bd96f257..000000000 --- a/test/e2e/stress-scale-tests/logs/cleanup_20250605_190904.log +++ /dev/null @@ -1,129 +0,0 @@ -2025-06-05 19:09:04,981 - INFO - Starting EFS CSI Driver test resource cleanup -2025-06-05 19:09:05,918 - INFO - Cleaning up resources in namespaces: ['default', 'efs-stress-test'] -2025-06-05 19:09:06,196 - INFO - Deleting pod/efs-scale-test-0 in namespace default -2025-06-05 19:09:06,287 - INFO - Deleting pod/efs-scale-test-1 in namespace default -2025-06-05 19:09:06,368 - INFO - Deleting pod/efs-scale-test-10 in namespace default -2025-06-05 19:09:06,463 - INFO - Deleting pod/efs-scale-test-11 in namespace default -2025-06-05 19:09:06,555 - INFO - Deleting pod/efs-scale-test-12 in namespace default -2025-06-05 19:09:06,645 - INFO - Deleting pod/efs-scale-test-13 in namespace default -2025-06-05 19:09:06,738 - INFO - Deleting pod/efs-scale-test-14 in namespace default -2025-06-05 19:09:06,820 - INFO - Deleting pod/efs-scale-test-15 in namespace default -2025-06-05 19:09:06,900 - INFO - Deleting pod/efs-scale-test-16 in namespace default -2025-06-05 19:09:06,983 - INFO - Deleting pod/efs-scale-test-17 in namespace default -2025-06-05 19:09:07,064 - INFO - Deleting pod/efs-scale-test-18 in namespace default -2025-06-05 19:09:07,151 - INFO - Deleting pod/efs-scale-test-19 in namespace default -2025-06-05 19:09:07,231 - INFO - Deleting pod/efs-scale-test-2 in namespace default -2025-06-05 19:09:07,310 - INFO - Deleting pod/efs-scale-test-3 in namespace default -2025-06-05 19:09:07,391 - INFO - Deleting pod/efs-scale-test-4 in namespace default -2025-06-05 19:09:07,471 - INFO - Deleting pod/efs-scale-test-5 in namespace default -2025-06-05 19:09:07,562 - INFO - Deleting pod/efs-scale-test-6 in namespace default -2025-06-05 19:09:07,650 - INFO - Deleting pod/efs-scale-test-7 in namespace default -2025-06-05 19:09:07,729 - INFO - Deleting pod/efs-scale-test-8 in namespace default -2025-06-05 19:09:07,813 - INFO - Deleting pod/efs-scale-test-9 in namespace default -2025-06-05 19:09:07,968 - INFO - Deleting pod/test-pod-3cc2d48d in namespace efs-stress-test -2025-06-05 19:09:08,055 - INFO - Deleting pod/test-pod-564344d0 in namespace efs-stress-test -2025-06-05 19:09:08,144 - INFO - Deleting pod/test-pod-a9935180 in namespace efs-stress-test -2025-06-05 19:09:08,225 - INFO - Deleting pod/test-pod-e1337987 in namespace efs-stress-test -2025-06-05 19:09:08,311 - INFO - Waiting 5 seconds for pods to start terminating before deleting PVCs... -2025-06-05 19:09:13,444 - INFO - Deleting pvc/efs-volume-efs-scale-test-0 in namespace default -2025-06-05 19:09:13,514 - INFO - Deleting pvc/efs-volume-efs-scale-test-1 in namespace default -2025-06-05 19:09:13,585 - INFO - Deleting pvc/efs-volume-efs-scale-test-10 in namespace default -2025-06-05 19:09:13,658 - INFO - Deleting pvc/efs-volume-efs-scale-test-11 in namespace default -2025-06-05 19:09:13,729 - INFO - Deleting pvc/efs-volume-efs-scale-test-12 in namespace default -2025-06-05 19:09:13,803 - INFO - Deleting pvc/efs-volume-efs-scale-test-13 in namespace default -2025-06-05 19:09:13,874 - INFO - Deleting pvc/efs-volume-efs-scale-test-14 in namespace default -2025-06-05 19:09:13,944 - INFO - Deleting pvc/efs-volume-efs-scale-test-15 in namespace default -2025-06-05 19:09:14,015 - INFO - Deleting pvc/efs-volume-efs-scale-test-16 in namespace default -2025-06-05 19:09:14,088 - INFO - Deleting pvc/efs-volume-efs-scale-test-17 in namespace default -2025-06-05 19:09:14,159 - INFO - Deleting pvc/efs-volume-efs-scale-test-18 in namespace default -2025-06-05 19:09:14,232 - INFO - Deleting pvc/efs-volume-efs-scale-test-19 in namespace default -2025-06-05 19:09:14,308 - INFO - Deleting pvc/efs-volume-efs-scale-test-2 in namespace default -2025-06-05 19:09:14,378 - INFO - Deleting pvc/efs-volume-efs-scale-test-3 in namespace default -2025-06-05 19:09:14,449 - INFO - Deleting pvc/efs-volume-efs-scale-test-4 in namespace default -2025-06-05 19:09:14,520 - INFO - Deleting pvc/efs-volume-efs-scale-test-5 in namespace default -2025-06-05 19:09:14,592 - INFO - Deleting pvc/efs-volume-efs-scale-test-6 in namespace default -2025-06-05 19:09:14,663 - INFO - Deleting pvc/efs-volume-efs-scale-test-7 in namespace default -2025-06-05 19:09:14,735 - INFO - Deleting pvc/efs-volume-efs-scale-test-8 in namespace default -2025-06-05 19:09:14,806 - INFO - Deleting pvc/efs-volume-efs-scale-test-9 in namespace default -2025-06-05 19:09:15,013 - INFO - Deleting pvc/test-pvc-0b5af805 in namespace efs-stress-test -2025-06-05 19:09:15,088 - INFO - Deleting pvc/test-pvc-0bf22cd9 in namespace efs-stress-test -2025-06-05 19:09:15,161 - INFO - Deleting pvc/test-pvc-1bb798a3 in namespace efs-stress-test -2025-06-05 19:09:15,237 - INFO - Deleting pvc/test-pvc-29a67853 in namespace efs-stress-test -2025-06-05 19:09:15,308 - INFO - Deleting pvc/test-pvc-36370699 in namespace efs-stress-test -2025-06-05 19:09:15,382 - INFO - Deleting pvc/test-pvc-3691f2a0 in namespace efs-stress-test -2025-06-05 19:09:15,454 - INFO - Deleting pvc/test-pvc-3d16a983 in namespace efs-stress-test -2025-06-05 19:09:15,526 - INFO - Deleting pvc/test-pvc-4184bff1 in namespace efs-stress-test -2025-06-05 19:09:15,600 - INFO - Deleting pvc/test-pvc-41d09b77 in namespace efs-stress-test -2025-06-05 19:09:15,672 - INFO - Deleting pvc/test-pvc-455abf8b in namespace efs-stress-test -2025-06-05 19:09:15,746 - INFO - Deleting pvc/test-pvc-51dbaa96 in namespace efs-stress-test -2025-06-05 19:09:15,820 - INFO - Deleting pvc/test-pvc-601432ce in namespace efs-stress-test -2025-06-05 19:09:15,891 - INFO - Deleting pvc/test-pvc-60b59274 in namespace efs-stress-test -2025-06-05 19:09:15,962 - INFO - Deleting pvc/test-pvc-63ada5ce in namespace efs-stress-test -2025-06-05 19:09:16,034 - INFO - Deleting pvc/test-pvc-64bf8174 in namespace efs-stress-test -2025-06-05 19:09:16,105 - INFO - Deleting pvc/test-pvc-6e8e6345 in namespace efs-stress-test -2025-06-05 19:09:16,179 - INFO - Deleting pvc/test-pvc-7ae6d5e7 in namespace efs-stress-test -2025-06-05 19:09:16,251 - INFO - Deleting pvc/test-pvc-b84d5df8 in namespace efs-stress-test -2025-06-05 19:09:16,322 - INFO - Deleting pvc/test-pvc-c196bb79 in namespace efs-stress-test -2025-06-05 19:09:16,400 - INFO - Deleting pvc/test-pvc-c1c2eedb in namespace efs-stress-test -2025-06-05 19:09:16,481 - INFO - Deleting pvc/test-pvc-c6976eae in namespace efs-stress-test -2025-06-05 19:09:16,552 - INFO - Deleting pvc/test-pvc-d0c53501 in namespace efs-stress-test -2025-06-05 19:09:16,630 - INFO - Deleting pvc/test-pvc-d3ee5325 in namespace efs-stress-test -2025-06-05 19:09:16,702 - INFO - Deleting pvc/test-pvc-faea1890 in namespace efs-stress-test -2025-06-05 19:09:16,775 - INFO - Deleting pvc/test-pvc-fe0fe649 in namespace efs-stress-test -2025-06-05 19:09:16,849 - INFO - Cleanup Summary: -2025-06-05 19:09:16,849 - INFO - Deleted 24 pods and 45 PVCs -2025-06-05 19:09:16,849 - INFO - All resources deleted successfully -2025-06-05 19:09:16,850 - INFO - Verifying resource deletion for up to 60 seconds... -2025-06-05 19:09:17,649 - INFO - Still waiting on 40 resources to be deleted... -2025-06-05 19:09:23,324 - INFO - Still waiting on 40 resources to be deleted... -2025-06-05 19:09:28,995 - INFO - Still waiting on 40 resources to be deleted... -2025-06-05 19:09:34,677 - INFO - Still waiting on 40 resources to be deleted... -2025-06-05 19:09:40,425 - INFO - Still waiting on 40 resources to be deleted... -2025-06-05 19:09:46,178 - INFO - Still waiting on 40 resources to be deleted... -2025-06-05 19:09:51,931 - INFO - Still waiting on 40 resources to be deleted... -2025-06-05 19:09:57,697 - INFO - Still waiting on 40 resources to be deleted... -2025-06-05 19:10:03,442 - INFO - Still waiting on 40 resources to be deleted... -2025-06-05 19:10:09,190 - INFO - Still waiting on 40 resources to be deleted... -2025-06-05 19:10:14,938 - INFO - Still waiting on 40 resources to be deleted... -2025-06-05 19:10:19,938 - ERROR - Timed out waiting for resource deletion. Remaining resources: -2025-06-05 19:10:19,939 - ERROR - - pod/default/efs-scale-test-0 -2025-06-05 19:10:19,939 - ERROR - - pod/default/efs-scale-test-1 -2025-06-05 19:10:19,939 - ERROR - - pod/default/efs-scale-test-10 -2025-06-05 19:10:19,939 - ERROR - - pod/default/efs-scale-test-11 -2025-06-05 19:10:19,939 - ERROR - - pod/default/efs-scale-test-12 -2025-06-05 19:10:19,939 - ERROR - - pod/default/efs-scale-test-13 -2025-06-05 19:10:19,939 - ERROR - - pod/default/efs-scale-test-14 -2025-06-05 19:10:19,939 - ERROR - - pod/default/efs-scale-test-15 -2025-06-05 19:10:19,939 - ERROR - - pod/default/efs-scale-test-16 -2025-06-05 19:10:19,939 - ERROR - - pod/default/efs-scale-test-17 -2025-06-05 19:10:19,939 - ERROR - - pod/default/efs-scale-test-18 -2025-06-05 19:10:19,939 - ERROR - - pod/default/efs-scale-test-19 -2025-06-05 19:10:19,939 - ERROR - - pod/default/efs-scale-test-2 -2025-06-05 19:10:19,939 - ERROR - - pod/default/efs-scale-test-3 -2025-06-05 19:10:19,939 - ERROR - - pod/default/efs-scale-test-4 -2025-06-05 19:10:19,939 - ERROR - - pod/default/efs-scale-test-5 -2025-06-05 19:10:19,939 - ERROR - - pod/default/efs-scale-test-6 -2025-06-05 19:10:19,939 - ERROR - - pod/default/efs-scale-test-7 -2025-06-05 19:10:19,939 - ERROR - - pod/default/efs-scale-test-8 -2025-06-05 19:10:19,939 - ERROR - - pod/default/efs-scale-test-9 -2025-06-05 19:10:19,939 - ERROR - - pvc/default/efs-volume-efs-scale-test-0 -2025-06-05 19:10:19,939 - ERROR - - pvc/default/efs-volume-efs-scale-test-1 -2025-06-05 19:10:19,939 - ERROR - - pvc/default/efs-volume-efs-scale-test-10 -2025-06-05 19:10:19,939 - ERROR - - pvc/default/efs-volume-efs-scale-test-11 -2025-06-05 19:10:19,939 - ERROR - - pvc/default/efs-volume-efs-scale-test-12 -2025-06-05 19:10:19,939 - ERROR - - pvc/default/efs-volume-efs-scale-test-13 -2025-06-05 19:10:19,939 - ERROR - - pvc/default/efs-volume-efs-scale-test-14 -2025-06-05 19:10:19,940 - ERROR - - pvc/default/efs-volume-efs-scale-test-15 -2025-06-05 19:10:19,940 - ERROR - - pvc/default/efs-volume-efs-scale-test-16 -2025-06-05 19:10:19,940 - ERROR - - pvc/default/efs-volume-efs-scale-test-17 -2025-06-05 19:10:19,940 - ERROR - - pvc/default/efs-volume-efs-scale-test-18 -2025-06-05 19:10:19,940 - ERROR - - pvc/default/efs-volume-efs-scale-test-19 -2025-06-05 19:10:19,940 - ERROR - - pvc/default/efs-volume-efs-scale-test-2 -2025-06-05 19:10:19,940 - ERROR - - pvc/default/efs-volume-efs-scale-test-3 -2025-06-05 19:10:19,940 - ERROR - - pvc/default/efs-volume-efs-scale-test-4 -2025-06-05 19:10:19,940 - ERROR - - pvc/default/efs-volume-efs-scale-test-5 -2025-06-05 19:10:19,940 - ERROR - - pvc/default/efs-volume-efs-scale-test-6 -2025-06-05 19:10:19,940 - ERROR - - pvc/default/efs-volume-efs-scale-test-7 -2025-06-05 19:10:19,940 - ERROR - - pvc/default/efs-volume-efs-scale-test-8 -2025-06-05 19:10:19,940 - ERROR - - pvc/default/efs-volume-efs-scale-test-9 -2025-06-05 19:10:19,940 - INFO - Cleanup process completed diff --git a/test/e2e/stress-scale-tests/logs/cluster_setup.log b/test/e2e/stress-scale-tests/logs/cluster_setup.log deleted file mode 100644 index 8a879c89f..000000000 --- a/test/e2e/stress-scale-tests/logs/cluster_setup.log +++ /dev/null @@ -1,383 +0,0 @@ -2025-06-04 14:02:25,306 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-04 14:02:25,306 - __main__ - INFO - Verifying AWS credentials -2025-06-04 14:02:25,307 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-04 14:02:25,308 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-04 14:02:25,308 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-04 14:02:25,309 - __main__ - INFO - Setting up EKS cluster and EFS CSI driver -2025-06-04 14:02:25,309 - __main__ - INFO - Using Kubernetes version from command line: 1.25 -2025-06-04 14:02:25,309 - __main__ - INFO - Using EFS CSI Driver version from command line: 2.1.0 -2025-06-04 14:02:25,322 - cluster_setup - INFO - Created cluster config at cluster-config/efs-csi-test-1749045745.yaml -2025-06-04 14:02:25,322 - cluster_setup - INFO - Verifying AWS credentials... -2025-06-04 14:02:25,323 - cluster_setup - INFO - Running: aws sts get-caller-identity -2025-06-04 14:02:26,166 - cluster_setup - WARNING - Command returned non-zero exit code: 254 -2025-06-04 14:02:26,167 - cluster_setup - WARNING - stderr: -An error occurred (ExpiredToken) when calling the GetCallerIdentity operation: The security token included in the request is expired - -2025-06-04 14:02:26,167 - cluster_setup - ERROR - AWS credentials verification failed. Please check your AWS configuration. -2025-06-04 14:02:26,167 - cluster_setup - ERROR - Error: -An error occurred (ExpiredToken) when calling the GetCallerIdentity operation: The security token included in the request is expired - -2025-06-04 14:03:03,790 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-04 14:03:03,790 - __main__ - INFO - Verifying AWS credentials -2025-06-04 14:03:03,791 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-04 14:03:03,791 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-04 14:03:03,792 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-04 14:03:03,792 - __main__ - INFO - Setting up EKS cluster and EFS CSI driver -2025-06-04 14:03:03,792 - __main__ - INFO - Using Kubernetes version from command line: 1.25 -2025-06-04 14:03:03,792 - __main__ - INFO - Using EFS CSI Driver version from command line: 2.1.0 -2025-06-04 14:03:03,807 - cluster_setup - INFO - Created cluster config at cluster-config/efs-csi-test-1749045783.yaml -2025-06-04 14:03:03,807 - cluster_setup - INFO - Verifying AWS credentials... -2025-06-04 14:03:03,807 - cluster_setup - INFO - Running: aws sts get-caller-identity -2025-06-04 14:03:04,654 - cluster_setup - INFO - Using AWS account: 745939127895 with user: AROA23LLPZZLQCLTVMVDC:ibhard-Isengard -2025-06-04 14:03:04,655 - cluster_setup - INFO - Running: eksctl version -2025-06-04 14:03:04,697 - cluster_setup - INFO - eksctl version: 0.208.0 -2025-06-04 14:03:04,697 - cluster_setup - INFO - Creating EKS cluster efs-csi-test-1749045783 in us-west-2... -2025-06-04 14:03:04,697 - cluster_setup - INFO - This may take 15-20 minutes to complete... -2025-06-04 14:03:04,697 - cluster_setup - INFO - Running: eksctl create cluster -f cluster-config/efs-csi-test-1749045783.yaml --verbose 4 -2025-06-04 14:03:05,304 - cluster_setup - WARNING - Command returned non-zero exit code: 1 -2025-06-04 14:03:05,304 - cluster_setup - WARNING - stderr: Error: resolving cluster version: invalid version, 1.25 is no longer supported, supported values: 1.26, 1.27, 1.28, 1.29, 1.30, 1.31, 1.32, 1.33 -see also: https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html - -2025-06-04 14:03:05,304 - cluster_setup - ERROR - Failed to create EKS cluster -2025-06-04 14:03:05,304 - cluster_setup - ERROR - Error details: Error: resolving cluster version: invalid version, 1.25 is no longer supported, supported values: 1.26, 1.27, 1.28, 1.29, 1.30, 1.31, 1.32, 1.33 -see also: https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html - -2025-06-04 14:03:05,304 - cluster_setup - ERROR - Common causes of failure: -2025-06-04 14:03:05,304 - cluster_setup - ERROR - 1. Insufficient IAM permissions -2025-06-04 14:03:05,304 - cluster_setup - ERROR - 2. Service quotas reached (check your AWS console) -2025-06-04 14:03:05,304 - cluster_setup - ERROR - 3. Network connectivity issues -2025-06-04 14:03:05,304 - cluster_setup - ERROR - 4. Region availability zone limitations -2025-06-04 14:03:05,304 - cluster_setup - ERROR - Detailed error log saved to: logs/eks-cluster-creation-error-efs-csi-test-1749045783.log -2025-06-04 14:43:42,712 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-04 14:43:42,712 - __main__ - INFO - Verifying AWS credentials -2025-06-04 14:43:42,713 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-04 14:43:42,714 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-04 14:43:42,714 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-04 14:43:42,715 - __main__ - INFO - Setting up EKS cluster and EFS CSI driver -2025-06-04 14:43:42,715 - __main__ - INFO - Using Kubernetes version from command line: 1.25 -2025-06-04 14:43:42,715 - __main__ - INFO - Using EFS CSI Driver version from command line: 2.1.0 -2025-06-04 14:43:42,728 - cluster_setup - INFO - Created cluster config at cluster-config/efs-csi-test-1749048222.yaml -2025-06-04 14:43:42,728 - cluster_setup - INFO - Verifying AWS credentials... -2025-06-04 14:43:42,728 - cluster_setup - INFO - Running: aws sts get-caller-identity -2025-06-04 14:43:43,584 - cluster_setup - INFO - Using AWS account: 745939127895 with user: AROA23LLPZZLQCLTVMVDC:ibhard-Isengard -2025-06-04 14:43:43,584 - cluster_setup - INFO - Running: eksctl version -2025-06-04 14:43:43,619 - cluster_setup - INFO - eksctl version: 0.208.0 -2025-06-04 14:43:43,619 - cluster_setup - INFO - Creating EKS cluster efs-csi-test-1749048222 in us-west-2... -2025-06-04 14:43:43,619 - cluster_setup - INFO - This may take 15-20 minutes to complete... -2025-06-04 14:43:43,619 - cluster_setup - INFO - Running: eksctl create cluster -f cluster-config/efs-csi-test-1749048222.yaml --verbose 4 -2025-06-04 14:43:44,231 - cluster_setup - WARNING - Command returned non-zero exit code: 1 -2025-06-04 14:43:44,231 - cluster_setup - WARNING - stderr: Error: resolving cluster version: invalid version, 1.25 is no longer supported, supported values: 1.26, 1.27, 1.28, 1.29, 1.30, 1.31, 1.32, 1.33 -see also: https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html - -2025-06-04 14:43:44,231 - cluster_setup - ERROR - Failed to create EKS cluster -2025-06-04 14:43:44,231 - cluster_setup - ERROR - Error details: Error: resolving cluster version: invalid version, 1.25 is no longer supported, supported values: 1.26, 1.27, 1.28, 1.29, 1.30, 1.31, 1.32, 1.33 -see also: https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html - -2025-06-04 14:43:44,231 - cluster_setup - ERROR - Common causes of failure: -2025-06-04 14:43:44,231 - cluster_setup - ERROR - 1. Insufficient IAM permissions -2025-06-04 14:43:44,231 - cluster_setup - ERROR - 2. Service quotas reached (check your AWS console) -2025-06-04 14:43:44,231 - cluster_setup - ERROR - 3. Network connectivity issues -2025-06-04 14:43:44,231 - cluster_setup - ERROR - 4. Region availability zone limitations -2025-06-04 14:43:44,231 - cluster_setup - ERROR - Detailed error log saved to: logs/eks-cluster-creation-error-efs-csi-test-1749048222.log -2025-06-04 14:43:55,254 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-04 14:43:55,254 - __main__ - INFO - Verifying AWS credentials -2025-06-04 14:43:55,255 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-04 14:43:55,255 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-04 14:43:55,256 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-04 14:43:55,256 - __main__ - INFO - Setting up EKS cluster and EFS CSI driver -2025-06-04 14:43:55,256 - __main__ - INFO - Using Kubernetes version from command line: 1.33 -2025-06-04 14:43:55,256 - __main__ - INFO - Using EFS CSI Driver version from command line: 2.1.0 -2025-06-04 14:43:55,269 - cluster_setup - INFO - Created cluster config at cluster-config/efs-csi-test-1749048235.yaml -2025-06-04 14:43:55,269 - cluster_setup - INFO - Verifying AWS credentials... -2025-06-04 14:43:55,269 - cluster_setup - INFO - Running: aws sts get-caller-identity -2025-06-04 14:43:56,102 - cluster_setup - INFO - Using AWS account: 745939127895 with user: AROA23LLPZZLQCLTVMVDC:ibhard-Isengard -2025-06-04 14:43:56,102 - cluster_setup - INFO - Running: eksctl version -2025-06-04 14:43:56,138 - cluster_setup - INFO - eksctl version: 0.208.0 -2025-06-04 14:43:56,138 - cluster_setup - INFO - Creating EKS cluster efs-csi-test-1749048235 in us-west-2... -2025-06-04 14:43:56,138 - cluster_setup - INFO - This may take 15-20 minutes to complete... -2025-06-04 14:43:56,138 - cluster_setup - INFO - Running: eksctl create cluster -f cluster-config/efs-csi-test-1749048235.yaml --verbose 4 -2025-06-04 14:43:57,393 - cluster_setup - WARNING - Command returned non-zero exit code: 1 -2025-06-04 14:43:57,393 - cluster_setup - WARNING - stderr: Error: unable to determine AMI to use: error getting AMI from SSM Parameter Store: operation error SSM: GetParameter, https response error StatusCode: 400, RequestID: a443a5ad-af0e-4afb-b46f-a021532755a5, ParameterNotFound: . please verify that AMI Family is supported - -2025-06-04 14:43:57,393 - cluster_setup - ERROR - Failed to create EKS cluster -2025-06-04 14:43:57,393 - cluster_setup - ERROR - Error details: Error: unable to determine AMI to use: error getting AMI from SSM Parameter Store: operation error SSM: GetParameter, https response error StatusCode: 400, RequestID: a443a5ad-af0e-4afb-b46f-a021532755a5, ParameterNotFound: . please verify that AMI Family is supported - -2025-06-04 14:43:57,393 - cluster_setup - ERROR - Common causes of failure: -2025-06-04 14:43:57,393 - cluster_setup - ERROR - 1. Insufficient IAM permissions -2025-06-04 14:43:57,393 - cluster_setup - ERROR - 2. Service quotas reached (check your AWS console) -2025-06-04 14:43:57,393 - cluster_setup - ERROR - 3. Network connectivity issues -2025-06-04 14:43:57,393 - cluster_setup - ERROR - 4. Region availability zone limitations -2025-06-04 14:43:57,393 - cluster_setup - ERROR - Detailed error log saved to: logs/eks-cluster-creation-error-efs-csi-test-1749048235.log -2025-06-04 14:52:14,794 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-04 14:52:14,794 - __main__ - INFO - Verifying AWS credentials -2025-06-04 14:52:14,795 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-04 14:52:14,795 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-04 14:52:14,796 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-04 14:52:14,796 - __main__ - INFO - Setting up EKS cluster and EFS CSI driver -2025-06-04 14:52:14,796 - __main__ - INFO - Using Kubernetes version from command line: 1.33 -2025-06-04 14:52:14,796 - __main__ - INFO - Using EFS CSI Driver version from command line: 2.1.0 -2025-06-04 14:52:14,809 - cluster_setup - INFO - Created cluster config at cluster-config/efs-csi-test-1749048734.yaml -2025-06-04 14:52:14,809 - cluster_setup - INFO - Verifying AWS credentials... -2025-06-04 14:52:14,809 - cluster_setup - INFO - Running: aws sts get-caller-identity -2025-06-04 14:52:15,649 - cluster_setup - INFO - Using AWS account: 745939127895 with user: AROA23LLPZZLQCLTVMVDC:ibhard-Isengard -2025-06-04 14:52:15,649 - cluster_setup - INFO - Running: eksctl version -2025-06-04 14:52:15,683 - cluster_setup - INFO - eksctl version: 0.208.0 -2025-06-04 14:52:15,683 - cluster_setup - INFO - Creating EKS cluster efs-csi-test-1749048734 in us-west-2... -2025-06-04 14:52:15,683 - cluster_setup - INFO - This may take 15-20 minutes to complete... -2025-06-04 14:52:15,683 - cluster_setup - INFO - Running: eksctl create cluster -f cluster-config/efs-csi-test-1749048734.yaml --verbose 4 -2025-06-04 14:52:16,955 - cluster_setup - WARNING - Command returned non-zero exit code: 1 -2025-06-04 14:52:16,955 - cluster_setup - WARNING - stderr: Error: unable to determine AMI to use: error getting AMI from SSM Parameter Store: operation error SSM: GetParameter, https response error StatusCode: 400, RequestID: 4650ea1c-e137-440e-835d-9214a82b7beb, ParameterNotFound: . please verify that AMI Family is supported - -2025-06-04 14:52:16,955 - cluster_setup - ERROR - Failed to create EKS cluster -2025-06-04 14:52:16,955 - cluster_setup - ERROR - Error details: Error: unable to determine AMI to use: error getting AMI from SSM Parameter Store: operation error SSM: GetParameter, https response error StatusCode: 400, RequestID: 4650ea1c-e137-440e-835d-9214a82b7beb, ParameterNotFound: . please verify that AMI Family is supported - -2025-06-04 14:52:16,955 - cluster_setup - ERROR - Common causes of failure: -2025-06-04 14:52:16,955 - cluster_setup - ERROR - 1. Insufficient IAM permissions -2025-06-04 14:52:16,956 - cluster_setup - ERROR - 2. Service quotas reached (check your AWS console) -2025-06-04 14:52:16,956 - cluster_setup - ERROR - 3. Network connectivity issues -2025-06-04 14:52:16,956 - cluster_setup - ERROR - 4. Region availability zone limitations -2025-06-04 14:52:16,956 - cluster_setup - ERROR - Detailed error log saved to: logs/eks-cluster-creation-error-efs-csi-test-1749048734.log -2025-06-04 15:33:53,994 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-04 15:33:53,994 - __main__ - INFO - Verifying AWS credentials -2025-06-04 15:33:53,995 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-04 15:33:53,995 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-04 15:33:53,996 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-04 15:33:53,996 - __main__ - INFO - Setting up EKS cluster and EFS CSI driver -2025-06-04 15:33:53,996 - __main__ - INFO - Using Kubernetes version from command line: 1.33 -2025-06-04 15:33:53,996 - __main__ - INFO - Using EFS CSI Driver version from command line: 2.1.0 -2025-06-04 15:33:54,008 - cluster_setup - INFO - Created cluster config at cluster-config/efs-csi-test-1749051234.yaml -2025-06-04 15:33:54,008 - cluster_setup - INFO - Verifying AWS credentials... -2025-06-04 15:33:54,008 - cluster_setup - INFO - Running: aws sts get-caller-identity -2025-06-04 15:33:54,759 - cluster_setup - WARNING - Command returned non-zero exit code: 254 -2025-06-04 15:33:54,759 - cluster_setup - WARNING - stderr: -An error occurred (ExpiredToken) when calling the GetCallerIdentity operation: The security token included in the request is expired - -2025-06-04 15:33:54,760 - cluster_setup - ERROR - AWS credentials verification failed. Please check your AWS configuration. -2025-06-04 15:33:54,760 - cluster_setup - ERROR - Error: -An error occurred (ExpiredToken) when calling the GetCallerIdentity operation: The security token included in the request is expired - -2025-06-04 15:35:25,473 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-04 15:35:25,474 - __main__ - INFO - Verifying AWS credentials -2025-06-04 15:35:25,475 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-04 15:35:25,475 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-04 15:35:25,475 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-04 15:35:25,476 - __main__ - INFO - Setting up EKS cluster and EFS CSI driver -2025-06-04 15:35:25,476 - __main__ - INFO - Using Kubernetes version from command line: 1.33 -2025-06-04 15:35:25,476 - __main__ - INFO - Using EFS CSI Driver version from command line: 2.1.0 -2025-06-04 15:35:25,488 - cluster_setup - INFO - Created cluster config at cluster-config/efs-csi-test-1749051325.yaml -2025-06-04 15:35:25,489 - cluster_setup - INFO - Verifying AWS credentials... -2025-06-04 15:35:25,489 - cluster_setup - INFO - Running: aws sts get-caller-identity -2025-06-04 15:35:26,251 - cluster_setup - INFO - Using AWS account: 745939127895 with user: AROA23LLPZZLQCLTVMVDC:ibhard-Isengard -2025-06-04 15:35:26,251 - cluster_setup - INFO - Running: eksctl version -2025-06-04 15:35:26,284 - cluster_setup - INFO - eksctl version: 0.208.0 -2025-06-04 15:35:26,285 - cluster_setup - INFO - Creating EKS cluster efs-csi-test-1749051325 in us-west-2... -2025-06-04 15:35:26,285 - cluster_setup - INFO - This may take 15-20 minutes to complete... -2025-06-04 15:35:26,285 - cluster_setup - INFO - Running: eksctl create cluster -f cluster-config/efs-csi-test-1749051325.yaml --verbose 4 -2025-06-04 15:35:27,569 - cluster_setup - WARNING - Command returned non-zero exit code: 1 -2025-06-04 15:35:27,569 - cluster_setup - WARNING - stderr: Error: unable to determine AMI to use: error getting AMI from SSM Parameter Store: operation error SSM: GetParameter, https response error StatusCode: 400, RequestID: 59f5b73c-2035-46bb-9074-986f681c13f8, ParameterNotFound: . please verify that AMI Family is supported - -2025-06-04 15:35:27,569 - cluster_setup - ERROR - Failed to create EKS cluster -2025-06-04 15:35:27,569 - cluster_setup - ERROR - Error details: Error: unable to determine AMI to use: error getting AMI from SSM Parameter Store: operation error SSM: GetParameter, https response error StatusCode: 400, RequestID: 59f5b73c-2035-46bb-9074-986f681c13f8, ParameterNotFound: . please verify that AMI Family is supported - -2025-06-04 15:35:27,570 - cluster_setup - ERROR - Common causes of failure: -2025-06-04 15:35:27,570 - cluster_setup - ERROR - 1. Insufficient IAM permissions -2025-06-04 15:35:27,570 - cluster_setup - ERROR - 2. Service quotas reached (check your AWS console) -2025-06-04 15:35:27,570 - cluster_setup - ERROR - 3. Network connectivity issues -2025-06-04 15:35:27,570 - cluster_setup - ERROR - 4. Region availability zone limitations -2025-06-04 15:35:27,570 - cluster_setup - ERROR - Detailed error log saved to: logs/eks-cluster-creation-error-efs-csi-test-1749051325.log -2025-06-04 15:36:14,552 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-04 15:36:14,552 - __main__ - INFO - Verifying AWS credentials -2025-06-04 15:36:14,553 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-04 15:36:14,553 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-04 15:36:14,554 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-04 15:36:14,554 - __main__ - INFO - Setting up EKS cluster and EFS CSI driver -2025-06-04 15:36:14,554 - __main__ - INFO - Using Kubernetes version from command line: 1.33 -2025-06-04 15:36:14,554 - __main__ - INFO - Using EFS CSI Driver version from command line: 2.1.0 -2025-06-04 15:36:14,567 - cluster_setup - INFO - Created cluster config at cluster-config/efs-csi-test-1749051374.yaml -2025-06-04 15:36:14,567 - cluster_setup - INFO - Verifying AWS credentials... -2025-06-04 15:36:14,567 - cluster_setup - INFO - Running: aws sts get-caller-identity -2025-06-04 15:36:15,333 - cluster_setup - INFO - Using AWS account: 745939127895 with user: AROA23LLPZZLQCLTVMVDC:ibhard-Isengard -2025-06-04 15:36:15,333 - cluster_setup - INFO - Running: eksctl version -2025-06-04 15:36:15,366 - cluster_setup - INFO - eksctl version: 0.208.0 -2025-06-04 15:36:15,366 - cluster_setup - INFO - Creating EKS cluster efs-csi-test-1749051374 in us-west-2... -2025-06-04 15:36:15,366 - cluster_setup - INFO - This may take 15-20 minutes to complete... -2025-06-04 15:36:15,366 - cluster_setup - INFO - Running: eksctl create cluster -f cluster-config/efs-csi-test-1749051374.yaml --verbose 4 -2025-06-04 15:36:16,734 - cluster_setup - WARNING - Command returned non-zero exit code: 1 -2025-06-04 15:36:16,734 - cluster_setup - WARNING - stderr: Error: unable to determine AMI to use: error getting AMI from SSM Parameter Store: operation error SSM: GetParameter, https response error StatusCode: 400, RequestID: 1189c451-cdbd-4c3b-9495-a627bfe2113d, ParameterNotFound: . please verify that AMI Family is supported - -2025-06-04 15:36:16,734 - cluster_setup - ERROR - Failed to create EKS cluster -2025-06-04 15:36:16,734 - cluster_setup - ERROR - Error details: Error: unable to determine AMI to use: error getting AMI from SSM Parameter Store: operation error SSM: GetParameter, https response error StatusCode: 400, RequestID: 1189c451-cdbd-4c3b-9495-a627bfe2113d, ParameterNotFound: . please verify that AMI Family is supported - -2025-06-04 15:36:16,734 - cluster_setup - ERROR - Common causes of failure: -2025-06-04 15:36:16,734 - cluster_setup - ERROR - 1. Insufficient IAM permissions -2025-06-04 15:36:16,734 - cluster_setup - ERROR - 2. Service quotas reached (check your AWS console) -2025-06-04 15:36:16,734 - cluster_setup - ERROR - 3. Network connectivity issues -2025-06-04 15:36:16,734 - cluster_setup - ERROR - 4. Region availability zone limitations -2025-06-04 15:36:16,734 - cluster_setup - ERROR - Detailed error log saved to: logs/eks-cluster-creation-error-efs-csi-test-1749051374.log -2025-06-04 15:36:31,995 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-04 15:36:31,996 - __main__ - INFO - Verifying AWS credentials -2025-06-04 15:36:31,997 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-04 15:36:31,997 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-04 15:36:31,997 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-04 15:36:31,998 - __main__ - INFO - Setting up EKS cluster and EFS CSI driver -2025-06-04 15:36:31,998 - __main__ - INFO - Using Kubernetes version from command line: 1.28 -2025-06-04 15:36:31,998 - __main__ - INFO - Using EFS CSI Driver version from command line: 2.1.0 -2025-06-04 15:36:32,010 - cluster_setup - INFO - Created cluster config at cluster-config/efs-csi-test-1749051392.yaml -2025-06-04 15:36:32,010 - cluster_setup - INFO - Verifying AWS credentials... -2025-06-04 15:36:32,011 - cluster_setup - INFO - Running: aws sts get-caller-identity -2025-06-04 15:36:32,772 - cluster_setup - INFO - Using AWS account: 745939127895 with user: AROA23LLPZZLQCLTVMVDC:ibhard-Isengard -2025-06-04 15:36:32,772 - cluster_setup - INFO - Running: eksctl version -2025-06-04 15:36:32,804 - cluster_setup - INFO - eksctl version: 0.208.0 -2025-06-04 15:36:32,804 - cluster_setup - INFO - Creating EKS cluster efs-csi-test-1749051392 in us-west-2... -2025-06-04 15:36:32,804 - cluster_setup - INFO - This may take 15-20 minutes to complete... -2025-06-04 15:36:32,804 - cluster_setup - INFO - Running: eksctl create cluster -f cluster-config/efs-csi-test-1749051392.yaml --verbose 4 -2025-06-04 15:37:05,360 - cluster_setup - WARNING - Command returned non-zero exit code: 1 -2025-06-04 15:37:05,360 - cluster_setup - WARNING - stderr: Error: failed to create cluster "efs-csi-test-1749051392" - -2025-06-04 15:37:05,360 - cluster_setup - ERROR - Failed to create EKS cluster -2025-06-04 15:37:05,360 - cluster_setup - ERROR - Error details: Error: failed to create cluster "efs-csi-test-1749051392" - -2025-06-04 15:37:05,360 - cluster_setup - ERROR - Common causes of failure: -2025-06-04 15:37:05,360 - cluster_setup - ERROR - 1. Insufficient IAM permissions -2025-06-04 15:37:05,360 - cluster_setup - ERROR - 2. Service quotas reached (check your AWS console) -2025-06-04 15:37:05,360 - cluster_setup - ERROR - 3. Network connectivity issues -2025-06-04 15:37:05,361 - cluster_setup - ERROR - 4. Region availability zone limitations -2025-06-04 15:37:05,361 - cluster_setup - ERROR - Detailed error log saved to: logs/eks-cluster-creation-error-efs-csi-test-1749051392.log -2025-06-04 20:43:00,283 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-04 20:43:00,283 - __main__ - INFO - Verifying AWS credentials -2025-06-04 20:43:00,284 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-04 20:43:00,285 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-04 20:43:00,285 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-04 20:43:00,285 - __main__ - INFO - Running orchestrator stress test suite -2025-06-04 20:43:00,285 - __main__ - INFO - Starting orchestrator for 300 seconds -2025-06-04 20:43:00,285 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-04 20:43:01,074 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-04 20:43:01,299 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-04 20:43:01,299 - __main__ - INFO - Test duration overridden to 300 seconds -2025-06-04 20:43:01,299 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) -2025-06-04 20:43:01,299 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-04 20:43:01,528 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-04 20:43:01,529 - tests.orchestrator - INFO - Selected operation: _verify_readwrite -2025-06-04 20:43:01,529 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test -2025-06-04 20:43:02,529 - tests.orchestrator - INFO - Selected operation: _attach_pod -2025-06-04 20:43:02,529 - tests.orchestrator - INFO - No PVCs available, skipping pod attachment -2025-06-04 20:43:03,529 - tests.orchestrator - INFO - Selected operation: _create_pvc -2025-06-04 20:43:03,530 - tests.orchestrator - INFO - Creating PVC: test-pvc-a8738d91 -2025-06-04 20:43:03,612 - tests.orchestrator - INFO - Created PVC: test-pvc-a8738d91 -2025-06-04 20:43:03,613 - tests.orchestrator - INFO - Waiting for PVC test-pvc-a8738d91 to be bound -2025-06-04 20:43:05,748 - tests.orchestrator - INFO - PVC test-pvc-a8738d91 is bound -2025-06-04 20:43:06,749 - tests.orchestrator - INFO - Selected operation: _create_pvc -2025-06-04 20:43:06,749 - tests.orchestrator - INFO - Creating PVC: test-pvc-ba881b7a -2025-06-04 20:43:06,823 - tests.orchestrator - INFO - Created PVC: test-pvc-ba881b7a -2025-06-04 20:43:06,824 - tests.orchestrator - INFO - Waiting for PVC test-pvc-ba881b7a to be bound -2025-06-04 20:43:08,959 - tests.orchestrator - INFO - PVC test-pvc-ba881b7a is bound -2025-06-04 20:43:09,959 - tests.orchestrator - INFO - Selected operation: _delete_pod -2025-06-04 20:43:09,959 - tests.orchestrator - INFO - No pods to delete -2025-06-04 20:43:10,959 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario -2025-06-04 20:43:10,960 - tests.orchestrator - INFO - Running specific scenario: _scenario_one_to_one -2025-06-04 20:43:10,960 - tests.orchestrator - INFO - Running scenario: One pod per PVC -2025-06-04 20:43:10,960 - tests.orchestrator - INFO - Creating 4 PVC-pod pairs -2025-06-04 20:43:11,032 - tests.orchestrator - INFO - Waiting for PVC one2one-43f0888a to be bound -2025-06-04 20:43:13,170 - tests.orchestrator - INFO - PVC one2one-43f0888a is bound -2025-06-04 20:43:13,170 - tests.orchestrator - INFO - Creating pod test-pod-f3b3a319 on PVC one2one-43f0888a -2025-06-04 20:43:13,279 - tests.orchestrator - INFO - Created pod: test-pod-f3b3a319 using PVC: one2one-43f0888a -2025-06-04 20:43:13,279 - tests.orchestrator - INFO - Waiting for pod test-pod-f3b3a319 to be ready -2025-06-04 20:44:15,334 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-f3b3a319 to be ready after 60s -2025-06-04 20:44:15,334 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-f3b3a319 to be ready -2025-06-04 20:44:15,411 - tests.orchestrator - INFO - Waiting for PVC one2one-aac689b5 to be bound -2025-06-04 20:44:17,545 - tests.orchestrator - INFO - PVC one2one-aac689b5 is bound -2025-06-04 20:44:17,545 - tests.orchestrator - INFO - Creating pod test-pod-80037404 on PVC one2one-aac689b5 -2025-06-04 20:44:17,620 - tests.orchestrator - INFO - Created pod: test-pod-80037404 using PVC: one2one-aac689b5 -2025-06-04 20:44:17,620 - tests.orchestrator - INFO - Waiting for pod test-pod-80037404 to be ready -2025-06-04 20:45:17,743 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-80037404 to be ready after 60s -2025-06-04 20:45:17,744 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-80037404 to be ready -2025-06-04 20:45:17,826 - tests.orchestrator - INFO - Waiting for PVC one2one-116b8e13 to be bound -2025-06-04 20:45:19,959 - tests.orchestrator - INFO - PVC one2one-116b8e13 is bound -2025-06-04 20:45:19,959 - tests.orchestrator - INFO - Creating pod test-pod-725477e8 on PVC one2one-116b8e13 -2025-06-04 20:45:20,032 - tests.orchestrator - INFO - Created pod: test-pod-725477e8 using PVC: one2one-116b8e13 -2025-06-04 20:45:20,032 - tests.orchestrator - INFO - Waiting for pod test-pod-725477e8 to be ready -2025-06-04 20:46:20,115 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-725477e8 to be ready after 60s -2025-06-04 20:46:20,116 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-725477e8 to be ready -2025-06-04 20:46:20,195 - tests.orchestrator - INFO - Waiting for PVC one2one-4e963a5b to be bound -2025-06-04 20:46:22,332 - tests.orchestrator - INFO - PVC one2one-4e963a5b is bound -2025-06-04 20:46:22,332 - tests.orchestrator - INFO - Creating pod test-pod-b778d0e5 on PVC one2one-4e963a5b -2025-06-04 20:46:22,407 - tests.orchestrator - INFO - Created pod: test-pod-b778d0e5 using PVC: one2one-4e963a5b -2025-06-04 20:46:22,408 - tests.orchestrator - INFO - Waiting for pod test-pod-b778d0e5 to be ready -2025-06-04 20:47:24,472 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b778d0e5 to be ready after 60s -2025-06-04 20:47:24,472 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b778d0e5 to be ready -2025-06-04 20:47:24,472 - tests.orchestrator - WARNING - Failed to create enough PVC-pod pairs, only created 0 -2025-06-04 20:47:25,473 - tests.orchestrator - INFO - Selected operation: _delete_pod -2025-06-04 20:47:25,473 - tests.orchestrator - INFO - Deleting pod: test-pod-725477e8 from PVC: one2one-116b8e13 -2025-06-04 20:47:25,551 - tests.orchestrator - INFO - Waiting for pod test-pod-725477e8 to be deleted -2025-06-04 20:47:25,616 - tests.orchestrator - INFO - Pod test-pod-725477e8 has been deleted -2025-06-04 20:47:25,616 - tests.orchestrator - INFO - Deleted pod: test-pod-725477e8 -2025-06-04 20:47:26,616 - tests.orchestrator - INFO - Selected operation: _delete_pod -2025-06-04 20:47:26,616 - tests.orchestrator - INFO - Deleting pod: test-pod-f3b3a319 from PVC: one2one-43f0888a -2025-06-04 20:47:26,699 - tests.orchestrator - INFO - Waiting for pod test-pod-f3b3a319 to be deleted -2025-06-04 20:47:26,766 - tests.orchestrator - INFO - Pod test-pod-f3b3a319 has been deleted -2025-06-04 20:47:26,766 - tests.orchestrator - INFO - Deleted pod: test-pod-f3b3a319 -2025-06-04 20:47:27,767 - tests.orchestrator - INFO - Selected operation: _delete_pod -2025-06-04 20:47:27,767 - tests.orchestrator - INFO - Deleting pod: test-pod-80037404 from PVC: one2one-aac689b5 -2025-06-04 20:47:27,853 - tests.orchestrator - INFO - Waiting for pod test-pod-80037404 to be deleted -2025-06-04 20:47:27,919 - tests.orchestrator - INFO - Pod test-pod-80037404 has been deleted -2025-06-04 20:47:27,919 - tests.orchestrator - INFO - Deleted pod: test-pod-80037404 -2025-06-04 20:47:28,919 - tests.orchestrator - INFO - Selected operation: _verify_readwrite -2025-06-04 20:47:28,920 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test -2025-06-04 20:47:29,920 - tests.orchestrator - INFO - Selected operation: _attach_pod -2025-06-04 20:47:29,920 - tests.orchestrator - INFO - Creating pod test-pod-715d3677 on PVC one2one-116b8e13 -2025-06-04 20:47:30,008 - tests.orchestrator - INFO - Created pod: test-pod-715d3677 using PVC: one2one-116b8e13 -2025-06-04 20:47:30,008 - tests.orchestrator - INFO - Waiting for pod test-pod-715d3677 to be ready -2025-06-04 20:48:32,062 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-715d3677 to be ready after 60s -2025-06-04 20:48:32,062 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-715d3677 to be ready -2025-06-04 20:48:33,062 - tests.orchestrator - INFO - Test completed in 331.76 seconds -2025-06-04 20:48:33,063 - tests.orchestrator - INFO - Cleaning up resources... -2025-06-04 20:48:33,063 - tests.orchestrator - INFO - Deleting pod: test-pod-715d3677 from PVC: one2one-116b8e13 -2025-06-04 20:48:33,144 - tests.orchestrator - INFO - Waiting for pod test-pod-715d3677 to be deleted -2025-06-04 20:48:33,208 - tests.orchestrator - INFO - Pod test-pod-715d3677 has been deleted -2025-06-04 20:48:33,209 - tests.orchestrator - INFO - Deleted pod: test-pod-715d3677 -2025-06-04 20:48:33,209 - tests.orchestrator - INFO - Deleting pod: test-pod-b778d0e5 from PVC: one2one-4e963a5b -2025-06-04 20:48:33,289 - tests.orchestrator - INFO - Waiting for pod test-pod-b778d0e5 to be deleted -2025-06-04 20:48:33,356 - tests.orchestrator - INFO - Pod test-pod-b778d0e5 has been deleted -2025-06-04 20:48:33,356 - tests.orchestrator - INFO - Deleted pod: test-pod-b778d0e5 -2025-06-04 20:48:33,356 - tests.orchestrator - INFO - Deleting PVC: test-pvc-a8738d91 -2025-06-04 20:48:33,430 - tests.orchestrator - INFO - Waiting for PVC test-pvc-a8738d91 to be deleted -2025-06-04 20:48:33,497 - tests.orchestrator - INFO - PVC test-pvc-a8738d91 has been deleted -2025-06-04 20:48:33,497 - tests.orchestrator - INFO - Deleted PVC: test-pvc-a8738d91 -2025-06-04 20:48:33,497 - tests.orchestrator - INFO - Deleting PVC: test-pvc-ba881b7a -2025-06-04 20:48:33,570 - tests.orchestrator - INFO - Waiting for PVC test-pvc-ba881b7a to be deleted -2025-06-04 20:48:33,637 - tests.orchestrator - INFO - PVC test-pvc-ba881b7a has been deleted -2025-06-04 20:48:33,637 - tests.orchestrator - INFO - Deleted PVC: test-pvc-ba881b7a -2025-06-04 20:48:33,637 - tests.orchestrator - INFO - Deleting PVC: one2one-43f0888a -2025-06-04 20:48:33,712 - tests.orchestrator - INFO - Waiting for PVC one2one-43f0888a to be deleted -2025-06-04 20:48:33,779 - tests.orchestrator - INFO - PVC one2one-43f0888a has been deleted -2025-06-04 20:48:33,780 - tests.orchestrator - INFO - Deleted PVC: one2one-43f0888a -2025-06-04 20:48:33,780 - tests.orchestrator - INFO - Deleting PVC: one2one-aac689b5 -2025-06-04 20:48:33,858 - tests.orchestrator - INFO - Waiting for PVC one2one-aac689b5 to be deleted -2025-06-04 20:48:33,925 - tests.orchestrator - INFO - PVC one2one-aac689b5 has been deleted -2025-06-04 20:48:33,925 - tests.orchestrator - INFO - Deleted PVC: one2one-aac689b5 -2025-06-04 20:48:33,925 - tests.orchestrator - INFO - Deleting PVC: one2one-116b8e13 -2025-06-04 20:48:34,000 - tests.orchestrator - INFO - Waiting for PVC one2one-116b8e13 to be deleted -2025-06-04 20:48:34,067 - tests.orchestrator - INFO - PVC one2one-116b8e13 has been deleted -2025-06-04 20:48:34,067 - tests.orchestrator - INFO - Deleted PVC: one2one-116b8e13 -2025-06-04 20:48:34,067 - tests.orchestrator - INFO - Deleting PVC: one2one-4e963a5b -2025-06-04 20:48:34,143 - tests.orchestrator - INFO - Waiting for PVC one2one-4e963a5b to be deleted -2025-06-04 20:48:34,209 - tests.orchestrator - INFO - PVC one2one-4e963a5b has been deleted -2025-06-04 20:48:34,210 - tests.orchestrator - INFO - Deleted PVC: one2one-4e963a5b -2025-06-04 20:48:34,210 - tests.orchestrator - INFO - Cleanup completed -2025-06-04 20:48:34,210 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-04 20:48:34,210 - tests.orchestrator - INFO - --- Operations --- -2025-06-04 20:48:34,210 - tests.orchestrator - INFO - create_pvc: 2 succeeded, 0 failed (100.0%) -2025-06-04 20:48:34,210 - tests.orchestrator - INFO - attach_pod: 5 succeeded, 0 failed (100.0%) -2025-06-04 20:48:34,210 - tests.orchestrator - INFO - delete_pod: 5 succeeded, 0 failed (100.0%) -2025-06-04 20:48:34,210 - tests.orchestrator - INFO - delete_pvc: 6 succeeded, 0 failed (100.0%) -2025-06-04 20:48:34,210 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 0 failed (0.0%) -2025-06-04 20:48:34,210 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 0 failed (0.0%) -2025-06-04 20:48:34,210 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-04 20:48:34,210 - tests.orchestrator - INFO - shared_volume_rw: No runs -2025-06-04 20:48:34,210 - tests.orchestrator - INFO - many_to_one: No runs -2025-06-04 20:48:34,210 - tests.orchestrator - INFO - one_to_one: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-04 20:48:34,210 - tests.orchestrator - INFO - concurrent_pvc: No runs -2025-06-04 20:48:34,210 - tests.orchestrator - INFO - ========================================= -2025-06-04 20:48:34,211 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250604_204834.json -2025-06-04 20:48:34,258 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250604_204834_20250604_204834.json -2025-06-04 20:48:34,258 - __main__ - INFO - All tests completed successfully diff --git a/test/e2e/stress-scale-tests/logs/efs_tests.log b/test/e2e/stress-scale-tests/logs/efs_tests.log deleted file mode 100644 index 9ec4f5fa2..000000000 --- a/test/e2e/stress-scale-tests/logs/efs_tests.log +++ /dev/null @@ -1,2869 +0,0 @@ -2025-06-03 21:00:10,432 - __main__ - INFO - Starting EFS CSI tests with configuration from config/test_config.yaml -2025-06-03 21:00:10,433 - __main__ - INFO - Verifying AWS credentials -2025-06-03 21:00:10,434 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-03 21:00:10,434 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-03 21:00:10,435 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-03 21:00:10,435 - __main__ - INFO - Running orchestrator stress test suite -2025-06-03 21:00:10,435 - __main__ - INFO - Starting orchestrator for 300 seconds -2025-06-03 21:00:10,435 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-03 21:00:11,173 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-03 21:00:11,448 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-03 21:00:11,448 - __main__ - INFO - Test duration overridden to 300 seconds -2025-06-03 21:00:11,448 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) -2025-06-03 21:00:11,448 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-03 21:00:11,865 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-03 21:00:11,865 - tests.orchestrator - INFO - Selected operation: _attach_pod -2025-06-03 21:00:11,865 - tests.orchestrator - INFO - No PVCs available, skipping pod attachment -2025-06-03 21:00:12,865 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario -2025-06-03 21:00:12,865 - tests.orchestrator - INFO - Running specific scenario: _scenario_many_to_one -2025-06-03 21:00:12,866 - tests.orchestrator - INFO - Running scenario: Many pods to one PVC -2025-06-03 21:00:12,969 - tests.orchestrator - INFO - Waiting for PVC many2one-8ccf2b6f to be bound -2025-06-03 21:00:15,141 - tests.orchestrator - INFO - PVC many2one-8ccf2b6f is bound -2025-06-03 21:00:15,141 - tests.orchestrator - INFO - Creating 4 pods for the same PVC -2025-06-03 21:00:15,141 - tests.orchestrator - INFO - Creating pod test-pod-5c9bbadf on PVC many2one-8ccf2b6f -2025-06-03 21:00:15,239 - tests.orchestrator - INFO - Created pod: test-pod-5c9bbadf using PVC: many2one-8ccf2b6f -2025-06-03 21:00:15,239 - tests.orchestrator - INFO - Waiting for pod test-pod-5c9bbadf to be ready -2025-06-03 21:01:15,739 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-5c9bbadf to be ready after 60s -2025-06-03 21:01:15,740 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-5c9bbadf to be ready -2025-06-03 21:01:15,740 - tests.orchestrator - INFO - Creating pod test-pod-a795ee8f on PVC many2one-8ccf2b6f -2025-06-03 21:01:15,842 - tests.orchestrator - INFO - Created pod: test-pod-a795ee8f using PVC: many2one-8ccf2b6f -2025-06-03 21:01:15,842 - tests.orchestrator - INFO - Waiting for pod test-pod-a795ee8f to be ready -2025-06-03 21:02:16,279 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a795ee8f to be ready after 60s -2025-06-03 21:02:16,279 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a795ee8f to be ready -2025-06-03 21:02:16,279 - tests.orchestrator - INFO - Creating pod test-pod-bb687a78 on PVC many2one-8ccf2b6f -2025-06-03 21:02:16,379 - tests.orchestrator - INFO - Created pod: test-pod-bb687a78 using PVC: many2one-8ccf2b6f -2025-06-03 21:02:16,379 - tests.orchestrator - INFO - Waiting for pod test-pod-bb687a78 to be ready -2025-06-03 21:03:16,816 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-bb687a78 to be ready after 60s -2025-06-03 21:03:16,816 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-bb687a78 to be ready -2025-06-03 21:03:16,817 - tests.orchestrator - INFO - Creating pod test-pod-da23ad5a on PVC many2one-8ccf2b6f -2025-06-03 21:03:16,917 - tests.orchestrator - INFO - Created pod: test-pod-da23ad5a using PVC: many2one-8ccf2b6f -2025-06-03 21:03:16,917 - tests.orchestrator - INFO - Waiting for pod test-pod-da23ad5a to be ready -2025-06-03 21:04:17,450 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-da23ad5a to be ready after 60s -2025-06-03 21:04:17,450 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-da23ad5a to be ready -2025-06-03 21:04:17,450 - tests.orchestrator - WARNING - Failed to create enough pods for scenario, only created 0 -2025-06-03 21:04:18,450 - tests.orchestrator - INFO - Selected operation: _verify_readwrite -2025-06-03 21:04:18,451 - tests.orchestrator - INFO - Testing read/write between pods test-pod-a795ee8f and test-pod-5c9bbadf sharing PVC many2one-8ccf2b6f -2025-06-03 21:04:18,534 - tests.orchestrator - ERROR - Failed in read/write verification: (400) -Reason: Bad Request -HTTP response headers: HTTPHeaderDict({'Audit-Id': 'e5f54d7e-0b5d-44ba-9610-9f446b015a24', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Tue, 03 Jun 2025 21:04:18 GMT', 'Content-Length': '166'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-a795ee8f does not have a host assigned","reason":"BadRequest","code":400} - - -2025-06-03 21:04:19,534 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario -2025-06-03 21:04:19,534 - tests.orchestrator - INFO - Running specific scenario: _scenario_one_to_one -2025-06-03 21:04:19,534 - tests.orchestrator - INFO - Running scenario: One pod per PVC -2025-06-03 21:04:19,534 - tests.orchestrator - INFO - Creating 5 PVC-pod pairs -2025-06-03 21:04:19,631 - tests.orchestrator - INFO - Waiting for PVC one2one-c1de2935 to be bound -2025-06-03 21:04:21,797 - tests.orchestrator - INFO - PVC one2one-c1de2935 is bound -2025-06-03 21:04:21,797 - tests.orchestrator - INFO - Creating pod test-pod-c8cbcc77 on PVC one2one-c1de2935 -2025-06-03 21:04:21,894 - tests.orchestrator - INFO - Created pod: test-pod-c8cbcc77 using PVC: one2one-c1de2935 -2025-06-03 21:04:21,894 - tests.orchestrator - INFO - Waiting for pod test-pod-c8cbcc77 to be ready -2025-06-03 21:05:22,524 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-c8cbcc77 to be ready after 60s -2025-06-03 21:05:22,524 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-c8cbcc77 to be ready -2025-06-03 21:05:22,615 - tests.orchestrator - INFO - Waiting for PVC one2one-2f5447c5 to be bound -2025-06-03 21:05:24,783 - tests.orchestrator - INFO - PVC one2one-2f5447c5 is bound -2025-06-03 21:05:24,783 - tests.orchestrator - INFO - Creating pod test-pod-9c65d0a8 on PVC one2one-2f5447c5 -2025-06-03 21:05:24,879 - tests.orchestrator - INFO - Created pod: test-pod-9c65d0a8 using PVC: one2one-2f5447c5 -2025-06-03 21:05:24,879 - tests.orchestrator - INFO - Waiting for pod test-pod-9c65d0a8 to be ready -2025-06-03 21:06:25,353 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9c65d0a8 to be ready after 60s -2025-06-03 21:06:25,353 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9c65d0a8 to be ready -2025-06-03 21:06:25,454 - tests.orchestrator - INFO - Waiting for PVC one2one-65a70cda to be bound -2025-06-03 21:06:27,620 - tests.orchestrator - INFO - PVC one2one-65a70cda is bound -2025-06-03 21:06:27,621 - tests.orchestrator - INFO - Creating pod test-pod-396a98c7 on PVC one2one-65a70cda -2025-06-03 21:06:27,709 - tests.orchestrator - INFO - Created pod: test-pod-396a98c7 using PVC: one2one-65a70cda -2025-06-03 21:06:27,709 - tests.orchestrator - INFO - Waiting for pod test-pod-396a98c7 to be ready -2025-06-03 21:07:28,272 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-396a98c7 to be ready after 60s -2025-06-03 21:07:28,272 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-396a98c7 to be ready -2025-06-03 21:07:28,365 - tests.orchestrator - INFO - Waiting for PVC one2one-ac0c0034 to be bound -2025-06-03 21:07:30,531 - tests.orchestrator - INFO - PVC one2one-ac0c0034 is bound -2025-06-03 21:07:30,531 - tests.orchestrator - INFO - Creating pod test-pod-707ffe1e on PVC one2one-ac0c0034 -2025-06-03 21:07:30,620 - tests.orchestrator - INFO - Created pod: test-pod-707ffe1e using PVC: one2one-ac0c0034 -2025-06-03 21:07:30,620 - tests.orchestrator - INFO - Waiting for pod test-pod-707ffe1e to be ready -2025-06-03 21:08:31,077 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-707ffe1e to be ready after 60s -2025-06-03 21:08:31,077 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-707ffe1e to be ready -2025-06-03 21:08:31,174 - tests.orchestrator - INFO - Waiting for PVC one2one-6960322d to be bound -2025-06-03 21:08:33,340 - tests.orchestrator - INFO - PVC one2one-6960322d is bound -2025-06-03 21:08:33,340 - tests.orchestrator - INFO - Creating pod test-pod-db166953 on PVC one2one-6960322d -2025-06-03 21:08:33,433 - tests.orchestrator - INFO - Created pod: test-pod-db166953 using PVC: one2one-6960322d -2025-06-03 21:08:33,433 - tests.orchestrator - INFO - Waiting for pod test-pod-db166953 to be ready -2025-06-03 21:09:34,745 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-db166953 to be ready after 60s -2025-06-03 21:09:34,745 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-db166953 to be ready -2025-06-03 21:09:34,746 - tests.orchestrator - WARNING - Failed to create enough PVC-pod pairs, only created 0 -2025-06-03 21:09:35,746 - tests.orchestrator - INFO - Test completed in 564.30 seconds -2025-06-03 21:09:35,746 - tests.orchestrator - INFO - Cleaning up resources... -2025-06-03 21:09:35,746 - tests.orchestrator - INFO - Deleting pod: test-pod-5c9bbadf from PVC: many2one-8ccf2b6f -2025-06-03 21:09:35,842 - tests.orchestrator - INFO - Waiting for pod test-pod-5c9bbadf to be deleted -2025-06-03 21:09:35,926 - tests.orchestrator - INFO - Pod test-pod-5c9bbadf has been deleted -2025-06-03 21:09:35,926 - tests.orchestrator - INFO - Deleted pod: test-pod-5c9bbadf -2025-06-03 21:09:35,926 - tests.orchestrator - INFO - Deleting pod: test-pod-a795ee8f from PVC: many2one-8ccf2b6f -2025-06-03 21:09:36,025 - tests.orchestrator - INFO - Waiting for pod test-pod-a795ee8f to be deleted -2025-06-03 21:09:36,106 - tests.orchestrator - INFO - Pod test-pod-a795ee8f has been deleted -2025-06-03 21:09:36,106 - tests.orchestrator - INFO - Deleted pod: test-pod-a795ee8f -2025-06-03 21:09:36,106 - tests.orchestrator - INFO - Deleting pod: test-pod-bb687a78 from PVC: many2one-8ccf2b6f -2025-06-03 21:09:36,202 - tests.orchestrator - INFO - Waiting for pod test-pod-bb687a78 to be deleted -2025-06-03 21:09:36,284 - tests.orchestrator - INFO - Pod test-pod-bb687a78 has been deleted -2025-06-03 21:09:36,284 - tests.orchestrator - INFO - Deleted pod: test-pod-bb687a78 -2025-06-03 21:09:36,284 - tests.orchestrator - INFO - Deleting pod: test-pod-da23ad5a from PVC: many2one-8ccf2b6f -2025-06-03 21:09:36,386 - tests.orchestrator - INFO - Waiting for pod test-pod-da23ad5a to be deleted -2025-06-03 21:09:36,466 - tests.orchestrator - INFO - Pod test-pod-da23ad5a has been deleted -2025-06-03 21:09:36,466 - tests.orchestrator - INFO - Deleted pod: test-pod-da23ad5a -2025-06-03 21:09:36,466 - tests.orchestrator - INFO - Deleting pod: test-pod-c8cbcc77 from PVC: one2one-c1de2935 -2025-06-03 21:09:36,565 - tests.orchestrator - INFO - Waiting for pod test-pod-c8cbcc77 to be deleted -2025-06-03 21:09:36,651 - tests.orchestrator - INFO - Pod test-pod-c8cbcc77 has been deleted -2025-06-03 21:09:36,652 - tests.orchestrator - INFO - Deleted pod: test-pod-c8cbcc77 -2025-06-03 21:09:36,652 - tests.orchestrator - INFO - Deleting pod: test-pod-9c65d0a8 from PVC: one2one-2f5447c5 -2025-06-03 21:09:36,747 - tests.orchestrator - INFO - Waiting for pod test-pod-9c65d0a8 to be deleted -2025-06-03 21:09:36,832 - tests.orchestrator - INFO - Pod test-pod-9c65d0a8 has been deleted -2025-06-03 21:09:36,833 - tests.orchestrator - INFO - Deleted pod: test-pod-9c65d0a8 -2025-06-03 21:09:36,833 - tests.orchestrator - INFO - Deleting pod: test-pod-396a98c7 from PVC: one2one-65a70cda -2025-06-03 21:09:36,929 - tests.orchestrator - INFO - Waiting for pod test-pod-396a98c7 to be deleted -2025-06-03 21:09:37,009 - tests.orchestrator - INFO - Pod test-pod-396a98c7 has been deleted -2025-06-03 21:09:37,009 - tests.orchestrator - INFO - Deleted pod: test-pod-396a98c7 -2025-06-03 21:09:37,009 - tests.orchestrator - INFO - Deleting pod: test-pod-707ffe1e from PVC: one2one-ac0c0034 -2025-06-03 21:09:37,104 - tests.orchestrator - INFO - Waiting for pod test-pod-707ffe1e to be deleted -2025-06-03 21:09:37,187 - tests.orchestrator - INFO - Pod test-pod-707ffe1e has been deleted -2025-06-03 21:09:37,187 - tests.orchestrator - INFO - Deleted pod: test-pod-707ffe1e -2025-06-03 21:09:37,187 - tests.orchestrator - INFO - Deleting pod: test-pod-db166953 from PVC: one2one-6960322d -2025-06-03 21:09:37,284 - tests.orchestrator - INFO - Waiting for pod test-pod-db166953 to be deleted -2025-06-03 21:09:37,368 - tests.orchestrator - INFO - Pod test-pod-db166953 has been deleted -2025-06-03 21:09:37,368 - tests.orchestrator - INFO - Deleted pod: test-pod-db166953 -2025-06-03 21:09:37,368 - tests.orchestrator - INFO - Deleting PVC: many2one-8ccf2b6f -2025-06-03 21:09:37,456 - tests.orchestrator - INFO - Waiting for PVC many2one-8ccf2b6f to be deleted -2025-06-03 21:09:37,538 - tests.orchestrator - INFO - PVC many2one-8ccf2b6f has been deleted -2025-06-03 21:09:37,539 - tests.orchestrator - INFO - Deleted PVC: many2one-8ccf2b6f -2025-06-03 21:09:37,539 - tests.orchestrator - INFO - Deleting PVC: one2one-c1de2935 -2025-06-03 21:09:37,627 - tests.orchestrator - INFO - Waiting for PVC one2one-c1de2935 to be deleted -2025-06-03 21:09:37,711 - tests.orchestrator - INFO - PVC one2one-c1de2935 has been deleted -2025-06-03 21:09:37,711 - tests.orchestrator - INFO - Deleted PVC: one2one-c1de2935 -2025-06-03 21:09:37,711 - tests.orchestrator - INFO - Deleting PVC: one2one-2f5447c5 -2025-06-03 21:09:37,802 - tests.orchestrator - INFO - Waiting for PVC one2one-2f5447c5 to be deleted -2025-06-03 21:09:37,885 - tests.orchestrator - INFO - PVC one2one-2f5447c5 has been deleted -2025-06-03 21:09:37,886 - tests.orchestrator - INFO - Deleted PVC: one2one-2f5447c5 -2025-06-03 21:09:37,886 - tests.orchestrator - INFO - Deleting PVC: one2one-65a70cda -2025-06-03 21:09:37,973 - tests.orchestrator - INFO - Waiting for PVC one2one-65a70cda to be deleted -2025-06-03 21:09:38,059 - tests.orchestrator - INFO - PVC one2one-65a70cda has been deleted -2025-06-03 21:09:38,059 - tests.orchestrator - INFO - Deleted PVC: one2one-65a70cda -2025-06-03 21:09:38,059 - tests.orchestrator - INFO - Deleting PVC: one2one-ac0c0034 -2025-06-03 21:09:38,148 - tests.orchestrator - INFO - Waiting for PVC one2one-ac0c0034 to be deleted -2025-06-03 21:09:38,230 - tests.orchestrator - INFO - PVC one2one-ac0c0034 has been deleted -2025-06-03 21:09:38,231 - tests.orchestrator - INFO - Deleted PVC: one2one-ac0c0034 -2025-06-03 21:09:38,231 - tests.orchestrator - INFO - Deleting PVC: one2one-6960322d -2025-06-03 21:09:38,318 - tests.orchestrator - INFO - Waiting for PVC one2one-6960322d to be deleted -2025-06-03 21:09:38,401 - tests.orchestrator - INFO - PVC one2one-6960322d has been deleted -2025-06-03 21:09:38,401 - tests.orchestrator - INFO - Deleted PVC: one2one-6960322d -2025-06-03 21:09:38,401 - tests.orchestrator - INFO - Cleanup completed -2025-06-03 21:09:38,401 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-03 21:09:38,401 - tests.orchestrator - INFO - --- Operations --- -2025-06-03 21:09:38,401 - tests.orchestrator - INFO - create_pvc: 0 succeeded, 0 failed (0.0%) -2025-06-03 21:09:38,402 - tests.orchestrator - INFO - attach_pod: 9 succeeded, 0 failed (100.0%) -2025-06-03 21:09:38,402 - tests.orchestrator - INFO - delete_pod: 9 succeeded, 0 failed (100.0%) -2025-06-03 21:09:38,402 - tests.orchestrator - INFO - delete_pvc: 6 succeeded, 0 failed (100.0%) -2025-06-03 21:09:38,402 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) -2025-06-03 21:09:38,402 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) -2025-06-03 21:09:38,402 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-03 21:09:38,402 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-03 21:09:38,402 - tests.orchestrator - INFO - many_to_one: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-03 21:09:38,402 - tests.orchestrator - INFO - one_to_one: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-03 21:09:38,402 - tests.orchestrator - INFO - concurrent_pvc: No runs -2025-06-03 21:09:38,402 - tests.orchestrator - INFO - ========================================= -2025-06-03 21:09:38,402 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250603_210938.json -2025-06-03 21:09:38,449 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250603_210938_20250603_210938.json -2025-06-03 21:09:38,492 - __main__ - INFO - Summary report generated: reports/general/efs_tests_20250603_210938_20250603_210938_summary.txt -2025-06-03 21:09:38,492 - __main__ - INFO - All tests completed successfully -2025-06-04 21:12:54,295 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-04 21:12:54,295 - __main__ - INFO - Verifying AWS credentials -2025-06-04 21:12:54,296 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-04 21:12:54,297 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-04 21:12:54,297 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-04 21:12:54,298 - __main__ - INFO - Running orchestrator stress test suite -2025-06-04 21:12:54,298 - __main__ - INFO - Starting orchestrator for 300 seconds -2025-06-04 21:12:54,298 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-04 21:12:55,097 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-04 21:12:55,314 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-04 21:12:55,314 - __main__ - INFO - Test duration overridden to 300 seconds -2025-06-04 21:12:55,314 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) -2025-06-04 21:12:55,314 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-04 21:12:55,503 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-04 21:12:55,504 - tests.orchestrator - INFO - Selected operation: _delete_pod -2025-06-04 21:12:55,504 - tests.orchestrator - INFO - No pods to delete -2025-06-04 21:12:56,504 - tests.orchestrator - INFO - Selected operation: _verify_readwrite -2025-06-04 21:12:56,504 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test -2025-06-04 21:12:57,504 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario -2025-06-04 21:12:57,504 - tests.orchestrator - INFO - Running specific scenario: _scenario_one_to_one -2025-06-04 21:12:57,504 - tests.orchestrator - INFO - Running scenario: One pod per PVC -2025-06-04 21:12:57,505 - tests.orchestrator - INFO - Creating 4 PVC-pod pairs -2025-06-04 21:12:57,584 - tests.orchestrator - INFO - Waiting for PVC one2one-fa7d38c4 to be bound -2025-06-04 21:12:59,720 - tests.orchestrator - INFO - PVC one2one-fa7d38c4 is bound -2025-06-04 21:12:59,721 - tests.orchestrator - INFO - Creating pod test-pod-f3bd2110 on PVC one2one-fa7d38c4 -2025-06-04 21:12:59,822 - tests.orchestrator - INFO - Created pod: test-pod-f3bd2110 using PVC: one2one-fa7d38c4 -2025-06-04 21:12:59,822 - tests.orchestrator - INFO - Waiting for pod test-pod-f3bd2110 to be ready -2025-06-04 21:13:46,839 - tests.orchestrator - INFO - Test interrupted by user -2025-06-04 21:13:46,839 - tests.orchestrator - INFO - Test completed in 51.52 seconds -2025-06-04 21:13:46,839 - tests.orchestrator - INFO - Cleaning up resources... -2025-06-04 21:13:46,839 - tests.orchestrator - INFO - Deleting pod: test-pod-f3bd2110 from PVC: one2one-fa7d38c4 -2025-06-04 21:13:46,924 - tests.orchestrator - INFO - Waiting for pod test-pod-f3bd2110 to be deleted -2025-06-04 21:13:46,989 - tests.orchestrator - INFO - Pod test-pod-f3bd2110 has been deleted -2025-06-04 21:13:46,989 - tests.orchestrator - INFO - Deleted pod: test-pod-f3bd2110 -2025-06-04 21:13:46,989 - tests.orchestrator - INFO - Deleting PVC: one2one-fa7d38c4 -2025-06-04 21:13:47,063 - tests.orchestrator - INFO - Waiting for PVC one2one-fa7d38c4 to be deleted -2025-06-04 21:13:47,130 - tests.orchestrator - INFO - PVC one2one-fa7d38c4 has been deleted -2025-06-04 21:13:47,130 - tests.orchestrator - INFO - Deleted PVC: one2one-fa7d38c4 -2025-06-04 21:13:47,130 - tests.orchestrator - INFO - Cleanup completed -2025-06-04 21:13:47,131 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-04 21:13:47,131 - tests.orchestrator - INFO - --- Operations --- -2025-06-04 21:13:47,131 - tests.orchestrator - INFO - create_pvc: 0 succeeded, 0 failed (0.0%) -2025-06-04 21:13:47,131 - tests.orchestrator - INFO - attach_pod: 1 succeeded, 0 failed (100.0%) -2025-06-04 21:13:47,131 - tests.orchestrator - INFO - delete_pod: 1 succeeded, 0 failed (100.0%) -2025-06-04 21:13:47,131 - tests.orchestrator - INFO - delete_pvc: 1 succeeded, 0 failed (100.0%) -2025-06-04 21:13:47,131 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 0 failed (0.0%) -2025-06-04 21:13:47,131 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 0 failed (0.0%) -2025-06-04 21:13:47,131 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-04 21:13:47,131 - tests.orchestrator - INFO - shared_volume_rw: No runs -2025-06-04 21:13:47,131 - tests.orchestrator - INFO - many_to_one: No runs -2025-06-04 21:13:47,131 - tests.orchestrator - INFO - one_to_one: 0 succeeded, 0 failed out of 1 runs (0.0%) -2025-06-04 21:13:47,131 - tests.orchestrator - INFO - concurrent_pvc: No runs -2025-06-04 21:13:47,131 - tests.orchestrator - INFO - ========================================= -2025-06-04 21:13:47,132 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250604_211347.json -2025-06-04 21:13:47,178 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250604_211347_20250604_211347.json -2025-06-04 21:13:47,178 - __main__ - INFO - All tests completed successfully -2025-06-04 21:13:52,325 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-04 21:13:52,325 - __main__ - INFO - Verifying AWS credentials -2025-06-04 21:13:52,326 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-04 21:13:52,327 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-04 21:13:52,327 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-04 21:13:52,328 - __main__ - INFO - Running orchestrator stress test suite -2025-06-04 21:13:52,328 - __main__ - INFO - Starting orchestrator for 120 seconds -2025-06-04 21:13:52,328 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-04 21:13:53,143 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-04 21:13:53,342 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-04 21:13:53,342 - __main__ - INFO - Test duration overridden to 120 seconds -2025-06-04 21:13:53,342 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 10/s) -2025-06-04 21:13:53,342 - tests.orchestrator - INFO - Starting orchestrator test for 120 seconds -2025-06-04 21:13:53,535 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-04 21:13:53,536 - tests.orchestrator - INFO - Selected operation: _verify_readwrite -2025-06-04 21:13:53,536 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test -2025-06-04 21:13:54,536 - tests.orchestrator - INFO - Selected operation: _delete_pvc -2025-06-04 21:13:54,536 - tests.orchestrator - INFO - No PVCs to delete -2025-06-04 21:13:55,536 - tests.orchestrator - INFO - Selected operation: _verify_readwrite -2025-06-04 21:13:55,537 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test -2025-06-04 21:13:56,537 - tests.orchestrator - INFO - Selected operation: _delete_pod -2025-06-04 21:13:56,537 - tests.orchestrator - INFO - No pods to delete -2025-06-04 21:13:57,537 - tests.orchestrator - INFO - Selected operation: _delete_pod -2025-06-04 21:13:57,537 - tests.orchestrator - INFO - No pods to delete -2025-06-04 21:13:58,538 - tests.orchestrator - INFO - Selected operation: _verify_readwrite -2025-06-04 21:13:58,538 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test -2025-06-04 21:13:59,538 - tests.orchestrator - INFO - Selected operation: _delete_pod -2025-06-04 21:13:59,538 - tests.orchestrator - INFO - No pods to delete -2025-06-04 21:14:00,538 - tests.orchestrator - INFO - Selected operation: _attach_pod -2025-06-04 21:14:00,539 - tests.orchestrator - INFO - No PVCs available, skipping pod attachment -2025-06-04 21:14:01,539 - tests.orchestrator - INFO - Selected operation: _delete_pod -2025-06-04 21:14:01,540 - tests.orchestrator - INFO - No pods to delete -2025-06-04 21:14:02,540 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario -2025-06-04 21:14:02,541 - tests.orchestrator - INFO - Running specific scenario: _scenario_concurrent_pvc -2025-06-04 21:14:02,541 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations -2025-06-04 21:14:02,541 - tests.orchestrator - INFO - Creating 4 PVCs in quick succession -2025-06-04 21:14:02,617 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-68e38df4 -2025-06-04 21:14:02,617 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-68e38df4 to be bound -2025-06-04 21:14:04,753 - tests.orchestrator - INFO - PVC concurrent-pvc-68e38df4 is bound -2025-06-04 21:14:04,825 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-90dd2e0a -2025-06-04 21:14:04,825 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-90dd2e0a to be bound -2025-06-04 21:14:06,959 - tests.orchestrator - INFO - PVC concurrent-pvc-90dd2e0a is bound -2025-06-04 21:14:07,028 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-09b78e7f -2025-06-04 21:14:07,028 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-09b78e7f to be bound -2025-06-04 21:14:09,163 - tests.orchestrator - INFO - PVC concurrent-pvc-09b78e7f is bound -2025-06-04 21:14:09,232 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-51c7b76a -2025-06-04 21:14:09,232 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-51c7b76a to be bound -2025-06-04 21:14:11,368 - tests.orchestrator - INFO - PVC concurrent-pvc-51c7b76a is bound -2025-06-04 21:14:11,368 - tests.orchestrator - INFO - Successfully created 4 PVCs -2025-06-04 21:14:11,368 - tests.orchestrator - INFO - Creating pod test-pod-60af97a5 on PVC concurrent-pvc-90dd2e0a -2025-06-04 21:14:11,458 - tests.orchestrator - INFO - Created pod: test-pod-60af97a5 using PVC: concurrent-pvc-90dd2e0a -2025-06-04 21:14:11,458 - tests.orchestrator - INFO - Waiting for pod test-pod-60af97a5 to be ready -2025-06-04 21:14:56,603 - tests.orchestrator - INFO - Test interrupted by user -2025-06-04 21:14:56,603 - tests.orchestrator - INFO - Test completed in 63.26 seconds -2025-06-04 21:14:56,603 - tests.orchestrator - INFO - Cleaning up resources... -2025-06-04 21:14:56,603 - tests.orchestrator - INFO - Deleting pod: test-pod-60af97a5 from PVC: concurrent-pvc-90dd2e0a -2025-06-04 21:14:56,689 - tests.orchestrator - INFO - Waiting for pod test-pod-60af97a5 to be deleted -2025-06-04 21:14:56,754 - tests.orchestrator - INFO - Pod test-pod-60af97a5 has been deleted -2025-06-04 21:14:56,754 - tests.orchestrator - INFO - Deleted pod: test-pod-60af97a5 -2025-06-04 21:14:56,754 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-68e38df4 -2025-06-04 21:14:56,838 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-68e38df4 to be deleted -2025-06-04 21:14:56,902 - tests.orchestrator - INFO - PVC concurrent-pvc-68e38df4 has been deleted -2025-06-04 21:14:56,903 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-68e38df4 -2025-06-04 21:14:56,903 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-90dd2e0a -2025-06-04 21:14:56,977 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-90dd2e0a to be deleted -2025-06-04 21:14:57,041 - tests.orchestrator - INFO - PVC concurrent-pvc-90dd2e0a has been deleted -2025-06-04 21:14:57,041 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-90dd2e0a -2025-06-04 21:14:57,041 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-09b78e7f -2025-06-04 21:14:57,116 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-09b78e7f to be deleted -2025-06-04 21:14:57,181 - tests.orchestrator - INFO - PVC concurrent-pvc-09b78e7f has been deleted -2025-06-04 21:14:57,181 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-09b78e7f -2025-06-04 21:14:57,181 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-51c7b76a -2025-06-04 21:14:57,257 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-51c7b76a to be deleted -2025-06-04 21:14:57,322 - tests.orchestrator - INFO - PVC concurrent-pvc-51c7b76a has been deleted -2025-06-04 21:14:57,322 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-51c7b76a -2025-06-04 21:14:57,322 - tests.orchestrator - INFO - Cleanup completed -2025-06-04 21:14:57,322 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-04 21:14:57,322 - tests.orchestrator - INFO - --- Operations --- -2025-06-04 21:14:57,322 - tests.orchestrator - INFO - create_pvc: 4 succeeded, 0 failed (100.0%) -2025-06-04 21:14:57,322 - tests.orchestrator - INFO - attach_pod: 1 succeeded, 0 failed (100.0%) -2025-06-04 21:14:57,322 - tests.orchestrator - INFO - delete_pod: 1 succeeded, 0 failed (100.0%) -2025-06-04 21:14:57,322 - tests.orchestrator - INFO - delete_pvc: 4 succeeded, 0 failed (100.0%) -2025-06-04 21:14:57,322 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 0 failed (0.0%) -2025-06-04 21:14:57,322 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 0 failed (0.0%) -2025-06-04 21:14:57,322 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-04 21:14:57,323 - tests.orchestrator - INFO - shared_volume_rw: No runs -2025-06-04 21:14:57,323 - tests.orchestrator - INFO - many_to_one: No runs -2025-06-04 21:14:57,323 - tests.orchestrator - INFO - one_to_one: No runs -2025-06-04 21:14:57,323 - tests.orchestrator - INFO - concurrent_pvc: 0 succeeded, 0 failed out of 1 runs (0.0%) -2025-06-04 21:14:57,323 - tests.orchestrator - INFO - ========================================= -2025-06-04 21:14:57,323 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250604_211457.json -2025-06-04 21:14:57,371 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250604_211457_20250604_211457.json -2025-06-04 21:14:57,371 - __main__ - INFO - All tests completed successfully -2025-06-05 13:28:01,933 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-05 13:28:01,933 - __main__ - INFO - Verifying AWS credentials -2025-06-05 13:28:01,934 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 13:28:01,935 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 13:28:01,935 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 13:28:01,935 - __main__ - INFO - Running orchestrator stress test suite -2025-06-05 13:28:01,935 - __main__ - INFO - Starting orchestrator for 400 seconds -2025-06-05 13:28:01,935 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-05 13:28:02,670 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 13:28:02,885 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 13:28:02,886 - __main__ - INFO - Test duration overridden to 400 seconds -2025-06-05 13:28:02,886 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 20/s) -2025-06-05 13:28:02,886 - tests.orchestrator - INFO - Starting orchestrator test for 400 seconds -2025-06-05 13:28:03,078 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 13:28:03,079 - tests.orchestrator - INFO - Selected operation: _create_pvc -2025-06-05 13:28:03,079 - tests.orchestrator - INFO - Creating PVC: test-pvc-87f8dca2 -2025-06-05 13:28:03,159 - tests.orchestrator - INFO - Created PVC: test-pvc-87f8dca2 -2025-06-05 13:28:03,159 - tests.orchestrator - INFO - Waiting for PVC test-pvc-87f8dca2 to be bound -2025-06-05 13:28:05,290 - tests.orchestrator - INFO - PVC test-pvc-87f8dca2 is bound -2025-06-05 13:28:06,290 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario -2025-06-05 13:28:06,290 - tests.orchestrator - INFO - Running specific scenario: _scenario_concurrent_pvc -2025-06-05 13:28:06,290 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations -2025-06-05 13:28:06,290 - tests.orchestrator - INFO - Creating 6 PVCs in quick succession -2025-06-05 13:28:06,363 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-27248b37 -2025-06-05 13:28:06,363 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-27248b37 to be bound -2025-06-05 13:28:08,495 - tests.orchestrator - INFO - PVC concurrent-pvc-27248b37 is bound -2025-06-05 13:28:08,561 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-0be57a62 -2025-06-05 13:28:08,562 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-0be57a62 to be bound -2025-06-05 13:28:10,696 - tests.orchestrator - INFO - PVC concurrent-pvc-0be57a62 is bound -2025-06-05 13:28:10,764 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-d46ca8c1 -2025-06-05 13:28:10,764 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-d46ca8c1 to be bound -2025-06-05 13:28:12,902 - tests.orchestrator - INFO - PVC concurrent-pvc-d46ca8c1 is bound -2025-06-05 13:28:12,972 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-4b94a252 -2025-06-05 13:28:12,973 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-4b94a252 to be bound -2025-06-05 13:28:15,113 - tests.orchestrator - INFO - PVC concurrent-pvc-4b94a252 is bound -2025-06-05 13:28:15,180 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-737583a5 -2025-06-05 13:28:15,181 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-737583a5 to be bound -2025-06-05 13:28:17,317 - tests.orchestrator - INFO - PVC concurrent-pvc-737583a5 is bound -2025-06-05 13:28:17,385 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-b6ec6167 -2025-06-05 13:28:17,385 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-b6ec6167 to be bound -2025-06-05 13:28:19,520 - tests.orchestrator - INFO - PVC concurrent-pvc-b6ec6167 is bound -2025-06-05 13:28:19,520 - tests.orchestrator - INFO - Successfully created 6 PVCs -2025-06-05 13:28:19,520 - tests.orchestrator - INFO - Creating pod test-pod-05839583 on PVC concurrent-pvc-4b94a252 -2025-06-05 13:28:19,605 - tests.orchestrator - INFO - Created pod: test-pod-05839583 using PVC: concurrent-pvc-4b94a252 -2025-06-05 13:28:19,605 - tests.orchestrator - INFO - Waiting for pod test-pod-05839583 to be ready -2025-06-05 13:29:21,656 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-05839583 to be ready after 60s -2025-06-05 13:29:21,657 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-05839583 to be ready -2025-06-05 13:29:21,657 - tests.orchestrator - INFO - Creating pod test-pod-c11d7609 on PVC concurrent-pvc-b6ec6167 -2025-06-05 13:29:21,746 - tests.orchestrator - INFO - Created pod: test-pod-c11d7609 using PVC: concurrent-pvc-b6ec6167 -2025-06-05 13:29:21,746 - tests.orchestrator - INFO - Waiting for pod test-pod-c11d7609 to be ready -2025-06-05 13:30:23,813 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-c11d7609 to be ready after 60s -2025-06-05 13:30:23,813 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-c11d7609 to be ready -2025-06-05 13:30:23,813 - tests.orchestrator - INFO - Creating pod test-pod-f3958de4 on PVC concurrent-pvc-27248b37 -2025-06-05 13:30:23,897 - tests.orchestrator - INFO - Created pod: test-pod-f3958de4 using PVC: concurrent-pvc-27248b37 -2025-06-05 13:30:23,897 - tests.orchestrator - INFO - Waiting for pod test-pod-f3958de4 to be ready -2025-06-05 13:31:23,958 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-f3958de4 to be ready after 60s -2025-06-05 13:31:23,958 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-f3958de4 to be ready -2025-06-05 13:31:23,958 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-0be57a62 -2025-06-05 13:31:24,029 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-0be57a62 to be deleted -2025-06-05 13:31:24,096 - tests.orchestrator - INFO - PVC concurrent-pvc-0be57a62 has been deleted -2025-06-05 13:31:24,096 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-0be57a62 -2025-06-05 13:31:24,096 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-4b94a252 -2025-06-05 13:31:24,096 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-4b94a252 -2025-06-05 13:31:24,096 - tests.orchestrator - INFO - Deleting pod: test-pod-05839583 from PVC: concurrent-pvc-4b94a252 -2025-06-05 13:31:24,177 - tests.orchestrator - INFO - Waiting for pod test-pod-05839583 to be deleted -2025-06-05 13:31:24,241 - tests.orchestrator - INFO - Pod test-pod-05839583 has been deleted -2025-06-05 13:31:24,241 - tests.orchestrator - INFO - Deleted pod: test-pod-05839583 -2025-06-05 13:31:24,314 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-4b94a252 to be deleted -2025-06-05 13:31:24,380 - tests.orchestrator - INFO - PVC concurrent-pvc-4b94a252 has been deleted -2025-06-05 13:31:24,380 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-4b94a252 -2025-06-05 13:31:24,381 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-27248b37 -2025-06-05 13:31:24,381 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-27248b37 -2025-06-05 13:31:24,381 - tests.orchestrator - INFO - Deleting pod: test-pod-f3958de4 from PVC: concurrent-pvc-27248b37 -2025-06-05 13:31:24,464 - tests.orchestrator - INFO - Waiting for pod test-pod-f3958de4 to be deleted -2025-06-05 13:31:24,528 - tests.orchestrator - INFO - Pod test-pod-f3958de4 has been deleted -2025-06-05 13:31:24,528 - tests.orchestrator - INFO - Deleted pod: test-pod-f3958de4 -2025-06-05 13:31:24,601 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-27248b37 to be deleted -2025-06-05 13:31:24,668 - tests.orchestrator - INFO - PVC concurrent-pvc-27248b37 has been deleted -2025-06-05 13:31:24,668 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-27248b37 -2025-06-05 13:31:24,668 - tests.orchestrator - INFO - Rapid PVC scenario completed -2025-06-05 13:31:25,668 - tests.orchestrator - INFO - Selected operation: _attach_pod -2025-06-05 13:31:25,669 - tests.orchestrator - INFO - Creating pod test-pod-4386bacf on PVC concurrent-pvc-737583a5 -2025-06-05 13:31:25,750 - tests.orchestrator - INFO - Created pod: test-pod-4386bacf using PVC: concurrent-pvc-737583a5 -2025-06-05 13:31:25,750 - tests.orchestrator - INFO - Waiting for pod test-pod-4386bacf to be ready -2025-06-05 13:32:27,796 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-4386bacf to be ready after 60s -2025-06-05 13:32:27,796 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-4386bacf to be ready -2025-06-05 13:32:28,797 - tests.orchestrator - INFO - Selected operation: _attach_pod -2025-06-05 13:32:28,798 - tests.orchestrator - INFO - Creating pod test-pod-93094317 on PVC concurrent-pvc-b6ec6167 -2025-06-05 13:32:28,887 - tests.orchestrator - INFO - Created pod: test-pod-93094317 using PVC: concurrent-pvc-b6ec6167 -2025-06-05 13:32:28,887 - tests.orchestrator - INFO - Waiting for pod test-pod-93094317 to be ready -2025-06-05 13:33:30,941 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-93094317 to be ready after 60s -2025-06-05 13:33:30,941 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-93094317 to be ready -2025-06-05 13:33:31,941 - tests.orchestrator - INFO - Selected operation: _create_pvc -2025-06-05 13:33:31,942 - tests.orchestrator - INFO - Creating PVC: test-pvc-cd15165b -2025-06-05 13:33:32,022 - tests.orchestrator - INFO - Created PVC: test-pvc-cd15165b -2025-06-05 13:33:32,022 - tests.orchestrator - INFO - Waiting for PVC test-pvc-cd15165b to be bound -2025-06-05 13:33:34,157 - tests.orchestrator - INFO - PVC test-pvc-cd15165b is bound -2025-06-05 13:33:35,157 - tests.orchestrator - INFO - Selected operation: _create_pvc -2025-06-05 13:33:35,157 - tests.orchestrator - INFO - Creating PVC: test-pvc-67687a38 -2025-06-05 13:33:35,229 - tests.orchestrator - INFO - Created PVC: test-pvc-67687a38 -2025-06-05 13:33:35,229 - tests.orchestrator - INFO - Waiting for PVC test-pvc-67687a38 to be bound -2025-06-05 13:33:37,363 - tests.orchestrator - INFO - PVC test-pvc-67687a38 is bound -2025-06-05 13:33:38,364 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario -2025-06-05 13:33:38,364 - tests.orchestrator - INFO - Running specific scenario: _scenario_concurrent_pvc -2025-06-05 13:33:38,364 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations -2025-06-05 13:33:38,364 - tests.orchestrator - INFO - Creating 3 PVCs in quick succession -2025-06-05 13:33:38,431 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-a6fe87d2 -2025-06-05 13:33:38,431 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-a6fe87d2 to be bound -2025-06-05 13:33:40,563 - tests.orchestrator - INFO - PVC concurrent-pvc-a6fe87d2 is bound -2025-06-05 13:33:40,631 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-eddc8eb9 -2025-06-05 13:33:40,631 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-eddc8eb9 to be bound -2025-06-05 13:33:42,765 - tests.orchestrator - INFO - PVC concurrent-pvc-eddc8eb9 is bound -2025-06-05 13:33:42,835 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-2bd04bf8 -2025-06-05 13:33:42,836 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-2bd04bf8 to be bound -2025-06-05 13:33:44,970 - tests.orchestrator - INFO - PVC concurrent-pvc-2bd04bf8 is bound -2025-06-05 13:33:44,971 - tests.orchestrator - INFO - Successfully created 3 PVCs -2025-06-05 13:33:44,971 - tests.orchestrator - INFO - Creating pod test-pod-b5b431f0 on PVC concurrent-pvc-2bd04bf8 -2025-06-05 13:33:45,045 - tests.orchestrator - INFO - Created pod: test-pod-b5b431f0 using PVC: concurrent-pvc-2bd04bf8 -2025-06-05 13:33:45,045 - tests.orchestrator - INFO - Waiting for pod test-pod-b5b431f0 to be ready -2025-06-05 13:34:45,136 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b5b431f0 to be ready after 60s -2025-06-05 13:34:45,136 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b5b431f0 to be ready -2025-06-05 13:34:45,137 - tests.orchestrator - INFO - Creating pod test-pod-a900f8ea on PVC concurrent-pvc-a6fe87d2 -2025-06-05 13:34:45,234 - tests.orchestrator - INFO - Created pod: test-pod-a900f8ea using PVC: concurrent-pvc-a6fe87d2 -2025-06-05 13:34:45,234 - tests.orchestrator - INFO - Waiting for pod test-pod-a900f8ea to be ready -2025-06-05 13:35:45,421 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a900f8ea to be ready after 60s -2025-06-05 13:35:45,421 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a900f8ea to be ready -2025-06-05 13:35:45,421 - tests.orchestrator - INFO - Creating pod test-pod-71d9daf8 on PVC concurrent-pvc-eddc8eb9 -2025-06-05 13:35:45,506 - tests.orchestrator - INFO - Created pod: test-pod-71d9daf8 using PVC: concurrent-pvc-eddc8eb9 -2025-06-05 13:35:45,506 - tests.orchestrator - INFO - Waiting for pod test-pod-71d9daf8 to be ready -2025-06-05 13:36:47,551 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-71d9daf8 to be ready after 60s -2025-06-05 13:36:47,551 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-71d9daf8 to be ready -2025-06-05 13:36:47,551 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-2bd04bf8 -2025-06-05 13:36:47,551 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-2bd04bf8 -2025-06-05 13:36:47,551 - tests.orchestrator - INFO - Deleting pod: test-pod-b5b431f0 from PVC: concurrent-pvc-2bd04bf8 -2025-06-05 13:36:47,632 - tests.orchestrator - INFO - Waiting for pod test-pod-b5b431f0 to be deleted -2025-06-05 13:36:47,699 - tests.orchestrator - INFO - Pod test-pod-b5b431f0 has been deleted -2025-06-05 13:36:47,699 - tests.orchestrator - INFO - Deleted pod: test-pod-b5b431f0 -2025-06-05 13:36:47,771 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-2bd04bf8 to be deleted -2025-06-05 13:36:47,839 - tests.orchestrator - INFO - PVC concurrent-pvc-2bd04bf8 has been deleted -2025-06-05 13:36:47,839 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-2bd04bf8 -2025-06-05 13:36:47,839 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-eddc8eb9 -2025-06-05 13:36:47,839 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-eddc8eb9 -2025-06-05 13:36:47,839 - tests.orchestrator - INFO - Deleting pod: test-pod-71d9daf8 from PVC: concurrent-pvc-eddc8eb9 -2025-06-05 13:36:47,919 - tests.orchestrator - INFO - Waiting for pod test-pod-71d9daf8 to be deleted -2025-06-05 13:36:47,984 - tests.orchestrator - INFO - Pod test-pod-71d9daf8 has been deleted -2025-06-05 13:36:47,984 - tests.orchestrator - INFO - Deleted pod: test-pod-71d9daf8 -2025-06-05 13:36:48,056 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-eddc8eb9 to be deleted -2025-06-05 13:36:48,122 - tests.orchestrator - INFO - PVC concurrent-pvc-eddc8eb9 has been deleted -2025-06-05 13:36:48,122 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-eddc8eb9 -2025-06-05 13:36:48,122 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-a6fe87d2 -2025-06-05 13:36:48,122 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-a6fe87d2 -2025-06-05 13:36:48,122 - tests.orchestrator - INFO - Deleting pod: test-pod-a900f8ea from PVC: concurrent-pvc-a6fe87d2 -2025-06-05 13:36:48,206 - tests.orchestrator - INFO - Waiting for pod test-pod-a900f8ea to be deleted -2025-06-05 13:36:48,270 - tests.orchestrator - INFO - Pod test-pod-a900f8ea has been deleted -2025-06-05 13:36:48,271 - tests.orchestrator - INFO - Deleted pod: test-pod-a900f8ea -2025-06-05 13:36:48,345 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-a6fe87d2 to be deleted -2025-06-05 13:36:48,412 - tests.orchestrator - INFO - PVC concurrent-pvc-a6fe87d2 has been deleted -2025-06-05 13:36:48,412 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-a6fe87d2 -2025-06-05 13:36:48,412 - tests.orchestrator - INFO - Rapid PVC scenario completed -2025-06-05 13:36:49,412 - tests.orchestrator - INFO - Test completed in 526.53 seconds -2025-06-05 13:36:49,412 - tests.orchestrator - INFO - Cleaning up resources... -2025-06-05 13:36:49,412 - tests.orchestrator - INFO - Deleting pod: test-pod-4386bacf from PVC: concurrent-pvc-737583a5 -2025-06-05 13:36:49,492 - tests.orchestrator - INFO - Waiting for pod test-pod-4386bacf to be deleted -2025-06-05 13:36:49,559 - tests.orchestrator - INFO - Pod test-pod-4386bacf has been deleted -2025-06-05 13:36:49,560 - tests.orchestrator - INFO - Deleted pod: test-pod-4386bacf -2025-06-05 13:36:49,560 - tests.orchestrator - INFO - Deleting pod: test-pod-c11d7609 from PVC: concurrent-pvc-b6ec6167 -2025-06-05 13:36:49,640 - tests.orchestrator - INFO - Waiting for pod test-pod-c11d7609 to be deleted -2025-06-05 13:36:49,706 - tests.orchestrator - INFO - Pod test-pod-c11d7609 has been deleted -2025-06-05 13:36:49,706 - tests.orchestrator - INFO - Deleted pod: test-pod-c11d7609 -2025-06-05 13:36:49,706 - tests.orchestrator - INFO - Deleting pod: test-pod-93094317 from PVC: concurrent-pvc-b6ec6167 -2025-06-05 13:36:49,783 - tests.orchestrator - INFO - Waiting for pod test-pod-93094317 to be deleted -2025-06-05 13:36:49,850 - tests.orchestrator - INFO - Pod test-pod-93094317 has been deleted -2025-06-05 13:36:49,850 - tests.orchestrator - INFO - Deleted pod: test-pod-93094317 -2025-06-05 13:36:49,850 - tests.orchestrator - INFO - Deleting PVC: test-pvc-87f8dca2 -2025-06-05 13:36:49,922 - tests.orchestrator - INFO - Waiting for PVC test-pvc-87f8dca2 to be deleted -2025-06-05 13:36:49,988 - tests.orchestrator - INFO - PVC test-pvc-87f8dca2 has been deleted -2025-06-05 13:36:49,989 - tests.orchestrator - INFO - Deleted PVC: test-pvc-87f8dca2 -2025-06-05 13:36:49,989 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-d46ca8c1 -2025-06-05 13:36:50,060 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-d46ca8c1 to be deleted -2025-06-05 13:36:50,126 - tests.orchestrator - INFO - PVC concurrent-pvc-d46ca8c1 has been deleted -2025-06-05 13:36:50,127 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-d46ca8c1 -2025-06-05 13:36:50,127 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-737583a5 -2025-06-05 13:36:50,197 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-737583a5 to be deleted -2025-06-05 13:36:50,264 - tests.orchestrator - INFO - PVC concurrent-pvc-737583a5 has been deleted -2025-06-05 13:36:50,264 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-737583a5 -2025-06-05 13:36:50,264 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-b6ec6167 -2025-06-05 13:36:50,336 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-b6ec6167 to be deleted -2025-06-05 13:36:50,403 - tests.orchestrator - INFO - PVC concurrent-pvc-b6ec6167 has been deleted -2025-06-05 13:36:50,403 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-b6ec6167 -2025-06-05 13:36:50,403 - tests.orchestrator - INFO - Deleting PVC: test-pvc-cd15165b -2025-06-05 13:36:50,477 - tests.orchestrator - INFO - Waiting for PVC test-pvc-cd15165b to be deleted -2025-06-05 13:36:50,544 - tests.orchestrator - INFO - PVC test-pvc-cd15165b has been deleted -2025-06-05 13:36:50,545 - tests.orchestrator - INFO - Deleted PVC: test-pvc-cd15165b -2025-06-05 13:36:50,545 - tests.orchestrator - INFO - Deleting PVC: test-pvc-67687a38 -2025-06-05 13:36:50,617 - tests.orchestrator - INFO - Waiting for PVC test-pvc-67687a38 to be deleted -2025-06-05 13:36:50,683 - tests.orchestrator - INFO - PVC test-pvc-67687a38 has been deleted -2025-06-05 13:36:50,683 - tests.orchestrator - INFO - Deleted PVC: test-pvc-67687a38 -2025-06-05 13:36:50,683 - tests.orchestrator - INFO - Cleanup completed -2025-06-05 13:36:50,683 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-05 13:36:50,683 - tests.orchestrator - INFO - --- Operations --- -2025-06-05 13:36:50,683 - tests.orchestrator - INFO - create_pvc: 12 succeeded, 0 failed (100.0%) -2025-06-05 13:36:50,683 - tests.orchestrator - INFO - attach_pod: 8 succeeded, 0 failed (100.0%) -2025-06-05 13:36:50,683 - tests.orchestrator - INFO - delete_pod: 8 succeeded, 0 failed (100.0%) -2025-06-05 13:36:50,684 - tests.orchestrator - INFO - delete_pvc: 12 succeeded, 0 failed (100.0%) -2025-06-05 13:36:50,684 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 0 failed (0.0%) -2025-06-05 13:36:50,684 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 0 failed (0.0%) -2025-06-05 13:36:50,684 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-05 13:36:50,684 - tests.orchestrator - INFO - shared_volume_rw: No runs -2025-06-05 13:36:50,684 - tests.orchestrator - INFO - many_to_one: No runs -2025-06-05 13:36:50,684 - tests.orchestrator - INFO - one_to_one: No runs -2025-06-05 13:36:50,684 - tests.orchestrator - INFO - concurrent_pvc: 2 succeeded, 0 failed out of 2 runs (100.0%) -2025-06-05 13:36:50,684 - tests.orchestrator - INFO - ========================================= -2025-06-05 13:36:50,685 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250605_133650.json -2025-06-05 13:36:50,728 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250605_133650_20250605_133650.json -2025-06-05 13:36:50,728 - __main__ - INFO - All tests completed successfully -2025-06-05 13:50:01,756 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-05 13:50:01,756 - __main__ - INFO - Verifying AWS credentials -2025-06-05 13:50:01,757 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 13:50:01,758 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 13:50:01,758 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 13:50:01,758 - __main__ - INFO - Running orchestrator stress test suite -2025-06-05 13:50:01,758 - __main__ - INFO - Starting orchestrator for 300 seconds -2025-06-05 13:50:01,759 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-05 13:50:02,494 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 13:50:02,715 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 13:50:02,715 - __main__ - INFO - Test duration overridden to 300 seconds -2025-06-05 13:50:02,715 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) -2025-06-05 13:50:02,715 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-05 13:50:02,906 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 13:50:02,906 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 13:50:02,906 - tests.orchestrator - INFO - Creating PVC: test-pvc-040b70c6 -2025-06-05 13:50:02,982 - tests.orchestrator - INFO - Created PVC: test-pvc-040b70c6 -2025-06-05 13:50:02,982 - tests.orchestrator - INFO - Waiting for PVC test-pvc-040b70c6 to be bound -2025-06-05 13:50:05,116 - tests.orchestrator - INFO - PVC test-pvc-040b70c6 is bound -2025-06-05 13:50:05,116 - tests.orchestrator - INFO - Creating PVC: test-pvc-e5eb6b62 -2025-06-05 13:50:05,184 - tests.orchestrator - INFO - Created PVC: test-pvc-e5eb6b62 -2025-06-05 13:50:05,184 - tests.orchestrator - INFO - Waiting for PVC test-pvc-e5eb6b62 to be bound -2025-06-05 13:50:07,318 - tests.orchestrator - INFO - PVC test-pvc-e5eb6b62 is bound -2025-06-05 13:50:07,319 - tests.orchestrator - INFO - Creating pod test-pod-094962b3 on PVC test-pvc-e5eb6b62 -2025-06-05 13:50:07,399 - tests.orchestrator - INFO - Created pod: test-pod-094962b3 using PVC: test-pvc-e5eb6b62 -2025-06-05 13:50:07,400 - tests.orchestrator - INFO - Waiting for pod test-pod-094962b3 to be ready -2025-06-05 13:51:09,437 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-094962b3 to be ready after 60s -2025-06-05 13:51:09,437 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-094962b3 to be ready -2025-06-05 13:51:09,437 - tests.orchestrator - INFO - Creating pod test-pod-1b064942 on PVC test-pvc-040b70c6 -2025-06-05 13:51:09,521 - tests.orchestrator - INFO - Created pod: test-pod-1b064942 using PVC: test-pvc-040b70c6 -2025-06-05 13:51:09,521 - tests.orchestrator - INFO - Waiting for pod test-pod-1b064942 to be ready -2025-06-05 13:52:11,558 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1b064942 to be ready after 60s -2025-06-05 13:52:11,558 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1b064942 to be ready -2025-06-05 13:52:11,559 - tests.orchestrator - INFO - Creating pod test-pod-ab3159d4 on PVC test-pvc-e5eb6b62 -2025-06-05 13:52:11,645 - tests.orchestrator - INFO - Created pod: test-pod-ab3159d4 using PVC: test-pvc-e5eb6b62 -2025-06-05 13:52:11,646 - tests.orchestrator - INFO - Waiting for pod test-pod-ab3159d4 to be ready -2025-06-05 13:53:13,682 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-ab3159d4 to be ready after 60s -2025-06-05 13:53:13,682 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-ab3159d4 to be ready -2025-06-05 13:53:13,683 - tests.orchestrator - INFO - Testing read/write between pods test-pod-094962b3 and test-pod-ab3159d4 sharing PVC test-pvc-e5eb6b62 -2025-06-05 13:53:13,751 - tests.orchestrator - ERROR - Failed in read/write verification: (400) -Reason: Bad Request -HTTP response headers: HTTPHeaderDict({'Audit-Id': '7c3ae37d-dd7e-42d1-97d7-9f45f47124cc', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 13:53:13 GMT', 'Content-Length': '166'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-094962b3 does not have a host assigned","reason":"BadRequest","code":400} - - -2025-06-05 13:53:13,751 - tests.orchestrator - INFO - Running specific scenario: _scenario_one_to_one -2025-06-05 13:53:13,751 - tests.orchestrator - INFO - Running scenario: One pod per PVC -2025-06-05 13:53:13,751 - tests.orchestrator - INFO - Creating 3 PVC-pod pairs -2025-06-05 13:53:13,829 - tests.orchestrator - INFO - Waiting for PVC one2one-9268308a to be bound -2025-06-05 13:53:15,962 - tests.orchestrator - INFO - PVC one2one-9268308a is bound -2025-06-05 13:53:15,962 - tests.orchestrator - INFO - Creating pod test-pod-2d27ea53 on PVC one2one-9268308a -2025-06-05 13:53:16,038 - tests.orchestrator - INFO - Created pod: test-pod-2d27ea53 using PVC: one2one-9268308a -2025-06-05 13:53:16,038 - tests.orchestrator - INFO - Waiting for pod test-pod-2d27ea53 to be ready -2025-06-05 13:54:16,048 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-2d27ea53 to be ready after 60s -2025-06-05 13:54:16,048 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-2d27ea53 to be ready -2025-06-05 13:54:16,124 - tests.orchestrator - INFO - Waiting for PVC one2one-63559411 to be bound -2025-06-05 13:54:18,258 - tests.orchestrator - INFO - PVC one2one-63559411 is bound -2025-06-05 13:54:18,258 - tests.orchestrator - INFO - Creating pod test-pod-5c1f7339 on PVC one2one-63559411 -2025-06-05 13:54:18,330 - tests.orchestrator - INFO - Created pod: test-pod-5c1f7339 using PVC: one2one-63559411 -2025-06-05 13:54:18,330 - tests.orchestrator - INFO - Waiting for pod test-pod-5c1f7339 to be ready -2025-06-05 13:55:20,367 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-5c1f7339 to be ready after 60s -2025-06-05 13:55:20,367 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-5c1f7339 to be ready -2025-06-05 13:55:20,444 - tests.orchestrator - INFO - Waiting for PVC one2one-24717940 to be bound -2025-06-05 13:55:22,579 - tests.orchestrator - INFO - PVC one2one-24717940 is bound -2025-06-05 13:55:22,580 - tests.orchestrator - INFO - Creating pod test-pod-a84638d7 on PVC one2one-24717940 -2025-06-05 13:55:22,655 - tests.orchestrator - INFO - Created pod: test-pod-a84638d7 using PVC: one2one-24717940 -2025-06-05 13:55:22,655 - tests.orchestrator - INFO - Waiting for pod test-pod-a84638d7 to be ready -2025-06-05 13:56:24,701 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a84638d7 to be ready after 60s -2025-06-05 13:56:24,701 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a84638d7 to be ready -2025-06-05 13:56:24,701 - tests.orchestrator - WARNING - Failed to create enough PVC-pod pairs, only created 0 -2025-06-05 13:56:24,701 - tests.orchestrator - INFO - Deleting pod: test-pod-094962b3 from PVC: test-pvc-e5eb6b62 -2025-06-05 13:56:24,780 - tests.orchestrator - INFO - Waiting for pod test-pod-094962b3 to be deleted -2025-06-05 13:56:24,846 - tests.orchestrator - INFO - Pod test-pod-094962b3 has been deleted -2025-06-05 13:56:24,846 - tests.orchestrator - INFO - Deleted pod: test-pod-094962b3 -2025-06-05 13:56:24,846 - tests.orchestrator - INFO - Deleting PVC: one2one-63559411 -2025-06-05 13:56:24,846 - tests.orchestrator - INFO - Deleting 1 pods using PVC one2one-63559411 -2025-06-05 13:56:24,846 - tests.orchestrator - INFO - Deleting pod: test-pod-5c1f7339 from PVC: one2one-63559411 -2025-06-05 13:56:24,932 - tests.orchestrator - INFO - Waiting for pod test-pod-5c1f7339 to be deleted -2025-06-05 13:56:24,997 - tests.orchestrator - INFO - Pod test-pod-5c1f7339 has been deleted -2025-06-05 13:56:24,997 - tests.orchestrator - INFO - Deleted pod: test-pod-5c1f7339 -2025-06-05 13:56:25,070 - tests.orchestrator - INFO - Waiting for PVC one2one-63559411 to be deleted -2025-06-05 13:56:25,135 - tests.orchestrator - INFO - PVC one2one-63559411 has been deleted -2025-06-05 13:56:25,136 - tests.orchestrator - INFO - Deleted PVC: one2one-63559411 -2025-06-05 13:56:25,136 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations -2025-06-05 13:56:25,136 - tests.orchestrator - INFO - Test completed in 382.42 seconds -2025-06-05 13:56:25,136 - tests.orchestrator - INFO - Cleaning up resources... -2025-06-05 13:56:25,136 - tests.orchestrator - INFO - Deleting pod: test-pod-1b064942 from PVC: test-pvc-040b70c6 -2025-06-05 13:56:25,213 - tests.orchestrator - INFO - Waiting for pod test-pod-1b064942 to be deleted -2025-06-05 13:56:25,279 - tests.orchestrator - INFO - Pod test-pod-1b064942 has been deleted -2025-06-05 13:56:25,279 - tests.orchestrator - INFO - Deleted pod: test-pod-1b064942 -2025-06-05 13:56:25,279 - tests.orchestrator - INFO - Deleting pod: test-pod-ab3159d4 from PVC: test-pvc-e5eb6b62 -2025-06-05 13:56:25,363 - tests.orchestrator - INFO - Waiting for pod test-pod-ab3159d4 to be deleted -2025-06-05 13:56:25,429 - tests.orchestrator - INFO - Pod test-pod-ab3159d4 has been deleted -2025-06-05 13:56:25,429 - tests.orchestrator - INFO - Deleted pod: test-pod-ab3159d4 -2025-06-05 13:56:25,429 - tests.orchestrator - INFO - Deleting pod: test-pod-2d27ea53 from PVC: one2one-9268308a -2025-06-05 13:56:25,507 - tests.orchestrator - INFO - Waiting for pod test-pod-2d27ea53 to be deleted -2025-06-05 13:56:25,573 - tests.orchestrator - INFO - Pod test-pod-2d27ea53 has been deleted -2025-06-05 13:56:25,573 - tests.orchestrator - INFO - Deleted pod: test-pod-2d27ea53 -2025-06-05 13:56:25,574 - tests.orchestrator - INFO - Deleting pod: test-pod-a84638d7 from PVC: one2one-24717940 -2025-06-05 13:56:25,658 - tests.orchestrator - INFO - Waiting for pod test-pod-a84638d7 to be deleted -2025-06-05 13:56:25,725 - tests.orchestrator - INFO - Pod test-pod-a84638d7 has been deleted -2025-06-05 13:56:25,726 - tests.orchestrator - INFO - Deleted pod: test-pod-a84638d7 -2025-06-05 13:56:25,726 - tests.orchestrator - INFO - Deleting PVC: test-pvc-040b70c6 -2025-06-05 13:56:25,800 - tests.orchestrator - INFO - Waiting for PVC test-pvc-040b70c6 to be deleted -2025-06-05 13:56:25,866 - tests.orchestrator - INFO - PVC test-pvc-040b70c6 has been deleted -2025-06-05 13:56:25,866 - tests.orchestrator - INFO - Deleted PVC: test-pvc-040b70c6 -2025-06-05 13:56:25,866 - tests.orchestrator - INFO - Deleting PVC: test-pvc-e5eb6b62 -2025-06-05 13:56:25,940 - tests.orchestrator - INFO - Waiting for PVC test-pvc-e5eb6b62 to be deleted -2025-06-05 13:56:26,006 - tests.orchestrator - INFO - PVC test-pvc-e5eb6b62 has been deleted -2025-06-05 13:56:26,006 - tests.orchestrator - INFO - Deleted PVC: test-pvc-e5eb6b62 -2025-06-05 13:56:26,006 - tests.orchestrator - INFO - Deleting PVC: one2one-9268308a -2025-06-05 13:56:26,081 - tests.orchestrator - INFO - Waiting for PVC one2one-9268308a to be deleted -2025-06-05 13:56:26,146 - tests.orchestrator - INFO - PVC one2one-9268308a has been deleted -2025-06-05 13:56:26,146 - tests.orchestrator - INFO - Deleted PVC: one2one-9268308a -2025-06-05 13:56:26,146 - tests.orchestrator - INFO - Deleting PVC: one2one-24717940 -2025-06-05 13:56:26,221 - tests.orchestrator - INFO - Waiting for PVC one2one-24717940 to be deleted -2025-06-05 13:56:26,287 - tests.orchestrator - INFO - PVC one2one-24717940 has been deleted -2025-06-05 13:56:26,287 - tests.orchestrator - INFO - Deleted PVC: one2one-24717940 -2025-06-05 13:56:26,287 - tests.orchestrator - INFO - Cleanup completed -2025-06-05 13:56:26,287 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-05 13:56:26,287 - tests.orchestrator - INFO - --- Operations --- -2025-06-05 13:56:26,287 - tests.orchestrator - INFO - create_pvc: 2 succeeded, 0 failed (100.0%) -2025-06-05 13:56:26,287 - tests.orchestrator - INFO - attach_pod: 6 succeeded, 0 failed (100.0%) -2025-06-05 13:56:26,287 - tests.orchestrator - INFO - delete_pod: 6 succeeded, 0 failed (100.0%) -2025-06-05 13:56:26,287 - tests.orchestrator - INFO - delete_pvc: 5 succeeded, 0 failed (100.0%) -2025-06-05 13:56:26,287 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) -2025-06-05 13:56:26,287 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) -2025-06-05 13:56:26,287 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-05 13:56:26,287 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-05 13:56:26,287 - tests.orchestrator - INFO - many_to_one: No runs -2025-06-05 13:56:26,287 - tests.orchestrator - INFO - one_to_one: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-05 13:56:26,288 - tests.orchestrator - INFO - concurrent_pvc: No runs -2025-06-05 13:56:26,288 - tests.orchestrator - INFO - ========================================= -2025-06-05 13:56:26,288 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250605_135626.json -2025-06-05 13:56:26,330 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250605_135626_20250605_135626.json -2025-06-05 13:56:26,331 - __main__ - INFO - All tests completed successfully -2025-06-05 14:04:07,833 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-05 14:04:07,833 - __main__ - INFO - Verifying AWS credentials -2025-06-05 14:04:07,834 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 14:04:07,834 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 14:04:07,835 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 14:04:07,835 - __main__ - INFO - Running orchestrator stress test suite -2025-06-05 14:04:07,835 - __main__ - INFO - Starting orchestrator for 300 seconds -2025-06-05 14:04:07,835 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-05 14:04:08,567 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 14:04:08,780 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 14:04:08,780 - __main__ - INFO - Test duration overridden to 300 seconds -2025-06-05 14:04:08,780 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) -2025-06-05 14:04:08,780 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-05 14:04:08,972 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 14:04:08,972 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 14:04:08,972 - tests.orchestrator - INFO - Creating PVC: test-pvc-847a8a56 -2025-06-05 14:04:09,057 - tests.orchestrator - INFO - Created PVC: test-pvc-847a8a56 -2025-06-05 14:04:09,057 - tests.orchestrator - INFO - Waiting for PVC test-pvc-847a8a56 to be bound -2025-06-05 14:04:11,188 - tests.orchestrator - INFO - PVC test-pvc-847a8a56 is bound -2025-06-05 14:04:11,189 - tests.orchestrator - INFO - Creating PVC: test-pvc-9dee4752 -2025-06-05 14:04:11,257 - tests.orchestrator - INFO - Created PVC: test-pvc-9dee4752 -2025-06-05 14:04:11,258 - tests.orchestrator - INFO - Waiting for PVC test-pvc-9dee4752 to be bound -2025-06-05 14:04:13,391 - tests.orchestrator - INFO - PVC test-pvc-9dee4752 is bound -2025-06-05 14:04:13,391 - tests.orchestrator - INFO - Creating pod test-pod-d215a119 on PVC test-pvc-9dee4752 -2025-06-05 14:04:13,474 - tests.orchestrator - INFO - Created pod: test-pod-d215a119 using PVC: test-pvc-9dee4752 -2025-06-05 14:04:13,474 - tests.orchestrator - INFO - Waiting for pod test-pod-d215a119 to be ready -2025-06-05 14:05:15,521 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-d215a119 to be ready after 60s -2025-06-05 14:05:15,521 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-d215a119 to be ready -2025-06-05 14:05:15,522 - tests.orchestrator - INFO - Creating pod test-pod-54364ba6 on PVC test-pvc-9dee4752 -2025-06-05 14:05:15,608 - tests.orchestrator - INFO - Created pod: test-pod-54364ba6 using PVC: test-pvc-9dee4752 -2025-06-05 14:05:15,608 - tests.orchestrator - INFO - Waiting for pod test-pod-54364ba6 to be ready -2025-06-05 14:06:15,655 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-54364ba6 to be ready after 60s -2025-06-05 14:06:15,655 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-54364ba6 to be ready -2025-06-05 14:06:15,655 - tests.orchestrator - INFO - Creating pod test-pod-98a62ebf on PVC test-pvc-9dee4752 -2025-06-05 14:06:15,742 - tests.orchestrator - INFO - Created pod: test-pod-98a62ebf using PVC: test-pvc-9dee4752 -2025-06-05 14:06:15,742 - tests.orchestrator - INFO - Waiting for pod test-pod-98a62ebf to be ready -2025-06-05 14:07:15,768 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-98a62ebf to be ready after 60s -2025-06-05 14:07:15,769 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-98a62ebf to be ready -2025-06-05 14:07:15,769 - tests.orchestrator - INFO - Testing read/write between pods test-pod-d215a119 and test-pod-98a62ebf sharing PVC test-pvc-9dee4752 -2025-06-05 14:07:15,836 - tests.orchestrator - ERROR - Failed in read/write verification: (400) -Reason: Bad Request -HTTP response headers: HTTPHeaderDict({'Audit-Id': '0fd252b7-6182-4f97-9a5a-1503836973da', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 14:07:15 GMT', 'Content-Length': '166'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-d215a119 does not have a host assigned","reason":"BadRequest","code":400} - - -2025-06-05 14:07:15,836 - tests.orchestrator - INFO - Running specific scenario: _scenario_one_to_one -2025-06-05 14:07:15,836 - tests.orchestrator - INFO - Running scenario: One pod per PVC -2025-06-05 14:07:15,836 - tests.orchestrator - INFO - Creating 4 PVC-pod pairs -2025-06-05 14:07:15,913 - tests.orchestrator - INFO - Waiting for PVC one2one-71f9fc22 to be bound -2025-06-05 14:07:18,066 - tests.orchestrator - INFO - PVC one2one-71f9fc22 is bound -2025-06-05 14:07:18,066 - tests.orchestrator - INFO - Creating pod test-pod-78c97483 on PVC one2one-71f9fc22 -2025-06-05 14:07:18,197 - tests.orchestrator - INFO - Created pod: test-pod-78c97483 using PVC: one2one-71f9fc22 -2025-06-05 14:07:18,197 - tests.orchestrator - INFO - Waiting for pod test-pod-78c97483 to be ready -2025-06-05 14:08:18,229 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-78c97483 to be ready after 60s -2025-06-05 14:08:18,229 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-78c97483 to be ready -2025-06-05 14:08:18,310 - tests.orchestrator - INFO - Waiting for PVC one2one-d55f8841 to be bound -2025-06-05 14:08:20,442 - tests.orchestrator - INFO - PVC one2one-d55f8841 is bound -2025-06-05 14:08:20,442 - tests.orchestrator - INFO - Creating pod test-pod-a400277a on PVC one2one-d55f8841 -2025-06-05 14:08:20,518 - tests.orchestrator - INFO - Created pod: test-pod-a400277a using PVC: one2one-d55f8841 -2025-06-05 14:08:20,519 - tests.orchestrator - INFO - Waiting for pod test-pod-a400277a to be ready -2025-06-05 14:09:22,561 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a400277a to be ready after 60s -2025-06-05 14:09:22,561 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a400277a to be ready -2025-06-05 14:09:22,638 - tests.orchestrator - INFO - Waiting for PVC one2one-f3257be9 to be bound -2025-06-05 14:09:24,772 - tests.orchestrator - INFO - PVC one2one-f3257be9 is bound -2025-06-05 14:09:24,773 - tests.orchestrator - INFO - Creating pod test-pod-1bfbc8b3 on PVC one2one-f3257be9 -2025-06-05 14:09:24,850 - tests.orchestrator - INFO - Created pod: test-pod-1bfbc8b3 using PVC: one2one-f3257be9 -2025-06-05 14:09:24,850 - tests.orchestrator - INFO - Waiting for pod test-pod-1bfbc8b3 to be ready -2025-06-05 14:10:26,912 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1bfbc8b3 to be ready after 60s -2025-06-05 14:10:26,913 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1bfbc8b3 to be ready -2025-06-05 14:10:26,992 - tests.orchestrator - INFO - Waiting for PVC one2one-d2044d7d to be bound -2025-06-05 14:10:29,127 - tests.orchestrator - INFO - PVC one2one-d2044d7d is bound -2025-06-05 14:10:29,127 - tests.orchestrator - INFO - Creating pod test-pod-6edcdac8 on PVC one2one-d2044d7d -2025-06-05 14:10:29,203 - tests.orchestrator - INFO - Created pod: test-pod-6edcdac8 using PVC: one2one-d2044d7d -2025-06-05 14:10:29,203 - tests.orchestrator - INFO - Waiting for pod test-pod-6edcdac8 to be ready -2025-06-05 14:11:29,292 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-6edcdac8 to be ready after 60s -2025-06-05 14:11:29,293 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-6edcdac8 to be ready -2025-06-05 14:11:29,293 - tests.orchestrator - WARNING - Failed to create enough PVC-pod pairs, only created 0 -2025-06-05 14:11:29,293 - tests.orchestrator - INFO - Deleting pod: test-pod-54364ba6 from PVC: test-pvc-9dee4752 -2025-06-05 14:11:29,378 - tests.orchestrator - INFO - Waiting for pod test-pod-54364ba6 to be deleted -2025-06-05 14:11:29,443 - tests.orchestrator - INFO - Pod test-pod-54364ba6 has been deleted -2025-06-05 14:11:29,443 - tests.orchestrator - INFO - Deleted pod: test-pod-54364ba6 -2025-06-05 14:11:29,443 - tests.orchestrator - INFO - Deleting PVC: one2one-71f9fc22 -2025-06-05 14:11:29,443 - tests.orchestrator - INFO - Deleting 1 pods using PVC one2one-71f9fc22 -2025-06-05 14:11:29,443 - tests.orchestrator - INFO - Deleting pod: test-pod-78c97483 from PVC: one2one-71f9fc22 -2025-06-05 14:11:29,521 - tests.orchestrator - INFO - Waiting for pod test-pod-78c97483 to be deleted -2025-06-05 14:11:29,588 - tests.orchestrator - INFO - Pod test-pod-78c97483 has been deleted -2025-06-05 14:11:29,588 - tests.orchestrator - INFO - Deleted pod: test-pod-78c97483 -2025-06-05 14:11:29,666 - tests.orchestrator - INFO - Waiting for PVC one2one-71f9fc22 to be deleted -2025-06-05 14:11:29,733 - tests.orchestrator - INFO - PVC one2one-71f9fc22 has been deleted -2025-06-05 14:11:29,733 - tests.orchestrator - INFO - Deleted PVC: one2one-71f9fc22 -2025-06-05 14:11:29,733 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations -2025-06-05 14:11:29,733 - tests.orchestrator - INFO - Test completed in 440.95 seconds -2025-06-05 14:11:29,733 - tests.orchestrator - INFO - Cleaning up resources... -2025-06-05 14:11:29,733 - tests.orchestrator - INFO - Deleting pod: test-pod-d215a119 from PVC: test-pvc-9dee4752 -2025-06-05 14:11:29,832 - tests.orchestrator - INFO - Waiting for pod test-pod-d215a119 to be deleted -2025-06-05 14:11:29,897 - tests.orchestrator - INFO - Pod test-pod-d215a119 has been deleted -2025-06-05 14:11:29,897 - tests.orchestrator - INFO - Deleted pod: test-pod-d215a119 -2025-06-05 14:11:29,897 - tests.orchestrator - INFO - Deleting pod: test-pod-98a62ebf from PVC: test-pvc-9dee4752 -2025-06-05 14:11:29,977 - tests.orchestrator - INFO - Waiting for pod test-pod-98a62ebf to be deleted -2025-06-05 14:11:30,044 - tests.orchestrator - INFO - Pod test-pod-98a62ebf has been deleted -2025-06-05 14:11:30,044 - tests.orchestrator - INFO - Deleted pod: test-pod-98a62ebf -2025-06-05 14:11:30,044 - tests.orchestrator - INFO - Deleting pod: test-pod-a400277a from PVC: one2one-d55f8841 -2025-06-05 14:11:30,134 - tests.orchestrator - INFO - Waiting for pod test-pod-a400277a to be deleted -2025-06-05 14:11:30,199 - tests.orchestrator - INFO - Pod test-pod-a400277a has been deleted -2025-06-05 14:11:30,199 - tests.orchestrator - INFO - Deleted pod: test-pod-a400277a -2025-06-05 14:11:30,199 - tests.orchestrator - INFO - Deleting pod: test-pod-1bfbc8b3 from PVC: one2one-f3257be9 -2025-06-05 14:11:30,279 - tests.orchestrator - INFO - Waiting for pod test-pod-1bfbc8b3 to be deleted -2025-06-05 14:11:30,346 - tests.orchestrator - INFO - Pod test-pod-1bfbc8b3 has been deleted -2025-06-05 14:11:30,346 - tests.orchestrator - INFO - Deleted pod: test-pod-1bfbc8b3 -2025-06-05 14:11:30,346 - tests.orchestrator - INFO - Deleting pod: test-pod-6edcdac8 from PVC: one2one-d2044d7d -2025-06-05 14:11:30,432 - tests.orchestrator - INFO - Waiting for pod test-pod-6edcdac8 to be deleted -2025-06-05 14:11:30,500 - tests.orchestrator - INFO - Pod test-pod-6edcdac8 has been deleted -2025-06-05 14:11:30,500 - tests.orchestrator - INFO - Deleted pod: test-pod-6edcdac8 -2025-06-05 14:11:30,500 - tests.orchestrator - INFO - Deleting PVC: test-pvc-847a8a56 -2025-06-05 14:11:30,572 - tests.orchestrator - INFO - Waiting for PVC test-pvc-847a8a56 to be deleted -2025-06-05 14:11:30,642 - tests.orchestrator - INFO - PVC test-pvc-847a8a56 has been deleted -2025-06-05 14:11:30,642 - tests.orchestrator - INFO - Deleted PVC: test-pvc-847a8a56 -2025-06-05 14:11:30,642 - tests.orchestrator - INFO - Deleting PVC: test-pvc-9dee4752 -2025-06-05 14:11:30,715 - tests.orchestrator - INFO - Waiting for PVC test-pvc-9dee4752 to be deleted -2025-06-05 14:11:30,782 - tests.orchestrator - INFO - PVC test-pvc-9dee4752 has been deleted -2025-06-05 14:11:30,782 - tests.orchestrator - INFO - Deleted PVC: test-pvc-9dee4752 -2025-06-05 14:11:30,782 - tests.orchestrator - INFO - Deleting PVC: one2one-d55f8841 -2025-06-05 14:11:30,860 - tests.orchestrator - INFO - Waiting for PVC one2one-d55f8841 to be deleted -2025-06-05 14:11:30,927 - tests.orchestrator - INFO - PVC one2one-d55f8841 has been deleted -2025-06-05 14:11:30,927 - tests.orchestrator - INFO - Deleted PVC: one2one-d55f8841 -2025-06-05 14:11:30,928 - tests.orchestrator - INFO - Deleting PVC: one2one-f3257be9 -2025-06-05 14:11:30,999 - tests.orchestrator - INFO - Waiting for PVC one2one-f3257be9 to be deleted -2025-06-05 14:11:31,068 - tests.orchestrator - INFO - PVC one2one-f3257be9 has been deleted -2025-06-05 14:11:31,068 - tests.orchestrator - INFO - Deleted PVC: one2one-f3257be9 -2025-06-05 14:11:31,068 - tests.orchestrator - INFO - Deleting PVC: one2one-d2044d7d -2025-06-05 14:11:31,142 - tests.orchestrator - INFO - Waiting for PVC one2one-d2044d7d to be deleted -2025-06-05 14:11:31,208 - tests.orchestrator - INFO - PVC one2one-d2044d7d has been deleted -2025-06-05 14:11:31,208 - tests.orchestrator - INFO - Deleted PVC: one2one-d2044d7d -2025-06-05 14:11:31,208 - tests.orchestrator - INFO - Cleanup completed -2025-06-05 14:11:31,208 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-05 14:11:31,208 - tests.orchestrator - INFO - --- Operations --- -2025-06-05 14:11:31,209 - tests.orchestrator - INFO - create_pvc: 2 succeeded, 0 failed (100.0%) -2025-06-05 14:11:31,209 - tests.orchestrator - INFO - attach_pod: 7 succeeded, 0 failed (100.0%) -2025-06-05 14:11:31,209 - tests.orchestrator - INFO - delete_pod: 7 succeeded, 0 failed (100.0%) -2025-06-05 14:11:31,209 - tests.orchestrator - INFO - delete_pvc: 6 succeeded, 0 failed (100.0%) -2025-06-05 14:11:31,209 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) -2025-06-05 14:11:31,209 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) -2025-06-05 14:11:31,209 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-05 14:11:31,209 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-05 14:11:31,209 - tests.orchestrator - INFO - many_to_one: No runs -2025-06-05 14:11:31,209 - tests.orchestrator - INFO - one_to_one: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-05 14:11:31,209 - tests.orchestrator - INFO - concurrent_pvc: No runs -2025-06-05 14:11:31,209 - tests.orchestrator - INFO - ========================================= -2025-06-05 14:11:31,210 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250605_141131.json -2025-06-05 14:11:31,252 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250605_141131_20250605_141131.json -2025-06-05 14:11:31,252 - __main__ - INFO - All tests completed successfully -2025-06-05 14:13:20,869 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-05 14:13:20,869 - __main__ - INFO - Verifying AWS credentials -2025-06-05 14:13:20,870 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 14:13:20,871 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 14:13:20,871 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 14:13:20,871 - __main__ - INFO - Running orchestrator stress test suite -2025-06-05 14:13:20,871 - __main__ - INFO - Starting orchestrator for 60 seconds -2025-06-05 14:13:20,872 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-05 14:13:21,602 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 14:13:21,814 - tests.orchestrator - ERROR - Error checking namespace: (401) -Reason: Unauthorized -HTTP response headers: HTTPHeaderDict({'Audit-Id': 'eb0b1f42-f08c-4b18-b778-a83d8eb055f7', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 14:13:21 GMT', 'Content-Length': '129'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} - - -2025-06-05 14:13:21,814 - __main__ - ERROR - Error running tests: (401) -Reason: Unauthorized -HTTP response headers: HTTPHeaderDict({'Audit-Id': 'eb0b1f42-f08c-4b18-b778-a83d8eb055f7', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 14:13:21 GMT', 'Content-Length': '129'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} - -Traceback (most recent call last): - File "/local/home/ibhard/efs-csi-driver-fork/aws-efs-csi-driver/test/e2e/stress-scale-tests/run_tests.py", line 209, in main - orchestrator = EFSCSIOrchestrator(config_file=orchestrator_config) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/local/home/ibhard/efs-csi-driver-fork/aws-efs-csi-driver/test/e2e/stress-scale-tests/tests/orchestrator.py", line 86, in __init__ - self._ensure_namespace_exists() - File "/local/home/ibhard/efs-csi-driver-fork/aws-efs-csi-driver/test/e2e/stress-scale-tests/tests/orchestrator.py", line 92, in _ensure_namespace_exists - self.core_v1.read_namespace(name=self.namespace) - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api/core_v1_api.py", line 22992, in read_namespace - return self.read_namespace_with_http_info(name, **kwargs) # noqa: E501 - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api/core_v1_api.py", line 23071, in read_namespace_with_http_info - return self.api_client.call_api( - ^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api_client.py", line 348, in call_api - return self.__call_api(resource_path, method, - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api_client.py", line 180, in __call_api - response_data = self.request( - ^^^^^^^^^^^^^ - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api_client.py", line 373, in request - return self.rest_client.GET(url, - ^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/rest.py", line 244, in GET - return self.request("GET", url, - ^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/rest.py", line 238, in request - raise ApiException(http_resp=r) -kubernetes.client.exceptions.ApiException: (401) -Reason: Unauthorized -HTTP response headers: HTTPHeaderDict({'Audit-Id': 'eb0b1f42-f08c-4b18-b778-a83d8eb055f7', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 14:13:21 GMT', 'Content-Length': '129'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} - - -2025-06-05 14:14:34,079 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-05 14:14:34,079 - __main__ - INFO - Verifying AWS credentials -2025-06-05 14:14:34,080 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 14:14:34,080 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 14:14:34,081 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 14:14:34,081 - __main__ - INFO - Running orchestrator stress test suite -2025-06-05 14:14:34,081 - __main__ - INFO - Starting orchestrator for 60 seconds -2025-06-05 14:14:34,081 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-05 14:14:34,816 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 14:14:35,029 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 14:14:35,029 - __main__ - INFO - Test duration overridden to 60 seconds -2025-06-05 14:14:35,029 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 20/s) -2025-06-05 14:14:35,029 - tests.orchestrator - INFO - Starting orchestrator test for 60 seconds -2025-06-05 14:14:35,218 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 14:14:35,219 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 14:14:35,219 - tests.orchestrator - INFO - Creating PVC: test-pvc-0673a51e -2025-06-05 14:14:35,294 - tests.orchestrator - INFO - Created PVC: test-pvc-0673a51e -2025-06-05 14:14:35,295 - tests.orchestrator - INFO - Waiting for PVC test-pvc-0673a51e to be bound -2025-06-05 14:14:37,429 - tests.orchestrator - INFO - PVC test-pvc-0673a51e is bound -2025-06-05 14:14:37,429 - tests.orchestrator - INFO - Creating PVC: test-pvc-ad878159 -2025-06-05 14:14:37,497 - tests.orchestrator - INFO - Created PVC: test-pvc-ad878159 -2025-06-05 14:14:37,497 - tests.orchestrator - INFO - Waiting for PVC test-pvc-ad878159 to be bound -2025-06-05 14:14:39,631 - tests.orchestrator - INFO - PVC test-pvc-ad878159 is bound -2025-06-05 14:14:39,632 - tests.orchestrator - INFO - Creating pod test-pod-97d4c158 on PVC test-pvc-ad878159 -2025-06-05 14:14:39,709 - tests.orchestrator - INFO - Created pod: test-pod-97d4c158 using PVC: test-pvc-ad878159 -2025-06-05 14:14:39,710 - tests.orchestrator - INFO - Waiting for pod test-pod-97d4c158 to be ready -2025-06-05 14:15:39,755 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-97d4c158 to be ready after 60s -2025-06-05 14:15:39,755 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-97d4c158 to be ready -2025-06-05 14:15:39,756 - tests.orchestrator - INFO - Creating pod test-pod-27dfc967 on PVC test-pvc-ad878159 -2025-06-05 14:15:39,836 - tests.orchestrator - INFO - Created pod: test-pod-27dfc967 using PVC: test-pvc-ad878159 -2025-06-05 14:15:39,837 - tests.orchestrator - INFO - Waiting for pod test-pod-27dfc967 to be ready -2025-06-05 14:16:39,914 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-27dfc967 to be ready after 60s -2025-06-05 14:16:39,915 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-27dfc967 to be ready -2025-06-05 14:16:39,915 - tests.orchestrator - INFO - Creating pod test-pod-41d8fee0 on PVC test-pvc-0673a51e -2025-06-05 14:16:39,996 - tests.orchestrator - INFO - Created pod: test-pod-41d8fee0 using PVC: test-pvc-0673a51e -2025-06-05 14:16:39,996 - tests.orchestrator - INFO - Waiting for pod test-pod-41d8fee0 to be ready -2025-06-05 14:17:42,051 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-41d8fee0 to be ready after 60s -2025-06-05 14:17:42,051 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-41d8fee0 to be ready -2025-06-05 14:17:42,051 - tests.orchestrator - INFO - Testing read/write between pods test-pod-97d4c158 and test-pod-27dfc967 sharing PVC test-pvc-ad878159 -2025-06-05 14:17:42,118 - tests.orchestrator - ERROR - Failed in read/write verification: (400) -Reason: Bad Request -HTTP response headers: HTTPHeaderDict({'Audit-Id': '82f9299f-d78b-4a35-8447-4b9382646785', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 14:17:42 GMT', 'Content-Length': '166'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-97d4c158 does not have a host assigned","reason":"BadRequest","code":400} - - -2025-06-05 14:17:42,118 - tests.orchestrator - INFO - Running specific scenario: _scenario_concurrent_pvc -2025-06-05 14:17:42,118 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations -2025-06-05 14:17:42,118 - tests.orchestrator - INFO - Creating 5 PVCs in quick succession -2025-06-05 14:17:42,197 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-af92f77f -2025-06-05 14:17:42,197 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-af92f77f to be bound -2025-06-05 14:17:44,333 - tests.orchestrator - INFO - PVC concurrent-pvc-af92f77f is bound -2025-06-05 14:17:44,403 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-2a91a331 -2025-06-05 14:17:44,403 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-2a91a331 to be bound -2025-06-05 14:17:46,537 - tests.orchestrator - INFO - PVC concurrent-pvc-2a91a331 is bound -2025-06-05 14:17:46,606 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-b13c79e0 -2025-06-05 14:17:46,606 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-b13c79e0 to be bound -2025-06-05 14:17:48,740 - tests.orchestrator - INFO - PVC concurrent-pvc-b13c79e0 is bound -2025-06-05 14:17:48,810 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-2e52864c -2025-06-05 14:17:48,810 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-2e52864c to be bound -2025-06-05 14:17:50,943 - tests.orchestrator - INFO - PVC concurrent-pvc-2e52864c is bound -2025-06-05 14:17:51,013 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-83f53c18 -2025-06-05 14:17:51,014 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-83f53c18 to be bound -2025-06-05 14:17:53,147 - tests.orchestrator - INFO - PVC concurrent-pvc-83f53c18 is bound -2025-06-05 14:17:53,147 - tests.orchestrator - INFO - Successfully created 5 PVCs -2025-06-05 14:17:53,147 - tests.orchestrator - INFO - Creating pod test-pod-783e20b8 on PVC concurrent-pvc-2a91a331 -2025-06-05 14:17:53,221 - tests.orchestrator - INFO - Created pod: test-pod-783e20b8 using PVC: concurrent-pvc-2a91a331 -2025-06-05 14:17:53,221 - tests.orchestrator - INFO - Waiting for pod test-pod-783e20b8 to be ready -2025-06-05 14:18:53,279 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-783e20b8 to be ready after 60s -2025-06-05 14:18:53,279 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-783e20b8 to be ready -2025-06-05 14:18:53,279 - tests.orchestrator - INFO - Creating pod test-pod-206a8855 on PVC concurrent-pvc-b13c79e0 -2025-06-05 14:18:53,362 - tests.orchestrator - INFO - Created pod: test-pod-206a8855 using PVC: concurrent-pvc-b13c79e0 -2025-06-05 14:18:53,362 - tests.orchestrator - INFO - Waiting for pod test-pod-206a8855 to be ready -2025-06-05 14:19:55,415 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-206a8855 to be ready after 60s -2025-06-05 14:19:55,416 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-206a8855 to be ready -2025-06-05 14:19:55,416 - tests.orchestrator - INFO - Creating pod test-pod-7df5d0f2 on PVC concurrent-pvc-af92f77f -2025-06-05 14:19:55,499 - tests.orchestrator - INFO - Created pod: test-pod-7df5d0f2 using PVC: concurrent-pvc-af92f77f -2025-06-05 14:19:55,499 - tests.orchestrator - INFO - Waiting for pod test-pod-7df5d0f2 to be ready -2025-06-05 14:20:57,532 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-7df5d0f2 to be ready after 60s -2025-06-05 14:20:57,532 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-7df5d0f2 to be ready -2025-06-05 14:20:57,532 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-2e52864c -2025-06-05 14:20:57,607 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-2e52864c to be deleted -2025-06-05 14:20:57,673 - tests.orchestrator - INFO - PVC concurrent-pvc-2e52864c has been deleted -2025-06-05 14:20:57,674 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-2e52864c -2025-06-05 14:20:57,674 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-af92f77f -2025-06-05 14:20:57,675 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-af92f77f -2025-06-05 14:20:57,675 - tests.orchestrator - INFO - Deleting pod: test-pod-7df5d0f2 from PVC: concurrent-pvc-af92f77f -2025-06-05 14:20:57,761 - tests.orchestrator - INFO - Waiting for pod test-pod-7df5d0f2 to be deleted -2025-06-05 14:20:57,827 - tests.orchestrator - INFO - Pod test-pod-7df5d0f2 has been deleted -2025-06-05 14:20:57,827 - tests.orchestrator - INFO - Deleted pod: test-pod-7df5d0f2 -2025-06-05 14:20:57,901 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-af92f77f to be deleted -2025-06-05 14:20:57,969 - tests.orchestrator - INFO - PVC concurrent-pvc-af92f77f has been deleted -2025-06-05 14:20:57,969 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-af92f77f -2025-06-05 14:20:57,969 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-83f53c18 -2025-06-05 14:20:58,041 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-83f53c18 to be deleted -2025-06-05 14:20:58,108 - tests.orchestrator - INFO - PVC concurrent-pvc-83f53c18 has been deleted -2025-06-05 14:20:58,108 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-83f53c18 -2025-06-05 14:20:58,108 - tests.orchestrator - INFO - Rapid PVC scenario completed -2025-06-05 14:20:58,108 - tests.orchestrator - INFO - Deleting pod: test-pod-97d4c158 from PVC: test-pvc-ad878159 -2025-06-05 14:20:58,188 - tests.orchestrator - INFO - Waiting for pod test-pod-97d4c158 to be deleted -2025-06-05 14:20:58,254 - tests.orchestrator - INFO - Pod test-pod-97d4c158 has been deleted -2025-06-05 14:20:58,254 - tests.orchestrator - INFO - Deleted pod: test-pod-97d4c158 -2025-06-05 14:20:58,254 - tests.orchestrator - INFO - Deleting PVC: test-pvc-ad878159 -2025-06-05 14:20:58,255 - tests.orchestrator - INFO - Deleting 1 pods using PVC test-pvc-ad878159 -2025-06-05 14:20:58,255 - tests.orchestrator - INFO - Deleting pod: test-pod-27dfc967 from PVC: test-pvc-ad878159 -2025-06-05 14:20:58,341 - tests.orchestrator - INFO - Waiting for pod test-pod-27dfc967 to be deleted -2025-06-05 14:20:58,407 - tests.orchestrator - INFO - Pod test-pod-27dfc967 has been deleted -2025-06-05 14:20:58,407 - tests.orchestrator - INFO - Deleted pod: test-pod-27dfc967 -2025-06-05 14:20:58,479 - tests.orchestrator - INFO - Waiting for PVC test-pvc-ad878159 to be deleted -2025-06-05 14:20:58,546 - tests.orchestrator - INFO - PVC test-pvc-ad878159 has been deleted -2025-06-05 14:20:58,546 - tests.orchestrator - INFO - Deleted PVC: test-pvc-ad878159 -2025-06-05 14:20:58,546 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations -2025-06-05 14:20:58,546 - tests.orchestrator - INFO - Test completed in 383.52 seconds -2025-06-05 14:20:58,547 - tests.orchestrator - INFO - Cleaning up resources... -2025-06-05 14:20:58,547 - tests.orchestrator - INFO - Deleting pod: test-pod-41d8fee0 from PVC: test-pvc-0673a51e -2025-06-05 14:20:58,633 - tests.orchestrator - INFO - Waiting for pod test-pod-41d8fee0 to be deleted -2025-06-05 14:20:58,700 - tests.orchestrator - INFO - Pod test-pod-41d8fee0 has been deleted -2025-06-05 14:20:58,700 - tests.orchestrator - INFO - Deleted pod: test-pod-41d8fee0 -2025-06-05 14:20:58,700 - tests.orchestrator - INFO - Deleting pod: test-pod-783e20b8 from PVC: concurrent-pvc-2a91a331 -2025-06-05 14:20:58,781 - tests.orchestrator - INFO - Waiting for pod test-pod-783e20b8 to be deleted -2025-06-05 14:20:58,847 - tests.orchestrator - INFO - Pod test-pod-783e20b8 has been deleted -2025-06-05 14:20:58,847 - tests.orchestrator - INFO - Deleted pod: test-pod-783e20b8 -2025-06-05 14:20:58,847 - tests.orchestrator - INFO - Deleting pod: test-pod-206a8855 from PVC: concurrent-pvc-b13c79e0 -2025-06-05 14:20:58,925 - tests.orchestrator - INFO - Waiting for pod test-pod-206a8855 to be deleted -2025-06-05 14:20:58,991 - tests.orchestrator - INFO - Pod test-pod-206a8855 has been deleted -2025-06-05 14:20:58,992 - tests.orchestrator - INFO - Deleted pod: test-pod-206a8855 -2025-06-05 14:20:58,992 - tests.orchestrator - INFO - Deleting PVC: test-pvc-0673a51e -2025-06-05 14:20:59,063 - tests.orchestrator - INFO - Waiting for PVC test-pvc-0673a51e to be deleted -2025-06-05 14:20:59,130 - tests.orchestrator - INFO - PVC test-pvc-0673a51e has been deleted -2025-06-05 14:20:59,130 - tests.orchestrator - INFO - Deleted PVC: test-pvc-0673a51e -2025-06-05 14:20:59,130 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-2a91a331 -2025-06-05 14:20:59,204 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-2a91a331 to be deleted -2025-06-05 14:20:59,270 - tests.orchestrator - INFO - PVC concurrent-pvc-2a91a331 has been deleted -2025-06-05 14:20:59,270 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-2a91a331 -2025-06-05 14:20:59,270 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-b13c79e0 -2025-06-05 14:20:59,344 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-b13c79e0 to be deleted -2025-06-05 14:20:59,410 - tests.orchestrator - INFO - PVC concurrent-pvc-b13c79e0 has been deleted -2025-06-05 14:20:59,410 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-b13c79e0 -2025-06-05 14:20:59,410 - tests.orchestrator - INFO - Cleanup completed -2025-06-05 14:20:59,410 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-05 14:20:59,410 - tests.orchestrator - INFO - --- Operations --- -2025-06-05 14:20:59,410 - tests.orchestrator - INFO - create_pvc: 7 succeeded, 0 failed (100.0%) -2025-06-05 14:20:59,410 - tests.orchestrator - INFO - attach_pod: 6 succeeded, 0 failed (100.0%) -2025-06-05 14:20:59,410 - tests.orchestrator - INFO - delete_pod: 6 succeeded, 0 failed (100.0%) -2025-06-05 14:20:59,410 - tests.orchestrator - INFO - delete_pvc: 7 succeeded, 0 failed (100.0%) -2025-06-05 14:20:59,410 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) -2025-06-05 14:20:59,411 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) -2025-06-05 14:20:59,411 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-05 14:20:59,411 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-05 14:20:59,411 - tests.orchestrator - INFO - many_to_one: No runs -2025-06-05 14:20:59,411 - tests.orchestrator - INFO - one_to_one: No runs -2025-06-05 14:20:59,411 - tests.orchestrator - INFO - concurrent_pvc: 1 succeeded, 0 failed out of 1 runs (100.0%) -2025-06-05 14:20:59,411 - tests.orchestrator - INFO - ========================================= -2025-06-05 14:20:59,411 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250605_142059.json -2025-06-05 14:20:59,453 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250605_142059_20250605_142059.json -2025-06-05 14:20:59,453 - __main__ - INFO - All tests completed successfully -2025-06-05 14:39:25,645 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-05 14:39:25,645 - __main__ - INFO - Verifying AWS credentials -2025-06-05 14:39:25,646 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 14:39:25,647 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 14:39:25,647 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 14:39:25,647 - __main__ - INFO - Running orchestrator stress test suite -2025-06-05 14:39:25,647 - __main__ - INFO - Starting orchestrator for 300 seconds -2025-06-05 14:39:25,647 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-05 14:39:26,410 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 14:39:26,657 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 14:39:26,657 - __main__ - INFO - Test duration overridden to 300 seconds -2025-06-05 14:39:26,657 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) -2025-06-05 14:39:26,657 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-05 14:39:26,852 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 14:39:26,853 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 14:39:26,853 - tests.orchestrator - INFO - Creating PVC: test-pvc-e5985cb1 -2025-06-05 14:39:26,928 - tests.orchestrator - INFO - Created PVC: test-pvc-e5985cb1 -2025-06-05 14:39:26,928 - tests.orchestrator - INFO - Waiting for PVC test-pvc-e5985cb1 to be bound -2025-06-05 14:39:29,061 - tests.orchestrator - INFO - PVC test-pvc-e5985cb1 is bound -2025-06-05 14:39:29,061 - tests.orchestrator - INFO - Creating PVC: test-pvc-83426f91 -2025-06-05 14:39:29,130 - tests.orchestrator - INFO - Created PVC: test-pvc-83426f91 -2025-06-05 14:39:29,131 - tests.orchestrator - INFO - Waiting for PVC test-pvc-83426f91 to be bound -2025-06-05 14:39:31,262 - tests.orchestrator - INFO - PVC test-pvc-83426f91 is bound -2025-06-05 14:39:31,263 - tests.orchestrator - INFO - Creating pod test-pod-d86424ba on PVC test-pvc-83426f91 -2025-06-05 14:39:31,342 - tests.orchestrator - INFO - Created pod: test-pod-d86424ba using PVC: test-pvc-83426f91 -2025-06-05 14:39:31,342 - tests.orchestrator - INFO - Waiting for pod test-pod-d86424ba to be ready -2025-06-05 14:40:31,371 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-d86424ba to be ready after 60s -2025-06-05 14:40:31,371 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-d86424ba to be ready -2025-06-05 14:40:31,371 - tests.orchestrator - INFO - Creating pod test-pod-b7e9d0a0 on PVC test-pvc-e5985cb1 -2025-06-05 14:40:31,454 - tests.orchestrator - INFO - Created pod: test-pod-b7e9d0a0 using PVC: test-pvc-e5985cb1 -2025-06-05 14:40:31,455 - tests.orchestrator - INFO - Waiting for pod test-pod-b7e9d0a0 to be ready -2025-06-05 14:41:31,485 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b7e9d0a0 to be ready after 60s -2025-06-05 14:41:31,485 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b7e9d0a0 to be ready -2025-06-05 14:41:31,485 - tests.orchestrator - INFO - Creating pod test-pod-c947d819 on PVC test-pvc-e5985cb1 -2025-06-05 14:41:31,586 - tests.orchestrator - INFO - Created pod: test-pod-c947d819 using PVC: test-pvc-e5985cb1 -2025-06-05 14:41:31,586 - tests.orchestrator - INFO - Waiting for pod test-pod-c947d819 to be ready -2025-06-05 14:42:33,647 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-c947d819 to be ready after 60s -2025-06-05 14:42:33,647 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-c947d819 to be ready -2025-06-05 14:42:33,647 - tests.orchestrator - INFO - Testing read/write between pods test-pod-b7e9d0a0 and test-pod-c947d819 sharing PVC test-pvc-e5985cb1 -2025-06-05 14:42:33,714 - tests.orchestrator - ERROR - Failed in read/write verification: (400) -Reason: Bad Request -HTTP response headers: HTTPHeaderDict({'Audit-Id': '8b1165bc-48b9-433e-8654-68c6095da9bd', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 14:42:33 GMT', 'Content-Length': '166'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-b7e9d0a0 does not have a host assigned","reason":"BadRequest","code":400} - - -2025-06-05 14:42:33,714 - tests.orchestrator - INFO - Running specific scenario: _scenario_concurrent_pvc -2025-06-05 14:42:33,714 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations -2025-06-05 14:42:33,714 - tests.orchestrator - INFO - Creating 7 PVCs in quick succession -2025-06-05 14:42:33,792 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-f6a663b2 -2025-06-05 14:42:33,792 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-f6a663b2 to be bound -2025-06-05 14:42:35,924 - tests.orchestrator - INFO - PVC concurrent-pvc-f6a663b2 is bound -2025-06-05 14:42:35,993 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-903eb13a -2025-06-05 14:42:35,994 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-903eb13a to be bound -2025-06-05 14:42:38,128 - tests.orchestrator - INFO - PVC concurrent-pvc-903eb13a is bound -2025-06-05 14:42:38,196 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-ba7e5e4d -2025-06-05 14:42:38,196 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-ba7e5e4d to be bound -2025-06-05 14:42:40,331 - tests.orchestrator - INFO - PVC concurrent-pvc-ba7e5e4d is bound -2025-06-05 14:42:40,397 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-5b29e9cd -2025-06-05 14:42:40,397 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-5b29e9cd to be bound -2025-06-05 14:42:42,533 - tests.orchestrator - INFO - PVC concurrent-pvc-5b29e9cd is bound -2025-06-05 14:42:42,603 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-248d7584 -2025-06-05 14:42:42,603 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-248d7584 to be bound -2025-06-05 14:42:44,740 - tests.orchestrator - INFO - PVC concurrent-pvc-248d7584 is bound -2025-06-05 14:42:44,808 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-0aa045b0 -2025-06-05 14:42:44,808 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-0aa045b0 to be bound -2025-06-05 14:42:46,944 - tests.orchestrator - INFO - PVC concurrent-pvc-0aa045b0 is bound -2025-06-05 14:42:47,013 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-b0004a21 -2025-06-05 14:42:47,013 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-b0004a21 to be bound -2025-06-05 14:42:49,147 - tests.orchestrator - INFO - PVC concurrent-pvc-b0004a21 is bound -2025-06-05 14:42:49,148 - tests.orchestrator - INFO - Successfully created 7 PVCs -2025-06-05 14:42:49,148 - tests.orchestrator - INFO - Creating pod test-pod-edd264c5 on PVC concurrent-pvc-0aa045b0 -2025-06-05 14:42:49,220 - tests.orchestrator - INFO - Created pod: test-pod-edd264c5 using PVC: concurrent-pvc-0aa045b0 -2025-06-05 14:42:49,220 - tests.orchestrator - INFO - Waiting for pod test-pod-edd264c5 to be ready -2025-06-05 14:43:49,221 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-edd264c5 to be ready after 60s -2025-06-05 14:43:49,221 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-edd264c5 to be ready -2025-06-05 14:43:49,221 - tests.orchestrator - INFO - Creating pod test-pod-9f0f38ef on PVC concurrent-pvc-248d7584 -2025-06-05 14:43:49,306 - tests.orchestrator - INFO - Created pod: test-pod-9f0f38ef using PVC: concurrent-pvc-248d7584 -2025-06-05 14:43:49,306 - tests.orchestrator - INFO - Waiting for pod test-pod-9f0f38ef to be ready -2025-06-05 14:44:51,350 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9f0f38ef to be ready after 60s -2025-06-05 14:44:51,351 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9f0f38ef to be ready -2025-06-05 14:44:51,351 - tests.orchestrator - INFO - Creating pod test-pod-6bde4264 on PVC concurrent-pvc-903eb13a -2025-06-05 14:44:51,437 - tests.orchestrator - INFO - Created pod: test-pod-6bde4264 using PVC: concurrent-pvc-903eb13a -2025-06-05 14:44:51,437 - tests.orchestrator - INFO - Waiting for pod test-pod-6bde4264 to be ready -2025-06-05 14:51:32,599 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-05 14:51:32,599 - __main__ - INFO - Verifying AWS credentials -2025-06-05 14:51:32,600 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 14:51:32,601 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 14:51:32,601 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 14:51:32,601 - __main__ - INFO - Running orchestrator stress test suite -2025-06-05 14:51:32,601 - __main__ - INFO - Starting orchestrator for 300 seconds -2025-06-05 14:51:32,602 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-05 14:51:33,341 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 14:51:33,563 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 14:51:33,563 - __main__ - INFO - Test duration overridden to 300 seconds -2025-06-05 14:51:33,563 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) -2025-06-05 14:51:33,563 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-05 14:51:33,753 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 14:51:33,753 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 14:51:33,754 - tests.orchestrator - INFO - Creating PVC: test-pvc-49392d7c -2025-06-05 14:51:33,831 - tests.orchestrator - INFO - Created PVC: test-pvc-49392d7c -2025-06-05 14:51:33,831 - tests.orchestrator - INFO - Waiting for PVC test-pvc-49392d7c to be bound -2025-06-05 14:51:35,964 - tests.orchestrator - INFO - PVC test-pvc-49392d7c is bound -2025-06-05 14:51:35,966 - tests.orchestrator - INFO - Creating PVC: test-pvc-5e912838 -2025-06-05 14:51:36,038 - tests.orchestrator - INFO - Created PVC: test-pvc-5e912838 -2025-06-05 14:51:36,038 - tests.orchestrator - INFO - Waiting for PVC test-pvc-5e912838 to be bound -2025-06-05 14:51:38,171 - tests.orchestrator - INFO - PVC test-pvc-5e912838 is bound -2025-06-05 14:51:38,171 - tests.orchestrator - INFO - Creating pod test-pod-a16fe7e2 on PVC test-pvc-5e912838 -2025-06-05 14:51:38,249 - tests.orchestrator - INFO - Created pod: test-pod-a16fe7e2 using PVC: test-pvc-5e912838 -2025-06-05 14:51:38,250 - tests.orchestrator - INFO - Waiting for pod test-pod-a16fe7e2 to be ready -2025-06-05 14:52:40,287 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a16fe7e2 to be ready after 60s -2025-06-05 14:52:40,287 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a16fe7e2 to be ready -2025-06-05 14:52:40,288 - tests.orchestrator - INFO - Creating pod test-pod-6cc653e7 on PVC test-pvc-49392d7c -2025-06-05 14:52:40,370 - tests.orchestrator - INFO - Created pod: test-pod-6cc653e7 using PVC: test-pvc-49392d7c -2025-06-05 14:52:40,370 - tests.orchestrator - INFO - Waiting for pod test-pod-6cc653e7 to be ready -2025-06-05 14:53:42,411 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-6cc653e7 to be ready after 60s -2025-06-05 14:53:42,411 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-6cc653e7 to be ready -2025-06-05 14:53:42,411 - tests.orchestrator - INFO - Creating pod test-pod-807f64e5 on PVC test-pvc-49392d7c -2025-06-05 14:53:42,494 - tests.orchestrator - INFO - Created pod: test-pod-807f64e5 using PVC: test-pvc-49392d7c -2025-06-05 14:53:42,494 - tests.orchestrator - INFO - Waiting for pod test-pod-807f64e5 to be ready -2025-06-05 14:54:44,529 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-807f64e5 to be ready after 60s -2025-06-05 14:54:44,529 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-807f64e5 to be ready -2025-06-05 14:54:44,529 - tests.orchestrator - INFO - Testing read/write between pods test-pod-807f64e5 and test-pod-6cc653e7 sharing PVC test-pvc-49392d7c -2025-06-05 14:54:44,596 - tests.orchestrator - ERROR - Failed in read/write verification: (400) -Reason: Bad Request -HTTP response headers: HTTPHeaderDict({'Audit-Id': '12fe6240-5ebe-4564-8452-9306b897c940', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 14:54:44 GMT', 'Content-Length': '166'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-807f64e5 does not have a host assigned","reason":"BadRequest","code":400} - - -2025-06-05 14:54:44,596 - tests.orchestrator - INFO - Running specific scenario: _scenario_many_to_one -2025-06-05 14:54:44,597 - tests.orchestrator - INFO - Running scenario: Many pods to one PVC -2025-06-05 14:54:44,678 - tests.orchestrator - INFO - Waiting for PVC many2one-7a10cfd9 to be bound -2025-06-05 14:54:46,810 - tests.orchestrator - INFO - PVC many2one-7a10cfd9 is bound -2025-06-05 14:54:46,811 - tests.orchestrator - INFO - Creating 3 pods for the same PVC -2025-06-05 14:54:46,811 - tests.orchestrator - INFO - Creating pod test-pod-6d648a79 on PVC many2one-7a10cfd9 -2025-06-05 14:54:46,884 - tests.orchestrator - INFO - Created pod: test-pod-6d648a79 using PVC: many2one-7a10cfd9 -2025-06-05 14:54:46,884 - tests.orchestrator - INFO - Waiting for pod test-pod-6d648a79 to be ready -2025-06-05 14:55:48,914 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-6d648a79 to be ready after 60s -2025-06-05 14:55:48,915 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-6d648a79 to be ready -2025-06-05 14:55:48,915 - tests.orchestrator - INFO - Creating pod test-pod-264dcb36 on PVC many2one-7a10cfd9 -2025-06-05 14:55:48,997 - tests.orchestrator - INFO - Created pod: test-pod-264dcb36 using PVC: many2one-7a10cfd9 -2025-06-05 14:55:48,997 - tests.orchestrator - INFO - Waiting for pod test-pod-264dcb36 to be ready -2025-06-05 14:56:49,023 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-264dcb36 to be ready after 60s -2025-06-05 14:56:49,024 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-264dcb36 to be ready -2025-06-05 14:56:49,024 - tests.orchestrator - INFO - Creating pod test-pod-b6a62203 on PVC many2one-7a10cfd9 -2025-06-05 14:56:49,106 - tests.orchestrator - INFO - Created pod: test-pod-b6a62203 using PVC: many2one-7a10cfd9 -2025-06-05 14:56:49,106 - tests.orchestrator - INFO - Waiting for pod test-pod-b6a62203 to be ready -2025-06-05 14:57:49,129 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b6a62203 to be ready after 60s -2025-06-05 14:57:49,129 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b6a62203 to be ready -2025-06-05 14:57:49,129 - tests.orchestrator - WARNING - Failed to create enough pods for scenario, only created 0 -2025-06-05 14:57:49,130 - tests.orchestrator - INFO - Deleting pod: test-pod-6cc653e7 from PVC: test-pvc-49392d7c -2025-06-05 14:57:49,207 - tests.orchestrator - INFO - Waiting for pod test-pod-6cc653e7 to be deleted -2025-06-05 14:57:49,272 - tests.orchestrator - INFO - Pod test-pod-6cc653e7 has been deleted -2025-06-05 14:57:49,272 - tests.orchestrator - INFO - Deleted pod: test-pod-6cc653e7 -2025-06-05 14:57:49,272 - tests.orchestrator - INFO - Deleting PVC: many2one-7a10cfd9 -2025-06-05 14:57:49,272 - tests.orchestrator - INFO - Deleting 3 pods using PVC many2one-7a10cfd9 -2025-06-05 14:57:49,272 - tests.orchestrator - INFO - Deleting pod: test-pod-6d648a79 from PVC: many2one-7a10cfd9 -2025-06-05 14:57:49,348 - tests.orchestrator - INFO - Waiting for pod test-pod-6d648a79 to be deleted -2025-06-05 14:57:49,414 - tests.orchestrator - INFO - Pod test-pod-6d648a79 has been deleted -2025-06-05 14:57:49,415 - tests.orchestrator - INFO - Deleted pod: test-pod-6d648a79 -2025-06-05 14:57:49,415 - tests.orchestrator - INFO - Deleting pod: test-pod-264dcb36 from PVC: many2one-7a10cfd9 -2025-06-05 14:57:49,498 - tests.orchestrator - INFO - Waiting for pod test-pod-264dcb36 to be deleted -2025-06-05 14:57:49,563 - tests.orchestrator - INFO - Pod test-pod-264dcb36 has been deleted -2025-06-05 14:57:49,564 - tests.orchestrator - INFO - Deleted pod: test-pod-264dcb36 -2025-06-05 14:57:49,564 - tests.orchestrator - INFO - Deleting pod: test-pod-b6a62203 from PVC: many2one-7a10cfd9 -2025-06-05 14:57:49,643 - tests.orchestrator - INFO - Waiting for pod test-pod-b6a62203 to be deleted -2025-06-05 14:57:49,710 - tests.orchestrator - INFO - Pod test-pod-b6a62203 has been deleted -2025-06-05 14:57:49,710 - tests.orchestrator - INFO - Deleted pod: test-pod-b6a62203 -2025-06-05 14:57:49,782 - tests.orchestrator - INFO - Waiting for PVC many2one-7a10cfd9 to be deleted -2025-06-05 14:57:49,848 - tests.orchestrator - INFO - PVC many2one-7a10cfd9 has been deleted -2025-06-05 14:57:49,848 - tests.orchestrator - INFO - Deleted PVC: many2one-7a10cfd9 -2025-06-05 14:57:49,848 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations -2025-06-05 14:57:49,849 - tests.orchestrator - INFO - Test completed in 376.28 seconds -2025-06-05 14:57:49,849 - tests.orchestrator - INFO - Cleaning up resources... -2025-06-05 14:57:49,849 - tests.orchestrator - INFO - Deleting pod: test-pod-807f64e5 from PVC: test-pvc-49392d7c -2025-06-05 14:57:49,935 - tests.orchestrator - INFO - Waiting for pod test-pod-807f64e5 to be deleted -2025-06-05 14:57:50,000 - tests.orchestrator - INFO - Pod test-pod-807f64e5 has been deleted -2025-06-05 14:57:50,001 - tests.orchestrator - INFO - Deleted pod: test-pod-807f64e5 -2025-06-05 14:57:50,001 - tests.orchestrator - INFO - Deleting pod: test-pod-a16fe7e2 from PVC: test-pvc-5e912838 -2025-06-05 14:57:50,081 - tests.orchestrator - INFO - Waiting for pod test-pod-a16fe7e2 to be deleted -2025-06-05 14:57:50,147 - tests.orchestrator - INFO - Pod test-pod-a16fe7e2 has been deleted -2025-06-05 14:57:50,147 - tests.orchestrator - INFO - Deleted pod: test-pod-a16fe7e2 -2025-06-05 14:57:50,147 - tests.orchestrator - INFO - Deleting PVC: test-pvc-49392d7c -2025-06-05 14:57:50,221 - tests.orchestrator - INFO - Waiting for PVC test-pvc-49392d7c to be deleted -2025-06-05 14:57:50,287 - tests.orchestrator - INFO - PVC test-pvc-49392d7c has been deleted -2025-06-05 14:57:50,288 - tests.orchestrator - INFO - Deleted PVC: test-pvc-49392d7c -2025-06-05 14:57:50,288 - tests.orchestrator - INFO - Deleting PVC: test-pvc-5e912838 -2025-06-05 14:57:50,360 - tests.orchestrator - INFO - Waiting for PVC test-pvc-5e912838 to be deleted -2025-06-05 14:57:50,426 - tests.orchestrator - INFO - PVC test-pvc-5e912838 has been deleted -2025-06-05 14:57:50,426 - tests.orchestrator - INFO - Deleted PVC: test-pvc-5e912838 -2025-06-05 14:57:50,426 - tests.orchestrator - INFO - Cleanup completed -2025-06-05 14:57:50,426 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-05 14:57:50,426 - tests.orchestrator - INFO - --- Operations --- -2025-06-05 14:57:50,426 - tests.orchestrator - INFO - create_pvc: 2 succeeded, 0 failed (100.0%) -2025-06-05 14:57:50,426 - tests.orchestrator - INFO - attach_pod: 6 succeeded, 0 failed (100.0%) -2025-06-05 14:57:50,427 - tests.orchestrator - INFO - delete_pod: 6 succeeded, 0 failed (100.0%) -2025-06-05 14:57:50,427 - tests.orchestrator - INFO - delete_pvc: 3 succeeded, 0 failed (100.0%) -2025-06-05 14:57:50,427 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) -2025-06-05 14:57:50,427 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) -2025-06-05 14:57:50,427 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-05 14:57:50,427 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-05 14:57:50,427 - tests.orchestrator - INFO - many_to_one: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-05 14:57:50,427 - tests.orchestrator - INFO - one_to_one: No runs -2025-06-05 14:57:50,427 - tests.orchestrator - INFO - concurrent_pvc: No runs -2025-06-05 14:57:50,427 - tests.orchestrator - INFO - ========================================= -2025-06-05 14:57:50,428 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250605_145750.json -2025-06-05 14:57:50,470 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250605_145750_20250605_145750.json -2025-06-05 14:57:50,470 - __main__ - INFO - All tests completed successfully -2025-06-05 14:59:37,084 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-05 14:59:37,084 - __main__ - INFO - Verifying AWS credentials -2025-06-05 14:59:37,085 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 14:59:37,085 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 14:59:37,086 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 14:59:37,086 - __main__ - INFO - Running orchestrator stress test suite -2025-06-05 14:59:37,086 - __main__ - INFO - Starting orchestrator for 300 seconds -2025-06-05 14:59:37,086 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-05 14:59:37,825 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 14:59:38,037 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 14:59:38,038 - __main__ - INFO - Test duration overridden to 300 seconds -2025-06-05 14:59:38,038 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) -2025-06-05 14:59:38,038 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-05 14:59:38,227 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 14:59:38,227 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 14:59:38,227 - tests.orchestrator - INFO - Creating PVC: test-pvc-fe252ee6 -2025-06-05 14:59:38,302 - tests.orchestrator - INFO - Created PVC: test-pvc-fe252ee6 -2025-06-05 14:59:38,303 - tests.orchestrator - INFO - Waiting for PVC test-pvc-fe252ee6 to be bound -2025-06-05 14:59:40,435 - tests.orchestrator - INFO - PVC test-pvc-fe252ee6 is bound -2025-06-05 14:59:40,435 - tests.orchestrator - INFO - Creating PVC: test-pvc-4a7f89b4 -2025-06-05 14:59:40,502 - tests.orchestrator - INFO - Created PVC: test-pvc-4a7f89b4 -2025-06-05 14:59:40,502 - tests.orchestrator - INFO - Waiting for PVC test-pvc-4a7f89b4 to be bound -2025-06-05 14:59:42,634 - tests.orchestrator - INFO - PVC test-pvc-4a7f89b4 is bound -2025-06-05 14:59:42,635 - tests.orchestrator - INFO - Creating pod test-pod-5311b1cc on PVC test-pvc-4a7f89b4 -2025-06-05 14:59:42,712 - tests.orchestrator - INFO - Created pod: test-pod-5311b1cc using PVC: test-pvc-4a7f89b4 -2025-06-05 14:59:42,712 - tests.orchestrator - INFO - Waiting for pod test-pod-5311b1cc to be ready -2025-06-05 15:00:44,719 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-5311b1cc to be ready after 60s -2025-06-05 15:00:44,719 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-5311b1cc to be ready -2025-06-05 15:00:44,720 - tests.orchestrator - INFO - Creating pod test-pod-71d7f585 on PVC test-pvc-fe252ee6 -2025-06-05 15:00:44,803 - tests.orchestrator - INFO - Created pod: test-pod-71d7f585 using PVC: test-pvc-fe252ee6 -2025-06-05 15:00:44,803 - tests.orchestrator - INFO - Waiting for pod test-pod-71d7f585 to be ready -2025-06-05 15:01:46,811 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-71d7f585 to be ready after 60s -2025-06-05 15:01:46,811 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-71d7f585 to be ready -2025-06-05 15:01:46,812 - tests.orchestrator - INFO - Creating pod test-pod-cc8d2266 on PVC test-pvc-4a7f89b4 -2025-06-05 15:01:46,893 - tests.orchestrator - INFO - Created pod: test-pod-cc8d2266 using PVC: test-pvc-4a7f89b4 -2025-06-05 15:01:46,894 - tests.orchestrator - INFO - Waiting for pod test-pod-cc8d2266 to be ready -2025-06-05 15:02:48,905 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-cc8d2266 to be ready after 60s -2025-06-05 15:02:48,905 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-cc8d2266 to be ready -2025-06-05 15:02:48,905 - tests.orchestrator - INFO - Testing read/write between pods test-pod-5311b1cc and test-pod-cc8d2266 sharing PVC test-pvc-4a7f89b4 -2025-06-05 15:02:48,971 - tests.orchestrator - ERROR - Failed in read/write verification: (400) -Reason: Bad Request -HTTP response headers: HTTPHeaderDict({'Audit-Id': '6fd01cb2-cb35-43e3-8bf2-4cba5eccfd8d', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 15:02:48 GMT', 'Content-Length': '166'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-5311b1cc does not have a host assigned","reason":"BadRequest","code":400} - - -2025-06-05 15:02:48,971 - tests.orchestrator - INFO - ============================================================ -2025-06-05 15:02:48,971 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_concurrent_pvc -2025-06-05 15:02:48,971 - tests.orchestrator - INFO - ============================================================ -2025-06-05 15:02:48,972 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations -2025-06-05 15:02:48,972 - tests.orchestrator - INFO - Creating 3 PVCs in quick succession -2025-06-05 15:02:49,049 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-02666bec -2025-06-05 15:02:49,049 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-02666bec to be bound -2025-06-05 15:02:51,181 - tests.orchestrator - INFO - PVC concurrent-pvc-02666bec is bound -2025-06-05 15:02:51,248 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-6592afb4 -2025-06-05 15:02:51,249 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-6592afb4 to be bound -2025-06-05 15:02:53,380 - tests.orchestrator - INFO - PVC concurrent-pvc-6592afb4 is bound -2025-06-05 15:02:53,448 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-2386ec8a -2025-06-05 15:02:53,449 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-2386ec8a to be bound -2025-06-05 15:02:55,580 - tests.orchestrator - INFO - PVC concurrent-pvc-2386ec8a is bound -2025-06-05 15:02:55,580 - tests.orchestrator - INFO - Successfully created 3 PVCs -2025-06-05 15:02:55,580 - tests.orchestrator - INFO - Creating pod test-pod-27ffffcb on PVC concurrent-pvc-2386ec8a -2025-06-05 15:02:55,653 - tests.orchestrator - INFO - Created pod: test-pod-27ffffcb using PVC: concurrent-pvc-2386ec8a -2025-06-05 15:02:55,653 - tests.orchestrator - INFO - Waiting for pod test-pod-27ffffcb to be ready -2025-06-05 15:03:57,662 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-27ffffcb to be ready after 60s -2025-06-05 15:03:57,662 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-27ffffcb to be ready -2025-06-05 15:03:57,662 - tests.orchestrator - INFO - Creating pod test-pod-00a03fb8 on PVC concurrent-pvc-02666bec -2025-06-05 15:03:57,743 - tests.orchestrator - INFO - Created pod: test-pod-00a03fb8 using PVC: concurrent-pvc-02666bec -2025-06-05 15:03:57,744 - tests.orchestrator - INFO - Waiting for pod test-pod-00a03fb8 to be ready -2025-06-05 15:04:59,752 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-00a03fb8 to be ready after 60s -2025-06-05 15:04:59,753 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-00a03fb8 to be ready -2025-06-05 15:04:59,753 - tests.orchestrator - INFO - Creating pod test-pod-87658b4a on PVC concurrent-pvc-6592afb4 -2025-06-05 15:04:59,832 - tests.orchestrator - INFO - Created pod: test-pod-87658b4a using PVC: concurrent-pvc-6592afb4 -2025-06-05 15:04:59,832 - tests.orchestrator - INFO - Waiting for pod test-pod-87658b4a to be ready -2025-06-05 15:06:01,847 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-87658b4a to be ready after 60s -2025-06-05 15:06:01,847 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-87658b4a to be ready -2025-06-05 15:06:01,847 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-6592afb4 -2025-06-05 15:06:01,847 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-6592afb4 -2025-06-05 15:06:01,847 - tests.orchestrator - INFO - Deleting pod: test-pod-87658b4a from PVC: concurrent-pvc-6592afb4 -2025-06-05 15:06:01,968 - tests.orchestrator - INFO - Waiting for pod test-pod-87658b4a to be deleted -2025-06-05 15:06:02,040 - tests.orchestrator - INFO - Pod test-pod-87658b4a has been deleted -2025-06-05 15:06:02,041 - tests.orchestrator - INFO - Deleted pod: test-pod-87658b4a -2025-06-05 15:06:02,126 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-6592afb4 to be deleted -2025-06-05 15:06:02,196 - tests.orchestrator - INFO - PVC concurrent-pvc-6592afb4 has been deleted -2025-06-05 15:06:02,196 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-6592afb4 -2025-06-05 15:06:02,196 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-02666bec -2025-06-05 15:06:02,196 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-02666bec -2025-06-05 15:06:02,196 - tests.orchestrator - INFO - Deleting pod: test-pod-00a03fb8 from PVC: concurrent-pvc-02666bec -2025-06-05 15:06:02,295 - tests.orchestrator - INFO - Waiting for pod test-pod-00a03fb8 to be deleted -2025-06-05 15:06:02,359 - tests.orchestrator - INFO - Pod test-pod-00a03fb8 has been deleted -2025-06-05 15:06:02,360 - tests.orchestrator - INFO - Deleted pod: test-pod-00a03fb8 -2025-06-05 15:06:02,433 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-02666bec to be deleted -2025-06-05 15:06:02,497 - tests.orchestrator - INFO - PVC concurrent-pvc-02666bec has been deleted -2025-06-05 15:06:02,498 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-02666bec -2025-06-05 15:06:02,498 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-2386ec8a -2025-06-05 15:06:02,498 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-2386ec8a -2025-06-05 15:06:02,498 - tests.orchestrator - INFO - Deleting pod: test-pod-27ffffcb from PVC: concurrent-pvc-2386ec8a -2025-06-05 15:06:02,575 - tests.orchestrator - INFO - Waiting for pod test-pod-27ffffcb to be deleted -2025-06-05 15:06:02,640 - tests.orchestrator - INFO - Pod test-pod-27ffffcb has been deleted -2025-06-05 15:06:02,640 - tests.orchestrator - INFO - Deleted pod: test-pod-27ffffcb -2025-06-05 15:06:02,713 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-2386ec8a to be deleted -2025-06-05 15:06:02,777 - tests.orchestrator - INFO - PVC concurrent-pvc-2386ec8a has been deleted -2025-06-05 15:06:02,777 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-2386ec8a -2025-06-05 15:06:02,778 - tests.orchestrator - INFO - Rapid PVC scenario completed -2025-06-05 15:06:02,778 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_concurrent_pvc -2025-06-05 15:06:02,778 - tests.orchestrator - INFO - ------------------------------------------------------------ -2025-06-05 15:06:02,778 - tests.orchestrator - INFO - Deleting pod: test-pod-71d7f585 from PVC: test-pvc-fe252ee6 -2025-06-05 15:06:02,870 - tests.orchestrator - INFO - Waiting for pod test-pod-71d7f585 to be deleted -2025-06-05 15:06:02,935 - tests.orchestrator - INFO - Pod test-pod-71d7f585 has been deleted -2025-06-05 15:06:02,935 - tests.orchestrator - INFO - Deleted pod: test-pod-71d7f585 -2025-06-05 15:06:02,935 - tests.orchestrator - INFO - Deleting PVC: test-pvc-4a7f89b4 -2025-06-05 15:06:02,935 - tests.orchestrator - INFO - Deleting 2 pods using PVC test-pvc-4a7f89b4 -2025-06-05 15:06:02,935 - tests.orchestrator - INFO - Deleting pod: test-pod-5311b1cc from PVC: test-pvc-4a7f89b4 -2025-06-05 15:06:03,025 - tests.orchestrator - INFO - Waiting for pod test-pod-5311b1cc to be deleted -2025-06-05 15:06:03,090 - tests.orchestrator - INFO - Pod test-pod-5311b1cc has been deleted -2025-06-05 15:06:03,090 - tests.orchestrator - INFO - Deleted pod: test-pod-5311b1cc -2025-06-05 15:06:03,090 - tests.orchestrator - INFO - Deleting pod: test-pod-cc8d2266 from PVC: test-pvc-4a7f89b4 -2025-06-05 15:06:03,172 - tests.orchestrator - INFO - Waiting for pod test-pod-cc8d2266 to be deleted -2025-06-05 15:06:03,238 - tests.orchestrator - INFO - Pod test-pod-cc8d2266 has been deleted -2025-06-05 15:06:03,238 - tests.orchestrator - INFO - Deleted pod: test-pod-cc8d2266 -2025-06-05 15:06:03,311 - tests.orchestrator - INFO - Waiting for PVC test-pvc-4a7f89b4 to be deleted -2025-06-05 15:06:03,376 - tests.orchestrator - INFO - PVC test-pvc-4a7f89b4 has been deleted -2025-06-05 15:06:03,376 - tests.orchestrator - INFO - Deleted PVC: test-pvc-4a7f89b4 -2025-06-05 15:06:03,376 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations -2025-06-05 15:06:03,376 - tests.orchestrator - INFO - Test completed in 385.34 seconds -2025-06-05 15:06:03,376 - tests.orchestrator - INFO - Cleaning up resources... -2025-06-05 15:06:03,376 - tests.orchestrator - INFO - Deleting PVC: test-pvc-fe252ee6 -2025-06-05 15:06:03,447 - tests.orchestrator - INFO - Waiting for PVC test-pvc-fe252ee6 to be deleted -2025-06-05 15:06:03,512 - tests.orchestrator - INFO - PVC test-pvc-fe252ee6 has been deleted -2025-06-05 15:06:03,512 - tests.orchestrator - INFO - Deleted PVC: test-pvc-fe252ee6 -2025-06-05 15:06:03,512 - tests.orchestrator - INFO - Cleanup completed -2025-06-05 15:06:03,512 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-05 15:06:03,513 - tests.orchestrator - INFO - --- Operations --- -2025-06-05 15:06:03,513 - tests.orchestrator - INFO - create_pvc: 5 succeeded, 0 failed (100.0%) -2025-06-05 15:06:03,513 - tests.orchestrator - INFO - attach_pod: 6 succeeded, 0 failed (100.0%) -2025-06-05 15:06:03,513 - tests.orchestrator - INFO - delete_pod: 6 succeeded, 0 failed (100.0%) -2025-06-05 15:06:03,513 - tests.orchestrator - INFO - delete_pvc: 5 succeeded, 0 failed (100.0%) -2025-06-05 15:06:03,513 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) -2025-06-05 15:06:03,513 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) -2025-06-05 15:06:03,513 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-05 15:06:03,513 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-05 15:06:03,513 - tests.orchestrator - INFO - many_to_one: No runs -2025-06-05 15:06:03,513 - tests.orchestrator - INFO - one_to_one: No runs -2025-06-05 15:06:03,513 - tests.orchestrator - INFO - concurrent_pvc: 1 succeeded, 0 failed out of 1 runs (100.0%) -2025-06-05 15:06:03,513 - tests.orchestrator - INFO - ========================================= -2025-06-05 15:06:03,514 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250605_150603.json -2025-06-05 15:06:03,557 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250605_150603_20250605_150603.json -2025-06-05 15:06:03,557 - __main__ - INFO - All tests completed successfully -2025-06-05 15:06:19,120 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-05 15:06:19,121 - __main__ - INFO - Verifying AWS credentials -2025-06-05 15:06:19,122 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 15:06:19,122 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 15:06:19,122 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 15:06:19,123 - __main__ - INFO - Running orchestrator stress test suite -2025-06-05 15:06:19,123 - __main__ - INFO - Starting orchestrator for 300 seconds -2025-06-05 15:06:19,123 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-05 15:06:19,870 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 15:06:20,072 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 15:06:20,072 - __main__ - INFO - Test duration overridden to 300 seconds -2025-06-05 15:06:20,072 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) -2025-06-05 15:06:20,072 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-05 15:06:20,260 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 15:06:20,261 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 15:06:20,261 - tests.orchestrator - INFO - Creating PVC: test-pvc-7aa57f8c -2025-06-05 15:06:20,336 - tests.orchestrator - INFO - Created PVC: test-pvc-7aa57f8c -2025-06-05 15:06:20,336 - tests.orchestrator - INFO - Waiting for PVC test-pvc-7aa57f8c to be bound -2025-06-05 15:06:22,470 - tests.orchestrator - INFO - PVC test-pvc-7aa57f8c is bound -2025-06-05 15:06:22,471 - tests.orchestrator - INFO - Creating PVC: test-pvc-afe1c1f0 -2025-06-05 15:06:22,538 - tests.orchestrator - INFO - Created PVC: test-pvc-afe1c1f0 -2025-06-05 15:06:22,538 - tests.orchestrator - INFO - Waiting for PVC test-pvc-afe1c1f0 to be bound -2025-06-05 15:06:24,671 - tests.orchestrator - INFO - PVC test-pvc-afe1c1f0 is bound -2025-06-05 15:06:24,671 - tests.orchestrator - INFO - Creating pod test-pod-d8058d9f on PVC test-pvc-7aa57f8c -2025-06-05 15:06:24,755 - tests.orchestrator - INFO - Created pod: test-pod-d8058d9f using PVC: test-pvc-7aa57f8c -2025-06-05 15:06:24,755 - tests.orchestrator - INFO - Waiting for pod test-pod-d8058d9f to be ready -2025-06-05 15:07:26,793 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-d8058d9f to be ready after 60s -2025-06-05 15:07:26,793 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-d8058d9f to be ready -2025-06-05 15:07:26,793 - tests.orchestrator - INFO - Creating pod test-pod-fdb5f5f7 on PVC test-pvc-afe1c1f0 -2025-06-05 15:07:26,879 - tests.orchestrator - INFO - Created pod: test-pod-fdb5f5f7 using PVC: test-pvc-afe1c1f0 -2025-06-05 15:07:26,880 - tests.orchestrator - INFO - Waiting for pod test-pod-fdb5f5f7 to be ready -2025-06-05 15:08:28,914 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-fdb5f5f7 to be ready after 60s -2025-06-05 15:08:28,915 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-fdb5f5f7 to be ready -2025-06-05 15:08:28,915 - tests.orchestrator - INFO - Creating pod test-pod-415052fa on PVC test-pvc-7aa57f8c -2025-06-05 15:08:28,998 - tests.orchestrator - INFO - Created pod: test-pod-415052fa using PVC: test-pvc-7aa57f8c -2025-06-05 15:08:28,998 - tests.orchestrator - INFO - Waiting for pod test-pod-415052fa to be ready -2025-06-05 15:09:31,031 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-415052fa to be ready after 60s -2025-06-05 15:09:31,031 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-415052fa to be ready -2025-06-05 15:09:31,031 - tests.orchestrator - INFO - Testing read/write between pods test-pod-d8058d9f and test-pod-415052fa sharing PVC test-pvc-7aa57f8c -2025-06-05 15:09:31,098 - tests.orchestrator - ERROR - Failed in read/write verification: (400) -Reason: Bad Request -HTTP response headers: HTTPHeaderDict({'Audit-Id': '9f5a799b-0705-4213-aaf5-1218c7e9571a', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 15:09:31 GMT', 'Content-Length': '166'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-d8058d9f does not have a host assigned","reason":"BadRequest","code":400} - - -2025-06-05 15:09:31,098 - tests.orchestrator - INFO - ============================================================ -2025-06-05 15:09:31,098 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_concurrent_pvc -2025-06-05 15:09:31,098 - tests.orchestrator - INFO - ============================================================ -2025-06-05 15:09:31,098 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations -2025-06-05 15:09:31,099 - tests.orchestrator - INFO - Creating 5 PVCs in quick succession -2025-06-05 15:09:31,177 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-a396d6c2 -2025-06-05 15:09:31,178 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-a396d6c2 to be bound -2025-06-05 15:09:33,310 - tests.orchestrator - INFO - PVC concurrent-pvc-a396d6c2 is bound -2025-06-05 15:09:33,381 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-85edcaa9 -2025-06-05 15:09:33,381 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-85edcaa9 to be bound -2025-06-05 15:09:35,516 - tests.orchestrator - INFO - PVC concurrent-pvc-85edcaa9 is bound -2025-06-05 15:09:35,583 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-bb7b8fe6 -2025-06-05 15:09:35,583 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-bb7b8fe6 to be bound -2025-06-05 15:09:37,719 - tests.orchestrator - INFO - PVC concurrent-pvc-bb7b8fe6 is bound -2025-06-05 15:09:37,790 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-a3d1cb36 -2025-06-05 15:09:37,790 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-a3d1cb36 to be bound -2025-06-05 15:09:39,924 - tests.orchestrator - INFO - PVC concurrent-pvc-a3d1cb36 is bound -2025-06-05 15:09:39,993 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-e1cb5975 -2025-06-05 15:09:39,994 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-e1cb5975 to be bound -2025-06-05 15:09:42,273 - tests.orchestrator - INFO - PVC concurrent-pvc-e1cb5975 is bound -2025-06-05 15:09:42,273 - tests.orchestrator - INFO - Successfully created 5 PVCs -2025-06-05 15:09:42,274 - tests.orchestrator - INFO - Creating pod test-pod-1ea724b8 on PVC concurrent-pvc-a3d1cb36 -2025-06-05 15:09:42,449 - tests.orchestrator - INFO - Created pod: test-pod-1ea724b8 using PVC: concurrent-pvc-a3d1cb36 -2025-06-05 15:09:42,449 - tests.orchestrator - INFO - Waiting for pod test-pod-1ea724b8 to be ready -2025-06-05 15:10:42,631 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1ea724b8 to be ready after 60s -2025-06-05 15:10:42,631 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1ea724b8 to be ready -2025-06-05 15:10:42,631 - tests.orchestrator - INFO - Creating pod test-pod-7f110176 on PVC concurrent-pvc-e1cb5975 -2025-06-05 15:10:42,732 - tests.orchestrator - INFO - Created pod: test-pod-7f110176 using PVC: concurrent-pvc-e1cb5975 -2025-06-05 15:10:42,732 - tests.orchestrator - INFO - Waiting for pod test-pod-7f110176 to be ready -2025-06-05 15:11:42,757 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-7f110176 to be ready after 60s -2025-06-05 15:11:42,757 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-7f110176 to be ready -2025-06-05 15:11:42,757 - tests.orchestrator - INFO - Creating pod test-pod-f48c7103 on PVC concurrent-pvc-85edcaa9 -2025-06-05 15:11:42,854 - tests.orchestrator - INFO - Created pod: test-pod-f48c7103 using PVC: concurrent-pvc-85edcaa9 -2025-06-05 15:11:42,854 - tests.orchestrator - INFO - Waiting for pod test-pod-f48c7103 to be ready -2025-06-05 15:12:44,885 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-f48c7103 to be ready after 60s -2025-06-05 15:12:44,886 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-f48c7103 to be ready -2025-06-05 15:12:44,886 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-a3d1cb36 -2025-06-05 15:12:44,886 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-a3d1cb36 -2025-06-05 15:12:44,886 - tests.orchestrator - INFO - Deleting pod: test-pod-1ea724b8 from PVC: concurrent-pvc-a3d1cb36 -2025-06-05 15:12:44,967 - tests.orchestrator - INFO - Waiting for pod test-pod-1ea724b8 to be deleted -2025-06-05 15:12:45,035 - tests.orchestrator - INFO - Pod test-pod-1ea724b8 has been deleted -2025-06-05 15:12:45,035 - tests.orchestrator - INFO - Deleted pod: test-pod-1ea724b8 -2025-06-05 15:12:45,120 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-a3d1cb36 to be deleted -2025-06-05 15:12:45,197 - tests.orchestrator - INFO - PVC concurrent-pvc-a3d1cb36 has been deleted -2025-06-05 15:12:45,198 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-a3d1cb36 -2025-06-05 15:12:45,198 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-85edcaa9 -2025-06-05 15:12:45,198 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-85edcaa9 -2025-06-05 15:12:45,198 - tests.orchestrator - INFO - Deleting pod: test-pod-f48c7103 from PVC: concurrent-pvc-85edcaa9 -2025-06-05 15:12:45,285 - tests.orchestrator - INFO - Waiting for pod test-pod-f48c7103 to be deleted -2025-06-05 15:12:45,353 - tests.orchestrator - INFO - Pod test-pod-f48c7103 has been deleted -2025-06-05 15:12:45,353 - tests.orchestrator - INFO - Deleted pod: test-pod-f48c7103 -2025-06-05 15:12:45,426 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-85edcaa9 to be deleted -2025-06-05 15:12:45,492 - tests.orchestrator - INFO - PVC concurrent-pvc-85edcaa9 has been deleted -2025-06-05 15:12:45,493 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-85edcaa9 -2025-06-05 15:12:45,493 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-a396d6c2 -2025-06-05 15:12:45,566 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-a396d6c2 to be deleted -2025-06-05 15:12:45,632 - tests.orchestrator - INFO - PVC concurrent-pvc-a396d6c2 has been deleted -2025-06-05 15:12:45,632 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-a396d6c2 -2025-06-05 15:12:45,633 - tests.orchestrator - INFO - Rapid PVC scenario completed -2025-06-05 15:12:45,633 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_concurrent_pvc -2025-06-05 15:12:45,633 - tests.orchestrator - INFO - ------------------------------------------------------------ -2025-06-05 15:12:45,633 - tests.orchestrator - INFO - Deleting pod: test-pod-fdb5f5f7 from PVC: test-pvc-afe1c1f0 -2025-06-05 15:12:45,715 - tests.orchestrator - INFO - Waiting for pod test-pod-fdb5f5f7 to be deleted -2025-06-05 15:12:45,783 - tests.orchestrator - INFO - Pod test-pod-fdb5f5f7 has been deleted -2025-06-05 15:12:45,783 - tests.orchestrator - INFO - Deleted pod: test-pod-fdb5f5f7 -2025-06-05 15:12:45,783 - tests.orchestrator - INFO - Deleting PVC: test-pvc-7aa57f8c -2025-06-05 15:12:45,783 - tests.orchestrator - INFO - Deleting 2 pods using PVC test-pvc-7aa57f8c -2025-06-05 15:12:45,784 - tests.orchestrator - INFO - Deleting pod: test-pod-d8058d9f from PVC: test-pvc-7aa57f8c -2025-06-05 15:12:45,867 - tests.orchestrator - INFO - Waiting for pod test-pod-d8058d9f to be deleted -2025-06-05 15:12:45,934 - tests.orchestrator - INFO - Pod test-pod-d8058d9f has been deleted -2025-06-05 15:12:45,934 - tests.orchestrator - INFO - Deleted pod: test-pod-d8058d9f -2025-06-05 15:12:45,934 - tests.orchestrator - INFO - Deleting pod: test-pod-415052fa from PVC: test-pvc-7aa57f8c -2025-06-05 15:12:46,015 - tests.orchestrator - INFO - Waiting for pod test-pod-415052fa to be deleted -2025-06-05 15:12:46,079 - tests.orchestrator - INFO - Pod test-pod-415052fa has been deleted -2025-06-05 15:12:46,080 - tests.orchestrator - INFO - Deleted pod: test-pod-415052fa -2025-06-05 15:12:46,154 - tests.orchestrator - INFO - Waiting for PVC test-pvc-7aa57f8c to be deleted -2025-06-05 15:12:46,220 - tests.orchestrator - INFO - PVC test-pvc-7aa57f8c has been deleted -2025-06-05 15:12:46,220 - tests.orchestrator - INFO - Deleted PVC: test-pvc-7aa57f8c -2025-06-05 15:12:46,220 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations -2025-06-05 15:12:46,221 - tests.orchestrator - INFO - Test completed in 386.15 seconds -2025-06-05 15:12:46,221 - tests.orchestrator - INFO - Cleaning up resources... -2025-06-05 15:12:46,221 - tests.orchestrator - INFO - Deleting pod: test-pod-7f110176 from PVC: concurrent-pvc-e1cb5975 -2025-06-05 15:12:46,308 - tests.orchestrator - INFO - Waiting for pod test-pod-7f110176 to be deleted -2025-06-05 15:12:46,378 - tests.orchestrator - INFO - Pod test-pod-7f110176 has been deleted -2025-06-05 15:12:46,379 - tests.orchestrator - INFO - Deleted pod: test-pod-7f110176 -2025-06-05 15:12:46,379 - tests.orchestrator - INFO - Deleting PVC: test-pvc-afe1c1f0 -2025-06-05 15:12:46,452 - tests.orchestrator - INFO - Waiting for PVC test-pvc-afe1c1f0 to be deleted -2025-06-05 15:12:46,519 - tests.orchestrator - INFO - PVC test-pvc-afe1c1f0 has been deleted -2025-06-05 15:12:46,519 - tests.orchestrator - INFO - Deleted PVC: test-pvc-afe1c1f0 -2025-06-05 15:12:46,519 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-bb7b8fe6 -2025-06-05 15:12:46,593 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-bb7b8fe6 to be deleted -2025-06-05 15:12:46,660 - tests.orchestrator - INFO - PVC concurrent-pvc-bb7b8fe6 has been deleted -2025-06-05 15:12:46,660 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-bb7b8fe6 -2025-06-05 15:12:46,660 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-e1cb5975 -2025-06-05 15:12:46,735 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-e1cb5975 to be deleted -2025-06-05 15:12:46,805 - tests.orchestrator - INFO - PVC concurrent-pvc-e1cb5975 has been deleted -2025-06-05 15:12:46,805 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-e1cb5975 -2025-06-05 15:12:46,809 - tests.orchestrator - INFO - Cleanup completed -2025-06-05 15:12:46,809 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-05 15:12:46,809 - tests.orchestrator - INFO - --- Operations --- -2025-06-05 15:12:46,809 - tests.orchestrator - INFO - create_pvc: 7 succeeded, 0 failed (100.0%) -2025-06-05 15:12:46,809 - tests.orchestrator - INFO - attach_pod: 6 succeeded, 0 failed (100.0%) -2025-06-05 15:12:46,809 - tests.orchestrator - INFO - delete_pod: 6 succeeded, 0 failed (100.0%) -2025-06-05 15:12:46,809 - tests.orchestrator - INFO - delete_pvc: 7 succeeded, 0 failed (100.0%) -2025-06-05 15:12:46,809 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) -2025-06-05 15:12:46,809 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) -2025-06-05 15:12:46,809 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-05 15:12:46,809 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-05 15:12:46,809 - tests.orchestrator - INFO - many_to_one: No runs -2025-06-05 15:12:46,809 - tests.orchestrator - INFO - one_to_one: No runs -2025-06-05 15:12:46,809 - tests.orchestrator - INFO - concurrent_pvc: 1 succeeded, 0 failed out of 1 runs (100.0%) -2025-06-05 15:12:46,809 - tests.orchestrator - INFO - ========================================= -2025-06-05 15:12:46,810 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250605_151246.json -2025-06-05 15:12:46,852 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250605_151246_20250605_151246.json -2025-06-05 15:12:46,852 - __main__ - INFO - All tests completed successfully -2025-06-05 15:15:29,024 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-05 15:15:29,024 - __main__ - INFO - Verifying AWS credentials -2025-06-05 15:15:29,025 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 15:15:29,025 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 15:15:29,026 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 15:15:29,026 - __main__ - INFO - Running orchestrator stress test suite -2025-06-05 15:15:29,026 - __main__ - INFO - Starting orchestrator for 300 seconds -2025-06-05 15:15:29,026 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-05 15:15:29,738 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 15:15:29,948 - tests.orchestrator - ERROR - Error checking namespace: (401) -Reason: Unauthorized -HTTP response headers: HTTPHeaderDict({'Audit-Id': '95b9f34d-de93-4da4-99dd-fc17111bd0ce', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 15:15:29 GMT', 'Content-Length': '129'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} - - -2025-06-05 15:15:29,950 - __main__ - ERROR - Error running tests: (401) -Reason: Unauthorized -HTTP response headers: HTTPHeaderDict({'Audit-Id': '95b9f34d-de93-4da4-99dd-fc17111bd0ce', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 15:15:29 GMT', 'Content-Length': '129'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} - -Traceback (most recent call last): - File "/local/home/ibhard/efs-csi-driver-fork/aws-efs-csi-driver/test/e2e/stress-scale-tests/run_tests.py", line 209, in main - orchestrator = EFSCSIOrchestrator(config_file=orchestrator_config) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/local/home/ibhard/efs-csi-driver-fork/aws-efs-csi-driver/test/e2e/stress-scale-tests/tests/orchestrator.py", line 86, in __init__ - self._ensure_namespace_exists() - File "/local/home/ibhard/efs-csi-driver-fork/aws-efs-csi-driver/test/e2e/stress-scale-tests/tests/orchestrator.py", line 92, in _ensure_namespace_exists - self.core_v1.read_namespace(name=self.namespace) - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api/core_v1_api.py", line 22992, in read_namespace - return self.read_namespace_with_http_info(name, **kwargs) # noqa: E501 - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api/core_v1_api.py", line 23071, in read_namespace_with_http_info - return self.api_client.call_api( - ^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api_client.py", line 348, in call_api - return self.__call_api(resource_path, method, - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api_client.py", line 180, in __call_api - response_data = self.request( - ^^^^^^^^^^^^^ - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api_client.py", line 373, in request - return self.rest_client.GET(url, - ^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/rest.py", line 244, in GET - return self.request("GET", url, - ^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/rest.py", line 238, in request - raise ApiException(http_resp=r) -kubernetes.client.exceptions.ApiException: (401) -Reason: Unauthorized -HTTP response headers: HTTPHeaderDict({'Audit-Id': '95b9f34d-de93-4da4-99dd-fc17111bd0ce', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 15:15:29 GMT', 'Content-Length': '129'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} - - -2025-06-05 15:16:04,243 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-05 15:16:04,243 - __main__ - INFO - Verifying AWS credentials -2025-06-05 15:16:04,244 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 15:16:04,244 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 15:16:04,244 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 15:16:04,245 - __main__ - INFO - Running orchestrator stress test suite -2025-06-05 15:16:04,245 - __main__ - INFO - Starting orchestrator for 300 seconds -2025-06-05 15:16:04,245 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-05 15:16:04,962 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 15:16:05,167 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 15:16:05,168 - __main__ - INFO - Test duration overridden to 300 seconds -2025-06-05 15:16:05,168 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) -2025-06-05 15:16:05,168 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-05 15:16:05,359 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 15:16:05,359 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 15:16:05,359 - tests.orchestrator - INFO - Creating PVC: test-pvc-20f868e5 -2025-06-05 15:16:05,438 - tests.orchestrator - INFO - Created PVC: test-pvc-20f868e5 -2025-06-05 15:16:05,438 - tests.orchestrator - INFO - Waiting for PVC test-pvc-20f868e5 to be bound -2025-06-05 15:16:07,570 - tests.orchestrator - INFO - PVC test-pvc-20f868e5 is bound -2025-06-05 15:16:07,570 - tests.orchestrator - INFO - Creating PVC: test-pvc-48e3d2d9 -2025-06-05 15:16:07,639 - tests.orchestrator - INFO - Created PVC: test-pvc-48e3d2d9 -2025-06-05 15:16:07,639 - tests.orchestrator - INFO - Waiting for PVC test-pvc-48e3d2d9 to be bound -2025-06-05 15:16:09,773 - tests.orchestrator - INFO - PVC test-pvc-48e3d2d9 is bound -2025-06-05 15:16:09,773 - tests.orchestrator - INFO - Creating pod test-pod-03b0f7fd on PVC test-pvc-20f868e5 -2025-06-05 15:16:09,857 - tests.orchestrator - INFO - Created pod: test-pod-03b0f7fd using PVC: test-pvc-20f868e5 -2025-06-05 15:16:09,857 - tests.orchestrator - INFO - Waiting for pod test-pod-03b0f7fd to be ready -2025-06-05 15:16:09,926 - tests.orchestrator - INFO - Pod test-pod-03b0f7fd phase: Pending -2025-06-05 15:16:40,974 - tests.orchestrator - INFO - Pod test-pod-03b0f7fd taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 15:16:40,974 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-03b0f7fd ===== -2025-06-05 15:16:41,041 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:16:41,124 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:16:41,125 - tests.orchestrator - INFO - [2025-06-05 15:16:09+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:16:41,199 - tests.orchestrator - INFO - No logs available -2025-06-05 15:16:41,199 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:16:41,199 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-20f868e5'} -2025-06-05 15:16:41,199 - tests.orchestrator - INFO - - kube-api-access-f6t7l: {} -2025-06-05 15:16:41,199 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-03b0f7fd ===== -2025-06-05 15:17:10,102 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-03b0f7fd to be ready after 60s -2025-06-05 15:17:10,102 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-03b0f7fd ===== -2025-06-05 15:17:10,172 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:17:10,245 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:17:10,245 - tests.orchestrator - INFO - [2025-06-05 15:16:09+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:17:10,312 - tests.orchestrator - INFO - No logs available -2025-06-05 15:17:10,312 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:17:10,312 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-20f868e5'} -2025-06-05 15:17:10,312 - tests.orchestrator - INFO - - kube-api-access-f6t7l: {} -2025-06-05 15:17:10,312 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-03b0f7fd ===== -2025-06-05 15:17:10,313 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-03b0f7fd to be ready -2025-06-05 15:17:10,313 - tests.orchestrator - INFO - Creating pod test-pod-264015e0 on PVC test-pvc-20f868e5 -2025-06-05 15:17:10,399 - tests.orchestrator - INFO - Created pod: test-pod-264015e0 using PVC: test-pvc-20f868e5 -2025-06-05 15:17:10,399 - tests.orchestrator - INFO - Waiting for pod test-pod-264015e0 to be ready -2025-06-05 15:17:10,467 - tests.orchestrator - INFO - Pod test-pod-264015e0 phase: Pending -2025-06-05 15:17:41,811 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-05 15:17:41,811 - __main__ - INFO - Verifying AWS credentials -2025-06-05 15:17:41,812 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 15:17:41,812 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 15:17:41,813 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 15:17:41,813 - __main__ - INFO - Running orchestrator stress test suite -2025-06-05 15:17:41,813 - __main__ - INFO - Starting orchestrator for 300 seconds -2025-06-05 15:17:41,813 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-05 15:17:42,550 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 15:17:42,784 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 15:17:42,784 - __main__ - INFO - Test duration overridden to 300 seconds -2025-06-05 15:17:42,784 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) -2025-06-05 15:17:42,784 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-05 15:17:42,975 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 15:17:42,976 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 15:17:42,976 - tests.orchestrator - INFO - Creating PVC: test-pvc-b967555b -2025-06-05 15:17:43,061 - tests.orchestrator - INFO - Created PVC: test-pvc-b967555b -2025-06-05 15:17:43,061 - tests.orchestrator - INFO - Waiting for PVC test-pvc-b967555b to be bound -2025-06-05 15:17:45,204 - tests.orchestrator - INFO - PVC test-pvc-b967555b is bound -2025-06-05 15:17:45,205 - tests.orchestrator - INFO - Creating PVC: test-pvc-6282a942 -2025-06-05 15:17:45,276 - tests.orchestrator - INFO - Created PVC: test-pvc-6282a942 -2025-06-05 15:17:45,276 - tests.orchestrator - INFO - Waiting for PVC test-pvc-6282a942 to be bound -2025-06-05 15:17:47,412 - tests.orchestrator - INFO - PVC test-pvc-6282a942 is bound -2025-06-05 15:17:47,412 - tests.orchestrator - INFO - Creating pod test-pod-9ef8c6d5 on PVC test-pvc-b967555b -2025-06-05 15:17:47,490 - tests.orchestrator - INFO - Created pod: test-pod-9ef8c6d5 using PVC: test-pvc-b967555b -2025-06-05 15:17:47,491 - tests.orchestrator - INFO - Waiting for pod test-pod-9ef8c6d5 to be ready -2025-06-05 15:17:47,559 - tests.orchestrator - INFO - Pod test-pod-9ef8c6d5 phase: Pending -2025-06-05 15:37:50,109 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-05 15:37:50,109 - __main__ - INFO - Verifying AWS credentials -2025-06-05 15:37:50,110 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 15:37:50,111 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 15:37:50,111 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 15:37:50,111 - __main__ - INFO - Running orchestrator stress test suite -2025-06-05 15:37:50,111 - __main__ - INFO - Starting orchestrator for 300 seconds -2025-06-05 15:37:50,112 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-05 15:37:50,853 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 15:37:51,073 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 15:37:51,074 - __main__ - INFO - Test duration overridden to 300 seconds -2025-06-05 15:37:51,074 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) -2025-06-05 15:37:51,074 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-05 15:37:51,263 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 15:37:51,263 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 15:37:51,263 - tests.orchestrator - INFO - Creating PVC: test-pvc-011557a1 -2025-06-05 15:37:51,341 - tests.orchestrator - INFO - Created PVC: test-pvc-011557a1 -2025-06-05 15:37:51,341 - tests.orchestrator - INFO - Waiting for PVC test-pvc-011557a1 to be bound -2025-06-05 15:37:53,474 - tests.orchestrator - INFO - PVC test-pvc-011557a1 is bound -2025-06-05 15:37:53,477 - tests.orchestrator - INFO - Creating PVC: test-pvc-b52b04ee -2025-06-05 15:37:53,544 - tests.orchestrator - INFO - Created PVC: test-pvc-b52b04ee -2025-06-05 15:37:53,545 - tests.orchestrator - INFO - Waiting for PVC test-pvc-b52b04ee to be bound -2025-06-05 15:37:55,677 - tests.orchestrator - INFO - PVC test-pvc-b52b04ee is bound -2025-06-05 15:37:55,677 - tests.orchestrator - INFO - Creating pod test-pod-1f8e6665 on PVC test-pvc-b52b04ee -2025-06-05 15:37:55,757 - tests.orchestrator - INFO - Created pod: test-pod-1f8e6665 using PVC: test-pvc-b52b04ee -2025-06-05 15:37:55,757 - tests.orchestrator - INFO - Waiting for pod test-pod-1f8e6665 to be ready -2025-06-05 15:37:55,827 - tests.orchestrator - INFO - Pod test-pod-1f8e6665 phase: Pending -2025-06-05 15:38:26,850 - tests.orchestrator - INFO - Pod test-pod-1f8e6665 taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 15:38:26,850 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-1f8e6665 ===== -2025-06-05 15:38:26,917 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:38:26,987 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:38:26,987 - tests.orchestrator - INFO - [2025-06-05 15:37:55+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:38:27,053 - tests.orchestrator - INFO - No logs available -2025-06-05 15:38:27,053 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:38:27,054 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-b52b04ee'} -2025-06-05 15:38:27,054 - tests.orchestrator - INFO - - kube-api-access-z4xbc: {} -2025-06-05 15:38:27,054 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-1f8e6665 ===== -2025-06-05 15:38:55,926 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1f8e6665 to be ready after 60s -2025-06-05 15:38:55,927 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-1f8e6665 ===== -2025-06-05 15:38:55,995 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:38:56,064 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:38:56,064 - tests.orchestrator - INFO - [2025-06-05 15:37:55+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:38:56,129 - tests.orchestrator - INFO - No logs available -2025-06-05 15:38:56,129 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:38:56,129 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-b52b04ee'} -2025-06-05 15:38:56,129 - tests.orchestrator - INFO - - kube-api-access-z4xbc: {} -2025-06-05 15:38:56,129 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-1f8e6665 ===== -2025-06-05 15:38:56,130 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1f8e6665 to be ready -2025-06-05 15:38:56,130 - tests.orchestrator - INFO - Creating pod test-pod-43a71634 on PVC test-pvc-b52b04ee -2025-06-05 15:38:56,213 - tests.orchestrator - INFO - Created pod: test-pod-43a71634 using PVC: test-pvc-b52b04ee -2025-06-05 15:38:56,213 - tests.orchestrator - INFO - Waiting for pod test-pod-43a71634 to be ready -2025-06-05 15:38:56,283 - tests.orchestrator - INFO - Pod test-pod-43a71634 phase: Pending -2025-06-05 15:39:27,290 - tests.orchestrator - INFO - Pod test-pod-43a71634 taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 15:39:27,290 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-43a71634 ===== -2025-06-05 15:39:27,357 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:39:27,428 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:39:27,428 - tests.orchestrator - INFO - [2025-06-05 15:38:56+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:39:27,493 - tests.orchestrator - INFO - No logs available -2025-06-05 15:39:27,494 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:39:27,494 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-b52b04ee'} -2025-06-05 15:39:27,494 - tests.orchestrator - INFO - - kube-api-access-fgg2x: {} -2025-06-05 15:39:27,494 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-43a71634 ===== -2025-06-05 15:39:56,469 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-43a71634 to be ready after 60s -2025-06-05 15:39:56,469 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-43a71634 ===== -2025-06-05 15:39:56,538 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:39:56,609 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:39:56,609 - tests.orchestrator - INFO - [2025-06-05 15:38:56+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:39:56,674 - tests.orchestrator - INFO - No logs available -2025-06-05 15:39:56,674 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:39:56,674 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-b52b04ee'} -2025-06-05 15:39:56,674 - tests.orchestrator - INFO - - kube-api-access-fgg2x: {} -2025-06-05 15:39:56,675 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-43a71634 ===== -2025-06-05 15:39:56,675 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-43a71634 to be ready -2025-06-05 15:39:56,675 - tests.orchestrator - INFO - Creating pod test-pod-9785ae47 on PVC test-pvc-011557a1 -2025-06-05 15:39:56,757 - tests.orchestrator - INFO - Created pod: test-pod-9785ae47 using PVC: test-pvc-011557a1 -2025-06-05 15:39:56,757 - tests.orchestrator - INFO - Waiting for pod test-pod-9785ae47 to be ready -2025-06-05 15:39:56,823 - tests.orchestrator - INFO - Pod test-pod-9785ae47 phase: Pending -2025-06-05 15:40:27,832 - tests.orchestrator - INFO - Pod test-pod-9785ae47 taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 15:40:27,832 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-9785ae47 ===== -2025-06-05 15:40:27,899 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:40:27,970 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:40:27,970 - tests.orchestrator - INFO - [2025-06-05 15:39:56+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:40:28,037 - tests.orchestrator - INFO - No logs available -2025-06-05 15:40:28,037 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:40:28,037 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-011557a1'} -2025-06-05 15:40:28,037 - tests.orchestrator - INFO - - kube-api-access-p2phq: {} -2025-06-05 15:40:28,037 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-9785ae47 ===== -2025-06-05 15:40:56,911 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9785ae47 to be ready after 60s -2025-06-05 15:40:56,912 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-9785ae47 ===== -2025-06-05 15:40:56,980 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:40:57,050 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:40:57,050 - tests.orchestrator - INFO - [2025-06-05 15:39:56+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:40:57,118 - tests.orchestrator - INFO - No logs available -2025-06-05 15:40:57,118 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:40:57,118 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-011557a1'} -2025-06-05 15:40:57,118 - tests.orchestrator - INFO - - kube-api-access-p2phq: {} -2025-06-05 15:40:57,118 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-9785ae47 ===== -2025-06-05 15:40:57,118 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9785ae47 to be ready -2025-06-05 15:40:57,119 - tests.orchestrator - INFO - Testing read/write between pods test-pod-1f8e6665 and test-pod-43a71634 sharing PVC test-pvc-b52b04ee -2025-06-05 15:40:57,185 - tests.orchestrator - ERROR - Failed in read/write verification: (400) -Reason: Bad Request -HTTP response headers: HTTPHeaderDict({'Audit-Id': '4713df17-943b-4cf0-b43b-7454ef741357', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 15:40:57 GMT', 'Content-Length': '166'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-1f8e6665 does not have a host assigned","reason":"BadRequest","code":400} - - -2025-06-05 15:40:57,185 - tests.orchestrator - INFO - ============================================================ -2025-06-05 15:40:57,185 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_many_to_one -2025-06-05 15:40:57,185 - tests.orchestrator - INFO - ============================================================ -2025-06-05 15:40:57,185 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -2025-06-05 15:40:57,185 - tests.orchestrator - INFO - STARTING MANY-TO-ONE SCENARIO DIAGNOSTICS -2025-06-05 15:40:57,185 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -2025-06-05 15:40:57,185 - tests.orchestrator - INFO - [MANY2ONE] STEP 1: Creating dedicated PVC: many2one-21027765 -2025-06-05 15:40:57,185 - tests.orchestrator - INFO - [MANY2ONE] Creating PVC manifest with storageClassName: efs-sc -2025-06-05 15:40:57,263 - tests.orchestrator - INFO - [MANY2ONE] PVC many2one-21027765 created successfully -2025-06-05 15:40:57,264 - tests.orchestrator - INFO - [MANY2ONE] Waiting for PVC many2one-21027765 to be bound... -2025-06-05 15:40:57,264 - tests.orchestrator - INFO - Waiting for PVC many2one-21027765 to be bound -2025-06-05 15:40:59,396 - tests.orchestrator - INFO - PVC many2one-21027765 is bound -2025-06-05 15:40:59,463 - tests.orchestrator - INFO - [MANY2ONE] PVC status: phase=Bound, capacity={'storage': '1Gi'} -2025-06-05 15:40:59,463 - tests.orchestrator - INFO - [MANY2ONE] STEP 2: Creating 3 pods for the same PVC many2one-21027765 -2025-06-05 15:40:59,463 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 1/3 for PVC many2one-21027765 -2025-06-05 15:40:59,463 - tests.orchestrator - INFO - Creating pod test-pod-052e963e on PVC many2one-21027765 -2025-06-05 15:40:59,538 - tests.orchestrator - INFO - Created pod: test-pod-052e963e using PVC: many2one-21027765 -2025-06-05 15:40:59,538 - tests.orchestrator - INFO - Waiting for pod test-pod-052e963e to be ready -2025-06-05 15:40:59,604 - tests.orchestrator - INFO - Pod test-pod-052e963e phase: Pending -2025-06-05 15:41:30,643 - tests.orchestrator - INFO - Pod test-pod-052e963e taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 15:41:30,643 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-052e963e ===== -2025-06-05 15:41:30,710 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:41:30,779 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:41:30,780 - tests.orchestrator - INFO - [2025-06-05 15:40:59+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:41:30,846 - tests.orchestrator - INFO - No logs available -2025-06-05 15:41:30,846 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:41:30,846 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-21027765'} -2025-06-05 15:41:30,846 - tests.orchestrator - INFO - - kube-api-access-h4dzw: {} -2025-06-05 15:41:30,846 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-052e963e ===== -2025-06-05 15:41:59,722 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-052e963e to be ready after 60s -2025-06-05 15:41:59,722 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-052e963e ===== -2025-06-05 15:41:59,789 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:41:59,859 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:41:59,859 - tests.orchestrator - INFO - [2025-06-05 15:40:59+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:41:59,925 - tests.orchestrator - INFO - No logs available -2025-06-05 15:41:59,925 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:41:59,925 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-21027765'} -2025-06-05 15:41:59,925 - tests.orchestrator - INFO - - kube-api-access-h4dzw: {} -2025-06-05 15:41:59,925 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-052e963e ===== -2025-06-05 15:41:59,926 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-052e963e to be ready -2025-06-05 15:41:59,926 - tests.orchestrator - ERROR - [MANY2ONE] Failed to create pod 1/3 -2025-06-05 15:41:59,926 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 2/3 for PVC many2one-21027765 -2025-06-05 15:41:59,926 - tests.orchestrator - INFO - Creating pod test-pod-dda7e82b on PVC many2one-21027765 -2025-06-05 15:42:00,007 - tests.orchestrator - INFO - Created pod: test-pod-dda7e82b using PVC: many2one-21027765 -2025-06-05 15:42:00,008 - tests.orchestrator - INFO - Waiting for pod test-pod-dda7e82b to be ready -2025-06-05 15:42:00,075 - tests.orchestrator - INFO - Pod test-pod-dda7e82b phase: Pending -2025-06-05 15:42:31,084 - tests.orchestrator - INFO - Pod test-pod-dda7e82b taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 15:42:31,084 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-dda7e82b ===== -2025-06-05 15:42:31,150 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:42:31,219 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:42:31,219 - tests.orchestrator - INFO - [2025-06-05 15:41:59+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:42:31,285 - tests.orchestrator - INFO - No logs available -2025-06-05 15:42:31,285 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:42:31,285 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-21027765'} -2025-06-05 15:42:31,285 - tests.orchestrator - INFO - - kube-api-access-bktns: {} -2025-06-05 15:42:31,285 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-dda7e82b ===== -2025-06-05 15:43:00,160 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-dda7e82b to be ready after 60s -2025-06-05 15:43:00,160 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-dda7e82b ===== -2025-06-05 15:43:00,227 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:43:00,296 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:43:00,296 - tests.orchestrator - INFO - [2025-06-05 15:41:59+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:43:00,362 - tests.orchestrator - INFO - No logs available -2025-06-05 15:43:00,362 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:43:00,362 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-21027765'} -2025-06-05 15:43:00,362 - tests.orchestrator - INFO - - kube-api-access-bktns: {} -2025-06-05 15:43:00,363 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-dda7e82b ===== -2025-06-05 15:43:00,363 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-dda7e82b to be ready -2025-06-05 15:43:00,363 - tests.orchestrator - ERROR - [MANY2ONE] Failed to create pod 2/3 -2025-06-05 15:43:00,363 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 3/3 for PVC many2one-21027765 -2025-06-05 15:43:00,363 - tests.orchestrator - INFO - Creating pod test-pod-9b6c5221 on PVC many2one-21027765 -2025-06-05 15:43:00,448 - tests.orchestrator - INFO - Created pod: test-pod-9b6c5221 using PVC: many2one-21027765 -2025-06-05 15:43:00,448 - tests.orchestrator - INFO - Waiting for pod test-pod-9b6c5221 to be ready -2025-06-05 15:43:00,514 - tests.orchestrator - INFO - Pod test-pod-9b6c5221 phase: Pending -2025-06-05 15:43:31,552 - tests.orchestrator - INFO - Pod test-pod-9b6c5221 taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 15:43:31,552 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-9b6c5221 ===== -2025-06-05 15:43:31,684 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:43:31,759 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:43:31,759 - tests.orchestrator - INFO - [2025-06-05 15:43:00+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:43:31,829 - tests.orchestrator - INFO - No logs available -2025-06-05 15:43:31,830 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:43:31,830 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-21027765'} -2025-06-05 15:43:31,830 - tests.orchestrator - INFO - - kube-api-access-mh6kx: {} -2025-06-05 15:43:31,830 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-9b6c5221 ===== -2025-06-05 15:44:00,708 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9b6c5221 to be ready after 60s -2025-06-05 15:44:00,708 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-9b6c5221 ===== -2025-06-05 15:44:00,776 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:44:00,846 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:44:00,846 - tests.orchestrator - INFO - [2025-06-05 15:43:00+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:44:00,911 - tests.orchestrator - INFO - No logs available -2025-06-05 15:44:00,911 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:44:00,911 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-21027765'} -2025-06-05 15:44:00,911 - tests.orchestrator - INFO - - kube-api-access-mh6kx: {} -2025-06-05 15:44:00,911 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-9b6c5221 ===== -2025-06-05 15:44:00,912 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9b6c5221 to be ready -2025-06-05 15:44:00,912 - tests.orchestrator - ERROR - [MANY2ONE] Failed to create pod 3/3 -2025-06-05 15:44:00,912 - tests.orchestrator - INFO - [MANY2ONE] Created 0/3 pods successfully -2025-06-05 15:44:00,912 - tests.orchestrator - ERROR - [MANY2ONE] FAILED: Insufficient pods created (0), need at least 2 for read/write test -2025-06-05 15:44:00,912 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_many_to_one -2025-06-05 15:44:00,912 - tests.orchestrator - INFO - ------------------------------------------------------------ -2025-06-05 15:44:00,912 - tests.orchestrator - INFO - Deleting pod: test-pod-052e963e from PVC: many2one-21027765 -2025-06-05 15:44:00,990 - tests.orchestrator - INFO - Waiting for pod test-pod-052e963e to be deleted -2025-06-05 15:44:01,055 - tests.orchestrator - INFO - Pod test-pod-052e963e has been deleted -2025-06-05 15:44:01,055 - tests.orchestrator - INFO - Deleted pod: test-pod-052e963e -2025-06-05 15:44:01,056 - tests.orchestrator - INFO - Deleting PVC: test-pvc-b52b04ee -2025-06-05 15:44:01,056 - tests.orchestrator - INFO - Deleting 2 pods using PVC test-pvc-b52b04ee -2025-06-05 15:44:01,056 - tests.orchestrator - INFO - Deleting pod: test-pod-1f8e6665 from PVC: test-pvc-b52b04ee -2025-06-05 15:44:01,140 - tests.orchestrator - INFO - Waiting for pod test-pod-1f8e6665 to be deleted -2025-06-05 15:44:01,205 - tests.orchestrator - INFO - Pod test-pod-1f8e6665 has been deleted -2025-06-05 15:44:01,205 - tests.orchestrator - INFO - Deleted pod: test-pod-1f8e6665 -2025-06-05 15:44:01,205 - tests.orchestrator - INFO - Deleting pod: test-pod-43a71634 from PVC: test-pvc-b52b04ee -2025-06-05 15:44:01,287 - tests.orchestrator - INFO - Waiting for pod test-pod-43a71634 to be deleted -2025-06-05 15:44:01,352 - tests.orchestrator - INFO - Pod test-pod-43a71634 has been deleted -2025-06-05 15:44:01,352 - tests.orchestrator - INFO - Deleted pod: test-pod-43a71634 -2025-06-05 15:44:01,426 - tests.orchestrator - INFO - Waiting for PVC test-pvc-b52b04ee to be deleted -2025-06-05 15:44:01,527 - tests.orchestrator - INFO - PVC test-pvc-b52b04ee has been deleted -2025-06-05 15:44:01,527 - tests.orchestrator - INFO - Deleted PVC: test-pvc-b52b04ee -2025-06-05 15:44:01,527 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations -2025-06-05 15:44:01,527 - tests.orchestrator - INFO - Test completed in 370.45 seconds -2025-06-05 15:44:01,527 - tests.orchestrator - INFO - Cleaning up resources... -2025-06-05 15:44:01,527 - tests.orchestrator - INFO - Deleting pod: test-pod-9785ae47 from PVC: test-pvc-011557a1 -2025-06-05 15:44:01,678 - tests.orchestrator - INFO - Waiting for pod test-pod-9785ae47 to be deleted -2025-06-05 15:44:01,750 - tests.orchestrator - INFO - Pod test-pod-9785ae47 has been deleted -2025-06-05 15:44:01,751 - tests.orchestrator - INFO - Deleted pod: test-pod-9785ae47 -2025-06-05 15:44:01,751 - tests.orchestrator - INFO - Deleting pod: test-pod-dda7e82b from PVC: many2one-21027765 -2025-06-05 15:44:01,841 - tests.orchestrator - INFO - Waiting for pod test-pod-dda7e82b to be deleted -2025-06-05 15:44:01,912 - tests.orchestrator - INFO - Pod test-pod-dda7e82b has been deleted -2025-06-05 15:44:01,912 - tests.orchestrator - INFO - Deleted pod: test-pod-dda7e82b -2025-06-05 15:44:01,912 - tests.orchestrator - INFO - Deleting pod: test-pod-9b6c5221 from PVC: many2one-21027765 -2025-06-05 15:44:02,038 - tests.orchestrator - INFO - Waiting for pod test-pod-9b6c5221 to be deleted -2025-06-05 15:44:02,118 - tests.orchestrator - INFO - Pod test-pod-9b6c5221 has been deleted -2025-06-05 15:44:02,118 - tests.orchestrator - INFO - Deleted pod: test-pod-9b6c5221 -2025-06-05 15:44:02,118 - tests.orchestrator - INFO - Deleting PVC: test-pvc-011557a1 -2025-06-05 15:44:02,196 - tests.orchestrator - INFO - Waiting for PVC test-pvc-011557a1 to be deleted -2025-06-05 15:44:02,268 - tests.orchestrator - INFO - PVC test-pvc-011557a1 has been deleted -2025-06-05 15:44:02,268 - tests.orchestrator - INFO - Deleted PVC: test-pvc-011557a1 -2025-06-05 15:44:02,268 - tests.orchestrator - INFO - Deleting PVC: many2one-21027765 -2025-06-05 15:44:02,341 - tests.orchestrator - INFO - Waiting for PVC many2one-21027765 to be deleted -2025-06-05 15:44:02,406 - tests.orchestrator - INFO - PVC many2one-21027765 has been deleted -2025-06-05 15:44:02,406 - tests.orchestrator - INFO - Deleted PVC: many2one-21027765 -2025-06-05 15:44:02,406 - tests.orchestrator - INFO - Cleanup completed -2025-06-05 15:44:02,406 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-05 15:44:02,407 - tests.orchestrator - INFO - --- Operations --- -2025-06-05 15:44:02,407 - tests.orchestrator - INFO - create_pvc: 2 succeeded, 0 failed (100.0%) -2025-06-05 15:44:02,407 - tests.orchestrator - INFO - attach_pod: 6 succeeded, 0 failed (100.0%) -2025-06-05 15:44:02,407 - tests.orchestrator - INFO - delete_pod: 6 succeeded, 0 failed (100.0%) -2025-06-05 15:44:02,407 - tests.orchestrator - INFO - delete_pvc: 3 succeeded, 0 failed (100.0%) -2025-06-05 15:44:02,407 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) -2025-06-05 15:44:02,407 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) -2025-06-05 15:44:02,407 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-05 15:44:02,407 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-05 15:44:02,408 - tests.orchestrator - INFO - many_to_one: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-05 15:44:02,408 - tests.orchestrator - INFO - one_to_one: No runs -2025-06-05 15:44:02,408 - tests.orchestrator - INFO - concurrent_pvc: No runs -2025-06-05 15:44:02,408 - tests.orchestrator - INFO - ========================================= -2025-06-05 15:44:02,408 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250605_154402.json -2025-06-05 15:44:02,451 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250605_154402_20250605_154402.json -2025-06-05 15:44:02,451 - __main__ - INFO - All tests completed successfully -2025-06-05 15:45:45,975 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-05 15:45:45,975 - __main__ - INFO - Verifying AWS credentials -2025-06-05 15:45:45,976 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 15:45:45,976 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 15:45:45,977 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 15:45:45,977 - __main__ - INFO - Running orchestrator stress test suite -2025-06-05 15:45:45,977 - __main__ - INFO - Starting orchestrator for 300 seconds -2025-06-05 15:45:45,977 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-05 15:45:46,726 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 15:45:46,943 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 15:45:46,944 - __main__ - INFO - Test duration overridden to 300 seconds -2025-06-05 15:45:46,944 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) -2025-06-05 15:45:46,944 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-05 15:45:47,132 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 15:45:47,133 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 15:45:47,133 - tests.orchestrator - INFO - Creating PVC: test-pvc-7dbf87c6 -2025-06-05 15:45:47,210 - tests.orchestrator - INFO - Created PVC: test-pvc-7dbf87c6 -2025-06-05 15:45:47,210 - tests.orchestrator - INFO - Waiting for PVC test-pvc-7dbf87c6 to be bound -2025-06-05 15:45:49,343 - tests.orchestrator - INFO - PVC test-pvc-7dbf87c6 is bound -2025-06-05 15:45:49,343 - tests.orchestrator - INFO - Creating PVC: test-pvc-83f66d6c -2025-06-05 15:45:49,411 - tests.orchestrator - INFO - Created PVC: test-pvc-83f66d6c -2025-06-05 15:45:49,411 - tests.orchestrator - INFO - Waiting for PVC test-pvc-83f66d6c to be bound -2025-06-05 15:45:51,548 - tests.orchestrator - INFO - PVC test-pvc-83f66d6c is bound -2025-06-05 15:45:51,549 - tests.orchestrator - INFO - Creating pod test-pod-c3e945dc on PVC test-pvc-7dbf87c6 -2025-06-05 15:45:51,631 - tests.orchestrator - INFO - Created pod: test-pod-c3e945dc using PVC: test-pvc-7dbf87c6 -2025-06-05 15:45:51,631 - tests.orchestrator - INFO - Waiting for pod test-pod-c3e945dc to be ready -2025-06-05 15:45:51,697 - tests.orchestrator - INFO - Pod test-pod-c3e945dc phase: Pending -2025-06-05 15:46:22,714 - tests.orchestrator - INFO - Pod test-pod-c3e945dc taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 15:46:22,714 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-c3e945dc ===== -2025-06-05 15:46:22,781 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:46:22,850 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:46:22,850 - tests.orchestrator - INFO - [2025-06-05 15:45:51+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:46:22,916 - tests.orchestrator - INFO - No logs available -2025-06-05 15:46:22,917 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:46:22,917 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-7dbf87c6'} -2025-06-05 15:46:22,917 - tests.orchestrator - INFO - - kube-api-access-vmkzs: {} -2025-06-05 15:46:22,917 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-c3e945dc ===== -2025-06-05 15:46:51,793 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-c3e945dc to be ready after 60s -2025-06-05 15:46:51,793 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-c3e945dc ===== -2025-06-05 15:46:51,860 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:46:51,930 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:46:51,930 - tests.orchestrator - INFO - [2025-06-05 15:45:51+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:46:51,996 - tests.orchestrator - INFO - No logs available -2025-06-05 15:46:51,996 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:46:51,996 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-7dbf87c6'} -2025-06-05 15:46:51,996 - tests.orchestrator - INFO - - kube-api-access-vmkzs: {} -2025-06-05 15:46:51,996 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-c3e945dc ===== -2025-06-05 15:46:51,997 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-c3e945dc to be ready -2025-06-05 15:46:51,997 - tests.orchestrator - INFO - Creating pod test-pod-7abe409e on PVC test-pvc-83f66d6c -2025-06-05 15:46:52,079 - tests.orchestrator - INFO - Created pod: test-pod-7abe409e using PVC: test-pvc-83f66d6c -2025-06-05 15:46:52,079 - tests.orchestrator - INFO - Waiting for pod test-pod-7abe409e to be ready -2025-06-05 15:46:52,146 - tests.orchestrator - INFO - Pod test-pod-7abe409e phase: Pending -2025-06-05 15:47:23,154 - tests.orchestrator - INFO - Pod test-pod-7abe409e taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 15:47:23,154 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-7abe409e ===== -2025-06-05 15:47:23,220 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:47:23,289 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:47:23,290 - tests.orchestrator - INFO - [2025-06-05 15:46:52+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:47:23,355 - tests.orchestrator - INFO - No logs available -2025-06-05 15:47:23,355 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:47:23,355 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-83f66d6c'} -2025-06-05 15:47:23,355 - tests.orchestrator - INFO - - kube-api-access-swfv7: {} -2025-06-05 15:47:23,355 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-7abe409e ===== -2025-06-05 15:47:52,325 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-7abe409e to be ready after 60s -2025-06-05 15:47:52,325 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-7abe409e ===== -2025-06-05 15:47:52,392 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:47:52,462 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:47:52,462 - tests.orchestrator - INFO - [2025-06-05 15:46:52+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:47:52,528 - tests.orchestrator - INFO - No logs available -2025-06-05 15:47:52,528 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:47:52,528 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-83f66d6c'} -2025-06-05 15:47:52,528 - tests.orchestrator - INFO - - kube-api-access-swfv7: {} -2025-06-05 15:47:52,528 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-7abe409e ===== -2025-06-05 15:47:52,528 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-7abe409e to be ready -2025-06-05 15:47:52,528 - tests.orchestrator - INFO - Creating pod test-pod-a9fc1f04 on PVC test-pvc-7dbf87c6 -2025-06-05 15:47:52,611 - tests.orchestrator - INFO - Created pod: test-pod-a9fc1f04 using PVC: test-pvc-7dbf87c6 -2025-06-05 15:47:52,612 - tests.orchestrator - INFO - Waiting for pod test-pod-a9fc1f04 to be ready -2025-06-05 15:47:52,678 - tests.orchestrator - INFO - Pod test-pod-a9fc1f04 phase: Pending -2025-06-05 15:48:23,696 - tests.orchestrator - INFO - Pod test-pod-a9fc1f04 taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 15:48:23,696 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-a9fc1f04 ===== -2025-06-05 15:48:23,762 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:48:23,831 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:48:23,832 - tests.orchestrator - INFO - [2025-06-05 15:47:52+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:48:23,898 - tests.orchestrator - INFO - No logs available -2025-06-05 15:48:23,898 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:48:23,898 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-7dbf87c6'} -2025-06-05 15:48:23,898 - tests.orchestrator - INFO - - kube-api-access-bh5hw: {} -2025-06-05 15:48:23,898 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-a9fc1f04 ===== -2025-06-05 15:48:52,778 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a9fc1f04 to be ready after 60s -2025-06-05 15:48:52,778 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-a9fc1f04 ===== -2025-06-05 15:48:52,845 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:48:52,917 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:48:52,917 - tests.orchestrator - INFO - [2025-06-05 15:47:52+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:48:52,983 - tests.orchestrator - INFO - No logs available -2025-06-05 15:48:52,983 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:48:52,983 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-7dbf87c6'} -2025-06-05 15:48:52,983 - tests.orchestrator - INFO - - kube-api-access-bh5hw: {} -2025-06-05 15:48:52,984 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-a9fc1f04 ===== -2025-06-05 15:48:52,984 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a9fc1f04 to be ready -2025-06-05 15:48:52,984 - tests.orchestrator - INFO - Testing read/write between pods test-pod-c3e945dc and test-pod-a9fc1f04 sharing PVC test-pvc-7dbf87c6 -2025-06-05 15:48:53,049 - tests.orchestrator - ERROR - Failed in read/write verification: (400) -Reason: Bad Request -HTTP response headers: HTTPHeaderDict({'Audit-Id': '0f444cd6-376b-4233-8e31-012234fb64c8', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 15:48:53 GMT', 'Content-Length': '166'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-c3e945dc does not have a host assigned","reason":"BadRequest","code":400} - - -2025-06-05 15:48:53,049 - tests.orchestrator - INFO - ============================================================ -2025-06-05 15:48:53,049 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_concurrent_pvc -2025-06-05 15:48:53,049 - tests.orchestrator - INFO - ============================================================ -2025-06-05 15:48:53,049 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations -2025-06-05 15:48:53,049 - tests.orchestrator - INFO - Creating 3 PVCs in quick succession -2025-06-05 15:48:53,127 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-74639d37 -2025-06-05 15:48:53,127 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-74639d37 to be bound -2025-06-05 15:48:55,259 - tests.orchestrator - INFO - PVC concurrent-pvc-74639d37 is bound -2025-06-05 15:48:55,328 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-be273a95 -2025-06-05 15:48:55,328 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-be273a95 to be bound -2025-06-05 15:48:57,460 - tests.orchestrator - INFO - PVC concurrent-pvc-be273a95 is bound -2025-06-05 15:48:57,530 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-421903bd -2025-06-05 15:48:57,530 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-421903bd to be bound -2025-06-05 15:48:59,662 - tests.orchestrator - INFO - PVC concurrent-pvc-421903bd is bound -2025-06-05 15:48:59,662 - tests.orchestrator - INFO - Successfully created 3 PVCs -2025-06-05 15:48:59,662 - tests.orchestrator - INFO - Creating pod test-pod-d257ffb1 on PVC concurrent-pvc-be273a95 -2025-06-05 15:48:59,737 - tests.orchestrator - INFO - Created pod: test-pod-d257ffb1 using PVC: concurrent-pvc-be273a95 -2025-06-05 15:48:59,737 - tests.orchestrator - INFO - Waiting for pod test-pod-d257ffb1 to be ready -2025-06-05 15:48:59,805 - tests.orchestrator - INFO - Pod test-pod-d257ffb1 phase: Pending -2025-06-05 15:49:30,827 - tests.orchestrator - INFO - Pod test-pod-d257ffb1 taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 15:49:30,828 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-d257ffb1 ===== -2025-06-05 15:49:30,894 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:49:30,964 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:49:30,964 - tests.orchestrator - INFO - [2025-06-05 15:48:59+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:49:31,030 - tests.orchestrator - INFO - No logs available -2025-06-05 15:49:31,030 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:49:31,030 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'concurrent-pvc-be273a95'} -2025-06-05 15:49:31,031 - tests.orchestrator - INFO - - kube-api-access-tr4s4: {} -2025-06-05 15:49:31,031 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-d257ffb1 ===== -2025-06-05 15:49:59,906 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-d257ffb1 to be ready after 60s -2025-06-05 15:49:59,907 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-d257ffb1 ===== -2025-06-05 15:49:59,974 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:50:00,045 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:50:00,045 - tests.orchestrator - INFO - [2025-06-05 15:48:59+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:50:00,111 - tests.orchestrator - INFO - No logs available -2025-06-05 15:50:00,112 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:50:00,112 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'concurrent-pvc-be273a95'} -2025-06-05 15:50:00,112 - tests.orchestrator - INFO - - kube-api-access-tr4s4: {} -2025-06-05 15:50:00,112 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-d257ffb1 ===== -2025-06-05 15:50:00,112 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-d257ffb1 to be ready -2025-06-05 15:50:00,112 - tests.orchestrator - INFO - Creating pod test-pod-8c9ea815 on PVC concurrent-pvc-421903bd -2025-06-05 15:50:00,195 - tests.orchestrator - INFO - Created pod: test-pod-8c9ea815 using PVC: concurrent-pvc-421903bd -2025-06-05 15:50:00,196 - tests.orchestrator - INFO - Waiting for pod test-pod-8c9ea815 to be ready -2025-06-05 15:50:00,263 - tests.orchestrator - INFO - Pod test-pod-8c9ea815 phase: Pending -2025-06-05 15:50:31,275 - tests.orchestrator - INFO - Pod test-pod-8c9ea815 taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 15:50:31,275 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-8c9ea815 ===== -2025-06-05 15:50:31,341 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:50:31,411 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:50:31,411 - tests.orchestrator - INFO - [2025-06-05 15:50:00+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:50:31,526 - tests.orchestrator - INFO - No logs available -2025-06-05 15:50:31,526 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:50:31,526 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'concurrent-pvc-421903bd'} -2025-06-05 15:50:31,526 - tests.orchestrator - INFO - - kube-api-access-mk5vl: {} -2025-06-05 15:50:31,527 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-8c9ea815 ===== -2025-06-05 15:51:00,401 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-8c9ea815 to be ready after 60s -2025-06-05 15:51:00,401 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-8c9ea815 ===== -2025-06-05 15:51:00,469 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:51:00,539 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:51:00,539 - tests.orchestrator - INFO - [2025-06-05 15:50:00+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:51:00,604 - tests.orchestrator - INFO - No logs available -2025-06-05 15:51:00,604 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:51:00,604 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'concurrent-pvc-421903bd'} -2025-06-05 15:51:00,604 - tests.orchestrator - INFO - - kube-api-access-mk5vl: {} -2025-06-05 15:51:00,605 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-8c9ea815 ===== -2025-06-05 15:51:00,605 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-8c9ea815 to be ready -2025-06-05 15:51:00,605 - tests.orchestrator - INFO - Creating pod test-pod-3106ab46 on PVC concurrent-pvc-74639d37 -2025-06-05 15:51:00,689 - tests.orchestrator - INFO - Created pod: test-pod-3106ab46 using PVC: concurrent-pvc-74639d37 -2025-06-05 15:51:00,689 - tests.orchestrator - INFO - Waiting for pod test-pod-3106ab46 to be ready -2025-06-05 15:51:00,755 - tests.orchestrator - INFO - Pod test-pod-3106ab46 phase: Pending -2025-06-05 15:51:31,779 - tests.orchestrator - INFO - Pod test-pod-3106ab46 taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 15:51:31,779 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-3106ab46 ===== -2025-06-05 15:51:31,852 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:51:31,927 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:51:31,927 - tests.orchestrator - INFO - [2025-06-05 15:51:00+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:51:31,998 - tests.orchestrator - INFO - No logs available -2025-06-05 15:51:31,998 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:51:31,998 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'concurrent-pvc-74639d37'} -2025-06-05 15:51:31,998 - tests.orchestrator - INFO - - kube-api-access-97jdn: {} -2025-06-05 15:51:31,998 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-3106ab46 ===== -2025-06-05 15:52:00,875 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-3106ab46 to be ready after 60s -2025-06-05 15:52:00,876 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-3106ab46 ===== -2025-06-05 15:52:00,943 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:52:01,013 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:52:01,013 - tests.orchestrator - INFO - [2025-06-05 15:51:00+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:52:01,079 - tests.orchestrator - INFO - No logs available -2025-06-05 15:52:01,079 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:52:01,079 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'concurrent-pvc-74639d37'} -2025-06-05 15:52:01,080 - tests.orchestrator - INFO - - kube-api-access-97jdn: {} -2025-06-05 15:52:01,080 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-3106ab46 ===== -2025-06-05 15:52:01,080 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-3106ab46 to be ready -2025-06-05 15:52:01,080 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-be273a95 -2025-06-05 15:52:01,080 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-be273a95 -2025-06-05 15:52:01,080 - tests.orchestrator - INFO - Deleting pod: test-pod-d257ffb1 from PVC: concurrent-pvc-be273a95 -2025-06-05 15:52:01,164 - tests.orchestrator - INFO - Waiting for pod test-pod-d257ffb1 to be deleted -2025-06-05 15:52:01,229 - tests.orchestrator - INFO - Pod test-pod-d257ffb1 has been deleted -2025-06-05 15:52:01,230 - tests.orchestrator - INFO - Deleted pod: test-pod-d257ffb1 -2025-06-05 15:52:01,301 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-be273a95 to be deleted -2025-06-05 15:52:01,366 - tests.orchestrator - INFO - PVC concurrent-pvc-be273a95 has been deleted -2025-06-05 15:52:01,367 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-be273a95 -2025-06-05 15:52:01,367 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-421903bd -2025-06-05 15:52:01,367 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-421903bd -2025-06-05 15:52:01,367 - tests.orchestrator - INFO - Deleting pod: test-pod-8c9ea815 from PVC: concurrent-pvc-421903bd -2025-06-05 15:52:01,548 - tests.orchestrator - INFO - Waiting for pod test-pod-8c9ea815 to be deleted -2025-06-05 15:52:01,774 - tests.orchestrator - INFO - Pod test-pod-8c9ea815 has been deleted -2025-06-05 15:52:01,774 - tests.orchestrator - INFO - Deleted pod: test-pod-8c9ea815 -2025-06-05 15:52:01,852 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-421903bd to be deleted -2025-06-05 15:52:01,924 - tests.orchestrator - INFO - PVC concurrent-pvc-421903bd has been deleted -2025-06-05 15:52:01,925 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-421903bd -2025-06-05 15:52:01,925 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-74639d37 -2025-06-05 15:52:01,925 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-74639d37 -2025-06-05 15:52:01,925 - tests.orchestrator - INFO - Deleting pod: test-pod-3106ab46 from PVC: concurrent-pvc-74639d37 -2025-06-05 15:52:02,021 - tests.orchestrator - INFO - Waiting for pod test-pod-3106ab46 to be deleted -2025-06-05 15:52:02,092 - tests.orchestrator - INFO - Pod test-pod-3106ab46 has been deleted -2025-06-05 15:52:02,092 - tests.orchestrator - INFO - Deleted pod: test-pod-3106ab46 -2025-06-05 15:52:02,172 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-74639d37 to be deleted -2025-06-05 15:52:02,238 - tests.orchestrator - INFO - PVC concurrent-pvc-74639d37 has been deleted -2025-06-05 15:52:02,238 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-74639d37 -2025-06-05 15:52:02,238 - tests.orchestrator - INFO - Rapid PVC scenario completed -2025-06-05 15:52:02,238 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_concurrent_pvc -2025-06-05 15:52:02,238 - tests.orchestrator - INFO - ------------------------------------------------------------ -2025-06-05 15:52:02,238 - tests.orchestrator - INFO - Deleting pod: test-pod-7abe409e from PVC: test-pvc-83f66d6c -2025-06-05 15:52:02,322 - tests.orchestrator - INFO - Waiting for pod test-pod-7abe409e to be deleted -2025-06-05 15:52:02,390 - tests.orchestrator - INFO - Pod test-pod-7abe409e has been deleted -2025-06-05 15:52:02,390 - tests.orchestrator - INFO - Deleted pod: test-pod-7abe409e -2025-06-05 15:52:02,390 - tests.orchestrator - INFO - Deleting PVC: test-pvc-7dbf87c6 -2025-06-05 15:52:02,390 - tests.orchestrator - INFO - Deleting 2 pods using PVC test-pvc-7dbf87c6 -2025-06-05 15:52:02,390 - tests.orchestrator - INFO - Deleting pod: test-pod-c3e945dc from PVC: test-pvc-7dbf87c6 -2025-06-05 15:52:02,471 - tests.orchestrator - INFO - Waiting for pod test-pod-c3e945dc to be deleted -2025-06-05 15:52:02,537 - tests.orchestrator - INFO - Pod test-pod-c3e945dc has been deleted -2025-06-05 15:52:02,537 - tests.orchestrator - INFO - Deleted pod: test-pod-c3e945dc -2025-06-05 15:52:02,537 - tests.orchestrator - INFO - Deleting pod: test-pod-a9fc1f04 from PVC: test-pvc-7dbf87c6 -2025-06-05 15:52:02,625 - tests.orchestrator - INFO - Waiting for pod test-pod-a9fc1f04 to be deleted -2025-06-05 15:52:02,691 - tests.orchestrator - INFO - Pod test-pod-a9fc1f04 has been deleted -2025-06-05 15:52:02,691 - tests.orchestrator - INFO - Deleted pod: test-pod-a9fc1f04 -2025-06-05 15:52:02,763 - tests.orchestrator - INFO - Waiting for PVC test-pvc-7dbf87c6 to be deleted -2025-06-05 15:52:02,828 - tests.orchestrator - INFO - PVC test-pvc-7dbf87c6 has been deleted -2025-06-05 15:52:02,828 - tests.orchestrator - INFO - Deleted PVC: test-pvc-7dbf87c6 -2025-06-05 15:52:02,828 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations -2025-06-05 15:52:02,828 - tests.orchestrator - INFO - Test completed in 375.88 seconds -2025-06-05 15:52:02,828 - tests.orchestrator - INFO - Cleaning up resources... -2025-06-05 15:52:02,828 - tests.orchestrator - INFO - Deleting PVC: test-pvc-83f66d6c -2025-06-05 15:52:02,903 - tests.orchestrator - INFO - Waiting for PVC test-pvc-83f66d6c to be deleted -2025-06-05 15:52:02,968 - tests.orchestrator - INFO - PVC test-pvc-83f66d6c has been deleted -2025-06-05 15:52:02,968 - tests.orchestrator - INFO - Deleted PVC: test-pvc-83f66d6c -2025-06-05 15:52:02,968 - tests.orchestrator - INFO - Cleanup completed -2025-06-05 15:52:02,968 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-05 15:52:02,968 - tests.orchestrator - INFO - --- Operations --- -2025-06-05 15:52:02,968 - tests.orchestrator - INFO - create_pvc: 5 succeeded, 0 failed (100.0%) -2025-06-05 15:52:02,968 - tests.orchestrator - INFO - attach_pod: 6 succeeded, 0 failed (100.0%) -2025-06-05 15:52:02,968 - tests.orchestrator - INFO - delete_pod: 6 succeeded, 0 failed (100.0%) -2025-06-05 15:52:02,968 - tests.orchestrator - INFO - delete_pvc: 5 succeeded, 0 failed (100.0%) -2025-06-05 15:52:02,968 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) -2025-06-05 15:52:02,968 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) -2025-06-05 15:52:02,968 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-05 15:52:02,969 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-05 15:52:02,969 - tests.orchestrator - INFO - many_to_one: No runs -2025-06-05 15:52:02,969 - tests.orchestrator - INFO - one_to_one: No runs -2025-06-05 15:52:02,969 - tests.orchestrator - INFO - concurrent_pvc: 1 succeeded, 0 failed out of 1 runs (100.0%) -2025-06-05 15:52:02,969 - tests.orchestrator - INFO - ========================================= -2025-06-05 15:52:02,969 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250605_155202.json -2025-06-05 15:52:03,011 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250605_155202_20250605_155202.json -2025-06-05 15:52:03,012 - __main__ - INFO - All tests completed successfully -2025-06-05 15:59:48,062 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-05 15:59:48,062 - __main__ - INFO - Verifying AWS credentials -2025-06-05 15:59:48,063 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 15:59:48,064 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 15:59:48,064 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 15:59:48,064 - __main__ - INFO - Running orchestrator stress test suite -2025-06-05 15:59:48,065 - __main__ - INFO - Starting orchestrator for 300 seconds -2025-06-05 15:59:48,065 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-05 15:59:48,809 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 15:59:49,025 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 15:59:49,027 - __main__ - INFO - Test duration overridden to 300 seconds -2025-06-05 15:59:49,027 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) -2025-06-05 15:59:49,027 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-05 15:59:49,218 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 15:59:49,218 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 15:59:49,218 - tests.orchestrator - INFO - Creating PVC: test-pvc-47b1c666 -2025-06-05 15:59:49,295 - tests.orchestrator - INFO - Created PVC: test-pvc-47b1c666 -2025-06-05 15:59:49,296 - tests.orchestrator - INFO - Waiting for PVC test-pvc-47b1c666 to be bound -2025-06-05 15:59:51,432 - tests.orchestrator - INFO - PVC test-pvc-47b1c666 is bound -2025-06-05 15:59:51,433 - tests.orchestrator - INFO - Creating PVC: test-pvc-13de15dd -2025-06-05 15:59:51,506 - tests.orchestrator - INFO - Created PVC: test-pvc-13de15dd -2025-06-05 15:59:51,507 - tests.orchestrator - INFO - Waiting for PVC test-pvc-13de15dd to be bound -2025-06-05 15:59:53,652 - tests.orchestrator - INFO - PVC test-pvc-13de15dd is bound -2025-06-05 15:59:53,652 - tests.orchestrator - INFO - Creating pod test-pod-b12f5a09 on PVC test-pvc-47b1c666 -2025-06-05 15:59:53,736 - tests.orchestrator - INFO - Created pod: test-pod-b12f5a09 using PVC: test-pvc-47b1c666 -2025-06-05 15:59:53,736 - tests.orchestrator - INFO - Waiting for pod test-pod-b12f5a09 to be ready -2025-06-05 15:59:53,809 - tests.orchestrator - INFO - Pod test-pod-b12f5a09 phase: Pending -2025-06-05 16:00:24,845 - tests.orchestrator - INFO - Pod test-pod-b12f5a09 taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 16:00:24,846 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-b12f5a09 ===== -2025-06-05 16:00:24,914 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:00:24,986 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:00:24,987 - tests.orchestrator - INFO - [2025-06-05 15:59:53+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:00:25,052 - tests.orchestrator - INFO - No logs available -2025-06-05 16:00:25,053 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:00:25,053 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-47b1c666'} -2025-06-05 16:00:25,053 - tests.orchestrator - INFO - - kube-api-access-q49n5: {} -2025-06-05 16:00:25,053 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-b12f5a09 ===== -2025-06-05 16:00:54,038 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b12f5a09 to be ready after 60s -2025-06-05 16:00:54,038 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-b12f5a09 ===== -2025-06-05 16:00:54,106 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:00:54,179 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:00:54,179 - tests.orchestrator - INFO - [2025-06-05 15:59:53+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:00:54,244 - tests.orchestrator - INFO - No logs available -2025-06-05 16:00:54,245 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:00:54,245 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-47b1c666'} -2025-06-05 16:00:54,245 - tests.orchestrator - INFO - - kube-api-access-q49n5: {} -2025-06-05 16:00:54,245 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-b12f5a09 ===== -2025-06-05 16:00:54,245 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b12f5a09 to be ready -2025-06-05 16:00:54,245 - tests.orchestrator - INFO - Creating pod test-pod-397db057 on PVC test-pvc-13de15dd -2025-06-05 16:00:54,335 - tests.orchestrator - INFO - Created pod: test-pod-397db057 using PVC: test-pvc-13de15dd -2025-06-05 16:00:54,336 - tests.orchestrator - INFO - Waiting for pod test-pod-397db057 to be ready -2025-06-05 16:00:54,401 - tests.orchestrator - INFO - Pod test-pod-397db057 phase: Pending -2025-06-05 16:01:25,423 - tests.orchestrator - INFO - Pod test-pod-397db057 taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 16:01:25,423 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-397db057 ===== -2025-06-05 16:01:25,491 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:01:25,563 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:01:25,563 - tests.orchestrator - INFO - [2025-06-05 16:00:54+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:01:25,629 - tests.orchestrator - INFO - No logs available -2025-06-05 16:01:25,629 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:01:25,629 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-13de15dd'} -2025-06-05 16:01:25,630 - tests.orchestrator - INFO - - kube-api-access-q4p66: {} -2025-06-05 16:01:25,630 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-397db057 ===== -2025-06-05 16:01:54,513 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-397db057 to be ready after 60s -2025-06-05 16:01:54,513 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-397db057 ===== -2025-06-05 16:01:54,581 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:01:54,652 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:01:54,653 - tests.orchestrator - INFO - [2025-06-05 16:00:54+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:01:54,717 - tests.orchestrator - INFO - No logs available -2025-06-05 16:01:54,718 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:01:54,718 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-13de15dd'} -2025-06-05 16:01:54,718 - tests.orchestrator - INFO - - kube-api-access-q4p66: {} -2025-06-05 16:01:54,718 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-397db057 ===== -2025-06-05 16:01:54,718 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-397db057 to be ready -2025-06-05 16:01:54,718 - tests.orchestrator - INFO - Creating pod test-pod-b378ede9 on PVC test-pvc-13de15dd -2025-06-05 16:01:54,802 - tests.orchestrator - INFO - Created pod: test-pod-b378ede9 using PVC: test-pvc-13de15dd -2025-06-05 16:01:54,802 - tests.orchestrator - INFO - Waiting for pod test-pod-b378ede9 to be ready -2025-06-05 16:01:54,868 - tests.orchestrator - INFO - Pod test-pod-b378ede9 phase: Pending -2025-06-05 16:02:25,904 - tests.orchestrator - INFO - Pod test-pod-b378ede9 taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 16:02:25,904 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-b378ede9 ===== -2025-06-05 16:02:25,971 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:02:26,043 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:02:26,043 - tests.orchestrator - INFO - [2025-06-05 16:01:54+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:02:26,109 - tests.orchestrator - INFO - No logs available -2025-06-05 16:02:26,110 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:02:26,110 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-13de15dd'} -2025-06-05 16:02:26,110 - tests.orchestrator - INFO - - kube-api-access-ggwzs: {} -2025-06-05 16:02:26,110 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-b378ede9 ===== -2025-06-05 16:02:54,996 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b378ede9 to be ready after 60s -2025-06-05 16:02:54,996 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-b378ede9 ===== -2025-06-05 16:02:55,065 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:02:55,137 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:02:55,137 - tests.orchestrator - INFO - [2025-06-05 16:01:54+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:02:55,204 - tests.orchestrator - INFO - No logs available -2025-06-05 16:02:55,204 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:02:55,204 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-13de15dd'} -2025-06-05 16:02:55,204 - tests.orchestrator - INFO - - kube-api-access-ggwzs: {} -2025-06-05 16:02:55,204 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-b378ede9 ===== -2025-06-05 16:02:55,205 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b378ede9 to be ready -2025-06-05 16:02:55,205 - tests.orchestrator - INFO - Testing read/write between pods test-pod-397db057 and test-pod-b378ede9 sharing PVC test-pvc-13de15dd -2025-06-05 16:02:55,270 - tests.orchestrator - ERROR - Failed in read/write verification: (400) -Reason: Bad Request -HTTP response headers: HTTPHeaderDict({'Audit-Id': 'c1161df1-c0d5-4407-a027-5613955ae526', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 16:02:55 GMT', 'Content-Length': '166'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-397db057 does not have a host assigned","reason":"BadRequest","code":400} - - -2025-06-05 16:02:55,270 - tests.orchestrator - INFO - ============================================================ -2025-06-05 16:02:55,270 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_many_to_one -2025-06-05 16:02:55,270 - tests.orchestrator - INFO - ============================================================ -2025-06-05 16:02:55,270 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -2025-06-05 16:02:55,270 - tests.orchestrator - INFO - STARTING MANY-TO-ONE SCENARIO DIAGNOSTICS -2025-06-05 16:02:55,270 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -2025-06-05 16:02:55,271 - tests.orchestrator - INFO - [MANY2ONE] STEP 1: Creating dedicated PVC: many2one-68cd2a7e -2025-06-05 16:02:55,271 - tests.orchestrator - INFO - [MANY2ONE] Creating PVC manifest with storageClassName: efs-sc -2025-06-05 16:02:55,352 - tests.orchestrator - INFO - [MANY2ONE] PVC many2one-68cd2a7e created successfully -2025-06-05 16:02:55,352 - tests.orchestrator - INFO - [MANY2ONE] Waiting for PVC many2one-68cd2a7e to be bound... -2025-06-05 16:02:55,352 - tests.orchestrator - INFO - Waiting for PVC many2one-68cd2a7e to be bound -2025-06-05 16:02:57,487 - tests.orchestrator - INFO - PVC many2one-68cd2a7e is bound -2025-06-05 16:02:57,552 - tests.orchestrator - INFO - [MANY2ONE] PVC status: phase=Bound, capacity={'storage': '1Gi'} -2025-06-05 16:02:57,552 - tests.orchestrator - INFO - [MANY2ONE] STEP 2: Creating 4 pods for the same PVC many2one-68cd2a7e -2025-06-05 16:02:57,552 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 1/4 for PVC many2one-68cd2a7e -2025-06-05 16:02:57,552 - tests.orchestrator - INFO - Creating pod test-pod-4518fc8f on PVC many2one-68cd2a7e -2025-06-05 16:02:57,632 - tests.orchestrator - INFO - Created pod: test-pod-4518fc8f using PVC: many2one-68cd2a7e -2025-06-05 16:02:57,633 - tests.orchestrator - INFO - Waiting for pod test-pod-4518fc8f to be ready -2025-06-05 16:02:57,699 - tests.orchestrator - INFO - Pod test-pod-4518fc8f phase: Pending -2025-06-05 16:03:28,731 - tests.orchestrator - INFO - Pod test-pod-4518fc8f taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 16:03:28,732 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-4518fc8f ===== -2025-06-05 16:03:28,800 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:03:28,872 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:03:28,872 - tests.orchestrator - INFO - [2025-06-05 16:02:57+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:03:28,937 - tests.orchestrator - INFO - No logs available -2025-06-05 16:03:28,937 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:03:28,937 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-68cd2a7e'} -2025-06-05 16:03:28,937 - tests.orchestrator - INFO - - kube-api-access-v9z8m: {} -2025-06-05 16:03:28,937 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-4518fc8f ===== -2025-06-05 16:03:57,829 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-4518fc8f to be ready after 60s -2025-06-05 16:03:57,829 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-4518fc8f ===== -2025-06-05 16:03:57,897 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:03:57,967 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:03:57,968 - tests.orchestrator - INFO - [2025-06-05 16:02:57+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:03:58,032 - tests.orchestrator - INFO - No logs available -2025-06-05 16:03:58,032 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:03:58,032 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-68cd2a7e'} -2025-06-05 16:03:58,032 - tests.orchestrator - INFO - - kube-api-access-v9z8m: {} -2025-06-05 16:03:58,032 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-4518fc8f ===== -2025-06-05 16:03:58,033 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-4518fc8f to be ready -2025-06-05 16:03:58,033 - tests.orchestrator - ERROR - [MANY2ONE] Failed to create pod 1/4 -2025-06-05 16:03:58,033 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 2/4 for PVC many2one-68cd2a7e -2025-06-05 16:03:58,033 - tests.orchestrator - INFO - Creating pod test-pod-0c1359ff on PVC many2one-68cd2a7e -2025-06-05 16:03:58,117 - tests.orchestrator - INFO - Created pod: test-pod-0c1359ff using PVC: many2one-68cd2a7e -2025-06-05 16:03:58,117 - tests.orchestrator - INFO - Waiting for pod test-pod-0c1359ff to be ready -2025-06-05 16:03:58,183 - tests.orchestrator - INFO - Pod test-pod-0c1359ff phase: Pending -2025-06-05 16:04:29,215 - tests.orchestrator - INFO - Pod test-pod-0c1359ff taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 16:04:29,215 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-0c1359ff ===== -2025-06-05 16:04:29,283 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:04:29,355 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:04:29,356 - tests.orchestrator - INFO - [2025-06-05 16:03:58+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:04:29,422 - tests.orchestrator - INFO - No logs available -2025-06-05 16:04:29,422 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:04:29,422 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-68cd2a7e'} -2025-06-05 16:04:29,422 - tests.orchestrator - INFO - - kube-api-access-6jtzc: {} -2025-06-05 16:04:29,422 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-0c1359ff ===== -2025-06-05 16:04:58,441 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-0c1359ff to be ready after 60s -2025-06-05 16:04:58,441 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-0c1359ff ===== -2025-06-05 16:04:58,507 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:04:58,577 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:04:58,577 - tests.orchestrator - INFO - [2025-06-05 16:03:58+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:04:58,644 - tests.orchestrator - INFO - No logs available -2025-06-05 16:04:58,644 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:04:58,644 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-68cd2a7e'} -2025-06-05 16:04:58,644 - tests.orchestrator - INFO - - kube-api-access-6jtzc: {} -2025-06-05 16:04:58,644 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-0c1359ff ===== -2025-06-05 16:04:58,644 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-0c1359ff to be ready -2025-06-05 16:04:58,644 - tests.orchestrator - ERROR - [MANY2ONE] Failed to create pod 2/4 -2025-06-05 16:04:58,644 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 3/4 for PVC many2one-68cd2a7e -2025-06-05 16:04:58,645 - tests.orchestrator - INFO - Creating pod test-pod-1e078739 on PVC many2one-68cd2a7e -2025-06-05 16:04:58,732 - tests.orchestrator - INFO - Created pod: test-pod-1e078739 using PVC: many2one-68cd2a7e -2025-06-05 16:04:58,732 - tests.orchestrator - INFO - Waiting for pod test-pod-1e078739 to be ready -2025-06-05 16:04:58,800 - tests.orchestrator - INFO - Pod test-pod-1e078739 phase: Pending -2025-06-05 16:05:29,826 - tests.orchestrator - INFO - Pod test-pod-1e078739 taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 16:05:29,826 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-1e078739 ===== -2025-06-05 16:05:29,893 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:05:29,963 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:05:29,963 - tests.orchestrator - INFO - [2025-06-05 16:04:58+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:05:30,027 - tests.orchestrator - INFO - No logs available -2025-06-05 16:05:30,027 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:05:30,028 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-68cd2a7e'} -2025-06-05 16:05:30,028 - tests.orchestrator - INFO - - kube-api-access-rksgf: {} -2025-06-05 16:05:30,028 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-1e078739 ===== -2025-06-05 16:05:58,917 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1e078739 to be ready after 60s -2025-06-05 16:05:58,917 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-1e078739 ===== -2025-06-05 16:05:58,985 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:05:59,057 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:05:59,057 - tests.orchestrator - INFO - [2025-06-05 16:04:58+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:05:59,122 - tests.orchestrator - INFO - No logs available -2025-06-05 16:05:59,122 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:05:59,122 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-68cd2a7e'} -2025-06-05 16:05:59,122 - tests.orchestrator - INFO - - kube-api-access-rksgf: {} -2025-06-05 16:05:59,122 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-1e078739 ===== -2025-06-05 16:05:59,122 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1e078739 to be ready -2025-06-05 16:05:59,122 - tests.orchestrator - ERROR - [MANY2ONE] Failed to create pod 3/4 -2025-06-05 16:05:59,122 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 4/4 for PVC many2one-68cd2a7e -2025-06-05 16:05:59,122 - tests.orchestrator - INFO - Creating pod test-pod-a5d14df4 on PVC many2one-68cd2a7e -2025-06-05 16:05:59,208 - tests.orchestrator - INFO - Created pod: test-pod-a5d14df4 using PVC: many2one-68cd2a7e -2025-06-05 16:05:59,208 - tests.orchestrator - INFO - Waiting for pod test-pod-a5d14df4 to be ready -2025-06-05 16:05:59,273 - tests.orchestrator - INFO - Pod test-pod-a5d14df4 phase: Pending -2025-06-05 16:06:30,377 - tests.orchestrator - INFO - Pod test-pod-a5d14df4 taking longer than expected to become ready (31.2s). Collecting diagnostics... -2025-06-05 16:06:30,377 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-a5d14df4 ===== -2025-06-05 16:06:30,444 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:06:30,516 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:06:30,517 - tests.orchestrator - INFO - [2025-06-05 16:05:59+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:06:30,583 - tests.orchestrator - INFO - No logs available -2025-06-05 16:06:30,583 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:06:30,584 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-68cd2a7e'} -2025-06-05 16:06:30,584 - tests.orchestrator - INFO - - kube-api-access-dcxmb: {} -2025-06-05 16:06:30,584 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-a5d14df4 ===== -2025-06-05 16:06:59,487 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a5d14df4 to be ready after 60s -2025-06-05 16:06:59,487 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-a5d14df4 ===== -2025-06-05 16:06:59,555 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:06:59,627 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:06:59,628 - tests.orchestrator - INFO - [2025-06-05 16:05:59+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:06:59,694 - tests.orchestrator - INFO - No logs available -2025-06-05 16:06:59,694 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:06:59,694 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-68cd2a7e'} -2025-06-05 16:06:59,695 - tests.orchestrator - INFO - - kube-api-access-dcxmb: {} -2025-06-05 16:06:59,695 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-a5d14df4 ===== -2025-06-05 16:06:59,695 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a5d14df4 to be ready -2025-06-05 16:06:59,695 - tests.orchestrator - ERROR - [MANY2ONE] Failed to create pod 4/4 -2025-06-05 16:06:59,695 - tests.orchestrator - INFO - [MANY2ONE] Created 0/4 pods successfully -2025-06-05 16:06:59,695 - tests.orchestrator - ERROR - [MANY2ONE] FAILED: Insufficient pods created (0), need at least 2 for read/write test -2025-06-05 16:06:59,695 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_many_to_one -2025-06-05 16:06:59,695 - tests.orchestrator - INFO - ------------------------------------------------------------ -2025-06-05 16:06:59,695 - tests.orchestrator - INFO - Deleting pod: test-pod-1e078739 from PVC: many2one-68cd2a7e -2025-06-05 16:06:59,777 - tests.orchestrator - INFO - Waiting for pod test-pod-1e078739 to be deleted -2025-06-05 16:06:59,841 - tests.orchestrator - INFO - Pod test-pod-1e078739 has been deleted -2025-06-05 16:06:59,842 - tests.orchestrator - INFO - Deleted pod: test-pod-1e078739 -2025-06-05 16:06:59,842 - tests.orchestrator - INFO - Deleting PVC: test-pvc-13de15dd -2025-06-05 16:06:59,842 - tests.orchestrator - INFO - Deleting 2 pods using PVC test-pvc-13de15dd -2025-06-05 16:06:59,842 - tests.orchestrator - INFO - Deleting pod: test-pod-397db057 from PVC: test-pvc-13de15dd -2025-06-05 16:06:59,923 - tests.orchestrator - INFO - Waiting for pod test-pod-397db057 to be deleted -2025-06-05 16:06:59,989 - tests.orchestrator - INFO - Pod test-pod-397db057 has been deleted -2025-06-05 16:06:59,989 - tests.orchestrator - INFO - Deleted pod: test-pod-397db057 -2025-06-05 16:06:59,989 - tests.orchestrator - INFO - Deleting pod: test-pod-b378ede9 from PVC: test-pvc-13de15dd -2025-06-05 16:07:00,077 - tests.orchestrator - INFO - Waiting for pod test-pod-b378ede9 to be deleted -2025-06-05 16:07:00,142 - tests.orchestrator - INFO - Pod test-pod-b378ede9 has been deleted -2025-06-05 16:07:00,142 - tests.orchestrator - INFO - Deleted pod: test-pod-b378ede9 -2025-06-05 16:07:00,213 - tests.orchestrator - INFO - Waiting for PVC test-pvc-13de15dd to be deleted -2025-06-05 16:07:00,280 - tests.orchestrator - INFO - PVC test-pvc-13de15dd has been deleted -2025-06-05 16:07:00,280 - tests.orchestrator - INFO - Deleted PVC: test-pvc-13de15dd -2025-06-05 16:07:00,280 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations -2025-06-05 16:07:00,280 - tests.orchestrator - INFO - Test completed in 431.25 seconds -2025-06-05 16:07:00,280 - tests.orchestrator - INFO - Cleaning up resources... -2025-06-05 16:07:00,280 - tests.orchestrator - INFO - Deleting pod: test-pod-b12f5a09 from PVC: test-pvc-47b1c666 -2025-06-05 16:07:00,364 - tests.orchestrator - INFO - Waiting for pod test-pod-b12f5a09 to be deleted -2025-06-05 16:07:00,430 - tests.orchestrator - INFO - Pod test-pod-b12f5a09 has been deleted -2025-06-05 16:07:00,430 - tests.orchestrator - INFO - Deleted pod: test-pod-b12f5a09 -2025-06-05 16:07:00,431 - tests.orchestrator - INFO - Deleting pod: test-pod-4518fc8f from PVC: many2one-68cd2a7e -2025-06-05 16:07:00,512 - tests.orchestrator - INFO - Waiting for pod test-pod-4518fc8f to be deleted -2025-06-05 16:07:00,579 - tests.orchestrator - INFO - Pod test-pod-4518fc8f has been deleted -2025-06-05 16:07:00,579 - tests.orchestrator - INFO - Deleted pod: test-pod-4518fc8f -2025-06-05 16:07:00,579 - tests.orchestrator - INFO - Deleting pod: test-pod-0c1359ff from PVC: many2one-68cd2a7e -2025-06-05 16:07:00,659 - tests.orchestrator - INFO - Waiting for pod test-pod-0c1359ff to be deleted -2025-06-05 16:07:00,725 - tests.orchestrator - INFO - Pod test-pod-0c1359ff has been deleted -2025-06-05 16:07:00,726 - tests.orchestrator - INFO - Deleted pod: test-pod-0c1359ff -2025-06-05 16:07:00,726 - tests.orchestrator - INFO - Deleting pod: test-pod-a5d14df4 from PVC: many2one-68cd2a7e -2025-06-05 16:07:00,813 - tests.orchestrator - INFO - Waiting for pod test-pod-a5d14df4 to be deleted -2025-06-05 16:07:00,878 - tests.orchestrator - INFO - Pod test-pod-a5d14df4 has been deleted -2025-06-05 16:07:00,878 - tests.orchestrator - INFO - Deleted pod: test-pod-a5d14df4 -2025-06-05 16:07:00,878 - tests.orchestrator - INFO - Deleting PVC: test-pvc-47b1c666 -2025-06-05 16:07:00,951 - tests.orchestrator - INFO - Waiting for PVC test-pvc-47b1c666 to be deleted -2025-06-05 16:07:01,017 - tests.orchestrator - INFO - PVC test-pvc-47b1c666 has been deleted -2025-06-05 16:07:01,018 - tests.orchestrator - INFO - Deleted PVC: test-pvc-47b1c666 -2025-06-05 16:07:01,018 - tests.orchestrator - INFO - Deleting PVC: many2one-68cd2a7e -2025-06-05 16:07:01,090 - tests.orchestrator - INFO - Waiting for PVC many2one-68cd2a7e to be deleted -2025-06-05 16:07:01,157 - tests.orchestrator - INFO - PVC many2one-68cd2a7e has been deleted -2025-06-05 16:07:01,157 - tests.orchestrator - INFO - Deleted PVC: many2one-68cd2a7e -2025-06-05 16:07:01,157 - tests.orchestrator - INFO - Cleanup completed -2025-06-05 16:07:01,157 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-05 16:07:01,157 - tests.orchestrator - INFO - --- Operations --- -2025-06-05 16:07:01,157 - tests.orchestrator - INFO - create_pvc: 2 succeeded, 0 failed (100.0%) -2025-06-05 16:07:01,157 - tests.orchestrator - INFO - attach_pod: 7 succeeded, 0 failed (100.0%) -2025-06-05 16:07:01,157 - tests.orchestrator - INFO - delete_pod: 7 succeeded, 0 failed (100.0%) -2025-06-05 16:07:01,157 - tests.orchestrator - INFO - delete_pvc: 3 succeeded, 0 failed (100.0%) -2025-06-05 16:07:01,157 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) -2025-06-05 16:07:01,157 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) -2025-06-05 16:07:01,157 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-05 16:07:01,158 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-05 16:07:01,158 - tests.orchestrator - INFO - many_to_one: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-05 16:07:01,158 - tests.orchestrator - INFO - one_to_one: No runs -2025-06-05 16:07:01,158 - tests.orchestrator - INFO - concurrent_pvc: No runs -2025-06-05 16:07:01,158 - tests.orchestrator - INFO - ========================================= -2025-06-05 16:07:01,158 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250605_160701.json -2025-06-05 16:07:01,200 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250605_160701_20250605_160701.json -2025-06-05 16:07:01,200 - __main__ - INFO - All tests completed successfully -2025-06-05 16:08:43,289 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-05 16:08:43,289 - __main__ - INFO - Verifying AWS credentials -2025-06-05 16:08:43,290 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 16:08:43,291 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 16:08:43,291 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 16:08:43,292 - __main__ - INFO - Running orchestrator stress test suite -2025-06-05 16:08:43,292 - __main__ - INFO - Starting orchestrator for 300 seconds -2025-06-05 16:08:43,292 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-05 16:08:44,245 - __main__ - INFO - Test duration overridden to 300 seconds -2025-06-05 16:08:44,246 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) -2025-06-05 16:15:09,979 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250605_161509.json -2025-06-05 16:15:10,022 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250605_161509_20250605_161509.json -2025-06-05 16:15:10,022 - __main__ - INFO - All tests completed successfully -2025-06-05 16:21:33,305 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-05 16:21:33,305 - __main__ - INFO - Verifying AWS credentials -2025-06-05 16:21:33,306 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 16:21:33,307 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 16:21:33,307 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 16:21:33,308 - __main__ - INFO - Running orchestrator stress test suite -2025-06-05 16:21:33,308 - __main__ - INFO - Starting orchestrator for 300 seconds -2025-06-05 16:21:33,308 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-05 16:21:34,256 - __main__ - ERROR - Error running tests: (401) -Reason: Unauthorized -HTTP response headers: HTTPHeaderDict({'Audit-Id': 'ae5fa875-33f1-44a7-a754-80136c368fa1', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 16:21:34 GMT', 'Content-Length': '129'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} - -Traceback (most recent call last): - File "/local/home/ibhard/efs-csi-driver-fork/aws-efs-csi-driver/test/e2e/stress-scale-tests/run_tests.py", line 209, in main - orchestrator = EFSCSIOrchestrator(config_file=orchestrator_config) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/local/home/ibhard/efs-csi-driver-fork/aws-efs-csi-driver/test/e2e/stress-scale-tests/tests/orchestrator.py", line 92, in __init__ - self._ensure_namespace_exists() - File "/local/home/ibhard/efs-csi-driver-fork/aws-efs-csi-driver/test/e2e/stress-scale-tests/tests/orchestrator.py", line 98, in _ensure_namespace_exists - self.core_v1.read_namespace(name=self.namespace) - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api/core_v1_api.py", line 22992, in read_namespace - return self.read_namespace_with_http_info(name, **kwargs) # noqa: E501 - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api/core_v1_api.py", line 23071, in read_namespace_with_http_info - return self.api_client.call_api( - ^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api_client.py", line 348, in call_api - return self.__call_api(resource_path, method, - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api_client.py", line 180, in __call_api - response_data = self.request( - ^^^^^^^^^^^^^ - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api_client.py", line 373, in request - return self.rest_client.GET(url, - ^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/rest.py", line 244, in GET - return self.request("GET", url, - ^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/rest.py", line 238, in request - raise ApiException(http_resp=r) -kubernetes.client.exceptions.ApiException: (401) -Reason: Unauthorized -HTTP response headers: HTTPHeaderDict({'Audit-Id': 'ae5fa875-33f1-44a7-a754-80136c368fa1', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 16:21:34 GMT', 'Content-Length': '129'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} - - -2025-06-05 16:21:55,167 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-05 16:21:55,168 - __main__ - INFO - Verifying AWS credentials -2025-06-05 16:21:55,169 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 16:21:55,169 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 16:21:55,169 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 16:21:55,170 - __main__ - INFO - Running orchestrator stress test suite -2025-06-05 16:21:55,170 - __main__ - INFO - Starting orchestrator for 300 seconds -2025-06-05 16:21:55,170 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-05 16:21:56,123 - __main__ - INFO - Test duration overridden to 300 seconds -2025-06-05 16:21:56,123 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) -2025-06-05 16:23:20,187 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-05 16:23:20,187 - __main__ - INFO - Verifying AWS credentials -2025-06-05 16:23:20,188 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 16:23:20,189 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 16:23:20,189 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 16:23:20,190 - __main__ - INFO - Running orchestrator stress test suite -2025-06-05 16:23:20,190 - __main__ - INFO - Starting orchestrator for 300 seconds -2025-06-05 16:23:20,190 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-05 16:23:21,135 - __main__ - INFO - Test duration overridden to 300 seconds -2025-06-05 16:23:21,135 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) -2025-06-05 16:23:53,806 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-05 16:23:53,806 - __main__ - INFO - Verifying AWS credentials -2025-06-05 16:23:53,807 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 16:23:53,808 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 16:23:53,808 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 16:23:53,809 - __main__ - INFO - Running orchestrator stress test suite -2025-06-05 16:23:53,809 - __main__ - INFO - Starting orchestrator for 300 seconds -2025-06-05 16:23:53,809 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-05 16:23:54,744 - __main__ - INFO - Test duration overridden to 300 seconds -2025-06-05 16:23:54,744 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) -2025-06-05 16:30:10,897 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250605_163010.json -2025-06-05 16:30:10,941 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250605_163010_20250605_163010.json -2025-06-05 16:30:10,941 - __main__ - INFO - All tests completed successfully -2025-06-05 16:31:13,588 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-05 16:31:13,588 - __main__ - INFO - Verifying AWS credentials -2025-06-05 16:31:13,589 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 16:31:13,590 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 16:31:13,590 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 16:31:13,590 - __main__ - INFO - Running orchestrator stress test suite -2025-06-05 16:31:13,590 - __main__ - INFO - Starting orchestrator for 300 seconds -2025-06-05 16:31:13,590 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-05 16:31:14,546 - __main__ - INFO - Test duration overridden to 300 seconds -2025-06-05 16:31:14,546 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) -2025-06-05 16:33:39,799 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-05 16:33:39,799 - __main__ - INFO - Verifying AWS credentials -2025-06-05 16:33:39,800 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 16:33:39,800 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 16:33:39,801 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 16:33:39,801 - __main__ - INFO - Running orchestrator stress test suite -2025-06-05 16:33:39,801 - __main__ - INFO - Starting orchestrator for 300 seconds -2025-06-05 16:33:39,801 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-05 16:33:40,752 - __main__ - INFO - Test duration overridden to 300 seconds -2025-06-05 16:33:40,752 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) -2025-06-05 16:35:01,084 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-05 16:35:01,084 - __main__ - INFO - Verifying AWS credentials -2025-06-05 16:35:01,085 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 16:35:01,085 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 16:35:01,086 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 16:35:01,086 - __main__ - INFO - Running orchestrator stress test suite -2025-06-05 16:35:01,086 - __main__ - INFO - Starting orchestrator for 300 seconds -2025-06-05 16:35:01,086 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-05 16:35:02,054 - __main__ - INFO - Test duration overridden to 300 seconds -2025-06-05 16:35:02,054 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) -2025-06-05 16:40:17,452 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250605_164017.json -2025-06-05 16:40:17,511 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250605_164017_20250605_164017.json -2025-06-05 16:40:17,511 - __main__ - INFO - All tests completed successfully -2025-06-05 16:47:08,706 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-05 16:47:08,706 - __main__ - INFO - Verifying AWS credentials -2025-06-05 16:47:08,707 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 16:47:08,708 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 16:47:08,708 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 16:47:08,708 - __main__ - INFO - Running orchestrator stress test suite -2025-06-05 16:47:08,709 - __main__ - INFO - Starting orchestrator for 300 seconds -2025-06-05 16:47:08,709 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-05 16:47:09,670 - __main__ - INFO - Test duration overridden to 300 seconds -2025-06-05 16:47:09,670 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) -2025-06-05 16:49:33,984 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-05 16:49:33,984 - __main__ - INFO - Verifying AWS credentials -2025-06-05 16:49:33,985 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 16:49:33,985 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 16:49:33,986 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 16:49:33,986 - __main__ - INFO - Running orchestrator stress test suite -2025-06-05 16:49:33,986 - __main__ - INFO - Starting orchestrator for 300 seconds -2025-06-05 16:49:33,986 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-05 16:49:34,939 - __main__ - INFO - Test duration overridden to 300 seconds -2025-06-05 16:49:34,939 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) -2025-06-05 16:55:52,177 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250605_165552.json -2025-06-05 16:55:52,220 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250605_165552_20250605_165552.json -2025-06-05 16:55:52,220 - __main__ - INFO - All tests completed successfully -2025-06-05 17:03:06,202 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-05 17:03:06,202 - __main__ - INFO - Verifying AWS credentials -2025-06-05 17:03:06,203 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 17:03:06,204 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 17:03:06,204 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 17:03:06,204 - __main__ - INFO - Running orchestrator stress test suite -2025-06-05 17:03:06,204 - __main__ - INFO - Starting orchestrator for 300 seconds -2025-06-05 17:03:06,205 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-05 17:03:07,165 - __main__ - INFO - Test duration overridden to 300 seconds -2025-06-05 17:03:07,165 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) -2025-06-05 17:03:42,089 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-05 17:03:42,089 - __main__ - INFO - Verifying AWS credentials -2025-06-05 17:03:42,090 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 17:03:42,090 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 17:03:42,091 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 17:03:42,091 - __main__ - INFO - Running orchestrator stress test suite -2025-06-05 17:03:42,091 - __main__ - INFO - Starting orchestrator for 300 seconds -2025-06-05 17:03:42,091 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-05 17:03:43,043 - __main__ - INFO - Test duration overridden to 300 seconds -2025-06-05 17:03:43,043 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) -2025-06-05 17:04:04,822 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-05 17:04:04,822 - __main__ - INFO - Verifying AWS credentials -2025-06-05 17:04:04,823 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 17:04:04,824 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 17:04:04,824 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 17:04:04,825 - __main__ - INFO - Running orchestrator stress test suite -2025-06-05 17:04:04,825 - __main__ - INFO - Starting orchestrator for 300 seconds -2025-06-05 17:04:04,825 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-05 17:04:05,759 - __main__ - INFO - Test duration overridden to 300 seconds -2025-06-05 17:04:05,759 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) -2025-06-05 17:05:02,167 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-05 17:05:02,167 - __main__ - INFO - Verifying AWS credentials -2025-06-05 17:05:02,168 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 17:05:02,168 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 17:05:02,168 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 17:05:02,169 - __main__ - INFO - Running orchestrator stress test suite -2025-06-05 17:05:02,169 - __main__ - INFO - Starting orchestrator for 300 seconds -2025-06-05 17:05:02,169 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-05 17:05:03,121 - __main__ - INFO - Test duration overridden to 300 seconds -2025-06-05 17:05:03,121 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) -2025-06-05 17:05:25,486 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-05 17:05:25,486 - __main__ - INFO - Verifying AWS credentials -2025-06-05 17:05:25,487 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 17:05:25,488 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 17:05:25,488 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 17:05:25,489 - __main__ - INFO - Running orchestrator stress test suite -2025-06-05 17:05:25,489 - __main__ - INFO - Starting orchestrator for 300 seconds -2025-06-05 17:05:25,489 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-05 17:05:26,435 - __main__ - INFO - Test duration overridden to 300 seconds -2025-06-05 17:05:26,435 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) -2025-06-05 17:12:16,582 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250605_171216.json -2025-06-05 17:12:16,625 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250605_171216_20250605_171216.json -2025-06-05 17:12:16,625 - __main__ - INFO - All tests completed successfully -2025-06-05 18:22:15,497 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-05 18:22:15,497 - __main__ - INFO - Verifying AWS credentials -2025-06-05 18:22:15,498 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 18:22:15,498 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 18:22:15,499 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 18:22:15,499 - __main__ - INFO - Running orchestrator stress test suite -2025-06-05 18:22:15,499 - __main__ - INFO - Starting orchestrator for 300 seconds -2025-06-05 18:22:15,499 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-05 18:22:16,529 - __main__ - ERROR - Error running tests: (401) -Reason: Unauthorized -HTTP response headers: HTTPHeaderDict({'Audit-Id': '1a400704-0ba2-4e3d-a421-03adbfa89ebe', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 18:22:16 GMT', 'Content-Length': '129'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} - -Traceback (most recent call last): - File "/local/home/ibhard/efs-csi-driver-fork/aws-efs-csi-driver/test/e2e/stress-scale-tests/run_tests.py", line 209, in main - orchestrator = EFSCSIOrchestrator(config_file=orchestrator_config) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/local/home/ibhard/efs-csi-driver-fork/aws-efs-csi-driver/test/e2e/stress-scale-tests/tests/orchestrator.py", line 92, in __init__ - self._ensure_namespace_exists() - File "/local/home/ibhard/efs-csi-driver-fork/aws-efs-csi-driver/test/e2e/stress-scale-tests/tests/orchestrator.py", line 98, in _ensure_namespace_exists - self.core_v1.read_namespace(name=self.namespace) - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api/core_v1_api.py", line 22992, in read_namespace - return self.read_namespace_with_http_info(name, **kwargs) # noqa: E501 - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api/core_v1_api.py", line 23071, in read_namespace_with_http_info - return self.api_client.call_api( - ^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api_client.py", line 348, in call_api - return self.__call_api(resource_path, method, - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api_client.py", line 180, in __call_api - response_data = self.request( - ^^^^^^^^^^^^^ - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api_client.py", line 373, in request - return self.rest_client.GET(url, - ^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/rest.py", line 244, in GET - return self.request("GET", url, - ^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/rest.py", line 238, in request - raise ApiException(http_resp=r) -kubernetes.client.exceptions.ApiException: (401) -Reason: Unauthorized -HTTP response headers: HTTPHeaderDict({'Audit-Id': '1a400704-0ba2-4e3d-a421-03adbfa89ebe', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 18:22:16 GMT', 'Content-Length': '129'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} - - -2025-06-05 18:22:53,399 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-05 18:22:53,399 - __main__ - INFO - Verifying AWS credentials -2025-06-05 18:22:53,400 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 18:22:53,401 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 18:22:53,401 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 18:22:53,402 - __main__ - INFO - Running orchestrator stress test suite -2025-06-05 18:22:53,402 - __main__ - INFO - Starting orchestrator for 300 seconds -2025-06-05 18:22:53,402 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-05 18:22:54,350 - __main__ - INFO - Test duration overridden to 300 seconds -2025-06-05 18:22:54,350 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) -2025-06-05 18:29:32,601 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250605_182932.json -2025-06-05 18:29:32,643 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250605_182932_20250605_182932.json -2025-06-05 18:29:32,643 - __main__ - INFO - All tests completed successfully -2025-06-05 18:38:35,023 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-05 18:38:35,023 - __main__ - INFO - Verifying AWS credentials -2025-06-05 18:38:35,024 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 18:38:35,025 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 18:38:35,025 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 18:38:35,025 - __main__ - INFO - Running orchestrator stress test suite -2025-06-05 18:38:35,025 - __main__ - INFO - Starting orchestrator for 7200 seconds -2025-06-05 18:38:35,026 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-05 18:38:35,987 - __main__ - INFO - Test duration overridden to 7200 seconds -2025-06-05 18:38:35,987 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) -2025-06-05 19:12:14,835 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-05 19:12:14,835 - __main__ - INFO - Verifying AWS credentials -2025-06-05 19:12:14,836 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 19:12:14,836 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 19:12:14,837 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 19:12:14,837 - __main__ - INFO - Running orchestrator stress test suite -2025-06-05 19:12:14,837 - __main__ - INFO - Starting orchestrator for 300 seconds -2025-06-05 19:12:14,837 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-05 19:12:15,782 - __main__ - INFO - Test duration overridden to 300 seconds -2025-06-05 19:12:15,782 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) -2025-06-05 19:28:23,860 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-05 19:28:23,861 - __main__ - INFO - Verifying AWS credentials -2025-06-05 19:28:23,862 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 19:28:23,862 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 19:28:23,862 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 19:28:23,863 - __main__ - INFO - Running orchestrator stress test suite -2025-06-05 19:28:23,863 - __main__ - INFO - Starting orchestrator for 300 seconds -2025-06-05 19:28:23,863 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-05 19:28:24,815 - __main__ - ERROR - Error running tests: (401) -Reason: Unauthorized -HTTP response headers: HTTPHeaderDict({'Audit-Id': '0dff4cee-4147-4a36-b942-2640bcda6774', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 19:28:24 GMT', 'Content-Length': '129'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} - -Traceback (most recent call last): - File "/local/home/ibhard/efs-csi-driver-fork/aws-efs-csi-driver/test/e2e/stress-scale-tests/run_tests.py", line 209, in main - orchestrator = EFSCSIOrchestrator(config_file=orchestrator_config) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/local/home/ibhard/efs-csi-driver-fork/aws-efs-csi-driver/test/e2e/stress-scale-tests/tests/orchestrator.py", line 92, in __init__ - self._ensure_namespace_exists() - File "/local/home/ibhard/efs-csi-driver-fork/aws-efs-csi-driver/test/e2e/stress-scale-tests/tests/orchestrator.py", line 98, in _ensure_namespace_exists - self.core_v1.read_namespace(name=self.namespace) - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api/core_v1_api.py", line 22992, in read_namespace - return self.read_namespace_with_http_info(name, **kwargs) # noqa: E501 - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api/core_v1_api.py", line 23071, in read_namespace_with_http_info - return self.api_client.call_api( - ^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api_client.py", line 348, in call_api - return self.__call_api(resource_path, method, - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api_client.py", line 180, in __call_api - response_data = self.request( - ^^^^^^^^^^^^^ - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api_client.py", line 373, in request - return self.rest_client.GET(url, - ^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/rest.py", line 244, in GET - return self.request("GET", url, - ^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/rest.py", line 238, in request - raise ApiException(http_resp=r) -kubernetes.client.exceptions.ApiException: (401) -Reason: Unauthorized -HTTP response headers: HTTPHeaderDict({'Audit-Id': '0dff4cee-4147-4a36-b942-2640bcda6774', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 19:28:24 GMT', 'Content-Length': '129'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} - - -2025-06-05 19:28:50,833 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-05 19:28:50,833 - __main__ - INFO - Verifying AWS credentials -2025-06-05 19:28:50,834 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 19:28:50,835 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 19:28:50,835 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 19:28:50,836 - __main__ - INFO - Running orchestrator stress test suite -2025-06-05 19:28:50,836 - __main__ - INFO - Starting orchestrator for 300 seconds -2025-06-05 19:28:50,836 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-05 19:28:51,800 - __main__ - INFO - Test duration overridden to 300 seconds -2025-06-05 19:28:51,800 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) -2025-06-05 19:34:07,404 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250605_193407.json -2025-06-05 19:34:07,447 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250605_193407_20250605_193407.json -2025-06-05 19:34:07,447 - __main__ - INFO - All tests completed successfully -2025-06-05 19:52:24,900 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-05 19:52:24,900 - __main__ - INFO - Verifying AWS credentials -2025-06-05 19:52:24,901 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 19:52:24,902 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 19:52:24,902 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 19:52:24,902 - __main__ - INFO - Running orchestrator stress test suite -2025-06-05 19:52:24,903 - __main__ - INFO - Starting orchestrator for 300 seconds -2025-06-05 19:52:24,903 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-05 19:52:25,880 - __main__ - INFO - Test duration overridden to 300 seconds -2025-06-05 19:52:25,880 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) -2025-06-05 20:01:24,756 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250605_200124.json -2025-06-05 20:01:24,798 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250605_200124_20250605_200124.json -2025-06-05 20:01:24,798 - __main__ - INFO - All tests completed successfully -2025-06-05 20:06:45,182 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-05 20:06:45,182 - __main__ - INFO - Verifying AWS credentials -2025-06-05 20:06:45,183 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 20:06:45,184 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 20:06:45,184 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-05 20:06:45,184 - __main__ - INFO - Running orchestrator stress test suite -2025-06-05 20:06:45,184 - __main__ - INFO - Starting orchestrator for 300 seconds -2025-06-05 20:06:45,185 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-05 20:06:46,152 - __main__ - INFO - Test duration overridden to 300 seconds -2025-06-05 20:06:46,152 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) -2025-06-05 20:17:16,472 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250605_201716.json -2025-06-05 20:17:16,514 - __main__ - INFO - JSON report generated: reports/general/efs_tests_20250605_201716_20250605_201716.json -2025-06-05 20:17:16,515 - __main__ - INFO - All tests completed successfully diff --git a/test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749045783.log b/test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749045783.log deleted file mode 100644 index b0f022fcd..000000000 --- a/test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749045783.log +++ /dev/null @@ -1,8 +0,0 @@ -STDOUT: -2025-06-04 14:03:04 [▶] Setting credentials expiry window to 30 minutes -2025-06-04 14:03:05 [▶] role ARN for the current session is "arn:aws:sts::745939127895:assumed-role/Admin/ibhard-Isengard" - - -STDERR: -Error: resolving cluster version: invalid version, 1.25 is no longer supported, supported values: 1.26, 1.27, 1.28, 1.29, 1.30, 1.31, 1.32, 1.33 -see also: https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html diff --git a/test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749048222.log b/test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749048222.log deleted file mode 100644 index 952f62886..000000000 --- a/test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749048222.log +++ /dev/null @@ -1,8 +0,0 @@ -STDOUT: -2025-06-04 14:43:43 [▶] Setting credentials expiry window to 30 minutes -2025-06-04 14:43:43 [▶] role ARN for the current session is "arn:aws:sts::745939127895:assumed-role/Admin/ibhard-Isengard" - - -STDERR: -Error: resolving cluster version: invalid version, 1.25 is no longer supported, supported values: 1.26, 1.27, 1.28, 1.29, 1.30, 1.31, 1.32, 1.33 -see also: https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html diff --git a/test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749048235.log b/test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749048235.log deleted file mode 100644 index 1cf3eac2b..000000000 --- a/test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749048235.log +++ /dev/null @@ -1,17 +0,0 @@ -STDOUT: -2025-06-04 14:43:56 [▶] Setting credentials expiry window to 30 minutes -2025-06-04 14:43:56 [▶] role ARN for the current session is "arn:aws:sts::745939127895:assumed-role/Admin/ibhard-Isengard" -2025-06-04 14:43:56 [ℹ] eksctl version 0.208.0 -2025-06-04 14:43:56 [ℹ] using region us-west-2 -2025-06-04 14:43:56 [!] Amazon EKS will no longer publish EKS-optimized Amazon Linux 2 (AL2) AMIs after November 26th, 2025. Additionally, Kubernetes version 1.32 is the last version for which Amazon EKS will release AL2 AMIs. From version 1.33 onwards, Amazon EKS will continue to release AL2023 and Bottlerocket based AMIs. The default AMI family when creating clusters and nodegroups in Eksctl will be changed to AL2023 in the future. -2025-06-04 14:43:56 [▶] determining availability zones -2025-06-04 14:43:57 [ℹ] setting availability zones to [us-west-2d us-west-2a us-west-2b] -2025-06-04 14:43:57 [▶] VPC CIDR (192.168.0.0/16) was divided into 8 subnets [192.168.0.0/19 192.168.32.0/19 192.168.64.0/19 192.168.96.0/19 192.168.128.0/19 192.168.160.0/19 192.168.192.0/19 192.168.224.0/19] -2025-06-04 14:43:57 [ℹ] subnets for us-west-2d - public:192.168.0.0/19 private:192.168.96.0/19 -2025-06-04 14:43:57 [ℹ] subnets for us-west-2a - public:192.168.32.0/19 private:192.168.128.0/19 -2025-06-04 14:43:57 [ℹ] subnets for us-west-2b - public:192.168.64.0/19 private:192.168.160.0/19 -2025-06-04 14:43:57 [▶] resolving AMI using SSM Parameter resolver for region us-west-2, instanceType t3.large and imageFamily AmazonLinux2 - - -STDERR: -Error: unable to determine AMI to use: error getting AMI from SSM Parameter Store: operation error SSM: GetParameter, https response error StatusCode: 400, RequestID: a443a5ad-af0e-4afb-b46f-a021532755a5, ParameterNotFound: . please verify that AMI Family is supported diff --git a/test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749048734.log b/test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749048734.log deleted file mode 100644 index 678235019..000000000 --- a/test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749048734.log +++ /dev/null @@ -1,17 +0,0 @@ -STDOUT: -2025-06-04 14:52:15 [▶] Setting credentials expiry window to 30 minutes -2025-06-04 14:52:15 [▶] role ARN for the current session is "arn:aws:sts::745939127895:assumed-role/Admin/ibhard-Isengard" -2025-06-04 14:52:16 [ℹ] eksctl version 0.208.0 -2025-06-04 14:52:16 [ℹ] using region us-west-2 -2025-06-04 14:52:16 [!] Amazon EKS will no longer publish EKS-optimized Amazon Linux 2 (AL2) AMIs after November 26th, 2025. Additionally, Kubernetes version 1.32 is the last version for which Amazon EKS will release AL2 AMIs. From version 1.33 onwards, Amazon EKS will continue to release AL2023 and Bottlerocket based AMIs. The default AMI family when creating clusters and nodegroups in Eksctl will be changed to AL2023 in the future. -2025-06-04 14:52:16 [▶] determining availability zones -2025-06-04 14:52:16 [ℹ] setting availability zones to [us-west-2a us-west-2c us-west-2d] -2025-06-04 14:52:16 [▶] VPC CIDR (192.168.0.0/16) was divided into 8 subnets [192.168.0.0/19 192.168.32.0/19 192.168.64.0/19 192.168.96.0/19 192.168.128.0/19 192.168.160.0/19 192.168.192.0/19 192.168.224.0/19] -2025-06-04 14:52:16 [ℹ] subnets for us-west-2a - public:192.168.0.0/19 private:192.168.96.0/19 -2025-06-04 14:52:16 [ℹ] subnets for us-west-2c - public:192.168.32.0/19 private:192.168.128.0/19 -2025-06-04 14:52:16 [ℹ] subnets for us-west-2d - public:192.168.64.0/19 private:192.168.160.0/19 -2025-06-04 14:52:16 [▶] resolving AMI using SSM Parameter resolver for region us-west-2, instanceType t3.large and imageFamily AmazonLinux2 - - -STDERR: -Error: unable to determine AMI to use: error getting AMI from SSM Parameter Store: operation error SSM: GetParameter, https response error StatusCode: 400, RequestID: 4650ea1c-e137-440e-835d-9214a82b7beb, ParameterNotFound: . please verify that AMI Family is supported diff --git a/test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749051325.log b/test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749051325.log deleted file mode 100644 index 5d9a498eb..000000000 --- a/test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749051325.log +++ /dev/null @@ -1,17 +0,0 @@ -STDOUT: -2025-06-04 15:35:26 [▶] Setting credentials expiry window to 30 minutes -2025-06-04 15:35:26 [▶] role ARN for the current session is "arn:aws:sts::745939127895:assumed-role/Admin/ibhard-Isengard" -2025-06-04 15:35:26 [ℹ] eksctl version 0.208.0 -2025-06-04 15:35:26 [ℹ] using region us-west-2 -2025-06-04 15:35:26 [!] Amazon EKS will no longer publish EKS-optimized Amazon Linux 2 (AL2) AMIs after November 26th, 2025. Additionally, Kubernetes version 1.32 is the last version for which Amazon EKS will release AL2 AMIs. From version 1.33 onwards, Amazon EKS will continue to release AL2023 and Bottlerocket based AMIs. The default AMI family when creating clusters and nodegroups in Eksctl will be changed to AL2023 in the future. -2025-06-04 15:35:26 [▶] determining availability zones -2025-06-04 15:35:27 [ℹ] setting availability zones to [us-west-2b us-west-2a us-west-2c] -2025-06-04 15:35:27 [▶] VPC CIDR (192.168.0.0/16) was divided into 8 subnets [192.168.0.0/19 192.168.32.0/19 192.168.64.0/19 192.168.96.0/19 192.168.128.0/19 192.168.160.0/19 192.168.192.0/19 192.168.224.0/19] -2025-06-04 15:35:27 [ℹ] subnets for us-west-2b - public:192.168.0.0/19 private:192.168.96.0/19 -2025-06-04 15:35:27 [ℹ] subnets for us-west-2a - public:192.168.32.0/19 private:192.168.128.0/19 -2025-06-04 15:35:27 [ℹ] subnets for us-west-2c - public:192.168.64.0/19 private:192.168.160.0/19 -2025-06-04 15:35:27 [▶] resolving AMI using SSM Parameter resolver for region us-west-2, instanceType t3.large and imageFamily AmazonLinux2 - - -STDERR: -Error: unable to determine AMI to use: error getting AMI from SSM Parameter Store: operation error SSM: GetParameter, https response error StatusCode: 400, RequestID: 59f5b73c-2035-46bb-9074-986f681c13f8, ParameterNotFound: . please verify that AMI Family is supported diff --git a/test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749051374.log b/test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749051374.log deleted file mode 100644 index 24ee6f202..000000000 --- a/test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749051374.log +++ /dev/null @@ -1,17 +0,0 @@ -STDOUT: -2025-06-04 15:36:15 [▶] Setting credentials expiry window to 30 minutes -2025-06-04 15:36:15 [▶] role ARN for the current session is "arn:aws:sts::745939127895:assumed-role/Admin/ibhard-Isengard" -2025-06-04 15:36:15 [ℹ] eksctl version 0.208.0 -2025-06-04 15:36:15 [ℹ] using region us-west-2 -2025-06-04 15:36:15 [!] Amazon EKS will no longer publish EKS-optimized Amazon Linux 2 (AL2) AMIs after November 26th, 2025. Additionally, Kubernetes version 1.32 is the last version for which Amazon EKS will release AL2 AMIs. From version 1.33 onwards, Amazon EKS will continue to release AL2023 and Bottlerocket based AMIs. The default AMI family when creating clusters and nodegroups in Eksctl will be changed to AL2023 in the future. -2025-06-04 15:36:15 [▶] determining availability zones -2025-06-04 15:36:16 [ℹ] setting availability zones to [us-west-2b us-west-2d us-west-2a] -2025-06-04 15:36:16 [▶] VPC CIDR (192.168.0.0/16) was divided into 8 subnets [192.168.0.0/19 192.168.32.0/19 192.168.64.0/19 192.168.96.0/19 192.168.128.0/19 192.168.160.0/19 192.168.192.0/19 192.168.224.0/19] -2025-06-04 15:36:16 [ℹ] subnets for us-west-2b - public:192.168.0.0/19 private:192.168.96.0/19 -2025-06-04 15:36:16 [ℹ] subnets for us-west-2d - public:192.168.32.0/19 private:192.168.128.0/19 -2025-06-04 15:36:16 [ℹ] subnets for us-west-2a - public:192.168.64.0/19 private:192.168.160.0/19 -2025-06-04 15:36:16 [▶] resolving AMI using SSM Parameter resolver for region us-west-2, instanceType t3.large and imageFamily AmazonLinux2 - - -STDERR: -Error: unable to determine AMI to use: error getting AMI from SSM Parameter Store: operation error SSM: GetParameter, https response error StatusCode: 400, RequestID: 1189c451-cdbd-4c3b-9495-a627bfe2113d, ParameterNotFound: . please verify that AMI Family is supported diff --git a/test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749051392.log b/test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749051392.log deleted file mode 100644 index 4efc1291f..000000000 --- a/test/e2e/stress-scale-tests/logs/eks-cluster-creation-error-efs-csi-test-1749051392.log +++ /dev/null @@ -1,254 +0,0 @@ -STDOUT: -2025-06-04 15:36:32 [▶] Setting credentials expiry window to 30 minutes -2025-06-04 15:36:33 [▶] role ARN for the current session is "arn:aws:sts::745939127895:assumed-role/Admin/ibhard-Isengard" -2025-06-04 15:36:33 [ℹ] eksctl version 0.208.0 -2025-06-04 15:36:33 [ℹ] using region us-west-2 -2025-06-04 15:36:33 [!] Amazon EKS will no longer publish EKS-optimized Amazon Linux 2 (AL2) AMIs after November 26th, 2025. Additionally, Kubernetes version 1.32 is the last version for which Amazon EKS will release AL2 AMIs. From version 1.33 onwards, Amazon EKS will continue to release AL2023 and Bottlerocket based AMIs. The default AMI family when creating clusters and nodegroups in Eksctl will be changed to AL2023 in the future. -2025-06-04 15:36:33 [▶] determining availability zones -2025-06-04 15:36:33 [ℹ] setting availability zones to [us-west-2d us-west-2a us-west-2c] -2025-06-04 15:36:33 [▶] VPC CIDR (192.168.0.0/16) was divided into 8 subnets [192.168.0.0/19 192.168.32.0/19 192.168.64.0/19 192.168.96.0/19 192.168.128.0/19 192.168.160.0/19 192.168.192.0/19 192.168.224.0/19] -2025-06-04 15:36:33 [ℹ] subnets for us-west-2d - public:192.168.0.0/19 private:192.168.96.0/19 -2025-06-04 15:36:33 [ℹ] subnets for us-west-2a - public:192.168.32.0/19 private:192.168.128.0/19 -2025-06-04 15:36:33 [ℹ] subnets for us-west-2c - public:192.168.64.0/19 private:192.168.160.0/19 -2025-06-04 15:36:33 [▶] resolving AMI using SSM Parameter resolver for region us-west-2, instanceType t3.large and imageFamily AmazonLinux2 -2025-06-04 15:36:34 [ℹ] nodegroup "ng-1" will use "ami-0ff76e62eff72da2a" [AmazonLinux2/1.28] -2025-06-04 15:36:34 [ℹ] using Kubernetes version 1.28 -2025-06-04 15:36:34 [ℹ] creating EKS cluster "efs-csi-test-1749051392" in "us-west-2" region with un-managed nodes -2025-06-04 15:36:34 [▶] cfg.json = \ -{ - "kind": "ClusterConfig", - "apiVersion": "eksctl.io/v1alpha5", - "metadata": { - "name": "efs-csi-test-1749051392", - "region": "us-west-2", - "version": "1.28" - }, - "iam": { - "withOIDC": true, - "serviceAccounts": [ - { - "metadata": { - "name": "efs-csi-controller-sa", - "namespace": "kube-system" - }, - "attachPolicyARNs": [ - "arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy" - ], - "wellKnownPolicies": { - "imageBuilder": false, - "autoScaler": false, - "awsLoadBalancerController": false, - "externalDNS": false, - "certManager": false, - "ebsCSIController": false, - "efsCSIController": false - } - } - ], - "vpcResourceControllerPolicy": true - }, - "accessConfig": { - "authenticationMode": "API_AND_CONFIG_MAP" - }, - "vpc": { - "cidr": "192.168.0.0/16", - "subnets": { - "private": { - "us-west-2a": { - "az": "us-west-2a", - "cidr": "192.168.128.0/19" - }, - "us-west-2c": { - "az": "us-west-2c", - "cidr": "192.168.160.0/19" - }, - "us-west-2d": { - "az": "us-west-2d", - "cidr": "192.168.96.0/19" - } - }, - "public": { - "us-west-2a": { - "az": "us-west-2a", - "cidr": "192.168.32.0/19" - }, - "us-west-2c": { - "az": "us-west-2c", - "cidr": "192.168.64.0/19" - }, - "us-west-2d": { - "az": "us-west-2d", - "cidr": "192.168.0.0/19" - } - } - }, - "manageSharedNodeSecurityGroupRules": true, - "autoAllocateIPv6": false, - "nat": { - "gateway": "Single" - }, - "clusterEndpoints": { - "privateAccess": false, - "publicAccess": true - } - }, - "addonsConfig": {}, - "privateCluster": { - "enabled": false, - "skipEndpointCreation": false - }, - "nodeGroups": [ - { - "name": "ng-1", - "amiFamily": "AmazonLinux2", - "instanceType": "t3.large", - "desiredCapacity": 3, - "minSize": 2, - "maxSize": 5, - "volumeSize": 80, - "ssh": { - "allow": false - }, - "labels": { - "alpha.eksctl.io/cluster-name": "efs-csi-test-1749051392", - "alpha.eksctl.io/nodegroup-name": "ng-1", - "efs-issue": "true", - "role": "general" - }, - "privateNetworking": false, - "tags": { - "nodegroup-role": "general" - }, - "iam": { - "attachPolicy": { - "Statement": [ - { - "Action": [ - "elasticfilesystem:DescribeAccessPoints", - "elasticfilesystem:DescribeFileSystems", - "elasticfilesystem:DescribeMountTargets", - "ec2:DescribeAvailabilityZones" - ], - "Effect": "Allow", - "Resource": "*" - }, - { - "Action": [ - "elasticfilesystem:CreateAccessPoint" - ], - "Condition": { - "StringLike": { - "aws:RequestTag/kubernetes.io/cluster/*": "owned" - } - }, - "Effect": "Allow", - "Resource": "*" - }, - { - "Action": [ - "elasticfilesystem:DeleteAccessPoint" - ], - "Condition": { - "StringEquals": { - "aws:ResourceTag/kubernetes.io/cluster/*": "owned" - } - }, - "Effect": "Allow", - "Resource": "*" - } - ], - "Version": "2012-10-17" - }, - "withAddonPolicies": { - "imageBuilder": false, - "autoScaler": false, - "externalDNS": false, - "certManager": false, - "appMesh": null, - "appMeshPreview": null, - "ebs": false, - "fsx": false, - "efs": false, - "awsLoadBalancerController": false, - "albIngress": false, - "xRay": false, - "cloudWatch": false - } - }, - "ami": "ami-0ff76e62eff72da2a", - "securityGroups": { - "withShared": true, - "withLocal": true - }, - "volumeType": "gp3", - "volumeName": "/dev/xvda", - "volumeEncrypted": false, - "volumeIOPS": 3000, - "volumeThroughput": 125, - "disableIMDSv1": true, - "disablePodIMDS": false, - "instanceSelector": {}, - "containerRuntime": "containerd" - } - ], - "availabilityZones": [ - "us-west-2d", - "us-west-2a", - "us-west-2c" - ] -} - -2025-06-04 15:36:34 [ℹ] 1 nodegroup (ng-1) was included (based on the include/exclude rules) -2025-06-04 15:36:34 [ℹ] will create a CloudFormation stack for cluster itself and 1 nodegroup stack(s) -2025-06-04 15:36:34 [ℹ] if you encounter any issues, check CloudFormation console or try 'eksctl utils describe-stacks --region=us-west-2 --cluster=efs-csi-test-1749051392' -2025-06-04 15:36:34 [ℹ] Kubernetes API endpoint access will use default of {publicAccess=true, privateAccess=false} for cluster "efs-csi-test-1749051392" in "us-west-2" -2025-06-04 15:36:34 [ℹ] CloudWatch logging will not be enabled for cluster "efs-csi-test-1749051392" in "us-west-2" -2025-06-04 15:36:34 [ℹ] you can enable it with 'eksctl utils update-cluster-logging --enable-types={SPECIFY-YOUR-LOG-TYPES-HERE (e.g. all)} --region=us-west-2 --cluster=efs-csi-test-1749051392' -2025-06-04 15:36:34 [ℹ] default addons kube-proxy, coredns, metrics-server, vpc-cni were not specified, will install them as EKS addons -2025-06-04 15:36:34 [ℹ] -2 sequential tasks: { create cluster control plane "efs-csi-test-1749051392", - 2 sequential sub-tasks: { - 5 sequential sub-tasks: { - 1 task: { create addons }, - wait for control plane to become ready, - associate IAM OIDC provider, - 2 sequential sub-tasks: { - create IAM role for serviceaccount "kube-system/efs-csi-controller-sa", - create serviceaccount "kube-system/efs-csi-controller-sa", - }, - update VPC CNI to use IRSA if required, - }, - create nodegroup "ng-1", - } -} -2025-06-04 15:36:34 [▶] started task: create cluster control plane "efs-csi-test-1749051392" -2025-06-04 15:36:34 [ℹ] building cluster stack "eksctl-efs-csi-test-1749051392-cluster" -2025-06-04 15:36:34 [▶] CreateStackInput = &cloudformation.CreateStackInput{StackName:(*string)(0xc000f5db00), Capabilities:[]types.Capability{"CAPABILITY_IAM"}, ClientRequestToken:(*string)(nil), DisableRollback:(*bool)(0xc00063aed8), EnableTerminationProtection:(*bool)(nil), NotificationARNs:[]string(nil), OnFailure:"", Parameters:[]types.Parameter(nil), ResourceTypes:[]string(nil), RetainExceptOnCreate:(*bool)(nil), RoleARN:(*string)(nil), RollbackConfiguration:(*types.RollbackConfiguration)(nil), StackPolicyBody:(*string)(nil), StackPolicyURL:(*string)(nil), Tags:[]types.Tag{types.Tag{Key:(*string)(0xc000cdb8d0), Value:(*string)(0xc000cdb8e0), noSmithyDocumentSerde:document.NoSerde{}}, types.Tag{Key:(*string)(0xc000cdb8f0), Value:(*string)(0xc000cdb900), noSmithyDocumentSerde:document.NoSerde{}}, types.Tag{Key:(*string)(0xc000cdb910), Value:(*string)(0xc000cdb920), noSmithyDocumentSerde:document.NoSerde{}}, types.Tag{Key:(*string)(0xc000f718c0), Value:(*string)(0xc000f718d0), noSmithyDocumentSerde:document.NoSerde{}}}, TemplateBody:(*string)(0xc000f718e0), TemplateURL:(*string)(nil), TimeoutInMinutes:(*int32)(nil), noSmithyDocumentSerde:document.NoSerde{}} -2025-06-04 15:36:34 [ℹ] deploying stack "eksctl-efs-csi-test-1749051392-cluster" -2025-06-04 15:37:04 [ℹ] waiting for CloudFormation stack "eksctl-efs-csi-test-1749051392-cluster" -2025-06-04 15:37:05 [✖] unexpected status "ROLLBACK_IN_PROGRESS" while waiting for CloudFormation stack "eksctl-efs-csi-test-1749051392-cluster" -2025-06-04 15:37:05 [✖] unexpected status "ROLLBACK_IN_PROGRESS" while waiting for CloudFormation stack "eksctl-efs-csi-test-1749051392-cluster" -2025-06-04 15:37:05 [ℹ] fetching stack events in attempt to troubleshoot the root cause of the failure -2025-06-04 15:37:05 [▶] AWS::IAM::Role/ServiceRole: DELETE_COMPLETE -2025-06-04 15:37:05 [▶] AWS::EC2::EIP/NATIP: DELETE_COMPLETE -2025-06-04 15:37:05 [!] AWS::IAM::Role/ServiceRole: DELETE_IN_PROGRESS -2025-06-04 15:37:05 [!] AWS::EC2::EIP/NATIP: DELETE_IN_PROGRESS -2025-06-04 15:37:05 [▶] AWS::CloudFormation::Stack/eksctl-efs-csi-test-1749051392-cluster: ROLLBACK_IN_PROGRESS – "The following resource(s) failed to create: [InternetGateway, ServiceRole, NATIP, VPC]. Rollback requested by user." -2025-06-04 15:37:05 [✖] AWS::EC2::EIP/NATIP: CREATE_FAILED – "Resource creation cancelled" -2025-06-04 15:37:05 [✖] AWS::IAM::Role/ServiceRole: CREATE_FAILED – "Resource creation cancelled" -2025-06-04 15:37:05 [▶] AWS::EC2::EIP/NATIP: CREATE_IN_PROGRESS – "Resource creation Initiated" -2025-06-04 15:37:05 [✖] AWS::EC2::VPC/VPC: CREATE_FAILED – "Resource handler returned message: \"The maximum number of VPCs has been reached. (Service: Ec2, Status Code: 400, Request ID: 12f2bdae-04a6-4aea-8573-1101de3244a7) (SDK Attempt Count: 1)\" (RequestToken: 06fa18f0-e50e-789e-41a0-492077692cf3, HandlerErrorCode: GeneralServiceException)" -2025-06-04 15:37:05 [✖] AWS::EC2::InternetGateway/InternetGateway: CREATE_FAILED – "Resource handler returned message: \"The maximum number of internet gateways has been reached. (Service: Ec2, Status Code: 400, Request ID: e67da527-78fd-41bd-be9f-ce04d9ba2a26) (SDK Attempt Count: 1)\" (RequestToken: 6f816c80-7115-5bdf-5120-3a7099dbe90d, HandlerErrorCode: ServiceLimitExceeded)" -2025-06-04 15:37:05 [▶] AWS::EC2::InternetGateway/InternetGateway: CREATE_IN_PROGRESS -2025-06-04 15:37:05 [▶] AWS::IAM::Role/ServiceRole: CREATE_IN_PROGRESS -2025-06-04 15:37:05 [▶] AWS::EC2::EIP/NATIP: CREATE_IN_PROGRESS -2025-06-04 15:37:05 [▶] AWS::EC2::VPC/VPC: CREATE_IN_PROGRESS -2025-06-04 15:37:05 [▶] AWS::CloudFormation::Stack/eksctl-efs-csi-test-1749051392-cluster: CREATE_IN_PROGRESS – "User Initiated" -2025-06-04 15:37:05 [▶] failed task: create cluster control plane "efs-csi-test-1749051392" (will not run other sequential tasks) -2025-06-04 15:37:05 [!] 1 error(s) occurred and cluster hasn't been created properly, you may wish to check CloudFormation console -2025-06-04 15:37:05 [ℹ] to cleanup resources, run 'eksctl delete cluster --region=us-west-2 --name=efs-csi-test-1749051392' -2025-06-04 15:37:05 [✖] ResourceNotReady: failed waiting for successful resource state - - -STDERR: -Error: failed to create cluster "efs-csi-test-1749051392" diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250603_145210.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250603_145210.log deleted file mode 100644 index 662b37b48..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250603_145210.log +++ /dev/null @@ -1,147 +0,0 @@ -2025-06-03 14:52:10,097 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-03 14:52:10,098 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-03 14:52:10,325 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-03 14:52:10,325 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario -2025-06-03 14:52:10,325 - tests.orchestrator - INFO - Running specific scenario: _scenario_many_to_one -2025-06-03 14:52:10,325 - tests.orchestrator - INFO - Running scenario: Many pods to one PVC -2025-06-03 14:52:10,528 - tests.orchestrator - INFO - Waiting for PVC many2one-020acd02 to be bound -2025-06-03 14:52:12,667 - tests.orchestrator - INFO - PVC many2one-020acd02 is bound -2025-06-03 14:52:12,668 - tests.orchestrator - INFO - Creating 3 pods for the same PVC -2025-06-03 14:52:12,669 - tests.orchestrator - INFO - Creating pod test-pod-ef805426 on PVC many2one-020acd02 -2025-06-03 14:52:12,771 - tests.orchestrator - INFO - Created pod: test-pod-ef805426 using PVC: many2one-020acd02 -2025-06-03 14:52:12,771 - tests.orchestrator - INFO - Waiting for pod test-pod-ef805426 to be ready -2025-06-03 14:52:25,251 - tests.orchestrator - INFO - Pod test-pod-ef805426 is ready -2025-06-03 14:52:25,252 - tests.orchestrator - INFO - Creating pod test-pod-a76bf50a on PVC many2one-020acd02 -2025-06-03 14:52:25,325 - tests.orchestrator - INFO - Created pod: test-pod-a76bf50a using PVC: many2one-020acd02 -2025-06-03 14:52:25,325 - tests.orchestrator - INFO - Waiting for pod test-pod-a76bf50a to be ready -2025-06-03 14:52:37,804 - tests.orchestrator - INFO - Pod test-pod-a76bf50a is ready -2025-06-03 14:52:37,804 - tests.orchestrator - INFO - Creating pod test-pod-6da904f7 on PVC many2one-020acd02 -2025-06-03 14:52:37,879 - tests.orchestrator - INFO - Created pod: test-pod-6da904f7 using PVC: many2one-020acd02 -2025-06-03 14:52:37,879 - tests.orchestrator - INFO - Waiting for pod test-pod-6da904f7 to be ready -2025-06-03 14:52:50,362 - tests.orchestrator - INFO - Pod test-pod-6da904f7 is ready -2025-06-03 14:52:50,434 - tests.orchestrator - ERROR - Error in many-to-one scenario: (400) -Reason: Bad Request -HTTP response headers: HTTPHeaderDict({'Audit-Id': '76d6f572-aa54-4107-b259-0d0f792714d5', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Tue, 03 Jun 2025 14:52:50 GMT', 'Content-Length': '139'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Upgrade request required","reason":"BadRequest","code":400} - - -2025-06-03 14:52:51,434 - tests.orchestrator - INFO - Selected operation: _attach_pod -2025-06-03 14:52:51,434 - tests.orchestrator - INFO - Creating pod test-pod-799aaf63 on PVC many2one-020acd02 -2025-06-03 14:52:51,531 - tests.orchestrator - INFO - Created pod: test-pod-799aaf63 using PVC: many2one-020acd02 -2025-06-03 14:52:51,531 - tests.orchestrator - INFO - Waiting for pod test-pod-799aaf63 to be ready -2025-06-03 14:52:57,803 - tests.orchestrator - INFO - Pod test-pod-799aaf63 is ready -2025-06-03 14:52:58,803 - tests.orchestrator - INFO - Selected operation: _verify_readwrite -2025-06-03 14:52:58,803 - tests.orchestrator - INFO - Testing read/write between pods test-pod-ef805426 and test-pod-a76bf50a sharing PVC many2one-020acd02 -2025-06-03 14:52:58,875 - tests.orchestrator - ERROR - Failed in read/write verification: (400) -Reason: Bad Request -HTTP response headers: HTTPHeaderDict({'Audit-Id': 'e1a62bf8-8f8a-41b5-851e-c5ce76b06ea8', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Tue, 03 Jun 2025 14:52:58 GMT', 'Content-Length': '139'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Upgrade request required","reason":"BadRequest","code":400} - - -2025-06-03 14:52:59,875 - tests.orchestrator - INFO - Selected operation: _delete_pod -2025-06-03 14:52:59,876 - tests.orchestrator - INFO - Deleting pod: test-pod-6da904f7 from PVC: many2one-020acd02 -2025-06-03 14:52:59,951 - tests.orchestrator - INFO - Waiting for pod test-pod-6da904f7 to be deleted -2025-06-03 14:53:32,578 - tests.orchestrator - INFO - Pod test-pod-6da904f7 has been deleted -2025-06-03 14:53:32,578 - tests.orchestrator - INFO - Deleted pod: test-pod-6da904f7 -2025-06-03 14:53:33,578 - tests.orchestrator - INFO - Selected operation: _create_pvc -2025-06-03 14:53:33,579 - tests.orchestrator - INFO - Creating PVC: test-pvc-4937168b -2025-06-03 14:53:33,660 - tests.orchestrator - INFO - Created PVC: test-pvc-4937168b -2025-06-03 14:53:33,661 - tests.orchestrator - INFO - Waiting for PVC test-pvc-4937168b to be bound -2025-06-03 14:53:35,792 - tests.orchestrator - INFO - PVC test-pvc-4937168b is bound -2025-06-03 14:53:36,793 - tests.orchestrator - INFO - Selected operation: _attach_pod -2025-06-03 14:53:36,793 - tests.orchestrator - INFO - Creating pod test-pod-dd8a0785 on PVC many2one-020acd02 -2025-06-03 14:53:36,868 - tests.orchestrator - INFO - Created pod: test-pod-dd8a0785 using PVC: many2one-020acd02 -2025-06-03 14:53:36,868 - tests.orchestrator - INFO - Waiting for pod test-pod-dd8a0785 to be ready -2025-06-03 14:53:49,354 - tests.orchestrator - INFO - Pod test-pod-dd8a0785 is ready -2025-06-03 14:53:50,355 - tests.orchestrator - INFO - Selected operation: _delete_pod -2025-06-03 14:53:50,355 - tests.orchestrator - INFO - Deleting pod: test-pod-799aaf63 from PVC: many2one-020acd02 -2025-06-03 14:53:50,431 - tests.orchestrator - INFO - Waiting for pod test-pod-799aaf63 to be deleted -2025-06-03 14:54:23,047 - tests.orchestrator - INFO - Pod test-pod-799aaf63 has been deleted -2025-06-03 14:54:23,047 - tests.orchestrator - INFO - Deleted pod: test-pod-799aaf63 -2025-06-03 14:54:24,047 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario -2025-06-03 14:54:24,048 - tests.orchestrator - INFO - Running specific scenario: _scenario_many_to_one -2025-06-03 14:54:24,048 - tests.orchestrator - INFO - Running scenario: Many pods to one PVC -2025-06-03 14:54:24,127 - tests.orchestrator - INFO - Waiting for PVC many2one-8b088f98 to be bound -2025-06-03 14:54:26,258 - tests.orchestrator - INFO - PVC many2one-8b088f98 is bound -2025-06-03 14:54:26,259 - tests.orchestrator - INFO - Creating 4 pods for the same PVC -2025-06-03 14:54:26,259 - tests.orchestrator - INFO - Creating pod test-pod-017db5e3 on PVC many2one-8b088f98 -2025-06-03 14:54:26,330 - tests.orchestrator - INFO - Created pod: test-pod-017db5e3 using PVC: many2one-8b088f98 -2025-06-03 14:54:26,330 - tests.orchestrator - INFO - Waiting for pod test-pod-017db5e3 to be ready -2025-06-03 14:54:32,600 - tests.orchestrator - INFO - Pod test-pod-017db5e3 is ready -2025-06-03 14:54:32,601 - tests.orchestrator - INFO - Creating pod test-pod-25f6ccfe on PVC many2one-8b088f98 -2025-06-03 14:54:32,676 - tests.orchestrator - INFO - Created pod: test-pod-25f6ccfe using PVC: many2one-8b088f98 -2025-06-03 14:54:32,676 - tests.orchestrator - INFO - Waiting for pod test-pod-25f6ccfe to be ready -2025-06-03 14:54:38,950 - tests.orchestrator - INFO - Pod test-pod-25f6ccfe is ready -2025-06-03 14:54:38,950 - tests.orchestrator - INFO - Creating pod test-pod-a3be1bc9 on PVC many2one-8b088f98 -2025-06-03 14:54:39,025 - tests.orchestrator - INFO - Created pod: test-pod-a3be1bc9 using PVC: many2one-8b088f98 -2025-06-03 14:54:39,025 - tests.orchestrator - INFO - Waiting for pod test-pod-a3be1bc9 to be ready -2025-06-03 14:54:51,508 - tests.orchestrator - INFO - Pod test-pod-a3be1bc9 is ready -2025-06-03 14:54:51,508 - tests.orchestrator - INFO - Creating pod test-pod-2da972ac on PVC many2one-8b088f98 -2025-06-03 14:54:51,601 - tests.orchestrator - INFO - Created pod: test-pod-2da972ac using PVC: many2one-8b088f98 -2025-06-03 14:54:51,602 - tests.orchestrator - INFO - Waiting for pod test-pod-2da972ac to be ready -2025-06-03 14:54:57,878 - tests.orchestrator - INFO - Pod test-pod-2da972ac is ready -2025-06-03 14:54:57,950 - tests.orchestrator - ERROR - Error in many-to-one scenario: (400) -Reason: Bad Request -HTTP response headers: HTTPHeaderDict({'Audit-Id': '458593e0-149a-4412-8f07-f85de3d6cf1d', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Tue, 03 Jun 2025 14:54:57 GMT', 'Content-Length': '139'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Upgrade request required","reason":"BadRequest","code":400} - - -2025-06-03 14:54:58,950 - tests.orchestrator - INFO - Selected operation: _delete_pvc -2025-06-03 14:54:58,950 - tests.orchestrator - INFO - Deleting PVC: many2one-8b088f98 -2025-06-03 14:54:58,950 - tests.orchestrator - INFO - Deleting 4 pods using PVC many2one-8b088f98 -2025-06-03 14:54:58,950 - tests.orchestrator - INFO - Deleting pod: test-pod-017db5e3 from PVC: many2one-8b088f98 -2025-06-03 14:54:59,025 - tests.orchestrator - INFO - Waiting for pod test-pod-017db5e3 to be deleted -2025-06-03 14:55:31,641 - tests.orchestrator - INFO - Pod test-pod-017db5e3 has been deleted -2025-06-03 14:55:31,641 - tests.orchestrator - INFO - Deleted pod: test-pod-017db5e3 -2025-06-03 14:55:31,641 - tests.orchestrator - INFO - Deleting pod: test-pod-25f6ccfe from PVC: many2one-8b088f98 -2025-06-03 14:55:31,722 - tests.orchestrator - INFO - Waiting for pod test-pod-25f6ccfe to be deleted -2025-06-03 14:56:04,344 - tests.orchestrator - INFO - Pod test-pod-25f6ccfe has been deleted -2025-06-03 14:56:04,344 - tests.orchestrator - INFO - Deleted pod: test-pod-25f6ccfe -2025-06-03 14:56:04,344 - tests.orchestrator - INFO - Deleting pod: test-pod-a3be1bc9 from PVC: many2one-8b088f98 -2025-06-03 14:56:04,420 - tests.orchestrator - INFO - Waiting for pod test-pod-a3be1bc9 to be deleted -2025-06-03 14:56:37,041 - tests.orchestrator - INFO - Pod test-pod-a3be1bc9 has been deleted -2025-06-03 14:56:37,042 - tests.orchestrator - INFO - Deleted pod: test-pod-a3be1bc9 -2025-06-03 14:56:37,042 - tests.orchestrator - INFO - Deleting pod: test-pod-2da972ac from PVC: many2one-8b088f98 -2025-06-03 14:56:37,119 - tests.orchestrator - INFO - Waiting for pod test-pod-2da972ac to be deleted -2025-06-03 14:57:09,741 - tests.orchestrator - INFO - Pod test-pod-2da972ac has been deleted -2025-06-03 14:57:09,741 - tests.orchestrator - INFO - Deleted pod: test-pod-2da972ac -2025-06-03 14:57:09,815 - tests.orchestrator - INFO - Waiting for PVC many2one-8b088f98 to be deleted -2025-06-03 14:57:09,880 - tests.orchestrator - INFO - PVC many2one-8b088f98 has been deleted -2025-06-03 14:57:09,880 - tests.orchestrator - INFO - Deleted PVC: many2one-8b088f98 -2025-06-03 14:57:10,880 - tests.orchestrator - INFO - Test completed in 300.78 seconds -2025-06-03 14:57:10,880 - tests.orchestrator - INFO - Cleaning up resources... -2025-06-03 14:57:10,880 - tests.orchestrator - INFO - Deleting pod: test-pod-ef805426 from PVC: many2one-020acd02 -2025-06-03 14:57:10,955 - tests.orchestrator - INFO - Waiting for pod test-pod-ef805426 to be deleted -2025-06-03 14:57:43,569 - tests.orchestrator - INFO - Pod test-pod-ef805426 has been deleted -2025-06-03 14:57:43,569 - tests.orchestrator - INFO - Deleted pod: test-pod-ef805426 -2025-06-03 14:57:43,569 - tests.orchestrator - INFO - Deleting pod: test-pod-a76bf50a from PVC: many2one-020acd02 -2025-06-03 14:57:43,647 - tests.orchestrator - INFO - Waiting for pod test-pod-a76bf50a to be deleted -2025-06-03 14:58:16,290 - tests.orchestrator - INFO - Pod test-pod-a76bf50a has been deleted -2025-06-03 14:58:16,290 - tests.orchestrator - INFO - Deleted pod: test-pod-a76bf50a -2025-06-03 14:58:16,290 - tests.orchestrator - INFO - Deleting pod: test-pod-dd8a0785 from PVC: many2one-020acd02 -2025-06-03 14:58:16,365 - tests.orchestrator - INFO - Waiting for pod test-pod-dd8a0785 to be deleted -2025-06-03 14:58:48,979 - tests.orchestrator - INFO - Pod test-pod-dd8a0785 has been deleted -2025-06-03 14:58:48,979 - tests.orchestrator - INFO - Deleted pod: test-pod-dd8a0785 -2025-06-03 14:58:48,979 - tests.orchestrator - INFO - Deleting PVC: many2one-020acd02 -2025-06-03 14:58:49,051 - tests.orchestrator - INFO - Waiting for PVC many2one-020acd02 to be deleted -2025-06-03 14:58:49,119 - tests.orchestrator - INFO - PVC many2one-020acd02 has been deleted -2025-06-03 14:58:49,119 - tests.orchestrator - INFO - Deleted PVC: many2one-020acd02 -2025-06-03 14:58:49,119 - tests.orchestrator - INFO - Deleting PVC: test-pvc-4937168b -2025-06-03 14:58:49,191 - tests.orchestrator - INFO - Waiting for PVC test-pvc-4937168b to be deleted -2025-06-03 14:58:49,257 - tests.orchestrator - INFO - PVC test-pvc-4937168b has been deleted -2025-06-03 14:58:49,257 - tests.orchestrator - INFO - Deleted PVC: test-pvc-4937168b -2025-06-03 14:58:49,258 - tests.orchestrator - INFO - Cleanup completed -2025-06-03 14:58:49,258 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-03 14:58:49,258 - tests.orchestrator - INFO - --- Operations --- -2025-06-03 14:58:49,258 - tests.orchestrator - INFO - create_pvc: 1 succeeded, 0 failed (100.0%) -2025-06-03 14:58:49,258 - tests.orchestrator - INFO - attach_pod: 9 succeeded, 0 failed (100.0%) -2025-06-03 14:58:49,258 - tests.orchestrator - INFO - delete_pod: 9 succeeded, 0 failed (100.0%) -2025-06-03 14:58:49,258 - tests.orchestrator - INFO - delete_pvc: 3 succeeded, 0 failed (100.0%) -2025-06-03 14:58:49,258 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) -2025-06-03 14:58:49,258 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) -2025-06-03 14:58:49,258 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-03 14:58:49,258 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-03 14:58:49,258 - tests.orchestrator - INFO - many_to_one: 0 succeeded, 2 failed out of 2 runs (0.0%) -2025-06-03 14:58:49,258 - tests.orchestrator - INFO - one_to_one: No runs -2025-06-03 14:58:49,258 - tests.orchestrator - INFO - concurrent_pvc: No runs -2025-06-03 14:58:49,258 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250603_154721.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250603_154721.log deleted file mode 100644 index 551c1f94b..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250603_154721.log +++ /dev/null @@ -1,8 +0,0 @@ -2025-06-03 15:47:21,097 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-03 15:47:21,097 - tests.orchestrator - INFO - Starting orchestrator test for 600 seconds -2025-06-03 15:47:21,297 - tests.orchestrator - ERROR - Error checking StorageClass: (401) -Reason: Unauthorized -HTTP response headers: HTTPHeaderDict({'Audit-Id': 'fb6b8162-e595-4821-8d1a-19fb10c247db', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Tue, 03 Jun 2025 15:47:21 GMT', 'Content-Length': '129'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} - - diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250603_154805.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250603_154805.log deleted file mode 100644 index 9b1f8b427..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250603_154805.log +++ /dev/null @@ -1,455 +0,0 @@ -2025-06-03 15:48:05,458 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-03 15:48:05,458 - tests.orchestrator - INFO - Starting orchestrator test for 600 seconds -2025-06-03 15:48:05,664 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-03 15:48:05,664 - tests.orchestrator - INFO - Selected operation: _create_pvc -2025-06-03 15:48:05,665 - tests.orchestrator - INFO - Creating PVC: test-pvc-9e0f63a9 -2025-06-03 15:48:05,866 - tests.orchestrator - INFO - Created PVC: test-pvc-9e0f63a9 -2025-06-03 15:48:05,866 - tests.orchestrator - INFO - Waiting for PVC test-pvc-9e0f63a9 to be bound -2025-06-03 15:48:08,002 - tests.orchestrator - INFO - PVC test-pvc-9e0f63a9 is bound -2025-06-03 15:48:09,003 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario -2025-06-03 15:48:09,003 - tests.orchestrator - INFO - Running specific scenario: _scenario_one_to_one -2025-06-03 15:48:09,003 - tests.orchestrator - INFO - Running scenario: One pod per PVC -2025-06-03 15:48:09,003 - tests.orchestrator - INFO - Creating 4 PVC-pod pairs -2025-06-03 15:48:09,071 - tests.orchestrator - INFO - Waiting for PVC one2one-53c40f0a to be bound -2025-06-03 15:48:11,204 - tests.orchestrator - INFO - PVC one2one-53c40f0a is bound -2025-06-03 15:48:11,204 - tests.orchestrator - INFO - Creating pod test-pod-9dcfaf3b on PVC one2one-53c40f0a -2025-06-03 15:48:11,286 - tests.orchestrator - INFO - Created pod: test-pod-9dcfaf3b using PVC: one2one-53c40f0a -2025-06-03 15:48:11,286 - tests.orchestrator - INFO - Waiting for pod test-pod-9dcfaf3b to be ready -2025-06-03 15:48:17,562 - tests.orchestrator - INFO - Pod test-pod-9dcfaf3b is ready -2025-06-03 15:48:17,631 - tests.orchestrator - INFO - Waiting for PVC one2one-e698727d to be bound -2025-06-03 15:48:19,766 - tests.orchestrator - INFO - PVC one2one-e698727d is bound -2025-06-03 15:48:19,766 - tests.orchestrator - INFO - Creating pod test-pod-256bf78d on PVC one2one-e698727d -2025-06-03 15:48:19,842 - tests.orchestrator - INFO - Created pod: test-pod-256bf78d using PVC: one2one-e698727d -2025-06-03 15:48:19,842 - tests.orchestrator - INFO - Waiting for pod test-pod-256bf78d to be ready -2025-06-03 15:48:32,323 - tests.orchestrator - INFO - Pod test-pod-256bf78d is ready -2025-06-03 15:48:32,394 - tests.orchestrator - INFO - Waiting for PVC one2one-ee84a0d2 to be bound -2025-06-03 15:48:34,527 - tests.orchestrator - INFO - PVC one2one-ee84a0d2 is bound -2025-06-03 15:48:34,527 - tests.orchestrator - INFO - Creating pod test-pod-d95dbc73 on PVC one2one-ee84a0d2 -2025-06-03 15:48:34,600 - tests.orchestrator - INFO - Created pod: test-pod-d95dbc73 using PVC: one2one-ee84a0d2 -2025-06-03 15:48:34,600 - tests.orchestrator - INFO - Waiting for pod test-pod-d95dbc73 to be ready -2025-06-03 15:48:40,873 - tests.orchestrator - INFO - Pod test-pod-d95dbc73 is ready -2025-06-03 15:48:40,950 - tests.orchestrator - INFO - Waiting for PVC one2one-b38cd705 to be bound -2025-06-03 15:48:43,088 - tests.orchestrator - INFO - PVC one2one-b38cd705 is bound -2025-06-03 15:48:43,088 - tests.orchestrator - INFO - Creating pod test-pod-04cae7e2 on PVC one2one-b38cd705 -2025-06-03 15:48:43,173 - tests.orchestrator - INFO - Created pod: test-pod-04cae7e2 using PVC: one2one-b38cd705 -2025-06-03 15:48:43,173 - tests.orchestrator - INFO - Waiting for pod test-pod-04cae7e2 to be ready -2025-06-03 15:48:49,452 - tests.orchestrator - INFO - Pod test-pod-04cae7e2 is ready -2025-06-03 15:48:49,524 - tests.orchestrator - ERROR - Error in one-to-one scenario: (400) -Reason: Bad Request -HTTP response headers: HTTPHeaderDict({'Audit-Id': 'be1d7a01-f2a7-47e7-9b5f-595caf3d4262', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Tue, 03 Jun 2025 15:48:49 GMT', 'Content-Length': '139'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Upgrade request required","reason":"BadRequest","code":400} - - -2025-06-03 15:48:50,524 - tests.orchestrator - INFO - Selected operation: _delete_pod -2025-06-03 15:48:50,524 - tests.orchestrator - INFO - Deleting pod: test-pod-256bf78d from PVC: one2one-e698727d -2025-06-03 15:48:50,602 - tests.orchestrator - INFO - Waiting for pod test-pod-256bf78d to be deleted -2025-06-03 15:49:23,225 - tests.orchestrator - INFO - Pod test-pod-256bf78d has been deleted -2025-06-03 15:49:23,225 - tests.orchestrator - INFO - Deleted pod: test-pod-256bf78d -2025-06-03 15:49:24,225 - tests.orchestrator - INFO - Selected operation: _delete_pod -2025-06-03 15:49:24,226 - tests.orchestrator - INFO - Deleting pod: test-pod-04cae7e2 from PVC: one2one-b38cd705 -2025-06-03 15:49:24,300 - tests.orchestrator - INFO - Waiting for pod test-pod-04cae7e2 to be deleted -2025-06-03 15:49:57,000 - tests.orchestrator - INFO - Pod test-pod-04cae7e2 has been deleted -2025-06-03 15:49:57,000 - tests.orchestrator - INFO - Deleted pod: test-pod-04cae7e2 -2025-06-03 15:49:58,001 - tests.orchestrator - INFO - Selected operation: _attach_pod -2025-06-03 15:49:58,001 - tests.orchestrator - INFO - Creating pod test-pod-2a900eb4 on PVC one2one-e698727d -2025-06-03 15:49:58,086 - tests.orchestrator - INFO - Created pod: test-pod-2a900eb4 using PVC: one2one-e698727d -2025-06-03 15:49:58,086 - tests.orchestrator - INFO - Waiting for pod test-pod-2a900eb4 to be ready -2025-06-03 15:50:10,568 - tests.orchestrator - INFO - Pod test-pod-2a900eb4 is ready -2025-06-03 15:50:11,568 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario -2025-06-03 15:50:11,569 - tests.orchestrator - INFO - Running specific scenario: _scenario_one_to_one -2025-06-03 15:50:11,569 - tests.orchestrator - INFO - Running scenario: One pod per PVC -2025-06-03 15:50:11,569 - tests.orchestrator - INFO - Creating 5 PVC-pod pairs -2025-06-03 15:50:11,833 - tests.orchestrator - INFO - Waiting for PVC one2one-e7169041 to be bound -2025-06-03 15:50:14,174 - tests.orchestrator - INFO - PVC one2one-e7169041 is bound -2025-06-03 15:50:14,174 - tests.orchestrator - INFO - Creating pod test-pod-5aeb9067 on PVC one2one-e7169041 -2025-06-03 15:50:14,249 - tests.orchestrator - INFO - Created pod: test-pod-5aeb9067 using PVC: one2one-e7169041 -2025-06-03 15:50:14,250 - tests.orchestrator - INFO - Waiting for pod test-pod-5aeb9067 to be ready -2025-06-03 15:50:20,524 - tests.orchestrator - INFO - Pod test-pod-5aeb9067 is ready -2025-06-03 15:50:20,593 - tests.orchestrator - INFO - Waiting for PVC one2one-bee0c24d to be bound -2025-06-03 15:50:22,729 - tests.orchestrator - INFO - PVC one2one-bee0c24d is bound -2025-06-03 15:50:22,729 - tests.orchestrator - INFO - Creating pod test-pod-62c10933 on PVC one2one-bee0c24d -2025-06-03 15:50:22,802 - tests.orchestrator - INFO - Created pod: test-pod-62c10933 using PVC: one2one-bee0c24d -2025-06-03 15:50:22,802 - tests.orchestrator - INFO - Waiting for pod test-pod-62c10933 to be ready -2025-06-03 15:50:35,280 - tests.orchestrator - INFO - Pod test-pod-62c10933 is ready -2025-06-03 15:50:35,359 - tests.orchestrator - INFO - Waiting for PVC one2one-74ea88f9 to be bound -2025-06-03 15:50:37,491 - tests.orchestrator - INFO - PVC one2one-74ea88f9 is bound -2025-06-03 15:50:37,491 - tests.orchestrator - INFO - Creating pod test-pod-7f739327 on PVC one2one-74ea88f9 -2025-06-03 15:50:37,567 - tests.orchestrator - INFO - Created pod: test-pod-7f739327 using PVC: one2one-74ea88f9 -2025-06-03 15:50:37,567 - tests.orchestrator - INFO - Waiting for pod test-pod-7f739327 to be ready -2025-06-03 15:50:50,197 - tests.orchestrator - INFO - Pod test-pod-7f739327 is ready -2025-06-03 15:50:50,269 - tests.orchestrator - INFO - Waiting for PVC one2one-11548b80 to be bound -2025-06-03 15:50:52,400 - tests.orchestrator - INFO - PVC one2one-11548b80 is bound -2025-06-03 15:50:52,401 - tests.orchestrator - INFO - Creating pod test-pod-1d7db6ad on PVC one2one-11548b80 -2025-06-03 15:50:52,477 - tests.orchestrator - INFO - Created pod: test-pod-1d7db6ad using PVC: one2one-11548b80 -2025-06-03 15:50:52,477 - tests.orchestrator - INFO - Waiting for pod test-pod-1d7db6ad to be ready -2025-06-03 15:51:04,957 - tests.orchestrator - INFO - Pod test-pod-1d7db6ad is ready -2025-06-03 15:51:05,026 - tests.orchestrator - INFO - Waiting for PVC one2one-2b9e9546 to be bound -2025-06-03 15:51:07,162 - tests.orchestrator - INFO - PVC one2one-2b9e9546 is bound -2025-06-03 15:51:07,162 - tests.orchestrator - INFO - Creating pod test-pod-431935c6 on PVC one2one-2b9e9546 -2025-06-03 15:51:07,246 - tests.orchestrator - INFO - Created pod: test-pod-431935c6 using PVC: one2one-2b9e9546 -2025-06-03 15:51:07,246 - tests.orchestrator - INFO - Waiting for pod test-pod-431935c6 to be ready -2025-06-03 15:51:19,728 - tests.orchestrator - INFO - Pod test-pod-431935c6 is ready -2025-06-03 15:51:19,800 - tests.orchestrator - ERROR - Error in one-to-one scenario: (400) -Reason: Bad Request -HTTP response headers: HTTPHeaderDict({'Audit-Id': 'ae626c0b-b622-4551-b0e8-5ab032b19ba9', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Tue, 03 Jun 2025 15:51:19 GMT', 'Content-Length': '139'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Upgrade request required","reason":"BadRequest","code":400} - - -2025-06-03 15:51:20,800 - tests.orchestrator - INFO - Selected operation: _delete_pod -2025-06-03 15:51:20,800 - tests.orchestrator - INFO - Deleting pod: test-pod-5aeb9067 from PVC: one2one-e7169041 -2025-06-03 15:51:20,876 - tests.orchestrator - INFO - Waiting for pod test-pod-5aeb9067 to be deleted -2025-06-03 15:51:53,497 - tests.orchestrator - INFO - Pod test-pod-5aeb9067 has been deleted -2025-06-03 15:51:53,498 - tests.orchestrator - INFO - Deleted pod: test-pod-5aeb9067 -2025-06-03 15:51:54,498 - tests.orchestrator - INFO - Selected operation: _attach_pod -2025-06-03 15:51:54,498 - tests.orchestrator - INFO - Creating pod test-pod-8f39fd3b on PVC one2one-b38cd705 -2025-06-03 15:51:54,585 - tests.orchestrator - INFO - Created pod: test-pod-8f39fd3b using PVC: one2one-b38cd705 -2025-06-03 15:51:54,585 - tests.orchestrator - INFO - Waiting for pod test-pod-8f39fd3b to be ready -2025-06-03 15:52:07,068 - tests.orchestrator - INFO - Pod test-pod-8f39fd3b is ready -2025-06-03 15:52:08,069 - tests.orchestrator - INFO - Selected operation: _delete_pod -2025-06-03 15:52:08,069 - tests.orchestrator - INFO - Deleting pod: test-pod-d95dbc73 from PVC: one2one-ee84a0d2 -2025-06-03 15:52:08,147 - tests.orchestrator - INFO - Waiting for pod test-pod-d95dbc73 to be deleted -2025-06-03 15:52:40,769 - tests.orchestrator - INFO - Pod test-pod-d95dbc73 has been deleted -2025-06-03 15:52:40,769 - tests.orchestrator - INFO - Deleted pod: test-pod-d95dbc73 -2025-06-03 15:52:41,769 - tests.orchestrator - INFO - Selected operation: _create_pvc -2025-06-03 15:52:41,769 - tests.orchestrator - INFO - Creating PVC: test-pvc-c2fea111 -2025-06-03 15:52:41,945 - tests.orchestrator - INFO - Created PVC: test-pvc-c2fea111 -2025-06-03 15:52:41,945 - tests.orchestrator - INFO - Waiting for PVC test-pvc-c2fea111 to be bound -2025-06-03 15:52:44,085 - tests.orchestrator - INFO - PVC test-pvc-c2fea111 is bound -2025-06-03 15:52:45,086 - tests.orchestrator - INFO - Selected operation: _verify_readwrite -2025-06-03 15:52:45,086 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test -2025-06-03 15:52:46,086 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario -2025-06-03 15:52:46,086 - tests.orchestrator - INFO - Running specific scenario: _scenario_many_to_one -2025-06-03 15:52:46,086 - tests.orchestrator - INFO - Running scenario: Many pods to one PVC -2025-06-03 15:52:46,157 - tests.orchestrator - INFO - Waiting for PVC many2one-5c9be389 to be bound -2025-06-03 15:52:48,291 - tests.orchestrator - INFO - PVC many2one-5c9be389 is bound -2025-06-03 15:52:48,291 - tests.orchestrator - INFO - Creating 4 pods for the same PVC -2025-06-03 15:52:48,291 - tests.orchestrator - INFO - Creating pod test-pod-7eb2e7e7 on PVC many2one-5c9be389 -2025-06-03 15:52:48,366 - tests.orchestrator - INFO - Created pod: test-pod-7eb2e7e7 using PVC: many2one-5c9be389 -2025-06-03 15:52:48,366 - tests.orchestrator - INFO - Waiting for pod test-pod-7eb2e7e7 to be ready -2025-06-03 15:52:54,649 - tests.orchestrator - INFO - Pod test-pod-7eb2e7e7 is ready -2025-06-03 15:52:54,649 - tests.orchestrator - INFO - Creating pod test-pod-506e6bf9 on PVC many2one-5c9be389 -2025-06-03 15:52:54,725 - tests.orchestrator - INFO - Created pod: test-pod-506e6bf9 using PVC: many2one-5c9be389 -2025-06-03 15:52:54,725 - tests.orchestrator - INFO - Waiting for pod test-pod-506e6bf9 to be ready -2025-06-03 15:53:07,211 - tests.orchestrator - INFO - Pod test-pod-506e6bf9 is ready -2025-06-03 15:53:07,211 - tests.orchestrator - INFO - Creating pod test-pod-6b52ef99 on PVC many2one-5c9be389 -2025-06-03 15:53:07,283 - tests.orchestrator - INFO - Created pod: test-pod-6b52ef99 using PVC: many2one-5c9be389 -2025-06-03 15:53:07,284 - tests.orchestrator - INFO - Waiting for pod test-pod-6b52ef99 to be ready -2025-06-03 15:53:19,854 - tests.orchestrator - INFO - Pod test-pod-6b52ef99 is ready -2025-06-03 15:53:19,854 - tests.orchestrator - INFO - Creating pod test-pod-fc6a3c0c on PVC many2one-5c9be389 -2025-06-03 15:53:19,942 - tests.orchestrator - INFO - Created pod: test-pod-fc6a3c0c using PVC: many2one-5c9be389 -2025-06-03 15:53:19,942 - tests.orchestrator - INFO - Waiting for pod test-pod-fc6a3c0c to be ready -2025-06-03 15:53:32,426 - tests.orchestrator - INFO - Pod test-pod-fc6a3c0c is ready -2025-06-03 15:53:32,495 - tests.orchestrator - ERROR - Error in many-to-one scenario: (400) -Reason: Bad Request -HTTP response headers: HTTPHeaderDict({'Audit-Id': '9cd3b5fb-b62b-4ee0-b08b-2a3e761b6d8c', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Tue, 03 Jun 2025 15:53:32 GMT', 'Content-Length': '139'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Upgrade request required","reason":"BadRequest","code":400} - - -2025-06-03 15:53:33,496 - tests.orchestrator - INFO - Selected operation: _attach_pod -2025-06-03 15:53:33,496 - tests.orchestrator - INFO - Creating pod test-pod-4f87753e on PVC one2one-b38cd705 -2025-06-03 15:53:33,569 - tests.orchestrator - INFO - Created pod: test-pod-4f87753e using PVC: one2one-b38cd705 -2025-06-03 15:53:33,569 - tests.orchestrator - INFO - Waiting for pod test-pod-4f87753e to be ready -2025-06-03 15:53:39,842 - tests.orchestrator - INFO - Pod test-pod-4f87753e is ready -2025-06-03 15:53:40,842 - tests.orchestrator - INFO - Selected operation: _verify_readwrite -2025-06-03 15:53:40,842 - tests.orchestrator - INFO - Testing read/write between pods test-pod-8f39fd3b and test-pod-4f87753e sharing PVC one2one-b38cd705 -2025-06-03 15:53:40,913 - tests.orchestrator - ERROR - Failed in read/write verification: (400) -Reason: Bad Request -HTTP response headers: HTTPHeaderDict({'Audit-Id': '571b0988-ce3a-498b-9d74-580abe8cc33a', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Tue, 03 Jun 2025 15:53:40 GMT', 'Content-Length': '139'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Upgrade request required","reason":"BadRequest","code":400} - - -2025-06-03 15:53:41,914 - tests.orchestrator - INFO - Selected operation: _delete_pod -2025-06-03 15:53:41,914 - tests.orchestrator - INFO - Deleting pod: test-pod-7f739327 from PVC: one2one-74ea88f9 -2025-06-03 15:53:41,991 - tests.orchestrator - INFO - Waiting for pod test-pod-7f739327 to be deleted -2025-06-03 15:54:14,614 - tests.orchestrator - INFO - Pod test-pod-7f739327 has been deleted -2025-06-03 15:54:14,614 - tests.orchestrator - INFO - Deleted pod: test-pod-7f739327 -2025-06-03 15:54:15,615 - tests.orchestrator - INFO - Selected operation: _delete_pod -2025-06-03 15:54:15,615 - tests.orchestrator - INFO - Deleting pod: test-pod-4f87753e from PVC: one2one-b38cd705 -2025-06-03 15:54:15,691 - tests.orchestrator - INFO - Waiting for pod test-pod-4f87753e to be deleted -2025-06-03 15:54:48,304 - tests.orchestrator - INFO - Pod test-pod-4f87753e has been deleted -2025-06-03 15:54:48,304 - tests.orchestrator - INFO - Deleted pod: test-pod-4f87753e -2025-06-03 15:54:49,304 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario -2025-06-03 15:54:49,304 - tests.orchestrator - INFO - Running specific scenario: _scenario_concurrent_pvc -2025-06-03 15:54:49,304 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations -2025-06-03 15:54:49,304 - tests.orchestrator - INFO - Creating 7 PVCs in quick succession -2025-06-03 15:54:49,382 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-ba4fefcb -2025-06-03 15:54:49,382 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-ba4fefcb to be bound -2025-06-03 15:54:51,518 - tests.orchestrator - INFO - PVC concurrent-pvc-ba4fefcb is bound -2025-06-03 15:54:51,586 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-1f973741 -2025-06-03 15:54:51,587 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-1f973741 to be bound -2025-06-03 15:54:53,725 - tests.orchestrator - INFO - PVC concurrent-pvc-1f973741 is bound -2025-06-03 15:54:53,793 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-3559ffc8 -2025-06-03 15:54:53,793 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-3559ffc8 to be bound -2025-06-03 15:54:55,928 - tests.orchestrator - INFO - PVC concurrent-pvc-3559ffc8 is bound -2025-06-03 15:54:55,996 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-f3060a1d -2025-06-03 15:54:55,996 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-f3060a1d to be bound -2025-06-03 15:54:58,130 - tests.orchestrator - INFO - PVC concurrent-pvc-f3060a1d is bound -2025-06-03 15:54:58,198 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-e44ab328 -2025-06-03 15:54:58,198 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-e44ab328 to be bound -2025-06-03 15:55:00,332 - tests.orchestrator - INFO - PVC concurrent-pvc-e44ab328 is bound -2025-06-03 15:55:00,403 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-2d6e60cb -2025-06-03 15:55:00,403 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-2d6e60cb to be bound -2025-06-03 15:55:02,538 - tests.orchestrator - INFO - PVC concurrent-pvc-2d6e60cb is bound -2025-06-03 15:55:02,612 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-c27e9e10 -2025-06-03 15:55:02,612 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-c27e9e10 to be bound -2025-06-03 15:55:04,754 - tests.orchestrator - INFO - PVC concurrent-pvc-c27e9e10 is bound -2025-06-03 15:55:04,754 - tests.orchestrator - INFO - Successfully created 7 PVCs -2025-06-03 15:55:04,754 - tests.orchestrator - INFO - Creating pod test-pod-f3855837 on PVC concurrent-pvc-3559ffc8 -2025-06-03 15:55:04,834 - tests.orchestrator - INFO - Created pod: test-pod-f3855837 using PVC: concurrent-pvc-3559ffc8 -2025-06-03 15:55:04,834 - tests.orchestrator - INFO - Waiting for pod test-pod-f3855837 to be ready -2025-06-03 15:55:17,347 - tests.orchestrator - INFO - Pod test-pod-f3855837 is ready -2025-06-03 15:55:17,347 - tests.orchestrator - INFO - Creating pod test-pod-8e0cc6ec on PVC concurrent-pvc-e44ab328 -2025-06-03 15:55:17,423 - tests.orchestrator - INFO - Created pod: test-pod-8e0cc6ec using PVC: concurrent-pvc-e44ab328 -2025-06-03 15:55:17,423 - tests.orchestrator - INFO - Waiting for pod test-pod-8e0cc6ec to be ready -2025-06-03 15:55:23,703 - tests.orchestrator - INFO - Pod test-pod-8e0cc6ec is ready -2025-06-03 15:55:23,703 - tests.orchestrator - INFO - Creating pod test-pod-f7e46b0a on PVC concurrent-pvc-1f973741 -2025-06-03 15:55:23,790 - tests.orchestrator - INFO - Created pod: test-pod-f7e46b0a using PVC: concurrent-pvc-1f973741 -2025-06-03 15:55:23,790 - tests.orchestrator - INFO - Waiting for pod test-pod-f7e46b0a to be ready -2025-06-03 15:55:36,266 - tests.orchestrator - INFO - Pod test-pod-f7e46b0a is ready -2025-06-03 15:55:36,266 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-ba4fefcb -2025-06-03 15:55:36,339 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-ba4fefcb to be deleted -2025-06-03 15:55:36,406 - tests.orchestrator - INFO - PVC concurrent-pvc-ba4fefcb has been deleted -2025-06-03 15:55:36,406 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-ba4fefcb -2025-06-03 15:55:36,406 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-1f973741 -2025-06-03 15:55:36,406 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-1f973741 -2025-06-03 15:55:36,406 - tests.orchestrator - INFO - Deleting pod: test-pod-f7e46b0a from PVC: concurrent-pvc-1f973741 -2025-06-03 15:55:36,500 - tests.orchestrator - INFO - Waiting for pod test-pod-f7e46b0a to be deleted -2025-06-03 15:56:09,115 - tests.orchestrator - INFO - Pod test-pod-f7e46b0a has been deleted -2025-06-03 15:56:09,115 - tests.orchestrator - INFO - Deleted pod: test-pod-f7e46b0a -2025-06-03 15:56:09,188 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-1f973741 to be deleted -2025-06-03 15:56:09,254 - tests.orchestrator - INFO - PVC concurrent-pvc-1f973741 has been deleted -2025-06-03 15:56:09,255 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-1f973741 -2025-06-03 15:56:09,255 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-3559ffc8 -2025-06-03 15:56:09,255 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-3559ffc8 -2025-06-03 15:56:09,255 - tests.orchestrator - INFO - Deleting pod: test-pod-f3855837 from PVC: concurrent-pvc-3559ffc8 -2025-06-03 15:56:09,329 - tests.orchestrator - INFO - Waiting for pod test-pod-f3855837 to be deleted -2025-06-03 15:56:41,952 - tests.orchestrator - INFO - Pod test-pod-f3855837 has been deleted -2025-06-03 15:56:41,952 - tests.orchestrator - INFO - Deleted pod: test-pod-f3855837 -2025-06-03 15:56:42,031 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-3559ffc8 to be deleted -2025-06-03 15:56:42,102 - tests.orchestrator - INFO - PVC concurrent-pvc-3559ffc8 has been deleted -2025-06-03 15:56:42,102 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-3559ffc8 -2025-06-03 15:56:42,103 - tests.orchestrator - INFO - Rapid PVC scenario completed -2025-06-03 15:56:43,103 - tests.orchestrator - INFO - Selected operation: _delete_pod -2025-06-03 15:56:43,103 - tests.orchestrator - INFO - Deleting pod: test-pod-6b52ef99 from PVC: many2one-5c9be389 -2025-06-03 15:56:43,182 - tests.orchestrator - INFO - Waiting for pod test-pod-6b52ef99 to be deleted -2025-06-03 15:57:16,611 - tests.orchestrator - INFO - Pod test-pod-6b52ef99 has been deleted -2025-06-03 15:57:16,611 - tests.orchestrator - INFO - Deleted pod: test-pod-6b52ef99 -2025-06-03 15:57:17,611 - tests.orchestrator - INFO - Selected operation: _attach_pod -2025-06-03 15:57:17,611 - tests.orchestrator - INFO - Creating pod test-pod-a819397c on PVC concurrent-pvc-c27e9e10 -2025-06-03 15:57:17,702 - tests.orchestrator - INFO - Created pod: test-pod-a819397c using PVC: concurrent-pvc-c27e9e10 -2025-06-03 15:57:17,702 - tests.orchestrator - INFO - Waiting for pod test-pod-a819397c to be ready -2025-06-03 15:57:30,182 - tests.orchestrator - INFO - Pod test-pod-a819397c is ready -2025-06-03 15:57:31,182 - tests.orchestrator - INFO - Selected operation: _create_pvc -2025-06-03 15:57:31,182 - tests.orchestrator - INFO - Creating PVC: test-pvc-ee48e109 -2025-06-03 15:57:31,250 - tests.orchestrator - INFO - Created PVC: test-pvc-ee48e109 -2025-06-03 15:57:31,250 - tests.orchestrator - INFO - Waiting for PVC test-pvc-ee48e109 to be bound -2025-06-03 15:57:33,384 - tests.orchestrator - INFO - PVC test-pvc-ee48e109 is bound -2025-06-03 15:57:34,384 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario -2025-06-03 15:57:34,385 - tests.orchestrator - INFO - Running specific scenario: _scenario_concurrent_pvc -2025-06-03 15:57:34,385 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations -2025-06-03 15:57:34,385 - tests.orchestrator - INFO - Creating 7 PVCs in quick succession -2025-06-03 15:57:34,455 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-51a18d7b -2025-06-03 15:57:34,456 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-51a18d7b to be bound -2025-06-03 15:57:36,594 - tests.orchestrator - INFO - PVC concurrent-pvc-51a18d7b is bound -2025-06-03 15:57:36,666 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-c7fabc72 -2025-06-03 15:57:36,666 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-c7fabc72 to be bound -2025-06-03 15:57:38,803 - tests.orchestrator - INFO - PVC concurrent-pvc-c7fabc72 is bound -2025-06-03 15:57:38,877 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-07a19245 -2025-06-03 15:57:38,877 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-07a19245 to be bound -2025-06-03 15:57:41,013 - tests.orchestrator - INFO - PVC concurrent-pvc-07a19245 is bound -2025-06-03 15:57:41,080 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-5d6cd5f4 -2025-06-03 15:57:41,080 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-5d6cd5f4 to be bound -2025-06-03 15:57:43,222 - tests.orchestrator - INFO - PVC concurrent-pvc-5d6cd5f4 is bound -2025-06-03 15:57:43,291 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-9cdc5f04 -2025-06-03 15:57:43,291 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-9cdc5f04 to be bound -2025-06-03 15:57:45,427 - tests.orchestrator - INFO - PVC concurrent-pvc-9cdc5f04 is bound -2025-06-03 15:57:45,494 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-0112d2c9 -2025-06-03 15:57:45,494 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-0112d2c9 to be bound -2025-06-03 15:57:47,630 - tests.orchestrator - INFO - PVC concurrent-pvc-0112d2c9 is bound -2025-06-03 15:57:47,708 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-96aa4689 -2025-06-03 15:57:47,708 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-96aa4689 to be bound -2025-06-03 15:57:49,840 - tests.orchestrator - INFO - PVC concurrent-pvc-96aa4689 is bound -2025-06-03 15:57:49,840 - tests.orchestrator - INFO - Successfully created 7 PVCs -2025-06-03 15:57:49,840 - tests.orchestrator - INFO - Creating pod test-pod-cc67cce8 on PVC concurrent-pvc-07a19245 -2025-06-03 15:57:49,915 - tests.orchestrator - INFO - Created pod: test-pod-cc67cce8 using PVC: concurrent-pvc-07a19245 -2025-06-03 15:57:49,915 - tests.orchestrator - INFO - Waiting for pod test-pod-cc67cce8 to be ready -2025-06-03 15:58:02,392 - tests.orchestrator - INFO - Pod test-pod-cc67cce8 is ready -2025-06-03 15:58:02,393 - tests.orchestrator - INFO - Creating pod test-pod-39578545 on PVC concurrent-pvc-9cdc5f04 -2025-06-03 15:58:02,484 - tests.orchestrator - INFO - Created pod: test-pod-39578545 using PVC: concurrent-pvc-9cdc5f04 -2025-06-03 15:58:02,484 - tests.orchestrator - INFO - Waiting for pod test-pod-39578545 to be ready -2025-06-03 15:58:08,761 - tests.orchestrator - INFO - Pod test-pod-39578545 is ready -2025-06-03 15:58:08,761 - tests.orchestrator - INFO - Creating pod test-pod-8bd839df on PVC concurrent-pvc-96aa4689 -2025-06-03 15:58:08,836 - tests.orchestrator - INFO - Created pod: test-pod-8bd839df using PVC: concurrent-pvc-96aa4689 -2025-06-03 15:58:08,836 - tests.orchestrator - INFO - Waiting for pod test-pod-8bd839df to be ready -2025-06-03 15:58:21,318 - tests.orchestrator - INFO - Pod test-pod-8bd839df is ready -2025-06-03 15:58:21,318 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-07a19245 -2025-06-03 15:58:21,319 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-07a19245 -2025-06-03 15:58:21,319 - tests.orchestrator - INFO - Deleting pod: test-pod-cc67cce8 from PVC: concurrent-pvc-07a19245 -2025-06-03 15:58:21,396 - tests.orchestrator - INFO - Waiting for pod test-pod-cc67cce8 to be deleted -2025-06-03 15:58:54,172 - tests.orchestrator - INFO - Pod test-pod-cc67cce8 has been deleted -2025-06-03 15:58:54,172 - tests.orchestrator - INFO - Deleted pod: test-pod-cc67cce8 -2025-06-03 15:58:54,243 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-07a19245 to be deleted -2025-06-03 15:58:54,310 - tests.orchestrator - INFO - PVC concurrent-pvc-07a19245 has been deleted -2025-06-03 15:58:54,311 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-07a19245 -2025-06-03 15:58:54,311 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-5d6cd5f4 -2025-06-03 15:58:54,382 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-5d6cd5f4 to be deleted -2025-06-03 15:58:54,450 - tests.orchestrator - INFO - PVC concurrent-pvc-5d6cd5f4 has been deleted -2025-06-03 15:58:54,450 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-5d6cd5f4 -2025-06-03 15:58:54,450 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-51a18d7b -2025-06-03 15:58:54,523 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-51a18d7b to be deleted -2025-06-03 15:58:54,589 - tests.orchestrator - INFO - PVC concurrent-pvc-51a18d7b has been deleted -2025-06-03 15:58:54,590 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-51a18d7b -2025-06-03 15:58:54,590 - tests.orchestrator - INFO - Rapid PVC scenario completed -2025-06-03 15:58:55,590 - tests.orchestrator - INFO - Test completed in 650.13 seconds -2025-06-03 15:58:55,590 - tests.orchestrator - INFO - Cleaning up resources... -2025-06-03 15:58:55,590 - tests.orchestrator - INFO - Deleting pod: test-pod-9dcfaf3b from PVC: one2one-53c40f0a -2025-06-03 15:58:55,667 - tests.orchestrator - INFO - Waiting for pod test-pod-9dcfaf3b to be deleted -2025-06-03 15:59:28,324 - tests.orchestrator - INFO - Pod test-pod-9dcfaf3b has been deleted -2025-06-03 15:59:28,324 - tests.orchestrator - INFO - Deleted pod: test-pod-9dcfaf3b -2025-06-03 15:59:28,324 - tests.orchestrator - INFO - Deleting pod: test-pod-2a900eb4 from PVC: one2one-e698727d -2025-06-03 15:59:28,400 - tests.orchestrator - INFO - Waiting for pod test-pod-2a900eb4 to be deleted -2025-06-03 16:00:01,019 - tests.orchestrator - INFO - Pod test-pod-2a900eb4 has been deleted -2025-06-03 16:00:01,019 - tests.orchestrator - INFO - Deleted pod: test-pod-2a900eb4 -2025-06-03 16:00:01,019 - tests.orchestrator - INFO - Deleting pod: test-pod-8f39fd3b from PVC: one2one-b38cd705 -2025-06-03 16:00:01,100 - tests.orchestrator - INFO - Waiting for pod test-pod-8f39fd3b to be deleted -2025-06-03 16:00:33,739 - tests.orchestrator - INFO - Pod test-pod-8f39fd3b has been deleted -2025-06-03 16:00:33,739 - tests.orchestrator - INFO - Deleted pod: test-pod-8f39fd3b -2025-06-03 16:00:33,739 - tests.orchestrator - INFO - Deleting pod: test-pod-62c10933 from PVC: one2one-bee0c24d -2025-06-03 16:00:33,814 - tests.orchestrator - INFO - Waiting for pod test-pod-62c10933 to be deleted -2025-06-03 16:01:06,447 - tests.orchestrator - INFO - Pod test-pod-62c10933 has been deleted -2025-06-03 16:01:06,448 - tests.orchestrator - INFO - Deleted pod: test-pod-62c10933 -2025-06-03 16:01:06,448 - tests.orchestrator - INFO - Deleting pod: test-pod-1d7db6ad from PVC: one2one-11548b80 -2025-06-03 16:01:06,527 - tests.orchestrator - INFO - Waiting for pod test-pod-1d7db6ad to be deleted -2025-06-03 16:01:39,146 - tests.orchestrator - INFO - Pod test-pod-1d7db6ad has been deleted -2025-06-03 16:01:39,146 - tests.orchestrator - INFO - Deleted pod: test-pod-1d7db6ad -2025-06-03 16:01:39,146 - tests.orchestrator - INFO - Deleting pod: test-pod-431935c6 from PVC: one2one-2b9e9546 -2025-06-03 16:01:39,222 - tests.orchestrator - INFO - Waiting for pod test-pod-431935c6 to be deleted -2025-06-03 16:02:11,990 - tests.orchestrator - INFO - Pod test-pod-431935c6 has been deleted -2025-06-03 16:02:11,990 - tests.orchestrator - INFO - Deleted pod: test-pod-431935c6 -2025-06-03 16:02:11,990 - tests.orchestrator - INFO - Deleting pod: test-pod-7eb2e7e7 from PVC: many2one-5c9be389 -2025-06-03 16:02:12,112 - tests.orchestrator - INFO - Waiting for pod test-pod-7eb2e7e7 to be deleted -2025-06-03 16:02:44,728 - tests.orchestrator - INFO - Pod test-pod-7eb2e7e7 has been deleted -2025-06-03 16:02:44,728 - tests.orchestrator - INFO - Deleted pod: test-pod-7eb2e7e7 -2025-06-03 16:02:44,728 - tests.orchestrator - INFO - Deleting pod: test-pod-506e6bf9 from PVC: many2one-5c9be389 -2025-06-03 16:02:44,801 - tests.orchestrator - INFO - Waiting for pod test-pod-506e6bf9 to be deleted -2025-06-03 16:03:17,416 - tests.orchestrator - INFO - Pod test-pod-506e6bf9 has been deleted -2025-06-03 16:03:17,417 - tests.orchestrator - INFO - Deleted pod: test-pod-506e6bf9 -2025-06-03 16:03:17,417 - tests.orchestrator - INFO - Deleting pod: test-pod-fc6a3c0c from PVC: many2one-5c9be389 -2025-06-03 16:03:17,495 - tests.orchestrator - INFO - Waiting for pod test-pod-fc6a3c0c to be deleted -2025-06-03 16:03:50,115 - tests.orchestrator - INFO - Pod test-pod-fc6a3c0c has been deleted -2025-06-03 16:03:50,115 - tests.orchestrator - INFO - Deleted pod: test-pod-fc6a3c0c -2025-06-03 16:03:50,115 - tests.orchestrator - INFO - Deleting pod: test-pod-8e0cc6ec from PVC: concurrent-pvc-e44ab328 -2025-06-03 16:03:50,193 - tests.orchestrator - INFO - Waiting for pod test-pod-8e0cc6ec to be deleted -2025-06-03 16:04:22,834 - tests.orchestrator - INFO - Pod test-pod-8e0cc6ec has been deleted -2025-06-03 16:04:22,834 - tests.orchestrator - INFO - Deleted pod: test-pod-8e0cc6ec -2025-06-03 16:04:22,834 - tests.orchestrator - INFO - Deleting pod: test-pod-a819397c from PVC: concurrent-pvc-c27e9e10 -2025-06-03 16:04:22,909 - tests.orchestrator - INFO - Waiting for pod test-pod-a819397c to be deleted -2025-06-03 16:04:55,527 - tests.orchestrator - INFO - Pod test-pod-a819397c has been deleted -2025-06-03 16:04:55,527 - tests.orchestrator - INFO - Deleted pod: test-pod-a819397c -2025-06-03 16:04:55,527 - tests.orchestrator - INFO - Deleting pod: test-pod-39578545 from PVC: concurrent-pvc-9cdc5f04 -2025-06-03 16:04:55,601 - tests.orchestrator - INFO - Waiting for pod test-pod-39578545 to be deleted -2025-06-03 16:05:28,321 - tests.orchestrator - INFO - Pod test-pod-39578545 has been deleted -2025-06-03 16:05:28,321 - tests.orchestrator - INFO - Deleted pod: test-pod-39578545 -2025-06-03 16:05:28,322 - tests.orchestrator - INFO - Deleting pod: test-pod-8bd839df from PVC: concurrent-pvc-96aa4689 -2025-06-03 16:05:28,399 - tests.orchestrator - INFO - Waiting for pod test-pod-8bd839df to be deleted -2025-06-03 16:06:01,015 - tests.orchestrator - INFO - Pod test-pod-8bd839df has been deleted -2025-06-03 16:06:01,015 - tests.orchestrator - INFO - Deleted pod: test-pod-8bd839df -2025-06-03 16:06:01,016 - tests.orchestrator - INFO - Deleting PVC: test-pvc-9e0f63a9 -2025-06-03 16:06:01,091 - tests.orchestrator - INFO - Waiting for PVC test-pvc-9e0f63a9 to be deleted -2025-06-03 16:06:01,156 - tests.orchestrator - INFO - PVC test-pvc-9e0f63a9 has been deleted -2025-06-03 16:06:01,156 - tests.orchestrator - INFO - Deleted PVC: test-pvc-9e0f63a9 -2025-06-03 16:06:01,156 - tests.orchestrator - INFO - Deleting PVC: one2one-53c40f0a -2025-06-03 16:06:01,249 - tests.orchestrator - INFO - Waiting for PVC one2one-53c40f0a to be deleted -2025-06-03 16:06:01,313 - tests.orchestrator - INFO - PVC one2one-53c40f0a has been deleted -2025-06-03 16:06:01,314 - tests.orchestrator - INFO - Deleted PVC: one2one-53c40f0a -2025-06-03 16:06:01,315 - tests.orchestrator - INFO - Deleting PVC: one2one-e698727d -2025-06-03 16:06:01,401 - tests.orchestrator - INFO - Waiting for PVC one2one-e698727d to be deleted -2025-06-03 16:06:01,608 - tests.orchestrator - INFO - PVC one2one-e698727d has been deleted -2025-06-03 16:06:01,608 - tests.orchestrator - INFO - Deleted PVC: one2one-e698727d -2025-06-03 16:06:01,608 - tests.orchestrator - INFO - Deleting PVC: one2one-ee84a0d2 -2025-06-03 16:06:01,694 - tests.orchestrator - INFO - Waiting for PVC one2one-ee84a0d2 to be deleted -2025-06-03 16:06:01,786 - tests.orchestrator - INFO - PVC one2one-ee84a0d2 has been deleted -2025-06-03 16:06:01,786 - tests.orchestrator - INFO - Deleted PVC: one2one-ee84a0d2 -2025-06-03 16:06:01,786 - tests.orchestrator - INFO - Deleting PVC: one2one-b38cd705 -2025-06-03 16:06:01,891 - tests.orchestrator - INFO - Waiting for PVC one2one-b38cd705 to be deleted -2025-06-03 16:06:02,000 - tests.orchestrator - INFO - PVC one2one-b38cd705 has been deleted -2025-06-03 16:06:02,000 - tests.orchestrator - INFO - Deleted PVC: one2one-b38cd705 -2025-06-03 16:06:02,000 - tests.orchestrator - INFO - Deleting PVC: one2one-e7169041 -2025-06-03 16:06:02,121 - tests.orchestrator - INFO - Waiting for PVC one2one-e7169041 to be deleted -2025-06-03 16:06:02,191 - tests.orchestrator - INFO - PVC one2one-e7169041 has been deleted -2025-06-03 16:06:02,191 - tests.orchestrator - INFO - Deleted PVC: one2one-e7169041 -2025-06-03 16:06:02,191 - tests.orchestrator - INFO - Deleting PVC: one2one-bee0c24d -2025-06-03 16:06:02,270 - tests.orchestrator - INFO - Waiting for PVC one2one-bee0c24d to be deleted -2025-06-03 16:06:02,335 - tests.orchestrator - INFO - PVC one2one-bee0c24d has been deleted -2025-06-03 16:06:02,335 - tests.orchestrator - INFO - Deleted PVC: one2one-bee0c24d -2025-06-03 16:06:02,336 - tests.orchestrator - INFO - Deleting PVC: one2one-74ea88f9 -2025-06-03 16:06:02,412 - tests.orchestrator - INFO - Waiting for PVC one2one-74ea88f9 to be deleted -2025-06-03 16:06:02,477 - tests.orchestrator - INFO - PVC one2one-74ea88f9 has been deleted -2025-06-03 16:06:02,477 - tests.orchestrator - INFO - Deleted PVC: one2one-74ea88f9 -2025-06-03 16:06:02,477 - tests.orchestrator - INFO - Deleting PVC: one2one-11548b80 -2025-06-03 16:06:02,551 - tests.orchestrator - INFO - Waiting for PVC one2one-11548b80 to be deleted -2025-06-03 16:06:02,616 - tests.orchestrator - INFO - PVC one2one-11548b80 has been deleted -2025-06-03 16:06:02,616 - tests.orchestrator - INFO - Deleted PVC: one2one-11548b80 -2025-06-03 16:06:02,616 - tests.orchestrator - INFO - Deleting PVC: one2one-2b9e9546 -2025-06-03 16:06:02,694 - tests.orchestrator - INFO - Waiting for PVC one2one-2b9e9546 to be deleted -2025-06-03 16:06:02,759 - tests.orchestrator - INFO - PVC one2one-2b9e9546 has been deleted -2025-06-03 16:06:02,760 - tests.orchestrator - INFO - Deleted PVC: one2one-2b9e9546 -2025-06-03 16:06:02,760 - tests.orchestrator - INFO - Deleting PVC: test-pvc-c2fea111 -2025-06-03 16:06:02,838 - tests.orchestrator - INFO - Waiting for PVC test-pvc-c2fea111 to be deleted -2025-06-03 16:06:02,903 - tests.orchestrator - INFO - PVC test-pvc-c2fea111 has been deleted -2025-06-03 16:06:02,903 - tests.orchestrator - INFO - Deleted PVC: test-pvc-c2fea111 -2025-06-03 16:06:02,904 - tests.orchestrator - INFO - Deleting PVC: many2one-5c9be389 -2025-06-03 16:06:02,982 - tests.orchestrator - INFO - Waiting for PVC many2one-5c9be389 to be deleted -2025-06-03 16:06:03,061 - tests.orchestrator - INFO - PVC many2one-5c9be389 has been deleted -2025-06-03 16:06:03,061 - tests.orchestrator - INFO - Deleted PVC: many2one-5c9be389 -2025-06-03 16:06:03,061 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-f3060a1d -2025-06-03 16:06:03,144 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-f3060a1d to be deleted -2025-06-03 16:06:03,212 - tests.orchestrator - INFO - PVC concurrent-pvc-f3060a1d has been deleted -2025-06-03 16:06:03,212 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-f3060a1d -2025-06-03 16:06:03,212 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-e44ab328 -2025-06-03 16:06:03,288 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-e44ab328 to be deleted -2025-06-03 16:06:03,360 - tests.orchestrator - INFO - PVC concurrent-pvc-e44ab328 has been deleted -2025-06-03 16:06:03,360 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-e44ab328 -2025-06-03 16:06:03,360 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-2d6e60cb -2025-06-03 16:06:03,455 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-2d6e60cb to be deleted -2025-06-03 16:06:03,530 - tests.orchestrator - INFO - PVC concurrent-pvc-2d6e60cb has been deleted -2025-06-03 16:06:03,530 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-2d6e60cb -2025-06-03 16:06:03,530 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-c27e9e10 -2025-06-03 16:06:03,609 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-c27e9e10 to be deleted -2025-06-03 16:06:03,674 - tests.orchestrator - INFO - PVC concurrent-pvc-c27e9e10 has been deleted -2025-06-03 16:06:03,675 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-c27e9e10 -2025-06-03 16:06:03,675 - tests.orchestrator - INFO - Deleting PVC: test-pvc-ee48e109 -2025-06-03 16:06:03,750 - tests.orchestrator - INFO - Waiting for PVC test-pvc-ee48e109 to be deleted -2025-06-03 16:06:03,823 - tests.orchestrator - INFO - PVC test-pvc-ee48e109 has been deleted -2025-06-03 16:06:03,823 - tests.orchestrator - INFO - Deleted PVC: test-pvc-ee48e109 -2025-06-03 16:06:03,823 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-c7fabc72 -2025-06-03 16:06:03,898 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-c7fabc72 to be deleted -2025-06-03 16:06:03,997 - tests.orchestrator - INFO - PVC concurrent-pvc-c7fabc72 has been deleted -2025-06-03 16:06:03,998 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-c7fabc72 -2025-06-03 16:06:03,998 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-9cdc5f04 -2025-06-03 16:06:04,072 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-9cdc5f04 to be deleted -2025-06-03 16:06:04,139 - tests.orchestrator - INFO - PVC concurrent-pvc-9cdc5f04 has been deleted -2025-06-03 16:06:04,139 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-9cdc5f04 -2025-06-03 16:06:04,139 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-0112d2c9 -2025-06-03 16:06:04,229 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-0112d2c9 to be deleted -2025-06-03 16:06:04,297 - tests.orchestrator - INFO - PVC concurrent-pvc-0112d2c9 has been deleted -2025-06-03 16:06:04,297 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-0112d2c9 -2025-06-03 16:06:04,298 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-96aa4689 -2025-06-03 16:06:04,388 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-96aa4689 to be deleted -2025-06-03 16:06:04,456 - tests.orchestrator - INFO - PVC concurrent-pvc-96aa4689 has been deleted -2025-06-03 16:06:04,456 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-96aa4689 -2025-06-03 16:06:04,456 - tests.orchestrator - INFO - Cleanup completed -2025-06-03 16:06:04,456 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-03 16:06:04,456 - tests.orchestrator - INFO - --- Operations --- -2025-06-03 16:06:04,456 - tests.orchestrator - INFO - create_pvc: 17 succeeded, 0 failed (100.0%) -2025-06-03 16:06:04,456 - tests.orchestrator - INFO - attach_pod: 23 succeeded, 0 failed (100.0%) -2025-06-03 16:06:04,456 - tests.orchestrator - INFO - delete_pod: 23 succeeded, 0 failed (100.0%) -2025-06-03 16:06:04,456 - tests.orchestrator - INFO - delete_pvc: 27 succeeded, 0 failed (100.0%) -2025-06-03 16:06:04,456 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) -2025-06-03 16:06:04,456 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) -2025-06-03 16:06:04,456 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-03 16:06:04,457 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-03 16:06:04,457 - tests.orchestrator - INFO - many_to_one: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-03 16:06:04,457 - tests.orchestrator - INFO - one_to_one: 0 succeeded, 2 failed out of 2 runs (0.0%) -2025-06-03 16:06:04,457 - tests.orchestrator - INFO - concurrent_pvc: 2 succeeded, 0 failed out of 2 runs (100.0%) -2025-06-03 16:06:04,457 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250603_174810.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250603_174810.log deleted file mode 100644 index 6b0890219..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250603_174810.log +++ /dev/null @@ -1,8 +0,0 @@ -2025-06-03 17:48:10,686 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-03 17:48:10,686 - tests.orchestrator - INFO - Starting orchestrator test for 50 seconds -2025-06-03 17:48:10,945 - tests.orchestrator - ERROR - Error checking StorageClass: (401) -Reason: Unauthorized -HTTP response headers: HTTPHeaderDict({'Audit-Id': '501ac0bb-a3bf-4a94-bd95-be7f3c4887ad', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Tue, 03 Jun 2025 17:48:10 GMT', 'Content-Length': '129'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} - - diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250603_175820.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250603_175820.log deleted file mode 100644 index ebe1eb6bb..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250603_175820.log +++ /dev/null @@ -1,77 +0,0 @@ -2025-06-03 17:58:20,784 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-03 17:58:20,784 - tests.orchestrator - INFO - Starting orchestrator test for 50 seconds -2025-06-03 17:58:21,041 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-03 17:58:21,041 - tests.orchestrator - INFO - Selected operation: _create_pvc -2025-06-03 17:58:21,041 - tests.orchestrator - INFO - Creating PVC: test-pvc-595330e0 -2025-06-03 17:58:21,286 - tests.orchestrator - INFO - Created PVC: test-pvc-595330e0 -2025-06-03 17:58:21,287 - tests.orchestrator - INFO - Waiting for PVC test-pvc-595330e0 to be bound -2025-06-03 17:58:23,445 - tests.orchestrator - INFO - PVC test-pvc-595330e0 is bound -2025-06-03 17:58:24,446 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario -2025-06-03 17:58:24,446 - tests.orchestrator - INFO - Running specific scenario: _scenario_many_to_one -2025-06-03 17:58:24,446 - tests.orchestrator - INFO - Running scenario: Many pods to one PVC -2025-06-03 17:58:24,530 - tests.orchestrator - INFO - Waiting for PVC many2one-8a476379 to be bound -2025-06-03 17:58:26,691 - tests.orchestrator - INFO - PVC many2one-8a476379 is bound -2025-06-03 17:58:26,691 - tests.orchestrator - INFO - Creating 4 pods for the same PVC -2025-06-03 17:58:26,691 - tests.orchestrator - INFO - Creating pod test-pod-7ca8c26c on PVC many2one-8a476379 -2025-06-03 17:58:26,797 - tests.orchestrator - INFO - Created pod: test-pod-7ca8c26c using PVC: many2one-8a476379 -2025-06-03 17:58:26,797 - tests.orchestrator - INFO - Waiting for pod test-pod-7ca8c26c to be ready -2025-06-03 17:58:39,374 - tests.orchestrator - INFO - Pod test-pod-7ca8c26c is ready -2025-06-03 17:58:39,375 - tests.orchestrator - INFO - Creating pod test-pod-f9ae1d98 on PVC many2one-8a476379 -2025-06-03 17:58:39,463 - tests.orchestrator - INFO - Created pod: test-pod-f9ae1d98 using PVC: many2one-8a476379 -2025-06-03 17:58:39,463 - tests.orchestrator - INFO - Waiting for pod test-pod-f9ae1d98 to be ready -2025-06-03 17:58:52,156 - tests.orchestrator - INFO - Pod test-pod-f9ae1d98 is ready -2025-06-03 17:58:52,156 - tests.orchestrator - INFO - Creating pod test-pod-c048c330 on PVC many2one-8a476379 -2025-06-03 17:58:52,257 - tests.orchestrator - INFO - Created pod: test-pod-c048c330 using PVC: many2one-8a476379 -2025-06-03 17:58:52,257 - tests.orchestrator - INFO - Waiting for pod test-pod-c048c330 to be ready -2025-06-03 17:59:04,830 - tests.orchestrator - INFO - Pod test-pod-c048c330 is ready -2025-06-03 17:59:04,830 - tests.orchestrator - INFO - Creating pod test-pod-2414b40d on PVC many2one-8a476379 -2025-06-03 17:59:04,917 - tests.orchestrator - INFO - Created pod: test-pod-2414b40d using PVC: many2one-8a476379 -2025-06-03 17:59:04,917 - tests.orchestrator - INFO - Waiting for pod test-pod-2414b40d to be ready -2025-06-03 17:59:17,501 - tests.orchestrator - INFO - Pod test-pod-2414b40d is ready -2025-06-03 17:59:17,585 - tests.orchestrator - ERROR - Error in many-to-one scenario: (400) -Reason: Bad Request -HTTP response headers: HTTPHeaderDict({'Audit-Id': 'b1a66ba3-d997-4bf0-9a5c-eb8d855e80b5', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Tue, 03 Jun 2025 17:59:17 GMT', 'Content-Length': '139'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Upgrade request required","reason":"BadRequest","code":400} - - -2025-06-03 17:59:18,585 - tests.orchestrator - INFO - Test completed in 57.80 seconds -2025-06-03 17:59:18,586 - tests.orchestrator - INFO - Cleaning up resources... -2025-06-03 17:59:18,586 - tests.orchestrator - INFO - Deleting pod: test-pod-7ca8c26c from PVC: many2one-8a476379 -2025-06-03 17:59:18,675 - tests.orchestrator - INFO - Waiting for pod test-pod-7ca8c26c to be deleted -2025-06-03 17:59:51,422 - tests.orchestrator - INFO - Pod test-pod-7ca8c26c has been deleted -2025-06-03 17:59:51,423 - tests.orchestrator - INFO - Deleted pod: test-pod-7ca8c26c -2025-06-03 17:59:51,423 - tests.orchestrator - INFO - Deleting pod: test-pod-f9ae1d98 from PVC: many2one-8a476379 -2025-06-03 17:59:51,528 - tests.orchestrator - INFO - Waiting for pod test-pod-f9ae1d98 to be deleted -2025-06-03 18:00:24,370 - tests.orchestrator - INFO - Pod test-pod-f9ae1d98 has been deleted -2025-06-03 18:00:24,370 - tests.orchestrator - INFO - Deleted pod: test-pod-f9ae1d98 -2025-06-03 18:00:24,370 - tests.orchestrator - INFO - Deleting pod: test-pod-c048c330 from PVC: many2one-8a476379 -2025-06-03 18:00:24,459 - tests.orchestrator - INFO - Waiting for pod test-pod-c048c330 to be deleted -2025-06-03 18:00:57,202 - tests.orchestrator - INFO - Pod test-pod-c048c330 has been deleted -2025-06-03 18:00:57,203 - tests.orchestrator - INFO - Deleted pod: test-pod-c048c330 -2025-06-03 18:00:57,203 - tests.orchestrator - INFO - Deleting pod: test-pod-2414b40d from PVC: many2one-8a476379 -2025-06-03 18:00:57,295 - tests.orchestrator - INFO - Waiting for pod test-pod-2414b40d to be deleted -2025-06-03 18:01:30,035 - tests.orchestrator - INFO - Pod test-pod-2414b40d has been deleted -2025-06-03 18:01:30,035 - tests.orchestrator - INFO - Deleted pod: test-pod-2414b40d -2025-06-03 18:01:30,035 - tests.orchestrator - INFO - Deleting PVC: test-pvc-595330e0 -2025-06-03 18:01:30,127 - tests.orchestrator - INFO - Waiting for PVC test-pvc-595330e0 to be deleted -2025-06-03 18:01:30,207 - tests.orchestrator - INFO - PVC test-pvc-595330e0 has been deleted -2025-06-03 18:01:30,207 - tests.orchestrator - INFO - Deleted PVC: test-pvc-595330e0 -2025-06-03 18:01:30,207 - tests.orchestrator - INFO - Deleting PVC: many2one-8a476379 -2025-06-03 18:01:30,298 - tests.orchestrator - INFO - Waiting for PVC many2one-8a476379 to be deleted -2025-06-03 18:01:30,377 - tests.orchestrator - INFO - PVC many2one-8a476379 has been deleted -2025-06-03 18:01:30,377 - tests.orchestrator - INFO - Deleted PVC: many2one-8a476379 -2025-06-03 18:01:30,377 - tests.orchestrator - INFO - Cleanup completed -2025-06-03 18:01:30,377 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-03 18:01:30,377 - tests.orchestrator - INFO - --- Operations --- -2025-06-03 18:01:30,377 - tests.orchestrator - INFO - create_pvc: 1 succeeded, 0 failed (100.0%) -2025-06-03 18:01:30,377 - tests.orchestrator - INFO - attach_pod: 4 succeeded, 0 failed (100.0%) -2025-06-03 18:01:30,377 - tests.orchestrator - INFO - delete_pod: 4 succeeded, 0 failed (100.0%) -2025-06-03 18:01:30,377 - tests.orchestrator - INFO - delete_pvc: 2 succeeded, 0 failed (100.0%) -2025-06-03 18:01:30,377 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 0 failed (0.0%) -2025-06-03 18:01:30,377 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 0 failed (0.0%) -2025-06-03 18:01:30,377 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-03 18:01:30,377 - tests.orchestrator - INFO - shared_volume_rw: No runs -2025-06-03 18:01:30,378 - tests.orchestrator - INFO - many_to_one: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-03 18:01:30,378 - tests.orchestrator - INFO - one_to_one: No runs -2025-06-03 18:01:30,378 - tests.orchestrator - INFO - concurrent_pvc: No runs -2025-06-03 18:01:30,378 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250603_204418.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250603_204418.log deleted file mode 100644 index 55b365575..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250603_204418.log +++ /dev/null @@ -1,8 +0,0 @@ -2025-06-03 20:44:18,852 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-03 20:44:18,853 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-03 20:44:19,110 - tests.orchestrator - ERROR - Error checking StorageClass: (401) -Reason: Unauthorized -HTTP response headers: HTTPHeaderDict({'Audit-Id': '75ffdf3f-c989-47e5-afa5-81c57f7481c8', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Tue, 03 Jun 2025 20:44:19 GMT', 'Content-Length': '129'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} - - diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250603_204433.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250603_204433.log deleted file mode 100644 index 6a692a266..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250603_204433.log +++ /dev/null @@ -1,94 +0,0 @@ -2025-06-03 20:44:33,955 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-03 20:44:33,956 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-03 20:44:34,200 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-03 20:44:34,200 - tests.orchestrator - INFO - Selected operation: _attach_pod -2025-06-03 20:44:34,201 - tests.orchestrator - INFO - No PVCs available, skipping pod attachment -2025-06-03 20:44:35,201 - tests.orchestrator - INFO - Selected operation: _verify_readwrite -2025-06-03 20:44:35,201 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test -2025-06-03 20:44:36,201 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario -2025-06-03 20:44:36,201 - tests.orchestrator - INFO - Running specific scenario: _scenario_concurrent_pvc -2025-06-03 20:44:36,201 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations -2025-06-03 20:44:36,202 - tests.orchestrator - INFO - Creating 4 PVCs in quick succession -2025-06-03 20:44:36,496 - tests.orchestrator - ERROR - Failed to create PVC concurrent-pvc-de3117b2 concurrently: (404) -Reason: Not Found -HTTP response headers: HTTPHeaderDict({'Audit-Id': '6de3c195-b2f2-4b32-8ecf-97264e52a31c', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'X-Kubernetes-Pf-Flowschema-Uid': 'a278f63c-a854-47c8-82d3-d9efaaac2f24', 'X-Kubernetes-Pf-Prioritylevel-Uid': '6f082086-057e-4745-9def-b60972ad437b', 'Date': 'Tue, 03 Jun 2025 20:44:36 GMT', 'Content-Length': '210'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"namespaces \"efs-stress-test\" not found","reason":"NotFound","details":{"name":"efs-stress-test","kind":"namespaces"},"code":404} - - -2025-06-03 20:44:36,629 - tests.orchestrator - ERROR - Failed to create PVC concurrent-pvc-85b7f827 concurrently: (404) -Reason: Not Found -HTTP response headers: HTTPHeaderDict({'Audit-Id': 'c0386aa9-45b5-4f3c-a98b-030be2caf4ee', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'X-Kubernetes-Pf-Flowschema-Uid': 'a278f63c-a854-47c8-82d3-d9efaaac2f24', 'X-Kubernetes-Pf-Prioritylevel-Uid': '6f082086-057e-4745-9def-b60972ad437b', 'Date': 'Tue, 03 Jun 2025 20:44:36 GMT', 'Content-Length': '210'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"namespaces \"efs-stress-test\" not found","reason":"NotFound","details":{"name":"efs-stress-test","kind":"namespaces"},"code":404} - - -2025-06-03 20:44:36,761 - tests.orchestrator - ERROR - Failed to create PVC concurrent-pvc-0fd0fcb2 concurrently: (404) -Reason: Not Found -HTTP response headers: HTTPHeaderDict({'Audit-Id': '8b45ed55-a0d9-42cf-8f9e-49802dd09685', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'X-Kubernetes-Pf-Flowschema-Uid': 'a278f63c-a854-47c8-82d3-d9efaaac2f24', 'X-Kubernetes-Pf-Prioritylevel-Uid': '6f082086-057e-4745-9def-b60972ad437b', 'Date': 'Tue, 03 Jun 2025 20:44:36 GMT', 'Content-Length': '210'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"namespaces \"efs-stress-test\" not found","reason":"NotFound","details":{"name":"efs-stress-test","kind":"namespaces"},"code":404} - - -2025-06-03 20:44:36,894 - tests.orchestrator - ERROR - Failed to create PVC concurrent-pvc-70957dd1 concurrently: (404) -Reason: Not Found -HTTP response headers: HTTPHeaderDict({'Audit-Id': 'eb053268-dd23-487f-a943-fe937d3e637c', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'X-Kubernetes-Pf-Flowschema-Uid': 'a278f63c-a854-47c8-82d3-d9efaaac2f24', 'X-Kubernetes-Pf-Prioritylevel-Uid': '6f082086-057e-4745-9def-b60972ad437b', 'Date': 'Tue, 03 Jun 2025 20:44:36 GMT', 'Content-Length': '210'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"namespaces \"efs-stress-test\" not found","reason":"NotFound","details":{"name":"efs-stress-test","kind":"namespaces"},"code":404} - - -2025-06-03 20:44:36,894 - tests.orchestrator - WARNING - Failed to create enough PVCs, only created 0 -2025-06-03 20:44:37,895 - tests.orchestrator - INFO - Selected operation: _attach_pod -2025-06-03 20:44:37,895 - tests.orchestrator - INFO - No PVCs available, skipping pod attachment -2025-06-03 20:44:38,895 - tests.orchestrator - INFO - Selected operation: _delete_pvc -2025-06-03 20:44:38,895 - tests.orchestrator - INFO - No PVCs to delete -2025-06-03 20:44:39,895 - tests.orchestrator - INFO - Selected operation: _create_pvc -2025-06-03 20:44:39,896 - tests.orchestrator - INFO - Creating PVC: test-pvc-a59e84da -2025-06-03 20:44:40,030 - tests.orchestrator - ERROR - Failed to create PVC: (404) -Reason: Not Found -HTTP response headers: HTTPHeaderDict({'Audit-Id': '024a4209-6aa7-489a-8bef-e93376c42c84', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'X-Kubernetes-Pf-Flowschema-Uid': 'a278f63c-a854-47c8-82d3-d9efaaac2f24', 'X-Kubernetes-Pf-Prioritylevel-Uid': '6f082086-057e-4745-9def-b60972ad437b', 'Date': 'Tue, 03 Jun 2025 20:44:39 GMT', 'Content-Length': '210'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"namespaces \"efs-stress-test\" not found","reason":"NotFound","details":{"name":"efs-stress-test","kind":"namespaces"},"code":404} - - -2025-06-03 20:44:41,030 - tests.orchestrator - INFO - Selected operation: _delete_pvc -2025-06-03 20:44:41,030 - tests.orchestrator - INFO - No PVCs to delete -2025-06-03 20:44:42,030 - tests.orchestrator - INFO - Selected operation: _create_pvc -2025-06-03 20:44:42,031 - tests.orchestrator - INFO - Creating PVC: test-pvc-f2989ffe -2025-06-03 20:44:42,163 - tests.orchestrator - ERROR - Failed to create PVC: (404) -Reason: Not Found -HTTP response headers: HTTPHeaderDict({'Audit-Id': 'baeae3c9-acf2-4389-8ecd-6a83618aecd7', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'X-Kubernetes-Pf-Flowschema-Uid': 'a278f63c-a854-47c8-82d3-d9efaaac2f24', 'X-Kubernetes-Pf-Prioritylevel-Uid': '6f082086-057e-4745-9def-b60972ad437b', 'Date': 'Tue, 03 Jun 2025 20:44:42 GMT', 'Content-Length': '210'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"namespaces \"efs-stress-test\" not found","reason":"NotFound","details":{"name":"efs-stress-test","kind":"namespaces"},"code":404} - - -2025-06-03 20:44:43,164 - tests.orchestrator - INFO - Selected operation: _delete_pvc -2025-06-03 20:44:43,164 - tests.orchestrator - INFO - No PVCs to delete -2025-06-03 20:44:44,164 - tests.orchestrator - INFO - Selected operation: _create_pvc -2025-06-03 20:44:44,164 - tests.orchestrator - INFO - Creating PVC: test-pvc-53fee8c6 -2025-06-03 20:44:44,298 - tests.orchestrator - ERROR - Failed to create PVC: (404) -Reason: Not Found -HTTP response headers: HTTPHeaderDict({'Audit-Id': 'df53a974-5bea-4379-89b2-408d521a999c', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'X-Kubernetes-Pf-Flowschema-Uid': 'a278f63c-a854-47c8-82d3-d9efaaac2f24', 'X-Kubernetes-Pf-Prioritylevel-Uid': '6f082086-057e-4745-9def-b60972ad437b', 'Date': 'Tue, 03 Jun 2025 20:44:44 GMT', 'Content-Length': '210'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"namespaces \"efs-stress-test\" not found","reason":"NotFound","details":{"name":"efs-stress-test","kind":"namespaces"},"code":404} - - -2025-06-03 20:44:45,298 - tests.orchestrator - INFO - Selected operation: _delete_pod -2025-06-03 20:44:45,299 - tests.orchestrator - INFO - No pods to delete -2025-06-03 20:44:46,299 - tests.orchestrator - INFO - Selected operation: _delete_pod -2025-06-03 20:44:46,299 - tests.orchestrator - INFO - No pods to delete -2025-06-03 20:44:47,299 - tests.orchestrator - INFO - Selected operation: _delete_pod -2025-06-03 20:44:47,299 - tests.orchestrator - INFO - No pods to delete -2025-06-03 20:44:48,299 - tests.orchestrator - INFO - Selected operation: _delete_pod -2025-06-03 20:44:48,300 - tests.orchestrator - INFO - No pods to delete -2025-06-03 20:44:48,747 - tests.orchestrator - INFO - Test interrupted by user -2025-06-03 20:44:48,747 - tests.orchestrator - INFO - Test completed in 14.79 seconds -2025-06-03 20:44:48,747 - tests.orchestrator - INFO - Cleaning up resources... -2025-06-03 20:44:48,747 - tests.orchestrator - INFO - Cleanup completed -2025-06-03 20:44:48,747 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-03 20:44:48,747 - tests.orchestrator - INFO - --- Operations --- -2025-06-03 20:44:48,747 - tests.orchestrator - INFO - create_pvc: 0 succeeded, 7 failed (0.0%) -2025-06-03 20:44:48,747 - tests.orchestrator - INFO - attach_pod: 0 succeeded, 0 failed (0.0%) -2025-06-03 20:44:48,747 - tests.orchestrator - INFO - delete_pod: 0 succeeded, 0 failed (0.0%) -2025-06-03 20:44:48,747 - tests.orchestrator - INFO - delete_pvc: 0 succeeded, 0 failed (0.0%) -2025-06-03 20:44:48,747 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 0 failed (0.0%) -2025-06-03 20:44:48,747 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 0 failed (0.0%) -2025-06-03 20:44:48,748 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-03 20:44:48,748 - tests.orchestrator - INFO - shared_volume_rw: No runs -2025-06-03 20:44:48,748 - tests.orchestrator - INFO - many_to_one: No runs -2025-06-03 20:44:48,748 - tests.orchestrator - INFO - one_to_one: No runs -2025-06-03 20:44:48,748 - tests.orchestrator - INFO - concurrent_pvc: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-03 20:44:48,748 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250603_204821.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250603_204821.log deleted file mode 100644 index e95a6a2ed..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250603_204821.log +++ /dev/null @@ -1,175 +0,0 @@ -2025-06-03 20:48:21,427 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-03 20:48:21,428 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-03 20:48:21,428 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-03 20:48:21,428 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-03 20:48:21,428 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-03 20:48:21,663 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-03 20:48:21,663 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-03 20:48:21,663 - tests.orchestrator - INFO - Selected operation: _attach_pod -2025-06-03 20:48:21,663 - tests.orchestrator - INFO - Selected operation: _attach_pod -2025-06-03 20:48:21,663 - tests.orchestrator - INFO - No PVCs available, skipping pod attachment -2025-06-03 20:48:21,663 - tests.orchestrator - INFO - No PVCs available, skipping pod attachment -2025-06-03 20:48:22,663 - tests.orchestrator - INFO - Selected operation: _create_pvc -2025-06-03 20:48:22,663 - tests.orchestrator - INFO - Selected operation: _create_pvc -2025-06-03 20:48:22,664 - tests.orchestrator - INFO - Creating PVC: test-pvc-b929a758 -2025-06-03 20:48:22,664 - tests.orchestrator - INFO - Creating PVC: test-pvc-b929a758 -2025-06-03 20:48:22,756 - tests.orchestrator - INFO - Created PVC: test-pvc-b929a758 -2025-06-03 20:48:22,756 - tests.orchestrator - INFO - Created PVC: test-pvc-b929a758 -2025-06-03 20:48:22,756 - tests.orchestrator - INFO - Waiting for PVC test-pvc-b929a758 to be bound -2025-06-03 20:48:22,756 - tests.orchestrator - INFO - Waiting for PVC test-pvc-b929a758 to be bound -2025-06-03 20:48:24,919 - tests.orchestrator - INFO - PVC test-pvc-b929a758 is bound -2025-06-03 20:48:24,919 - tests.orchestrator - INFO - PVC test-pvc-b929a758 is bound -2025-06-03 20:48:25,919 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario -2025-06-03 20:48:25,919 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario -2025-06-03 20:48:25,919 - tests.orchestrator - INFO - Running specific scenario: _scenario_many_to_one -2025-06-03 20:48:25,919 - tests.orchestrator - INFO - Running specific scenario: _scenario_many_to_one -2025-06-03 20:48:25,920 - tests.orchestrator - INFO - Running scenario: Many pods to one PVC -2025-06-03 20:48:25,920 - tests.orchestrator - INFO - Running scenario: Many pods to one PVC -2025-06-03 20:48:26,002 - tests.orchestrator - INFO - Waiting for PVC many2one-5e21f77a to be bound -2025-06-03 20:48:26,002 - tests.orchestrator - INFO - Waiting for PVC many2one-5e21f77a to be bound -2025-06-03 20:48:28,166 - tests.orchestrator - INFO - PVC many2one-5e21f77a is bound -2025-06-03 20:48:28,166 - tests.orchestrator - INFO - PVC many2one-5e21f77a is bound -2025-06-03 20:48:28,167 - tests.orchestrator - INFO - Creating 5 pods for the same PVC -2025-06-03 20:48:28,167 - tests.orchestrator - INFO - Creating 5 pods for the same PVC -2025-06-03 20:48:28,167 - tests.orchestrator - INFO - Creating pod test-pod-63513a27 on PVC many2one-5e21f77a -2025-06-03 20:48:28,167 - tests.orchestrator - INFO - Creating pod test-pod-63513a27 on PVC many2one-5e21f77a -2025-06-03 20:48:28,261 - tests.orchestrator - INFO - Created pod: test-pod-63513a27 using PVC: many2one-5e21f77a -2025-06-03 20:48:28,261 - tests.orchestrator - INFO - Created pod: test-pod-63513a27 using PVC: many2one-5e21f77a -2025-06-03 20:48:28,262 - tests.orchestrator - INFO - Waiting for pod test-pod-63513a27 to be ready -2025-06-03 20:48:28,262 - tests.orchestrator - INFO - Waiting for pod test-pod-63513a27 to be ready -2025-06-03 20:49:28,686 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-63513a27 to be ready after 60s -2025-06-03 20:49:28,686 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-63513a27 to be ready after 60s -2025-06-03 20:49:28,687 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-63513a27 to be ready -2025-06-03 20:49:28,687 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-63513a27 to be ready -2025-06-03 20:49:28,687 - tests.orchestrator - INFO - Creating pod test-pod-2c5c5642 on PVC many2one-5e21f77a -2025-06-03 20:49:28,687 - tests.orchestrator - INFO - Creating pod test-pod-2c5c5642 on PVC many2one-5e21f77a -2025-06-03 20:49:28,783 - tests.orchestrator - INFO - Created pod: test-pod-2c5c5642 using PVC: many2one-5e21f77a -2025-06-03 20:49:28,783 - tests.orchestrator - INFO - Created pod: test-pod-2c5c5642 using PVC: many2one-5e21f77a -2025-06-03 20:49:28,783 - tests.orchestrator - INFO - Waiting for pod test-pod-2c5c5642 to be ready -2025-06-03 20:49:28,783 - tests.orchestrator - INFO - Waiting for pod test-pod-2c5c5642 to be ready -2025-06-03 20:50:29,204 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-2c5c5642 to be ready after 60s -2025-06-03 20:50:29,204 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-2c5c5642 to be ready after 60s -2025-06-03 20:50:29,204 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-2c5c5642 to be ready -2025-06-03 20:50:29,204 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-2c5c5642 to be ready -2025-06-03 20:50:29,204 - tests.orchestrator - INFO - Creating pod test-pod-9d6a21f8 on PVC many2one-5e21f77a -2025-06-03 20:50:29,204 - tests.orchestrator - INFO - Creating pod test-pod-9d6a21f8 on PVC many2one-5e21f77a -2025-06-03 20:50:29,299 - tests.orchestrator - INFO - Created pod: test-pod-9d6a21f8 using PVC: many2one-5e21f77a -2025-06-03 20:50:29,299 - tests.orchestrator - INFO - Created pod: test-pod-9d6a21f8 using PVC: many2one-5e21f77a -2025-06-03 20:50:29,299 - tests.orchestrator - INFO - Waiting for pod test-pod-9d6a21f8 to be ready -2025-06-03 20:50:29,299 - tests.orchestrator - INFO - Waiting for pod test-pod-9d6a21f8 to be ready -2025-06-03 20:51:29,793 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9d6a21f8 to be ready after 60s -2025-06-03 20:51:29,793 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9d6a21f8 to be ready after 60s -2025-06-03 20:51:29,794 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9d6a21f8 to be ready -2025-06-03 20:51:29,794 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9d6a21f8 to be ready -2025-06-03 20:51:29,794 - tests.orchestrator - INFO - Creating pod test-pod-1bd90f06 on PVC many2one-5e21f77a -2025-06-03 20:51:29,794 - tests.orchestrator - INFO - Creating pod test-pod-1bd90f06 on PVC many2one-5e21f77a -2025-06-03 20:51:29,892 - tests.orchestrator - INFO - Created pod: test-pod-1bd90f06 using PVC: many2one-5e21f77a -2025-06-03 20:51:29,892 - tests.orchestrator - INFO - Created pod: test-pod-1bd90f06 using PVC: many2one-5e21f77a -2025-06-03 20:51:29,893 - tests.orchestrator - INFO - Waiting for pod test-pod-1bd90f06 to be ready -2025-06-03 20:51:29,893 - tests.orchestrator - INFO - Waiting for pod test-pod-1bd90f06 to be ready -2025-06-03 20:52:30,297 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1bd90f06 to be ready after 60s -2025-06-03 20:52:30,297 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1bd90f06 to be ready after 60s -2025-06-03 20:52:30,297 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1bd90f06 to be ready -2025-06-03 20:52:30,297 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1bd90f06 to be ready -2025-06-03 20:52:30,297 - tests.orchestrator - INFO - Creating pod test-pod-cb8a2172 on PVC many2one-5e21f77a -2025-06-03 20:52:30,297 - tests.orchestrator - INFO - Creating pod test-pod-cb8a2172 on PVC many2one-5e21f77a -2025-06-03 20:52:30,394 - tests.orchestrator - INFO - Created pod: test-pod-cb8a2172 using PVC: many2one-5e21f77a -2025-06-03 20:52:30,394 - tests.orchestrator - INFO - Created pod: test-pod-cb8a2172 using PVC: many2one-5e21f77a -2025-06-03 20:52:30,395 - tests.orchestrator - INFO - Waiting for pod test-pod-cb8a2172 to be ready -2025-06-03 20:52:30,395 - tests.orchestrator - INFO - Waiting for pod test-pod-cb8a2172 to be ready -2025-06-03 20:53:30,865 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-cb8a2172 to be ready after 60s -2025-06-03 20:53:30,865 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-cb8a2172 to be ready after 60s -2025-06-03 20:53:30,866 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-cb8a2172 to be ready -2025-06-03 20:53:30,866 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-cb8a2172 to be ready -2025-06-03 20:53:30,866 - tests.orchestrator - WARNING - Failed to create enough pods for scenario, only created 0 -2025-06-03 20:53:30,866 - tests.orchestrator - WARNING - Failed to create enough pods for scenario, only created 0 -2025-06-03 20:53:31,866 - tests.orchestrator - INFO - Test completed in 310.44 seconds -2025-06-03 20:53:31,866 - tests.orchestrator - INFO - Test completed in 310.44 seconds -2025-06-03 20:53:31,866 - tests.orchestrator - INFO - Cleaning up resources... -2025-06-03 20:53:31,866 - tests.orchestrator - INFO - Cleaning up resources... -2025-06-03 20:53:31,866 - tests.orchestrator - INFO - Deleting pod: test-pod-63513a27 from PVC: many2one-5e21f77a -2025-06-03 20:53:31,866 - tests.orchestrator - INFO - Deleting pod: test-pod-63513a27 from PVC: many2one-5e21f77a -2025-06-03 20:53:31,978 - tests.orchestrator - INFO - Waiting for pod test-pod-63513a27 to be deleted -2025-06-03 20:53:31,978 - tests.orchestrator - INFO - Waiting for pod test-pod-63513a27 to be deleted -2025-06-03 20:53:32,062 - tests.orchestrator - INFO - Pod test-pod-63513a27 has been deleted -2025-06-03 20:53:32,062 - tests.orchestrator - INFO - Pod test-pod-63513a27 has been deleted -2025-06-03 20:53:32,062 - tests.orchestrator - INFO - Deleted pod: test-pod-63513a27 -2025-06-03 20:53:32,062 - tests.orchestrator - INFO - Deleted pod: test-pod-63513a27 -2025-06-03 20:53:32,062 - tests.orchestrator - INFO - Deleting pod: test-pod-2c5c5642 from PVC: many2one-5e21f77a -2025-06-03 20:53:32,062 - tests.orchestrator - INFO - Deleting pod: test-pod-2c5c5642 from PVC: many2one-5e21f77a -2025-06-03 20:53:32,163 - tests.orchestrator - INFO - Waiting for pod test-pod-2c5c5642 to be deleted -2025-06-03 20:53:32,163 - tests.orchestrator - INFO - Waiting for pod test-pod-2c5c5642 to be deleted -2025-06-03 20:53:32,248 - tests.orchestrator - INFO - Pod test-pod-2c5c5642 has been deleted -2025-06-03 20:53:32,248 - tests.orchestrator - INFO - Pod test-pod-2c5c5642 has been deleted -2025-06-03 20:53:32,249 - tests.orchestrator - INFO - Deleted pod: test-pod-2c5c5642 -2025-06-03 20:53:32,249 - tests.orchestrator - INFO - Deleted pod: test-pod-2c5c5642 -2025-06-03 20:53:32,249 - tests.orchestrator - INFO - Deleting pod: test-pod-9d6a21f8 from PVC: many2one-5e21f77a -2025-06-03 20:53:32,249 - tests.orchestrator - INFO - Deleting pod: test-pod-9d6a21f8 from PVC: many2one-5e21f77a -2025-06-03 20:53:32,357 - tests.orchestrator - INFO - Waiting for pod test-pod-9d6a21f8 to be deleted -2025-06-03 20:53:32,357 - tests.orchestrator - INFO - Waiting for pod test-pod-9d6a21f8 to be deleted -2025-06-03 20:53:32,446 - tests.orchestrator - INFO - Pod test-pod-9d6a21f8 has been deleted -2025-06-03 20:53:32,446 - tests.orchestrator - INFO - Pod test-pod-9d6a21f8 has been deleted -2025-06-03 20:53:32,446 - tests.orchestrator - INFO - Deleted pod: test-pod-9d6a21f8 -2025-06-03 20:53:32,446 - tests.orchestrator - INFO - Deleted pod: test-pod-9d6a21f8 -2025-06-03 20:53:32,446 - tests.orchestrator - INFO - Deleting pod: test-pod-1bd90f06 from PVC: many2one-5e21f77a -2025-06-03 20:53:32,446 - tests.orchestrator - INFO - Deleting pod: test-pod-1bd90f06 from PVC: many2one-5e21f77a -2025-06-03 20:53:32,565 - tests.orchestrator - INFO - Waiting for pod test-pod-1bd90f06 to be deleted -2025-06-03 20:53:32,565 - tests.orchestrator - INFO - Waiting for pod test-pod-1bd90f06 to be deleted -2025-06-03 20:53:32,647 - tests.orchestrator - INFO - Pod test-pod-1bd90f06 has been deleted -2025-06-03 20:53:32,647 - tests.orchestrator - INFO - Pod test-pod-1bd90f06 has been deleted -2025-06-03 20:53:32,647 - tests.orchestrator - INFO - Deleted pod: test-pod-1bd90f06 -2025-06-03 20:53:32,647 - tests.orchestrator - INFO - Deleted pod: test-pod-1bd90f06 -2025-06-03 20:53:32,647 - tests.orchestrator - INFO - Deleting pod: test-pod-cb8a2172 from PVC: many2one-5e21f77a -2025-06-03 20:53:32,647 - tests.orchestrator - INFO - Deleting pod: test-pod-cb8a2172 from PVC: many2one-5e21f77a -2025-06-03 20:53:32,744 - tests.orchestrator - INFO - Waiting for pod test-pod-cb8a2172 to be deleted -2025-06-03 20:53:32,744 - tests.orchestrator - INFO - Waiting for pod test-pod-cb8a2172 to be deleted -2025-06-03 20:53:32,824 - tests.orchestrator - INFO - Pod test-pod-cb8a2172 has been deleted -2025-06-03 20:53:32,824 - tests.orchestrator - INFO - Pod test-pod-cb8a2172 has been deleted -2025-06-03 20:53:32,824 - tests.orchestrator - INFO - Deleted pod: test-pod-cb8a2172 -2025-06-03 20:53:32,824 - tests.orchestrator - INFO - Deleted pod: test-pod-cb8a2172 -2025-06-03 20:53:32,824 - tests.orchestrator - INFO - Deleting PVC: test-pvc-b929a758 -2025-06-03 20:53:32,824 - tests.orchestrator - INFO - Deleting PVC: test-pvc-b929a758 -2025-06-03 20:53:32,911 - tests.orchestrator - INFO - Waiting for PVC test-pvc-b929a758 to be deleted -2025-06-03 20:53:32,911 - tests.orchestrator - INFO - Waiting for PVC test-pvc-b929a758 to be deleted -2025-06-03 20:53:32,991 - tests.orchestrator - INFO - PVC test-pvc-b929a758 has been deleted -2025-06-03 20:53:32,991 - tests.orchestrator - INFO - PVC test-pvc-b929a758 has been deleted -2025-06-03 20:53:32,992 - tests.orchestrator - INFO - Deleted PVC: test-pvc-b929a758 -2025-06-03 20:53:32,992 - tests.orchestrator - INFO - Deleted PVC: test-pvc-b929a758 -2025-06-03 20:53:32,992 - tests.orchestrator - INFO - Deleting PVC: many2one-5e21f77a -2025-06-03 20:53:32,992 - tests.orchestrator - INFO - Deleting PVC: many2one-5e21f77a -2025-06-03 20:53:33,079 - tests.orchestrator - INFO - Waiting for PVC many2one-5e21f77a to be deleted -2025-06-03 20:53:33,079 - tests.orchestrator - INFO - Waiting for PVC many2one-5e21f77a to be deleted -2025-06-03 20:53:33,159 - tests.orchestrator - INFO - PVC many2one-5e21f77a has been deleted -2025-06-03 20:53:33,159 - tests.orchestrator - INFO - PVC many2one-5e21f77a has been deleted -2025-06-03 20:53:33,159 - tests.orchestrator - INFO - Deleted PVC: many2one-5e21f77a -2025-06-03 20:53:33,159 - tests.orchestrator - INFO - Deleted PVC: many2one-5e21f77a -2025-06-03 20:53:33,159 - tests.orchestrator - INFO - Cleanup completed -2025-06-03 20:53:33,159 - tests.orchestrator - INFO - Cleanup completed -2025-06-03 20:53:33,160 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-03 20:53:33,160 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-03 20:53:33,160 - tests.orchestrator - INFO - --- Operations --- -2025-06-03 20:53:33,160 - tests.orchestrator - INFO - --- Operations --- -2025-06-03 20:53:33,160 - tests.orchestrator - INFO - create_pvc: 1 succeeded, 0 failed (100.0%) -2025-06-03 20:53:33,160 - tests.orchestrator - INFO - create_pvc: 1 succeeded, 0 failed (100.0%) -2025-06-03 20:53:33,160 - tests.orchestrator - INFO - attach_pod: 5 succeeded, 0 failed (100.0%) -2025-06-03 20:53:33,160 - tests.orchestrator - INFO - attach_pod: 5 succeeded, 0 failed (100.0%) -2025-06-03 20:53:33,160 - tests.orchestrator - INFO - delete_pod: 5 succeeded, 0 failed (100.0%) -2025-06-03 20:53:33,160 - tests.orchestrator - INFO - delete_pod: 5 succeeded, 0 failed (100.0%) -2025-06-03 20:53:33,160 - tests.orchestrator - INFO - delete_pvc: 2 succeeded, 0 failed (100.0%) -2025-06-03 20:53:33,160 - tests.orchestrator - INFO - delete_pvc: 2 succeeded, 0 failed (100.0%) -2025-06-03 20:53:33,160 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 0 failed (0.0%) -2025-06-03 20:53:33,160 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 0 failed (0.0%) -2025-06-03 20:53:33,160 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 0 failed (0.0%) -2025-06-03 20:53:33,160 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 0 failed (0.0%) -2025-06-03 20:53:33,160 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-03 20:53:33,160 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-03 20:53:33,160 - tests.orchestrator - INFO - shared_volume_rw: No runs -2025-06-03 20:53:33,160 - tests.orchestrator - INFO - shared_volume_rw: No runs -2025-06-03 20:53:33,160 - tests.orchestrator - INFO - many_to_one: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-03 20:53:33,160 - tests.orchestrator - INFO - many_to_one: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-03 20:53:33,160 - tests.orchestrator - INFO - one_to_one: No runs -2025-06-03 20:53:33,160 - tests.orchestrator - INFO - one_to_one: No runs -2025-06-03 20:53:33,160 - tests.orchestrator - INFO - concurrent_pvc: No runs -2025-06-03 20:53:33,160 - tests.orchestrator - INFO - concurrent_pvc: No runs -2025-06-03 20:53:33,160 - tests.orchestrator - INFO - ========================================= -2025-06-03 20:53:33,160 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250603_210011.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250603_210011.log deleted file mode 100644 index 0bfa029ed..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250603_210011.log +++ /dev/null @@ -1,158 +0,0 @@ -2025-06-03 21:00:11,173 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-03 21:00:11,448 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-03 21:00:11,448 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-03 21:00:11,865 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-03 21:00:11,865 - tests.orchestrator - INFO - Selected operation: _attach_pod -2025-06-03 21:00:11,865 - tests.orchestrator - INFO - No PVCs available, skipping pod attachment -2025-06-03 21:00:12,865 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario -2025-06-03 21:00:12,865 - tests.orchestrator - INFO - Running specific scenario: _scenario_many_to_one -2025-06-03 21:00:12,866 - tests.orchestrator - INFO - Running scenario: Many pods to one PVC -2025-06-03 21:00:12,969 - tests.orchestrator - INFO - Waiting for PVC many2one-8ccf2b6f to be bound -2025-06-03 21:00:15,141 - tests.orchestrator - INFO - PVC many2one-8ccf2b6f is bound -2025-06-03 21:00:15,141 - tests.orchestrator - INFO - Creating 4 pods for the same PVC -2025-06-03 21:00:15,141 - tests.orchestrator - INFO - Creating pod test-pod-5c9bbadf on PVC many2one-8ccf2b6f -2025-06-03 21:00:15,239 - tests.orchestrator - INFO - Created pod: test-pod-5c9bbadf using PVC: many2one-8ccf2b6f -2025-06-03 21:00:15,239 - tests.orchestrator - INFO - Waiting for pod test-pod-5c9bbadf to be ready -2025-06-03 21:01:15,739 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-5c9bbadf to be ready after 60s -2025-06-03 21:01:15,740 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-5c9bbadf to be ready -2025-06-03 21:01:15,740 - tests.orchestrator - INFO - Creating pod test-pod-a795ee8f on PVC many2one-8ccf2b6f -2025-06-03 21:01:15,842 - tests.orchestrator - INFO - Created pod: test-pod-a795ee8f using PVC: many2one-8ccf2b6f -2025-06-03 21:01:15,842 - tests.orchestrator - INFO - Waiting for pod test-pod-a795ee8f to be ready -2025-06-03 21:02:16,279 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a795ee8f to be ready after 60s -2025-06-03 21:02:16,279 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a795ee8f to be ready -2025-06-03 21:02:16,279 - tests.orchestrator - INFO - Creating pod test-pod-bb687a78 on PVC many2one-8ccf2b6f -2025-06-03 21:02:16,379 - tests.orchestrator - INFO - Created pod: test-pod-bb687a78 using PVC: many2one-8ccf2b6f -2025-06-03 21:02:16,379 - tests.orchestrator - INFO - Waiting for pod test-pod-bb687a78 to be ready -2025-06-03 21:03:16,816 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-bb687a78 to be ready after 60s -2025-06-03 21:03:16,816 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-bb687a78 to be ready -2025-06-03 21:03:16,817 - tests.orchestrator - INFO - Creating pod test-pod-da23ad5a on PVC many2one-8ccf2b6f -2025-06-03 21:03:16,917 - tests.orchestrator - INFO - Created pod: test-pod-da23ad5a using PVC: many2one-8ccf2b6f -2025-06-03 21:03:16,917 - tests.orchestrator - INFO - Waiting for pod test-pod-da23ad5a to be ready -2025-06-03 21:04:17,450 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-da23ad5a to be ready after 60s -2025-06-03 21:04:17,450 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-da23ad5a to be ready -2025-06-03 21:04:17,450 - tests.orchestrator - WARNING - Failed to create enough pods for scenario, only created 0 -2025-06-03 21:04:18,450 - tests.orchestrator - INFO - Selected operation: _verify_readwrite -2025-06-03 21:04:18,451 - tests.orchestrator - INFO - Testing read/write between pods test-pod-a795ee8f and test-pod-5c9bbadf sharing PVC many2one-8ccf2b6f -2025-06-03 21:04:18,534 - tests.orchestrator - ERROR - Failed in read/write verification: (400) -Reason: Bad Request -HTTP response headers: HTTPHeaderDict({'Audit-Id': 'e5f54d7e-0b5d-44ba-9610-9f446b015a24', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Tue, 03 Jun 2025 21:04:18 GMT', 'Content-Length': '166'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-a795ee8f does not have a host assigned","reason":"BadRequest","code":400} - - -2025-06-03 21:04:19,534 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario -2025-06-03 21:04:19,534 - tests.orchestrator - INFO - Running specific scenario: _scenario_one_to_one -2025-06-03 21:04:19,534 - tests.orchestrator - INFO - Running scenario: One pod per PVC -2025-06-03 21:04:19,534 - tests.orchestrator - INFO - Creating 5 PVC-pod pairs -2025-06-03 21:04:19,631 - tests.orchestrator - INFO - Waiting for PVC one2one-c1de2935 to be bound -2025-06-03 21:04:21,797 - tests.orchestrator - INFO - PVC one2one-c1de2935 is bound -2025-06-03 21:04:21,797 - tests.orchestrator - INFO - Creating pod test-pod-c8cbcc77 on PVC one2one-c1de2935 -2025-06-03 21:04:21,894 - tests.orchestrator - INFO - Created pod: test-pod-c8cbcc77 using PVC: one2one-c1de2935 -2025-06-03 21:04:21,894 - tests.orchestrator - INFO - Waiting for pod test-pod-c8cbcc77 to be ready -2025-06-03 21:05:22,524 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-c8cbcc77 to be ready after 60s -2025-06-03 21:05:22,524 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-c8cbcc77 to be ready -2025-06-03 21:05:22,615 - tests.orchestrator - INFO - Waiting for PVC one2one-2f5447c5 to be bound -2025-06-03 21:05:24,783 - tests.orchestrator - INFO - PVC one2one-2f5447c5 is bound -2025-06-03 21:05:24,783 - tests.orchestrator - INFO - Creating pod test-pod-9c65d0a8 on PVC one2one-2f5447c5 -2025-06-03 21:05:24,879 - tests.orchestrator - INFO - Created pod: test-pod-9c65d0a8 using PVC: one2one-2f5447c5 -2025-06-03 21:05:24,879 - tests.orchestrator - INFO - Waiting for pod test-pod-9c65d0a8 to be ready -2025-06-03 21:06:25,353 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9c65d0a8 to be ready after 60s -2025-06-03 21:06:25,353 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9c65d0a8 to be ready -2025-06-03 21:06:25,454 - tests.orchestrator - INFO - Waiting for PVC one2one-65a70cda to be bound -2025-06-03 21:06:27,620 - tests.orchestrator - INFO - PVC one2one-65a70cda is bound -2025-06-03 21:06:27,621 - tests.orchestrator - INFO - Creating pod test-pod-396a98c7 on PVC one2one-65a70cda -2025-06-03 21:06:27,709 - tests.orchestrator - INFO - Created pod: test-pod-396a98c7 using PVC: one2one-65a70cda -2025-06-03 21:06:27,709 - tests.orchestrator - INFO - Waiting for pod test-pod-396a98c7 to be ready -2025-06-03 21:07:28,272 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-396a98c7 to be ready after 60s -2025-06-03 21:07:28,272 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-396a98c7 to be ready -2025-06-03 21:07:28,365 - tests.orchestrator - INFO - Waiting for PVC one2one-ac0c0034 to be bound -2025-06-03 21:07:30,531 - tests.orchestrator - INFO - PVC one2one-ac0c0034 is bound -2025-06-03 21:07:30,531 - tests.orchestrator - INFO - Creating pod test-pod-707ffe1e on PVC one2one-ac0c0034 -2025-06-03 21:07:30,620 - tests.orchestrator - INFO - Created pod: test-pod-707ffe1e using PVC: one2one-ac0c0034 -2025-06-03 21:07:30,620 - tests.orchestrator - INFO - Waiting for pod test-pod-707ffe1e to be ready -2025-06-03 21:08:31,077 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-707ffe1e to be ready after 60s -2025-06-03 21:08:31,077 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-707ffe1e to be ready -2025-06-03 21:08:31,174 - tests.orchestrator - INFO - Waiting for PVC one2one-6960322d to be bound -2025-06-03 21:08:33,340 - tests.orchestrator - INFO - PVC one2one-6960322d is bound -2025-06-03 21:08:33,340 - tests.orchestrator - INFO - Creating pod test-pod-db166953 on PVC one2one-6960322d -2025-06-03 21:08:33,433 - tests.orchestrator - INFO - Created pod: test-pod-db166953 using PVC: one2one-6960322d -2025-06-03 21:08:33,433 - tests.orchestrator - INFO - Waiting for pod test-pod-db166953 to be ready -2025-06-03 21:09:34,745 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-db166953 to be ready after 60s -2025-06-03 21:09:34,745 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-db166953 to be ready -2025-06-03 21:09:34,746 - tests.orchestrator - WARNING - Failed to create enough PVC-pod pairs, only created 0 -2025-06-03 21:09:35,746 - tests.orchestrator - INFO - Test completed in 564.30 seconds -2025-06-03 21:09:35,746 - tests.orchestrator - INFO - Cleaning up resources... -2025-06-03 21:09:35,746 - tests.orchestrator - INFO - Deleting pod: test-pod-5c9bbadf from PVC: many2one-8ccf2b6f -2025-06-03 21:09:35,842 - tests.orchestrator - INFO - Waiting for pod test-pod-5c9bbadf to be deleted -2025-06-03 21:09:35,926 - tests.orchestrator - INFO - Pod test-pod-5c9bbadf has been deleted -2025-06-03 21:09:35,926 - tests.orchestrator - INFO - Deleted pod: test-pod-5c9bbadf -2025-06-03 21:09:35,926 - tests.orchestrator - INFO - Deleting pod: test-pod-a795ee8f from PVC: many2one-8ccf2b6f -2025-06-03 21:09:36,025 - tests.orchestrator - INFO - Waiting for pod test-pod-a795ee8f to be deleted -2025-06-03 21:09:36,106 - tests.orchestrator - INFO - Pod test-pod-a795ee8f has been deleted -2025-06-03 21:09:36,106 - tests.orchestrator - INFO - Deleted pod: test-pod-a795ee8f -2025-06-03 21:09:36,106 - tests.orchestrator - INFO - Deleting pod: test-pod-bb687a78 from PVC: many2one-8ccf2b6f -2025-06-03 21:09:36,202 - tests.orchestrator - INFO - Waiting for pod test-pod-bb687a78 to be deleted -2025-06-03 21:09:36,284 - tests.orchestrator - INFO - Pod test-pod-bb687a78 has been deleted -2025-06-03 21:09:36,284 - tests.orchestrator - INFO - Deleted pod: test-pod-bb687a78 -2025-06-03 21:09:36,284 - tests.orchestrator - INFO - Deleting pod: test-pod-da23ad5a from PVC: many2one-8ccf2b6f -2025-06-03 21:09:36,386 - tests.orchestrator - INFO - Waiting for pod test-pod-da23ad5a to be deleted -2025-06-03 21:09:36,466 - tests.orchestrator - INFO - Pod test-pod-da23ad5a has been deleted -2025-06-03 21:09:36,466 - tests.orchestrator - INFO - Deleted pod: test-pod-da23ad5a -2025-06-03 21:09:36,466 - tests.orchestrator - INFO - Deleting pod: test-pod-c8cbcc77 from PVC: one2one-c1de2935 -2025-06-03 21:09:36,565 - tests.orchestrator - INFO - Waiting for pod test-pod-c8cbcc77 to be deleted -2025-06-03 21:09:36,651 - tests.orchestrator - INFO - Pod test-pod-c8cbcc77 has been deleted -2025-06-03 21:09:36,652 - tests.orchestrator - INFO - Deleted pod: test-pod-c8cbcc77 -2025-06-03 21:09:36,652 - tests.orchestrator - INFO - Deleting pod: test-pod-9c65d0a8 from PVC: one2one-2f5447c5 -2025-06-03 21:09:36,747 - tests.orchestrator - INFO - Waiting for pod test-pod-9c65d0a8 to be deleted -2025-06-03 21:09:36,832 - tests.orchestrator - INFO - Pod test-pod-9c65d0a8 has been deleted -2025-06-03 21:09:36,833 - tests.orchestrator - INFO - Deleted pod: test-pod-9c65d0a8 -2025-06-03 21:09:36,833 - tests.orchestrator - INFO - Deleting pod: test-pod-396a98c7 from PVC: one2one-65a70cda -2025-06-03 21:09:36,929 - tests.orchestrator - INFO - Waiting for pod test-pod-396a98c7 to be deleted -2025-06-03 21:09:37,009 - tests.orchestrator - INFO - Pod test-pod-396a98c7 has been deleted -2025-06-03 21:09:37,009 - tests.orchestrator - INFO - Deleted pod: test-pod-396a98c7 -2025-06-03 21:09:37,009 - tests.orchestrator - INFO - Deleting pod: test-pod-707ffe1e from PVC: one2one-ac0c0034 -2025-06-03 21:09:37,104 - tests.orchestrator - INFO - Waiting for pod test-pod-707ffe1e to be deleted -2025-06-03 21:09:37,187 - tests.orchestrator - INFO - Pod test-pod-707ffe1e has been deleted -2025-06-03 21:09:37,187 - tests.orchestrator - INFO - Deleted pod: test-pod-707ffe1e -2025-06-03 21:09:37,187 - tests.orchestrator - INFO - Deleting pod: test-pod-db166953 from PVC: one2one-6960322d -2025-06-03 21:09:37,284 - tests.orchestrator - INFO - Waiting for pod test-pod-db166953 to be deleted -2025-06-03 21:09:37,368 - tests.orchestrator - INFO - Pod test-pod-db166953 has been deleted -2025-06-03 21:09:37,368 - tests.orchestrator - INFO - Deleted pod: test-pod-db166953 -2025-06-03 21:09:37,368 - tests.orchestrator - INFO - Deleting PVC: many2one-8ccf2b6f -2025-06-03 21:09:37,456 - tests.orchestrator - INFO - Waiting for PVC many2one-8ccf2b6f to be deleted -2025-06-03 21:09:37,538 - tests.orchestrator - INFO - PVC many2one-8ccf2b6f has been deleted -2025-06-03 21:09:37,539 - tests.orchestrator - INFO - Deleted PVC: many2one-8ccf2b6f -2025-06-03 21:09:37,539 - tests.orchestrator - INFO - Deleting PVC: one2one-c1de2935 -2025-06-03 21:09:37,627 - tests.orchestrator - INFO - Waiting for PVC one2one-c1de2935 to be deleted -2025-06-03 21:09:37,711 - tests.orchestrator - INFO - PVC one2one-c1de2935 has been deleted -2025-06-03 21:09:37,711 - tests.orchestrator - INFO - Deleted PVC: one2one-c1de2935 -2025-06-03 21:09:37,711 - tests.orchestrator - INFO - Deleting PVC: one2one-2f5447c5 -2025-06-03 21:09:37,802 - tests.orchestrator - INFO - Waiting for PVC one2one-2f5447c5 to be deleted -2025-06-03 21:09:37,885 - tests.orchestrator - INFO - PVC one2one-2f5447c5 has been deleted -2025-06-03 21:09:37,886 - tests.orchestrator - INFO - Deleted PVC: one2one-2f5447c5 -2025-06-03 21:09:37,886 - tests.orchestrator - INFO - Deleting PVC: one2one-65a70cda -2025-06-03 21:09:37,973 - tests.orchestrator - INFO - Waiting for PVC one2one-65a70cda to be deleted -2025-06-03 21:09:38,059 - tests.orchestrator - INFO - PVC one2one-65a70cda has been deleted -2025-06-03 21:09:38,059 - tests.orchestrator - INFO - Deleted PVC: one2one-65a70cda -2025-06-03 21:09:38,059 - tests.orchestrator - INFO - Deleting PVC: one2one-ac0c0034 -2025-06-03 21:09:38,148 - tests.orchestrator - INFO - Waiting for PVC one2one-ac0c0034 to be deleted -2025-06-03 21:09:38,230 - tests.orchestrator - INFO - PVC one2one-ac0c0034 has been deleted -2025-06-03 21:09:38,231 - tests.orchestrator - INFO - Deleted PVC: one2one-ac0c0034 -2025-06-03 21:09:38,231 - tests.orchestrator - INFO - Deleting PVC: one2one-6960322d -2025-06-03 21:09:38,318 - tests.orchestrator - INFO - Waiting for PVC one2one-6960322d to be deleted -2025-06-03 21:09:38,401 - tests.orchestrator - INFO - PVC one2one-6960322d has been deleted -2025-06-03 21:09:38,401 - tests.orchestrator - INFO - Deleted PVC: one2one-6960322d -2025-06-03 21:09:38,401 - tests.orchestrator - INFO - Cleanup completed -2025-06-03 21:09:38,401 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-03 21:09:38,401 - tests.orchestrator - INFO - --- Operations --- -2025-06-03 21:09:38,401 - tests.orchestrator - INFO - create_pvc: 0 succeeded, 0 failed (0.0%) -2025-06-03 21:09:38,402 - tests.orchestrator - INFO - attach_pod: 9 succeeded, 0 failed (100.0%) -2025-06-03 21:09:38,402 - tests.orchestrator - INFO - delete_pod: 9 succeeded, 0 failed (100.0%) -2025-06-03 21:09:38,402 - tests.orchestrator - INFO - delete_pvc: 6 succeeded, 0 failed (100.0%) -2025-06-03 21:09:38,402 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) -2025-06-03 21:09:38,402 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) -2025-06-03 21:09:38,402 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-03 21:09:38,402 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-03 21:09:38,402 - tests.orchestrator - INFO - many_to_one: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-03 21:09:38,402 - tests.orchestrator - INFO - one_to_one: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-03 21:09:38,402 - tests.orchestrator - INFO - concurrent_pvc: No runs -2025-06-03 21:09:38,402 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250604_204301.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250604_204301.log deleted file mode 100644 index 75ab26ee1..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250604_204301.log +++ /dev/null @@ -1,125 +0,0 @@ -2025-06-04 20:43:01,074 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-04 20:43:01,299 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-04 20:43:01,299 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-04 20:43:01,528 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-04 20:43:01,529 - tests.orchestrator - INFO - Selected operation: _verify_readwrite -2025-06-04 20:43:01,529 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test -2025-06-04 20:43:02,529 - tests.orchestrator - INFO - Selected operation: _attach_pod -2025-06-04 20:43:02,529 - tests.orchestrator - INFO - No PVCs available, skipping pod attachment -2025-06-04 20:43:03,529 - tests.orchestrator - INFO - Selected operation: _create_pvc -2025-06-04 20:43:03,530 - tests.orchestrator - INFO - Creating PVC: test-pvc-a8738d91 -2025-06-04 20:43:03,612 - tests.orchestrator - INFO - Created PVC: test-pvc-a8738d91 -2025-06-04 20:43:03,613 - tests.orchestrator - INFO - Waiting for PVC test-pvc-a8738d91 to be bound -2025-06-04 20:43:05,748 - tests.orchestrator - INFO - PVC test-pvc-a8738d91 is bound -2025-06-04 20:43:06,749 - tests.orchestrator - INFO - Selected operation: _create_pvc -2025-06-04 20:43:06,749 - tests.orchestrator - INFO - Creating PVC: test-pvc-ba881b7a -2025-06-04 20:43:06,823 - tests.orchestrator - INFO - Created PVC: test-pvc-ba881b7a -2025-06-04 20:43:06,824 - tests.orchestrator - INFO - Waiting for PVC test-pvc-ba881b7a to be bound -2025-06-04 20:43:08,959 - tests.orchestrator - INFO - PVC test-pvc-ba881b7a is bound -2025-06-04 20:43:09,959 - tests.orchestrator - INFO - Selected operation: _delete_pod -2025-06-04 20:43:09,959 - tests.orchestrator - INFO - No pods to delete -2025-06-04 20:43:10,959 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario -2025-06-04 20:43:10,960 - tests.orchestrator - INFO - Running specific scenario: _scenario_one_to_one -2025-06-04 20:43:10,960 - tests.orchestrator - INFO - Running scenario: One pod per PVC -2025-06-04 20:43:10,960 - tests.orchestrator - INFO - Creating 4 PVC-pod pairs -2025-06-04 20:43:11,032 - tests.orchestrator - INFO - Waiting for PVC one2one-43f0888a to be bound -2025-06-04 20:43:13,170 - tests.orchestrator - INFO - PVC one2one-43f0888a is bound -2025-06-04 20:43:13,170 - tests.orchestrator - INFO - Creating pod test-pod-f3b3a319 on PVC one2one-43f0888a -2025-06-04 20:43:13,279 - tests.orchestrator - INFO - Created pod: test-pod-f3b3a319 using PVC: one2one-43f0888a -2025-06-04 20:43:13,279 - tests.orchestrator - INFO - Waiting for pod test-pod-f3b3a319 to be ready -2025-06-04 20:44:15,334 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-f3b3a319 to be ready after 60s -2025-06-04 20:44:15,334 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-f3b3a319 to be ready -2025-06-04 20:44:15,411 - tests.orchestrator - INFO - Waiting for PVC one2one-aac689b5 to be bound -2025-06-04 20:44:17,545 - tests.orchestrator - INFO - PVC one2one-aac689b5 is bound -2025-06-04 20:44:17,545 - tests.orchestrator - INFO - Creating pod test-pod-80037404 on PVC one2one-aac689b5 -2025-06-04 20:44:17,620 - tests.orchestrator - INFO - Created pod: test-pod-80037404 using PVC: one2one-aac689b5 -2025-06-04 20:44:17,620 - tests.orchestrator - INFO - Waiting for pod test-pod-80037404 to be ready -2025-06-04 20:45:17,743 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-80037404 to be ready after 60s -2025-06-04 20:45:17,744 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-80037404 to be ready -2025-06-04 20:45:17,826 - tests.orchestrator - INFO - Waiting for PVC one2one-116b8e13 to be bound -2025-06-04 20:45:19,959 - tests.orchestrator - INFO - PVC one2one-116b8e13 is bound -2025-06-04 20:45:19,959 - tests.orchestrator - INFO - Creating pod test-pod-725477e8 on PVC one2one-116b8e13 -2025-06-04 20:45:20,032 - tests.orchestrator - INFO - Created pod: test-pod-725477e8 using PVC: one2one-116b8e13 -2025-06-04 20:45:20,032 - tests.orchestrator - INFO - Waiting for pod test-pod-725477e8 to be ready -2025-06-04 20:46:20,115 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-725477e8 to be ready after 60s -2025-06-04 20:46:20,116 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-725477e8 to be ready -2025-06-04 20:46:20,195 - tests.orchestrator - INFO - Waiting for PVC one2one-4e963a5b to be bound -2025-06-04 20:46:22,332 - tests.orchestrator - INFO - PVC one2one-4e963a5b is bound -2025-06-04 20:46:22,332 - tests.orchestrator - INFO - Creating pod test-pod-b778d0e5 on PVC one2one-4e963a5b -2025-06-04 20:46:22,407 - tests.orchestrator - INFO - Created pod: test-pod-b778d0e5 using PVC: one2one-4e963a5b -2025-06-04 20:46:22,408 - tests.orchestrator - INFO - Waiting for pod test-pod-b778d0e5 to be ready -2025-06-04 20:47:24,472 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b778d0e5 to be ready after 60s -2025-06-04 20:47:24,472 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b778d0e5 to be ready -2025-06-04 20:47:24,472 - tests.orchestrator - WARNING - Failed to create enough PVC-pod pairs, only created 0 -2025-06-04 20:47:25,473 - tests.orchestrator - INFO - Selected operation: _delete_pod -2025-06-04 20:47:25,473 - tests.orchestrator - INFO - Deleting pod: test-pod-725477e8 from PVC: one2one-116b8e13 -2025-06-04 20:47:25,551 - tests.orchestrator - INFO - Waiting for pod test-pod-725477e8 to be deleted -2025-06-04 20:47:25,616 - tests.orchestrator - INFO - Pod test-pod-725477e8 has been deleted -2025-06-04 20:47:25,616 - tests.orchestrator - INFO - Deleted pod: test-pod-725477e8 -2025-06-04 20:47:26,616 - tests.orchestrator - INFO - Selected operation: _delete_pod -2025-06-04 20:47:26,616 - tests.orchestrator - INFO - Deleting pod: test-pod-f3b3a319 from PVC: one2one-43f0888a -2025-06-04 20:47:26,699 - tests.orchestrator - INFO - Waiting for pod test-pod-f3b3a319 to be deleted -2025-06-04 20:47:26,766 - tests.orchestrator - INFO - Pod test-pod-f3b3a319 has been deleted -2025-06-04 20:47:26,766 - tests.orchestrator - INFO - Deleted pod: test-pod-f3b3a319 -2025-06-04 20:47:27,767 - tests.orchestrator - INFO - Selected operation: _delete_pod -2025-06-04 20:47:27,767 - tests.orchestrator - INFO - Deleting pod: test-pod-80037404 from PVC: one2one-aac689b5 -2025-06-04 20:47:27,853 - tests.orchestrator - INFO - Waiting for pod test-pod-80037404 to be deleted -2025-06-04 20:47:27,919 - tests.orchestrator - INFO - Pod test-pod-80037404 has been deleted -2025-06-04 20:47:27,919 - tests.orchestrator - INFO - Deleted pod: test-pod-80037404 -2025-06-04 20:47:28,919 - tests.orchestrator - INFO - Selected operation: _verify_readwrite -2025-06-04 20:47:28,920 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test -2025-06-04 20:47:29,920 - tests.orchestrator - INFO - Selected operation: _attach_pod -2025-06-04 20:47:29,920 - tests.orchestrator - INFO - Creating pod test-pod-715d3677 on PVC one2one-116b8e13 -2025-06-04 20:47:30,008 - tests.orchestrator - INFO - Created pod: test-pod-715d3677 using PVC: one2one-116b8e13 -2025-06-04 20:47:30,008 - tests.orchestrator - INFO - Waiting for pod test-pod-715d3677 to be ready -2025-06-04 20:48:32,062 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-715d3677 to be ready after 60s -2025-06-04 20:48:32,062 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-715d3677 to be ready -2025-06-04 20:48:33,062 - tests.orchestrator - INFO - Test completed in 331.76 seconds -2025-06-04 20:48:33,063 - tests.orchestrator - INFO - Cleaning up resources... -2025-06-04 20:48:33,063 - tests.orchestrator - INFO - Deleting pod: test-pod-715d3677 from PVC: one2one-116b8e13 -2025-06-04 20:48:33,144 - tests.orchestrator - INFO - Waiting for pod test-pod-715d3677 to be deleted -2025-06-04 20:48:33,208 - tests.orchestrator - INFO - Pod test-pod-715d3677 has been deleted -2025-06-04 20:48:33,209 - tests.orchestrator - INFO - Deleted pod: test-pod-715d3677 -2025-06-04 20:48:33,209 - tests.orchestrator - INFO - Deleting pod: test-pod-b778d0e5 from PVC: one2one-4e963a5b -2025-06-04 20:48:33,289 - tests.orchestrator - INFO - Waiting for pod test-pod-b778d0e5 to be deleted -2025-06-04 20:48:33,356 - tests.orchestrator - INFO - Pod test-pod-b778d0e5 has been deleted -2025-06-04 20:48:33,356 - tests.orchestrator - INFO - Deleted pod: test-pod-b778d0e5 -2025-06-04 20:48:33,356 - tests.orchestrator - INFO - Deleting PVC: test-pvc-a8738d91 -2025-06-04 20:48:33,430 - tests.orchestrator - INFO - Waiting for PVC test-pvc-a8738d91 to be deleted -2025-06-04 20:48:33,497 - tests.orchestrator - INFO - PVC test-pvc-a8738d91 has been deleted -2025-06-04 20:48:33,497 - tests.orchestrator - INFO - Deleted PVC: test-pvc-a8738d91 -2025-06-04 20:48:33,497 - tests.orchestrator - INFO - Deleting PVC: test-pvc-ba881b7a -2025-06-04 20:48:33,570 - tests.orchestrator - INFO - Waiting for PVC test-pvc-ba881b7a to be deleted -2025-06-04 20:48:33,637 - tests.orchestrator - INFO - PVC test-pvc-ba881b7a has been deleted -2025-06-04 20:48:33,637 - tests.orchestrator - INFO - Deleted PVC: test-pvc-ba881b7a -2025-06-04 20:48:33,637 - tests.orchestrator - INFO - Deleting PVC: one2one-43f0888a -2025-06-04 20:48:33,712 - tests.orchestrator - INFO - Waiting for PVC one2one-43f0888a to be deleted -2025-06-04 20:48:33,779 - tests.orchestrator - INFO - PVC one2one-43f0888a has been deleted -2025-06-04 20:48:33,780 - tests.orchestrator - INFO - Deleted PVC: one2one-43f0888a -2025-06-04 20:48:33,780 - tests.orchestrator - INFO - Deleting PVC: one2one-aac689b5 -2025-06-04 20:48:33,858 - tests.orchestrator - INFO - Waiting for PVC one2one-aac689b5 to be deleted -2025-06-04 20:48:33,925 - tests.orchestrator - INFO - PVC one2one-aac689b5 has been deleted -2025-06-04 20:48:33,925 - tests.orchestrator - INFO - Deleted PVC: one2one-aac689b5 -2025-06-04 20:48:33,925 - tests.orchestrator - INFO - Deleting PVC: one2one-116b8e13 -2025-06-04 20:48:34,000 - tests.orchestrator - INFO - Waiting for PVC one2one-116b8e13 to be deleted -2025-06-04 20:48:34,067 - tests.orchestrator - INFO - PVC one2one-116b8e13 has been deleted -2025-06-04 20:48:34,067 - tests.orchestrator - INFO - Deleted PVC: one2one-116b8e13 -2025-06-04 20:48:34,067 - tests.orchestrator - INFO - Deleting PVC: one2one-4e963a5b -2025-06-04 20:48:34,143 - tests.orchestrator - INFO - Waiting for PVC one2one-4e963a5b to be deleted -2025-06-04 20:48:34,209 - tests.orchestrator - INFO - PVC one2one-4e963a5b has been deleted -2025-06-04 20:48:34,210 - tests.orchestrator - INFO - Deleted PVC: one2one-4e963a5b -2025-06-04 20:48:34,210 - tests.orchestrator - INFO - Cleanup completed -2025-06-04 20:48:34,210 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-04 20:48:34,210 - tests.orchestrator - INFO - --- Operations --- -2025-06-04 20:48:34,210 - tests.orchestrator - INFO - create_pvc: 2 succeeded, 0 failed (100.0%) -2025-06-04 20:48:34,210 - tests.orchestrator - INFO - attach_pod: 5 succeeded, 0 failed (100.0%) -2025-06-04 20:48:34,210 - tests.orchestrator - INFO - delete_pod: 5 succeeded, 0 failed (100.0%) -2025-06-04 20:48:34,210 - tests.orchestrator - INFO - delete_pvc: 6 succeeded, 0 failed (100.0%) -2025-06-04 20:48:34,210 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 0 failed (0.0%) -2025-06-04 20:48:34,210 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 0 failed (0.0%) -2025-06-04 20:48:34,210 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-04 20:48:34,210 - tests.orchestrator - INFO - shared_volume_rw: No runs -2025-06-04 20:48:34,210 - tests.orchestrator - INFO - many_to_one: No runs -2025-06-04 20:48:34,210 - tests.orchestrator - INFO - one_to_one: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-04 20:48:34,210 - tests.orchestrator - INFO - concurrent_pvc: No runs -2025-06-04 20:48:34,210 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250604_211255.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250604_211255.log deleted file mode 100644 index 025920f41..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250604_211255.log +++ /dev/null @@ -1,43 +0,0 @@ -2025-06-04 21:12:55,097 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-04 21:12:55,314 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-04 21:12:55,314 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-04 21:12:55,503 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-04 21:12:55,504 - tests.orchestrator - INFO - Selected operation: _delete_pod -2025-06-04 21:12:55,504 - tests.orchestrator - INFO - No pods to delete -2025-06-04 21:12:56,504 - tests.orchestrator - INFO - Selected operation: _verify_readwrite -2025-06-04 21:12:56,504 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test -2025-06-04 21:12:57,504 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario -2025-06-04 21:12:57,504 - tests.orchestrator - INFO - Running specific scenario: _scenario_one_to_one -2025-06-04 21:12:57,504 - tests.orchestrator - INFO - Running scenario: One pod per PVC -2025-06-04 21:12:57,505 - tests.orchestrator - INFO - Creating 4 PVC-pod pairs -2025-06-04 21:12:57,584 - tests.orchestrator - INFO - Waiting for PVC one2one-fa7d38c4 to be bound -2025-06-04 21:12:59,720 - tests.orchestrator - INFO - PVC one2one-fa7d38c4 is bound -2025-06-04 21:12:59,721 - tests.orchestrator - INFO - Creating pod test-pod-f3bd2110 on PVC one2one-fa7d38c4 -2025-06-04 21:12:59,822 - tests.orchestrator - INFO - Created pod: test-pod-f3bd2110 using PVC: one2one-fa7d38c4 -2025-06-04 21:12:59,822 - tests.orchestrator - INFO - Waiting for pod test-pod-f3bd2110 to be ready -2025-06-04 21:13:46,839 - tests.orchestrator - INFO - Test interrupted by user -2025-06-04 21:13:46,839 - tests.orchestrator - INFO - Test completed in 51.52 seconds -2025-06-04 21:13:46,839 - tests.orchestrator - INFO - Cleaning up resources... -2025-06-04 21:13:46,839 - tests.orchestrator - INFO - Deleting pod: test-pod-f3bd2110 from PVC: one2one-fa7d38c4 -2025-06-04 21:13:46,924 - tests.orchestrator - INFO - Waiting for pod test-pod-f3bd2110 to be deleted -2025-06-04 21:13:46,989 - tests.orchestrator - INFO - Pod test-pod-f3bd2110 has been deleted -2025-06-04 21:13:46,989 - tests.orchestrator - INFO - Deleted pod: test-pod-f3bd2110 -2025-06-04 21:13:46,989 - tests.orchestrator - INFO - Deleting PVC: one2one-fa7d38c4 -2025-06-04 21:13:47,063 - tests.orchestrator - INFO - Waiting for PVC one2one-fa7d38c4 to be deleted -2025-06-04 21:13:47,130 - tests.orchestrator - INFO - PVC one2one-fa7d38c4 has been deleted -2025-06-04 21:13:47,130 - tests.orchestrator - INFO - Deleted PVC: one2one-fa7d38c4 -2025-06-04 21:13:47,130 - tests.orchestrator - INFO - Cleanup completed -2025-06-04 21:13:47,131 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-04 21:13:47,131 - tests.orchestrator - INFO - --- Operations --- -2025-06-04 21:13:47,131 - tests.orchestrator - INFO - create_pvc: 0 succeeded, 0 failed (0.0%) -2025-06-04 21:13:47,131 - tests.orchestrator - INFO - attach_pod: 1 succeeded, 0 failed (100.0%) -2025-06-04 21:13:47,131 - tests.orchestrator - INFO - delete_pod: 1 succeeded, 0 failed (100.0%) -2025-06-04 21:13:47,131 - tests.orchestrator - INFO - delete_pvc: 1 succeeded, 0 failed (100.0%) -2025-06-04 21:13:47,131 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 0 failed (0.0%) -2025-06-04 21:13:47,131 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 0 failed (0.0%) -2025-06-04 21:13:47,131 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-04 21:13:47,131 - tests.orchestrator - INFO - shared_volume_rw: No runs -2025-06-04 21:13:47,131 - tests.orchestrator - INFO - many_to_one: No runs -2025-06-04 21:13:47,131 - tests.orchestrator - INFO - one_to_one: 0 succeeded, 0 failed out of 1 runs (0.0%) -2025-06-04 21:13:47,131 - tests.orchestrator - INFO - concurrent_pvc: No runs -2025-06-04 21:13:47,131 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250604_211353.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250604_211353.log deleted file mode 100644 index 55a38aabb..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250604_211353.log +++ /dev/null @@ -1,80 +0,0 @@ -2025-06-04 21:13:53,143 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-04 21:13:53,342 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-04 21:13:53,342 - tests.orchestrator - INFO - Starting orchestrator test for 120 seconds -2025-06-04 21:13:53,535 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-04 21:13:53,536 - tests.orchestrator - INFO - Selected operation: _verify_readwrite -2025-06-04 21:13:53,536 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test -2025-06-04 21:13:54,536 - tests.orchestrator - INFO - Selected operation: _delete_pvc -2025-06-04 21:13:54,536 - tests.orchestrator - INFO - No PVCs to delete -2025-06-04 21:13:55,536 - tests.orchestrator - INFO - Selected operation: _verify_readwrite -2025-06-04 21:13:55,537 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test -2025-06-04 21:13:56,537 - tests.orchestrator - INFO - Selected operation: _delete_pod -2025-06-04 21:13:56,537 - tests.orchestrator - INFO - No pods to delete -2025-06-04 21:13:57,537 - tests.orchestrator - INFO - Selected operation: _delete_pod -2025-06-04 21:13:57,537 - tests.orchestrator - INFO - No pods to delete -2025-06-04 21:13:58,538 - tests.orchestrator - INFO - Selected operation: _verify_readwrite -2025-06-04 21:13:58,538 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test -2025-06-04 21:13:59,538 - tests.orchestrator - INFO - Selected operation: _delete_pod -2025-06-04 21:13:59,538 - tests.orchestrator - INFO - No pods to delete -2025-06-04 21:14:00,538 - tests.orchestrator - INFO - Selected operation: _attach_pod -2025-06-04 21:14:00,539 - tests.orchestrator - INFO - No PVCs available, skipping pod attachment -2025-06-04 21:14:01,539 - tests.orchestrator - INFO - Selected operation: _delete_pod -2025-06-04 21:14:01,540 - tests.orchestrator - INFO - No pods to delete -2025-06-04 21:14:02,540 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario -2025-06-04 21:14:02,541 - tests.orchestrator - INFO - Running specific scenario: _scenario_concurrent_pvc -2025-06-04 21:14:02,541 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations -2025-06-04 21:14:02,541 - tests.orchestrator - INFO - Creating 4 PVCs in quick succession -2025-06-04 21:14:02,617 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-68e38df4 -2025-06-04 21:14:02,617 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-68e38df4 to be bound -2025-06-04 21:14:04,753 - tests.orchestrator - INFO - PVC concurrent-pvc-68e38df4 is bound -2025-06-04 21:14:04,825 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-90dd2e0a -2025-06-04 21:14:04,825 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-90dd2e0a to be bound -2025-06-04 21:14:06,959 - tests.orchestrator - INFO - PVC concurrent-pvc-90dd2e0a is bound -2025-06-04 21:14:07,028 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-09b78e7f -2025-06-04 21:14:07,028 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-09b78e7f to be bound -2025-06-04 21:14:09,163 - tests.orchestrator - INFO - PVC concurrent-pvc-09b78e7f is bound -2025-06-04 21:14:09,232 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-51c7b76a -2025-06-04 21:14:09,232 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-51c7b76a to be bound -2025-06-04 21:14:11,368 - tests.orchestrator - INFO - PVC concurrent-pvc-51c7b76a is bound -2025-06-04 21:14:11,368 - tests.orchestrator - INFO - Successfully created 4 PVCs -2025-06-04 21:14:11,368 - tests.orchestrator - INFO - Creating pod test-pod-60af97a5 on PVC concurrent-pvc-90dd2e0a -2025-06-04 21:14:11,458 - tests.orchestrator - INFO - Created pod: test-pod-60af97a5 using PVC: concurrent-pvc-90dd2e0a -2025-06-04 21:14:11,458 - tests.orchestrator - INFO - Waiting for pod test-pod-60af97a5 to be ready -2025-06-04 21:14:56,603 - tests.orchestrator - INFO - Test interrupted by user -2025-06-04 21:14:56,603 - tests.orchestrator - INFO - Test completed in 63.26 seconds -2025-06-04 21:14:56,603 - tests.orchestrator - INFO - Cleaning up resources... -2025-06-04 21:14:56,603 - tests.orchestrator - INFO - Deleting pod: test-pod-60af97a5 from PVC: concurrent-pvc-90dd2e0a -2025-06-04 21:14:56,689 - tests.orchestrator - INFO - Waiting for pod test-pod-60af97a5 to be deleted -2025-06-04 21:14:56,754 - tests.orchestrator - INFO - Pod test-pod-60af97a5 has been deleted -2025-06-04 21:14:56,754 - tests.orchestrator - INFO - Deleted pod: test-pod-60af97a5 -2025-06-04 21:14:56,754 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-68e38df4 -2025-06-04 21:14:56,838 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-68e38df4 to be deleted -2025-06-04 21:14:56,902 - tests.orchestrator - INFO - PVC concurrent-pvc-68e38df4 has been deleted -2025-06-04 21:14:56,903 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-68e38df4 -2025-06-04 21:14:56,903 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-90dd2e0a -2025-06-04 21:14:56,977 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-90dd2e0a to be deleted -2025-06-04 21:14:57,041 - tests.orchestrator - INFO - PVC concurrent-pvc-90dd2e0a has been deleted -2025-06-04 21:14:57,041 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-90dd2e0a -2025-06-04 21:14:57,041 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-09b78e7f -2025-06-04 21:14:57,116 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-09b78e7f to be deleted -2025-06-04 21:14:57,181 - tests.orchestrator - INFO - PVC concurrent-pvc-09b78e7f has been deleted -2025-06-04 21:14:57,181 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-09b78e7f -2025-06-04 21:14:57,181 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-51c7b76a -2025-06-04 21:14:57,257 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-51c7b76a to be deleted -2025-06-04 21:14:57,322 - tests.orchestrator - INFO - PVC concurrent-pvc-51c7b76a has been deleted -2025-06-04 21:14:57,322 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-51c7b76a -2025-06-04 21:14:57,322 - tests.orchestrator - INFO - Cleanup completed -2025-06-04 21:14:57,322 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-04 21:14:57,322 - tests.orchestrator - INFO - --- Operations --- -2025-06-04 21:14:57,322 - tests.orchestrator - INFO - create_pvc: 4 succeeded, 0 failed (100.0%) -2025-06-04 21:14:57,322 - tests.orchestrator - INFO - attach_pod: 1 succeeded, 0 failed (100.0%) -2025-06-04 21:14:57,322 - tests.orchestrator - INFO - delete_pod: 1 succeeded, 0 failed (100.0%) -2025-06-04 21:14:57,322 - tests.orchestrator - INFO - delete_pvc: 4 succeeded, 0 failed (100.0%) -2025-06-04 21:14:57,322 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 0 failed (0.0%) -2025-06-04 21:14:57,322 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 0 failed (0.0%) -2025-06-04 21:14:57,322 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-04 21:14:57,323 - tests.orchestrator - INFO - shared_volume_rw: No runs -2025-06-04 21:14:57,323 - tests.orchestrator - INFO - many_to_one: No runs -2025-06-04 21:14:57,323 - tests.orchestrator - INFO - one_to_one: No runs -2025-06-04 21:14:57,323 - tests.orchestrator - INFO - concurrent_pvc: 0 succeeded, 0 failed out of 1 runs (0.0%) -2025-06-04 21:14:57,323 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_132802.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_132802.log deleted file mode 100644 index 9cd4da221..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_132802.log +++ /dev/null @@ -1,202 +0,0 @@ -2025-06-05 13:28:02,670 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 13:28:02,885 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 13:28:02,886 - tests.orchestrator - INFO - Starting orchestrator test for 400 seconds -2025-06-05 13:28:03,078 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 13:28:03,079 - tests.orchestrator - INFO - Selected operation: _create_pvc -2025-06-05 13:28:03,079 - tests.orchestrator - INFO - Creating PVC: test-pvc-87f8dca2 -2025-06-05 13:28:03,159 - tests.orchestrator - INFO - Created PVC: test-pvc-87f8dca2 -2025-06-05 13:28:03,159 - tests.orchestrator - INFO - Waiting for PVC test-pvc-87f8dca2 to be bound -2025-06-05 13:28:05,290 - tests.orchestrator - INFO - PVC test-pvc-87f8dca2 is bound -2025-06-05 13:28:06,290 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario -2025-06-05 13:28:06,290 - tests.orchestrator - INFO - Running specific scenario: _scenario_concurrent_pvc -2025-06-05 13:28:06,290 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations -2025-06-05 13:28:06,290 - tests.orchestrator - INFO - Creating 6 PVCs in quick succession -2025-06-05 13:28:06,363 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-27248b37 -2025-06-05 13:28:06,363 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-27248b37 to be bound -2025-06-05 13:28:08,495 - tests.orchestrator - INFO - PVC concurrent-pvc-27248b37 is bound -2025-06-05 13:28:08,561 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-0be57a62 -2025-06-05 13:28:08,562 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-0be57a62 to be bound -2025-06-05 13:28:10,696 - tests.orchestrator - INFO - PVC concurrent-pvc-0be57a62 is bound -2025-06-05 13:28:10,764 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-d46ca8c1 -2025-06-05 13:28:10,764 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-d46ca8c1 to be bound -2025-06-05 13:28:12,902 - tests.orchestrator - INFO - PVC concurrent-pvc-d46ca8c1 is bound -2025-06-05 13:28:12,972 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-4b94a252 -2025-06-05 13:28:12,973 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-4b94a252 to be bound -2025-06-05 13:28:15,113 - tests.orchestrator - INFO - PVC concurrent-pvc-4b94a252 is bound -2025-06-05 13:28:15,180 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-737583a5 -2025-06-05 13:28:15,181 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-737583a5 to be bound -2025-06-05 13:28:17,317 - tests.orchestrator - INFO - PVC concurrent-pvc-737583a5 is bound -2025-06-05 13:28:17,385 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-b6ec6167 -2025-06-05 13:28:17,385 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-b6ec6167 to be bound -2025-06-05 13:28:19,520 - tests.orchestrator - INFO - PVC concurrent-pvc-b6ec6167 is bound -2025-06-05 13:28:19,520 - tests.orchestrator - INFO - Successfully created 6 PVCs -2025-06-05 13:28:19,520 - tests.orchestrator - INFO - Creating pod test-pod-05839583 on PVC concurrent-pvc-4b94a252 -2025-06-05 13:28:19,605 - tests.orchestrator - INFO - Created pod: test-pod-05839583 using PVC: concurrent-pvc-4b94a252 -2025-06-05 13:28:19,605 - tests.orchestrator - INFO - Waiting for pod test-pod-05839583 to be ready -2025-06-05 13:29:21,656 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-05839583 to be ready after 60s -2025-06-05 13:29:21,657 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-05839583 to be ready -2025-06-05 13:29:21,657 - tests.orchestrator - INFO - Creating pod test-pod-c11d7609 on PVC concurrent-pvc-b6ec6167 -2025-06-05 13:29:21,746 - tests.orchestrator - INFO - Created pod: test-pod-c11d7609 using PVC: concurrent-pvc-b6ec6167 -2025-06-05 13:29:21,746 - tests.orchestrator - INFO - Waiting for pod test-pod-c11d7609 to be ready -2025-06-05 13:30:23,813 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-c11d7609 to be ready after 60s -2025-06-05 13:30:23,813 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-c11d7609 to be ready -2025-06-05 13:30:23,813 - tests.orchestrator - INFO - Creating pod test-pod-f3958de4 on PVC concurrent-pvc-27248b37 -2025-06-05 13:30:23,897 - tests.orchestrator - INFO - Created pod: test-pod-f3958de4 using PVC: concurrent-pvc-27248b37 -2025-06-05 13:30:23,897 - tests.orchestrator - INFO - Waiting for pod test-pod-f3958de4 to be ready -2025-06-05 13:31:23,958 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-f3958de4 to be ready after 60s -2025-06-05 13:31:23,958 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-f3958de4 to be ready -2025-06-05 13:31:23,958 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-0be57a62 -2025-06-05 13:31:24,029 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-0be57a62 to be deleted -2025-06-05 13:31:24,096 - tests.orchestrator - INFO - PVC concurrent-pvc-0be57a62 has been deleted -2025-06-05 13:31:24,096 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-0be57a62 -2025-06-05 13:31:24,096 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-4b94a252 -2025-06-05 13:31:24,096 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-4b94a252 -2025-06-05 13:31:24,096 - tests.orchestrator - INFO - Deleting pod: test-pod-05839583 from PVC: concurrent-pvc-4b94a252 -2025-06-05 13:31:24,177 - tests.orchestrator - INFO - Waiting for pod test-pod-05839583 to be deleted -2025-06-05 13:31:24,241 - tests.orchestrator - INFO - Pod test-pod-05839583 has been deleted -2025-06-05 13:31:24,241 - tests.orchestrator - INFO - Deleted pod: test-pod-05839583 -2025-06-05 13:31:24,314 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-4b94a252 to be deleted -2025-06-05 13:31:24,380 - tests.orchestrator - INFO - PVC concurrent-pvc-4b94a252 has been deleted -2025-06-05 13:31:24,380 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-4b94a252 -2025-06-05 13:31:24,381 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-27248b37 -2025-06-05 13:31:24,381 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-27248b37 -2025-06-05 13:31:24,381 - tests.orchestrator - INFO - Deleting pod: test-pod-f3958de4 from PVC: concurrent-pvc-27248b37 -2025-06-05 13:31:24,464 - tests.orchestrator - INFO - Waiting for pod test-pod-f3958de4 to be deleted -2025-06-05 13:31:24,528 - tests.orchestrator - INFO - Pod test-pod-f3958de4 has been deleted -2025-06-05 13:31:24,528 - tests.orchestrator - INFO - Deleted pod: test-pod-f3958de4 -2025-06-05 13:31:24,601 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-27248b37 to be deleted -2025-06-05 13:31:24,668 - tests.orchestrator - INFO - PVC concurrent-pvc-27248b37 has been deleted -2025-06-05 13:31:24,668 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-27248b37 -2025-06-05 13:31:24,668 - tests.orchestrator - INFO - Rapid PVC scenario completed -2025-06-05 13:31:25,668 - tests.orchestrator - INFO - Selected operation: _attach_pod -2025-06-05 13:31:25,669 - tests.orchestrator - INFO - Creating pod test-pod-4386bacf on PVC concurrent-pvc-737583a5 -2025-06-05 13:31:25,750 - tests.orchestrator - INFO - Created pod: test-pod-4386bacf using PVC: concurrent-pvc-737583a5 -2025-06-05 13:31:25,750 - tests.orchestrator - INFO - Waiting for pod test-pod-4386bacf to be ready -2025-06-05 13:32:27,796 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-4386bacf to be ready after 60s -2025-06-05 13:32:27,796 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-4386bacf to be ready -2025-06-05 13:32:28,797 - tests.orchestrator - INFO - Selected operation: _attach_pod -2025-06-05 13:32:28,798 - tests.orchestrator - INFO - Creating pod test-pod-93094317 on PVC concurrent-pvc-b6ec6167 -2025-06-05 13:32:28,887 - tests.orchestrator - INFO - Created pod: test-pod-93094317 using PVC: concurrent-pvc-b6ec6167 -2025-06-05 13:32:28,887 - tests.orchestrator - INFO - Waiting for pod test-pod-93094317 to be ready -2025-06-05 13:33:30,941 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-93094317 to be ready after 60s -2025-06-05 13:33:30,941 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-93094317 to be ready -2025-06-05 13:33:31,941 - tests.orchestrator - INFO - Selected operation: _create_pvc -2025-06-05 13:33:31,942 - tests.orchestrator - INFO - Creating PVC: test-pvc-cd15165b -2025-06-05 13:33:32,022 - tests.orchestrator - INFO - Created PVC: test-pvc-cd15165b -2025-06-05 13:33:32,022 - tests.orchestrator - INFO - Waiting for PVC test-pvc-cd15165b to be bound -2025-06-05 13:33:34,157 - tests.orchestrator - INFO - PVC test-pvc-cd15165b is bound -2025-06-05 13:33:35,157 - tests.orchestrator - INFO - Selected operation: _create_pvc -2025-06-05 13:33:35,157 - tests.orchestrator - INFO - Creating PVC: test-pvc-67687a38 -2025-06-05 13:33:35,229 - tests.orchestrator - INFO - Created PVC: test-pvc-67687a38 -2025-06-05 13:33:35,229 - tests.orchestrator - INFO - Waiting for PVC test-pvc-67687a38 to be bound -2025-06-05 13:33:37,363 - tests.orchestrator - INFO - PVC test-pvc-67687a38 is bound -2025-06-05 13:33:38,364 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario -2025-06-05 13:33:38,364 - tests.orchestrator - INFO - Running specific scenario: _scenario_concurrent_pvc -2025-06-05 13:33:38,364 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations -2025-06-05 13:33:38,364 - tests.orchestrator - INFO - Creating 3 PVCs in quick succession -2025-06-05 13:33:38,431 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-a6fe87d2 -2025-06-05 13:33:38,431 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-a6fe87d2 to be bound -2025-06-05 13:33:40,563 - tests.orchestrator - INFO - PVC concurrent-pvc-a6fe87d2 is bound -2025-06-05 13:33:40,631 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-eddc8eb9 -2025-06-05 13:33:40,631 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-eddc8eb9 to be bound -2025-06-05 13:33:42,765 - tests.orchestrator - INFO - PVC concurrent-pvc-eddc8eb9 is bound -2025-06-05 13:33:42,835 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-2bd04bf8 -2025-06-05 13:33:42,836 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-2bd04bf8 to be bound -2025-06-05 13:33:44,970 - tests.orchestrator - INFO - PVC concurrent-pvc-2bd04bf8 is bound -2025-06-05 13:33:44,971 - tests.orchestrator - INFO - Successfully created 3 PVCs -2025-06-05 13:33:44,971 - tests.orchestrator - INFO - Creating pod test-pod-b5b431f0 on PVC concurrent-pvc-2bd04bf8 -2025-06-05 13:33:45,045 - tests.orchestrator - INFO - Created pod: test-pod-b5b431f0 using PVC: concurrent-pvc-2bd04bf8 -2025-06-05 13:33:45,045 - tests.orchestrator - INFO - Waiting for pod test-pod-b5b431f0 to be ready -2025-06-05 13:34:45,136 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b5b431f0 to be ready after 60s -2025-06-05 13:34:45,136 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b5b431f0 to be ready -2025-06-05 13:34:45,137 - tests.orchestrator - INFO - Creating pod test-pod-a900f8ea on PVC concurrent-pvc-a6fe87d2 -2025-06-05 13:34:45,234 - tests.orchestrator - INFO - Created pod: test-pod-a900f8ea using PVC: concurrent-pvc-a6fe87d2 -2025-06-05 13:34:45,234 - tests.orchestrator - INFO - Waiting for pod test-pod-a900f8ea to be ready -2025-06-05 13:35:45,421 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a900f8ea to be ready after 60s -2025-06-05 13:35:45,421 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a900f8ea to be ready -2025-06-05 13:35:45,421 - tests.orchestrator - INFO - Creating pod test-pod-71d9daf8 on PVC concurrent-pvc-eddc8eb9 -2025-06-05 13:35:45,506 - tests.orchestrator - INFO - Created pod: test-pod-71d9daf8 using PVC: concurrent-pvc-eddc8eb9 -2025-06-05 13:35:45,506 - tests.orchestrator - INFO - Waiting for pod test-pod-71d9daf8 to be ready -2025-06-05 13:36:47,551 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-71d9daf8 to be ready after 60s -2025-06-05 13:36:47,551 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-71d9daf8 to be ready -2025-06-05 13:36:47,551 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-2bd04bf8 -2025-06-05 13:36:47,551 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-2bd04bf8 -2025-06-05 13:36:47,551 - tests.orchestrator - INFO - Deleting pod: test-pod-b5b431f0 from PVC: concurrent-pvc-2bd04bf8 -2025-06-05 13:36:47,632 - tests.orchestrator - INFO - Waiting for pod test-pod-b5b431f0 to be deleted -2025-06-05 13:36:47,699 - tests.orchestrator - INFO - Pod test-pod-b5b431f0 has been deleted -2025-06-05 13:36:47,699 - tests.orchestrator - INFO - Deleted pod: test-pod-b5b431f0 -2025-06-05 13:36:47,771 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-2bd04bf8 to be deleted -2025-06-05 13:36:47,839 - tests.orchestrator - INFO - PVC concurrent-pvc-2bd04bf8 has been deleted -2025-06-05 13:36:47,839 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-2bd04bf8 -2025-06-05 13:36:47,839 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-eddc8eb9 -2025-06-05 13:36:47,839 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-eddc8eb9 -2025-06-05 13:36:47,839 - tests.orchestrator - INFO - Deleting pod: test-pod-71d9daf8 from PVC: concurrent-pvc-eddc8eb9 -2025-06-05 13:36:47,919 - tests.orchestrator - INFO - Waiting for pod test-pod-71d9daf8 to be deleted -2025-06-05 13:36:47,984 - tests.orchestrator - INFO - Pod test-pod-71d9daf8 has been deleted -2025-06-05 13:36:47,984 - tests.orchestrator - INFO - Deleted pod: test-pod-71d9daf8 -2025-06-05 13:36:48,056 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-eddc8eb9 to be deleted -2025-06-05 13:36:48,122 - tests.orchestrator - INFO - PVC concurrent-pvc-eddc8eb9 has been deleted -2025-06-05 13:36:48,122 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-eddc8eb9 -2025-06-05 13:36:48,122 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-a6fe87d2 -2025-06-05 13:36:48,122 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-a6fe87d2 -2025-06-05 13:36:48,122 - tests.orchestrator - INFO - Deleting pod: test-pod-a900f8ea from PVC: concurrent-pvc-a6fe87d2 -2025-06-05 13:36:48,206 - tests.orchestrator - INFO - Waiting for pod test-pod-a900f8ea to be deleted -2025-06-05 13:36:48,270 - tests.orchestrator - INFO - Pod test-pod-a900f8ea has been deleted -2025-06-05 13:36:48,271 - tests.orchestrator - INFO - Deleted pod: test-pod-a900f8ea -2025-06-05 13:36:48,345 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-a6fe87d2 to be deleted -2025-06-05 13:36:48,412 - tests.orchestrator - INFO - PVC concurrent-pvc-a6fe87d2 has been deleted -2025-06-05 13:36:48,412 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-a6fe87d2 -2025-06-05 13:36:48,412 - tests.orchestrator - INFO - Rapid PVC scenario completed -2025-06-05 13:36:49,412 - tests.orchestrator - INFO - Test completed in 526.53 seconds -2025-06-05 13:36:49,412 - tests.orchestrator - INFO - Cleaning up resources... -2025-06-05 13:36:49,412 - tests.orchestrator - INFO - Deleting pod: test-pod-4386bacf from PVC: concurrent-pvc-737583a5 -2025-06-05 13:36:49,492 - tests.orchestrator - INFO - Waiting for pod test-pod-4386bacf to be deleted -2025-06-05 13:36:49,559 - tests.orchestrator - INFO - Pod test-pod-4386bacf has been deleted -2025-06-05 13:36:49,560 - tests.orchestrator - INFO - Deleted pod: test-pod-4386bacf -2025-06-05 13:36:49,560 - tests.orchestrator - INFO - Deleting pod: test-pod-c11d7609 from PVC: concurrent-pvc-b6ec6167 -2025-06-05 13:36:49,640 - tests.orchestrator - INFO - Waiting for pod test-pod-c11d7609 to be deleted -2025-06-05 13:36:49,706 - tests.orchestrator - INFO - Pod test-pod-c11d7609 has been deleted -2025-06-05 13:36:49,706 - tests.orchestrator - INFO - Deleted pod: test-pod-c11d7609 -2025-06-05 13:36:49,706 - tests.orchestrator - INFO - Deleting pod: test-pod-93094317 from PVC: concurrent-pvc-b6ec6167 -2025-06-05 13:36:49,783 - tests.orchestrator - INFO - Waiting for pod test-pod-93094317 to be deleted -2025-06-05 13:36:49,850 - tests.orchestrator - INFO - Pod test-pod-93094317 has been deleted -2025-06-05 13:36:49,850 - tests.orchestrator - INFO - Deleted pod: test-pod-93094317 -2025-06-05 13:36:49,850 - tests.orchestrator - INFO - Deleting PVC: test-pvc-87f8dca2 -2025-06-05 13:36:49,922 - tests.orchestrator - INFO - Waiting for PVC test-pvc-87f8dca2 to be deleted -2025-06-05 13:36:49,988 - tests.orchestrator - INFO - PVC test-pvc-87f8dca2 has been deleted -2025-06-05 13:36:49,989 - tests.orchestrator - INFO - Deleted PVC: test-pvc-87f8dca2 -2025-06-05 13:36:49,989 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-d46ca8c1 -2025-06-05 13:36:50,060 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-d46ca8c1 to be deleted -2025-06-05 13:36:50,126 - tests.orchestrator - INFO - PVC concurrent-pvc-d46ca8c1 has been deleted -2025-06-05 13:36:50,127 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-d46ca8c1 -2025-06-05 13:36:50,127 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-737583a5 -2025-06-05 13:36:50,197 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-737583a5 to be deleted -2025-06-05 13:36:50,264 - tests.orchestrator - INFO - PVC concurrent-pvc-737583a5 has been deleted -2025-06-05 13:36:50,264 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-737583a5 -2025-06-05 13:36:50,264 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-b6ec6167 -2025-06-05 13:36:50,336 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-b6ec6167 to be deleted -2025-06-05 13:36:50,403 - tests.orchestrator - INFO - PVC concurrent-pvc-b6ec6167 has been deleted -2025-06-05 13:36:50,403 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-b6ec6167 -2025-06-05 13:36:50,403 - tests.orchestrator - INFO - Deleting PVC: test-pvc-cd15165b -2025-06-05 13:36:50,477 - tests.orchestrator - INFO - Waiting for PVC test-pvc-cd15165b to be deleted -2025-06-05 13:36:50,544 - tests.orchestrator - INFO - PVC test-pvc-cd15165b has been deleted -2025-06-05 13:36:50,545 - tests.orchestrator - INFO - Deleted PVC: test-pvc-cd15165b -2025-06-05 13:36:50,545 - tests.orchestrator - INFO - Deleting PVC: test-pvc-67687a38 -2025-06-05 13:36:50,617 - tests.orchestrator - INFO - Waiting for PVC test-pvc-67687a38 to be deleted -2025-06-05 13:36:50,683 - tests.orchestrator - INFO - PVC test-pvc-67687a38 has been deleted -2025-06-05 13:36:50,683 - tests.orchestrator - INFO - Deleted PVC: test-pvc-67687a38 -2025-06-05 13:36:50,683 - tests.orchestrator - INFO - Cleanup completed -2025-06-05 13:36:50,683 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-05 13:36:50,683 - tests.orchestrator - INFO - --- Operations --- -2025-06-05 13:36:50,683 - tests.orchestrator - INFO - create_pvc: 12 succeeded, 0 failed (100.0%) -2025-06-05 13:36:50,683 - tests.orchestrator - INFO - attach_pod: 8 succeeded, 0 failed (100.0%) -2025-06-05 13:36:50,683 - tests.orchestrator - INFO - delete_pod: 8 succeeded, 0 failed (100.0%) -2025-06-05 13:36:50,684 - tests.orchestrator - INFO - delete_pvc: 12 succeeded, 0 failed (100.0%) -2025-06-05 13:36:50,684 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 0 failed (0.0%) -2025-06-05 13:36:50,684 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 0 failed (0.0%) -2025-06-05 13:36:50,684 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-05 13:36:50,684 - tests.orchestrator - INFO - shared_volume_rw: No runs -2025-06-05 13:36:50,684 - tests.orchestrator - INFO - many_to_one: No runs -2025-06-05 13:36:50,684 - tests.orchestrator - INFO - one_to_one: No runs -2025-06-05 13:36:50,684 - tests.orchestrator - INFO - concurrent_pvc: 2 succeeded, 0 failed out of 2 runs (100.0%) -2025-06-05 13:36:50,684 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_135002.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_135002.log deleted file mode 100644 index 570bd57de..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_135002.log +++ /dev/null @@ -1,123 +0,0 @@ -2025-06-05 13:50:02,494 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 13:50:02,715 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 13:50:02,715 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-05 13:50:02,906 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 13:50:02,906 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 13:50:02,906 - tests.orchestrator - INFO - Creating PVC: test-pvc-040b70c6 -2025-06-05 13:50:02,982 - tests.orchestrator - INFO - Created PVC: test-pvc-040b70c6 -2025-06-05 13:50:02,982 - tests.orchestrator - INFO - Waiting for PVC test-pvc-040b70c6 to be bound -2025-06-05 13:50:05,116 - tests.orchestrator - INFO - PVC test-pvc-040b70c6 is bound -2025-06-05 13:50:05,116 - tests.orchestrator - INFO - Creating PVC: test-pvc-e5eb6b62 -2025-06-05 13:50:05,184 - tests.orchestrator - INFO - Created PVC: test-pvc-e5eb6b62 -2025-06-05 13:50:05,184 - tests.orchestrator - INFO - Waiting for PVC test-pvc-e5eb6b62 to be bound -2025-06-05 13:50:07,318 - tests.orchestrator - INFO - PVC test-pvc-e5eb6b62 is bound -2025-06-05 13:50:07,319 - tests.orchestrator - INFO - Creating pod test-pod-094962b3 on PVC test-pvc-e5eb6b62 -2025-06-05 13:50:07,399 - tests.orchestrator - INFO - Created pod: test-pod-094962b3 using PVC: test-pvc-e5eb6b62 -2025-06-05 13:50:07,400 - tests.orchestrator - INFO - Waiting for pod test-pod-094962b3 to be ready -2025-06-05 13:51:09,437 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-094962b3 to be ready after 60s -2025-06-05 13:51:09,437 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-094962b3 to be ready -2025-06-05 13:51:09,437 - tests.orchestrator - INFO - Creating pod test-pod-1b064942 on PVC test-pvc-040b70c6 -2025-06-05 13:51:09,521 - tests.orchestrator - INFO - Created pod: test-pod-1b064942 using PVC: test-pvc-040b70c6 -2025-06-05 13:51:09,521 - tests.orchestrator - INFO - Waiting for pod test-pod-1b064942 to be ready -2025-06-05 13:52:11,558 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1b064942 to be ready after 60s -2025-06-05 13:52:11,558 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1b064942 to be ready -2025-06-05 13:52:11,559 - tests.orchestrator - INFO - Creating pod test-pod-ab3159d4 on PVC test-pvc-e5eb6b62 -2025-06-05 13:52:11,645 - tests.orchestrator - INFO - Created pod: test-pod-ab3159d4 using PVC: test-pvc-e5eb6b62 -2025-06-05 13:52:11,646 - tests.orchestrator - INFO - Waiting for pod test-pod-ab3159d4 to be ready -2025-06-05 13:53:13,682 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-ab3159d4 to be ready after 60s -2025-06-05 13:53:13,682 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-ab3159d4 to be ready -2025-06-05 13:53:13,683 - tests.orchestrator - INFO - Testing read/write between pods test-pod-094962b3 and test-pod-ab3159d4 sharing PVC test-pvc-e5eb6b62 -2025-06-05 13:53:13,751 - tests.orchestrator - ERROR - Failed in read/write verification: (400) -Reason: Bad Request -HTTP response headers: HTTPHeaderDict({'Audit-Id': '7c3ae37d-dd7e-42d1-97d7-9f45f47124cc', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 13:53:13 GMT', 'Content-Length': '166'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-094962b3 does not have a host assigned","reason":"BadRequest","code":400} - - -2025-06-05 13:53:13,751 - tests.orchestrator - INFO - Running specific scenario: _scenario_one_to_one -2025-06-05 13:53:13,751 - tests.orchestrator - INFO - Running scenario: One pod per PVC -2025-06-05 13:53:13,751 - tests.orchestrator - INFO - Creating 3 PVC-pod pairs -2025-06-05 13:53:13,829 - tests.orchestrator - INFO - Waiting for PVC one2one-9268308a to be bound -2025-06-05 13:53:15,962 - tests.orchestrator - INFO - PVC one2one-9268308a is bound -2025-06-05 13:53:15,962 - tests.orchestrator - INFO - Creating pod test-pod-2d27ea53 on PVC one2one-9268308a -2025-06-05 13:53:16,038 - tests.orchestrator - INFO - Created pod: test-pod-2d27ea53 using PVC: one2one-9268308a -2025-06-05 13:53:16,038 - tests.orchestrator - INFO - Waiting for pod test-pod-2d27ea53 to be ready -2025-06-05 13:54:16,048 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-2d27ea53 to be ready after 60s -2025-06-05 13:54:16,048 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-2d27ea53 to be ready -2025-06-05 13:54:16,124 - tests.orchestrator - INFO - Waiting for PVC one2one-63559411 to be bound -2025-06-05 13:54:18,258 - tests.orchestrator - INFO - PVC one2one-63559411 is bound -2025-06-05 13:54:18,258 - tests.orchestrator - INFO - Creating pod test-pod-5c1f7339 on PVC one2one-63559411 -2025-06-05 13:54:18,330 - tests.orchestrator - INFO - Created pod: test-pod-5c1f7339 using PVC: one2one-63559411 -2025-06-05 13:54:18,330 - tests.orchestrator - INFO - Waiting for pod test-pod-5c1f7339 to be ready -2025-06-05 13:55:20,367 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-5c1f7339 to be ready after 60s -2025-06-05 13:55:20,367 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-5c1f7339 to be ready -2025-06-05 13:55:20,444 - tests.orchestrator - INFO - Waiting for PVC one2one-24717940 to be bound -2025-06-05 13:55:22,579 - tests.orchestrator - INFO - PVC one2one-24717940 is bound -2025-06-05 13:55:22,580 - tests.orchestrator - INFO - Creating pod test-pod-a84638d7 on PVC one2one-24717940 -2025-06-05 13:55:22,655 - tests.orchestrator - INFO - Created pod: test-pod-a84638d7 using PVC: one2one-24717940 -2025-06-05 13:55:22,655 - tests.orchestrator - INFO - Waiting for pod test-pod-a84638d7 to be ready -2025-06-05 13:56:24,701 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a84638d7 to be ready after 60s -2025-06-05 13:56:24,701 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a84638d7 to be ready -2025-06-05 13:56:24,701 - tests.orchestrator - WARNING - Failed to create enough PVC-pod pairs, only created 0 -2025-06-05 13:56:24,701 - tests.orchestrator - INFO - Deleting pod: test-pod-094962b3 from PVC: test-pvc-e5eb6b62 -2025-06-05 13:56:24,780 - tests.orchestrator - INFO - Waiting for pod test-pod-094962b3 to be deleted -2025-06-05 13:56:24,846 - tests.orchestrator - INFO - Pod test-pod-094962b3 has been deleted -2025-06-05 13:56:24,846 - tests.orchestrator - INFO - Deleted pod: test-pod-094962b3 -2025-06-05 13:56:24,846 - tests.orchestrator - INFO - Deleting PVC: one2one-63559411 -2025-06-05 13:56:24,846 - tests.orchestrator - INFO - Deleting 1 pods using PVC one2one-63559411 -2025-06-05 13:56:24,846 - tests.orchestrator - INFO - Deleting pod: test-pod-5c1f7339 from PVC: one2one-63559411 -2025-06-05 13:56:24,932 - tests.orchestrator - INFO - Waiting for pod test-pod-5c1f7339 to be deleted -2025-06-05 13:56:24,997 - tests.orchestrator - INFO - Pod test-pod-5c1f7339 has been deleted -2025-06-05 13:56:24,997 - tests.orchestrator - INFO - Deleted pod: test-pod-5c1f7339 -2025-06-05 13:56:25,070 - tests.orchestrator - INFO - Waiting for PVC one2one-63559411 to be deleted -2025-06-05 13:56:25,135 - tests.orchestrator - INFO - PVC one2one-63559411 has been deleted -2025-06-05 13:56:25,136 - tests.orchestrator - INFO - Deleted PVC: one2one-63559411 -2025-06-05 13:56:25,136 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations -2025-06-05 13:56:25,136 - tests.orchestrator - INFO - Test completed in 382.42 seconds -2025-06-05 13:56:25,136 - tests.orchestrator - INFO - Cleaning up resources... -2025-06-05 13:56:25,136 - tests.orchestrator - INFO - Deleting pod: test-pod-1b064942 from PVC: test-pvc-040b70c6 -2025-06-05 13:56:25,213 - tests.orchestrator - INFO - Waiting for pod test-pod-1b064942 to be deleted -2025-06-05 13:56:25,279 - tests.orchestrator - INFO - Pod test-pod-1b064942 has been deleted -2025-06-05 13:56:25,279 - tests.orchestrator - INFO - Deleted pod: test-pod-1b064942 -2025-06-05 13:56:25,279 - tests.orchestrator - INFO - Deleting pod: test-pod-ab3159d4 from PVC: test-pvc-e5eb6b62 -2025-06-05 13:56:25,363 - tests.orchestrator - INFO - Waiting for pod test-pod-ab3159d4 to be deleted -2025-06-05 13:56:25,429 - tests.orchestrator - INFO - Pod test-pod-ab3159d4 has been deleted -2025-06-05 13:56:25,429 - tests.orchestrator - INFO - Deleted pod: test-pod-ab3159d4 -2025-06-05 13:56:25,429 - tests.orchestrator - INFO - Deleting pod: test-pod-2d27ea53 from PVC: one2one-9268308a -2025-06-05 13:56:25,507 - tests.orchestrator - INFO - Waiting for pod test-pod-2d27ea53 to be deleted -2025-06-05 13:56:25,573 - tests.orchestrator - INFO - Pod test-pod-2d27ea53 has been deleted -2025-06-05 13:56:25,573 - tests.orchestrator - INFO - Deleted pod: test-pod-2d27ea53 -2025-06-05 13:56:25,574 - tests.orchestrator - INFO - Deleting pod: test-pod-a84638d7 from PVC: one2one-24717940 -2025-06-05 13:56:25,658 - tests.orchestrator - INFO - Waiting for pod test-pod-a84638d7 to be deleted -2025-06-05 13:56:25,725 - tests.orchestrator - INFO - Pod test-pod-a84638d7 has been deleted -2025-06-05 13:56:25,726 - tests.orchestrator - INFO - Deleted pod: test-pod-a84638d7 -2025-06-05 13:56:25,726 - tests.orchestrator - INFO - Deleting PVC: test-pvc-040b70c6 -2025-06-05 13:56:25,800 - tests.orchestrator - INFO - Waiting for PVC test-pvc-040b70c6 to be deleted -2025-06-05 13:56:25,866 - tests.orchestrator - INFO - PVC test-pvc-040b70c6 has been deleted -2025-06-05 13:56:25,866 - tests.orchestrator - INFO - Deleted PVC: test-pvc-040b70c6 -2025-06-05 13:56:25,866 - tests.orchestrator - INFO - Deleting PVC: test-pvc-e5eb6b62 -2025-06-05 13:56:25,940 - tests.orchestrator - INFO - Waiting for PVC test-pvc-e5eb6b62 to be deleted -2025-06-05 13:56:26,006 - tests.orchestrator - INFO - PVC test-pvc-e5eb6b62 has been deleted -2025-06-05 13:56:26,006 - tests.orchestrator - INFO - Deleted PVC: test-pvc-e5eb6b62 -2025-06-05 13:56:26,006 - tests.orchestrator - INFO - Deleting PVC: one2one-9268308a -2025-06-05 13:56:26,081 - tests.orchestrator - INFO - Waiting for PVC one2one-9268308a to be deleted -2025-06-05 13:56:26,146 - tests.orchestrator - INFO - PVC one2one-9268308a has been deleted -2025-06-05 13:56:26,146 - tests.orchestrator - INFO - Deleted PVC: one2one-9268308a -2025-06-05 13:56:26,146 - tests.orchestrator - INFO - Deleting PVC: one2one-24717940 -2025-06-05 13:56:26,221 - tests.orchestrator - INFO - Waiting for PVC one2one-24717940 to be deleted -2025-06-05 13:56:26,287 - tests.orchestrator - INFO - PVC one2one-24717940 has been deleted -2025-06-05 13:56:26,287 - tests.orchestrator - INFO - Deleted PVC: one2one-24717940 -2025-06-05 13:56:26,287 - tests.orchestrator - INFO - Cleanup completed -2025-06-05 13:56:26,287 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-05 13:56:26,287 - tests.orchestrator - INFO - --- Operations --- -2025-06-05 13:56:26,287 - tests.orchestrator - INFO - create_pvc: 2 succeeded, 0 failed (100.0%) -2025-06-05 13:56:26,287 - tests.orchestrator - INFO - attach_pod: 6 succeeded, 0 failed (100.0%) -2025-06-05 13:56:26,287 - tests.orchestrator - INFO - delete_pod: 6 succeeded, 0 failed (100.0%) -2025-06-05 13:56:26,287 - tests.orchestrator - INFO - delete_pvc: 5 succeeded, 0 failed (100.0%) -2025-06-05 13:56:26,287 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) -2025-06-05 13:56:26,287 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) -2025-06-05 13:56:26,287 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-05 13:56:26,287 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-05 13:56:26,287 - tests.orchestrator - INFO - many_to_one: No runs -2025-06-05 13:56:26,287 - tests.orchestrator - INFO - one_to_one: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-05 13:56:26,288 - tests.orchestrator - INFO - concurrent_pvc: No runs -2025-06-05 13:56:26,288 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_140408.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_140408.log deleted file mode 100644 index 51b5d693f..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_140408.log +++ /dev/null @@ -1,138 +0,0 @@ -2025-06-05 14:04:08,567 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 14:04:08,780 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 14:04:08,780 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-05 14:04:08,972 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 14:04:08,972 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 14:04:08,972 - tests.orchestrator - INFO - Creating PVC: test-pvc-847a8a56 -2025-06-05 14:04:09,057 - tests.orchestrator - INFO - Created PVC: test-pvc-847a8a56 -2025-06-05 14:04:09,057 - tests.orchestrator - INFO - Waiting for PVC test-pvc-847a8a56 to be bound -2025-06-05 14:04:11,188 - tests.orchestrator - INFO - PVC test-pvc-847a8a56 is bound -2025-06-05 14:04:11,189 - tests.orchestrator - INFO - Creating PVC: test-pvc-9dee4752 -2025-06-05 14:04:11,257 - tests.orchestrator - INFO - Created PVC: test-pvc-9dee4752 -2025-06-05 14:04:11,258 - tests.orchestrator - INFO - Waiting for PVC test-pvc-9dee4752 to be bound -2025-06-05 14:04:13,391 - tests.orchestrator - INFO - PVC test-pvc-9dee4752 is bound -2025-06-05 14:04:13,391 - tests.orchestrator - INFO - Creating pod test-pod-d215a119 on PVC test-pvc-9dee4752 -2025-06-05 14:04:13,474 - tests.orchestrator - INFO - Created pod: test-pod-d215a119 using PVC: test-pvc-9dee4752 -2025-06-05 14:04:13,474 - tests.orchestrator - INFO - Waiting for pod test-pod-d215a119 to be ready -2025-06-05 14:05:15,521 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-d215a119 to be ready after 60s -2025-06-05 14:05:15,521 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-d215a119 to be ready -2025-06-05 14:05:15,522 - tests.orchestrator - INFO - Creating pod test-pod-54364ba6 on PVC test-pvc-9dee4752 -2025-06-05 14:05:15,608 - tests.orchestrator - INFO - Created pod: test-pod-54364ba6 using PVC: test-pvc-9dee4752 -2025-06-05 14:05:15,608 - tests.orchestrator - INFO - Waiting for pod test-pod-54364ba6 to be ready -2025-06-05 14:06:15,655 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-54364ba6 to be ready after 60s -2025-06-05 14:06:15,655 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-54364ba6 to be ready -2025-06-05 14:06:15,655 - tests.orchestrator - INFO - Creating pod test-pod-98a62ebf on PVC test-pvc-9dee4752 -2025-06-05 14:06:15,742 - tests.orchestrator - INFO - Created pod: test-pod-98a62ebf using PVC: test-pvc-9dee4752 -2025-06-05 14:06:15,742 - tests.orchestrator - INFO - Waiting for pod test-pod-98a62ebf to be ready -2025-06-05 14:07:15,768 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-98a62ebf to be ready after 60s -2025-06-05 14:07:15,769 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-98a62ebf to be ready -2025-06-05 14:07:15,769 - tests.orchestrator - INFO - Testing read/write between pods test-pod-d215a119 and test-pod-98a62ebf sharing PVC test-pvc-9dee4752 -2025-06-05 14:07:15,836 - tests.orchestrator - ERROR - Failed in read/write verification: (400) -Reason: Bad Request -HTTP response headers: HTTPHeaderDict({'Audit-Id': '0fd252b7-6182-4f97-9a5a-1503836973da', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 14:07:15 GMT', 'Content-Length': '166'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-d215a119 does not have a host assigned","reason":"BadRequest","code":400} - - -2025-06-05 14:07:15,836 - tests.orchestrator - INFO - Running specific scenario: _scenario_one_to_one -2025-06-05 14:07:15,836 - tests.orchestrator - INFO - Running scenario: One pod per PVC -2025-06-05 14:07:15,836 - tests.orchestrator - INFO - Creating 4 PVC-pod pairs -2025-06-05 14:07:15,913 - tests.orchestrator - INFO - Waiting for PVC one2one-71f9fc22 to be bound -2025-06-05 14:07:18,066 - tests.orchestrator - INFO - PVC one2one-71f9fc22 is bound -2025-06-05 14:07:18,066 - tests.orchestrator - INFO - Creating pod test-pod-78c97483 on PVC one2one-71f9fc22 -2025-06-05 14:07:18,197 - tests.orchestrator - INFO - Created pod: test-pod-78c97483 using PVC: one2one-71f9fc22 -2025-06-05 14:07:18,197 - tests.orchestrator - INFO - Waiting for pod test-pod-78c97483 to be ready -2025-06-05 14:08:18,229 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-78c97483 to be ready after 60s -2025-06-05 14:08:18,229 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-78c97483 to be ready -2025-06-05 14:08:18,310 - tests.orchestrator - INFO - Waiting for PVC one2one-d55f8841 to be bound -2025-06-05 14:08:20,442 - tests.orchestrator - INFO - PVC one2one-d55f8841 is bound -2025-06-05 14:08:20,442 - tests.orchestrator - INFO - Creating pod test-pod-a400277a on PVC one2one-d55f8841 -2025-06-05 14:08:20,518 - tests.orchestrator - INFO - Created pod: test-pod-a400277a using PVC: one2one-d55f8841 -2025-06-05 14:08:20,519 - tests.orchestrator - INFO - Waiting for pod test-pod-a400277a to be ready -2025-06-05 14:09:22,561 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a400277a to be ready after 60s -2025-06-05 14:09:22,561 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a400277a to be ready -2025-06-05 14:09:22,638 - tests.orchestrator - INFO - Waiting for PVC one2one-f3257be9 to be bound -2025-06-05 14:09:24,772 - tests.orchestrator - INFO - PVC one2one-f3257be9 is bound -2025-06-05 14:09:24,773 - tests.orchestrator - INFO - Creating pod test-pod-1bfbc8b3 on PVC one2one-f3257be9 -2025-06-05 14:09:24,850 - tests.orchestrator - INFO - Created pod: test-pod-1bfbc8b3 using PVC: one2one-f3257be9 -2025-06-05 14:09:24,850 - tests.orchestrator - INFO - Waiting for pod test-pod-1bfbc8b3 to be ready -2025-06-05 14:10:26,912 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1bfbc8b3 to be ready after 60s -2025-06-05 14:10:26,913 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1bfbc8b3 to be ready -2025-06-05 14:10:26,992 - tests.orchestrator - INFO - Waiting for PVC one2one-d2044d7d to be bound -2025-06-05 14:10:29,127 - tests.orchestrator - INFO - PVC one2one-d2044d7d is bound -2025-06-05 14:10:29,127 - tests.orchestrator - INFO - Creating pod test-pod-6edcdac8 on PVC one2one-d2044d7d -2025-06-05 14:10:29,203 - tests.orchestrator - INFO - Created pod: test-pod-6edcdac8 using PVC: one2one-d2044d7d -2025-06-05 14:10:29,203 - tests.orchestrator - INFO - Waiting for pod test-pod-6edcdac8 to be ready -2025-06-05 14:11:29,292 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-6edcdac8 to be ready after 60s -2025-06-05 14:11:29,293 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-6edcdac8 to be ready -2025-06-05 14:11:29,293 - tests.orchestrator - WARNING - Failed to create enough PVC-pod pairs, only created 0 -2025-06-05 14:11:29,293 - tests.orchestrator - INFO - Deleting pod: test-pod-54364ba6 from PVC: test-pvc-9dee4752 -2025-06-05 14:11:29,378 - tests.orchestrator - INFO - Waiting for pod test-pod-54364ba6 to be deleted -2025-06-05 14:11:29,443 - tests.orchestrator - INFO - Pod test-pod-54364ba6 has been deleted -2025-06-05 14:11:29,443 - tests.orchestrator - INFO - Deleted pod: test-pod-54364ba6 -2025-06-05 14:11:29,443 - tests.orchestrator - INFO - Deleting PVC: one2one-71f9fc22 -2025-06-05 14:11:29,443 - tests.orchestrator - INFO - Deleting 1 pods using PVC one2one-71f9fc22 -2025-06-05 14:11:29,443 - tests.orchestrator - INFO - Deleting pod: test-pod-78c97483 from PVC: one2one-71f9fc22 -2025-06-05 14:11:29,521 - tests.orchestrator - INFO - Waiting for pod test-pod-78c97483 to be deleted -2025-06-05 14:11:29,588 - tests.orchestrator - INFO - Pod test-pod-78c97483 has been deleted -2025-06-05 14:11:29,588 - tests.orchestrator - INFO - Deleted pod: test-pod-78c97483 -2025-06-05 14:11:29,666 - tests.orchestrator - INFO - Waiting for PVC one2one-71f9fc22 to be deleted -2025-06-05 14:11:29,733 - tests.orchestrator - INFO - PVC one2one-71f9fc22 has been deleted -2025-06-05 14:11:29,733 - tests.orchestrator - INFO - Deleted PVC: one2one-71f9fc22 -2025-06-05 14:11:29,733 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations -2025-06-05 14:11:29,733 - tests.orchestrator - INFO - Test completed in 440.95 seconds -2025-06-05 14:11:29,733 - tests.orchestrator - INFO - Cleaning up resources... -2025-06-05 14:11:29,733 - tests.orchestrator - INFO - Deleting pod: test-pod-d215a119 from PVC: test-pvc-9dee4752 -2025-06-05 14:11:29,832 - tests.orchestrator - INFO - Waiting for pod test-pod-d215a119 to be deleted -2025-06-05 14:11:29,897 - tests.orchestrator - INFO - Pod test-pod-d215a119 has been deleted -2025-06-05 14:11:29,897 - tests.orchestrator - INFO - Deleted pod: test-pod-d215a119 -2025-06-05 14:11:29,897 - tests.orchestrator - INFO - Deleting pod: test-pod-98a62ebf from PVC: test-pvc-9dee4752 -2025-06-05 14:11:29,977 - tests.orchestrator - INFO - Waiting for pod test-pod-98a62ebf to be deleted -2025-06-05 14:11:30,044 - tests.orchestrator - INFO - Pod test-pod-98a62ebf has been deleted -2025-06-05 14:11:30,044 - tests.orchestrator - INFO - Deleted pod: test-pod-98a62ebf -2025-06-05 14:11:30,044 - tests.orchestrator - INFO - Deleting pod: test-pod-a400277a from PVC: one2one-d55f8841 -2025-06-05 14:11:30,134 - tests.orchestrator - INFO - Waiting for pod test-pod-a400277a to be deleted -2025-06-05 14:11:30,199 - tests.orchestrator - INFO - Pod test-pod-a400277a has been deleted -2025-06-05 14:11:30,199 - tests.orchestrator - INFO - Deleted pod: test-pod-a400277a -2025-06-05 14:11:30,199 - tests.orchestrator - INFO - Deleting pod: test-pod-1bfbc8b3 from PVC: one2one-f3257be9 -2025-06-05 14:11:30,279 - tests.orchestrator - INFO - Waiting for pod test-pod-1bfbc8b3 to be deleted -2025-06-05 14:11:30,346 - tests.orchestrator - INFO - Pod test-pod-1bfbc8b3 has been deleted -2025-06-05 14:11:30,346 - tests.orchestrator - INFO - Deleted pod: test-pod-1bfbc8b3 -2025-06-05 14:11:30,346 - tests.orchestrator - INFO - Deleting pod: test-pod-6edcdac8 from PVC: one2one-d2044d7d -2025-06-05 14:11:30,432 - tests.orchestrator - INFO - Waiting for pod test-pod-6edcdac8 to be deleted -2025-06-05 14:11:30,500 - tests.orchestrator - INFO - Pod test-pod-6edcdac8 has been deleted -2025-06-05 14:11:30,500 - tests.orchestrator - INFO - Deleted pod: test-pod-6edcdac8 -2025-06-05 14:11:30,500 - tests.orchestrator - INFO - Deleting PVC: test-pvc-847a8a56 -2025-06-05 14:11:30,572 - tests.orchestrator - INFO - Waiting for PVC test-pvc-847a8a56 to be deleted -2025-06-05 14:11:30,642 - tests.orchestrator - INFO - PVC test-pvc-847a8a56 has been deleted -2025-06-05 14:11:30,642 - tests.orchestrator - INFO - Deleted PVC: test-pvc-847a8a56 -2025-06-05 14:11:30,642 - tests.orchestrator - INFO - Deleting PVC: test-pvc-9dee4752 -2025-06-05 14:11:30,715 - tests.orchestrator - INFO - Waiting for PVC test-pvc-9dee4752 to be deleted -2025-06-05 14:11:30,782 - tests.orchestrator - INFO - PVC test-pvc-9dee4752 has been deleted -2025-06-05 14:11:30,782 - tests.orchestrator - INFO - Deleted PVC: test-pvc-9dee4752 -2025-06-05 14:11:30,782 - tests.orchestrator - INFO - Deleting PVC: one2one-d55f8841 -2025-06-05 14:11:30,860 - tests.orchestrator - INFO - Waiting for PVC one2one-d55f8841 to be deleted -2025-06-05 14:11:30,927 - tests.orchestrator - INFO - PVC one2one-d55f8841 has been deleted -2025-06-05 14:11:30,927 - tests.orchestrator - INFO - Deleted PVC: one2one-d55f8841 -2025-06-05 14:11:30,928 - tests.orchestrator - INFO - Deleting PVC: one2one-f3257be9 -2025-06-05 14:11:30,999 - tests.orchestrator - INFO - Waiting for PVC one2one-f3257be9 to be deleted -2025-06-05 14:11:31,068 - tests.orchestrator - INFO - PVC one2one-f3257be9 has been deleted -2025-06-05 14:11:31,068 - tests.orchestrator - INFO - Deleted PVC: one2one-f3257be9 -2025-06-05 14:11:31,068 - tests.orchestrator - INFO - Deleting PVC: one2one-d2044d7d -2025-06-05 14:11:31,142 - tests.orchestrator - INFO - Waiting for PVC one2one-d2044d7d to be deleted -2025-06-05 14:11:31,208 - tests.orchestrator - INFO - PVC one2one-d2044d7d has been deleted -2025-06-05 14:11:31,208 - tests.orchestrator - INFO - Deleted PVC: one2one-d2044d7d -2025-06-05 14:11:31,208 - tests.orchestrator - INFO - Cleanup completed -2025-06-05 14:11:31,208 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-05 14:11:31,208 - tests.orchestrator - INFO - --- Operations --- -2025-06-05 14:11:31,209 - tests.orchestrator - INFO - create_pvc: 2 succeeded, 0 failed (100.0%) -2025-06-05 14:11:31,209 - tests.orchestrator - INFO - attach_pod: 7 succeeded, 0 failed (100.0%) -2025-06-05 14:11:31,209 - tests.orchestrator - INFO - delete_pod: 7 succeeded, 0 failed (100.0%) -2025-06-05 14:11:31,209 - tests.orchestrator - INFO - delete_pvc: 6 succeeded, 0 failed (100.0%) -2025-06-05 14:11:31,209 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) -2025-06-05 14:11:31,209 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) -2025-06-05 14:11:31,209 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-05 14:11:31,209 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-05 14:11:31,209 - tests.orchestrator - INFO - many_to_one: No runs -2025-06-05 14:11:31,209 - tests.orchestrator - INFO - one_to_one: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-05 14:11:31,209 - tests.orchestrator - INFO - concurrent_pvc: No runs -2025-06-05 14:11:31,209 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_141321.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_141321.log deleted file mode 100644 index cf6449873..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_141321.log +++ /dev/null @@ -1,7 +0,0 @@ -2025-06-05 14:13:21,602 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 14:13:21,814 - tests.orchestrator - ERROR - Error checking namespace: (401) -Reason: Unauthorized -HTTP response headers: HTTPHeaderDict({'Audit-Id': 'eb0b1f42-f08c-4b18-b778-a83d8eb055f7', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 14:13:21 GMT', 'Content-Length': '129'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} - - diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_141434.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_141434.log deleted file mode 100644 index 2964606c2..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_141434.log +++ /dev/null @@ -1,142 +0,0 @@ -2025-06-05 14:14:34,816 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 14:14:35,029 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 14:14:35,029 - tests.orchestrator - INFO - Starting orchestrator test for 60 seconds -2025-06-05 14:14:35,218 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 14:14:35,219 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 14:14:35,219 - tests.orchestrator - INFO - Creating PVC: test-pvc-0673a51e -2025-06-05 14:14:35,294 - tests.orchestrator - INFO - Created PVC: test-pvc-0673a51e -2025-06-05 14:14:35,295 - tests.orchestrator - INFO - Waiting for PVC test-pvc-0673a51e to be bound -2025-06-05 14:14:37,429 - tests.orchestrator - INFO - PVC test-pvc-0673a51e is bound -2025-06-05 14:14:37,429 - tests.orchestrator - INFO - Creating PVC: test-pvc-ad878159 -2025-06-05 14:14:37,497 - tests.orchestrator - INFO - Created PVC: test-pvc-ad878159 -2025-06-05 14:14:37,497 - tests.orchestrator - INFO - Waiting for PVC test-pvc-ad878159 to be bound -2025-06-05 14:14:39,631 - tests.orchestrator - INFO - PVC test-pvc-ad878159 is bound -2025-06-05 14:14:39,632 - tests.orchestrator - INFO - Creating pod test-pod-97d4c158 on PVC test-pvc-ad878159 -2025-06-05 14:14:39,709 - tests.orchestrator - INFO - Created pod: test-pod-97d4c158 using PVC: test-pvc-ad878159 -2025-06-05 14:14:39,710 - tests.orchestrator - INFO - Waiting for pod test-pod-97d4c158 to be ready -2025-06-05 14:15:39,755 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-97d4c158 to be ready after 60s -2025-06-05 14:15:39,755 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-97d4c158 to be ready -2025-06-05 14:15:39,756 - tests.orchestrator - INFO - Creating pod test-pod-27dfc967 on PVC test-pvc-ad878159 -2025-06-05 14:15:39,836 - tests.orchestrator - INFO - Created pod: test-pod-27dfc967 using PVC: test-pvc-ad878159 -2025-06-05 14:15:39,837 - tests.orchestrator - INFO - Waiting for pod test-pod-27dfc967 to be ready -2025-06-05 14:16:39,914 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-27dfc967 to be ready after 60s -2025-06-05 14:16:39,915 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-27dfc967 to be ready -2025-06-05 14:16:39,915 - tests.orchestrator - INFO - Creating pod test-pod-41d8fee0 on PVC test-pvc-0673a51e -2025-06-05 14:16:39,996 - tests.orchestrator - INFO - Created pod: test-pod-41d8fee0 using PVC: test-pvc-0673a51e -2025-06-05 14:16:39,996 - tests.orchestrator - INFO - Waiting for pod test-pod-41d8fee0 to be ready -2025-06-05 14:17:42,051 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-41d8fee0 to be ready after 60s -2025-06-05 14:17:42,051 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-41d8fee0 to be ready -2025-06-05 14:17:42,051 - tests.orchestrator - INFO - Testing read/write between pods test-pod-97d4c158 and test-pod-27dfc967 sharing PVC test-pvc-ad878159 -2025-06-05 14:17:42,118 - tests.orchestrator - ERROR - Failed in read/write verification: (400) -Reason: Bad Request -HTTP response headers: HTTPHeaderDict({'Audit-Id': '82f9299f-d78b-4a35-8447-4b9382646785', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 14:17:42 GMT', 'Content-Length': '166'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-97d4c158 does not have a host assigned","reason":"BadRequest","code":400} - - -2025-06-05 14:17:42,118 - tests.orchestrator - INFO - Running specific scenario: _scenario_concurrent_pvc -2025-06-05 14:17:42,118 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations -2025-06-05 14:17:42,118 - tests.orchestrator - INFO - Creating 5 PVCs in quick succession -2025-06-05 14:17:42,197 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-af92f77f -2025-06-05 14:17:42,197 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-af92f77f to be bound -2025-06-05 14:17:44,333 - tests.orchestrator - INFO - PVC concurrent-pvc-af92f77f is bound -2025-06-05 14:17:44,403 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-2a91a331 -2025-06-05 14:17:44,403 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-2a91a331 to be bound -2025-06-05 14:17:46,537 - tests.orchestrator - INFO - PVC concurrent-pvc-2a91a331 is bound -2025-06-05 14:17:46,606 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-b13c79e0 -2025-06-05 14:17:46,606 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-b13c79e0 to be bound -2025-06-05 14:17:48,740 - tests.orchestrator - INFO - PVC concurrent-pvc-b13c79e0 is bound -2025-06-05 14:17:48,810 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-2e52864c -2025-06-05 14:17:48,810 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-2e52864c to be bound -2025-06-05 14:17:50,943 - tests.orchestrator - INFO - PVC concurrent-pvc-2e52864c is bound -2025-06-05 14:17:51,013 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-83f53c18 -2025-06-05 14:17:51,014 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-83f53c18 to be bound -2025-06-05 14:17:53,147 - tests.orchestrator - INFO - PVC concurrent-pvc-83f53c18 is bound -2025-06-05 14:17:53,147 - tests.orchestrator - INFO - Successfully created 5 PVCs -2025-06-05 14:17:53,147 - tests.orchestrator - INFO - Creating pod test-pod-783e20b8 on PVC concurrent-pvc-2a91a331 -2025-06-05 14:17:53,221 - tests.orchestrator - INFO - Created pod: test-pod-783e20b8 using PVC: concurrent-pvc-2a91a331 -2025-06-05 14:17:53,221 - tests.orchestrator - INFO - Waiting for pod test-pod-783e20b8 to be ready -2025-06-05 14:18:53,279 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-783e20b8 to be ready after 60s -2025-06-05 14:18:53,279 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-783e20b8 to be ready -2025-06-05 14:18:53,279 - tests.orchestrator - INFO - Creating pod test-pod-206a8855 on PVC concurrent-pvc-b13c79e0 -2025-06-05 14:18:53,362 - tests.orchestrator - INFO - Created pod: test-pod-206a8855 using PVC: concurrent-pvc-b13c79e0 -2025-06-05 14:18:53,362 - tests.orchestrator - INFO - Waiting for pod test-pod-206a8855 to be ready -2025-06-05 14:19:55,415 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-206a8855 to be ready after 60s -2025-06-05 14:19:55,416 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-206a8855 to be ready -2025-06-05 14:19:55,416 - tests.orchestrator - INFO - Creating pod test-pod-7df5d0f2 on PVC concurrent-pvc-af92f77f -2025-06-05 14:19:55,499 - tests.orchestrator - INFO - Created pod: test-pod-7df5d0f2 using PVC: concurrent-pvc-af92f77f -2025-06-05 14:19:55,499 - tests.orchestrator - INFO - Waiting for pod test-pod-7df5d0f2 to be ready -2025-06-05 14:20:57,532 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-7df5d0f2 to be ready after 60s -2025-06-05 14:20:57,532 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-7df5d0f2 to be ready -2025-06-05 14:20:57,532 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-2e52864c -2025-06-05 14:20:57,607 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-2e52864c to be deleted -2025-06-05 14:20:57,673 - tests.orchestrator - INFO - PVC concurrent-pvc-2e52864c has been deleted -2025-06-05 14:20:57,674 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-2e52864c -2025-06-05 14:20:57,674 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-af92f77f -2025-06-05 14:20:57,675 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-af92f77f -2025-06-05 14:20:57,675 - tests.orchestrator - INFO - Deleting pod: test-pod-7df5d0f2 from PVC: concurrent-pvc-af92f77f -2025-06-05 14:20:57,761 - tests.orchestrator - INFO - Waiting for pod test-pod-7df5d0f2 to be deleted -2025-06-05 14:20:57,827 - tests.orchestrator - INFO - Pod test-pod-7df5d0f2 has been deleted -2025-06-05 14:20:57,827 - tests.orchestrator - INFO - Deleted pod: test-pod-7df5d0f2 -2025-06-05 14:20:57,901 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-af92f77f to be deleted -2025-06-05 14:20:57,969 - tests.orchestrator - INFO - PVC concurrent-pvc-af92f77f has been deleted -2025-06-05 14:20:57,969 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-af92f77f -2025-06-05 14:20:57,969 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-83f53c18 -2025-06-05 14:20:58,041 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-83f53c18 to be deleted -2025-06-05 14:20:58,108 - tests.orchestrator - INFO - PVC concurrent-pvc-83f53c18 has been deleted -2025-06-05 14:20:58,108 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-83f53c18 -2025-06-05 14:20:58,108 - tests.orchestrator - INFO - Rapid PVC scenario completed -2025-06-05 14:20:58,108 - tests.orchestrator - INFO - Deleting pod: test-pod-97d4c158 from PVC: test-pvc-ad878159 -2025-06-05 14:20:58,188 - tests.orchestrator - INFO - Waiting for pod test-pod-97d4c158 to be deleted -2025-06-05 14:20:58,254 - tests.orchestrator - INFO - Pod test-pod-97d4c158 has been deleted -2025-06-05 14:20:58,254 - tests.orchestrator - INFO - Deleted pod: test-pod-97d4c158 -2025-06-05 14:20:58,254 - tests.orchestrator - INFO - Deleting PVC: test-pvc-ad878159 -2025-06-05 14:20:58,255 - tests.orchestrator - INFO - Deleting 1 pods using PVC test-pvc-ad878159 -2025-06-05 14:20:58,255 - tests.orchestrator - INFO - Deleting pod: test-pod-27dfc967 from PVC: test-pvc-ad878159 -2025-06-05 14:20:58,341 - tests.orchestrator - INFO - Waiting for pod test-pod-27dfc967 to be deleted -2025-06-05 14:20:58,407 - tests.orchestrator - INFO - Pod test-pod-27dfc967 has been deleted -2025-06-05 14:20:58,407 - tests.orchestrator - INFO - Deleted pod: test-pod-27dfc967 -2025-06-05 14:20:58,479 - tests.orchestrator - INFO - Waiting for PVC test-pvc-ad878159 to be deleted -2025-06-05 14:20:58,546 - tests.orchestrator - INFO - PVC test-pvc-ad878159 has been deleted -2025-06-05 14:20:58,546 - tests.orchestrator - INFO - Deleted PVC: test-pvc-ad878159 -2025-06-05 14:20:58,546 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations -2025-06-05 14:20:58,546 - tests.orchestrator - INFO - Test completed in 383.52 seconds -2025-06-05 14:20:58,547 - tests.orchestrator - INFO - Cleaning up resources... -2025-06-05 14:20:58,547 - tests.orchestrator - INFO - Deleting pod: test-pod-41d8fee0 from PVC: test-pvc-0673a51e -2025-06-05 14:20:58,633 - tests.orchestrator - INFO - Waiting for pod test-pod-41d8fee0 to be deleted -2025-06-05 14:20:58,700 - tests.orchestrator - INFO - Pod test-pod-41d8fee0 has been deleted -2025-06-05 14:20:58,700 - tests.orchestrator - INFO - Deleted pod: test-pod-41d8fee0 -2025-06-05 14:20:58,700 - tests.orchestrator - INFO - Deleting pod: test-pod-783e20b8 from PVC: concurrent-pvc-2a91a331 -2025-06-05 14:20:58,781 - tests.orchestrator - INFO - Waiting for pod test-pod-783e20b8 to be deleted -2025-06-05 14:20:58,847 - tests.orchestrator - INFO - Pod test-pod-783e20b8 has been deleted -2025-06-05 14:20:58,847 - tests.orchestrator - INFO - Deleted pod: test-pod-783e20b8 -2025-06-05 14:20:58,847 - tests.orchestrator - INFO - Deleting pod: test-pod-206a8855 from PVC: concurrent-pvc-b13c79e0 -2025-06-05 14:20:58,925 - tests.orchestrator - INFO - Waiting for pod test-pod-206a8855 to be deleted -2025-06-05 14:20:58,991 - tests.orchestrator - INFO - Pod test-pod-206a8855 has been deleted -2025-06-05 14:20:58,992 - tests.orchestrator - INFO - Deleted pod: test-pod-206a8855 -2025-06-05 14:20:58,992 - tests.orchestrator - INFO - Deleting PVC: test-pvc-0673a51e -2025-06-05 14:20:59,063 - tests.orchestrator - INFO - Waiting for PVC test-pvc-0673a51e to be deleted -2025-06-05 14:20:59,130 - tests.orchestrator - INFO - PVC test-pvc-0673a51e has been deleted -2025-06-05 14:20:59,130 - tests.orchestrator - INFO - Deleted PVC: test-pvc-0673a51e -2025-06-05 14:20:59,130 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-2a91a331 -2025-06-05 14:20:59,204 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-2a91a331 to be deleted -2025-06-05 14:20:59,270 - tests.orchestrator - INFO - PVC concurrent-pvc-2a91a331 has been deleted -2025-06-05 14:20:59,270 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-2a91a331 -2025-06-05 14:20:59,270 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-b13c79e0 -2025-06-05 14:20:59,344 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-b13c79e0 to be deleted -2025-06-05 14:20:59,410 - tests.orchestrator - INFO - PVC concurrent-pvc-b13c79e0 has been deleted -2025-06-05 14:20:59,410 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-b13c79e0 -2025-06-05 14:20:59,410 - tests.orchestrator - INFO - Cleanup completed -2025-06-05 14:20:59,410 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-05 14:20:59,410 - tests.orchestrator - INFO - --- Operations --- -2025-06-05 14:20:59,410 - tests.orchestrator - INFO - create_pvc: 7 succeeded, 0 failed (100.0%) -2025-06-05 14:20:59,410 - tests.orchestrator - INFO - attach_pod: 6 succeeded, 0 failed (100.0%) -2025-06-05 14:20:59,410 - tests.orchestrator - INFO - delete_pod: 6 succeeded, 0 failed (100.0%) -2025-06-05 14:20:59,410 - tests.orchestrator - INFO - delete_pvc: 7 succeeded, 0 failed (100.0%) -2025-06-05 14:20:59,410 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) -2025-06-05 14:20:59,411 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) -2025-06-05 14:20:59,411 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-05 14:20:59,411 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-05 14:20:59,411 - tests.orchestrator - INFO - many_to_one: No runs -2025-06-05 14:20:59,411 - tests.orchestrator - INFO - one_to_one: No runs -2025-06-05 14:20:59,411 - tests.orchestrator - INFO - concurrent_pvc: 1 succeeded, 0 failed out of 1 runs (100.0%) -2025-06-05 14:20:59,411 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_143926.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_143926.log deleted file mode 100644 index 35607442a..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_143926.log +++ /dev/null @@ -1,73 +0,0 @@ -2025-06-05 14:39:26,410 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 14:39:26,657 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 14:39:26,657 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-05 14:39:26,852 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 14:39:26,853 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 14:39:26,853 - tests.orchestrator - INFO - Creating PVC: test-pvc-e5985cb1 -2025-06-05 14:39:26,928 - tests.orchestrator - INFO - Created PVC: test-pvc-e5985cb1 -2025-06-05 14:39:26,928 - tests.orchestrator - INFO - Waiting for PVC test-pvc-e5985cb1 to be bound -2025-06-05 14:39:29,061 - tests.orchestrator - INFO - PVC test-pvc-e5985cb1 is bound -2025-06-05 14:39:29,061 - tests.orchestrator - INFO - Creating PVC: test-pvc-83426f91 -2025-06-05 14:39:29,130 - tests.orchestrator - INFO - Created PVC: test-pvc-83426f91 -2025-06-05 14:39:29,131 - tests.orchestrator - INFO - Waiting for PVC test-pvc-83426f91 to be bound -2025-06-05 14:39:31,262 - tests.orchestrator - INFO - PVC test-pvc-83426f91 is bound -2025-06-05 14:39:31,263 - tests.orchestrator - INFO - Creating pod test-pod-d86424ba on PVC test-pvc-83426f91 -2025-06-05 14:39:31,342 - tests.orchestrator - INFO - Created pod: test-pod-d86424ba using PVC: test-pvc-83426f91 -2025-06-05 14:39:31,342 - tests.orchestrator - INFO - Waiting for pod test-pod-d86424ba to be ready -2025-06-05 14:40:31,371 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-d86424ba to be ready after 60s -2025-06-05 14:40:31,371 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-d86424ba to be ready -2025-06-05 14:40:31,371 - tests.orchestrator - INFO - Creating pod test-pod-b7e9d0a0 on PVC test-pvc-e5985cb1 -2025-06-05 14:40:31,454 - tests.orchestrator - INFO - Created pod: test-pod-b7e9d0a0 using PVC: test-pvc-e5985cb1 -2025-06-05 14:40:31,455 - tests.orchestrator - INFO - Waiting for pod test-pod-b7e9d0a0 to be ready -2025-06-05 14:41:31,485 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b7e9d0a0 to be ready after 60s -2025-06-05 14:41:31,485 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b7e9d0a0 to be ready -2025-06-05 14:41:31,485 - tests.orchestrator - INFO - Creating pod test-pod-c947d819 on PVC test-pvc-e5985cb1 -2025-06-05 14:41:31,586 - tests.orchestrator - INFO - Created pod: test-pod-c947d819 using PVC: test-pvc-e5985cb1 -2025-06-05 14:41:31,586 - tests.orchestrator - INFO - Waiting for pod test-pod-c947d819 to be ready -2025-06-05 14:42:33,647 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-c947d819 to be ready after 60s -2025-06-05 14:42:33,647 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-c947d819 to be ready -2025-06-05 14:42:33,647 - tests.orchestrator - INFO - Testing read/write between pods test-pod-b7e9d0a0 and test-pod-c947d819 sharing PVC test-pvc-e5985cb1 -2025-06-05 14:42:33,714 - tests.orchestrator - ERROR - Failed in read/write verification: (400) -Reason: Bad Request -HTTP response headers: HTTPHeaderDict({'Audit-Id': '8b1165bc-48b9-433e-8654-68c6095da9bd', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 14:42:33 GMT', 'Content-Length': '166'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-b7e9d0a0 does not have a host assigned","reason":"BadRequest","code":400} - - -2025-06-05 14:42:33,714 - tests.orchestrator - INFO - Running specific scenario: _scenario_concurrent_pvc -2025-06-05 14:42:33,714 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations -2025-06-05 14:42:33,714 - tests.orchestrator - INFO - Creating 7 PVCs in quick succession -2025-06-05 14:42:33,792 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-f6a663b2 -2025-06-05 14:42:33,792 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-f6a663b2 to be bound -2025-06-05 14:42:35,924 - tests.orchestrator - INFO - PVC concurrent-pvc-f6a663b2 is bound -2025-06-05 14:42:35,993 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-903eb13a -2025-06-05 14:42:35,994 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-903eb13a to be bound -2025-06-05 14:42:38,128 - tests.orchestrator - INFO - PVC concurrent-pvc-903eb13a is bound -2025-06-05 14:42:38,196 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-ba7e5e4d -2025-06-05 14:42:38,196 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-ba7e5e4d to be bound -2025-06-05 14:42:40,331 - tests.orchestrator - INFO - PVC concurrent-pvc-ba7e5e4d is bound -2025-06-05 14:42:40,397 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-5b29e9cd -2025-06-05 14:42:40,397 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-5b29e9cd to be bound -2025-06-05 14:42:42,533 - tests.orchestrator - INFO - PVC concurrent-pvc-5b29e9cd is bound -2025-06-05 14:42:42,603 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-248d7584 -2025-06-05 14:42:42,603 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-248d7584 to be bound -2025-06-05 14:42:44,740 - tests.orchestrator - INFO - PVC concurrent-pvc-248d7584 is bound -2025-06-05 14:42:44,808 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-0aa045b0 -2025-06-05 14:42:44,808 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-0aa045b0 to be bound -2025-06-05 14:42:46,944 - tests.orchestrator - INFO - PVC concurrent-pvc-0aa045b0 is bound -2025-06-05 14:42:47,013 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-b0004a21 -2025-06-05 14:42:47,013 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-b0004a21 to be bound -2025-06-05 14:42:49,147 - tests.orchestrator - INFO - PVC concurrent-pvc-b0004a21 is bound -2025-06-05 14:42:49,148 - tests.orchestrator - INFO - Successfully created 7 PVCs -2025-06-05 14:42:49,148 - tests.orchestrator - INFO - Creating pod test-pod-edd264c5 on PVC concurrent-pvc-0aa045b0 -2025-06-05 14:42:49,220 - tests.orchestrator - INFO - Created pod: test-pod-edd264c5 using PVC: concurrent-pvc-0aa045b0 -2025-06-05 14:42:49,220 - tests.orchestrator - INFO - Waiting for pod test-pod-edd264c5 to be ready -2025-06-05 14:43:49,221 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-edd264c5 to be ready after 60s -2025-06-05 14:43:49,221 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-edd264c5 to be ready -2025-06-05 14:43:49,221 - tests.orchestrator - INFO - Creating pod test-pod-9f0f38ef on PVC concurrent-pvc-248d7584 -2025-06-05 14:43:49,306 - tests.orchestrator - INFO - Created pod: test-pod-9f0f38ef using PVC: concurrent-pvc-248d7584 -2025-06-05 14:43:49,306 - tests.orchestrator - INFO - Waiting for pod test-pod-9f0f38ef to be ready -2025-06-05 14:44:51,350 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9f0f38ef to be ready after 60s -2025-06-05 14:44:51,351 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9f0f38ef to be ready -2025-06-05 14:44:51,351 - tests.orchestrator - INFO - Creating pod test-pod-6bde4264 on PVC concurrent-pvc-903eb13a -2025-06-05 14:44:51,437 - tests.orchestrator - INFO - Created pod: test-pod-6bde4264 using PVC: concurrent-pvc-903eb13a -2025-06-05 14:44:51,437 - tests.orchestrator - INFO - Waiting for pod test-pod-6bde4264 to be ready diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_145133.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_145133.log deleted file mode 100644 index a89fdcdab..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_145133.log +++ /dev/null @@ -1,111 +0,0 @@ -2025-06-05 14:51:33,341 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 14:51:33,563 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 14:51:33,563 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-05 14:51:33,753 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 14:51:33,753 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 14:51:33,754 - tests.orchestrator - INFO - Creating PVC: test-pvc-49392d7c -2025-06-05 14:51:33,831 - tests.orchestrator - INFO - Created PVC: test-pvc-49392d7c -2025-06-05 14:51:33,831 - tests.orchestrator - INFO - Waiting for PVC test-pvc-49392d7c to be bound -2025-06-05 14:51:35,964 - tests.orchestrator - INFO - PVC test-pvc-49392d7c is bound -2025-06-05 14:51:35,966 - tests.orchestrator - INFO - Creating PVC: test-pvc-5e912838 -2025-06-05 14:51:36,038 - tests.orchestrator - INFO - Created PVC: test-pvc-5e912838 -2025-06-05 14:51:36,038 - tests.orchestrator - INFO - Waiting for PVC test-pvc-5e912838 to be bound -2025-06-05 14:51:38,171 - tests.orchestrator - INFO - PVC test-pvc-5e912838 is bound -2025-06-05 14:51:38,171 - tests.orchestrator - INFO - Creating pod test-pod-a16fe7e2 on PVC test-pvc-5e912838 -2025-06-05 14:51:38,249 - tests.orchestrator - INFO - Created pod: test-pod-a16fe7e2 using PVC: test-pvc-5e912838 -2025-06-05 14:51:38,250 - tests.orchestrator - INFO - Waiting for pod test-pod-a16fe7e2 to be ready -2025-06-05 14:52:40,287 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a16fe7e2 to be ready after 60s -2025-06-05 14:52:40,287 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a16fe7e2 to be ready -2025-06-05 14:52:40,288 - tests.orchestrator - INFO - Creating pod test-pod-6cc653e7 on PVC test-pvc-49392d7c -2025-06-05 14:52:40,370 - tests.orchestrator - INFO - Created pod: test-pod-6cc653e7 using PVC: test-pvc-49392d7c -2025-06-05 14:52:40,370 - tests.orchestrator - INFO - Waiting for pod test-pod-6cc653e7 to be ready -2025-06-05 14:53:42,411 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-6cc653e7 to be ready after 60s -2025-06-05 14:53:42,411 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-6cc653e7 to be ready -2025-06-05 14:53:42,411 - tests.orchestrator - INFO - Creating pod test-pod-807f64e5 on PVC test-pvc-49392d7c -2025-06-05 14:53:42,494 - tests.orchestrator - INFO - Created pod: test-pod-807f64e5 using PVC: test-pvc-49392d7c -2025-06-05 14:53:42,494 - tests.orchestrator - INFO - Waiting for pod test-pod-807f64e5 to be ready -2025-06-05 14:54:44,529 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-807f64e5 to be ready after 60s -2025-06-05 14:54:44,529 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-807f64e5 to be ready -2025-06-05 14:54:44,529 - tests.orchestrator - INFO - Testing read/write between pods test-pod-807f64e5 and test-pod-6cc653e7 sharing PVC test-pvc-49392d7c -2025-06-05 14:54:44,596 - tests.orchestrator - ERROR - Failed in read/write verification: (400) -Reason: Bad Request -HTTP response headers: HTTPHeaderDict({'Audit-Id': '12fe6240-5ebe-4564-8452-9306b897c940', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 14:54:44 GMT', 'Content-Length': '166'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-807f64e5 does not have a host assigned","reason":"BadRequest","code":400} - - -2025-06-05 14:54:44,596 - tests.orchestrator - INFO - Running specific scenario: _scenario_many_to_one -2025-06-05 14:54:44,597 - tests.orchestrator - INFO - Running scenario: Many pods to one PVC -2025-06-05 14:54:44,678 - tests.orchestrator - INFO - Waiting for PVC many2one-7a10cfd9 to be bound -2025-06-05 14:54:46,810 - tests.orchestrator - INFO - PVC many2one-7a10cfd9 is bound -2025-06-05 14:54:46,811 - tests.orchestrator - INFO - Creating 3 pods for the same PVC -2025-06-05 14:54:46,811 - tests.orchestrator - INFO - Creating pod test-pod-6d648a79 on PVC many2one-7a10cfd9 -2025-06-05 14:54:46,884 - tests.orchestrator - INFO - Created pod: test-pod-6d648a79 using PVC: many2one-7a10cfd9 -2025-06-05 14:54:46,884 - tests.orchestrator - INFO - Waiting for pod test-pod-6d648a79 to be ready -2025-06-05 14:55:48,914 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-6d648a79 to be ready after 60s -2025-06-05 14:55:48,915 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-6d648a79 to be ready -2025-06-05 14:55:48,915 - tests.orchestrator - INFO - Creating pod test-pod-264dcb36 on PVC many2one-7a10cfd9 -2025-06-05 14:55:48,997 - tests.orchestrator - INFO - Created pod: test-pod-264dcb36 using PVC: many2one-7a10cfd9 -2025-06-05 14:55:48,997 - tests.orchestrator - INFO - Waiting for pod test-pod-264dcb36 to be ready -2025-06-05 14:56:49,023 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-264dcb36 to be ready after 60s -2025-06-05 14:56:49,024 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-264dcb36 to be ready -2025-06-05 14:56:49,024 - tests.orchestrator - INFO - Creating pod test-pod-b6a62203 on PVC many2one-7a10cfd9 -2025-06-05 14:56:49,106 - tests.orchestrator - INFO - Created pod: test-pod-b6a62203 using PVC: many2one-7a10cfd9 -2025-06-05 14:56:49,106 - tests.orchestrator - INFO - Waiting for pod test-pod-b6a62203 to be ready -2025-06-05 14:57:49,129 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b6a62203 to be ready after 60s -2025-06-05 14:57:49,129 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b6a62203 to be ready -2025-06-05 14:57:49,129 - tests.orchestrator - WARNING - Failed to create enough pods for scenario, only created 0 -2025-06-05 14:57:49,130 - tests.orchestrator - INFO - Deleting pod: test-pod-6cc653e7 from PVC: test-pvc-49392d7c -2025-06-05 14:57:49,207 - tests.orchestrator - INFO - Waiting for pod test-pod-6cc653e7 to be deleted -2025-06-05 14:57:49,272 - tests.orchestrator - INFO - Pod test-pod-6cc653e7 has been deleted -2025-06-05 14:57:49,272 - tests.orchestrator - INFO - Deleted pod: test-pod-6cc653e7 -2025-06-05 14:57:49,272 - tests.orchestrator - INFO - Deleting PVC: many2one-7a10cfd9 -2025-06-05 14:57:49,272 - tests.orchestrator - INFO - Deleting 3 pods using PVC many2one-7a10cfd9 -2025-06-05 14:57:49,272 - tests.orchestrator - INFO - Deleting pod: test-pod-6d648a79 from PVC: many2one-7a10cfd9 -2025-06-05 14:57:49,348 - tests.orchestrator - INFO - Waiting for pod test-pod-6d648a79 to be deleted -2025-06-05 14:57:49,414 - tests.orchestrator - INFO - Pod test-pod-6d648a79 has been deleted -2025-06-05 14:57:49,415 - tests.orchestrator - INFO - Deleted pod: test-pod-6d648a79 -2025-06-05 14:57:49,415 - tests.orchestrator - INFO - Deleting pod: test-pod-264dcb36 from PVC: many2one-7a10cfd9 -2025-06-05 14:57:49,498 - tests.orchestrator - INFO - Waiting for pod test-pod-264dcb36 to be deleted -2025-06-05 14:57:49,563 - tests.orchestrator - INFO - Pod test-pod-264dcb36 has been deleted -2025-06-05 14:57:49,564 - tests.orchestrator - INFO - Deleted pod: test-pod-264dcb36 -2025-06-05 14:57:49,564 - tests.orchestrator - INFO - Deleting pod: test-pod-b6a62203 from PVC: many2one-7a10cfd9 -2025-06-05 14:57:49,643 - tests.orchestrator - INFO - Waiting for pod test-pod-b6a62203 to be deleted -2025-06-05 14:57:49,710 - tests.orchestrator - INFO - Pod test-pod-b6a62203 has been deleted -2025-06-05 14:57:49,710 - tests.orchestrator - INFO - Deleted pod: test-pod-b6a62203 -2025-06-05 14:57:49,782 - tests.orchestrator - INFO - Waiting for PVC many2one-7a10cfd9 to be deleted -2025-06-05 14:57:49,848 - tests.orchestrator - INFO - PVC many2one-7a10cfd9 has been deleted -2025-06-05 14:57:49,848 - tests.orchestrator - INFO - Deleted PVC: many2one-7a10cfd9 -2025-06-05 14:57:49,848 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations -2025-06-05 14:57:49,849 - tests.orchestrator - INFO - Test completed in 376.28 seconds -2025-06-05 14:57:49,849 - tests.orchestrator - INFO - Cleaning up resources... -2025-06-05 14:57:49,849 - tests.orchestrator - INFO - Deleting pod: test-pod-807f64e5 from PVC: test-pvc-49392d7c -2025-06-05 14:57:49,935 - tests.orchestrator - INFO - Waiting for pod test-pod-807f64e5 to be deleted -2025-06-05 14:57:50,000 - tests.orchestrator - INFO - Pod test-pod-807f64e5 has been deleted -2025-06-05 14:57:50,001 - tests.orchestrator - INFO - Deleted pod: test-pod-807f64e5 -2025-06-05 14:57:50,001 - tests.orchestrator - INFO - Deleting pod: test-pod-a16fe7e2 from PVC: test-pvc-5e912838 -2025-06-05 14:57:50,081 - tests.orchestrator - INFO - Waiting for pod test-pod-a16fe7e2 to be deleted -2025-06-05 14:57:50,147 - tests.orchestrator - INFO - Pod test-pod-a16fe7e2 has been deleted -2025-06-05 14:57:50,147 - tests.orchestrator - INFO - Deleted pod: test-pod-a16fe7e2 -2025-06-05 14:57:50,147 - tests.orchestrator - INFO - Deleting PVC: test-pvc-49392d7c -2025-06-05 14:57:50,221 - tests.orchestrator - INFO - Waiting for PVC test-pvc-49392d7c to be deleted -2025-06-05 14:57:50,287 - tests.orchestrator - INFO - PVC test-pvc-49392d7c has been deleted -2025-06-05 14:57:50,288 - tests.orchestrator - INFO - Deleted PVC: test-pvc-49392d7c -2025-06-05 14:57:50,288 - tests.orchestrator - INFO - Deleting PVC: test-pvc-5e912838 -2025-06-05 14:57:50,360 - tests.orchestrator - INFO - Waiting for PVC test-pvc-5e912838 to be deleted -2025-06-05 14:57:50,426 - tests.orchestrator - INFO - PVC test-pvc-5e912838 has been deleted -2025-06-05 14:57:50,426 - tests.orchestrator - INFO - Deleted PVC: test-pvc-5e912838 -2025-06-05 14:57:50,426 - tests.orchestrator - INFO - Cleanup completed -2025-06-05 14:57:50,426 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-05 14:57:50,426 - tests.orchestrator - INFO - --- Operations --- -2025-06-05 14:57:50,426 - tests.orchestrator - INFO - create_pvc: 2 succeeded, 0 failed (100.0%) -2025-06-05 14:57:50,426 - tests.orchestrator - INFO - attach_pod: 6 succeeded, 0 failed (100.0%) -2025-06-05 14:57:50,427 - tests.orchestrator - INFO - delete_pod: 6 succeeded, 0 failed (100.0%) -2025-06-05 14:57:50,427 - tests.orchestrator - INFO - delete_pvc: 3 succeeded, 0 failed (100.0%) -2025-06-05 14:57:50,427 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) -2025-06-05 14:57:50,427 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) -2025-06-05 14:57:50,427 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-05 14:57:50,427 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-05 14:57:50,427 - tests.orchestrator - INFO - many_to_one: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-05 14:57:50,427 - tests.orchestrator - INFO - one_to_one: No runs -2025-06-05 14:57:50,427 - tests.orchestrator - INFO - concurrent_pvc: No runs -2025-06-05 14:57:50,427 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_145937.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_145937.log deleted file mode 100644 index 5711d3fb8..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_145937.log +++ /dev/null @@ -1,134 +0,0 @@ -2025-06-05 14:59:37,825 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 14:59:38,037 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 14:59:38,038 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-05 14:59:38,227 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 14:59:38,227 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 14:59:38,227 - tests.orchestrator - INFO - Creating PVC: test-pvc-fe252ee6 -2025-06-05 14:59:38,302 - tests.orchestrator - INFO - Created PVC: test-pvc-fe252ee6 -2025-06-05 14:59:38,303 - tests.orchestrator - INFO - Waiting for PVC test-pvc-fe252ee6 to be bound -2025-06-05 14:59:40,435 - tests.orchestrator - INFO - PVC test-pvc-fe252ee6 is bound -2025-06-05 14:59:40,435 - tests.orchestrator - INFO - Creating PVC: test-pvc-4a7f89b4 -2025-06-05 14:59:40,502 - tests.orchestrator - INFO - Created PVC: test-pvc-4a7f89b4 -2025-06-05 14:59:40,502 - tests.orchestrator - INFO - Waiting for PVC test-pvc-4a7f89b4 to be bound -2025-06-05 14:59:42,634 - tests.orchestrator - INFO - PVC test-pvc-4a7f89b4 is bound -2025-06-05 14:59:42,635 - tests.orchestrator - INFO - Creating pod test-pod-5311b1cc on PVC test-pvc-4a7f89b4 -2025-06-05 14:59:42,712 - tests.orchestrator - INFO - Created pod: test-pod-5311b1cc using PVC: test-pvc-4a7f89b4 -2025-06-05 14:59:42,712 - tests.orchestrator - INFO - Waiting for pod test-pod-5311b1cc to be ready -2025-06-05 15:00:44,719 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-5311b1cc to be ready after 60s -2025-06-05 15:00:44,719 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-5311b1cc to be ready -2025-06-05 15:00:44,720 - tests.orchestrator - INFO - Creating pod test-pod-71d7f585 on PVC test-pvc-fe252ee6 -2025-06-05 15:00:44,803 - tests.orchestrator - INFO - Created pod: test-pod-71d7f585 using PVC: test-pvc-fe252ee6 -2025-06-05 15:00:44,803 - tests.orchestrator - INFO - Waiting for pod test-pod-71d7f585 to be ready -2025-06-05 15:01:46,811 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-71d7f585 to be ready after 60s -2025-06-05 15:01:46,811 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-71d7f585 to be ready -2025-06-05 15:01:46,812 - tests.orchestrator - INFO - Creating pod test-pod-cc8d2266 on PVC test-pvc-4a7f89b4 -2025-06-05 15:01:46,893 - tests.orchestrator - INFO - Created pod: test-pod-cc8d2266 using PVC: test-pvc-4a7f89b4 -2025-06-05 15:01:46,894 - tests.orchestrator - INFO - Waiting for pod test-pod-cc8d2266 to be ready -2025-06-05 15:02:48,905 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-cc8d2266 to be ready after 60s -2025-06-05 15:02:48,905 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-cc8d2266 to be ready -2025-06-05 15:02:48,905 - tests.orchestrator - INFO - Testing read/write between pods test-pod-5311b1cc and test-pod-cc8d2266 sharing PVC test-pvc-4a7f89b4 -2025-06-05 15:02:48,971 - tests.orchestrator - ERROR - Failed in read/write verification: (400) -Reason: Bad Request -HTTP response headers: HTTPHeaderDict({'Audit-Id': '6fd01cb2-cb35-43e3-8bf2-4cba5eccfd8d', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 15:02:48 GMT', 'Content-Length': '166'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-5311b1cc does not have a host assigned","reason":"BadRequest","code":400} - - -2025-06-05 15:02:48,971 - tests.orchestrator - INFO - ============================================================ -2025-06-05 15:02:48,971 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_concurrent_pvc -2025-06-05 15:02:48,971 - tests.orchestrator - INFO - ============================================================ -2025-06-05 15:02:48,972 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations -2025-06-05 15:02:48,972 - tests.orchestrator - INFO - Creating 3 PVCs in quick succession -2025-06-05 15:02:49,049 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-02666bec -2025-06-05 15:02:49,049 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-02666bec to be bound -2025-06-05 15:02:51,181 - tests.orchestrator - INFO - PVC concurrent-pvc-02666bec is bound -2025-06-05 15:02:51,248 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-6592afb4 -2025-06-05 15:02:51,249 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-6592afb4 to be bound -2025-06-05 15:02:53,380 - tests.orchestrator - INFO - PVC concurrent-pvc-6592afb4 is bound -2025-06-05 15:02:53,448 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-2386ec8a -2025-06-05 15:02:53,449 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-2386ec8a to be bound -2025-06-05 15:02:55,580 - tests.orchestrator - INFO - PVC concurrent-pvc-2386ec8a is bound -2025-06-05 15:02:55,580 - tests.orchestrator - INFO - Successfully created 3 PVCs -2025-06-05 15:02:55,580 - tests.orchestrator - INFO - Creating pod test-pod-27ffffcb on PVC concurrent-pvc-2386ec8a -2025-06-05 15:02:55,653 - tests.orchestrator - INFO - Created pod: test-pod-27ffffcb using PVC: concurrent-pvc-2386ec8a -2025-06-05 15:02:55,653 - tests.orchestrator - INFO - Waiting for pod test-pod-27ffffcb to be ready -2025-06-05 15:03:57,662 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-27ffffcb to be ready after 60s -2025-06-05 15:03:57,662 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-27ffffcb to be ready -2025-06-05 15:03:57,662 - tests.orchestrator - INFO - Creating pod test-pod-00a03fb8 on PVC concurrent-pvc-02666bec -2025-06-05 15:03:57,743 - tests.orchestrator - INFO - Created pod: test-pod-00a03fb8 using PVC: concurrent-pvc-02666bec -2025-06-05 15:03:57,744 - tests.orchestrator - INFO - Waiting for pod test-pod-00a03fb8 to be ready -2025-06-05 15:04:59,752 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-00a03fb8 to be ready after 60s -2025-06-05 15:04:59,753 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-00a03fb8 to be ready -2025-06-05 15:04:59,753 - tests.orchestrator - INFO - Creating pod test-pod-87658b4a on PVC concurrent-pvc-6592afb4 -2025-06-05 15:04:59,832 - tests.orchestrator - INFO - Created pod: test-pod-87658b4a using PVC: concurrent-pvc-6592afb4 -2025-06-05 15:04:59,832 - tests.orchestrator - INFO - Waiting for pod test-pod-87658b4a to be ready -2025-06-05 15:06:01,847 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-87658b4a to be ready after 60s -2025-06-05 15:06:01,847 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-87658b4a to be ready -2025-06-05 15:06:01,847 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-6592afb4 -2025-06-05 15:06:01,847 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-6592afb4 -2025-06-05 15:06:01,847 - tests.orchestrator - INFO - Deleting pod: test-pod-87658b4a from PVC: concurrent-pvc-6592afb4 -2025-06-05 15:06:01,968 - tests.orchestrator - INFO - Waiting for pod test-pod-87658b4a to be deleted -2025-06-05 15:06:02,040 - tests.orchestrator - INFO - Pod test-pod-87658b4a has been deleted -2025-06-05 15:06:02,041 - tests.orchestrator - INFO - Deleted pod: test-pod-87658b4a -2025-06-05 15:06:02,126 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-6592afb4 to be deleted -2025-06-05 15:06:02,196 - tests.orchestrator - INFO - PVC concurrent-pvc-6592afb4 has been deleted -2025-06-05 15:06:02,196 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-6592afb4 -2025-06-05 15:06:02,196 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-02666bec -2025-06-05 15:06:02,196 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-02666bec -2025-06-05 15:06:02,196 - tests.orchestrator - INFO - Deleting pod: test-pod-00a03fb8 from PVC: concurrent-pvc-02666bec -2025-06-05 15:06:02,295 - tests.orchestrator - INFO - Waiting for pod test-pod-00a03fb8 to be deleted -2025-06-05 15:06:02,359 - tests.orchestrator - INFO - Pod test-pod-00a03fb8 has been deleted -2025-06-05 15:06:02,360 - tests.orchestrator - INFO - Deleted pod: test-pod-00a03fb8 -2025-06-05 15:06:02,433 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-02666bec to be deleted -2025-06-05 15:06:02,497 - tests.orchestrator - INFO - PVC concurrent-pvc-02666bec has been deleted -2025-06-05 15:06:02,498 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-02666bec -2025-06-05 15:06:02,498 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-2386ec8a -2025-06-05 15:06:02,498 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-2386ec8a -2025-06-05 15:06:02,498 - tests.orchestrator - INFO - Deleting pod: test-pod-27ffffcb from PVC: concurrent-pvc-2386ec8a -2025-06-05 15:06:02,575 - tests.orchestrator - INFO - Waiting for pod test-pod-27ffffcb to be deleted -2025-06-05 15:06:02,640 - tests.orchestrator - INFO - Pod test-pod-27ffffcb has been deleted -2025-06-05 15:06:02,640 - tests.orchestrator - INFO - Deleted pod: test-pod-27ffffcb -2025-06-05 15:06:02,713 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-2386ec8a to be deleted -2025-06-05 15:06:02,777 - tests.orchestrator - INFO - PVC concurrent-pvc-2386ec8a has been deleted -2025-06-05 15:06:02,777 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-2386ec8a -2025-06-05 15:06:02,778 - tests.orchestrator - INFO - Rapid PVC scenario completed -2025-06-05 15:06:02,778 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_concurrent_pvc -2025-06-05 15:06:02,778 - tests.orchestrator - INFO - ------------------------------------------------------------ -2025-06-05 15:06:02,778 - tests.orchestrator - INFO - Deleting pod: test-pod-71d7f585 from PVC: test-pvc-fe252ee6 -2025-06-05 15:06:02,870 - tests.orchestrator - INFO - Waiting for pod test-pod-71d7f585 to be deleted -2025-06-05 15:06:02,935 - tests.orchestrator - INFO - Pod test-pod-71d7f585 has been deleted -2025-06-05 15:06:02,935 - tests.orchestrator - INFO - Deleted pod: test-pod-71d7f585 -2025-06-05 15:06:02,935 - tests.orchestrator - INFO - Deleting PVC: test-pvc-4a7f89b4 -2025-06-05 15:06:02,935 - tests.orchestrator - INFO - Deleting 2 pods using PVC test-pvc-4a7f89b4 -2025-06-05 15:06:02,935 - tests.orchestrator - INFO - Deleting pod: test-pod-5311b1cc from PVC: test-pvc-4a7f89b4 -2025-06-05 15:06:03,025 - tests.orchestrator - INFO - Waiting for pod test-pod-5311b1cc to be deleted -2025-06-05 15:06:03,090 - tests.orchestrator - INFO - Pod test-pod-5311b1cc has been deleted -2025-06-05 15:06:03,090 - tests.orchestrator - INFO - Deleted pod: test-pod-5311b1cc -2025-06-05 15:06:03,090 - tests.orchestrator - INFO - Deleting pod: test-pod-cc8d2266 from PVC: test-pvc-4a7f89b4 -2025-06-05 15:06:03,172 - tests.orchestrator - INFO - Waiting for pod test-pod-cc8d2266 to be deleted -2025-06-05 15:06:03,238 - tests.orchestrator - INFO - Pod test-pod-cc8d2266 has been deleted -2025-06-05 15:06:03,238 - tests.orchestrator - INFO - Deleted pod: test-pod-cc8d2266 -2025-06-05 15:06:03,311 - tests.orchestrator - INFO - Waiting for PVC test-pvc-4a7f89b4 to be deleted -2025-06-05 15:06:03,376 - tests.orchestrator - INFO - PVC test-pvc-4a7f89b4 has been deleted -2025-06-05 15:06:03,376 - tests.orchestrator - INFO - Deleted PVC: test-pvc-4a7f89b4 -2025-06-05 15:06:03,376 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations -2025-06-05 15:06:03,376 - tests.orchestrator - INFO - Test completed in 385.34 seconds -2025-06-05 15:06:03,376 - tests.orchestrator - INFO - Cleaning up resources... -2025-06-05 15:06:03,376 - tests.orchestrator - INFO - Deleting PVC: test-pvc-fe252ee6 -2025-06-05 15:06:03,447 - tests.orchestrator - INFO - Waiting for PVC test-pvc-fe252ee6 to be deleted -2025-06-05 15:06:03,512 - tests.orchestrator - INFO - PVC test-pvc-fe252ee6 has been deleted -2025-06-05 15:06:03,512 - tests.orchestrator - INFO - Deleted PVC: test-pvc-fe252ee6 -2025-06-05 15:06:03,512 - tests.orchestrator - INFO - Cleanup completed -2025-06-05 15:06:03,512 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-05 15:06:03,513 - tests.orchestrator - INFO - --- Operations --- -2025-06-05 15:06:03,513 - tests.orchestrator - INFO - create_pvc: 5 succeeded, 0 failed (100.0%) -2025-06-05 15:06:03,513 - tests.orchestrator - INFO - attach_pod: 6 succeeded, 0 failed (100.0%) -2025-06-05 15:06:03,513 - tests.orchestrator - INFO - delete_pod: 6 succeeded, 0 failed (100.0%) -2025-06-05 15:06:03,513 - tests.orchestrator - INFO - delete_pvc: 5 succeeded, 0 failed (100.0%) -2025-06-05 15:06:03,513 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) -2025-06-05 15:06:03,513 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) -2025-06-05 15:06:03,513 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-05 15:06:03,513 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-05 15:06:03,513 - tests.orchestrator - INFO - many_to_one: No runs -2025-06-05 15:06:03,513 - tests.orchestrator - INFO - one_to_one: No runs -2025-06-05 15:06:03,513 - tests.orchestrator - INFO - concurrent_pvc: 1 succeeded, 0 failed out of 1 runs (100.0%) -2025-06-05 15:06:03,513 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_150619.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_150619.log deleted file mode 100644 index 78223d473..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_150619.log +++ /dev/null @@ -1,147 +0,0 @@ -2025-06-05 15:06:19,870 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 15:06:20,072 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 15:06:20,072 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-05 15:06:20,260 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 15:06:20,261 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 15:06:20,261 - tests.orchestrator - INFO - Creating PVC: test-pvc-7aa57f8c -2025-06-05 15:06:20,336 - tests.orchestrator - INFO - Created PVC: test-pvc-7aa57f8c -2025-06-05 15:06:20,336 - tests.orchestrator - INFO - Waiting for PVC test-pvc-7aa57f8c to be bound -2025-06-05 15:06:22,470 - tests.orchestrator - INFO - PVC test-pvc-7aa57f8c is bound -2025-06-05 15:06:22,471 - tests.orchestrator - INFO - Creating PVC: test-pvc-afe1c1f0 -2025-06-05 15:06:22,538 - tests.orchestrator - INFO - Created PVC: test-pvc-afe1c1f0 -2025-06-05 15:06:22,538 - tests.orchestrator - INFO - Waiting for PVC test-pvc-afe1c1f0 to be bound -2025-06-05 15:06:24,671 - tests.orchestrator - INFO - PVC test-pvc-afe1c1f0 is bound -2025-06-05 15:06:24,671 - tests.orchestrator - INFO - Creating pod test-pod-d8058d9f on PVC test-pvc-7aa57f8c -2025-06-05 15:06:24,755 - tests.orchestrator - INFO - Created pod: test-pod-d8058d9f using PVC: test-pvc-7aa57f8c -2025-06-05 15:06:24,755 - tests.orchestrator - INFO - Waiting for pod test-pod-d8058d9f to be ready -2025-06-05 15:07:26,793 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-d8058d9f to be ready after 60s -2025-06-05 15:07:26,793 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-d8058d9f to be ready -2025-06-05 15:07:26,793 - tests.orchestrator - INFO - Creating pod test-pod-fdb5f5f7 on PVC test-pvc-afe1c1f0 -2025-06-05 15:07:26,879 - tests.orchestrator - INFO - Created pod: test-pod-fdb5f5f7 using PVC: test-pvc-afe1c1f0 -2025-06-05 15:07:26,880 - tests.orchestrator - INFO - Waiting for pod test-pod-fdb5f5f7 to be ready -2025-06-05 15:08:28,914 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-fdb5f5f7 to be ready after 60s -2025-06-05 15:08:28,915 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-fdb5f5f7 to be ready -2025-06-05 15:08:28,915 - tests.orchestrator - INFO - Creating pod test-pod-415052fa on PVC test-pvc-7aa57f8c -2025-06-05 15:08:28,998 - tests.orchestrator - INFO - Created pod: test-pod-415052fa using PVC: test-pvc-7aa57f8c -2025-06-05 15:08:28,998 - tests.orchestrator - INFO - Waiting for pod test-pod-415052fa to be ready -2025-06-05 15:09:31,031 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-415052fa to be ready after 60s -2025-06-05 15:09:31,031 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-415052fa to be ready -2025-06-05 15:09:31,031 - tests.orchestrator - INFO - Testing read/write between pods test-pod-d8058d9f and test-pod-415052fa sharing PVC test-pvc-7aa57f8c -2025-06-05 15:09:31,098 - tests.orchestrator - ERROR - Failed in read/write verification: (400) -Reason: Bad Request -HTTP response headers: HTTPHeaderDict({'Audit-Id': '9f5a799b-0705-4213-aaf5-1218c7e9571a', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 15:09:31 GMT', 'Content-Length': '166'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-d8058d9f does not have a host assigned","reason":"BadRequest","code":400} - - -2025-06-05 15:09:31,098 - tests.orchestrator - INFO - ============================================================ -2025-06-05 15:09:31,098 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_concurrent_pvc -2025-06-05 15:09:31,098 - tests.orchestrator - INFO - ============================================================ -2025-06-05 15:09:31,098 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations -2025-06-05 15:09:31,099 - tests.orchestrator - INFO - Creating 5 PVCs in quick succession -2025-06-05 15:09:31,177 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-a396d6c2 -2025-06-05 15:09:31,178 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-a396d6c2 to be bound -2025-06-05 15:09:33,310 - tests.orchestrator - INFO - PVC concurrent-pvc-a396d6c2 is bound -2025-06-05 15:09:33,381 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-85edcaa9 -2025-06-05 15:09:33,381 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-85edcaa9 to be bound -2025-06-05 15:09:35,516 - tests.orchestrator - INFO - PVC concurrent-pvc-85edcaa9 is bound -2025-06-05 15:09:35,583 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-bb7b8fe6 -2025-06-05 15:09:35,583 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-bb7b8fe6 to be bound -2025-06-05 15:09:37,719 - tests.orchestrator - INFO - PVC concurrent-pvc-bb7b8fe6 is bound -2025-06-05 15:09:37,790 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-a3d1cb36 -2025-06-05 15:09:37,790 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-a3d1cb36 to be bound -2025-06-05 15:09:39,924 - tests.orchestrator - INFO - PVC concurrent-pvc-a3d1cb36 is bound -2025-06-05 15:09:39,993 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-e1cb5975 -2025-06-05 15:09:39,994 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-e1cb5975 to be bound -2025-06-05 15:09:42,273 - tests.orchestrator - INFO - PVC concurrent-pvc-e1cb5975 is bound -2025-06-05 15:09:42,273 - tests.orchestrator - INFO - Successfully created 5 PVCs -2025-06-05 15:09:42,274 - tests.orchestrator - INFO - Creating pod test-pod-1ea724b8 on PVC concurrent-pvc-a3d1cb36 -2025-06-05 15:09:42,449 - tests.orchestrator - INFO - Created pod: test-pod-1ea724b8 using PVC: concurrent-pvc-a3d1cb36 -2025-06-05 15:09:42,449 - tests.orchestrator - INFO - Waiting for pod test-pod-1ea724b8 to be ready -2025-06-05 15:10:42,631 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1ea724b8 to be ready after 60s -2025-06-05 15:10:42,631 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1ea724b8 to be ready -2025-06-05 15:10:42,631 - tests.orchestrator - INFO - Creating pod test-pod-7f110176 on PVC concurrent-pvc-e1cb5975 -2025-06-05 15:10:42,732 - tests.orchestrator - INFO - Created pod: test-pod-7f110176 using PVC: concurrent-pvc-e1cb5975 -2025-06-05 15:10:42,732 - tests.orchestrator - INFO - Waiting for pod test-pod-7f110176 to be ready -2025-06-05 15:11:42,757 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-7f110176 to be ready after 60s -2025-06-05 15:11:42,757 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-7f110176 to be ready -2025-06-05 15:11:42,757 - tests.orchestrator - INFO - Creating pod test-pod-f48c7103 on PVC concurrent-pvc-85edcaa9 -2025-06-05 15:11:42,854 - tests.orchestrator - INFO - Created pod: test-pod-f48c7103 using PVC: concurrent-pvc-85edcaa9 -2025-06-05 15:11:42,854 - tests.orchestrator - INFO - Waiting for pod test-pod-f48c7103 to be ready -2025-06-05 15:12:44,885 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-f48c7103 to be ready after 60s -2025-06-05 15:12:44,886 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-f48c7103 to be ready -2025-06-05 15:12:44,886 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-a3d1cb36 -2025-06-05 15:12:44,886 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-a3d1cb36 -2025-06-05 15:12:44,886 - tests.orchestrator - INFO - Deleting pod: test-pod-1ea724b8 from PVC: concurrent-pvc-a3d1cb36 -2025-06-05 15:12:44,967 - tests.orchestrator - INFO - Waiting for pod test-pod-1ea724b8 to be deleted -2025-06-05 15:12:45,035 - tests.orchestrator - INFO - Pod test-pod-1ea724b8 has been deleted -2025-06-05 15:12:45,035 - tests.orchestrator - INFO - Deleted pod: test-pod-1ea724b8 -2025-06-05 15:12:45,120 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-a3d1cb36 to be deleted -2025-06-05 15:12:45,197 - tests.orchestrator - INFO - PVC concurrent-pvc-a3d1cb36 has been deleted -2025-06-05 15:12:45,198 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-a3d1cb36 -2025-06-05 15:12:45,198 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-85edcaa9 -2025-06-05 15:12:45,198 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-85edcaa9 -2025-06-05 15:12:45,198 - tests.orchestrator - INFO - Deleting pod: test-pod-f48c7103 from PVC: concurrent-pvc-85edcaa9 -2025-06-05 15:12:45,285 - tests.orchestrator - INFO - Waiting for pod test-pod-f48c7103 to be deleted -2025-06-05 15:12:45,353 - tests.orchestrator - INFO - Pod test-pod-f48c7103 has been deleted -2025-06-05 15:12:45,353 - tests.orchestrator - INFO - Deleted pod: test-pod-f48c7103 -2025-06-05 15:12:45,426 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-85edcaa9 to be deleted -2025-06-05 15:12:45,492 - tests.orchestrator - INFO - PVC concurrent-pvc-85edcaa9 has been deleted -2025-06-05 15:12:45,493 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-85edcaa9 -2025-06-05 15:12:45,493 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-a396d6c2 -2025-06-05 15:12:45,566 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-a396d6c2 to be deleted -2025-06-05 15:12:45,632 - tests.orchestrator - INFO - PVC concurrent-pvc-a396d6c2 has been deleted -2025-06-05 15:12:45,632 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-a396d6c2 -2025-06-05 15:12:45,633 - tests.orchestrator - INFO - Rapid PVC scenario completed -2025-06-05 15:12:45,633 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_concurrent_pvc -2025-06-05 15:12:45,633 - tests.orchestrator - INFO - ------------------------------------------------------------ -2025-06-05 15:12:45,633 - tests.orchestrator - INFO - Deleting pod: test-pod-fdb5f5f7 from PVC: test-pvc-afe1c1f0 -2025-06-05 15:12:45,715 - tests.orchestrator - INFO - Waiting for pod test-pod-fdb5f5f7 to be deleted -2025-06-05 15:12:45,783 - tests.orchestrator - INFO - Pod test-pod-fdb5f5f7 has been deleted -2025-06-05 15:12:45,783 - tests.orchestrator - INFO - Deleted pod: test-pod-fdb5f5f7 -2025-06-05 15:12:45,783 - tests.orchestrator - INFO - Deleting PVC: test-pvc-7aa57f8c -2025-06-05 15:12:45,783 - tests.orchestrator - INFO - Deleting 2 pods using PVC test-pvc-7aa57f8c -2025-06-05 15:12:45,784 - tests.orchestrator - INFO - Deleting pod: test-pod-d8058d9f from PVC: test-pvc-7aa57f8c -2025-06-05 15:12:45,867 - tests.orchestrator - INFO - Waiting for pod test-pod-d8058d9f to be deleted -2025-06-05 15:12:45,934 - tests.orchestrator - INFO - Pod test-pod-d8058d9f has been deleted -2025-06-05 15:12:45,934 - tests.orchestrator - INFO - Deleted pod: test-pod-d8058d9f -2025-06-05 15:12:45,934 - tests.orchestrator - INFO - Deleting pod: test-pod-415052fa from PVC: test-pvc-7aa57f8c -2025-06-05 15:12:46,015 - tests.orchestrator - INFO - Waiting for pod test-pod-415052fa to be deleted -2025-06-05 15:12:46,079 - tests.orchestrator - INFO - Pod test-pod-415052fa has been deleted -2025-06-05 15:12:46,080 - tests.orchestrator - INFO - Deleted pod: test-pod-415052fa -2025-06-05 15:12:46,154 - tests.orchestrator - INFO - Waiting for PVC test-pvc-7aa57f8c to be deleted -2025-06-05 15:12:46,220 - tests.orchestrator - INFO - PVC test-pvc-7aa57f8c has been deleted -2025-06-05 15:12:46,220 - tests.orchestrator - INFO - Deleted PVC: test-pvc-7aa57f8c -2025-06-05 15:12:46,220 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations -2025-06-05 15:12:46,221 - tests.orchestrator - INFO - Test completed in 386.15 seconds -2025-06-05 15:12:46,221 - tests.orchestrator - INFO - Cleaning up resources... -2025-06-05 15:12:46,221 - tests.orchestrator - INFO - Deleting pod: test-pod-7f110176 from PVC: concurrent-pvc-e1cb5975 -2025-06-05 15:12:46,308 - tests.orchestrator - INFO - Waiting for pod test-pod-7f110176 to be deleted -2025-06-05 15:12:46,378 - tests.orchestrator - INFO - Pod test-pod-7f110176 has been deleted -2025-06-05 15:12:46,379 - tests.orchestrator - INFO - Deleted pod: test-pod-7f110176 -2025-06-05 15:12:46,379 - tests.orchestrator - INFO - Deleting PVC: test-pvc-afe1c1f0 -2025-06-05 15:12:46,452 - tests.orchestrator - INFO - Waiting for PVC test-pvc-afe1c1f0 to be deleted -2025-06-05 15:12:46,519 - tests.orchestrator - INFO - PVC test-pvc-afe1c1f0 has been deleted -2025-06-05 15:12:46,519 - tests.orchestrator - INFO - Deleted PVC: test-pvc-afe1c1f0 -2025-06-05 15:12:46,519 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-bb7b8fe6 -2025-06-05 15:12:46,593 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-bb7b8fe6 to be deleted -2025-06-05 15:12:46,660 - tests.orchestrator - INFO - PVC concurrent-pvc-bb7b8fe6 has been deleted -2025-06-05 15:12:46,660 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-bb7b8fe6 -2025-06-05 15:12:46,660 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-e1cb5975 -2025-06-05 15:12:46,735 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-e1cb5975 to be deleted -2025-06-05 15:12:46,805 - tests.orchestrator - INFO - PVC concurrent-pvc-e1cb5975 has been deleted -2025-06-05 15:12:46,805 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-e1cb5975 -2025-06-05 15:12:46,809 - tests.orchestrator - INFO - Cleanup completed -2025-06-05 15:12:46,809 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-05 15:12:46,809 - tests.orchestrator - INFO - --- Operations --- -2025-06-05 15:12:46,809 - tests.orchestrator - INFO - create_pvc: 7 succeeded, 0 failed (100.0%) -2025-06-05 15:12:46,809 - tests.orchestrator - INFO - attach_pod: 6 succeeded, 0 failed (100.0%) -2025-06-05 15:12:46,809 - tests.orchestrator - INFO - delete_pod: 6 succeeded, 0 failed (100.0%) -2025-06-05 15:12:46,809 - tests.orchestrator - INFO - delete_pvc: 7 succeeded, 0 failed (100.0%) -2025-06-05 15:12:46,809 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) -2025-06-05 15:12:46,809 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) -2025-06-05 15:12:46,809 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-05 15:12:46,809 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-05 15:12:46,809 - tests.orchestrator - INFO - many_to_one: No runs -2025-06-05 15:12:46,809 - tests.orchestrator - INFO - one_to_one: No runs -2025-06-05 15:12:46,809 - tests.orchestrator - INFO - concurrent_pvc: 1 succeeded, 0 failed out of 1 runs (100.0%) -2025-06-05 15:12:46,809 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_151529.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_151529.log deleted file mode 100644 index fb9e26ae4..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_151529.log +++ /dev/null @@ -1,7 +0,0 @@ -2025-06-05 15:15:29,738 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 15:15:29,948 - tests.orchestrator - ERROR - Error checking namespace: (401) -Reason: Unauthorized -HTTP response headers: HTTPHeaderDict({'Audit-Id': '95b9f34d-de93-4da4-99dd-fc17111bd0ce', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 15:15:29 GMT', 'Content-Length': '129'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} - - diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_151604.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_151604.log deleted file mode 100644 index ead869f56..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_151604.log +++ /dev/null @@ -1,42 +0,0 @@ -2025-06-05 15:16:04,962 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 15:16:05,167 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 15:16:05,168 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-05 15:16:05,359 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 15:16:05,359 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 15:16:05,359 - tests.orchestrator - INFO - Creating PVC: test-pvc-20f868e5 -2025-06-05 15:16:05,438 - tests.orchestrator - INFO - Created PVC: test-pvc-20f868e5 -2025-06-05 15:16:05,438 - tests.orchestrator - INFO - Waiting for PVC test-pvc-20f868e5 to be bound -2025-06-05 15:16:07,570 - tests.orchestrator - INFO - PVC test-pvc-20f868e5 is bound -2025-06-05 15:16:07,570 - tests.orchestrator - INFO - Creating PVC: test-pvc-48e3d2d9 -2025-06-05 15:16:07,639 - tests.orchestrator - INFO - Created PVC: test-pvc-48e3d2d9 -2025-06-05 15:16:07,639 - tests.orchestrator - INFO - Waiting for PVC test-pvc-48e3d2d9 to be bound -2025-06-05 15:16:09,773 - tests.orchestrator - INFO - PVC test-pvc-48e3d2d9 is bound -2025-06-05 15:16:09,773 - tests.orchestrator - INFO - Creating pod test-pod-03b0f7fd on PVC test-pvc-20f868e5 -2025-06-05 15:16:09,857 - tests.orchestrator - INFO - Created pod: test-pod-03b0f7fd using PVC: test-pvc-20f868e5 -2025-06-05 15:16:09,857 - tests.orchestrator - INFO - Waiting for pod test-pod-03b0f7fd to be ready -2025-06-05 15:16:09,926 - tests.orchestrator - INFO - Pod test-pod-03b0f7fd phase: Pending -2025-06-05 15:16:40,974 - tests.orchestrator - INFO - Pod test-pod-03b0f7fd taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 15:16:40,974 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-03b0f7fd ===== -2025-06-05 15:16:41,041 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:16:41,124 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:16:41,125 - tests.orchestrator - INFO - [2025-06-05 15:16:09+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:16:41,199 - tests.orchestrator - INFO - No logs available -2025-06-05 15:16:41,199 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:16:41,199 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-20f868e5'} -2025-06-05 15:16:41,199 - tests.orchestrator - INFO - - kube-api-access-f6t7l: {} -2025-06-05 15:16:41,199 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-03b0f7fd ===== -2025-06-05 15:17:10,102 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-03b0f7fd to be ready after 60s -2025-06-05 15:17:10,102 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-03b0f7fd ===== -2025-06-05 15:17:10,172 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:17:10,245 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:17:10,245 - tests.orchestrator - INFO - [2025-06-05 15:16:09+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:17:10,312 - tests.orchestrator - INFO - No logs available -2025-06-05 15:17:10,312 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:17:10,312 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-20f868e5'} -2025-06-05 15:17:10,312 - tests.orchestrator - INFO - - kube-api-access-f6t7l: {} -2025-06-05 15:17:10,312 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-03b0f7fd ===== -2025-06-05 15:17:10,313 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-03b0f7fd to be ready -2025-06-05 15:17:10,313 - tests.orchestrator - INFO - Creating pod test-pod-264015e0 on PVC test-pvc-20f868e5 -2025-06-05 15:17:10,399 - tests.orchestrator - INFO - Created pod: test-pod-264015e0 using PVC: test-pvc-20f868e5 -2025-06-05 15:17:10,399 - tests.orchestrator - INFO - Waiting for pod test-pod-264015e0 to be ready -2025-06-05 15:17:10,467 - tests.orchestrator - INFO - Pod test-pod-264015e0 phase: Pending diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_151742.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_151742.log deleted file mode 100644 index 596cad530..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_151742.log +++ /dev/null @@ -1,17 +0,0 @@ -2025-06-05 15:17:42,550 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 15:17:42,784 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 15:17:42,784 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-05 15:17:42,975 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 15:17:42,976 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 15:17:42,976 - tests.orchestrator - INFO - Creating PVC: test-pvc-b967555b -2025-06-05 15:17:43,061 - tests.orchestrator - INFO - Created PVC: test-pvc-b967555b -2025-06-05 15:17:43,061 - tests.orchestrator - INFO - Waiting for PVC test-pvc-b967555b to be bound -2025-06-05 15:17:45,204 - tests.orchestrator - INFO - PVC test-pvc-b967555b is bound -2025-06-05 15:17:45,205 - tests.orchestrator - INFO - Creating PVC: test-pvc-6282a942 -2025-06-05 15:17:45,276 - tests.orchestrator - INFO - Created PVC: test-pvc-6282a942 -2025-06-05 15:17:45,276 - tests.orchestrator - INFO - Waiting for PVC test-pvc-6282a942 to be bound -2025-06-05 15:17:47,412 - tests.orchestrator - INFO - PVC test-pvc-6282a942 is bound -2025-06-05 15:17:47,412 - tests.orchestrator - INFO - Creating pod test-pod-9ef8c6d5 on PVC test-pvc-b967555b -2025-06-05 15:17:47,490 - tests.orchestrator - INFO - Created pod: test-pod-9ef8c6d5 using PVC: test-pvc-b967555b -2025-06-05 15:17:47,491 - tests.orchestrator - INFO - Waiting for pod test-pod-9ef8c6d5 to be ready -2025-06-05 15:17:47,559 - tests.orchestrator - INFO - Pod test-pod-9ef8c6d5 phase: Pending diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_153750.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_153750.log deleted file mode 100644 index 4fba39f91..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_153750.log +++ /dev/null @@ -1,249 +0,0 @@ -2025-06-05 15:37:50,853 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 15:37:51,073 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 15:37:51,074 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-05 15:37:51,263 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 15:37:51,263 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 15:37:51,263 - tests.orchestrator - INFO - Creating PVC: test-pvc-011557a1 -2025-06-05 15:37:51,341 - tests.orchestrator - INFO - Created PVC: test-pvc-011557a1 -2025-06-05 15:37:51,341 - tests.orchestrator - INFO - Waiting for PVC test-pvc-011557a1 to be bound -2025-06-05 15:37:53,474 - tests.orchestrator - INFO - PVC test-pvc-011557a1 is bound -2025-06-05 15:37:53,477 - tests.orchestrator - INFO - Creating PVC: test-pvc-b52b04ee -2025-06-05 15:37:53,544 - tests.orchestrator - INFO - Created PVC: test-pvc-b52b04ee -2025-06-05 15:37:53,545 - tests.orchestrator - INFO - Waiting for PVC test-pvc-b52b04ee to be bound -2025-06-05 15:37:55,677 - tests.orchestrator - INFO - PVC test-pvc-b52b04ee is bound -2025-06-05 15:37:55,677 - tests.orchestrator - INFO - Creating pod test-pod-1f8e6665 on PVC test-pvc-b52b04ee -2025-06-05 15:37:55,757 - tests.orchestrator - INFO - Created pod: test-pod-1f8e6665 using PVC: test-pvc-b52b04ee -2025-06-05 15:37:55,757 - tests.orchestrator - INFO - Waiting for pod test-pod-1f8e6665 to be ready -2025-06-05 15:37:55,827 - tests.orchestrator - INFO - Pod test-pod-1f8e6665 phase: Pending -2025-06-05 15:38:26,850 - tests.orchestrator - INFO - Pod test-pod-1f8e6665 taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 15:38:26,850 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-1f8e6665 ===== -2025-06-05 15:38:26,917 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:38:26,987 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:38:26,987 - tests.orchestrator - INFO - [2025-06-05 15:37:55+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:38:27,053 - tests.orchestrator - INFO - No logs available -2025-06-05 15:38:27,053 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:38:27,054 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-b52b04ee'} -2025-06-05 15:38:27,054 - tests.orchestrator - INFO - - kube-api-access-z4xbc: {} -2025-06-05 15:38:27,054 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-1f8e6665 ===== -2025-06-05 15:38:55,926 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1f8e6665 to be ready after 60s -2025-06-05 15:38:55,927 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-1f8e6665 ===== -2025-06-05 15:38:55,995 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:38:56,064 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:38:56,064 - tests.orchestrator - INFO - [2025-06-05 15:37:55+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:38:56,129 - tests.orchestrator - INFO - No logs available -2025-06-05 15:38:56,129 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:38:56,129 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-b52b04ee'} -2025-06-05 15:38:56,129 - tests.orchestrator - INFO - - kube-api-access-z4xbc: {} -2025-06-05 15:38:56,129 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-1f8e6665 ===== -2025-06-05 15:38:56,130 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1f8e6665 to be ready -2025-06-05 15:38:56,130 - tests.orchestrator - INFO - Creating pod test-pod-43a71634 on PVC test-pvc-b52b04ee -2025-06-05 15:38:56,213 - tests.orchestrator - INFO - Created pod: test-pod-43a71634 using PVC: test-pvc-b52b04ee -2025-06-05 15:38:56,213 - tests.orchestrator - INFO - Waiting for pod test-pod-43a71634 to be ready -2025-06-05 15:38:56,283 - tests.orchestrator - INFO - Pod test-pod-43a71634 phase: Pending -2025-06-05 15:39:27,290 - tests.orchestrator - INFO - Pod test-pod-43a71634 taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 15:39:27,290 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-43a71634 ===== -2025-06-05 15:39:27,357 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:39:27,428 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:39:27,428 - tests.orchestrator - INFO - [2025-06-05 15:38:56+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:39:27,493 - tests.orchestrator - INFO - No logs available -2025-06-05 15:39:27,494 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:39:27,494 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-b52b04ee'} -2025-06-05 15:39:27,494 - tests.orchestrator - INFO - - kube-api-access-fgg2x: {} -2025-06-05 15:39:27,494 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-43a71634 ===== -2025-06-05 15:39:56,469 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-43a71634 to be ready after 60s -2025-06-05 15:39:56,469 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-43a71634 ===== -2025-06-05 15:39:56,538 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:39:56,609 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:39:56,609 - tests.orchestrator - INFO - [2025-06-05 15:38:56+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:39:56,674 - tests.orchestrator - INFO - No logs available -2025-06-05 15:39:56,674 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:39:56,674 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-b52b04ee'} -2025-06-05 15:39:56,674 - tests.orchestrator - INFO - - kube-api-access-fgg2x: {} -2025-06-05 15:39:56,675 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-43a71634 ===== -2025-06-05 15:39:56,675 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-43a71634 to be ready -2025-06-05 15:39:56,675 - tests.orchestrator - INFO - Creating pod test-pod-9785ae47 on PVC test-pvc-011557a1 -2025-06-05 15:39:56,757 - tests.orchestrator - INFO - Created pod: test-pod-9785ae47 using PVC: test-pvc-011557a1 -2025-06-05 15:39:56,757 - tests.orchestrator - INFO - Waiting for pod test-pod-9785ae47 to be ready -2025-06-05 15:39:56,823 - tests.orchestrator - INFO - Pod test-pod-9785ae47 phase: Pending -2025-06-05 15:40:27,832 - tests.orchestrator - INFO - Pod test-pod-9785ae47 taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 15:40:27,832 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-9785ae47 ===== -2025-06-05 15:40:27,899 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:40:27,970 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:40:27,970 - tests.orchestrator - INFO - [2025-06-05 15:39:56+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:40:28,037 - tests.orchestrator - INFO - No logs available -2025-06-05 15:40:28,037 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:40:28,037 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-011557a1'} -2025-06-05 15:40:28,037 - tests.orchestrator - INFO - - kube-api-access-p2phq: {} -2025-06-05 15:40:28,037 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-9785ae47 ===== -2025-06-05 15:40:56,911 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9785ae47 to be ready after 60s -2025-06-05 15:40:56,912 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-9785ae47 ===== -2025-06-05 15:40:56,980 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:40:57,050 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:40:57,050 - tests.orchestrator - INFO - [2025-06-05 15:39:56+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:40:57,118 - tests.orchestrator - INFO - No logs available -2025-06-05 15:40:57,118 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:40:57,118 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-011557a1'} -2025-06-05 15:40:57,118 - tests.orchestrator - INFO - - kube-api-access-p2phq: {} -2025-06-05 15:40:57,118 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-9785ae47 ===== -2025-06-05 15:40:57,118 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9785ae47 to be ready -2025-06-05 15:40:57,119 - tests.orchestrator - INFO - Testing read/write between pods test-pod-1f8e6665 and test-pod-43a71634 sharing PVC test-pvc-b52b04ee -2025-06-05 15:40:57,185 - tests.orchestrator - ERROR - Failed in read/write verification: (400) -Reason: Bad Request -HTTP response headers: HTTPHeaderDict({'Audit-Id': '4713df17-943b-4cf0-b43b-7454ef741357', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 15:40:57 GMT', 'Content-Length': '166'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-1f8e6665 does not have a host assigned","reason":"BadRequest","code":400} - - -2025-06-05 15:40:57,185 - tests.orchestrator - INFO - ============================================================ -2025-06-05 15:40:57,185 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_many_to_one -2025-06-05 15:40:57,185 - tests.orchestrator - INFO - ============================================================ -2025-06-05 15:40:57,185 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -2025-06-05 15:40:57,185 - tests.orchestrator - INFO - STARTING MANY-TO-ONE SCENARIO DIAGNOSTICS -2025-06-05 15:40:57,185 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -2025-06-05 15:40:57,185 - tests.orchestrator - INFO - [MANY2ONE] STEP 1: Creating dedicated PVC: many2one-21027765 -2025-06-05 15:40:57,185 - tests.orchestrator - INFO - [MANY2ONE] Creating PVC manifest with storageClassName: efs-sc -2025-06-05 15:40:57,263 - tests.orchestrator - INFO - [MANY2ONE] PVC many2one-21027765 created successfully -2025-06-05 15:40:57,264 - tests.orchestrator - INFO - [MANY2ONE] Waiting for PVC many2one-21027765 to be bound... -2025-06-05 15:40:57,264 - tests.orchestrator - INFO - Waiting for PVC many2one-21027765 to be bound -2025-06-05 15:40:59,396 - tests.orchestrator - INFO - PVC many2one-21027765 is bound -2025-06-05 15:40:59,463 - tests.orchestrator - INFO - [MANY2ONE] PVC status: phase=Bound, capacity={'storage': '1Gi'} -2025-06-05 15:40:59,463 - tests.orchestrator - INFO - [MANY2ONE] STEP 2: Creating 3 pods for the same PVC many2one-21027765 -2025-06-05 15:40:59,463 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 1/3 for PVC many2one-21027765 -2025-06-05 15:40:59,463 - tests.orchestrator - INFO - Creating pod test-pod-052e963e on PVC many2one-21027765 -2025-06-05 15:40:59,538 - tests.orchestrator - INFO - Created pod: test-pod-052e963e using PVC: many2one-21027765 -2025-06-05 15:40:59,538 - tests.orchestrator - INFO - Waiting for pod test-pod-052e963e to be ready -2025-06-05 15:40:59,604 - tests.orchestrator - INFO - Pod test-pod-052e963e phase: Pending -2025-06-05 15:41:30,643 - tests.orchestrator - INFO - Pod test-pod-052e963e taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 15:41:30,643 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-052e963e ===== -2025-06-05 15:41:30,710 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:41:30,779 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:41:30,780 - tests.orchestrator - INFO - [2025-06-05 15:40:59+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:41:30,846 - tests.orchestrator - INFO - No logs available -2025-06-05 15:41:30,846 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:41:30,846 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-21027765'} -2025-06-05 15:41:30,846 - tests.orchestrator - INFO - - kube-api-access-h4dzw: {} -2025-06-05 15:41:30,846 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-052e963e ===== -2025-06-05 15:41:59,722 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-052e963e to be ready after 60s -2025-06-05 15:41:59,722 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-052e963e ===== -2025-06-05 15:41:59,789 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:41:59,859 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:41:59,859 - tests.orchestrator - INFO - [2025-06-05 15:40:59+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:41:59,925 - tests.orchestrator - INFO - No logs available -2025-06-05 15:41:59,925 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:41:59,925 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-21027765'} -2025-06-05 15:41:59,925 - tests.orchestrator - INFO - - kube-api-access-h4dzw: {} -2025-06-05 15:41:59,925 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-052e963e ===== -2025-06-05 15:41:59,926 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-052e963e to be ready -2025-06-05 15:41:59,926 - tests.orchestrator - ERROR - [MANY2ONE] Failed to create pod 1/3 -2025-06-05 15:41:59,926 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 2/3 for PVC many2one-21027765 -2025-06-05 15:41:59,926 - tests.orchestrator - INFO - Creating pod test-pod-dda7e82b on PVC many2one-21027765 -2025-06-05 15:42:00,007 - tests.orchestrator - INFO - Created pod: test-pod-dda7e82b using PVC: many2one-21027765 -2025-06-05 15:42:00,008 - tests.orchestrator - INFO - Waiting for pod test-pod-dda7e82b to be ready -2025-06-05 15:42:00,075 - tests.orchestrator - INFO - Pod test-pod-dda7e82b phase: Pending -2025-06-05 15:42:31,084 - tests.orchestrator - INFO - Pod test-pod-dda7e82b taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 15:42:31,084 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-dda7e82b ===== -2025-06-05 15:42:31,150 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:42:31,219 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:42:31,219 - tests.orchestrator - INFO - [2025-06-05 15:41:59+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:42:31,285 - tests.orchestrator - INFO - No logs available -2025-06-05 15:42:31,285 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:42:31,285 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-21027765'} -2025-06-05 15:42:31,285 - tests.orchestrator - INFO - - kube-api-access-bktns: {} -2025-06-05 15:42:31,285 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-dda7e82b ===== -2025-06-05 15:43:00,160 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-dda7e82b to be ready after 60s -2025-06-05 15:43:00,160 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-dda7e82b ===== -2025-06-05 15:43:00,227 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:43:00,296 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:43:00,296 - tests.orchestrator - INFO - [2025-06-05 15:41:59+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:43:00,362 - tests.orchestrator - INFO - No logs available -2025-06-05 15:43:00,362 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:43:00,362 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-21027765'} -2025-06-05 15:43:00,362 - tests.orchestrator - INFO - - kube-api-access-bktns: {} -2025-06-05 15:43:00,363 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-dda7e82b ===== -2025-06-05 15:43:00,363 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-dda7e82b to be ready -2025-06-05 15:43:00,363 - tests.orchestrator - ERROR - [MANY2ONE] Failed to create pod 2/3 -2025-06-05 15:43:00,363 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 3/3 for PVC many2one-21027765 -2025-06-05 15:43:00,363 - tests.orchestrator - INFO - Creating pod test-pod-9b6c5221 on PVC many2one-21027765 -2025-06-05 15:43:00,448 - tests.orchestrator - INFO - Created pod: test-pod-9b6c5221 using PVC: many2one-21027765 -2025-06-05 15:43:00,448 - tests.orchestrator - INFO - Waiting for pod test-pod-9b6c5221 to be ready -2025-06-05 15:43:00,514 - tests.orchestrator - INFO - Pod test-pod-9b6c5221 phase: Pending -2025-06-05 15:43:31,552 - tests.orchestrator - INFO - Pod test-pod-9b6c5221 taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 15:43:31,552 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-9b6c5221 ===== -2025-06-05 15:43:31,684 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:43:31,759 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:43:31,759 - tests.orchestrator - INFO - [2025-06-05 15:43:00+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:43:31,829 - tests.orchestrator - INFO - No logs available -2025-06-05 15:43:31,830 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:43:31,830 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-21027765'} -2025-06-05 15:43:31,830 - tests.orchestrator - INFO - - kube-api-access-mh6kx: {} -2025-06-05 15:43:31,830 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-9b6c5221 ===== -2025-06-05 15:44:00,708 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9b6c5221 to be ready after 60s -2025-06-05 15:44:00,708 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-9b6c5221 ===== -2025-06-05 15:44:00,776 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:44:00,846 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:44:00,846 - tests.orchestrator - INFO - [2025-06-05 15:43:00+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:44:00,911 - tests.orchestrator - INFO - No logs available -2025-06-05 15:44:00,911 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:44:00,911 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-21027765'} -2025-06-05 15:44:00,911 - tests.orchestrator - INFO - - kube-api-access-mh6kx: {} -2025-06-05 15:44:00,911 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-9b6c5221 ===== -2025-06-05 15:44:00,912 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9b6c5221 to be ready -2025-06-05 15:44:00,912 - tests.orchestrator - ERROR - [MANY2ONE] Failed to create pod 3/3 -2025-06-05 15:44:00,912 - tests.orchestrator - INFO - [MANY2ONE] Created 0/3 pods successfully -2025-06-05 15:44:00,912 - tests.orchestrator - ERROR - [MANY2ONE] FAILED: Insufficient pods created (0), need at least 2 for read/write test -2025-06-05 15:44:00,912 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_many_to_one -2025-06-05 15:44:00,912 - tests.orchestrator - INFO - ------------------------------------------------------------ -2025-06-05 15:44:00,912 - tests.orchestrator - INFO - Deleting pod: test-pod-052e963e from PVC: many2one-21027765 -2025-06-05 15:44:00,990 - tests.orchestrator - INFO - Waiting for pod test-pod-052e963e to be deleted -2025-06-05 15:44:01,055 - tests.orchestrator - INFO - Pod test-pod-052e963e has been deleted -2025-06-05 15:44:01,055 - tests.orchestrator - INFO - Deleted pod: test-pod-052e963e -2025-06-05 15:44:01,056 - tests.orchestrator - INFO - Deleting PVC: test-pvc-b52b04ee -2025-06-05 15:44:01,056 - tests.orchestrator - INFO - Deleting 2 pods using PVC test-pvc-b52b04ee -2025-06-05 15:44:01,056 - tests.orchestrator - INFO - Deleting pod: test-pod-1f8e6665 from PVC: test-pvc-b52b04ee -2025-06-05 15:44:01,140 - tests.orchestrator - INFO - Waiting for pod test-pod-1f8e6665 to be deleted -2025-06-05 15:44:01,205 - tests.orchestrator - INFO - Pod test-pod-1f8e6665 has been deleted -2025-06-05 15:44:01,205 - tests.orchestrator - INFO - Deleted pod: test-pod-1f8e6665 -2025-06-05 15:44:01,205 - tests.orchestrator - INFO - Deleting pod: test-pod-43a71634 from PVC: test-pvc-b52b04ee -2025-06-05 15:44:01,287 - tests.orchestrator - INFO - Waiting for pod test-pod-43a71634 to be deleted -2025-06-05 15:44:01,352 - tests.orchestrator - INFO - Pod test-pod-43a71634 has been deleted -2025-06-05 15:44:01,352 - tests.orchestrator - INFO - Deleted pod: test-pod-43a71634 -2025-06-05 15:44:01,426 - tests.orchestrator - INFO - Waiting for PVC test-pvc-b52b04ee to be deleted -2025-06-05 15:44:01,527 - tests.orchestrator - INFO - PVC test-pvc-b52b04ee has been deleted -2025-06-05 15:44:01,527 - tests.orchestrator - INFO - Deleted PVC: test-pvc-b52b04ee -2025-06-05 15:44:01,527 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations -2025-06-05 15:44:01,527 - tests.orchestrator - INFO - Test completed in 370.45 seconds -2025-06-05 15:44:01,527 - tests.orchestrator - INFO - Cleaning up resources... -2025-06-05 15:44:01,527 - tests.orchestrator - INFO - Deleting pod: test-pod-9785ae47 from PVC: test-pvc-011557a1 -2025-06-05 15:44:01,678 - tests.orchestrator - INFO - Waiting for pod test-pod-9785ae47 to be deleted -2025-06-05 15:44:01,750 - tests.orchestrator - INFO - Pod test-pod-9785ae47 has been deleted -2025-06-05 15:44:01,751 - tests.orchestrator - INFO - Deleted pod: test-pod-9785ae47 -2025-06-05 15:44:01,751 - tests.orchestrator - INFO - Deleting pod: test-pod-dda7e82b from PVC: many2one-21027765 -2025-06-05 15:44:01,841 - tests.orchestrator - INFO - Waiting for pod test-pod-dda7e82b to be deleted -2025-06-05 15:44:01,912 - tests.orchestrator - INFO - Pod test-pod-dda7e82b has been deleted -2025-06-05 15:44:01,912 - tests.orchestrator - INFO - Deleted pod: test-pod-dda7e82b -2025-06-05 15:44:01,912 - tests.orchestrator - INFO - Deleting pod: test-pod-9b6c5221 from PVC: many2one-21027765 -2025-06-05 15:44:02,038 - tests.orchestrator - INFO - Waiting for pod test-pod-9b6c5221 to be deleted -2025-06-05 15:44:02,118 - tests.orchestrator - INFO - Pod test-pod-9b6c5221 has been deleted -2025-06-05 15:44:02,118 - tests.orchestrator - INFO - Deleted pod: test-pod-9b6c5221 -2025-06-05 15:44:02,118 - tests.orchestrator - INFO - Deleting PVC: test-pvc-011557a1 -2025-06-05 15:44:02,196 - tests.orchestrator - INFO - Waiting for PVC test-pvc-011557a1 to be deleted -2025-06-05 15:44:02,268 - tests.orchestrator - INFO - PVC test-pvc-011557a1 has been deleted -2025-06-05 15:44:02,268 - tests.orchestrator - INFO - Deleted PVC: test-pvc-011557a1 -2025-06-05 15:44:02,268 - tests.orchestrator - INFO - Deleting PVC: many2one-21027765 -2025-06-05 15:44:02,341 - tests.orchestrator - INFO - Waiting for PVC many2one-21027765 to be deleted -2025-06-05 15:44:02,406 - tests.orchestrator - INFO - PVC many2one-21027765 has been deleted -2025-06-05 15:44:02,406 - tests.orchestrator - INFO - Deleted PVC: many2one-21027765 -2025-06-05 15:44:02,406 - tests.orchestrator - INFO - Cleanup completed -2025-06-05 15:44:02,406 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-05 15:44:02,407 - tests.orchestrator - INFO - --- Operations --- -2025-06-05 15:44:02,407 - tests.orchestrator - INFO - create_pvc: 2 succeeded, 0 failed (100.0%) -2025-06-05 15:44:02,407 - tests.orchestrator - INFO - attach_pod: 6 succeeded, 0 failed (100.0%) -2025-06-05 15:44:02,407 - tests.orchestrator - INFO - delete_pod: 6 succeeded, 0 failed (100.0%) -2025-06-05 15:44:02,407 - tests.orchestrator - INFO - delete_pvc: 3 succeeded, 0 failed (100.0%) -2025-06-05 15:44:02,407 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) -2025-06-05 15:44:02,407 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) -2025-06-05 15:44:02,407 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-05 15:44:02,407 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-05 15:44:02,408 - tests.orchestrator - INFO - many_to_one: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-05 15:44:02,408 - tests.orchestrator - INFO - one_to_one: No runs -2025-06-05 15:44:02,408 - tests.orchestrator - INFO - concurrent_pvc: No runs -2025-06-05 15:44:02,408 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_154546.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_154546.log deleted file mode 100644 index 6863cb59c..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_154546.log +++ /dev/null @@ -1,254 +0,0 @@ -2025-06-05 15:45:46,726 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 15:45:46,943 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 15:45:46,944 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-05 15:45:47,132 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 15:45:47,133 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 15:45:47,133 - tests.orchestrator - INFO - Creating PVC: test-pvc-7dbf87c6 -2025-06-05 15:45:47,210 - tests.orchestrator - INFO - Created PVC: test-pvc-7dbf87c6 -2025-06-05 15:45:47,210 - tests.orchestrator - INFO - Waiting for PVC test-pvc-7dbf87c6 to be bound -2025-06-05 15:45:49,343 - tests.orchestrator - INFO - PVC test-pvc-7dbf87c6 is bound -2025-06-05 15:45:49,343 - tests.orchestrator - INFO - Creating PVC: test-pvc-83f66d6c -2025-06-05 15:45:49,411 - tests.orchestrator - INFO - Created PVC: test-pvc-83f66d6c -2025-06-05 15:45:49,411 - tests.orchestrator - INFO - Waiting for PVC test-pvc-83f66d6c to be bound -2025-06-05 15:45:51,548 - tests.orchestrator - INFO - PVC test-pvc-83f66d6c is bound -2025-06-05 15:45:51,549 - tests.orchestrator - INFO - Creating pod test-pod-c3e945dc on PVC test-pvc-7dbf87c6 -2025-06-05 15:45:51,631 - tests.orchestrator - INFO - Created pod: test-pod-c3e945dc using PVC: test-pvc-7dbf87c6 -2025-06-05 15:45:51,631 - tests.orchestrator - INFO - Waiting for pod test-pod-c3e945dc to be ready -2025-06-05 15:45:51,697 - tests.orchestrator - INFO - Pod test-pod-c3e945dc phase: Pending -2025-06-05 15:46:22,714 - tests.orchestrator - INFO - Pod test-pod-c3e945dc taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 15:46:22,714 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-c3e945dc ===== -2025-06-05 15:46:22,781 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:46:22,850 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:46:22,850 - tests.orchestrator - INFO - [2025-06-05 15:45:51+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:46:22,916 - tests.orchestrator - INFO - No logs available -2025-06-05 15:46:22,917 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:46:22,917 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-7dbf87c6'} -2025-06-05 15:46:22,917 - tests.orchestrator - INFO - - kube-api-access-vmkzs: {} -2025-06-05 15:46:22,917 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-c3e945dc ===== -2025-06-05 15:46:51,793 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-c3e945dc to be ready after 60s -2025-06-05 15:46:51,793 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-c3e945dc ===== -2025-06-05 15:46:51,860 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:46:51,930 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:46:51,930 - tests.orchestrator - INFO - [2025-06-05 15:45:51+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:46:51,996 - tests.orchestrator - INFO - No logs available -2025-06-05 15:46:51,996 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:46:51,996 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-7dbf87c6'} -2025-06-05 15:46:51,996 - tests.orchestrator - INFO - - kube-api-access-vmkzs: {} -2025-06-05 15:46:51,996 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-c3e945dc ===== -2025-06-05 15:46:51,997 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-c3e945dc to be ready -2025-06-05 15:46:51,997 - tests.orchestrator - INFO - Creating pod test-pod-7abe409e on PVC test-pvc-83f66d6c -2025-06-05 15:46:52,079 - tests.orchestrator - INFO - Created pod: test-pod-7abe409e using PVC: test-pvc-83f66d6c -2025-06-05 15:46:52,079 - tests.orchestrator - INFO - Waiting for pod test-pod-7abe409e to be ready -2025-06-05 15:46:52,146 - tests.orchestrator - INFO - Pod test-pod-7abe409e phase: Pending -2025-06-05 15:47:23,154 - tests.orchestrator - INFO - Pod test-pod-7abe409e taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 15:47:23,154 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-7abe409e ===== -2025-06-05 15:47:23,220 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:47:23,289 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:47:23,290 - tests.orchestrator - INFO - [2025-06-05 15:46:52+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:47:23,355 - tests.orchestrator - INFO - No logs available -2025-06-05 15:47:23,355 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:47:23,355 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-83f66d6c'} -2025-06-05 15:47:23,355 - tests.orchestrator - INFO - - kube-api-access-swfv7: {} -2025-06-05 15:47:23,355 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-7abe409e ===== -2025-06-05 15:47:52,325 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-7abe409e to be ready after 60s -2025-06-05 15:47:52,325 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-7abe409e ===== -2025-06-05 15:47:52,392 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:47:52,462 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:47:52,462 - tests.orchestrator - INFO - [2025-06-05 15:46:52+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:47:52,528 - tests.orchestrator - INFO - No logs available -2025-06-05 15:47:52,528 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:47:52,528 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-83f66d6c'} -2025-06-05 15:47:52,528 - tests.orchestrator - INFO - - kube-api-access-swfv7: {} -2025-06-05 15:47:52,528 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-7abe409e ===== -2025-06-05 15:47:52,528 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-7abe409e to be ready -2025-06-05 15:47:52,528 - tests.orchestrator - INFO - Creating pod test-pod-a9fc1f04 on PVC test-pvc-7dbf87c6 -2025-06-05 15:47:52,611 - tests.orchestrator - INFO - Created pod: test-pod-a9fc1f04 using PVC: test-pvc-7dbf87c6 -2025-06-05 15:47:52,612 - tests.orchestrator - INFO - Waiting for pod test-pod-a9fc1f04 to be ready -2025-06-05 15:47:52,678 - tests.orchestrator - INFO - Pod test-pod-a9fc1f04 phase: Pending -2025-06-05 15:48:23,696 - tests.orchestrator - INFO - Pod test-pod-a9fc1f04 taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 15:48:23,696 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-a9fc1f04 ===== -2025-06-05 15:48:23,762 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:48:23,831 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:48:23,832 - tests.orchestrator - INFO - [2025-06-05 15:47:52+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:48:23,898 - tests.orchestrator - INFO - No logs available -2025-06-05 15:48:23,898 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:48:23,898 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-7dbf87c6'} -2025-06-05 15:48:23,898 - tests.orchestrator - INFO - - kube-api-access-bh5hw: {} -2025-06-05 15:48:23,898 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-a9fc1f04 ===== -2025-06-05 15:48:52,778 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a9fc1f04 to be ready after 60s -2025-06-05 15:48:52,778 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-a9fc1f04 ===== -2025-06-05 15:48:52,845 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:48:52,917 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:48:52,917 - tests.orchestrator - INFO - [2025-06-05 15:47:52+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:48:52,983 - tests.orchestrator - INFO - No logs available -2025-06-05 15:48:52,983 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:48:52,983 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-7dbf87c6'} -2025-06-05 15:48:52,983 - tests.orchestrator - INFO - - kube-api-access-bh5hw: {} -2025-06-05 15:48:52,984 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-a9fc1f04 ===== -2025-06-05 15:48:52,984 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a9fc1f04 to be ready -2025-06-05 15:48:52,984 - tests.orchestrator - INFO - Testing read/write between pods test-pod-c3e945dc and test-pod-a9fc1f04 sharing PVC test-pvc-7dbf87c6 -2025-06-05 15:48:53,049 - tests.orchestrator - ERROR - Failed in read/write verification: (400) -Reason: Bad Request -HTTP response headers: HTTPHeaderDict({'Audit-Id': '0f444cd6-376b-4233-8e31-012234fb64c8', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 15:48:53 GMT', 'Content-Length': '166'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-c3e945dc does not have a host assigned","reason":"BadRequest","code":400} - - -2025-06-05 15:48:53,049 - tests.orchestrator - INFO - ============================================================ -2025-06-05 15:48:53,049 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_concurrent_pvc -2025-06-05 15:48:53,049 - tests.orchestrator - INFO - ============================================================ -2025-06-05 15:48:53,049 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations -2025-06-05 15:48:53,049 - tests.orchestrator - INFO - Creating 3 PVCs in quick succession -2025-06-05 15:48:53,127 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-74639d37 -2025-06-05 15:48:53,127 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-74639d37 to be bound -2025-06-05 15:48:55,259 - tests.orchestrator - INFO - PVC concurrent-pvc-74639d37 is bound -2025-06-05 15:48:55,328 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-be273a95 -2025-06-05 15:48:55,328 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-be273a95 to be bound -2025-06-05 15:48:57,460 - tests.orchestrator - INFO - PVC concurrent-pvc-be273a95 is bound -2025-06-05 15:48:57,530 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-421903bd -2025-06-05 15:48:57,530 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-421903bd to be bound -2025-06-05 15:48:59,662 - tests.orchestrator - INFO - PVC concurrent-pvc-421903bd is bound -2025-06-05 15:48:59,662 - tests.orchestrator - INFO - Successfully created 3 PVCs -2025-06-05 15:48:59,662 - tests.orchestrator - INFO - Creating pod test-pod-d257ffb1 on PVC concurrent-pvc-be273a95 -2025-06-05 15:48:59,737 - tests.orchestrator - INFO - Created pod: test-pod-d257ffb1 using PVC: concurrent-pvc-be273a95 -2025-06-05 15:48:59,737 - tests.orchestrator - INFO - Waiting for pod test-pod-d257ffb1 to be ready -2025-06-05 15:48:59,805 - tests.orchestrator - INFO - Pod test-pod-d257ffb1 phase: Pending -2025-06-05 15:49:30,827 - tests.orchestrator - INFO - Pod test-pod-d257ffb1 taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 15:49:30,828 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-d257ffb1 ===== -2025-06-05 15:49:30,894 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:49:30,964 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:49:30,964 - tests.orchestrator - INFO - [2025-06-05 15:48:59+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:49:31,030 - tests.orchestrator - INFO - No logs available -2025-06-05 15:49:31,030 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:49:31,030 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'concurrent-pvc-be273a95'} -2025-06-05 15:49:31,031 - tests.orchestrator - INFO - - kube-api-access-tr4s4: {} -2025-06-05 15:49:31,031 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-d257ffb1 ===== -2025-06-05 15:49:59,906 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-d257ffb1 to be ready after 60s -2025-06-05 15:49:59,907 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-d257ffb1 ===== -2025-06-05 15:49:59,974 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:50:00,045 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:50:00,045 - tests.orchestrator - INFO - [2025-06-05 15:48:59+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:50:00,111 - tests.orchestrator - INFO - No logs available -2025-06-05 15:50:00,112 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:50:00,112 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'concurrent-pvc-be273a95'} -2025-06-05 15:50:00,112 - tests.orchestrator - INFO - - kube-api-access-tr4s4: {} -2025-06-05 15:50:00,112 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-d257ffb1 ===== -2025-06-05 15:50:00,112 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-d257ffb1 to be ready -2025-06-05 15:50:00,112 - tests.orchestrator - INFO - Creating pod test-pod-8c9ea815 on PVC concurrent-pvc-421903bd -2025-06-05 15:50:00,195 - tests.orchestrator - INFO - Created pod: test-pod-8c9ea815 using PVC: concurrent-pvc-421903bd -2025-06-05 15:50:00,196 - tests.orchestrator - INFO - Waiting for pod test-pod-8c9ea815 to be ready -2025-06-05 15:50:00,263 - tests.orchestrator - INFO - Pod test-pod-8c9ea815 phase: Pending -2025-06-05 15:50:31,275 - tests.orchestrator - INFO - Pod test-pod-8c9ea815 taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 15:50:31,275 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-8c9ea815 ===== -2025-06-05 15:50:31,341 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:50:31,411 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:50:31,411 - tests.orchestrator - INFO - [2025-06-05 15:50:00+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:50:31,526 - tests.orchestrator - INFO - No logs available -2025-06-05 15:50:31,526 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:50:31,526 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'concurrent-pvc-421903bd'} -2025-06-05 15:50:31,526 - tests.orchestrator - INFO - - kube-api-access-mk5vl: {} -2025-06-05 15:50:31,527 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-8c9ea815 ===== -2025-06-05 15:51:00,401 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-8c9ea815 to be ready after 60s -2025-06-05 15:51:00,401 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-8c9ea815 ===== -2025-06-05 15:51:00,469 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:51:00,539 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:51:00,539 - tests.orchestrator - INFO - [2025-06-05 15:50:00+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:51:00,604 - tests.orchestrator - INFO - No logs available -2025-06-05 15:51:00,604 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:51:00,604 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'concurrent-pvc-421903bd'} -2025-06-05 15:51:00,604 - tests.orchestrator - INFO - - kube-api-access-mk5vl: {} -2025-06-05 15:51:00,605 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-8c9ea815 ===== -2025-06-05 15:51:00,605 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-8c9ea815 to be ready -2025-06-05 15:51:00,605 - tests.orchestrator - INFO - Creating pod test-pod-3106ab46 on PVC concurrent-pvc-74639d37 -2025-06-05 15:51:00,689 - tests.orchestrator - INFO - Created pod: test-pod-3106ab46 using PVC: concurrent-pvc-74639d37 -2025-06-05 15:51:00,689 - tests.orchestrator - INFO - Waiting for pod test-pod-3106ab46 to be ready -2025-06-05 15:51:00,755 - tests.orchestrator - INFO - Pod test-pod-3106ab46 phase: Pending -2025-06-05 15:51:31,779 - tests.orchestrator - INFO - Pod test-pod-3106ab46 taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 15:51:31,779 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-3106ab46 ===== -2025-06-05 15:51:31,852 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:51:31,927 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:51:31,927 - tests.orchestrator - INFO - [2025-06-05 15:51:00+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:51:31,998 - tests.orchestrator - INFO - No logs available -2025-06-05 15:51:31,998 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:51:31,998 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'concurrent-pvc-74639d37'} -2025-06-05 15:51:31,998 - tests.orchestrator - INFO - - kube-api-access-97jdn: {} -2025-06-05 15:51:31,998 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-3106ab46 ===== -2025-06-05 15:52:00,875 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-3106ab46 to be ready after 60s -2025-06-05 15:52:00,876 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-3106ab46 ===== -2025-06-05 15:52:00,943 - tests.orchestrator - INFO - No container statuses available -2025-06-05 15:52:01,013 - tests.orchestrator - INFO - Pod events: -2025-06-05 15:52:01,013 - tests.orchestrator - INFO - [2025-06-05 15:51:00+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 15:52:01,079 - tests.orchestrator - INFO - No logs available -2025-06-05 15:52:01,079 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 15:52:01,079 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'concurrent-pvc-74639d37'} -2025-06-05 15:52:01,080 - tests.orchestrator - INFO - - kube-api-access-97jdn: {} -2025-06-05 15:52:01,080 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-3106ab46 ===== -2025-06-05 15:52:01,080 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-3106ab46 to be ready -2025-06-05 15:52:01,080 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-be273a95 -2025-06-05 15:52:01,080 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-be273a95 -2025-06-05 15:52:01,080 - tests.orchestrator - INFO - Deleting pod: test-pod-d257ffb1 from PVC: concurrent-pvc-be273a95 -2025-06-05 15:52:01,164 - tests.orchestrator - INFO - Waiting for pod test-pod-d257ffb1 to be deleted -2025-06-05 15:52:01,229 - tests.orchestrator - INFO - Pod test-pod-d257ffb1 has been deleted -2025-06-05 15:52:01,230 - tests.orchestrator - INFO - Deleted pod: test-pod-d257ffb1 -2025-06-05 15:52:01,301 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-be273a95 to be deleted -2025-06-05 15:52:01,366 - tests.orchestrator - INFO - PVC concurrent-pvc-be273a95 has been deleted -2025-06-05 15:52:01,367 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-be273a95 -2025-06-05 15:52:01,367 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-421903bd -2025-06-05 15:52:01,367 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-421903bd -2025-06-05 15:52:01,367 - tests.orchestrator - INFO - Deleting pod: test-pod-8c9ea815 from PVC: concurrent-pvc-421903bd -2025-06-05 15:52:01,548 - tests.orchestrator - INFO - Waiting for pod test-pod-8c9ea815 to be deleted -2025-06-05 15:52:01,774 - tests.orchestrator - INFO - Pod test-pod-8c9ea815 has been deleted -2025-06-05 15:52:01,774 - tests.orchestrator - INFO - Deleted pod: test-pod-8c9ea815 -2025-06-05 15:52:01,852 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-421903bd to be deleted -2025-06-05 15:52:01,924 - tests.orchestrator - INFO - PVC concurrent-pvc-421903bd has been deleted -2025-06-05 15:52:01,925 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-421903bd -2025-06-05 15:52:01,925 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-74639d37 -2025-06-05 15:52:01,925 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-74639d37 -2025-06-05 15:52:01,925 - tests.orchestrator - INFO - Deleting pod: test-pod-3106ab46 from PVC: concurrent-pvc-74639d37 -2025-06-05 15:52:02,021 - tests.orchestrator - INFO - Waiting for pod test-pod-3106ab46 to be deleted -2025-06-05 15:52:02,092 - tests.orchestrator - INFO - Pod test-pod-3106ab46 has been deleted -2025-06-05 15:52:02,092 - tests.orchestrator - INFO - Deleted pod: test-pod-3106ab46 -2025-06-05 15:52:02,172 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-74639d37 to be deleted -2025-06-05 15:52:02,238 - tests.orchestrator - INFO - PVC concurrent-pvc-74639d37 has been deleted -2025-06-05 15:52:02,238 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-74639d37 -2025-06-05 15:52:02,238 - tests.orchestrator - INFO - Rapid PVC scenario completed -2025-06-05 15:52:02,238 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_concurrent_pvc -2025-06-05 15:52:02,238 - tests.orchestrator - INFO - ------------------------------------------------------------ -2025-06-05 15:52:02,238 - tests.orchestrator - INFO - Deleting pod: test-pod-7abe409e from PVC: test-pvc-83f66d6c -2025-06-05 15:52:02,322 - tests.orchestrator - INFO - Waiting for pod test-pod-7abe409e to be deleted -2025-06-05 15:52:02,390 - tests.orchestrator - INFO - Pod test-pod-7abe409e has been deleted -2025-06-05 15:52:02,390 - tests.orchestrator - INFO - Deleted pod: test-pod-7abe409e -2025-06-05 15:52:02,390 - tests.orchestrator - INFO - Deleting PVC: test-pvc-7dbf87c6 -2025-06-05 15:52:02,390 - tests.orchestrator - INFO - Deleting 2 pods using PVC test-pvc-7dbf87c6 -2025-06-05 15:52:02,390 - tests.orchestrator - INFO - Deleting pod: test-pod-c3e945dc from PVC: test-pvc-7dbf87c6 -2025-06-05 15:52:02,471 - tests.orchestrator - INFO - Waiting for pod test-pod-c3e945dc to be deleted -2025-06-05 15:52:02,537 - tests.orchestrator - INFO - Pod test-pod-c3e945dc has been deleted -2025-06-05 15:52:02,537 - tests.orchestrator - INFO - Deleted pod: test-pod-c3e945dc -2025-06-05 15:52:02,537 - tests.orchestrator - INFO - Deleting pod: test-pod-a9fc1f04 from PVC: test-pvc-7dbf87c6 -2025-06-05 15:52:02,625 - tests.orchestrator - INFO - Waiting for pod test-pod-a9fc1f04 to be deleted -2025-06-05 15:52:02,691 - tests.orchestrator - INFO - Pod test-pod-a9fc1f04 has been deleted -2025-06-05 15:52:02,691 - tests.orchestrator - INFO - Deleted pod: test-pod-a9fc1f04 -2025-06-05 15:52:02,763 - tests.orchestrator - INFO - Waiting for PVC test-pvc-7dbf87c6 to be deleted -2025-06-05 15:52:02,828 - tests.orchestrator - INFO - PVC test-pvc-7dbf87c6 has been deleted -2025-06-05 15:52:02,828 - tests.orchestrator - INFO - Deleted PVC: test-pvc-7dbf87c6 -2025-06-05 15:52:02,828 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations -2025-06-05 15:52:02,828 - tests.orchestrator - INFO - Test completed in 375.88 seconds -2025-06-05 15:52:02,828 - tests.orchestrator - INFO - Cleaning up resources... -2025-06-05 15:52:02,828 - tests.orchestrator - INFO - Deleting PVC: test-pvc-83f66d6c -2025-06-05 15:52:02,903 - tests.orchestrator - INFO - Waiting for PVC test-pvc-83f66d6c to be deleted -2025-06-05 15:52:02,968 - tests.orchestrator - INFO - PVC test-pvc-83f66d6c has been deleted -2025-06-05 15:52:02,968 - tests.orchestrator - INFO - Deleted PVC: test-pvc-83f66d6c -2025-06-05 15:52:02,968 - tests.orchestrator - INFO - Cleanup completed -2025-06-05 15:52:02,968 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-05 15:52:02,968 - tests.orchestrator - INFO - --- Operations --- -2025-06-05 15:52:02,968 - tests.orchestrator - INFO - create_pvc: 5 succeeded, 0 failed (100.0%) -2025-06-05 15:52:02,968 - tests.orchestrator - INFO - attach_pod: 6 succeeded, 0 failed (100.0%) -2025-06-05 15:52:02,968 - tests.orchestrator - INFO - delete_pod: 6 succeeded, 0 failed (100.0%) -2025-06-05 15:52:02,968 - tests.orchestrator - INFO - delete_pvc: 5 succeeded, 0 failed (100.0%) -2025-06-05 15:52:02,968 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) -2025-06-05 15:52:02,968 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) -2025-06-05 15:52:02,968 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-05 15:52:02,969 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-05 15:52:02,969 - tests.orchestrator - INFO - many_to_one: No runs -2025-06-05 15:52:02,969 - tests.orchestrator - INFO - one_to_one: No runs -2025-06-05 15:52:02,969 - tests.orchestrator - INFO - concurrent_pvc: 1 succeeded, 0 failed out of 1 runs (100.0%) -2025-06-05 15:52:02,969 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_155948.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_155948.log deleted file mode 100644 index eb30e4717..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_155948.log +++ /dev/null @@ -1,280 +0,0 @@ -2025-06-05 15:59:48,809 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 15:59:49,025 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 15:59:49,027 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-05 15:59:49,218 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 15:59:49,218 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 15:59:49,218 - tests.orchestrator - INFO - Creating PVC: test-pvc-47b1c666 -2025-06-05 15:59:49,295 - tests.orchestrator - INFO - Created PVC: test-pvc-47b1c666 -2025-06-05 15:59:49,296 - tests.orchestrator - INFO - Waiting for PVC test-pvc-47b1c666 to be bound -2025-06-05 15:59:51,432 - tests.orchestrator - INFO - PVC test-pvc-47b1c666 is bound -2025-06-05 15:59:51,433 - tests.orchestrator - INFO - Creating PVC: test-pvc-13de15dd -2025-06-05 15:59:51,506 - tests.orchestrator - INFO - Created PVC: test-pvc-13de15dd -2025-06-05 15:59:51,507 - tests.orchestrator - INFO - Waiting for PVC test-pvc-13de15dd to be bound -2025-06-05 15:59:53,652 - tests.orchestrator - INFO - PVC test-pvc-13de15dd is bound -2025-06-05 15:59:53,652 - tests.orchestrator - INFO - Creating pod test-pod-b12f5a09 on PVC test-pvc-47b1c666 -2025-06-05 15:59:53,736 - tests.orchestrator - INFO - Created pod: test-pod-b12f5a09 using PVC: test-pvc-47b1c666 -2025-06-05 15:59:53,736 - tests.orchestrator - INFO - Waiting for pod test-pod-b12f5a09 to be ready -2025-06-05 15:59:53,809 - tests.orchestrator - INFO - Pod test-pod-b12f5a09 phase: Pending -2025-06-05 16:00:24,845 - tests.orchestrator - INFO - Pod test-pod-b12f5a09 taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 16:00:24,846 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-b12f5a09 ===== -2025-06-05 16:00:24,914 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:00:24,986 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:00:24,987 - tests.orchestrator - INFO - [2025-06-05 15:59:53+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:00:25,052 - tests.orchestrator - INFO - No logs available -2025-06-05 16:00:25,053 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:00:25,053 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-47b1c666'} -2025-06-05 16:00:25,053 - tests.orchestrator - INFO - - kube-api-access-q49n5: {} -2025-06-05 16:00:25,053 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-b12f5a09 ===== -2025-06-05 16:00:54,038 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b12f5a09 to be ready after 60s -2025-06-05 16:00:54,038 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-b12f5a09 ===== -2025-06-05 16:00:54,106 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:00:54,179 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:00:54,179 - tests.orchestrator - INFO - [2025-06-05 15:59:53+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:00:54,244 - tests.orchestrator - INFO - No logs available -2025-06-05 16:00:54,245 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:00:54,245 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-47b1c666'} -2025-06-05 16:00:54,245 - tests.orchestrator - INFO - - kube-api-access-q49n5: {} -2025-06-05 16:00:54,245 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-b12f5a09 ===== -2025-06-05 16:00:54,245 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b12f5a09 to be ready -2025-06-05 16:00:54,245 - tests.orchestrator - INFO - Creating pod test-pod-397db057 on PVC test-pvc-13de15dd -2025-06-05 16:00:54,335 - tests.orchestrator - INFO - Created pod: test-pod-397db057 using PVC: test-pvc-13de15dd -2025-06-05 16:00:54,336 - tests.orchestrator - INFO - Waiting for pod test-pod-397db057 to be ready -2025-06-05 16:00:54,401 - tests.orchestrator - INFO - Pod test-pod-397db057 phase: Pending -2025-06-05 16:01:25,423 - tests.orchestrator - INFO - Pod test-pod-397db057 taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 16:01:25,423 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-397db057 ===== -2025-06-05 16:01:25,491 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:01:25,563 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:01:25,563 - tests.orchestrator - INFO - [2025-06-05 16:00:54+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:01:25,629 - tests.orchestrator - INFO - No logs available -2025-06-05 16:01:25,629 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:01:25,629 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-13de15dd'} -2025-06-05 16:01:25,630 - tests.orchestrator - INFO - - kube-api-access-q4p66: {} -2025-06-05 16:01:25,630 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-397db057 ===== -2025-06-05 16:01:54,513 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-397db057 to be ready after 60s -2025-06-05 16:01:54,513 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-397db057 ===== -2025-06-05 16:01:54,581 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:01:54,652 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:01:54,653 - tests.orchestrator - INFO - [2025-06-05 16:00:54+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:01:54,717 - tests.orchestrator - INFO - No logs available -2025-06-05 16:01:54,718 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:01:54,718 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-13de15dd'} -2025-06-05 16:01:54,718 - tests.orchestrator - INFO - - kube-api-access-q4p66: {} -2025-06-05 16:01:54,718 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-397db057 ===== -2025-06-05 16:01:54,718 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-397db057 to be ready -2025-06-05 16:01:54,718 - tests.orchestrator - INFO - Creating pod test-pod-b378ede9 on PVC test-pvc-13de15dd -2025-06-05 16:01:54,802 - tests.orchestrator - INFO - Created pod: test-pod-b378ede9 using PVC: test-pvc-13de15dd -2025-06-05 16:01:54,802 - tests.orchestrator - INFO - Waiting for pod test-pod-b378ede9 to be ready -2025-06-05 16:01:54,868 - tests.orchestrator - INFO - Pod test-pod-b378ede9 phase: Pending -2025-06-05 16:02:25,904 - tests.orchestrator - INFO - Pod test-pod-b378ede9 taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 16:02:25,904 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-b378ede9 ===== -2025-06-05 16:02:25,971 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:02:26,043 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:02:26,043 - tests.orchestrator - INFO - [2025-06-05 16:01:54+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:02:26,109 - tests.orchestrator - INFO - No logs available -2025-06-05 16:02:26,110 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:02:26,110 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-13de15dd'} -2025-06-05 16:02:26,110 - tests.orchestrator - INFO - - kube-api-access-ggwzs: {} -2025-06-05 16:02:26,110 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-b378ede9 ===== -2025-06-05 16:02:54,996 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b378ede9 to be ready after 60s -2025-06-05 16:02:54,996 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-b378ede9 ===== -2025-06-05 16:02:55,065 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:02:55,137 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:02:55,137 - tests.orchestrator - INFO - [2025-06-05 16:01:54+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:02:55,204 - tests.orchestrator - INFO - No logs available -2025-06-05 16:02:55,204 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:02:55,204 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-13de15dd'} -2025-06-05 16:02:55,204 - tests.orchestrator - INFO - - kube-api-access-ggwzs: {} -2025-06-05 16:02:55,204 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-b378ede9 ===== -2025-06-05 16:02:55,205 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b378ede9 to be ready -2025-06-05 16:02:55,205 - tests.orchestrator - INFO - Testing read/write between pods test-pod-397db057 and test-pod-b378ede9 sharing PVC test-pvc-13de15dd -2025-06-05 16:02:55,270 - tests.orchestrator - ERROR - Failed in read/write verification: (400) -Reason: Bad Request -HTTP response headers: HTTPHeaderDict({'Audit-Id': 'c1161df1-c0d5-4407-a027-5613955ae526', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 16:02:55 GMT', 'Content-Length': '166'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-397db057 does not have a host assigned","reason":"BadRequest","code":400} - - -2025-06-05 16:02:55,270 - tests.orchestrator - INFO - ============================================================ -2025-06-05 16:02:55,270 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_many_to_one -2025-06-05 16:02:55,270 - tests.orchestrator - INFO - ============================================================ -2025-06-05 16:02:55,270 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -2025-06-05 16:02:55,270 - tests.orchestrator - INFO - STARTING MANY-TO-ONE SCENARIO DIAGNOSTICS -2025-06-05 16:02:55,270 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -2025-06-05 16:02:55,271 - tests.orchestrator - INFO - [MANY2ONE] STEP 1: Creating dedicated PVC: many2one-68cd2a7e -2025-06-05 16:02:55,271 - tests.orchestrator - INFO - [MANY2ONE] Creating PVC manifest with storageClassName: efs-sc -2025-06-05 16:02:55,352 - tests.orchestrator - INFO - [MANY2ONE] PVC many2one-68cd2a7e created successfully -2025-06-05 16:02:55,352 - tests.orchestrator - INFO - [MANY2ONE] Waiting for PVC many2one-68cd2a7e to be bound... -2025-06-05 16:02:55,352 - tests.orchestrator - INFO - Waiting for PVC many2one-68cd2a7e to be bound -2025-06-05 16:02:57,487 - tests.orchestrator - INFO - PVC many2one-68cd2a7e is bound -2025-06-05 16:02:57,552 - tests.orchestrator - INFO - [MANY2ONE] PVC status: phase=Bound, capacity={'storage': '1Gi'} -2025-06-05 16:02:57,552 - tests.orchestrator - INFO - [MANY2ONE] STEP 2: Creating 4 pods for the same PVC many2one-68cd2a7e -2025-06-05 16:02:57,552 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 1/4 for PVC many2one-68cd2a7e -2025-06-05 16:02:57,552 - tests.orchestrator - INFO - Creating pod test-pod-4518fc8f on PVC many2one-68cd2a7e -2025-06-05 16:02:57,632 - tests.orchestrator - INFO - Created pod: test-pod-4518fc8f using PVC: many2one-68cd2a7e -2025-06-05 16:02:57,633 - tests.orchestrator - INFO - Waiting for pod test-pod-4518fc8f to be ready -2025-06-05 16:02:57,699 - tests.orchestrator - INFO - Pod test-pod-4518fc8f phase: Pending -2025-06-05 16:03:28,731 - tests.orchestrator - INFO - Pod test-pod-4518fc8f taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 16:03:28,732 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-4518fc8f ===== -2025-06-05 16:03:28,800 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:03:28,872 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:03:28,872 - tests.orchestrator - INFO - [2025-06-05 16:02:57+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:03:28,937 - tests.orchestrator - INFO - No logs available -2025-06-05 16:03:28,937 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:03:28,937 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-68cd2a7e'} -2025-06-05 16:03:28,937 - tests.orchestrator - INFO - - kube-api-access-v9z8m: {} -2025-06-05 16:03:28,937 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-4518fc8f ===== -2025-06-05 16:03:57,829 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-4518fc8f to be ready after 60s -2025-06-05 16:03:57,829 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-4518fc8f ===== -2025-06-05 16:03:57,897 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:03:57,967 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:03:57,968 - tests.orchestrator - INFO - [2025-06-05 16:02:57+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:03:58,032 - tests.orchestrator - INFO - No logs available -2025-06-05 16:03:58,032 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:03:58,032 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-68cd2a7e'} -2025-06-05 16:03:58,032 - tests.orchestrator - INFO - - kube-api-access-v9z8m: {} -2025-06-05 16:03:58,032 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-4518fc8f ===== -2025-06-05 16:03:58,033 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-4518fc8f to be ready -2025-06-05 16:03:58,033 - tests.orchestrator - ERROR - [MANY2ONE] Failed to create pod 1/4 -2025-06-05 16:03:58,033 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 2/4 for PVC many2one-68cd2a7e -2025-06-05 16:03:58,033 - tests.orchestrator - INFO - Creating pod test-pod-0c1359ff on PVC many2one-68cd2a7e -2025-06-05 16:03:58,117 - tests.orchestrator - INFO - Created pod: test-pod-0c1359ff using PVC: many2one-68cd2a7e -2025-06-05 16:03:58,117 - tests.orchestrator - INFO - Waiting for pod test-pod-0c1359ff to be ready -2025-06-05 16:03:58,183 - tests.orchestrator - INFO - Pod test-pod-0c1359ff phase: Pending -2025-06-05 16:04:29,215 - tests.orchestrator - INFO - Pod test-pod-0c1359ff taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 16:04:29,215 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-0c1359ff ===== -2025-06-05 16:04:29,283 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:04:29,355 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:04:29,356 - tests.orchestrator - INFO - [2025-06-05 16:03:58+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:04:29,422 - tests.orchestrator - INFO - No logs available -2025-06-05 16:04:29,422 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:04:29,422 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-68cd2a7e'} -2025-06-05 16:04:29,422 - tests.orchestrator - INFO - - kube-api-access-6jtzc: {} -2025-06-05 16:04:29,422 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-0c1359ff ===== -2025-06-05 16:04:58,441 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-0c1359ff to be ready after 60s -2025-06-05 16:04:58,441 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-0c1359ff ===== -2025-06-05 16:04:58,507 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:04:58,577 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:04:58,577 - tests.orchestrator - INFO - [2025-06-05 16:03:58+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:04:58,644 - tests.orchestrator - INFO - No logs available -2025-06-05 16:04:58,644 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:04:58,644 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-68cd2a7e'} -2025-06-05 16:04:58,644 - tests.orchestrator - INFO - - kube-api-access-6jtzc: {} -2025-06-05 16:04:58,644 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-0c1359ff ===== -2025-06-05 16:04:58,644 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-0c1359ff to be ready -2025-06-05 16:04:58,644 - tests.orchestrator - ERROR - [MANY2ONE] Failed to create pod 2/4 -2025-06-05 16:04:58,644 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 3/4 for PVC many2one-68cd2a7e -2025-06-05 16:04:58,645 - tests.orchestrator - INFO - Creating pod test-pod-1e078739 on PVC many2one-68cd2a7e -2025-06-05 16:04:58,732 - tests.orchestrator - INFO - Created pod: test-pod-1e078739 using PVC: many2one-68cd2a7e -2025-06-05 16:04:58,732 - tests.orchestrator - INFO - Waiting for pod test-pod-1e078739 to be ready -2025-06-05 16:04:58,800 - tests.orchestrator - INFO - Pod test-pod-1e078739 phase: Pending -2025-06-05 16:05:29,826 - tests.orchestrator - INFO - Pod test-pod-1e078739 taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 16:05:29,826 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-1e078739 ===== -2025-06-05 16:05:29,893 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:05:29,963 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:05:29,963 - tests.orchestrator - INFO - [2025-06-05 16:04:58+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:05:30,027 - tests.orchestrator - INFO - No logs available -2025-06-05 16:05:30,027 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:05:30,028 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-68cd2a7e'} -2025-06-05 16:05:30,028 - tests.orchestrator - INFO - - kube-api-access-rksgf: {} -2025-06-05 16:05:30,028 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-1e078739 ===== -2025-06-05 16:05:58,917 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1e078739 to be ready after 60s -2025-06-05 16:05:58,917 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-1e078739 ===== -2025-06-05 16:05:58,985 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:05:59,057 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:05:59,057 - tests.orchestrator - INFO - [2025-06-05 16:04:58+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:05:59,122 - tests.orchestrator - INFO - No logs available -2025-06-05 16:05:59,122 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:05:59,122 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-68cd2a7e'} -2025-06-05 16:05:59,122 - tests.orchestrator - INFO - - kube-api-access-rksgf: {} -2025-06-05 16:05:59,122 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-1e078739 ===== -2025-06-05 16:05:59,122 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1e078739 to be ready -2025-06-05 16:05:59,122 - tests.orchestrator - ERROR - [MANY2ONE] Failed to create pod 3/4 -2025-06-05 16:05:59,122 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 4/4 for PVC many2one-68cd2a7e -2025-06-05 16:05:59,122 - tests.orchestrator - INFO - Creating pod test-pod-a5d14df4 on PVC many2one-68cd2a7e -2025-06-05 16:05:59,208 - tests.orchestrator - INFO - Created pod: test-pod-a5d14df4 using PVC: many2one-68cd2a7e -2025-06-05 16:05:59,208 - tests.orchestrator - INFO - Waiting for pod test-pod-a5d14df4 to be ready -2025-06-05 16:05:59,273 - tests.orchestrator - INFO - Pod test-pod-a5d14df4 phase: Pending -2025-06-05 16:06:30,377 - tests.orchestrator - INFO - Pod test-pod-a5d14df4 taking longer than expected to become ready (31.2s). Collecting diagnostics... -2025-06-05 16:06:30,377 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-a5d14df4 ===== -2025-06-05 16:06:30,444 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:06:30,516 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:06:30,517 - tests.orchestrator - INFO - [2025-06-05 16:05:59+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:06:30,583 - tests.orchestrator - INFO - No logs available -2025-06-05 16:06:30,583 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:06:30,584 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-68cd2a7e'} -2025-06-05 16:06:30,584 - tests.orchestrator - INFO - - kube-api-access-dcxmb: {} -2025-06-05 16:06:30,584 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-a5d14df4 ===== -2025-06-05 16:06:59,487 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a5d14df4 to be ready after 60s -2025-06-05 16:06:59,487 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-a5d14df4 ===== -2025-06-05 16:06:59,555 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:06:59,627 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:06:59,628 - tests.orchestrator - INFO - [2025-06-05 16:05:59+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:06:59,694 - tests.orchestrator - INFO - No logs available -2025-06-05 16:06:59,694 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:06:59,694 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'many2one-68cd2a7e'} -2025-06-05 16:06:59,695 - tests.orchestrator - INFO - - kube-api-access-dcxmb: {} -2025-06-05 16:06:59,695 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-a5d14df4 ===== -2025-06-05 16:06:59,695 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-a5d14df4 to be ready -2025-06-05 16:06:59,695 - tests.orchestrator - ERROR - [MANY2ONE] Failed to create pod 4/4 -2025-06-05 16:06:59,695 - tests.orchestrator - INFO - [MANY2ONE] Created 0/4 pods successfully -2025-06-05 16:06:59,695 - tests.orchestrator - ERROR - [MANY2ONE] FAILED: Insufficient pods created (0), need at least 2 for read/write test -2025-06-05 16:06:59,695 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_many_to_one -2025-06-05 16:06:59,695 - tests.orchestrator - INFO - ------------------------------------------------------------ -2025-06-05 16:06:59,695 - tests.orchestrator - INFO - Deleting pod: test-pod-1e078739 from PVC: many2one-68cd2a7e -2025-06-05 16:06:59,777 - tests.orchestrator - INFO - Waiting for pod test-pod-1e078739 to be deleted -2025-06-05 16:06:59,841 - tests.orchestrator - INFO - Pod test-pod-1e078739 has been deleted -2025-06-05 16:06:59,842 - tests.orchestrator - INFO - Deleted pod: test-pod-1e078739 -2025-06-05 16:06:59,842 - tests.orchestrator - INFO - Deleting PVC: test-pvc-13de15dd -2025-06-05 16:06:59,842 - tests.orchestrator - INFO - Deleting 2 pods using PVC test-pvc-13de15dd -2025-06-05 16:06:59,842 - tests.orchestrator - INFO - Deleting pod: test-pod-397db057 from PVC: test-pvc-13de15dd -2025-06-05 16:06:59,923 - tests.orchestrator - INFO - Waiting for pod test-pod-397db057 to be deleted -2025-06-05 16:06:59,989 - tests.orchestrator - INFO - Pod test-pod-397db057 has been deleted -2025-06-05 16:06:59,989 - tests.orchestrator - INFO - Deleted pod: test-pod-397db057 -2025-06-05 16:06:59,989 - tests.orchestrator - INFO - Deleting pod: test-pod-b378ede9 from PVC: test-pvc-13de15dd -2025-06-05 16:07:00,077 - tests.orchestrator - INFO - Waiting for pod test-pod-b378ede9 to be deleted -2025-06-05 16:07:00,142 - tests.orchestrator - INFO - Pod test-pod-b378ede9 has been deleted -2025-06-05 16:07:00,142 - tests.orchestrator - INFO - Deleted pod: test-pod-b378ede9 -2025-06-05 16:07:00,213 - tests.orchestrator - INFO - Waiting for PVC test-pvc-13de15dd to be deleted -2025-06-05 16:07:00,280 - tests.orchestrator - INFO - PVC test-pvc-13de15dd has been deleted -2025-06-05 16:07:00,280 - tests.orchestrator - INFO - Deleted PVC: test-pvc-13de15dd -2025-06-05 16:07:00,280 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations -2025-06-05 16:07:00,280 - tests.orchestrator - INFO - Test completed in 431.25 seconds -2025-06-05 16:07:00,280 - tests.orchestrator - INFO - Cleaning up resources... -2025-06-05 16:07:00,280 - tests.orchestrator - INFO - Deleting pod: test-pod-b12f5a09 from PVC: test-pvc-47b1c666 -2025-06-05 16:07:00,364 - tests.orchestrator - INFO - Waiting for pod test-pod-b12f5a09 to be deleted -2025-06-05 16:07:00,430 - tests.orchestrator - INFO - Pod test-pod-b12f5a09 has been deleted -2025-06-05 16:07:00,430 - tests.orchestrator - INFO - Deleted pod: test-pod-b12f5a09 -2025-06-05 16:07:00,431 - tests.orchestrator - INFO - Deleting pod: test-pod-4518fc8f from PVC: many2one-68cd2a7e -2025-06-05 16:07:00,512 - tests.orchestrator - INFO - Waiting for pod test-pod-4518fc8f to be deleted -2025-06-05 16:07:00,579 - tests.orchestrator - INFO - Pod test-pod-4518fc8f has been deleted -2025-06-05 16:07:00,579 - tests.orchestrator - INFO - Deleted pod: test-pod-4518fc8f -2025-06-05 16:07:00,579 - tests.orchestrator - INFO - Deleting pod: test-pod-0c1359ff from PVC: many2one-68cd2a7e -2025-06-05 16:07:00,659 - tests.orchestrator - INFO - Waiting for pod test-pod-0c1359ff to be deleted -2025-06-05 16:07:00,725 - tests.orchestrator - INFO - Pod test-pod-0c1359ff has been deleted -2025-06-05 16:07:00,726 - tests.orchestrator - INFO - Deleted pod: test-pod-0c1359ff -2025-06-05 16:07:00,726 - tests.orchestrator - INFO - Deleting pod: test-pod-a5d14df4 from PVC: many2one-68cd2a7e -2025-06-05 16:07:00,813 - tests.orchestrator - INFO - Waiting for pod test-pod-a5d14df4 to be deleted -2025-06-05 16:07:00,878 - tests.orchestrator - INFO - Pod test-pod-a5d14df4 has been deleted -2025-06-05 16:07:00,878 - tests.orchestrator - INFO - Deleted pod: test-pod-a5d14df4 -2025-06-05 16:07:00,878 - tests.orchestrator - INFO - Deleting PVC: test-pvc-47b1c666 -2025-06-05 16:07:00,951 - tests.orchestrator - INFO - Waiting for PVC test-pvc-47b1c666 to be deleted -2025-06-05 16:07:01,017 - tests.orchestrator - INFO - PVC test-pvc-47b1c666 has been deleted -2025-06-05 16:07:01,018 - tests.orchestrator - INFO - Deleted PVC: test-pvc-47b1c666 -2025-06-05 16:07:01,018 - tests.orchestrator - INFO - Deleting PVC: many2one-68cd2a7e -2025-06-05 16:07:01,090 - tests.orchestrator - INFO - Waiting for PVC many2one-68cd2a7e to be deleted -2025-06-05 16:07:01,157 - tests.orchestrator - INFO - PVC many2one-68cd2a7e has been deleted -2025-06-05 16:07:01,157 - tests.orchestrator - INFO - Deleted PVC: many2one-68cd2a7e -2025-06-05 16:07:01,157 - tests.orchestrator - INFO - Cleanup completed -2025-06-05 16:07:01,157 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-05 16:07:01,157 - tests.orchestrator - INFO - --- Operations --- -2025-06-05 16:07:01,157 - tests.orchestrator - INFO - create_pvc: 2 succeeded, 0 failed (100.0%) -2025-06-05 16:07:01,157 - tests.orchestrator - INFO - attach_pod: 7 succeeded, 0 failed (100.0%) -2025-06-05 16:07:01,157 - tests.orchestrator - INFO - delete_pod: 7 succeeded, 0 failed (100.0%) -2025-06-05 16:07:01,157 - tests.orchestrator - INFO - delete_pvc: 3 succeeded, 0 failed (100.0%) -2025-06-05 16:07:01,157 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) -2025-06-05 16:07:01,157 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) -2025-06-05 16:07:01,157 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-05 16:07:01,158 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-05 16:07:01,158 - tests.orchestrator - INFO - many_to_one: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-05 16:07:01,158 - tests.orchestrator - INFO - one_to_one: No runs -2025-06-05 16:07:01,158 - tests.orchestrator - INFO - concurrent_pvc: No runs -2025-06-05 16:07:01,158 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_160844.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_160844.log deleted file mode 100644 index 2336bab83..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_160844.log +++ /dev/null @@ -1,281 +0,0 @@ -2025-06-05 16:08:44,038 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 16:08:44,245 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 16:08:44,246 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-05 16:08:44,436 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 16:08:44,436 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 16:08:44,436 - tests.orchestrator - INFO - Creating PVC: test-pvc-6fb3132b -2025-06-05 16:08:44,509 - tests.orchestrator - INFO - Created PVC: test-pvc-6fb3132b -2025-06-05 16:08:44,510 - tests.orchestrator - INFO - Waiting for PVC test-pvc-6fb3132b to be bound -2025-06-05 16:08:46,643 - tests.orchestrator - INFO - PVC test-pvc-6fb3132b is bound -2025-06-05 16:08:46,643 - tests.orchestrator - INFO - Creating PVC: test-pvc-989dfd43 -2025-06-05 16:08:46,714 - tests.orchestrator - INFO - Created PVC: test-pvc-989dfd43 -2025-06-05 16:08:46,714 - tests.orchestrator - INFO - Waiting for PVC test-pvc-989dfd43 to be bound -2025-06-05 16:08:48,850 - tests.orchestrator - INFO - PVC test-pvc-989dfd43 is bound -2025-06-05 16:08:48,851 - tests.orchestrator - INFO - Creating pod test-pod-f073851f on PVC test-pvc-6fb3132b -2025-06-05 16:08:48,934 - tests.orchestrator - INFO - Created pod: test-pod-f073851f using PVC: test-pvc-6fb3132b -2025-06-05 16:08:48,934 - tests.orchestrator - INFO - Waiting for pod test-pod-f073851f to be ready -2025-06-05 16:08:49,001 - tests.orchestrator - INFO - Pod test-pod-f073851f phase: Pending -2025-06-05 16:09:20,151 - tests.orchestrator - INFO - Pod test-pod-f073851f taking longer than expected to become ready (31.2s). Collecting diagnostics... -2025-06-05 16:09:20,151 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-f073851f ===== -2025-06-05 16:09:20,217 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:09:20,287 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:09:20,287 - tests.orchestrator - INFO - [2025-06-05 16:08:48+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:09:20,354 - tests.orchestrator - INFO - No logs available -2025-06-05 16:09:20,354 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:09:20,354 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-6fb3132b'} -2025-06-05 16:09:20,354 - tests.orchestrator - INFO - - kube-api-access-l7zvz: {} -2025-06-05 16:09:20,355 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-f073851f ===== -2025-06-05 16:09:49,251 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-f073851f to be ready after 60s -2025-06-05 16:09:49,251 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-f073851f ===== -2025-06-05 16:09:49,317 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:09:49,389 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:09:49,390 - tests.orchestrator - INFO - [2025-06-05 16:08:48+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:09:49,456 - tests.orchestrator - INFO - No logs available -2025-06-05 16:09:49,456 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:09:49,456 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-6fb3132b'} -2025-06-05 16:09:49,456 - tests.orchestrator - INFO - - kube-api-access-l7zvz: {} -2025-06-05 16:09:49,457 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-f073851f ===== -2025-06-05 16:09:49,457 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-f073851f to be ready -2025-06-05 16:09:49,457 - tests.orchestrator - INFO - Creating pod test-pod-6b63ccc1 on PVC test-pvc-6fb3132b -2025-06-05 16:09:49,543 - tests.orchestrator - INFO - Created pod: test-pod-6b63ccc1 using PVC: test-pvc-6fb3132b -2025-06-05 16:09:49,543 - tests.orchestrator - INFO - Waiting for pod test-pod-6b63ccc1 to be ready -2025-06-05 16:09:49,612 - tests.orchestrator - INFO - Pod test-pod-6b63ccc1 phase: Pending -2025-06-05 16:10:20,641 - tests.orchestrator - INFO - Pod test-pod-6b63ccc1 taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 16:10:20,641 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-6b63ccc1 ===== -2025-06-05 16:10:20,710 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:10:20,782 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:10:20,782 - tests.orchestrator - INFO - [2025-06-05 16:09:49+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:10:20,849 - tests.orchestrator - INFO - No logs available -2025-06-05 16:10:20,849 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:10:20,849 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-6fb3132b'} -2025-06-05 16:10:20,849 - tests.orchestrator - INFO - - kube-api-access-8pzqd: {} -2025-06-05 16:10:20,849 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-6b63ccc1 ===== -2025-06-05 16:10:49,843 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-6b63ccc1 to be ready after 60s -2025-06-05 16:10:49,843 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-6b63ccc1 ===== -2025-06-05 16:10:49,909 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:10:49,979 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:10:49,979 - tests.orchestrator - INFO - [2025-06-05 16:09:49+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:10:50,046 - tests.orchestrator - INFO - No logs available -2025-06-05 16:10:50,046 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:10:50,046 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-6fb3132b'} -2025-06-05 16:10:50,046 - tests.orchestrator - INFO - - kube-api-access-8pzqd: {} -2025-06-05 16:10:50,046 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-6b63ccc1 ===== -2025-06-05 16:10:50,046 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-6b63ccc1 to be ready -2025-06-05 16:10:50,046 - tests.orchestrator - INFO - Creating pod test-pod-91cf9293 on PVC test-pvc-989dfd43 -2025-06-05 16:10:50,129 - tests.orchestrator - INFO - Created pod: test-pod-91cf9293 using PVC: test-pvc-989dfd43 -2025-06-05 16:10:50,129 - tests.orchestrator - INFO - Waiting for pod test-pod-91cf9293 to be ready -2025-06-05 16:10:50,197 - tests.orchestrator - INFO - Pod test-pod-91cf9293 phase: Pending -2025-06-05 16:11:21,219 - tests.orchestrator - INFO - Pod test-pod-91cf9293 taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 16:11:21,219 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-91cf9293 ===== -2025-06-05 16:11:21,287 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:11:21,355 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:11:21,356 - tests.orchestrator - INFO - [2025-06-05 16:10:50+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:11:21,422 - tests.orchestrator - INFO - No logs available -2025-06-05 16:11:21,422 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:11:21,422 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-989dfd43'} -2025-06-05 16:11:21,423 - tests.orchestrator - INFO - - kube-api-access-z69bk: {} -2025-06-05 16:11:21,423 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-91cf9293 ===== -2025-06-05 16:11:50,358 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-91cf9293 to be ready after 60s -2025-06-05 16:11:50,358 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-91cf9293 ===== -2025-06-05 16:11:50,425 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:11:50,494 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:11:50,495 - tests.orchestrator - INFO - [2025-06-05 16:10:50+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:11:50,561 - tests.orchestrator - INFO - No logs available -2025-06-05 16:11:50,561 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:11:50,561 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-989dfd43'} -2025-06-05 16:11:50,561 - tests.orchestrator - INFO - - kube-api-access-z69bk: {} -2025-06-05 16:11:50,561 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-91cf9293 ===== -2025-06-05 16:11:50,561 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-91cf9293 to be ready -2025-06-05 16:11:50,562 - tests.orchestrator - INFO - Testing read/write between pods test-pod-6b63ccc1 and test-pod-f073851f sharing PVC test-pvc-6fb3132b -2025-06-05 16:11:50,628 - tests.orchestrator - ERROR - Failed in read/write verification: (400) -Reason: Bad Request -HTTP response headers: HTTPHeaderDict({'Audit-Id': 'ced94a61-feed-4098-84b1-2d400a53a883', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 16:11:50 GMT', 'Content-Length': '166'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-6b63ccc1 does not have a host assigned","reason":"BadRequest","code":400} - - -2025-06-05 16:11:50,629 - tests.orchestrator - INFO - ============================================================ -2025-06-05 16:11:50,629 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_concurrent_pvc -2025-06-05 16:11:50,629 - tests.orchestrator - INFO - ============================================================ -2025-06-05 16:11:50,629 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations -2025-06-05 16:11:50,629 - tests.orchestrator - INFO - Creating 7 PVCs in quick succession -2025-06-05 16:11:50,709 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-7711078c -2025-06-05 16:11:50,710 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-7711078c to be bound -2025-06-05 16:11:52,845 - tests.orchestrator - INFO - PVC concurrent-pvc-7711078c is bound -2025-06-05 16:11:52,913 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-4da2d71f -2025-06-05 16:11:52,913 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-4da2d71f to be bound -2025-06-05 16:11:55,058 - tests.orchestrator - INFO - PVC concurrent-pvc-4da2d71f is bound -2025-06-05 16:11:55,126 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-58107a6b -2025-06-05 16:11:55,126 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-58107a6b to be bound -2025-06-05 16:11:57,266 - tests.orchestrator - INFO - PVC concurrent-pvc-58107a6b is bound -2025-06-05 16:11:57,333 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-0bbb89d4 -2025-06-05 16:11:57,333 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-0bbb89d4 to be bound -2025-06-05 16:11:59,469 - tests.orchestrator - INFO - PVC concurrent-pvc-0bbb89d4 is bound -2025-06-05 16:11:59,537 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-aad17783 -2025-06-05 16:11:59,537 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-aad17783 to be bound -2025-06-05 16:12:01,702 - tests.orchestrator - INFO - PVC concurrent-pvc-aad17783 is bound -2025-06-05 16:12:01,782 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-792a8a18 -2025-06-05 16:12:01,782 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-792a8a18 to be bound -2025-06-05 16:12:03,930 - tests.orchestrator - INFO - PVC concurrent-pvc-792a8a18 is bound -2025-06-05 16:12:03,999 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-95d0108f -2025-06-05 16:12:03,999 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-95d0108f to be bound -2025-06-05 16:12:06,134 - tests.orchestrator - INFO - PVC concurrent-pvc-95d0108f is bound -2025-06-05 16:12:06,134 - tests.orchestrator - INFO - Successfully created 7 PVCs -2025-06-05 16:12:06,134 - tests.orchestrator - INFO - Creating pod test-pod-2497c925 on PVC concurrent-pvc-58107a6b -2025-06-05 16:12:06,207 - tests.orchestrator - INFO - Created pod: test-pod-2497c925 using PVC: concurrent-pvc-58107a6b -2025-06-05 16:12:06,207 - tests.orchestrator - INFO - Waiting for pod test-pod-2497c925 to be ready -2025-06-05 16:12:06,275 - tests.orchestrator - INFO - Pod test-pod-2497c925 phase: Pending -2025-06-05 16:12:37,298 - tests.orchestrator - INFO - Pod test-pod-2497c925 taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 16:12:37,298 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-2497c925 ===== -2025-06-05 16:12:37,365 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:12:37,437 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:12:37,437 - tests.orchestrator - INFO - [2025-06-05 16:12:06+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:12:37,504 - tests.orchestrator - INFO - No logs available -2025-06-05 16:12:37,504 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:12:37,504 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'concurrent-pvc-58107a6b'} -2025-06-05 16:12:37,504 - tests.orchestrator - INFO - - kube-api-access-bfxtj: {} -2025-06-05 16:12:37,504 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-2497c925 ===== -2025-06-05 16:13:06,471 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-2497c925 to be ready after 60s -2025-06-05 16:13:06,471 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-2497c925 ===== -2025-06-05 16:13:06,541 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:13:06,616 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:13:06,616 - tests.orchestrator - INFO - [2025-06-05 16:12:06+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:13:06,684 - tests.orchestrator - INFO - No logs available -2025-06-05 16:13:06,684 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:13:06,685 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'concurrent-pvc-58107a6b'} -2025-06-05 16:13:06,685 - tests.orchestrator - INFO - - kube-api-access-bfxtj: {} -2025-06-05 16:13:06,685 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-2497c925 ===== -2025-06-05 16:13:06,685 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-2497c925 to be ready -2025-06-05 16:13:06,685 - tests.orchestrator - INFO - Creating pod test-pod-3bfa61d5 on PVC concurrent-pvc-4da2d71f -2025-06-05 16:13:06,776 - tests.orchestrator - INFO - Created pod: test-pod-3bfa61d5 using PVC: concurrent-pvc-4da2d71f -2025-06-05 16:13:06,777 - tests.orchestrator - INFO - Waiting for pod test-pod-3bfa61d5 to be ready -2025-06-05 16:13:06,847 - tests.orchestrator - INFO - Pod test-pod-3bfa61d5 phase: Pending -2025-06-05 16:13:37,879 - tests.orchestrator - INFO - Pod test-pod-3bfa61d5 taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 16:13:37,879 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-3bfa61d5 ===== -2025-06-05 16:13:37,945 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:13:38,018 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:13:38,018 - tests.orchestrator - INFO - [2025-06-05 16:13:06+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:13:38,085 - tests.orchestrator - INFO - No logs available -2025-06-05 16:13:38,085 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:13:38,085 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'concurrent-pvc-4da2d71f'} -2025-06-05 16:13:38,086 - tests.orchestrator - INFO - - kube-api-access-8b465: {} -2025-06-05 16:13:38,086 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-3bfa61d5 ===== -2025-06-05 16:14:06,978 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-3bfa61d5 to be ready after 60s -2025-06-05 16:14:06,978 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-3bfa61d5 ===== -2025-06-05 16:14:07,046 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:14:07,118 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:14:07,119 - tests.orchestrator - INFO - [2025-06-05 16:13:06+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:14:07,186 - tests.orchestrator - INFO - No logs available -2025-06-05 16:14:07,186 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:14:07,186 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'concurrent-pvc-4da2d71f'} -2025-06-05 16:14:07,186 - tests.orchestrator - INFO - - kube-api-access-8b465: {} -2025-06-05 16:14:07,186 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-3bfa61d5 ===== -2025-06-05 16:14:07,186 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-3bfa61d5 to be ready -2025-06-05 16:14:07,186 - tests.orchestrator - INFO - Creating pod test-pod-d5f1e1e8 on PVC concurrent-pvc-792a8a18 -2025-06-05 16:14:07,273 - tests.orchestrator - INFO - Created pod: test-pod-d5f1e1e8 using PVC: concurrent-pvc-792a8a18 -2025-06-05 16:14:07,273 - tests.orchestrator - INFO - Waiting for pod test-pod-d5f1e1e8 to be ready -2025-06-05 16:14:07,342 - tests.orchestrator - INFO - Pod test-pod-d5f1e1e8 phase: Pending -2025-06-05 16:14:38,491 - tests.orchestrator - INFO - Pod test-pod-d5f1e1e8 taking longer than expected to become ready (31.2s). Collecting diagnostics... -2025-06-05 16:14:38,491 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-d5f1e1e8 ===== -2025-06-05 16:14:38,557 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:14:38,629 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:14:38,629 - tests.orchestrator - INFO - [2025-06-05 16:14:07+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:14:38,696 - tests.orchestrator - INFO - No logs available -2025-06-05 16:14:38,696 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:14:38,696 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'concurrent-pvc-792a8a18'} -2025-06-05 16:14:38,696 - tests.orchestrator - INFO - - kube-api-access-7h2vf: {} -2025-06-05 16:14:38,696 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-d5f1e1e8 ===== -2025-06-05 16:15:07,592 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-d5f1e1e8 to be ready after 60s -2025-06-05 16:15:07,593 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-d5f1e1e8 ===== -2025-06-05 16:15:07,662 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:15:07,735 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:15:07,735 - tests.orchestrator - INFO - [2025-06-05 16:14:07+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:15:07,803 - tests.orchestrator - INFO - No logs available -2025-06-05 16:15:07,803 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:15:07,804 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'concurrent-pvc-792a8a18'} -2025-06-05 16:15:07,804 - tests.orchestrator - INFO - - kube-api-access-7h2vf: {} -2025-06-05 16:15:07,804 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-d5f1e1e8 ===== -2025-06-05 16:15:07,804 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-d5f1e1e8 to be ready -2025-06-05 16:15:07,804 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-58107a6b -2025-06-05 16:15:07,804 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-58107a6b -2025-06-05 16:15:07,804 - tests.orchestrator - INFO - Deleting pod: test-pod-2497c925 from PVC: concurrent-pvc-58107a6b -2025-06-05 16:15:07,888 - tests.orchestrator - INFO - Waiting for pod test-pod-2497c925 to be deleted -2025-06-05 16:15:07,953 - tests.orchestrator - INFO - Pod test-pod-2497c925 has been deleted -2025-06-05 16:15:07,953 - tests.orchestrator - INFO - Deleted pod: test-pod-2497c925 -2025-06-05 16:15:08,028 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-58107a6b to be deleted -2025-06-05 16:15:08,107 - tests.orchestrator - INFO - PVC concurrent-pvc-58107a6b has been deleted -2025-06-05 16:15:08,108 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-58107a6b -2025-06-05 16:15:08,108 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-4da2d71f -2025-06-05 16:15:08,108 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-4da2d71f -2025-06-05 16:15:08,108 - tests.orchestrator - INFO - Deleting pod: test-pod-3bfa61d5 from PVC: concurrent-pvc-4da2d71f -2025-06-05 16:15:08,193 - tests.orchestrator - INFO - Waiting for pod test-pod-3bfa61d5 to be deleted -2025-06-05 16:15:08,258 - tests.orchestrator - INFO - Pod test-pod-3bfa61d5 has been deleted -2025-06-05 16:15:08,258 - tests.orchestrator - INFO - Deleted pod: test-pod-3bfa61d5 -2025-06-05 16:15:08,332 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-4da2d71f to be deleted -2025-06-05 16:15:08,397 - tests.orchestrator - INFO - PVC concurrent-pvc-4da2d71f has been deleted -2025-06-05 16:15:08,397 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-4da2d71f -2025-06-05 16:15:08,397 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-792a8a18 -2025-06-05 16:15:08,397 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-792a8a18 -2025-06-05 16:15:08,397 - tests.orchestrator - INFO - Deleting pod: test-pod-d5f1e1e8 from PVC: concurrent-pvc-792a8a18 -2025-06-05 16:15:08,484 - tests.orchestrator - INFO - Waiting for pod test-pod-d5f1e1e8 to be deleted -2025-06-05 16:15:08,549 - tests.orchestrator - INFO - Pod test-pod-d5f1e1e8 has been deleted -2025-06-05 16:15:08,549 - tests.orchestrator - INFO - Deleted pod: test-pod-d5f1e1e8 -2025-06-05 16:15:08,623 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-792a8a18 to be deleted -2025-06-05 16:15:08,689 - tests.orchestrator - INFO - PVC concurrent-pvc-792a8a18 has been deleted -2025-06-05 16:15:08,689 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-792a8a18 -2025-06-05 16:15:08,689 - tests.orchestrator - INFO - Rapid PVC scenario completed -2025-06-05 16:15:08,689 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_concurrent_pvc -2025-06-05 16:15:08,689 - tests.orchestrator - INFO - ------------------------------------------------------------ -2025-06-05 16:15:08,689 - tests.orchestrator - INFO - Deleting pod: test-pod-f073851f from PVC: test-pvc-6fb3132b -2025-06-05 16:15:08,770 - tests.orchestrator - INFO - Waiting for pod test-pod-f073851f to be deleted -2025-06-05 16:15:08,837 - tests.orchestrator - INFO - Pod test-pod-f073851f has been deleted -2025-06-05 16:15:08,837 - tests.orchestrator - INFO - Deleted pod: test-pod-f073851f -2025-06-05 16:15:08,837 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-aad17783 -2025-06-05 16:15:08,912 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-aad17783 to be deleted -2025-06-05 16:15:08,977 - tests.orchestrator - INFO - PVC concurrent-pvc-aad17783 has been deleted -2025-06-05 16:15:08,977 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-aad17783 -2025-06-05 16:15:08,977 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations -2025-06-05 16:15:08,977 - tests.orchestrator - INFO - Test completed in 384.73 seconds -2025-06-05 16:15:08,977 - tests.orchestrator - INFO - Cleaning up resources... -2025-06-05 16:15:08,977 - tests.orchestrator - INFO - Deleting pod: test-pod-6b63ccc1 from PVC: test-pvc-6fb3132b -2025-06-05 16:15:09,060 - tests.orchestrator - INFO - Waiting for pod test-pod-6b63ccc1 to be deleted -2025-06-05 16:15:09,127 - tests.orchestrator - INFO - Pod test-pod-6b63ccc1 has been deleted -2025-06-05 16:15:09,127 - tests.orchestrator - INFO - Deleted pod: test-pod-6b63ccc1 -2025-06-05 16:15:09,127 - tests.orchestrator - INFO - Deleting pod: test-pod-91cf9293 from PVC: test-pvc-989dfd43 -2025-06-05 16:15:09,214 - tests.orchestrator - INFO - Waiting for pod test-pod-91cf9293 to be deleted -2025-06-05 16:15:09,280 - tests.orchestrator - INFO - Pod test-pod-91cf9293 has been deleted -2025-06-05 16:15:09,280 - tests.orchestrator - INFO - Deleted pod: test-pod-91cf9293 -2025-06-05 16:15:09,280 - tests.orchestrator - INFO - Deleting PVC: test-pvc-6fb3132b -2025-06-05 16:15:09,355 - tests.orchestrator - INFO - Waiting for PVC test-pvc-6fb3132b to be deleted -2025-06-05 16:15:09,420 - tests.orchestrator - INFO - PVC test-pvc-6fb3132b has been deleted -2025-06-05 16:15:09,420 - tests.orchestrator - INFO - Deleted PVC: test-pvc-6fb3132b -2025-06-05 16:15:09,420 - tests.orchestrator - INFO - Deleting PVC: test-pvc-989dfd43 -2025-06-05 16:15:09,496 - tests.orchestrator - INFO - Waiting for PVC test-pvc-989dfd43 to be deleted -2025-06-05 16:15:09,561 - tests.orchestrator - INFO - PVC test-pvc-989dfd43 has been deleted -2025-06-05 16:15:09,561 - tests.orchestrator - INFO - Deleted PVC: test-pvc-989dfd43 -2025-06-05 16:15:09,561 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-7711078c -2025-06-05 16:15:09,635 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-7711078c to be deleted -2025-06-05 16:15:09,700 - tests.orchestrator - INFO - PVC concurrent-pvc-7711078c has been deleted -2025-06-05 16:15:09,701 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-7711078c -2025-06-05 16:15:09,701 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-0bbb89d4 -2025-06-05 16:15:09,775 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-0bbb89d4 to be deleted -2025-06-05 16:15:09,840 - tests.orchestrator - INFO - PVC concurrent-pvc-0bbb89d4 has been deleted -2025-06-05 16:15:09,840 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-0bbb89d4 -2025-06-05 16:15:09,840 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-95d0108f -2025-06-05 16:15:09,913 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-95d0108f to be deleted -2025-06-05 16:15:09,978 - tests.orchestrator - INFO - PVC concurrent-pvc-95d0108f has been deleted -2025-06-05 16:15:09,978 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-95d0108f -2025-06-05 16:15:09,978 - tests.orchestrator - INFO - Cleanup completed -2025-06-05 16:15:09,978 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-05 16:15:09,978 - tests.orchestrator - INFO - --- Operations --- -2025-06-05 16:15:09,978 - tests.orchestrator - INFO - create_pvc: 9 succeeded, 0 failed (100.0%) -2025-06-05 16:15:09,979 - tests.orchestrator - INFO - attach_pod: 6 succeeded, 0 failed (100.0%) -2025-06-05 16:15:09,979 - tests.orchestrator - INFO - delete_pod: 6 succeeded, 0 failed (100.0%) -2025-06-05 16:15:09,979 - tests.orchestrator - INFO - delete_pvc: 9 succeeded, 0 failed (100.0%) -2025-06-05 16:15:09,979 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) -2025-06-05 16:15:09,979 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) -2025-06-05 16:15:09,979 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-05 16:15:09,979 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-05 16:15:09,979 - tests.orchestrator - INFO - many_to_one: No runs -2025-06-05 16:15:09,979 - tests.orchestrator - INFO - one_to_one: No runs -2025-06-05 16:15:09,979 - tests.orchestrator - INFO - concurrent_pvc: 1 succeeded, 0 failed out of 1 runs (100.0%) -2025-06-05 16:15:09,979 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_162134.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_162134.log deleted file mode 100644 index 02712b5c3..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_162134.log +++ /dev/null @@ -1,7 +0,0 @@ -2025-06-05 16:21:34,044 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 16:21:34,256 - tests.orchestrator - ERROR - Error checking namespace: (401) -Reason: Unauthorized -HTTP response headers: HTTPHeaderDict({'Audit-Id': 'ae5fa875-33f1-44a7-a754-80136c368fa1', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 16:21:34 GMT', 'Content-Length': '129'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} - - diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_162155.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_162155.log deleted file mode 100644 index e7d55845b..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_162155.log +++ /dev/null @@ -1,52 +0,0 @@ -2025-06-05 16:21:55,920 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 16:21:56,123 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 16:21:56,123 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-05 16:21:56,313 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 16:21:56,313 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 16:21:56,313 - tests.orchestrator - INFO - Creating PVC: test-pvc-a253186e -2025-06-05 16:21:56,391 - tests.orchestrator - INFO - Created PVC: test-pvc-a253186e -2025-06-05 16:21:56,391 - tests.orchestrator - INFO - Waiting for PVC test-pvc-a253186e to be bound -2025-06-05 16:21:58,525 - tests.orchestrator - INFO - PVC test-pvc-a253186e is bound -2025-06-05 16:21:58,526 - tests.orchestrator - INFO - Creating PVC: test-pvc-98b7ae8a -2025-06-05 16:21:58,593 - tests.orchestrator - INFO - Created PVC: test-pvc-98b7ae8a -2025-06-05 16:21:58,594 - tests.orchestrator - INFO - Waiting for PVC test-pvc-98b7ae8a to be bound -2025-06-05 16:22:00,726 - tests.orchestrator - INFO - PVC test-pvc-98b7ae8a is bound -2025-06-05 16:22:00,726 - tests.orchestrator - INFO - Creating pod test-pod-955db556 on PVC test-pvc-a253186e -2025-06-05 16:22:00,807 - tests.orchestrator - INFO - Created pod: test-pod-955db556 using PVC: test-pvc-a253186e -2025-06-05 16:22:00,807 - tests.orchestrator - INFO - Waiting for pod test-pod-955db556 to be ready -2025-06-05 16:22:00,874 - tests.orchestrator - INFO - Pod test-pod-955db556 phase: Pending -2025-06-05 16:22:31,890 - tests.orchestrator - INFO - Pod test-pod-955db556 taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 16:22:31,890 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-955db556 ===== -2025-06-05 16:22:31,957 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:22:32,031 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:22:32,031 - tests.orchestrator - INFO - [2025-06-05 16:22:00+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:22:32,103 - tests.orchestrator - INFO - No logs available -2025-06-05 16:22:32,103 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:22:32,104 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-a253186e'} -2025-06-05 16:22:32,104 - tests.orchestrator - INFO - - kube-api-access-tlm4p: {} -2025-06-05 16:22:32,104 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-955db556 ===== -2025-06-05 16:23:00,974 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-955db556 to be ready after 60s -2025-06-05 16:23:00,974 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-955db556 ===== -2025-06-05 16:23:01,041 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:23:01,110 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:23:01,110 - tests.orchestrator - INFO - [2025-06-05 16:22:00+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:23:01,176 - tests.orchestrator - INFO - No logs available -2025-06-05 16:23:01,176 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:23:01,176 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-a253186e'} -2025-06-05 16:23:01,176 - tests.orchestrator - INFO - - kube-api-access-tlm4p: {} -2025-06-05 16:23:01,176 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-955db556 ===== -2025-06-05 16:23:01,176 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-955db556 to be ready -2025-06-05 16:23:01,177 - tests.orchestrator - INFO - Creating pod test-pod-28a31e9a on PVC test-pvc-98b7ae8a -2025-06-05 16:23:01,259 - tests.orchestrator - INFO - Created pod: test-pod-28a31e9a using PVC: test-pvc-98b7ae8a -2025-06-05 16:23:01,259 - tests.orchestrator - INFO - Waiting for pod test-pod-28a31e9a to be ready -2025-06-05 16:23:01,326 - tests.orchestrator - INFO - Pod test-pod-28a31e9a phase: Pending -2025-06-05 16:23:32,335 - tests.orchestrator - INFO - Pod test-pod-28a31e9a taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 16:23:32,335 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-28a31e9a ===== -2025-06-05 16:23:32,410 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:23:32,482 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:23:32,483 - tests.orchestrator - INFO - [2025-06-05 16:23:01+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:23:32,550 - tests.orchestrator - INFO - No logs available -2025-06-05 16:23:32,550 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:23:32,550 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-98b7ae8a'} -2025-06-05 16:23:32,550 - tests.orchestrator - INFO - - kube-api-access-gfrq6: {} -2025-06-05 16:23:32,550 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-28a31e9a ===== diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_162320.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_162320.log deleted file mode 100644 index e7442a34e..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_162320.log +++ /dev/null @@ -1,27 +0,0 @@ -2025-06-05 16:23:20,930 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 16:23:21,135 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 16:23:21,135 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-05 16:23:21,326 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 16:23:21,326 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 16:23:21,326 - tests.orchestrator - INFO - Creating PVC: test-pvc-3f880237 -2025-06-05 16:23:21,404 - tests.orchestrator - INFO - Created PVC: test-pvc-3f880237 -2025-06-05 16:23:21,404 - tests.orchestrator - INFO - Waiting for PVC test-pvc-3f880237 to be bound -2025-06-05 16:23:23,540 - tests.orchestrator - INFO - PVC test-pvc-3f880237 is bound -2025-06-05 16:23:23,540 - tests.orchestrator - INFO - Creating PVC: test-pvc-e3594f67 -2025-06-05 16:23:23,611 - tests.orchestrator - INFO - Created PVC: test-pvc-e3594f67 -2025-06-05 16:23:23,611 - tests.orchestrator - INFO - Waiting for PVC test-pvc-e3594f67 to be bound -2025-06-05 16:23:25,746 - tests.orchestrator - INFO - PVC test-pvc-e3594f67 is bound -2025-06-05 16:23:25,747 - tests.orchestrator - INFO - Creating pod test-pod-0fcf8a5b on PVC test-pvc-3f880237 -2025-06-05 16:23:25,829 - tests.orchestrator - INFO - Created pod: test-pod-0fcf8a5b using PVC: test-pvc-3f880237 -2025-06-05 16:23:25,829 - tests.orchestrator - INFO - Waiting for pod test-pod-0fcf8a5b to be ready -2025-06-05 16:23:25,895 - tests.orchestrator - INFO - Pod test-pod-0fcf8a5b phase: Pending -2025-06-05 16:23:56,949 - tests.orchestrator - INFO - Pod test-pod-0fcf8a5b taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 16:23:56,950 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-0fcf8a5b ===== -2025-06-05 16:23:57,018 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:23:57,090 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:23:57,090 - tests.orchestrator - INFO - [2025-06-05 16:23:25+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:23:57,157 - tests.orchestrator - INFO - No logs available -2025-06-05 16:23:57,157 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:23:57,157 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-3f880237'} -2025-06-05 16:23:57,157 - tests.orchestrator - INFO - - kube-api-access-pj9g4: {} -2025-06-05 16:23:57,157 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-0fcf8a5b ===== diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_162354.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_162354.log deleted file mode 100644 index 9c680a51e..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_162354.log +++ /dev/null @@ -1,247 +0,0 @@ -2025-06-05 16:23:54,549 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 16:23:54,744 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 16:23:54,744 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-05 16:23:54,939 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 16:23:54,939 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 16:23:54,939 - tests.orchestrator - INFO - Creating PVC: test-pvc-d9e565cc -2025-06-05 16:23:55,018 - tests.orchestrator - INFO - Created PVC: test-pvc-d9e565cc -2025-06-05 16:23:55,018 - tests.orchestrator - INFO - Waiting for PVC test-pvc-d9e565cc to be bound -2025-06-05 16:23:57,153 - tests.orchestrator - INFO - PVC test-pvc-d9e565cc is bound -2025-06-05 16:23:57,153 - tests.orchestrator - INFO - Creating PVC: test-pvc-7efe733c -2025-06-05 16:23:57,224 - tests.orchestrator - INFO - Created PVC: test-pvc-7efe733c -2025-06-05 16:23:57,224 - tests.orchestrator - INFO - Waiting for PVC test-pvc-7efe733c to be bound -2025-06-05 16:23:59,358 - tests.orchestrator - INFO - PVC test-pvc-7efe733c is bound -2025-06-05 16:23:59,359 - tests.orchestrator - INFO - Creating pod test-pod-dcc5bed9 on PVC test-pvc-7efe733c -2025-06-05 16:23:59,435 - tests.orchestrator - INFO - Created pod: test-pod-dcc5bed9 using PVC: test-pvc-7efe733c -2025-06-05 16:23:59,435 - tests.orchestrator - INFO - Waiting for pod test-pod-dcc5bed9 to be ready -2025-06-05 16:23:59,501 - tests.orchestrator - INFO - Pod test-pod-dcc5bed9 phase: Pending -2025-06-05 16:24:30,649 - tests.orchestrator - INFO - Pod test-pod-dcc5bed9 taking longer than expected to become ready (31.2s). Collecting diagnostics... -2025-06-05 16:24:30,649 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-dcc5bed9 ===== -2025-06-05 16:24:30,717 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:24:30,789 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:24:30,790 - tests.orchestrator - INFO - [2025-06-05 16:23:59+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:24:30,854 - tests.orchestrator - INFO - No logs available -2025-06-05 16:24:30,854 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:24:30,854 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-7efe733c'} -2025-06-05 16:24:30,854 - tests.orchestrator - INFO - - kube-api-access-b8bk8: {} -2025-06-05 16:24:30,854 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-dcc5bed9 ===== -2025-06-05 16:24:59,749 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-dcc5bed9 to be ready after 60s -2025-06-05 16:24:59,749 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-dcc5bed9 ===== -2025-06-05 16:24:59,815 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:24:59,888 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:24:59,888 - tests.orchestrator - INFO - [2025-06-05 16:23:59+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:24:59,954 - tests.orchestrator - INFO - No logs available -2025-06-05 16:24:59,954 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:24:59,955 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-7efe733c'} -2025-06-05 16:24:59,955 - tests.orchestrator - INFO - - kube-api-access-b8bk8: {} -2025-06-05 16:24:59,955 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-dcc5bed9 ===== -2025-06-05 16:24:59,955 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-dcc5bed9 to be ready -2025-06-05 16:24:59,955 - tests.orchestrator - INFO - Creating pod test-pod-9af93280 on PVC test-pvc-7efe733c -2025-06-05 16:25:00,043 - tests.orchestrator - INFO - Created pod: test-pod-9af93280 using PVC: test-pvc-7efe733c -2025-06-05 16:25:00,043 - tests.orchestrator - INFO - Waiting for pod test-pod-9af93280 to be ready -2025-06-05 16:25:00,113 - tests.orchestrator - INFO - Pod test-pod-9af93280 phase: Pending -2025-06-05 16:25:31,135 - tests.orchestrator - INFO - Pod test-pod-9af93280 taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 16:25:31,135 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-9af93280 ===== -2025-06-05 16:25:31,201 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:25:31,273 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:25:31,273 - tests.orchestrator - INFO - [2025-06-05 16:25:00+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:25:31,339 - tests.orchestrator - INFO - No logs available -2025-06-05 16:25:31,340 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:25:31,340 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-7efe733c'} -2025-06-05 16:25:31,340 - tests.orchestrator - INFO - - kube-api-access-6qzmx: {} -2025-06-05 16:25:31,340 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-9af93280 ===== -2025-06-05 16:26:00,299 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9af93280 to be ready after 60s -2025-06-05 16:26:00,299 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-9af93280 ===== -2025-06-05 16:26:00,365 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:26:00,436 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:26:00,436 - tests.orchestrator - INFO - [2025-06-05 16:25:00+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:26:00,504 - tests.orchestrator - INFO - No logs available -2025-06-05 16:26:00,504 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:26:00,504 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-7efe733c'} -2025-06-05 16:26:00,504 - tests.orchestrator - INFO - - kube-api-access-6qzmx: {} -2025-06-05 16:26:00,504 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-9af93280 ===== -2025-06-05 16:26:00,504 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9af93280 to be ready -2025-06-05 16:26:00,504 - tests.orchestrator - INFO - Creating pod test-pod-5feeb276 on PVC test-pvc-7efe733c -2025-06-05 16:26:00,589 - tests.orchestrator - INFO - Created pod: test-pod-5feeb276 using PVC: test-pvc-7efe733c -2025-06-05 16:26:00,590 - tests.orchestrator - INFO - Waiting for pod test-pod-5feeb276 to be ready -2025-06-05 16:26:00,657 - tests.orchestrator - INFO - Pod test-pod-5feeb276 phase: Pending -2025-06-05 16:26:31,674 - tests.orchestrator - INFO - Pod test-pod-5feeb276 taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 16:26:31,674 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-5feeb276 ===== -2025-06-05 16:26:31,743 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:26:31,818 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:26:31,818 - tests.orchestrator - INFO - [2025-06-05 16:26:00+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:26:31,884 - tests.orchestrator - INFO - No logs available -2025-06-05 16:26:31,884 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:26:31,884 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-7efe733c'} -2025-06-05 16:26:31,884 - tests.orchestrator - INFO - - kube-api-access-78f5j: {} -2025-06-05 16:26:31,884 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-5feeb276 ===== -2025-06-05 16:27:00,780 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-5feeb276 to be ready after 60s -2025-06-05 16:27:00,781 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-5feeb276 ===== -2025-06-05 16:27:00,848 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:27:00,917 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:27:00,917 - tests.orchestrator - INFO - [2025-06-05 16:26:00+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:27:00,982 - tests.orchestrator - INFO - No logs available -2025-06-05 16:27:00,982 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:27:00,982 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-7efe733c'} -2025-06-05 16:27:00,982 - tests.orchestrator - INFO - - kube-api-access-78f5j: {} -2025-06-05 16:27:00,982 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-5feeb276 ===== -2025-06-05 16:27:00,982 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-5feeb276 to be ready -2025-06-05 16:27:00,982 - tests.orchestrator - INFO - Testing read/write between pods test-pod-5feeb276 and test-pod-dcc5bed9 sharing PVC test-pvc-7efe733c -2025-06-05 16:27:01,050 - tests.orchestrator - ERROR - Failed in read/write verification: (400) -Reason: Bad Request -HTTP response headers: HTTPHeaderDict({'Audit-Id': '957ef347-88e9-4e31-8db6-db8deac109f5', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 16:27:01 GMT', 'Content-Length': '166'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-5feeb276 does not have a host assigned","reason":"BadRequest","code":400} - - -2025-06-05 16:27:01,050 - tests.orchestrator - INFO - ============================================================ -2025-06-05 16:27:01,051 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_one_to_one -2025-06-05 16:27:01,051 - tests.orchestrator - INFO - ============================================================ -2025-06-05 16:27:01,051 - tests.orchestrator - INFO - Running scenario: One pod per PVC -2025-06-05 16:27:01,051 - tests.orchestrator - INFO - Creating 3 PVC-pod pairs -2025-06-05 16:27:01,134 - tests.orchestrator - INFO - Waiting for PVC one2one-afd352b2 to be bound -2025-06-05 16:27:03,267 - tests.orchestrator - INFO - PVC one2one-afd352b2 is bound -2025-06-05 16:27:03,267 - tests.orchestrator - INFO - Creating pod test-pod-95808816 on PVC one2one-afd352b2 -2025-06-05 16:27:03,344 - tests.orchestrator - INFO - Created pod: test-pod-95808816 using PVC: one2one-afd352b2 -2025-06-05 16:27:03,344 - tests.orchestrator - INFO - Waiting for pod test-pod-95808816 to be ready -2025-06-05 16:27:03,412 - tests.orchestrator - INFO - Pod test-pod-95808816 phase: Pending -2025-06-05 16:27:34,490 - tests.orchestrator - INFO - Pod test-pod-95808816 taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 16:27:34,490 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-95808816 ===== -2025-06-05 16:27:34,558 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:27:34,627 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:27:34,627 - tests.orchestrator - INFO - [2025-06-05 16:27:03+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:27:34,692 - tests.orchestrator - INFO - No logs available -2025-06-05 16:27:34,692 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:27:34,692 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'one2one-afd352b2'} -2025-06-05 16:27:34,692 - tests.orchestrator - INFO - - kube-api-access-rmslv: {} -2025-06-05 16:27:34,692 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-95808816 ===== -2025-06-05 16:28:03,588 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-95808816 to be ready after 60s -2025-06-05 16:28:03,589 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-95808816 ===== -2025-06-05 16:28:03,656 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:28:03,727 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:28:03,727 - tests.orchestrator - INFO - [2025-06-05 16:27:03+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:28:03,792 - tests.orchestrator - INFO - No logs available -2025-06-05 16:28:03,792 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:28:03,792 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'one2one-afd352b2'} -2025-06-05 16:28:03,792 - tests.orchestrator - INFO - - kube-api-access-rmslv: {} -2025-06-05 16:28:03,792 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-95808816 ===== -2025-06-05 16:28:03,792 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-95808816 to be ready -2025-06-05 16:28:03,870 - tests.orchestrator - INFO - Waiting for PVC one2one-cc87f2c5 to be bound -2025-06-05 16:28:06,005 - tests.orchestrator - INFO - PVC one2one-cc87f2c5 is bound -2025-06-05 16:28:06,005 - tests.orchestrator - INFO - Creating pod test-pod-b29bd9ef on PVC one2one-cc87f2c5 -2025-06-05 16:28:06,081 - tests.orchestrator - INFO - Created pod: test-pod-b29bd9ef using PVC: one2one-cc87f2c5 -2025-06-05 16:28:06,081 - tests.orchestrator - INFO - Waiting for pod test-pod-b29bd9ef to be ready -2025-06-05 16:28:06,149 - tests.orchestrator - INFO - Pod test-pod-b29bd9ef phase: Pending -2025-06-05 16:28:37,171 - tests.orchestrator - INFO - Pod test-pod-b29bd9ef taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 16:28:37,171 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-b29bd9ef ===== -2025-06-05 16:28:37,238 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:28:37,308 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:28:37,308 - tests.orchestrator - INFO - [2025-06-05 16:28:06+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:28:37,373 - tests.orchestrator - INFO - No logs available -2025-06-05 16:28:37,373 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:28:37,373 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'one2one-cc87f2c5'} -2025-06-05 16:28:37,373 - tests.orchestrator - INFO - - kube-api-access-wlwbk: {} -2025-06-05 16:28:37,373 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-b29bd9ef ===== -2025-06-05 16:29:06,383 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b29bd9ef to be ready after 60s -2025-06-05 16:29:06,383 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-b29bd9ef ===== -2025-06-05 16:29:06,451 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:29:06,520 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:29:06,521 - tests.orchestrator - INFO - [2025-06-05 16:28:06+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:29:06,587 - tests.orchestrator - INFO - No logs available -2025-06-05 16:29:06,587 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:29:06,587 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'one2one-cc87f2c5'} -2025-06-05 16:29:06,587 - tests.orchestrator - INFO - - kube-api-access-wlwbk: {} -2025-06-05 16:29:06,587 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-b29bd9ef ===== -2025-06-05 16:29:06,587 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-b29bd9ef to be ready -2025-06-05 16:29:06,670 - tests.orchestrator - INFO - Waiting for PVC one2one-010b3efe to be bound -2025-06-05 16:29:08,808 - tests.orchestrator - INFO - PVC one2one-010b3efe is bound -2025-06-05 16:29:08,808 - tests.orchestrator - INFO - Creating pod test-pod-1fbeeb48 on PVC one2one-010b3efe -2025-06-05 16:29:08,885 - tests.orchestrator - INFO - Created pod: test-pod-1fbeeb48 using PVC: one2one-010b3efe -2025-06-05 16:29:08,885 - tests.orchestrator - INFO - Waiting for pod test-pod-1fbeeb48 to be ready -2025-06-05 16:29:08,950 - tests.orchestrator - INFO - Pod test-pod-1fbeeb48 phase: Pending -2025-06-05 16:29:39,968 - tests.orchestrator - INFO - Pod test-pod-1fbeeb48 taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 16:29:39,968 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-1fbeeb48 ===== -2025-06-05 16:29:40,037 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:29:40,110 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:29:40,111 - tests.orchestrator - INFO - [2025-06-05 16:29:08+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:29:40,178 - tests.orchestrator - INFO - No logs available -2025-06-05 16:29:40,178 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:29:40,179 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'one2one-010b3efe'} -2025-06-05 16:29:40,179 - tests.orchestrator - INFO - - kube-api-access-q67hr: {} -2025-06-05 16:29:40,179 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-1fbeeb48 ===== -2025-06-05 16:30:09,074 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1fbeeb48 to be ready after 60s -2025-06-05 16:30:09,074 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-1fbeeb48 ===== -2025-06-05 16:30:09,143 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:30:09,215 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:30:09,215 - tests.orchestrator - INFO - [2025-06-05 16:29:08+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:30:09,280 - tests.orchestrator - INFO - No logs available -2025-06-05 16:30:09,280 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:30:09,280 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'one2one-010b3efe'} -2025-06-05 16:30:09,280 - tests.orchestrator - INFO - - kube-api-access-q67hr: {} -2025-06-05 16:30:09,280 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-1fbeeb48 ===== -2025-06-05 16:30:09,280 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-1fbeeb48 to be ready -2025-06-05 16:30:09,280 - tests.orchestrator - WARNING - Failed to create enough PVC-pod pairs, only created 0 -2025-06-05 16:30:09,280 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_one_to_one -2025-06-05 16:30:09,280 - tests.orchestrator - INFO - ------------------------------------------------------------ -2025-06-05 16:30:09,280 - tests.orchestrator - INFO - Deleting pod: test-pod-b29bd9ef from PVC: one2one-cc87f2c5 -2025-06-05 16:30:09,361 - tests.orchestrator - INFO - Waiting for pod test-pod-b29bd9ef to be deleted -2025-06-05 16:30:09,427 - tests.orchestrator - INFO - Pod test-pod-b29bd9ef has been deleted -2025-06-05 16:30:09,428 - tests.orchestrator - INFO - Deleted pod: test-pod-b29bd9ef -2025-06-05 16:30:09,428 - tests.orchestrator - INFO - Deleting PVC: one2one-afd352b2 -2025-06-05 16:30:09,428 - tests.orchestrator - INFO - Deleting 1 pods using PVC one2one-afd352b2 -2025-06-05 16:30:09,428 - tests.orchestrator - INFO - Deleting pod: test-pod-95808816 from PVC: one2one-afd352b2 -2025-06-05 16:30:09,509 - tests.orchestrator - INFO - Waiting for pod test-pod-95808816 to be deleted -2025-06-05 16:30:09,575 - tests.orchestrator - INFO - Pod test-pod-95808816 has been deleted -2025-06-05 16:30:09,576 - tests.orchestrator - INFO - Deleted pod: test-pod-95808816 -2025-06-05 16:30:09,650 - tests.orchestrator - INFO - Waiting for PVC one2one-afd352b2 to be deleted -2025-06-05 16:30:09,715 - tests.orchestrator - INFO - PVC one2one-afd352b2 has been deleted -2025-06-05 16:30:09,715 - tests.orchestrator - INFO - Deleted PVC: one2one-afd352b2 -2025-06-05 16:30:09,716 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations -2025-06-05 16:30:09,716 - tests.orchestrator - INFO - Test completed in 374.97 seconds -2025-06-05 16:30:09,716 - tests.orchestrator - INFO - Cleaning up resources... -2025-06-05 16:30:09,717 - tests.orchestrator - INFO - Deleting pod: test-pod-dcc5bed9 from PVC: test-pvc-7efe733c -2025-06-05 16:30:09,800 - tests.orchestrator - INFO - Waiting for pod test-pod-dcc5bed9 to be deleted -2025-06-05 16:30:09,868 - tests.orchestrator - INFO - Pod test-pod-dcc5bed9 has been deleted -2025-06-05 16:30:09,868 - tests.orchestrator - INFO - Deleted pod: test-pod-dcc5bed9 -2025-06-05 16:30:09,868 - tests.orchestrator - INFO - Deleting pod: test-pod-9af93280 from PVC: test-pvc-7efe733c -2025-06-05 16:30:09,955 - tests.orchestrator - INFO - Waiting for pod test-pod-9af93280 to be deleted -2025-06-05 16:30:10,021 - tests.orchestrator - INFO - Pod test-pod-9af93280 has been deleted -2025-06-05 16:30:10,022 - tests.orchestrator - INFO - Deleted pod: test-pod-9af93280 -2025-06-05 16:30:10,022 - tests.orchestrator - INFO - Deleting pod: test-pod-5feeb276 from PVC: test-pvc-7efe733c -2025-06-05 16:30:10,106 - tests.orchestrator - INFO - Waiting for pod test-pod-5feeb276 to be deleted -2025-06-05 16:30:10,173 - tests.orchestrator - INFO - Pod test-pod-5feeb276 has been deleted -2025-06-05 16:30:10,173 - tests.orchestrator - INFO - Deleted pod: test-pod-5feeb276 -2025-06-05 16:30:10,173 - tests.orchestrator - INFO - Deleting pod: test-pod-1fbeeb48 from PVC: one2one-010b3efe -2025-06-05 16:30:10,260 - tests.orchestrator - INFO - Waiting for pod test-pod-1fbeeb48 to be deleted -2025-06-05 16:30:10,325 - tests.orchestrator - INFO - Pod test-pod-1fbeeb48 has been deleted -2025-06-05 16:30:10,325 - tests.orchestrator - INFO - Deleted pod: test-pod-1fbeeb48 -2025-06-05 16:30:10,326 - tests.orchestrator - INFO - Deleting PVC: test-pvc-d9e565cc -2025-06-05 16:30:10,400 - tests.orchestrator - INFO - Waiting for PVC test-pvc-d9e565cc to be deleted -2025-06-05 16:30:10,464 - tests.orchestrator - INFO - PVC test-pvc-d9e565cc has been deleted -2025-06-05 16:30:10,464 - tests.orchestrator - INFO - Deleted PVC: test-pvc-d9e565cc -2025-06-05 16:30:10,464 - tests.orchestrator - INFO - Deleting PVC: test-pvc-7efe733c -2025-06-05 16:30:10,538 - tests.orchestrator - INFO - Waiting for PVC test-pvc-7efe733c to be deleted -2025-06-05 16:30:10,602 - tests.orchestrator - INFO - PVC test-pvc-7efe733c has been deleted -2025-06-05 16:30:10,602 - tests.orchestrator - INFO - Deleted PVC: test-pvc-7efe733c -2025-06-05 16:30:10,603 - tests.orchestrator - INFO - Deleting PVC: one2one-cc87f2c5 -2025-06-05 16:30:10,693 - tests.orchestrator - INFO - Waiting for PVC one2one-cc87f2c5 to be deleted -2025-06-05 16:30:10,757 - tests.orchestrator - INFO - PVC one2one-cc87f2c5 has been deleted -2025-06-05 16:30:10,757 - tests.orchestrator - INFO - Deleted PVC: one2one-cc87f2c5 -2025-06-05 16:30:10,757 - tests.orchestrator - INFO - Deleting PVC: one2one-010b3efe -2025-06-05 16:30:10,831 - tests.orchestrator - INFO - Waiting for PVC one2one-010b3efe to be deleted -2025-06-05 16:30:10,896 - tests.orchestrator - INFO - PVC one2one-010b3efe has been deleted -2025-06-05 16:30:10,896 - tests.orchestrator - INFO - Deleted PVC: one2one-010b3efe -2025-06-05 16:30:10,896 - tests.orchestrator - INFO - Cleanup completed -2025-06-05 16:30:10,896 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-05 16:30:10,896 - tests.orchestrator - INFO - --- Operations --- -2025-06-05 16:30:10,896 - tests.orchestrator - INFO - create_pvc: 2 succeeded, 0 failed (100.0%) -2025-06-05 16:30:10,896 - tests.orchestrator - INFO - attach_pod: 6 succeeded, 0 failed (100.0%) -2025-06-05 16:30:10,896 - tests.orchestrator - INFO - delete_pod: 6 succeeded, 0 failed (100.0%) -2025-06-05 16:30:10,897 - tests.orchestrator - INFO - delete_pvc: 5 succeeded, 0 failed (100.0%) -2025-06-05 16:30:10,897 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) -2025-06-05 16:30:10,897 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) -2025-06-05 16:30:10,897 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-05 16:30:10,897 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-05 16:30:10,897 - tests.orchestrator - INFO - many_to_one: No runs -2025-06-05 16:30:10,897 - tests.orchestrator - INFO - one_to_one: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-05 16:30:10,897 - tests.orchestrator - INFO - concurrent_pvc: No runs -2025-06-05 16:30:10,897 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_163114.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_163114.log deleted file mode 100644 index 1377e47c0..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_163114.log +++ /dev/null @@ -1,67 +0,0 @@ -2025-06-05 16:31:14,327 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 16:31:14,546 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 16:31:14,546 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-05 16:31:14,735 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 16:31:14,735 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 16:31:14,736 - tests.orchestrator - INFO - Creating PVC: test-pvc-dbe6630f -2025-06-05 16:31:14,812 - tests.orchestrator - INFO - Created PVC: test-pvc-dbe6630f -2025-06-05 16:31:14,812 - tests.orchestrator - INFO - Waiting for PVC test-pvc-dbe6630f to be bound -2025-06-05 16:31:16,944 - tests.orchestrator - INFO - PVC test-pvc-dbe6630f is bound -2025-06-05 16:31:16,944 - tests.orchestrator - INFO - Creating PVC: test-pvc-2381ce5e -2025-06-05 16:31:17,012 - tests.orchestrator - INFO - Created PVC: test-pvc-2381ce5e -2025-06-05 16:31:17,012 - tests.orchestrator - INFO - Waiting for PVC test-pvc-2381ce5e to be bound -2025-06-05 16:31:19,144 - tests.orchestrator - INFO - PVC test-pvc-2381ce5e is bound -2025-06-05 16:31:19,144 - tests.orchestrator - INFO - Creating pod test-pod-4022a58d on PVC test-pvc-2381ce5e -2025-06-05 16:31:19,223 - tests.orchestrator - INFO - Created pod: test-pod-4022a58d using PVC: test-pvc-2381ce5e -2025-06-05 16:31:19,223 - tests.orchestrator - INFO - Waiting for pod test-pod-4022a58d to be ready -2025-06-05 16:31:19,289 - tests.orchestrator - INFO - Pod test-pod-4022a58d phase: Pending -2025-06-05 16:31:50,320 - tests.orchestrator - INFO - Pod test-pod-4022a58d taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 16:31:50,320 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-4022a58d ===== -2025-06-05 16:31:50,387 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:31:50,457 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:31:50,457 - tests.orchestrator - INFO - [2025-06-05 16:31:19+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:31:50,524 - tests.orchestrator - INFO - No logs available -2025-06-05 16:31:50,525 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:31:50,525 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-2381ce5e'} -2025-06-05 16:31:50,525 - tests.orchestrator - INFO - - kube-api-access-xrgfl: {} -2025-06-05 16:31:50,525 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-4022a58d ===== -2025-06-05 16:32:19,393 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-4022a58d to be ready after 60s -2025-06-05 16:32:19,393 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-4022a58d ===== -2025-06-05 16:32:19,461 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:32:19,530 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:32:19,530 - tests.orchestrator - INFO - [2025-06-05 16:31:19+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:32:19,595 - tests.orchestrator - INFO - No logs available -2025-06-05 16:32:19,595 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:32:19,595 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-2381ce5e'} -2025-06-05 16:32:19,595 - tests.orchestrator - INFO - - kube-api-access-xrgfl: {} -2025-06-05 16:32:19,595 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-4022a58d ===== -2025-06-05 16:32:19,595 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-4022a58d to be ready -2025-06-05 16:32:19,595 - tests.orchestrator - INFO - Creating pod test-pod-9de3bebb on PVC test-pvc-dbe6630f -2025-06-05 16:32:19,678 - tests.orchestrator - INFO - Created pod: test-pod-9de3bebb using PVC: test-pvc-dbe6630f -2025-06-05 16:32:19,678 - tests.orchestrator - INFO - Waiting for pod test-pod-9de3bebb to be ready -2025-06-05 16:32:19,745 - tests.orchestrator - INFO - Pod test-pod-9de3bebb phase: Pending -2025-06-05 16:32:50,757 - tests.orchestrator - INFO - Pod test-pod-9de3bebb taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 16:32:50,757 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-9de3bebb ===== -2025-06-05 16:32:50,823 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:32:50,895 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:32:50,895 - tests.orchestrator - INFO - [2025-06-05 16:32:19+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:32:50,960 - tests.orchestrator - INFO - No logs available -2025-06-05 16:32:50,960 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:32:50,960 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-dbe6630f'} -2025-06-05 16:32:50,960 - tests.orchestrator - INFO - - kube-api-access-vfnnb: {} -2025-06-05 16:32:50,960 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-9de3bebb ===== -2025-06-05 16:33:19,831 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9de3bebb to be ready after 60s -2025-06-05 16:33:19,831 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-9de3bebb ===== -2025-06-05 16:33:19,897 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:33:19,966 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:33:19,966 - tests.orchestrator - INFO - [2025-06-05 16:32:19+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:33:20,031 - tests.orchestrator - INFO - No logs available -2025-06-05 16:33:20,031 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:33:20,031 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-dbe6630f'} -2025-06-05 16:33:20,031 - tests.orchestrator - INFO - - kube-api-access-vfnnb: {} -2025-06-05 16:33:20,031 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-9de3bebb ===== -2025-06-05 16:33:20,031 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9de3bebb to be ready -2025-06-05 16:33:20,032 - tests.orchestrator - INFO - Creating pod test-pod-4eee03ff on PVC test-pvc-2381ce5e -2025-06-05 16:33:20,115 - tests.orchestrator - INFO - Created pod: test-pod-4eee03ff using PVC: test-pvc-2381ce5e -2025-06-05 16:33:20,115 - tests.orchestrator - INFO - Waiting for pod test-pod-4eee03ff to be ready -2025-06-05 16:33:20,181 - tests.orchestrator - INFO - Pod test-pod-4eee03ff phase: Pending diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_163340.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_163340.log deleted file mode 100644 index ef9c60d94..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_163340.log +++ /dev/null @@ -1,42 +0,0 @@ -2025-06-05 16:33:40,546 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 16:33:40,752 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 16:33:40,752 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-05 16:33:40,943 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 16:33:40,944 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 16:33:40,944 - tests.orchestrator - INFO - Creating PVC: test-pvc-567589be -2025-06-05 16:33:41,017 - tests.orchestrator - INFO - Created PVC: test-pvc-567589be -2025-06-05 16:33:41,017 - tests.orchestrator - INFO - Waiting for PVC test-pvc-567589be to be bound -2025-06-05 16:33:43,150 - tests.orchestrator - INFO - PVC test-pvc-567589be is bound -2025-06-05 16:33:43,150 - tests.orchestrator - INFO - Creating PVC: test-pvc-4348540c -2025-06-05 16:33:43,218 - tests.orchestrator - INFO - Created PVC: test-pvc-4348540c -2025-06-05 16:33:43,218 - tests.orchestrator - INFO - Waiting for PVC test-pvc-4348540c to be bound -2025-06-05 16:33:45,350 - tests.orchestrator - INFO - PVC test-pvc-4348540c is bound -2025-06-05 16:33:45,350 - tests.orchestrator - INFO - Creating pod test-pod-0dadc981 on PVC test-pvc-567589be -2025-06-05 16:33:45,423 - tests.orchestrator - INFO - Created pod: test-pod-0dadc981 using PVC: test-pvc-567589be -2025-06-05 16:33:45,424 - tests.orchestrator - INFO - Waiting for pod test-pod-0dadc981 to be ready -2025-06-05 16:33:45,490 - tests.orchestrator - INFO - Pod test-pod-0dadc981 phase: Pending -2025-06-05 16:34:16,508 - tests.orchestrator - INFO - Pod test-pod-0dadc981 taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 16:34:16,508 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-0dadc981 ===== -2025-06-05 16:34:16,574 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:34:16,646 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:34:16,646 - tests.orchestrator - INFO - [2025-06-05 16:33:45+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:34:16,711 - tests.orchestrator - INFO - No logs available -2025-06-05 16:34:16,712 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:34:16,712 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-567589be'} -2025-06-05 16:34:16,712 - tests.orchestrator - INFO - - kube-api-access-jhmdz: {} -2025-06-05 16:34:16,712 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-0dadc981 ===== -2025-06-05 16:34:45,586 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-0dadc981 to be ready after 60s -2025-06-05 16:34:45,586 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-0dadc981 ===== -2025-06-05 16:34:45,653 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:34:45,725 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:34:45,725 - tests.orchestrator - INFO - [2025-06-05 16:33:45+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:34:45,791 - tests.orchestrator - INFO - No logs available -2025-06-05 16:34:45,791 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:34:45,791 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-567589be'} -2025-06-05 16:34:45,791 - tests.orchestrator - INFO - - kube-api-access-jhmdz: {} -2025-06-05 16:34:45,791 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-0dadc981 ===== -2025-06-05 16:34:45,791 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-0dadc981 to be ready -2025-06-05 16:34:45,791 - tests.orchestrator - INFO - Creating pod test-pod-74abb322 on PVC test-pvc-4348540c -2025-06-05 16:34:45,873 - tests.orchestrator - INFO - Created pod: test-pod-74abb322 using PVC: test-pvc-4348540c -2025-06-05 16:34:45,873 - tests.orchestrator - INFO - Waiting for pod test-pod-74abb322 to be ready -2025-06-05 16:34:45,939 - tests.orchestrator - INFO - Pod test-pod-74abb322 phase: Pending diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_163501.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_163501.log deleted file mode 100644 index 7c87c876e..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_163501.log +++ /dev/null @@ -1,205 +0,0 @@ -2025-06-05 16:35:01,819 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 16:35:02,054 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 16:35:02,054 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-05 16:35:02,250 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 16:35:02,250 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 16:35:02,250 - tests.orchestrator - INFO - Creating PVC: test-pvc-1f8e9142 -2025-06-05 16:35:02,318 - tests.orchestrator - INFO - Created PVC: test-pvc-1f8e9142 -2025-06-05 16:35:02,318 - tests.orchestrator - INFO - Waiting for PVC test-pvc-1f8e9142 to be bound -2025-06-05 16:35:04,451 - tests.orchestrator - INFO - PVC test-pvc-1f8e9142 is bound -2025-06-05 16:35:04,451 - tests.orchestrator - INFO - Creating PVC: test-pvc-6d74da5a -2025-06-05 16:35:04,519 - tests.orchestrator - INFO - Created PVC: test-pvc-6d74da5a -2025-06-05 16:35:04,520 - tests.orchestrator - INFO - Waiting for PVC test-pvc-6d74da5a to be bound -2025-06-05 16:35:06,652 - tests.orchestrator - INFO - PVC test-pvc-6d74da5a is bound -2025-06-05 16:35:06,652 - tests.orchestrator - INFO - Creating pod test-pod-dff251c2 on PVC test-pvc-6d74da5a -2025-06-05 16:35:06,728 - tests.orchestrator - INFO - Created pod: test-pod-dff251c2 using PVC: test-pvc-6d74da5a -2025-06-05 16:35:06,728 - tests.orchestrator - INFO - Waiting for pod test-pod-dff251c2 to be ready -2025-06-05 16:35:06,794 - tests.orchestrator - INFO - Pod test-pod-dff251c2 phase: Pending -2025-06-05 16:35:37,873 - tests.orchestrator - INFO - Pod test-pod-dff251c2 taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 16:35:37,873 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-dff251c2 ===== -2025-06-05 16:35:37,941 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:35:38,013 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:35:38,013 - tests.orchestrator - INFO - [2025-06-05 16:35:06+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:35:38,078 - tests.orchestrator - INFO - No logs available -2025-06-05 16:35:38,078 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:35:38,078 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-6d74da5a'} -2025-06-05 16:35:38,079 - tests.orchestrator - INFO - - kube-api-access-4d7rh: {} -2025-06-05 16:35:38,079 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-dff251c2 ===== -2025-06-05 16:36:06,966 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-dff251c2 to be ready after 60s -2025-06-05 16:36:06,966 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-dff251c2 ===== -2025-06-05 16:36:07,034 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:36:07,104 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:36:07,104 - tests.orchestrator - INFO - [2025-06-05 16:35:06+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:36:07,170 - tests.orchestrator - INFO - No logs available -2025-06-05 16:36:07,170 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:36:07,170 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-6d74da5a'} -2025-06-05 16:36:07,170 - tests.orchestrator - INFO - - kube-api-access-4d7rh: {} -2025-06-05 16:36:07,170 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-dff251c2 ===== -2025-06-05 16:36:07,170 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-dff251c2 to be ready -2025-06-05 16:36:07,170 - tests.orchestrator - INFO - Creating pod test-pod-9ee2d595 on PVC test-pvc-6d74da5a -2025-06-05 16:36:07,253 - tests.orchestrator - INFO - Created pod: test-pod-9ee2d595 using PVC: test-pvc-6d74da5a -2025-06-05 16:36:07,253 - tests.orchestrator - INFO - Waiting for pod test-pod-9ee2d595 to be ready -2025-06-05 16:36:07,321 - tests.orchestrator - INFO - Pod test-pod-9ee2d595 phase: Pending -2025-06-05 16:36:38,341 - tests.orchestrator - INFO - Pod test-pod-9ee2d595 taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 16:36:38,341 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-9ee2d595 ===== -2025-06-05 16:36:38,408 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:36:38,478 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:36:38,478 - tests.orchestrator - INFO - [2025-06-05 16:36:07+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:36:38,544 - tests.orchestrator - INFO - No logs available -2025-06-05 16:36:38,544 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:36:38,544 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-6d74da5a'} -2025-06-05 16:36:38,544 - tests.orchestrator - INFO - - kube-api-access-vll4f: {} -2025-06-05 16:36:38,544 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-9ee2d595 ===== -2025-06-05 16:37:07,418 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9ee2d595 to be ready after 60s -2025-06-05 16:37:07,418 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-9ee2d595 ===== -2025-06-05 16:37:07,485 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:37:07,558 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:37:07,558 - tests.orchestrator - INFO - [2025-06-05 16:36:07+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:37:07,624 - tests.orchestrator - INFO - No logs available -2025-06-05 16:37:07,624 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:37:07,624 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-6d74da5a'} -2025-06-05 16:37:07,624 - tests.orchestrator - INFO - - kube-api-access-vll4f: {} -2025-06-05 16:37:07,624 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-9ee2d595 ===== -2025-06-05 16:37:07,625 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-9ee2d595 to be ready -2025-06-05 16:37:07,625 - tests.orchestrator - INFO - Creating pod test-pod-38614e6b on PVC test-pvc-1f8e9142 -2025-06-05 16:37:07,706 - tests.orchestrator - INFO - Created pod: test-pod-38614e6b using PVC: test-pvc-1f8e9142 -2025-06-05 16:37:07,707 - tests.orchestrator - INFO - Waiting for pod test-pod-38614e6b to be ready -2025-06-05 16:37:07,773 - tests.orchestrator - INFO - Pod test-pod-38614e6b phase: Pending -2025-06-05 16:37:38,780 - tests.orchestrator - INFO - Pod test-pod-38614e6b taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 16:37:38,781 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-38614e6b ===== -2025-06-05 16:37:38,848 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:37:38,919 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:37:38,919 - tests.orchestrator - INFO - [2025-06-05 16:37:07+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:37:38,984 - tests.orchestrator - INFO - No logs available -2025-06-05 16:37:38,984 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:37:38,984 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-1f8e9142'} -2025-06-05 16:37:38,984 - tests.orchestrator - INFO - - kube-api-access-c44px: {} -2025-06-05 16:37:38,984 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-38614e6b ===== -2025-06-05 16:38:07,879 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-38614e6b to be ready after 60s -2025-06-05 16:38:07,879 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-38614e6b ===== -2025-06-05 16:38:07,947 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:38:08,019 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:38:08,019 - tests.orchestrator - INFO - [2025-06-05 16:37:07+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:38:08,084 - tests.orchestrator - INFO - No logs available -2025-06-05 16:38:08,084 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:38:08,084 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-1f8e9142'} -2025-06-05 16:38:08,084 - tests.orchestrator - INFO - - kube-api-access-c44px: {} -2025-06-05 16:38:08,084 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-38614e6b ===== -2025-06-05 16:38:08,084 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-38614e6b to be ready -2025-06-05 16:38:08,084 - tests.orchestrator - INFO - Testing read/write between pods test-pod-9ee2d595 and test-pod-dff251c2 sharing PVC test-pvc-6d74da5a -2025-06-05 16:38:08,150 - tests.orchestrator - ERROR - Failed in read/write verification: (400) -Reason: Bad Request -HTTP response headers: HTTPHeaderDict({'Audit-Id': '2b5c9bbe-dcea-4a5d-82b0-b2b0e2833174', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 16:38:08 GMT', 'Content-Length': '166'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"pod test-pod-9ee2d595 does not have a host assigned","reason":"BadRequest","code":400} - - -2025-06-05 16:38:08,150 - tests.orchestrator - INFO - Deleting pod: test-pod-dff251c2 from PVC: test-pvc-6d74da5a -2025-06-05 16:38:08,235 - tests.orchestrator - INFO - Waiting for pod test-pod-dff251c2 to be deleted -2025-06-05 16:38:08,300 - tests.orchestrator - INFO - Pod test-pod-dff251c2 has been deleted -2025-06-05 16:38:08,300 - tests.orchestrator - INFO - Deleted pod: test-pod-dff251c2 -2025-06-05 16:38:08,300 - tests.orchestrator - INFO - Deleting PVC: test-pvc-6d74da5a -2025-06-05 16:38:08,300 - tests.orchestrator - INFO - Deleting 1 pods using PVC test-pvc-6d74da5a -2025-06-05 16:38:08,300 - tests.orchestrator - INFO - Deleting pod: test-pod-9ee2d595 from PVC: test-pvc-6d74da5a -2025-06-05 16:38:08,378 - tests.orchestrator - INFO - Waiting for pod test-pod-9ee2d595 to be deleted -2025-06-05 16:38:08,444 - tests.orchestrator - INFO - Pod test-pod-9ee2d595 has been deleted -2025-06-05 16:38:08,444 - tests.orchestrator - INFO - Deleted pod: test-pod-9ee2d595 -2025-06-05 16:38:08,520 - tests.orchestrator - INFO - Waiting for PVC test-pvc-6d74da5a to be deleted -2025-06-05 16:38:08,585 - tests.orchestrator - INFO - PVC test-pvc-6d74da5a has been deleted -2025-06-05 16:38:08,585 - tests.orchestrator - INFO - Deleted PVC: test-pvc-6d74da5a -2025-06-05 16:38:08,585 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations -2025-06-05 16:38:08,585 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 1 times) -2025-06-05 16:38:08,585 - tests.orchestrator - INFO - Creating PVC: test-pvc-cf863d95 -2025-06-05 16:38:08,662 - tests.orchestrator - INFO - Created PVC: test-pvc-cf863d95 -2025-06-05 16:38:08,662 - tests.orchestrator - INFO - Waiting for PVC test-pvc-cf863d95 to be bound -2025-06-05 16:38:10,795 - tests.orchestrator - INFO - PVC test-pvc-cf863d95 is bound -2025-06-05 16:38:11,795 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 1 times) -2025-06-05 16:38:11,795 - tests.orchestrator - INFO - Deleting pod: test-pod-38614e6b from PVC: test-pvc-1f8e9142 -2025-06-05 16:38:11,879 - tests.orchestrator - INFO - Waiting for pod test-pod-38614e6b to be deleted -2025-06-05 16:38:11,944 - tests.orchestrator - INFO - Pod test-pod-38614e6b has been deleted -2025-06-05 16:38:11,944 - tests.orchestrator - INFO - Deleted pod: test-pod-38614e6b -2025-06-05 16:38:12,944 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 1 times) -2025-06-05 16:38:12,944 - tests.orchestrator - INFO - Deleting PVC: test-pvc-1f8e9142 -2025-06-05 16:38:13,016 - tests.orchestrator - INFO - Waiting for PVC test-pvc-1f8e9142 to be deleted -2025-06-05 16:38:13,082 - tests.orchestrator - INFO - PVC test-pvc-1f8e9142 has been deleted -2025-06-05 16:38:13,082 - tests.orchestrator - INFO - Deleted PVC: test-pvc-1f8e9142 -2025-06-05 16:38:14,082 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 1 times) -2025-06-05 16:38:14,083 - tests.orchestrator - INFO - Creating pod test-pod-3d503390 on PVC test-pvc-cf863d95 -2025-06-05 16:38:14,156 - tests.orchestrator - INFO - Created pod: test-pod-3d503390 using PVC: test-pvc-cf863d95 -2025-06-05 16:38:14,156 - tests.orchestrator - INFO - Waiting for pod test-pod-3d503390 to be ready -2025-06-05 16:38:14,224 - tests.orchestrator - INFO - Pod test-pod-3d503390 phase: Pending -2025-06-05 16:38:45,253 - tests.orchestrator - INFO - Pod test-pod-3d503390 taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 16:38:45,254 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-3d503390 ===== -2025-06-05 16:38:45,320 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:38:45,413 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:38:45,413 - tests.orchestrator - INFO - [2025-06-05 16:38:14+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:38:45,481 - tests.orchestrator - INFO - No logs available -2025-06-05 16:38:45,481 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:38:45,481 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-cf863d95'} -2025-06-05 16:38:45,481 - tests.orchestrator - INFO - - kube-api-access-j6x6f: {} -2025-06-05 16:38:45,481 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-3d503390 ===== -2025-06-05 16:39:14,367 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-3d503390 to be ready after 60s -2025-06-05 16:39:14,368 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-3d503390 ===== -2025-06-05 16:39:14,435 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:39:14,506 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:39:14,506 - tests.orchestrator - INFO - [2025-06-05 16:38:14+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:39:14,572 - tests.orchestrator - INFO - No logs available -2025-06-05 16:39:14,572 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:39:14,572 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-cf863d95'} -2025-06-05 16:39:14,572 - tests.orchestrator - INFO - - kube-api-access-j6x6f: {} -2025-06-05 16:39:14,572 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-3d503390 ===== -2025-06-05 16:39:14,572 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-3d503390 to be ready -2025-06-05 16:39:15,572 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 2 times) -2025-06-05 16:39:15,573 - tests.orchestrator - INFO - Creating pod test-pod-bf6753dc on PVC test-pvc-cf863d95 -2025-06-05 16:39:15,656 - tests.orchestrator - INFO - Created pod: test-pod-bf6753dc using PVC: test-pvc-cf863d95 -2025-06-05 16:39:15,656 - tests.orchestrator - INFO - Waiting for pod test-pod-bf6753dc to be ready -2025-06-05 16:39:15,723 - tests.orchestrator - INFO - Pod test-pod-bf6753dc phase: Pending -2025-06-05 16:39:46,730 - tests.orchestrator - INFO - Pod test-pod-bf6753dc taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 16:39:46,731 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-bf6753dc ===== -2025-06-05 16:39:46,797 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:39:46,867 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:39:46,867 - tests.orchestrator - INFO - [2025-06-05 16:39:15+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:39:46,932 - tests.orchestrator - INFO - No logs available -2025-06-05 16:39:46,933 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:39:46,933 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-cf863d95'} -2025-06-05 16:39:46,933 - tests.orchestrator - INFO - - kube-api-access-2ptll: {} -2025-06-05 16:39:46,933 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-bf6753dc ===== -2025-06-05 16:40:15,809 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-bf6753dc to be ready after 60s -2025-06-05 16:40:15,809 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-bf6753dc ===== -2025-06-05 16:40:15,877 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:40:15,949 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:40:15,949 - tests.orchestrator - INFO - [2025-06-05 16:39:15+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:40:16,015 - tests.orchestrator - INFO - No logs available -2025-06-05 16:40:16,015 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:40:16,015 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-cf863d95'} -2025-06-05 16:40:16,015 - tests.orchestrator - INFO - - kube-api-access-2ptll: {} -2025-06-05 16:40:16,015 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-bf6753dc ===== -2025-06-05 16:40:16,015 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-bf6753dc to be ready -2025-06-05 16:40:17,016 - tests.orchestrator - INFO - Test completed in 314.96 seconds -2025-06-05 16:40:17,016 - tests.orchestrator - INFO - Cleaning up resources... -2025-06-05 16:40:17,016 - tests.orchestrator - INFO - Deleting pod: test-pod-3d503390 from PVC: test-pvc-cf863d95 -2025-06-05 16:40:17,103 - tests.orchestrator - INFO - Waiting for pod test-pod-3d503390 to be deleted -2025-06-05 16:40:17,168 - tests.orchestrator - INFO - Pod test-pod-3d503390 has been deleted -2025-06-05 16:40:17,168 - tests.orchestrator - INFO - Deleted pod: test-pod-3d503390 -2025-06-05 16:40:17,168 - tests.orchestrator - INFO - Deleting pod: test-pod-bf6753dc from PVC: test-pvc-cf863d95 -2025-06-05 16:40:17,247 - tests.orchestrator - INFO - Waiting for pod test-pod-bf6753dc to be deleted -2025-06-05 16:40:17,312 - tests.orchestrator - INFO - Pod test-pod-bf6753dc has been deleted -2025-06-05 16:40:17,313 - tests.orchestrator - INFO - Deleted pod: test-pod-bf6753dc -2025-06-05 16:40:17,313 - tests.orchestrator - INFO - Deleting PVC: test-pvc-cf863d95 -2025-06-05 16:40:17,385 - tests.orchestrator - INFO - Waiting for PVC test-pvc-cf863d95 to be deleted -2025-06-05 16:40:17,450 - tests.orchestrator - INFO - PVC test-pvc-cf863d95 has been deleted -2025-06-05 16:40:17,450 - tests.orchestrator - INFO - Deleted PVC: test-pvc-cf863d95 -2025-06-05 16:40:17,450 - tests.orchestrator - INFO - Cleanup completed -2025-06-05 16:40:17,450 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-05 16:40:17,451 - tests.orchestrator - INFO - --- Operations --- -2025-06-05 16:40:17,451 - tests.orchestrator - INFO - create_pvc: 3 succeeded, 0 failed (100.0%) -2025-06-05 16:40:17,451 - tests.orchestrator - INFO - attach_pod: 5 succeeded, 0 failed (100.0%) -2025-06-05 16:40:17,451 - tests.orchestrator - INFO - delete_pod: 5 succeeded, 0 failed (100.0%) -2025-06-05 16:40:17,451 - tests.orchestrator - INFO - delete_pvc: 3 succeeded, 0 failed (100.0%) -2025-06-05 16:40:17,451 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) -2025-06-05 16:40:17,451 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) -2025-06-05 16:40:17,451 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-05 16:40:17,451 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-05 16:40:17,451 - tests.orchestrator - INFO - many_to_one: No runs -2025-06-05 16:40:17,451 - tests.orchestrator - INFO - one_to_one: No runs -2025-06-05 16:40:17,451 - tests.orchestrator - INFO - concurrent_pvc: No runs -2025-06-05 16:40:17,451 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_164709.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_164709.log deleted file mode 100644 index 793f7d128..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_164709.log +++ /dev/null @@ -1,67 +0,0 @@ -2025-06-05 16:47:09,450 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 16:47:09,670 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 16:47:09,670 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-05 16:47:09,860 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 16:47:09,860 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 16:47:09,860 - tests.orchestrator - INFO - Creating PVC: test-pvc-2686f57c -2025-06-05 16:47:09,941 - tests.orchestrator - INFO - Created PVC: test-pvc-2686f57c -2025-06-05 16:47:09,941 - tests.orchestrator - INFO - Waiting for PVC test-pvc-2686f57c to be bound -2025-06-05 16:47:12,088 - tests.orchestrator - INFO - PVC test-pvc-2686f57c is bound -2025-06-05 16:47:12,088 - tests.orchestrator - INFO - Creating PVC: test-pvc-269e9543 -2025-06-05 16:47:12,161 - tests.orchestrator - INFO - Created PVC: test-pvc-269e9543 -2025-06-05 16:47:12,161 - tests.orchestrator - INFO - Waiting for PVC test-pvc-269e9543 to be bound -2025-06-05 16:47:14,297 - tests.orchestrator - INFO - PVC test-pvc-269e9543 is bound -2025-06-05 16:47:14,297 - tests.orchestrator - INFO - Creating pod test-pod-60374c6e on PVC test-pvc-269e9543 -2025-06-05 16:47:14,379 - tests.orchestrator - INFO - Created pod: test-pod-60374c6e using PVC: test-pvc-269e9543 -2025-06-05 16:47:14,379 - tests.orchestrator - INFO - Waiting for pod test-pod-60374c6e to be ready -2025-06-05 16:47:14,447 - tests.orchestrator - INFO - Pod test-pod-60374c6e phase: Pending -2025-06-05 16:47:45,472 - tests.orchestrator - INFO - Pod test-pod-60374c6e taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 16:47:45,472 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-60374c6e ===== -2025-06-05 16:47:45,538 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:47:45,612 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:47:45,613 - tests.orchestrator - INFO - [2025-06-05 16:47:14+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:47:45,680 - tests.orchestrator - INFO - No logs available -2025-06-05 16:47:45,680 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:47:45,680 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-269e9543'} -2025-06-05 16:47:45,680 - tests.orchestrator - INFO - - kube-api-access-cnr5h: {} -2025-06-05 16:47:45,680 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-60374c6e ===== -2025-06-05 16:48:14,576 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-60374c6e to be ready after 60s -2025-06-05 16:48:14,576 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-60374c6e ===== -2025-06-05 16:48:14,643 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:48:14,718 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:48:14,718 - tests.orchestrator - INFO - [2025-06-05 16:47:14+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:48:14,785 - tests.orchestrator - INFO - No logs available -2025-06-05 16:48:14,785 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:48:14,786 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-269e9543'} -2025-06-05 16:48:14,786 - tests.orchestrator - INFO - - kube-api-access-cnr5h: {} -2025-06-05 16:48:14,786 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-60374c6e ===== -2025-06-05 16:48:14,786 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-60374c6e to be ready -2025-06-05 16:48:14,786 - tests.orchestrator - INFO - Creating pod test-pod-2f20e8f1 on PVC test-pvc-2686f57c -2025-06-05 16:48:14,872 - tests.orchestrator - INFO - Created pod: test-pod-2f20e8f1 using PVC: test-pvc-2686f57c -2025-06-05 16:48:14,872 - tests.orchestrator - INFO - Waiting for pod test-pod-2f20e8f1 to be ready -2025-06-05 16:48:14,941 - tests.orchestrator - INFO - Pod test-pod-2f20e8f1 phase: Pending -2025-06-05 16:48:46,015 - tests.orchestrator - INFO - Pod test-pod-2f20e8f1 taking longer than expected to become ready (31.1s). Collecting diagnostics... -2025-06-05 16:48:46,015 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-2f20e8f1 ===== -2025-06-05 16:48:46,081 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:48:46,155 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:48:46,155 - tests.orchestrator - INFO - [2025-06-05 16:48:14+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:48:46,222 - tests.orchestrator - INFO - No logs available -2025-06-05 16:48:46,222 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:48:46,222 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-2686f57c'} -2025-06-05 16:48:46,222 - tests.orchestrator - INFO - - kube-api-access-nqmbn: {} -2025-06-05 16:48:46,222 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-2f20e8f1 ===== -2025-06-05 16:49:15,114 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-2f20e8f1 to be ready after 60s -2025-06-05 16:49:15,114 - tests.orchestrator - INFO - ===== DIAGNOSTICS FOR POD test-pod-2f20e8f1 ===== -2025-06-05 16:49:15,181 - tests.orchestrator - INFO - No container statuses available -2025-06-05 16:49:15,256 - tests.orchestrator - INFO - Pod events: -2025-06-05 16:49:15,256 - tests.orchestrator - INFO - [2025-06-05 16:48:14+00:00] Warning/FailedScheduling: 0/2 nodes are available: 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/2 nodes are available: 2 Preemption is not helpful for scheduling. -2025-06-05 16:49:15,323 - tests.orchestrator - INFO - No logs available -2025-06-05 16:49:15,323 - tests.orchestrator - INFO - Pod volumes: -2025-06-05 16:49:15,324 - tests.orchestrator - INFO - - efs-volume: {'type': 'PVC', 'claim_name': 'test-pvc-2686f57c'} -2025-06-05 16:49:15,324 - tests.orchestrator - INFO - - kube-api-access-nqmbn: {} -2025-06-05 16:49:15,324 - tests.orchestrator - INFO - ===== END DIAGNOSTICS FOR POD test-pod-2f20e8f1 ===== -2025-06-05 16:49:15,324 - tests.orchestrator - WARNING - Timeout waiting for pod test-pod-2f20e8f1 to be ready -2025-06-05 16:49:15,324 - tests.orchestrator - INFO - Creating pod test-pod-988a488e on PVC test-pvc-269e9543 -2025-06-05 16:49:15,409 - tests.orchestrator - INFO - Created pod: test-pod-988a488e using PVC: test-pvc-269e9543 -2025-06-05 16:49:15,409 - tests.orchestrator - INFO - Waiting for pod test-pod-988a488e to be ready -2025-06-05 16:49:15,477 - tests.orchestrator - INFO - Pod test-pod-988a488e phase: Pending diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_164934.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_164934.log deleted file mode 100644 index 1301857d1..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_164934.log +++ /dev/null @@ -1,246 +0,0 @@ -2025-06-05 16:49:34,727 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 16:49:34,939 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 16:49:34,939 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-05 16:49:35,128 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 16:49:35,128 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 16:49:35,128 - tests.orchestrator - INFO - Creating PVC: test-pvc-410242e8 -2025-06-05 16:49:35,205 - tests.orchestrator - INFO - Created PVC: test-pvc-410242e8 -2025-06-05 16:49:35,205 - tests.orchestrator - INFO - Waiting for PVC test-pvc-410242e8 to be bound -2025-06-05 16:49:37,342 - tests.orchestrator - INFO - PVC test-pvc-410242e8 is bound -2025-06-05 16:49:37,342 - tests.orchestrator - INFO - Creating PVC: test-pvc-7b3bb2da -2025-06-05 16:49:37,410 - tests.orchestrator - INFO - Created PVC: test-pvc-7b3bb2da -2025-06-05 16:49:37,410 - tests.orchestrator - INFO - Waiting for PVC test-pvc-7b3bb2da to be bound -2025-06-05 16:49:39,543 - tests.orchestrator - INFO - PVC test-pvc-7b3bb2da is bound -2025-06-05 16:49:39,544 - tests.orchestrator - INFO - Creating pod test-pod-005090d9 on PVC test-pvc-7b3bb2da -2025-06-05 16:49:39,624 - tests.orchestrator - INFO - Created pod: test-pod-005090d9 using PVC: test-pvc-7b3bb2da -2025-06-05 16:49:39,625 - tests.orchestrator - INFO - Waiting for pod test-pod-005090d9 to be ready -2025-06-05 16:49:39,693 - tests.orchestrator - INFO - Pod test-pod-005090d9 phase: Pending -2025-06-05 16:49:43,827 - tests.orchestrator - INFO - Pod test-pod-005090d9 phase: Running -2025-06-05 16:49:43,827 - tests.orchestrator - INFO - Pod test-pod-005090d9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:49:45,895 - tests.orchestrator - INFO - Pod test-pod-005090d9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:49:47,962 - tests.orchestrator - INFO - Pod test-pod-005090d9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:49:50,030 - tests.orchestrator - INFO - Pod test-pod-005090d9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:49:52,098 - tests.orchestrator - INFO - Pod test-pod-005090d9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:49:54,164 - tests.orchestrator - INFO - Pod test-pod-005090d9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:49:56,232 - tests.orchestrator - INFO - Pod test-pod-005090d9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:49:58,300 - tests.orchestrator - INFO - Pod test-pod-005090d9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:50:00,367 - tests.orchestrator - INFO - Pod test-pod-005090d9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:50:02,435 - tests.orchestrator - INFO - Pod test-pod-005090d9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 16:50:02,435 - tests.orchestrator - INFO - Pod test-pod-005090d9 is ready -2025-06-05 16:50:02,435 - tests.orchestrator - INFO - Creating pod test-pod-4d7d2fab on PVC test-pvc-7b3bb2da -2025-06-05 16:50:02,511 - tests.orchestrator - INFO - Created pod: test-pod-4d7d2fab using PVC: test-pvc-7b3bb2da -2025-06-05 16:50:02,511 - tests.orchestrator - INFO - Waiting for pod test-pod-4d7d2fab to be ready -2025-06-05 16:50:02,579 - tests.orchestrator - INFO - Pod test-pod-4d7d2fab phase: Pending -2025-06-05 16:50:04,645 - tests.orchestrator - INFO - Pod test-pod-4d7d2fab phase: Running -2025-06-05 16:50:04,645 - tests.orchestrator - INFO - Pod test-pod-4d7d2fab conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:50:06,713 - tests.orchestrator - INFO - Pod test-pod-4d7d2fab conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:50:08,781 - tests.orchestrator - INFO - Pod test-pod-4d7d2fab conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:50:10,848 - tests.orchestrator - INFO - Pod test-pod-4d7d2fab conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:50:12,915 - tests.orchestrator - INFO - Pod test-pod-4d7d2fab conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:50:14,984 - tests.orchestrator - INFO - Pod test-pod-4d7d2fab conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:50:17,051 - tests.orchestrator - INFO - Pod test-pod-4d7d2fab conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:50:19,118 - tests.orchestrator - INFO - Pod test-pod-4d7d2fab conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:50:21,187 - tests.orchestrator - INFO - Pod test-pod-4d7d2fab conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:50:23,254 - tests.orchestrator - INFO - Pod test-pod-4d7d2fab conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:50:25,321 - tests.orchestrator - INFO - Pod test-pod-4d7d2fab conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 16:50:25,321 - tests.orchestrator - INFO - Pod test-pod-4d7d2fab is ready -2025-06-05 16:50:25,321 - tests.orchestrator - INFO - Creating pod test-pod-5ae56b7a on PVC test-pvc-7b3bb2da -2025-06-05 16:50:25,404 - tests.orchestrator - INFO - Created pod: test-pod-5ae56b7a using PVC: test-pvc-7b3bb2da -2025-06-05 16:50:25,404 - tests.orchestrator - INFO - Waiting for pod test-pod-5ae56b7a to be ready -2025-06-05 16:50:25,472 - tests.orchestrator - INFO - Pod test-pod-5ae56b7a phase: Pending -2025-06-05 16:50:27,539 - tests.orchestrator - INFO - Pod test-pod-5ae56b7a phase: Running -2025-06-05 16:50:27,539 - tests.orchestrator - INFO - Pod test-pod-5ae56b7a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:50:29,606 - tests.orchestrator - INFO - Pod test-pod-5ae56b7a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:50:31,799 - tests.orchestrator - INFO - Pod test-pod-5ae56b7a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:50:33,866 - tests.orchestrator - INFO - Pod test-pod-5ae56b7a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:50:35,933 - tests.orchestrator - INFO - Pod test-pod-5ae56b7a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:50:38,001 - tests.orchestrator - INFO - Pod test-pod-5ae56b7a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:50:40,068 - tests.orchestrator - INFO - Pod test-pod-5ae56b7a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:50:42,136 - tests.orchestrator - INFO - Pod test-pod-5ae56b7a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:50:44,203 - tests.orchestrator - INFO - Pod test-pod-5ae56b7a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:50:46,270 - tests.orchestrator - INFO - Pod test-pod-5ae56b7a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 16:50:46,270 - tests.orchestrator - INFO - Pod test-pod-5ae56b7a is ready -2025-06-05 16:50:46,270 - tests.orchestrator - INFO - Testing read/write between pods test-pod-005090d9 and test-pod-4d7d2fab sharing PVC test-pvc-7b3bb2da -2025-06-05 16:50:46,341 - tests.orchestrator - ERROR - Failed in read/write verification: (400) -Reason: Bad Request -HTTP response headers: HTTPHeaderDict({'Audit-Id': '76467c30-4547-40d8-b95c-8757f3c9c795', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 16:50:46 GMT', 'Content-Length': '139'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Upgrade request required","reason":"BadRequest","code":400} - - -2025-06-05 16:50:46,341 - tests.orchestrator - INFO - Deleting pod: test-pod-005090d9 from PVC: test-pvc-7b3bb2da -2025-06-05 16:50:46,416 - tests.orchestrator - INFO - Waiting for pod test-pod-005090d9 to be deleted -2025-06-05 16:51:19,031 - tests.orchestrator - INFO - Pod test-pod-005090d9 has been deleted -2025-06-05 16:51:19,031 - tests.orchestrator - INFO - Deleted pod: test-pod-005090d9 -2025-06-05 16:51:19,031 - tests.orchestrator - INFO - Deleting PVC: test-pvc-7b3bb2da -2025-06-05 16:51:19,031 - tests.orchestrator - INFO - Deleting 2 pods using PVC test-pvc-7b3bb2da -2025-06-05 16:51:19,031 - tests.orchestrator - INFO - Deleting pod: test-pod-4d7d2fab from PVC: test-pvc-7b3bb2da -2025-06-05 16:51:19,108 - tests.orchestrator - INFO - Waiting for pod test-pod-4d7d2fab to be deleted -2025-06-05 16:51:51,715 - tests.orchestrator - INFO - Pod test-pod-4d7d2fab has been deleted -2025-06-05 16:51:51,715 - tests.orchestrator - INFO - Deleted pod: test-pod-4d7d2fab -2025-06-05 16:51:51,715 - tests.orchestrator - INFO - Deleting pod: test-pod-5ae56b7a from PVC: test-pvc-7b3bb2da -2025-06-05 16:51:51,790 - tests.orchestrator - INFO - Waiting for pod test-pod-5ae56b7a to be deleted -2025-06-05 16:52:24,395 - tests.orchestrator - INFO - Pod test-pod-5ae56b7a has been deleted -2025-06-05 16:52:24,395 - tests.orchestrator - INFO - Deleted pod: test-pod-5ae56b7a -2025-06-05 16:52:24,466 - tests.orchestrator - INFO - Waiting for PVC test-pvc-7b3bb2da to be deleted -2025-06-05 16:52:24,531 - tests.orchestrator - INFO - PVC test-pvc-7b3bb2da has been deleted -2025-06-05 16:52:24,531 - tests.orchestrator - INFO - Deleted PVC: test-pvc-7b3bb2da -2025-06-05 16:52:24,531 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations -2025-06-05 16:52:24,531 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 1 times) -2025-06-05 16:52:24,531 - tests.orchestrator - INFO - Creating PVC: test-pvc-614c6d80 -2025-06-05 16:52:24,607 - tests.orchestrator - INFO - Created PVC: test-pvc-614c6d80 -2025-06-05 16:52:24,608 - tests.orchestrator - INFO - Waiting for PVC test-pvc-614c6d80 to be bound -2025-06-05 16:52:26,740 - tests.orchestrator - INFO - PVC test-pvc-614c6d80 is bound -2025-06-05 16:52:27,740 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 2 times) -2025-06-05 16:52:27,740 - tests.orchestrator - INFO - Creating PVC: test-pvc-18e6fc84 -2025-06-05 16:52:27,807 - tests.orchestrator - INFO - Created PVC: test-pvc-18e6fc84 -2025-06-05 16:52:27,807 - tests.orchestrator - INFO - Waiting for PVC test-pvc-18e6fc84 to be bound -2025-06-05 16:52:29,940 - tests.orchestrator - INFO - PVC test-pvc-18e6fc84 is bound -2025-06-05 16:52:30,940 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 1 times) -2025-06-05 16:52:30,940 - tests.orchestrator - INFO - Creating pod test-pod-367335c6 on PVC test-pvc-410242e8 -2025-06-05 16:52:31,020 - tests.orchestrator - INFO - Created pod: test-pod-367335c6 using PVC: test-pvc-410242e8 -2025-06-05 16:52:31,021 - tests.orchestrator - INFO - Waiting for pod test-pod-367335c6 to be ready -2025-06-05 16:52:31,088 - tests.orchestrator - INFO - Pod test-pod-367335c6 phase: Pending -2025-06-05 16:52:35,223 - tests.orchestrator - INFO - Pod test-pod-367335c6 phase: Running -2025-06-05 16:52:35,224 - tests.orchestrator - INFO - Pod test-pod-367335c6 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:52:37,292 - tests.orchestrator - INFO - Pod test-pod-367335c6 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:52:39,358 - tests.orchestrator - INFO - Pod test-pod-367335c6 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:52:41,427 - tests.orchestrator - INFO - Pod test-pod-367335c6 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:52:43,493 - tests.orchestrator - INFO - Pod test-pod-367335c6 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:52:45,560 - tests.orchestrator - INFO - Pod test-pod-367335c6 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:52:47,628 - tests.orchestrator - INFO - Pod test-pod-367335c6 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:52:49,696 - tests.orchestrator - INFO - Pod test-pod-367335c6 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:52:51,763 - tests.orchestrator - INFO - Pod test-pod-367335c6 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:52:53,830 - tests.orchestrator - INFO - Pod test-pod-367335c6 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 16:52:53,830 - tests.orchestrator - INFO - Pod test-pod-367335c6 is ready -2025-06-05 16:52:54,831 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 1 times) -2025-06-05 16:52:54,831 - tests.orchestrator - INFO - Deleting PVC: test-pvc-410242e8 -2025-06-05 16:52:54,831 - tests.orchestrator - INFO - Deleting 1 pods using PVC test-pvc-410242e8 -2025-06-05 16:52:54,831 - tests.orchestrator - INFO - Deleting pod: test-pod-367335c6 from PVC: test-pvc-410242e8 -2025-06-05 16:52:54,907 - tests.orchestrator - INFO - Waiting for pod test-pod-367335c6 to be deleted -2025-06-05 16:53:27,511 - tests.orchestrator - INFO - Pod test-pod-367335c6 has been deleted -2025-06-05 16:53:27,511 - tests.orchestrator - INFO - Deleted pod: test-pod-367335c6 -2025-06-05 16:53:27,583 - tests.orchestrator - INFO - Waiting for PVC test-pvc-410242e8 to be deleted -2025-06-05 16:53:27,649 - tests.orchestrator - INFO - PVC test-pvc-410242e8 has been deleted -2025-06-05 16:53:27,649 - tests.orchestrator - INFO - Deleted PVC: test-pvc-410242e8 -2025-06-05 16:53:28,649 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 3 times) -2025-06-05 16:53:28,649 - tests.orchestrator - INFO - Creating PVC: test-pvc-be750096 -2025-06-05 16:53:28,727 - tests.orchestrator - INFO - Created PVC: test-pvc-be750096 -2025-06-05 16:53:28,727 - tests.orchestrator - INFO - Waiting for PVC test-pvc-be750096 to be bound -2025-06-05 16:53:30,858 - tests.orchestrator - INFO - PVC test-pvc-be750096 is bound -2025-06-05 16:53:31,858 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 4 times) -2025-06-05 16:53:31,858 - tests.orchestrator - INFO - Creating PVC: test-pvc-6cb60179 -2025-06-05 16:53:31,939 - tests.orchestrator - INFO - Created PVC: test-pvc-6cb60179 -2025-06-05 16:53:31,939 - tests.orchestrator - INFO - Waiting for PVC test-pvc-6cb60179 to be bound -2025-06-05 16:53:34,074 - tests.orchestrator - INFO - PVC test-pvc-6cb60179 is bound -2025-06-05 16:53:35,075 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 1 times) -2025-06-05 16:53:35,075 - tests.orchestrator - INFO - No pods to delete -2025-06-05 16:53:36,075 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 2 times) -2025-06-05 16:53:36,075 - tests.orchestrator - INFO - No pods to delete -2025-06-05 16:53:37,075 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 2 times) -2025-06-05 16:53:37,075 - tests.orchestrator - INFO - Deleting PVC: test-pvc-be750096 -2025-06-05 16:53:37,148 - tests.orchestrator - INFO - Waiting for PVC test-pvc-be750096 to be deleted -2025-06-05 16:53:37,212 - tests.orchestrator - INFO - PVC test-pvc-be750096 has been deleted -2025-06-05 16:53:37,212 - tests.orchestrator - INFO - Deleted PVC: test-pvc-be750096 -2025-06-05 16:53:38,212 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 5 times) -2025-06-05 16:53:38,212 - tests.orchestrator - INFO - Creating PVC: test-pvc-ee8508e1 -2025-06-05 16:53:38,282 - tests.orchestrator - INFO - Created PVC: test-pvc-ee8508e1 -2025-06-05 16:53:38,282 - tests.orchestrator - INFO - Waiting for PVC test-pvc-ee8508e1 to be bound -2025-06-05 16:53:40,414 - tests.orchestrator - INFO - PVC test-pvc-ee8508e1 is bound -2025-06-05 16:53:41,414 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 1 times) -2025-06-05 16:53:41,414 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test -2025-06-05 16:53:42,416 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 2 times) -2025-06-05 16:53:42,416 - tests.orchestrator - INFO - Creating pod test-pod-6b2c2055 on PVC test-pvc-18e6fc84 -2025-06-05 16:53:42,490 - tests.orchestrator - INFO - Created pod: test-pod-6b2c2055 using PVC: test-pvc-18e6fc84 -2025-06-05 16:53:42,490 - tests.orchestrator - INFO - Waiting for pod test-pod-6b2c2055 to be ready -2025-06-05 16:53:42,557 - tests.orchestrator - INFO - Pod test-pod-6b2c2055 phase: Pending -2025-06-05 16:53:44,625 - tests.orchestrator - INFO - Pod test-pod-6b2c2055 phase: Running -2025-06-05 16:53:44,625 - tests.orchestrator - INFO - Pod test-pod-6b2c2055 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:53:46,692 - tests.orchestrator - INFO - Pod test-pod-6b2c2055 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:53:48,760 - tests.orchestrator - INFO - Pod test-pod-6b2c2055 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:53:50,827 - tests.orchestrator - INFO - Pod test-pod-6b2c2055 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:53:52,895 - tests.orchestrator - INFO - Pod test-pod-6b2c2055 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:53:54,963 - tests.orchestrator - INFO - Pod test-pod-6b2c2055 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:53:57,031 - tests.orchestrator - INFO - Pod test-pod-6b2c2055 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:53:59,098 - tests.orchestrator - INFO - Pod test-pod-6b2c2055 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:54:01,166 - tests.orchestrator - INFO - Pod test-pod-6b2c2055 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:54:03,233 - tests.orchestrator - INFO - Pod test-pod-6b2c2055 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:54:05,301 - tests.orchestrator - INFO - Pod test-pod-6b2c2055 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 16:54:05,301 - tests.orchestrator - INFO - Pod test-pod-6b2c2055 is ready -2025-06-05 16:54:06,301 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 6 times) -2025-06-05 16:54:06,301 - tests.orchestrator - INFO - Creating PVC: test-pvc-9c79d5b3 -2025-06-05 16:54:06,378 - tests.orchestrator - INFO - Created PVC: test-pvc-9c79d5b3 -2025-06-05 16:54:06,379 - tests.orchestrator - INFO - Waiting for PVC test-pvc-9c79d5b3 to be bound -2025-06-05 16:54:08,513 - tests.orchestrator - INFO - PVC test-pvc-9c79d5b3 is bound -2025-06-05 16:54:09,514 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 3 times) -2025-06-05 16:54:09,514 - tests.orchestrator - INFO - Deleting PVC: test-pvc-ee8508e1 -2025-06-05 16:54:09,586 - tests.orchestrator - INFO - Waiting for PVC test-pvc-ee8508e1 to be deleted -2025-06-05 16:54:09,651 - tests.orchestrator - INFO - PVC test-pvc-ee8508e1 has been deleted -2025-06-05 16:54:09,651 - tests.orchestrator - INFO - Deleted PVC: test-pvc-ee8508e1 -2025-06-05 16:54:10,651 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 3 times) -2025-06-05 16:54:10,651 - tests.orchestrator - INFO - Creating pod test-pod-71e98828 on PVC test-pvc-9c79d5b3 -2025-06-05 16:54:10,724 - tests.orchestrator - INFO - Created pod: test-pod-71e98828 using PVC: test-pvc-9c79d5b3 -2025-06-05 16:54:10,725 - tests.orchestrator - INFO - Waiting for pod test-pod-71e98828 to be ready -2025-06-05 16:54:10,792 - tests.orchestrator - INFO - Pod test-pod-71e98828 phase: Pending -2025-06-05 16:54:12,859 - tests.orchestrator - INFO - Pod test-pod-71e98828 phase: Running -2025-06-05 16:54:12,859 - tests.orchestrator - INFO - Pod test-pod-71e98828 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:54:14,928 - tests.orchestrator - INFO - Pod test-pod-71e98828 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:54:16,996 - tests.orchestrator - INFO - Pod test-pod-71e98828 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:54:19,062 - tests.orchestrator - INFO - Pod test-pod-71e98828 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:54:21,130 - tests.orchestrator - INFO - Pod test-pod-71e98828 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:54:23,198 - tests.orchestrator - INFO - Pod test-pod-71e98828 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:54:25,265 - tests.orchestrator - INFO - Pod test-pod-71e98828 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:54:27,332 - tests.orchestrator - INFO - Pod test-pod-71e98828 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:54:29,400 - tests.orchestrator - INFO - Pod test-pod-71e98828 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:54:31,568 - tests.orchestrator - INFO - Pod test-pod-71e98828 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 16:54:33,635 - tests.orchestrator - INFO - Pod test-pod-71e98828 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 16:54:33,636 - tests.orchestrator - INFO - Pod test-pod-71e98828 is ready -2025-06-05 16:54:34,636 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 3 times) -2025-06-05 16:54:34,636 - tests.orchestrator - INFO - Deleting pod: test-pod-71e98828 from PVC: test-pvc-9c79d5b3 -2025-06-05 16:54:34,710 - tests.orchestrator - INFO - Waiting for pod test-pod-71e98828 to be deleted -2025-06-05 16:55:07,317 - tests.orchestrator - INFO - Pod test-pod-71e98828 has been deleted -2025-06-05 16:55:07,317 - tests.orchestrator - INFO - Deleted pod: test-pod-71e98828 -2025-06-05 16:55:08,318 - tests.orchestrator - INFO - Test completed in 333.38 seconds -2025-06-05 16:55:08,318 - tests.orchestrator - INFO - ===== STARTING COMPREHENSIVE CLEANUP ===== -2025-06-05 16:55:08,318 - tests.orchestrator - INFO - Deleting 1 pods (force=False)... -2025-06-05 16:55:08,318 - tests.orchestrator - INFO - Deleting pod: test-pod-6b2c2055 from PVC: test-pvc-18e6fc84 -2025-06-05 16:55:08,393 - tests.orchestrator - INFO - Waiting for pod test-pod-6b2c2055 to be deleted -2025-06-05 16:55:41,002 - tests.orchestrator - INFO - Pod test-pod-6b2c2055 has been deleted -2025-06-05 16:55:41,002 - tests.orchestrator - INFO - Deleted pod: test-pod-6b2c2055 -2025-06-05 16:55:46,003 - tests.orchestrator - INFO - Deleting 4 PVCs (force=False)... -2025-06-05 16:55:46,003 - tests.orchestrator - INFO - Deleting PVC: test-pvc-614c6d80 -2025-06-05 16:55:46,074 - tests.orchestrator - INFO - Waiting for PVC test-pvc-614c6d80 to be deleted -2025-06-05 16:55:46,139 - tests.orchestrator - INFO - PVC test-pvc-614c6d80 has been deleted -2025-06-05 16:55:46,139 - tests.orchestrator - INFO - Deleted PVC: test-pvc-614c6d80 -2025-06-05 16:55:46,139 - tests.orchestrator - INFO - Deleting PVC: test-pvc-18e6fc84 -2025-06-05 16:55:46,210 - tests.orchestrator - INFO - Waiting for PVC test-pvc-18e6fc84 to be deleted -2025-06-05 16:55:46,276 - tests.orchestrator - INFO - PVC test-pvc-18e6fc84 has been deleted -2025-06-05 16:55:46,276 - tests.orchestrator - INFO - Deleted PVC: test-pvc-18e6fc84 -2025-06-05 16:55:46,277 - tests.orchestrator - INFO - Deleting PVC: test-pvc-6cb60179 -2025-06-05 16:55:46,347 - tests.orchestrator - INFO - Waiting for PVC test-pvc-6cb60179 to be deleted -2025-06-05 16:55:46,413 - tests.orchestrator - INFO - PVC test-pvc-6cb60179 has been deleted -2025-06-05 16:55:46,413 - tests.orchestrator - INFO - Deleted PVC: test-pvc-6cb60179 -2025-06-05 16:55:46,413 - tests.orchestrator - INFO - Deleting PVC: test-pvc-9c79d5b3 -2025-06-05 16:55:46,488 - tests.orchestrator - INFO - Waiting for PVC test-pvc-9c79d5b3 to be deleted -2025-06-05 16:55:46,567 - tests.orchestrator - INFO - PVC test-pvc-9c79d5b3 has been deleted -2025-06-05 16:55:46,568 - tests.orchestrator - INFO - Deleted PVC: test-pvc-9c79d5b3 -2025-06-05 16:55:46,843 - tests.orchestrator - WARNING - First cleanup pass incomplete. Remaining resources: ['pod/test-pod-0dadc981', 'pod/test-pod-0fcf8a5b', 'pod/test-pod-28a31e9a', 'pod/test-pod-2f20e8f1', 'pod/test-pod-4022a58d', 'pod/test-pod-4eee03ff', 'pod/test-pod-60374c6e', 'pod/test-pod-74abb322', 'pod/test-pod-955db556', 'pod/test-pod-988a488e', 'pod/test-pod-9de3bebb', 'pvc/concurrent-pvc-0aa045b0', 'pvc/concurrent-pvc-248d7584', 'pvc/concurrent-pvc-5b29e9cd', 'pvc/concurrent-pvc-903eb13a', 'pvc/concurrent-pvc-b0004a21', 'pvc/concurrent-pvc-ba7e5e4d', 'pvc/concurrent-pvc-f6a663b2', 'pvc/test-pvc-20f868e5', 'pvc/test-pvc-2381ce5e', 'pvc/test-pvc-2686f57c', 'pvc/test-pvc-269e9543', 'pvc/test-pvc-3f880237', 'pvc/test-pvc-4348540c', 'pvc/test-pvc-48e3d2d9', 'pvc/test-pvc-567589be', 'pvc/test-pvc-6282a942', 'pvc/test-pvc-83426f91', 'pvc/test-pvc-98b7ae8a', 'pvc/test-pvc-a253186e', 'pvc/test-pvc-b967555b', 'pvc/test-pvc-dbe6630f', 'pvc/test-pvc-e3594f67', 'pvc/test-pvc-e5985cb1'] -2025-06-05 16:55:46,843 - tests.orchestrator - INFO - Attempting force deletion of remaining resources... -2025-06-05 16:55:46,843 - tests.orchestrator - INFO - Deleting 0 pods (force=True)... -2025-06-05 16:55:51,843 - tests.orchestrator - INFO - Deleting 0 PVCs (force=True)... -2025-06-05 16:55:52,175 - tests.orchestrator - ERROR - Cleanup incomplete. Remaining resources after force deletion: ['pod/test-pod-0dadc981', 'pod/test-pod-0fcf8a5b', 'pod/test-pod-28a31e9a', 'pod/test-pod-2f20e8f1', 'pod/test-pod-4022a58d', 'pod/test-pod-4eee03ff', 'pod/test-pod-60374c6e', 'pod/test-pod-74abb322', 'pod/test-pod-955db556', 'pod/test-pod-988a488e', 'pod/test-pod-9de3bebb', 'pvc/concurrent-pvc-0aa045b0', 'pvc/concurrent-pvc-248d7584', 'pvc/concurrent-pvc-5b29e9cd', 'pvc/concurrent-pvc-903eb13a', 'pvc/concurrent-pvc-b0004a21', 'pvc/concurrent-pvc-ba7e5e4d', 'pvc/concurrent-pvc-f6a663b2', 'pvc/test-pvc-20f868e5', 'pvc/test-pvc-2381ce5e', 'pvc/test-pvc-2686f57c', 'pvc/test-pvc-269e9543', 'pvc/test-pvc-3f880237', 'pvc/test-pvc-4348540c', 'pvc/test-pvc-48e3d2d9', 'pvc/test-pvc-567589be', 'pvc/test-pvc-6282a942', 'pvc/test-pvc-83426f91', 'pvc/test-pvc-98b7ae8a', 'pvc/test-pvc-a253186e', 'pvc/test-pvc-b967555b', 'pvc/test-pvc-dbe6630f', 'pvc/test-pvc-e3594f67', 'pvc/test-pvc-e5985cb1'] -2025-06-05 16:55:52,175 - tests.orchestrator - INFO - Cleanup completed successfully in 43.86 seconds -2025-06-05 16:55:52,175 - tests.orchestrator - INFO - ===== CLEANUP PROCESS FINISHED ===== -2025-06-05 16:55:52,176 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-05 16:55:52,176 - tests.orchestrator - INFO - --- Operations --- -2025-06-05 16:55:52,176 - tests.orchestrator - INFO - create_pvc: 8 succeeded, 0 failed (100.0%) -2025-06-05 16:55:52,176 - tests.orchestrator - INFO - attach_pod: 6 succeeded, 0 failed (100.0%) -2025-06-05 16:55:52,176 - tests.orchestrator - INFO - delete_pod: 6 succeeded, 0 failed (100.0%) -2025-06-05 16:55:52,176 - tests.orchestrator - INFO - delete_pvc: 8 succeeded, 0 failed (100.0%) -2025-06-05 16:55:52,176 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) -2025-06-05 16:55:52,176 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) -2025-06-05 16:55:52,176 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-05 16:55:52,176 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-05 16:55:52,176 - tests.orchestrator - INFO - many_to_one: No runs -2025-06-05 16:55:52,176 - tests.orchestrator - INFO - one_to_one: No runs -2025-06-05 16:55:52,176 - tests.orchestrator - INFO - concurrent_pvc: No runs -2025-06-05 16:55:52,176 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_170306.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_170306.log deleted file mode 100644 index aea06a813..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_170306.log +++ /dev/null @@ -1,17 +0,0 @@ -2025-06-05 17:03:06,949 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 17:03:07,165 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 17:03:07,165 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-05 17:03:07,357 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 17:03:07,357 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 17:03:07,357 - tests.orchestrator - INFO - Creating PVC: test-pvc-63ada5ce -2025-06-05 17:03:07,436 - tests.orchestrator - INFO - Created PVC: test-pvc-63ada5ce -2025-06-05 17:03:07,436 - tests.orchestrator - INFO - Waiting for PVC test-pvc-63ada5ce to be bound -2025-06-05 17:03:09,571 - tests.orchestrator - INFO - PVC test-pvc-63ada5ce is bound -2025-06-05 17:03:09,571 - tests.orchestrator - INFO - Creating PVC: test-pvc-601432ce -2025-06-05 17:03:09,638 - tests.orchestrator - INFO - Created PVC: test-pvc-601432ce -2025-06-05 17:03:09,638 - tests.orchestrator - INFO - Waiting for PVC test-pvc-601432ce to be bound -2025-06-05 17:03:11,896 - tests.orchestrator - INFO - PVC test-pvc-601432ce is bound -2025-06-05 17:03:11,896 - tests.orchestrator - INFO - Creating pod test-pod-564344d0 on PVC test-pvc-63ada5ce -2025-06-05 17:03:12,343 - tests.orchestrator - INFO - Created pod: test-pod-564344d0 using PVC: test-pvc-63ada5ce -2025-06-05 17:03:12,343 - tests.orchestrator - INFO - Waiting for pod test-pod-564344d0 to be ready -2025-06-05 17:03:12,413 - tests.orchestrator - INFO - Pod test-pod-564344d0 phase: Pending diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_170342.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_170342.log deleted file mode 100644 index 525986131..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_170342.log +++ /dev/null @@ -1,17 +0,0 @@ -2025-06-05 17:03:42,827 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 17:03:43,043 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 17:03:43,044 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-05 17:03:43,233 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 17:03:43,233 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 17:03:43,233 - tests.orchestrator - INFO - Creating PVC: test-pvc-455abf8b -2025-06-05 17:03:43,309 - tests.orchestrator - INFO - Created PVC: test-pvc-455abf8b -2025-06-05 17:03:43,309 - tests.orchestrator - INFO - Waiting for PVC test-pvc-455abf8b to be bound -2025-06-05 17:03:45,443 - tests.orchestrator - INFO - PVC test-pvc-455abf8b is bound -2025-06-05 17:03:45,443 - tests.orchestrator - INFO - Creating PVC: test-pvc-51dbaa96 -2025-06-05 17:03:45,510 - tests.orchestrator - INFO - Created PVC: test-pvc-51dbaa96 -2025-06-05 17:03:45,511 - tests.orchestrator - INFO - Waiting for PVC test-pvc-51dbaa96 to be bound -2025-06-05 17:03:47,644 - tests.orchestrator - INFO - PVC test-pvc-51dbaa96 is bound -2025-06-05 17:03:47,644 - tests.orchestrator - INFO - Creating pod test-pod-e1337987 on PVC test-pvc-455abf8b -2025-06-05 17:03:47,724 - tests.orchestrator - INFO - Created pod: test-pod-e1337987 using PVC: test-pvc-455abf8b -2025-06-05 17:03:47,724 - tests.orchestrator - INFO - Waiting for pod test-pod-e1337987 to be ready -2025-06-05 17:03:47,791 - tests.orchestrator - INFO - Pod test-pod-e1337987 phase: Pending diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_170405.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_170405.log deleted file mode 100644 index 2e9f3bea5..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_170405.log +++ /dev/null @@ -1,39 +0,0 @@ -2025-06-05 17:04:05,563 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 17:04:05,759 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 17:04:05,759 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-05 17:04:05,948 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 17:04:05,948 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 17:04:05,948 - tests.orchestrator - INFO - Creating PVC: test-pvc-0b5af805 -2025-06-05 17:04:06,017 - tests.orchestrator - INFO - Created PVC: test-pvc-0b5af805 -2025-06-05 17:04:06,017 - tests.orchestrator - INFO - Waiting for PVC test-pvc-0b5af805 to be bound -2025-06-05 17:04:08,153 - tests.orchestrator - INFO - PVC test-pvc-0b5af805 is bound -2025-06-05 17:04:08,153 - tests.orchestrator - INFO - Creating PVC: test-pvc-d3ee5325 -2025-06-05 17:04:08,222 - tests.orchestrator - INFO - Created PVC: test-pvc-d3ee5325 -2025-06-05 17:04:08,222 - tests.orchestrator - INFO - Waiting for PVC test-pvc-d3ee5325 to be bound -2025-06-05 17:04:10,358 - tests.orchestrator - INFO - PVC test-pvc-d3ee5325 is bound -2025-06-05 17:04:10,358 - tests.orchestrator - INFO - Creating pod test-pod-3cc2d48d on PVC test-pvc-0b5af805 -2025-06-05 17:04:10,433 - tests.orchestrator - INFO - Created pod: test-pod-3cc2d48d using PVC: test-pvc-0b5af805 -2025-06-05 17:04:10,433 - tests.orchestrator - INFO - Waiting for pod test-pod-3cc2d48d to be ready -2025-06-05 17:04:10,500 - tests.orchestrator - INFO - Pod test-pod-3cc2d48d phase: Pending -2025-06-05 17:04:12,570 - tests.orchestrator - INFO - Pod test-pod-3cc2d48d phase: Running -2025-06-05 17:04:12,570 - tests.orchestrator - INFO - Pod test-pod-3cc2d48d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:04:14,639 - tests.orchestrator - INFO - Pod test-pod-3cc2d48d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:04:16,707 - tests.orchestrator - INFO - Pod test-pod-3cc2d48d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:04:18,775 - tests.orchestrator - INFO - Pod test-pod-3cc2d48d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:04:20,845 - tests.orchestrator - INFO - Pod test-pod-3cc2d48d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:04:22,913 - tests.orchestrator - INFO - Pod test-pod-3cc2d48d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:04:24,982 - tests.orchestrator - INFO - Pod test-pod-3cc2d48d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:04:27,051 - tests.orchestrator - INFO - Pod test-pod-3cc2d48d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 17:04:27,051 - tests.orchestrator - INFO - Pod test-pod-3cc2d48d is ready -2025-06-05 17:04:27,051 - tests.orchestrator - INFO - Creating pod test-pod-a9935180 on PVC test-pvc-0b5af805 -2025-06-05 17:04:27,135 - tests.orchestrator - INFO - Created pod: test-pod-a9935180 using PVC: test-pvc-0b5af805 -2025-06-05 17:04:27,135 - tests.orchestrator - INFO - Waiting for pod test-pod-a9935180 to be ready -2025-06-05 17:04:27,202 - tests.orchestrator - INFO - Pod test-pod-a9935180 phase: Pending -2025-06-05 17:04:29,271 - tests.orchestrator - INFO - Pod test-pod-a9935180 phase: Running -2025-06-05 17:04:29,271 - tests.orchestrator - INFO - Pod test-pod-a9935180 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:04:31,340 - tests.orchestrator - INFO - Pod test-pod-a9935180 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:04:33,409 - tests.orchestrator - INFO - Pod test-pod-a9935180 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:04:35,477 - tests.orchestrator - INFO - Pod test-pod-a9935180 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:04:37,547 - tests.orchestrator - INFO - Pod test-pod-a9935180 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:04:39,615 - tests.orchestrator - INFO - Pod test-pod-a9935180 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:04:41,683 - tests.orchestrator - INFO - Pod test-pod-a9935180 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_170502.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_170502.log deleted file mode 100644 index b4d515d9c..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_170502.log +++ /dev/null @@ -1,12 +0,0 @@ -2025-06-05 17:05:02,906 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 17:05:03,121 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 17:05:03,121 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-05 17:05:03,312 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 17:05:03,312 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 17:05:03,313 - tests.orchestrator - INFO - Creating PVC: test-pvc-3691f2a0 -2025-06-05 17:05:03,396 - tests.orchestrator - INFO - Created PVC: test-pvc-3691f2a0 -2025-06-05 17:05:03,396 - tests.orchestrator - INFO - Waiting for PVC test-pvc-3691f2a0 to be bound -2025-06-05 17:05:05,534 - tests.orchestrator - INFO - PVC test-pvc-3691f2a0 is bound -2025-06-05 17:05:05,534 - tests.orchestrator - INFO - Creating PVC: test-pvc-64bf8174 -2025-06-05 17:05:05,608 - tests.orchestrator - INFO - Created PVC: test-pvc-64bf8174 -2025-06-05 17:05:05,608 - tests.orchestrator - INFO - Waiting for PVC test-pvc-64bf8174 to be bound diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_170526.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_170526.log deleted file mode 100644 index af01d9c7c..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_170526.log +++ /dev/null @@ -1,234 +0,0 @@ -2025-06-05 17:05:26,238 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 17:05:26,435 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 17:05:26,436 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-05 17:05:26,638 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 17:05:26,638 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 17:05:26,638 - tests.orchestrator - INFO - Creating PVC: test-pvc-9dae599f -2025-06-05 17:05:26,711 - tests.orchestrator - INFO - Created PVC: test-pvc-9dae599f -2025-06-05 17:05:26,712 - tests.orchestrator - INFO - Waiting for PVC test-pvc-9dae599f to be bound -2025-06-05 17:05:28,848 - tests.orchestrator - INFO - PVC test-pvc-9dae599f is bound -2025-06-05 17:05:28,849 - tests.orchestrator - INFO - Creating PVC: test-pvc-759dbae5 -2025-06-05 17:05:28,916 - tests.orchestrator - INFO - Created PVC: test-pvc-759dbae5 -2025-06-05 17:05:28,916 - tests.orchestrator - INFO - Waiting for PVC test-pvc-759dbae5 to be bound -2025-06-05 17:05:31,051 - tests.orchestrator - INFO - PVC test-pvc-759dbae5 is bound -2025-06-05 17:05:31,052 - tests.orchestrator - INFO - Creating pod test-pod-60858a96 on PVC test-pvc-9dae599f -2025-06-05 17:05:31,134 - tests.orchestrator - INFO - Created pod: test-pod-60858a96 using PVC: test-pvc-9dae599f -2025-06-05 17:05:31,134 - tests.orchestrator - INFO - Waiting for pod test-pod-60858a96 to be ready -2025-06-05 17:05:31,203 - tests.orchestrator - INFO - Pod test-pod-60858a96 phase: Pending -2025-06-05 17:05:35,349 - tests.orchestrator - INFO - Pod test-pod-60858a96 phase: Running -2025-06-05 17:05:35,349 - tests.orchestrator - INFO - Pod test-pod-60858a96 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:05:37,418 - tests.orchestrator - INFO - Pod test-pod-60858a96 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:05:39,486 - tests.orchestrator - INFO - Pod test-pod-60858a96 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:05:41,723 - tests.orchestrator - INFO - Pod test-pod-60858a96 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:05:43,792 - tests.orchestrator - INFO - Pod test-pod-60858a96 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:05:45,860 - tests.orchestrator - INFO - Pod test-pod-60858a96 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:05:47,929 - tests.orchestrator - INFO - Pod test-pod-60858a96 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 17:05:47,929 - tests.orchestrator - INFO - Pod test-pod-60858a96 is ready -2025-06-05 17:05:47,929 - tests.orchestrator - INFO - Creating pod test-pod-e1e1abfd on PVC test-pvc-759dbae5 -2025-06-05 17:05:48,053 - tests.orchestrator - INFO - Created pod: test-pod-e1e1abfd using PVC: test-pvc-759dbae5 -2025-06-05 17:05:48,053 - tests.orchestrator - INFO - Waiting for pod test-pod-e1e1abfd to be ready -2025-06-05 17:05:48,139 - tests.orchestrator - INFO - Pod test-pod-e1e1abfd phase: Pending -2025-06-05 17:05:52,277 - tests.orchestrator - INFO - Pod test-pod-e1e1abfd phase: Running -2025-06-05 17:05:52,278 - tests.orchestrator - INFO - Pod test-pod-e1e1abfd conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:05:54,345 - tests.orchestrator - INFO - Pod test-pod-e1e1abfd conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:05:56,414 - tests.orchestrator - INFO - Pod test-pod-e1e1abfd conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:05:58,484 - tests.orchestrator - INFO - Pod test-pod-e1e1abfd conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:06:00,553 - tests.orchestrator - INFO - Pod test-pod-e1e1abfd conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:06:02,623 - tests.orchestrator - INFO - Pod test-pod-e1e1abfd conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:06:04,693 - tests.orchestrator - INFO - Pod test-pod-e1e1abfd conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:06:06,766 - tests.orchestrator - INFO - Pod test-pod-e1e1abfd conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:06:08,838 - tests.orchestrator - INFO - Pod test-pod-e1e1abfd conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:06:10,908 - tests.orchestrator - INFO - Pod test-pod-e1e1abfd conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 17:06:10,908 - tests.orchestrator - INFO - Pod test-pod-e1e1abfd is ready -2025-06-05 17:06:10,908 - tests.orchestrator - INFO - Creating pod test-pod-177eb3e7 on PVC test-pvc-9dae599f -2025-06-05 17:06:10,984 - tests.orchestrator - INFO - Created pod: test-pod-177eb3e7 using PVC: test-pvc-9dae599f -2025-06-05 17:06:10,984 - tests.orchestrator - INFO - Waiting for pod test-pod-177eb3e7 to be ready -2025-06-05 17:06:11,053 - tests.orchestrator - INFO - Pod test-pod-177eb3e7 phase: Pending -2025-06-05 17:06:13,125 - tests.orchestrator - INFO - Pod test-pod-177eb3e7 phase: Running -2025-06-05 17:06:13,125 - tests.orchestrator - INFO - Pod test-pod-177eb3e7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:06:15,195 - tests.orchestrator - INFO - Pod test-pod-177eb3e7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:06:17,263 - tests.orchestrator - INFO - Pod test-pod-177eb3e7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:06:19,332 - tests.orchestrator - INFO - Pod test-pod-177eb3e7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:06:21,402 - tests.orchestrator - INFO - Pod test-pod-177eb3e7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:06:23,470 - tests.orchestrator - INFO - Pod test-pod-177eb3e7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:06:25,540 - tests.orchestrator - INFO - Pod test-pod-177eb3e7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:06:27,607 - tests.orchestrator - INFO - Pod test-pod-177eb3e7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:06:29,677 - tests.orchestrator - INFO - Pod test-pod-177eb3e7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:06:31,747 - tests.orchestrator - INFO - Pod test-pod-177eb3e7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:06:33,819 - tests.orchestrator - INFO - Pod test-pod-177eb3e7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 17:06:33,819 - tests.orchestrator - INFO - Pod test-pod-177eb3e7 is ready -2025-06-05 17:06:33,819 - tests.orchestrator - INFO - Testing read/write between pods test-pod-177eb3e7 and test-pod-60858a96 sharing PVC test-pvc-9dae599f -2025-06-05 17:06:33,887 - tests.orchestrator - ERROR - Failed in read/write verification: (400) -Reason: Bad Request -HTTP response headers: HTTPHeaderDict({'Audit-Id': 'b5f76f0a-dcce-47f0-8633-e17708ea40ac', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 17:06:33 GMT', 'Content-Length': '139'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Upgrade request required","reason":"BadRequest","code":400} - - -2025-06-05 17:06:33,887 - tests.orchestrator - INFO - Deleting pod: test-pod-60858a96 from PVC: test-pvc-9dae599f -2025-06-05 17:06:33,966 - tests.orchestrator - INFO - Waiting for pod test-pod-60858a96 to be deleted -2025-06-05 17:07:06,621 - tests.orchestrator - INFO - Pod test-pod-60858a96 has been deleted -2025-06-05 17:07:06,621 - tests.orchestrator - INFO - Deleted pod: test-pod-60858a96 -2025-06-05 17:07:06,621 - tests.orchestrator - INFO - Deleting PVC: test-pvc-9dae599f -2025-06-05 17:07:06,621 - tests.orchestrator - INFO - Deleting 1 pods using PVC test-pvc-9dae599f -2025-06-05 17:07:06,621 - tests.orchestrator - INFO - Deleting pod: test-pod-177eb3e7 from PVC: test-pvc-9dae599f -2025-06-05 17:07:06,699 - tests.orchestrator - INFO - Waiting for pod test-pod-177eb3e7 to be deleted -2025-06-05 17:07:39,317 - tests.orchestrator - INFO - Pod test-pod-177eb3e7 has been deleted -2025-06-05 17:07:39,317 - tests.orchestrator - INFO - Deleted pod: test-pod-177eb3e7 -2025-06-05 17:07:39,393 - tests.orchestrator - INFO - Waiting for PVC test-pvc-9dae599f to be deleted -2025-06-05 17:07:39,461 - tests.orchestrator - INFO - PVC test-pvc-9dae599f has been deleted -2025-06-05 17:07:39,461 - tests.orchestrator - INFO - Deleted PVC: test-pvc-9dae599f -2025-06-05 17:07:39,461 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations -2025-06-05 17:07:39,461 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 1 times) -2025-06-05 17:07:39,461 - tests.orchestrator - INFO - Creating pod test-pod-3ce2f282 on PVC test-pvc-759dbae5 -2025-06-05 17:07:39,585 - tests.orchestrator - INFO - Created pod: test-pod-3ce2f282 using PVC: test-pvc-759dbae5 -2025-06-05 17:07:39,585 - tests.orchestrator - INFO - Waiting for pod test-pod-3ce2f282 to be ready -2025-06-05 17:07:39,655 - tests.orchestrator - INFO - Pod test-pod-3ce2f282 phase: Pending -2025-06-05 17:07:41,879 - tests.orchestrator - INFO - Pod test-pod-3ce2f282 phase: Running -2025-06-05 17:07:41,879 - tests.orchestrator - INFO - Pod test-pod-3ce2f282 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:07:43,948 - tests.orchestrator - INFO - Pod test-pod-3ce2f282 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:07:46,017 - tests.orchestrator - INFO - Pod test-pod-3ce2f282 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:07:48,085 - tests.orchestrator - INFO - Pod test-pod-3ce2f282 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:07:50,155 - tests.orchestrator - INFO - Pod test-pod-3ce2f282 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:07:52,225 - tests.orchestrator - INFO - Pod test-pod-3ce2f282 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:07:54,292 - tests.orchestrator - INFO - Pod test-pod-3ce2f282 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:07:56,363 - tests.orchestrator - INFO - Pod test-pod-3ce2f282 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:07:58,434 - tests.orchestrator - INFO - Pod test-pod-3ce2f282 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:08:00,502 - tests.orchestrator - INFO - Pod test-pod-3ce2f282 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:08:02,573 - tests.orchestrator - INFO - Pod test-pod-3ce2f282 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 17:08:02,573 - tests.orchestrator - INFO - Pod test-pod-3ce2f282 is ready -2025-06-05 17:08:03,574 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 1 times) -2025-06-05 17:08:03,574 - tests.orchestrator - INFO - Deleting pod: test-pod-3ce2f282 from PVC: test-pvc-759dbae5 -2025-06-05 17:08:03,651 - tests.orchestrator - INFO - Waiting for pod test-pod-3ce2f282 to be deleted -2025-06-05 17:08:36,384 - tests.orchestrator - INFO - Pod test-pod-3ce2f282 has been deleted -2025-06-05 17:08:36,384 - tests.orchestrator - INFO - Deleted pod: test-pod-3ce2f282 -2025-06-05 17:08:37,384 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 1 times) -2025-06-05 17:08:37,384 - tests.orchestrator - INFO - Creating PVC: test-pvc-e7dcdff0 -2025-06-05 17:08:37,462 - tests.orchestrator - INFO - Created PVC: test-pvc-e7dcdff0 -2025-06-05 17:08:37,463 - tests.orchestrator - INFO - Waiting for PVC test-pvc-e7dcdff0 to be bound -2025-06-05 17:08:39,598 - tests.orchestrator - INFO - PVC test-pvc-e7dcdff0 is bound -2025-06-05 17:08:40,599 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 2 times) -2025-06-05 17:08:40,599 - tests.orchestrator - INFO - Creating PVC: test-pvc-1511fcaf -2025-06-05 17:08:40,670 - tests.orchestrator - INFO - Created PVC: test-pvc-1511fcaf -2025-06-05 17:08:40,670 - tests.orchestrator - INFO - Waiting for PVC test-pvc-1511fcaf to be bound -2025-06-05 17:08:42,808 - tests.orchestrator - INFO - PVC test-pvc-1511fcaf is bound -2025-06-05 17:08:43,808 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 3 times) -2025-06-05 17:08:43,809 - tests.orchestrator - INFO - Creating PVC: test-pvc-47e77b85 -2025-06-05 17:08:43,878 - tests.orchestrator - INFO - Created PVC: test-pvc-47e77b85 -2025-06-05 17:08:43,879 - tests.orchestrator - INFO - Waiting for PVC test-pvc-47e77b85 to be bound -2025-06-05 17:08:46,014 - tests.orchestrator - INFO - PVC test-pvc-47e77b85 is bound -2025-06-05 17:08:47,015 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 1 times) -2025-06-05 17:08:47,015 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test -2025-06-05 17:08:48,015 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 2 times) -2025-06-05 17:08:48,015 - tests.orchestrator - INFO - Creating pod test-pod-94de82f9 on PVC test-pvc-1511fcaf -2025-06-05 17:08:48,137 - tests.orchestrator - INFO - Created pod: test-pod-94de82f9 using PVC: test-pvc-1511fcaf -2025-06-05 17:08:48,137 - tests.orchestrator - INFO - Waiting for pod test-pod-94de82f9 to be ready -2025-06-05 17:08:48,224 - tests.orchestrator - INFO - Pod test-pod-94de82f9 phase: Pending -2025-06-05 17:08:52,367 - tests.orchestrator - INFO - Pod test-pod-94de82f9 phase: Running -2025-06-05 17:08:52,367 - tests.orchestrator - INFO - Pod test-pod-94de82f9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:08:54,435 - tests.orchestrator - INFO - Pod test-pod-94de82f9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:08:56,504 - tests.orchestrator - INFO - Pod test-pod-94de82f9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:08:58,574 - tests.orchestrator - INFO - Pod test-pod-94de82f9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:09:00,641 - tests.orchestrator - INFO - Pod test-pod-94de82f9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:09:02,713 - tests.orchestrator - INFO - Pod test-pod-94de82f9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:09:04,782 - tests.orchestrator - INFO - Pod test-pod-94de82f9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 17:09:04,782 - tests.orchestrator - INFO - Pod test-pod-94de82f9 is ready -2025-06-05 17:09:05,782 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 3 times) -2025-06-05 17:09:05,782 - tests.orchestrator - INFO - Creating pod test-pod-42c169cc on PVC test-pvc-1511fcaf -2025-06-05 17:09:05,861 - tests.orchestrator - INFO - Created pod: test-pod-42c169cc using PVC: test-pvc-1511fcaf -2025-06-05 17:09:05,861 - tests.orchestrator - INFO - Waiting for pod test-pod-42c169cc to be ready -2025-06-05 17:09:05,928 - tests.orchestrator - INFO - Pod test-pod-42c169cc phase: Pending -2025-06-05 17:09:07,998 - tests.orchestrator - INFO - Pod test-pod-42c169cc phase: Running -2025-06-05 17:09:07,998 - tests.orchestrator - INFO - Pod test-pod-42c169cc conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:09:10,071 - tests.orchestrator - INFO - Pod test-pod-42c169cc conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:09:12,149 - tests.orchestrator - INFO - Pod test-pod-42c169cc conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:09:14,219 - tests.orchestrator - INFO - Pod test-pod-42c169cc conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:09:16,290 - tests.orchestrator - INFO - Pod test-pod-42c169cc conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:09:18,358 - tests.orchestrator - INFO - Pod test-pod-42c169cc conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:09:20,429 - tests.orchestrator - INFO - Pod test-pod-42c169cc conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:09:22,498 - tests.orchestrator - INFO - Pod test-pod-42c169cc conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:09:24,566 - tests.orchestrator - INFO - Pod test-pod-42c169cc conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:09:26,636 - tests.orchestrator - INFO - Pod test-pod-42c169cc conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 17:09:26,637 - tests.orchestrator - INFO - Pod test-pod-42c169cc is ready -2025-06-05 17:09:27,637 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 4 times) -2025-06-05 17:09:27,637 - tests.orchestrator - INFO - Creating PVC: test-pvc-5d63ba99 -2025-06-05 17:09:27,713 - tests.orchestrator - INFO - Created PVC: test-pvc-5d63ba99 -2025-06-05 17:09:27,714 - tests.orchestrator - INFO - Waiting for PVC test-pvc-5d63ba99 to be bound -2025-06-05 17:09:29,848 - tests.orchestrator - INFO - PVC test-pvc-5d63ba99 is bound -2025-06-05 17:09:30,848 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 2 times) -2025-06-05 17:09:30,848 - tests.orchestrator - INFO - Deleting pod: test-pod-e1e1abfd from PVC: test-pvc-759dbae5 -2025-06-05 17:09:30,923 - tests.orchestrator - INFO - Waiting for pod test-pod-e1e1abfd to be deleted -2025-06-05 17:10:03,546 - tests.orchestrator - INFO - Pod test-pod-e1e1abfd has been deleted -2025-06-05 17:10:03,546 - tests.orchestrator - INFO - Deleted pod: test-pod-e1e1abfd -2025-06-05 17:10:04,546 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 4 times) -2025-06-05 17:10:04,546 - tests.orchestrator - INFO - Creating pod test-pod-21da98c0 on PVC test-pvc-47e77b85 -2025-06-05 17:10:04,641 - tests.orchestrator - INFO - Created pod: test-pod-21da98c0 using PVC: test-pvc-47e77b85 -2025-06-05 17:10:04,641 - tests.orchestrator - INFO - Waiting for pod test-pod-21da98c0 to be ready -2025-06-05 17:10:04,710 - tests.orchestrator - INFO - Pod test-pod-21da98c0 phase: Pending -2025-06-05 17:10:06,780 - tests.orchestrator - INFO - Pod test-pod-21da98c0 phase: Running -2025-06-05 17:10:06,780 - tests.orchestrator - INFO - Pod test-pod-21da98c0 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:10:08,850 - tests.orchestrator - INFO - Pod test-pod-21da98c0 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:10:10,920 - tests.orchestrator - INFO - Pod test-pod-21da98c0 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:10:12,990 - tests.orchestrator - INFO - Pod test-pod-21da98c0 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:10:15,059 - tests.orchestrator - INFO - Pod test-pod-21da98c0 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:10:17,130 - tests.orchestrator - INFO - Pod test-pod-21da98c0 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:10:19,198 - tests.orchestrator - INFO - Pod test-pod-21da98c0 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 17:10:21,265 - tests.orchestrator - INFO - Pod test-pod-21da98c0 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 17:10:21,265 - tests.orchestrator - INFO - Pod test-pod-21da98c0 is ready -2025-06-05 17:10:22,266 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 3 times) -2025-06-05 17:10:22,266 - tests.orchestrator - INFO - Deleting pod: test-pod-42c169cc from PVC: test-pvc-1511fcaf -2025-06-05 17:10:22,353 - tests.orchestrator - INFO - Waiting for pod test-pod-42c169cc to be deleted -2025-06-05 17:10:54,992 - tests.orchestrator - INFO - Pod test-pod-42c169cc has been deleted -2025-06-05 17:10:54,992 - tests.orchestrator - INFO - Deleted pod: test-pod-42c169cc -2025-06-05 17:10:55,992 - tests.orchestrator - INFO - Test completed in 329.56 seconds -2025-06-05 17:10:55,993 - tests.orchestrator - INFO - ===== STARTING COMPREHENSIVE CLEANUP ===== -2025-06-05 17:10:55,993 - tests.orchestrator - INFO - Deleting 2 pods (force=False)... -2025-06-05 17:10:55,993 - tests.orchestrator - INFO - Deleting pod: test-pod-94de82f9 from PVC: test-pvc-1511fcaf -2025-06-05 17:10:56,071 - tests.orchestrator - INFO - Waiting for pod test-pod-94de82f9 to be deleted -2025-06-05 17:11:28,699 - tests.orchestrator - INFO - Pod test-pod-94de82f9 has been deleted -2025-06-05 17:11:28,699 - tests.orchestrator - INFO - Deleted pod: test-pod-94de82f9 -2025-06-05 17:11:28,699 - tests.orchestrator - INFO - Deleting pod: test-pod-21da98c0 from PVC: test-pvc-47e77b85 -2025-06-05 17:11:28,774 - tests.orchestrator - INFO - Waiting for pod test-pod-21da98c0 to be deleted -2025-06-05 17:12:01,396 - tests.orchestrator - INFO - Pod test-pod-21da98c0 has been deleted -2025-06-05 17:12:01,396 - tests.orchestrator - INFO - Deleted pod: test-pod-21da98c0 -2025-06-05 17:12:06,396 - tests.orchestrator - INFO - Deleting 5 PVCs (force=False)... -2025-06-05 17:12:06,396 - tests.orchestrator - INFO - Deleting PVC: test-pvc-759dbae5 -2025-06-05 17:12:06,474 - tests.orchestrator - INFO - Waiting for PVC test-pvc-759dbae5 to be deleted -2025-06-05 17:12:10,610 - tests.orchestrator - INFO - PVC test-pvc-759dbae5 has been deleted -2025-06-05 17:12:10,610 - tests.orchestrator - INFO - Deleted PVC: test-pvc-759dbae5 -2025-06-05 17:12:10,610 - tests.orchestrator - INFO - Deleting PVC: test-pvc-e7dcdff0 -2025-06-05 17:12:10,682 - tests.orchestrator - INFO - Waiting for PVC test-pvc-e7dcdff0 to be deleted -2025-06-05 17:12:10,753 - tests.orchestrator - INFO - PVC test-pvc-e7dcdff0 has been deleted -2025-06-05 17:12:10,753 - tests.orchestrator - INFO - Deleted PVC: test-pvc-e7dcdff0 -2025-06-05 17:12:10,753 - tests.orchestrator - INFO - Deleting PVC: test-pvc-1511fcaf -2025-06-05 17:12:10,826 - tests.orchestrator - INFO - Waiting for PVC test-pvc-1511fcaf to be deleted -2025-06-05 17:12:10,894 - tests.orchestrator - INFO - PVC test-pvc-1511fcaf has been deleted -2025-06-05 17:12:10,894 - tests.orchestrator - INFO - Deleted PVC: test-pvc-1511fcaf -2025-06-05 17:12:10,894 - tests.orchestrator - INFO - Deleting PVC: test-pvc-47e77b85 -2025-06-05 17:12:10,967 - tests.orchestrator - INFO - Waiting for PVC test-pvc-47e77b85 to be deleted -2025-06-05 17:12:11,032 - tests.orchestrator - INFO - PVC test-pvc-47e77b85 has been deleted -2025-06-05 17:12:11,032 - tests.orchestrator - INFO - Deleted PVC: test-pvc-47e77b85 -2025-06-05 17:12:11,032 - tests.orchestrator - INFO - Deleting PVC: test-pvc-5d63ba99 -2025-06-05 17:12:11,107 - tests.orchestrator - INFO - Waiting for PVC test-pvc-5d63ba99 to be deleted -2025-06-05 17:12:11,173 - tests.orchestrator - INFO - PVC test-pvc-5d63ba99 has been deleted -2025-06-05 17:12:11,173 - tests.orchestrator - INFO - Deleted PVC: test-pvc-5d63ba99 -2025-06-05 17:12:11,377 - tests.orchestrator - WARNING - First cleanup pass incomplete. Remaining resources: ['pod/test-pod-3cc2d48d', 'pod/test-pod-564344d0', 'pod/test-pod-a9935180', 'pod/test-pod-e1337987', 'pvc/test-pvc-0b5af805', 'pvc/test-pvc-3691f2a0', 'pvc/test-pvc-455abf8b', 'pvc/test-pvc-51dbaa96', 'pvc/test-pvc-601432ce', 'pvc/test-pvc-63ada5ce', 'pvc/test-pvc-64bf8174', 'pvc/test-pvc-d3ee5325'] -2025-06-05 17:12:11,377 - tests.orchestrator - INFO - Attempting force deletion of remaining resources... -2025-06-05 17:12:11,377 - tests.orchestrator - INFO - Deleting 0 pods (force=True)... -2025-06-05 17:12:16,377 - tests.orchestrator - INFO - Deleting 0 PVCs (force=True)... -2025-06-05 17:12:16,581 - tests.orchestrator - ERROR - Cleanup incomplete. Remaining resources after force deletion: ['pod/test-pod-3cc2d48d', 'pod/test-pod-564344d0', 'pod/test-pod-a9935180', 'pod/test-pod-e1337987', 'pvc/test-pvc-0b5af805', 'pvc/test-pvc-3691f2a0', 'pvc/test-pvc-455abf8b', 'pvc/test-pvc-51dbaa96', 'pvc/test-pvc-601432ce', 'pvc/test-pvc-63ada5ce', 'pvc/test-pvc-64bf8174', 'pvc/test-pvc-d3ee5325'] -2025-06-05 17:12:16,581 - tests.orchestrator - INFO - Cleanup completed successfully in 80.59 seconds -2025-06-05 17:12:16,582 - tests.orchestrator - INFO - ===== CLEANUP PROCESS FINISHED ===== -2025-06-05 17:12:16,582 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-05 17:12:16,582 - tests.orchestrator - INFO - --- Operations --- -2025-06-05 17:12:16,582 - tests.orchestrator - INFO - create_pvc: 6 succeeded, 0 failed (100.0%) -2025-06-05 17:12:16,582 - tests.orchestrator - INFO - attach_pod: 7 succeeded, 0 failed (100.0%) -2025-06-05 17:12:16,582 - tests.orchestrator - INFO - delete_pod: 7 succeeded, 0 failed (100.0%) -2025-06-05 17:12:16,582 - tests.orchestrator - INFO - delete_pvc: 6 succeeded, 0 failed (100.0%) -2025-06-05 17:12:16,582 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) -2025-06-05 17:12:16,582 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) -2025-06-05 17:12:16,582 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-05 17:12:16,582 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-05 17:12:16,582 - tests.orchestrator - INFO - many_to_one: No runs -2025-06-05 17:12:16,582 - tests.orchestrator - INFO - one_to_one: No runs -2025-06-05 17:12:16,582 - tests.orchestrator - INFO - concurrent_pvc: No runs -2025-06-05 17:12:16,582 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_182216.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_182216.log deleted file mode 100644 index 79d31d520..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_182216.log +++ /dev/null @@ -1,7 +0,0 @@ -2025-06-05 18:22:16,318 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 18:22:16,528 - tests.orchestrator - ERROR - Error checking namespace: (401) -Reason: Unauthorized -HTTP response headers: HTTPHeaderDict({'Audit-Id': '1a400704-0ba2-4e3d-a421-03adbfa89ebe', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 18:22:16 GMT', 'Content-Length': '129'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} - - diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_182254.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_182254.log deleted file mode 100644 index e9292aaa6..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_182254.log +++ /dev/null @@ -1,207 +0,0 @@ -2025-06-05 18:22:54,149 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 18:22:54,350 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 18:22:54,351 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-05 18:22:54,543 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 18:22:54,543 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 18:22:54,543 - tests.orchestrator - INFO - Creating PVC: test-pvc-7a09d3a9 -2025-06-05 18:22:54,620 - tests.orchestrator - INFO - Created PVC: test-pvc-7a09d3a9 -2025-06-05 18:22:54,620 - tests.orchestrator - INFO - Waiting for PVC test-pvc-7a09d3a9 to be bound -2025-06-05 18:22:56,752 - tests.orchestrator - INFO - PVC test-pvc-7a09d3a9 is bound -2025-06-05 18:22:56,752 - tests.orchestrator - INFO - Creating PVC: test-pvc-af47e7e8 -2025-06-05 18:22:56,820 - tests.orchestrator - INFO - Created PVC: test-pvc-af47e7e8 -2025-06-05 18:22:56,820 - tests.orchestrator - INFO - Waiting for PVC test-pvc-af47e7e8 to be bound -2025-06-05 18:22:58,953 - tests.orchestrator - INFO - PVC test-pvc-af47e7e8 is bound -2025-06-05 18:22:58,953 - tests.orchestrator - INFO - Creating pod test-pod-09840b6a on PVC test-pvc-7a09d3a9 -2025-06-05 18:22:59,033 - tests.orchestrator - INFO - Created pod: test-pod-09840b6a using PVC: test-pvc-7a09d3a9 -2025-06-05 18:22:59,033 - tests.orchestrator - INFO - Waiting for pod test-pod-09840b6a to be ready -2025-06-05 18:22:59,100 - tests.orchestrator - INFO - Pod test-pod-09840b6a phase: Pending -2025-06-05 18:23:03,237 - tests.orchestrator - INFO - Pod test-pod-09840b6a phase: Running -2025-06-05 18:23:03,237 - tests.orchestrator - INFO - Pod test-pod-09840b6a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:23:05,303 - tests.orchestrator - INFO - Pod test-pod-09840b6a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:23:07,371 - tests.orchestrator - INFO - Pod test-pod-09840b6a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:23:09,439 - tests.orchestrator - INFO - Pod test-pod-09840b6a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:23:11,506 - tests.orchestrator - INFO - Pod test-pod-09840b6a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:23:13,575 - tests.orchestrator - INFO - Pod test-pod-09840b6a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:23:15,642 - tests.orchestrator - INFO - Pod test-pod-09840b6a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:23:17,710 - tests.orchestrator - INFO - Pod test-pod-09840b6a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:23:19,778 - tests.orchestrator - INFO - Pod test-pod-09840b6a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 18:23:19,778 - tests.orchestrator - INFO - Pod test-pod-09840b6a is ready -2025-06-05 18:23:19,778 - tests.orchestrator - INFO - Creating pod test-pod-606ca02c on PVC test-pvc-af47e7e8 -2025-06-05 18:23:19,851 - tests.orchestrator - INFO - Created pod: test-pod-606ca02c using PVC: test-pvc-af47e7e8 -2025-06-05 18:23:19,851 - tests.orchestrator - INFO - Waiting for pod test-pod-606ca02c to be ready -2025-06-05 18:23:19,918 - tests.orchestrator - INFO - Pod test-pod-606ca02c phase: Pending -2025-06-05 18:23:24,056 - tests.orchestrator - INFO - Pod test-pod-606ca02c phase: Running -2025-06-05 18:23:24,056 - tests.orchestrator - INFO - Pod test-pod-606ca02c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:23:26,123 - tests.orchestrator - INFO - Pod test-pod-606ca02c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:23:28,191 - tests.orchestrator - INFO - Pod test-pod-606ca02c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:23:30,260 - tests.orchestrator - INFO - Pod test-pod-606ca02c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:23:32,329 - tests.orchestrator - INFO - Pod test-pod-606ca02c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:23:34,397 - tests.orchestrator - INFO - Pod test-pod-606ca02c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:23:36,470 - tests.orchestrator - INFO - Pod test-pod-606ca02c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:23:38,538 - tests.orchestrator - INFO - Pod test-pod-606ca02c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:23:40,606 - tests.orchestrator - INFO - Pod test-pod-606ca02c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:23:42,674 - tests.orchestrator - INFO - Pod test-pod-606ca02c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 18:23:42,674 - tests.orchestrator - INFO - Pod test-pod-606ca02c is ready -2025-06-05 18:23:42,674 - tests.orchestrator - INFO - Creating pod test-pod-6e1e895f on PVC test-pvc-af47e7e8 -2025-06-05 18:23:42,758 - tests.orchestrator - INFO - Created pod: test-pod-6e1e895f using PVC: test-pvc-af47e7e8 -2025-06-05 18:23:42,758 - tests.orchestrator - INFO - Waiting for pod test-pod-6e1e895f to be ready -2025-06-05 18:23:42,825 - tests.orchestrator - INFO - Pod test-pod-6e1e895f phase: Pending -2025-06-05 18:23:44,892 - tests.orchestrator - INFO - Pod test-pod-6e1e895f phase: Running -2025-06-05 18:23:44,892 - tests.orchestrator - INFO - Pod test-pod-6e1e895f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:23:46,961 - tests.orchestrator - INFO - Pod test-pod-6e1e895f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:23:49,028 - tests.orchestrator - INFO - Pod test-pod-6e1e895f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:23:51,095 - tests.orchestrator - INFO - Pod test-pod-6e1e895f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:23:53,163 - tests.orchestrator - INFO - Pod test-pod-6e1e895f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:23:55,230 - tests.orchestrator - INFO - Pod test-pod-6e1e895f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:23:57,298 - tests.orchestrator - INFO - Pod test-pod-6e1e895f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:23:59,366 - tests.orchestrator - INFO - Pod test-pod-6e1e895f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:24:01,442 - tests.orchestrator - INFO - Pod test-pod-6e1e895f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:24:03,509 - tests.orchestrator - INFO - Pod test-pod-6e1e895f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 18:24:03,509 - tests.orchestrator - INFO - Pod test-pod-6e1e895f is ready -2025-06-05 18:24:03,510 - tests.orchestrator - INFO - Testing read/write between pods test-pod-6e1e895f and test-pod-606ca02c sharing PVC test-pvc-af47e7e8 -2025-06-05 18:24:03,580 - tests.orchestrator - ERROR - Failed in read/write verification: (400) -Reason: Bad Request -HTTP response headers: HTTPHeaderDict({'Audit-Id': 'ebf31bfa-8019-42fb-ae12-2d291ecd43e4', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 18:24:03 GMT', 'Content-Length': '139'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Upgrade request required","reason":"BadRequest","code":400} - - -2025-06-05 18:24:03,580 - tests.orchestrator - INFO - Deleting pod: test-pod-606ca02c from PVC: test-pvc-af47e7e8 -2025-06-05 18:24:03,653 - tests.orchestrator - INFO - Waiting for pod test-pod-606ca02c to be deleted -2025-06-05 18:24:36,256 - tests.orchestrator - INFO - Pod test-pod-606ca02c has been deleted -2025-06-05 18:24:36,256 - tests.orchestrator - INFO - Deleted pod: test-pod-606ca02c -2025-06-05 18:24:36,256 - tests.orchestrator - INFO - Deleting PVC: test-pvc-af47e7e8 -2025-06-05 18:24:36,256 - tests.orchestrator - INFO - Deleting 1 pods using PVC test-pvc-af47e7e8 -2025-06-05 18:24:36,256 - tests.orchestrator - INFO - Deleting pod: test-pod-6e1e895f from PVC: test-pvc-af47e7e8 -2025-06-05 18:24:36,332 - tests.orchestrator - INFO - Waiting for pod test-pod-6e1e895f to be deleted -2025-06-05 18:25:08,936 - tests.orchestrator - INFO - Pod test-pod-6e1e895f has been deleted -2025-06-05 18:25:08,936 - tests.orchestrator - INFO - Deleted pod: test-pod-6e1e895f -2025-06-05 18:25:09,008 - tests.orchestrator - INFO - Waiting for PVC test-pvc-af47e7e8 to be deleted -2025-06-05 18:25:09,073 - tests.orchestrator - INFO - PVC test-pvc-af47e7e8 has been deleted -2025-06-05 18:25:09,074 - tests.orchestrator - INFO - Deleted PVC: test-pvc-af47e7e8 -2025-06-05 18:25:09,074 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations -2025-06-05 18:25:09,074 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 1 times) -2025-06-05 18:25:09,074 - tests.orchestrator - INFO - Creating pod test-pod-d40af63e on PVC test-pvc-7a09d3a9 -2025-06-05 18:25:09,153 - tests.orchestrator - INFO - Created pod: test-pod-d40af63e using PVC: test-pvc-7a09d3a9 -2025-06-05 18:25:09,153 - tests.orchestrator - INFO - Waiting for pod test-pod-d40af63e to be ready -2025-06-05 18:25:09,220 - tests.orchestrator - INFO - Pod test-pod-d40af63e phase: Pending -2025-06-05 18:25:13,353 - tests.orchestrator - INFO - Pod test-pod-d40af63e phase: Running -2025-06-05 18:25:13,353 - tests.orchestrator - INFO - Pod test-pod-d40af63e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:25:15,421 - tests.orchestrator - INFO - Pod test-pod-d40af63e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:25:17,489 - tests.orchestrator - INFO - Pod test-pod-d40af63e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:25:19,556 - tests.orchestrator - INFO - Pod test-pod-d40af63e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:25:21,625 - tests.orchestrator - INFO - Pod test-pod-d40af63e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:25:23,693 - tests.orchestrator - INFO - Pod test-pod-d40af63e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:25:25,761 - tests.orchestrator - INFO - Pod test-pod-d40af63e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 18:25:25,762 - tests.orchestrator - INFO - Pod test-pod-d40af63e is ready -2025-06-05 18:25:26,762 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 2 times) -2025-06-05 18:25:26,762 - tests.orchestrator - INFO - Creating pod test-pod-bf04a30a on PVC test-pvc-7a09d3a9 -2025-06-05 18:25:26,834 - tests.orchestrator - INFO - Created pod: test-pod-bf04a30a using PVC: test-pvc-7a09d3a9 -2025-06-05 18:25:26,834 - tests.orchestrator - INFO - Waiting for pod test-pod-bf04a30a to be ready -2025-06-05 18:25:26,901 - tests.orchestrator - INFO - Pod test-pod-bf04a30a phase: Pending -2025-06-05 18:25:28,970 - tests.orchestrator - INFO - Pod test-pod-bf04a30a phase: Running -2025-06-05 18:25:28,970 - tests.orchestrator - INFO - Pod test-pod-bf04a30a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:25:31,038 - tests.orchestrator - INFO - Pod test-pod-bf04a30a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:25:33,105 - tests.orchestrator - INFO - Pod test-pod-bf04a30a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:25:35,173 - tests.orchestrator - INFO - Pod test-pod-bf04a30a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:25:37,240 - tests.orchestrator - INFO - Pod test-pod-bf04a30a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:25:39,308 - tests.orchestrator - INFO - Pod test-pod-bf04a30a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:25:41,376 - tests.orchestrator - INFO - Pod test-pod-bf04a30a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:25:43,443 - tests.orchestrator - INFO - Pod test-pod-bf04a30a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:25:45,511 - tests.orchestrator - INFO - Pod test-pod-bf04a30a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:25:47,579 - tests.orchestrator - INFO - Pod test-pod-bf04a30a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 18:25:47,579 - tests.orchestrator - INFO - Pod test-pod-bf04a30a is ready -2025-06-05 18:25:48,579 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 1 times) -2025-06-05 18:25:48,579 - tests.orchestrator - INFO - Creating PVC: test-pvc-5faad9c0 -2025-06-05 18:25:48,657 - tests.orchestrator - INFO - Created PVC: test-pvc-5faad9c0 -2025-06-05 18:25:48,657 - tests.orchestrator - INFO - Waiting for PVC test-pvc-5faad9c0 to be bound -2025-06-05 18:25:50,789 - tests.orchestrator - INFO - PVC test-pvc-5faad9c0 is bound -2025-06-05 18:25:51,789 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 1 times) -2025-06-05 18:25:51,789 - tests.orchestrator - INFO - Deleting PVC: test-pvc-7a09d3a9 -2025-06-05 18:25:51,789 - tests.orchestrator - INFO - Deleting 3 pods using PVC test-pvc-7a09d3a9 -2025-06-05 18:25:51,789 - tests.orchestrator - INFO - Deleting pod: test-pod-09840b6a from PVC: test-pvc-7a09d3a9 -2025-06-05 18:25:51,863 - tests.orchestrator - INFO - Waiting for pod test-pod-09840b6a to be deleted -2025-06-05 18:26:24,469 - tests.orchestrator - INFO - Pod test-pod-09840b6a has been deleted -2025-06-05 18:26:24,469 - tests.orchestrator - INFO - Deleted pod: test-pod-09840b6a -2025-06-05 18:26:24,469 - tests.orchestrator - INFO - Deleting pod: test-pod-d40af63e from PVC: test-pvc-7a09d3a9 -2025-06-05 18:26:24,543 - tests.orchestrator - INFO - Waiting for pod test-pod-d40af63e to be deleted -2025-06-05 18:26:57,150 - tests.orchestrator - INFO - Pod test-pod-d40af63e has been deleted -2025-06-05 18:26:57,150 - tests.orchestrator - INFO - Deleted pod: test-pod-d40af63e -2025-06-05 18:26:57,151 - tests.orchestrator - INFO - Deleting pod: test-pod-bf04a30a from PVC: test-pvc-7a09d3a9 -2025-06-05 18:26:57,225 - tests.orchestrator - INFO - Waiting for pod test-pod-bf04a30a to be deleted -2025-06-05 18:27:29,830 - tests.orchestrator - INFO - Pod test-pod-bf04a30a has been deleted -2025-06-05 18:27:29,830 - tests.orchestrator - INFO - Deleted pod: test-pod-bf04a30a -2025-06-05 18:27:29,900 - tests.orchestrator - INFO - Waiting for PVC test-pvc-7a09d3a9 to be deleted -2025-06-05 18:27:29,967 - tests.orchestrator - INFO - PVC test-pvc-7a09d3a9 has been deleted -2025-06-05 18:27:29,968 - tests.orchestrator - INFO - Deleted PVC: test-pvc-7a09d3a9 -2025-06-05 18:27:30,968 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 3 times) -2025-06-05 18:27:30,968 - tests.orchestrator - INFO - Creating pod test-pod-8c05e159 on PVC test-pvc-5faad9c0 -2025-06-05 18:27:31,056 - tests.orchestrator - INFO - Created pod: test-pod-8c05e159 using PVC: test-pvc-5faad9c0 -2025-06-05 18:27:31,056 - tests.orchestrator - INFO - Waiting for pod test-pod-8c05e159 to be ready -2025-06-05 18:27:31,123 - tests.orchestrator - INFO - Pod test-pod-8c05e159 phase: Pending -2025-06-05 18:27:33,192 - tests.orchestrator - INFO - Pod test-pod-8c05e159 phase: Running -2025-06-05 18:27:33,192 - tests.orchestrator - INFO - Pod test-pod-8c05e159 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:27:35,259 - tests.orchestrator - INFO - Pod test-pod-8c05e159 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:27:37,326 - tests.orchestrator - INFO - Pod test-pod-8c05e159 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:27:39,394 - tests.orchestrator - INFO - Pod test-pod-8c05e159 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:27:41,463 - tests.orchestrator - INFO - Pod test-pod-8c05e159 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:27:43,531 - tests.orchestrator - INFO - Pod test-pod-8c05e159 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:27:45,599 - tests.orchestrator - INFO - Pod test-pod-8c05e159 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:27:47,667 - tests.orchestrator - INFO - Pod test-pod-8c05e159 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:27:49,734 - tests.orchestrator - INFO - Pod test-pod-8c05e159 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:27:51,802 - tests.orchestrator - INFO - Pod test-pod-8c05e159 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 18:27:51,803 - tests.orchestrator - INFO - Pod test-pod-8c05e159 is ready -2025-06-05 18:27:52,803 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 4 times) -2025-06-05 18:27:52,803 - tests.orchestrator - INFO - Creating pod test-pod-7e1c33ef on PVC test-pvc-5faad9c0 -2025-06-05 18:27:52,877 - tests.orchestrator - INFO - Created pod: test-pod-7e1c33ef using PVC: test-pvc-5faad9c0 -2025-06-05 18:27:52,877 - tests.orchestrator - INFO - Waiting for pod test-pod-7e1c33ef to be ready -2025-06-05 18:27:52,943 - tests.orchestrator - INFO - Pod test-pod-7e1c33ef phase: Pending -2025-06-05 18:27:55,011 - tests.orchestrator - INFO - Pod test-pod-7e1c33ef phase: Running -2025-06-05 18:27:55,011 - tests.orchestrator - INFO - Pod test-pod-7e1c33ef conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:27:57,079 - tests.orchestrator - INFO - Pod test-pod-7e1c33ef conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:27:59,146 - tests.orchestrator - INFO - Pod test-pod-7e1c33ef conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:28:01,216 - tests.orchestrator - INFO - Pod test-pod-7e1c33ef conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:28:03,284 - tests.orchestrator - INFO - Pod test-pod-7e1c33ef conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:28:05,352 - tests.orchestrator - INFO - Pod test-pod-7e1c33ef conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:28:07,421 - tests.orchestrator - INFO - Pod test-pod-7e1c33ef conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:28:09,487 - tests.orchestrator - INFO - Pod test-pod-7e1c33ef conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:28:11,563 - tests.orchestrator - INFO - Pod test-pod-7e1c33ef conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:28:13,632 - tests.orchestrator - INFO - Pod test-pod-7e1c33ef conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:28:15,700 - tests.orchestrator - INFO - Pod test-pod-7e1c33ef conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 18:28:15,700 - tests.orchestrator - INFO - Pod test-pod-7e1c33ef is ready -2025-06-05 18:28:16,700 - tests.orchestrator - INFO - Test completed in 322.35 seconds -2025-06-05 18:28:16,701 - tests.orchestrator - INFO - ===== STARTING COMPREHENSIVE CLEANUP ===== -2025-06-05 18:28:16,701 - tests.orchestrator - INFO - Deleting 2 pods (force=False)... -2025-06-05 18:28:16,701 - tests.orchestrator - INFO - Deleting pod: test-pod-8c05e159 from PVC: test-pvc-5faad9c0 -2025-06-05 18:28:16,774 - tests.orchestrator - INFO - Waiting for pod test-pod-8c05e159 to be deleted -2025-06-05 18:28:49,382 - tests.orchestrator - INFO - Pod test-pod-8c05e159 has been deleted -2025-06-05 18:28:49,382 - tests.orchestrator - INFO - Deleted pod: test-pod-8c05e159 -2025-06-05 18:28:49,382 - tests.orchestrator - INFO - Deleting pod: test-pod-7e1c33ef from PVC: test-pvc-5faad9c0 -2025-06-05 18:28:49,455 - tests.orchestrator - INFO - Waiting for pod test-pod-7e1c33ef to be deleted -2025-06-05 18:29:22,059 - tests.orchestrator - INFO - Pod test-pod-7e1c33ef has been deleted -2025-06-05 18:29:22,059 - tests.orchestrator - INFO - Deleted pod: test-pod-7e1c33ef -2025-06-05 18:29:27,059 - tests.orchestrator - INFO - Deleting 1 PVCs (force=False)... -2025-06-05 18:29:27,059 - tests.orchestrator - INFO - Deleting PVC: test-pvc-5faad9c0 -2025-06-05 18:29:27,131 - tests.orchestrator - INFO - Waiting for PVC test-pvc-5faad9c0 to be deleted -2025-06-05 18:29:27,195 - tests.orchestrator - INFO - PVC test-pvc-5faad9c0 has been deleted -2025-06-05 18:29:27,195 - tests.orchestrator - INFO - Deleted PVC: test-pvc-5faad9c0 -2025-06-05 18:29:27,397 - tests.orchestrator - WARNING - First cleanup pass incomplete. Remaining resources: ['pod/test-pod-3cc2d48d', 'pod/test-pod-564344d0', 'pod/test-pod-a9935180', 'pod/test-pod-e1337987', 'pvc/test-pvc-0b5af805', 'pvc/test-pvc-3691f2a0', 'pvc/test-pvc-455abf8b', 'pvc/test-pvc-51dbaa96', 'pvc/test-pvc-601432ce', 'pvc/test-pvc-63ada5ce', 'pvc/test-pvc-64bf8174', 'pvc/test-pvc-d3ee5325'] -2025-06-05 18:29:27,397 - tests.orchestrator - INFO - Attempting force deletion of remaining resources... -2025-06-05 18:29:27,397 - tests.orchestrator - INFO - Deleting 0 pods (force=True)... -2025-06-05 18:29:32,397 - tests.orchestrator - INFO - Deleting 0 PVCs (force=True)... -2025-06-05 18:29:32,600 - tests.orchestrator - ERROR - Cleanup incomplete. Remaining resources after force deletion: ['pod/test-pod-3cc2d48d', 'pod/test-pod-564344d0', 'pod/test-pod-a9935180', 'pod/test-pod-e1337987', 'pvc/test-pvc-0b5af805', 'pvc/test-pvc-3691f2a0', 'pvc/test-pvc-455abf8b', 'pvc/test-pvc-51dbaa96', 'pvc/test-pvc-601432ce', 'pvc/test-pvc-63ada5ce', 'pvc/test-pvc-64bf8174', 'pvc/test-pvc-d3ee5325'] -2025-06-05 18:29:32,600 - tests.orchestrator - INFO - Cleanup completed successfully in 75.90 seconds -2025-06-05 18:29:32,600 - tests.orchestrator - INFO - ===== CLEANUP PROCESS FINISHED ===== -2025-06-05 18:29:32,600 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-05 18:29:32,600 - tests.orchestrator - INFO - --- Operations --- -2025-06-05 18:29:32,600 - tests.orchestrator - INFO - create_pvc: 3 succeeded, 0 failed (100.0%) -2025-06-05 18:29:32,600 - tests.orchestrator - INFO - attach_pod: 7 succeeded, 0 failed (100.0%) -2025-06-05 18:29:32,600 - tests.orchestrator - INFO - delete_pod: 7 succeeded, 0 failed (100.0%) -2025-06-05 18:29:32,600 - tests.orchestrator - INFO - delete_pvc: 3 succeeded, 0 failed (100.0%) -2025-06-05 18:29:32,601 - tests.orchestrator - INFO - verify_read_write: Writes 0 succeeded, 1 failed (0.0%) -2025-06-05 18:29:32,601 - tests.orchestrator - INFO - verify_read_write: Reads 0 succeeded, 1 failed (0.0%) -2025-06-05 18:29:32,601 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-05 18:29:32,601 - tests.orchestrator - INFO - shared_volume_rw: 0 succeeded, 1 failed out of 1 runs (0.0%) -2025-06-05 18:29:32,601 - tests.orchestrator - INFO - many_to_one: No runs -2025-06-05 18:29:32,601 - tests.orchestrator - INFO - one_to_one: No runs -2025-06-05 18:29:32,601 - tests.orchestrator - INFO - concurrent_pvc: No runs -2025-06-05 18:29:32,601 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_183835.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_183835.log deleted file mode 100644 index 034a984a2..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_183835.log +++ /dev/null @@ -1,983 +0,0 @@ -2025-06-05 18:38:35,776 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 18:38:35,987 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 18:38:35,987 - tests.orchestrator - INFO - Starting orchestrator test for 7200 seconds -2025-06-05 18:38:36,176 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 18:38:36,176 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 18:38:36,176 - tests.orchestrator - INFO - Creating PVC: test-pvc-21c303a0 -2025-06-05 18:38:36,254 - tests.orchestrator - INFO - Created PVC: test-pvc-21c303a0 -2025-06-05 18:38:36,254 - tests.orchestrator - INFO - Waiting for PVC test-pvc-21c303a0 to be bound -2025-06-05 18:38:38,387 - tests.orchestrator - INFO - PVC test-pvc-21c303a0 is bound -2025-06-05 18:38:38,387 - tests.orchestrator - INFO - Creating PVC: test-pvc-7f1f1d08 -2025-06-05 18:38:38,456 - tests.orchestrator - INFO - Created PVC: test-pvc-7f1f1d08 -2025-06-05 18:38:38,456 - tests.orchestrator - INFO - Waiting for PVC test-pvc-7f1f1d08 to be bound -2025-06-05 18:38:40,589 - tests.orchestrator - INFO - PVC test-pvc-7f1f1d08 is bound -2025-06-05 18:38:40,589 - tests.orchestrator - INFO - Creating pod test-pod-b63b288d on PVC test-pvc-21c303a0 -2025-06-05 18:38:40,671 - tests.orchestrator - INFO - Created pod: test-pod-b63b288d using PVC: test-pvc-21c303a0 -2025-06-05 18:38:40,671 - tests.orchestrator - INFO - Waiting for pod test-pod-b63b288d to be ready -2025-06-05 18:38:40,739 - tests.orchestrator - INFO - Pod test-pod-b63b288d phase: Pending -2025-06-05 18:38:44,876 - tests.orchestrator - INFO - Pod test-pod-b63b288d phase: Running -2025-06-05 18:38:44,876 - tests.orchestrator - INFO - Pod test-pod-b63b288d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:38:46,944 - tests.orchestrator - INFO - Pod test-pod-b63b288d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:38:49,012 - tests.orchestrator - INFO - Pod test-pod-b63b288d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:38:51,080 - tests.orchestrator - INFO - Pod test-pod-b63b288d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:38:53,150 - tests.orchestrator - INFO - Pod test-pod-b63b288d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:38:55,221 - tests.orchestrator - INFO - Pod test-pod-b63b288d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:38:57,289 - tests.orchestrator - INFO - Pod test-pod-b63b288d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:38:59,357 - tests.orchestrator - INFO - Pod test-pod-b63b288d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:39:01,428 - tests.orchestrator - INFO - Pod test-pod-b63b288d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 18:39:01,428 - tests.orchestrator - INFO - Pod test-pod-b63b288d is ready -2025-06-05 18:39:01,428 - tests.orchestrator - INFO - Creating pod test-pod-4deff76d on PVC test-pvc-21c303a0 -2025-06-05 18:39:01,570 - tests.orchestrator - INFO - Created pod: test-pod-4deff76d using PVC: test-pvc-21c303a0 -2025-06-05 18:39:01,570 - tests.orchestrator - INFO - Waiting for pod test-pod-4deff76d to be ready -2025-06-05 18:39:01,694 - tests.orchestrator - INFO - Pod test-pod-4deff76d phase: Pending -2025-06-05 18:39:03,763 - tests.orchestrator - INFO - Pod test-pod-4deff76d phase: Running -2025-06-05 18:39:03,763 - tests.orchestrator - INFO - Pod test-pod-4deff76d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:39:05,831 - tests.orchestrator - INFO - Pod test-pod-4deff76d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:39:07,901 - tests.orchestrator - INFO - Pod test-pod-4deff76d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:39:09,970 - tests.orchestrator - INFO - Pod test-pod-4deff76d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:39:12,038 - tests.orchestrator - INFO - Pod test-pod-4deff76d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:39:14,107 - tests.orchestrator - INFO - Pod test-pod-4deff76d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:39:16,176 - tests.orchestrator - INFO - Pod test-pod-4deff76d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:39:18,244 - tests.orchestrator - INFO - Pod test-pod-4deff76d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 18:39:18,244 - tests.orchestrator - INFO - Pod test-pod-4deff76d is ready -2025-06-05 18:39:18,245 - tests.orchestrator - INFO - Creating pod test-pod-674c680f on PVC test-pvc-21c303a0 -2025-06-05 18:39:18,327 - tests.orchestrator - INFO - Created pod: test-pod-674c680f using PVC: test-pvc-21c303a0 -2025-06-05 18:39:18,327 - tests.orchestrator - INFO - Waiting for pod test-pod-674c680f to be ready -2025-06-05 18:39:18,395 - tests.orchestrator - INFO - Pod test-pod-674c680f phase: Pending -2025-06-05 18:39:20,464 - tests.orchestrator - INFO - Pod test-pod-674c680f phase: Running -2025-06-05 18:39:20,464 - tests.orchestrator - INFO - Pod test-pod-674c680f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:39:22,532 - tests.orchestrator - INFO - Pod test-pod-674c680f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:39:24,601 - tests.orchestrator - INFO - Pod test-pod-674c680f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:39:26,669 - tests.orchestrator - INFO - Pod test-pod-674c680f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:39:28,737 - tests.orchestrator - INFO - Pod test-pod-674c680f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:39:30,806 - tests.orchestrator - INFO - Pod test-pod-674c680f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:39:32,874 - tests.orchestrator - INFO - Pod test-pod-674c680f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:39:34,942 - tests.orchestrator - INFO - Pod test-pod-674c680f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:39:37,011 - tests.orchestrator - INFO - Pod test-pod-674c680f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:39:39,080 - tests.orchestrator - INFO - Pod test-pod-674c680f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 18:39:39,080 - tests.orchestrator - INFO - Pod test-pod-674c680f is ready -2025-06-05 18:39:39,081 - tests.orchestrator - INFO - Testing read/write between pods test-pod-4deff76d and test-pod-674c680f sharing PVC test-pvc-21c303a0 -2025-06-05 18:39:39,150 - tests.orchestrator - ERROR - Failed in read/write verification: (400) -Reason: Bad Request -HTTP response headers: HTTPHeaderDict({'Audit-Id': '6945d221-5c2e-4401-a73f-a03b2205822c', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 18:39:39 GMT', 'Content-Length': '139'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Upgrade request required","reason":"BadRequest","code":400} - - -2025-06-05 18:39:39,150 - tests.orchestrator - INFO - Deleting pod: test-pod-b63b288d from PVC: test-pvc-21c303a0 -2025-06-05 18:39:39,226 - tests.orchestrator - INFO - Waiting for pod test-pod-b63b288d to be deleted -2025-06-05 18:40:11,843 - tests.orchestrator - INFO - Pod test-pod-b63b288d has been deleted -2025-06-05 18:40:11,844 - tests.orchestrator - INFO - Deleted pod: test-pod-b63b288d -2025-06-05 18:40:11,844 - tests.orchestrator - INFO - Deleting PVC: test-pvc-7f1f1d08 -2025-06-05 18:40:11,916 - tests.orchestrator - INFO - Waiting for PVC test-pvc-7f1f1d08 to be deleted -2025-06-05 18:40:16,049 - tests.orchestrator - INFO - PVC test-pvc-7f1f1d08 has been deleted -2025-06-05 18:40:16,050 - tests.orchestrator - INFO - Deleted PVC: test-pvc-7f1f1d08 -2025-06-05 18:40:16,050 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations -2025-06-05 18:40:16,050 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 1 times) -2025-06-05 18:40:16,050 - tests.orchestrator - INFO - Creating pod test-pod-ebc67d81 on PVC test-pvc-21c303a0 -2025-06-05 18:40:16,133 - tests.orchestrator - INFO - Created pod: test-pod-ebc67d81 using PVC: test-pvc-21c303a0 -2025-06-05 18:40:16,133 - tests.orchestrator - INFO - Waiting for pod test-pod-ebc67d81 to be ready -2025-06-05 18:40:16,200 - tests.orchestrator - INFO - Pod test-pod-ebc67d81 phase: Pending -2025-06-05 18:40:20,337 - tests.orchestrator - INFO - Pod test-pod-ebc67d81 phase: Running -2025-06-05 18:40:20,337 - tests.orchestrator - INFO - Pod test-pod-ebc67d81 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:40:22,404 - tests.orchestrator - INFO - Pod test-pod-ebc67d81 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:40:24,473 - tests.orchestrator - INFO - Pod test-pod-ebc67d81 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:40:26,548 - tests.orchestrator - INFO - Pod test-pod-ebc67d81 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:40:28,616 - tests.orchestrator - INFO - Pod test-pod-ebc67d81 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:40:30,684 - tests.orchestrator - INFO - Pod test-pod-ebc67d81 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:40:32,753 - tests.orchestrator - INFO - Pod test-pod-ebc67d81 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:40:34,822 - tests.orchestrator - INFO - Pod test-pod-ebc67d81 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:40:36,891 - tests.orchestrator - INFO - Pod test-pod-ebc67d81 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 18:40:36,891 - tests.orchestrator - INFO - Pod test-pod-ebc67d81 is ready -2025-06-05 18:40:37,891 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 1 times) -2025-06-05 18:40:37,891 - tests.orchestrator - INFO - Deleting pod: test-pod-ebc67d81 from PVC: test-pvc-21c303a0 -2025-06-05 18:40:37,968 - tests.orchestrator - INFO - Waiting for pod test-pod-ebc67d81 to be deleted -2025-06-05 18:41:10,581 - tests.orchestrator - INFO - Pod test-pod-ebc67d81 has been deleted -2025-06-05 18:41:10,581 - tests.orchestrator - INFO - Deleted pod: test-pod-ebc67d81 -2025-06-05 18:41:11,581 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 1 times) -2025-06-05 18:41:11,582 - tests.orchestrator - INFO - Creating PVC: test-pvc-9c78a2d7 -2025-06-05 18:41:11,661 - tests.orchestrator - INFO - Created PVC: test-pvc-9c78a2d7 -2025-06-05 18:41:11,661 - tests.orchestrator - INFO - Waiting for PVC test-pvc-9c78a2d7 to be bound -2025-06-05 18:41:13,794 - tests.orchestrator - INFO - PVC test-pvc-9c78a2d7 is bound -2025-06-05 18:41:14,795 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 2 times) -2025-06-05 18:41:14,795 - tests.orchestrator - INFO - Deleting pod: test-pod-4deff76d from PVC: test-pvc-21c303a0 -2025-06-05 18:41:14,871 - tests.orchestrator - INFO - Waiting for pod test-pod-4deff76d to be deleted -2025-06-05 18:41:47,485 - tests.orchestrator - INFO - Pod test-pod-4deff76d has been deleted -2025-06-05 18:41:47,485 - tests.orchestrator - INFO - Deleted pod: test-pod-4deff76d -2025-06-05 18:41:48,485 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 1 times) -2025-06-05 18:41:48,485 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test -2025-06-05 18:41:49,485 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 3 times) -2025-06-05 18:41:49,485 - tests.orchestrator - INFO - Deleting pod: test-pod-674c680f from PVC: test-pvc-21c303a0 -2025-06-05 18:41:49,560 - tests.orchestrator - INFO - Waiting for pod test-pod-674c680f to be deleted -2025-06-05 18:42:22,181 - tests.orchestrator - INFO - Pod test-pod-674c680f has been deleted -2025-06-05 18:42:22,181 - tests.orchestrator - INFO - Deleted pod: test-pod-674c680f -2025-06-05 18:42:23,181 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 1 times) -2025-06-05 18:42:23,181 - tests.orchestrator - INFO - Deleting PVC: test-pvc-21c303a0 -2025-06-05 18:42:23,255 - tests.orchestrator - INFO - Waiting for PVC test-pvc-21c303a0 to be deleted -2025-06-05 18:42:23,320 - tests.orchestrator - INFO - PVC test-pvc-21c303a0 has been deleted -2025-06-05 18:42:23,320 - tests.orchestrator - INFO - Deleted PVC: test-pvc-21c303a0 -2025-06-05 18:42:24,320 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 2 times) -2025-06-05 18:42:24,320 - tests.orchestrator - INFO - Creating PVC: test-pvc-f072232d -2025-06-05 18:42:24,396 - tests.orchestrator - INFO - Created PVC: test-pvc-f072232d -2025-06-05 18:42:24,396 - tests.orchestrator - INFO - Waiting for PVC test-pvc-f072232d to be bound -2025-06-05 18:42:26,534 - tests.orchestrator - INFO - PVC test-pvc-f072232d is bound -2025-06-05 18:42:27,534 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 3 times) -2025-06-05 18:42:27,534 - tests.orchestrator - INFO - Creating PVC: test-pvc-5da89758 -2025-06-05 18:42:27,603 - tests.orchestrator - INFO - Created PVC: test-pvc-5da89758 -2025-06-05 18:42:27,603 - tests.orchestrator - INFO - Waiting for PVC test-pvc-5da89758 to be bound -2025-06-05 18:42:29,737 - tests.orchestrator - INFO - PVC test-pvc-5da89758 is bound -2025-06-05 18:42:30,737 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 4 times) -2025-06-05 18:42:30,737 - tests.orchestrator - INFO - Creating PVC: test-pvc-24af11bc -2025-06-05 18:42:30,806 - tests.orchestrator - INFO - Created PVC: test-pvc-24af11bc -2025-06-05 18:42:30,806 - tests.orchestrator - INFO - Waiting for PVC test-pvc-24af11bc to be bound -2025-06-05 18:42:32,939 - tests.orchestrator - INFO - PVC test-pvc-24af11bc is bound -2025-06-05 18:42:33,939 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 4 times) -2025-06-05 18:42:33,939 - tests.orchestrator - INFO - No pods to delete -2025-06-05 18:42:34,939 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 5 times) -2025-06-05 18:42:34,939 - tests.orchestrator - INFO - Creating PVC: test-pvc-29a67853 -2025-06-05 18:42:35,009 - tests.orchestrator - INFO - Created PVC: test-pvc-29a67853 -2025-06-05 18:42:35,010 - tests.orchestrator - INFO - Waiting for PVC test-pvc-29a67853 to be bound -2025-06-05 18:42:37,142 - tests.orchestrator - INFO - PVC test-pvc-29a67853 is bound -2025-06-05 18:42:38,142 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 5 times) -2025-06-05 18:42:38,142 - tests.orchestrator - INFO - No pods to delete -2025-06-05 18:42:39,142 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 6 times) -2025-06-05 18:42:39,143 - tests.orchestrator - INFO - Creating PVC: test-pvc-0b86173e -2025-06-05 18:42:39,212 - tests.orchestrator - INFO - Created PVC: test-pvc-0b86173e -2025-06-05 18:42:39,212 - tests.orchestrator - INFO - Waiting for PVC test-pvc-0b86173e to be bound -2025-06-05 18:42:41,345 - tests.orchestrator - INFO - PVC test-pvc-0b86173e is bound -2025-06-05 18:42:42,345 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 6 times) -2025-06-05 18:42:42,346 - tests.orchestrator - INFO - No pods to delete -2025-06-05 18:42:43,346 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 2 times) -2025-06-05 18:42:43,346 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test -2025-06-05 18:42:44,346 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 3 times) -2025-06-05 18:42:44,346 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test -2025-06-05 18:42:45,346 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 7 times) -2025-06-05 18:42:45,347 - tests.orchestrator - INFO - Creating PVC: test-pvc-41d09b77 -2025-06-05 18:42:45,416 - tests.orchestrator - INFO - Created PVC: test-pvc-41d09b77 -2025-06-05 18:42:45,416 - tests.orchestrator - INFO - Waiting for PVC test-pvc-41d09b77 to be bound -2025-06-05 18:42:47,550 - tests.orchestrator - INFO - PVC test-pvc-41d09b77 is bound -2025-06-05 18:42:48,550 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 8 times) -2025-06-05 18:42:48,551 - tests.orchestrator - INFO - Creating PVC: test-pvc-f1842f61 -2025-06-05 18:42:48,618 - tests.orchestrator - INFO - Created PVC: test-pvc-f1842f61 -2025-06-05 18:42:48,619 - tests.orchestrator - INFO - Waiting for PVC test-pvc-f1842f61 to be bound -2025-06-05 18:42:50,752 - tests.orchestrator - INFO - PVC test-pvc-f1842f61 is bound -2025-06-05 18:42:51,753 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 7 times) -2025-06-05 18:42:51,753 - tests.orchestrator - INFO - No pods to delete -2025-06-05 18:42:52,753 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 8 times) -2025-06-05 18:42:52,753 - tests.orchestrator - INFO - No pods to delete -2025-06-05 18:42:53,753 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 2 times) -2025-06-05 18:42:53,754 - tests.orchestrator - INFO - Creating pod test-pod-e254da33 on PVC test-pvc-9c78a2d7 -2025-06-05 18:42:53,834 - tests.orchestrator - INFO - Created pod: test-pod-e254da33 using PVC: test-pvc-9c78a2d7 -2025-06-05 18:42:53,834 - tests.orchestrator - INFO - Waiting for pod test-pod-e254da33 to be ready -2025-06-05 18:42:53,902 - tests.orchestrator - INFO - Pod test-pod-e254da33 phase: Pending -2025-06-05 18:42:55,971 - tests.orchestrator - INFO - Pod test-pod-e254da33 phase: Running -2025-06-05 18:42:55,971 - tests.orchestrator - INFO - Pod test-pod-e254da33 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:42:58,039 - tests.orchestrator - INFO - Pod test-pod-e254da33 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:43:00,107 - tests.orchestrator - INFO - Pod test-pod-e254da33 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:43:02,185 - tests.orchestrator - INFO - Pod test-pod-e254da33 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:43:04,253 - tests.orchestrator - INFO - Pod test-pod-e254da33 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:43:06,322 - tests.orchestrator - INFO - Pod test-pod-e254da33 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:43:08,391 - tests.orchestrator - INFO - Pod test-pod-e254da33 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:43:10,459 - tests.orchestrator - INFO - Pod test-pod-e254da33 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:43:12,527 - tests.orchestrator - INFO - Pod test-pod-e254da33 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:43:14,596 - tests.orchestrator - INFO - Pod test-pod-e254da33 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 18:43:14,596 - tests.orchestrator - INFO - Pod test-pod-e254da33 is ready -2025-06-05 18:43:15,596 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 9 times) -2025-06-05 18:43:15,596 - tests.orchestrator - INFO - Creating PVC: test-pvc-ded65224 -2025-06-05 18:43:15,673 - tests.orchestrator - INFO - Created PVC: test-pvc-ded65224 -2025-06-05 18:43:15,674 - tests.orchestrator - INFO - Waiting for PVC test-pvc-ded65224 to be bound -2025-06-05 18:43:17,807 - tests.orchestrator - INFO - PVC test-pvc-ded65224 is bound -2025-06-05 18:43:18,807 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 2 times) -2025-06-05 18:43:18,807 - tests.orchestrator - INFO - Deleting PVC: test-pvc-f072232d -2025-06-05 18:43:18,881 - tests.orchestrator - INFO - Waiting for PVC test-pvc-f072232d to be deleted -2025-06-05 18:43:18,946 - tests.orchestrator - INFO - PVC test-pvc-f072232d has been deleted -2025-06-05 18:43:18,946 - tests.orchestrator - INFO - Deleted PVC: test-pvc-f072232d -2025-06-05 18:43:19,946 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 9 times) -2025-06-05 18:43:19,947 - tests.orchestrator - INFO - Deleting pod: test-pod-e254da33 from PVC: test-pvc-9c78a2d7 -2025-06-05 18:43:20,021 - tests.orchestrator - INFO - Waiting for pod test-pod-e254da33 to be deleted -2025-06-05 18:43:52,636 - tests.orchestrator - INFO - Pod test-pod-e254da33 has been deleted -2025-06-05 18:43:52,637 - tests.orchestrator - INFO - Deleted pod: test-pod-e254da33 -2025-06-05 18:43:53,637 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 3 times) -2025-06-05 18:43:53,637 - tests.orchestrator - INFO - Deleting PVC: test-pvc-ded65224 -2025-06-05 18:43:53,712 - tests.orchestrator - INFO - Waiting for PVC test-pvc-ded65224 to be deleted -2025-06-05 18:43:53,778 - tests.orchestrator - INFO - PVC test-pvc-ded65224 has been deleted -2025-06-05 18:43:53,778 - tests.orchestrator - INFO - Deleted PVC: test-pvc-ded65224 -2025-06-05 18:43:54,778 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 4 times) -2025-06-05 18:43:54,778 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test -2025-06-05 18:43:55,778 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 10 times) -2025-06-05 18:43:55,778 - tests.orchestrator - INFO - No pods to delete -2025-06-05 18:43:56,779 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 3 times) -2025-06-05 18:43:56,779 - tests.orchestrator - INFO - Creating pod test-pod-c065ef92 on PVC test-pvc-29a67853 -2025-06-05 18:43:56,863 - tests.orchestrator - INFO - Created pod: test-pod-c065ef92 using PVC: test-pvc-29a67853 -2025-06-05 18:43:56,863 - tests.orchestrator - INFO - Waiting for pod test-pod-c065ef92 to be ready -2025-06-05 18:43:56,931 - tests.orchestrator - INFO - Pod test-pod-c065ef92 phase: Pending -2025-06-05 18:43:58,999 - tests.orchestrator - INFO - Pod test-pod-c065ef92 phase: Running -2025-06-05 18:43:58,999 - tests.orchestrator - INFO - Pod test-pod-c065ef92 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:44:01,068 - tests.orchestrator - INFO - Pod test-pod-c065ef92 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:44:03,137 - tests.orchestrator - INFO - Pod test-pod-c065ef92 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:44:05,205 - tests.orchestrator - INFO - Pod test-pod-c065ef92 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:44:07,273 - tests.orchestrator - INFO - Pod test-pod-c065ef92 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:44:09,342 - tests.orchestrator - INFO - Pod test-pod-c065ef92 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:44:11,410 - tests.orchestrator - INFO - Pod test-pod-c065ef92 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:44:13,478 - tests.orchestrator - INFO - Pod test-pod-c065ef92 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:44:15,546 - tests.orchestrator - INFO - Pod test-pod-c065ef92 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:44:17,614 - tests.orchestrator - INFO - Pod test-pod-c065ef92 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 18:44:17,614 - tests.orchestrator - INFO - Pod test-pod-c065ef92 is ready -2025-06-05 18:44:18,614 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 4 times) -2025-06-05 18:44:18,614 - tests.orchestrator - INFO - Deleting PVC: test-pvc-5da89758 -2025-06-05 18:44:18,686 - tests.orchestrator - INFO - Waiting for PVC test-pvc-5da89758 to be deleted -2025-06-05 18:44:18,753 - tests.orchestrator - INFO - PVC test-pvc-5da89758 has been deleted -2025-06-05 18:44:18,753 - tests.orchestrator - INFO - Deleted PVC: test-pvc-5da89758 -2025-06-05 18:44:19,753 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 5 times) -2025-06-05 18:44:19,753 - tests.orchestrator - INFO - Deleting PVC: test-pvc-24af11bc -2025-06-05 18:44:19,824 - tests.orchestrator - INFO - Waiting for PVC test-pvc-24af11bc to be deleted -2025-06-05 18:44:19,890 - tests.orchestrator - INFO - PVC test-pvc-24af11bc has been deleted -2025-06-05 18:44:19,890 - tests.orchestrator - INFO - Deleted PVC: test-pvc-24af11bc -2025-06-05 18:44:20,890 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 4 times) -2025-06-05 18:44:20,891 - tests.orchestrator - INFO - Creating pod test-pod-b81d1ccb on PVC test-pvc-29a67853 -2025-06-05 18:44:20,965 - tests.orchestrator - INFO - Created pod: test-pod-b81d1ccb using PVC: test-pvc-29a67853 -2025-06-05 18:44:20,965 - tests.orchestrator - INFO - Waiting for pod test-pod-b81d1ccb to be ready -2025-06-05 18:44:21,033 - tests.orchestrator - INFO - Pod test-pod-b81d1ccb phase: Pending -2025-06-05 18:44:23,101 - tests.orchestrator - INFO - Pod test-pod-b81d1ccb phase: Running -2025-06-05 18:44:23,101 - tests.orchestrator - INFO - Pod test-pod-b81d1ccb conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:44:25,170 - tests.orchestrator - INFO - Pod test-pod-b81d1ccb conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:44:27,240 - tests.orchestrator - INFO - Pod test-pod-b81d1ccb conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:44:29,307 - tests.orchestrator - INFO - Pod test-pod-b81d1ccb conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:44:31,375 - tests.orchestrator - INFO - Pod test-pod-b81d1ccb conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:44:33,444 - tests.orchestrator - INFO - Pod test-pod-b81d1ccb conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:44:35,512 - tests.orchestrator - INFO - Pod test-pod-b81d1ccb conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:44:37,580 - tests.orchestrator - INFO - Pod test-pod-b81d1ccb conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:44:39,649 - tests.orchestrator - INFO - Pod test-pod-b81d1ccb conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:44:41,717 - tests.orchestrator - INFO - Pod test-pod-b81d1ccb conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:44:43,785 - tests.orchestrator - INFO - Pod test-pod-b81d1ccb conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 18:44:43,785 - tests.orchestrator - INFO - Pod test-pod-b81d1ccb is ready -2025-06-05 18:44:44,785 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 10 times) -2025-06-05 18:44:44,785 - tests.orchestrator - INFO - Creating PVC: test-pvc-60b59274 -2025-06-05 18:44:44,862 - tests.orchestrator - INFO - Created PVC: test-pvc-60b59274 -2025-06-05 18:44:44,862 - tests.orchestrator - INFO - Waiting for PVC test-pvc-60b59274 to be bound -2025-06-05 18:44:46,995 - tests.orchestrator - INFO - PVC test-pvc-60b59274 is bound -2025-06-05 18:44:47,995 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 5 times) -2025-06-05 18:44:47,996 - tests.orchestrator - INFO - Testing read/write between pods test-pod-b81d1ccb and test-pod-c065ef92 sharing PVC test-pvc-29a67853 -2025-06-05 18:44:48,066 - tests.orchestrator - ERROR - Failed in read/write verification: (400) -Reason: Bad Request -HTTP response headers: HTTPHeaderDict({'Audit-Id': 'a677ce6d-d92f-45e9-89ad-e0bdd2abd606', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 18:44:48 GMT', 'Content-Length': '139'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Upgrade request required","reason":"BadRequest","code":400} - - -2025-06-05 18:44:49,066 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 11 times) -2025-06-05 18:44:49,066 - tests.orchestrator - INFO - Deleting pod: test-pod-c065ef92 from PVC: test-pvc-29a67853 -2025-06-05 18:44:49,141 - tests.orchestrator - INFO - Waiting for pod test-pod-c065ef92 to be deleted -2025-06-05 18:45:21,752 - tests.orchestrator - INFO - Pod test-pod-c065ef92 has been deleted -2025-06-05 18:45:21,752 - tests.orchestrator - INFO - Deleted pod: test-pod-c065ef92 -2025-06-05 18:45:22,753 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 6 times) -2025-06-05 18:45:22,754 - tests.orchestrator - INFO - Deleting PVC: test-pvc-0b86173e -2025-06-05 18:45:22,826 - tests.orchestrator - INFO - Waiting for PVC test-pvc-0b86173e to be deleted -2025-06-05 18:45:22,893 - tests.orchestrator - INFO - PVC test-pvc-0b86173e has been deleted -2025-06-05 18:45:22,893 - tests.orchestrator - INFO - Deleted PVC: test-pvc-0b86173e -2025-06-05 18:45:23,893 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 5 times) -2025-06-05 18:45:23,893 - tests.orchestrator - INFO - Creating pod test-pod-077370c2 on PVC test-pvc-41d09b77 -2025-06-05 18:45:23,976 - tests.orchestrator - INFO - Created pod: test-pod-077370c2 using PVC: test-pvc-41d09b77 -2025-06-05 18:45:23,976 - tests.orchestrator - INFO - Waiting for pod test-pod-077370c2 to be ready -2025-06-05 18:45:24,044 - tests.orchestrator - INFO - Pod test-pod-077370c2 phase: Pending -2025-06-05 18:45:26,113 - tests.orchestrator - INFO - Pod test-pod-077370c2 phase: Running -2025-06-05 18:45:26,113 - tests.orchestrator - INFO - Pod test-pod-077370c2 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:45:28,181 - tests.orchestrator - INFO - Pod test-pod-077370c2 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:45:30,250 - tests.orchestrator - INFO - Pod test-pod-077370c2 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:45:32,319 - tests.orchestrator - INFO - Pod test-pod-077370c2 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:45:34,387 - tests.orchestrator - INFO - Pod test-pod-077370c2 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:45:36,454 - tests.orchestrator - INFO - Pod test-pod-077370c2 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:45:38,523 - tests.orchestrator - INFO - Pod test-pod-077370c2 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:45:40,591 - tests.orchestrator - INFO - Pod test-pod-077370c2 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:45:42,666 - tests.orchestrator - INFO - Pod test-pod-077370c2 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:45:44,734 - tests.orchestrator - INFO - Pod test-pod-077370c2 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 18:45:44,734 - tests.orchestrator - INFO - Pod test-pod-077370c2 is ready -2025-06-05 18:45:45,735 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 6 times) -2025-06-05 18:45:45,735 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test -2025-06-05 18:45:46,735 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 12 times) -2025-06-05 18:45:46,735 - tests.orchestrator - INFO - Deleting pod: test-pod-b81d1ccb from PVC: test-pvc-29a67853 -2025-06-05 18:45:46,810 - tests.orchestrator - INFO - Waiting for pod test-pod-b81d1ccb to be deleted -2025-06-05 18:46:19,436 - tests.orchestrator - INFO - Pod test-pod-b81d1ccb has been deleted -2025-06-05 18:46:19,436 - tests.orchestrator - INFO - Deleted pod: test-pod-b81d1ccb -2025-06-05 18:46:20,437 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 7 times) -2025-06-05 18:46:20,437 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test -2025-06-05 18:46:21,437 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 11 times) -2025-06-05 18:46:21,437 - tests.orchestrator - INFO - Creating PVC: test-pvc-33a8ceec -2025-06-05 18:46:21,523 - tests.orchestrator - INFO - Created PVC: test-pvc-33a8ceec -2025-06-05 18:46:21,523 - tests.orchestrator - INFO - Waiting for PVC test-pvc-33a8ceec to be bound -2025-06-05 18:46:23,658 - tests.orchestrator - INFO - PVC test-pvc-33a8ceec is bound -2025-06-05 18:46:24,658 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 6 times) -2025-06-05 18:46:24,658 - tests.orchestrator - INFO - Creating pod test-pod-f1a6e117 on PVC test-pvc-9c78a2d7 -2025-06-05 18:46:24,733 - tests.orchestrator - INFO - Created pod: test-pod-f1a6e117 using PVC: test-pvc-9c78a2d7 -2025-06-05 18:46:24,734 - tests.orchestrator - INFO - Waiting for pod test-pod-f1a6e117 to be ready -2025-06-05 18:46:24,801 - tests.orchestrator - INFO - Pod test-pod-f1a6e117 phase: Pending -2025-06-05 18:46:28,938 - tests.orchestrator - INFO - Pod test-pod-f1a6e117 phase: Running -2025-06-05 18:46:28,938 - tests.orchestrator - INFO - Pod test-pod-f1a6e117 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:46:31,006 - tests.orchestrator - INFO - Pod test-pod-f1a6e117 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:46:33,074 - tests.orchestrator - INFO - Pod test-pod-f1a6e117 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:46:35,143 - tests.orchestrator - INFO - Pod test-pod-f1a6e117 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:46:37,211 - tests.orchestrator - INFO - Pod test-pod-f1a6e117 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:46:39,279 - tests.orchestrator - INFO - Pod test-pod-f1a6e117 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:46:41,348 - tests.orchestrator - INFO - Pod test-pod-f1a6e117 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:46:43,417 - tests.orchestrator - INFO - Pod test-pod-f1a6e117 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:46:45,485 - tests.orchestrator - INFO - Pod test-pod-f1a6e117 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 18:46:45,485 - tests.orchestrator - INFO - Pod test-pod-f1a6e117 is ready -2025-06-05 18:46:46,485 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 7 times) -2025-06-05 18:46:46,486 - tests.orchestrator - INFO - Creating pod test-pod-ba1a0549 on PVC test-pvc-41d09b77 -2025-06-05 18:46:46,563 - tests.orchestrator - INFO - Created pod: test-pod-ba1a0549 using PVC: test-pvc-41d09b77 -2025-06-05 18:46:46,563 - tests.orchestrator - INFO - Waiting for pod test-pod-ba1a0549 to be ready -2025-06-05 18:46:46,632 - tests.orchestrator - INFO - Pod test-pod-ba1a0549 phase: Pending -2025-06-05 18:46:48,701 - tests.orchestrator - INFO - Pod test-pod-ba1a0549 phase: Running -2025-06-05 18:46:48,701 - tests.orchestrator - INFO - Pod test-pod-ba1a0549 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:46:50,769 - tests.orchestrator - INFO - Pod test-pod-ba1a0549 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:46:52,838 - tests.orchestrator - INFO - Pod test-pod-ba1a0549 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:46:54,906 - tests.orchestrator - INFO - Pod test-pod-ba1a0549 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:46:56,974 - tests.orchestrator - INFO - Pod test-pod-ba1a0549 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:46:59,043 - tests.orchestrator - INFO - Pod test-pod-ba1a0549 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:47:01,111 - tests.orchestrator - INFO - Pod test-pod-ba1a0549 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:47:03,180 - tests.orchestrator - INFO - Pod test-pod-ba1a0549 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 18:47:03,180 - tests.orchestrator - INFO - Pod test-pod-ba1a0549 is ready -2025-06-05 18:47:04,180 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 8 times) -2025-06-05 18:47:04,180 - tests.orchestrator - INFO - Testing read/write between pods test-pod-077370c2 and test-pod-ba1a0549 sharing PVC test-pvc-41d09b77 -2025-06-05 18:47:04,252 - tests.orchestrator - ERROR - Failed in read/write verification: (400) -Reason: Bad Request -HTTP response headers: HTTPHeaderDict({'Audit-Id': 'f65371f5-232a-415d-a016-435e1f074adf', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 18:47:04 GMT', 'Content-Length': '139'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Upgrade request required","reason":"BadRequest","code":400} - - -2025-06-05 18:47:05,253 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 13 times) -2025-06-05 18:47:05,253 - tests.orchestrator - INFO - Deleting pod: test-pod-ba1a0549 from PVC: test-pvc-41d09b77 -2025-06-05 18:47:05,328 - tests.orchestrator - INFO - Waiting for pod test-pod-ba1a0549 to be deleted -2025-06-05 18:47:38,669 - tests.orchestrator - INFO - Pod test-pod-ba1a0549 has been deleted -2025-06-05 18:47:38,669 - tests.orchestrator - INFO - Deleted pod: test-pod-ba1a0549 -2025-06-05 18:47:39,669 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 14 times) -2025-06-05 18:47:39,670 - tests.orchestrator - INFO - Deleting pod: test-pod-077370c2 from PVC: test-pvc-41d09b77 -2025-06-05 18:47:39,746 - tests.orchestrator - INFO - Waiting for pod test-pod-077370c2 to be deleted -2025-06-05 18:48:12,360 - tests.orchestrator - INFO - Pod test-pod-077370c2 has been deleted -2025-06-05 18:48:12,360 - tests.orchestrator - INFO - Deleted pod: test-pod-077370c2 -2025-06-05 18:48:13,360 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 15 times) -2025-06-05 18:48:13,360 - tests.orchestrator - INFO - Deleting pod: test-pod-f1a6e117 from PVC: test-pvc-9c78a2d7 -2025-06-05 18:48:13,437 - tests.orchestrator - INFO - Waiting for pod test-pod-f1a6e117 to be deleted -2025-06-05 18:48:46,049 - tests.orchestrator - INFO - Pod test-pod-f1a6e117 has been deleted -2025-06-05 18:48:46,049 - tests.orchestrator - INFO - Deleted pod: test-pod-f1a6e117 -2025-06-05 18:48:47,049 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 9 times) -2025-06-05 18:48:47,050 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test -2025-06-05 18:48:48,050 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 7 times) -2025-06-05 18:48:48,050 - tests.orchestrator - INFO - Deleting PVC: test-pvc-9c78a2d7 -2025-06-05 18:48:48,123 - tests.orchestrator - INFO - Waiting for PVC test-pvc-9c78a2d7 to be deleted -2025-06-05 18:48:48,188 - tests.orchestrator - INFO - PVC test-pvc-9c78a2d7 has been deleted -2025-06-05 18:48:48,189 - tests.orchestrator - INFO - Deleted PVC: test-pvc-9c78a2d7 -2025-06-05 18:48:49,189 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 8 times) -2025-06-05 18:48:49,189 - tests.orchestrator - INFO - Creating pod test-pod-f445578c on PVC test-pvc-29a67853 -2025-06-05 18:48:49,275 - tests.orchestrator - INFO - Created pod: test-pod-f445578c using PVC: test-pvc-29a67853 -2025-06-05 18:48:49,275 - tests.orchestrator - INFO - Waiting for pod test-pod-f445578c to be ready -2025-06-05 18:48:49,343 - tests.orchestrator - INFO - Pod test-pod-f445578c phase: Pending -2025-06-05 18:48:51,411 - tests.orchestrator - INFO - Pod test-pod-f445578c phase: Running -2025-06-05 18:48:51,412 - tests.orchestrator - INFO - Pod test-pod-f445578c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:48:53,479 - tests.orchestrator - INFO - Pod test-pod-f445578c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:48:55,548 - tests.orchestrator - INFO - Pod test-pod-f445578c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:48:57,617 - tests.orchestrator - INFO - Pod test-pod-f445578c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:48:59,684 - tests.orchestrator - INFO - Pod test-pod-f445578c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:49:01,806 - tests.orchestrator - INFO - Pod test-pod-f445578c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:49:03,875 - tests.orchestrator - INFO - Pod test-pod-f445578c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:49:05,943 - tests.orchestrator - INFO - Pod test-pod-f445578c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:49:08,011 - tests.orchestrator - INFO - Pod test-pod-f445578c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:49:10,080 - tests.orchestrator - INFO - Pod test-pod-f445578c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 18:49:10,080 - tests.orchestrator - INFO - Pod test-pod-f445578c is ready -2025-06-05 18:49:11,080 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 12 times) -2025-06-05 18:49:11,080 - tests.orchestrator - INFO - Creating PVC: test-pvc-6e8e6345 -2025-06-05 18:49:11,150 - tests.orchestrator - INFO - Created PVC: test-pvc-6e8e6345 -2025-06-05 18:49:11,150 - tests.orchestrator - INFO - Waiting for PVC test-pvc-6e8e6345 to be bound -2025-06-05 18:49:13,283 - tests.orchestrator - INFO - PVC test-pvc-6e8e6345 is bound -2025-06-05 18:49:14,283 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 9 times) -2025-06-05 18:49:14,283 - tests.orchestrator - INFO - Creating pod test-pod-28044324 on PVC test-pvc-41d09b77 -2025-06-05 18:49:14,357 - tests.orchestrator - INFO - Created pod: test-pod-28044324 using PVC: test-pvc-41d09b77 -2025-06-05 18:49:14,358 - tests.orchestrator - INFO - Waiting for pod test-pod-28044324 to be ready -2025-06-05 18:49:14,425 - tests.orchestrator - INFO - Pod test-pod-28044324 phase: Pending -2025-06-05 18:49:16,500 - tests.orchestrator - INFO - Pod test-pod-28044324 phase: Running -2025-06-05 18:49:16,500 - tests.orchestrator - INFO - Pod test-pod-28044324 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:49:18,569 - tests.orchestrator - INFO - Pod test-pod-28044324 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:49:20,636 - tests.orchestrator - INFO - Pod test-pod-28044324 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:49:22,705 - tests.orchestrator - INFO - Pod test-pod-28044324 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:49:24,775 - tests.orchestrator - INFO - Pod test-pod-28044324 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:49:26,842 - tests.orchestrator - INFO - Pod test-pod-28044324 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:49:28,911 - tests.orchestrator - INFO - Pod test-pod-28044324 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:49:30,980 - tests.orchestrator - INFO - Pod test-pod-28044324 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:49:33,064 - tests.orchestrator - INFO - Pod test-pod-28044324 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:49:35,131 - tests.orchestrator - INFO - Pod test-pod-28044324 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 18:49:35,132 - tests.orchestrator - INFO - Pod test-pod-28044324 is ready -2025-06-05 18:49:36,132 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 13 times) -2025-06-05 18:49:36,132 - tests.orchestrator - INFO - Creating PVC: test-pvc-b84d5df8 -2025-06-05 18:49:36,208 - tests.orchestrator - INFO - Created PVC: test-pvc-b84d5df8 -2025-06-05 18:49:36,208 - tests.orchestrator - INFO - Waiting for PVC test-pvc-b84d5df8 to be bound -2025-06-05 18:49:38,342 - tests.orchestrator - INFO - PVC test-pvc-b84d5df8 is bound -2025-06-05 18:49:39,342 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 10 times) -2025-06-05 18:49:39,342 - tests.orchestrator - INFO - Creating pod test-pod-5a3a12f1 on PVC test-pvc-41d09b77 -2025-06-05 18:49:39,418 - tests.orchestrator - INFO - Created pod: test-pod-5a3a12f1 using PVC: test-pvc-41d09b77 -2025-06-05 18:49:39,418 - tests.orchestrator - INFO - Waiting for pod test-pod-5a3a12f1 to be ready -2025-06-05 18:49:39,485 - tests.orchestrator - INFO - Pod test-pod-5a3a12f1 phase: Pending -2025-06-05 18:49:41,571 - tests.orchestrator - INFO - Pod test-pod-5a3a12f1 phase: Running -2025-06-05 18:49:41,571 - tests.orchestrator - INFO - Pod test-pod-5a3a12f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:49:43,642 - tests.orchestrator - INFO - Pod test-pod-5a3a12f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:49:45,710 - tests.orchestrator - INFO - Pod test-pod-5a3a12f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:49:47,777 - tests.orchestrator - INFO - Pod test-pod-5a3a12f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:49:49,847 - tests.orchestrator - INFO - Pod test-pod-5a3a12f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:49:51,914 - tests.orchestrator - INFO - Pod test-pod-5a3a12f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:49:53,982 - tests.orchestrator - INFO - Pod test-pod-5a3a12f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:49:56,051 - tests.orchestrator - INFO - Pod test-pod-5a3a12f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 18:49:56,052 - tests.orchestrator - INFO - Pod test-pod-5a3a12f1 is ready -2025-06-05 18:49:57,052 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 14 times) -2025-06-05 18:49:57,052 - tests.orchestrator - INFO - Creating PVC: test-pvc-634bfb7f -2025-06-05 18:49:57,120 - tests.orchestrator - INFO - Created PVC: test-pvc-634bfb7f -2025-06-05 18:49:57,121 - tests.orchestrator - INFO - Waiting for PVC test-pvc-634bfb7f to be bound -2025-06-05 18:49:59,254 - tests.orchestrator - INFO - PVC test-pvc-634bfb7f is bound -2025-06-05 18:50:00,254 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 10 times) -2025-06-05 18:50:00,254 - tests.orchestrator - INFO - Testing read/write between pods test-pod-5a3a12f1 and test-pod-28044324 sharing PVC test-pvc-41d09b77 -2025-06-05 18:50:00,324 - tests.orchestrator - ERROR - Failed in read/write verification: (400) -Reason: Bad Request -HTTP response headers: HTTPHeaderDict({'Audit-Id': '72eb5033-42c4-4730-9e3d-d9327f160c1c', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 18:50:00 GMT', 'Content-Length': '139'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Upgrade request required","reason":"BadRequest","code":400} - - -2025-06-05 18:50:01,324 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 8 times) -2025-06-05 18:50:01,324 - tests.orchestrator - INFO - Deleting PVC: test-pvc-33a8ceec -2025-06-05 18:50:01,401 - tests.orchestrator - INFO - Waiting for PVC test-pvc-33a8ceec to be deleted -2025-06-05 18:50:01,530 - tests.orchestrator - INFO - PVC test-pvc-33a8ceec has been deleted -2025-06-05 18:50:01,530 - tests.orchestrator - INFO - Deleted PVC: test-pvc-33a8ceec -2025-06-05 18:50:02,530 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 11 times) -2025-06-05 18:50:02,530 - tests.orchestrator - INFO - Creating pod test-pod-ace60ac9 on PVC test-pvc-634bfb7f -2025-06-05 18:50:02,605 - tests.orchestrator - INFO - Created pod: test-pod-ace60ac9 using PVC: test-pvc-634bfb7f -2025-06-05 18:50:02,605 - tests.orchestrator - INFO - Waiting for pod test-pod-ace60ac9 to be ready -2025-06-05 18:50:02,673 - tests.orchestrator - INFO - Pod test-pod-ace60ac9 phase: Pending -2025-06-05 18:50:04,742 - tests.orchestrator - INFO - Pod test-pod-ace60ac9 phase: Running -2025-06-05 18:50:04,742 - tests.orchestrator - INFO - Pod test-pod-ace60ac9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:50:06,809 - tests.orchestrator - INFO - Pod test-pod-ace60ac9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:50:08,877 - tests.orchestrator - INFO - Pod test-pod-ace60ac9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:50:10,946 - tests.orchestrator - INFO - Pod test-pod-ace60ac9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:50:13,014 - tests.orchestrator - INFO - Pod test-pod-ace60ac9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:50:15,082 - tests.orchestrator - INFO - Pod test-pod-ace60ac9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:50:17,151 - tests.orchestrator - INFO - Pod test-pod-ace60ac9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:50:19,220 - tests.orchestrator - INFO - Pod test-pod-ace60ac9 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 18:50:19,220 - tests.orchestrator - INFO - Pod test-pod-ace60ac9 is ready -2025-06-05 18:50:20,220 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 9 times) -2025-06-05 18:50:20,220 - tests.orchestrator - INFO - Deleting PVC: test-pvc-634bfb7f -2025-06-05 18:50:20,221 - tests.orchestrator - INFO - Deleting 1 pods using PVC test-pvc-634bfb7f -2025-06-05 18:50:20,221 - tests.orchestrator - INFO - Deleting pod: test-pod-ace60ac9 from PVC: test-pvc-634bfb7f -2025-06-05 18:50:20,296 - tests.orchestrator - INFO - Waiting for pod test-pod-ace60ac9 to be deleted -2025-06-05 18:50:52,910 - tests.orchestrator - INFO - Pod test-pod-ace60ac9 has been deleted -2025-06-05 18:50:52,910 - tests.orchestrator - INFO - Deleted pod: test-pod-ace60ac9 -2025-06-05 18:50:52,983 - tests.orchestrator - INFO - Waiting for PVC test-pvc-634bfb7f to be deleted -2025-06-05 18:50:53,052 - tests.orchestrator - INFO - PVC test-pvc-634bfb7f has been deleted -2025-06-05 18:50:53,052 - tests.orchestrator - INFO - Deleted PVC: test-pvc-634bfb7f -2025-06-05 18:50:54,052 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 16 times) -2025-06-05 18:50:54,053 - tests.orchestrator - INFO - Deleting pod: test-pod-f445578c from PVC: test-pvc-29a67853 -2025-06-05 18:50:54,129 - tests.orchestrator - INFO - Waiting for pod test-pod-f445578c to be deleted -2025-06-05 18:51:26,740 - tests.orchestrator - INFO - Pod test-pod-f445578c has been deleted -2025-06-05 18:51:26,741 - tests.orchestrator - INFO - Deleted pod: test-pod-f445578c -2025-06-05 18:51:27,741 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 17 times) -2025-06-05 18:51:27,741 - tests.orchestrator - INFO - Deleting pod: test-pod-5a3a12f1 from PVC: test-pvc-41d09b77 -2025-06-05 18:51:27,817 - tests.orchestrator - INFO - Waiting for pod test-pod-5a3a12f1 to be deleted -2025-06-05 18:52:00,433 - tests.orchestrator - INFO - Pod test-pod-5a3a12f1 has been deleted -2025-06-05 18:52:00,433 - tests.orchestrator - INFO - Deleted pod: test-pod-5a3a12f1 -2025-06-05 18:52:01,433 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 12 times) -2025-06-05 18:52:01,434 - tests.orchestrator - INFO - Creating pod test-pod-66a16102 on PVC test-pvc-29a67853 -2025-06-05 18:52:01,788 - tests.orchestrator - INFO - Created pod: test-pod-66a16102 using PVC: test-pvc-29a67853 -2025-06-05 18:52:01,788 - tests.orchestrator - INFO - Waiting for pod test-pod-66a16102 to be ready -2025-06-05 18:52:01,860 - tests.orchestrator - INFO - Pod test-pod-66a16102 phase: Pending -2025-06-05 18:52:03,929 - tests.orchestrator - INFO - Pod test-pod-66a16102 phase: Running -2025-06-05 18:52:03,929 - tests.orchestrator - INFO - Pod test-pod-66a16102 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:52:05,997 - tests.orchestrator - INFO - Pod test-pod-66a16102 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:52:08,065 - tests.orchestrator - INFO - Pod test-pod-66a16102 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:52:10,135 - tests.orchestrator - INFO - Pod test-pod-66a16102 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:52:12,202 - tests.orchestrator - INFO - Pod test-pod-66a16102 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:52:14,271 - tests.orchestrator - INFO - Pod test-pod-66a16102 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:52:16,339 - tests.orchestrator - INFO - Pod test-pod-66a16102 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:52:18,407 - tests.orchestrator - INFO - Pod test-pod-66a16102 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:52:20,476 - tests.orchestrator - INFO - Pod test-pod-66a16102 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:52:22,544 - tests.orchestrator - INFO - Pod test-pod-66a16102 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 18:52:22,544 - tests.orchestrator - INFO - Pod test-pod-66a16102 is ready -2025-06-05 18:52:23,544 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 13 times) -2025-06-05 18:52:23,545 - tests.orchestrator - INFO - Creating pod test-pod-075367b4 on PVC test-pvc-41d09b77 -2025-06-05 18:52:23,620 - tests.orchestrator - INFO - Created pod: test-pod-075367b4 using PVC: test-pvc-41d09b77 -2025-06-05 18:52:23,620 - tests.orchestrator - INFO - Waiting for pod test-pod-075367b4 to be ready -2025-06-05 18:52:23,687 - tests.orchestrator - INFO - Pod test-pod-075367b4 phase: Pending -2025-06-05 18:52:25,757 - tests.orchestrator - INFO - Pod test-pod-075367b4 phase: Running -2025-06-05 18:52:25,757 - tests.orchestrator - INFO - Pod test-pod-075367b4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:52:27,825 - tests.orchestrator - INFO - Pod test-pod-075367b4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:52:29,893 - tests.orchestrator - INFO - Pod test-pod-075367b4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:52:31,966 - tests.orchestrator - INFO - Pod test-pod-075367b4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:52:34,035 - tests.orchestrator - INFO - Pod test-pod-075367b4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:52:36,103 - tests.orchestrator - INFO - Pod test-pod-075367b4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:52:38,173 - tests.orchestrator - INFO - Pod test-pod-075367b4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:52:40,242 - tests.orchestrator - INFO - Pod test-pod-075367b4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 18:52:40,242 - tests.orchestrator - INFO - Pod test-pod-075367b4 is ready -2025-06-05 18:52:41,243 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 14 times) -2025-06-05 18:52:41,243 - tests.orchestrator - INFO - Creating pod test-pod-dc5857c7 on PVC test-pvc-29a67853 -2025-06-05 18:52:41,326 - tests.orchestrator - INFO - Created pod: test-pod-dc5857c7 using PVC: test-pvc-29a67853 -2025-06-05 18:52:41,326 - tests.orchestrator - INFO - Waiting for pod test-pod-dc5857c7 to be ready -2025-06-05 18:52:41,394 - tests.orchestrator - INFO - Pod test-pod-dc5857c7 phase: Pending -2025-06-05 18:52:43,462 - tests.orchestrator - INFO - Pod test-pod-dc5857c7 phase: Running -2025-06-05 18:52:43,463 - tests.orchestrator - INFO - Pod test-pod-dc5857c7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:52:45,530 - tests.orchestrator - INFO - Pod test-pod-dc5857c7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:52:47,598 - tests.orchestrator - INFO - Pod test-pod-dc5857c7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:52:49,667 - tests.orchestrator - INFO - Pod test-pod-dc5857c7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:52:51,735 - tests.orchestrator - INFO - Pod test-pod-dc5857c7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:52:53,803 - tests.orchestrator - INFO - Pod test-pod-dc5857c7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:52:55,872 - tests.orchestrator - INFO - Pod test-pod-dc5857c7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:52:57,940 - tests.orchestrator - INFO - Pod test-pod-dc5857c7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:53:00,009 - tests.orchestrator - INFO - Pod test-pod-dc5857c7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:53:02,085 - tests.orchestrator - INFO - Pod test-pod-dc5857c7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 18:53:02,086 - tests.orchestrator - INFO - Pod test-pod-dc5857c7 is ready -2025-06-05 18:53:03,086 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 18 times) -2025-06-05 18:53:03,086 - tests.orchestrator - INFO - Deleting pod: test-pod-66a16102 from PVC: test-pvc-29a67853 -2025-06-05 18:53:03,163 - tests.orchestrator - INFO - Waiting for pod test-pod-66a16102 to be deleted -2025-06-05 18:53:35,844 - tests.orchestrator - INFO - Pod test-pod-66a16102 has been deleted -2025-06-05 18:53:35,844 - tests.orchestrator - INFO - Deleted pod: test-pod-66a16102 -2025-06-05 18:53:36,844 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 15 times) -2025-06-05 18:53:36,844 - tests.orchestrator - INFO - Creating pod test-pod-ab6b7029 on PVC test-pvc-41d09b77 -2025-06-05 18:53:36,927 - tests.orchestrator - INFO - Created pod: test-pod-ab6b7029 using PVC: test-pvc-41d09b77 -2025-06-05 18:53:36,927 - tests.orchestrator - INFO - Waiting for pod test-pod-ab6b7029 to be ready -2025-06-05 18:53:36,994 - tests.orchestrator - INFO - Pod test-pod-ab6b7029 phase: Pending -2025-06-05 18:53:39,063 - tests.orchestrator - INFO - Pod test-pod-ab6b7029 phase: Running -2025-06-05 18:53:39,064 - tests.orchestrator - INFO - Pod test-pod-ab6b7029 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:53:41,131 - tests.orchestrator - INFO - Pod test-pod-ab6b7029 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:53:43,199 - tests.orchestrator - INFO - Pod test-pod-ab6b7029 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:53:45,268 - tests.orchestrator - INFO - Pod test-pod-ab6b7029 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:53:47,336 - tests.orchestrator - INFO - Pod test-pod-ab6b7029 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:53:49,404 - tests.orchestrator - INFO - Pod test-pod-ab6b7029 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:53:51,473 - tests.orchestrator - INFO - Pod test-pod-ab6b7029 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:53:53,541 - tests.orchestrator - INFO - Pod test-pod-ab6b7029 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:53:55,609 - tests.orchestrator - INFO - Pod test-pod-ab6b7029 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:53:57,678 - tests.orchestrator - INFO - Pod test-pod-ab6b7029 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:53:59,746 - tests.orchestrator - INFO - Pod test-pod-ab6b7029 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 18:53:59,746 - tests.orchestrator - INFO - Pod test-pod-ab6b7029 is ready -2025-06-05 18:54:00,746 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 19 times) -2025-06-05 18:54:00,746 - tests.orchestrator - INFO - Deleting pod: test-pod-075367b4 from PVC: test-pvc-41d09b77 -2025-06-05 18:54:00,824 - tests.orchestrator - INFO - Waiting for pod test-pod-075367b4 to be deleted -2025-06-05 18:54:33,435 - tests.orchestrator - INFO - Pod test-pod-075367b4 has been deleted -2025-06-05 18:54:33,436 - tests.orchestrator - INFO - Deleted pod: test-pod-075367b4 -2025-06-05 18:54:34,436 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 20 times) -2025-06-05 18:54:34,436 - tests.orchestrator - INFO - Deleting pod: test-pod-28044324 from PVC: test-pvc-41d09b77 -2025-06-05 18:54:34,511 - tests.orchestrator - INFO - Waiting for pod test-pod-28044324 to be deleted -2025-06-05 18:55:07,139 - tests.orchestrator - INFO - Pod test-pod-28044324 has been deleted -2025-06-05 18:55:07,139 - tests.orchestrator - INFO - Deleted pod: test-pod-28044324 -2025-06-05 18:55:08,139 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 15 times) -2025-06-05 18:55:08,140 - tests.orchestrator - INFO - Creating PVC: test-pvc-0bf22cd9 -2025-06-05 18:55:08,219 - tests.orchestrator - INFO - Created PVC: test-pvc-0bf22cd9 -2025-06-05 18:55:08,219 - tests.orchestrator - INFO - Waiting for PVC test-pvc-0bf22cd9 to be bound -2025-06-05 18:55:10,352 - tests.orchestrator - INFO - PVC test-pvc-0bf22cd9 is bound -2025-06-05 18:55:11,352 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 10 times) -2025-06-05 18:55:11,352 - tests.orchestrator - INFO - Deleting PVC: test-pvc-f1842f61 -2025-06-05 18:55:11,424 - tests.orchestrator - INFO - Waiting for PVC test-pvc-f1842f61 to be deleted -2025-06-05 18:55:11,492 - tests.orchestrator - INFO - PVC test-pvc-f1842f61 has been deleted -2025-06-05 18:55:11,492 - tests.orchestrator - INFO - Deleted PVC: test-pvc-f1842f61 -2025-06-05 18:55:12,492 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 16 times) -2025-06-05 18:55:12,492 - tests.orchestrator - INFO - Creating pod test-pod-670c9b02 on PVC test-pvc-60b59274 -2025-06-05 18:55:12,572 - tests.orchestrator - INFO - Created pod: test-pod-670c9b02 using PVC: test-pvc-60b59274 -2025-06-05 18:55:12,572 - tests.orchestrator - INFO - Waiting for pod test-pod-670c9b02 to be ready -2025-06-05 18:55:12,639 - tests.orchestrator - INFO - Pod test-pod-670c9b02 phase: Pending -2025-06-05 18:55:16,774 - tests.orchestrator - INFO - Pod test-pod-670c9b02 phase: Running -2025-06-05 18:55:16,774 - tests.orchestrator - INFO - Pod test-pod-670c9b02 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:55:18,842 - tests.orchestrator - INFO - Pod test-pod-670c9b02 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:55:20,911 - tests.orchestrator - INFO - Pod test-pod-670c9b02 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:55:22,979 - tests.orchestrator - INFO - Pod test-pod-670c9b02 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:55:25,047 - tests.orchestrator - INFO - Pod test-pod-670c9b02 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:55:27,116 - tests.orchestrator - INFO - Pod test-pod-670c9b02 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:55:29,184 - tests.orchestrator - INFO - Pod test-pod-670c9b02 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:55:31,252 - tests.orchestrator - INFO - Pod test-pod-670c9b02 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:55:33,321 - tests.orchestrator - INFO - Pod test-pod-670c9b02 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:55:35,390 - tests.orchestrator - INFO - Pod test-pod-670c9b02 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 18:55:35,390 - tests.orchestrator - INFO - Pod test-pod-670c9b02 is ready -2025-06-05 18:55:36,391 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 17 times) -2025-06-05 18:55:36,391 - tests.orchestrator - INFO - Creating pod test-pod-392b4b23 on PVC test-pvc-b84d5df8 -2025-06-05 18:55:36,465 - tests.orchestrator - INFO - Created pod: test-pod-392b4b23 using PVC: test-pvc-b84d5df8 -2025-06-05 18:55:36,465 - tests.orchestrator - INFO - Waiting for pod test-pod-392b4b23 to be ready -2025-06-05 18:55:36,535 - tests.orchestrator - INFO - Pod test-pod-392b4b23 phase: Pending -2025-06-05 18:55:38,603 - tests.orchestrator - INFO - Pod test-pod-392b4b23 phase: Running -2025-06-05 18:55:38,603 - tests.orchestrator - INFO - Pod test-pod-392b4b23 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:55:40,672 - tests.orchestrator - INFO - Pod test-pod-392b4b23 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:55:42,740 - tests.orchestrator - INFO - Pod test-pod-392b4b23 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:55:44,809 - tests.orchestrator - INFO - Pod test-pod-392b4b23 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:55:46,878 - tests.orchestrator - INFO - Pod test-pod-392b4b23 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:55:48,946 - tests.orchestrator - INFO - Pod test-pod-392b4b23 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:55:51,014 - tests.orchestrator - INFO - Pod test-pod-392b4b23 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:55:53,084 - tests.orchestrator - INFO - Pod test-pod-392b4b23 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 18:55:53,084 - tests.orchestrator - INFO - Pod test-pod-392b4b23 is ready -2025-06-05 18:55:54,085 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 16 times) -2025-06-05 18:55:54,085 - tests.orchestrator - INFO - Creating PVC: test-pvc-54d1de91 -2025-06-05 18:55:54,160 - tests.orchestrator - INFO - Created PVC: test-pvc-54d1de91 -2025-06-05 18:55:54,160 - tests.orchestrator - INFO - Waiting for PVC test-pvc-54d1de91 to be bound -2025-06-05 18:55:56,294 - tests.orchestrator - INFO - PVC test-pvc-54d1de91 is bound -2025-06-05 18:55:57,295 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 17 times) -2025-06-05 18:55:57,295 - tests.orchestrator - INFO - Creating PVC: test-pvc-4184bff1 -2025-06-05 18:55:57,364 - tests.orchestrator - INFO - Created PVC: test-pvc-4184bff1 -2025-06-05 18:55:57,364 - tests.orchestrator - INFO - Waiting for PVC test-pvc-4184bff1 to be bound -2025-06-05 18:55:59,498 - tests.orchestrator - INFO - PVC test-pvc-4184bff1 is bound -2025-06-05 18:56:00,498 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 21 times) -2025-06-05 18:56:00,498 - tests.orchestrator - INFO - Deleting pod: test-pod-392b4b23 from PVC: test-pvc-b84d5df8 -2025-06-05 18:56:00,575 - tests.orchestrator - INFO - Waiting for pod test-pod-392b4b23 to be deleted -2025-06-05 18:56:33,187 - tests.orchestrator - INFO - Pod test-pod-392b4b23 has been deleted -2025-06-05 18:56:33,187 - tests.orchestrator - INFO - Deleted pod: test-pod-392b4b23 -2025-06-05 18:56:34,187 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 18 times) -2025-06-05 18:56:34,187 - tests.orchestrator - INFO - Creating pod test-pod-01b55f76 on PVC test-pvc-41d09b77 -2025-06-05 18:56:34,271 - tests.orchestrator - INFO - Created pod: test-pod-01b55f76 using PVC: test-pvc-41d09b77 -2025-06-05 18:56:34,271 - tests.orchestrator - INFO - Waiting for pod test-pod-01b55f76 to be ready -2025-06-05 18:56:34,339 - tests.orchestrator - INFO - Pod test-pod-01b55f76 phase: Pending -2025-06-05 18:56:36,408 - tests.orchestrator - INFO - Pod test-pod-01b55f76 phase: Running -2025-06-05 18:56:36,408 - tests.orchestrator - INFO - Pod test-pod-01b55f76 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:56:39,222 - tests.orchestrator - INFO - Pod test-pod-01b55f76 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:56:41,289 - tests.orchestrator - INFO - Pod test-pod-01b55f76 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:56:43,358 - tests.orchestrator - INFO - Pod test-pod-01b55f76 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:56:45,426 - tests.orchestrator - INFO - Pod test-pod-01b55f76 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:56:47,494 - tests.orchestrator - INFO - Pod test-pod-01b55f76 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:56:49,563 - tests.orchestrator - INFO - Pod test-pod-01b55f76 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:56:51,631 - tests.orchestrator - INFO - Pod test-pod-01b55f76 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 18:56:51,632 - tests.orchestrator - INFO - Pod test-pod-01b55f76 is ready -2025-06-05 18:56:52,632 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 19 times) -2025-06-05 18:56:52,632 - tests.orchestrator - INFO - Creating pod test-pod-3ab2a061 on PVC test-pvc-4184bff1 -2025-06-05 18:56:52,706 - tests.orchestrator - INFO - Created pod: test-pod-3ab2a061 using PVC: test-pvc-4184bff1 -2025-06-05 18:56:52,706 - tests.orchestrator - INFO - Waiting for pod test-pod-3ab2a061 to be ready -2025-06-05 18:56:52,774 - tests.orchestrator - INFO - Pod test-pod-3ab2a061 phase: Pending -2025-06-05 18:56:54,842 - tests.orchestrator - INFO - Pod test-pod-3ab2a061 phase: Running -2025-06-05 18:56:54,842 - tests.orchestrator - INFO - Pod test-pod-3ab2a061 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:56:56,911 - tests.orchestrator - INFO - Pod test-pod-3ab2a061 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:56:58,978 - tests.orchestrator - INFO - Pod test-pod-3ab2a061 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:57:01,047 - tests.orchestrator - INFO - Pod test-pod-3ab2a061 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:57:03,116 - tests.orchestrator - INFO - Pod test-pod-3ab2a061 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:57:05,183 - tests.orchestrator - INFO - Pod test-pod-3ab2a061 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:57:07,252 - tests.orchestrator - INFO - Pod test-pod-3ab2a061 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:57:09,320 - tests.orchestrator - INFO - Pod test-pod-3ab2a061 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:57:11,389 - tests.orchestrator - INFO - Pod test-pod-3ab2a061 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:57:13,457 - tests.orchestrator - INFO - Pod test-pod-3ab2a061 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 18:57:13,457 - tests.orchestrator - INFO - Pod test-pod-3ab2a061 is ready -2025-06-05 18:57:14,458 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 11 times) -2025-06-05 18:57:14,458 - tests.orchestrator - INFO - Testing read/write between pods test-pod-ab6b7029 and test-pod-01b55f76 sharing PVC test-pvc-41d09b77 -2025-06-05 18:57:14,528 - tests.orchestrator - ERROR - Failed in read/write verification: (400) -Reason: Bad Request -HTTP response headers: HTTPHeaderDict({'Audit-Id': '7c8180b4-3675-4d2d-a309-c082223d6092', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 18:57:14 GMT', 'Content-Length': '139'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Upgrade request required","reason":"BadRequest","code":400} - - -2025-06-05 18:57:15,529 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 20 times) -2025-06-05 18:57:15,529 - tests.orchestrator - INFO - Creating pod test-pod-afdd5b8d on PVC test-pvc-29a67853 -2025-06-05 18:57:15,611 - tests.orchestrator - INFO - Created pod: test-pod-afdd5b8d using PVC: test-pvc-29a67853 -2025-06-05 18:57:15,612 - tests.orchestrator - INFO - Waiting for pod test-pod-afdd5b8d to be ready -2025-06-05 18:57:15,680 - tests.orchestrator - INFO - Pod test-pod-afdd5b8d phase: Pending -2025-06-05 18:57:17,749 - tests.orchestrator - INFO - Pod test-pod-afdd5b8d phase: Running -2025-06-05 18:57:17,749 - tests.orchestrator - INFO - Pod test-pod-afdd5b8d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:57:19,817 - tests.orchestrator - INFO - Pod test-pod-afdd5b8d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:57:21,885 - tests.orchestrator - INFO - Pod test-pod-afdd5b8d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:57:23,954 - tests.orchestrator - INFO - Pod test-pod-afdd5b8d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:57:26,022 - tests.orchestrator - INFO - Pod test-pod-afdd5b8d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:57:28,091 - tests.orchestrator - INFO - Pod test-pod-afdd5b8d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:57:30,160 - tests.orchestrator - INFO - Pod test-pod-afdd5b8d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:57:32,235 - tests.orchestrator - INFO - Pod test-pod-afdd5b8d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 18:57:32,235 - tests.orchestrator - INFO - Pod test-pod-afdd5b8d is ready -2025-06-05 18:57:33,235 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 22 times) -2025-06-05 18:57:33,236 - tests.orchestrator - INFO - Deleting pod: test-pod-3ab2a061 from PVC: test-pvc-4184bff1 -2025-06-05 18:57:33,312 - tests.orchestrator - INFO - Waiting for pod test-pod-3ab2a061 to be deleted -2025-06-05 18:58:05,947 - tests.orchestrator - INFO - Pod test-pod-3ab2a061 has been deleted -2025-06-05 18:58:05,947 - tests.orchestrator - INFO - Deleted pod: test-pod-3ab2a061 -2025-06-05 18:58:06,947 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 23 times) -2025-06-05 18:58:06,947 - tests.orchestrator - INFO - Deleting pod: test-pod-670c9b02 from PVC: test-pvc-60b59274 -2025-06-05 18:58:07,022 - tests.orchestrator - INFO - Waiting for pod test-pod-670c9b02 to be deleted -2025-06-05 18:58:39,738 - tests.orchestrator - INFO - Pod test-pod-670c9b02 has been deleted -2025-06-05 18:58:39,738 - tests.orchestrator - INFO - Deleted pod: test-pod-670c9b02 -2025-06-05 18:58:40,738 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 18 times) -2025-06-05 18:58:40,738 - tests.orchestrator - INFO - Creating PVC: test-pvc-fc8a37f7 -2025-06-05 18:58:40,817 - tests.orchestrator - INFO - Created PVC: test-pvc-fc8a37f7 -2025-06-05 18:58:40,817 - tests.orchestrator - INFO - Waiting for PVC test-pvc-fc8a37f7 to be bound -2025-06-05 18:58:42,953 - tests.orchestrator - INFO - PVC test-pvc-fc8a37f7 is bound -2025-06-05 18:58:43,953 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 12 times) -2025-06-05 18:58:43,953 - tests.orchestrator - INFO - Testing read/write between pods test-pod-afdd5b8d and test-pod-dc5857c7 sharing PVC test-pvc-29a67853 -2025-06-05 18:58:44,024 - tests.orchestrator - ERROR - Failed in read/write verification: (400) -Reason: Bad Request -HTTP response headers: HTTPHeaderDict({'Audit-Id': 'b3898348-0d5a-437c-b7e8-9d9ee4242002', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 18:58:43 GMT', 'Content-Length': '139'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Upgrade request required","reason":"BadRequest","code":400} - - -2025-06-05 18:58:45,024 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 24 times) -2025-06-05 18:58:45,024 - tests.orchestrator - INFO - Deleting pod: test-pod-ab6b7029 from PVC: test-pvc-41d09b77 -2025-06-05 18:58:45,100 - tests.orchestrator - INFO - Waiting for pod test-pod-ab6b7029 to be deleted -2025-06-05 18:59:17,791 - tests.orchestrator - INFO - Pod test-pod-ab6b7029 has been deleted -2025-06-05 18:59:17,792 - tests.orchestrator - INFO - Deleted pod: test-pod-ab6b7029 -2025-06-05 18:59:18,792 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 21 times) -2025-06-05 18:59:18,792 - tests.orchestrator - INFO - Creating pod test-pod-9b9f8243 on PVC test-pvc-0bf22cd9 -2025-06-05 18:59:18,884 - tests.orchestrator - INFO - Created pod: test-pod-9b9f8243 using PVC: test-pvc-0bf22cd9 -2025-06-05 18:59:18,884 - tests.orchestrator - INFO - Waiting for pod test-pod-9b9f8243 to be ready -2025-06-05 18:59:18,954 - tests.orchestrator - INFO - Pod test-pod-9b9f8243 phase: Pending -2025-06-05 18:59:23,090 - tests.orchestrator - INFO - Pod test-pod-9b9f8243 phase: Running -2025-06-05 18:59:23,090 - tests.orchestrator - INFO - Pod test-pod-9b9f8243 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:59:25,159 - tests.orchestrator - INFO - Pod test-pod-9b9f8243 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:59:27,226 - tests.orchestrator - INFO - Pod test-pod-9b9f8243 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:59:29,295 - tests.orchestrator - INFO - Pod test-pod-9b9f8243 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:59:31,364 - tests.orchestrator - INFO - Pod test-pod-9b9f8243 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:59:33,432 - tests.orchestrator - INFO - Pod test-pod-9b9f8243 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:59:35,500 - tests.orchestrator - INFO - Pod test-pod-9b9f8243 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:59:37,569 - tests.orchestrator - INFO - Pod test-pod-9b9f8243 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:59:39,637 - tests.orchestrator - INFO - Pod test-pod-9b9f8243 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 18:59:41,704 - tests.orchestrator - INFO - Pod test-pod-9b9f8243 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 18:59:41,705 - tests.orchestrator - INFO - Pod test-pod-9b9f8243 is ready -2025-06-05 18:59:42,705 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 25 times) -2025-06-05 18:59:42,705 - tests.orchestrator - INFO - Deleting pod: test-pod-9b9f8243 from PVC: test-pvc-0bf22cd9 -2025-06-05 18:59:42,781 - tests.orchestrator - INFO - Waiting for pod test-pod-9b9f8243 to be deleted -2025-06-05 19:00:15,395 - tests.orchestrator - INFO - Pod test-pod-9b9f8243 has been deleted -2025-06-05 19:00:15,395 - tests.orchestrator - INFO - Deleted pod: test-pod-9b9f8243 -2025-06-05 19:00:16,397 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 26 times) -2025-06-05 19:00:16,397 - tests.orchestrator - INFO - Deleting pod: test-pod-afdd5b8d from PVC: test-pvc-29a67853 -2025-06-05 19:00:16,479 - tests.orchestrator - INFO - Waiting for pod test-pod-afdd5b8d to be deleted -2025-06-05 19:00:49,098 - tests.orchestrator - INFO - Pod test-pod-afdd5b8d has been deleted -2025-06-05 19:00:49,098 - tests.orchestrator - INFO - Deleted pod: test-pod-afdd5b8d -2025-06-05 19:00:50,098 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 13 times) -2025-06-05 19:00:50,098 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test -2025-06-05 19:00:51,098 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 14 times) -2025-06-05 19:00:51,098 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test -2025-06-05 19:00:52,099 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 19 times) -2025-06-05 19:00:52,099 - tests.orchestrator - INFO - Creating PVC: test-pvc-7ae6d5e7 -2025-06-05 19:00:52,178 - tests.orchestrator - INFO - Created PVC: test-pvc-7ae6d5e7 -2025-06-05 19:00:52,178 - tests.orchestrator - INFO - Waiting for PVC test-pvc-7ae6d5e7 to be bound -2025-06-05 19:00:54,312 - tests.orchestrator - INFO - PVC test-pvc-7ae6d5e7 is bound -2025-06-05 19:00:55,312 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 15 times) -2025-06-05 19:00:55,312 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test -2025-06-05 19:00:56,313 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 27 times) -2025-06-05 19:00:56,313 - tests.orchestrator - INFO - Deleting pod: test-pod-01b55f76 from PVC: test-pvc-41d09b77 -2025-06-05 19:00:56,387 - tests.orchestrator - INFO - Waiting for pod test-pod-01b55f76 to be deleted -2025-06-05 19:01:28,999 - tests.orchestrator - INFO - Pod test-pod-01b55f76 has been deleted -2025-06-05 19:01:28,999 - tests.orchestrator - INFO - Deleted pod: test-pod-01b55f76 -2025-06-05 19:01:29,999 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 16 times) -2025-06-05 19:01:29,999 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test -2025-06-05 19:01:30,999 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 20 times) -2025-06-05 19:01:30,999 - tests.orchestrator - INFO - Creating PVC: test-pvc-c196bb79 -2025-06-05 19:01:31,077 - tests.orchestrator - INFO - Created PVC: test-pvc-c196bb79 -2025-06-05 19:01:31,077 - tests.orchestrator - INFO - Waiting for PVC test-pvc-c196bb79 to be bound -2025-06-05 19:01:33,210 - tests.orchestrator - INFO - PVC test-pvc-c196bb79 is bound -2025-06-05 19:01:34,210 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 21 times) -2025-06-05 19:01:34,210 - tests.orchestrator - INFO - Creating PVC: test-pvc-d0c53501 -2025-06-05 19:01:34,279 - tests.orchestrator - INFO - Created PVC: test-pvc-d0c53501 -2025-06-05 19:01:34,279 - tests.orchestrator - INFO - Waiting for PVC test-pvc-d0c53501 to be bound -2025-06-05 19:01:36,412 - tests.orchestrator - INFO - PVC test-pvc-d0c53501 is bound -2025-06-05 19:01:37,412 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 28 times) -2025-06-05 19:01:37,412 - tests.orchestrator - INFO - Deleting pod: test-pod-dc5857c7 from PVC: test-pvc-29a67853 -2025-06-05 19:01:37,488 - tests.orchestrator - INFO - Waiting for pod test-pod-dc5857c7 to be deleted -2025-06-05 19:02:10,113 - tests.orchestrator - INFO - Pod test-pod-dc5857c7 has been deleted -2025-06-05 19:02:10,113 - tests.orchestrator - INFO - Deleted pod: test-pod-dc5857c7 -2025-06-05 19:02:11,113 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 22 times) -2025-06-05 19:02:11,113 - tests.orchestrator - INFO - Creating PVC: test-pvc-9bad69f7 -2025-06-05 19:02:11,192 - tests.orchestrator - INFO - Created PVC: test-pvc-9bad69f7 -2025-06-05 19:02:11,192 - tests.orchestrator - INFO - Waiting for PVC test-pvc-9bad69f7 to be bound -2025-06-05 19:02:13,324 - tests.orchestrator - INFO - PVC test-pvc-9bad69f7 is bound -2025-06-05 19:02:14,324 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 23 times) -2025-06-05 19:02:14,324 - tests.orchestrator - INFO - Creating PVC: test-pvc-3d16a983 -2025-06-05 19:02:14,394 - tests.orchestrator - INFO - Created PVC: test-pvc-3d16a983 -2025-06-05 19:02:14,394 - tests.orchestrator - INFO - Waiting for PVC test-pvc-3d16a983 to be bound -2025-06-05 19:02:16,527 - tests.orchestrator - INFO - PVC test-pvc-3d16a983 is bound -2025-06-05 19:02:17,528 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 11 times) -2025-06-05 19:02:17,528 - tests.orchestrator - INFO - Deleting PVC: test-pvc-54d1de91 -2025-06-05 19:02:17,603 - tests.orchestrator - INFO - Waiting for PVC test-pvc-54d1de91 to be deleted -2025-06-05 19:02:21,735 - tests.orchestrator - INFO - PVC test-pvc-54d1de91 has been deleted -2025-06-05 19:02:21,735 - tests.orchestrator - INFO - Deleted PVC: test-pvc-54d1de91 -2025-06-05 19:02:22,736 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 22 times) -2025-06-05 19:02:22,736 - tests.orchestrator - INFO - Creating pod test-pod-b11af190 on PVC test-pvc-d0c53501 -2025-06-05 19:02:22,817 - tests.orchestrator - INFO - Created pod: test-pod-b11af190 using PVC: test-pvc-d0c53501 -2025-06-05 19:02:22,818 - tests.orchestrator - INFO - Waiting for pod test-pod-b11af190 to be ready -2025-06-05 19:02:22,885 - tests.orchestrator - INFO - Pod test-pod-b11af190 phase: Pending -2025-06-05 19:02:24,955 - tests.orchestrator - INFO - Pod test-pod-b11af190 phase: Running -2025-06-05 19:02:24,955 - tests.orchestrator - INFO - Pod test-pod-b11af190 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:02:27,024 - tests.orchestrator - INFO - Pod test-pod-b11af190 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:02:29,093 - tests.orchestrator - INFO - Pod test-pod-b11af190 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:02:31,161 - tests.orchestrator - INFO - Pod test-pod-b11af190 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:02:33,230 - tests.orchestrator - INFO - Pod test-pod-b11af190 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:02:35,299 - tests.orchestrator - INFO - Pod test-pod-b11af190 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:02:37,368 - tests.orchestrator - INFO - Pod test-pod-b11af190 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:02:39,437 - tests.orchestrator - INFO - Pod test-pod-b11af190 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:02:41,507 - tests.orchestrator - INFO - Pod test-pod-b11af190 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:02:43,575 - tests.orchestrator - INFO - Pod test-pod-b11af190 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:02:45,644 - tests.orchestrator - INFO - Pod test-pod-b11af190 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 19:02:45,644 - tests.orchestrator - INFO - Pod test-pod-b11af190 is ready -2025-06-05 19:02:46,646 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 29 times) -2025-06-05 19:02:46,646 - tests.orchestrator - INFO - Deleting pod: test-pod-b11af190 from PVC: test-pvc-d0c53501 -2025-06-05 19:02:46,725 - tests.orchestrator - INFO - Waiting for pod test-pod-b11af190 to be deleted -2025-06-05 19:03:19,340 - tests.orchestrator - INFO - Pod test-pod-b11af190 has been deleted -2025-06-05 19:03:19,341 - tests.orchestrator - INFO - Deleted pod: test-pod-b11af190 -2025-06-05 19:03:20,341 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 23 times) -2025-06-05 19:03:20,341 - tests.orchestrator - INFO - Creating pod test-pod-02a62d88 on PVC test-pvc-9bad69f7 -2025-06-05 19:03:20,425 - tests.orchestrator - INFO - Created pod: test-pod-02a62d88 using PVC: test-pvc-9bad69f7 -2025-06-05 19:03:20,425 - tests.orchestrator - INFO - Waiting for pod test-pod-02a62d88 to be ready -2025-06-05 19:03:20,492 - tests.orchestrator - INFO - Pod test-pod-02a62d88 phase: Pending -2025-06-05 19:03:22,560 - tests.orchestrator - INFO - Pod test-pod-02a62d88 phase: Running -2025-06-05 19:03:22,560 - tests.orchestrator - INFO - Pod test-pod-02a62d88 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:03:24,629 - tests.orchestrator - INFO - Pod test-pod-02a62d88 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:03:26,697 - tests.orchestrator - INFO - Pod test-pod-02a62d88 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:03:28,765 - tests.orchestrator - INFO - Pod test-pod-02a62d88 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:03:30,834 - tests.orchestrator - INFO - Pod test-pod-02a62d88 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:03:32,902 - tests.orchestrator - INFO - Pod test-pod-02a62d88 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:03:34,970 - tests.orchestrator - INFO - Pod test-pod-02a62d88 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:03:37,041 - tests.orchestrator - INFO - Pod test-pod-02a62d88 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 19:03:37,041 - tests.orchestrator - INFO - Pod test-pod-02a62d88 is ready -2025-06-05 19:03:38,042 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 30 times) -2025-06-05 19:03:38,042 - tests.orchestrator - INFO - Deleting pod: test-pod-02a62d88 from PVC: test-pvc-9bad69f7 -2025-06-05 19:03:38,117 - tests.orchestrator - INFO - Waiting for pod test-pod-02a62d88 to be deleted -2025-06-05 19:04:10,749 - tests.orchestrator - INFO - Pod test-pod-02a62d88 has been deleted -2025-06-05 19:04:10,749 - tests.orchestrator - INFO - Deleted pod: test-pod-02a62d88 -2025-06-05 19:04:11,749 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 12 times) -2025-06-05 19:04:11,749 - tests.orchestrator - INFO - Deleting PVC: test-pvc-fc8a37f7 -2025-06-05 19:04:11,822 - tests.orchestrator - INFO - Waiting for PVC test-pvc-fc8a37f7 to be deleted -2025-06-05 19:04:15,956 - tests.orchestrator - INFO - PVC test-pvc-fc8a37f7 has been deleted -2025-06-05 19:04:15,956 - tests.orchestrator - INFO - Deleted PVC: test-pvc-fc8a37f7 -2025-06-05 19:04:16,957 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 17 times) -2025-06-05 19:04:16,957 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test -2025-06-05 19:04:17,957 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 31 times) -2025-06-05 19:04:17,957 - tests.orchestrator - INFO - No pods to delete -2025-06-05 19:04:18,958 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 18 times) -2025-06-05 19:04:18,959 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test -2025-06-05 19:04:19,959 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 24 times) -2025-06-05 19:04:19,959 - tests.orchestrator - INFO - Creating pod test-pod-fbf9cd30 on PVC test-pvc-29a67853 -2025-06-05 19:04:20,042 - tests.orchestrator - INFO - Created pod: test-pod-fbf9cd30 using PVC: test-pvc-29a67853 -2025-06-05 19:04:20,042 - tests.orchestrator - INFO - Waiting for pod test-pod-fbf9cd30 to be ready -2025-06-05 19:04:20,109 - tests.orchestrator - INFO - Pod test-pod-fbf9cd30 phase: Pending -2025-06-05 19:04:22,177 - tests.orchestrator - INFO - Pod test-pod-fbf9cd30 phase: Running -2025-06-05 19:04:22,177 - tests.orchestrator - INFO - Pod test-pod-fbf9cd30 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:04:24,246 - tests.orchestrator - INFO - Pod test-pod-fbf9cd30 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:04:26,314 - tests.orchestrator - INFO - Pod test-pod-fbf9cd30 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:04:28,382 - tests.orchestrator - INFO - Pod test-pod-fbf9cd30 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:04:30,451 - tests.orchestrator - INFO - Pod test-pod-fbf9cd30 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:04:32,518 - tests.orchestrator - INFO - Pod test-pod-fbf9cd30 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:04:34,586 - tests.orchestrator - INFO - Pod test-pod-fbf9cd30 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:04:36,656 - tests.orchestrator - INFO - Pod test-pod-fbf9cd30 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:04:38,723 - tests.orchestrator - INFO - Pod test-pod-fbf9cd30 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:04:40,791 - tests.orchestrator - INFO - Pod test-pod-fbf9cd30 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 19:04:40,791 - tests.orchestrator - INFO - Pod test-pod-fbf9cd30 is ready -2025-06-05 19:04:41,792 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 24 times) -2025-06-05 19:04:41,792 - tests.orchestrator - INFO - Creating PVC: test-pvc-c1c2eedb -2025-06-05 19:04:41,862 - tests.orchestrator - INFO - Created PVC: test-pvc-c1c2eedb -2025-06-05 19:04:41,863 - tests.orchestrator - INFO - Waiting for PVC test-pvc-c1c2eedb to be bound -2025-06-05 19:04:43,996 - tests.orchestrator - INFO - PVC test-pvc-c1c2eedb is bound -2025-06-05 19:04:44,997 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 32 times) -2025-06-05 19:04:44,997 - tests.orchestrator - INFO - Deleting pod: test-pod-fbf9cd30 from PVC: test-pvc-29a67853 -2025-06-05 19:04:45,072 - tests.orchestrator - INFO - Waiting for pod test-pod-fbf9cd30 to be deleted -2025-06-05 19:05:17,689 - tests.orchestrator - INFO - Pod test-pod-fbf9cd30 has been deleted -2025-06-05 19:05:17,689 - tests.orchestrator - INFO - Deleted pod: test-pod-fbf9cd30 -2025-06-05 19:05:18,689 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 25 times) -2025-06-05 19:05:18,689 - tests.orchestrator - INFO - Creating PVC: test-pvc-2ddd2ed2 -2025-06-05 19:05:18,768 - tests.orchestrator - INFO - Created PVC: test-pvc-2ddd2ed2 -2025-06-05 19:05:18,768 - tests.orchestrator - INFO - Waiting for PVC test-pvc-2ddd2ed2 to be bound -2025-06-05 19:05:20,901 - tests.orchestrator - INFO - PVC test-pvc-2ddd2ed2 is bound -2025-06-05 19:05:21,902 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 19 times) -2025-06-05 19:05:21,902 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test -2025-06-05 19:05:22,902 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 25 times) -2025-06-05 19:05:22,902 - tests.orchestrator - INFO - Creating pod test-pod-7012ed49 on PVC test-pvc-0bf22cd9 -2025-06-05 19:05:22,978 - tests.orchestrator - INFO - Created pod: test-pod-7012ed49 using PVC: test-pvc-0bf22cd9 -2025-06-05 19:05:22,978 - tests.orchestrator - INFO - Waiting for pod test-pod-7012ed49 to be ready -2025-06-05 19:05:23,047 - tests.orchestrator - INFO - Pod test-pod-7012ed49 phase: Pending -2025-06-05 19:05:27,183 - tests.orchestrator - INFO - Pod test-pod-7012ed49 phase: Running -2025-06-05 19:05:27,183 - tests.orchestrator - INFO - Pod test-pod-7012ed49 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:05:29,252 - tests.orchestrator - INFO - Pod test-pod-7012ed49 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:05:31,319 - tests.orchestrator - INFO - Pod test-pod-7012ed49 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:05:33,388 - tests.orchestrator - INFO - Pod test-pod-7012ed49 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:05:35,456 - tests.orchestrator - INFO - Pod test-pod-7012ed49 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:05:37,524 - tests.orchestrator - INFO - Pod test-pod-7012ed49 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:05:40,341 - tests.orchestrator - INFO - Pod test-pod-7012ed49 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:05:42,410 - tests.orchestrator - INFO - Pod test-pod-7012ed49 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:05:44,479 - tests.orchestrator - INFO - Pod test-pod-7012ed49 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 19:05:44,479 - tests.orchestrator - INFO - Pod test-pod-7012ed49 is ready -2025-06-05 19:05:45,479 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 20 times) -2025-06-05 19:05:45,479 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test -2025-06-05 19:05:46,480 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 33 times) -2025-06-05 19:05:46,480 - tests.orchestrator - INFO - Deleting pod: test-pod-7012ed49 from PVC: test-pvc-0bf22cd9 -2025-06-05 19:05:46,561 - tests.orchestrator - INFO - Waiting for pod test-pod-7012ed49 to be deleted -2025-06-05 19:06:19,176 - tests.orchestrator - INFO - Pod test-pod-7012ed49 has been deleted -2025-06-05 19:06:19,176 - tests.orchestrator - INFO - Deleted pod: test-pod-7012ed49 -2025-06-05 19:06:20,176 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 13 times) -2025-06-05 19:06:20,177 - tests.orchestrator - INFO - Deleting PVC: test-pvc-9bad69f7 -2025-06-05 19:06:20,249 - tests.orchestrator - INFO - Waiting for PVC test-pvc-9bad69f7 to be deleted -2025-06-05 19:06:20,316 - tests.orchestrator - INFO - PVC test-pvc-9bad69f7 has been deleted -2025-06-05 19:06:20,316 - tests.orchestrator - INFO - Deleted PVC: test-pvc-9bad69f7 -2025-06-05 19:06:21,316 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 34 times) -2025-06-05 19:06:21,317 - tests.orchestrator - INFO - No pods to delete -2025-06-05 19:06:22,317 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 26 times) -2025-06-05 19:06:22,317 - tests.orchestrator - INFO - Creating pod test-pod-d45ef83e on PVC test-pvc-6e8e6345 -2025-06-05 19:06:22,400 - tests.orchestrator - INFO - Created pod: test-pod-d45ef83e using PVC: test-pvc-6e8e6345 -2025-06-05 19:06:22,400 - tests.orchestrator - INFO - Waiting for pod test-pod-d45ef83e to be ready -2025-06-05 19:06:22,468 - tests.orchestrator - INFO - Pod test-pod-d45ef83e phase: Pending -2025-06-05 19:06:24,537 - tests.orchestrator - INFO - Pod test-pod-d45ef83e phase: Running -2025-06-05 19:06:24,538 - tests.orchestrator - INFO - Pod test-pod-d45ef83e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:06:26,611 - tests.orchestrator - INFO - Pod test-pod-d45ef83e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:06:28,680 - tests.orchestrator - INFO - Pod test-pod-d45ef83e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:06:30,748 - tests.orchestrator - INFO - Pod test-pod-d45ef83e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:06:32,816 - tests.orchestrator - INFO - Pod test-pod-d45ef83e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:06:34,885 - tests.orchestrator - INFO - Pod test-pod-d45ef83e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:06:36,953 - tests.orchestrator - INFO - Pod test-pod-d45ef83e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:06:39,022 - tests.orchestrator - INFO - Pod test-pod-d45ef83e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:06:41,091 - tests.orchestrator - INFO - Pod test-pod-d45ef83e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:06:43,158 - tests.orchestrator - INFO - Pod test-pod-d45ef83e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:06:45,226 - tests.orchestrator - INFO - Pod test-pod-d45ef83e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 19:06:45,226 - tests.orchestrator - INFO - Pod test-pod-d45ef83e is ready -2025-06-05 19:06:46,226 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 26 times) -2025-06-05 19:06:46,228 - tests.orchestrator - INFO - Creating PVC: test-pvc-1bb798a3 -2025-06-05 19:06:46,297 - tests.orchestrator - INFO - Created PVC: test-pvc-1bb798a3 -2025-06-05 19:06:46,297 - tests.orchestrator - INFO - Waiting for PVC test-pvc-1bb798a3 to be bound -2025-06-05 19:06:48,430 - tests.orchestrator - INFO - PVC test-pvc-1bb798a3 is bound -2025-06-05 19:06:49,430 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 27 times) -2025-06-05 19:06:49,431 - tests.orchestrator - INFO - Creating PVC: test-pvc-36370699 -2025-06-05 19:06:49,499 - tests.orchestrator - INFO - Created PVC: test-pvc-36370699 -2025-06-05 19:06:49,499 - tests.orchestrator - INFO - Waiting for PVC test-pvc-36370699 to be bound -2025-06-05 19:06:51,633 - tests.orchestrator - INFO - PVC test-pvc-36370699 is bound -2025-06-05 19:06:52,633 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 35 times) -2025-06-05 19:06:52,633 - tests.orchestrator - INFO - Deleting pod: test-pod-d45ef83e from PVC: test-pvc-6e8e6345 -2025-06-05 19:06:52,708 - tests.orchestrator - INFO - Waiting for pod test-pod-d45ef83e to be deleted -2025-06-05 19:07:25,324 - tests.orchestrator - INFO - Pod test-pod-d45ef83e has been deleted -2025-06-05 19:07:25,324 - tests.orchestrator - INFO - Deleted pod: test-pod-d45ef83e -2025-06-05 19:07:26,324 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 36 times) -2025-06-05 19:07:26,324 - tests.orchestrator - INFO - No pods to delete -2025-06-05 19:07:27,324 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 28 times) -2025-06-05 19:07:27,325 - tests.orchestrator - INFO - Creating PVC: test-pvc-faea1890 -2025-06-05 19:07:27,403 - tests.orchestrator - INFO - Created PVC: test-pvc-faea1890 -2025-06-05 19:07:27,403 - tests.orchestrator - INFO - Waiting for PVC test-pvc-faea1890 to be bound -2025-06-05 19:07:29,537 - tests.orchestrator - INFO - PVC test-pvc-faea1890 is bound -2025-06-05 19:07:30,537 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 21 times) -2025-06-05 19:07:30,538 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test -2025-06-05 19:07:31,538 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 37 times) -2025-06-05 19:07:31,538 - tests.orchestrator - INFO - No pods to delete -2025-06-05 19:07:32,538 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 22 times) -2025-06-05 19:07:32,538 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test -2025-06-05 19:07:33,539 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 29 times) -2025-06-05 19:07:33,539 - tests.orchestrator - INFO - Creating PVC: test-pvc-c6976eae -2025-06-05 19:07:33,607 - tests.orchestrator - INFO - Created PVC: test-pvc-c6976eae -2025-06-05 19:07:33,608 - tests.orchestrator - INFO - Waiting for PVC test-pvc-c6976eae to be bound -2025-06-05 19:07:35,740 - tests.orchestrator - INFO - PVC test-pvc-c6976eae is bound -2025-06-05 19:07:36,740 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 23 times) -2025-06-05 19:07:36,741 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test -2025-06-05 19:07:37,741 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 27 times) -2025-06-05 19:07:37,741 - tests.orchestrator - INFO - Creating pod test-pod-bb3a758b on PVC test-pvc-faea1890 -2025-06-05 19:07:37,833 - tests.orchestrator - INFO - Created pod: test-pod-bb3a758b using PVC: test-pvc-faea1890 -2025-06-05 19:07:37,833 - tests.orchestrator - INFO - Waiting for pod test-pod-bb3a758b to be ready -2025-06-05 19:07:37,904 - tests.orchestrator - INFO - Pod test-pod-bb3a758b phase: Pending -2025-06-05 19:07:42,041 - tests.orchestrator - INFO - Pod test-pod-bb3a758b phase: Running -2025-06-05 19:07:42,041 - tests.orchestrator - INFO - Pod test-pod-bb3a758b conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:07:44,110 - tests.orchestrator - INFO - Pod test-pod-bb3a758b conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:07:46,178 - tests.orchestrator - INFO - Pod test-pod-bb3a758b conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:07:48,247 - tests.orchestrator - INFO - Pod test-pod-bb3a758b conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:07:50,316 - tests.orchestrator - INFO - Pod test-pod-bb3a758b conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:07:52,383 - tests.orchestrator - INFO - Pod test-pod-bb3a758b conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:07:54,451 - tests.orchestrator - INFO - Pod test-pod-bb3a758b conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:07:56,524 - tests.orchestrator - INFO - Pod test-pod-bb3a758b conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:07:58,591 - tests.orchestrator - INFO - Pod test-pod-bb3a758b conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:08:00,658 - tests.orchestrator - INFO - Pod test-pod-bb3a758b conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 19:08:00,659 - tests.orchestrator - INFO - Pod test-pod-bb3a758b is ready -2025-06-05 19:08:01,659 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 24 times) -2025-06-05 19:08:01,659 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test -2025-06-05 19:08:02,659 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 38 times) -2025-06-05 19:08:02,659 - tests.orchestrator - INFO - Deleting pod: test-pod-bb3a758b from PVC: test-pvc-faea1890 -2025-06-05 19:08:02,736 - tests.orchestrator - INFO - Waiting for pod test-pod-bb3a758b to be deleted -2025-06-05 19:08:35,351 - tests.orchestrator - INFO - Pod test-pod-bb3a758b has been deleted -2025-06-05 19:08:35,352 - tests.orchestrator - INFO - Deleted pod: test-pod-bb3a758b -2025-06-05 19:08:36,352 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 39 times) -2025-06-05 19:08:36,352 - tests.orchestrator - INFO - No pods to delete -2025-06-05 19:08:37,352 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 14 times) -2025-06-05 19:08:37,352 - tests.orchestrator - INFO - Deleting PVC: test-pvc-2ddd2ed2 -2025-06-05 19:08:37,424 - tests.orchestrator - INFO - Waiting for PVC test-pvc-2ddd2ed2 to be deleted -2025-06-05 19:08:37,490 - tests.orchestrator - INFO - PVC test-pvc-2ddd2ed2 has been deleted -2025-06-05 19:08:37,490 - tests.orchestrator - INFO - Deleted PVC: test-pvc-2ddd2ed2 -2025-06-05 19:08:38,490 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 30 times) -2025-06-05 19:08:38,491 - tests.orchestrator - INFO - Creating PVC: test-pvc-fe0fe649 -2025-06-05 19:08:38,567 - tests.orchestrator - INFO - Created PVC: test-pvc-fe0fe649 -2025-06-05 19:08:38,567 - tests.orchestrator - INFO - Waiting for PVC test-pvc-fe0fe649 to be bound -2025-06-05 19:08:40,700 - tests.orchestrator - INFO - PVC test-pvc-fe0fe649 is bound diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_191215.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_191215.log deleted file mode 100644 index 40bba3461..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_191215.log +++ /dev/null @@ -1,158 +0,0 @@ -2025-06-05 19:12:15,577 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 19:12:15,781 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 19:12:15,782 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-05 19:12:15,974 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 19:12:15,974 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 19:12:15,974 - tests.orchestrator - INFO - Creating PVC: test-pvc-0bb0d20c -2025-06-05 19:12:16,053 - tests.orchestrator - INFO - Created PVC: test-pvc-0bb0d20c -2025-06-05 19:12:16,053 - tests.orchestrator - INFO - Waiting for PVC test-pvc-0bb0d20c to be bound -2025-06-05 19:12:18,183 - tests.orchestrator - INFO - PVC test-pvc-0bb0d20c is bound -2025-06-05 19:12:18,183 - tests.orchestrator - INFO - Creating PVC: test-pvc-5076c293 -2025-06-05 19:12:18,253 - tests.orchestrator - INFO - Created PVC: test-pvc-5076c293 -2025-06-05 19:12:18,253 - tests.orchestrator - INFO - Waiting for PVC test-pvc-5076c293 to be bound -2025-06-05 19:12:20,383 - tests.orchestrator - INFO - PVC test-pvc-5076c293 is bound -2025-06-05 19:12:20,383 - tests.orchestrator - INFO - Creating pod test-pod-fd8d328f on PVC test-pvc-5076c293 -2025-06-05 19:12:20,469 - tests.orchestrator - INFO - Created pod: test-pod-fd8d328f using PVC: test-pvc-5076c293 -2025-06-05 19:12:20,469 - tests.orchestrator - INFO - Waiting for pod test-pod-fd8d328f to be ready -2025-06-05 19:12:20,535 - tests.orchestrator - INFO - Pod test-pod-fd8d328f phase: Pending -2025-06-05 19:12:22,606 - tests.orchestrator - INFO - Pod test-pod-fd8d328f phase: Running -2025-06-05 19:12:22,606 - tests.orchestrator - INFO - Pod test-pod-fd8d328f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:12:24,675 - tests.orchestrator - INFO - Pod test-pod-fd8d328f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:12:26,743 - tests.orchestrator - INFO - Pod test-pod-fd8d328f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:12:28,812 - tests.orchestrator - INFO - Pod test-pod-fd8d328f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:12:30,880 - tests.orchestrator - INFO - Pod test-pod-fd8d328f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:12:32,948 - tests.orchestrator - INFO - Pod test-pod-fd8d328f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:12:35,017 - tests.orchestrator - INFO - Pod test-pod-fd8d328f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:12:37,085 - tests.orchestrator - INFO - Pod test-pod-fd8d328f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 19:12:37,085 - tests.orchestrator - INFO - Pod test-pod-fd8d328f is ready -2025-06-05 19:12:37,085 - tests.orchestrator - INFO - Creating pod test-pod-a2dd05ca on PVC test-pvc-0bb0d20c -2025-06-05 19:12:37,162 - tests.orchestrator - INFO - Created pod: test-pod-a2dd05ca using PVC: test-pvc-0bb0d20c -2025-06-05 19:12:37,162 - tests.orchestrator - INFO - Waiting for pod test-pod-a2dd05ca to be ready -2025-06-05 19:12:37,229 - tests.orchestrator - INFO - Pod test-pod-a2dd05ca phase: Pending -2025-06-05 19:12:39,298 - tests.orchestrator - INFO - Pod test-pod-a2dd05ca phase: Running -2025-06-05 19:12:39,298 - tests.orchestrator - INFO - Pod test-pod-a2dd05ca conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:12:41,367 - tests.orchestrator - INFO - Pod test-pod-a2dd05ca conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:12:43,436 - tests.orchestrator - INFO - Pod test-pod-a2dd05ca conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:12:45,505 - tests.orchestrator - INFO - Pod test-pod-a2dd05ca conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:12:47,574 - tests.orchestrator - INFO - Pod test-pod-a2dd05ca conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:12:49,643 - tests.orchestrator - INFO - Pod test-pod-a2dd05ca conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:12:51,713 - tests.orchestrator - INFO - Pod test-pod-a2dd05ca conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:12:53,781 - tests.orchestrator - INFO - Pod test-pod-a2dd05ca conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 19:12:53,782 - tests.orchestrator - INFO - Pod test-pod-a2dd05ca is ready -2025-06-05 19:12:53,782 - tests.orchestrator - INFO - Creating pod test-pod-284d58ca on PVC test-pvc-5076c293 -2025-06-05 19:12:53,869 - tests.orchestrator - INFO - Created pod: test-pod-284d58ca using PVC: test-pvc-5076c293 -2025-06-05 19:12:53,869 - tests.orchestrator - INFO - Waiting for pod test-pod-284d58ca to be ready -2025-06-05 19:12:53,938 - tests.orchestrator - INFO - Pod test-pod-284d58ca phase: Pending -2025-06-05 19:12:58,073 - tests.orchestrator - INFO - Pod test-pod-284d58ca phase: Running -2025-06-05 19:12:58,073 - tests.orchestrator - INFO - Pod test-pod-284d58ca conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:13:00,143 - tests.orchestrator - INFO - Pod test-pod-284d58ca conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:13:02,214 - tests.orchestrator - INFO - Pod test-pod-284d58ca conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:13:04,282 - tests.orchestrator - INFO - Pod test-pod-284d58ca conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:13:06,351 - tests.orchestrator - INFO - Pod test-pod-284d58ca conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:13:08,422 - tests.orchestrator - INFO - Pod test-pod-284d58ca conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:13:10,488 - tests.orchestrator - INFO - Pod test-pod-284d58ca conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:13:12,559 - tests.orchestrator - INFO - Pod test-pod-284d58ca conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:13:14,628 - tests.orchestrator - INFO - Pod test-pod-284d58ca conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:13:16,697 - tests.orchestrator - INFO - Pod test-pod-284d58ca conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 19:13:16,697 - tests.orchestrator - INFO - Pod test-pod-284d58ca is ready -2025-06-05 19:13:16,697 - tests.orchestrator - INFO - Testing read/write between pods test-pod-284d58ca and test-pod-fd8d328f sharing PVC test-pvc-5076c293 -2025-06-05 19:13:16,697 - tests.orchestrator - INFO - Executing write command: kubectl exec -n efs-stress-test test-pod-284d58ca -- /bin/sh -c 'echo "Test content: e2446e14-a753-4aa8-80af-e120861c76a8" > /data/test-421a110b.txt' -2025-06-05 19:13:17,954 - tests.orchestrator - INFO - Pod test-pod-284d58ca wrote to /data/test-421a110b.txt -2025-06-05 19:13:19,954 - tests.orchestrator - INFO - Executing read command: kubectl exec -n efs-stress-test test-pod-fd8d328f -- cat /data/test-421a110b.txt -2025-06-05 19:13:21,152 - tests.orchestrator - INFO - Read result: 'Test content: e2446e14-a753-4aa8-80af-e120861c76a8' -2025-06-05 19:13:21,152 - tests.orchestrator - INFO - Pod test-pod-fd8d328f successfully read content written by test-pod-284d58ca -2025-06-05 19:13:21,153 - tests.orchestrator - INFO - Deleting pod: test-pod-284d58ca from PVC: test-pvc-5076c293 -2025-06-05 19:13:21,229 - tests.orchestrator - INFO - Waiting for pod test-pod-284d58ca to be deleted -2025-06-05 19:13:53,872 - tests.orchestrator - INFO - Pod test-pod-284d58ca has been deleted -2025-06-05 19:13:53,872 - tests.orchestrator - INFO - Deleted pod: test-pod-284d58ca -2025-06-05 19:13:53,872 - tests.orchestrator - INFO - Deleting PVC: test-pvc-5076c293 -2025-06-05 19:13:53,872 - tests.orchestrator - INFO - Deleting 1 pods using PVC test-pvc-5076c293 -2025-06-05 19:13:53,872 - tests.orchestrator - INFO - Deleting pod: test-pod-fd8d328f from PVC: test-pvc-5076c293 -2025-06-05 19:13:53,950 - tests.orchestrator - INFO - Waiting for pod test-pod-fd8d328f to be deleted -2025-06-05 19:14:26,581 - tests.orchestrator - INFO - Pod test-pod-fd8d328f has been deleted -2025-06-05 19:14:26,581 - tests.orchestrator - INFO - Deleted pod: test-pod-fd8d328f -2025-06-05 19:14:26,658 - tests.orchestrator - INFO - Waiting for PVC test-pvc-5076c293 to be deleted -2025-06-05 19:14:30,797 - tests.orchestrator - INFO - PVC test-pvc-5076c293 has been deleted -2025-06-05 19:14:30,797 - tests.orchestrator - INFO - Deleted PVC: test-pvc-5076c293 -2025-06-05 19:14:30,797 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations -2025-06-05 19:14:30,797 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 1 times) -2025-06-05 19:14:30,797 - tests.orchestrator - INFO - Deleting pod: test-pod-a2dd05ca from PVC: test-pvc-0bb0d20c -2025-06-05 19:14:30,876 - tests.orchestrator - INFO - Waiting for pod test-pod-a2dd05ca to be deleted -2025-06-05 19:15:03,503 - tests.orchestrator - INFO - Pod test-pod-a2dd05ca has been deleted -2025-06-05 19:15:03,503 - tests.orchestrator - INFO - Deleted pod: test-pod-a2dd05ca -2025-06-05 19:15:04,503 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 1 times) -2025-06-05 19:15:04,504 - tests.orchestrator - INFO - Creating PVC: test-pvc-2127e71a -2025-06-05 19:15:04,583 - tests.orchestrator - INFO - Created PVC: test-pvc-2127e71a -2025-06-05 19:15:04,583 - tests.orchestrator - INFO - Waiting for PVC test-pvc-2127e71a to be bound -2025-06-05 19:15:06,716 - tests.orchestrator - INFO - PVC test-pvc-2127e71a is bound -2025-06-05 19:15:07,716 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 2 times) -2025-06-05 19:15:07,716 - tests.orchestrator - INFO - No pods to delete -2025-06-05 19:15:08,716 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 2 times) -2025-06-05 19:15:08,717 - tests.orchestrator - INFO - Creating PVC: test-pvc-a0df5547 -2025-06-05 19:15:08,786 - tests.orchestrator - INFO - Created PVC: test-pvc-a0df5547 -2025-06-05 19:15:08,786 - tests.orchestrator - INFO - Waiting for PVC test-pvc-a0df5547 to be bound -2025-06-05 19:15:10,934 - tests.orchestrator - INFO - PVC test-pvc-a0df5547 is bound -2025-06-05 19:15:11,934 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 1 times) -2025-06-05 19:15:11,935 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test -2025-06-05 19:15:12,935 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 3 times) -2025-06-05 19:15:12,935 - tests.orchestrator - INFO - Creating PVC: test-pvc-ecb12f74 -2025-06-05 19:15:13,010 - tests.orchestrator - INFO - Created PVC: test-pvc-ecb12f74 -2025-06-05 19:15:13,010 - tests.orchestrator - INFO - Waiting for PVC test-pvc-ecb12f74 to be bound -2025-06-05 19:15:15,146 - tests.orchestrator - INFO - PVC test-pvc-ecb12f74 is bound -2025-06-05 19:15:16,146 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 4 times) -2025-06-05 19:15:16,146 - tests.orchestrator - INFO - Creating PVC: test-pvc-31fbd501 -2025-06-05 19:15:16,214 - tests.orchestrator - INFO - Created PVC: test-pvc-31fbd501 -2025-06-05 19:15:16,214 - tests.orchestrator - INFO - Waiting for PVC test-pvc-31fbd501 to be bound -2025-06-05 19:15:18,349 - tests.orchestrator - INFO - PVC test-pvc-31fbd501 is bound -2025-06-05 19:15:19,349 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 5 times) -2025-06-05 19:15:19,349 - tests.orchestrator - INFO - Creating PVC: test-pvc-89991955 -2025-06-05 19:15:19,416 - tests.orchestrator - INFO - Created PVC: test-pvc-89991955 -2025-06-05 19:15:19,416 - tests.orchestrator - INFO - Waiting for PVC test-pvc-89991955 to be bound -2025-06-05 19:15:21,555 - tests.orchestrator - INFO - PVC test-pvc-89991955 is bound -2025-06-05 19:15:22,555 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 2 times) -2025-06-05 19:15:22,556 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test -2025-06-05 19:15:23,556 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 1 times) -2025-06-05 19:15:23,556 - tests.orchestrator - INFO - Deleting PVC: test-pvc-89991955 -2025-06-05 19:15:23,631 - tests.orchestrator - INFO - Waiting for PVC test-pvc-89991955 to be deleted -2025-06-05 19:15:23,698 - tests.orchestrator - INFO - PVC test-pvc-89991955 has been deleted -2025-06-05 19:15:23,698 - tests.orchestrator - INFO - Deleted PVC: test-pvc-89991955 -2025-06-05 19:15:24,699 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 3 times) -2025-06-05 19:15:24,699 - tests.orchestrator - INFO - No pods to delete -2025-06-05 19:15:25,699 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 6 times) -2025-06-05 19:15:25,699 - tests.orchestrator - INFO - Creating PVC: test-pvc-2cf5b7a2 -2025-06-05 19:15:25,768 - tests.orchestrator - INFO - Created PVC: test-pvc-2cf5b7a2 -2025-06-05 19:15:25,768 - tests.orchestrator - INFO - Waiting for PVC test-pvc-2cf5b7a2 to be bound -2025-06-05 19:15:27,904 - tests.orchestrator - INFO - PVC test-pvc-2cf5b7a2 is bound -2025-06-05 19:15:28,904 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 7 times) -2025-06-05 19:15:28,904 - tests.orchestrator - INFO - Creating PVC: test-pvc-528b4f5c -2025-06-05 19:15:28,973 - tests.orchestrator - INFO - Created PVC: test-pvc-528b4f5c -2025-06-05 19:15:28,973 - tests.orchestrator - INFO - Waiting for PVC test-pvc-528b4f5c to be bound -2025-06-05 19:15:31,115 - tests.orchestrator - INFO - PVC test-pvc-528b4f5c is bound -2025-06-05 19:15:32,115 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 1 times) -2025-06-05 19:15:32,115 - tests.orchestrator - INFO - Creating pod test-pod-0d0fddfe on PVC test-pvc-2cf5b7a2 -2025-06-05 19:15:32,198 - tests.orchestrator - INFO - Created pod: test-pod-0d0fddfe using PVC: test-pvc-2cf5b7a2 -2025-06-05 19:15:32,199 - tests.orchestrator - INFO - Waiting for pod test-pod-0d0fddfe to be ready -2025-06-05 19:15:32,267 - tests.orchestrator - INFO - Pod test-pod-0d0fddfe phase: Pending -2025-06-05 19:15:36,403 - tests.orchestrator - INFO - Pod test-pod-0d0fddfe phase: Running -2025-06-05 19:15:36,403 - tests.orchestrator - INFO - Pod test-pod-0d0fddfe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:15:38,472 - tests.orchestrator - INFO - Pod test-pod-0d0fddfe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:15:40,540 - tests.orchestrator - INFO - Pod test-pod-0d0fddfe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:15:42,613 - tests.orchestrator - INFO - Pod test-pod-0d0fddfe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:15:44,682 - tests.orchestrator - INFO - Pod test-pod-0d0fddfe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:15:46,750 - tests.orchestrator - INFO - Pod test-pod-0d0fddfe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:15:48,818 - tests.orchestrator - INFO - Pod test-pod-0d0fddfe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:15:50,886 - tests.orchestrator - INFO - Pod test-pod-0d0fddfe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:15:52,955 - tests.orchestrator - INFO - Pod test-pod-0d0fddfe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 19:15:52,955 - tests.orchestrator - INFO - Pod test-pod-0d0fddfe is ready -2025-06-05 19:15:53,955 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 2 times) -2025-06-05 19:15:53,955 - tests.orchestrator - INFO - Deleting PVC: test-pvc-a0df5547 -2025-06-05 19:15:54,028 - tests.orchestrator - INFO - Waiting for PVC test-pvc-a0df5547 to be deleted -2025-06-05 19:15:54,099 - tests.orchestrator - INFO - PVC test-pvc-a0df5547 has been deleted -2025-06-05 19:15:54,099 - tests.orchestrator - INFO - Deleted PVC: test-pvc-a0df5547 -2025-06-05 19:15:55,100 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 2 times) -2025-06-05 19:15:55,100 - tests.orchestrator - INFO - Creating pod test-pod-158cdfa4 on PVC test-pvc-31fbd501 -2025-06-05 19:15:55,181 - tests.orchestrator - INFO - Created pod: test-pod-158cdfa4 using PVC: test-pvc-31fbd501 -2025-06-05 19:15:55,181 - tests.orchestrator - INFO - Waiting for pod test-pod-158cdfa4 to be ready -2025-06-05 19:15:55,248 - tests.orchestrator - INFO - Pod test-pod-158cdfa4 phase: Pending -2025-06-05 19:15:59,387 - tests.orchestrator - INFO - Pod test-pod-158cdfa4 phase: Running -2025-06-05 19:15:59,387 - tests.orchestrator - INFO - Pod test-pod-158cdfa4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_192824.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_192824.log deleted file mode 100644 index 834b6116e..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_192824.log +++ /dev/null @@ -1,7 +0,0 @@ -2025-06-05 19:28:24,606 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 19:28:24,815 - tests.orchestrator - ERROR - Error checking namespace: (401) -Reason: Unauthorized -HTTP response headers: HTTPHeaderDict({'Audit-Id': '0dff4cee-4147-4a36-b942-2640bcda6774', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 19:28:24 GMT', 'Content-Length': '129'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} - - diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_192851.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_192851.log deleted file mode 100644 index 164f8db03..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_192851.log +++ /dev/null @@ -1,207 +0,0 @@ -2025-06-05 19:28:51,597 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 19:28:51,800 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 19:28:51,800 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-05 19:28:51,990 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 19:28:51,990 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 19:28:51,991 - tests.orchestrator - INFO - Creating PVC: test-pvc-4046c78c -2025-06-05 19:28:52,067 - tests.orchestrator - INFO - Created PVC: test-pvc-4046c78c -2025-06-05 19:28:52,067 - tests.orchestrator - INFO - Waiting for PVC test-pvc-4046c78c to be bound -2025-06-05 19:28:54,199 - tests.orchestrator - INFO - PVC test-pvc-4046c78c is bound -2025-06-05 19:28:54,200 - tests.orchestrator - INFO - Creating PVC: test-pvc-d086aade -2025-06-05 19:28:54,267 - tests.orchestrator - INFO - Created PVC: test-pvc-d086aade -2025-06-05 19:28:54,267 - tests.orchestrator - INFO - Waiting for PVC test-pvc-d086aade to be bound -2025-06-05 19:28:56,399 - tests.orchestrator - INFO - PVC test-pvc-d086aade is bound -2025-06-05 19:28:56,399 - tests.orchestrator - INFO - Creating pod test-pod-7c93ffdb on PVC test-pvc-d086aade -2025-06-05 19:28:56,484 - tests.orchestrator - INFO - Created pod: test-pod-7c93ffdb using PVC: test-pvc-d086aade -2025-06-05 19:28:56,484 - tests.orchestrator - INFO - Waiting for pod test-pod-7c93ffdb to be ready -2025-06-05 19:28:56,552 - tests.orchestrator - INFO - Pod test-pod-7c93ffdb phase: Pending -2025-06-05 19:29:00,688 - tests.orchestrator - INFO - Pod test-pod-7c93ffdb phase: Running -2025-06-05 19:29:00,688 - tests.orchestrator - INFO - Pod test-pod-7c93ffdb conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:29:02,755 - tests.orchestrator - INFO - Pod test-pod-7c93ffdb conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:29:04,823 - tests.orchestrator - INFO - Pod test-pod-7c93ffdb conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:29:06,896 - tests.orchestrator - INFO - Pod test-pod-7c93ffdb conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:29:08,964 - tests.orchestrator - INFO - Pod test-pod-7c93ffdb conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:29:11,031 - tests.orchestrator - INFO - Pod test-pod-7c93ffdb conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:29:13,099 - tests.orchestrator - INFO - Pod test-pod-7c93ffdb conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:29:15,166 - tests.orchestrator - INFO - Pod test-pod-7c93ffdb conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:29:17,234 - tests.orchestrator - INFO - Pod test-pod-7c93ffdb conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:29:19,302 - tests.orchestrator - INFO - Pod test-pod-7c93ffdb conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 19:29:19,302 - tests.orchestrator - INFO - Pod test-pod-7c93ffdb is ready -2025-06-05 19:29:19,302 - tests.orchestrator - INFO - Creating pod test-pod-7340c349 on PVC test-pvc-d086aade -2025-06-05 19:29:19,376 - tests.orchestrator - INFO - Created pod: test-pod-7340c349 using PVC: test-pvc-d086aade -2025-06-05 19:29:19,376 - tests.orchestrator - INFO - Waiting for pod test-pod-7340c349 to be ready -2025-06-05 19:29:19,443 - tests.orchestrator - INFO - Pod test-pod-7340c349 phase: Pending -2025-06-05 19:29:21,512 - tests.orchestrator - INFO - Pod test-pod-7340c349 phase: Running -2025-06-05 19:29:21,512 - tests.orchestrator - INFO - Pod test-pod-7340c349 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:29:23,580 - tests.orchestrator - INFO - Pod test-pod-7340c349 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:29:25,647 - tests.orchestrator - INFO - Pod test-pod-7340c349 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:29:27,715 - tests.orchestrator - INFO - Pod test-pod-7340c349 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:29:29,783 - tests.orchestrator - INFO - Pod test-pod-7340c349 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:29:31,854 - tests.orchestrator - INFO - Pod test-pod-7340c349 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:29:33,922 - tests.orchestrator - INFO - Pod test-pod-7340c349 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:29:35,989 - tests.orchestrator - INFO - Pod test-pod-7340c349 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:29:38,057 - tests.orchestrator - INFO - Pod test-pod-7340c349 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:29:40,125 - tests.orchestrator - INFO - Pod test-pod-7340c349 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 19:29:40,125 - tests.orchestrator - INFO - Pod test-pod-7340c349 is ready -2025-06-05 19:29:40,125 - tests.orchestrator - INFO - Creating pod test-pod-b4873308 on PVC test-pvc-4046c78c -2025-06-05 19:29:40,208 - tests.orchestrator - INFO - Created pod: test-pod-b4873308 using PVC: test-pvc-4046c78c -2025-06-05 19:29:40,208 - tests.orchestrator - INFO - Waiting for pod test-pod-b4873308 to be ready -2025-06-05 19:29:40,275 - tests.orchestrator - INFO - Pod test-pod-b4873308 phase: Pending -2025-06-05 19:29:42,343 - tests.orchestrator - INFO - Pod test-pod-b4873308 phase: Running -2025-06-05 19:29:42,343 - tests.orchestrator - INFO - Pod test-pod-b4873308 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:29:44,411 - tests.orchestrator - INFO - Pod test-pod-b4873308 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:29:46,478 - tests.orchestrator - INFO - Pod test-pod-b4873308 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:29:48,546 - tests.orchestrator - INFO - Pod test-pod-b4873308 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:29:50,614 - tests.orchestrator - INFO - Pod test-pod-b4873308 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:29:52,681 - tests.orchestrator - INFO - Pod test-pod-b4873308 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:29:54,749 - tests.orchestrator - INFO - Pod test-pod-b4873308 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:29:56,817 - tests.orchestrator - INFO - Pod test-pod-b4873308 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 19:29:56,817 - tests.orchestrator - INFO - Pod test-pod-b4873308 is ready -2025-06-05 19:29:56,817 - tests.orchestrator - INFO - Testing read/write between pods test-pod-7340c349 and test-pod-7c93ffdb sharing PVC test-pvc-d086aade -2025-06-05 19:29:56,817 - tests.orchestrator - INFO - Executing write command: kubectl exec -n efs-stress-test test-pod-7340c349 -- /bin/sh -c 'echo "Test content: 0a7dd7bd-4b4d-493f-9b70-64073a2558bf" > /data/test-7c27f5ae.txt' -2025-06-05 19:29:58,091 - tests.orchestrator - INFO - Pod test-pod-7340c349 wrote to /data/test-7c27f5ae.txt -2025-06-05 19:30:00,091 - tests.orchestrator - INFO - Executing read command: kubectl exec -n efs-stress-test test-pod-7c93ffdb -- cat /data/test-7c27f5ae.txt -2025-06-05 19:30:01,307 - tests.orchestrator - INFO - Read result: 'Test content: 0a7dd7bd-4b4d-493f-9b70-64073a2558bf' -2025-06-05 19:30:01,307 - tests.orchestrator - INFO - Pod test-pod-7c93ffdb successfully read content written by test-pod-7340c349 -2025-06-05 19:30:01,307 - tests.orchestrator - INFO - Deleting pod: test-pod-b4873308 from PVC: test-pvc-4046c78c -2025-06-05 19:30:01,397 - tests.orchestrator - INFO - Waiting for pod test-pod-b4873308 to be deleted -2025-06-05 19:30:34,119 - tests.orchestrator - INFO - Pod test-pod-b4873308 has been deleted -2025-06-05 19:30:34,119 - tests.orchestrator - INFO - Deleted pod: test-pod-b4873308 -2025-06-05 19:30:34,119 - tests.orchestrator - INFO - Deleting PVC: test-pvc-4046c78c -2025-06-05 19:30:34,189 - tests.orchestrator - INFO - Waiting for PVC test-pvc-4046c78c to be deleted -2025-06-05 19:30:34,254 - tests.orchestrator - INFO - PVC test-pvc-4046c78c has been deleted -2025-06-05 19:30:34,255 - tests.orchestrator - INFO - Deleted PVC: test-pvc-4046c78c -2025-06-05 19:30:34,255 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations -2025-06-05 19:30:34,255 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 1 times) -2025-06-05 19:30:34,255 - tests.orchestrator - INFO - Creating PVC: test-pvc-2da07839 -2025-06-05 19:30:34,332 - tests.orchestrator - INFO - Created PVC: test-pvc-2da07839 -2025-06-05 19:30:34,332 - tests.orchestrator - INFO - Waiting for PVC test-pvc-2da07839 to be bound -2025-06-05 19:30:36,474 - tests.orchestrator - INFO - PVC test-pvc-2da07839 is bound -2025-06-05 19:30:37,474 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 1 times) -2025-06-05 19:30:37,474 - tests.orchestrator - INFO - Creating pod test-pod-ada6dc83 on PVC test-pvc-2da07839 -2025-06-05 19:30:37,547 - tests.orchestrator - INFO - Created pod: test-pod-ada6dc83 using PVC: test-pvc-2da07839 -2025-06-05 19:30:37,547 - tests.orchestrator - INFO - Waiting for pod test-pod-ada6dc83 to be ready -2025-06-05 19:30:37,614 - tests.orchestrator - INFO - Pod test-pod-ada6dc83 phase: Pending -2025-06-05 19:30:41,747 - tests.orchestrator - INFO - Pod test-pod-ada6dc83 phase: Running -2025-06-05 19:30:41,747 - tests.orchestrator - INFO - Pod test-pod-ada6dc83 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:30:43,815 - tests.orchestrator - INFO - Pod test-pod-ada6dc83 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:30:45,883 - tests.orchestrator - INFO - Pod test-pod-ada6dc83 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:30:47,950 - tests.orchestrator - INFO - Pod test-pod-ada6dc83 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:30:50,019 - tests.orchestrator - INFO - Pod test-pod-ada6dc83 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:30:52,088 - tests.orchestrator - INFO - Pod test-pod-ada6dc83 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:30:54,154 - tests.orchestrator - INFO - Pod test-pod-ada6dc83 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:30:56,222 - tests.orchestrator - INFO - Pod test-pod-ada6dc83 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:30:58,290 - tests.orchestrator - INFO - Pod test-pod-ada6dc83 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:31:00,358 - tests.orchestrator - INFO - Pod test-pod-ada6dc83 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 19:31:00,358 - tests.orchestrator - INFO - Pod test-pod-ada6dc83 is ready -2025-06-05 19:31:01,359 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 1 times) -2025-06-05 19:31:01,359 - tests.orchestrator - INFO - Deleting pod: test-pod-7340c349 from PVC: test-pvc-d086aade -2025-06-05 19:31:01,460 - tests.orchestrator - INFO - Waiting for pod test-pod-7340c349 to be deleted -2025-06-05 19:31:34,130 - tests.orchestrator - INFO - Pod test-pod-7340c349 has been deleted -2025-06-05 19:31:34,130 - tests.orchestrator - INFO - Deleted pod: test-pod-7340c349 -2025-06-05 19:31:35,130 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 2 times) -2025-06-05 19:31:35,131 - tests.orchestrator - INFO - Deleting pod: test-pod-7c93ffdb from PVC: test-pvc-d086aade -2025-06-05 19:31:35,204 - tests.orchestrator - INFO - Waiting for pod test-pod-7c93ffdb to be deleted -2025-06-05 19:32:07,811 - tests.orchestrator - INFO - Pod test-pod-7c93ffdb has been deleted -2025-06-05 19:32:07,811 - tests.orchestrator - INFO - Deleted pod: test-pod-7c93ffdb -2025-06-05 19:32:08,811 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 2 times) -2025-06-05 19:32:08,812 - tests.orchestrator - INFO - Creating pod test-pod-98d8b402 on PVC test-pvc-d086aade -2025-06-05 19:32:08,902 - tests.orchestrator - INFO - Created pod: test-pod-98d8b402 using PVC: test-pvc-d086aade -2025-06-05 19:32:08,902 - tests.orchestrator - INFO - Waiting for pod test-pod-98d8b402 to be ready -2025-06-05 19:32:08,972 - tests.orchestrator - INFO - Pod test-pod-98d8b402 phase: Pending -2025-06-05 19:32:13,114 - tests.orchestrator - INFO - Pod test-pod-98d8b402 phase: Running -2025-06-05 19:32:13,115 - tests.orchestrator - INFO - Pod test-pod-98d8b402 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:32:15,182 - tests.orchestrator - INFO - Pod test-pod-98d8b402 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:32:17,250 - tests.orchestrator - INFO - Pod test-pod-98d8b402 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:32:19,317 - tests.orchestrator - INFO - Pod test-pod-98d8b402 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:32:21,384 - tests.orchestrator - INFO - Pod test-pod-98d8b402 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:32:23,453 - tests.orchestrator - INFO - Pod test-pod-98d8b402 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:32:25,520 - tests.orchestrator - INFO - Pod test-pod-98d8b402 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:32:27,588 - tests.orchestrator - INFO - Pod test-pod-98d8b402 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:32:29,656 - tests.orchestrator - INFO - Pod test-pod-98d8b402 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:32:31,766 - tests.orchestrator - INFO - Pod test-pod-98d8b402 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 19:32:31,766 - tests.orchestrator - INFO - Pod test-pod-98d8b402 is ready -2025-06-05 19:32:32,766 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 2 times) -2025-06-05 19:32:32,766 - tests.orchestrator - INFO - Creating PVC: test-pvc-4a306e31 -2025-06-05 19:32:32,835 - tests.orchestrator - INFO - Created PVC: test-pvc-4a306e31 -2025-06-05 19:32:32,835 - tests.orchestrator - INFO - Waiting for PVC test-pvc-4a306e31 to be bound -2025-06-05 19:32:34,969 - tests.orchestrator - INFO - PVC test-pvc-4a306e31 is bound -2025-06-05 19:32:35,969 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 3 times) -2025-06-05 19:32:35,969 - tests.orchestrator - INFO - Deleting pod: test-pod-98d8b402 from PVC: test-pvc-d086aade -2025-06-05 19:32:36,045 - tests.orchestrator - INFO - Waiting for pod test-pod-98d8b402 to be deleted -2025-06-05 19:33:08,653 - tests.orchestrator - INFO - Pod test-pod-98d8b402 has been deleted -2025-06-05 19:33:08,653 - tests.orchestrator - INFO - Deleted pod: test-pod-98d8b402 -2025-06-05 19:33:09,653 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 3 times) -2025-06-05 19:33:09,653 - tests.orchestrator - INFO - Creating PVC: test-pvc-1d05425d -2025-06-05 19:33:09,730 - tests.orchestrator - INFO - Created PVC: test-pvc-1d05425d -2025-06-05 19:33:09,730 - tests.orchestrator - INFO - Waiting for PVC test-pvc-1d05425d to be bound -2025-06-05 19:33:11,862 - tests.orchestrator - INFO - PVC test-pvc-1d05425d is bound -2025-06-05 19:33:12,863 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 4 times) -2025-06-05 19:33:12,863 - tests.orchestrator - INFO - Creating PVC: test-pvc-16969258 -2025-06-05 19:33:12,931 - tests.orchestrator - INFO - Created PVC: test-pvc-16969258 -2025-06-05 19:33:12,931 - tests.orchestrator - INFO - Waiting for PVC test-pvc-16969258 to be bound -2025-06-05 19:33:15,064 - tests.orchestrator - INFO - PVC test-pvc-16969258 is bound -2025-06-05 19:33:16,064 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 1 times) -2025-06-05 19:33:16,064 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test -2025-06-05 19:33:17,065 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 1 times) -2025-06-05 19:33:17,065 - tests.orchestrator - INFO - Deleting PVC: test-pvc-d086aade -2025-06-05 19:33:17,139 - tests.orchestrator - INFO - Waiting for PVC test-pvc-d086aade to be deleted -2025-06-05 19:33:17,204 - tests.orchestrator - INFO - PVC test-pvc-d086aade has been deleted -2025-06-05 19:33:17,204 - tests.orchestrator - INFO - Deleted PVC: test-pvc-d086aade -2025-06-05 19:33:18,204 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 5 times) -2025-06-05 19:33:18,204 - tests.orchestrator - INFO - Creating PVC: test-pvc-8e9317d0 -2025-06-05 19:33:18,272 - tests.orchestrator - INFO - Created PVC: test-pvc-8e9317d0 -2025-06-05 19:33:18,272 - tests.orchestrator - INFO - Waiting for PVC test-pvc-8e9317d0 to be bound -2025-06-05 19:33:20,404 - tests.orchestrator - INFO - PVC test-pvc-8e9317d0 is bound -2025-06-05 19:33:21,405 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 2 times) -2025-06-05 19:33:21,405 - tests.orchestrator - INFO - No shared PVCs with multiple pods for read/write test -2025-06-05 19:33:22,405 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 4 times) -2025-06-05 19:33:22,405 - tests.orchestrator - INFO - Deleting pod: test-pod-ada6dc83 from PVC: test-pvc-2da07839 -2025-06-05 19:33:22,481 - tests.orchestrator - INFO - Waiting for pod test-pod-ada6dc83 to be deleted -2025-06-05 19:33:55,087 - tests.orchestrator - INFO - Pod test-pod-ada6dc83 has been deleted -2025-06-05 19:33:55,088 - tests.orchestrator - INFO - Deleted pod: test-pod-ada6dc83 -2025-06-05 19:33:56,088 - tests.orchestrator - INFO - Test completed in 304.29 seconds -2025-06-05 19:33:56,088 - tests.orchestrator - INFO - ===== STARTING COMPREHENSIVE CLEANUP ===== -2025-06-05 19:33:56,088 - tests.orchestrator - INFO - Deleting 0 pods (force=False)... -2025-06-05 19:34:01,088 - tests.orchestrator - INFO - Deleting 5 PVCs (force=False)... -2025-06-05 19:34:01,088 - tests.orchestrator - INFO - Deleting PVC: test-pvc-2da07839 -2025-06-05 19:34:01,160 - tests.orchestrator - INFO - Waiting for PVC test-pvc-2da07839 to be deleted -2025-06-05 19:34:01,226 - tests.orchestrator - INFO - PVC test-pvc-2da07839 has been deleted -2025-06-05 19:34:01,226 - tests.orchestrator - INFO - Deleted PVC: test-pvc-2da07839 -2025-06-05 19:34:01,226 - tests.orchestrator - INFO - Deleting PVC: test-pvc-4a306e31 -2025-06-05 19:34:01,298 - tests.orchestrator - INFO - Waiting for PVC test-pvc-4a306e31 to be deleted -2025-06-05 19:34:01,364 - tests.orchestrator - INFO - PVC test-pvc-4a306e31 has been deleted -2025-06-05 19:34:01,364 - tests.orchestrator - INFO - Deleted PVC: test-pvc-4a306e31 -2025-06-05 19:34:01,364 - tests.orchestrator - INFO - Deleting PVC: test-pvc-1d05425d -2025-06-05 19:34:01,514 - tests.orchestrator - INFO - Waiting for PVC test-pvc-1d05425d to be deleted -2025-06-05 19:34:01,667 - tests.orchestrator - INFO - PVC test-pvc-1d05425d has been deleted -2025-06-05 19:34:01,667 - tests.orchestrator - INFO - Deleted PVC: test-pvc-1d05425d -2025-06-05 19:34:01,667 - tests.orchestrator - INFO - Deleting PVC: test-pvc-16969258 -2025-06-05 19:34:01,774 - tests.orchestrator - INFO - Waiting for PVC test-pvc-16969258 to be deleted -2025-06-05 19:34:01,847 - tests.orchestrator - INFO - PVC test-pvc-16969258 has been deleted -2025-06-05 19:34:01,847 - tests.orchestrator - INFO - Deleted PVC: test-pvc-16969258 -2025-06-05 19:34:01,847 - tests.orchestrator - INFO - Deleting PVC: test-pvc-8e9317d0 -2025-06-05 19:34:01,925 - tests.orchestrator - INFO - Waiting for PVC test-pvc-8e9317d0 to be deleted -2025-06-05 19:34:01,996 - tests.orchestrator - INFO - PVC test-pvc-8e9317d0 has been deleted -2025-06-05 19:34:01,996 - tests.orchestrator - INFO - Deleted PVC: test-pvc-8e9317d0 -2025-06-05 19:34:02,206 - tests.orchestrator - WARNING - First cleanup pass incomplete. Remaining resources: ['pod/test-pod-0d0fddfe', 'pod/test-pod-158cdfa4', 'pvc/test-pvc-0bb0d20c', 'pvc/test-pvc-2127e71a', 'pvc/test-pvc-2cf5b7a2', 'pvc/test-pvc-31fbd501', 'pvc/test-pvc-528b4f5c', 'pvc/test-pvc-ecb12f74'] -2025-06-05 19:34:02,206 - tests.orchestrator - INFO - Attempting force deletion of remaining resources... -2025-06-05 19:34:02,206 - tests.orchestrator - INFO - Deleting 0 pods (force=True)... -2025-06-05 19:34:07,206 - tests.orchestrator - INFO - Deleting 0 PVCs (force=True)... -2025-06-05 19:34:07,403 - tests.orchestrator - ERROR - Cleanup incomplete. Remaining resources after force deletion: ['pod/test-pod-0d0fddfe', 'pod/test-pod-158cdfa4', 'pvc/test-pvc-0bb0d20c', 'pvc/test-pvc-2127e71a', 'pvc/test-pvc-2cf5b7a2', 'pvc/test-pvc-31fbd501', 'pvc/test-pvc-528b4f5c', 'pvc/test-pvc-ecb12f74'] -2025-06-05 19:34:07,403 - tests.orchestrator - INFO - Cleanup completed successfully in 11.32 seconds -2025-06-05 19:34:07,404 - tests.orchestrator - INFO - ===== CLEANUP PROCESS FINISHED ===== -2025-06-05 19:34:07,404 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-05 19:34:07,404 - tests.orchestrator - INFO - --- Operations --- -2025-06-05 19:34:07,404 - tests.orchestrator - INFO - create_pvc: 7 succeeded, 0 failed (100.0%) -2025-06-05 19:34:07,404 - tests.orchestrator - INFO - attach_pod: 5 succeeded, 0 failed (100.0%) -2025-06-05 19:34:07,404 - tests.orchestrator - INFO - delete_pod: 5 succeeded, 0 failed (100.0%) -2025-06-05 19:34:07,404 - tests.orchestrator - INFO - delete_pvc: 7 succeeded, 0 failed (100.0%) -2025-06-05 19:34:07,404 - tests.orchestrator - INFO - verify_read_write: Writes 1 succeeded, 0 failed (100.0%) -2025-06-05 19:34:07,404 - tests.orchestrator - INFO - verify_read_write: Reads 1 succeeded, 0 failed (100.0%) -2025-06-05 19:34:07,404 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-05 19:34:07,404 - tests.orchestrator - INFO - shared_volume_rw: 1 succeeded, 0 failed out of 1 runs (100.0%) -2025-06-05 19:34:07,404 - tests.orchestrator - INFO - many_to_one: No runs -2025-06-05 19:34:07,404 - tests.orchestrator - INFO - one_to_one: No runs -2025-06-05 19:34:07,404 - tests.orchestrator - INFO - concurrent_pvc: No runs -2025-06-05 19:34:07,404 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_195225.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_195225.log deleted file mode 100644 index cf0578c8f..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_195225.log +++ /dev/null @@ -1,335 +0,0 @@ -2025-06-05 19:52:25,645 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 19:52:25,880 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 19:52:25,881 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-05 19:52:26,070 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 19:52:26,071 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 19:52:26,071 - tests.orchestrator - INFO - Creating PVC: test-pvc-b5f8638e -2025-06-05 19:52:26,146 - tests.orchestrator - INFO - Created PVC: test-pvc-b5f8638e -2025-06-05 19:52:26,146 - tests.orchestrator - INFO - Waiting for PVC test-pvc-b5f8638e to be bound -2025-06-05 19:52:28,278 - tests.orchestrator - INFO - PVC test-pvc-b5f8638e is bound -2025-06-05 19:52:28,278 - tests.orchestrator - INFO - Creating PVC: test-pvc-81ab07bf -2025-06-05 19:52:28,347 - tests.orchestrator - INFO - Created PVC: test-pvc-81ab07bf -2025-06-05 19:52:28,347 - tests.orchestrator - INFO - Waiting for PVC test-pvc-81ab07bf to be bound -2025-06-05 19:52:30,483 - tests.orchestrator - INFO - PVC test-pvc-81ab07bf is bound -2025-06-05 19:52:30,483 - tests.orchestrator - INFO - Creating pod test-pod-a16d9846 on PVC test-pvc-81ab07bf -2025-06-05 19:52:30,563 - tests.orchestrator - INFO - Created pod: test-pod-a16d9846 using PVC: test-pvc-81ab07bf -2025-06-05 19:52:30,563 - tests.orchestrator - INFO - Waiting for pod test-pod-a16d9846 to be ready -2025-06-05 19:52:30,632 - tests.orchestrator - INFO - Pod test-pod-a16d9846 phase: Pending -2025-06-05 19:52:32,699 - tests.orchestrator - INFO - Pod test-pod-a16d9846 phase: Running -2025-06-05 19:52:32,700 - tests.orchestrator - INFO - Pod test-pod-a16d9846 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:52:34,768 - tests.orchestrator - INFO - Pod test-pod-a16d9846 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:52:36,837 - tests.orchestrator - INFO - Pod test-pod-a16d9846 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:52:38,905 - tests.orchestrator - INFO - Pod test-pod-a16d9846 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:52:40,973 - tests.orchestrator - INFO - Pod test-pod-a16d9846 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:52:43,042 - tests.orchestrator - INFO - Pod test-pod-a16d9846 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:52:45,109 - tests.orchestrator - INFO - Pod test-pod-a16d9846 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:52:47,178 - tests.orchestrator - INFO - Pod test-pod-a16d9846 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 19:52:47,178 - tests.orchestrator - INFO - Pod test-pod-a16d9846 is ready -2025-06-05 19:52:47,178 - tests.orchestrator - INFO - Creating pod test-pod-28ec4f7d on PVC test-pvc-b5f8638e -2025-06-05 19:52:47,253 - tests.orchestrator - INFO - Created pod: test-pod-28ec4f7d using PVC: test-pvc-b5f8638e -2025-06-05 19:52:47,253 - tests.orchestrator - INFO - Waiting for pod test-pod-28ec4f7d to be ready -2025-06-05 19:52:47,321 - tests.orchestrator - INFO - Pod test-pod-28ec4f7d phase: Pending -2025-06-05 19:52:51,458 - tests.orchestrator - INFO - Pod test-pod-28ec4f7d phase: Running -2025-06-05 19:52:51,458 - tests.orchestrator - INFO - Pod test-pod-28ec4f7d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:52:53,527 - tests.orchestrator - INFO - Pod test-pod-28ec4f7d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:52:55,594 - tests.orchestrator - INFO - Pod test-pod-28ec4f7d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:52:57,662 - tests.orchestrator - INFO - Pod test-pod-28ec4f7d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:52:59,731 - tests.orchestrator - INFO - Pod test-pod-28ec4f7d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:53:01,804 - tests.orchestrator - INFO - Pod test-pod-28ec4f7d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:53:03,871 - tests.orchestrator - INFO - Pod test-pod-28ec4f7d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 19:53:03,872 - tests.orchestrator - INFO - Pod test-pod-28ec4f7d is ready -2025-06-05 19:53:03,872 - tests.orchestrator - INFO - Creating pod test-pod-044934ff on PVC test-pvc-b5f8638e -2025-06-05 19:53:03,956 - tests.orchestrator - INFO - Created pod: test-pod-044934ff using PVC: test-pvc-b5f8638e -2025-06-05 19:53:03,956 - tests.orchestrator - INFO - Waiting for pod test-pod-044934ff to be ready -2025-06-05 19:53:04,024 - tests.orchestrator - INFO - Pod test-pod-044934ff phase: Pending -2025-06-05 19:53:08,160 - tests.orchestrator - INFO - Pod test-pod-044934ff phase: Running -2025-06-05 19:53:08,161 - tests.orchestrator - INFO - Pod test-pod-044934ff conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:53:10,230 - tests.orchestrator - INFO - Pod test-pod-044934ff conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:53:12,298 - tests.orchestrator - INFO - Pod test-pod-044934ff conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:53:14,366 - tests.orchestrator - INFO - Pod test-pod-044934ff conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:53:16,438 - tests.orchestrator - INFO - Pod test-pod-044934ff conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:53:18,506 - tests.orchestrator - INFO - Pod test-pod-044934ff conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:53:20,574 - tests.orchestrator - INFO - Pod test-pod-044934ff conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:53:22,644 - tests.orchestrator - INFO - Pod test-pod-044934ff conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:53:24,712 - tests.orchestrator - INFO - Pod test-pod-044934ff conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:53:26,780 - tests.orchestrator - INFO - Pod test-pod-044934ff conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 19:53:26,781 - tests.orchestrator - INFO - Pod test-pod-044934ff is ready -2025-06-05 19:53:26,781 - tests.orchestrator - INFO - Testing read/write between pods test-pod-044934ff and test-pod-28ec4f7d sharing PVC test-pvc-b5f8638e -2025-06-05 19:53:26,781 - tests.orchestrator - INFO - Executing write command: kubectl exec -n efs-stress-test test-pod-044934ff -- /bin/sh -c 'echo "Test content: 90b0f8ff-b5eb-4887-86da-4de1b6273ee1" > /data/test-309c7d73.txt' -2025-06-05 19:53:28,021 - tests.orchestrator - INFO - Pod test-pod-044934ff wrote to /data/test-309c7d73.txt -2025-06-05 19:53:30,021 - tests.orchestrator - INFO - Executing read command: kubectl exec -n efs-stress-test test-pod-28ec4f7d -- cat /data/test-309c7d73.txt -2025-06-05 19:53:31,221 - tests.orchestrator - INFO - Read result: 'Test content: 90b0f8ff-b5eb-4887-86da-4de1b6273ee1' -2025-06-05 19:53:31,221 - tests.orchestrator - INFO - Pod test-pod-28ec4f7d successfully read content written by test-pod-044934ff -2025-06-05 19:53:31,221 - tests.orchestrator - INFO - Deleting pod: test-pod-a16d9846 from PVC: test-pvc-81ab07bf -2025-06-05 19:53:31,296 - tests.orchestrator - INFO - Waiting for pod test-pod-a16d9846 to be deleted -2025-06-05 19:54:03,907 - tests.orchestrator - INFO - Pod test-pod-a16d9846 has been deleted -2025-06-05 19:54:03,907 - tests.orchestrator - INFO - Deleted pod: test-pod-a16d9846 -2025-06-05 19:54:03,907 - tests.orchestrator - INFO - Deleting PVC: test-pvc-81ab07bf -2025-06-05 19:54:03,979 - tests.orchestrator - INFO - Waiting for PVC test-pvc-81ab07bf to be deleted -2025-06-05 19:54:04,044 - tests.orchestrator - INFO - PVC test-pvc-81ab07bf has been deleted -2025-06-05 19:54:04,044 - tests.orchestrator - INFO - Deleted PVC: test-pvc-81ab07bf -2025-06-05 19:54:04,044 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations -2025-06-05 19:54:04,044 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 1 times) -2025-06-05 19:54:04,045 - tests.orchestrator - INFO - Testing read/write between pods test-pod-044934ff and test-pod-28ec4f7d sharing PVC test-pvc-b5f8638e -2025-06-05 19:54:04,045 - tests.orchestrator - INFO - Executing write command: kubectl exec -n efs-stress-test test-pod-044934ff -- /bin/sh -c 'echo "Test content: eafc327a-6e13-4907-8a5f-8caa39cd7559" > /data/test-cc6083d7.txt' -2025-06-05 19:54:05,250 - tests.orchestrator - INFO - Pod test-pod-044934ff wrote to /data/test-cc6083d7.txt -2025-06-05 19:54:07,250 - tests.orchestrator - INFO - Executing read command: kubectl exec -n efs-stress-test test-pod-28ec4f7d -- cat /data/test-cc6083d7.txt -2025-06-05 19:54:08,454 - tests.orchestrator - INFO - Read result: 'Test content: eafc327a-6e13-4907-8a5f-8caa39cd7559' -2025-06-05 19:54:08,454 - tests.orchestrator - INFO - Pod test-pod-28ec4f7d successfully read content written by test-pod-044934ff -2025-06-05 19:54:09,454 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario (selected 1 times) -2025-06-05 19:54:09,454 - tests.orchestrator - INFO - ============================================================ -2025-06-05 19:54:09,454 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_concurrent_pvc -2025-06-05 19:54:09,454 - tests.orchestrator - INFO - ============================================================ -2025-06-05 19:54:09,454 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations -2025-06-05 19:54:09,454 - tests.orchestrator - INFO - Creating 5 PVCs in quick succession -2025-06-05 19:54:09,531 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-1bfddf93 -2025-06-05 19:54:09,531 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-1bfddf93 to be bound -2025-06-05 19:54:11,664 - tests.orchestrator - INFO - PVC concurrent-pvc-1bfddf93 is bound -2025-06-05 19:54:11,732 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-0edb68b1 -2025-06-05 19:54:11,732 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-0edb68b1 to be bound -2025-06-05 19:54:13,866 - tests.orchestrator - INFO - PVC concurrent-pvc-0edb68b1 is bound -2025-06-05 19:54:13,933 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-25754791 -2025-06-05 19:54:13,933 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-25754791 to be bound -2025-06-05 19:54:16,067 - tests.orchestrator - INFO - PVC concurrent-pvc-25754791 is bound -2025-06-05 19:54:16,135 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-e4c09a13 -2025-06-05 19:54:16,135 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-e4c09a13 to be bound -2025-06-05 19:54:18,270 - tests.orchestrator - INFO - PVC concurrent-pvc-e4c09a13 is bound -2025-06-05 19:54:18,339 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-3f3c40ea -2025-06-05 19:54:18,339 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-3f3c40ea to be bound -2025-06-05 19:54:20,472 - tests.orchestrator - INFO - PVC concurrent-pvc-3f3c40ea is bound -2025-06-05 19:54:20,472 - tests.orchestrator - INFO - Successfully created 5 PVCs -2025-06-05 19:54:20,472 - tests.orchestrator - INFO - Creating pod test-pod-a0c335d2 on PVC concurrent-pvc-0edb68b1 -2025-06-05 19:54:20,547 - tests.orchestrator - INFO - Created pod: test-pod-a0c335d2 using PVC: concurrent-pvc-0edb68b1 -2025-06-05 19:54:20,547 - tests.orchestrator - INFO - Waiting for pod test-pod-a0c335d2 to be ready -2025-06-05 19:54:20,614 - tests.orchestrator - INFO - Pod test-pod-a0c335d2 phase: Pending -2025-06-05 19:54:22,683 - tests.orchestrator - INFO - Pod test-pod-a0c335d2 phase: Running -2025-06-05 19:54:22,683 - tests.orchestrator - INFO - Pod test-pod-a0c335d2 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:54:24,752 - tests.orchestrator - INFO - Pod test-pod-a0c335d2 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:54:26,819 - tests.orchestrator - INFO - Pod test-pod-a0c335d2 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:54:28,888 - tests.orchestrator - INFO - Pod test-pod-a0c335d2 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:54:30,956 - tests.orchestrator - INFO - Pod test-pod-a0c335d2 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:54:33,024 - tests.orchestrator - INFO - Pod test-pod-a0c335d2 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:54:35,092 - tests.orchestrator - INFO - Pod test-pod-a0c335d2 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:54:37,161 - tests.orchestrator - INFO - Pod test-pod-a0c335d2 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:54:39,228 - tests.orchestrator - INFO - Pod test-pod-a0c335d2 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:54:41,296 - tests.orchestrator - INFO - Pod test-pod-a0c335d2 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:54:43,365 - tests.orchestrator - INFO - Pod test-pod-a0c335d2 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 19:54:43,365 - tests.orchestrator - INFO - Pod test-pod-a0c335d2 is ready -2025-06-05 19:54:43,365 - tests.orchestrator - INFO - Creating pod test-pod-a24dabfe on PVC concurrent-pvc-3f3c40ea -2025-06-05 19:54:43,449 - tests.orchestrator - INFO - Created pod: test-pod-a24dabfe using PVC: concurrent-pvc-3f3c40ea -2025-06-05 19:54:43,449 - tests.orchestrator - INFO - Waiting for pod test-pod-a24dabfe to be ready -2025-06-05 19:54:43,516 - tests.orchestrator - INFO - Pod test-pod-a24dabfe phase: Pending -2025-06-05 19:54:47,656 - tests.orchestrator - INFO - Pod test-pod-a24dabfe phase: Running -2025-06-05 19:54:47,656 - tests.orchestrator - INFO - Pod test-pod-a24dabfe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:54:49,725 - tests.orchestrator - INFO - Pod test-pod-a24dabfe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:54:51,793 - tests.orchestrator - INFO - Pod test-pod-a24dabfe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:54:53,862 - tests.orchestrator - INFO - Pod test-pod-a24dabfe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:54:55,930 - tests.orchestrator - INFO - Pod test-pod-a24dabfe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:54:57,998 - tests.orchestrator - INFO - Pod test-pod-a24dabfe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:55:00,068 - tests.orchestrator - INFO - Pod test-pod-a24dabfe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:55:02,150 - tests.orchestrator - INFO - Pod test-pod-a24dabfe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:55:04,218 - tests.orchestrator - INFO - Pod test-pod-a24dabfe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:55:06,287 - tests.orchestrator - INFO - Pod test-pod-a24dabfe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 19:55:06,287 - tests.orchestrator - INFO - Pod test-pod-a24dabfe is ready -2025-06-05 19:55:06,287 - tests.orchestrator - INFO - Creating pod test-pod-cbfe1bf0 on PVC concurrent-pvc-25754791 -2025-06-05 19:55:06,364 - tests.orchestrator - INFO - Created pod: test-pod-cbfe1bf0 using PVC: concurrent-pvc-25754791 -2025-06-05 19:55:06,364 - tests.orchestrator - INFO - Waiting for pod test-pod-cbfe1bf0 to be ready -2025-06-05 19:55:06,431 - tests.orchestrator - INFO - Pod test-pod-cbfe1bf0 phase: Pending -2025-06-05 19:55:10,568 - tests.orchestrator - INFO - Pod test-pod-cbfe1bf0 phase: Running -2025-06-05 19:55:10,568 - tests.orchestrator - INFO - Pod test-pod-cbfe1bf0 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:55:12,636 - tests.orchestrator - INFO - Pod test-pod-cbfe1bf0 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:55:14,704 - tests.orchestrator - INFO - Pod test-pod-cbfe1bf0 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:55:16,773 - tests.orchestrator - INFO - Pod test-pod-cbfe1bf0 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:55:18,840 - tests.orchestrator - INFO - Pod test-pod-cbfe1bf0 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:55:20,909 - tests.orchestrator - INFO - Pod test-pod-cbfe1bf0 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:55:22,978 - tests.orchestrator - INFO - Pod test-pod-cbfe1bf0 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:55:25,045 - tests.orchestrator - INFO - Pod test-pod-cbfe1bf0 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:55:27,114 - tests.orchestrator - INFO - Pod test-pod-cbfe1bf0 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 19:55:27,114 - tests.orchestrator - INFO - Pod test-pod-cbfe1bf0 is ready -2025-06-05 19:55:27,114 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-1bfddf93 -2025-06-05 19:55:27,186 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-1bfddf93 to be deleted -2025-06-05 19:55:27,252 - tests.orchestrator - INFO - PVC concurrent-pvc-1bfddf93 has been deleted -2025-06-05 19:55:27,252 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-1bfddf93 -2025-06-05 19:55:27,253 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-25754791 -2025-06-05 19:55:27,253 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-25754791 -2025-06-05 19:55:27,253 - tests.orchestrator - INFO - Deleting pod: test-pod-cbfe1bf0 from PVC: concurrent-pvc-25754791 -2025-06-05 19:55:27,328 - tests.orchestrator - INFO - Waiting for pod test-pod-cbfe1bf0 to be deleted -2025-06-05 19:55:59,981 - tests.orchestrator - INFO - Pod test-pod-cbfe1bf0 has been deleted -2025-06-05 19:55:59,981 - tests.orchestrator - INFO - Deleted pod: test-pod-cbfe1bf0 -2025-06-05 19:56:00,058 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-25754791 to be deleted -2025-06-05 19:56:00,122 - tests.orchestrator - INFO - PVC concurrent-pvc-25754791 has been deleted -2025-06-05 19:56:00,123 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-25754791 -2025-06-05 19:56:00,123 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-3f3c40ea -2025-06-05 19:56:00,123 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-3f3c40ea -2025-06-05 19:56:00,123 - tests.orchestrator - INFO - Deleting pod: test-pod-a24dabfe from PVC: concurrent-pvc-3f3c40ea -2025-06-05 19:56:00,197 - tests.orchestrator - INFO - Waiting for pod test-pod-a24dabfe to be deleted -2025-06-05 19:56:32,809 - tests.orchestrator - INFO - Pod test-pod-a24dabfe has been deleted -2025-06-05 19:56:32,809 - tests.orchestrator - INFO - Deleted pod: test-pod-a24dabfe -2025-06-05 19:56:32,881 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-3f3c40ea to be deleted -2025-06-05 19:56:32,947 - tests.orchestrator - INFO - PVC concurrent-pvc-3f3c40ea has been deleted -2025-06-05 19:56:32,947 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-3f3c40ea -2025-06-05 19:56:32,947 - tests.orchestrator - INFO - Rapid PVC scenario completed -2025-06-05 19:56:32,947 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_concurrent_pvc -2025-06-05 19:56:32,947 - tests.orchestrator - INFO - ------------------------------------------------------------ -2025-06-05 19:56:33,947 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario (selected 2 times) -2025-06-05 19:56:33,947 - tests.orchestrator - INFO - ============================================================ -2025-06-05 19:56:33,947 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_concurrent_pvc -2025-06-05 19:56:33,947 - tests.orchestrator - INFO - ============================================================ -2025-06-05 19:56:33,948 - tests.orchestrator - INFO - Running scenario: Rapid PVC operations -2025-06-05 19:56:33,948 - tests.orchestrator - INFO - Creating 6 PVCs in quick succession -2025-06-05 19:56:34,025 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-cbac8cde -2025-06-05 19:56:34,025 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-cbac8cde to be bound -2025-06-05 19:56:36,158 - tests.orchestrator - INFO - PVC concurrent-pvc-cbac8cde is bound -2025-06-05 19:56:36,227 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-12483e38 -2025-06-05 19:56:36,227 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-12483e38 to be bound -2025-06-05 19:56:38,360 - tests.orchestrator - INFO - PVC concurrent-pvc-12483e38 is bound -2025-06-05 19:56:38,428 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-ab9c791a -2025-06-05 19:56:38,428 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-ab9c791a to be bound -2025-06-05 19:56:40,563 - tests.orchestrator - INFO - PVC concurrent-pvc-ab9c791a is bound -2025-06-05 19:56:40,630 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-ac5d027e -2025-06-05 19:56:40,630 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-ac5d027e to be bound -2025-06-05 19:56:42,763 - tests.orchestrator - INFO - PVC concurrent-pvc-ac5d027e is bound -2025-06-05 19:56:42,832 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-ee19f53f -2025-06-05 19:56:42,832 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-ee19f53f to be bound -2025-06-05 19:56:44,965 - tests.orchestrator - INFO - PVC concurrent-pvc-ee19f53f is bound -2025-06-05 19:56:45,033 - tests.orchestrator - INFO - Created PVC: concurrent-pvc-f87f399d -2025-06-05 19:56:45,033 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-f87f399d to be bound -2025-06-05 19:56:47,168 - tests.orchestrator - INFO - PVC concurrent-pvc-f87f399d is bound -2025-06-05 19:56:47,168 - tests.orchestrator - INFO - Successfully created 6 PVCs -2025-06-05 19:56:47,168 - tests.orchestrator - INFO - Creating pod test-pod-025216b5 on PVC concurrent-pvc-ab9c791a -2025-06-05 19:56:47,249 - tests.orchestrator - INFO - Created pod: test-pod-025216b5 using PVC: concurrent-pvc-ab9c791a -2025-06-05 19:56:47,249 - tests.orchestrator - INFO - Waiting for pod test-pod-025216b5 to be ready -2025-06-05 19:56:47,317 - tests.orchestrator - INFO - Pod test-pod-025216b5 phase: Pending -2025-06-05 19:56:49,385 - tests.orchestrator - INFO - Pod test-pod-025216b5 phase: Running -2025-06-05 19:56:49,385 - tests.orchestrator - INFO - Pod test-pod-025216b5 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:56:51,460 - tests.orchestrator - INFO - Pod test-pod-025216b5 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:56:53,529 - tests.orchestrator - INFO - Pod test-pod-025216b5 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:56:55,598 - tests.orchestrator - INFO - Pod test-pod-025216b5 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:56:57,666 - tests.orchestrator - INFO - Pod test-pod-025216b5 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:56:59,735 - tests.orchestrator - INFO - Pod test-pod-025216b5 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:57:01,809 - tests.orchestrator - INFO - Pod test-pod-025216b5 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:57:03,877 - tests.orchestrator - INFO - Pod test-pod-025216b5 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:57:05,946 - tests.orchestrator - INFO - Pod test-pod-025216b5 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:57:08,014 - tests.orchestrator - INFO - Pod test-pod-025216b5 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 19:57:08,014 - tests.orchestrator - INFO - Pod test-pod-025216b5 is ready -2025-06-05 19:57:08,014 - tests.orchestrator - INFO - Creating pod test-pod-96bcc503 on PVC concurrent-pvc-12483e38 -2025-06-05 19:57:08,097 - tests.orchestrator - INFO - Created pod: test-pod-96bcc503 using PVC: concurrent-pvc-12483e38 -2025-06-05 19:57:08,097 - tests.orchestrator - INFO - Waiting for pod test-pod-96bcc503 to be ready -2025-06-05 19:57:08,165 - tests.orchestrator - INFO - Pod test-pod-96bcc503 phase: Pending -2025-06-05 19:57:12,300 - tests.orchestrator - INFO - Pod test-pod-96bcc503 phase: Running -2025-06-05 19:57:12,300 - tests.orchestrator - INFO - Pod test-pod-96bcc503 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:57:14,370 - tests.orchestrator - INFO - Pod test-pod-96bcc503 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:57:16,442 - tests.orchestrator - INFO - Pod test-pod-96bcc503 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:57:18,510 - tests.orchestrator - INFO - Pod test-pod-96bcc503 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:57:20,578 - tests.orchestrator - INFO - Pod test-pod-96bcc503 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:57:22,646 - tests.orchestrator - INFO - Pod test-pod-96bcc503 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:57:24,715 - tests.orchestrator - INFO - Pod test-pod-96bcc503 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:57:26,783 - tests.orchestrator - INFO - Pod test-pod-96bcc503 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:57:28,852 - tests.orchestrator - INFO - Pod test-pod-96bcc503 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 19:57:28,852 - tests.orchestrator - INFO - Pod test-pod-96bcc503 is ready -2025-06-05 19:57:28,854 - tests.orchestrator - INFO - Creating pod test-pod-1bba039d on PVC concurrent-pvc-ee19f53f -2025-06-05 19:57:28,927 - tests.orchestrator - INFO - Created pod: test-pod-1bba039d using PVC: concurrent-pvc-ee19f53f -2025-06-05 19:57:28,927 - tests.orchestrator - INFO - Waiting for pod test-pod-1bba039d to be ready -2025-06-05 19:57:28,995 - tests.orchestrator - INFO - Pod test-pod-1bba039d phase: Pending -2025-06-05 19:57:33,132 - tests.orchestrator - INFO - Pod test-pod-1bba039d phase: Running -2025-06-05 19:57:33,132 - tests.orchestrator - INFO - Pod test-pod-1bba039d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:57:35,200 - tests.orchestrator - INFO - Pod test-pod-1bba039d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:57:37,270 - tests.orchestrator - INFO - Pod test-pod-1bba039d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:57:39,339 - tests.orchestrator - INFO - Pod test-pod-1bba039d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:57:41,409 - tests.orchestrator - INFO - Pod test-pod-1bba039d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:57:43,476 - tests.orchestrator - INFO - Pod test-pod-1bba039d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:57:45,545 - tests.orchestrator - INFO - Pod test-pod-1bba039d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:57:47,614 - tests.orchestrator - INFO - Pod test-pod-1bba039d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:57:49,682 - tests.orchestrator - INFO - Pod test-pod-1bba039d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 19:57:51,749 - tests.orchestrator - INFO - Pod test-pod-1bba039d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 19:57:51,749 - tests.orchestrator - INFO - Pod test-pod-1bba039d is ready -2025-06-05 19:57:51,749 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-cbac8cde -2025-06-05 19:57:51,821 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-cbac8cde to be deleted -2025-06-05 19:57:51,887 - tests.orchestrator - INFO - PVC concurrent-pvc-cbac8cde has been deleted -2025-06-05 19:57:51,887 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-cbac8cde -2025-06-05 19:57:51,887 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-12483e38 -2025-06-05 19:57:51,888 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-12483e38 -2025-06-05 19:57:51,888 - tests.orchestrator - INFO - Deleting pod: test-pod-96bcc503 from PVC: concurrent-pvc-12483e38 -2025-06-05 19:57:51,966 - tests.orchestrator - INFO - Waiting for pod test-pod-96bcc503 to be deleted -2025-06-05 19:58:24,579 - tests.orchestrator - INFO - Pod test-pod-96bcc503 has been deleted -2025-06-05 19:58:24,579 - tests.orchestrator - INFO - Deleted pod: test-pod-96bcc503 -2025-06-05 19:58:24,652 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-12483e38 to be deleted -2025-06-05 19:58:24,718 - tests.orchestrator - INFO - PVC concurrent-pvc-12483e38 has been deleted -2025-06-05 19:58:24,718 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-12483e38 -2025-06-05 19:58:24,718 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-ee19f53f -2025-06-05 19:58:24,718 - tests.orchestrator - INFO - Deleting 1 pods using PVC concurrent-pvc-ee19f53f -2025-06-05 19:58:24,718 - tests.orchestrator - INFO - Deleting pod: test-pod-1bba039d from PVC: concurrent-pvc-ee19f53f -2025-06-05 19:58:24,793 - tests.orchestrator - INFO - Waiting for pod test-pod-1bba039d to be deleted -2025-06-05 19:58:57,410 - tests.orchestrator - INFO - Pod test-pod-1bba039d has been deleted -2025-06-05 19:58:57,410 - tests.orchestrator - INFO - Deleted pod: test-pod-1bba039d -2025-06-05 19:58:57,483 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-ee19f53f to be deleted -2025-06-05 19:59:01,743 - tests.orchestrator - INFO - PVC concurrent-pvc-ee19f53f has been deleted -2025-06-05 19:59:01,743 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-ee19f53f -2025-06-05 19:59:01,743 - tests.orchestrator - INFO - Rapid PVC scenario completed -2025-06-05 19:59:01,743 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_concurrent_pvc -2025-06-05 19:59:01,743 - tests.orchestrator - INFO - ------------------------------------------------------------ -2025-06-05 19:59:02,743 - tests.orchestrator - INFO - Test completed in 396.86 seconds -2025-06-05 19:59:02,743 - tests.orchestrator - INFO - ===== STARTING COMPREHENSIVE CLEANUP ===== -2025-06-05 19:59:02,744 - tests.orchestrator - INFO - Deleting 4 pods (force=False)... -2025-06-05 19:59:02,744 - tests.orchestrator - INFO - Deleting pod: test-pod-28ec4f7d from PVC: test-pvc-b5f8638e -2025-06-05 19:59:02,820 - tests.orchestrator - INFO - Waiting for pod test-pod-28ec4f7d to be deleted -2025-06-05 19:59:35,451 - tests.orchestrator - INFO - Pod test-pod-28ec4f7d has been deleted -2025-06-05 19:59:35,451 - tests.orchestrator - INFO - Deleted pod: test-pod-28ec4f7d -2025-06-05 19:59:35,452 - tests.orchestrator - INFO - Deleting pod: test-pod-044934ff from PVC: test-pvc-b5f8638e -2025-06-05 19:59:35,528 - tests.orchestrator - INFO - Waiting for pod test-pod-044934ff to be deleted -2025-06-05 20:00:08,148 - tests.orchestrator - INFO - Pod test-pod-044934ff has been deleted -2025-06-05 20:00:08,148 - tests.orchestrator - INFO - Deleted pod: test-pod-044934ff -2025-06-05 20:00:08,148 - tests.orchestrator - INFO - Deleting pod: test-pod-a0c335d2 from PVC: concurrent-pvc-0edb68b1 -2025-06-05 20:00:08,225 - tests.orchestrator - INFO - Waiting for pod test-pod-a0c335d2 to be deleted -2025-06-05 20:00:40,838 - tests.orchestrator - INFO - Pod test-pod-a0c335d2 has been deleted -2025-06-05 20:00:40,838 - tests.orchestrator - INFO - Deleted pod: test-pod-a0c335d2 -2025-06-05 20:00:40,838 - tests.orchestrator - INFO - Deleting pod: test-pod-025216b5 from PVC: concurrent-pvc-ab9c791a -2025-06-05 20:00:40,913 - tests.orchestrator - INFO - Waiting for pod test-pod-025216b5 to be deleted -2025-06-05 20:01:13,524 - tests.orchestrator - INFO - Pod test-pod-025216b5 has been deleted -2025-06-05 20:01:13,524 - tests.orchestrator - INFO - Deleted pod: test-pod-025216b5 -2025-06-05 20:01:18,524 - tests.orchestrator - INFO - Deleting 6 PVCs (force=False)... -2025-06-05 20:01:18,524 - tests.orchestrator - INFO - Deleting PVC: test-pvc-b5f8638e -2025-06-05 20:01:18,598 - tests.orchestrator - INFO - Waiting for PVC test-pvc-b5f8638e to be deleted -2025-06-05 20:01:18,663 - tests.orchestrator - INFO - PVC test-pvc-b5f8638e has been deleted -2025-06-05 20:01:18,663 - tests.orchestrator - INFO - Deleted PVC: test-pvc-b5f8638e -2025-06-05 20:01:18,663 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-0edb68b1 -2025-06-05 20:01:18,736 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-0edb68b1 to be deleted -2025-06-05 20:01:18,801 - tests.orchestrator - INFO - PVC concurrent-pvc-0edb68b1 has been deleted -2025-06-05 20:01:18,801 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-0edb68b1 -2025-06-05 20:01:18,801 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-e4c09a13 -2025-06-05 20:01:18,875 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-e4c09a13 to be deleted -2025-06-05 20:01:18,940 - tests.orchestrator - INFO - PVC concurrent-pvc-e4c09a13 has been deleted -2025-06-05 20:01:18,941 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-e4c09a13 -2025-06-05 20:01:18,942 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-ab9c791a -2025-06-05 20:01:19,016 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-ab9c791a to be deleted -2025-06-05 20:01:19,081 - tests.orchestrator - INFO - PVC concurrent-pvc-ab9c791a has been deleted -2025-06-05 20:01:19,082 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-ab9c791a -2025-06-05 20:01:19,082 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-ac5d027e -2025-06-05 20:01:19,155 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-ac5d027e to be deleted -2025-06-05 20:01:19,219 - tests.orchestrator - INFO - PVC concurrent-pvc-ac5d027e has been deleted -2025-06-05 20:01:19,220 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-ac5d027e -2025-06-05 20:01:19,220 - tests.orchestrator - INFO - Deleting PVC: concurrent-pvc-f87f399d -2025-06-05 20:01:19,292 - tests.orchestrator - INFO - Waiting for PVC concurrent-pvc-f87f399d to be deleted -2025-06-05 20:01:19,357 - tests.orchestrator - INFO - PVC concurrent-pvc-f87f399d has been deleted -2025-06-05 20:01:19,357 - tests.orchestrator - INFO - Deleted PVC: concurrent-pvc-f87f399d -2025-06-05 20:01:19,556 - tests.orchestrator - WARNING - First cleanup pass incomplete. Remaining resources: ['pod/test-pod-0d0fddfe', 'pod/test-pod-158cdfa4', 'pvc/test-pvc-0bb0d20c', 'pvc/test-pvc-2127e71a', 'pvc/test-pvc-2cf5b7a2', 'pvc/test-pvc-31fbd501', 'pvc/test-pvc-528b4f5c', 'pvc/test-pvc-ecb12f74'] -2025-06-05 20:01:19,557 - tests.orchestrator - INFO - Attempting force deletion of remaining resources... -2025-06-05 20:01:19,557 - tests.orchestrator - INFO - Deleting 0 pods (force=True)... -2025-06-05 20:01:24,557 - tests.orchestrator - INFO - Deleting 0 PVCs (force=True)... -2025-06-05 20:01:24,754 - tests.orchestrator - ERROR - Cleanup incomplete. Remaining resources after force deletion: ['pod/test-pod-0d0fddfe', 'pod/test-pod-158cdfa4', 'pvc/test-pvc-0bb0d20c', 'pvc/test-pvc-2127e71a', 'pvc/test-pvc-2cf5b7a2', 'pvc/test-pvc-31fbd501', 'pvc/test-pvc-528b4f5c', 'pvc/test-pvc-ecb12f74'] -2025-06-05 20:01:24,755 - tests.orchestrator - INFO - Cleanup completed successfully in 142.01 seconds -2025-06-05 20:01:24,755 - tests.orchestrator - INFO - ===== CLEANUP PROCESS FINISHED ===== -2025-06-05 20:01:24,755 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-05 20:01:24,755 - tests.orchestrator - INFO - --- Operations --- -2025-06-05 20:01:24,755 - tests.orchestrator - INFO - create_pvc: 13 succeeded, 0 failed (100.0%) -2025-06-05 20:01:24,755 - tests.orchestrator - INFO - attach_pod: 9 succeeded, 0 failed (100.0%) -2025-06-05 20:01:24,755 - tests.orchestrator - INFO - delete_pod: 9 succeeded, 0 failed (100.0%) -2025-06-05 20:01:24,755 - tests.orchestrator - INFO - delete_pvc: 13 succeeded, 0 failed (100.0%) -2025-06-05 20:01:24,755 - tests.orchestrator - INFO - verify_read_write: Writes 2 succeeded, 0 failed (100.0%) -2025-06-05 20:01:24,755 - tests.orchestrator - INFO - verify_read_write: Reads 2 succeeded, 0 failed (100.0%) -2025-06-05 20:01:24,755 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-05 20:01:24,755 - tests.orchestrator - INFO - shared_volume_rw: 2 succeeded, 0 failed out of 2 runs (100.0%) -2025-06-05 20:01:24,755 - tests.orchestrator - INFO - many_to_one: No runs -2025-06-05 20:01:24,755 - tests.orchestrator - INFO - one_to_one: No runs -2025-06-05 20:01:24,755 - tests.orchestrator - INFO - concurrent_pvc: 2 succeeded, 0 failed out of 2 runs (100.0%) -2025-06-05 20:01:24,755 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_200645.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_200645.log deleted file mode 100644 index 1aefe7d2e..000000000 --- a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_200645.log +++ /dev/null @@ -1,348 +0,0 @@ -2025-06-05 20:06:45,933 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 20:06:46,152 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 20:06:46,152 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-05 20:06:46,343 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 20:06:46,343 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 20:06:46,343 - tests.orchestrator - INFO - Creating PVC: test-pvc-779d01ad -2025-06-05 20:06:46,420 - tests.orchestrator - INFO - Created PVC: test-pvc-779d01ad -2025-06-05 20:06:46,420 - tests.orchestrator - INFO - Waiting for PVC test-pvc-779d01ad to be bound -2025-06-05 20:06:48,554 - tests.orchestrator - INFO - PVC test-pvc-779d01ad is bound -2025-06-05 20:06:48,554 - tests.orchestrator - INFO - Creating PVC: test-pvc-69cf206a -2025-06-05 20:06:48,623 - tests.orchestrator - INFO - Created PVC: test-pvc-69cf206a -2025-06-05 20:06:48,624 - tests.orchestrator - INFO - Waiting for PVC test-pvc-69cf206a to be bound -2025-06-05 20:06:50,757 - tests.orchestrator - INFO - PVC test-pvc-69cf206a is bound -2025-06-05 20:06:50,757 - tests.orchestrator - INFO - Creating pod test-pod-88b786b1 on PVC test-pvc-69cf206a -2025-06-05 20:06:50,838 - tests.orchestrator - INFO - Created pod: test-pod-88b786b1 using PVC: test-pvc-69cf206a -2025-06-05 20:06:50,838 - tests.orchestrator - INFO - Waiting for pod test-pod-88b786b1 to be ready -2025-06-05 20:06:50,907 - tests.orchestrator - INFO - Pod test-pod-88b786b1 phase: Pending -2025-06-05 20:06:55,042 - tests.orchestrator - INFO - Pod test-pod-88b786b1 phase: Running -2025-06-05 20:06:55,043 - tests.orchestrator - INFO - Pod test-pod-88b786b1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:06:57,112 - tests.orchestrator - INFO - Pod test-pod-88b786b1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:06:59,189 - tests.orchestrator - INFO - Pod test-pod-88b786b1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:07:01,258 - tests.orchestrator - INFO - Pod test-pod-88b786b1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:07:03,327 - tests.orchestrator - INFO - Pod test-pod-88b786b1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:07:05,395 - tests.orchestrator - INFO - Pod test-pod-88b786b1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:07:07,466 - tests.orchestrator - INFO - Pod test-pod-88b786b1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:07:09,535 - tests.orchestrator - INFO - Pod test-pod-88b786b1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:07:11,603 - tests.orchestrator - INFO - Pod test-pod-88b786b1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 20:07:11,603 - tests.orchestrator - INFO - Pod test-pod-88b786b1 is ready -2025-06-05 20:07:11,603 - tests.orchestrator - INFO - Creating pod test-pod-da70686e on PVC test-pvc-779d01ad -2025-06-05 20:07:11,677 - tests.orchestrator - INFO - Created pod: test-pod-da70686e using PVC: test-pvc-779d01ad -2025-06-05 20:07:11,677 - tests.orchestrator - INFO - Waiting for pod test-pod-da70686e to be ready -2025-06-05 20:07:11,745 - tests.orchestrator - INFO - Pod test-pod-da70686e phase: Pending -2025-06-05 20:07:15,881 - tests.orchestrator - INFO - Pod test-pod-da70686e phase: Running -2025-06-05 20:07:15,881 - tests.orchestrator - INFO - Pod test-pod-da70686e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:07:17,949 - tests.orchestrator - INFO - Pod test-pod-da70686e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:07:20,018 - tests.orchestrator - INFO - Pod test-pod-da70686e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:07:22,087 - tests.orchestrator - INFO - Pod test-pod-da70686e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:07:24,155 - tests.orchestrator - INFO - Pod test-pod-da70686e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:07:26,224 - tests.orchestrator - INFO - Pod test-pod-da70686e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:07:28,293 - tests.orchestrator - INFO - Pod test-pod-da70686e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:07:30,361 - tests.orchestrator - INFO - Pod test-pod-da70686e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:07:32,429 - tests.orchestrator - INFO - Pod test-pod-da70686e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:07:34,498 - tests.orchestrator - INFO - Pod test-pod-da70686e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 20:07:34,498 - tests.orchestrator - INFO - Pod test-pod-da70686e is ready -2025-06-05 20:07:34,498 - tests.orchestrator - INFO - Creating pod test-pod-a0822d95 on PVC test-pvc-779d01ad -2025-06-05 20:07:34,580 - tests.orchestrator - INFO - Created pod: test-pod-a0822d95 using PVC: test-pvc-779d01ad -2025-06-05 20:07:34,580 - tests.orchestrator - INFO - Waiting for pod test-pod-a0822d95 to be ready -2025-06-05 20:07:34,649 - tests.orchestrator - INFO - Pod test-pod-a0822d95 phase: Pending -2025-06-05 20:07:36,718 - tests.orchestrator - INFO - Pod test-pod-a0822d95 phase: Running -2025-06-05 20:07:36,718 - tests.orchestrator - INFO - Pod test-pod-a0822d95 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:07:38,787 - tests.orchestrator - INFO - Pod test-pod-a0822d95 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:07:40,855 - tests.orchestrator - INFO - Pod test-pod-a0822d95 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:07:42,923 - tests.orchestrator - INFO - Pod test-pod-a0822d95 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:07:44,992 - tests.orchestrator - INFO - Pod test-pod-a0822d95 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:07:47,060 - tests.orchestrator - INFO - Pod test-pod-a0822d95 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:07:49,131 - tests.orchestrator - INFO - Pod test-pod-a0822d95 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:07:51,201 - tests.orchestrator - INFO - Pod test-pod-a0822d95 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:07:53,268 - tests.orchestrator - INFO - Pod test-pod-a0822d95 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:07:55,337 - tests.orchestrator - INFO - Pod test-pod-a0822d95 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:07:57,405 - tests.orchestrator - INFO - Pod test-pod-a0822d95 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 20:07:57,405 - tests.orchestrator - INFO - Pod test-pod-a0822d95 is ready -2025-06-05 20:07:57,406 - tests.orchestrator - INFO - Testing read/write between pods test-pod-da70686e and test-pod-a0822d95 sharing PVC test-pvc-779d01ad -2025-06-05 20:07:57,406 - tests.orchestrator - INFO - Executing write command: kubectl exec -n efs-stress-test test-pod-da70686e -- /bin/sh -c 'echo "Test content: e5a1b64d-ce63-4074-93b0-af11de5623c7" > /data/test-85c4aefc.txt' -2025-06-05 20:07:58,672 - tests.orchestrator - INFO - Pod test-pod-da70686e wrote to /data/test-85c4aefc.txt -2025-06-05 20:08:00,672 - tests.orchestrator - INFO - Executing read command: kubectl exec -n efs-stress-test test-pod-a0822d95 -- cat /data/test-85c4aefc.txt -2025-06-05 20:08:01,892 - tests.orchestrator - INFO - Read result: 'Test content: e5a1b64d-ce63-4074-93b0-af11de5623c7' -2025-06-05 20:08:01,892 - tests.orchestrator - INFO - Pod test-pod-a0822d95 successfully read content written by test-pod-da70686e -2025-06-05 20:08:01,892 - tests.orchestrator - INFO - Deleting pod: test-pod-88b786b1 from PVC: test-pvc-69cf206a -2025-06-05 20:08:01,981 - tests.orchestrator - INFO - Waiting for pod test-pod-88b786b1 to be deleted -2025-06-05 20:08:34,617 - tests.orchestrator - INFO - Pod test-pod-88b786b1 has been deleted -2025-06-05 20:08:34,617 - tests.orchestrator - INFO - Deleted pod: test-pod-88b786b1 -2025-06-05 20:08:34,617 - tests.orchestrator - INFO - Deleting PVC: test-pvc-779d01ad -2025-06-05 20:08:34,617 - tests.orchestrator - INFO - Deleting 2 pods using PVC test-pvc-779d01ad -2025-06-05 20:08:34,617 - tests.orchestrator - INFO - Deleting pod: test-pod-da70686e from PVC: test-pvc-779d01ad -2025-06-05 20:08:34,693 - tests.orchestrator - INFO - Waiting for pod test-pod-da70686e to be deleted -2025-06-05 20:09:07,311 - tests.orchestrator - INFO - Pod test-pod-da70686e has been deleted -2025-06-05 20:09:07,311 - tests.orchestrator - INFO - Deleted pod: test-pod-da70686e -2025-06-05 20:09:07,311 - tests.orchestrator - INFO - Deleting pod: test-pod-a0822d95 from PVC: test-pvc-779d01ad -2025-06-05 20:09:07,387 - tests.orchestrator - INFO - Waiting for pod test-pod-a0822d95 to be deleted -2025-06-05 20:09:39,999 - tests.orchestrator - INFO - Pod test-pod-a0822d95 has been deleted -2025-06-05 20:09:39,999 - tests.orchestrator - INFO - Deleted pod: test-pod-a0822d95 -2025-06-05 20:09:40,073 - tests.orchestrator - INFO - Waiting for PVC test-pvc-779d01ad to be deleted -2025-06-05 20:09:44,206 - tests.orchestrator - INFO - PVC test-pvc-779d01ad has been deleted -2025-06-05 20:09:44,206 - tests.orchestrator - INFO - Deleted PVC: test-pvc-779d01ad -2025-06-05 20:09:44,206 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations -2025-06-05 20:09:44,206 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario (selected 1 times) -2025-06-05 20:09:44,206 - tests.orchestrator - INFO - ============================================================ -2025-06-05 20:09:44,206 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_many_to_one -2025-06-05 20:09:44,206 - tests.orchestrator - INFO - ============================================================ -2025-06-05 20:09:44,206 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -2025-06-05 20:09:44,206 - tests.orchestrator - INFO - STARTING MANY-TO-ONE SCENARIO DIAGNOSTICS -2025-06-05 20:09:44,206 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -2025-06-05 20:09:44,206 - tests.orchestrator - INFO - [MANY2ONE] STEP 1: Creating dedicated PVC: many2one-45c123ce -2025-06-05 20:09:44,206 - tests.orchestrator - INFO - [MANY2ONE] Creating PVC manifest with storageClassName: efs-sc -2025-06-05 20:09:44,285 - tests.orchestrator - INFO - [MANY2ONE] PVC many2one-45c123ce created successfully -2025-06-05 20:09:44,285 - tests.orchestrator - INFO - [MANY2ONE] Waiting for PVC many2one-45c123ce to be bound... -2025-06-05 20:09:44,285 - tests.orchestrator - INFO - Waiting for PVC many2one-45c123ce to be bound -2025-06-05 20:09:46,420 - tests.orchestrator - INFO - PVC many2one-45c123ce is bound -2025-06-05 20:09:46,489 - tests.orchestrator - INFO - [MANY2ONE] PVC status: phase=Bound, capacity={'storage': '1Gi'} -2025-06-05 20:09:46,489 - tests.orchestrator - INFO - [MANY2ONE] STEP 2: Creating 4 pods for the same PVC many2one-45c123ce -2025-06-05 20:09:46,489 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 1/4 for PVC many2one-45c123ce -2025-06-05 20:09:46,489 - tests.orchestrator - INFO - Creating pod test-pod-d7eb09ac on PVC many2one-45c123ce -2025-06-05 20:09:46,575 - tests.orchestrator - INFO - Created pod: test-pod-d7eb09ac using PVC: many2one-45c123ce -2025-06-05 20:09:46,576 - tests.orchestrator - INFO - Waiting for pod test-pod-d7eb09ac to be ready -2025-06-05 20:09:46,644 - tests.orchestrator - INFO - Pod test-pod-d7eb09ac phase: Pending -2025-06-05 20:09:48,712 - tests.orchestrator - INFO - Pod test-pod-d7eb09ac phase: Running -2025-06-05 20:09:48,712 - tests.orchestrator - INFO - Pod test-pod-d7eb09ac conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:09:50,781 - tests.orchestrator - INFO - Pod test-pod-d7eb09ac conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:09:52,850 - tests.orchestrator - INFO - Pod test-pod-d7eb09ac conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:09:54,918 - tests.orchestrator - INFO - Pod test-pod-d7eb09ac conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:09:56,986 - tests.orchestrator - INFO - Pod test-pod-d7eb09ac conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:09:59,055 - tests.orchestrator - INFO - Pod test-pod-d7eb09ac conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:10:01,123 - tests.orchestrator - INFO - Pod test-pod-d7eb09ac conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:10:03,193 - tests.orchestrator - INFO - Pod test-pod-d7eb09ac conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 20:10:03,193 - tests.orchestrator - INFO - Pod test-pod-d7eb09ac is ready -2025-06-05 20:10:03,193 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-d7eb09ac -2025-06-05 20:10:03,193 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 2/4 for PVC many2one-45c123ce -2025-06-05 20:10:03,193 - tests.orchestrator - INFO - Creating pod test-pod-440ae01d on PVC many2one-45c123ce -2025-06-05 20:10:03,268 - tests.orchestrator - INFO - Created pod: test-pod-440ae01d using PVC: many2one-45c123ce -2025-06-05 20:10:03,268 - tests.orchestrator - INFO - Waiting for pod test-pod-440ae01d to be ready -2025-06-05 20:10:03,337 - tests.orchestrator - INFO - Pod test-pod-440ae01d phase: Pending -2025-06-05 20:10:07,474 - tests.orchestrator - INFO - Pod test-pod-440ae01d phase: Running -2025-06-05 20:10:07,474 - tests.orchestrator - INFO - Pod test-pod-440ae01d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:10:09,543 - tests.orchestrator - INFO - Pod test-pod-440ae01d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:10:11,611 - tests.orchestrator - INFO - Pod test-pod-440ae01d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:10:13,679 - tests.orchestrator - INFO - Pod test-pod-440ae01d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:10:15,747 - tests.orchestrator - INFO - Pod test-pod-440ae01d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:10:17,816 - tests.orchestrator - INFO - Pod test-pod-440ae01d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:10:19,885 - tests.orchestrator - INFO - Pod test-pod-440ae01d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 20:10:19,885 - tests.orchestrator - INFO - Pod test-pod-440ae01d is ready -2025-06-05 20:10:19,885 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-440ae01d -2025-06-05 20:10:19,885 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 3/4 for PVC many2one-45c123ce -2025-06-05 20:10:19,885 - tests.orchestrator - INFO - Creating pod test-pod-d651a57e on PVC many2one-45c123ce -2025-06-05 20:10:19,968 - tests.orchestrator - INFO - Created pod: test-pod-d651a57e using PVC: many2one-45c123ce -2025-06-05 20:10:19,968 - tests.orchestrator - INFO - Waiting for pod test-pod-d651a57e to be ready -2025-06-05 20:10:20,036 - tests.orchestrator - INFO - Pod test-pod-d651a57e phase: Pending -2025-06-05 20:10:22,105 - tests.orchestrator - INFO - Pod test-pod-d651a57e phase: Running -2025-06-05 20:10:22,105 - tests.orchestrator - INFO - Pod test-pod-d651a57e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:10:24,174 - tests.orchestrator - INFO - Pod test-pod-d651a57e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:10:26,242 - tests.orchestrator - INFO - Pod test-pod-d651a57e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:10:28,310 - tests.orchestrator - INFO - Pod test-pod-d651a57e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:10:30,379 - tests.orchestrator - INFO - Pod test-pod-d651a57e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:10:32,447 - tests.orchestrator - INFO - Pod test-pod-d651a57e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:10:34,515 - tests.orchestrator - INFO - Pod test-pod-d651a57e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:10:36,584 - tests.orchestrator - INFO - Pod test-pod-d651a57e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:10:38,652 - tests.orchestrator - INFO - Pod test-pod-d651a57e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:10:40,720 - tests.orchestrator - INFO - Pod test-pod-d651a57e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 20:10:40,720 - tests.orchestrator - INFO - Pod test-pod-d651a57e is ready -2025-06-05 20:10:40,720 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-d651a57e -2025-06-05 20:10:40,721 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 4/4 for PVC many2one-45c123ce -2025-06-05 20:10:40,721 - tests.orchestrator - INFO - Creating pod test-pod-3929c90d on PVC many2one-45c123ce -2025-06-05 20:10:40,796 - tests.orchestrator - INFO - Created pod: test-pod-3929c90d using PVC: many2one-45c123ce -2025-06-05 20:10:40,796 - tests.orchestrator - INFO - Waiting for pod test-pod-3929c90d to be ready -2025-06-05 20:10:40,864 - tests.orchestrator - INFO - Pod test-pod-3929c90d phase: Pending -2025-06-05 20:10:45,001 - tests.orchestrator - INFO - Pod test-pod-3929c90d phase: Running -2025-06-05 20:10:45,001 - tests.orchestrator - INFO - Pod test-pod-3929c90d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:10:47,070 - tests.orchestrator - INFO - Pod test-pod-3929c90d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:10:49,137 - tests.orchestrator - INFO - Pod test-pod-3929c90d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:10:51,206 - tests.orchestrator - INFO - Pod test-pod-3929c90d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:10:53,275 - tests.orchestrator - INFO - Pod test-pod-3929c90d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:10:55,343 - tests.orchestrator - INFO - Pod test-pod-3929c90d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:10:57,412 - tests.orchestrator - INFO - Pod test-pod-3929c90d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:10:59,481 - tests.orchestrator - INFO - Pod test-pod-3929c90d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:11:01,576 - tests.orchestrator - INFO - Pod test-pod-3929c90d conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 20:11:01,576 - tests.orchestrator - INFO - Pod test-pod-3929c90d is ready -2025-06-05 20:11:01,576 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-3929c90d -2025-06-05 20:11:01,576 - tests.orchestrator - INFO - [MANY2ONE] Created 4/4 pods successfully -2025-06-05 20:11:01,576 - tests.orchestrator - INFO - [MANY2ONE] STEP 3: Testing read/write operations -2025-06-05 20:11:01,576 - tests.orchestrator - INFO - [MANY2ONE] Writer pod: test-pod-440ae01d, Reader pod: test-pod-d651a57e -2025-06-05 20:11:01,576 - tests.orchestrator - INFO - [MANY2ONE] Test file: /data/many2one-c2ee41a1.txt -2025-06-05 20:11:01,576 - tests.orchestrator - INFO - [MANY2ONE] Test content: Many2One test content: 036ddf73-684a-4b98-b06f-4292a885f843 -2025-06-05 20:11:01,576 - tests.orchestrator - INFO - [MANY2ONE] Executing write command: kubectl exec -n efs-stress-test test-pod-440ae01d -- /bin/sh -c 'echo "Many2One test content: 036ddf73-684a-4b98-b06f-4292a885f843" > /data/many2one-c2ee41a1.txt' -2025-06-05 20:11:03,979 - tests.orchestrator - INFO - [MANY2ONE] Writer pod ls check: '-rw-r--r-- 1 50006 50006 60 Jun 5 20:11 /data/many2one-c2ee41a1.txt' -2025-06-05 20:11:03,979 - tests.orchestrator - INFO - [MANY2ONE] Sleeping for 5 seconds to ensure filesystem sync -2025-06-05 20:11:08,979 - tests.orchestrator - INFO - [MANY2ONE] Executing read command: kubectl exec -n efs-stress-test test-pod-d651a57e -- cat /data/many2one-c2ee41a1.txt -2025-06-05 20:11:10,142 - tests.orchestrator - INFO - [MANY2ONE] Read command result: 'Many2One test content: 036ddf73-684a-4b98-b06f-4292a885f843' -2025-06-05 20:11:10,142 - tests.orchestrator - INFO - [MANY2ONE] SUCCESS: Many-to-one scenario successful with 4 pods -2025-06-05 20:11:10,142 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -2025-06-05 20:11:10,142 - tests.orchestrator - INFO - COMPLETED MANY-TO-ONE SCENARIO DIAGNOSTICS -2025-06-05 20:11:10,142 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -2025-06-05 20:11:10,142 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_many_to_one -2025-06-05 20:11:10,142 - tests.orchestrator - INFO - ------------------------------------------------------------ -2025-06-05 20:11:11,143 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 1 times) -2025-06-05 20:11:11,143 - tests.orchestrator - INFO - Creating pod test-pod-3f02ab55 on PVC many2one-45c123ce -2025-06-05 20:11:11,230 - tests.orchestrator - INFO - Created pod: test-pod-3f02ab55 using PVC: many2one-45c123ce -2025-06-05 20:11:11,230 - tests.orchestrator - INFO - Waiting for pod test-pod-3f02ab55 to be ready -2025-06-05 20:11:11,298 - tests.orchestrator - INFO - Pod test-pod-3f02ab55 phase: Pending -2025-06-05 20:11:15,434 - tests.orchestrator - INFO - Pod test-pod-3f02ab55 phase: Running -2025-06-05 20:11:15,434 - tests.orchestrator - INFO - Pod test-pod-3f02ab55 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:11:17,502 - tests.orchestrator - INFO - Pod test-pod-3f02ab55 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:11:19,571 - tests.orchestrator - INFO - Pod test-pod-3f02ab55 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:11:21,639 - tests.orchestrator - INFO - Pod test-pod-3f02ab55 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:11:23,708 - tests.orchestrator - INFO - Pod test-pod-3f02ab55 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:11:25,777 - tests.orchestrator - INFO - Pod test-pod-3f02ab55 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:11:27,844 - tests.orchestrator - INFO - Pod test-pod-3f02ab55 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:11:29,913 - tests.orchestrator - INFO - Pod test-pod-3f02ab55 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:11:31,984 - tests.orchestrator - INFO - Pod test-pod-3f02ab55 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 20:11:31,984 - tests.orchestrator - INFO - Pod test-pod-3f02ab55 is ready -2025-06-05 20:11:32,984 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario (selected 2 times) -2025-06-05 20:11:32,985 - tests.orchestrator - INFO - ============================================================ -2025-06-05 20:11:32,985 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_many_to_one -2025-06-05 20:11:32,985 - tests.orchestrator - INFO - ============================================================ -2025-06-05 20:11:32,985 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -2025-06-05 20:11:32,985 - tests.orchestrator - INFO - STARTING MANY-TO-ONE SCENARIO DIAGNOSTICS -2025-06-05 20:11:32,985 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -2025-06-05 20:11:32,985 - tests.orchestrator - INFO - [MANY2ONE] STEP 1: Creating dedicated PVC: many2one-89ac31d4 -2025-06-05 20:11:32,985 - tests.orchestrator - INFO - [MANY2ONE] Creating PVC manifest with storageClassName: efs-sc -2025-06-05 20:11:33,054 - tests.orchestrator - INFO - [MANY2ONE] PVC many2one-89ac31d4 created successfully -2025-06-05 20:11:33,054 - tests.orchestrator - INFO - [MANY2ONE] Waiting for PVC many2one-89ac31d4 to be bound... -2025-06-05 20:11:33,054 - tests.orchestrator - INFO - Waiting for PVC many2one-89ac31d4 to be bound -2025-06-05 20:11:35,189 - tests.orchestrator - INFO - PVC many2one-89ac31d4 is bound -2025-06-05 20:11:35,256 - tests.orchestrator - INFO - [MANY2ONE] PVC status: phase=Bound, capacity={'storage': '1Gi'} -2025-06-05 20:11:35,256 - tests.orchestrator - INFO - [MANY2ONE] STEP 2: Creating 3 pods for the same PVC many2one-89ac31d4 -2025-06-05 20:11:35,256 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 1/3 for PVC many2one-89ac31d4 -2025-06-05 20:11:35,256 - tests.orchestrator - INFO - Creating pod test-pod-09cac2b7 on PVC many2one-89ac31d4 -2025-06-05 20:11:35,328 - tests.orchestrator - INFO - Created pod: test-pod-09cac2b7 using PVC: many2one-89ac31d4 -2025-06-05 20:11:35,328 - tests.orchestrator - INFO - Waiting for pod test-pod-09cac2b7 to be ready -2025-06-05 20:11:35,395 - tests.orchestrator - INFO - Pod test-pod-09cac2b7 phase: Pending -2025-06-05 20:11:37,464 - tests.orchestrator - INFO - Pod test-pod-09cac2b7 phase: Running -2025-06-05 20:11:37,464 - tests.orchestrator - INFO - Pod test-pod-09cac2b7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:11:39,533 - tests.orchestrator - INFO - Pod test-pod-09cac2b7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:11:41,600 - tests.orchestrator - INFO - Pod test-pod-09cac2b7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:11:43,668 - tests.orchestrator - INFO - Pod test-pod-09cac2b7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:11:45,737 - tests.orchestrator - INFO - Pod test-pod-09cac2b7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:11:47,805 - tests.orchestrator - INFO - Pod test-pod-09cac2b7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:11:49,873 - tests.orchestrator - INFO - Pod test-pod-09cac2b7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:11:51,942 - tests.orchestrator - INFO - Pod test-pod-09cac2b7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:11:54,009 - tests.orchestrator - INFO - Pod test-pod-09cac2b7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:11:56,078 - tests.orchestrator - INFO - Pod test-pod-09cac2b7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 20:11:56,078 - tests.orchestrator - INFO - Pod test-pod-09cac2b7 is ready -2025-06-05 20:11:56,078 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-09cac2b7 -2025-06-05 20:11:56,078 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 2/3 for PVC many2one-89ac31d4 -2025-06-05 20:11:56,078 - tests.orchestrator - INFO - Creating pod test-pod-fc15ec0f on PVC many2one-89ac31d4 -2025-06-05 20:11:56,163 - tests.orchestrator - INFO - Created pod: test-pod-fc15ec0f using PVC: many2one-89ac31d4 -2025-06-05 20:11:56,163 - tests.orchestrator - INFO - Waiting for pod test-pod-fc15ec0f to be ready -2025-06-05 20:11:56,231 - tests.orchestrator - INFO - Pod test-pod-fc15ec0f phase: Pending -2025-06-05 20:12:00,366 - tests.orchestrator - INFO - Pod test-pod-fc15ec0f phase: Running -2025-06-05 20:12:00,366 - tests.orchestrator - INFO - Pod test-pod-fc15ec0f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:12:02,435 - tests.orchestrator - INFO - Pod test-pod-fc15ec0f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:12:04,503 - tests.orchestrator - INFO - Pod test-pod-fc15ec0f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:12:06,575 - tests.orchestrator - INFO - Pod test-pod-fc15ec0f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:12:08,643 - tests.orchestrator - INFO - Pod test-pod-fc15ec0f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:12:10,712 - tests.orchestrator - INFO - Pod test-pod-fc15ec0f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:12:12,780 - tests.orchestrator - INFO - Pod test-pod-fc15ec0f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 20:12:12,780 - tests.orchestrator - INFO - Pod test-pod-fc15ec0f is ready -2025-06-05 20:12:12,780 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-fc15ec0f -2025-06-05 20:12:12,780 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 3/3 for PVC many2one-89ac31d4 -2025-06-05 20:12:12,780 - tests.orchestrator - INFO - Creating pod test-pod-5a86841b on PVC many2one-89ac31d4 -2025-06-05 20:12:12,855 - tests.orchestrator - INFO - Created pod: test-pod-5a86841b using PVC: many2one-89ac31d4 -2025-06-05 20:12:12,855 - tests.orchestrator - INFO - Waiting for pod test-pod-5a86841b to be ready -2025-06-05 20:12:12,923 - tests.orchestrator - INFO - Pod test-pod-5a86841b phase: Pending -2025-06-05 20:12:17,061 - tests.orchestrator - INFO - Pod test-pod-5a86841b phase: Running -2025-06-05 20:12:17,061 - tests.orchestrator - INFO - Pod test-pod-5a86841b conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:12:19,128 - tests.orchestrator - INFO - Pod test-pod-5a86841b conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:12:21,196 - tests.orchestrator - INFO - Pod test-pod-5a86841b conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:12:23,265 - tests.orchestrator - INFO - Pod test-pod-5a86841b conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:12:25,333 - tests.orchestrator - INFO - Pod test-pod-5a86841b conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:12:27,401 - tests.orchestrator - INFO - Pod test-pod-5a86841b conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:12:29,470 - tests.orchestrator - INFO - Pod test-pod-5a86841b conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:12:31,563 - tests.orchestrator - INFO - Pod test-pod-5a86841b conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 20:12:33,631 - tests.orchestrator - INFO - Pod test-pod-5a86841b conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 20:12:33,631 - tests.orchestrator - INFO - Pod test-pod-5a86841b is ready -2025-06-05 20:12:33,631 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-5a86841b -2025-06-05 20:12:33,631 - tests.orchestrator - INFO - [MANY2ONE] Created 3/3 pods successfully -2025-06-05 20:12:33,631 - tests.orchestrator - INFO - [MANY2ONE] STEP 3: Testing read/write operations -2025-06-05 20:12:33,631 - tests.orchestrator - INFO - [MANY2ONE] Writer pod: test-pod-09cac2b7, Reader pod: test-pod-5a86841b -2025-06-05 20:12:33,631 - tests.orchestrator - INFO - [MANY2ONE] Test file: /data/many2one-9e3adf41.txt -2025-06-05 20:12:33,631 - tests.orchestrator - INFO - [MANY2ONE] Test content: Many2One test content: eeb4c284-9658-4254-984e-6a8c4fda8c94 -2025-06-05 20:12:33,632 - tests.orchestrator - INFO - [MANY2ONE] Executing write command: kubectl exec -n efs-stress-test test-pod-09cac2b7 -- /bin/sh -c 'echo "Many2One test content: eeb4c284-9658-4254-984e-6a8c4fda8c94" > /data/many2one-9e3adf41.txt' -2025-06-05 20:12:36,048 - tests.orchestrator - INFO - [MANY2ONE] Writer pod ls check: '-rw-r--r-- 1 50014 50014 60 Jun 5 20:12 /data/many2one-9e3adf41.txt' -2025-06-05 20:12:36,048 - tests.orchestrator - INFO - [MANY2ONE] Sleeping for 5 seconds to ensure filesystem sync -2025-06-05 20:12:41,048 - tests.orchestrator - INFO - [MANY2ONE] Executing read command: kubectl exec -n efs-stress-test test-pod-5a86841b -- cat /data/many2one-9e3adf41.txt -2025-06-05 20:12:42,254 - tests.orchestrator - INFO - [MANY2ONE] Read command result: 'Many2One test content: eeb4c284-9658-4254-984e-6a8c4fda8c94' -2025-06-05 20:12:42,254 - tests.orchestrator - INFO - [MANY2ONE] SUCCESS: Many-to-one scenario successful with 3 pods -2025-06-05 20:12:42,254 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -2025-06-05 20:12:42,254 - tests.orchestrator - INFO - COMPLETED MANY-TO-ONE SCENARIO DIAGNOSTICS -2025-06-05 20:12:42,254 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -2025-06-05 20:12:42,254 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_many_to_one -2025-06-05 20:12:42,254 - tests.orchestrator - INFO - ------------------------------------------------------------ -2025-06-05 20:12:43,255 - tests.orchestrator - INFO - Test completed in 357.10 seconds -2025-06-05 20:12:43,255 - tests.orchestrator - INFO - ===== STARTING COMPREHENSIVE CLEANUP ===== -2025-06-05 20:12:43,255 - tests.orchestrator - INFO - Deleting 8 pods (force=False)... -2025-06-05 20:12:43,255 - tests.orchestrator - INFO - Deleting pod: test-pod-d7eb09ac from PVC: many2one-45c123ce -2025-06-05 20:12:43,331 - tests.orchestrator - INFO - Waiting for pod test-pod-d7eb09ac to be deleted -2025-06-05 20:13:15,956 - tests.orchestrator - INFO - Pod test-pod-d7eb09ac has been deleted -2025-06-05 20:13:15,956 - tests.orchestrator - INFO - Deleted pod: test-pod-d7eb09ac -2025-06-05 20:13:15,956 - tests.orchestrator - INFO - Deleting pod: test-pod-440ae01d from PVC: many2one-45c123ce -2025-06-05 20:13:16,032 - tests.orchestrator - INFO - Waiting for pod test-pod-440ae01d to be deleted -2025-06-05 20:13:48,650 - tests.orchestrator - INFO - Pod test-pod-440ae01d has been deleted -2025-06-05 20:13:48,650 - tests.orchestrator - INFO - Deleted pod: test-pod-440ae01d -2025-06-05 20:13:48,650 - tests.orchestrator - INFO - Deleting pod: test-pod-d651a57e from PVC: many2one-45c123ce -2025-06-05 20:13:48,727 - tests.orchestrator - INFO - Waiting for pod test-pod-d651a57e to be deleted -2025-06-05 20:14:21,356 - tests.orchestrator - INFO - Pod test-pod-d651a57e has been deleted -2025-06-05 20:14:21,356 - tests.orchestrator - INFO - Deleted pod: test-pod-d651a57e -2025-06-05 20:14:21,356 - tests.orchestrator - INFO - Deleting pod: test-pod-3929c90d from PVC: many2one-45c123ce -2025-06-05 20:14:21,432 - tests.orchestrator - INFO - Waiting for pod test-pod-3929c90d to be deleted -2025-06-05 20:14:54,051 - tests.orchestrator - INFO - Pod test-pod-3929c90d has been deleted -2025-06-05 20:14:54,051 - tests.orchestrator - INFO - Deleted pod: test-pod-3929c90d -2025-06-05 20:14:54,051 - tests.orchestrator - INFO - Deleting pod: test-pod-3f02ab55 from PVC: many2one-45c123ce -2025-06-05 20:14:54,127 - tests.orchestrator - INFO - Waiting for pod test-pod-3f02ab55 to be deleted -2025-06-05 20:15:26,738 - tests.orchestrator - INFO - Pod test-pod-3f02ab55 has been deleted -2025-06-05 20:15:26,739 - tests.orchestrator - INFO - Deleted pod: test-pod-3f02ab55 -2025-06-05 20:15:26,739 - tests.orchestrator - INFO - Deleting pod: test-pod-09cac2b7 from PVC: many2one-89ac31d4 -2025-06-05 20:15:26,813 - tests.orchestrator - INFO - Waiting for pod test-pod-09cac2b7 to be deleted -2025-06-05 20:16:00,160 - tests.orchestrator - INFO - Pod test-pod-09cac2b7 has been deleted -2025-06-05 20:16:00,161 - tests.orchestrator - INFO - Deleted pod: test-pod-09cac2b7 -2025-06-05 20:16:00,161 - tests.orchestrator - INFO - Deleting pod: test-pod-fc15ec0f from PVC: many2one-89ac31d4 -2025-06-05 20:16:00,237 - tests.orchestrator - INFO - Waiting for pod test-pod-fc15ec0f to be deleted -2025-06-05 20:16:32,849 - tests.orchestrator - INFO - Pod test-pod-fc15ec0f has been deleted -2025-06-05 20:16:32,849 - tests.orchestrator - INFO - Deleted pod: test-pod-fc15ec0f -2025-06-05 20:16:32,849 - tests.orchestrator - INFO - Deleting pod: test-pod-5a86841b from PVC: many2one-89ac31d4 -2025-06-05 20:16:32,925 - tests.orchestrator - INFO - Waiting for pod test-pod-5a86841b to be deleted -2025-06-05 20:17:05,648 - tests.orchestrator - INFO - Pod test-pod-5a86841b has been deleted -2025-06-05 20:17:05,648 - tests.orchestrator - INFO - Deleted pod: test-pod-5a86841b -2025-06-05 20:17:10,649 - tests.orchestrator - INFO - Deleting 3 PVCs (force=False)... -2025-06-05 20:17:10,649 - tests.orchestrator - INFO - Deleting PVC: test-pvc-69cf206a -2025-06-05 20:17:10,724 - tests.orchestrator - INFO - Waiting for PVC test-pvc-69cf206a to be deleted -2025-06-05 20:17:10,792 - tests.orchestrator - INFO - PVC test-pvc-69cf206a has been deleted -2025-06-05 20:17:10,792 - tests.orchestrator - INFO - Deleted PVC: test-pvc-69cf206a -2025-06-05 20:17:10,792 - tests.orchestrator - INFO - Deleting PVC: many2one-45c123ce -2025-06-05 20:17:10,865 - tests.orchestrator - INFO - Waiting for PVC many2one-45c123ce to be deleted -2025-06-05 20:17:10,933 - tests.orchestrator - INFO - PVC many2one-45c123ce has been deleted -2025-06-05 20:17:10,933 - tests.orchestrator - INFO - Deleted PVC: many2one-45c123ce -2025-06-05 20:17:10,934 - tests.orchestrator - INFO - Deleting PVC: many2one-89ac31d4 -2025-06-05 20:17:11,006 - tests.orchestrator - INFO - Waiting for PVC many2one-89ac31d4 to be deleted -2025-06-05 20:17:11,071 - tests.orchestrator - INFO - PVC many2one-89ac31d4 has been deleted -2025-06-05 20:17:11,071 - tests.orchestrator - INFO - Deleted PVC: many2one-89ac31d4 -2025-06-05 20:17:11,272 - tests.orchestrator - WARNING - First cleanup pass incomplete. Remaining resources: ['pod/test-pod-0d0fddfe', 'pod/test-pod-158cdfa4', 'pvc/test-pvc-0bb0d20c', 'pvc/test-pvc-2127e71a', 'pvc/test-pvc-2cf5b7a2', 'pvc/test-pvc-31fbd501', 'pvc/test-pvc-528b4f5c', 'pvc/test-pvc-ecb12f74'] -2025-06-05 20:17:11,272 - tests.orchestrator - INFO - Attempting force deletion of remaining resources... -2025-06-05 20:17:11,272 - tests.orchestrator - INFO - Deleting 0 pods (force=True)... -2025-06-05 20:17:16,272 - tests.orchestrator - INFO - Deleting 0 PVCs (force=True)... -2025-06-05 20:17:16,471 - tests.orchestrator - ERROR - Cleanup incomplete. Remaining resources after force deletion: ['pod/test-pod-0d0fddfe', 'pod/test-pod-158cdfa4', 'pvc/test-pvc-0bb0d20c', 'pvc/test-pvc-2127e71a', 'pvc/test-pvc-2cf5b7a2', 'pvc/test-pvc-31fbd501', 'pvc/test-pvc-528b4f5c', 'pvc/test-pvc-ecb12f74'] -2025-06-05 20:17:16,471 - tests.orchestrator - INFO - Cleanup completed successfully in 273.22 seconds -2025-06-05 20:17:16,471 - tests.orchestrator - INFO - ===== CLEANUP PROCESS FINISHED ===== -2025-06-05 20:17:16,471 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-05 20:17:16,471 - tests.orchestrator - INFO - --- Operations --- -2025-06-05 20:17:16,472 - tests.orchestrator - INFO - create_pvc: 2 succeeded, 0 failed (100.0%) -2025-06-05 20:17:16,472 - tests.orchestrator - INFO - attach_pod: 11 succeeded, 0 failed (100.0%) -2025-06-05 20:17:16,472 - tests.orchestrator - INFO - delete_pod: 11 succeeded, 0 failed (100.0%) -2025-06-05 20:17:16,472 - tests.orchestrator - INFO - delete_pvc: 4 succeeded, 0 failed (100.0%) -2025-06-05 20:17:16,472 - tests.orchestrator - INFO - verify_read_write: Writes 1 succeeded, 0 failed (100.0%) -2025-06-05 20:17:16,472 - tests.orchestrator - INFO - verify_read_write: Reads 1 succeeded, 0 failed (100.0%) -2025-06-05 20:17:16,472 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-05 20:17:16,472 - tests.orchestrator - INFO - shared_volume_rw: 1 succeeded, 0 failed out of 1 runs (100.0%) -2025-06-05 20:17:16,472 - tests.orchestrator - INFO - many_to_one: 2 succeeded, 0 failed out of 2 runs (100.0%) -2025-06-05 20:17:16,472 - tests.orchestrator - INFO - one_to_one: No runs -2025-06-05 20:17:16,472 - tests.orchestrator - INFO - concurrent_pvc: No runs -2025-06-05 20:17:16,472 - tests.orchestrator - INFO - ========================================= diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_210820.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_210820.log new file mode 100644 index 000000000..efa109bb9 --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_210820.log @@ -0,0 +1,7 @@ +2025-06-05 21:08:20,584 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 21:08:20,807 - tests.orchestrator - ERROR - Error checking namespace: (401) +Reason: Unauthorized +HTTP response headers: HTTPHeaderDict({'Audit-Id': '0bd8e6e9-7167-4275-a18c-06f08e3ee80c', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 21:08:20 GMT', 'Content-Length': '129'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} + + diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_210842.log b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_210842.log new file mode 100644 index 000000000..58576fca5 --- /dev/null +++ b/test/e2e/stress-scale-tests/logs/orchestrator_20250605_210842.log @@ -0,0 +1,70 @@ +2025-06-05 21:08:42,081 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-05 21:08:42,282 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-05 21:08:42,282 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-05 21:08:42,475 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-05 21:08:42,475 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-05 21:08:42,475 - tests.orchestrator - INFO - Creating PVC: test-pvc-dfd12d43 +2025-06-05 21:08:42,554 - tests.orchestrator - INFO - Created PVC: test-pvc-dfd12d43 +2025-06-05 21:08:42,554 - tests.orchestrator - INFO - Waiting for PVC test-pvc-dfd12d43 to be bound +2025-06-05 21:08:44,686 - tests.orchestrator - INFO - PVC test-pvc-dfd12d43 is bound +2025-06-05 21:08:44,687 - tests.orchestrator - INFO - Creating PVC: test-pvc-c542ee58 +2025-06-05 21:08:44,755 - tests.orchestrator - INFO - Created PVC: test-pvc-c542ee58 +2025-06-05 21:08:44,755 - tests.orchestrator - INFO - Waiting for PVC test-pvc-c542ee58 to be bound +2025-06-05 21:08:46,886 - tests.orchestrator - INFO - PVC test-pvc-c542ee58 is bound +2025-06-05 21:08:46,887 - tests.orchestrator - INFO - Creating pod test-pod-62d8b803 on PVC test-pvc-c542ee58 +2025-06-05 21:08:46,966 - tests.orchestrator - INFO - Created pod: test-pod-62d8b803 using PVC: test-pvc-c542ee58 +2025-06-05 21:08:46,966 - tests.orchestrator - INFO - Waiting for pod test-pod-62d8b803 to be ready +2025-06-05 21:08:47,032 - tests.orchestrator - INFO - Pod test-pod-62d8b803 phase: Pending +2025-06-05 21:08:49,100 - tests.orchestrator - INFO - Pod test-pod-62d8b803 phase: Running +2025-06-05 21:08:49,100 - tests.orchestrator - INFO - Pod test-pod-62d8b803 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 21:08:51,168 - tests.orchestrator - INFO - Pod test-pod-62d8b803 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 21:08:53,236 - tests.orchestrator - INFO - Pod test-pod-62d8b803 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 21:08:55,304 - tests.orchestrator - INFO - Pod test-pod-62d8b803 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 21:08:57,372 - tests.orchestrator - INFO - Pod test-pod-62d8b803 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 21:08:59,439 - tests.orchestrator - INFO - Pod test-pod-62d8b803 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 21:09:01,598 - tests.orchestrator - INFO - Pod test-pod-62d8b803 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 21:09:03,669 - tests.orchestrator - INFO - Pod test-pod-62d8b803 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 21:09:05,735 - tests.orchestrator - INFO - Pod test-pod-62d8b803 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 21:09:07,804 - tests.orchestrator - INFO - Pod test-pod-62d8b803 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 21:09:09,873 - tests.orchestrator - INFO - Pod test-pod-62d8b803 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 21:09:09,873 - tests.orchestrator - INFO - Pod test-pod-62d8b803 is ready +2025-06-05 21:09:09,873 - tests.orchestrator - INFO - Creating pod test-pod-87095965 on PVC test-pvc-c542ee58 +2025-06-05 21:09:09,947 - tests.orchestrator - INFO - Created pod: test-pod-87095965 using PVC: test-pvc-c542ee58 +2025-06-05 21:09:09,947 - tests.orchestrator - INFO - Waiting for pod test-pod-87095965 to be ready +2025-06-05 21:09:10,014 - tests.orchestrator - INFO - Pod test-pod-87095965 phase: Pending +2025-06-05 21:09:12,083 - tests.orchestrator - INFO - Pod test-pod-87095965 phase: Running +2025-06-05 21:09:12,083 - tests.orchestrator - INFO - Pod test-pod-87095965 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 21:09:14,152 - tests.orchestrator - INFO - Pod test-pod-87095965 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 21:09:16,219 - tests.orchestrator - INFO - Pod test-pod-87095965 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 21:09:18,288 - tests.orchestrator - INFO - Pod test-pod-87095965 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 21:09:20,355 - tests.orchestrator - INFO - Pod test-pod-87095965 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 21:09:22,423 - tests.orchestrator - INFO - Pod test-pod-87095965 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 21:09:24,491 - tests.orchestrator - INFO - Pod test-pod-87095965 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 21:09:26,557 - tests.orchestrator - INFO - Pod test-pod-87095965 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 21:09:28,625 - tests.orchestrator - INFO - Pod test-pod-87095965 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 21:09:30,692 - tests.orchestrator - INFO - Pod test-pod-87095965 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 21:09:30,692 - tests.orchestrator - INFO - Pod test-pod-87095965 is ready +2025-06-05 21:09:30,693 - tests.orchestrator - INFO - Creating pod test-pod-4044e577 on PVC test-pvc-c542ee58 +2025-06-05 21:09:30,774 - tests.orchestrator - INFO - Created pod: test-pod-4044e577 using PVC: test-pvc-c542ee58 +2025-06-05 21:09:30,774 - tests.orchestrator - INFO - Waiting for pod test-pod-4044e577 to be ready +2025-06-05 21:09:30,840 - tests.orchestrator - INFO - Pod test-pod-4044e577 phase: Pending +2025-06-05 21:09:32,908 - tests.orchestrator - INFO - Pod test-pod-4044e577 phase: Running +2025-06-05 21:09:32,908 - tests.orchestrator - INFO - Pod test-pod-4044e577 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 21:09:34,976 - tests.orchestrator - INFO - Pod test-pod-4044e577 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 21:09:37,045 - tests.orchestrator - INFO - Pod test-pod-4044e577 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 21:09:39,115 - tests.orchestrator - INFO - Pod test-pod-4044e577 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 21:09:41,211 - tests.orchestrator - INFO - Pod test-pod-4044e577 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 21:09:43,278 - tests.orchestrator - INFO - Pod test-pod-4044e577 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 21:09:45,346 - tests.orchestrator - INFO - Pod test-pod-4044e577 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 21:09:47,414 - tests.orchestrator - INFO - Pod test-pod-4044e577 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 21:09:49,483 - tests.orchestrator - INFO - Pod test-pod-4044e577 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-05 21:09:51,551 - tests.orchestrator - INFO - Pod test-pod-4044e577 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-05 21:09:51,551 - tests.orchestrator - INFO - Pod test-pod-4044e577 is ready +2025-06-05 21:09:51,551 - tests.orchestrator - INFO - Testing read/write between pods test-pod-4044e577 and test-pod-87095965 sharing PVC test-pvc-c542ee58 +2025-06-05 21:09:51,551 - tests.orchestrator - INFO - Executing write command: kubectl exec -n efs-stress-test test-pod-4044e577 -- /bin/sh -c 'echo "Test content: 3c9e6d0b-4c30-48ec-81e0-6a10b24ddcba" > /data/test-d1bd009a.txt' +2025-06-05 21:09:52,943 - tests.orchestrator - INFO - Pod test-pod-4044e577 wrote to /data/test-d1bd009a.txt +2025-06-05 21:09:54,943 - tests.orchestrator - INFO - Executing read command: kubectl exec -n efs-stress-test test-pod-87095965 -- cat /data/test-d1bd009a.txt +2025-06-05 21:09:56,145 - tests.orchestrator - INFO - Read result: 'Test content: 3c9e6d0b-4c30-48ec-81e0-6a10b24ddcba' +2025-06-05 21:09:56,145 - tests.orchestrator - INFO - Pod test-pod-87095965 successfully read content written by test-pod-4044e577 +2025-06-05 21:09:56,145 - tests.orchestrator - INFO - Deleting pod: test-pod-62d8b803 from PVC: test-pvc-c542ee58 +2025-06-05 21:09:56,221 - tests.orchestrator - INFO - Waiting for pod test-pod-62d8b803 to be deleted diff --git a/test/e2e/stress-scale-tests/reports/chaos/efs_chaos_20250603_135616.json b/test/e2e/stress-scale-tests/reports/chaos/efs_chaos_20250603_135616.json deleted file mode 100644 index 66bfed3bb..000000000 --- a/test/e2e/stress-scale-tests/reports/chaos/efs_chaos_20250603_135616.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "api_flood": { - "test": "rapid_api_flood", - "pvcs_requested": 50, - "pvcs_created": 50, - "success_rate": 100.0, - "duration": 0.46927380561828613, - "access_points_used": true - }, - "race_conditions": { - "test": "race_conditions", - "iterations": 30, - "successes": 17, - "failures": 13, - "exceptions": 0, - "duration": 6.168970346450806 - }, - "resource_exhaustion": { - "test": "resource_exhaustion", - "pods_requested": 75, - "pods_created": 75, - "pod_success_rate": 100.0, - "access_points_requested": 25, - "access_points_created": 0, - "ap_success_rate": 0.0, - "duration": 2.9329886436462402 - }, - "error": "type object 'Thread' has no attribute '_wait'" -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_145849_20250603_145849.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_145849_20250603_145849.json deleted file mode 100644 index 7d530574e..000000000 --- a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_145849_20250603_145849.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "test_name": "efs_tests_20250603_145849", - "test_type": "general", - "timestamp": "20250603_145849", - "system_info": { - "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", - "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", - "python_version": "3.12.10", - "cpu_count": 8, - "memory_total_gb": 30.88, - "timestamp": "2025-06-03T14:58:49.261345", - "kubernetes_version": "Unknown", - "aws_region": "us-west-1", - "efs_filesystem_id": "fs-02b74eafb14930828" - }, - "results": { - "orchestrator": { - "test_duration": 1748962729.2581227, - "operations": { - "create_pvc": { - "success": 1, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 9, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 9, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 3, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 1, - "read_success": 0, - "read_fail": 1, - "write_success_rate": 0.0, - "read_success_rate": 0.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "many_to_one": { - "runs": 2, - "success": 0, - "fail": 2, - "success_rate": 0.0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - } - } - }, - "metrics": { - "operations": {}, - "system": {}, - "kubernetes": {}, - "csi": {}, - "controller": { - "request_latency": {}, - "operation_counts": {}, - "success_rates": {}, - "volume_attach_timing": {} - }, - "node": { - "mount_timing": {}, - "mount_errors": {}, - "resource_utilization": {} - }, - "efs": { - "access_point_timing": {}, - "mount_completion_timing": {}, - "api_throttling_incidents": [] - }, - "k8s_events": { - "volume_events": [], - "binding_times": {}, - "pod_startup_delays": {} - } - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_145849_20250603_145849_summary.txt b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_145849_20250603_145849_summary.txt deleted file mode 100644 index a3794720e..000000000 --- a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_145849_20250603_145849_summary.txt +++ /dev/null @@ -1,61 +0,0 @@ -================================================================================ -EFS CSI DRIVER TEST REPORT: EFS_TESTS_20250603_145849 -================================================================================ - -SYSTEM INFORMATION --------------------------------------------------------------------------------- -Hostname: dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com -Platform: Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26 -Python Version: 3.12.10 -CPU Count: 8 -Memory Total: 30.88 GB -Kubernetes Version: Unknown -AWS Region: us-west-1 -EFS Filesystem ID: fs-02b74eafb14930828 -Test Timestamp: 2025-06-03T14:58:49.307018 - -TEST RESULTS --------------------------------------------------------------------------------- - -ORCHESTRATOR ----------------------------------------- -test_duration: 1748962729.2581227 -operations: - create_pvc: {'success': 1, 'fail': 0, 'success_rate': 100.0} - attach_pod: {'success': 9, 'fail': 0, 'success_rate': 100.0} - delete_pod: {'success': 9, 'fail': 0, 'success_rate': 100.0} - delete_pvc: {'success': 3, 'fail': 0, 'success_rate': 100.0} - verify_read_write: {'write_success': 0, 'write_fail': 1, 'read_success': 0, 'read_fail': 1, 'write_success_rate': 0.0, 'read_success_rate': 0.0} -scenarios: - shared_volume_rw: {'runs': 1, 'success': 0, 'fail': 1, 'success_rate': 0.0} - many_to_one: {'runs': 2, 'success': 0, 'fail': 2, 'success_rate': 0.0} - one_to_one: {'runs': 0, 'success': 0, 'fail': 0, 'success_rate': 0} - concurrent_pvc: {'runs': 0, 'success': 0, 'fail': 0, 'success_rate': 0} - -METRICS ----------------------------------------- -operations: -system: -kubernetes: -csi: -controller: - request_latency: {} - operation_counts: defaultdict(, {}) - success_rates: defaultdict(. at 0x7f77f5c3ede0>, {}) - volume_attach_timing: {} -node: - mount_timing: {} - mount_errors: defaultdict(, {}) - resource_utilization: {} -efs: - access_point_timing: {} - mount_completion_timing: {} - api_throttling_incidents: [] -k8s_events: - volume_events: [] - binding_times: {} - pod_startup_delays: {} - -================================================================================ -END OF REPORT: 2025-06-03T14:58:49.351704 -================================================================================ diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_160604_20250603_160604.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_160604_20250603_160604.json deleted file mode 100644 index de426e9b7..000000000 --- a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_160604_20250603_160604.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "test_name": "efs_tests_20250603_160604", - "test_type": "general", - "timestamp": "20250603_160604", - "system_info": { - "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", - "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", - "python_version": "3.12.10", - "cpu_count": 8, - "memory_total_gb": 30.88, - "timestamp": "2025-06-03T16:06:04.459863", - "kubernetes_version": "Unknown", - "aws_region": "us-west-1", - "efs_filesystem_id": "fs-02b74eafb14930828" - }, - "results": { - "orchestrator": { - "test_duration": 1748966764.4565506, - "operations": { - "create_pvc": { - "success": 17, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 23, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 23, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 27, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 1, - "read_success": 0, - "read_fail": 1, - "write_success_rate": 0.0, - "read_success_rate": 0.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "many_to_one": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "one_to_one": { - "runs": 2, - "success": 0, - "fail": 2, - "success_rate": 0.0 - }, - "concurrent_pvc": { - "runs": 2, - "success": 2, - "fail": 0, - "success_rate": 100.0 - } - } - }, - "metrics": { - "operations": {}, - "system": {}, - "kubernetes": {}, - "csi": {}, - "controller": { - "request_latency": {}, - "operation_counts": {}, - "success_rates": {}, - "volume_attach_timing": {} - }, - "node": { - "mount_timing": {}, - "mount_errors": {}, - "resource_utilization": {} - }, - "efs": { - "access_point_timing": {}, - "mount_completion_timing": {}, - "api_throttling_incidents": [] - }, - "k8s_events": { - "volume_events": [], - "binding_times": {}, - "pod_startup_delays": {} - } - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_160604_20250603_160604_summary.txt b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_160604_20250603_160604_summary.txt deleted file mode 100644 index 29bc48b33..000000000 --- a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_160604_20250603_160604_summary.txt +++ /dev/null @@ -1,61 +0,0 @@ -================================================================================ -EFS CSI DRIVER TEST REPORT: EFS_TESTS_20250603_160604 -================================================================================ - -SYSTEM INFORMATION --------------------------------------------------------------------------------- -Hostname: dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com -Platform: Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26 -Python Version: 3.12.10 -CPU Count: 8 -Memory Total: 30.88 GB -Kubernetes Version: Unknown -AWS Region: us-west-1 -EFS Filesystem ID: fs-02b74eafb14930828 -Test Timestamp: 2025-06-03T16:06:04.505249 - -TEST RESULTS --------------------------------------------------------------------------------- - -ORCHESTRATOR ----------------------------------------- -test_duration: 1748966764.4565506 -operations: - create_pvc: {'success': 17, 'fail': 0, 'success_rate': 100.0} - attach_pod: {'success': 23, 'fail': 0, 'success_rate': 100.0} - delete_pod: {'success': 23, 'fail': 0, 'success_rate': 100.0} - delete_pvc: {'success': 27, 'fail': 0, 'success_rate': 100.0} - verify_read_write: {'write_success': 0, 'write_fail': 1, 'read_success': 0, 'read_fail': 1, 'write_success_rate': 0.0, 'read_success_rate': 0.0} -scenarios: - shared_volume_rw: {'runs': 1, 'success': 0, 'fail': 1, 'success_rate': 0.0} - many_to_one: {'runs': 1, 'success': 0, 'fail': 1, 'success_rate': 0.0} - one_to_one: {'runs': 2, 'success': 0, 'fail': 2, 'success_rate': 0.0} - concurrent_pvc: {'runs': 2, 'success': 2, 'fail': 0, 'success_rate': 100.0} - -METRICS ----------------------------------------- -operations: -system: -kubernetes: -csi: -controller: - request_latency: {} - operation_counts: defaultdict(, {}) - success_rates: defaultdict(. at 0x7fa9e4df2de0>, {}) - volume_attach_timing: {} -node: - mount_timing: {} - mount_errors: defaultdict(, {}) - resource_utilization: {} -efs: - access_point_timing: {} - mount_completion_timing: {} - api_throttling_incidents: [] -k8s_events: - volume_events: [] - binding_times: {} - pod_startup_delays: {} - -================================================================================ -END OF REPORT: 2025-06-03T16:06:04.550169 -================================================================================ diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_180130_20250603_180130.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_180130_20250603_180130.json deleted file mode 100644 index 6c58d6804..000000000 --- a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_180130_20250603_180130.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "test_name": "efs_tests_20250603_180130", - "test_type": "general", - "timestamp": "20250603_180130", - "system_info": { - "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", - "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", - "python_version": "3.12.10", - "cpu_count": 8, - "memory_total_gb": 30.88, - "timestamp": "2025-06-03T18:01:30.380488", - "kubernetes_version": "Unknown", - "aws_region": "us-west-1", - "efs_filesystem_id": "fs-02b74eafb14930828" - }, - "results": { - "orchestrator": { - "test_duration": 1748973690.377472, - "operations": { - "create_pvc": { - "success": 1, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 4, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 4, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 2, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 0, - "read_success": 0, - "read_fail": 0, - "write_success_rate": 0, - "read_success_rate": 0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "many_to_one": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - } - } - }, - "metrics": { - "operations": {}, - "system": {}, - "kubernetes": {}, - "csi": {}, - "controller": { - "request_latency": {}, - "operation_counts": {}, - "success_rates": {}, - "volume_attach_timing": {} - }, - "node": { - "mount_timing": {}, - "mount_errors": {}, - "resource_utilization": {} - }, - "efs": { - "access_point_timing": {}, - "mount_completion_timing": {}, - "api_throttling_incidents": [] - }, - "k8s_events": { - "volume_events": [], - "binding_times": {}, - "pod_startup_delays": {} - } - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_180130_20250603_180130_summary.txt b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_180130_20250603_180130_summary.txt deleted file mode 100644 index 7cfca2169..000000000 --- a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_180130_20250603_180130_summary.txt +++ /dev/null @@ -1,61 +0,0 @@ -================================================================================ -EFS CSI DRIVER TEST REPORT: EFS_TESTS_20250603_180130 -================================================================================ - -SYSTEM INFORMATION --------------------------------------------------------------------------------- -Hostname: dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com -Platform: Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26 -Python Version: 3.12.10 -CPU Count: 8 -Memory Total: 30.88 GB -Kubernetes Version: Unknown -AWS Region: us-west-1 -EFS Filesystem ID: fs-02b74eafb14930828 -Test Timestamp: 2025-06-03T18:01:30.426564 - -TEST RESULTS --------------------------------------------------------------------------------- - -ORCHESTRATOR ----------------------------------------- -test_duration: 1748973690.377472 -operations: - create_pvc: {'success': 1, 'fail': 0, 'success_rate': 100.0} - attach_pod: {'success': 4, 'fail': 0, 'success_rate': 100.0} - delete_pod: {'success': 4, 'fail': 0, 'success_rate': 100.0} - delete_pvc: {'success': 2, 'fail': 0, 'success_rate': 100.0} - verify_read_write: {'write_success': 0, 'write_fail': 0, 'read_success': 0, 'read_fail': 0, 'write_success_rate': 0, 'read_success_rate': 0} -scenarios: - shared_volume_rw: {'runs': 0, 'success': 0, 'fail': 0, 'success_rate': 0} - many_to_one: {'runs': 1, 'success': 0, 'fail': 1, 'success_rate': 0.0} - one_to_one: {'runs': 0, 'success': 0, 'fail': 0, 'success_rate': 0} - concurrent_pvc: {'runs': 0, 'success': 0, 'fail': 0, 'success_rate': 0} - -METRICS ----------------------------------------- -operations: -system: -kubernetes: -csi: -controller: - request_latency: {} - operation_counts: defaultdict(, {}) - success_rates: defaultdict(. at 0x7f892f362d40>, {}) - volume_attach_timing: {} -node: - mount_timing: {} - mount_errors: defaultdict(, {}) - resource_utilization: {} -efs: - access_point_timing: {} - mount_completion_timing: {} - api_throttling_incidents: [] -k8s_events: - volume_events: [] - binding_times: {} - pod_startup_delays: {} - -================================================================================ -END OF REPORT: 2025-06-03T18:01:30.469590 -================================================================================ diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_204448_20250603_204448.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_204448_20250603_204448.json deleted file mode 100644 index acc51b261..000000000 --- a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_204448_20250603_204448.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "test_name": "efs_tests_20250603_204448", - "test_type": "general", - "timestamp": "20250603_204448", - "system_info": { - "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", - "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", - "python_version": "3.12.10", - "cpu_count": 8, - "memory_total_gb": 30.88, - "timestamp": "2025-06-03T20:44:48.750764", - "kubernetes_version": "Unknown", - "aws_region": "us-west-1", - "efs_filesystem_id": "fs-02b74eafb14930828" - }, - "results": { - "orchestrator": { - "test_duration": 1748983488.7475276, - "operations": { - "create_pvc": { - "success": 0, - "fail": 7, - "success_rate": 0.0 - }, - "attach_pod": { - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "delete_pod": { - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "delete_pvc": { - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 0, - "read_success": 0, - "read_fail": 0, - "write_success_rate": 0, - "read_success_rate": 0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "many_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - } - } - }, - "metrics": { - "operations": {}, - "system": {}, - "kubernetes": {}, - "csi": {}, - "controller": { - "request_latency": {}, - "operation_counts": {}, - "success_rates": {}, - "volume_attach_timing": {} - }, - "node": { - "mount_timing": {}, - "mount_errors": {}, - "resource_utilization": {} - }, - "efs": { - "access_point_timing": {}, - "mount_completion_timing": {}, - "api_throttling_incidents": [] - }, - "k8s_events": { - "volume_events": [], - "binding_times": {}, - "pod_startup_delays": {} - } - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_204448_20250603_204448_summary.txt b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_204448_20250603_204448_summary.txt deleted file mode 100644 index bec1dc247..000000000 --- a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_204448_20250603_204448_summary.txt +++ /dev/null @@ -1,61 +0,0 @@ -================================================================================ -EFS CSI DRIVER TEST REPORT: EFS_TESTS_20250603_204448 -================================================================================ - -SYSTEM INFORMATION --------------------------------------------------------------------------------- -Hostname: dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com -Platform: Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26 -Python Version: 3.12.10 -CPU Count: 8 -Memory Total: 30.88 GB -Kubernetes Version: Unknown -AWS Region: us-west-1 -EFS Filesystem ID: fs-02b74eafb14930828 -Test Timestamp: 2025-06-03T20:44:48.794425 - -TEST RESULTS --------------------------------------------------------------------------------- - -ORCHESTRATOR ----------------------------------------- -test_duration: 1748983488.7475276 -operations: - create_pvc: {'success': 0, 'fail': 7, 'success_rate': 0.0} - attach_pod: {'success': 0, 'fail': 0, 'success_rate': 0} - delete_pod: {'success': 0, 'fail': 0, 'success_rate': 0} - delete_pvc: {'success': 0, 'fail': 0, 'success_rate': 0} - verify_read_write: {'write_success': 0, 'write_fail': 0, 'read_success': 0, 'read_fail': 0, 'write_success_rate': 0, 'read_success_rate': 0} -scenarios: - shared_volume_rw: {'runs': 0, 'success': 0, 'fail': 0, 'success_rate': 0} - many_to_one: {'runs': 0, 'success': 0, 'fail': 0, 'success_rate': 0} - one_to_one: {'runs': 0, 'success': 0, 'fail': 0, 'success_rate': 0} - concurrent_pvc: {'runs': 1, 'success': 0, 'fail': 1, 'success_rate': 0.0} - -METRICS ----------------------------------------- -operations: -system: -kubernetes: -csi: -controller: - request_latency: {} - operation_counts: defaultdict(, {}) - success_rates: defaultdict(. at 0x7f9ccfd9b6a0>, {}) - volume_attach_timing: {} -node: - mount_timing: {} - mount_errors: defaultdict(, {}) - resource_utilization: {} -efs: - access_point_timing: {} - mount_completion_timing: {} - api_throttling_incidents: [] -k8s_events: - volume_events: [] - binding_times: {} - pod_startup_delays: {} - -================================================================================ -END OF REPORT: 2025-06-03T20:44:48.840194 -================================================================================ diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_205333_20250603_205333.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_205333_20250603_205333.json deleted file mode 100644 index 8c9fc25e7..000000000 --- a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_205333_20250603_205333.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "test_name": "efs_tests_20250603_205333", - "test_type": "general", - "timestamp": "20250603_205333", - "system_info": { - "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", - "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", - "python_version": "3.12.10", - "cpu_count": 8, - "memory_total_gb": 30.88, - "timestamp": "2025-06-03T20:53:33.163444", - "kubernetes_version": "Unknown", - "aws_region": "us-west-1", - "efs_filesystem_id": "fs-02b74eafb14930828" - }, - "results": { - "orchestrator": { - "test_duration": 1748984013.159983, - "operations": { - "create_pvc": { - "success": 1, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 5, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 5, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 2, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 0, - "read_success": 0, - "read_fail": 0, - "write_success_rate": 0, - "read_success_rate": 0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "many_to_one": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - } - } - }, - "metrics": { - "operations": {}, - "system": {}, - "kubernetes": {}, - "csi": {}, - "controller": { - "request_latency": {}, - "operation_counts": {}, - "success_rates": {}, - "volume_attach_timing": {} - }, - "node": { - "mount_timing": {}, - "mount_errors": {}, - "resource_utilization": {} - }, - "efs": { - "access_point_timing": {}, - "mount_completion_timing": {}, - "api_throttling_incidents": [] - }, - "k8s_events": { - "volume_events": [], - "binding_times": {}, - "pod_startup_delays": {} - } - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_205333_20250603_205333_summary.txt b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_205333_20250603_205333_summary.txt deleted file mode 100644 index 17b8c4bb2..000000000 --- a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_205333_20250603_205333_summary.txt +++ /dev/null @@ -1,61 +0,0 @@ -================================================================================ -EFS CSI DRIVER TEST REPORT: EFS_TESTS_20250603_205333 -================================================================================ - -SYSTEM INFORMATION --------------------------------------------------------------------------------- -Hostname: dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com -Platform: Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26 -Python Version: 3.12.10 -CPU Count: 8 -Memory Total: 30.88 GB -Kubernetes Version: Unknown -AWS Region: us-west-1 -EFS Filesystem ID: fs-02b74eafb14930828 -Test Timestamp: 2025-06-03T20:53:33.207801 - -TEST RESULTS --------------------------------------------------------------------------------- - -ORCHESTRATOR ----------------------------------------- -test_duration: 1748984013.159983 -operations: - create_pvc: {'success': 1, 'fail': 0, 'success_rate': 100.0} - attach_pod: {'success': 5, 'fail': 0, 'success_rate': 100.0} - delete_pod: {'success': 5, 'fail': 0, 'success_rate': 100.0} - delete_pvc: {'success': 2, 'fail': 0, 'success_rate': 100.0} - verify_read_write: {'write_success': 0, 'write_fail': 0, 'read_success': 0, 'read_fail': 0, 'write_success_rate': 0, 'read_success_rate': 0} -scenarios: - shared_volume_rw: {'runs': 0, 'success': 0, 'fail': 0, 'success_rate': 0} - many_to_one: {'runs': 1, 'success': 0, 'fail': 1, 'success_rate': 0.0} - one_to_one: {'runs': 0, 'success': 0, 'fail': 0, 'success_rate': 0} - concurrent_pvc: {'runs': 0, 'success': 0, 'fail': 0, 'success_rate': 0} - -METRICS ----------------------------------------- -operations: -system: -kubernetes: -csi: -controller: - request_latency: {} - operation_counts: defaultdict(, {}) - success_rates: defaultdict(. at 0x7fa4a85236a0>, {}) - volume_attach_timing: {} -node: - mount_timing: {} - mount_errors: defaultdict(, {}) - resource_utilization: {} -efs: - access_point_timing: {} - mount_completion_timing: {} - api_throttling_incidents: [] -k8s_events: - volume_events: [] - binding_times: {} - pod_startup_delays: {} - -================================================================================ -END OF REPORT: 2025-06-03T20:53:33.262686 -================================================================================ diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_210938_20250603_210938.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_210938_20250603_210938.json deleted file mode 100644 index e60fce0c4..000000000 --- a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_210938_20250603_210938.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "test_name": "efs_tests_20250603_210938", - "test_type": "general", - "timestamp": "20250603_210938", - "system_info": { - "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", - "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", - "python_version": "3.12.10", - "cpu_count": 8, - "memory_total_gb": 30.88, - "timestamp": "2025-06-03T21:09:38.404991", - "kubernetes_version": "Unknown", - "aws_region": "us-west-1", - "efs_filesystem_id": "fs-02b74eafb14930828" - }, - "results": { - "orchestrator": { - "test_duration": 1748984978.40179, - "operations": { - "create_pvc": { - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "attach_pod": { - "success": 9, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 9, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 1, - "read_success": 0, - "read_fail": 1, - "write_success_rate": 0.0, - "read_success_rate": 0.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "many_to_one": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "one_to_one": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "concurrent_pvc": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - } - } - }, - "metrics": { - "operations": {}, - "system": {}, - "kubernetes": {}, - "csi": {}, - "controller": { - "request_latency": {}, - "operation_counts": {}, - "success_rates": {}, - "volume_attach_timing": {} - }, - "node": { - "mount_timing": {}, - "mount_errors": {}, - "resource_utilization": {} - }, - "efs": { - "access_point_timing": {}, - "mount_completion_timing": {}, - "api_throttling_incidents": [] - }, - "k8s_events": { - "volume_events": [], - "binding_times": {}, - "pod_startup_delays": {} - } - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_210938_20250603_210938_summary.txt b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_210938_20250603_210938_summary.txt deleted file mode 100644 index 6cd5590c3..000000000 --- a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250603_210938_20250603_210938_summary.txt +++ /dev/null @@ -1,61 +0,0 @@ -================================================================================ -EFS CSI DRIVER TEST REPORT: EFS_TESTS_20250603_210938 -================================================================================ - -SYSTEM INFORMATION --------------------------------------------------------------------------------- -Hostname: dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com -Platform: Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26 -Python Version: 3.12.10 -CPU Count: 8 -Memory Total: 30.88 GB -Kubernetes Version: Unknown -AWS Region: us-west-1 -EFS Filesystem ID: fs-02b74eafb14930828 -Test Timestamp: 2025-06-03T21:09:38.449707 - -TEST RESULTS --------------------------------------------------------------------------------- - -ORCHESTRATOR ----------------------------------------- -test_duration: 1748984978.40179 -operations: - create_pvc: {'success': 0, 'fail': 0, 'success_rate': 0} - attach_pod: {'success': 9, 'fail': 0, 'success_rate': 100.0} - delete_pod: {'success': 9, 'fail': 0, 'success_rate': 100.0} - delete_pvc: {'success': 6, 'fail': 0, 'success_rate': 100.0} - verify_read_write: {'write_success': 0, 'write_fail': 1, 'read_success': 0, 'read_fail': 1, 'write_success_rate': 0.0, 'read_success_rate': 0.0} -scenarios: - shared_volume_rw: {'runs': 1, 'success': 0, 'fail': 1, 'success_rate': 0.0} - many_to_one: {'runs': 1, 'success': 0, 'fail': 1, 'success_rate': 0.0} - one_to_one: {'runs': 1, 'success': 0, 'fail': 1, 'success_rate': 0.0} - concurrent_pvc: {'runs': 0, 'success': 0, 'fail': 0, 'success_rate': 0} - -METRICS ----------------------------------------- -operations: -system: -kubernetes: -csi: -controller: - request_latency: {} - operation_counts: defaultdict(, {}) - success_rates: defaultdict(. at 0x7f6d73f9d120>, {}) - volume_attach_timing: {} -node: - mount_timing: {} - mount_errors: defaultdict(, {}) - resource_utilization: {} -efs: - access_point_timing: {} - mount_completion_timing: {} - api_throttling_incidents: [] -k8s_events: - volume_events: [] - binding_times: {} - pod_startup_delays: {} - -================================================================================ -END OF REPORT: 2025-06-03T21:09:38.492151 -================================================================================ diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250604_204834_20250604_204834.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250604_204834_20250604_204834.json deleted file mode 100644 index 57194f1c9..000000000 --- a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250604_204834_20250604_204834.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "test_name": "efs_tests_20250604_204834", - "test_type": "general", - "timestamp": "20250604_204834", - "system_info": { - "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", - "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", - "python_version": "3.12.10", - "cpu_count": 8, - "memory_total_gb": 30.88, - "timestamp": "2025-06-04T20:48:34.213634", - "kubernetes_version": "Unknown", - "aws_region": "us-west-1", - "efs_filesystem_id": "fs-02b74eafb14930828" - }, - "results": { - "orchestrator": { - "test_duration": 1749070114.2102087, - "operations": { - "create_pvc": { - "success": 2, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 5, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 5, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 0, - "read_success": 0, - "read_fail": 0, - "write_success_rate": 0, - "read_success_rate": 0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "many_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "one_to_one": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "concurrent_pvc": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - } - } - }, - "metrics": { - "operations": {}, - "system": {}, - "kubernetes": {}, - "csi": {}, - "controller": { - "request_latency": {}, - "operation_counts": {}, - "success_rates": {}, - "volume_attach_timing": {} - }, - "node": { - "mount_timing": {}, - "mount_errors": {}, - "resource_utilization": {} - }, - "efs": { - "access_point_timing": {}, - "mount_completion_timing": {}, - "api_throttling_incidents": [] - }, - "k8s_events": { - "volume_events": [], - "binding_times": {}, - "pod_startup_delays": {} - } - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250604_211347_20250604_211347.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250604_211347_20250604_211347.json deleted file mode 100644 index ef6d374f1..000000000 --- a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250604_211347_20250604_211347.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "test_name": "efs_tests_20250604_211347", - "test_type": "general", - "timestamp": "20250604_211347", - "system_info": { - "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", - "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", - "python_version": "3.12.10", - "cpu_count": 8, - "memory_total_gb": 30.88, - "timestamp": "2025-06-04T21:13:47.134029", - "kubernetes_version": "Unknown", - "aws_region": "us-west-1", - "efs_filesystem_id": "fs-02b74eafb14930828" - }, - "results": { - "orchestrator": { - "test_duration": 1749071627.1310592, - "operations": { - "create_pvc": { - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "attach_pod": { - "success": 1, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 1, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 1, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 0, - "read_success": 0, - "read_fail": 0, - "write_success_rate": 0, - "read_success_rate": 0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "many_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "one_to_one": { - "runs": 1, - "success": 0, - "fail": 0, - "success_rate": 0.0 - }, - "concurrent_pvc": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - } - } - }, - "metrics": { - "operations": {}, - "system": {}, - "kubernetes": {}, - "csi": {}, - "controller": { - "request_latency": {}, - "operation_counts": {}, - "success_rates": {}, - "volume_attach_timing": {} - }, - "node": { - "mount_timing": {}, - "mount_errors": {}, - "resource_utilization": {} - }, - "efs": { - "access_point_timing": {}, - "mount_completion_timing": {}, - "api_throttling_incidents": [] - }, - "k8s_events": { - "volume_events": [], - "binding_times": {}, - "pod_startup_delays": {} - } - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250604_211457_20250604_211457.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250604_211457_20250604_211457.json deleted file mode 100644 index 7ac9cea43..000000000 --- a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250604_211457_20250604_211457.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "test_name": "efs_tests_20250604_211457", - "test_type": "general", - "timestamp": "20250604_211457", - "system_info": { - "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", - "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", - "python_version": "3.12.10", - "cpu_count": 8, - "memory_total_gb": 30.88, - "timestamp": "2025-06-04T21:14:57.325771", - "kubernetes_version": "Unknown", - "aws_region": "us-west-1", - "efs_filesystem_id": "fs-02b74eafb14930828" - }, - "results": { - "orchestrator": { - "test_duration": 1749071697.3223903, - "operations": { - "create_pvc": { - "success": 4, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 1, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 1, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 4, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 0, - "read_success": 0, - "read_fail": 0, - "write_success_rate": 0, - "read_success_rate": 0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "many_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 1, - "success": 0, - "fail": 0, - "success_rate": 0.0 - } - } - }, - "metrics": { - "operations": {}, - "system": {}, - "kubernetes": {}, - "csi": {}, - "controller": { - "request_latency": {}, - "operation_counts": {}, - "success_rates": {}, - "volume_attach_timing": {} - }, - "node": { - "mount_timing": {}, - "mount_errors": {}, - "resource_utilization": {} - }, - "efs": { - "access_point_timing": {}, - "mount_completion_timing": {}, - "api_throttling_incidents": [] - }, - "k8s_events": { - "volume_events": [], - "binding_times": {}, - "pod_startup_delays": {} - } - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_133650_20250605_133650.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_133650_20250605_133650.json deleted file mode 100644 index e8751550c..000000000 --- a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_133650_20250605_133650.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "test_name": "efs_tests_20250605_133650", - "test_type": "general", - "timestamp": "20250605_133650", - "system_info": { - "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", - "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", - "python_version": "3.12.10", - "cpu_count": 8, - "memory_total_gb": 30.88, - "timestamp": "2025-06-05T13:36:50.687414", - "kubernetes_version": "Unknown", - "aws_region": "Unknown", - "efs_filesystem_id": "Unknown" - }, - "results": { - "orchestrator": { - "test_duration": 1749130610.6835227, - "operations": { - "create_pvc": { - "success": 12, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 8, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 8, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 12, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 0, - "read_success": 0, - "read_fail": 0, - "write_success_rate": 0, - "read_success_rate": 0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "many_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 2, - "success": 2, - "fail": 0, - "success_rate": 100.0 - } - } - }, - "metrics": { - "operations": {}, - "system": {}, - "kubernetes": {}, - "csi": {}, - "controller": { - "request_latency": {}, - "operation_counts": {}, - "success_rates": {}, - "volume_attach_timing": {} - }, - "node": { - "mount_timing": {}, - "mount_errors": {}, - "resource_utilization": {} - }, - "efs": { - "access_point_timing": {}, - "mount_completion_timing": {}, - "api_throttling_incidents": [] - }, - "k8s_events": { - "volume_events": [], - "binding_times": {}, - "pod_startup_delays": {} - } - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_135626_20250605_135626.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_135626_20250605_135626.json deleted file mode 100644 index 5e51eaa30..000000000 --- a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_135626_20250605_135626.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "test_name": "efs_tests_20250605_135626", - "test_type": "general", - "timestamp": "20250605_135626", - "system_info": { - "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", - "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", - "python_version": "3.12.10", - "cpu_count": 8, - "memory_total_gb": 30.88, - "timestamp": "2025-06-05T13:56:26.290855", - "kubernetes_version": "Unknown", - "aws_region": "Unknown", - "efs_filesystem_id": "Unknown" - }, - "results": { - "orchestrator": { - "test_duration": 1749131786.2873476, - "operations": { - "create_pvc": { - "success": 2, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 5, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 1, - "read_success": 0, - "read_fail": 1, - "write_success_rate": 0.0, - "read_success_rate": 0.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "many_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "one_to_one": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "concurrent_pvc": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - } - } - }, - "metrics": { - "operations": {}, - "system": {}, - "kubernetes": {}, - "csi": {}, - "controller": { - "request_latency": {}, - "operation_counts": {}, - "success_rates": {}, - "volume_attach_timing": {} - }, - "node": { - "mount_timing": {}, - "mount_errors": {}, - "resource_utilization": {} - }, - "efs": { - "access_point_timing": {}, - "mount_completion_timing": {}, - "api_throttling_incidents": [] - }, - "k8s_events": { - "volume_events": [], - "binding_times": {}, - "pod_startup_delays": {} - } - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_141131_20250605_141131.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_141131_20250605_141131.json deleted file mode 100644 index d2243619b..000000000 --- a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_141131_20250605_141131.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "test_name": "efs_tests_20250605_141131", - "test_type": "general", - "timestamp": "20250605_141131", - "system_info": { - "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", - "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", - "python_version": "3.12.10", - "cpu_count": 8, - "memory_total_gb": 30.88, - "timestamp": "2025-06-05T14:11:31.212354", - "kubernetes_version": "Unknown", - "aws_region": "Unknown", - "efs_filesystem_id": "Unknown" - }, - "results": { - "orchestrator": { - "test_duration": 1749132691.2088823, - "operations": { - "create_pvc": { - "success": 2, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 7, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 7, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 1, - "read_success": 0, - "read_fail": 1, - "write_success_rate": 0.0, - "read_success_rate": 0.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "many_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "one_to_one": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "concurrent_pvc": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - } - } - }, - "metrics": { - "operations": {}, - "system": {}, - "kubernetes": {}, - "csi": {}, - "controller": { - "request_latency": {}, - "operation_counts": {}, - "success_rates": {}, - "volume_attach_timing": {} - }, - "node": { - "mount_timing": {}, - "mount_errors": {}, - "resource_utilization": {} - }, - "efs": { - "access_point_timing": {}, - "mount_completion_timing": {}, - "api_throttling_incidents": [] - }, - "k8s_events": { - "volume_events": [], - "binding_times": {}, - "pod_startup_delays": {} - } - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_142059_20250605_142059.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_142059_20250605_142059.json deleted file mode 100644 index 9aebf3995..000000000 --- a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_142059_20250605_142059.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "test_name": "efs_tests_20250605_142059", - "test_type": "general", - "timestamp": "20250605_142059", - "system_info": { - "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", - "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", - "python_version": "3.12.10", - "cpu_count": 8, - "memory_total_gb": 30.88, - "timestamp": "2025-06-05T14:20:59.414086", - "kubernetes_version": "Unknown", - "aws_region": "Unknown", - "efs_filesystem_id": "Unknown" - }, - "results": { - "orchestrator": { - "test_duration": 1749133259.4106185, - "operations": { - "create_pvc": { - "success": 7, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 7, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 1, - "read_success": 0, - "read_fail": 1, - "write_success_rate": 0.0, - "read_success_rate": 0.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "many_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 1, - "success": 1, - "fail": 0, - "success_rate": 100.0 - } - } - }, - "metrics": { - "operations": {}, - "system": {}, - "kubernetes": {}, - "csi": {}, - "controller": { - "request_latency": {}, - "operation_counts": {}, - "success_rates": {}, - "volume_attach_timing": {} - }, - "node": { - "mount_timing": {}, - "mount_errors": {}, - "resource_utilization": {} - }, - "efs": { - "access_point_timing": {}, - "mount_completion_timing": {}, - "api_throttling_incidents": [] - }, - "k8s_events": { - "volume_events": [], - "binding_times": {}, - "pod_startup_delays": {} - } - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_145750_20250605_145750.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_145750_20250605_145750.json deleted file mode 100644 index ec26fc187..000000000 --- a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_145750_20250605_145750.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "test_name": "efs_tests_20250605_145750", - "test_type": "general", - "timestamp": "20250605_145750", - "system_info": { - "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", - "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", - "python_version": "3.12.10", - "cpu_count": 8, - "memory_total_gb": 30.88, - "timestamp": "2025-06-05T14:57:50.430436", - "kubernetes_version": "Unknown", - "aws_region": "Unknown", - "efs_filesystem_id": "Unknown" - }, - "results": { - "orchestrator": { - "test_duration": 1749135470.4266138, - "operations": { - "create_pvc": { - "success": 2, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 3, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 1, - "read_success": 0, - "read_fail": 1, - "write_success_rate": 0.0, - "read_success_rate": 0.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "many_to_one": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - } - } - }, - "metrics": { - "operations": {}, - "system": {}, - "kubernetes": {}, - "csi": {}, - "controller": { - "request_latency": {}, - "operation_counts": {}, - "success_rates": {}, - "volume_attach_timing": {} - }, - "node": { - "mount_timing": {}, - "mount_errors": {}, - "resource_utilization": {} - }, - "efs": { - "access_point_timing": {}, - "mount_completion_timing": {}, - "api_throttling_incidents": [] - }, - "k8s_events": { - "volume_events": [], - "binding_times": {}, - "pod_startup_delays": {} - } - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_150603_20250605_150603.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_150603_20250605_150603.json deleted file mode 100644 index 072a7ceb4..000000000 --- a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_150603_20250605_150603.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "test_name": "efs_tests_20250605_150603", - "test_type": "general", - "timestamp": "20250605_150603", - "system_info": { - "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", - "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", - "python_version": "3.12.10", - "cpu_count": 8, - "memory_total_gb": 30.88, - "timestamp": "2025-06-05T15:06:03.516189", - "kubernetes_version": "Unknown", - "aws_region": "Unknown", - "efs_filesystem_id": "Unknown" - }, - "results": { - "orchestrator": { - "test_duration": 1749135963.512927, - "operations": { - "create_pvc": { - "success": 5, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 5, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 1, - "read_success": 0, - "read_fail": 1, - "write_success_rate": 0.0, - "read_success_rate": 0.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "many_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 1, - "success": 1, - "fail": 0, - "success_rate": 100.0 - } - } - }, - "metrics": { - "operations": {}, - "system": {}, - "kubernetes": {}, - "csi": {}, - "controller": { - "request_latency": {}, - "operation_counts": {}, - "success_rates": {}, - "volume_attach_timing": {} - }, - "node": { - "mount_timing": {}, - "mount_errors": {}, - "resource_utilization": {} - }, - "efs": { - "access_point_timing": {}, - "mount_completion_timing": {}, - "api_throttling_incidents": [] - }, - "k8s_events": { - "volume_events": [], - "binding_times": {}, - "pod_startup_delays": {} - } - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_151246_20250605_151246.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_151246_20250605_151246.json deleted file mode 100644 index dd620e847..000000000 --- a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_151246_20250605_151246.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "test_name": "efs_tests_20250605_151246", - "test_type": "general", - "timestamp": "20250605_151246", - "system_info": { - "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", - "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", - "python_version": "3.12.10", - "cpu_count": 8, - "memory_total_gb": 30.88, - "timestamp": "2025-06-05T15:12:46.812571", - "kubernetes_version": "Unknown", - "aws_region": "Unknown", - "efs_filesystem_id": "Unknown" - }, - "results": { - "orchestrator": { - "test_duration": 1749136366.8090806, - "operations": { - "create_pvc": { - "success": 7, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 7, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 1, - "read_success": 0, - "read_fail": 1, - "write_success_rate": 0.0, - "read_success_rate": 0.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "many_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 1, - "success": 1, - "fail": 0, - "success_rate": 100.0 - } - } - }, - "metrics": { - "operations": {}, - "system": {}, - "kubernetes": {}, - "csi": {}, - "controller": { - "request_latency": {}, - "operation_counts": {}, - "success_rates": {}, - "volume_attach_timing": {} - }, - "node": { - "mount_timing": {}, - "mount_errors": {}, - "resource_utilization": {} - }, - "efs": { - "access_point_timing": {}, - "mount_completion_timing": {}, - "api_throttling_incidents": [] - }, - "k8s_events": { - "volume_events": [], - "binding_times": {}, - "pod_startup_delays": {} - } - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_154402_20250605_154402.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_154402_20250605_154402.json deleted file mode 100644 index 74c5aca15..000000000 --- a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_154402_20250605_154402.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "test_name": "efs_tests_20250605_154402", - "test_type": "general", - "timestamp": "20250605_154402", - "system_info": { - "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", - "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", - "python_version": "3.12.10", - "cpu_count": 8, - "memory_total_gb": 30.88, - "timestamp": "2025-06-05T15:44:02.410888", - "kubernetes_version": "Unknown", - "aws_region": "Unknown", - "efs_filesystem_id": "Unknown" - }, - "results": { - "orchestrator": { - "test_duration": 1749138242.406942, - "operations": { - "create_pvc": { - "success": 2, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 3, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 1, - "read_success": 0, - "read_fail": 1, - "write_success_rate": 0.0, - "read_success_rate": 0.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "many_to_one": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - } - } - }, - "metrics": { - "operations": {}, - "system": {}, - "kubernetes": {}, - "csi": {}, - "controller": { - "request_latency": {}, - "operation_counts": {}, - "success_rates": {}, - "volume_attach_timing": {} - }, - "node": { - "mount_timing": {}, - "mount_errors": {}, - "resource_utilization": {} - }, - "efs": { - "access_point_timing": {}, - "mount_completion_timing": {}, - "api_throttling_incidents": [] - }, - "k8s_events": { - "volume_events": [], - "binding_times": {}, - "pod_startup_delays": {} - } - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_155202_20250605_155202.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_155202_20250605_155202.json deleted file mode 100644 index bfceb1589..000000000 --- a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_155202_20250605_155202.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "test_name": "efs_tests_20250605_155202", - "test_type": "general", - "timestamp": "20250605_155202", - "system_info": { - "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", - "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", - "python_version": "3.12.10", - "cpu_count": 8, - "memory_total_gb": 30.88, - "timestamp": "2025-06-05T15:52:02.971763", - "kubernetes_version": "Unknown", - "aws_region": "Unknown", - "efs_filesystem_id": "Unknown" - }, - "results": { - "orchestrator": { - "test_duration": 1749138722.968492, - "operations": { - "create_pvc": { - "success": 5, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 5, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 1, - "read_success": 0, - "read_fail": 1, - "write_success_rate": 0.0, - "read_success_rate": 0.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "many_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 1, - "success": 1, - "fail": 0, - "success_rate": 100.0 - } - } - }, - "metrics": { - "operations": {}, - "system": {}, - "kubernetes": {}, - "csi": {}, - "controller": { - "request_latency": {}, - "operation_counts": {}, - "success_rates": {}, - "volume_attach_timing": {} - }, - "node": { - "mount_timing": {}, - "mount_errors": {}, - "resource_utilization": {} - }, - "efs": { - "access_point_timing": {}, - "mount_completion_timing": {}, - "api_throttling_incidents": [] - }, - "k8s_events": { - "volume_events": [], - "binding_times": {}, - "pod_startup_delays": {} - } - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_160701_20250605_160701.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_160701_20250605_160701.json deleted file mode 100644 index 6504484d1..000000000 --- a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_160701_20250605_160701.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "test_name": "efs_tests_20250605_160701", - "test_type": "general", - "timestamp": "20250605_160701", - "system_info": { - "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", - "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", - "python_version": "3.12.10", - "cpu_count": 8, - "memory_total_gb": 30.88, - "timestamp": "2025-06-05T16:07:01.160981", - "kubernetes_version": "Unknown", - "aws_region": "Unknown", - "efs_filesystem_id": "Unknown" - }, - "results": { - "orchestrator": { - "test_duration": 1749139621.1574326, - "operations": { - "create_pvc": { - "success": 2, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 7, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 7, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 3, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 1, - "read_success": 0, - "read_fail": 1, - "write_success_rate": 0.0, - "read_success_rate": 0.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "many_to_one": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - } - } - }, - "metrics": { - "operations": {}, - "system": {}, - "kubernetes": {}, - "csi": {}, - "controller": { - "request_latency": {}, - "operation_counts": {}, - "success_rates": {}, - "volume_attach_timing": {} - }, - "node": { - "mount_timing": {}, - "mount_errors": {}, - "resource_utilization": {} - }, - "efs": { - "access_point_timing": {}, - "mount_completion_timing": {}, - "api_throttling_incidents": [] - }, - "k8s_events": { - "volume_events": [], - "binding_times": {}, - "pod_startup_delays": {} - } - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_161509_20250605_161509.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_161509_20250605_161509.json deleted file mode 100644 index b41af0ee6..000000000 --- a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_161509_20250605_161509.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "test_name": "efs_tests_20250605_161509", - "test_type": "general", - "timestamp": "20250605_161509", - "system_info": { - "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", - "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", - "python_version": "3.12.10", - "cpu_count": 8, - "memory_total_gb": 30.88, - "timestamp": "2025-06-05T16:15:09.982171", - "kubernetes_version": "Unknown", - "aws_region": "Unknown", - "efs_filesystem_id": "Unknown" - }, - "results": { - "orchestrator": { - "test_duration": 1749140109.9788902, - "operations": { - "create_pvc": { - "success": 9, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 9, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 1, - "read_success": 0, - "read_fail": 1, - "write_success_rate": 0.0, - "read_success_rate": 0.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "many_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 1, - "success": 1, - "fail": 0, - "success_rate": 100.0 - } - } - }, - "metrics": { - "operations": {}, - "system": {}, - "kubernetes": {}, - "csi": {}, - "controller": { - "request_latency": {}, - "operation_counts": {}, - "success_rates": {}, - "volume_attach_timing": {} - }, - "node": { - "mount_timing": {}, - "mount_errors": {}, - "resource_utilization": {} - }, - "efs": { - "access_point_timing": {}, - "mount_completion_timing": {}, - "api_throttling_incidents": [] - }, - "k8s_events": { - "volume_events": [], - "binding_times": {}, - "pod_startup_delays": {} - } - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_163010_20250605_163010.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_163010_20250605_163010.json deleted file mode 100644 index 2ab04a1d2..000000000 --- a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_163010_20250605_163010.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "test_name": "efs_tests_20250605_163010", - "test_type": "general", - "timestamp": "20250605_163010", - "system_info": { - "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", - "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", - "python_version": "3.12.10", - "cpu_count": 8, - "memory_total_gb": 30.88, - "timestamp": "2025-06-05T16:30:10.899998", - "kubernetes_version": "Unknown", - "aws_region": "Unknown", - "efs_filesystem_id": "Unknown" - }, - "results": { - "orchestrator": { - "test_duration": 1749141010.8966942, - "operations": { - "create_pvc": { - "success": 2, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 5, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 1, - "read_success": 0, - "read_fail": 1, - "write_success_rate": 0.0, - "read_success_rate": 0.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "many_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "one_to_one": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "concurrent_pvc": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - } - } - }, - "metrics": { - "operations": {}, - "system": {}, - "kubernetes": {}, - "csi": {}, - "controller": { - "request_latency": {}, - "operation_counts": {}, - "success_rates": {}, - "volume_attach_timing": {} - }, - "node": { - "mount_timing": {}, - "mount_errors": {}, - "resource_utilization": {} - }, - "efs": { - "access_point_timing": {}, - "mount_completion_timing": {}, - "api_throttling_incidents": [] - }, - "k8s_events": { - "volume_events": [], - "binding_times": {}, - "pod_startup_delays": {} - } - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_164017_20250605_164017.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_164017_20250605_164017.json deleted file mode 100644 index 8dfa7eb5e..000000000 --- a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_164017_20250605_164017.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "test_name": "efs_tests_20250605_164017", - "test_type": "general", - "timestamp": "20250605_164017", - "system_info": { - "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", - "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", - "python_version": "3.12.10", - "cpu_count": 8, - "memory_total_gb": 30.88, - "timestamp": "2025-06-05T16:40:17.454819", - "kubernetes_version": "Unknown", - "aws_region": "Unknown", - "efs_filesystem_id": "Unknown" - }, - "results": { - "orchestrator": { - "test_duration": 1749141617.450957, - "operations": { - "create_pvc": { - "success": 3, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 5, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 5, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 3, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 1, - "read_success": 0, - "read_fail": 1, - "write_success_rate": 0.0, - "read_success_rate": 0.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "many_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - } - } - }, - "metrics": { - "operations": {}, - "system": {}, - "kubernetes": {}, - "csi": {}, - "controller": { - "request_latency": {}, - "operation_counts": {}, - "success_rates": {}, - "volume_attach_timing": {} - }, - "node": { - "mount_timing": {}, - "mount_errors": {}, - "resource_utilization": {} - }, - "efs": { - "access_point_timing": {}, - "mount_completion_timing": {}, - "api_throttling_incidents": [] - }, - "k8s_events": { - "volume_events": [], - "binding_times": {}, - "pod_startup_delays": {} - } - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_165552_20250605_165552.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_165552_20250605_165552.json deleted file mode 100644 index 826a5bd2b..000000000 --- a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_165552_20250605_165552.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "test_name": "efs_tests_20250605_165552", - "test_type": "general", - "timestamp": "20250605_165552", - "system_info": { - "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", - "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", - "python_version": "3.12.10", - "cpu_count": 8, - "memory_total_gb": 30.88, - "timestamp": "2025-06-05T16:55:52.179504", - "kubernetes_version": "Unknown", - "aws_region": "Unknown", - "efs_filesystem_id": "Unknown" - }, - "results": { - "orchestrator": { - "test_duration": 1749142552.17605, - "operations": { - "create_pvc": { - "success": 8, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 8, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 1, - "read_success": 0, - "read_fail": 1, - "write_success_rate": 0.0, - "read_success_rate": 0.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "many_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - } - } - }, - "metrics": { - "operations": {}, - "system": {}, - "kubernetes": {}, - "csi": {}, - "controller": { - "request_latency": {}, - "operation_counts": {}, - "success_rates": {}, - "volume_attach_timing": {} - }, - "node": { - "mount_timing": {}, - "mount_errors": {}, - "resource_utilization": {} - }, - "efs": { - "access_point_timing": {}, - "mount_completion_timing": {}, - "api_throttling_incidents": [] - }, - "k8s_events": { - "volume_events": [], - "binding_times": {}, - "pod_startup_delays": {} - } - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_171216_20250605_171216.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_171216_20250605_171216.json deleted file mode 100644 index a629ca01a..000000000 --- a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_171216_20250605_171216.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "test_name": "efs_tests_20250605_171216", - "test_type": "general", - "timestamp": "20250605_171216", - "system_info": { - "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", - "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", - "python_version": "3.12.10", - "cpu_count": 8, - "memory_total_gb": 30.88, - "timestamp": "2025-06-05T17:12:16.584971", - "kubernetes_version": "Unknown", - "aws_region": "Unknown", - "efs_filesystem_id": "Unknown" - }, - "results": { - "orchestrator": { - "test_duration": 1749143536.5820618, - "operations": { - "create_pvc": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 7, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 7, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 1, - "read_success": 0, - "read_fail": 1, - "write_success_rate": 0.0, - "read_success_rate": 0.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "many_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - } - } - }, - "metrics": { - "operations": {}, - "system": {}, - "kubernetes": {}, - "csi": {}, - "controller": { - "request_latency": {}, - "operation_counts": {}, - "success_rates": {}, - "volume_attach_timing": {} - }, - "node": { - "mount_timing": {}, - "mount_errors": {}, - "resource_utilization": {} - }, - "efs": { - "access_point_timing": {}, - "mount_completion_timing": {}, - "api_throttling_incidents": [] - }, - "k8s_events": { - "volume_events": [], - "binding_times": {}, - "pod_startup_delays": {} - } - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_182932_20250605_182932.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_182932_20250605_182932.json deleted file mode 100644 index fcfbb965f..000000000 --- a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_182932_20250605_182932.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "test_name": "efs_tests_20250605_182932", - "test_type": "general", - "timestamp": "20250605_182932", - "system_info": { - "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", - "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", - "python_version": "3.12.10", - "cpu_count": 8, - "memory_total_gb": 30.88, - "timestamp": "2025-06-05T18:29:32.603793", - "kubernetes_version": "Unknown", - "aws_region": "Unknown", - "efs_filesystem_id": "Unknown" - }, - "results": { - "orchestrator": { - "test_duration": 1749148172.6007035, - "operations": { - "create_pvc": { - "success": 3, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 7, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 7, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 3, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 1, - "read_success": 0, - "read_fail": 1, - "write_success_rate": 0.0, - "read_success_rate": 0.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "many_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - } - } - }, - "metrics": { - "operations": {}, - "system": {}, - "kubernetes": {}, - "csi": {}, - "controller": { - "request_latency": {}, - "operation_counts": {}, - "success_rates": {}, - "volume_attach_timing": {} - }, - "node": { - "mount_timing": {}, - "mount_errors": {}, - "resource_utilization": {} - }, - "efs": { - "access_point_timing": {}, - "mount_completion_timing": {}, - "api_throttling_incidents": [] - }, - "k8s_events": { - "volume_events": [], - "binding_times": {}, - "pod_startup_delays": {} - } - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_193407_20250605_193407.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_193407_20250605_193407.json deleted file mode 100644 index 331016cd9..000000000 --- a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_193407_20250605_193407.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "test_name": "efs_tests_20250605_193407", - "test_type": "general", - "timestamp": "20250605_193407", - "system_info": { - "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", - "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", - "python_version": "3.12.10", - "cpu_count": 8, - "memory_total_gb": 30.88, - "timestamp": "2025-06-05T19:34:07.407073", - "kubernetes_version": "Unknown", - "aws_region": "Unknown", - "efs_filesystem_id": "Unknown" - }, - "results": { - "orchestrator": { - "test_duration": 1749152047.404046, - "operations": { - "create_pvc": { - "success": 7, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 5, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 5, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 7, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 1, - "write_fail": 0, - "read_success": 1, - "read_fail": 0, - "write_success_rate": 100.0, - "read_success_rate": 100.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 1, - "success": 1, - "fail": 0, - "success_rate": 100.0 - }, - "many_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - } - } - }, - "metrics": { - "operations": {}, - "system": {}, - "kubernetes": {}, - "csi": {}, - "controller": { - "request_latency": {}, - "operation_counts": {}, - "success_rates": {}, - "volume_attach_timing": {} - }, - "node": { - "mount_timing": {}, - "mount_errors": {}, - "resource_utilization": {} - }, - "efs": { - "access_point_timing": {}, - "mount_completion_timing": {}, - "api_throttling_incidents": [] - }, - "k8s_events": { - "volume_events": [], - "binding_times": {}, - "pod_startup_delays": {} - } - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_200124_20250605_200124.json b/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_200124_20250605_200124.json deleted file mode 100644 index dfa7a6bda..000000000 --- a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_200124_20250605_200124.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "test_name": "efs_tests_20250605_200124", - "test_type": "general", - "timestamp": "20250605_200124", - "system_info": { - "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", - "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", - "python_version": "3.12.10", - "cpu_count": 8, - "memory_total_gb": 30.88, - "timestamp": "2025-06-05T20:01:24.758278", - "kubernetes_version": "Unknown", - "aws_region": "Unknown", - "efs_filesystem_id": "Unknown" - }, - "results": { - "orchestrator": { - "test_duration": 1749153684.7552128, - "operations": { - "create_pvc": { - "success": 13, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 9, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 9, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 13, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 2, - "write_fail": 0, - "read_success": 2, - "read_fail": 0, - "write_success_rate": 100.0, - "read_success_rate": 100.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 2, - "success": 2, - "fail": 0, - "success_rate": 100.0 - }, - "many_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 2, - "success": 2, - "fail": 0, - "success_rate": 100.0 - } - } - }, - "metrics": { - "operations": {}, - "system": {}, - "kubernetes": {}, - "csi": {}, - "controller": { - "request_latency": {}, - "operation_counts": {}, - "success_rates": {}, - "volume_attach_timing": {} - }, - "node": { - "mount_timing": {}, - "mount_errors": {}, - "resource_utilization": {} - }, - "efs": { - "access_point_timing": {}, - "mount_completion_timing": {}, - "api_throttling_incidents": [] - }, - "k8s_events": { - "volume_events": [], - "binding_times": {}, - "pod_startup_delays": {} - } - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_145849.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_145849.json deleted file mode 100644 index d77f05172..000000000 --- a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_145849.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "test_duration": 1748962729.2581227, - "operations": { - "create_pvc": { - "success": 1, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 9, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 9, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 3, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 1, - "read_success": 0, - "read_fail": 1, - "write_success_rate": 0.0, - "read_success_rate": 0.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "many_to_one": { - "runs": 2, - "success": 0, - "fail": 2, - "success_rate": 0.0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_160604.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_160604.json deleted file mode 100644 index bfafde25a..000000000 --- a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_160604.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "test_duration": 1748966764.4565506, - "operations": { - "create_pvc": { - "success": 17, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 23, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 23, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 27, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 1, - "read_success": 0, - "read_fail": 1, - "write_success_rate": 0.0, - "read_success_rate": 0.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "many_to_one": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "one_to_one": { - "runs": 2, - "success": 0, - "fail": 2, - "success_rate": 0.0 - }, - "concurrent_pvc": { - "runs": 2, - "success": 2, - "fail": 0, - "success_rate": 100.0 - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_180130.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_180130.json deleted file mode 100644 index 7d0710919..000000000 --- a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_180130.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "test_duration": 1748973690.377472, - "operations": { - "create_pvc": { - "success": 1, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 4, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 4, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 2, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 0, - "read_success": 0, - "read_fail": 0, - "write_success_rate": 0, - "read_success_rate": 0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "many_to_one": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_204448.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_204448.json deleted file mode 100644 index 2b13f3158..000000000 --- a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_204448.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "test_duration": 1748983488.7475276, - "operations": { - "create_pvc": { - "success": 0, - "fail": 7, - "success_rate": 0.0 - }, - "attach_pod": { - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "delete_pod": { - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "delete_pvc": { - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 0, - "read_success": 0, - "read_fail": 0, - "write_success_rate": 0, - "read_success_rate": 0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "many_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_205333.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_205333.json deleted file mode 100644 index b20e31af6..000000000 --- a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_205333.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "test_duration": 1748984013.159983, - "operations": { - "create_pvc": { - "success": 1, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 5, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 5, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 2, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 0, - "read_success": 0, - "read_fail": 0, - "write_success_rate": 0, - "read_success_rate": 0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "many_to_one": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_210938.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_210938.json deleted file mode 100644 index 77933bb92..000000000 --- a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250603_210938.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "test_duration": 1748984978.40179, - "operations": { - "create_pvc": { - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "attach_pod": { - "success": 9, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 9, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 1, - "read_success": 0, - "read_fail": 1, - "write_success_rate": 0.0, - "read_success_rate": 0.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "many_to_one": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "one_to_one": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "concurrent_pvc": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250604_204834.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250604_204834.json deleted file mode 100644 index 69e4dfcad..000000000 --- a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250604_204834.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "test_duration": 1749070114.2102087, - "operations": { - "create_pvc": { - "success": 2, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 5, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 5, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 0, - "read_success": 0, - "read_fail": 0, - "write_success_rate": 0, - "read_success_rate": 0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "many_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "one_to_one": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "concurrent_pvc": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250604_211347.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250604_211347.json deleted file mode 100644 index c44b60ca6..000000000 --- a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250604_211347.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "test_duration": 1749071627.1310592, - "operations": { - "create_pvc": { - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "attach_pod": { - "success": 1, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 1, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 1, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 0, - "read_success": 0, - "read_fail": 0, - "write_success_rate": 0, - "read_success_rate": 0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "many_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "one_to_one": { - "runs": 1, - "success": 0, - "fail": 0, - "success_rate": 0.0 - }, - "concurrent_pvc": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250604_211457.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250604_211457.json deleted file mode 100644 index 793073ebd..000000000 --- a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250604_211457.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "test_duration": 1749071697.3223903, - "operations": { - "create_pvc": { - "success": 4, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 1, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 1, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 4, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 0, - "read_success": 0, - "read_fail": 0, - "write_success_rate": 0, - "read_success_rate": 0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "many_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 1, - "success": 0, - "fail": 0, - "success_rate": 0.0 - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_133650.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_133650.json deleted file mode 100644 index 86ea6b2eb..000000000 --- a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_133650.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "test_duration": 1749130610.6835227, - "operations": { - "create_pvc": { - "success": 12, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 8, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 8, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 12, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 0, - "read_success": 0, - "read_fail": 0, - "write_success_rate": 0, - "read_success_rate": 0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "many_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 2, - "success": 2, - "fail": 0, - "success_rate": 100.0 - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_135626.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_135626.json deleted file mode 100644 index a05a9878f..000000000 --- a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_135626.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "test_duration": 1749131786.2873476, - "operations": { - "create_pvc": { - "success": 2, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 5, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 1, - "read_success": 0, - "read_fail": 1, - "write_success_rate": 0.0, - "read_success_rate": 0.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "many_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "one_to_one": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "concurrent_pvc": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_141131.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_141131.json deleted file mode 100644 index 8fd7476b9..000000000 --- a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_141131.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "test_duration": 1749132691.2088823, - "operations": { - "create_pvc": { - "success": 2, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 7, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 7, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 1, - "read_success": 0, - "read_fail": 1, - "write_success_rate": 0.0, - "read_success_rate": 0.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "many_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "one_to_one": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "concurrent_pvc": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_142059.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_142059.json deleted file mode 100644 index fda6ab914..000000000 --- a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_142059.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "test_duration": 1749133259.4106185, - "operations": { - "create_pvc": { - "success": 7, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 7, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 1, - "read_success": 0, - "read_fail": 1, - "write_success_rate": 0.0, - "read_success_rate": 0.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "many_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 1, - "success": 1, - "fail": 0, - "success_rate": 100.0 - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_145750.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_145750.json deleted file mode 100644 index 889fb64be..000000000 --- a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_145750.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "test_duration": 1749135470.4266138, - "operations": { - "create_pvc": { - "success": 2, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 3, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 1, - "read_success": 0, - "read_fail": 1, - "write_success_rate": 0.0, - "read_success_rate": 0.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "many_to_one": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_150603.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_150603.json deleted file mode 100644 index 96d858b04..000000000 --- a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_150603.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "test_duration": 1749135963.512927, - "operations": { - "create_pvc": { - "success": 5, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 5, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 1, - "read_success": 0, - "read_fail": 1, - "write_success_rate": 0.0, - "read_success_rate": 0.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "many_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 1, - "success": 1, - "fail": 0, - "success_rate": 100.0 - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_151246.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_151246.json deleted file mode 100644 index dcafe6de2..000000000 --- a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_151246.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "test_duration": 1749136366.8090806, - "operations": { - "create_pvc": { - "success": 7, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 7, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 1, - "read_success": 0, - "read_fail": 1, - "write_success_rate": 0.0, - "read_success_rate": 0.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "many_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 1, - "success": 1, - "fail": 0, - "success_rate": 100.0 - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_154402.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_154402.json deleted file mode 100644 index 4b6eff392..000000000 --- a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_154402.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "test_duration": 1749138242.406942, - "operations": { - "create_pvc": { - "success": 2, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 3, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 1, - "read_success": 0, - "read_fail": 1, - "write_success_rate": 0.0, - "read_success_rate": 0.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "many_to_one": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_155202.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_155202.json deleted file mode 100644 index 8952e04d1..000000000 --- a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_155202.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "test_duration": 1749138722.968492, - "operations": { - "create_pvc": { - "success": 5, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 5, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 1, - "read_success": 0, - "read_fail": 1, - "write_success_rate": 0.0, - "read_success_rate": 0.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "many_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 1, - "success": 1, - "fail": 0, - "success_rate": 100.0 - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_160701.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_160701.json deleted file mode 100644 index a440b0e36..000000000 --- a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_160701.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "test_duration": 1749139621.1574326, - "operations": { - "create_pvc": { - "success": 2, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 7, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 7, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 3, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 1, - "read_success": 0, - "read_fail": 1, - "write_success_rate": 0.0, - "read_success_rate": 0.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "many_to_one": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_161509.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_161509.json deleted file mode 100644 index 6e139bc85..000000000 --- a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_161509.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "test_duration": 1749140109.9788902, - "operations": { - "create_pvc": { - "success": 9, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 9, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 1, - "read_success": 0, - "read_fail": 1, - "write_success_rate": 0.0, - "read_success_rate": 0.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "many_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 1, - "success": 1, - "fail": 0, - "success_rate": 100.0 - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_163010.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_163010.json deleted file mode 100644 index 29617dfab..000000000 --- a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_163010.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "test_duration": 1749141010.8966942, - "operations": { - "create_pvc": { - "success": 2, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 5, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 1, - "read_success": 0, - "read_fail": 1, - "write_success_rate": 0.0, - "read_success_rate": 0.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "many_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "one_to_one": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "concurrent_pvc": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_164017.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_164017.json deleted file mode 100644 index 8f2fe7194..000000000 --- a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_164017.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "test_duration": 1749141617.450957, - "operations": { - "create_pvc": { - "success": 3, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 5, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 5, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 3, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 1, - "read_success": 0, - "read_fail": 1, - "write_success_rate": 0.0, - "read_success_rate": 0.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "many_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_165552.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_165552.json deleted file mode 100644 index 3e2159340..000000000 --- a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_165552.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "test_duration": 1749142552.17605, - "operations": { - "create_pvc": { - "success": 8, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 8, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 1, - "read_success": 0, - "read_fail": 1, - "write_success_rate": 0.0, - "read_success_rate": 0.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "many_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_171216.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_171216.json deleted file mode 100644 index a0fa1ac4b..000000000 --- a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_171216.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "test_duration": 1749143536.5820618, - "operations": { - "create_pvc": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 7, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 7, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 1, - "read_success": 0, - "read_fail": 1, - "write_success_rate": 0.0, - "read_success_rate": 0.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "many_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_182932.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_182932.json deleted file mode 100644 index a02f395fe..000000000 --- a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_182932.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "test_duration": 1749148172.6007035, - "operations": { - "create_pvc": { - "success": 3, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 7, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 7, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 3, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 0, - "write_fail": 1, - "read_success": 0, - "read_fail": 1, - "write_success_rate": 0.0, - "read_success_rate": 0.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 1, - "success": 0, - "fail": 1, - "success_rate": 0.0 - }, - "many_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_193407.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_193407.json deleted file mode 100644 index 606ae9787..000000000 --- a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_193407.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "test_duration": 1749152047.404046, - "operations": { - "create_pvc": { - "success": 7, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 5, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 5, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 7, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 1, - "write_fail": 0, - "read_success": 1, - "read_fail": 0, - "write_success_rate": 100.0, - "read_success_rate": 100.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 1, - "success": 1, - "fail": 0, - "success_rate": 100.0 - }, - "many_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_200124.json b/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_200124.json deleted file mode 100644 index 5080be634..000000000 --- a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_200124.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "test_duration": 1749153684.7552128, - "operations": { - "create_pvc": { - "success": 13, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 9, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 9, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 13, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 2, - "write_fail": 0, - "read_success": 2, - "read_fail": 0, - "write_success_rate": 100.0, - "read_success_rate": 100.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 2, - "success": 2, - "fail": 0, - "success_rate": 100.0 - }, - "many_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 2, - "success": 2, - "fail": 0, - "success_rate": 100.0 - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/scalability/efs_tests_20250602_195427_20250602_195427.json b/test/e2e/stress-scale-tests/reports/scalability/efs_tests_20250602_195427_20250602_195427.json deleted file mode 100644 index 90e53bbd7..000000000 --- a/test/e2e/stress-scale-tests/reports/scalability/efs_tests_20250602_195427_20250602_195427.json +++ /dev/null @@ -1,100 +0,0 @@ -{ - "test_name": "efs_tests_20250602_195427", - "test_type": "scalability", - "timestamp": "20250602_195427", - "system_info": { - "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", - "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", - "python_version": "3.12.10", - "cpu_count": 8, - "memory_total_gb": 30.88, - "timestamp": "2025-06-02T19:54:27.467966", - "kubernetes_version": "Unknown", - "aws_region": "us-west-1", - "efs_filesystem_id": "fs-02b74eafb14930828" - }, - "results": { - "scalability": { - "pod_scaling": { - "5": { - "duration": 5.151206731796265, - "success": true, - "pods_ready": 5 - }, - "10": { - "duration": 5.1346681118011475, - "success": true, - "pods_ready": 10 - }, - "15": { - "duration": 10.208761215209961, - "success": true, - "pods_ready": 15 - }, - "20": { - "duration": 10.21699595451355, - "success": true, - "pods_ready": 20 - }, - "25": { - "duration": 25.416666746139526, - "success": true, - "pods_ready": 25 - }, - "30": { - "duration": 10.209507942199707, - "success": true, - "pods_ready": 30 - }, - "35": { - "duration": 30.496685028076172, - "success": true, - "pods_ready": 35 - }, - "40": { - "duration": 15.27869462966919, - "success": true, - "pods_ready": 40 - }, - "45": { - "duration": 304.97394347190857, - "success": false, - "pods_ready": 44 - }, - "50": { - "duration": 304.2392809391022, - "success": false, - "pods_ready": 44 - } - }, - "volume_scaling": { - "1": { - "duration": 5.21432638168335, - "success": true, - "volumes_ready": 1 - }, - "6": { - "duration": 16.051532745361328, - "success": true, - "volumes_ready": 6 - }, - "11": { - "duration": 11.683096647262573, - "success": true, - "volumes_ready": 11 - }, - "16": { - "duration": 12.391531705856323, - "success": true, - "volumes_ready": 16 - } - } - }, - "metrics": { - "operations": {}, - "system": {}, - "kubernetes": {}, - "csi": {} - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/scalability/efs_tests_20250602_195427_20250602_195427_summary.txt b/test/e2e/stress-scale-tests/reports/scalability/efs_tests_20250602_195427_20250602_195427_summary.txt deleted file mode 100644 index 5ea1adb78..000000000 --- a/test/e2e/stress-scale-tests/reports/scalability/efs_tests_20250602_195427_20250602_195427_summary.txt +++ /dev/null @@ -1,22 +0,0 @@ -================================================================================ -EFS CSI DRIVER TEST REPORT: EFS_TESTS_20250602_195427 -================================================================================ - -SYSTEM INFORMATION --------------------------------------------------------------------------------- -Hostname: dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com -Platform: Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26 -Python Version: 3.12.10 -CPU Count: 8 -Memory Total: 30.88 GB -Kubernetes Version: Unknown -AWS Region: us-west-1 -EFS Filesystem ID: fs-02b74eafb14930828 -Test Timestamp: 2025-06-02T19:54:27.512219 - -TEST RESULTS --------------------------------------------------------------------------------- - -================================================================================ -END OF REPORT: 2025-06-02T19:54:27.556124 -================================================================================ diff --git a/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_183718_20250602_183718.json b/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_183718_20250602_183718.json deleted file mode 100644 index 23db83d18..000000000 --- a/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_183718_20250602_183718.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "test_name": "efs_tests_20250602_183718", - "test_type": "stress", - "timestamp": "20250602_183718", - "system_info": { - "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", - "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", - "python_version": "3.12.10", - "cpu_count": 8, - "memory_total_gb": 30.88, - "timestamp": "2025-06-02T18:37:18.745829", - "kubernetes_version": "Unknown", - "aws_region": "us-west-1", - "efs_filesystem_id": "fs-02b74eafb14930828" - }, - "results": { - "stress": { - "single_pvc": { - "operation": "concurrent_mount_single_pvc", - "duration": 4.789013147354126, - "pods": 20, - "pods_created": 20 - }, - "multi_pvc": { - "operation": "concurrent_mount_multiple_pvc", - "duration": 8.21653962135315, - "pvcs_created": 10, - "pods_created": 20, - "total_pods": 20 - }, - "sequential_write": { - "status": "running", - "duration": 184.95399236679077, - "pod": "sequential-write-test-pod", - "pvc": "io-test-pvc-sequential-write" - }, - "random_write": { - "status": "running", - "duration": 185.02785754203796, - "pod": "random-write-test-pod", - "pvc": "io-test-pvc-random-write" - }, - "mixed_io": { - "status": "running", - "duration": 180.6199607849121, - "pod": "mixed-io-test-pod", - "pvc": "io-test-pvc-mixed-io" - }, - "total_duration": 552.7662425041199 - }, - "metrics": { - "operations": {}, - "system": {}, - "kubernetes": {}, - "csi": {} - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_183718_20250602_183718_summary.txt b/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_183718_20250602_183718_summary.txt deleted file mode 100644 index 6cecb22e9..000000000 --- a/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_183718_20250602_183718_summary.txt +++ /dev/null @@ -1,40 +0,0 @@ -================================================================================ -EFS CSI DRIVER TEST REPORT: EFS_TESTS_20250602_183718 -================================================================================ - -SYSTEM INFORMATION --------------------------------------------------------------------------------- -Hostname: dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com -Platform: Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26 -Python Version: 3.12.10 -CPU Count: 8 -Memory Total: 30.88 GB -Kubernetes Version: Unknown -AWS Region: us-west-1 -EFS Filesystem ID: fs-02b74eafb14930828 -Test Timestamp: 2025-06-02T18:37:18.790768 - -TEST RESULTS --------------------------------------------------------------------------------- - -STRESS RESULTS ----------------------------------------- - -Sequential Write Test: - Status: running - Duration: 184.95399236679077 seconds - -Random Write Test: - Status: running - Duration: 185.02785754203796 seconds - -Mixed I/O Test: - Status: running - Duration: 180.6199607849121 seconds - -METRICS RESULTS ----------------------------------------- - -================================================================================ -END OF REPORT: 2025-06-02T18:37:18.834481 -================================================================================ diff --git a/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_201648_20250602_201648.json b/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_201648_20250602_201648.json deleted file mode 100644 index 4260f4bfd..000000000 --- a/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_201648_20250602_201648.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "test_name": "efs_tests_20250602_201648", - "test_type": "stress", - "timestamp": "20250602_201648", - "system_info": { - "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", - "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", - "python_version": "3.12.10", - "cpu_count": 8, - "memory_total_gb": 30.88, - "timestamp": "2025-06-02T20:16:48.627010", - "kubernetes_version": "Unknown", - "aws_region": "us-west-1", - "efs_filesystem_id": "fs-02b74eafb14930828" - }, - "results": { - "concurrent_api": { - "single_pvc": { - "operation": "concurrent_mount_single_pvc", - "duration": 4.969545602798462, - "pods": 20, - "pods_created": 20 - }, - "multi_pvc": { - "operation": "concurrent_mount_multiple_pvc", - "duration": 7.825149059295654, - "pvcs_created": 10, - "pods_created": 20, - "total_pods": 20 - }, - "sequential_write": { - "status": "running", - "duration": 185.06655764579773, - "pod": "sequential-write-test-pod", - "pvc": "io-test-pvc-sequential-write" - }, - "random_write": { - "status": "running", - "duration": 185.08776235580444, - "pod": "random-write-test-pod", - "pvc": "io-test-pvc-random-write" - }, - "mixed_io": { - "status": "running", - "duration": 180.72687149047852, - "pod": "mixed-io-test-pod", - "pvc": "io-test-pvc-mixed-io" - }, - "total_duration": 552.9746143817902 - }, - "metrics": { - "operations": {}, - "system": {}, - "kubernetes": {}, - "csi": {} - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_201648_20250602_201648_summary.txt b/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_201648_20250602_201648_summary.txt deleted file mode 100644 index 835064c5e..000000000 --- a/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_201648_20250602_201648_summary.txt +++ /dev/null @@ -1,40 +0,0 @@ -================================================================================ -EFS CSI DRIVER TEST REPORT: EFS_TESTS_20250602_201648 -================================================================================ - -SYSTEM INFORMATION --------------------------------------------------------------------------------- -Hostname: dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com -Platform: Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26 -Python Version: 3.12.10 -CPU Count: 8 -Memory Total: 30.88 GB -Kubernetes Version: Unknown -AWS Region: us-west-1 -EFS Filesystem ID: fs-02b74eafb14930828 -Test Timestamp: 2025-06-02T20:16:48.670978 - -TEST RESULTS --------------------------------------------------------------------------------- - -CONCURRENT_API RESULTS ----------------------------------------- - -Sequential Write Test: - Status: running - Duration: 185.06655764579773 seconds - -Random Write Test: - Status: running - Duration: 185.08776235580444 seconds - -Mixed I/O Test: - Status: running - Duration: 180.72687149047852 seconds - -METRICS RESULTS ----------------------------------------- - -================================================================================ -END OF REPORT: 2025-06-02T20:16:48.713866 -================================================================================ diff --git a/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_210512_20250602_210512.json b/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_210512_20250602_210512.json deleted file mode 100644 index 3c97bcf97..000000000 --- a/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_210512_20250602_210512.json +++ /dev/null @@ -1,79 +0,0 @@ -{ - "test_name": "efs_tests_20250602_210512", - "test_type": "stress", - "timestamp": "20250602_210512", - "system_info": { - "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", - "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", - "python_version": "3.12.10", - "cpu_count": 8, - "memory_total_gb": 30.88, - "timestamp": "2025-06-02T21:05:12.574471", - "kubernetes_version": "Unknown", - "aws_region": "us-west-1", - "efs_filesystem_id": "fs-02b74eafb14930828" - }, - "results": { - "concurrent_api": { - "single_pvc": { - "operation": "concurrent_mount_single_pvc", - "duration": 5.6173036098480225, - "pods": 20, - "pods_created": 20 - }, - "multi_pvc": { - "operation": "concurrent_mount_multiple_pvc", - "duration": 8.005990028381348, - "pvcs_created": 10, - "pods_created": 20, - "total_pods": 20 - }, - "sequential_write": { - "status": "running", - "duration": 184.9669418334961, - "pod": "sequential-write-test-pod", - "pvc": "io-test-pvc-sequential-write" - }, - "random_write": { - "status": "running", - "duration": 185.1048445701599, - "pod": "random-write-test-pod", - "pvc": "io-test-pvc-random-write" - }, - "mixed_io": { - "status": "running", - "duration": 180.83833074569702, - "pod": "mixed-io-test-pod", - "pvc": "io-test-pvc-mixed-io" - }, - "total_duration": 552.9123649597168 - }, - "metrics": { - "operations": {}, - "system": {}, - "kubernetes": {}, - "csi": {}, - "controller": { - "request_latency": {}, - "operation_counts": {}, - "success_rates": {}, - "volume_attach_timing": {} - }, - "node": { - "mount_timing": {}, - "mount_errors": {}, - "resource_utilization": {} - }, - "efs": { - "access_point_timing": {}, - "mount_completion_timing": {}, - "api_throttling_incidents": [] - }, - "k8s_events": { - "volume_events": [], - "binding_times": {}, - "pod_startup_delays": {} - } - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_210512_20250602_210512_summary.txt b/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_210512_20250602_210512_summary.txt deleted file mode 100644 index 95f882868..000000000 --- a/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250602_210512_20250602_210512_summary.txt +++ /dev/null @@ -1,40 +0,0 @@ -================================================================================ -EFS CSI DRIVER TEST REPORT: EFS_TESTS_20250602_210512 -================================================================================ - -SYSTEM INFORMATION --------------------------------------------------------------------------------- -Hostname: dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com -Platform: Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26 -Python Version: 3.12.10 -CPU Count: 8 -Memory Total: 30.88 GB -Kubernetes Version: Unknown -AWS Region: us-west-1 -EFS Filesystem ID: fs-02b74eafb14930828 -Test Timestamp: 2025-06-02T21:05:12.619174 - -TEST RESULTS --------------------------------------------------------------------------------- - -CONCURRENT_API RESULTS ----------------------------------------- - -Sequential Write Test: - Status: running - Duration: 184.9669418334961 seconds - -Random Write Test: - Status: running - Duration: 185.1048445701599 seconds - -Mixed I/O Test: - Status: running - Duration: 180.83833074569702 seconds - -METRICS RESULTS ----------------------------------------- - -================================================================================ -END OF REPORT: 2025-06-02T21:05:12.663439 -================================================================================ diff --git a/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250603_135616_20250603_135616.json b/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250603_135616_20250603_135616.json deleted file mode 100644 index 474ba6725..000000000 --- a/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250603_135616_20250603_135616.json +++ /dev/null @@ -1,216 +0,0 @@ -{ - "test_name": "efs_tests_20250603_135616", - "test_type": "stress", - "timestamp": "20250603_135616", - "system_info": { - "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", - "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", - "python_version": "3.12.10", - "cpu_count": 8, - "memory_total_gb": 30.88, - "timestamp": "2025-06-03T13:56:16.918247", - "kubernetes_version": "Unknown", - "aws_region": "us-west-1", - "efs_filesystem_id": "fs-02b74eafb14930828" - }, - "results": { - "stress": { - "single_pvc": { - "operation": "concurrent_mount_single_pvc", - "duration": 4.935153245925903, - "pods": 20, - "pods_created": 20 - }, - "multi_pvc": { - "operation": "concurrent_mount_multiple_pvc", - "duration": 7.949950933456421, - "pvcs_created": 10, - "pods_created": 20, - "total_pods": 20 - }, - "sequential_write": { - "status": "running", - "duration": 184.93946361541748, - "pod": "sequential-write-test-pod", - "pvc": "io-test-pvc-sequential-write" - }, - "random_write": { - "status": "running", - "duration": 184.98985695838928, - "pod": "random-write-test-pod", - "pvc": "io-test-pvc-random-write" - }, - "mixed_io": { - "status": "running", - "duration": 180.7335159778595, - "pod": "mixed-io-test-pod", - "pvc": "io-test-pvc-mixed-io" - }, - "total_duration": 552.7618699073792 - }, - "scalability": { - "pod_scaling": { - "5": { - "duration": 5.142866611480713, - "success": true, - "pods_ready": 5 - }, - "10": { - "duration": 5.132429361343384, - "success": true, - "pods_ready": 10 - }, - "15": { - "duration": 10.2158362865448, - "success": true, - "pods_ready": 15 - }, - "20": { - "duration": 10.207310199737549, - "success": true, - "pods_ready": 20 - }, - "25": { - "duration": 25.424506664276123, - "success": true, - "pods_ready": 25 - }, - "30": { - "duration": 10.213911771774292, - "success": true, - "pods_ready": 30 - }, - "35": { - "duration": 25.42242169380188, - "success": true, - "pods_ready": 35 - }, - "40": { - "duration": 15.276111841201782, - "success": true, - "pods_ready": 40 - }, - "45": { - "duration": 305.03443932533264, - "success": false, - "pods_ready": 44 - }, - "50": { - "duration": 304.205361366272, - "success": false, - "pods_ready": 44 - } - }, - "volume_scaling": { - "1": { - "duration": 5.215110540390015, - "success": true, - "volumes_ready": 1 - }, - "6": { - "duration": 10.974458932876587, - "success": true, - "volumes_ready": 6 - }, - "11": { - "duration": 11.669871091842651, - "success": true, - "volumes_ready": 11 - }, - "16": { - "duration": 12.36742639541626, - "success": true, - "volumes_ready": 16 - } - } - }, - "access_points": { - "access_point_scaling": { - "2": { - "duration": 5.353680372238159, - "success": true, - "access_points_ready": 2 - }, - "7": { - "duration": 6.05517315864563, - "success": true, - "access_points_ready": 7 - }, - "12": { - "duration": 6.7946085929870605, - "success": true, - "access_points_ready": 12 - }, - "17": { - "duration": 7.489370346069336, - "success": true, - "access_points_ready": 17 - } - } - }, - "statefulset": { - "statefulset_scaling": { - "replicas": 20, - "duration": 10.138212442398071, - "success": true, - "pods_ready": 20 - } - }, - "chaos": { - "api_flood": { - "test": "rapid_api_flood", - "pvcs_requested": 50, - "pvcs_created": 50, - "success_rate": 100.0, - "duration": 0.46927380561828613, - "access_points_used": true - }, - "race_conditions": { - "test": "race_conditions", - "iterations": 30, - "successes": 17, - "failures": 13, - "exceptions": 0, - "duration": 6.168970346450806 - }, - "resource_exhaustion": { - "test": "resource_exhaustion", - "pods_requested": 75, - "pods_created": 75, - "pod_success_rate": 100.0, - "access_points_requested": 25, - "access_points_created": 0, - "ap_success_rate": 0.0, - "duration": 2.9329886436462402 - }, - "error": "type object 'Thread' has no attribute '_wait'" - }, - "metrics": { - "operations": {}, - "system": {}, - "kubernetes": {}, - "csi": {}, - "controller": { - "request_latency": {}, - "operation_counts": {}, - "success_rates": {}, - "volume_attach_timing": {} - }, - "node": { - "mount_timing": {}, - "mount_errors": {}, - "resource_utilization": {} - }, - "efs": { - "access_point_timing": {}, - "mount_completion_timing": {}, - "api_throttling_incidents": [] - }, - "k8s_events": { - "volume_events": [], - "binding_times": {}, - "pod_startup_delays": {} - } - } - } -} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250603_135616_20250603_135616_summary.txt b/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250603_135616_20250603_135616_summary.txt deleted file mode 100644 index 23b550ff2..000000000 --- a/test/e2e/stress-scale-tests/reports/stress/efs_tests_20250603_135616_20250603_135616_summary.txt +++ /dev/null @@ -1,52 +0,0 @@ -================================================================================ -EFS CSI DRIVER TEST REPORT: EFS_TESTS_20250603_135616 -================================================================================ - -SYSTEM INFORMATION --------------------------------------------------------------------------------- -Hostname: dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com -Platform: Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26 -Python Version: 3.12.10 -CPU Count: 8 -Memory Total: 30.88 GB -Kubernetes Version: Unknown -AWS Region: us-west-1 -EFS Filesystem ID: fs-02b74eafb14930828 -Test Timestamp: 2025-06-03T13:56:16.964029 - -TEST RESULTS --------------------------------------------------------------------------------- - -STRESS RESULTS ----------------------------------------- - -Sequential Write Test: - Status: running - Duration: 184.93946361541748 seconds - -Random Write Test: - Status: running - Duration: 184.98985695838928 seconds - -Mixed I/O Test: - Status: running - Duration: 180.7335159778595 seconds - -SCALABILITY RESULTS ----------------------------------------- - -ACCESS_POINTS RESULTS ----------------------------------------- - -STATEFULSET RESULTS ----------------------------------------- - -CHAOS RESULTS ----------------------------------------- - -METRICS RESULTS ----------------------------------------- - -================================================================================ -END OF REPORT: 2025-06-03T13:56:17.008291 -================================================================================ diff --git a/test/e2e/stress-scale-tests/run_tests.py b/test/e2e/stress-scale-tests/run_tests.py index 8b51b9d2e..a17fcf87c 100644 --- a/test/e2e/stress-scale-tests/run_tests.py +++ b/test/e2e/stress-scale-tests/run_tests.py @@ -238,73 +238,6 @@ def main(): results['orchestrator'] = orchestrator_results - if args.test_suite in ['chaos', 'all']: - logger.info("Running chaos tests") - - if args.dry_run: - logger.info("DRY RUN MODE: Would run chaos tests to stress the CSI driver") - chaos_results = { - 'api_flood': {'status': 'would_run'}, - 'race_conditions': {'status': 'would_run'}, - 'resource_exhaustion': {'status': 'would_run'}, - 'mixed_chaos': {'status': 'would_run'} - } - else: - # Import here to avoid loading it unnecessarily for other tests - from tests.chaos_test import ChaosTester - - logger.warning("CAUTION: Running chaos tests that may cause system instability") - chaos_tester = ChaosTester(args.config) - chaos_results = chaos_tester.run_all_tests() - - # Generate chaos test specific reports - chaos_report_dir = os.path.join(report_dir, 'chaos') - os.makedirs(chaos_report_dir, exist_ok=True) - - # Write a dedicated report for chaos tests - chaos_timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - chaos_report_path = os.path.join(chaos_report_dir, f"efs_chaos_{chaos_timestamp}.json") - with open(chaos_report_path, 'w') as f: - import json - json.dump(chaos_results, f, indent=2) - logger.info(f"Chaos test report generated: {chaos_report_path}") - - results['chaos'] = chaos_results - - # Collect CSI metrics - metrics_collector.collect_csi_metrics(config) - results['metrics'] = metrics_collector.get_all_metrics() - - # Generate reports - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - report_formats = config.get('reporting', {}).get('formats', ['json']) - - if 'json' in report_formats: - json_path = report_generator.generate_json_report(results, f"efs_tests_{timestamp}") - logger.info(f"JSON report generated: {json_path}") - - if 'summary' in report_formats: - summary_path = report_generator.generate_summary_report(results, f"efs_tests_{timestamp}") - logger.info(f"Summary report generated: {summary_path}") - - logger.info("All tests completed successfully") - - # Clean up resources if cluster was set up and --skip-cleanup wasn't specified - DISABLED - if args.setup_cluster and not args.skip_cleanup: - logger.warning("Cluster cleanup functionality is disabled") - """ - logger.info("Cleaning up cluster resources") - try: - cluster_setup = ClusterSetup(config_file=args.config) - cluster_setup.cleanup() - logger.info("Cleanup completed successfully") - except Exception as e: - logger.error(f"Error during cleanup: {e}") - # Don't exit with failure, as the tests themselves succeeded - """ - elif args.setup_cluster and args.skip_cleanup: - logger.warning("Cluster cleanup functionality is disabled") - except Exception as e: logger.error(f"Error running tests: {e}", exc_info=True) From a5cc6e93c5584daec3538c0997ac5281b9937944 Mon Sep 17 00:00:00 2001 From: Ishaan Bhardwaj Date: Mon, 9 Jun 2025 15:24:58 +0000 Subject: [PATCH 03/13] Update configuration and run script for stress testing framework --- test/e2e/stress-scale-tests/config/orchestrator_config.yaml | 6 ++++-- test/e2e/stress-scale-tests/run_tests.py | 1 - 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/test/e2e/stress-scale-tests/config/orchestrator_config.yaml b/test/e2e/stress-scale-tests/config/orchestrator_config.yaml index 2b34d2e52..ca967ad03 100644 --- a/test/e2e/stress-scale-tests/config/orchestrator_config.yaml +++ b/test/e2e/stress-scale-tests/config/orchestrator_config.yaml @@ -115,10 +115,12 @@ storage_class: fileSystemId: fs-024e09bcfd37c74aa gidRangeEnd: '2000' gidRangeStart: '1500' - provisioningMode: efs-ap + provisioningMode: efs-ap # EFS access point mode + uidRangeEnd: '2000' + uidRangeStart: '1500' reclaim_policy: Delete volume_binding_mode: Immediate test: - duration: 300 + duration: 1200 namespace: efs-stress-test operation_interval: 3 diff --git a/test/e2e/stress-scale-tests/run_tests.py b/test/e2e/stress-scale-tests/run_tests.py index a17fcf87c..8c5be5a11 100644 --- a/test/e2e/stress-scale-tests/run_tests.py +++ b/test/e2e/stress-scale-tests/run_tests.py @@ -58,7 +58,6 @@ def parse_args(): parser.add_argument( '--duration', type=int, - default=300, help='Duration in seconds for test execution' ) parser.add_argument( From fb332542e4d494c90007a90bda19cace0384bfbc Mon Sep 17 00:00:00 2001 From: Ishaan Bhardwaj Date: Mon, 9 Jun 2025 21:05:34 +0000 Subject: [PATCH 04/13] Move stress-scale-tests from test/e2e/ to test/ directory and improve documentation - Move stress testing framework to be a peer to e2e tests rather than a subdirectory - Update scenario names in README.md to be more descriptive - Add virtual environment setup instructions - Update credential error message to be more suitable for external users --- .../__pycache__/__init__.cpython-312.pyc | Bin 200 -> 0 bytes .../__pycache__/__init__.cpython-312.pyc | Bin 200 -> 0 bytes test/{e2e => }/stress-scale-tests/README.md | 24 +- .../__pycache__/cluster_setup.cpython-312.pyc | Bin .../__pycache__/run_tests.cpython-312.pyc | Bin 0 -> 9544 bytes .../cleanup_test_resources.py | 0 .../config/orchestrator_config.yaml | 2 +- .../logs/cleanup_20250609_161210.log | 8 + test/stress-scale-tests/logs/efs_tests.log | 134 +++ .../logs/orchestrator_20250605_210820.log | 0 .../logs/orchestrator_20250605_210842.log | 0 .../logs/orchestrator_20250609_142216.log | 12 + .../logs/orchestrator_20250609_142258.log | 12 + .../logs/orchestrator_20250609_142522.log | 1058 +++++++++++++++++ .../logs/orchestrator_20250609_182011.log | 7 + .../logs/orchestrator_20250609_183658.log | 7 + .../logs/orchestrator_20250609_183729.log | 111 ++ .../prometheus_exporter.pid | 1 + ...tests_20250605_201716_20250605_201716.json | 0 .../efs_orchestrator_20250605_201716.json | 0 .../efs_orchestrator_20250609_145633.json | 59 + .../efs_orchestrator_20250609_184030.json | 59 + .../stress-scale-tests/requirements.txt | 0 .../{e2e => }/stress-scale-tests/run_tests.py | 7 +- .../test-reports/sample-test-report.json | 145 +++ .../stress-scale-tests/tests/__init__.py | 0 .../__pycache__/__init__.cpython-312.pyc | Bin 0 -> 196 bytes .../__pycache__/base_test.cpython-312.pyc | Bin .../__pycache__/chaos_test.cpython-312.pyc | Bin .../concurrent_api_test.cpython-312.pyc | Bin .../__pycache__/io_tests.cpython-312.pyc | Bin .../__pycache__/mount_tests.cpython-312.pyc | Bin .../__pycache__/orchestrator.cpython-312.pyc | Bin 75796 -> 75792 bytes .../run_stress_tests.cpython-312.pyc | Bin .../test_access_point_scaling.cpython-312.pyc | Bin ...ic_operations.cpython-312-pytest-8.3.5.pyc | Bin .../test_scalability.cpython-312.pyc | Bin .../test_statefulset_scaling.cpython-312.pyc | Bin ...ss_operations.cpython-312-pytest-8.3.5.pyc | Bin .../test_stress_operations.cpython-312.pyc | Bin .../stress-scale-tests/tests/orchestrator.py | 0 .../stress-scale-tests/utils/__init__.py | 0 .../__pycache__/__init__.cpython-312.pyc | Bin 0 -> 196 bytes .../metrics_collector.cpython-312.pyc | Bin 17194 -> 17190 bytes .../report_generator.cpython-312.pyc | Bin 18806 -> 18802 bytes .../utils/metrics_collector.py | 0 .../utils/report_generator.py | 0 47 files changed, 1634 insertions(+), 12 deletions(-) delete mode 100644 test/e2e/stress-scale-tests/tests/__pycache__/__init__.cpython-312.pyc delete mode 100644 test/e2e/stress-scale-tests/utils/__pycache__/__init__.cpython-312.pyc rename test/{e2e => }/stress-scale-tests/README.md (80%) rename test/{e2e => }/stress-scale-tests/__pycache__/cluster_setup.cpython-312.pyc (100%) create mode 100644 test/stress-scale-tests/__pycache__/run_tests.cpython-312.pyc rename test/{e2e => }/stress-scale-tests/cleanup_test_resources.py (100%) rename test/{e2e => }/stress-scale-tests/config/orchestrator_config.yaml (99%) create mode 100644 test/stress-scale-tests/logs/cleanup_20250609_161210.log create mode 100644 test/stress-scale-tests/logs/efs_tests.log rename test/{e2e => }/stress-scale-tests/logs/orchestrator_20250605_210820.log (100%) rename test/{e2e => }/stress-scale-tests/logs/orchestrator_20250605_210842.log (100%) create mode 100644 test/stress-scale-tests/logs/orchestrator_20250609_142216.log create mode 100644 test/stress-scale-tests/logs/orchestrator_20250609_142258.log create mode 100644 test/stress-scale-tests/logs/orchestrator_20250609_142522.log create mode 100644 test/stress-scale-tests/logs/orchestrator_20250609_182011.log create mode 100644 test/stress-scale-tests/logs/orchestrator_20250609_183658.log create mode 100644 test/stress-scale-tests/logs/orchestrator_20250609_183729.log create mode 100644 test/stress-scale-tests/prometheus_exporter.pid rename test/{e2e => }/stress-scale-tests/reports/general/efs_tests_20250605_201716_20250605_201716.json (100%) rename test/{e2e => }/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_201716.json (100%) create mode 100644 test/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250609_145633.json create mode 100644 test/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250609_184030.json rename test/{e2e => }/stress-scale-tests/requirements.txt (100%) rename test/{e2e => }/stress-scale-tests/run_tests.py (96%) create mode 100644 test/stress-scale-tests/test-reports/sample-test-report.json rename test/{e2e => }/stress-scale-tests/tests/__init__.py (100%) create mode 100644 test/stress-scale-tests/tests/__pycache__/__init__.cpython-312.pyc rename test/{e2e => }/stress-scale-tests/tests/__pycache__/base_test.cpython-312.pyc (100%) rename test/{e2e => }/stress-scale-tests/tests/__pycache__/chaos_test.cpython-312.pyc (100%) rename test/{e2e => }/stress-scale-tests/tests/__pycache__/concurrent_api_test.cpython-312.pyc (100%) rename test/{e2e => }/stress-scale-tests/tests/__pycache__/io_tests.cpython-312.pyc (100%) rename test/{e2e => }/stress-scale-tests/tests/__pycache__/mount_tests.cpython-312.pyc (100%) rename test/{e2e => }/stress-scale-tests/tests/__pycache__/orchestrator.cpython-312.pyc (98%) rename test/{e2e => }/stress-scale-tests/tests/__pycache__/run_stress_tests.cpython-312.pyc (100%) rename test/{e2e => }/stress-scale-tests/tests/__pycache__/test_access_point_scaling.cpython-312.pyc (100%) rename test/{e2e => }/stress-scale-tests/tests/__pycache__/test_basic_operations.cpython-312-pytest-8.3.5.pyc (100%) rename test/{e2e => }/stress-scale-tests/tests/__pycache__/test_scalability.cpython-312.pyc (100%) rename test/{e2e => }/stress-scale-tests/tests/__pycache__/test_statefulset_scaling.cpython-312.pyc (100%) rename test/{e2e => }/stress-scale-tests/tests/__pycache__/test_stress_operations.cpython-312-pytest-8.3.5.pyc (100%) rename test/{e2e => }/stress-scale-tests/tests/__pycache__/test_stress_operations.cpython-312.pyc (100%) rename test/{e2e => }/stress-scale-tests/tests/orchestrator.py (100%) rename test/{e2e => }/stress-scale-tests/utils/__init__.py (100%) create mode 100644 test/stress-scale-tests/utils/__pycache__/__init__.cpython-312.pyc rename test/{e2e => }/stress-scale-tests/utils/__pycache__/metrics_collector.cpython-312.pyc (96%) rename test/{e2e => }/stress-scale-tests/utils/__pycache__/report_generator.cpython-312.pyc (97%) rename test/{e2e => }/stress-scale-tests/utils/metrics_collector.py (100%) rename test/{e2e => }/stress-scale-tests/utils/report_generator.py (100%) diff --git a/test/e2e/stress-scale-tests/tests/__pycache__/__init__.cpython-312.pyc b/test/e2e/stress-scale-tests/tests/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index 649fc6e4c3571841f822be955d8b0327656a813b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 200 zcmX@j%ge<81V3NdW`O9&AOanHW&w&!XQ*V*Wb|9fP{ah}eFmxdm9C$YpPZPZpOK%N zs-KyZkywupIBb3iz=>Pl3HA%pK6qgjEsy$%s>_Z DCone& diff --git a/test/e2e/stress-scale-tests/utils/__pycache__/__init__.cpython-312.pyc b/test/e2e/stress-scale-tests/utils/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index fb113996f76cbcc85af4ac5eb7fe86b0f6012f3e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 200 zcmX@j%ge<81V3NdW`O9&AOanHW&w&!XQ*V*Wb|9fP{ah}eFmxdm9C$YpPZPZpOK%N zs-KyZkywupIBb3iz=>Pl3HA%pK6q-5!pP83g5+AQuPF--EnAUfTa_)#mgCq08g|SXNtF4}-Wgiv zMD%*wEoyHASl%w6ERvw!V1e4O(764n7AUX|P+*I86O1edVkZh3VBL@WX&o0XlAoS? zheJ||)C5H+XzuIWbI(2ZyzaUG?shvU_@!SxHT4%Q6!iz(Fn^X@<*Q#qWq}eYk&aU1 zMoEv;q-`0ukdhf^NXd?~P+Fqan04GrV^}6y60?olV%#`K+HBMwbBsGk-5PbqT;ncM zFNwNio^cPPy&?x+pJ<0~spx=jndpRXx#SmJ*V-uxaTZD{f0QGUrc!i@4!FO> z?U4YjbE2R~N+c!)EP6>e8j<1(l!^F6WD-Wp22YOlj}4te;VDU0kf0QRoFX+m&SlwTM zHET+xSu<%LqXG_97?YIb3?EHQPDbLB{8d4gL_QJc4cSP-!ABBt2M<5;=|hw95mLil z2G9|{p9oS8m=_~qJV`()0;t<^$aQN}x-La^c4*|}Ih~z|L?yKh#^f$(LJlb)d2%N- zFFpium2O02C6u_P?rUimWWWcKEAv6Vr9Cd>s{~z9J0{7pFbRMG203B3ddWlr#RNs? zri8c{m5>~8>CB|0=r;HWih>Y6aNUxSbuK1clf(#;b#_KjrgR(BF>2ijEQy4T9qP`L zK=f$?neHA_kR-&+rX6se0X9lV=RzSoHxvr62#bX7fPIAwZs~ShH#ouJD&~WXMIUqY z>o>ciiLemuno7i^uE^CX0g0GB!LS?&iYRhjLP6NnwJu>+4sPu4!W`@Z^#v{jWxyc? z2~WFFGM?k^OiFjl_`E}~Fg|b81nXi!lz#=EThzLXDyy5XT&TTQ`_9mPv{JhJ)}O5T zs@}8Rul>7*Uo||oEmf{Gomlbp-5SbzYwq5>bMu`Et@X%?_tjgcUzAW*+g}a;WcU}B z1=~H_g7=>HKDXivJRbOzJ^YMyuCi6jY*p4%mGurjbCs{U8kb#-0O1O(;a1bKt7*m6 zyl!R69RF>zSxa7&QPA(KTXxpHcX`FxsQAlTLkYDOBc(&bA@uzFIFs8ekbNCfUEu;*Nr8+lC8uV*U4kYJfExH?3(e z@Z7AKW(^DzubvqdkSu|TiN!$Q_-G_9@dBDm#=v#S#ITUh1rTQxFS1Ewnz#v6ExnM8 z^Wf!paC1CS3vh5wQIf+bGJ~y^$_0alf_~j-lwIK6^6oIFyqzf_7FBCU!9ws#Vxy7d zu_A+ewTaS%kc=uiJ0(SDR9BFwVNgy+6e+;yu41&hH9RGN^oy4Uqm~bdk8MzyTMNK7DiEm?O^R@2;bmAbDx>ubom`E@%1umXUq2*AJQ z_GR7sx0&w4qrTFtyK|fA4z% z`lqDuH9i7ji6jvWa72iLK!Uzq7ori+@e~i?6V^L{mm{&6sKob;4)G9!M8Cul7D$>e zgrpxO!M25PEC+_*qR#B=?$KSZ$Au(V1{6^xapYvciZH8mdp~INrJlZ-i0+BvP!Pvc z@{ACcbo=0qutaRO&Oo#S=8+KqjD`^jp#%y+r;K?G;S>>Ry%g+t_*$rlcJvyI3-B+$ z0UwBaScldB%;R76G%R}>Ry_P|Cd)Zjx!Pr}_Q7i!SDWFwvflFlS#$cIQ&y{g&0qBn z{Wm9X+p|vJ{Lr$q{(i&b!^=%a*UGD(dCR|9cLL1UGLFLkyfr{QwD%NF z8cGt=ZFn057sXjD1buISsV)eLdfsvD2mYx$@f!UX1_uU5E)Mmb8RG{pj}AeTKX-v2 z8oAVWW@tcV-aOQ;j{d+aA5-@_u7E|}yh(maxGsTnn~A{2dGH-rJ0fwSAD!y3V-!p~ z4rhoWrM5zVothMvlQ2)2l2Qc03`&4Ij;8XFh9q{Xy^cQO3iA9W=!zJ=hgVRl;0mVz zA--Jk!lV$1cdF-qXqGr|hQT_6J^w>6J|M@2-nEdfBE16}03sqpL&dU#PQru){L5G= zZ&A-!`|V(ctzKj8n!D$5v*tRGVGlv;cI!{i;!el$410o9E@#*)q%xdg&tQn)r)>4N zjc-0L(F?c#VihZxSun`hWRM?(y4VJ7`hhxHjIR9@nAiU^qp-o`T0mtj=3G$Naue~7 z3zrdgj-9SB`wJzQ=`Cq_mwG=x7p@#z&n;H*Dif!>JVGAW{hIJbYZL=m88JTS$>;*Gq$GIiTG;hu>l%k`MTbgbv^qV!& zDZ0$FNL!U=vwzOI=r-}EOOzI~-`siH7NOFd9bfNo47}+ldahIR7PNPplkl3D$@LJ& zRhxdOdYhTP!c4JLEECJ$v!w0wbo}Ku@P43g_a@}fe3ud{-p}W(Af;HDb|^TYC%@@V zvsNgldkS^4Hs>&@k%QZuQPrFS)~eoCYB$Qq7Fw2eY_qc(jJa6ql~)RME!LWprJY-I z0CjKOKj#e9=J9N;M?1EjxtXT`b-=__DAAa5*ldZc@@k>JHN5JiD045SoC1vv=DcmV zQ#a?FK5D`gN}#cR>vdqH;r*h}0OlVr%rk3w>9>D}wDg%X3gw(tY&3U}b}2z~WExxf zO}{yp$S*dT&|qZ-O@L|a95($-s-}lF4M0=eX>u>^{(sQy5}W7Tz?)s5EjeqS_7r&r zkJw`NV=o}KLTbMy?Va;ZpZzv1bM`q}Y+GzEq-3xsDE0;0oMvFovr*Oo6DKy}15Q~| zmS(DllId9sO;QeGO*I0 z8MVUoU^6vuU)z$FP?~|=Zsl=y8+3sLbeOy)c5*B4(E!ps@@-DbWq^Atud>IK1{z&IH z-_@=$ML-H3Hgg3=T!vF48)xtnC=uh;noAOjOr*AosarP%*(6*n3O{q9r+TR0$fH0? z2j}@Gl5rzBAB`v}J|aW1UKXxKA!H5mI3Y2E*CL?mP%dP}iE&6sL0AfLrV-`}klR7z zzGr4g$QI09c0^v10JrXdtKUpg2|;>Yw;A1#h(N=@pMZs=zz}v|YAYn{ zNG7K^UB(;PZ4%BJIWqP5z=bRPh1W;;v*!i|kMM7hoD*b$AOtUD?@02AM-X8rV)&lP zysrL*`HbS5U;KsXT6%km-( ztT4_|=KwKz`O(>FFjNTNFc7}x=uA8@t8<`D6F3_K!}wrWF&UfDUDynSatsXkbQY%y zb$Ws@37x>ROYrLvJaajWQzE+T%hhuymOy%G%y=t?)Oqf)7%RLCnQ>Y7Z;%YLw|qmd zA;Uz(^r{UbMa$YT{h)n7c=c8=_$?1j1JN?Ru0Td;!_T;;vN15|C_p4HOs<&TOa9;1|E7M)`{agJH=6OChs0{zXzN%?L1H|eFszh2h(OBw$gUzArZ9RB&?wF8Hjn*Mq0N$Hb|zq|6$m5=IIj-6XM zFp4L?lAqlF``P$ZeWd?U&SrR|>^*^iI z^|xn!dFI`b)w)-f>t1<$Zl!Ku-o6Gi4?aBj;n9ajv;K~3W!nS${kI;sX#PWAS(!a$ zny>wZgQ~36YWF_w&r}>*>if*!m<65;-5b&x_CD@g66S|8{u3{)ROMkf7xT=HX}y=U zV{d46movUAU$h>%eQJK>DZIu?H03(*GD(AoFT0z+d4}_!7SIUJt@(lvga>1qrzgYh z&5P_^=g*wqtXl!(>zAh)SVOBg^vw$&<*Im2`9Q*Q(q6Ua!TAUMT1EHLzT2)0cLG@C z@~t}SpF$>8Tx~u0wDlmCzmYp5+K!VM*Qs?2=k~quQ+Ai38F!s`V3{Vs__Z7YrG2~d zY$x?W<=G}?iG$j|G@dRQ*+>1prDmj^`F(E_)IV;rp6#%G+=_=k?zE4zFdrXm9^si! zcq`ODX}6B_*ggr`0P>T4_R%`#lVg1qqg9MnWrYTQ(8ops9`q{=`+OM|MVIgg2i&L| ze~4vu6keG~Cfo2WWhBt+EVfXFRj-7Xxv1Q^*?1Cfp;sA$-?F2`jSx>{yw>0w6~Y%K zV#E**oDsf36{X_1qp^gTj7q)e$1s4SQF#nL>kLiPf1vE2QI5|j=Vz4XGs^opRsWw< z%YRWlpHq83r}jPPD(SZ8-3RH$=ZJRE{^xZZUGv;OV4?l%XIP4L-BzD6HUGhKe|d79 Kq1alo=>Gz2)%~Ra literal 0 HcmV?d00001 diff --git a/test/e2e/stress-scale-tests/cleanup_test_resources.py b/test/stress-scale-tests/cleanup_test_resources.py similarity index 100% rename from test/e2e/stress-scale-tests/cleanup_test_resources.py rename to test/stress-scale-tests/cleanup_test_resources.py diff --git a/test/e2e/stress-scale-tests/config/orchestrator_config.yaml b/test/stress-scale-tests/config/orchestrator_config.yaml similarity index 99% rename from test/e2e/stress-scale-tests/config/orchestrator_config.yaml rename to test/stress-scale-tests/config/orchestrator_config.yaml index ca967ad03..7a8880132 100644 --- a/test/e2e/stress-scale-tests/config/orchestrator_config.yaml +++ b/test/stress-scale-tests/config/orchestrator_config.yaml @@ -121,6 +121,6 @@ storage_class: reclaim_policy: Delete volume_binding_mode: Immediate test: - duration: 1200 + duration: 120 namespace: efs-stress-test operation_interval: 3 diff --git a/test/stress-scale-tests/logs/cleanup_20250609_161210.log b/test/stress-scale-tests/logs/cleanup_20250609_161210.log new file mode 100644 index 000000000..35962ed2c --- /dev/null +++ b/test/stress-scale-tests/logs/cleanup_20250609_161210.log @@ -0,0 +1,8 @@ +2025-06-09 16:12:10,963 - INFO - Starting EFS CSI Driver test resource cleanup +2025-06-09 16:12:12,039 - ERROR - Failed to list namespaces: (401) +Reason: Unauthorized +HTTP response headers: HTTPHeaderDict({'Audit-Id': '1a009cf9-0d14-4835-a132-092db20a6853', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Mon, 09 Jun 2025 16:12:12 GMT', 'Content-Length': '129'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} + + +2025-06-09 16:12:12,040 - INFO - Cleanup process completed diff --git a/test/stress-scale-tests/logs/efs_tests.log b/test/stress-scale-tests/logs/efs_tests.log new file mode 100644 index 000000000..c861cbbb6 --- /dev/null +++ b/test/stress-scale-tests/logs/efs_tests.log @@ -0,0 +1,134 @@ +2025-06-09 14:22:15,661 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-09 14:22:15,661 - __main__ - INFO - Verifying AWS credentials +2025-06-09 14:22:15,663 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-09 14:22:15,663 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-09 14:22:15,664 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-09 14:22:15,664 - __main__ - INFO - Running orchestrator stress test suite +2025-06-09 14:22:15,664 - __main__ - INFO - Starting orchestrator for 300 seconds +2025-06-09 14:22:15,664 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-09 14:22:16,871 - __main__ - INFO - Test duration overridden to 300 seconds +2025-06-09 14:22:16,871 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) +2025-06-09 14:22:57,994 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-09 14:22:57,994 - __main__ - INFO - Verifying AWS credentials +2025-06-09 14:22:57,995 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-09 14:22:57,996 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-09 14:22:57,996 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-09 14:22:57,996 - __main__ - INFO - Running orchestrator stress test suite +2025-06-09 14:22:57,997 - __main__ - INFO - Starting orchestrator for 300 seconds +2025-06-09 14:22:57,997 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-09 14:22:59,034 - __main__ - INFO - Test duration overridden to 300 seconds +2025-06-09 14:22:59,034 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) +2025-06-09 14:25:21,361 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-09 14:25:21,361 - __main__ - INFO - Verifying AWS credentials +2025-06-09 14:25:21,362 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-09 14:25:21,363 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-09 14:25:21,363 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-09 14:25:21,364 - __main__ - INFO - Running orchestrator stress test suite +2025-06-09 14:25:21,364 - __main__ - INFO - Starting orchestrator for None seconds +2025-06-09 14:25:21,364 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-09 14:25:22,404 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) +2025-06-09 14:56:33,409 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250609_145633.json +2025-06-09 18:20:10,833 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-09 18:20:10,833 - __main__ - INFO - Verifying AWS credentials +2025-06-09 18:20:10,834 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-09 18:20:10,834 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-09 18:20:10,835 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-09 18:20:10,835 - __main__ - INFO - Running orchestrator stress test suite +2025-06-09 18:20:10,835 - __main__ - INFO - Starting orchestrator for None seconds +2025-06-09 18:20:10,835 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-09 18:20:11,779 - __main__ - ERROR - Error running tests: (401) +Reason: Unauthorized +HTTP response headers: HTTPHeaderDict({'Audit-Id': 'd5d12f12-977a-4faf-bd61-ad049e1a9ec2', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Mon, 09 Jun 2025 18:20:11 GMT', 'Content-Length': '129'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} + +Traceback (most recent call last): + File "/local/home/ibhard/efs-csi-driver-fork/aws-efs-csi-driver/test/e2e/stress-scale-tests/run_tests.py", line 208, in main + orchestrator = EFSCSIOrchestrator(config_file=orchestrator_config) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/local/home/ibhard/efs-csi-driver-fork/aws-efs-csi-driver/test/e2e/stress-scale-tests/tests/orchestrator.py", line 91, in __init__ + self._ensure_namespace_exists() + File "/local/home/ibhard/efs-csi-driver-fork/aws-efs-csi-driver/test/e2e/stress-scale-tests/tests/orchestrator.py", line 97, in _ensure_namespace_exists + self.core_v1.read_namespace(name=self.namespace) + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api/core_v1_api.py", line 22992, in read_namespace + return self.read_namespace_with_http_info(name, **kwargs) # noqa: E501 + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api/core_v1_api.py", line 23071, in read_namespace_with_http_info + return self.api_client.call_api( + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api_client.py", line 348, in call_api + return self.__call_api(resource_path, method, + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api_client.py", line 180, in __call_api + response_data = self.request( + ^^^^^^^^^^^^^ + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api_client.py", line 373, in request + return self.rest_client.GET(url, + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/rest.py", line 244, in GET + return self.request("GET", url, + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/rest.py", line 238, in request + raise ApiException(http_resp=r) +kubernetes.client.exceptions.ApiException: (401) +Reason: Unauthorized +HTTP response headers: HTTPHeaderDict({'Audit-Id': 'd5d12f12-977a-4faf-bd61-ad049e1a9ec2', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Mon, 09 Jun 2025 18:20:11 GMT', 'Content-Length': '129'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} + + +2025-06-09 18:36:58,115 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-09 18:36:58,115 - __main__ - INFO - Verifying AWS credentials +2025-06-09 18:36:58,116 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-09 18:36:58,116 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-09 18:36:58,117 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-09 18:36:58,117 - __main__ - INFO - Running orchestrator stress test suite +2025-06-09 18:36:58,117 - __main__ - INFO - Starting orchestrator for None seconds +2025-06-09 18:36:58,117 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-09 18:36:59,092 - __main__ - ERROR - Error running tests: (401) +Reason: Unauthorized +HTTP response headers: HTTPHeaderDict({'Audit-Id': 'd3c86ac1-4d01-4ad7-a9bc-d37363c9419a', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Mon, 09 Jun 2025 18:36:59 GMT', 'Content-Length': '129'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} + +Traceback (most recent call last): + File "/local/home/ibhard/efs-csi-driver-fork/aws-efs-csi-driver/test/e2e/stress-scale-tests/run_tests.py", line 208, in main + orchestrator = EFSCSIOrchestrator(config_file=orchestrator_config) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/local/home/ibhard/efs-csi-driver-fork/aws-efs-csi-driver/test/e2e/stress-scale-tests/tests/orchestrator.py", line 91, in __init__ + self._ensure_namespace_exists() + File "/local/home/ibhard/efs-csi-driver-fork/aws-efs-csi-driver/test/e2e/stress-scale-tests/tests/orchestrator.py", line 97, in _ensure_namespace_exists + self.core_v1.read_namespace(name=self.namespace) + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api/core_v1_api.py", line 22992, in read_namespace + return self.read_namespace_with_http_info(name, **kwargs) # noqa: E501 + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api/core_v1_api.py", line 23071, in read_namespace_with_http_info + return self.api_client.call_api( + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api_client.py", line 348, in call_api + return self.__call_api(resource_path, method, + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api_client.py", line 180, in __call_api + response_data = self.request( + ^^^^^^^^^^^^^ + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api_client.py", line 373, in request + return self.rest_client.GET(url, + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/rest.py", line 244, in GET + return self.request("GET", url, + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/rest.py", line 238, in request + raise ApiException(http_resp=r) +kubernetes.client.exceptions.ApiException: (401) +Reason: Unauthorized +HTTP response headers: HTTPHeaderDict({'Audit-Id': 'd3c86ac1-4d01-4ad7-a9bc-d37363c9419a', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Mon, 09 Jun 2025 18:36:59 GMT', 'Content-Length': '129'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} + + +2025-06-09 18:37:28,448 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml +2025-06-09 18:37:28,448 - __main__ - INFO - Verifying AWS credentials +2025-06-09 18:37:28,449 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-09 18:37:28,449 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-09 18:37:28,449 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces +2025-06-09 18:37:28,450 - __main__ - INFO - Running orchestrator stress test suite +2025-06-09 18:37:28,450 - __main__ - INFO - Starting orchestrator for None seconds +2025-06-09 18:37:28,450 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml +2025-06-09 18:37:29,435 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) +2025-06-09 18:40:30,388 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250609_184030.json diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_210820.log b/test/stress-scale-tests/logs/orchestrator_20250605_210820.log similarity index 100% rename from test/e2e/stress-scale-tests/logs/orchestrator_20250605_210820.log rename to test/stress-scale-tests/logs/orchestrator_20250605_210820.log diff --git a/test/e2e/stress-scale-tests/logs/orchestrator_20250605_210842.log b/test/stress-scale-tests/logs/orchestrator_20250605_210842.log similarity index 100% rename from test/e2e/stress-scale-tests/logs/orchestrator_20250605_210842.log rename to test/stress-scale-tests/logs/orchestrator_20250605_210842.log diff --git a/test/stress-scale-tests/logs/orchestrator_20250609_142216.log b/test/stress-scale-tests/logs/orchestrator_20250609_142216.log new file mode 100644 index 000000000..afaa08b91 --- /dev/null +++ b/test/stress-scale-tests/logs/orchestrator_20250609_142216.log @@ -0,0 +1,12 @@ +2025-06-09 14:22:16,647 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-09 14:22:16,871 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-09 14:22:16,872 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-09 14:22:17,065 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-09 14:22:17,065 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-09 14:22:17,065 - tests.orchestrator - INFO - Creating PVC: test-pvc-4ada8088 +2025-06-09 14:22:17,141 - tests.orchestrator - INFO - Created PVC: test-pvc-4ada8088 +2025-06-09 14:22:17,185 - tests.orchestrator - INFO - Waiting for PVC test-pvc-4ada8088 to be bound +2025-06-09 14:22:19,316 - tests.orchestrator - INFO - PVC test-pvc-4ada8088 is bound +2025-06-09 14:22:19,317 - tests.orchestrator - INFO - Creating PVC: test-pvc-5c13c143 +2025-06-09 14:22:19,386 - tests.orchestrator - INFO - Created PVC: test-pvc-5c13c143 +2025-06-09 14:22:19,386 - tests.orchestrator - INFO - Waiting for PVC test-pvc-5c13c143 to be bound diff --git a/test/stress-scale-tests/logs/orchestrator_20250609_142258.log b/test/stress-scale-tests/logs/orchestrator_20250609_142258.log new file mode 100644 index 000000000..e4a908e42 --- /dev/null +++ b/test/stress-scale-tests/logs/orchestrator_20250609_142258.log @@ -0,0 +1,12 @@ +2025-06-09 14:22:58,840 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-09 14:22:59,033 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-09 14:22:59,034 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds +2025-06-09 14:22:59,227 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-09 14:22:59,228 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-09 14:22:59,228 - tests.orchestrator - INFO - Creating PVC: test-pvc-02e00a44 +2025-06-09 14:22:59,305 - tests.orchestrator - INFO - Created PVC: test-pvc-02e00a44 +2025-06-09 14:22:59,305 - tests.orchestrator - INFO - Waiting for PVC test-pvc-02e00a44 to be bound +2025-06-09 14:23:01,462 - tests.orchestrator - INFO - PVC test-pvc-02e00a44 is bound +2025-06-09 14:23:01,462 - tests.orchestrator - INFO - Creating PVC: test-pvc-4b26f6f6 +2025-06-09 14:23:01,585 - tests.orchestrator - INFO - Created PVC: test-pvc-4b26f6f6 +2025-06-09 14:23:01,585 - tests.orchestrator - INFO - Waiting for PVC test-pvc-4b26f6f6 to be bound diff --git a/test/stress-scale-tests/logs/orchestrator_20250609_142522.log b/test/stress-scale-tests/logs/orchestrator_20250609_142522.log new file mode 100644 index 000000000..f1adc59d4 --- /dev/null +++ b/test/stress-scale-tests/logs/orchestrator_20250609_142522.log @@ -0,0 +1,1058 @@ +2025-06-09 14:25:22,187 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-09 14:25:22,403 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-09 14:25:22,404 - tests.orchestrator - INFO - Starting orchestrator test for 1200 seconds +2025-06-09 14:25:22,592 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-09 14:25:22,592 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-09 14:25:22,592 - tests.orchestrator - INFO - Creating PVC: test-pvc-da1ffeef +2025-06-09 14:25:22,670 - tests.orchestrator - INFO - Created PVC: test-pvc-da1ffeef +2025-06-09 14:25:22,670 - tests.orchestrator - INFO - Waiting for PVC test-pvc-da1ffeef to be bound +2025-06-09 14:25:24,806 - tests.orchestrator - INFO - PVC test-pvc-da1ffeef is bound +2025-06-09 14:25:24,806 - tests.orchestrator - INFO - Creating PVC: test-pvc-add96f02 +2025-06-09 14:25:24,873 - tests.orchestrator - INFO - Created PVC: test-pvc-add96f02 +2025-06-09 14:25:24,873 - tests.orchestrator - INFO - Waiting for PVC test-pvc-add96f02 to be bound +2025-06-09 14:25:27,007 - tests.orchestrator - INFO - PVC test-pvc-add96f02 is bound +2025-06-09 14:25:27,007 - tests.orchestrator - INFO - Creating pod test-pod-74c06ab7 on PVC test-pvc-add96f02 +2025-06-09 14:25:27,091 - tests.orchestrator - INFO - Created pod: test-pod-74c06ab7 using PVC: test-pvc-add96f02 +2025-06-09 14:25:27,091 - tests.orchestrator - INFO - Waiting for pod test-pod-74c06ab7 to be ready +2025-06-09 14:25:27,157 - tests.orchestrator - INFO - Pod test-pod-74c06ab7 phase: Pending +2025-06-09 14:25:29,226 - tests.orchestrator - INFO - Pod test-pod-74c06ab7 phase: Running +2025-06-09 14:25:29,227 - tests.orchestrator - INFO - Pod test-pod-74c06ab7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:25:31,295 - tests.orchestrator - INFO - Pod test-pod-74c06ab7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:25:33,362 - tests.orchestrator - INFO - Pod test-pod-74c06ab7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:25:35,431 - tests.orchestrator - INFO - Pod test-pod-74c06ab7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:25:37,500 - tests.orchestrator - INFO - Pod test-pod-74c06ab7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:25:39,567 - tests.orchestrator - INFO - Pod test-pod-74c06ab7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:25:41,692 - tests.orchestrator - INFO - Pod test-pod-74c06ab7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:25:43,760 - tests.orchestrator - INFO - Pod test-pod-74c06ab7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:25:45,827 - tests.orchestrator - INFO - Pod test-pod-74c06ab7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:25:47,895 - tests.orchestrator - INFO - Pod test-pod-74c06ab7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:25:49,964 - tests.orchestrator - INFO - Pod test-pod-74c06ab7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-09 14:25:49,964 - tests.orchestrator - INFO - Pod test-pod-74c06ab7 is ready +2025-06-09 14:25:49,964 - tests.orchestrator - INFO - Creating pod test-pod-667e05fd on PVC test-pvc-add96f02 +2025-06-09 14:25:50,042 - tests.orchestrator - INFO - Created pod: test-pod-667e05fd using PVC: test-pvc-add96f02 +2025-06-09 14:25:50,043 - tests.orchestrator - INFO - Waiting for pod test-pod-667e05fd to be ready +2025-06-09 14:25:50,111 - tests.orchestrator - INFO - Pod test-pod-667e05fd phase: Pending +2025-06-09 14:25:52,177 - tests.orchestrator - INFO - Pod test-pod-667e05fd phase: Running +2025-06-09 14:25:52,177 - tests.orchestrator - INFO - Pod test-pod-667e05fd conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:25:54,246 - tests.orchestrator - INFO - Pod test-pod-667e05fd conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:25:56,315 - tests.orchestrator - INFO - Pod test-pod-667e05fd conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:25:58,385 - tests.orchestrator - INFO - Pod test-pod-667e05fd conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:26:00,454 - tests.orchestrator - INFO - Pod test-pod-667e05fd conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:26:02,523 - tests.orchestrator - INFO - Pod test-pod-667e05fd conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:26:04,592 - tests.orchestrator - INFO - Pod test-pod-667e05fd conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:26:06,662 - tests.orchestrator - INFO - Pod test-pod-667e05fd conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:26:08,730 - tests.orchestrator - INFO - Pod test-pod-667e05fd conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:26:10,799 - tests.orchestrator - INFO - Pod test-pod-667e05fd conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:26:12,875 - tests.orchestrator - INFO - Pod test-pod-667e05fd conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-09 14:26:12,875 - tests.orchestrator - INFO - Pod test-pod-667e05fd is ready +2025-06-09 14:26:12,875 - tests.orchestrator - INFO - Creating pod test-pod-00c7ec79 on PVC test-pvc-add96f02 +2025-06-09 14:26:12,971 - tests.orchestrator - INFO - Created pod: test-pod-00c7ec79 using PVC: test-pvc-add96f02 +2025-06-09 14:26:12,971 - tests.orchestrator - INFO - Waiting for pod test-pod-00c7ec79 to be ready +2025-06-09 14:26:13,050 - tests.orchestrator - INFO - Pod test-pod-00c7ec79 phase: Pending +2025-06-09 14:26:15,118 - tests.orchestrator - INFO - Pod test-pod-00c7ec79 phase: Running +2025-06-09 14:26:15,118 - tests.orchestrator - INFO - Pod test-pod-00c7ec79 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:26:17,187 - tests.orchestrator - INFO - Pod test-pod-00c7ec79 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:26:19,256 - tests.orchestrator - INFO - Pod test-pod-00c7ec79 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:26:21,322 - tests.orchestrator - INFO - Pod test-pod-00c7ec79 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:26:23,392 - tests.orchestrator - INFO - Pod test-pod-00c7ec79 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:26:25,460 - tests.orchestrator - INFO - Pod test-pod-00c7ec79 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:26:27,527 - tests.orchestrator - INFO - Pod test-pod-00c7ec79 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:26:29,596 - tests.orchestrator - INFO - Pod test-pod-00c7ec79 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:26:31,666 - tests.orchestrator - INFO - Pod test-pod-00c7ec79 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:26:33,733 - tests.orchestrator - INFO - Pod test-pod-00c7ec79 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-09 14:26:33,733 - tests.orchestrator - INFO - Pod test-pod-00c7ec79 is ready +2025-06-09 14:26:33,734 - tests.orchestrator - INFO - Testing read/write between pods test-pod-00c7ec79 and test-pod-667e05fd sharing PVC test-pvc-add96f02 +2025-06-09 14:26:33,734 - tests.orchestrator - INFO - Executing write command: kubectl exec -n efs-stress-test test-pod-00c7ec79 -- /bin/sh -c 'echo "Test content: bf00aef3-7774-4aae-9288-d929351fe781" > /data/test-e4d325ac.txt' +2025-06-09 14:26:35,176 - tests.orchestrator - INFO - Pod test-pod-00c7ec79 wrote to /data/test-e4d325ac.txt +2025-06-09 14:26:37,176 - tests.orchestrator - INFO - Executing read command: kubectl exec -n efs-stress-test test-pod-667e05fd -- cat /data/test-e4d325ac.txt +2025-06-09 14:26:38,472 - tests.orchestrator - INFO - Read result: 'Test content: bf00aef3-7774-4aae-9288-d929351fe781' +2025-06-09 14:26:38,472 - tests.orchestrator - INFO - Pod test-pod-667e05fd successfully read content written by test-pod-00c7ec79 +2025-06-09 14:26:38,472 - tests.orchestrator - INFO - Deleting pod: test-pod-74c06ab7 from PVC: test-pvc-add96f02 +2025-06-09 14:26:38,548 - tests.orchestrator - INFO - Waiting for pod test-pod-74c06ab7 to be deleted +2025-06-09 14:27:11,161 - tests.orchestrator - INFO - Pod test-pod-74c06ab7 has been deleted +2025-06-09 14:27:11,161 - tests.orchestrator - INFO - Deleted pod: test-pod-74c06ab7 +2025-06-09 14:27:11,161 - tests.orchestrator - INFO - Deleting PVC: test-pvc-add96f02 +2025-06-09 14:27:11,161 - tests.orchestrator - INFO - Deleting 2 pods using PVC test-pvc-add96f02 +2025-06-09 14:27:11,161 - tests.orchestrator - INFO - Deleting pod: test-pod-667e05fd from PVC: test-pvc-add96f02 +2025-06-09 14:27:11,238 - tests.orchestrator - INFO - Waiting for pod test-pod-667e05fd to be deleted +2025-06-09 14:27:43,855 - tests.orchestrator - INFO - Pod test-pod-667e05fd has been deleted +2025-06-09 14:27:43,855 - tests.orchestrator - INFO - Deleted pod: test-pod-667e05fd +2025-06-09 14:27:43,855 - tests.orchestrator - INFO - Deleting pod: test-pod-00c7ec79 from PVC: test-pvc-add96f02 +2025-06-09 14:27:43,930 - tests.orchestrator - INFO - Waiting for pod test-pod-00c7ec79 to be deleted +2025-06-09 14:28:16,549 - tests.orchestrator - INFO - Pod test-pod-00c7ec79 has been deleted +2025-06-09 14:28:16,549 - tests.orchestrator - INFO - Deleted pod: test-pod-00c7ec79 +2025-06-09 14:28:16,623 - tests.orchestrator - INFO - Waiting for PVC test-pvc-add96f02 to be deleted +2025-06-09 14:28:16,695 - tests.orchestrator - INFO - PVC test-pvc-add96f02 has been deleted +2025-06-09 14:28:16,695 - tests.orchestrator - INFO - Deleted PVC: test-pvc-add96f02 +2025-06-09 14:28:16,695 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations +2025-06-09 14:28:16,695 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 1 times) +2025-06-09 14:28:16,695 - tests.orchestrator - INFO - Creating PVC: test-pvc-867df8a7 +2025-06-09 14:28:16,776 - tests.orchestrator - INFO - Created PVC: test-pvc-867df8a7 +2025-06-09 14:28:16,776 - tests.orchestrator - INFO - Waiting for PVC test-pvc-867df8a7 to be bound +2025-06-09 14:28:18,914 - tests.orchestrator - INFO - PVC test-pvc-867df8a7 is bound +2025-06-09 14:28:19,915 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 1 times) +2025-06-09 14:28:19,915 - tests.orchestrator - INFO - Creating pod test-pod-6fd060e1 on PVC test-pvc-da1ffeef +2025-06-09 14:28:19,990 - tests.orchestrator - INFO - Created pod: test-pod-6fd060e1 using PVC: test-pvc-da1ffeef +2025-06-09 14:28:19,991 - tests.orchestrator - INFO - Waiting for pod test-pod-6fd060e1 to be ready +2025-06-09 14:28:20,059 - tests.orchestrator - INFO - Pod test-pod-6fd060e1 phase: Pending +2025-06-09 14:28:24,195 - tests.orchestrator - INFO - Pod test-pod-6fd060e1 phase: Running +2025-06-09 14:28:24,196 - tests.orchestrator - INFO - Pod test-pod-6fd060e1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:28:26,264 - tests.orchestrator - INFO - Pod test-pod-6fd060e1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:28:28,330 - tests.orchestrator - INFO - Pod test-pod-6fd060e1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:28:30,399 - tests.orchestrator - INFO - Pod test-pod-6fd060e1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:28:32,467 - tests.orchestrator - INFO - Pod test-pod-6fd060e1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:28:34,534 - tests.orchestrator - INFO - Pod test-pod-6fd060e1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:28:36,603 - tests.orchestrator - INFO - Pod test-pod-6fd060e1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:28:38,672 - tests.orchestrator - INFO - Pod test-pod-6fd060e1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:28:40,738 - tests.orchestrator - INFO - Pod test-pod-6fd060e1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-09 14:28:40,739 - tests.orchestrator - INFO - Pod test-pod-6fd060e1 is ready +2025-06-09 14:28:41,739 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 2 times) +2025-06-09 14:28:41,739 - tests.orchestrator - INFO - Creating PVC: test-pvc-ff5ff1dd +2025-06-09 14:28:41,847 - tests.orchestrator - INFO - Created PVC: test-pvc-ff5ff1dd +2025-06-09 14:28:41,847 - tests.orchestrator - INFO - Waiting for PVC test-pvc-ff5ff1dd to be bound +2025-06-09 14:28:43,985 - tests.orchestrator - INFO - PVC test-pvc-ff5ff1dd is bound +2025-06-09 14:28:44,985 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 2 times) +2025-06-09 14:28:44,985 - tests.orchestrator - INFO - Creating pod test-pod-fe9965de on PVC test-pvc-ff5ff1dd +2025-06-09 14:28:45,063 - tests.orchestrator - INFO - Created pod: test-pod-fe9965de using PVC: test-pvc-ff5ff1dd +2025-06-09 14:28:45,063 - tests.orchestrator - INFO - Waiting for pod test-pod-fe9965de to be ready +2025-06-09 14:28:45,133 - tests.orchestrator - INFO - Pod test-pod-fe9965de phase: Pending +2025-06-09 14:28:49,271 - tests.orchestrator - INFO - Pod test-pod-fe9965de phase: Running +2025-06-09 14:28:49,271 - tests.orchestrator - INFO - Pod test-pod-fe9965de conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:28:51,338 - tests.orchestrator - INFO - Pod test-pod-fe9965de conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:28:53,406 - tests.orchestrator - INFO - Pod test-pod-fe9965de conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:28:55,475 - tests.orchestrator - INFO - Pod test-pod-fe9965de conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:28:57,542 - tests.orchestrator - INFO - Pod test-pod-fe9965de conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:28:59,611 - tests.orchestrator - INFO - Pod test-pod-fe9965de conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:29:01,682 - tests.orchestrator - INFO - Pod test-pod-fe9965de conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:29:03,754 - tests.orchestrator - INFO - Pod test-pod-fe9965de conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:29:05,823 - tests.orchestrator - INFO - Pod test-pod-fe9965de conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:29:07,892 - tests.orchestrator - INFO - Pod test-pod-fe9965de conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-09 14:29:07,892 - tests.orchestrator - INFO - Pod test-pod-fe9965de is ready +2025-06-09 14:29:08,892 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 3 times) +2025-06-09 14:29:08,892 - tests.orchestrator - INFO - Creating PVC: test-pvc-19553927 +2025-06-09 14:29:08,970 - tests.orchestrator - INFO - Created PVC: test-pvc-19553927 +2025-06-09 14:29:08,970 - tests.orchestrator - INFO - Waiting for PVC test-pvc-19553927 to be bound +2025-06-09 14:29:11,105 - tests.orchestrator - INFO - PVC test-pvc-19553927 is bound +2025-06-09 14:29:12,105 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 4 times) +2025-06-09 14:29:12,105 - tests.orchestrator - INFO - Creating PVC: test-pvc-d5437f32 +2025-06-09 14:29:12,179 - tests.orchestrator - INFO - Created PVC: test-pvc-d5437f32 +2025-06-09 14:29:12,179 - tests.orchestrator - INFO - Waiting for PVC test-pvc-d5437f32 to be bound +2025-06-09 14:29:14,315 - tests.orchestrator - INFO - PVC test-pvc-d5437f32 is bound +2025-06-09 14:29:15,315 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario (selected 1 times) +2025-06-09 14:29:15,315 - tests.orchestrator - INFO - ============================================================ +2025-06-09 14:29:15,315 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_many_to_one +2025-06-09 14:29:15,316 - tests.orchestrator - INFO - ============================================================ +2025-06-09 14:29:15,316 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +2025-06-09 14:29:15,316 - tests.orchestrator - INFO - STARTING MANY-TO-ONE SCENARIO DIAGNOSTICS +2025-06-09 14:29:15,316 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +2025-06-09 14:29:15,316 - tests.orchestrator - INFO - [MANY2ONE] STEP 1: Creating dedicated PVC: many2one-654a0929 +2025-06-09 14:29:15,316 - tests.orchestrator - INFO - [MANY2ONE] Creating PVC manifest with storageClassName: efs-sc +2025-06-09 14:29:15,386 - tests.orchestrator - INFO - [MANY2ONE] PVC many2one-654a0929 created successfully +2025-06-09 14:29:15,386 - tests.orchestrator - INFO - [MANY2ONE] Waiting for PVC many2one-654a0929 to be bound... +2025-06-09 14:29:15,386 - tests.orchestrator - INFO - Waiting for PVC many2one-654a0929 to be bound +2025-06-09 14:29:17,517 - tests.orchestrator - INFO - PVC many2one-654a0929 is bound +2025-06-09 14:29:17,584 - tests.orchestrator - INFO - [MANY2ONE] PVC status: phase=Bound, capacity={'storage': '1Gi'} +2025-06-09 14:29:17,585 - tests.orchestrator - INFO - [MANY2ONE] STEP 2: Creating 4 pods for the same PVC many2one-654a0929 +2025-06-09 14:29:17,585 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 1/4 for PVC many2one-654a0929 +2025-06-09 14:29:17,585 - tests.orchestrator - INFO - Creating pod test-pod-5559abbe on PVC many2one-654a0929 +2025-06-09 14:29:17,660 - tests.orchestrator - INFO - Created pod: test-pod-5559abbe using PVC: many2one-654a0929 +2025-06-09 14:29:17,660 - tests.orchestrator - INFO - Waiting for pod test-pod-5559abbe to be ready +2025-06-09 14:29:17,728 - tests.orchestrator - INFO - Pod test-pod-5559abbe phase: Pending +2025-06-09 14:29:19,795 - tests.orchestrator - INFO - Pod test-pod-5559abbe phase: Running +2025-06-09 14:29:19,796 - tests.orchestrator - INFO - Pod test-pod-5559abbe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:29:21,866 - tests.orchestrator - INFO - Pod test-pod-5559abbe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:29:23,932 - tests.orchestrator - INFO - Pod test-pod-5559abbe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:29:26,001 - tests.orchestrator - INFO - Pod test-pod-5559abbe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:29:28,070 - tests.orchestrator - INFO - Pod test-pod-5559abbe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:29:30,137 - tests.orchestrator - INFO - Pod test-pod-5559abbe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:29:32,206 - tests.orchestrator - INFO - Pod test-pod-5559abbe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:29:34,275 - tests.orchestrator - INFO - Pod test-pod-5559abbe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:29:36,342 - tests.orchestrator - INFO - Pod test-pod-5559abbe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:29:38,411 - tests.orchestrator - INFO - Pod test-pod-5559abbe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-09 14:29:38,411 - tests.orchestrator - INFO - Pod test-pod-5559abbe is ready +2025-06-09 14:29:38,411 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-5559abbe +2025-06-09 14:29:38,411 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 2/4 for PVC many2one-654a0929 +2025-06-09 14:29:38,411 - tests.orchestrator - INFO - Creating pod test-pod-f9432ba6 on PVC many2one-654a0929 +2025-06-09 14:29:38,486 - tests.orchestrator - INFO - Created pod: test-pod-f9432ba6 using PVC: many2one-654a0929 +2025-06-09 14:29:38,486 - tests.orchestrator - INFO - Waiting for pod test-pod-f9432ba6 to be ready +2025-06-09 14:29:38,554 - tests.orchestrator - INFO - Pod test-pod-f9432ba6 phase: Pending +2025-06-09 14:29:40,622 - tests.orchestrator - INFO - Pod test-pod-f9432ba6 phase: Running +2025-06-09 14:29:40,623 - tests.orchestrator - INFO - Pod test-pod-f9432ba6 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:29:42,697 - tests.orchestrator - INFO - Pod test-pod-f9432ba6 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:29:44,767 - tests.orchestrator - INFO - Pod test-pod-f9432ba6 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:29:46,836 - tests.orchestrator - INFO - Pod test-pod-f9432ba6 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:29:48,903 - tests.orchestrator - INFO - Pod test-pod-f9432ba6 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:29:50,971 - tests.orchestrator - INFO - Pod test-pod-f9432ba6 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:29:53,041 - tests.orchestrator - INFO - Pod test-pod-f9432ba6 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:29:55,109 - tests.orchestrator - INFO - Pod test-pod-f9432ba6 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-09 14:29:55,109 - tests.orchestrator - INFO - Pod test-pod-f9432ba6 is ready +2025-06-09 14:29:55,110 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-f9432ba6 +2025-06-09 14:29:55,110 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 3/4 for PVC many2one-654a0929 +2025-06-09 14:29:55,110 - tests.orchestrator - INFO - Creating pod test-pod-8843b6a8 on PVC many2one-654a0929 +2025-06-09 14:29:55,197 - tests.orchestrator - INFO - Created pod: test-pod-8843b6a8 using PVC: many2one-654a0929 +2025-06-09 14:29:55,197 - tests.orchestrator - INFO - Waiting for pod test-pod-8843b6a8 to be ready +2025-06-09 14:29:55,263 - tests.orchestrator - INFO - Pod test-pod-8843b6a8 phase: Pending +2025-06-09 14:29:59,400 - tests.orchestrator - INFO - Pod test-pod-8843b6a8 phase: Running +2025-06-09 14:29:59,400 - tests.orchestrator - INFO - Pod test-pod-8843b6a8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:30:01,467 - tests.orchestrator - INFO - Pod test-pod-8843b6a8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:30:03,536 - tests.orchestrator - INFO - Pod test-pod-8843b6a8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:30:05,605 - tests.orchestrator - INFO - Pod test-pod-8843b6a8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:30:07,674 - tests.orchestrator - INFO - Pod test-pod-8843b6a8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:30:09,742 - tests.orchestrator - INFO - Pod test-pod-8843b6a8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:30:11,950 - tests.orchestrator - INFO - Pod test-pod-8843b6a8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:30:14,021 - tests.orchestrator - INFO - Pod test-pod-8843b6a8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:30:16,090 - tests.orchestrator - INFO - Pod test-pod-8843b6a8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-09 14:30:16,090 - tests.orchestrator - INFO - Pod test-pod-8843b6a8 is ready +2025-06-09 14:30:16,090 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-8843b6a8 +2025-06-09 14:30:16,090 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 4/4 for PVC many2one-654a0929 +2025-06-09 14:30:16,090 - tests.orchestrator - INFO - Creating pod test-pod-5c8cee78 on PVC many2one-654a0929 +2025-06-09 14:30:16,166 - tests.orchestrator - INFO - Created pod: test-pod-5c8cee78 using PVC: many2one-654a0929 +2025-06-09 14:30:16,167 - tests.orchestrator - INFO - Waiting for pod test-pod-5c8cee78 to be ready +2025-06-09 14:30:16,235 - tests.orchestrator - INFO - Pod test-pod-5c8cee78 phase: Pending +2025-06-09 14:30:20,371 - tests.orchestrator - INFO - Pod test-pod-5c8cee78 phase: Running +2025-06-09 14:30:20,371 - tests.orchestrator - INFO - Pod test-pod-5c8cee78 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:30:22,440 - tests.orchestrator - INFO - Pod test-pod-5c8cee78 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:30:24,509 - tests.orchestrator - INFO - Pod test-pod-5c8cee78 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:30:26,589 - tests.orchestrator - INFO - Pod test-pod-5c8cee78 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:30:28,659 - tests.orchestrator - INFO - Pod test-pod-5c8cee78 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:30:30,727 - tests.orchestrator - INFO - Pod test-pod-5c8cee78 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:30:32,794 - tests.orchestrator - INFO - Pod test-pod-5c8cee78 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-09 14:30:32,795 - tests.orchestrator - INFO - Pod test-pod-5c8cee78 is ready +2025-06-09 14:30:32,795 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-5c8cee78 +2025-06-09 14:30:32,795 - tests.orchestrator - INFO - [MANY2ONE] Created 4/4 pods successfully +2025-06-09 14:30:32,795 - tests.orchestrator - INFO - [MANY2ONE] STEP 3: Testing read/write operations +2025-06-09 14:30:32,795 - tests.orchestrator - INFO - [MANY2ONE] Writer pod: test-pod-f9432ba6, Reader pod: test-pod-5559abbe +2025-06-09 14:30:32,795 - tests.orchestrator - INFO - [MANY2ONE] Test file: /data/many2one-a1b870f0.txt +2025-06-09 14:30:32,795 - tests.orchestrator - INFO - [MANY2ONE] Test content: Many2One test content: 9db66010-5289-410b-815e-3c54cfd68970 +2025-06-09 14:30:32,795 - tests.orchestrator - INFO - [MANY2ONE] Executing write command: kubectl exec -n efs-stress-test test-pod-f9432ba6 -- /bin/sh -c 'echo "Many2One test content: 9db66010-5289-410b-815e-3c54cfd68970" > /data/many2one-a1b870f0.txt' +2025-06-09 14:30:35,420 - tests.orchestrator - INFO - [MANY2ONE] Writer pod ls check: '-rw-r--r-- 1 50027 50027 60 Jun 9 14:30 /data/many2one-a1b870f0.txt' +2025-06-09 14:30:35,420 - tests.orchestrator - INFO - [MANY2ONE] Sleeping for 5 seconds to ensure filesystem sync +2025-06-09 14:30:40,420 - tests.orchestrator - INFO - [MANY2ONE] Executing read command: kubectl exec -n efs-stress-test test-pod-5559abbe -- cat /data/many2one-a1b870f0.txt +2025-06-09 14:30:41,956 - tests.orchestrator - INFO - [MANY2ONE] Read command result: 'Many2One test content: 9db66010-5289-410b-815e-3c54cfd68970' +2025-06-09 14:30:41,956 - tests.orchestrator - INFO - [MANY2ONE] SUCCESS: Many-to-one scenario successful with 4 pods +2025-06-09 14:30:41,956 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +2025-06-09 14:30:41,956 - tests.orchestrator - INFO - COMPLETED MANY-TO-ONE SCENARIO DIAGNOSTICS +2025-06-09 14:30:41,956 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +2025-06-09 14:30:41,956 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_many_to_one +2025-06-09 14:30:41,956 - tests.orchestrator - INFO - ------------------------------------------------------------ +2025-06-09 14:30:42,957 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 1 times) +2025-06-09 14:30:42,957 - tests.orchestrator - INFO - Testing read/write between pods test-pod-8843b6a8 and test-pod-f9432ba6 sharing PVC many2one-654a0929 +2025-06-09 14:30:42,957 - tests.orchestrator - INFO - Executing write command: kubectl exec -n efs-stress-test test-pod-8843b6a8 -- /bin/sh -c 'echo "Test content: 9401f428-fb98-4a34-a95d-1a15171af885" > /data/test-a691b284.txt' +2025-06-09 14:30:44,295 - tests.orchestrator - INFO - Pod test-pod-8843b6a8 wrote to /data/test-a691b284.txt +2025-06-09 14:30:46,295 - tests.orchestrator - INFO - Executing read command: kubectl exec -n efs-stress-test test-pod-f9432ba6 -- cat /data/test-a691b284.txt +2025-06-09 14:30:47,601 - tests.orchestrator - INFO - Read result: 'Test content: 9401f428-fb98-4a34-a95d-1a15171af885' +2025-06-09 14:30:47,601 - tests.orchestrator - INFO - Pod test-pod-f9432ba6 successfully read content written by test-pod-8843b6a8 +2025-06-09 14:30:48,601 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 2 times) +2025-06-09 14:30:48,602 - tests.orchestrator - INFO - Testing read/write between pods test-pod-5c8cee78 and test-pod-5559abbe sharing PVC many2one-654a0929 +2025-06-09 14:30:48,602 - tests.orchestrator - INFO - Executing write command: kubectl exec -n efs-stress-test test-pod-5c8cee78 -- /bin/sh -c 'echo "Test content: c297a021-7006-47e4-9a4b-06e221180f62" > /data/test-787c9f72.txt' +2025-06-09 14:30:49,904 - tests.orchestrator - INFO - Pod test-pod-5c8cee78 wrote to /data/test-787c9f72.txt +2025-06-09 14:30:51,905 - tests.orchestrator - INFO - Executing read command: kubectl exec -n efs-stress-test test-pod-5559abbe -- cat /data/test-787c9f72.txt +2025-06-09 14:30:53,226 - tests.orchestrator - INFO - Read result: 'Test content: c297a021-7006-47e4-9a4b-06e221180f62' +2025-06-09 14:30:53,226 - tests.orchestrator - INFO - Pod test-pod-5559abbe successfully read content written by test-pod-5c8cee78 +2025-06-09 14:30:54,226 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario (selected 2 times) +2025-06-09 14:30:54,226 - tests.orchestrator - INFO - ============================================================ +2025-06-09 14:30:54,226 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_many_to_one +2025-06-09 14:30:54,226 - tests.orchestrator - INFO - ============================================================ +2025-06-09 14:30:54,226 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +2025-06-09 14:30:54,226 - tests.orchestrator - INFO - STARTING MANY-TO-ONE SCENARIO DIAGNOSTICS +2025-06-09 14:30:54,227 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +2025-06-09 14:30:54,227 - tests.orchestrator - INFO - [MANY2ONE] STEP 1: Creating dedicated PVC: many2one-f8c35bac +2025-06-09 14:30:54,227 - tests.orchestrator - INFO - [MANY2ONE] Creating PVC manifest with storageClassName: efs-sc +2025-06-09 14:30:54,305 - tests.orchestrator - INFO - [MANY2ONE] PVC many2one-f8c35bac created successfully +2025-06-09 14:30:54,305 - tests.orchestrator - INFO - [MANY2ONE] Waiting for PVC many2one-f8c35bac to be bound... +2025-06-09 14:30:54,305 - tests.orchestrator - INFO - Waiting for PVC many2one-f8c35bac to be bound +2025-06-09 14:30:56,439 - tests.orchestrator - INFO - PVC many2one-f8c35bac is bound +2025-06-09 14:30:56,506 - tests.orchestrator - INFO - [MANY2ONE] PVC status: phase=Bound, capacity={'storage': '1Gi'} +2025-06-09 14:30:56,506 - tests.orchestrator - INFO - [MANY2ONE] STEP 2: Creating 5 pods for the same PVC many2one-f8c35bac +2025-06-09 14:30:56,506 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 1/5 for PVC many2one-f8c35bac +2025-06-09 14:30:56,506 - tests.orchestrator - INFO - Creating pod test-pod-5e4161c8 on PVC many2one-f8c35bac +2025-06-09 14:30:56,582 - tests.orchestrator - INFO - Created pod: test-pod-5e4161c8 using PVC: many2one-f8c35bac +2025-06-09 14:30:56,582 - tests.orchestrator - INFO - Waiting for pod test-pod-5e4161c8 to be ready +2025-06-09 14:30:56,652 - tests.orchestrator - INFO - Pod test-pod-5e4161c8 phase: Pending +2025-06-09 14:30:58,721 - tests.orchestrator - INFO - Pod test-pod-5e4161c8 phase: Running +2025-06-09 14:30:58,721 - tests.orchestrator - INFO - Pod test-pod-5e4161c8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:31:00,788 - tests.orchestrator - INFO - Pod test-pod-5e4161c8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:31:02,857 - tests.orchestrator - INFO - Pod test-pod-5e4161c8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:31:04,927 - tests.orchestrator - INFO - Pod test-pod-5e4161c8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:31:06,993 - tests.orchestrator - INFO - Pod test-pod-5e4161c8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:31:09,062 - tests.orchestrator - INFO - Pod test-pod-5e4161c8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:31:11,131 - tests.orchestrator - INFO - Pod test-pod-5e4161c8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:31:13,205 - tests.orchestrator - INFO - Pod test-pod-5e4161c8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:31:15,274 - tests.orchestrator - INFO - Pod test-pod-5e4161c8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:31:17,343 - tests.orchestrator - INFO - Pod test-pod-5e4161c8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:31:19,410 - tests.orchestrator - INFO - Pod test-pod-5e4161c8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-09 14:31:19,410 - tests.orchestrator - INFO - Pod test-pod-5e4161c8 is ready +2025-06-09 14:31:19,410 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-5e4161c8 +2025-06-09 14:31:19,410 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 2/5 for PVC many2one-f8c35bac +2025-06-09 14:31:19,411 - tests.orchestrator - INFO - Creating pod test-pod-592a8021 on PVC many2one-f8c35bac +2025-06-09 14:31:19,487 - tests.orchestrator - INFO - Created pod: test-pod-592a8021 using PVC: many2one-f8c35bac +2025-06-09 14:31:19,487 - tests.orchestrator - INFO - Waiting for pod test-pod-592a8021 to be ready +2025-06-09 14:31:19,554 - tests.orchestrator - INFO - Pod test-pod-592a8021 phase: Pending +2025-06-09 14:31:21,624 - tests.orchestrator - INFO - Pod test-pod-592a8021 phase: Running +2025-06-09 14:31:21,624 - tests.orchestrator - INFO - Pod test-pod-592a8021 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:31:23,690 - tests.orchestrator - INFO - Pod test-pod-592a8021 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:31:25,759 - tests.orchestrator - INFO - Pod test-pod-592a8021 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:31:27,828 - tests.orchestrator - INFO - Pod test-pod-592a8021 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:31:29,895 - tests.orchestrator - INFO - Pod test-pod-592a8021 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:31:31,964 - tests.orchestrator - INFO - Pod test-pod-592a8021 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:31:34,032 - tests.orchestrator - INFO - Pod test-pod-592a8021 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:31:36,099 - tests.orchestrator - INFO - Pod test-pod-592a8021 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:31:38,168 - tests.orchestrator - INFO - Pod test-pod-592a8021 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:31:40,237 - tests.orchestrator - INFO - Pod test-pod-592a8021 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:31:42,309 - tests.orchestrator - INFO - Pod test-pod-592a8021 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-09 14:31:42,309 - tests.orchestrator - INFO - Pod test-pod-592a8021 is ready +2025-06-09 14:31:42,309 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-592a8021 +2025-06-09 14:31:42,309 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 3/5 for PVC many2one-f8c35bac +2025-06-09 14:31:42,309 - tests.orchestrator - INFO - Creating pod test-pod-953c034c on PVC many2one-f8c35bac +2025-06-09 14:31:42,407 - tests.orchestrator - INFO - Created pod: test-pod-953c034c using PVC: many2one-f8c35bac +2025-06-09 14:31:42,407 - tests.orchestrator - INFO - Waiting for pod test-pod-953c034c to be ready +2025-06-09 14:31:42,476 - tests.orchestrator - INFO - Pod test-pod-953c034c phase: Pending +2025-06-09 14:31:44,545 - tests.orchestrator - INFO - Pod test-pod-953c034c phase: Running +2025-06-09 14:31:44,545 - tests.orchestrator - INFO - Pod test-pod-953c034c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:31:46,614 - tests.orchestrator - INFO - Pod test-pod-953c034c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:31:48,681 - tests.orchestrator - INFO - Pod test-pod-953c034c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:31:50,750 - tests.orchestrator - INFO - Pod test-pod-953c034c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:31:52,819 - tests.orchestrator - INFO - Pod test-pod-953c034c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:31:54,886 - tests.orchestrator - INFO - Pod test-pod-953c034c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:31:56,955 - tests.orchestrator - INFO - Pod test-pod-953c034c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:31:59,024 - tests.orchestrator - INFO - Pod test-pod-953c034c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:32:01,091 - tests.orchestrator - INFO - Pod test-pod-953c034c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:32:03,160 - tests.orchestrator - INFO - Pod test-pod-953c034c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-09 14:32:03,160 - tests.orchestrator - INFO - Pod test-pod-953c034c is ready +2025-06-09 14:32:03,160 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-953c034c +2025-06-09 14:32:03,160 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 4/5 for PVC many2one-f8c35bac +2025-06-09 14:32:03,160 - tests.orchestrator - INFO - Creating pod test-pod-c20fb8e7 on PVC many2one-f8c35bac +2025-06-09 14:32:03,238 - tests.orchestrator - INFO - Created pod: test-pod-c20fb8e7 using PVC: many2one-f8c35bac +2025-06-09 14:32:03,238 - tests.orchestrator - INFO - Waiting for pod test-pod-c20fb8e7 to be ready +2025-06-09 14:32:03,306 - tests.orchestrator - INFO - Pod test-pod-c20fb8e7 phase: Pending +2025-06-09 14:32:05,376 - tests.orchestrator - INFO - Pod test-pod-c20fb8e7 phase: Running +2025-06-09 14:32:05,376 - tests.orchestrator - INFO - Pod test-pod-c20fb8e7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:32:07,443 - tests.orchestrator - INFO - Pod test-pod-c20fb8e7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:32:09,511 - tests.orchestrator - INFO - Pod test-pod-c20fb8e7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:32:11,639 - tests.orchestrator - INFO - Pod test-pod-c20fb8e7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:32:13,706 - tests.orchestrator - INFO - Pod test-pod-c20fb8e7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:32:15,775 - tests.orchestrator - INFO - Pod test-pod-c20fb8e7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:32:17,844 - tests.orchestrator - INFO - Pod test-pod-c20fb8e7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:32:19,911 - tests.orchestrator - INFO - Pod test-pod-c20fb8e7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:32:21,980 - tests.orchestrator - INFO - Pod test-pod-c20fb8e7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:32:24,050 - tests.orchestrator - INFO - Pod test-pod-c20fb8e7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-09 14:32:24,050 - tests.orchestrator - INFO - Pod test-pod-c20fb8e7 is ready +2025-06-09 14:32:24,050 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-c20fb8e7 +2025-06-09 14:32:24,050 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 5/5 for PVC many2one-f8c35bac +2025-06-09 14:32:24,051 - tests.orchestrator - INFO - Creating pod test-pod-1b84e5d8 on PVC many2one-f8c35bac +2025-06-09 14:32:24,133 - tests.orchestrator - INFO - Created pod: test-pod-1b84e5d8 using PVC: many2one-f8c35bac +2025-06-09 14:32:24,133 - tests.orchestrator - INFO - Waiting for pod test-pod-1b84e5d8 to be ready +2025-06-09 14:32:24,202 - tests.orchestrator - INFO - Pod test-pod-1b84e5d8 phase: Pending +2025-06-09 14:32:26,269 - tests.orchestrator - INFO - Pod test-pod-1b84e5d8 phase: Running +2025-06-09 14:32:26,269 - tests.orchestrator - INFO - Pod test-pod-1b84e5d8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:32:28,355 - tests.orchestrator - INFO - Pod test-pod-1b84e5d8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:32:30,424 - tests.orchestrator - INFO - Pod test-pod-1b84e5d8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:32:32,491 - tests.orchestrator - INFO - Pod test-pod-1b84e5d8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:32:34,559 - tests.orchestrator - INFO - Pod test-pod-1b84e5d8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:32:36,628 - tests.orchestrator - INFO - Pod test-pod-1b84e5d8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:32:38,695 - tests.orchestrator - INFO - Pod test-pod-1b84e5d8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:32:40,764 - tests.orchestrator - INFO - Pod test-pod-1b84e5d8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:32:42,836 - tests.orchestrator - INFO - Pod test-pod-1b84e5d8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:32:44,903 - tests.orchestrator - INFO - Pod test-pod-1b84e5d8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-09 14:32:44,903 - tests.orchestrator - INFO - Pod test-pod-1b84e5d8 is ready +2025-06-09 14:32:44,903 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-1b84e5d8 +2025-06-09 14:32:44,903 - tests.orchestrator - INFO - [MANY2ONE] Created 5/5 pods successfully +2025-06-09 14:32:44,903 - tests.orchestrator - INFO - [MANY2ONE] STEP 3: Testing read/write operations +2025-06-09 14:32:44,903 - tests.orchestrator - INFO - [MANY2ONE] Writer pod: test-pod-1b84e5d8, Reader pod: test-pod-5e4161c8 +2025-06-09 14:32:44,903 - tests.orchestrator - INFO - [MANY2ONE] Test file: /data/many2one-0e18d56e.txt +2025-06-09 14:32:44,903 - tests.orchestrator - INFO - [MANY2ONE] Test content: Many2One test content: 2a157b23-d25d-4ab9-84e0-29eb3fec41a6 +2025-06-09 14:32:44,903 - tests.orchestrator - INFO - [MANY2ONE] Executing write command: kubectl exec -n efs-stress-test test-pod-1b84e5d8 -- /bin/sh -c 'echo "Many2One test content: 2a157b23-d25d-4ab9-84e0-29eb3fec41a6" > /data/many2one-0e18d56e.txt' +2025-06-09 14:32:47,545 - tests.orchestrator - INFO - [MANY2ONE] Writer pod ls check: '-rw-r--r-- 1 50028 50028 60 Jun 9 14:32 /data/many2one-0e18d56e.txt' +2025-06-09 14:32:47,545 - tests.orchestrator - INFO - [MANY2ONE] Sleeping for 5 seconds to ensure filesystem sync +2025-06-09 14:32:52,545 - tests.orchestrator - INFO - [MANY2ONE] Executing read command: kubectl exec -n efs-stress-test test-pod-5e4161c8 -- cat /data/many2one-0e18d56e.txt +2025-06-09 14:32:53,861 - tests.orchestrator - INFO - [MANY2ONE] Read command result: 'Many2One test content: 2a157b23-d25d-4ab9-84e0-29eb3fec41a6' +2025-06-09 14:32:53,861 - tests.orchestrator - INFO - [MANY2ONE] SUCCESS: Many-to-one scenario successful with 5 pods +2025-06-09 14:32:53,861 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +2025-06-09 14:32:53,861 - tests.orchestrator - INFO - COMPLETED MANY-TO-ONE SCENARIO DIAGNOSTICS +2025-06-09 14:32:53,861 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +2025-06-09 14:32:53,862 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_many_to_one +2025-06-09 14:32:53,862 - tests.orchestrator - INFO - ------------------------------------------------------------ +2025-06-09 14:32:54,862 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 1 times) +2025-06-09 14:32:54,862 - tests.orchestrator - INFO - Deleting PVC: test-pvc-da1ffeef +2025-06-09 14:32:54,862 - tests.orchestrator - INFO - Deleting 1 pods using PVC test-pvc-da1ffeef +2025-06-09 14:32:54,862 - tests.orchestrator - INFO - Deleting pod: test-pod-6fd060e1 from PVC: test-pvc-da1ffeef +2025-06-09 14:32:54,938 - tests.orchestrator - INFO - Waiting for pod test-pod-6fd060e1 to be deleted +2025-06-09 14:33:27,550 - tests.orchestrator - INFO - Pod test-pod-6fd060e1 has been deleted +2025-06-09 14:33:27,550 - tests.orchestrator - INFO - Deleted pod: test-pod-6fd060e1 +2025-06-09 14:33:27,625 - tests.orchestrator - INFO - Waiting for PVC test-pvc-da1ffeef to be deleted +2025-06-09 14:33:27,689 - tests.orchestrator - INFO - PVC test-pvc-da1ffeef has been deleted +2025-06-09 14:33:27,690 - tests.orchestrator - INFO - Deleted PVC: test-pvc-da1ffeef +2025-06-09 14:33:28,690 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 3 times) +2025-06-09 14:33:28,690 - tests.orchestrator - INFO - Testing read/write between pods test-pod-8843b6a8 and test-pod-5c8cee78 sharing PVC many2one-654a0929 +2025-06-09 14:33:28,690 - tests.orchestrator - INFO - Executing write command: kubectl exec -n efs-stress-test test-pod-8843b6a8 -- /bin/sh -c 'echo "Test content: dda92edc-ba48-49c3-88a3-1d4d7f4c3d27" > /data/test-be592f5a.txt' +2025-06-09 14:33:30,002 - tests.orchestrator - INFO - Pod test-pod-8843b6a8 wrote to /data/test-be592f5a.txt +2025-06-09 14:33:32,002 - tests.orchestrator - INFO - Executing read command: kubectl exec -n efs-stress-test test-pod-5c8cee78 -- cat /data/test-be592f5a.txt +2025-06-09 14:33:33,296 - tests.orchestrator - INFO - Read result: 'Test content: dda92edc-ba48-49c3-88a3-1d4d7f4c3d27' +2025-06-09 14:33:33,296 - tests.orchestrator - INFO - Pod test-pod-5c8cee78 successfully read content written by test-pod-8843b6a8 +2025-06-09 14:33:34,296 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 4 times) +2025-06-09 14:33:34,296 - tests.orchestrator - INFO - Testing read/write between pods test-pod-592a8021 and test-pod-953c034c sharing PVC many2one-f8c35bac +2025-06-09 14:33:34,296 - tests.orchestrator - INFO - Executing write command: kubectl exec -n efs-stress-test test-pod-592a8021 -- /bin/sh -c 'echo "Test content: 384e4dbc-dfb3-4109-b698-b34081fcf2f1" > /data/test-bddd9de4.txt' +2025-06-09 14:33:35,613 - tests.orchestrator - INFO - Pod test-pod-592a8021 wrote to /data/test-bddd9de4.txt +2025-06-09 14:33:37,613 - tests.orchestrator - INFO - Executing read command: kubectl exec -n efs-stress-test test-pod-953c034c -- cat /data/test-bddd9de4.txt +2025-06-09 14:33:38,931 - tests.orchestrator - INFO - Read result: 'Test content: 384e4dbc-dfb3-4109-b698-b34081fcf2f1' +2025-06-09 14:33:38,931 - tests.orchestrator - INFO - Pod test-pod-953c034c successfully read content written by test-pod-592a8021 +2025-06-09 14:33:39,931 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 1 times) +2025-06-09 14:33:39,931 - tests.orchestrator - INFO - Deleting pod: test-pod-5c8cee78 from PVC: many2one-654a0929 +2025-06-09 14:33:40,009 - tests.orchestrator - INFO - Waiting for pod test-pod-5c8cee78 to be deleted +2025-06-09 14:34:12,623 - tests.orchestrator - INFO - Pod test-pod-5c8cee78 has been deleted +2025-06-09 14:34:12,623 - tests.orchestrator - INFO - Deleted pod: test-pod-5c8cee78 +2025-06-09 14:34:13,624 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario (selected 3 times) +2025-06-09 14:34:13,624 - tests.orchestrator - INFO - ============================================================ +2025-06-09 14:34:13,624 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_many_to_one +2025-06-09 14:34:13,624 - tests.orchestrator - INFO - ============================================================ +2025-06-09 14:34:13,624 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +2025-06-09 14:34:13,624 - tests.orchestrator - INFO - STARTING MANY-TO-ONE SCENARIO DIAGNOSTICS +2025-06-09 14:34:13,624 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +2025-06-09 14:34:13,624 - tests.orchestrator - INFO - [MANY2ONE] STEP 1: Creating dedicated PVC: many2one-3cfcc636 +2025-06-09 14:34:13,624 - tests.orchestrator - INFO - [MANY2ONE] Creating PVC manifest with storageClassName: efs-sc +2025-06-09 14:34:13,703 - tests.orchestrator - INFO - [MANY2ONE] PVC many2one-3cfcc636 created successfully +2025-06-09 14:34:13,703 - tests.orchestrator - INFO - [MANY2ONE] Waiting for PVC many2one-3cfcc636 to be bound... +2025-06-09 14:34:13,703 - tests.orchestrator - INFO - Waiting for PVC many2one-3cfcc636 to be bound +2025-06-09 14:34:15,838 - tests.orchestrator - INFO - PVC many2one-3cfcc636 is bound +2025-06-09 14:34:15,905 - tests.orchestrator - INFO - [MANY2ONE] PVC status: phase=Bound, capacity={'storage': '1Gi'} +2025-06-09 14:34:15,905 - tests.orchestrator - INFO - [MANY2ONE] STEP 2: Creating 5 pods for the same PVC many2one-3cfcc636 +2025-06-09 14:34:15,905 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 1/5 for PVC many2one-3cfcc636 +2025-06-09 14:34:15,905 - tests.orchestrator - INFO - Creating pod test-pod-e2f780f1 on PVC many2one-3cfcc636 +2025-06-09 14:34:15,983 - tests.orchestrator - INFO - Created pod: test-pod-e2f780f1 using PVC: many2one-3cfcc636 +2025-06-09 14:34:15,983 - tests.orchestrator - INFO - Waiting for pod test-pod-e2f780f1 to be ready +2025-06-09 14:34:16,051 - tests.orchestrator - INFO - Pod test-pod-e2f780f1 phase: Pending +2025-06-09 14:34:18,131 - tests.orchestrator - INFO - Pod test-pod-e2f780f1 phase: Running +2025-06-09 14:34:18,131 - tests.orchestrator - INFO - Pod test-pod-e2f780f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:34:20,200 - tests.orchestrator - INFO - Pod test-pod-e2f780f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:34:23,114 - tests.orchestrator - INFO - Pod test-pod-e2f780f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:34:25,181 - tests.orchestrator - INFO - Pod test-pod-e2f780f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:34:27,250 - tests.orchestrator - INFO - Pod test-pod-e2f780f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:34:29,319 - tests.orchestrator - INFO - Pod test-pod-e2f780f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:34:31,386 - tests.orchestrator - INFO - Pod test-pod-e2f780f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:34:33,455 - tests.orchestrator - INFO - Pod test-pod-e2f780f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:34:35,523 - tests.orchestrator - INFO - Pod test-pod-e2f780f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:34:37,590 - tests.orchestrator - INFO - Pod test-pod-e2f780f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-09 14:34:37,590 - tests.orchestrator - INFO - Pod test-pod-e2f780f1 is ready +2025-06-09 14:34:37,590 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-e2f780f1 +2025-06-09 14:34:37,590 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 2/5 for PVC many2one-3cfcc636 +2025-06-09 14:34:37,591 - tests.orchestrator - INFO - Creating pod test-pod-1245ee3c on PVC many2one-3cfcc636 +2025-06-09 14:34:37,667 - tests.orchestrator - INFO - Created pod: test-pod-1245ee3c using PVC: many2one-3cfcc636 +2025-06-09 14:34:37,667 - tests.orchestrator - INFO - Waiting for pod test-pod-1245ee3c to be ready +2025-06-09 14:34:37,733 - tests.orchestrator - INFO - Pod test-pod-1245ee3c phase: Pending +2025-06-09 14:34:41,868 - tests.orchestrator - INFO - Pod test-pod-1245ee3c phase: Running +2025-06-09 14:34:41,868 - tests.orchestrator - INFO - Pod test-pod-1245ee3c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:34:43,937 - tests.orchestrator - INFO - Pod test-pod-1245ee3c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:34:46,006 - tests.orchestrator - INFO - Pod test-pod-1245ee3c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:34:48,073 - tests.orchestrator - INFO - Pod test-pod-1245ee3c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:34:50,142 - tests.orchestrator - INFO - Pod test-pod-1245ee3c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:34:52,211 - tests.orchestrator - INFO - Pod test-pod-1245ee3c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:34:54,278 - tests.orchestrator - INFO - Pod test-pod-1245ee3c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:34:56,347 - tests.orchestrator - INFO - Pod test-pod-1245ee3c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:34:58,416 - tests.orchestrator - INFO - Pod test-pod-1245ee3c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-09 14:34:58,416 - tests.orchestrator - INFO - Pod test-pod-1245ee3c is ready +2025-06-09 14:34:58,416 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-1245ee3c +2025-06-09 14:34:58,416 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 3/5 for PVC many2one-3cfcc636 +2025-06-09 14:34:58,416 - tests.orchestrator - INFO - Creating pod test-pod-b6422666 on PVC many2one-3cfcc636 +2025-06-09 14:34:58,500 - tests.orchestrator - INFO - Created pod: test-pod-b6422666 using PVC: many2one-3cfcc636 +2025-06-09 14:34:58,500 - tests.orchestrator - INFO - Waiting for pod test-pod-b6422666 to be ready +2025-06-09 14:34:58,581 - tests.orchestrator - INFO - Pod test-pod-b6422666 phase: Pending +2025-06-09 14:35:00,651 - tests.orchestrator - INFO - Pod test-pod-b6422666 phase: Running +2025-06-09 14:35:00,651 - tests.orchestrator - INFO - Pod test-pod-b6422666 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:35:02,720 - tests.orchestrator - INFO - Pod test-pod-b6422666 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:35:04,790 - tests.orchestrator - INFO - Pod test-pod-b6422666 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:35:06,857 - tests.orchestrator - INFO - Pod test-pod-b6422666 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:35:08,926 - tests.orchestrator - INFO - Pod test-pod-b6422666 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:35:10,995 - tests.orchestrator - INFO - Pod test-pod-b6422666 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:35:13,064 - tests.orchestrator - INFO - Pod test-pod-b6422666 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:35:15,133 - tests.orchestrator - INFO - Pod test-pod-b6422666 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:35:17,202 - tests.orchestrator - INFO - Pod test-pod-b6422666 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:35:19,268 - tests.orchestrator - INFO - Pod test-pod-b6422666 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:35:21,337 - tests.orchestrator - INFO - Pod test-pod-b6422666 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-09 14:35:21,337 - tests.orchestrator - INFO - Pod test-pod-b6422666 is ready +2025-06-09 14:35:21,338 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-b6422666 +2025-06-09 14:35:21,338 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 4/5 for PVC many2one-3cfcc636 +2025-06-09 14:35:21,338 - tests.orchestrator - INFO - Creating pod test-pod-043741f4 on PVC many2one-3cfcc636 +2025-06-09 14:35:21,414 - tests.orchestrator - INFO - Created pod: test-pod-043741f4 using PVC: many2one-3cfcc636 +2025-06-09 14:35:21,414 - tests.orchestrator - INFO - Waiting for pod test-pod-043741f4 to be ready +2025-06-09 14:35:21,494 - tests.orchestrator - INFO - Pod test-pod-043741f4 phase: Pending +2025-06-09 14:35:23,563 - tests.orchestrator - INFO - Pod test-pod-043741f4 phase: Running +2025-06-09 14:35:23,563 - tests.orchestrator - INFO - Pod test-pod-043741f4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:35:25,631 - tests.orchestrator - INFO - Pod test-pod-043741f4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:35:27,700 - tests.orchestrator - INFO - Pod test-pod-043741f4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:35:29,768 - tests.orchestrator - INFO - Pod test-pod-043741f4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:35:31,835 - tests.orchestrator - INFO - Pod test-pod-043741f4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:35:33,904 - tests.orchestrator - INFO - Pod test-pod-043741f4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:35:35,972 - tests.orchestrator - INFO - Pod test-pod-043741f4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:35:38,040 - tests.orchestrator - INFO - Pod test-pod-043741f4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:35:40,109 - tests.orchestrator - INFO - Pod test-pod-043741f4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:35:42,185 - tests.orchestrator - INFO - Pod test-pod-043741f4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-09 14:35:42,185 - tests.orchestrator - INFO - Pod test-pod-043741f4 is ready +2025-06-09 14:35:42,186 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-043741f4 +2025-06-09 14:35:42,186 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 5/5 for PVC many2one-3cfcc636 +2025-06-09 14:35:42,186 - tests.orchestrator - INFO - Creating pod test-pod-9b313857 on PVC many2one-3cfcc636 +2025-06-09 14:35:42,282 - tests.orchestrator - INFO - Created pod: test-pod-9b313857 using PVC: many2one-3cfcc636 +2025-06-09 14:35:42,282 - tests.orchestrator - INFO - Waiting for pod test-pod-9b313857 to be ready +2025-06-09 14:35:42,354 - tests.orchestrator - INFO - Pod test-pod-9b313857 phase: Pending +2025-06-09 14:35:44,421 - tests.orchestrator - INFO - Pod test-pod-9b313857 phase: Running +2025-06-09 14:35:44,421 - tests.orchestrator - INFO - Pod test-pod-9b313857 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:35:46,492 - tests.orchestrator - INFO - Pod test-pod-9b313857 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:35:48,561 - tests.orchestrator - INFO - Pod test-pod-9b313857 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:35:50,627 - tests.orchestrator - INFO - Pod test-pod-9b313857 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:35:52,696 - tests.orchestrator - INFO - Pod test-pod-9b313857 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:35:54,764 - tests.orchestrator - INFO - Pod test-pod-9b313857 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:35:56,832 - tests.orchestrator - INFO - Pod test-pod-9b313857 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:35:58,901 - tests.orchestrator - INFO - Pod test-pod-9b313857 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:36:00,969 - tests.orchestrator - INFO - Pod test-pod-9b313857 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:36:03,036 - tests.orchestrator - INFO - Pod test-pod-9b313857 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-09 14:36:03,036 - tests.orchestrator - INFO - Pod test-pod-9b313857 is ready +2025-06-09 14:36:03,037 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-9b313857 +2025-06-09 14:36:03,037 - tests.orchestrator - INFO - [MANY2ONE] Created 5/5 pods successfully +2025-06-09 14:36:03,037 - tests.orchestrator - INFO - [MANY2ONE] STEP 3: Testing read/write operations +2025-06-09 14:36:03,037 - tests.orchestrator - INFO - [MANY2ONE] Writer pod: test-pod-b6422666, Reader pod: test-pod-043741f4 +2025-06-09 14:36:03,037 - tests.orchestrator - INFO - [MANY2ONE] Test file: /data/many2one-a602e48d.txt +2025-06-09 14:36:03,037 - tests.orchestrator - INFO - [MANY2ONE] Test content: Many2One test content: 1c0e68f4-7c6d-4048-abc6-aac816db2715 +2025-06-09 14:36:03,037 - tests.orchestrator - INFO - [MANY2ONE] Executing write command: kubectl exec -n efs-stress-test test-pod-b6422666 -- /bin/sh -c 'echo "Many2One test content: 1c0e68f4-7c6d-4048-abc6-aac816db2715" > /data/many2one-a602e48d.txt' +2025-06-09 14:36:05,660 - tests.orchestrator - INFO - [MANY2ONE] Writer pod ls check: '-rw-r--r-- 1 50020 50020 60 Jun 9 14:36 /data/many2one-a602e48d.txt' +2025-06-09 14:36:05,660 - tests.orchestrator - INFO - [MANY2ONE] Sleeping for 5 seconds to ensure filesystem sync +2025-06-09 14:36:10,660 - tests.orchestrator - INFO - [MANY2ONE] Executing read command: kubectl exec -n efs-stress-test test-pod-043741f4 -- cat /data/many2one-a602e48d.txt +2025-06-09 14:36:12,140 - tests.orchestrator - INFO - [MANY2ONE] Read command result: 'Many2One test content: 1c0e68f4-7c6d-4048-abc6-aac816db2715' +2025-06-09 14:36:12,140 - tests.orchestrator - INFO - [MANY2ONE] SUCCESS: Many-to-one scenario successful with 5 pods +2025-06-09 14:36:12,141 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +2025-06-09 14:36:12,141 - tests.orchestrator - INFO - COMPLETED MANY-TO-ONE SCENARIO DIAGNOSTICS +2025-06-09 14:36:12,141 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +2025-06-09 14:36:12,141 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_many_to_one +2025-06-09 14:36:12,141 - tests.orchestrator - INFO - ------------------------------------------------------------ +2025-06-09 14:36:13,141 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 5 times) +2025-06-09 14:36:13,141 - tests.orchestrator - INFO - Testing read/write between pods test-pod-c20fb8e7 and test-pod-953c034c sharing PVC many2one-f8c35bac +2025-06-09 14:36:13,141 - tests.orchestrator - INFO - Executing write command: kubectl exec -n efs-stress-test test-pod-c20fb8e7 -- /bin/sh -c 'echo "Test content: 9424e180-93c5-45ac-9c19-1f641c94e5dd" > /data/test-68c72f8a.txt' +2025-06-09 14:36:14,451 - tests.orchestrator - INFO - Pod test-pod-c20fb8e7 wrote to /data/test-68c72f8a.txt +2025-06-09 14:36:16,451 - tests.orchestrator - INFO - Executing read command: kubectl exec -n efs-stress-test test-pod-953c034c -- cat /data/test-68c72f8a.txt +2025-06-09 14:36:17,756 - tests.orchestrator - INFO - Read result: 'Test content: 9424e180-93c5-45ac-9c19-1f641c94e5dd' +2025-06-09 14:36:17,756 - tests.orchestrator - INFO - Pod test-pod-953c034c successfully read content written by test-pod-c20fb8e7 +2025-06-09 14:36:18,756 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 3 times) +2025-06-09 14:36:18,756 - tests.orchestrator - INFO - Creating pod test-pod-1cd15e4f on PVC test-pvc-ff5ff1dd +2025-06-09 14:36:18,840 - tests.orchestrator - INFO - Created pod: test-pod-1cd15e4f using PVC: test-pvc-ff5ff1dd +2025-06-09 14:36:18,840 - tests.orchestrator - INFO - Waiting for pod test-pod-1cd15e4f to be ready +2025-06-09 14:36:18,909 - tests.orchestrator - INFO - Pod test-pod-1cd15e4f phase: Pending +2025-06-09 14:36:20,975 - tests.orchestrator - INFO - Pod test-pod-1cd15e4f phase: Running +2025-06-09 14:36:20,976 - tests.orchestrator - INFO - Pod test-pod-1cd15e4f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:36:23,044 - tests.orchestrator - INFO - Pod test-pod-1cd15e4f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:36:25,114 - tests.orchestrator - INFO - Pod test-pod-1cd15e4f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:36:27,181 - tests.orchestrator - INFO - Pod test-pod-1cd15e4f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:36:29,249 - tests.orchestrator - INFO - Pod test-pod-1cd15e4f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:36:31,318 - tests.orchestrator - INFO - Pod test-pod-1cd15e4f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:36:33,385 - tests.orchestrator - INFO - Pod test-pod-1cd15e4f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:36:35,454 - tests.orchestrator - INFO - Pod test-pod-1cd15e4f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:36:37,523 - tests.orchestrator - INFO - Pod test-pod-1cd15e4f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:36:39,590 - tests.orchestrator - INFO - Pod test-pod-1cd15e4f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-09 14:36:39,590 - tests.orchestrator - INFO - Pod test-pod-1cd15e4f is ready +2025-06-09 14:36:40,590 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 4 times) +2025-06-09 14:36:40,590 - tests.orchestrator - INFO - Creating pod test-pod-8d9fce38 on PVC test-pvc-19553927 +2025-06-09 14:36:40,666 - tests.orchestrator - INFO - Created pod: test-pod-8d9fce38 using PVC: test-pvc-19553927 +2025-06-09 14:36:40,666 - tests.orchestrator - INFO - Waiting for pod test-pod-8d9fce38 to be ready +2025-06-09 14:36:40,732 - tests.orchestrator - INFO - Pod test-pod-8d9fce38 phase: Pending +2025-06-09 14:36:42,804 - tests.orchestrator - INFO - Pod test-pod-8d9fce38 phase: Running +2025-06-09 14:36:42,805 - tests.orchestrator - INFO - Pod test-pod-8d9fce38 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:36:44,873 - tests.orchestrator - INFO - Pod test-pod-8d9fce38 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:36:46,942 - tests.orchestrator - INFO - Pod test-pod-8d9fce38 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:36:49,011 - tests.orchestrator - INFO - Pod test-pod-8d9fce38 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:36:51,078 - tests.orchestrator - INFO - Pod test-pod-8d9fce38 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:36:53,147 - tests.orchestrator - INFO - Pod test-pod-8d9fce38 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:36:55,215 - tests.orchestrator - INFO - Pod test-pod-8d9fce38 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:36:57,282 - tests.orchestrator - INFO - Pod test-pod-8d9fce38 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:36:59,351 - tests.orchestrator - INFO - Pod test-pod-8d9fce38 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:37:01,419 - tests.orchestrator - INFO - Pod test-pod-8d9fce38 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-09 14:37:01,420 - tests.orchestrator - INFO - Pod test-pod-8d9fce38 is ready +2025-06-09 14:37:02,420 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 5 times) +2025-06-09 14:37:02,420 - tests.orchestrator - INFO - Creating PVC: test-pvc-7fb8de82 +2025-06-09 14:37:02,503 - tests.orchestrator - INFO - Created PVC: test-pvc-7fb8de82 +2025-06-09 14:37:02,503 - tests.orchestrator - INFO - Waiting for PVC test-pvc-7fb8de82 to be bound +2025-06-09 14:37:04,638 - tests.orchestrator - INFO - PVC test-pvc-7fb8de82 is bound +2025-06-09 14:37:05,638 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 2 times) +2025-06-09 14:37:05,638 - tests.orchestrator - INFO - Deleting PVC: many2one-f8c35bac +2025-06-09 14:37:05,638 - tests.orchestrator - INFO - Deleting 5 pods using PVC many2one-f8c35bac +2025-06-09 14:37:05,638 - tests.orchestrator - INFO - Deleting pod: test-pod-5e4161c8 from PVC: many2one-f8c35bac +2025-06-09 14:37:05,716 - tests.orchestrator - INFO - Waiting for pod test-pod-5e4161c8 to be deleted +2025-06-09 14:37:38,325 - tests.orchestrator - INFO - Pod test-pod-5e4161c8 has been deleted +2025-06-09 14:37:38,325 - tests.orchestrator - INFO - Deleted pod: test-pod-5e4161c8 +2025-06-09 14:37:38,325 - tests.orchestrator - INFO - Deleting pod: test-pod-592a8021 from PVC: many2one-f8c35bac +2025-06-09 14:37:38,399 - tests.orchestrator - INFO - Waiting for pod test-pod-592a8021 to be deleted +2025-06-09 14:38:11,021 - tests.orchestrator - INFO - Pod test-pod-592a8021 has been deleted +2025-06-09 14:38:11,021 - tests.orchestrator - INFO - Deleted pod: test-pod-592a8021 +2025-06-09 14:38:11,021 - tests.orchestrator - INFO - Deleting pod: test-pod-953c034c from PVC: many2one-f8c35bac +2025-06-09 14:38:11,099 - tests.orchestrator - INFO - Waiting for pod test-pod-953c034c to be deleted +2025-06-09 14:38:43,711 - tests.orchestrator - INFO - Pod test-pod-953c034c has been deleted +2025-06-09 14:38:43,711 - tests.orchestrator - INFO - Deleted pod: test-pod-953c034c +2025-06-09 14:38:43,711 - tests.orchestrator - INFO - Deleting pod: test-pod-c20fb8e7 from PVC: many2one-f8c35bac +2025-06-09 14:38:43,788 - tests.orchestrator - INFO - Waiting for pod test-pod-c20fb8e7 to be deleted +2025-06-09 14:39:16,404 - tests.orchestrator - INFO - Pod test-pod-c20fb8e7 has been deleted +2025-06-09 14:39:16,404 - tests.orchestrator - INFO - Deleted pod: test-pod-c20fb8e7 +2025-06-09 14:39:16,405 - tests.orchestrator - INFO - Deleting pod: test-pod-1b84e5d8 from PVC: many2one-f8c35bac +2025-06-09 14:39:16,485 - tests.orchestrator - INFO - Waiting for pod test-pod-1b84e5d8 to be deleted +2025-06-09 14:39:49,101 - tests.orchestrator - INFO - Pod test-pod-1b84e5d8 has been deleted +2025-06-09 14:39:49,101 - tests.orchestrator - INFO - Deleted pod: test-pod-1b84e5d8 +2025-06-09 14:39:49,177 - tests.orchestrator - INFO - Waiting for PVC many2one-f8c35bac to be deleted +2025-06-09 14:39:49,243 - tests.orchestrator - INFO - PVC many2one-f8c35bac has been deleted +2025-06-09 14:39:49,243 - tests.orchestrator - INFO - Deleted PVC: many2one-f8c35bac +2025-06-09 14:39:50,244 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 3 times) +2025-06-09 14:39:50,244 - tests.orchestrator - INFO - Deleting PVC: test-pvc-19553927 +2025-06-09 14:39:50,244 - tests.orchestrator - INFO - Deleting 1 pods using PVC test-pvc-19553927 +2025-06-09 14:39:50,244 - tests.orchestrator - INFO - Deleting pod: test-pod-8d9fce38 from PVC: test-pvc-19553927 +2025-06-09 14:39:50,319 - tests.orchestrator - INFO - Waiting for pod test-pod-8d9fce38 to be deleted +2025-06-09 14:40:22,937 - tests.orchestrator - INFO - Pod test-pod-8d9fce38 has been deleted +2025-06-09 14:40:22,937 - tests.orchestrator - INFO - Deleted pod: test-pod-8d9fce38 +2025-06-09 14:40:23,009 - tests.orchestrator - INFO - Waiting for PVC test-pvc-19553927 to be deleted +2025-06-09 14:40:23,075 - tests.orchestrator - INFO - PVC test-pvc-19553927 has been deleted +2025-06-09 14:40:23,075 - tests.orchestrator - INFO - Deleted PVC: test-pvc-19553927 +2025-06-09 14:40:24,075 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 6 times) +2025-06-09 14:40:24,076 - tests.orchestrator - INFO - Creating PVC: test-pvc-fc084aac +2025-06-09 14:40:24,147 - tests.orchestrator - INFO - Created PVC: test-pvc-fc084aac +2025-06-09 14:40:24,148 - tests.orchestrator - INFO - Waiting for PVC test-pvc-fc084aac to be bound +2025-06-09 14:40:26,281 - tests.orchestrator - INFO - PVC test-pvc-fc084aac is bound +2025-06-09 14:40:27,282 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 6 times) +2025-06-09 14:40:27,282 - tests.orchestrator - INFO - Testing read/write between pods test-pod-043741f4 and test-pod-9b313857 sharing PVC many2one-3cfcc636 +2025-06-09 14:40:27,282 - tests.orchestrator - INFO - Executing write command: kubectl exec -n efs-stress-test test-pod-043741f4 -- /bin/sh -c 'echo "Test content: 0283c180-fcf2-42d9-a801-d8be4bba70dc" > /data/test-140b67fc.txt' +2025-06-09 14:40:28,646 - tests.orchestrator - INFO - Pod test-pod-043741f4 wrote to /data/test-140b67fc.txt +2025-06-09 14:40:30,647 - tests.orchestrator - INFO - Executing read command: kubectl exec -n efs-stress-test test-pod-9b313857 -- cat /data/test-140b67fc.txt +2025-06-09 14:40:31,981 - tests.orchestrator - INFO - Read result: 'Test content: 0283c180-fcf2-42d9-a801-d8be4bba70dc' +2025-06-09 14:40:31,981 - tests.orchestrator - INFO - Pod test-pod-9b313857 successfully read content written by test-pod-043741f4 +2025-06-09 14:40:32,981 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 7 times) +2025-06-09 14:40:32,981 - tests.orchestrator - INFO - Creating PVC: test-pvc-2e916b1d +2025-06-09 14:40:33,050 - tests.orchestrator - INFO - Created PVC: test-pvc-2e916b1d +2025-06-09 14:40:33,050 - tests.orchestrator - INFO - Waiting for PVC test-pvc-2e916b1d to be bound +2025-06-09 14:40:35,185 - tests.orchestrator - INFO - PVC test-pvc-2e916b1d is bound +2025-06-09 14:40:36,185 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario (selected 4 times) +2025-06-09 14:40:36,185 - tests.orchestrator - INFO - ============================================================ +2025-06-09 14:40:36,185 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_many_to_one +2025-06-09 14:40:36,185 - tests.orchestrator - INFO - ============================================================ +2025-06-09 14:40:36,185 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +2025-06-09 14:40:36,185 - tests.orchestrator - INFO - STARTING MANY-TO-ONE SCENARIO DIAGNOSTICS +2025-06-09 14:40:36,185 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +2025-06-09 14:40:36,185 - tests.orchestrator - INFO - [MANY2ONE] STEP 1: Creating dedicated PVC: many2one-7fc11e0a +2025-06-09 14:40:36,185 - tests.orchestrator - INFO - [MANY2ONE] Creating PVC manifest with storageClassName: efs-sc +2025-06-09 14:40:36,254 - tests.orchestrator - INFO - [MANY2ONE] PVC many2one-7fc11e0a created successfully +2025-06-09 14:40:36,254 - tests.orchestrator - INFO - [MANY2ONE] Waiting for PVC many2one-7fc11e0a to be bound... +2025-06-09 14:40:36,254 - tests.orchestrator - INFO - Waiting for PVC many2one-7fc11e0a to be bound +2025-06-09 14:40:38,388 - tests.orchestrator - INFO - PVC many2one-7fc11e0a is bound +2025-06-09 14:40:38,455 - tests.orchestrator - INFO - [MANY2ONE] PVC status: phase=Bound, capacity={'storage': '1Gi'} +2025-06-09 14:40:38,456 - tests.orchestrator - INFO - [MANY2ONE] STEP 2: Creating 3 pods for the same PVC many2one-7fc11e0a +2025-06-09 14:40:38,456 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 1/3 for PVC many2one-7fc11e0a +2025-06-09 14:40:38,456 - tests.orchestrator - INFO - Creating pod test-pod-2c32e834 on PVC many2one-7fc11e0a +2025-06-09 14:40:38,536 - tests.orchestrator - INFO - Created pod: test-pod-2c32e834 using PVC: many2one-7fc11e0a +2025-06-09 14:40:38,536 - tests.orchestrator - INFO - Waiting for pod test-pod-2c32e834 to be ready +2025-06-09 14:40:38,605 - tests.orchestrator - INFO - Pod test-pod-2c32e834 phase: Pending +2025-06-09 14:40:42,749 - tests.orchestrator - INFO - Pod test-pod-2c32e834 phase: Running +2025-06-09 14:40:42,749 - tests.orchestrator - INFO - Pod test-pod-2c32e834 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:40:44,818 - tests.orchestrator - INFO - Pod test-pod-2c32e834 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:40:46,886 - tests.orchestrator - INFO - Pod test-pod-2c32e834 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:40:48,953 - tests.orchestrator - INFO - Pod test-pod-2c32e834 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:40:51,022 - tests.orchestrator - INFO - Pod test-pod-2c32e834 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:40:53,090 - tests.orchestrator - INFO - Pod test-pod-2c32e834 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:40:55,158 - tests.orchestrator - INFO - Pod test-pod-2c32e834 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-09 14:40:55,158 - tests.orchestrator - INFO - Pod test-pod-2c32e834 is ready +2025-06-09 14:40:55,158 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-2c32e834 +2025-06-09 14:40:55,158 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 2/3 for PVC many2one-7fc11e0a +2025-06-09 14:40:55,158 - tests.orchestrator - INFO - Creating pod test-pod-d04ff60a on PVC many2one-7fc11e0a +2025-06-09 14:40:55,244 - tests.orchestrator - INFO - Created pod: test-pod-d04ff60a using PVC: many2one-7fc11e0a +2025-06-09 14:40:55,244 - tests.orchestrator - INFO - Waiting for pod test-pod-d04ff60a to be ready +2025-06-09 14:40:55,310 - tests.orchestrator - INFO - Pod test-pod-d04ff60a phase: Pending +2025-06-09 14:40:57,378 - tests.orchestrator - INFO - Pod test-pod-d04ff60a phase: Running +2025-06-09 14:40:57,379 - tests.orchestrator - INFO - Pod test-pod-d04ff60a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:40:59,447 - tests.orchestrator - INFO - Pod test-pod-d04ff60a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:41:01,571 - tests.orchestrator - INFO - Pod test-pod-d04ff60a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:41:03,640 - tests.orchestrator - INFO - Pod test-pod-d04ff60a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:41:05,709 - tests.orchestrator - INFO - Pod test-pod-d04ff60a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:41:07,776 - tests.orchestrator - INFO - Pod test-pod-d04ff60a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:41:09,844 - tests.orchestrator - INFO - Pod test-pod-d04ff60a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:41:12,112 - tests.orchestrator - INFO - Pod test-pod-d04ff60a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:41:14,179 - tests.orchestrator - INFO - Pod test-pod-d04ff60a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:41:16,247 - tests.orchestrator - INFO - Pod test-pod-d04ff60a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-09 14:41:16,247 - tests.orchestrator - INFO - Pod test-pod-d04ff60a is ready +2025-06-09 14:41:16,248 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-d04ff60a +2025-06-09 14:41:16,248 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 3/3 for PVC many2one-7fc11e0a +2025-06-09 14:41:16,248 - tests.orchestrator - INFO - Creating pod test-pod-723a440a on PVC many2one-7fc11e0a +2025-06-09 14:41:16,323 - tests.orchestrator - INFO - Created pod: test-pod-723a440a using PVC: many2one-7fc11e0a +2025-06-09 14:41:16,323 - tests.orchestrator - INFO - Waiting for pod test-pod-723a440a to be ready +2025-06-09 14:41:16,392 - tests.orchestrator - INFO - Pod test-pod-723a440a phase: Pending +2025-06-09 14:41:18,459 - tests.orchestrator - INFO - Pod test-pod-723a440a phase: Running +2025-06-09 14:41:18,459 - tests.orchestrator - INFO - Pod test-pod-723a440a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:41:20,527 - tests.orchestrator - INFO - Pod test-pod-723a440a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:41:22,596 - tests.orchestrator - INFO - Pod test-pod-723a440a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:41:24,663 - tests.orchestrator - INFO - Pod test-pod-723a440a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:41:26,742 - tests.orchestrator - INFO - Pod test-pod-723a440a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:41:28,811 - tests.orchestrator - INFO - Pod test-pod-723a440a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:41:30,878 - tests.orchestrator - INFO - Pod test-pod-723a440a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:41:32,946 - tests.orchestrator - INFO - Pod test-pod-723a440a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:41:35,015 - tests.orchestrator - INFO - Pod test-pod-723a440a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:41:37,085 - tests.orchestrator - INFO - Pod test-pod-723a440a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-09 14:41:37,085 - tests.orchestrator - INFO - Pod test-pod-723a440a is ready +2025-06-09 14:41:37,085 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-723a440a +2025-06-09 14:41:37,085 - tests.orchestrator - INFO - [MANY2ONE] Created 3/3 pods successfully +2025-06-09 14:41:37,085 - tests.orchestrator - INFO - [MANY2ONE] STEP 3: Testing read/write operations +2025-06-09 14:41:37,085 - tests.orchestrator - INFO - [MANY2ONE] Writer pod: test-pod-2c32e834, Reader pod: test-pod-723a440a +2025-06-09 14:41:37,085 - tests.orchestrator - INFO - [MANY2ONE] Test file: /data/many2one-a38a9757.txt +2025-06-09 14:41:37,085 - tests.orchestrator - INFO - [MANY2ONE] Test content: Many2One test content: bab6d8de-2f87-4a4c-a89a-55c41d6666a9 +2025-06-09 14:41:37,085 - tests.orchestrator - INFO - [MANY2ONE] Executing write command: kubectl exec -n efs-stress-test test-pod-2c32e834 -- /bin/sh -c 'echo "Many2One test content: bab6d8de-2f87-4a4c-a89a-55c41d6666a9" > /data/many2one-a38a9757.txt' +2025-06-09 14:41:39,710 - tests.orchestrator - INFO - [MANY2ONE] Writer pod ls check: '-rw-r--r-- 1 50030 50030 60 Jun 9 14:41 /data/many2one-a38a9757.txt' +2025-06-09 14:41:39,710 - tests.orchestrator - INFO - [MANY2ONE] Sleeping for 5 seconds to ensure filesystem sync +2025-06-09 14:41:44,710 - tests.orchestrator - INFO - [MANY2ONE] Executing read command: kubectl exec -n efs-stress-test test-pod-723a440a -- cat /data/many2one-a38a9757.txt +2025-06-09 14:41:46,012 - tests.orchestrator - INFO - [MANY2ONE] Read command result: 'Many2One test content: bab6d8de-2f87-4a4c-a89a-55c41d6666a9' +2025-06-09 14:41:46,012 - tests.orchestrator - INFO - [MANY2ONE] SUCCESS: Many-to-one scenario successful with 3 pods +2025-06-09 14:41:46,012 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +2025-06-09 14:41:46,012 - tests.orchestrator - INFO - COMPLETED MANY-TO-ONE SCENARIO DIAGNOSTICS +2025-06-09 14:41:46,012 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +2025-06-09 14:41:46,012 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_many_to_one +2025-06-09 14:41:46,012 - tests.orchestrator - INFO - ------------------------------------------------------------ +2025-06-09 14:41:47,012 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario (selected 5 times) +2025-06-09 14:41:47,013 - tests.orchestrator - INFO - ============================================================ +2025-06-09 14:41:47,013 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_many_to_one +2025-06-09 14:41:47,013 - tests.orchestrator - INFO - ============================================================ +2025-06-09 14:41:47,013 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +2025-06-09 14:41:47,013 - tests.orchestrator - INFO - STARTING MANY-TO-ONE SCENARIO DIAGNOSTICS +2025-06-09 14:41:47,013 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +2025-06-09 14:41:47,013 - tests.orchestrator - INFO - [MANY2ONE] STEP 1: Creating dedicated PVC: many2one-cbec9511 +2025-06-09 14:41:47,013 - tests.orchestrator - INFO - [MANY2ONE] Creating PVC manifest with storageClassName: efs-sc +2025-06-09 14:41:47,092 - tests.orchestrator - INFO - [MANY2ONE] PVC many2one-cbec9511 created successfully +2025-06-09 14:41:47,092 - tests.orchestrator - INFO - [MANY2ONE] Waiting for PVC many2one-cbec9511 to be bound... +2025-06-09 14:41:47,092 - tests.orchestrator - INFO - Waiting for PVC many2one-cbec9511 to be bound +2025-06-09 14:41:49,223 - tests.orchestrator - INFO - PVC many2one-cbec9511 is bound +2025-06-09 14:41:49,290 - tests.orchestrator - INFO - [MANY2ONE] PVC status: phase=Bound, capacity={'storage': '1Gi'} +2025-06-09 14:41:49,290 - tests.orchestrator - INFO - [MANY2ONE] STEP 2: Creating 5 pods for the same PVC many2one-cbec9511 +2025-06-09 14:41:49,290 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 1/5 for PVC many2one-cbec9511 +2025-06-09 14:41:49,290 - tests.orchestrator - INFO - Creating pod test-pod-9ec3d1f1 on PVC many2one-cbec9511 +2025-06-09 14:41:49,367 - tests.orchestrator - INFO - Created pod: test-pod-9ec3d1f1 using PVC: many2one-cbec9511 +2025-06-09 14:41:49,367 - tests.orchestrator - INFO - Waiting for pod test-pod-9ec3d1f1 to be ready +2025-06-09 14:41:49,436 - tests.orchestrator - INFO - Pod test-pod-9ec3d1f1 phase: Pending +2025-06-09 14:41:51,507 - tests.orchestrator - INFO - Pod test-pod-9ec3d1f1 phase: Running +2025-06-09 14:41:51,508 - tests.orchestrator - INFO - Pod test-pod-9ec3d1f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:41:53,574 - tests.orchestrator - INFO - Pod test-pod-9ec3d1f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:41:55,643 - tests.orchestrator - INFO - Pod test-pod-9ec3d1f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:41:57,712 - tests.orchestrator - INFO - Pod test-pod-9ec3d1f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:41:59,778 - tests.orchestrator - INFO - Pod test-pod-9ec3d1f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:42:01,847 - tests.orchestrator - INFO - Pod test-pod-9ec3d1f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:42:03,916 - tests.orchestrator - INFO - Pod test-pod-9ec3d1f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:42:05,983 - tests.orchestrator - INFO - Pod test-pod-9ec3d1f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:42:08,051 - tests.orchestrator - INFO - Pod test-pod-9ec3d1f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:42:10,120 - tests.orchestrator - INFO - Pod test-pod-9ec3d1f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-09 14:42:10,120 - tests.orchestrator - INFO - Pod test-pod-9ec3d1f1 is ready +2025-06-09 14:42:10,120 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-9ec3d1f1 +2025-06-09 14:42:10,120 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 2/5 for PVC many2one-cbec9511 +2025-06-09 14:42:10,121 - tests.orchestrator - INFO - Creating pod test-pod-9389b3d8 on PVC many2one-cbec9511 +2025-06-09 14:42:10,194 - tests.orchestrator - INFO - Created pod: test-pod-9389b3d8 using PVC: many2one-cbec9511 +2025-06-09 14:42:10,194 - tests.orchestrator - INFO - Waiting for pod test-pod-9389b3d8 to be ready +2025-06-09 14:42:10,262 - tests.orchestrator - INFO - Pod test-pod-9389b3d8 phase: Pending +2025-06-09 14:42:12,333 - tests.orchestrator - INFO - Pod test-pod-9389b3d8 phase: Running +2025-06-09 14:42:12,334 - tests.orchestrator - INFO - Pod test-pod-9389b3d8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:42:14,402 - tests.orchestrator - INFO - Pod test-pod-9389b3d8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:42:16,471 - tests.orchestrator - INFO - Pod test-pod-9389b3d8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:42:18,538 - tests.orchestrator - INFO - Pod test-pod-9389b3d8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:42:20,606 - tests.orchestrator - INFO - Pod test-pod-9389b3d8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:42:22,675 - tests.orchestrator - INFO - Pod test-pod-9389b3d8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:42:24,742 - tests.orchestrator - INFO - Pod test-pod-9389b3d8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:42:26,821 - tests.orchestrator - INFO - Pod test-pod-9389b3d8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-09 14:42:26,821 - tests.orchestrator - INFO - Pod test-pod-9389b3d8 is ready +2025-06-09 14:42:26,821 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-9389b3d8 +2025-06-09 14:42:26,821 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 3/5 for PVC many2one-cbec9511 +2025-06-09 14:42:26,821 - tests.orchestrator - INFO - Creating pod test-pod-6b550453 on PVC many2one-cbec9511 +2025-06-09 14:42:26,906 - tests.orchestrator - INFO - Created pod: test-pod-6b550453 using PVC: many2one-cbec9511 +2025-06-09 14:42:26,906 - tests.orchestrator - INFO - Waiting for pod test-pod-6b550453 to be ready +2025-06-09 14:42:26,974 - tests.orchestrator - INFO - Pod test-pod-6b550453 phase: Pending +2025-06-09 14:42:31,111 - tests.orchestrator - INFO - Pod test-pod-6b550453 phase: Running +2025-06-09 14:42:31,111 - tests.orchestrator - INFO - Pod test-pod-6b550453 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:42:33,180 - tests.orchestrator - INFO - Pod test-pod-6b550453 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:42:35,246 - tests.orchestrator - INFO - Pod test-pod-6b550453 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:42:37,315 - tests.orchestrator - INFO - Pod test-pod-6b550453 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:42:39,384 - tests.orchestrator - INFO - Pod test-pod-6b550453 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:42:41,452 - tests.orchestrator - INFO - Pod test-pod-6b550453 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:42:43,520 - tests.orchestrator - INFO - Pod test-pod-6b550453 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:42:45,590 - tests.orchestrator - INFO - Pod test-pod-6b550453 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:42:47,658 - tests.orchestrator - INFO - Pod test-pod-6b550453 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-09 14:42:47,658 - tests.orchestrator - INFO - Pod test-pod-6b550453 is ready +2025-06-09 14:42:47,658 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-6b550453 +2025-06-09 14:42:47,658 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 4/5 for PVC many2one-cbec9511 +2025-06-09 14:42:47,658 - tests.orchestrator - INFO - Creating pod test-pod-9c4bbb5c on PVC many2one-cbec9511 +2025-06-09 14:42:47,735 - tests.orchestrator - INFO - Created pod: test-pod-9c4bbb5c using PVC: many2one-cbec9511 +2025-06-09 14:42:47,735 - tests.orchestrator - INFO - Waiting for pod test-pod-9c4bbb5c to be ready +2025-06-09 14:42:47,805 - tests.orchestrator - INFO - Pod test-pod-9c4bbb5c phase: Pending +2025-06-09 14:42:51,942 - tests.orchestrator - INFO - Pod test-pod-9c4bbb5c phase: Running +2025-06-09 14:42:51,943 - tests.orchestrator - INFO - Pod test-pod-9c4bbb5c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:42:54,010 - tests.orchestrator - INFO - Pod test-pod-9c4bbb5c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:42:56,078 - tests.orchestrator - INFO - Pod test-pod-9c4bbb5c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:42:58,147 - tests.orchestrator - INFO - Pod test-pod-9c4bbb5c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:43:00,214 - tests.orchestrator - INFO - Pod test-pod-9c4bbb5c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:43:02,283 - tests.orchestrator - INFO - Pod test-pod-9c4bbb5c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:43:04,352 - tests.orchestrator - INFO - Pod test-pod-9c4bbb5c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:43:06,419 - tests.orchestrator - INFO - Pod test-pod-9c4bbb5c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:43:08,488 - tests.orchestrator - INFO - Pod test-pod-9c4bbb5c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-09 14:43:08,488 - tests.orchestrator - INFO - Pod test-pod-9c4bbb5c is ready +2025-06-09 14:43:08,488 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-9c4bbb5c +2025-06-09 14:43:08,488 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 5/5 for PVC many2one-cbec9511 +2025-06-09 14:43:08,488 - tests.orchestrator - INFO - Creating pod test-pod-0eeb1063 on PVC many2one-cbec9511 +2025-06-09 14:43:08,574 - tests.orchestrator - INFO - Created pod: test-pod-0eeb1063 using PVC: many2one-cbec9511 +2025-06-09 14:43:08,574 - tests.orchestrator - INFO - Waiting for pod test-pod-0eeb1063 to be ready +2025-06-09 14:43:08,642 - tests.orchestrator - INFO - Pod test-pod-0eeb1063 phase: Pending +2025-06-09 14:43:12,781 - tests.orchestrator - INFO - Pod test-pod-0eeb1063 phase: Running +2025-06-09 14:43:12,781 - tests.orchestrator - INFO - Pod test-pod-0eeb1063 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:43:14,849 - tests.orchestrator - INFO - Pod test-pod-0eeb1063 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:43:16,918 - tests.orchestrator - INFO - Pod test-pod-0eeb1063 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:43:18,985 - tests.orchestrator - INFO - Pod test-pod-0eeb1063 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:43:21,054 - tests.orchestrator - INFO - Pod test-pod-0eeb1063 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:43:23,956 - tests.orchestrator - INFO - Pod test-pod-0eeb1063 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:43:26,023 - tests.orchestrator - INFO - Pod test-pod-0eeb1063 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-09 14:43:26,023 - tests.orchestrator - INFO - Pod test-pod-0eeb1063 is ready +2025-06-09 14:43:26,023 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-0eeb1063 +2025-06-09 14:43:26,023 - tests.orchestrator - INFO - [MANY2ONE] Created 5/5 pods successfully +2025-06-09 14:43:26,023 - tests.orchestrator - INFO - [MANY2ONE] STEP 3: Testing read/write operations +2025-06-09 14:43:26,023 - tests.orchestrator - INFO - [MANY2ONE] Writer pod: test-pod-6b550453, Reader pod: test-pod-9c4bbb5c +2025-06-09 14:43:26,023 - tests.orchestrator - INFO - [MANY2ONE] Test file: /data/many2one-79e58722.txt +2025-06-09 14:43:26,023 - tests.orchestrator - INFO - [MANY2ONE] Test content: Many2One test content: 4c5425ca-afa1-4ae9-99e0-0cae3ac50968 +2025-06-09 14:43:26,023 - tests.orchestrator - INFO - [MANY2ONE] Executing write command: kubectl exec -n efs-stress-test test-pod-6b550453 -- /bin/sh -c 'echo "Many2One test content: 4c5425ca-afa1-4ae9-99e0-0cae3ac50968" > /data/many2one-79e58722.txt' +2025-06-09 14:43:28,655 - tests.orchestrator - INFO - [MANY2ONE] Writer pod ls check: '-rw-r--r-- 1 50031 50031 60 Jun 9 14:43 /data/many2one-79e58722.txt' +2025-06-09 14:43:28,656 - tests.orchestrator - INFO - [MANY2ONE] Sleeping for 5 seconds to ensure filesystem sync +2025-06-09 14:43:33,656 - tests.orchestrator - INFO - [MANY2ONE] Executing read command: kubectl exec -n efs-stress-test test-pod-9c4bbb5c -- cat /data/many2one-79e58722.txt +2025-06-09 14:43:34,981 - tests.orchestrator - INFO - [MANY2ONE] Read command result: 'Many2One test content: 4c5425ca-afa1-4ae9-99e0-0cae3ac50968' +2025-06-09 14:43:34,981 - tests.orchestrator - INFO - [MANY2ONE] SUCCESS: Many-to-one scenario successful with 5 pods +2025-06-09 14:43:34,981 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +2025-06-09 14:43:34,981 - tests.orchestrator - INFO - COMPLETED MANY-TO-ONE SCENARIO DIAGNOSTICS +2025-06-09 14:43:34,981 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +2025-06-09 14:43:34,981 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_many_to_one +2025-06-09 14:43:34,981 - tests.orchestrator - INFO - ------------------------------------------------------------ +2025-06-09 14:43:35,981 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 2 times) +2025-06-09 14:43:35,982 - tests.orchestrator - INFO - Deleting pod: test-pod-723a440a from PVC: many2one-7fc11e0a +2025-06-09 14:43:36,058 - tests.orchestrator - INFO - Waiting for pod test-pod-723a440a to be deleted +2025-06-09 14:44:08,668 - tests.orchestrator - INFO - Pod test-pod-723a440a has been deleted +2025-06-09 14:44:08,668 - tests.orchestrator - INFO - Deleted pod: test-pod-723a440a +2025-06-09 14:44:09,669 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario (selected 6 times) +2025-06-09 14:44:09,669 - tests.orchestrator - INFO - ============================================================ +2025-06-09 14:44:09,669 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_many_to_one +2025-06-09 14:44:09,669 - tests.orchestrator - INFO - ============================================================ +2025-06-09 14:44:09,669 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +2025-06-09 14:44:09,669 - tests.orchestrator - INFO - STARTING MANY-TO-ONE SCENARIO DIAGNOSTICS +2025-06-09 14:44:09,669 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +2025-06-09 14:44:09,669 - tests.orchestrator - INFO - [MANY2ONE] STEP 1: Creating dedicated PVC: many2one-7c4a7e37 +2025-06-09 14:44:09,669 - tests.orchestrator - INFO - [MANY2ONE] Creating PVC manifest with storageClassName: efs-sc +2025-06-09 14:44:09,744 - tests.orchestrator - INFO - [MANY2ONE] PVC many2one-7c4a7e37 created successfully +2025-06-09 14:44:09,744 - tests.orchestrator - INFO - [MANY2ONE] Waiting for PVC many2one-7c4a7e37 to be bound... +2025-06-09 14:44:09,744 - tests.orchestrator - INFO - Waiting for PVC many2one-7c4a7e37 to be bound +2025-06-09 14:44:11,920 - tests.orchestrator - INFO - PVC many2one-7c4a7e37 is bound +2025-06-09 14:44:11,987 - tests.orchestrator - INFO - [MANY2ONE] PVC status: phase=Bound, capacity={'storage': '1Gi'} +2025-06-09 14:44:11,988 - tests.orchestrator - INFO - [MANY2ONE] STEP 2: Creating 3 pods for the same PVC many2one-7c4a7e37 +2025-06-09 14:44:11,988 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 1/3 for PVC many2one-7c4a7e37 +2025-06-09 14:44:11,988 - tests.orchestrator - INFO - Creating pod test-pod-c4f3501e on PVC many2one-7c4a7e37 +2025-06-09 14:44:12,073 - tests.orchestrator - INFO - Created pod: test-pod-c4f3501e using PVC: many2one-7c4a7e37 +2025-06-09 14:44:12,073 - tests.orchestrator - INFO - Waiting for pod test-pod-c4f3501e to be ready +2025-06-09 14:44:12,145 - tests.orchestrator - INFO - Pod test-pod-c4f3501e phase: Pending +2025-06-09 14:44:14,214 - tests.orchestrator - INFO - Pod test-pod-c4f3501e phase: Running +2025-06-09 14:44:14,214 - tests.orchestrator - INFO - Pod test-pod-c4f3501e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:44:16,286 - tests.orchestrator - INFO - Pod test-pod-c4f3501e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:44:18,357 - tests.orchestrator - INFO - Pod test-pod-c4f3501e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:44:20,430 - tests.orchestrator - INFO - Pod test-pod-c4f3501e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:44:22,500 - tests.orchestrator - INFO - Pod test-pod-c4f3501e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:44:24,567 - tests.orchestrator - INFO - Pod test-pod-c4f3501e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:44:26,638 - tests.orchestrator - INFO - Pod test-pod-c4f3501e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:44:28,707 - tests.orchestrator - INFO - Pod test-pod-c4f3501e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:44:30,773 - tests.orchestrator - INFO - Pod test-pod-c4f3501e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:44:32,842 - tests.orchestrator - INFO - Pod test-pod-c4f3501e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-09 14:44:32,842 - tests.orchestrator - INFO - Pod test-pod-c4f3501e is ready +2025-06-09 14:44:32,842 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-c4f3501e +2025-06-09 14:44:32,842 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 2/3 for PVC many2one-7c4a7e37 +2025-06-09 14:44:32,842 - tests.orchestrator - INFO - Creating pod test-pod-b5fa3100 on PVC many2one-7c4a7e37 +2025-06-09 14:44:32,919 - tests.orchestrator - INFO - Created pod: test-pod-b5fa3100 using PVC: many2one-7c4a7e37 +2025-06-09 14:44:32,919 - tests.orchestrator - INFO - Waiting for pod test-pod-b5fa3100 to be ready +2025-06-09 14:44:32,987 - tests.orchestrator - INFO - Pod test-pod-b5fa3100 phase: Pending +2025-06-09 14:44:37,122 - tests.orchestrator - INFO - Pod test-pod-b5fa3100 phase: Running +2025-06-09 14:44:37,122 - tests.orchestrator - INFO - Pod test-pod-b5fa3100 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:44:39,192 - tests.orchestrator - INFO - Pod test-pod-b5fa3100 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:44:41,262 - tests.orchestrator - INFO - Pod test-pod-b5fa3100 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:44:43,329 - tests.orchestrator - INFO - Pod test-pod-b5fa3100 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:44:45,397 - tests.orchestrator - INFO - Pod test-pod-b5fa3100 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:44:47,466 - tests.orchestrator - INFO - Pod test-pod-b5fa3100 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:44:49,533 - tests.orchestrator - INFO - Pod test-pod-b5fa3100 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:44:51,606 - tests.orchestrator - INFO - Pod test-pod-b5fa3100 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:44:53,675 - tests.orchestrator - INFO - Pod test-pod-b5fa3100 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:44:55,742 - tests.orchestrator - INFO - Pod test-pod-b5fa3100 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-09 14:44:55,742 - tests.orchestrator - INFO - Pod test-pod-b5fa3100 is ready +2025-06-09 14:44:55,742 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-b5fa3100 +2025-06-09 14:44:55,742 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 3/3 for PVC many2one-7c4a7e37 +2025-06-09 14:44:55,742 - tests.orchestrator - INFO - Creating pod test-pod-00d66dda on PVC many2one-7c4a7e37 +2025-06-09 14:44:55,827 - tests.orchestrator - INFO - Created pod: test-pod-00d66dda using PVC: many2one-7c4a7e37 +2025-06-09 14:44:55,827 - tests.orchestrator - INFO - Waiting for pod test-pod-00d66dda to be ready +2025-06-09 14:44:55,894 - tests.orchestrator - INFO - Pod test-pod-00d66dda phase: Pending +2025-06-09 14:44:57,962 - tests.orchestrator - INFO - Pod test-pod-00d66dda phase: Running +2025-06-09 14:44:57,962 - tests.orchestrator - INFO - Pod test-pod-00d66dda conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:45:00,029 - tests.orchestrator - INFO - Pod test-pod-00d66dda conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:45:02,098 - tests.orchestrator - INFO - Pod test-pod-00d66dda conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:45:04,167 - tests.orchestrator - INFO - Pod test-pod-00d66dda conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:45:06,234 - tests.orchestrator - INFO - Pod test-pod-00d66dda conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:45:08,302 - tests.orchestrator - INFO - Pod test-pod-00d66dda conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:45:10,371 - tests.orchestrator - INFO - Pod test-pod-00d66dda conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:45:12,438 - tests.orchestrator - INFO - Pod test-pod-00d66dda conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:45:14,507 - tests.orchestrator - INFO - Pod test-pod-00d66dda conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 14:45:16,578 - tests.orchestrator - INFO - Pod test-pod-00d66dda conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-09 14:45:16,578 - tests.orchestrator - INFO - Pod test-pod-00d66dda is ready +2025-06-09 14:45:16,578 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-00d66dda +2025-06-09 14:45:16,578 - tests.orchestrator - INFO - [MANY2ONE] Created 3/3 pods successfully +2025-06-09 14:45:16,578 - tests.orchestrator - INFO - [MANY2ONE] STEP 3: Testing read/write operations +2025-06-09 14:45:16,578 - tests.orchestrator - INFO - [MANY2ONE] Writer pod: test-pod-00d66dda, Reader pod: test-pod-c4f3501e +2025-06-09 14:45:16,578 - tests.orchestrator - INFO - [MANY2ONE] Test file: /data/many2one-7884b2d1.txt +2025-06-09 14:45:16,578 - tests.orchestrator - INFO - [MANY2ONE] Test content: Many2One test content: c20849b6-5111-4950-bbd6-2142ee259beb +2025-06-09 14:45:16,578 - tests.orchestrator - INFO - [MANY2ONE] Executing write command: kubectl exec -n efs-stress-test test-pod-00d66dda -- /bin/sh -c 'echo "Many2One test content: c20849b6-5111-4950-bbd6-2142ee259beb" > /data/many2one-7884b2d1.txt' +2025-06-09 14:45:19,195 - tests.orchestrator - INFO - [MANY2ONE] Writer pod ls check: '-rw-r--r-- 1 50032 50032 60 Jun 9 14:45 /data/many2one-7884b2d1.txt' +2025-06-09 14:45:19,195 - tests.orchestrator - INFO - [MANY2ONE] Sleeping for 5 seconds to ensure filesystem sync +2025-06-09 14:45:24,195 - tests.orchestrator - INFO - [MANY2ONE] Executing read command: kubectl exec -n efs-stress-test test-pod-c4f3501e -- cat /data/many2one-7884b2d1.txt +2025-06-09 14:45:25,483 - tests.orchestrator - INFO - [MANY2ONE] Read command result: 'Many2One test content: c20849b6-5111-4950-bbd6-2142ee259beb' +2025-06-09 14:45:25,484 - tests.orchestrator - INFO - [MANY2ONE] SUCCESS: Many-to-one scenario successful with 3 pods +2025-06-09 14:45:25,484 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +2025-06-09 14:45:25,484 - tests.orchestrator - INFO - COMPLETED MANY-TO-ONE SCENARIO DIAGNOSTICS +2025-06-09 14:45:25,484 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +2025-06-09 14:45:25,484 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_many_to_one +2025-06-09 14:45:25,484 - tests.orchestrator - INFO - ------------------------------------------------------------ +2025-06-09 14:45:26,484 - tests.orchestrator - INFO - Test completed in 1204.08 seconds +2025-06-09 14:45:26,484 - tests.orchestrator - INFO - ===== STARTING COMPREHENSIVE CLEANUP ===== +2025-06-09 14:45:26,484 - tests.orchestrator - INFO - Deleting 20 pods (force=False)... +2025-06-09 14:45:26,484 - tests.orchestrator - INFO - Deleting pod: test-pod-fe9965de from PVC: test-pvc-ff5ff1dd +2025-06-09 14:45:26,561 - tests.orchestrator - INFO - Waiting for pod test-pod-fe9965de to be deleted +2025-06-09 14:45:59,174 - tests.orchestrator - INFO - Pod test-pod-fe9965de has been deleted +2025-06-09 14:45:59,174 - tests.orchestrator - INFO - Deleted pod: test-pod-fe9965de +2025-06-09 14:45:59,174 - tests.orchestrator - INFO - Deleting pod: test-pod-1cd15e4f from PVC: test-pvc-ff5ff1dd +2025-06-09 14:45:59,249 - tests.orchestrator - INFO - Waiting for pod test-pod-1cd15e4f to be deleted +2025-06-09 14:46:31,992 - tests.orchestrator - INFO - Pod test-pod-1cd15e4f has been deleted +2025-06-09 14:46:31,993 - tests.orchestrator - INFO - Deleted pod: test-pod-1cd15e4f +2025-06-09 14:46:31,993 - tests.orchestrator - INFO - Deleting pod: test-pod-5559abbe from PVC: many2one-654a0929 +2025-06-09 14:46:32,069 - tests.orchestrator - INFO - Waiting for pod test-pod-5559abbe to be deleted +2025-06-09 14:47:04,680 - tests.orchestrator - INFO - Pod test-pod-5559abbe has been deleted +2025-06-09 14:47:04,680 - tests.orchestrator - INFO - Deleted pod: test-pod-5559abbe +2025-06-09 14:47:04,680 - tests.orchestrator - INFO - Deleting pod: test-pod-f9432ba6 from PVC: many2one-654a0929 +2025-06-09 14:47:04,757 - tests.orchestrator - INFO - Waiting for pod test-pod-f9432ba6 to be deleted +2025-06-09 14:47:37,373 - tests.orchestrator - INFO - Pod test-pod-f9432ba6 has been deleted +2025-06-09 14:47:37,373 - tests.orchestrator - INFO - Deleted pod: test-pod-f9432ba6 +2025-06-09 14:47:37,373 - tests.orchestrator - INFO - Deleting pod: test-pod-8843b6a8 from PVC: many2one-654a0929 +2025-06-09 14:47:37,449 - tests.orchestrator - INFO - Waiting for pod test-pod-8843b6a8 to be deleted +2025-06-09 14:48:10,139 - tests.orchestrator - INFO - Pod test-pod-8843b6a8 has been deleted +2025-06-09 14:48:10,139 - tests.orchestrator - INFO - Deleted pod: test-pod-8843b6a8 +2025-06-09 14:48:10,139 - tests.orchestrator - INFO - Deleting pod: test-pod-e2f780f1 from PVC: many2one-3cfcc636 +2025-06-09 14:48:10,214 - tests.orchestrator - INFO - Waiting for pod test-pod-e2f780f1 to be deleted +2025-06-09 14:48:42,843 - tests.orchestrator - INFO - Pod test-pod-e2f780f1 has been deleted +2025-06-09 14:48:42,843 - tests.orchestrator - INFO - Deleted pod: test-pod-e2f780f1 +2025-06-09 14:48:42,843 - tests.orchestrator - INFO - Deleting pod: test-pod-1245ee3c from PVC: many2one-3cfcc636 +2025-06-09 14:48:42,924 - tests.orchestrator - INFO - Waiting for pod test-pod-1245ee3c to be deleted +2025-06-09 14:49:15,536 - tests.orchestrator - INFO - Pod test-pod-1245ee3c has been deleted +2025-06-09 14:49:15,537 - tests.orchestrator - INFO - Deleted pod: test-pod-1245ee3c +2025-06-09 14:49:15,537 - tests.orchestrator - INFO - Deleting pod: test-pod-b6422666 from PVC: many2one-3cfcc636 +2025-06-09 14:49:15,615 - tests.orchestrator - INFO - Waiting for pod test-pod-b6422666 to be deleted +2025-06-09 14:49:48,223 - tests.orchestrator - INFO - Pod test-pod-b6422666 has been deleted +2025-06-09 14:49:48,223 - tests.orchestrator - INFO - Deleted pod: test-pod-b6422666 +2025-06-09 14:49:48,223 - tests.orchestrator - INFO - Deleting pod: test-pod-043741f4 from PVC: many2one-3cfcc636 +2025-06-09 14:49:48,299 - tests.orchestrator - INFO - Waiting for pod test-pod-043741f4 to be deleted +2025-06-09 14:50:20,916 - tests.orchestrator - INFO - Pod test-pod-043741f4 has been deleted +2025-06-09 14:50:20,916 - tests.orchestrator - INFO - Deleted pod: test-pod-043741f4 +2025-06-09 14:50:20,916 - tests.orchestrator - INFO - Deleting pod: test-pod-9b313857 from PVC: many2one-3cfcc636 +2025-06-09 14:50:20,992 - tests.orchestrator - INFO - Waiting for pod test-pod-9b313857 to be deleted +2025-06-09 14:50:53,621 - tests.orchestrator - INFO - Pod test-pod-9b313857 has been deleted +2025-06-09 14:50:53,621 - tests.orchestrator - INFO - Deleted pod: test-pod-9b313857 +2025-06-09 14:50:53,621 - tests.orchestrator - INFO - Deleting pod: test-pod-2c32e834 from PVC: many2one-7fc11e0a +2025-06-09 14:50:53,696 - tests.orchestrator - INFO - Waiting for pod test-pod-2c32e834 to be deleted +2025-06-09 14:51:26,311 - tests.orchestrator - INFO - Pod test-pod-2c32e834 has been deleted +2025-06-09 14:51:26,311 - tests.orchestrator - INFO - Deleted pod: test-pod-2c32e834 +2025-06-09 14:51:26,311 - tests.orchestrator - INFO - Deleting pod: test-pod-d04ff60a from PVC: many2one-7fc11e0a +2025-06-09 14:51:26,388 - tests.orchestrator - INFO - Waiting for pod test-pod-d04ff60a to be deleted +2025-06-09 14:51:59,004 - tests.orchestrator - INFO - Pod test-pod-d04ff60a has been deleted +2025-06-09 14:51:59,004 - tests.orchestrator - INFO - Deleted pod: test-pod-d04ff60a +2025-06-09 14:51:59,004 - tests.orchestrator - INFO - Deleting pod: test-pod-9ec3d1f1 from PVC: many2one-cbec9511 +2025-06-09 14:51:59,082 - tests.orchestrator - INFO - Waiting for pod test-pod-9ec3d1f1 to be deleted +2025-06-09 14:52:32,541 - tests.orchestrator - INFO - Pod test-pod-9ec3d1f1 has been deleted +2025-06-09 14:52:32,541 - tests.orchestrator - INFO - Deleted pod: test-pod-9ec3d1f1 +2025-06-09 14:52:32,541 - tests.orchestrator - INFO - Deleting pod: test-pod-9389b3d8 from PVC: many2one-cbec9511 +2025-06-09 14:52:32,616 - tests.orchestrator - INFO - Waiting for pod test-pod-9389b3d8 to be deleted +2025-06-09 14:53:05,227 - tests.orchestrator - INFO - Pod test-pod-9389b3d8 has been deleted +2025-06-09 14:53:05,228 - tests.orchestrator - INFO - Deleted pod: test-pod-9389b3d8 +2025-06-09 14:53:05,228 - tests.orchestrator - INFO - Deleting pod: test-pod-6b550453 from PVC: many2one-cbec9511 +2025-06-09 14:53:05,303 - tests.orchestrator - INFO - Waiting for pod test-pod-6b550453 to be deleted +2025-06-09 14:53:37,916 - tests.orchestrator - INFO - Pod test-pod-6b550453 has been deleted +2025-06-09 14:53:37,916 - tests.orchestrator - INFO - Deleted pod: test-pod-6b550453 +2025-06-09 14:53:37,916 - tests.orchestrator - INFO - Deleting pod: test-pod-9c4bbb5c from PVC: many2one-cbec9511 +2025-06-09 14:53:37,993 - tests.orchestrator - INFO - Waiting for pod test-pod-9c4bbb5c to be deleted +2025-06-09 14:54:10,610 - tests.orchestrator - INFO - Pod test-pod-9c4bbb5c has been deleted +2025-06-09 14:54:10,610 - tests.orchestrator - INFO - Deleted pod: test-pod-9c4bbb5c +2025-06-09 14:54:10,610 - tests.orchestrator - INFO - Deleting pod: test-pod-0eeb1063 from PVC: many2one-cbec9511 +2025-06-09 14:54:10,691 - tests.orchestrator - INFO - Waiting for pod test-pod-0eeb1063 to be deleted +2025-06-09 14:54:43,304 - tests.orchestrator - INFO - Pod test-pod-0eeb1063 has been deleted +2025-06-09 14:54:43,305 - tests.orchestrator - INFO - Deleted pod: test-pod-0eeb1063 +2025-06-09 14:54:43,305 - tests.orchestrator - INFO - Deleting pod: test-pod-c4f3501e from PVC: many2one-7c4a7e37 +2025-06-09 14:54:43,380 - tests.orchestrator - INFO - Waiting for pod test-pod-c4f3501e to be deleted +2025-06-09 14:55:16,030 - tests.orchestrator - INFO - Pod test-pod-c4f3501e has been deleted +2025-06-09 14:55:16,030 - tests.orchestrator - INFO - Deleted pod: test-pod-c4f3501e +2025-06-09 14:55:16,030 - tests.orchestrator - INFO - Deleting pod: test-pod-b5fa3100 from PVC: many2one-7c4a7e37 +2025-06-09 14:55:16,106 - tests.orchestrator - INFO - Waiting for pod test-pod-b5fa3100 to be deleted +2025-06-09 14:55:48,720 - tests.orchestrator - INFO - Pod test-pod-b5fa3100 has been deleted +2025-06-09 14:55:48,721 - tests.orchestrator - INFO - Deleted pod: test-pod-b5fa3100 +2025-06-09 14:55:48,721 - tests.orchestrator - INFO - Deleting pod: test-pod-00d66dda from PVC: many2one-7c4a7e37 +2025-06-09 14:55:48,795 - tests.orchestrator - INFO - Waiting for pod test-pod-00d66dda to be deleted +2025-06-09 14:56:21,410 - tests.orchestrator - INFO - Pod test-pod-00d66dda has been deleted +2025-06-09 14:56:21,411 - tests.orchestrator - INFO - Deleted pod: test-pod-00d66dda +2025-06-09 14:56:26,411 - tests.orchestrator - INFO - Deleting 11 PVCs (force=False)... +2025-06-09 14:56:26,411 - tests.orchestrator - INFO - Deleting PVC: test-pvc-867df8a7 +2025-06-09 14:56:26,506 - tests.orchestrator - INFO - Waiting for PVC test-pvc-867df8a7 to be deleted +2025-06-09 14:56:26,580 - tests.orchestrator - INFO - PVC test-pvc-867df8a7 has been deleted +2025-06-09 14:56:26,580 - tests.orchestrator - INFO - Deleted PVC: test-pvc-867df8a7 +2025-06-09 14:56:26,580 - tests.orchestrator - INFO - Deleting PVC: test-pvc-ff5ff1dd +2025-06-09 14:56:26,672 - tests.orchestrator - INFO - Waiting for PVC test-pvc-ff5ff1dd to be deleted +2025-06-09 14:56:26,740 - tests.orchestrator - INFO - PVC test-pvc-ff5ff1dd has been deleted +2025-06-09 14:56:26,740 - tests.orchestrator - INFO - Deleted PVC: test-pvc-ff5ff1dd +2025-06-09 14:56:26,740 - tests.orchestrator - INFO - Deleting PVC: test-pvc-d5437f32 +2025-06-09 14:56:26,816 - tests.orchestrator - INFO - Waiting for PVC test-pvc-d5437f32 to be deleted +2025-06-09 14:56:26,886 - tests.orchestrator - INFO - PVC test-pvc-d5437f32 has been deleted +2025-06-09 14:56:26,887 - tests.orchestrator - INFO - Deleted PVC: test-pvc-d5437f32 +2025-06-09 14:56:26,887 - tests.orchestrator - INFO - Deleting PVC: many2one-654a0929 +2025-06-09 14:56:26,958 - tests.orchestrator - INFO - Waiting for PVC many2one-654a0929 to be deleted +2025-06-09 14:56:27,023 - tests.orchestrator - INFO - PVC many2one-654a0929 has been deleted +2025-06-09 14:56:27,023 - tests.orchestrator - INFO - Deleted PVC: many2one-654a0929 +2025-06-09 14:56:27,023 - tests.orchestrator - INFO - Deleting PVC: many2one-3cfcc636 +2025-06-09 14:56:27,098 - tests.orchestrator - INFO - Waiting for PVC many2one-3cfcc636 to be deleted +2025-06-09 14:56:27,164 - tests.orchestrator - INFO - PVC many2one-3cfcc636 has been deleted +2025-06-09 14:56:27,164 - tests.orchestrator - INFO - Deleted PVC: many2one-3cfcc636 +2025-06-09 14:56:27,164 - tests.orchestrator - INFO - Deleting PVC: test-pvc-7fb8de82 +2025-06-09 14:56:27,234 - tests.orchestrator - INFO - Waiting for PVC test-pvc-7fb8de82 to be deleted +2025-06-09 14:56:27,301 - tests.orchestrator - INFO - PVC test-pvc-7fb8de82 has been deleted +2025-06-09 14:56:27,301 - tests.orchestrator - INFO - Deleted PVC: test-pvc-7fb8de82 +2025-06-09 14:56:27,302 - tests.orchestrator - INFO - Deleting PVC: test-pvc-fc084aac +2025-06-09 14:56:27,374 - tests.orchestrator - INFO - Waiting for PVC test-pvc-fc084aac to be deleted +2025-06-09 14:56:27,440 - tests.orchestrator - INFO - PVC test-pvc-fc084aac has been deleted +2025-06-09 14:56:27,441 - tests.orchestrator - INFO - Deleted PVC: test-pvc-fc084aac +2025-06-09 14:56:27,441 - tests.orchestrator - INFO - Deleting PVC: test-pvc-2e916b1d +2025-06-09 14:56:27,511 - tests.orchestrator - INFO - Waiting for PVC test-pvc-2e916b1d to be deleted +2025-06-09 14:56:27,575 - tests.orchestrator - INFO - PVC test-pvc-2e916b1d has been deleted +2025-06-09 14:56:27,576 - tests.orchestrator - INFO - Deleted PVC: test-pvc-2e916b1d +2025-06-09 14:56:27,576 - tests.orchestrator - INFO - Deleting PVC: many2one-7fc11e0a +2025-06-09 14:56:27,649 - tests.orchestrator - INFO - Waiting for PVC many2one-7fc11e0a to be deleted +2025-06-09 14:56:27,717 - tests.orchestrator - INFO - PVC many2one-7fc11e0a has been deleted +2025-06-09 14:56:27,717 - tests.orchestrator - INFO - Deleted PVC: many2one-7fc11e0a +2025-06-09 14:56:27,717 - tests.orchestrator - INFO - Deleting PVC: many2one-cbec9511 +2025-06-09 14:56:27,790 - tests.orchestrator - INFO - Waiting for PVC many2one-cbec9511 to be deleted +2025-06-09 14:56:27,860 - tests.orchestrator - INFO - PVC many2one-cbec9511 has been deleted +2025-06-09 14:56:27,860 - tests.orchestrator - INFO - Deleted PVC: many2one-cbec9511 +2025-06-09 14:56:27,860 - tests.orchestrator - INFO - Deleting PVC: many2one-7c4a7e37 +2025-06-09 14:56:27,931 - tests.orchestrator - INFO - Waiting for PVC many2one-7c4a7e37 to be deleted +2025-06-09 14:56:27,996 - tests.orchestrator - INFO - PVC many2one-7c4a7e37 has been deleted +2025-06-09 14:56:27,996 - tests.orchestrator - INFO - Deleted PVC: many2one-7c4a7e37 +2025-06-09 14:56:28,201 - tests.orchestrator - WARNING - First cleanup pass incomplete. Remaining resources: ['pod/test-pod-0d0fddfe', 'pod/test-pod-158cdfa4', 'pod/test-pod-4044e577', 'pod/test-pod-87095965', 'pvc/test-pvc-02e00a44', 'pvc/test-pvc-0bb0d20c', 'pvc/test-pvc-2127e71a', 'pvc/test-pvc-2cf5b7a2', 'pvc/test-pvc-31fbd501', 'pvc/test-pvc-4ada8088', 'pvc/test-pvc-4b26f6f6', 'pvc/test-pvc-528b4f5c', 'pvc/test-pvc-5c13c143', 'pvc/test-pvc-c542ee58', 'pvc/test-pvc-dfd12d43', 'pvc/test-pvc-ecb12f74'] +2025-06-09 14:56:28,201 - tests.orchestrator - INFO - Attempting force deletion of remaining resources... +2025-06-09 14:56:28,202 - tests.orchestrator - INFO - Deleting 0 pods (force=True)... +2025-06-09 14:56:33,202 - tests.orchestrator - INFO - Deleting 0 PVCs (force=True)... +2025-06-09 14:56:33,407 - tests.orchestrator - ERROR - Cleanup incomplete. Remaining resources after force deletion: ['pod/test-pod-0d0fddfe', 'pod/test-pod-158cdfa4', 'pod/test-pod-4044e577', 'pod/test-pod-87095965', 'pvc/test-pvc-02e00a44', 'pvc/test-pvc-0bb0d20c', 'pvc/test-pvc-2127e71a', 'pvc/test-pvc-2cf5b7a2', 'pvc/test-pvc-31fbd501', 'pvc/test-pvc-4ada8088', 'pvc/test-pvc-4b26f6f6', 'pvc/test-pvc-528b4f5c', 'pvc/test-pvc-5c13c143', 'pvc/test-pvc-c542ee58', 'pvc/test-pvc-dfd12d43', 'pvc/test-pvc-ecb12f74'] +2025-06-09 14:56:33,408 - tests.orchestrator - INFO - Cleanup completed successfully in 666.92 seconds +2025-06-09 14:56:33,408 - tests.orchestrator - INFO - ===== CLEANUP PROCESS FINISHED ===== +2025-06-09 14:56:33,408 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-09 14:56:33,408 - tests.orchestrator - INFO - --- Operations --- +2025-06-09 14:56:33,408 - tests.orchestrator - INFO - create_pvc: 9 succeeded, 0 failed (100.0%) +2025-06-09 14:56:33,408 - tests.orchestrator - INFO - attach_pod: 32 succeeded, 0 failed (100.0%) +2025-06-09 14:56:33,408 - tests.orchestrator - INFO - delete_pod: 32 succeeded, 0 failed (100.0%) +2025-06-09 14:56:33,408 - tests.orchestrator - INFO - delete_pvc: 15 succeeded, 0 failed (100.0%) +2025-06-09 14:56:33,408 - tests.orchestrator - INFO - verify_read_write: Writes 7 succeeded, 0 failed (100.0%) +2025-06-09 14:56:33,408 - tests.orchestrator - INFO - verify_read_write: Reads 7 succeeded, 0 failed (100.0%) +2025-06-09 14:56:33,408 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-09 14:56:33,408 - tests.orchestrator - INFO - shared_volume_rw: 7 succeeded, 0 failed out of 7 runs (100.0%) +2025-06-09 14:56:33,408 - tests.orchestrator - INFO - many_to_one: 6 succeeded, 0 failed out of 6 runs (100.0%) +2025-06-09 14:56:33,408 - tests.orchestrator - INFO - one_to_one: No runs +2025-06-09 14:56:33,408 - tests.orchestrator - INFO - concurrent_pvc: No runs +2025-06-09 14:56:33,408 - tests.orchestrator - INFO - ========================================= diff --git a/test/stress-scale-tests/logs/orchestrator_20250609_182011.log b/test/stress-scale-tests/logs/orchestrator_20250609_182011.log new file mode 100644 index 000000000..e370848f9 --- /dev/null +++ b/test/stress-scale-tests/logs/orchestrator_20250609_182011.log @@ -0,0 +1,7 @@ +2025-06-09 18:20:11,585 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-09 18:20:11,779 - tests.orchestrator - ERROR - Error checking namespace: (401) +Reason: Unauthorized +HTTP response headers: HTTPHeaderDict({'Audit-Id': 'd5d12f12-977a-4faf-bd61-ad049e1a9ec2', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Mon, 09 Jun 2025 18:20:11 GMT', 'Content-Length': '129'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} + + diff --git a/test/stress-scale-tests/logs/orchestrator_20250609_183658.log b/test/stress-scale-tests/logs/orchestrator_20250609_183658.log new file mode 100644 index 000000000..8e649feca --- /dev/null +++ b/test/stress-scale-tests/logs/orchestrator_20250609_183658.log @@ -0,0 +1,7 @@ +2025-06-09 18:36:58,870 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-09 18:36:59,092 - tests.orchestrator - ERROR - Error checking namespace: (401) +Reason: Unauthorized +HTTP response headers: HTTPHeaderDict({'Audit-Id': 'd3c86ac1-4d01-4ad7-a9bc-d37363c9419a', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Mon, 09 Jun 2025 18:36:59 GMT', 'Content-Length': '129'}) +HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} + + diff --git a/test/stress-scale-tests/logs/orchestrator_20250609_183729.log b/test/stress-scale-tests/logs/orchestrator_20250609_183729.log new file mode 100644 index 000000000..584a1fdbf --- /dev/null +++ b/test/stress-scale-tests/logs/orchestrator_20250609_183729.log @@ -0,0 +1,111 @@ +2025-06-09 18:37:29,228 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized +2025-06-09 18:37:29,434 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists +2025-06-09 18:37:29,435 - tests.orchestrator - INFO - Starting orchestrator test for 120 seconds +2025-06-09 18:37:29,623 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists +2025-06-09 18:37:29,623 - tests.orchestrator - INFO - Running each operation type once to ensure coverage +2025-06-09 18:37:29,623 - tests.orchestrator - INFO - Creating PVC: test-pvc-c676e061 +2025-06-09 18:37:29,703 - tests.orchestrator - INFO - Created PVC: test-pvc-c676e061 +2025-06-09 18:37:29,703 - tests.orchestrator - INFO - Waiting for PVC test-pvc-c676e061 to be bound +2025-06-09 18:37:31,843 - tests.orchestrator - INFO - PVC test-pvc-c676e061 is bound +2025-06-09 18:37:31,843 - tests.orchestrator - INFO - Creating PVC: test-pvc-8b3b85d1 +2025-06-09 18:37:31,913 - tests.orchestrator - INFO - Created PVC: test-pvc-8b3b85d1 +2025-06-09 18:37:31,913 - tests.orchestrator - INFO - Waiting for PVC test-pvc-8b3b85d1 to be bound +2025-06-09 18:37:34,049 - tests.orchestrator - INFO - PVC test-pvc-8b3b85d1 is bound +2025-06-09 18:37:34,049 - tests.orchestrator - INFO - Creating pod test-pod-b3edaf2c on PVC test-pvc-c676e061 +2025-06-09 18:37:34,132 - tests.orchestrator - INFO - Created pod: test-pod-b3edaf2c using PVC: test-pvc-c676e061 +2025-06-09 18:37:34,132 - tests.orchestrator - INFO - Waiting for pod test-pod-b3edaf2c to be ready +2025-06-09 18:37:34,199 - tests.orchestrator - INFO - Pod test-pod-b3edaf2c phase: Pending +2025-06-09 18:37:36,269 - tests.orchestrator - INFO - Pod test-pod-b3edaf2c phase: Running +2025-06-09 18:37:36,269 - tests.orchestrator - INFO - Pod test-pod-b3edaf2c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 18:37:38,338 - tests.orchestrator - INFO - Pod test-pod-b3edaf2c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 18:37:40,405 - tests.orchestrator - INFO - Pod test-pod-b3edaf2c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 18:37:42,479 - tests.orchestrator - INFO - Pod test-pod-b3edaf2c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 18:37:44,549 - tests.orchestrator - INFO - Pod test-pod-b3edaf2c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 18:37:46,616 - tests.orchestrator - INFO - Pod test-pod-b3edaf2c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 18:37:48,685 - tests.orchestrator - INFO - Pod test-pod-b3edaf2c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 18:37:50,755 - tests.orchestrator - INFO - Pod test-pod-b3edaf2c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-09 18:37:50,755 - tests.orchestrator - INFO - Pod test-pod-b3edaf2c is ready +2025-06-09 18:37:50,755 - tests.orchestrator - INFO - Creating pod test-pod-486f01f4 on PVC test-pvc-c676e061 +2025-06-09 18:37:50,829 - tests.orchestrator - INFO - Created pod: test-pod-486f01f4 using PVC: test-pvc-c676e061 +2025-06-09 18:37:50,829 - tests.orchestrator - INFO - Waiting for pod test-pod-486f01f4 to be ready +2025-06-09 18:37:50,902 - tests.orchestrator - INFO - Pod test-pod-486f01f4 phase: Pending +2025-06-09 18:37:55,040 - tests.orchestrator - INFO - Pod test-pod-486f01f4 phase: Running +2025-06-09 18:37:55,040 - tests.orchestrator - INFO - Pod test-pod-486f01f4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 18:37:57,109 - tests.orchestrator - INFO - Pod test-pod-486f01f4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 18:37:59,176 - tests.orchestrator - INFO - Pod test-pod-486f01f4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 18:38:01,245 - tests.orchestrator - INFO - Pod test-pod-486f01f4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 18:38:03,314 - tests.orchestrator - INFO - Pod test-pod-486f01f4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 18:38:05,383 - tests.orchestrator - INFO - Pod test-pod-486f01f4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 18:38:07,452 - tests.orchestrator - INFO - Pod test-pod-486f01f4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 18:38:09,521 - tests.orchestrator - INFO - Pod test-pod-486f01f4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 18:38:11,707 - tests.orchestrator - INFO - Pod test-pod-486f01f4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-09 18:38:11,707 - tests.orchestrator - INFO - Pod test-pod-486f01f4 is ready +2025-06-09 18:38:11,707 - tests.orchestrator - INFO - Creating pod test-pod-f51dba0f on PVC test-pvc-8b3b85d1 +2025-06-09 18:38:11,942 - tests.orchestrator - INFO - Created pod: test-pod-f51dba0f using PVC: test-pvc-8b3b85d1 +2025-06-09 18:38:11,943 - tests.orchestrator - INFO - Waiting for pod test-pod-f51dba0f to be ready +2025-06-09 18:38:12,057 - tests.orchestrator - INFO - Pod test-pod-f51dba0f phase: Pending +2025-06-09 18:38:14,126 - tests.orchestrator - INFO - Pod test-pod-f51dba0f phase: Running +2025-06-09 18:38:14,126 - tests.orchestrator - INFO - Pod test-pod-f51dba0f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 18:38:16,196 - tests.orchestrator - INFO - Pod test-pod-f51dba0f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 18:38:18,263 - tests.orchestrator - INFO - Pod test-pod-f51dba0f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 18:38:20,336 - tests.orchestrator - INFO - Pod test-pod-f51dba0f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 18:38:22,405 - tests.orchestrator - INFO - Pod test-pod-f51dba0f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 18:38:24,472 - tests.orchestrator - INFO - Pod test-pod-f51dba0f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 18:38:26,544 - tests.orchestrator - INFO - Pod test-pod-f51dba0f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 18:38:28,613 - tests.orchestrator - INFO - Pod test-pod-f51dba0f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 18:38:30,681 - tests.orchestrator - INFO - Pod test-pod-f51dba0f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True +2025-06-09 18:38:32,751 - tests.orchestrator - INFO - Pod test-pod-f51dba0f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True +2025-06-09 18:38:32,751 - tests.orchestrator - INFO - Pod test-pod-f51dba0f is ready +2025-06-09 18:38:32,751 - tests.orchestrator - INFO - Testing read/write between pods test-pod-b3edaf2c and test-pod-486f01f4 sharing PVC test-pvc-c676e061 +2025-06-09 18:38:32,751 - tests.orchestrator - INFO - Executing write command: kubectl exec -n efs-stress-test test-pod-b3edaf2c -- /bin/sh -c 'echo "Test content: 101c0ddd-87ba-424d-8a9d-5dbe582e6e47" > /data/test-69dc76bb.txt' +2025-06-09 18:38:34,042 - tests.orchestrator - INFO - Pod test-pod-b3edaf2c wrote to /data/test-69dc76bb.txt +2025-06-09 18:38:36,042 - tests.orchestrator - INFO - Executing read command: kubectl exec -n efs-stress-test test-pod-486f01f4 -- cat /data/test-69dc76bb.txt +2025-06-09 18:38:37,308 - tests.orchestrator - INFO - Read result: 'Test content: 101c0ddd-87ba-424d-8a9d-5dbe582e6e47' +2025-06-09 18:38:37,308 - tests.orchestrator - INFO - Pod test-pod-486f01f4 successfully read content written by test-pod-b3edaf2c +2025-06-09 18:38:37,308 - tests.orchestrator - INFO - Deleting pod: test-pod-486f01f4 from PVC: test-pvc-c676e061 +2025-06-09 18:38:37,387 - tests.orchestrator - INFO - Waiting for pod test-pod-486f01f4 to be deleted +2025-06-09 18:39:10,176 - tests.orchestrator - INFO - Pod test-pod-486f01f4 has been deleted +2025-06-09 18:39:10,177 - tests.orchestrator - INFO - Deleted pod: test-pod-486f01f4 +2025-06-09 18:39:10,177 - tests.orchestrator - INFO - Deleting PVC: test-pvc-8b3b85d1 +2025-06-09 18:39:10,177 - tests.orchestrator - INFO - Deleting 1 pods using PVC test-pvc-8b3b85d1 +2025-06-09 18:39:10,177 - tests.orchestrator - INFO - Deleting pod: test-pod-f51dba0f from PVC: test-pvc-8b3b85d1 +2025-06-09 18:39:10,251 - tests.orchestrator - INFO - Waiting for pod test-pod-f51dba0f to be deleted +2025-06-09 18:39:42,876 - tests.orchestrator - INFO - Pod test-pod-f51dba0f has been deleted +2025-06-09 18:39:42,876 - tests.orchestrator - INFO - Deleted pod: test-pod-f51dba0f +2025-06-09 18:39:42,953 - tests.orchestrator - INFO - Waiting for PVC test-pvc-8b3b85d1 to be deleted +2025-06-09 18:39:47,093 - tests.orchestrator - INFO - PVC test-pvc-8b3b85d1 has been deleted +2025-06-09 18:39:47,093 - tests.orchestrator - INFO - Deleted PVC: test-pvc-8b3b85d1 +2025-06-09 18:39:47,093 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations +2025-06-09 18:39:47,093 - tests.orchestrator - INFO - Test completed in 137.66 seconds +2025-06-09 18:39:47,093 - tests.orchestrator - INFO - ===== STARTING COMPREHENSIVE CLEANUP ===== +2025-06-09 18:39:47,093 - tests.orchestrator - INFO - Deleting 1 pods (force=False)... +2025-06-09 18:39:47,093 - tests.orchestrator - INFO - Deleting pod: test-pod-b3edaf2c from PVC: test-pvc-c676e061 +2025-06-09 18:39:47,169 - tests.orchestrator - INFO - Waiting for pod test-pod-b3edaf2c to be deleted +2025-06-09 18:40:19,821 - tests.orchestrator - INFO - Pod test-pod-b3edaf2c has been deleted +2025-06-09 18:40:19,821 - tests.orchestrator - INFO - Deleted pod: test-pod-b3edaf2c +2025-06-09 18:40:24,822 - tests.orchestrator - INFO - Deleting 1 PVCs (force=False)... +2025-06-09 18:40:24,822 - tests.orchestrator - INFO - Deleting PVC: test-pvc-c676e061 +2025-06-09 18:40:24,898 - tests.orchestrator - INFO - Waiting for PVC test-pvc-c676e061 to be deleted +2025-06-09 18:40:24,965 - tests.orchestrator - INFO - PVC test-pvc-c676e061 has been deleted +2025-06-09 18:40:24,965 - tests.orchestrator - INFO - Deleted PVC: test-pvc-c676e061 +2025-06-09 18:40:25,180 - tests.orchestrator - WARNING - First cleanup pass incomplete. Remaining resources: ['pod/test-pod-0d0fddfe', 'pod/test-pod-158cdfa4', 'pod/test-pod-4044e577', 'pod/test-pod-87095965', 'pvc/test-pvc-02e00a44', 'pvc/test-pvc-0bb0d20c', 'pvc/test-pvc-2127e71a', 'pvc/test-pvc-2cf5b7a2', 'pvc/test-pvc-31fbd501', 'pvc/test-pvc-4ada8088', 'pvc/test-pvc-4b26f6f6', 'pvc/test-pvc-528b4f5c', 'pvc/test-pvc-5c13c143', 'pvc/test-pvc-c542ee58', 'pvc/test-pvc-dfd12d43', 'pvc/test-pvc-ecb12f74'] +2025-06-09 18:40:25,180 - tests.orchestrator - INFO - Attempting force deletion of remaining resources... +2025-06-09 18:40:25,180 - tests.orchestrator - INFO - Deleting 0 pods (force=True)... +2025-06-09 18:40:30,180 - tests.orchestrator - INFO - Deleting 0 PVCs (force=True)... +2025-06-09 18:40:30,387 - tests.orchestrator - ERROR - Cleanup incomplete. Remaining resources after force deletion: ['pod/test-pod-0d0fddfe', 'pod/test-pod-158cdfa4', 'pod/test-pod-4044e577', 'pod/test-pod-87095965', 'pvc/test-pvc-02e00a44', 'pvc/test-pvc-0bb0d20c', 'pvc/test-pvc-2127e71a', 'pvc/test-pvc-2cf5b7a2', 'pvc/test-pvc-31fbd501', 'pvc/test-pvc-4ada8088', 'pvc/test-pvc-4b26f6f6', 'pvc/test-pvc-528b4f5c', 'pvc/test-pvc-5c13c143', 'pvc/test-pvc-c542ee58', 'pvc/test-pvc-dfd12d43', 'pvc/test-pvc-ecb12f74'] +2025-06-09 18:40:30,387 - tests.orchestrator - INFO - Cleanup completed successfully in 43.29 seconds +2025-06-09 18:40:30,387 - tests.orchestrator - INFO - ===== CLEANUP PROCESS FINISHED ===== +2025-06-09 18:40:30,387 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== +2025-06-09 18:40:30,387 - tests.orchestrator - INFO - --- Operations --- +2025-06-09 18:40:30,387 - tests.orchestrator - INFO - create_pvc: 2 succeeded, 0 failed (100.0%) +2025-06-09 18:40:30,387 - tests.orchestrator - INFO - attach_pod: 3 succeeded, 0 failed (100.0%) +2025-06-09 18:40:30,387 - tests.orchestrator - INFO - delete_pod: 3 succeeded, 0 failed (100.0%) +2025-06-09 18:40:30,387 - tests.orchestrator - INFO - delete_pvc: 2 succeeded, 0 failed (100.0%) +2025-06-09 18:40:30,387 - tests.orchestrator - INFO - verify_read_write: Writes 1 succeeded, 0 failed (100.0%) +2025-06-09 18:40:30,388 - tests.orchestrator - INFO - verify_read_write: Reads 1 succeeded, 0 failed (100.0%) +2025-06-09 18:40:30,388 - tests.orchestrator - INFO - --- Scenarios --- +2025-06-09 18:40:30,388 - tests.orchestrator - INFO - shared_volume_rw: 1 succeeded, 0 failed out of 1 runs (100.0%) +2025-06-09 18:40:30,388 - tests.orchestrator - INFO - many_to_one: No runs +2025-06-09 18:40:30,388 - tests.orchestrator - INFO - one_to_one: No runs +2025-06-09 18:40:30,388 - tests.orchestrator - INFO - concurrent_pvc: No runs +2025-06-09 18:40:30,388 - tests.orchestrator - INFO - ========================================= diff --git a/test/stress-scale-tests/prometheus_exporter.pid b/test/stress-scale-tests/prometheus_exporter.pid new file mode 100644 index 000000000..6b22aaccc --- /dev/null +++ b/test/stress-scale-tests/prometheus_exporter.pid @@ -0,0 +1 @@ +26896 diff --git a/test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_201716_20250605_201716.json b/test/stress-scale-tests/reports/general/efs_tests_20250605_201716_20250605_201716.json similarity index 100% rename from test/e2e/stress-scale-tests/reports/general/efs_tests_20250605_201716_20250605_201716.json rename to test/stress-scale-tests/reports/general/efs_tests_20250605_201716_20250605_201716.json diff --git a/test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_201716.json b/test/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_201716.json similarity index 100% rename from test/e2e/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_201716.json rename to test/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_201716.json diff --git a/test/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250609_145633.json b/test/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250609_145633.json new file mode 100644 index 000000000..ecaa3f823 --- /dev/null +++ b/test/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250609_145633.json @@ -0,0 +1,59 @@ +{ + "test_duration": 1749480993.4082596, + "operations": { + "create_pvc": { + "success": 9, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 32, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 32, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 15, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 7, + "write_fail": 0, + "read_success": 7, + "read_fail": 0, + "write_success_rate": 100.0, + "read_success_rate": 100.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 7, + "success": 7, + "fail": 0, + "success_rate": 100.0 + }, + "many_to_one": { + "runs": 6, + "success": 6, + "fail": 0, + "success_rate": 100.0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + } + } +} \ No newline at end of file diff --git a/test/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250609_184030.json b/test/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250609_184030.json new file mode 100644 index 000000000..5472774ff --- /dev/null +++ b/test/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250609_184030.json @@ -0,0 +1,59 @@ +{ + "test_duration": 1749494430.3875926, + "operations": { + "create_pvc": { + "success": 2, + "fail": 0, + "success_rate": 100.0 + }, + "attach_pod": { + "success": 3, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pod": { + "success": 3, + "fail": 0, + "success_rate": 100.0 + }, + "delete_pvc": { + "success": 2, + "fail": 0, + "success_rate": 100.0 + }, + "verify_read_write": { + "write_success": 1, + "write_fail": 0, + "read_success": 1, + "read_fail": 0, + "write_success_rate": 100.0, + "read_success_rate": 100.0 + } + }, + "scenarios": { + "shared_volume_rw": { + "runs": 1, + "success": 1, + "fail": 0, + "success_rate": 100.0 + }, + "many_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "one_to_one": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + }, + "concurrent_pvc": { + "runs": 0, + "success": 0, + "fail": 0, + "success_rate": 0 + } + } +} \ No newline at end of file diff --git a/test/e2e/stress-scale-tests/requirements.txt b/test/stress-scale-tests/requirements.txt similarity index 100% rename from test/e2e/stress-scale-tests/requirements.txt rename to test/stress-scale-tests/requirements.txt diff --git a/test/e2e/stress-scale-tests/run_tests.py b/test/stress-scale-tests/run_tests.py similarity index 96% rename from test/e2e/stress-scale-tests/run_tests.py rename to test/stress-scale-tests/run_tests.py index 8c5be5a11..d5a84838c 100644 --- a/test/e2e/stress-scale-tests/run_tests.py +++ b/test/stress-scale-tests/run_tests.py @@ -114,11 +114,8 @@ def print_credential_renewal_instructions(): print(f"{'AWS CREDENTIALS EXPIRED OR INVALID':^80}") print("="*80) print("\nYour AWS credentials have expired or are invalid.") - print("\nTo renew your credentials, run:") - print("\n ada credentials update --provider isengard --role=Admin --once --account YOUR_ACCOUNT_ID") - print("\nOr for temporary credentials:") - print("\n aws sts get-session-token") - print("\nAfter renewing credentials, try running the tests again.") + print("\nPlease check your AWS credentials and ensure they are properly configured.") + print("\nAfter renewing your credentials, try running the tests again.") print("="*80 + "\n") def main(): diff --git a/test/stress-scale-tests/test-reports/sample-test-report.json b/test/stress-scale-tests/test-reports/sample-test-report.json new file mode 100644 index 000000000..d75b5ff76 --- /dev/null +++ b/test/stress-scale-tests/test-reports/sample-test-report.json @@ -0,0 +1,145 @@ +{ + "test_name": "EFS CSI Stress Test", + "test_type": "stress", + "timestamp": "20250609_180200", + "system_info": { + "hostname": "test-vm", + "platform": "Linux-5.10.0", + "kubernetes_version": "v1.25", + "aws_region": "us-west-2", + "efs_filesystem_id": "fs-1234abcd" + }, + "results": { + "test_duration": 300, + "test_info": { + "test_name": "EFS CSI Stress Test", + "test_type": "stress", + "duration": 300 + }, + "operations": { + "create_pvc_1": { + "start_time": 1717686500.0, + "end_time": 1717686502.4, + "duration": 2.4, + "status": "success" + }, + "create_pvc_2": { + "start_time": 1717686505.0, + "end_time": 1717686508.1, + "duration": 3.1, + "status": "success" + }, + "attach_pod_1": { + "start_time": 1717686510.0, + "end_time": 1717686518.5, + "duration": 8.5, + "status": "success" + }, + "attach_pod_2": { + "start_time": 1717686520.0, + "end_time": 1717686527.2, + "duration": 7.2, + "status": "success" + }, + "verify_readwrite_1": { + "start_time": 1717686530.0, + "end_time": 1717686531.3, + "duration": 1.3, + "status": "success" + }, + "delete_pod_1": { + "start_time": 1717686540.0, + "end_time": 1717686541.9, + "duration": 1.9, + "status": "success" + }, + "delete_pvc_1": { + "start_time": 1717686550.0, + "end_time": 1717686551.7, + "duration": 1.7, + "status": "success" + } + }, + "system": { + "create_pvc_1": { + "start_cpu_percent": 15.2, + "end_cpu_percent": 18.7, + "start_memory_percent": 45.6, + "end_memory_percent": 46.8, + "start_disk_read_bytes": 10240000, + "end_disk_read_bytes": 10241024, + "start_disk_write_bytes": 5120000, + "end_disk_write_bytes": 5125120 + }, + "attach_pod_1": { + "start_cpu_percent": 18.9, + "end_cpu_percent": 25.4, + "start_memory_percent": 47.2, + "end_memory_percent": 48.9, + "start_disk_read_bytes": 10245000, + "end_disk_read_bytes": 10250000, + "start_disk_write_bytes": 5130000, + "end_disk_write_bytes": 5140000 + } + }, + "test_scenarios": { + "many_to_one": { + "success_rate": 95.5, + "duration": 45.2 + }, + "one_to_one": { + "success_rate": 98.2, + "duration": 32.7 + }, + "concurrent_pvc": { + "success_rate": 92.1, + "duration": 28.9 + } + }, + "efs": { + "access_point_timing": { + "fsap-1234abcd": 1.8, + "fsap-5678efgh": 2.1 + }, + "mount_completion_timing": { + "test-pod-1": { + "test-pvc-1": 6.4 + }, + "test-pod-2": { + "test-pvc-2": 5.9 + } + }, + "mount_errors": { + "worker-node-1": 0, + "worker-node-2": 1 + }, + "api_throttling_incidents": [ + { + "timestamp": 1717686525.0, + "operation_type": "CreateAccessPoint", + "error_message": "Rate exceeded" + } + ] + }, + "k8s_events": { + "volume_events": [ + { + "timestamp": 1717686500.0, + "name": "test-pvc-1", + "kind": "PersistentVolumeClaim", + "reason": "ProvisioningSucceeded", + "message": "Successfully provisioned volume", + "count": 1 + } + ], + "binding_times": { + "test-pvc-1-test-pv-1": 0.8, + "test-pvc-2-test-pv-2": 0.9 + }, + "pod_startup_delays": { + "test-pod-1": 7.2, + "test-pod-2": 6.8 + } + } + } +} diff --git a/test/e2e/stress-scale-tests/tests/__init__.py b/test/stress-scale-tests/tests/__init__.py similarity index 100% rename from test/e2e/stress-scale-tests/tests/__init__.py rename to test/stress-scale-tests/tests/__init__.py diff --git a/test/stress-scale-tests/tests/__pycache__/__init__.cpython-312.pyc b/test/stress-scale-tests/tests/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..209026e0da3d68809337344a369e4e076819ca11 GIT binary patch literal 196 zcmX@j%ge<81hv-g86f&Gh(HIQS%4zb87dhx8U0o=6fpsLpFwJVCF|$pCnx6UXXNLm z>SrcpBo?LUr=}I_CKqSwrW9qCr55R?xjMSrcpBo?LUr=}I_CKqSwrW9qCr55R? C?=-gn literal 0 HcmV?d00001 diff --git a/test/e2e/stress-scale-tests/utils/__pycache__/metrics_collector.cpython-312.pyc b/test/stress-scale-tests/utils/__pycache__/metrics_collector.cpython-312.pyc similarity index 96% rename from test/e2e/stress-scale-tests/utils/__pycache__/metrics_collector.cpython-312.pyc rename to test/stress-scale-tests/utils/__pycache__/metrics_collector.cpython-312.pyc index 9de533713aedb21cb36218d9935f971860ab2a92..21968d08f381549ea6040a684662da977ce4483f 100644 GIT binary patch delta 33 ncmZ40#<;AFk^3|+FBbz4)LOf5P@J0Px`qivR!s diff --git a/test/e2e/stress-scale-tests/utils/metrics_collector.py b/test/stress-scale-tests/utils/metrics_collector.py similarity index 100% rename from test/e2e/stress-scale-tests/utils/metrics_collector.py rename to test/stress-scale-tests/utils/metrics_collector.py diff --git a/test/e2e/stress-scale-tests/utils/report_generator.py b/test/stress-scale-tests/utils/report_generator.py similarity index 100% rename from test/e2e/stress-scale-tests/utils/report_generator.py rename to test/stress-scale-tests/utils/report_generator.py From b4a46e502f63d70599570619b804a75f5fe3b4d2 Mon Sep 17 00:00:00 2001 From: Ishaan Bhardwaj Date: Wed, 11 Jun 2025 20:00:50 +0000 Subject: [PATCH 05/13] Improve operation interval handling and add detailed failure logging --- test/stress-scale-tests/run_tests.py | 103 ++++- test/stress-scale-tests/tests/orchestrator.py | 167 +++++++- .../log_integration.cpython-312.pyc | Bin 0 -> 15868 bytes .../metrics_collector.cpython-312.pyc | Bin 17190 -> 25216 bytes .../report_generator.cpython-312.pyc | Bin 18802 -> 17838 bytes .../utils/log_integration.py | 366 ++++++++++++++++++ .../utils/metrics_collector.py | 215 +++++++++- .../utils/report_generator.py | 35 +- 8 files changed, 836 insertions(+), 50 deletions(-) create mode 100644 test/stress-scale-tests/utils/__pycache__/log_integration.cpython-312.pyc create mode 100644 test/stress-scale-tests/utils/log_integration.py diff --git a/test/stress-scale-tests/run_tests.py b/test/stress-scale-tests/run_tests.py index d5a84838c..04b591473 100644 --- a/test/stress-scale-tests/run_tests.py +++ b/test/stress-scale-tests/run_tests.py @@ -10,6 +10,7 @@ from tests.orchestrator import EFSCSIOrchestrator from utils.report_generator import ReportGenerator from utils.metrics_collector import MetricsCollector +from utils.log_integration import collect_logs_on_test_failure # Commented out to remove dependency on cluster setup # from cluster_setup import ClusterSetup @@ -61,16 +62,21 @@ def parse_args(): help='Duration in seconds for test execution' ) parser.add_argument( - '--rate', + '--interval', type=int, - default=5, - help='Operations per second for tests' + default=None, + help='Seconds to wait between operations (overrides config value)' ) parser.add_argument( '--dry-run', action='store_true', help='Print what would be done without executing tests' ) + parser.add_argument( + '--driver-pod-name', + default=None, + help='Name of the EFS CSI driver pod for log collection (optional)' + ) # Cluster setup options - kept for compatibility but functionality is disabled parser.add_argument( @@ -202,41 +208,108 @@ def main(): orchestrator_config = args.config logger.info(f"Using orchestrator config: {orchestrator_config}") - orchestrator = EFSCSIOrchestrator(config_file=orchestrator_config) + # Get driver pod name from config if not specified on command line + driver_pod_name = args.driver_pod_name + if driver_pod_name is None and 'driver' in config and 'pod_name' in config['driver']: + driver_pod_name = config['driver']['pod_name'] + if driver_pod_name: + logger.info(f"Using driver pod name from config: {driver_pod_name}") + + # Pass the metrics collector and driver pod name to the orchestrator + orchestrator = EFSCSIOrchestrator( + config_file=orchestrator_config, + metrics_collector=metrics_collector, + driver_pod_name=driver_pod_name + ) # Override default test parameters if specified if args.duration: orchestrator.test_duration = args.duration logger.info(f"Test duration overridden to {args.duration} seconds") - if args.rate: # Use rate as operation interval (inverse relationship) - orchestrator.operation_interval = max(1, int(1 / args.rate)) - logger.info(f"Operation interval overridden to {orchestrator.operation_interval} seconds (from rate {args.rate}/s)") + if args.interval is not None: # Only override if explicitly specified + orchestrator.operation_interval = args.interval + logger.info(f"Operation interval overridden to {orchestrator.operation_interval} seconds") + else: + logger.info(f"Using operation interval from config: {orchestrator.operation_interval} seconds") # Run the orchestrator test orchestrator_results = orchestrator.run_test() - # Generate orchestrator test specific report + # Generate one consolidated orchestrator report with all information orchestrator_report_dir = os.path.join(report_dir, 'orchestrator') os.makedirs(orchestrator_report_dir, exist_ok=True) - + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - orchestrator_report_path = os.path.join( + test_name = f"efs_orchestrator_{timestamp}" + + # Create report path + report_path = os.path.join( orchestrator_report_dir, - f"efs_orchestrator_{timestamp}.json" + f"{test_name}.json" ) - with open(orchestrator_report_path, 'w') as f: + # Add metadata, system info, and metrics to the results + system_info = report_generator._collect_system_info() + + # Get all metrics from the metrics collector + collected_metrics = metrics_collector.get_all_metrics() + + full_report = { + "test_name": test_name, + "test_type": "orchestrator", + "timestamp": timestamp, + "system_info": system_info, + "results": orchestrator_results, + "metrics": { + "file_performance": collected_metrics.get("file_performance", {}) + } + } + + logger.info("File performance metrics collected:") + for volume, metrics in collected_metrics.get("file_performance", {}).get("by_volume", {}).items(): + logger.info(f" Volume: {volume}") + # Log read metrics if available + if metrics["iops"].get("read") is not None: + logger.info(f" Read IOPS: {metrics['iops']['read']:.2f}") + if metrics["throughput"].get("read") is not None: + logger.info(f" Read Throughput: {metrics['throughput']['read']:.2f} MB/s") + + # Log write metrics if available + if metrics["iops"].get("write") is not None: + logger.info(f" Write IOPS: {metrics['iops']['write']:.2f}") + if metrics["throughput"].get("write") is not None: + logger.info(f" Write Throughput: {metrics['throughput']['write']:.2f} MB/s") + + with open(report_path, 'w') as f: import json - json.dump(orchestrator_results, f, indent=2) - - logger.info(f"Orchestrator report generated: {orchestrator_report_path}") + json.dump(full_report, f, indent=2) + + logger.info(f"Orchestrator report generated: {report_path}") results['orchestrator'] = orchestrator_results except Exception as e: logger.error(f"Error running tests: {e}", exc_info=True) + # Collect logs on failure + logger.info("Collecting logs due to test failure") + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + test_name = f"efs_orchestrator_failure_{timestamp}" + + # Get driver pod name from config if not specified on command line + driver_pod_name = args.driver_pod_name + if driver_pod_name is None and 'driver' in config and 'pod_name' in config['driver']: + driver_pod_name = config['driver']['pod_name'] + if driver_pod_name: + logger.info(f"Using driver pod name from config: {driver_pod_name}") + + logs_path = collect_logs_on_test_failure(test_name, metrics_collector, driver_pod_name=driver_pod_name) + if logs_path: + logger.info(f"Failure logs collected to: {logs_path}") + else: + logger.warning("Failed to collect logs") + # Even if tests failed, try to clean up if requested - DISABLED if args.setup_cluster and not args.skip_cleanup: logger.warning("Cluster cleanup functionality is disabled") diff --git a/test/stress-scale-tests/tests/orchestrator.py b/test/stress-scale-tests/tests/orchestrator.py index ebaf5dbc5..6a5d4818e 100644 --- a/test/stress-scale-tests/tests/orchestrator.py +++ b/test/stress-scale-tests/tests/orchestrator.py @@ -8,12 +8,16 @@ import os from kubernetes import client, config from datetime import datetime +from utils.log_integration import collect_logs_on_test_failure class EFSCSIOrchestrator: """Orchestrator for testing EFS CSI driver operations""" - def __init__(self, config_file='config/orchestrator_config.yaml', namespace=None): + def __init__(self, config_file='config/orchestrator_config.yaml', namespace=None, metrics_collector=None, driver_pod_name=None): """Initialize the orchestrator with configuration""" + # Store driver pod name for log collection + self.driver_pod_name = driver_pod_name + # Load configuration with open(config_file, 'r') as f: self.config = yaml.safe_load(f) @@ -24,6 +28,10 @@ def __init__(self, config_file='config/orchestrator_config.yaml', namespace=None self.apps_v1 = client.AppsV1Api() self.storage_v1 = client.StorageV1Api() + # Initialize metrics collector if provided, or create a new one + from utils.metrics_collector import MetricsCollector + self.metrics_collector = metrics_collector or MetricsCollector() + # Set namespace from config or use default self.namespace = namespace or self.config['test'].get('namespace', 'default') @@ -162,7 +170,7 @@ def run_test(self): self._verify_readwrite() # Run a specific scenario - # self._run_specific_scenario() + self._run_specific_scenario() # Delete a pod self._delete_pod() @@ -197,6 +205,29 @@ def run_test(self): self.logger.info("Test interrupted by user") except Exception as e: self.logger.error(f"Unexpected error during test: {e}", exc_info=True) + + # Collect logs for unexpected failure with comprehensive resource tracking + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + test_name = f"orchestrator_unexpected_failure_{timestamp}" + + # Gather information about all active resources for diagnostic logging + failed_resources = [] + + # Add all PVCs currently being tracked + for pvc_name in self.pvcs: + failed_resources.append({"type": "pvc", "name": pvc_name, "namespace": self.namespace}) + + # Add all pods associated with each PVC + for pod_name in self.pods.get(pvc_name, []): + failed_resources.append({"type": "pod", "name": pod_name, "namespace": self.namespace}) + + logs_path = collect_logs_on_test_failure( + test_name, + self.metrics_collector, + self.driver_pod_name, + failed_resources=failed_resources + ) + self.logger.info(f"Collected comprehensive failure logs to: {logs_path}") finally: # Get test duration elapsed = time.time() - start_time @@ -557,6 +588,8 @@ def _verify_readwrite(self): Note: Using kubectl subprocesses instead of Kubernetes API to avoid WebSocket upgrade errors + + Also collects performance metrics for file operations. """ # Find PVCs that have multiple pods shared_pvcs = [(pvc, pods) for pvc, pods in self.pods.items() if len(pods) >= 2] @@ -575,13 +608,18 @@ def _verify_readwrite(self): reader_pod = random.choice([p for p in pod_names if p != writer_pod]) test_file = f"test-{uuid.uuid4().hex[:8]}.txt" - test_content = f"Test content: {uuid.uuid4()}" + test_content = f"Test content: {uuid.uuid4()}" * 50 # Make content larger for better measurements + content_size_bytes = len(test_content.encode('utf-8')) self.logger.info(f"Testing read/write between pods {writer_pod} and {reader_pod} sharing PVC {pvc_name}") + self.logger.info(f"File size: {content_size_bytes} bytes") try: import subprocess + # Track write operation with metrics + write_op_start = time.time() + # Write with first pod using kubectl write_cmd = f"kubectl exec -n {self.namespace} {writer_pod} -- /bin/sh -c 'echo \"{test_content}\" > /data/{test_file}'" self.logger.info(f"Executing write command: {write_cmd}") @@ -595,6 +633,18 @@ def _verify_readwrite(self): text=True ) + # Measure write operation duration + write_duration = time.time() - write_op_start + write_ops_count = 1 # One write operation + + # Track metrics for write operation + self.metrics_collector.track_file_operation_latency(pvc_name, "write", write_duration) + self.metrics_collector.track_file_operation_iops(pvc_name, "write", write_ops_count, write_duration) + self.metrics_collector.track_file_operation_throughput(pvc_name, "write", content_size_bytes, write_duration) + + self.logger.info(f"Write operation completed in {write_duration:.3f}s") + self.logger.info(f"Write throughput: {(content_size_bytes / 1024 / 1024) / write_duration:.2f} MB/s") + # Update results self.results['verify_write']['success'] += 1 self.logger.info(f"Pod {writer_pod} wrote to /data/{test_file}") @@ -602,6 +652,9 @@ def _verify_readwrite(self): # Sleep briefly to ensure the write completes and is visible time.sleep(2) + # Track read operation with metrics + read_op_start = time.time() + # Read with second pod using kubectl read_cmd = f"kubectl exec -n {self.namespace} {reader_pod} -- cat /data/{test_file}" self.logger.info(f"Executing read command: {read_cmd}") @@ -615,8 +668,20 @@ def _verify_readwrite(self): text=True ) + # Measure read operation duration + read_duration = time.time() - read_op_start + read_ops_count = 1 # One read operation + + # Track metrics for read operation + self.metrics_collector.track_file_operation_latency(pvc_name, "read", read_duration) + self.metrics_collector.track_file_operation_iops(pvc_name, "read", read_ops_count, read_duration) + self.metrics_collector.track_file_operation_throughput(pvc_name, "read", content_size_bytes, read_duration) + resp = read_process.stdout.strip() - self.logger.info(f"Read result: '{resp}'") + + self.logger.info(f"Read operation completed in {read_duration:.3f}s") + self.logger.info(f"Read throughput: {(content_size_bytes / 1024 / 1024) / read_duration:.2f} MB/s") + self.logger.info(f"Read result length: {len(resp)} bytes") # Verify content if test_content in resp: @@ -626,11 +691,34 @@ def _verify_readwrite(self): # Update scenario tracking self.scenarios['shared_volume_rw']['runs'] += 1 self.scenarios['shared_volume_rw']['success'] += 1 + + # Add metadata operation metrics - perform ls to measure metadata operations + meta_op_start = time.time() + ls_cmd = f"kubectl exec -n {self.namespace} {reader_pod} -- ls -la /data/" + + ls_process = subprocess.run( + ls_cmd, + shell=True, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + + meta_duration = time.time() - meta_op_start + meta_ops_count = 1 # One metadata operation + + # Track metrics for metadata operation + self.metrics_collector.track_file_operation_latency(pvc_name, "metadata", meta_duration) + self.metrics_collector.track_file_operation_iops(pvc_name, "metadata", meta_ops_count, meta_duration) + + self.logger.info(f"Metadata operation (ls) completed in {meta_duration:.3f}s") + else: self.results['verify_read']['fail'] += 1 self.scenarios['shared_volume_rw']['runs'] += 1 self.scenarios['shared_volume_rw']['fail'] += 1 - self.logger.error(f"Pod {reader_pod} failed to read content written by {writer_pod}. Got: '{resp}'") + self.logger.error(f"Pod {reader_pod} failed to read content written by {writer_pod}. Got different content length: {len(resp)} vs expected {len(test_content)}") except subprocess.CalledProcessError as e: self.logger.error(f"Command execution failed: {e}") @@ -660,9 +748,9 @@ def _run_specific_scenario(self): Randomly select from the required scenarios """ scenarios = [ - self._scenario_many_to_one - #self._scenario_one_to_one, - #self._scenario_concurrent_pvc + self._scenario_many_to_one, + self._scenario_one_to_one, + self._scenario_concurrent_pvc ] # Pick a random scenario @@ -855,6 +943,26 @@ def _scenario_many_to_one(self): self.logger.info(f"[MANY2ONE] Reader pod mount info: '{reader_mount_process.stdout.strip()}'") self.scenarios['many_to_one']['fail'] += 1 + # Collect logs for failure diagnostics with detailed information about failed resources + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + test_name = f"many2one_failure_{timestamp}" + + # Track failed resources for detailed logging + failed_resources = [ + # Add any pods involved in the scenario + {"type": "pod", "name": writer_pod, "namespace": self.namespace}, + {"type": "pod", "name": reader_pod, "namespace": self.namespace}, + # Add the PVC + {"type": "pvc", "name": pvc_name, "namespace": self.namespace} + ] + + logs_path = collect_logs_on_test_failure( + test_name, + self.metrics_collector, + self.driver_pod_name, + failed_resources=failed_resources + ) + self.logger.info(f"Collected detailed failure logs to: {logs_path}") except subprocess.CalledProcessError as e: self.logger.error(f"[MANY2ONE] Command execution failed: {e}") @@ -997,6 +1105,26 @@ def _scenario_one_to_one(self): self.logger.error("[ONE2ONE] One-to-one scenario failed") self.scenarios['one_to_one']['fail'] += 1 + # Collect logs for failure diagnostics with detailed information + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + test_name = f"one2one_failure_{timestamp}" + + # Track all failed resources for detailed logging + failed_resources = [] + + # Add all PVC-pod pairs + for pvc_name, pod_name in pairs: + failed_resources.append({"type": "pod", "name": pod_name, "namespace": self.namespace}) + failed_resources.append({"type": "pvc", "name": pvc_name, "namespace": self.namespace}) + + logs_path = collect_logs_on_test_failure( + test_name, + self.metrics_collector, + self.driver_pod_name, + failed_resources=failed_resources + ) + self.logger.info(f"Collected detailed failure logs to: {logs_path}") + except Exception as e: self.logger.error(f"[ONE2ONE] Unhandled error in one-to-one scenario: {e}") self.scenarios['one_to_one']['fail'] += 1 @@ -1053,6 +1181,29 @@ def _scenario_concurrent_pvc(self): except Exception as e: self.logger.error(f"Error in rapid PVC scenario: {e}") self.scenarios['concurrent_pvc']['fail'] += 1 + + # Collect logs for failure diagnostics with detailed information + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + test_name = f"concurrent_pvc_failure_{timestamp}" + + # Track all resources involved in this scenario + failed_resources = [] + + # Add all created PVCs + for pvc_name in created_pvcs: + failed_resources.append({"type": "pvc", "name": pvc_name, "namespace": self.namespace}) + + # Add pods using those PVCs + for pod_name in self.pods.get(pvc_name, []): + failed_resources.append({"type": "pod", "name": pod_name, "namespace": self.namespace}) + + logs_path = collect_logs_on_test_failure( + test_name, + self.metrics_collector, + self.driver_pod_name, + failed_resources=failed_resources + ) + self.logger.info(f"Collected detailed failure logs to: {logs_path}") def _create_pvc_for_concurrent(self, pvc_name): """ diff --git a/test/stress-scale-tests/utils/__pycache__/log_integration.cpython-312.pyc b/test/stress-scale-tests/utils/__pycache__/log_integration.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..eef6a24a82835b6f15a28fa591c051a8410db46d GIT binary patch literal 15868 zcmcJ0Yj6`;mS(2jvfi>STk?bS17u-a=KV0(F2GRW7Z_a7yh<&UvXPM`SEdYDNHy$6 z?>1}@fxXiaHZxs;E2bw)z{pHx0AFk@^ZWlE@lSZbjR=Fc4cKI{^+y%Q#xc=-p zS4v41Hq;|J!8h;Y+;h)8_mTH}_vkB~u84x?=9ed~HtnaV|APtfk!2z;{x?LXD3)Sr zKXoDfrZ3PWEx8~eZ|MaodCM-y$Xk9vPTqg1Vq) z4XhkeMXUl+8rH}vA*E$atP0*bubx$3YoI7@hlKJ)7l{XO-OM9ihBaeAC~nExG#`W$ph#kxb@kZ;86k*4dBPzfQX zUuc2b6cx(wq^JmJCJY4;nw313ewwL6QxOU02xU18i~1zY(gDwop~OCak5=WMW+bNb`pVLUy&P9Kl;N#MZ+bmr zAur=*Ji(C>cYtN^?*`|Cyx>@9bS%V#g3O@L?@iG~0}e?_M&j~8{}_MOAxp{mt6sl9 zB@20PhEfVX#Ckc7yrHr~%3-fK>=37cpwFR5$wxU~Ae1WN$F7WWL64W`Q&Mg$kW!o( z=sfw}05bG@Z^CQT?+bk*O(`>Saj0TW1p$vk!!XS1aEm|aar;}Y21mRtzAIPV9NXd@ zdHGNaAL5|-W*&HWn=yxP84LORJg^;dfso!I z&K>dv1J2R$l);-;luHz+6ok;CH19)jn|dx&-?2}0-gQogV!NKnw*5(|T~?Xz>+k9( zZp^Upl7AXrYCpNqep2|^rLR?&pQnp{H8yo?^484WL|J3J{-4529j6vLP6_8PKU2NG z4E*XR24+gWmTh?fEl_KwG)^nj!izl+z!Ik;)ZfFDjL;BDKa#UDoYqlk-c*-HB#-5} zV=sR2)TU$FSn|@Hg*Nd{0wS? z32$MV0}N(0^W%KTJHj-37*zDAe{2ZqHa7!jm`y1fNSj0Bqh4l^X(m$0yr8;}EZX@O zDXreMBBkUtFIAMhM5x?18~R$-$yuPDC%ZL3BD(=C!?@*dK*RY|K!{H$vv{fOl8;Iv zl40ag@T5mQe>;rOx2tm-$)_3mC@g*63_NfN7Cfv}+*n{Um4tL+@vvTu<=!H{VN)(o zjIlD2PgKgv$!t_aq_A1bqOuTTt-6n- z5#?jOC_il3ISK=zFcwnar-zl~)yTDG+ORb<;Z^(tc$F0JsuLxL4@oJ=Y{`8v-yl)h z8@QBzZ!XpXE;$|9NWUtuUof(j8_PuIWEIXywi;%oDzsVb&w59ch>G1LY8b{%9Z?V4 zMf%*ERkP~Hj7SH^q=J#zVBOaIm^mKFu93Azpr6L;Ewfr7u4Jj$p7;O`Ts-A|NecrL1pW}0X6;9Qydd5+F*CA zjTE7_VP@6kM)^F9bF}EgqH!2)eBYo&zK(i#E1WKi@MH5H?lzeA{95Xl; z@Zgcc4Tqa?JXvLm7$D;rb_~u&Lku28$QKq)Bf$XSMuJ1!5Z_8-*Z@+IPhcz42cJYH zI7r$fl^fvPBOc$fpL0jHYQfiZErX-S&M-&LoM6&K4198#2Hzml7Yultm>WL7pK*ua zTR)mP7{d1*N^-nOBd5KgF)olN>?wEXDt0%0oFLlpvguBT+}stn-%sj=ZAa3oLNyST zU`Mb^AY*u2UyxZxy1^iPyqg%0I}iwl)+lsUxB2|$kTCOmQunoaLga#!^N3MOGZJyz+f;%-w2nA#~#+pdpO?}FVRZaP;mU=@B^}H z6p4qtp)_4t0UgsEWX9bi{@2<=@p5M@(*LlnuyXEz6}E`-LY|8pU0qlyT-1O|ByzwIdn5z@6PbB2b(tFjxdLtz*WrJM8M(JpN7$*mZe{xF~; zqkeA)rVBsjL3?3v%Wx{1gISyt=W(A2erj<#Ltp`rEp5# zQYqTY;|UP=j?9xj5SL~&m>w2ba4r;v>jBy-_~ZW(PN}!4mljH2x}>XK&{hBXW-YJsWsp4x!WUHyBD=R z$>Oq?3d&@i(oSmsR=;GlFBt7HL&CW4_K9SPb=omqF>`3fH*--iZ@=BWEK^_u6Pg+J znXD1&S<0u*O`e;&G;0bl>wkl+N!fW zohwFaQ^WMdm?AbFyEa=QRJ7meSvHkVRZdn;`(wuzO*`X!(pWsDoK#M$WBVtRi^kn3 z%q|#3UGe?yyWN8A&=<9TuzzO%L(^wX^S6XcR}+VQ!l7ZI{Mw?{zg%28wRdvw)WOMv zGn&|?L~$$VO=q}&{O<9IOV6~muZC#o{(@=8>MPJ!Is&@bvtZi2`U?7cKrpqEh*K~% zuf8%v%Ytd=chJi`Cf_~RlzdA`FKS94{>5(diLKJ#DKY-L3d4!L(%)?*@peO( zO!|8r#(!T^++~#he%BU=|G`$$RVn?0^N6caV6L#g7wJOZnDnGGH`DF`< z@0EAa$}bOM{E1B7rBgmpD=}`AcUhHBN=dv@-nCizq=v*B%DOtFPdcQ)^DBwEyHWa; z#?rk-`qdUW#68f7?1P{O?m;hp0q2S-G%etre%3P{SwOPzAD~$oGcpP&*LnyRXInpi zJwsD8g3i3Hb=hCH^>KL}6+#4vJaBN^9i>CKrIBYC(U^iKlr>{N&FF1i<5hu1M7|*n zM?86y{*WG*r{8%;JZ0E%3_%TdV7;@5h?oF67V>gocfnbJf9;bAr>W7=HvFa|YK2G( zhbJHo&j`bOFo0i|Hv26n;Bh1T#ch|JBi@i3U{1FaKbBGZz$a1yily`(_b42?ysk8A zN07Jh#UI#yAjo>ZZ_9mQTlxcxpHBSx<_?Ulj6=;KmcXI@3G@emUWoIJa)&Tu8wT4k z*nz=r2mo1Bq(5+6HzavPMly&Ew+o3dh+xTM2mqFm**^_lWj}ITNaA9B#IjZZ3BJ-D%16RKcwEj8H)d7IkOop zs|-sj>w?Pq>xP7?Zb{`>P&tIAV{;$Od4#Q9i>mItT?s#;|GE7zO?^gV_&KdU+#&tE zsPS;C^z&9Z#MeN32$qr+|33jc{Sjc1G$LJZ4X=d=BNCYJdB$B1G+GaR7T5o979Ii_ zGfy7yBg+Cm2tz6&GHe3?8G}eWY!qX;cjOI#Om6kA2OJHT<+zBkNE&)l!&=TjYiyAi z12bL`%ECvih5!N?5GJjUh$0_|fm;svC)NgFCJSTzXt|8vLoQRHT&(#cYR#7%$h8w^Tx8)$75E?Y(8D0mH zV=IWfIwE@|fZ_lULH(-imr)r}t_S^ak@fS3YsA)a@9fH`%jJnNu`H^Js5+_3CjpRG z|Fk*8S|e%#Ur9tAY&c?@^7_tTt&bFM9<_V@GU^BDvt}KA7DbBI)#v5)^WUV;sOBeX zs#csg8)~R7qQN=!X07~0DeB)UMgAP!{I-=fK?kZ}b#2M4uClj`mpyL|G^{gT+K6`D z@%o4L^WQjLx`>YZ~V&QEX5mxp!xS68hf%TEFDp z&o{{52sWPEfc_U^eyCpDsR;zE2mP!E@$ZP}Q9q#Do#Kcf0L~hs`VaNva6-U$o_(;( z-43Qk5h5%ZER$j3$PO2h8N3tk*AaB$M#$wd5r|6cj z9`B^HO*0$=V;I&WNsB1Ai>rmq0S-UOVWTSz*%F-18M+zbwqUkNw2QKUE{7+>ung@p z0^|0Di(H7&Itfs#73mSv4Xd&MZjtx87w}MI9Jb`sr1Oy*UXnKt`n-O&nfKytJM_dC zxE}Oh_p&Fi3_}L7k#ZcTo88x z)wTW3>XKNIVGZlF}2~6S_kl0YuDSuj?WLX|Na@7Z$)=Q-#ATaOV@P%r?g-ZG#Na3f$010Waud}BeBf3lRg}w0 zsk8BvhRnhYjhZCV4ZvV$a+MIpyfmVKW+-lFc~1fUofRDS6V46ve+Pg30zheif0u8F ze{Y$I&IIHA@n}3a_r5T2Svd2)V010Ye!el(T|3h@(-Jqu+u|*Arn&udje@za5b$1+ z4bp&d=uE$=ow85brU^b%qNBZfkoYW!ubo!+KQyDG1eRFn5~)Z zo$Z*fpC6yUCX`=DmRbPN*69J()t{wbYALTPS(_d;;lANyp5#= zs|Or~YMTJsuGsw0`M_Bqvs*A;$(Ol8f2SKou2?eEJT=rLEA2~_&V@?nQsu6N%3X=d zJrnBXQtPj4AKD+-A2vN`nzXu^4f>`2ZaOR5sYkpXRP`=C$KZtCBvqthD|WKOSY{GwyjIHZ40(- z3EPed<)4ae1xlaQC903jOBby@!l{0u=+@c2vksxYGheg4^vfd3R-Hduv-(8!yYt11 z)?T6iG^w~dJ`_JcdsV1^H@|A%>$(uOy_2vUn%lW(dUt+1QO1t={&*wI*Y>kGYtI3) z4zIHkFTT8zFutm8Neb>=j^k`r8}4iFY6MHioMtvEl=LpD`jQ5lpt8L>M3&-$?!DDl zpvY77g6`z%D_BYIK&)f+)i#_j3s%qSt7;suh0-o81lb9>pTgVN92cSHQQ zJInjk(l0bSApS*LX}?AKJM$44B<3w;{dMyB=0mNJcw#s8Z<9XRhg4r}E$wfUe${bU z0*Qa2O{dk;e=%dhzsk&~OQrv6!MLEZoUV}y)m0cjDDPJ&7ibm47u53pV&#IK#4YlE zM!8Ty;&t-=X7xe?iEoqlw<;HQlX#o_v_kn*qJ+9n)$-G3tZUqO;i%qVgT+z)fd;8g0%!EUMXnM#pBI?4sHnauO+uXd>Fj z1`$z<0+PYg_qyMr2gLLXo`~*KF#Pa=`qws?8Afy{VoiuW0fHbGYXG{8Hznlx-dMMa z@G2%amQ)<&sQxV@pkF%zdDQE8>gW0d2<8NglHo_lsf%P$^r-Pi)Ys?LH@&6)Iv8!l zC~BFrhR`TXYq?&G<=zov#01(Y5$6I;-Mf`n11-mb2WyC$2kQSWL#H^O41Q6-J%YR1 zA3mlEwp8R7HRp|uv7li{M$82x`NLZ?vzBaY#UY2-Mx>Z66Z7HLq=YSJt^U$zX{2pU551ruBEw%1ChmDy2vx-_f!V+8N{-Mys{p ziIlBz+$@ij52ImI@I(}E85=Z#3Z6*zzHE)l=lAPRO{A=#KV`3@Z1;Yd#iq82jRQ7S z9kqa#&{`{a*s6$SEmy#$tWnE{mV49bcMg{-bkV%`Ub-zut$U&nuOf)cjjP@O<4*e` z4g!3|H<4e4_yQjP7+ey=H8kh*@CbHhU9eN2?Ov2<4GDF=*|~HSai|WlbV)gu`-1J<<3YM@Yp0Xos>&J72l|jwjs>oPS6d(@BL?z$*fo+P;6s5GpIcn*b1#u&FmQ!+19!W#~H^wxod$$Us~$!w2Avtx%I| z#Gna+u;U1L402wWzdQ()6_V&Ur*ec&992II6G4}_943;I0zgDyf|P=EmIq%$m^A2* zC-?FAQt(&NEiX|&j7q!;0Ei;bSnp`a2?p{ zI}oH)SKPc82h3sR4r7KXL!DbTX(vxFR>|Tk=&k4r5YIsIhcu`Rl95Gg(n*!L5(=h3 z$|M$~y#%py=q*cDjSFTjijSExbX0+UlY*2YU6Rs>aw3UVOrS3$fql3!l(!pul=WLg z#}}>(vUofd;u>G~Nkcf<-~W()1p~x`QCt+*yQ*>!yE%7$F>2IGqGjbI;|1v z4hqJ$MOpiX=#9xTF+Oo^#v0onYZMH7ZXXw&5LGkq(=YTdDQhAwcV$A;p}CHP<;?Aq z--2Ug`Et4Kq4I(9q4t4x=4V37p+x!NJ391H1XFRMH`e(~wf%W{!=2;F%KC>r4|-yj zxHVCE;7;#r>2ikfxEOaN$`9T-@toNWo`Cq@R`*gVBZ$YhdO$pWbNbP_kIy}N|Ks7kjC8CU#Jd?fA?Ixh*8mluuiFK=mhH1zS%Bmc+#c-!1%@P|Az zAiQ@)81M*(*~AvFV60mp_=b-}iOp{)x_INS=Fe?%|~e zmxQecW(N{ghbNA$$!wbqC8|0h6MQspOx~D^PDW?WE|xVdmsdYDJTSx*am5__b$R!5 zNTZg{#?C$~-}SAIs%l-?Oj#X&*+Z$UD-w-*@3O5ji+0Z|A(Q9#96Jwug zcRsJEdbsz&-iHSt9E@p%-NzFZC(!?~bizM#WYN?RJM@jQB55p{QctP{TjP@Lz=G{S z!qztX!Mtfgy=d%yC8x}e74T@xxI#9@dSdNDdF!I~z)BHSwrS$RbVw+!zumK}Hr}ti zTM2xREvg%1d{U*kue__AQ2nO%k^N))Qtgg~+8yzIiQ0pAl#8l1Kw^rTz{OIoNxRP; znC+Qu7mQttvhJj&Opukms)B|EQ^R)vZWyiKQnY%n=tNtwJzf(p7OcCMw0jn`d%jzN z-=OV9FrcVOP&cmfZ7|j69f#_A<fR>lAN3`@cIhAOa*Vggd-o~PHJa=euJEqS4Q?KOtpM8%K;X>g2D0;{qkg#H@aLrg z)8*YmKnQ+Za#EO5q_4PATJKG8U-o;+nhY2B;B`*HHcTIL(D-XVA4ue;+@h?khVB7S0jsydPzM+)tMBlNK z+<(DBEd-xX2Wz3tq6D1se>;_pHLN~Y--3W_fJhJyd5Zz%mYl=d4+ z2a!KhRWD>p+WxIsK_8`G7%$Uw$+sU$D(R9J2g4A5aZ}nu)8#AgODLK4PWY+R`fswb z+wU&P$`)j0Nqy;k|6PC5a1BhiP89FE-Sa1zTF`Eq(N2FT6gi*CT2_iF JnT?40{{fQGZXN&t literal 0 HcmV?d00001 diff --git a/test/stress-scale-tests/utils/__pycache__/metrics_collector.cpython-312.pyc b/test/stress-scale-tests/utils/__pycache__/metrics_collector.cpython-312.pyc index 21968d08f381549ea6040a684662da977ce4483f..ba51e3a5f99500639667e7e0aceb72b330fa3b85 100644 GIT binary patch delta 8118 zcmd5>c~BfzdY|sOdJYaT+{`cy0>cO-4uK?u1QJ3Lx&R3o*~s!}m~PNWFhhCXs3A2+zo8n^a8^$91~sJRwD6|mbU^i-jnhLN18ofa8aM;g8o6@L2yZ6N z1aD@pob00Pl%2dnmDh5GP*ubg0WIc=ftGM3Kuft&pk-VcQ0uhX!IIageiH*DQN!Pp zKc>k?)NeutD9cYEKT*0(#cRG{8>hRW4m_u9=<9U)bri4F%js{>X@ipY2YAQ?Z6If8 zEB@{JD*YUDLmN~%O$ptBl0@lzDY@*hR9F-5i6sQ?oJ8HU!C?&l4_!q&-&e;p0`KBt z+KYlGz{i+zKH%b90oVJqzyiBa0mPw=X*|9Oe@s8-3h-X{bj%nyC-^2u&rM7Q1PgG) zEaSY(KPm9zyf@&y=<#yCi~e3mE-BXri!@^%e;}4S;u+(e6TC3u6UJR$H&1?`eSvb2 zMeQ!Snj&9l-wNfaN~+%LjuboY!|%yR$)@|(nmd|^b@L;=rXc&N`AeFMGc-Tas>`yU z8c}Lrq_`H94n#^Es8k)X)`QXtY_4j!6NqeVdE_t_R2b(pcNyE$E#W`WO;c1Y(K6N4 zc2dXmQ%A!=CQMViNTaE^x)xVv8_rvQ(It32-cg|wtM&kK(86{scad>Z2Q?JFW7p4|?a zcX9%@iFYt*yocn534tH+ToMN5mvMgFCrromoX3B`>G6TOFE$iXdq_dvX6kZyU*5O$ z^*9e@5S3Wki*iQ)PU5I3oOe}=>gM46uE??liV&H zqppXG%U;s7EAvH7I{&mkz>lw+aZKa(O-u_o7l%4+4!z{Iy-#-^s{MaJerjqAH?lWr zs+~xcg-!ArZ^q90P)sL_#jDp$bc^Ew05P~3$KW+XkNc;i{!BPjZBk;@H5;qPz! zYpOD-CBDuv!<27qa-4U1I7POHj8vbb^;Gz0)y;HEJBatfrpjVl;3d(o!RzCAr`I*k z3v<|8dcT#mC+H1Sc&c^>z4aJ2;%*41fb_y_QHwkUS!rUs|18Bv9&YNTs=|$Rgvyfp zZGbl9BVv7lPVT6O{8#-ebSp)ER&5RQn}1AI{1ADx%2MKhV$8&pTNz(mqPAgx>I?@P zKA?2+pa^MbDzu{Rw9&*jgJA7Xh^wiO>J9(2=?P7pBY)YlO%dEn9=05%0%S{TS!fEo z%JFz%5vO?0=^H-Jy92^)6mMLmSw@y{2kSmS@*$FMBS|7KzO=$Qto=hEF&#ez9>pKi z%D=)}Q0LbHVdY22RgtCEz0~Qjaa#$Uh5aQ~^Dju_j$uj&lN|$8*56@6Vrj^ewn-`& z_U!y6t;5X~yrg6IaYlxe0hajc?xx~@hWeNZe4*fpFX2BT=P$^E-KVKPBD*^7ZPY?V z;$fg5N3uVMcy(13gs?hA2`ss6E~L+^iOXTxN(brdDo*{bI^mBg)wCw4;j|aJL9s5V z4r*q=o6l;!G^d>f-+o?$ay2@1>_J+l)Ft|oQc}-ksZ?&Yo5r0@@G|;1FL|ZhmK)Tj zdP;E6ObNCblF^6UEH5DcVPW!uG>pY?BW-lb;7vG)cB4FaQ&0<2FbApYs&{oM<AU zlExx5qdtVK5?( zVs-2MsfCjZ+YT&NFPRo+#nKn=F@r)5%-XF1l6aCH;Xc%4^av%HJwn{>8Eu*^lg=!& zVvYvc9Cfle>Vs-9M<#sbXkQrk_37}i&m&f?NHY2>3fj!~ZKSFsK&pB(x;K%jHu|ZS- z2wM@trHJ8G5RMDYY|ez@c;hb-$g`jXfSmiafSiEqHIKy9-YS=v9Sda=vp>olm6)ST z&b!Q+2aGv|r~?T|4K3z~o&93}pm=In9CVAN+&zX@P?a&x(9eo9otaUUh%@G(n$uyN zQE29vvnr05)8j}Q2=_8F2n}*fFlGRu$<>sM38~Y<^FlGS7K)Bl@Ee{E@O~%ku-8Ar z3j)uj#|OEQO>0~wy4KyU4-P09LcJM7!$t%k0{XMEwLt(QTRe~{e&3@4ObRccgbevC z2tJ8(6QJJ69_wmQkT3i@FeTC-!1!<+rg{MVsBB{kES z$pBLA56}jF41-EK^JdA^{0wK#i|oi<^XO`4Zf~4#k?h+Ssw8`d*mXiGKk;VYgL3;^ zx-;H?ZA@b7ZuLq``@#|M#ZjhDV*12>u#~f_EG1zmXBIoft^sl2jQFZYJab+wy>O2i zBd?mvvOCgEq9QHzD$8IJ9@RiHZPIHLS4&*w#i1ZIs=AFK8=hHpQ0>=oa3cc$;38o_ zmCTz|!DOa1s|h4e;X%!97=)DC6RwQldqe*tUCO*jF0ggy9HHZRfYemvFsx0J# z&e0M1`J{97!-EfJ@<05v)1glb86i0(@&h-VJAFcc=Y$S;5$b_BAh*YIDu5W2$Lehr&xxXlqRr+OXg^fF=wz@}tBXo7*)OQr11LW*Toh66-&+<(Pb$ z@bB}Djll`YCu{+MXY>CK?eAZq8?e9p=Z!%jOTpZs2l>U(e20|pxCNwT{#7ZzJDT4k z<@YR=iGwdi@?V}aJj^M6P*k=o|A%}hKYK-`$;y73t+H0$bVSPPMW*<^IY-Q8Bj$=Z z>f^kEcU;#tUKu6zbNc)8OVyQgWb>LA$gwK&t8N~YEcKAct@?#Na*fc3!6=*R&tY2M zAScqtbE3?KN=<7rnP-}DCQTLt-~f1ASMw@saZSNJ!YH49g7HuAZqJCv4F{G6IJ2-W zZ~QP|K%C2`0dm<`_03JDF0?nj*D2T7B#&XF*XQL~&j{=FWj59|cGD}T@$auY9muAj z_D_z>cCbE&GI>If(mUmD{c94a!W2H^E2HH9HD3bcbVeJvQUBOiA-HeNdB9Tips+Mr zSSJ3z5oZ zDW_Rvn*VkkgQa?bHQ5KCq{!b8}_ zUL}ut48`?0+K}oZ1*XudbbLLUQZh4}p951V_GU!ZL^`rE<|ZNKq?t%g09jTxYHgCN zP4o4VwR16Bvi3!-1Cn(>Jn0l&?ueBWnUaS&tb#4kQWeRmzM~fFcSmYFq?`_s=}=N~ z*wo~MxA*eZ&n=)=RUG9}d8k<`06gUi)9pC#EX~o~sxxQh!~(B{<_ikQ=}rq-Y}V01 zxFsrQvlA%-1y?>P4u(@w6qzTK;(2RW1AL(Ke?g}PE_pC4t$FZ@DUr}TlSB8F9Qp(Y z7ou;dK~Xa2N%gm;tIQn9uHXv>4SJGU(lhm};n<0c&{a~=GnJ&qtPEMcP}3Xad`U}w z-(HfV;f#rSffl)4r?ivL+eMaVkh+N4p z!0pMyEvNG3jsH1q!Kz}ps#K;{nU<5+4fzabzF~k6l|?@8bd^zLSonfd zR7&_c)31wK<_<2Ki=yUo$y|Od5Vgbcqh)?jvhP|rCE52!?T01%;fVd{UGuT^Y%fdp z&PBas?}^%vN%mtAd*3~C|B6;s-1>x3Swl5*M&s#CdY#ovns}Dr112S8uWb0fsErm`+Y%j{{yCk-2v0dyRim<0azcryB-J00k zEm`+3j!4#HQR{KZdR#nlMs%KySY5KNX35qpZrv~04lG`jY<*GNfMio#-*Cj{mbJG^ zY^%7fM`8~xZIaj*qwLEP`?5I1SBsF6&+nqLMRV27(~`MqzAj?!SW&C2 zH7kwe^F7_0mf0PTEJjTCj0U6ZFCA2`gF4`#PC6*~5XFzUCdUE}o6t%|_SS@UAXxSx zIfCRkl08Vii{uKD?<2W}uOF%lWQDKKjR zOG37ae6qJJ_fMhy4b?}Qw#S;?+U6&(k&>=6LywPAto9W3xSPt+j!=&)3$^3a*pbA zkPTdtWQo%#ZTjvV}`l*~%rGT!mq%gKanpJM?*-Z#yo*FrKuU z#qEF(;J02k<)>%LQ+$^*S5=Zeh2m&Vh}lcJyyP>gZ|HZ5x*?wf9mY9aGB(1;_=7Pl zUJ+0-od=s*ZO#MK;ew?R{J3HnhECOHRYhpW8}_QkIv$Jd<|r6njI&fCb)0pPD!}0r znV-fXd|;2jel_d}i&771!g#8py0wFB3~Vz5>!UexFG%rBMoGy&{YC_u=*QKH{cu#Z zRsJo)Aevo=9;?@EXEJg;r}#`|w!PX!hAGb_USflqkW`XsmM<9PMD|21k>=qpxI7TR zt1dqz)LX8TMmMcPE23aZD!Dk18fQTqtEq-Ae7U9zym-B)lXv0wn#ZjvvP~1ZFi<-U zXVvA}qxzoW^0b86d^V>f|F^(`elnfOvwCjh)0LInj{)yt>07SOzJU82t?G(*Q9RVf zWo9GLt}z$K5O>phypuEsIed~9r{G9u6c*1V(y3T3&xh?~pgBiyxp5qBtNLAQ;N5iI zXm7zho1UIYDzSuIbcx`-rl-U^f;!OLE_PB0Lwqkax5lz`=2Ti%VySpiVW%lmdAe5* zy6|R8MEpxoVRuOEI6wxp5$p*LS^`DVs@DRScKYJKb~x=H;^%M0kI<~;-o@uiB|GDj zlS(!l%cK*jTv_3>dq<&Gy|wpiXn32<4aE*qeFalycCuKmR%ry)wh?GjjqTSUq}BI2 zS=klF(kzzb(c{Mzs+97URl2chWX!JMA03AwpoTlwMHs`@U_jC$1#vn!2vhiFusStO zQJsktZ}O}n$I=s1%4Ci$k$cmBBC{Zg%Y+XJR|$E-=E2<9X*NdMIgWy)%qpo|wqVfS z>?~5}pAJRffVvRcCfX@Qsk!gs%A*qyRd@7^fc+Xxl-h%*!ZRSNe};b)CGE(`xZ3-q zNjqF-=0|T|Th&cUXXPITOKh}pg=^SXc%tuFNaOc?KLg;R+a-O$)t3dVbb9d8!ya6G z#shPB$Lhw429G!=;P4i=PMES`UzHob+47J{hWAY){I7v8HShszFkFrO&bVj)Az{>G{)4fV_h2^eb#_{nQ z-B#9ur^lN{AE9X7gnfho0$pEhp70vsJmF2kTZBu5cL?thJ|KKTxK8+-uuAwz!1{Q7 z1>F+!!fk!`eSM>$eSHXrd()kKUIePg+|Q4_vojnqINaJdSYrs25&+aBs!{KJgT#+tQkc$VG4MDOG$G zU()Y>!G8{%;EVe%@qZ(S_(#FcbBbNoiN*}uqGA^<=$1wdn~Jba3M)Eg;Ihd>Ptr@m z5Lbj5*cORS9g|Utd-7Rjy1NS&Z+V{W^@Ka_k39_?eQ@!sv8Tr_{18n%3wYWNRi{H; zax?)X(t}X~W$G})B)!$Ow&Rj?vifk(Xa4a%^j|=KaBBl0-6Y*$NJ=7LSVi4pnPixp zM&23ZwLpGL%qZz3&W95n{TI*&AkFZPk+X*O#H%;21(vG(DA5rwmi470UFI zIx`8L7tzy}hgz$7kVlzTDuWp-Yc3A^-)^DHW|+ zp#aatvTaz50*4K0OkZ+Lsxx3IcYK%>!(!y-I#{F|}a$Q4~Vo<|^@>m5Ox@{B~ zl)xrTFAD#vUZKJV`bip?FgLXU_Af!HNfy7Y<*0@&9gqtA+le?|NwiB}R{uUW)w1UrklkI)?vPbJKk~wnGKPTOXw0D^=vb?kebXWo zqvIB?FEgAX4u3H6-eD|PCK(j60mDEEiX_(&ZX>*jkmlJ_vC!MNL>**~-#9fND8kb1 z<4-3JwBiUf&giezKTh43V`A^liD(p+-yJ)$xQa0_@0#YRx@a%Fi|Ed8Frrqh5F7W| zt=3&?eZUlg3ds!wf>4vjaOLnl<8f(?50A$uP#eMKX2=_Wjd^M+5sFrB(tnb#0HwAu^>g#I?6fHzAn?^yS1oiFI zYw_?z=Xz6st4uoepCX*Px(?hloVAM-&M(wqRCU8SoELQ-b_Lc=jJT~Z>96;0N85mq zY^nc?c1WhI(JONF(?(X?lSf;?rfyv%f`la)x?yRc`p<;vd$ra$ROos~71yh(@PrdB zlMj*lobgHS`drA+Kqsqw}w)`UA@t_Xt+HZCST;ViX$fIaiHdI3X3| zxJm6daTQl92n7V3MuLqZO9&WY?L)I|~lq949CpI7&4CH5jv_-L0|}W~s^(c9k@M*(t|3 z?x!YL+jQQn*(&en&6>R;-vTRMDtWgV;xI!TCc+t!RAh_=WFwY}Zeui#i(KB!{?vmj zI)onv1QW+#!YT`wbq1<)U$D>le%5L6vlDd#?6bNKCRx3kJ*p;Z(p*MSC2dBX6%{!- zuRAv=+4}=cqV5uBXC+mgNvGtbrs~dgD!%Srvyf6U32A-o#A(EnvLb8Q+4a-`Qw!Yv za6cOX_z6c~1M=L7Mo1E2?#|}pTqTw7k?VwT?>0l^|n}Q+Td`8Num`OpPHSo}@hUcZEbS_Ota(mb})~!pyVI>6tsUyL4%Pw$K+W`K? zeRqMo?~W8hV=JDkp6hMb+LonrOxB z6<`s%1t4IeTj8OTGaKp9F3`Q0$bPV@UVsvjmhheNm9xurA9#P`y%Bxb+I?sId%7LW zx($;;MJ2%NW-*!7Etoc)mo+J&>JC|zld$E~a^~k=xY|L*Zg5{2(;;l{@ml6fwC7Pn$S*YoW{L<2`#Iayo#ltZ6l)$qF4T zOjY+05-q&#`p~kp7wsp|ZYw82CyR79C<0IwaxyL@wOF8sU&1PlSl^~M#kw*G)f4tHoH~tCo}9p{UkY6@O!?owa%g? z4v?#!Q8S5zNV7rhoJQx_Cf`5me}o3dF-6qoQZz9n&^>HZTazCN=A2{4riHOoCaHmb z5H&SNq(q8lg{hP#Dxe%L&`3DyLQoJA2)KE4hIu+(BUcKEj-T0iiszo`T?ffowq^Ur z^%OnY0dnn*myeE5ABnOjT}{mFkDOG|)Qb=TfHsh#niQum;lK=ikiHHQY`<|P=%wnRK!q9)SwprLOdEV0}EhPGdVsNx*LhRbMXG{c_wd&%boci@1% z?ku?Es-CAg&*pcAx_%B?>`HP*lF-d5d=)Br{uO#5Gb*vmeG_DX{iUy)d|R;gFY`Rb_QUQDbk*h2 z1A{ae9-wcqzYlbTE`TCeE0n$;Lh1YApcplM6X7icBivu0bcqcPwv)HnD}(K$7%#>B ziP3ie|05jTA*L(c$;dOkif>#)C?%PFGT3QdgmG>Md(zj;z8!2JodxHRZsutVsM$Y% zyzTY%xOF>~(uzDQsuaH{6&ffm;T(>MvUy0z*(Z&R!wFbtk#j^ckhapHDU->K2&XUO z1kbrZbN%+149*mBq{|{s+Mz5E$yt(UQhNGcI{Kl zFB_YI^m3*lN6sV&@Pw1czpw%VRX>QL)Qf(L%VYPaeE{scQ9sGD&0{;r@xqa@e)5uW za*e*0LZ=);24MjK^H0wqWC3on+YzsC6r_vXBg@`r78^gjR`2A!Yu-(K>)I$`e~k=G zpGJ41aT>kVXw-*M_Zon1ON-hmMLtRKSB@S+IF7(0`~ks@Py?XbhZCuIMv+G7U3NAa Otexgb_Zo+2IQ list of read IOPS measurements + "write": defaultdict(list), # PV/PVC name -> list of write IOPS measurements + "metadata": defaultdict(list) # PV/PVC name -> list of metadata IOPS measurements + }, + "latency": { + "read": defaultdict(list), # PV/PVC name -> list of read latency measurements + "write": defaultdict(list), # PV/PVC name -> list of write latency measurements + "metadata": defaultdict(list) # PV/PVC name -> list of metadata latency measurements + }, + "throughput": { + "read": defaultdict(list), # PV/PVC name -> list of read throughput measurements + "write": defaultdict(list), # PV/PVC name -> list of write throughput measurements + }, + "operation_counts": { + "read": defaultdict(int), # PV/PVC name -> count of read operations + "write": defaultdict(int), # PV/PVC name -> count of write operations + "metadata": defaultdict(int) # PV/PVC name -> count of metadata operations + }, + "measurement_windows": defaultdict(list) # PV/PVC name -> list of measurement timestamps + } + self.logger = logging.getLogger(__name__) def start_operation(self, name=None): @@ -402,12 +426,200 @@ def parse_prometheus_metrics(self, metrics_text): return parsed_metrics + def track_file_operation_iops(self, pv_pvc_name, operation_type, operations_count, duration_seconds): + """Track IOPS for a file operation + + Args: + pv_pvc_name: Name of the PV or PVC + operation_type: Type of operation ('read', 'write', 'metadata') + operations_count: Number of operations performed + duration_seconds: Duration of the measurement in seconds + """ + if operation_type not in ['read', 'write', 'metadata']: + self.logger.warning(f"Invalid operation type: {operation_type}. Must be 'read', 'write', or 'metadata'") + return + + # Calculate IOPS + if duration_seconds > 0: + iops = operations_count / duration_seconds + else: + iops = 0 + + # Update the operation counts + self.file_performance["operation_counts"][operation_type][pv_pvc_name] += operations_count + + # Record the IOPS measurement + self.file_performance["iops"][operation_type][pv_pvc_name].append(iops) + + # Record measurement timestamp + self.file_performance["measurement_windows"][pv_pvc_name].append({ + "timestamp": time.time(), + "operation_type": operation_type, + "duration_seconds": duration_seconds + }) + + self.logger.debug(f"Recorded {operation_type} IOPS: {iops} for {pv_pvc_name}") + + def track_file_operation_latency(self, pv_pvc_name, operation_type, latency_seconds): + """Track latency for a file operation + + Args: + pv_pvc_name: Name of the PV or PVC + operation_type: Type of operation ('read', 'write', 'metadata') + latency_seconds: Latency of the operation in seconds + """ + if operation_type not in ['read', 'write', 'metadata']: + self.logger.warning(f"Invalid operation type: {operation_type}. Must be 'read', 'write', or 'metadata'") + return + + # Record the latency measurement + self.file_performance["latency"][operation_type][pv_pvc_name].append(latency_seconds) + + self.logger.debug(f"Recorded {operation_type} latency: {latency_seconds}s for {pv_pvc_name}") + + def track_file_operation_throughput(self, pv_pvc_name, operation_type, bytes_transferred, duration_seconds): + """Track throughput for a file operation + + Args: + pv_pvc_name: Name of the PV or PVC + operation_type: Type of operation ('read', 'write') + bytes_transferred: Number of bytes transferred + duration_seconds: Duration of the transfer in seconds + """ + if operation_type not in ['read', 'write']: + self.logger.warning(f"Invalid operation type: {operation_type}. Must be 'read' or 'write'") + return + + # Calculate throughput (MB/s) + if duration_seconds > 0: + throughput_mbps = (bytes_transferred / 1024 / 1024) / duration_seconds + else: + throughput_mbps = 0 + + # Record the throughput measurement + self.file_performance["throughput"][operation_type][pv_pvc_name].append(throughput_mbps) + + self.logger.debug(f"Recorded {operation_type} throughput: {throughput_mbps} MB/s for {pv_pvc_name}") + + def calculate_latency_percentiles(self, pv_pvc_name, operation_type): + """Calculate percentiles for latency measurements + + Args: + pv_pvc_name: Name of the PV or PVC + operation_type: Type of operation ('read', 'write', 'metadata') + + Returns: + Dictionary with p50, p95, p99 percentiles + """ + if operation_type not in ['read', 'write', 'metadata']: + self.logger.warning(f"Invalid operation type: {operation_type}. Must be 'read', 'write', or 'metadata'") + return {} + + latencies = self.file_performance["latency"][operation_type].get(pv_pvc_name, []) + + if not latencies: + return { + "p50": None, + "p95": None, + "p99": None + } + + # Sort latencies for percentile calculation + sorted_latencies = sorted(latencies) + n = len(sorted_latencies) + + p50_idx = int(n * 0.5) + p95_idx = int(n * 0.95) + p99_idx = int(n * 0.99) + + return { + "p50": sorted_latencies[p50_idx], + "p95": sorted_latencies[p95_idx], + "p99": sorted_latencies[p99_idx] + } + + def calculate_average_iops(self, pv_pvc_name, operation_type): + """Calculate average IOPS for a specific PV/PVC and operation type + + Args: + pv_pvc_name: Name of the PV or PVC + operation_type: Type of operation ('read', 'write', 'metadata') + + Returns: + Average IOPS value or None if no measurements + """ + if operation_type not in ['read', 'write', 'metadata']: + self.logger.warning(f"Invalid operation type: {operation_type}. Must be 'read', 'write', or 'metadata'") + return None + + iops_values = self.file_performance["iops"][operation_type].get(pv_pvc_name, []) + + if not iops_values: + return None + + return sum(iops_values) / len(iops_values) + + def calculate_average_throughput(self, pv_pvc_name, operation_type): + """Calculate average throughput for a specific PV/PVC and operation type + + Args: + pv_pvc_name: Name of the PV or PVC + operation_type: Type of operation ('read', 'write') + + Returns: + Average throughput in MB/s or None if no measurements + """ + if operation_type not in ['read', 'write']: + self.logger.warning(f"Invalid operation type: {operation_type}. Must be 'read' or 'write'") + return None + + throughput_values = self.file_performance["throughput"][operation_type].get(pv_pvc_name, []) + + if not throughput_values: + return None + + return sum(throughput_values) / len(throughput_values) + def get_all_metrics(self): """Get all collected metrics Returns: Dictionary of all metrics """ + # Calculate summarized file performance metrics + file_performance_summary = { + "by_volume": {} + } + + # Collect all unique PV/PVC names + all_volumes = set() + for op_type in ['read', 'write', 'metadata']: + all_volumes.update(self.file_performance["iops"][op_type].keys()) + all_volumes.update(self.file_performance["latency"][op_type].keys()) + + for pv_pvc_name in all_volumes: + file_performance_summary["by_volume"][pv_pvc_name] = { + "iops": { + "read": self.calculate_average_iops(pv_pvc_name, "read"), + "write": self.calculate_average_iops(pv_pvc_name, "write"), + "metadata": self.calculate_average_iops(pv_pvc_name, "metadata") + }, + "throughput": { + "read": self.calculate_average_throughput(pv_pvc_name, "read"), + "write": self.calculate_average_throughput(pv_pvc_name, "write") + }, + "latency": { + "read": self.calculate_latency_percentiles(pv_pvc_name, "read"), + "write": self.calculate_latency_percentiles(pv_pvc_name, "write"), + "metadata": self.calculate_latency_percentiles(pv_pvc_name, "metadata") + }, + "operation_counts": { + "read": self.file_performance["operation_counts"]["read"].get(pv_pvc_name, 0), + "write": self.file_performance["operation_counts"]["write"].get(pv_pvc_name, 0), + "metadata": self.file_performance["operation_counts"]["metadata"].get(pv_pvc_name, 0) + } + } + return { "operations": self.operations, "system": self.system_metrics, @@ -416,5 +628,6 @@ def get_all_metrics(self): "controller": self.controller_metrics, "node": self.node_metrics, "efs": self.efs_metrics, - "k8s_events": self.k8s_events + "k8s_events": self.k8s_events, + "file_performance": file_performance_summary } diff --git a/test/stress-scale-tests/utils/report_generator.py b/test/stress-scale-tests/utils/report_generator.py index a7435575a..b93b098a5 100644 --- a/test/stress-scale-tests/utils/report_generator.py +++ b/test/stress-scale-tests/utils/report_generator.py @@ -35,36 +35,19 @@ def _get_output_dir(self, test_type): def _determine_test_type(self, test_results): """Determine the test type from the results structure - + Args: test_results: Dictionary containing test results - + Returns: Test type string """ - # Check for specific test keys - if "statefulset_scaling" in test_results: - return "statefulset" - elif "access_point_scaling" in test_results: - return "access_points" - elif "pod_scaling" in test_results or "volume_scaling" in test_results: - return "scalability" - elif any(key.endswith("_io") for key in test_results.keys()): - return "stress" - - # Check nested dictionaries - for key, value in test_results.items(): - if isinstance(value, dict): - if "statefulset_scaling" in value: - return "statefulset" - elif "access_point_scaling" in value: - return "access_points" - elif "pod_scaling" in value or "volume_scaling" in value: - return "scalability" - elif any(k.endswith("_io") for k in value.keys()): - return "stress" - - return "general" + # Check for orchestrator results (operations, scenarios, etc.) + if "operations" in test_results and "scenarios" in test_results: + return "orchestrator" + + # Default to orchestrator if nothing else matches + return "orchestrator" def _collect_system_info(self): """Collect system information for the report @@ -349,4 +332,4 @@ def _write_generic_results(self, file, results): else: file.write(f"{key}: {value}\n") else: - file.write(f"{result}\n") \ No newline at end of file + file.write(f"{result}\n") From 6c26919b4f865b780765e126e9b21d430dded8aa Mon Sep 17 00:00:00 2001 From: Ishaan Bhardwaj Date: Tue, 17 Jun 2025 17:16:55 +0000 Subject: [PATCH 06/13] Refactor testing framework: Modularize code for better maintainability --- .../aws_efs_csi_driver_testing_framework.md | 737 ++++++ test/stress-scale-tests/run_tests.py | 452 ++-- test/stress-scale-tests/tests/orchestrator.py | 2203 +++++++++-------- 3 files changed, 2140 insertions(+), 1252 deletions(-) create mode 100644 test/stress-scale-tests/documentation/aws_efs_csi_driver_testing_framework.md diff --git a/test/stress-scale-tests/documentation/aws_efs_csi_driver_testing_framework.md b/test/stress-scale-tests/documentation/aws_efs_csi_driver_testing_framework.md new file mode 100644 index 000000000..444e73616 --- /dev/null +++ b/test/stress-scale-tests/documentation/aws_efs_csi_driver_testing_framework.md @@ -0,0 +1,737 @@ +# AWS EFS CSI Driver Stress & Scale Testing Framework Documentation + +## 1. System Overview + +The AWS EFS CSI Driver stress & scale testing framework is designed to rigorously test the Elastic File System CSI driver in Kubernetes environments. It simulates various workloads and scenarios to verify the driver's reliability, performance, and behavior under different conditions. + +## 2. Architecture + +The framework is built with a modular architecture consisting of the following components: + +``` +aws-efs-csi-driver/test/stress-scale-tests/ +├── run_tests.py # Main entry point script +├── tests/ +│ └── orchestrator.py # Core test orchestration logic +├── utils/ +│ ├── metrics_collector.py # Collects performance metrics +│ ├── log_integration.py # Handles log collection +│ └── report_generator.py # Generates test reports +├── monitoring/ +│ ├── setup_monitoring.sh # Sets up monitoring stack +│ ├── prometheus_exporter.py# Exports metrics to Prometheus +│ ├── dashboards/ # Grafana dashboards +│ └── kubernetes/ # K8s monitoring manifests +├── config/ +│ └── orchestrator_config.yaml # Test configuration +└── logs/ # Test logs directory +``` + +## 3. Execution Flow + +### 3.1 Test Initialization + +1. The process begins when a user runs `run_tests.py`, typically with configuration options. +2. The script parses command-line arguments and loads the configuration from YAML files. +3. The system verifies AWS credentials and sets up logging. +4. It initializes the metrics collector and report generators. +5. Based on the selected test suite (orchestrator in this case), it initializes the `EFSCSIOrchestrator`. + +### 3.2 Test Execution + +1. The orchestrator sets up a Kubernetes storage class if needed. +2. It runs initial "warm-up" operations to ensure coverage of all operation types. +3. The main testing loop begins, where it: + - Randomly selects operations based on configured weights + - Executes operations (create PVCs, attach pods, test read/write, etc.) + - Collects metrics and logs for each operation + - Continues until the specified test duration is reached +4. During execution, specific test scenarios are also run periodically: + - Many-to-one: Multiple pods mounting a single PVC + - One-to-one: One pod per PVC + - Concurrent PVC: Rapid creation and deletion of PVCs + +### 3.3 Test Completion + +1. After the test completes or is interrupted, a comprehensive cleanup is performed. +2. Test results and metrics are compiled into a report. +3. The report is saved to the reports directory. + +## 4. Key Components + +### 4.1 EFSCSIOrchestrator + +The core component that manages test execution. Key responsibilities: + +- **Resource Management**: Creates, tracks, and cleans up Kubernetes resources (PVCs, pods). +- **Operation Execution**: Performs operations like creating PVCs, attaching pods, verifying read/write. +- **Scenario Execution**: Runs specific test scenarios (many-to-one, one-to-one, concurrent PVC). +- **Metrics Collection**: Tracks performance metrics for operations. +- **Error Handling**: Detects and logs failures, collects diagnostic information. + +#### Detailed Class Structure + +The `EFSCSIOrchestrator` class is initialized with: + +```python +def __init__(self, config_file='config/orchestrator_config.yaml', namespace=None, metrics_collector=None, driver_pod_name=None): +``` + +The initialization process is modularized into several helper methods: +- `_init_configuration`: Loads configuration from file +- `_init_kubernetes_clients`: Sets up Kubernetes API clients +- `_init_metrics_collector`: Initializes metrics collection +- `_init_logging`: Configures logging infrastructure +- `_init_test_parameters`: Sets test runtime parameters +- `_init_resource_tracking`: Initializes tracking structures + +Key methods include: +- `run_test()`: Main entry point that runs the test for the configured duration +- `_get_operations_and_weights()`: Sets up weighted operations for random selection +- `_run_random_operation()`: Selects and executes a random operation based on weights +- Resource operations: `_create_pvc()`, `_attach_pod()`, `_delete_pod()`, `_delete_pvc()` +- Validation methods: `_verify_readwrite()`, `_wait_for_pod_ready()`, `_wait_for_pvc_bound()` + +### 4.2 Operations Framework + +The orchestrator executes operations based on weighted probabilities: + +1. **Create PVC** (`_create_pvc`): Creates a PersistentVolumeClaim using EFS CSI driver. +2. **Attach Pod** (`_attach_pod`): Attaches a pod to a PVC with appropriate volume mounts. +3. **Delete Pod** (`_delete_pod`): Removes pods and tracks resource cleanup. +4. **Delete PVC** (`_delete_pvc`): Removes PVCs after ensuring no pods are using them. +5. **Verify ReadWrite** (`_verify_readwrite`): Tests read/write operations between pods. +6. **Run Specific Scenario** (`_run_specific_scenario`): Executes one of the defined test scenarios. + +#### Pod Manifest Building + +Pod manifests are built in a modular fashion using configuration values: + +- `_build_pod_manifest`: Orchestrates the pod manifest creation process +- `_build_container_spec`: Creates container specifications using values from config +- `_build_readiness_probe`: Sets up readiness probe with configured parameters +- `_build_container_resources`: Sets resource limits based on config values +- `_build_pod_spec`: Creates the complete pod spec with volume mounts, tolerations, etc. + +#### PVC Creation + +PVC creation is also modularized to use configuration values: + +- `_build_pvc_manifest`: Creates the PVC manifest based on configuration +- `_add_pvc_metadata`: Adds metadata like annotations and labels +- `_create_and_wait_for_pvc`: Creates the PVC and waits for it to be bound + +#### Operation Selection Process + +Operations are selected using a weighted random selection process: + +```python +def _run_random_operation(self, operations, cumulative_weights, total_weight, operation_counts): + random_val = random.uniform(0, total_weight) + for i, (operation, _) in enumerate(operations): + if random_val <= cumulative_weights[i]: + op_name = operation.__name__ + operation_counts[op_name] = operation_counts.get(op_name, 0) + 1 + self.logger.info(f"Selected operation: {op_name} (selected {operation_counts[op_name]} times)") + operation() + break +``` + +### 4.3 Test Scenarios + +Special scenarios that test specific aspects of the EFS CSI driver: + +1. **Many-to-One** (`_scenario_many_to_one`): Tests multiple pods mounting a single PVC. + - Creates a dedicated PVC + - Creates multiple pods all mounting the same volume + - Tests read/write operations between pods to verify data consistency + - Collects diagnostics if failures occur + - Uses configuration values from `scenarios.many_to_one.min_pods` and `max_pods` + +2. **One-to-One** (`_scenario_one_to_one`): Tests one pod per PVC. + - Creates multiple PVCs + - Attaches one pod to each PVC + - Verifies each pod can write to its volume + - Uses configuration values from `scenarios.one_to_one.min_pairs` and `max_pairs` + +3. **Concurrent PVC** (`_scenario_concurrent_pvc`): Tests rapid PVC creation and deletion. + - Creates multiple PVCs in quick succession + - Creates pods for some PVCs + - Deletes PVCs rapidly to test resource cleanup + - Uses configuration values from `scenarios.concurrent_pvc.min_pvcs` and `max_pvcs` + +#### Modular Scenario Implementation + +The many-to-one scenario is implemented in a modular fashion, breaking down the process into logical steps: + +```python +def _scenario_many_to_one(self): + # Record scenario run + self.scenarios['many_to_one']['runs'] += 1 + + try: + # Step 1: Create PVC + pvc_name = self._create_many_to_one_pvc() + if not pvc_name: + self.scenarios['many_to_one']['fail'] += 1 + return + + # Step 2: Create multiple pods + pod_names = self._create_many_to_one_pods(pvc_name) + if len(pod_names) < 2: + self.scenarios['many_to_one']['fail'] += 1 + return + + # Step 3: Test read/write between pods + success = self._test_many_to_one_readwrite(pvc_name, pod_names) + if success: + self.scenarios['many_to_one']['success'] += 1 + else: + self.scenarios['many_to_one']['fail'] += 1 + # Collect diagnostic information + if len(pod_names) >= 2: + writer_pod = pod_names[0] + reader_pod = pod_names[1] + self._collect_many_to_one_failure_logs(pvc_name, writer_pod, reader_pod) + + except Exception as e: + self.scenarios['many_to_one']['fail'] += 1 + # Log exception details + + # Clean up will be handled by the orchestrator's cleanup method +``` + +#### Configuration-Driven Scenarios + +All scenarios use configuration values instead of hardcoded values: + +```yaml +scenarios: + concurrent_pvc: + enabled: true + max_pvcs: 30 + min_pvcs: 20 + many_to_one: + enabled: true + max_pods: 20 + min_pods: 10 + one_to_one: + enabled: true + max_pairs: 20 + min_pairs: 10 +``` + +These values control the number of resources created during each scenario, allowing for easy adjustment without code changes. + +### 4.4 Metrics Collection + +The `MetricsCollector` class tracks performance metrics: + +- **File Operation Latency**: Time taken for read/write/metadata operations. +- **IOPS**: I/O operations per second for different operation types. +- **Throughput**: Data transfer rates for read/write operations. +- **Success Rates**: Success/failure rates for various operations. + +#### Metric Types + +```python +# Example metric collection for file operations +def track_file_operation_latency(self, pvc_name, operation_type, latency): + """Track latency for file operations + + Args: + pvc_name: Name of PVC + operation_type: Type of operation ('read', 'write', 'metadata') + latency: Operation latency in seconds + """ + # Record metrics in internal data structures for later reporting +``` + +### 4.5 Monitoring + +Two monitoring options are available: + +1. **Kubernetes Deployment**: Deploys Prometheus and Grafana in the cluster. + - Automatically collects metrics from the exporter + - Provides pre-configured dashboards for visualization + +2. **Standalone Mode**: Runs a local Prometheus exporter. + - Simpler setup for development environments + - Exports metrics on a local endpoint (default: http://localhost:8000/metrics) + +#### Monitoring Setup Process + +```bash +# Start monitoring +./monitoring/setup_monitoring.sh + +# Option 1: Kubernetes deployment +kubectl apply -f monitoring/kubernetes/namespace.yaml +kubectl apply -f monitoring/kubernetes/prometheus-configmap.yaml +kubectl apply -f monitoring/kubernetes/prometheus-deployment.yaml +kubectl apply -f monitoring/kubernetes/grafana-dashboards.yaml +kubectl apply -f monitoring/kubernetes/grafana-deployment.yaml +kubectl apply -f monitoring/kubernetes/metrics-exporter.yaml + +# Option 2: Standalone exporter +pip install prometheus-client psutil pyyaml +python monitoring/prometheus_exporter.py --port 8000 --watch-dir reports/orchestrator/ +``` + +## 5. Running the Tests + +### 5.1 Basic Execution + +```bash +python run_tests.py +``` + +### 5.2 Common Options + +```bash +python run_tests.py --config config/orchestrator_config.yaml --duration 3600 --interval 5 +``` + +Parameters: +- `--config`: Path to configuration file +- `--duration`: Test duration in seconds +- `--interval`: Seconds to wait between operations +- `--driver-pod-name`: Name of EFS CSI driver pod (for log collection) + +### 5.3 Setting Up Monitoring + +```bash +# Start monitoring +./monitoring/setup_monitoring.sh + +# Option 1: Kubernetes deployment (requires kubectl access) +# Access Grafana at http://localhost:3000 after running: +kubectl port-forward -n monitoring svc/grafana 3000:3000 + +# Option 2: Standalone exporter +# Metrics available at http://localhost:8000/metrics +``` + +## 6. Configuration + +The main configuration file (`config/orchestrator_config.yaml`) controls test behavior: + +```yaml +# Test parameters +test: + namespace: "default" + duration: # Test duration in seconds + operation_interval: 3 # Seconds between operations + +# Resource limits +resource_limits: + max_pvcs: + max_pods_per_pvc: + total_max_pods: + +# Operation weights (probability distribution) +operation_weights: + create_pvc: + attach_pod: + delete_pod: + delete_pvc: + verify_readwrite: + run_specific_scenario: + +# Pod configuration +pod_config: + image: "alpine:latest" + command: ["/bin/sh", "-c"] + args: ["touch /data/pod-ready && while true; do sleep 30; done"] + readiness_probe: + initial_delay_seconds: + period_seconds: + node_selector: + topology.kubernetes.io/zone: us-west-1b + tolerations: + - effect: NoSchedule + key: instance + operator: Equal + value: core + +# Scenario-specific configuration +scenarios: + concurrent_pvc: + enabled: true + max_pvcs: + min_pvcs: + many_to_one: + enabled: true + max_pods: + min_pods: + one_to_one: + enabled: true + max_pairs: + min_pairs: +``` + +### 6.1 Custom Configuration + +You can create custom configurations for specific test cases. The framework will look for: + +1. Command-line specified config: `--config custom_config.yaml` +2. Dedicated orchestrator config: `config/orchestrator_config.yaml` +3. Default config: The main config passed to run_tests.py + +### 6.2 Configuration Usage in Code + +The framework is designed to use configuration values wherever possible, reducing hardcoded values: + +1. **Pod Configuration**: The `_build_pod_manifest` function uses configuration values for: + - Container image + - Commands and arguments + - Readiness probe parameters + - Resource limits + - Node selector and tolerations + +2. **PVC Configuration**: The `_create_pvc` function uses configuration for: + - Storage class name + - Access modes + - Storage size + - Annotations and labels + +3. **Scenario Configuration**: Test scenarios use configuration for resource counts: + - Many-to-one: Number of pods per PVC + - One-to-one: Number of PVC-pod pairs + - Concurrent PVC: Number of PVCs to create/delete + +4. **Test Parameters**: Test behavior is configured through: + - Test duration + - Operation interval + - Resource limits + - Operation weights + +## 7. Understanding Test Results + +### 7.1 Report Structure + +Test reports are generated in `reports/orchestrator/` with metrics including: + +- **Operations**: Success/failure counts and rates for each operation type +- **Scenarios**: Results for each test scenario +- **File Performance**: Metrics for read/write operations + - IOPS (operations per second) + - Latency (operation duration) + - Throughput (MB/s) + +Example report structure: +```json +{ + "test_name": "efs_orchestrator_20250613_162210", + "test_type": "orchestrator", + "timestamp": "20250613_162210", + "system_info": { + "kubernetes_version": "1.24.7", + "nodes": 3, + "platform": "eks" + }, + "results": { + "operations": { + "create_pvc": {"success": 45, "fail": 2}, + "attach_pod": {"success": 120, "fail": 5}, + "delete_pod": {"success": 110, "fail": 0}, + "delete_pvc": {"success": 40, "fail": 0}, + "verify_write": {"success": 80, "fail": 3}, + "verify_read": {"success": 77, "fail": 3} + }, + "scenarios": { + "shared_volume_rw": {"runs": 25, "success": 22, "fail": 3}, + "many_to_one": {"runs": 10, "success": 9, "fail": 1}, + "one_to_one": {"runs": 8, "success": 8, "fail": 0}, + "concurrent_pvc": {"runs": 5, "success": 4, "fail": 1} + } + }, + "metrics": { + "file_performance": { + "overall": { + "read": { + "latency_avg": 0.0325, + "iops_avg": 30.76, + "throughput_avg": 5.43 + }, + "write": { + "latency_avg": 0.0412, + "iops_avg": 24.27, + "throughput_avg": 4.21 + } + } + } + } +} +``` + +### 7.2 Logs + +Logs are available in `logs/` directory: +- `orchestrator_YYYYMMDD_HHMMSS.log`: Regular execution logs +- `efs_orchestrator_failure_*`: Logs collected on test failures + +#### Log Analysis Example + +Regular execution logs contain detailed operation information: +``` +2025-06-13 16:22:10,342 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 4 times) +2025-06-13 16:22:10,456 - tests.orchestrator - INFO - Created PVC: test-pvc-3a8f21c5 +``` + +Failure logs contain comprehensive diagnostic information: +``` +2025-06-13 16:22:15,789 - tests.orchestrator - ERROR - [MANY2ONE] FAILED: Pods cannot share data +2025-06-13 16:22:15,790 - tests.orchestrator - INFO - [MANY2ONE] Writer pod mount info: 'fs-12345abc.efs.us-west-2.amazonaws.com:/ on /data type nfs4' +``` + +## 8. Troubleshooting + +### 8.1 Common Issues + +1. **PVC Creation Failures**: + - Check StorageClass configuration + - Verify EFS filesystem exists and is accessible + - Check IAM permissions for the EFS CSI driver + +2. **Pod Readiness Issues**: + - Check pod events with `kubectl describe pod ` + - Verify EFS mount permissions + - Check if readiness probe is succeeding + +3. **Read/Write Test Failures**: + - Check for network connectivity issues + - Verify mount points inside pods + - Check EFS access point settings + +### 8.2 Diagnostic Tools + +The framework includes built-in diagnostic tools: + +1. **Log Collection**: Automatically collects relevant logs on failures +2. **Mount Information**: Gathers mount details from pods for analysis +3. **Resource State**: Collects descriptive information about Kubernetes resources + +## 9. Advanced Topics + +### 9.1 Modifying Test Scenarios + +To add or modify test scenarios: + +1. Create a new method in `EFSCSIOrchestrator` following the pattern of existing scenarios: +```python +def _scenario_my_new_test(self): + """ + Test description here + """ + self.logger.info("Running scenario: My New Test") + self.scenarios['my_new_test']['runs'] += 1 + + try: + # Implementation here + if success: + self.scenarios['my_new_test']['success'] += 1 + else: + self.scenarios['my_new_test']['fail'] += 1 + except Exception as e: + self.logger.error(f"Scenario failed: {e}") + self.scenarios['my_new_test']['fail'] += 1 +``` + +2. Add the scenario to the list in `_run_specific_scenario`: +```python +def _run_specific_scenario(self): + """Run a specific test scenario""" + scenarios = [ + self._scenario_many_to_one, + self._scenario_one_to_one, + self._scenario_concurrent_pvc, + self._scenario_my_new_test # Add your new scenario here + ] + scenario = random.choice(scenarios) + self.logger.info(f"Running specific scenario: {scenario.__name__}") + scenario() +``` + +3. Update scenario tracking in `__init__`: +```python +self.scenarios = { + 'shared_volume_rw': {'runs': 0, 'success': 0, 'fail': 0}, + 'many_to_one': {'runs': 0, 'success': 0, 'fail': 0}, + 'one_to_one': {'runs': 0, 'success': 0, 'fail': 0}, + 'concurrent_pvc': {'runs': 0, 'success': 0, 'fail': 0}, + 'my_new_test': {'runs': 0, 'success': 0, 'fail': 0} # Add tracking for your new scenario +} +``` + +4. Update scenario configuration in the YAML file: +```yaml +scenarios: + # ... existing scenarios + my_new_test: + enabled: true + custom_param1: value1 + custom_param2: value2 +``` + +5. Update weights in configuration if needed. + +### 9.2 Custom Metrics + +To add custom metrics: + +1. Extend the `MetricsCollector` class in `utils/metrics_collector.py`: +```python +def track_my_custom_metric(self, pvc_name, value): + """Track a custom metric + + Args: + pvc_name: Name of PVC + value: Metric value + """ + if 'custom_metrics' not in self.metrics: + self.metrics['custom_metrics'] = {} + + if pvc_name not in self.metrics['custom_metrics']: + self.metrics['custom_metrics'][pvc_name] = [] + + self.metrics['custom_metrics'][pvc_name].append(value) +``` + +2. Update the Prometheus exporter in `monitoring/prometheus_exporter.py` to expose new metrics: +```python +# Add a new Prometheus metric +my_custom_metric = Gauge( + 'efs_csi_custom_metric', + 'Description of your custom metric', + ['pvc_name'] +) + +# Update it in process_metrics function +def process_metrics(metrics_data): + # Process existing metrics + # ... + + # Process custom metrics + if 'custom_metrics' in metrics_data: + for pvc_name, values in metrics_data['custom_metrics'].items(): + if values: + my_custom_metric.labels(pvc_name=pvc_name).set(sum(values) / len(values)) +``` + +3. Update dashboards to visualize the new metrics: + - Edit relevant JSON files in `monitoring/dashboards/` + - Add new panels that query your custom metrics + +### 9.3 Failure Analysis + +For detailed failure analysis: + +1. Examine logs in `logs/efs_orchestrator_failure_*` directories: + - Review `failed_resources` folder for resource state at failure time + - Check pod and PVC events + - Look for error patterns in driver logs + +2. Check collected metrics in `reports/orchestrator/*_metrics.json`: + - Look for performance anomalies before failures + - Check for patterns in failed operations + +3. Review pod and PVC events collected during failures: + - `kubectl describe` output saved during failures + - Container logs and events + +4. Create targeted tests for specific failure cases: + - Modify configuration for specific scenarios + - Reduce random factors to focus on problem areas + +### 9.4 Code Modularization Principles + +The framework follows these modularization principles: + +1. **Single Responsibility**: Each function has a clear and focused purpose +2. **Function Size**: Functions are kept small (under 50 lines) for better maintainability +3. **Configuration Over Code**: Use configuration values instead of hardcoded constants +4. **Descriptive Naming**: Function and variable names clearly communicate their purpose +5. **Helper Methods**: Complex operations are broken down into helper methods +6. **Error Handling**: Errors are caught at appropriate levels and properly logged + +Example of modularization: + +```python +# Instead of a single large function: +def build_complex_widget(config, name): + # 100+ lines of code doing many things + +# Break it down into smaller functions: +def build_complex_widget(config, name): + """Orchestrate the widget building process""" + base = create_widget_base(name) + add_widget_components(base, config) + configure_widget_behavior(base, config) + return base + +def create_widget_base(name): + """Create the foundation of the widget""" + # 10-15 lines focused on just this task + +def add_widget_components(widget, config): + """Add all necessary components to the widget""" + # 15-20 lines focused on component addition + +def configure_widget_behavior(widget, config): + """Set up the widget's behavior based on config""" + # 15-20 lines focused on behavior configuration +``` + +## 10. Component Relationships + +``` +┌─────────────────┐ ┌───────────────────┐ ┌────────────────┐ +│ run_tests.py │────▶│ EFSCSIOrchestrator│────▶│Test Operations │ +└─────────────────┘ └───────────────────┘ └────────────────┘ + │ │ │ + ▼ ▼ ▼ +┌─────────────────┐ ┌───────────────────┐ ┌────────────────┐ +│ReportGenerator │◀────│ MetricsCollector │◀────│Test Scenarios │ +└─────────────────┘ └───────────────────┘ └────────────────┘ + │ ▲ │ + │ │ │ + ▼ │ ▼ +┌─────────────────────────┐ │ ┌────────────────────────┐ +│ Reports & │ │ │ K8s Resources │ +│ Dashboards │ │ │ (PVCs, Pods, etc.) │ +└─────────────────────────┘ │ └────────────────────────┘ + │ + ┌────────────────┐ + │ Monitoring │ + │ Stack │ + └────────────────┘ +``` + +### 10.1 Data Flow + +1. `run_tests.py` initializes components and kicks off testing +2. `EFSCSIOrchestrator` executes operations and scenarios +3. Test operations and scenarios interact with Kubernetes resources +4. `MetricsCollector` gathers metrics from operations and scenarios +5. `ReportGenerator` creates reports from collected metrics +6. Reports are stored in files and exposed via the monitoring stack +7. Dashboards visualize the metrics for analysis + +### 10.2 Control Flow + +1. User invokes `run_tests.py` with configuration options +2. `run_tests.py` initializes the orchestrator +3. Orchestrator runs for the specified duration, executing operations +4. Orchestrator tracks resources and metrics during execution +5. After completion, orchestrator generates a report +6. Metrics are available during and after test execution via monitoring + + +## 12. Conclusion + +The AWS EFS CSI Driver Stress & Scale Testing Framework provides a comprehensive, configurable platform for testing the EFS CSI driver under various conditions. Its modular architecture allows for easy extension and customization, while the built-in metrics collection and monitoring provide valuable insights into driver performance and behavior. + +The recent modularization efforts have significantly improved code maintainability and configurability, making it easier to extend the framework and adapt it to different testing needs. + +By following this documentation, developers should be able to understand the system architecture, run tests, analyze results, and extend the framework for their specific testing needs. diff --git a/test/stress-scale-tests/run_tests.py b/test/stress-scale-tests/run_tests.py index 04b591473..9baf9a424 100644 --- a/test/stress-scale-tests/run_tests.py +++ b/test/stress-scale-tests/run_tests.py @@ -72,11 +72,12 @@ def parse_args(): action='store_true', help='Print what would be done without executing tests' ) - parser.add_argument( - '--driver-pod-name', - default=None, - help='Name of the EFS CSI driver pod for log collection (optional)' - ) + # Driver pod name functionality commented out as not currently used + # parser.add_argument( + # '--driver-pod-name', + # default=None, + # help='Name of the EFS CSI driver pod for log collection (optional)' + # ) # Cluster setup options - kept for compatibility but functionality is disabled parser.add_argument( @@ -124,205 +125,292 @@ def print_credential_renewal_instructions(): print("\nAfter renewing your credentials, try running the tests again.") print("="*80 + "\n") -def main(): - """Main entry point""" - args = parse_args() +def load_config(config_path): + """Load configuration from YAML file - # Load configuration + Args: + config_path: Path to configuration file + + Returns: + Loaded configuration as dictionary + """ try: - with open(args.config, 'r') as f: + with open(config_path, 'r') as f: config = yaml.safe_load(f) + return config except Exception as e: print(f"Error loading configuration: {e}") sys.exit(1) + +def get_driver_pod_name(args, config): + """Get driver pod name from config - # Setup logging - logger = setup_logging(config) - logger.info(f"Starting EFS CSI tests with configuration from {args.config}") - - # Verify credentials before proceeding - logger.info("Verifying AWS credentials") - if not check_credentials(): - logger.error("AWS credentials are expired or invalid") - print_credential_renewal_instructions() - sys.exit(1) + Args: + args: Parsed command line arguments + config: Configuration dictionary + + Returns: + Driver pod name or None + """ + # Since driver-pod-name arg is commented out, we only check config + driver_pod_name = None + if 'driver' in config and 'pod_name' in config['driver']: + driver_pod_name = config['driver']['pod_name'] + return driver_pod_name + +def handle_setup_cluster(args, config, logger): + """Handle cluster setup functionality - # Set up cluster if requested - DISABLED + Args: + args: Command line arguments + config: Configuration dictionary + logger: Logger instance + """ if args.setup_cluster: logger.warning("Cluster setup functionality is disabled - skipping setup") logger.warning("Please make sure you have a working cluster with EFS CSI driver installed") + +def initialize_components(config): + """Initialize report generator and metrics collector + + Args: + config: Configuration dictionary - # The following code has been commented out - """ - # Update config with command-line overrides - if args.kubernetes_version: - config['cluster']['kubernetes_version'] = args.kubernetes_version - logger.info(f"Using Kubernetes version from command line: {args.kubernetes_version}") - - if args.driver_version: - config['driver']['version'] = args.driver_version - logger.info(f"Using EFS CSI Driver version from command line: {args.driver_version}") - - # Write updated config back to file - with open(args.config, 'w') as f: - yaml.dump(config, f) - - # Initialize and run cluster setup - try: - cluster_setup = ClusterSetup(config_file=args.config) - setup_result = cluster_setup.setup() - logger.info(f"Cluster setup completed: {setup_result}") - except Exception as e: - logger.error(f"Cluster setup failed: {e}") - sys.exit(1) - """ - - # Initialize report generator + Returns: + report_generator, metrics_collector, report_dir + """ report_dir = config.get('reporting', {}).get('output_dir', 'reports') report_generator = ReportGenerator(output_dir=report_dir) - - # Initialize metrics collector metrics_collector = MetricsCollector() + return report_generator, metrics_collector, report_dir + +def run_orchestrator_test(args, config, logger, metrics_collector, report_generator, report_dir): + """Run the orchestrator test suite + + Args: + args: Command line arguments + config: Configuration dictionary + logger: Logger instance + metrics_collector: Instance of MetricsCollector + report_generator: Instance of ReportGenerator + report_dir: Path to report directory - # Run tests based on test suite + Returns: + Orchestrator test results + """ results = {} - try: - if args.test_suite in ['orchestrator', 'all']: - logger.info("Running orchestrator stress test suite") - - if args.dry_run: - logger.info("DRY RUN MODE: Would run orchestrator with randomized operations") - orchestrator_results = { - "status": "would_run", - "description": "Would run the orchestrator with randomized operations" - } - else: - logger.info(f"Starting orchestrator for {args.duration} seconds") - - # Create orchestrator instance - use dedicated orchestrator config file if it exists, - # otherwise fall back to the main config - orchestrator_config = 'config/orchestrator_config.yaml' - if not os.path.exists(orchestrator_config): - logger.warning(f"Orchestrator config file {orchestrator_config} not found, falling back to {args.config}") - orchestrator_config = args.config - - logger.info(f"Using orchestrator config: {orchestrator_config}") - # Get driver pod name from config if not specified on command line - driver_pod_name = args.driver_pod_name - if driver_pod_name is None and 'driver' in config and 'pod_name' in config['driver']: - driver_pod_name = config['driver']['pod_name'] - if driver_pod_name: - logger.info(f"Using driver pod name from config: {driver_pod_name}") - - # Pass the metrics collector and driver pod name to the orchestrator - orchestrator = EFSCSIOrchestrator( - config_file=orchestrator_config, - metrics_collector=metrics_collector, - driver_pod_name=driver_pod_name - ) - - # Override default test parameters if specified - if args.duration: - orchestrator.test_duration = args.duration - logger.info(f"Test duration overridden to {args.duration} seconds") - - if args.interval is not None: # Only override if explicitly specified - orchestrator.operation_interval = args.interval - logger.info(f"Operation interval overridden to {orchestrator.operation_interval} seconds") - else: - logger.info(f"Using operation interval from config: {orchestrator.operation_interval} seconds") - - # Run the orchestrator test - orchestrator_results = orchestrator.run_test() - - # Generate one consolidated orchestrator report with all information - orchestrator_report_dir = os.path.join(report_dir, 'orchestrator') - os.makedirs(orchestrator_report_dir, exist_ok=True) + if args.test_suite not in ['orchestrator', 'all']: + return results - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - test_name = f"efs_orchestrator_{timestamp}" - - # Create report path - report_path = os.path.join( - orchestrator_report_dir, - f"{test_name}.json" - ) - - # Add metadata, system info, and metrics to the results - system_info = report_generator._collect_system_info() - - # Get all metrics from the metrics collector - collected_metrics = metrics_collector.get_all_metrics() - - full_report = { - "test_name": test_name, - "test_type": "orchestrator", - "timestamp": timestamp, - "system_info": system_info, - "results": orchestrator_results, - "metrics": { - "file_performance": collected_metrics.get("file_performance", {}) - } - } - - logger.info("File performance metrics collected:") - for volume, metrics in collected_metrics.get("file_performance", {}).get("by_volume", {}).items(): - logger.info(f" Volume: {volume}") - # Log read metrics if available - if metrics["iops"].get("read") is not None: - logger.info(f" Read IOPS: {metrics['iops']['read']:.2f}") - if metrics["throughput"].get("read") is not None: - logger.info(f" Read Throughput: {metrics['throughput']['read']:.2f} MB/s") - - # Log write metrics if available - if metrics["iops"].get("write") is not None: - logger.info(f" Write IOPS: {metrics['iops']['write']:.2f}") - if metrics["throughput"].get("write") is not None: - logger.info(f" Write Throughput: {metrics['throughput']['write']:.2f} MB/s") - - with open(report_path, 'w') as f: - import json - json.dump(full_report, f, indent=2) - - logger.info(f"Orchestrator report generated: {report_path}") - - results['orchestrator'] = orchestrator_results - - except Exception as e: - logger.error(f"Error running tests: {e}", exc_info=True) - - # Collect logs on failure - logger.info("Collecting logs due to test failure") - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - test_name = f"efs_orchestrator_failure_{timestamp}" + logger.info("Running orchestrator stress test suite") + + if args.dry_run: + logger.info("DRY RUN MODE: Would run orchestrator with randomized operations") + return { + 'orchestrator': { + "status": "would_run", + "description": "Would run the orchestrator with randomized operations" + } + } + + # Set up the orchestrator + orchestrator = setup_orchestrator(args, config, logger, metrics_collector) + + # Run the test + logger.info(f"Starting orchestrator for {args.duration if args.duration else 'default'} seconds") + test_results = orchestrator.run_test() + + # Generate and save the test report + generate_test_report( + test_results, + report_dir, + report_generator, + metrics_collector, + logger + ) + + return {'orchestrator': test_results} + +def setup_orchestrator(args, config, logger, metrics_collector): + """Set up the orchestrator for testing + + Args: + args: Command line arguments + config: Configuration dictionary + logger: Logger instance + metrics_collector: Instance of MetricsCollector - # Get driver pod name from config if not specified on command line - driver_pod_name = args.driver_pod_name - if driver_pod_name is None and 'driver' in config and 'pod_name' in config['driver']: - driver_pod_name = config['driver']['pod_name'] - if driver_pod_name: - logger.info(f"Using driver pod name from config: {driver_pod_name}") - - logs_path = collect_logs_on_test_failure(test_name, metrics_collector, driver_pod_name=driver_pod_name) - if logs_path: - logger.info(f"Failure logs collected to: {logs_path}") - else: - logger.warning("Failed to collect logs") + Returns: + Configured orchestrator instance + """ + # Get orchestrator config file + orchestrator_config = 'config/orchestrator_config.yaml' + if not os.path.exists(orchestrator_config): + logger.warning(f"Orchestrator config file {orchestrator_config} not found, falling back to {args.config}") + orchestrator_config = args.config + + logger.info(f"Using orchestrator config: {orchestrator_config}") + + # Driver pod name functionality commented out as not currently needed + # driver_pod_name = get_driver_pod_name(args, config) + # if driver_pod_name: + # logger.info(f"Using driver pod name from config: {driver_pod_name}") + + # Create orchestrator (driver_pod_name parameter commented out) + orchestrator = EFSCSIOrchestrator( + config_file=orchestrator_config, + metrics_collector=metrics_collector, + # driver_pod_name=driver_pod_name + ) + + # Override default test parameters if specified + if args.duration: + orchestrator.test_duration = args.duration + logger.info(f"Test duration overridden to {args.duration} seconds") - # Even if tests failed, try to clean up if requested - DISABLED - if args.setup_cluster and not args.skip_cleanup: - logger.warning("Cluster cleanup functionality is disabled") - """ - try: - logger.info("Attempting cleanup after test failure") - cluster_setup = ClusterSetup(config_file=args.config) - cluster_setup.cleanup() - logger.info("Cleanup completed") - except Exception as cleanup_error: - logger.error(f"Cleanup after failure error: {cleanup_error}") - """ + if args.interval is not None: # Only override if explicitly specified + orchestrator.operation_interval = args.interval + logger.info(f"Operation interval overridden to {orchestrator.operation_interval} seconds") + else: + logger.info(f"Using operation interval from config: {orchestrator.operation_interval} seconds") + return orchestrator + +def generate_test_report(test_results, report_dir, report_generator, metrics_collector, logger): + """Generate a test report with metrics and results + + Args: + test_results: Results from the test run + report_dir: Directory to store reports + report_generator: Report generator instance + metrics_collector: Metrics collector instance + logger: Logger instance + """ + # Make report directory + orchestrator_report_dir = os.path.join(report_dir, 'orchestrator') + os.makedirs(orchestrator_report_dir, exist_ok=True) + + # Create timestamp and test name + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + test_name = f"efs_orchestrator_{timestamp}" + + # Create report path + report_path = os.path.join(orchestrator_report_dir, f"{test_name}.json") + + # Add metadata, system info, and metrics to the results + system_info = report_generator._collect_system_info() + collected_metrics = metrics_collector.get_all_metrics() + + # Create full report + full_report = { + "test_name": test_name, + "test_type": "orchestrator", + "timestamp": timestamp, + "system_info": system_info, + "results": test_results, + "metrics": { + "file_performance": collected_metrics.get("file_performance", {}) + } + } + + # Log metrics + log_performance_metrics(collected_metrics, logger) + + # Write report to file + with open(report_path, 'w') as f: + import json + json.dump(full_report, f, indent=2) + + logger.info(f"Orchestrator report generated: {report_path}") + +def log_performance_metrics(collected_metrics, logger): + """Log performance metrics + + Args: + collected_metrics: Metrics data + logger: Logger instance + """ + logger.info("File performance metrics collected:") + for volume, metrics in collected_metrics.get("file_performance", {}).get("by_volume", {}).items(): + logger.info(f" Volume: {volume}") + # Log read metrics if available + if metrics["iops"].get("read") is not None: + logger.info(f" Read IOPS: {metrics['iops']['read']:.2f}") + if metrics["throughput"].get("read") is not None: + logger.info(f" Read Throughput: {metrics['throughput']['read']:.2f} MB/s") + + # Log write metrics if available + if metrics["iops"].get("write") is not None: + logger.info(f" Write IOPS: {metrics['iops']['write']:.2f}") + if metrics["throughput"].get("write") is not None: + logger.info(f" Write Throughput: {metrics['throughput']['write']:.2f} MB/s") + +def handle_test_failure(e, args, config, metrics_collector, logger): + """Handle test failure + + Args: + e: Exception + args: Command line arguments + config: Configuration dictionary + metrics_collector: Metrics collector instance + logger: Logger instance + """ + logger.error(f"Error running tests: {e}", exc_info=True) + + # Collect logs on failure + logger.info("Collecting logs due to test failure") + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + test_name = f"efs_orchestrator_failure_{timestamp}" + + # Driver pod name functionality commented out + # driver_pod_name = get_driver_pod_name(args, config) + # if driver_pod_name: + # logger.info(f"Using driver pod name from config: {driver_pod_name}") + + logs_path = collect_logs_on_test_failure(test_name, metrics_collector) # driver_pod_name parameter removed + if logs_path: + logger.info(f"Failure logs collected to: {logs_path}") + else: + logger.warning("Failed to collect logs") + + # Even if tests failed, try to clean up if requested - DISABLED + if args.setup_cluster and not args.skip_cleanup: + logger.warning("Cluster cleanup functionality is disabled") + +def main(): + """Main entry point""" + # Parse command line arguments + args = parse_args() + + # Load configuration + config = load_config(args.config) + + # Setup logging + logger = setup_logging(config) + logger.info(f"Starting EFS CSI tests with configuration from {args.config}") + + # Verify credentials before proceeding + logger.info("Verifying AWS credentials") + if not check_credentials(): + logger.error("AWS credentials are expired or invalid") + print_credential_renewal_instructions() + sys.exit(1) + + # Handle cluster setup if requested + handle_setup_cluster(args, config, logger) + + # Initialize components + report_generator, metrics_collector, report_dir = initialize_components(config) + + # Run tests + try: + results = run_orchestrator_test( + args, config, logger, metrics_collector, report_generator, report_dir + ) + return results + except Exception as e: + handle_test_failure(e, args, config, metrics_collector, logger) sys.exit(1) if __name__ == "__main__": diff --git a/test/stress-scale-tests/tests/orchestrator.py b/test/stress-scale-tests/tests/orchestrator.py index 6a5d4818e..86e9a5bdc 100644 --- a/test/stress-scale-tests/tests/orchestrator.py +++ b/test/stress-scale-tests/tests/orchestrator.py @@ -6,6 +6,7 @@ import logging import uuid import os +import boto3 from kubernetes import client, config from datetime import datetime from utils.log_integration import collect_logs_on_test_failure @@ -18,24 +19,42 @@ def __init__(self, config_file='config/orchestrator_config.yaml', namespace=None # Store driver pod name for log collection self.driver_pod_name = driver_pod_name + # Initialize configuration, clients, logging, and resources + self._init_configuration(config_file, namespace) + self._init_kubernetes_clients() + self._init_metrics_collector(metrics_collector) + self._init_logging() + self._init_test_parameters() + self._init_resource_tracking() + + self.logger.info("EFS CSI Orchestrator initialized") + + # Create namespace if it doesn't exist + self._ensure_namespace_exists() + + def _init_configuration(self, config_file, namespace): + """Initialize configuration from file""" # Load configuration with open(config_file, 'r') as f: self.config = yaml.safe_load(f) - - # Initialize Kubernetes client + + # Set namespace from config or use default + self.namespace = namespace or self.config['test'].get('namespace', 'default') + + def _init_kubernetes_clients(self): + """Initialize Kubernetes API clients""" config.load_kube_config() self.core_v1 = client.CoreV1Api() self.apps_v1 = client.AppsV1Api() self.storage_v1 = client.StorageV1Api() - - # Initialize metrics collector if provided, or create a new one + + def _init_metrics_collector(self, metrics_collector): + """Initialize metrics collector""" from utils.metrics_collector import MetricsCollector self.metrics_collector = metrics_collector or MetricsCollector() - - # Set namespace from config or use default - self.namespace = namespace or self.config['test'].get('namespace', 'default') - - # Set up logging based on configuration + + def _init_logging(self): + """Set up logging based on configuration""" self.logger = logging.getLogger(__name__) # Remove any existing handlers to prevent duplicates self.logger.handlers.clear() @@ -59,7 +78,9 @@ def __init__(self, config_file='config/orchestrator_config.yaml', namespace=None file_handler = logging.FileHandler(f'logs/orchestrator_{datetime.now().strftime("%Y%m%d_%H%M%S")}.log') file_handler.setFormatter(formatter) self.logger.addHandler(file_handler) - + + def _init_test_parameters(self): + """Initialize test parameters from configuration""" # Test parameters self.test_duration = self.config['test'].get('duration', 3600) # seconds self.operation_interval = self.config['test'].get('operation_interval', 3) # seconds @@ -68,8 +89,10 @@ def __init__(self, config_file='config/orchestrator_config.yaml', namespace=None resource_limits = self.config.get('resource_limits', {}) self.max_pvcs = resource_limits.get('max_pvcs', 100) self.max_pods_per_pvc = resource_limits.get('max_pods_per_pvc', 50) - self.total_max_pods = resource_limits.get('total_max_pods', 30) - + self.total_max_pods = resource_limits.get('total_max_pods', 250) + + def _init_resource_tracking(self): + """Initialize resource tracking data structures""" # Resource tracking self.pvcs = [] # List of PVC names self.pods = {} # Maps pvc_name -> list of pod_names @@ -93,11 +116,6 @@ def __init__(self, config_file='config/orchestrator_config.yaml', namespace=None 'concurrent_pvc': {'runs': 0, 'success': 0, 'fail': 0} } - self.logger.info("EFS CSI Orchestrator initialized") - - # Create namespace if it doesn't exist - self._ensure_namespace_exists() - def _ensure_namespace_exists(self): """Create the namespace if it doesn't exist already""" try: @@ -128,11 +146,26 @@ def run_test(self): """ self.logger.info(f"Starting orchestrator test for {self.test_duration} seconds") start_time = time.time() - - # Ensure storage class exists self._ensure_storage_class() - - # Get operation weights from config or use defaults + operations, weights = self._get_operations_and_weights() + cumulative_weights, total_weight = self._get_cumulative_weights(weights) + self._run_initial_operations() + operation_counts = {op.__name__: 0 for op, _ in operations} + try: + while time.time() - start_time < self.test_duration: + self._run_random_operation(operations, cumulative_weights, total_weight, operation_counts) + time.sleep(self.operation_interval) + except KeyboardInterrupt: + self.logger.info("Test interrupted by user") + except Exception as e: + self._handle_unexpected_test_error(e) + finally: + elapsed = time.time() - start_time + self.logger.info(f"Test completed in {elapsed:.2f} seconds") + self._cleanup() + return self._generate_report() + + def _get_operations_and_weights(self): weights = self.config.get('operation_weights', {}) operations = [ (self._create_pvc, weights.get('create_pvc', 25)), @@ -142,105 +175,60 @@ def run_test(self): (self._verify_readwrite, weights.get('verify_readwrite', 15)), (self._run_specific_scenario, weights.get('run_specific_scenario', 20)) ] - - # Extract operations and weights into separate lists for cleaner selection operation_funcs, weights = zip(*operations) - - # Pre-calculate cumulative weights + return operations, weights + + def _get_cumulative_weights(self, weights): cumulative_weights = [] current_sum = 0 for weight in weights: current_sum += weight cumulative_weights.append(current_sum) total_weight = cumulative_weights[-1] - - # First, ensure we run all operations at least once in a controlled sequence + return cumulative_weights, total_weight + + def _run_initial_operations(self): self.logger.info("Running each operation type once to ensure coverage") - - # First create some PVCs self._create_pvc() self._create_pvc() - - # Then attach pods to those PVCs to create pairs self._attach_pod() self._attach_pod() - self._attach_pod() # One more to ensure we have multiple pods per PVC - - # Now we can run verify_readwrite which requires at least 2 pods on same PVC + self._attach_pod() self._verify_readwrite() - - # Run a specific scenario self._run_specific_scenario() - - # Delete a pod self._delete_pod() - - # Delete a PVC self._delete_pvc() - self.logger.info("Completed initial operation sequence, continuing with randomized operations") - - # Track operation selections to verify distribution - operation_counts = {op.__name__: 0 for op, _ in operations} - - # Run operations randomly until test_duration is reached - try: - while time.time() - start_time < self.test_duration: - # Select a random operation based on pre-calculated cumulative weights - random_val = random.uniform(0, total_weight) - - # Find the operation corresponding to the random value - for i, (operation, _) in enumerate(operations): - if random_val <= cumulative_weights[i]: - op_name = operation.__name__ - operation_counts[op_name] = operation_counts.get(op_name, 0) + 1 - self.logger.info(f"Selected operation: {op_name} (selected {operation_counts[op_name]} times)") - operation() - break - - # Wait between operations to avoid overwhelming the system - time.sleep(self.operation_interval) - - except KeyboardInterrupt: - self.logger.info("Test interrupted by user") - except Exception as e: - self.logger.error(f"Unexpected error during test: {e}", exc_info=True) - - # Collect logs for unexpected failure with comprehensive resource tracking - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - test_name = f"orchestrator_unexpected_failure_{timestamp}" - - # Gather information about all active resources for diagnostic logging - failed_resources = [] - - # Add all PVCs currently being tracked - for pvc_name in self.pvcs: - failed_resources.append({"type": "pvc", "name": pvc_name, "namespace": self.namespace}) - - # Add all pods associated with each PVC - for pod_name in self.pods.get(pvc_name, []): - failed_resources.append({"type": "pod", "name": pod_name, "namespace": self.namespace}) - - logs_path = collect_logs_on_test_failure( - test_name, - self.metrics_collector, - self.driver_pod_name, - failed_resources=failed_resources - ) - self.logger.info(f"Collected comprehensive failure logs to: {logs_path}") - finally: - # Get test duration - elapsed = time.time() - start_time - self.logger.info(f"Test completed in {elapsed:.2f} seconds") - - # Clean up resources - self._cleanup() - - # Return test results - return self._generate_report() - + + def _run_random_operation(self, operations, cumulative_weights, total_weight, operation_counts): + random_val = random.uniform(0, total_weight) + for i, (operation, _) in enumerate(operations): + if random_val <= cumulative_weights[i]: + op_name = operation.__name__ + operation_counts[op_name] = operation_counts.get(op_name, 0) + 1 + self.logger.info(f"Selected operation: {op_name} (selected {operation_counts[op_name]} times)") + operation() + break + + def _handle_unexpected_test_error(self, e): + self.logger.error(f"Unexpected error during test: {e}", exc_info=True) + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + test_name = f"orchestrator_unexpected_failure_{timestamp}" + failed_resources = [] + for pvc_name in self.pvcs: + failed_resources.append({"type": "pvc", "name": pvc_name, "namespace": self.namespace}) + for pod_name in self.pods.get(pvc_name, []): + failed_resources.append({"type": "pod", "name": pod_name, "namespace": self.namespace}) + logs_path = collect_logs_on_test_failure( + test_name, + self.metrics_collector, + self.driver_pod_name, + failed_resources=failed_resources + ) + self.logger.info(f"Collected comprehensive failure logs to: {logs_path}") + def _create_pvc(self): - """Create a PVC with access point""" + """Create a PVC with access point using configured values""" # Check if we've reached the maximum PVC count if len(self.pvcs) >= self.max_pvcs: self.logger.info("Maximum PVC count reached, skipping creation") @@ -250,83 +238,253 @@ def _create_pvc(self): self.logger.info(f"Creating PVC: {pvc_name}") try: - # Create PVC manifest - pvc_manifest = { - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "metadata": {"name": pvc_name}, - "spec": { - "accessModes": ["ReadWriteMany"], - "storageClassName": "efs-sc", - "resources": { - "requests": {"storage": "1Gi"} - } - } - } - - # Create PVC - self.core_v1.create_namespaced_persistent_volume_claim( - namespace=self.namespace, - body=pvc_manifest - ) - - # Track the PVC - self.pvcs.append(pvc_name) - self.pods[pvc_name] = [] + # Build the PVC manifest from config + pvc_manifest = self._build_pvc_manifest(pvc_name) - # Update results - self.results['create_pvc']['success'] += 1 - self.logger.info(f"Created PVC: {pvc_name}") + # Create and wait for PVC to be bound + success = self._create_and_wait_for_pvc(pvc_name, pvc_manifest) - # Wait for PVC to be bound - if not self._wait_for_pvc_bound(pvc_name, timeout=30): - self.logger.warning(f"Timeout waiting for PVC {pvc_name} to be bound") + if not success: + self.logger.warning(f"PVC {pvc_name} creation process did not complete successfully") except Exception as e: self.results['create_pvc']['fail'] += 1 self.logger.error(f"Failed to create PVC: {e}") + def _build_pvc_manifest(self, pvc_name): + """Build a PVC manifest based on configuration""" + pvc_config = self.config.get('pvc_config', {}) + + # Get storage class name from config + sc_name = self.config.get('storage_class', {}).get('name', 'efs-sc') + + # Get access modes from config or use default + access_modes = pvc_config.get('access_modes', ["ReadWriteMany"]) + + # Get storage size from config or use default + storage_size = pvc_config.get('storage_size', "1Gi") + + # Create base manifest + pvc_manifest = { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": {"name": pvc_name}, + "spec": { + "accessModes": access_modes, + "storageClassName": sc_name, + "resources": { + "requests": {"storage": storage_size} + } + } + } + + # Add metadata from config + self._add_pvc_metadata(pvc_manifest, pvc_config) + + return pvc_manifest + + def _add_pvc_metadata(self, pvc_manifest, pvc_config): + """Add metadata like annotations and labels to PVC manifest""" + # Add annotations if configured + pvc_annotations = pvc_config.get('annotations', {}) + if pvc_annotations: + pvc_manifest['metadata']['annotations'] = pvc_annotations + + # Add labels if configured + pvc_labels = pvc_config.get('labels', {}) + if pvc_labels: + pvc_manifest['metadata']['labels'] = pvc_labels + + return pvc_manifest + + def _create_and_wait_for_pvc(self, pvc_name, pvc_manifest): + """Create PVC and wait for it to be bound""" + # Create PVC + self.core_v1.create_namespaced_persistent_volume_claim( + namespace=self.namespace, + body=pvc_manifest + ) + + # Track the PVC + self.pvcs.append(pvc_name) + self.pods[pvc_name] = [] + + # Update results + self.results['create_pvc']['success'] += 1 + sc_name = pvc_manifest['spec']['storageClassName'] + self.logger.info(f"Created PVC: {pvc_name} with storage class {sc_name}") + + # Get timeout value from config or use default + retry_config = self.config.get('retries', {}) + pvc_bind_timeout = retry_config.get('pvc_bind_timeout', 30) + + # Wait for PVC to be bound + return self._wait_for_pvc_bound(pvc_name, timeout=pvc_bind_timeout) + def _attach_pod(self, pvc_name=None): """ - Attach a pod to a PVC - If pvc_name is provided, attach to that specific PVC - Otherwise, select a random PVC + Attach a pod to a PVC. If pvc_name is provided, attach to that specific PVC. Otherwise, select a random PVC. """ - # If no PVCs exist, skip if not self.pvcs: self.logger.info("No PVCs available, skipping pod attachment") return None - - # If we've reached max total pod count, skip if self.current_pod_count >= self.total_max_pods: self.logger.info("Maximum total pod count reached, skipping attachment") return None - - # Select PVC (either specified or random) + pvc_name = self._select_pvc_for_pod(pvc_name) + if pvc_name is None: + return None + pod_name = f"test-pod-{uuid.uuid4().hex[:8]}" + pod_manifest = self._build_pod_manifest(pod_name, pvc_name) + try: + self.core_v1.create_namespaced_pod(namespace=self.namespace, body=pod_manifest) + self._track_new_pod(pvc_name, pod_name) + self.logger.info(f"Created pod: {pod_name} using PVC: {pvc_name}") + if not self._wait_for_pod_ready(pod_name, timeout=60): + self.logger.warning(f"Timeout waiting for pod {pod_name} to be ready") + return None + return pod_name + except Exception as e: + self.results['attach_pod']['fail'] += 1 + self.logger.error(f"Failed to create pod: {e}") + return None + + def _select_pvc_for_pod(self, pvc_name): if pvc_name is None or pvc_name not in self.pvcs: pvc_name = random.choice(self.pvcs) - - # Check if this PVC has reached max pods if len(self.pods[pvc_name]) >= self.max_pods_per_pvc: self.logger.info(f"PVC {pvc_name} has reached max pods ({self.max_pods_per_pvc}), skipping") return None + return pvc_name + + def _build_pod_manifest(self, pod_name, pvc_name): + """Build pod manifest using configuration values""" + pod_config = self.config.get('pod_config', {}) + + # Build the container specification + container = self._build_container_spec(pod_config) + + # Build pod metadata + metadata = self._build_pod_metadata(pod_name, pod_config) + + # Build pod spec + pod_spec = self._build_pod_spec(container, pvc_name, pod_config) + + # Combine into complete manifest + manifest = { + "apiVersion": "v1", + "kind": "Pod", + "metadata": metadata, + "spec": pod_spec + } + + return manifest + + def _build_container_spec(self, pod_config): + """Build the container specification from config""" + # Determine command arguments + args = pod_config.get('args') + if not args: + startup_script = self._get_pod_startup_script() + args = [startup_script] + + # Create base container spec + container = { + "name": "test-container", + "image": pod_config.get('image', 'alpine:latest'), + "volumeMounts": [{"name": "efs-volume", "mountPath": "/data"}], + } + + # Add command if specified in config + if 'command' in pod_config: + container["command"] = pod_config['command'] + elif not args: + # Default command if args not specified and command not in config + container["command"] = ["/bin/sh", "-c"] + + # Add args if available + if args: + container["args"] = args - pod_name = f"test-pod-{uuid.uuid4().hex[:8]}" - self.logger.info(f"Creating pod {pod_name} on PVC {pvc_name}") + # Add readiness probe + container["readinessProbe"] = self._build_readiness_probe(pod_config) - try: - # Get pod configuration from config - pod_config = self.config.get('pod_config', {}) + # Add resource constraints + container["resources"] = self._build_container_resources(pod_config) + + return container + + def _build_readiness_probe(self, pod_config): + """Build readiness probe configuration from pod config""" + readiness_probe = pod_config.get('readiness_probe', {}) + return { + "exec": { + "command": ["/bin/sh", "-c", "cat /data/pod-ready 2>/dev/null || cat /tmp/ready/pod-ready 2>/dev/null"] + }, + "initialDelaySeconds": readiness_probe.get('initial_delay_seconds', 15), + "periodSeconds": readiness_probe.get('period_seconds', 5), + "failureThreshold": readiness_probe.get('failure_threshold', 6), + "timeoutSeconds": readiness_probe.get('timeout_seconds', 5) + } + + def _build_container_resources(self, pod_config): + """Build container resources configuration from config""" + container_resources = self.config.get('pod_resources', {}) + return { + "requests": container_resources.get('requests', {"cpu": "100m", "memory": "64Mi"}), + "limits": container_resources.get('limits', {"cpu": "200m", "memory": "128Mi"}) + } + + def _build_pod_metadata(self, pod_name, pod_config): + """Build pod metadata from config""" + metadata = { + "name": pod_name, + "labels": {"app": "efs-test", "component": "stress-test"} + } + + # Add custom labels if specified + custom_labels = pod_config.get('labels', {}) + if custom_labels: + metadata['labels'].update(custom_labels) - # Create better startup script with diagnostic capabilities - startup_script = """ + return metadata + + def _build_pod_spec(self, container, pvc_name, pod_config): + """Build pod spec from container and config""" + pod_spec = { + "containers": [container], + "volumes": [{ + "name": "efs-volume", + "persistentVolumeClaim": {"claimName": pvc_name} + }], + "tolerations": [ + {"key": "node.kubernetes.io/not-ready", "operator": "Exists", "effect": "NoExecute", "tolerationSeconds": 300}, + {"key": "node.kubernetes.io/unreachable", "operator": "Exists", "effect": "NoExecute", "tolerationSeconds": 300} + ] + } + + # Add additional tolerations from config + if 'tolerations' in pod_config: + pod_spec['tolerations'].extend(pod_config['tolerations']) + + # Add node selector if specified in config + if 'node_selector' in pod_config: + pod_spec['nodeSelector'] = pod_config['node_selector'] + self.logger.info(f"Using node selector: {pod_config['node_selector']}") + + # Add any additional pod spec settings from config + pod_spec_settings = pod_config.get('pod_spec', {}) + for key, value in pod_spec_settings.items(): + if key not in pod_spec: + pod_spec[key] = value + + return pod_spec + + def _get_pod_startup_script(self): + return """ #!/bin/sh -# Diagnostic information collection echo "Pod $(hostname) starting up" -echo "Attempting to access /data directory" ls -la /data || echo "ERROR: Cannot access /data directory" - -# Try to create readiness file with retries for i in 1 2 3 4 5; do echo "Attempt $i to create readiness file" if touch /data/pod-ready; then @@ -336,294 +494,178 @@ def _attach_pod(self, pvc_name=None): echo "Failed to create readiness file on attempt $i" if [ $i -eq 5 ]; then echo "All attempts failed, creating alternative readiness file" - # Create alternative readiness file that doesn't depend on volume mount mkdir -p /tmp/ready && touch /tmp/ready/pod-ready fi sleep 2 fi done - -# Stay alive -while true; do - sleep 30 -done +while true; do sleep 30; done """ - # Create pod manifest with enhanced readiness check and debugging - pod_manifest = { - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "name": pod_name, - "labels": { - "app": "efs-test", - "component": "stress-test" - } - }, - "spec": { - "containers": [{ - "name": "test-container", - "image": pod_config.get('image', 'alpine:latest'), - "command": ["/bin/sh", "-c"], - "args": [startup_script], - "volumeMounts": [{ - "name": "efs-volume", - "mountPath": "/data" - }], - "readinessProbe": { - "exec": { - "command": ["/bin/sh", "-c", "cat /data/pod-ready 2>/dev/null || cat /tmp/ready/pod-ready 2>/dev/null"] - }, - "initialDelaySeconds": 15, - "periodSeconds": 5, - "failureThreshold": 6, - "timeoutSeconds": 5 - }, - "resources": { - "requests": { - "cpu": "100m", - "memory": "64Mi" - }, - "limits": { - "cpu": "200m", - "memory": "128Mi" - } - } - }], - "volumes": [{ - "name": "efs-volume", - "persistentVolumeClaim": { - "claimName": pvc_name - } - }], - "tolerations": [{ - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "effect": "NoExecute", - "tolerationSeconds": 300 - }, { - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "effect": "NoExecute", - "tolerationSeconds": 300 - }] - } - } - - # Add any additional tolerations from config - if 'tolerations' in pod_config: - pod_manifest['spec']['tolerations'].extend(pod_config['tolerations']) - - # Add node selector only if explicitly defined in config - # Otherwise, let Kubernetes schedule freely - if 'node_selector' in pod_config: - pod_manifest['spec']['nodeSelector'] = pod_config['node_selector'] - - # Create pod in Kubernetes - self.core_v1.create_namespaced_pod( - namespace=self.namespace, - body=pod_manifest - ) - - # Track the pod - self.pods[pvc_name].append(pod_name) - self.current_pod_count += 1 - - # Update results - self.results['attach_pod']['success'] += 1 - self.logger.info(f"Created pod: {pod_name} using PVC: {pvc_name}") - - # Wait for pod to be ready - if not self._wait_for_pod_ready(pod_name, timeout=60): - self.logger.warning(f"Timeout waiting for pod {pod_name} to be ready") - return None - - return pod_name - - except Exception as e: - self.results['attach_pod']['fail'] += 1 - self.logger.error(f"Failed to create pod: {e}") - return None - + + def _track_new_pod(self, pvc_name, pod_name): + self.pods[pvc_name].append(pod_name) + self.current_pod_count += 1 + self.results['attach_pod']['success'] += 1 + def _delete_pod(self, pod_name=None, pvc_name=None, force=False): """ - Delete a pod - If pod_name and pvc_name are provided, delete that specific pod - Otherwise, select a random pod - - Args: - pod_name: Name of pod to delete - pvc_name: Name of PVC associated with pod - force: If True, use force deletion with grace period 0 + Delete a pod. If pod_name and pvc_name are provided, delete that specific pod. Otherwise, select a random pod. """ - # Find all pods if not specified - if pod_name is None or pvc_name is None: - all_pods = [] - for pvc, pod_list in self.pods.items(): - all_pods.extend([(pvc, pod) for pod in pod_list]) - - if not all_pods: - self.logger.info("No pods to delete") - return False - - # Pick a random pod - pvc_name, pod_name = random.choice(all_pods) - elif pod_name not in self.pods.get(pvc_name, []): - self.logger.warning(f"Pod {pod_name} not found in PVC {pvc_name}") + pvc_name, pod_name = self._select_pod_for_deletion(pod_name, pvc_name) + if not pod_name: return False - self.logger.info(f"Deleting pod: {pod_name} from PVC: {pvc_name}") - try: - # Set up delete options - if force: - # Force delete with grace period 0 seconds - grace_period_seconds = 0 - propagation_policy = 'Background' - self.logger.info(f"Force deleting pod {pod_name} with grace period 0") - else: - # Normal delete with default grace period (usually 30s) - grace_period_seconds = None - propagation_policy = 'Foreground' - - # Delete the pod - self.core_v1.delete_namespaced_pod( - name=pod_name, - namespace=self.namespace, - grace_period_seconds=grace_period_seconds, - propagation_policy=propagation_policy - ) - - # Wait for the pod to be deleted + self._delete_pod_k8s(pod_name, force) if not self._wait_for_pod_deleted(pod_name): self.logger.warning(f"Timeout waiting for pod {pod_name} to be deleted") return False - - # Remove from tracking - if pod_name in self.pods[pvc_name]: - self.pods[pvc_name].remove(pod_name) - self.current_pod_count -= 1 - - # Update results - self.results['delete_pod']['success'] += 1 + self._untrack_deleted_pod(pvc_name, pod_name) self.logger.info(f"Deleted pod: {pod_name}") + self.results['delete_pod']['success'] += 1 return True - except Exception as e: self.results['delete_pod']['fail'] += 1 self.logger.error(f"Failed to delete pod {pod_name}: {e}") return False - + + def _select_pod_for_deletion(self, pod_name, pvc_name): + if pod_name is None or pvc_name is None: + all_pods = [(pvc, pod) for pvc, pod_list in self.pods.items() for pod in pod_list] + if not all_pods: + self.logger.info("No pods to delete") + return (None, None) + return random.choice(all_pods) + elif pod_name not in self.pods.get(pvc_name, []): + self.logger.warning(f"Pod {pod_name} not found in PVC {pvc_name}") + return (None, None) + return (pvc_name, pod_name) + + def _delete_pod_k8s(self, pod_name, force): + if force: + grace_period_seconds = 0 + propagation_policy = 'Background' + self.logger.info(f"Force deleting pod {pod_name} with grace period 0") + else: + grace_period_seconds = None + propagation_policy = 'Foreground' + self.core_v1.delete_namespaced_pod( + name=pod_name, + namespace=self.namespace, + grace_period_seconds=grace_period_seconds, + propagation_policy=propagation_policy + ) + + def _untrack_deleted_pod(self, pvc_name, pod_name): + if pod_name in self.pods.get(pvc_name, []): + self.pods[pvc_name].remove(pod_name) + self.current_pod_count -= 1 + def _delete_pvc(self, pvc_name=None, force=False): """ - Delete a PVC - If pvc_name is provided, delete that specific PVC - Otherwise, select a random PVC - - Args: - pvc_name: Name of PVC to delete - force: If True, use force deletion with grace period 0 + Delete a PVC. If pvc_name is provided, delete that specific PVC. Otherwise, select a random PVC. """ - if not self.pvcs: - self.logger.info("No PVCs to delete") + pvc_name = self._select_pvc_for_deletion(pvc_name) + if not pvc_name: return False - - # Pick a PVC if not specified - if pvc_name is None or pvc_name not in self.pvcs: - pvc_name = random.choice(self.pvcs) - self.logger.info(f"Deleting PVC: {pvc_name}") - - # Ensure no pods are using it - if self.pods[pvc_name]: - # Delete all pods using this PVC - self.logger.info(f"Deleting {len(self.pods[pvc_name])} pods using PVC {pvc_name}") - for pod_name in list(self.pods[pvc_name]): # Create a copy of the list to avoid modification during iteration - self._delete_pod(pod_name, pvc_name) - + self._delete_all_pods_for_pvc(pvc_name) try: - # Set up delete options - if force: - # Force delete with grace period 0 seconds - grace_period_seconds = 0 - propagation_policy = 'Background' - self.logger.info(f"Force deleting PVC {pvc_name} with grace period 0") - else: - # Normal delete with default grace period - grace_period_seconds = None - propagation_policy = 'Foreground' - - # Delete the PVC - self.core_v1.delete_namespaced_persistent_volume_claim( - name=pvc_name, - namespace=self.namespace, - grace_period_seconds=grace_period_seconds, - propagation_policy=propagation_policy - ) - - # Wait for the PVC to be deleted + self._delete_pvc_k8s(pvc_name, force) if not self._wait_for_pvc_deleted(pvc_name): self.logger.warning(f"Timeout waiting for PVC {pvc_name} to be deleted") return False - - # Remove from tracking - if pvc_name in self.pvcs: - self.pvcs.remove(pvc_name) - if pvc_name in self.pods: - del self.pods[pvc_name] - - # Update results - self.results['delete_pvc']['success'] += 1 + self._untrack_deleted_pvc(pvc_name) self.logger.info(f"Deleted PVC: {pvc_name}") + self.results['delete_pvc']['success'] += 1 return True - except Exception as e: self.results['delete_pvc']['fail'] += 1 self.logger.error(f"Failed to delete PVC {pvc_name}: {e}") return False - + + def _select_pvc_for_deletion(self, pvc_name): + if not self.pvcs: + self.logger.info("No PVCs to delete") + return None + if pvc_name is None or pvc_name not in self.pvcs: + return random.choice(self.pvcs) + return pvc_name + + def _delete_all_pods_for_pvc(self, pvc_name): + if self.pods.get(pvc_name): + self.logger.info(f"Deleting {len(self.pods[pvc_name])} pods using PVC {pvc_name}") + for pod_name in list(self.pods[pvc_name]): + self._delete_pod(pod_name, pvc_name) + + def _delete_pvc_k8s(self, pvc_name, force): + if force: + grace_period_seconds = 0 + propagation_policy = 'Background' + self.logger.info(f"Force deleting PVC {pvc_name} with grace period 0") + else: + grace_period_seconds = None + propagation_policy = 'Foreground' + self.core_v1.delete_namespaced_persistent_volume_claim( + name=pvc_name, + namespace=self.namespace, + grace_period_seconds=grace_period_seconds, + propagation_policy=propagation_policy + ) + + def _untrack_deleted_pvc(self, pvc_name): + if pvc_name in self.pvcs: + self.pvcs.remove(pvc_name) + if pvc_name in self.pods: + del self.pods[pvc_name] + def _verify_readwrite(self): """ Verify read/write operations between pods sharing a PVC This tests that pods sharing the same volume can see each other's writes - - Note: Using kubectl subprocesses instead of Kubernetes API to avoid - WebSocket upgrade errors - - Also collects performance metrics for file operations. """ # Find PVCs that have multiple pods shared_pvcs = [(pvc, pods) for pvc, pods in self.pods.items() if len(pods) >= 2] - if not shared_pvcs: self.logger.info("No shared PVCs with multiple pods for read/write test") return - - # Pick a random shared PVC pvc_name, pod_names = random.choice(shared_pvcs) if len(pod_names) < 2: return - - # Pick two distinct pods writer_pod = random.choice(pod_names) reader_pod = random.choice([p for p in pod_names if p != writer_pod]) - test_file = f"test-{uuid.uuid4().hex[:8]}.txt" - test_content = f"Test content: {uuid.uuid4()}" * 50 # Make content larger for better measurements + test_content = f"Test content: {uuid.uuid4()}" * 50 content_size_bytes = len(test_content.encode('utf-8')) - self.logger.info(f"Testing read/write between pods {writer_pod} and {reader_pod} sharing PVC {pvc_name}") self.logger.info(f"File size: {content_size_bytes} bytes") - try: - import subprocess - - # Track write operation with metrics - write_op_start = time.time() - - # Write with first pod using kubectl - write_cmd = f"kubectl exec -n {self.namespace} {writer_pod} -- /bin/sh -c 'echo \"{test_content}\" > /data/{test_file}'" - self.logger.info(f"Executing write command: {write_cmd}") - + write_success, write_duration = self._run_write_op(writer_pod, test_file, test_content, pvc_name, content_size_bytes) + if not write_success: + self._track_rw_failure('write') + self._track_scenario_failure('shared_volume_rw') + return + time.sleep(2) + read_success, read_duration, resp = self._run_read_op(reader_pod, test_file, test_content, pvc_name, content_size_bytes) + if read_success: + self._track_rw_success('read') + self._track_scenario_success('shared_volume_rw') + self._run_metadata_ls(reader_pod, pvc_name) + else: + self._track_rw_failure('read') + self._track_scenario_failure('shared_volume_rw') + self.logger.error(f"Pod {reader_pod} failed to read content written by {writer_pod}. Got different content length: {len(resp)} vs expected {len(test_content)}") + except Exception as e: + self.logger.error(f"Failed in read/write verification: {e}") + self._track_rw_failure('write') + self._track_scenario_failure('shared_volume_rw') + + def _run_write_op(self, writer_pod, test_file, test_content, pvc_name, content_size_bytes): + import subprocess + write_op_start = time.time() + write_cmd = f"kubectl exec -n {self.namespace} {writer_pod} -- /bin/sh -c 'echo \"{test_content}\" > /data/{test_file}'" + self.logger.info(f"Executing write command: {write_cmd}") + try: write_process = subprocess.run( write_cmd, shell=True, @@ -632,33 +674,26 @@ def _verify_readwrite(self): stderr=subprocess.PIPE, text=True ) - - # Measure write operation duration write_duration = time.time() - write_op_start - write_ops_count = 1 # One write operation - - # Track metrics for write operation self.metrics_collector.track_file_operation_latency(pvc_name, "write", write_duration) - self.metrics_collector.track_file_operation_iops(pvc_name, "write", write_ops_count, write_duration) + self.metrics_collector.track_file_operation_iops(pvc_name, "write", 1, write_duration) self.metrics_collector.track_file_operation_throughput(pvc_name, "write", content_size_bytes, write_duration) - self.logger.info(f"Write operation completed in {write_duration:.3f}s") self.logger.info(f"Write throughput: {(content_size_bytes / 1024 / 1024) / write_duration:.2f} MB/s") - - # Update results - self.results['verify_write']['success'] += 1 + self._track_rw_success('write') self.logger.info(f"Pod {writer_pod} wrote to /data/{test_file}") - - # Sleep briefly to ensure the write completes and is visible - time.sleep(2) - - # Track read operation with metrics - read_op_start = time.time() - - # Read with second pod using kubectl - read_cmd = f"kubectl exec -n {self.namespace} {reader_pod} -- cat /data/{test_file}" - self.logger.info(f"Executing read command: {read_cmd}") - + return True, write_duration + except subprocess.CalledProcessError as e: + self.logger.error(f"Write command execution failed: {e}") + self.logger.error(f"Command stderr: {e.stderr}") + return False, 0 + + def _run_read_op(self, reader_pod, test_file, test_content, pvc_name, content_size_bytes): + import subprocess + read_op_start = time.time() + read_cmd = f"kubectl exec -n {self.namespace} {reader_pod} -- cat /data/{test_file}" + self.logger.info(f"Executing read command: {read_cmd}") + try: read_process = subprocess.run( read_cmd, shell=True, @@ -667,81 +702,65 @@ def _verify_readwrite(self): stderr=subprocess.PIPE, text=True ) - - # Measure read operation duration read_duration = time.time() - read_op_start - read_ops_count = 1 # One read operation - - # Track metrics for read operation self.metrics_collector.track_file_operation_latency(pvc_name, "read", read_duration) - self.metrics_collector.track_file_operation_iops(pvc_name, "read", read_ops_count, read_duration) + self.metrics_collector.track_file_operation_iops(pvc_name, "read", 1, read_duration) self.metrics_collector.track_file_operation_throughput(pvc_name, "read", content_size_bytes, read_duration) - resp = read_process.stdout.strip() - self.logger.info(f"Read operation completed in {read_duration:.3f}s") self.logger.info(f"Read throughput: {(content_size_bytes / 1024 / 1024) / read_duration:.2f} MB/s") self.logger.info(f"Read result length: {len(resp)} bytes") - - # Verify content if test_content in resp: - self.results['verify_read']['success'] += 1 - self.logger.info(f"Pod {reader_pod} successfully read content written by {writer_pod}") - - # Update scenario tracking - self.scenarios['shared_volume_rw']['runs'] += 1 - self.scenarios['shared_volume_rw']['success'] += 1 - - # Add metadata operation metrics - perform ls to measure metadata operations - meta_op_start = time.time() - ls_cmd = f"kubectl exec -n {self.namespace} {reader_pod} -- ls -la /data/" - - ls_process = subprocess.run( - ls_cmd, - shell=True, - check=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True - ) - - meta_duration = time.time() - meta_op_start - meta_ops_count = 1 # One metadata operation - - # Track metrics for metadata operation - self.metrics_collector.track_file_operation_latency(pvc_name, "metadata", meta_duration) - self.metrics_collector.track_file_operation_iops(pvc_name, "metadata", meta_ops_count, meta_duration) - - self.logger.info(f"Metadata operation (ls) completed in {meta_duration:.3f}s") - + self.logger.info(f"Pod {reader_pod} successfully read content written by writer pod") + return True, read_duration, resp else: - self.results['verify_read']['fail'] += 1 - self.scenarios['shared_volume_rw']['runs'] += 1 - self.scenarios['shared_volume_rw']['fail'] += 1 - self.logger.error(f"Pod {reader_pod} failed to read content written by {writer_pod}. Got different content length: {len(resp)} vs expected {len(test_content)}") - + return False, read_duration, resp except subprocess.CalledProcessError as e: - self.logger.error(f"Command execution failed: {e}") + self.logger.error(f"Read command execution failed: {e}") self.logger.error(f"Command stderr: {e.stderr}") - if 'write_process' not in locals(): - self.results['verify_write']['fail'] += 1 - else: - self.results['verify_read']['fail'] += 1 - - # Update scenario tracking - self.scenarios['shared_volume_rw']['runs'] += 1 - self.scenarios['shared_volume_rw']['fail'] += 1 - except Exception as e: - self.logger.error(f"Failed in read/write verification: {e}") - if 'write_process' not in locals(): - self.results['verify_write']['fail'] += 1 - else: - self.results['verify_read']['fail'] += 1 - - # Update scenario tracking - self.scenarios['shared_volume_rw']['runs'] += 1 - self.scenarios['shared_volume_rw']['fail'] += 1 - + return False, 0, '' + + def _run_metadata_ls(self, reader_pod, pvc_name): + import subprocess + meta_op_start = time.time() + ls_cmd = f"kubectl exec -n {self.namespace} {reader_pod} -- ls -la /data/" + try: + ls_process = subprocess.run( + ls_cmd, + shell=True, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + meta_duration = time.time() - meta_op_start + self.metrics_collector.track_file_operation_latency(pvc_name, "metadata", meta_duration) + self.metrics_collector.track_file_operation_iops(pvc_name, "metadata", 1, meta_duration) + self.logger.info(f"Metadata operation (ls) completed in {meta_duration:.3f}s") + except subprocess.CalledProcessError as e: + self.logger.error(f"Metadata ls command failed: {e}") + self.logger.error(f"Command stderr: {e}") + + def _track_rw_success(self, op_type): + if op_type == 'write': + self.results['verify_write']['success'] += 1 + elif op_type == 'read': + self.results['verify_read']['success'] += 1 + + def _track_rw_failure(self, op_type): + if op_type == 'write': + self.results['verify_write']['fail'] += 1 + elif op_type == 'read': + self.results['verify_read']['fail'] += 1 + + def _track_scenario_success(self, scenario): + self.scenarios[scenario]['runs'] += 1 + self.scenarios[scenario]['success'] += 1 + + def _track_scenario_failure(self, scenario): + self.scenarios[scenario]['runs'] += 1 + self.scenarios[scenario]['fail'] += 1 + def _run_specific_scenario(self): """ Run a specific test scenario @@ -781,28 +800,62 @@ def _scenario_many_to_one(self): self.logger.info("STARTING MANY-TO-ONE SCENARIO DIAGNOSTICS") self.logger.info("+" * 80) self.scenarios['many_to_one']['runs'] += 1 - - # Create PVC + try: + pvc_name = self._create_many_to_one_pvc() + if not pvc_name: + self.scenarios['many_to_one']['fail'] += 1 + return + pod_names = self._create_many_to_one_pods(pvc_name) + if len(pod_names) < 2: + self.logger.error(f"[MANY2ONE] FAILED: Insufficient pods created ({len(pod_names)}), need at least 2 for read/write test") + self.scenarios['many_to_one']['fail'] += 1 + return + success = self._test_many_to_one_rw(pvc_name, pod_names) + if success: + self.logger.info(f"[MANY2ONE] SUCCESS: Many-to-one scenario successful with {len(pod_names)} pods") + self.scenarios['many_to_one']['success'] += 1 + else: + self.scenarios['many_to_one']['fail'] += 1 + self._collect_many_to_one_failure_logs(pvc_name, pod_names) + except Exception as e: + self.logger.error(f"[MANY2ONE] FAILED: Unhandled error in many-to-one scenario: {e}") + self.scenarios['many_to_one']['fail'] += 1 + self.logger.info("+" * 80) + self.logger.info("COMPLETED MANY-TO-ONE SCENARIO DIAGNOSTICS") + self.logger.info("+" * 80) + + def _create_many_to_one_pvc(self): + """Create a PVC for many-to-one scenario using configuration""" + # Generate PVC name with unique identifier pvc_name = f"many2one-{uuid.uuid4().hex[:8]}" + self.logger.info(f"[MANY2ONE] STEP 1: Creating dedicated PVC: {pvc_name}") try: - # Create PVC manifest + # Get configuration values + scenario_config = self.config.get('scenarios', {}).get('many_to_one', {}) + sc_name = self.config.get('storage_class', {}).get('name', 'efs-sc') + + # Create PVC manifest using config values pvc_manifest = { "apiVersion": "v1", "kind": "PersistentVolumeClaim", "metadata": {"name": pvc_name}, "spec": { - "accessModes": ["ReadWriteMany"], - "storageClassName": "efs-sc", + "accessModes": ["ReadWriteMany"], # This is generally fixed for EFS + "storageClassName": sc_name, "resources": { - "requests": {"storage": "1Gi"} + "requests": {"storage": "1Gi"} # Size doesn't matter for EFS but required in PVC spec } } } + # Add annotations if configured + pvc_annotations = scenario_config.get('pvc_annotations', {}) + if pvc_annotations: + pvc_manifest['metadata']['annotations'] = pvc_annotations + # Create PVC - self.logger.info(f"[MANY2ONE] Creating PVC manifest with storageClassName: {pvc_manifest['spec']['storageClassName']}") self.core_v1.create_namespaced_persistent_volume_claim( namespace=self.namespace, body=pvc_manifest @@ -811,95 +864,229 @@ def _scenario_many_to_one(self): # Track the PVC self.pvcs.append(pvc_name) self.pods[pvc_name] = [] - self.logger.info(f"[MANY2ONE] PVC {pvc_name} created successfully") - # Wait for PVC to be bound - self.logger.info(f"[MANY2ONE] Waiting for PVC {pvc_name} to be bound...") - if not self._wait_for_pvc_bound(pvc_name, timeout=30): - self.logger.error(f"[MANY2ONE] FAILED: Timeout waiting for PVC {pvc_name} to be bound") - self.scenarios['many_to_one']['fail'] += 1 - return - - # Get PVC status to verify it's bound correctly - try: - pvc_status = self.core_v1.read_namespaced_persistent_volume_claim_status( - name=pvc_name, - namespace=self.namespace - ) - self.logger.info(f"[MANY2ONE] PVC status: phase={pvc_status.status.phase}, capacity={pvc_status.status.capacity}") - except Exception as e: - self.logger.warning(f"[MANY2ONE] Could not get PVC status: {e}") - - # Create multiple pods (3-5) - num_pods = random.randint(3, 5) - self.logger.info(f"[MANY2ONE] STEP 2: Creating {num_pods} pods for the same PVC {pvc_name}") - - pod_names = [] - for i in range(num_pods): - self.logger.info(f"[MANY2ONE] Creating pod {i+1}/{num_pods} for PVC {pvc_name}") - pod_name = self._attach_pod(pvc_name) - if pod_name: - self.logger.info(f"[MANY2ONE] Successfully created and attached pod {pod_name}") - pod_names.append(pod_name) - else: - self.logger.error(f"[MANY2ONE] Failed to create pod {i+1}/{num_pods}") - - self.logger.info(f"[MANY2ONE] Created {len(pod_names)}/{num_pods} pods successfully") - - if len(pod_names) < 2: - self.logger.error(f"[MANY2ONE] FAILED: Insufficient pods created ({len(pod_names)}), need at least 2 for read/write test") - self.scenarios['many_to_one']['fail'] += 1 - return - - # Test read/write between two pods - test_file = f"many2one-{uuid.uuid4().hex[:8]}.txt" - test_content = f"Many2One test content: {uuid.uuid4()}" - - # Select two random pods - writer_pod = random.choice(pod_names) - reader_pod = random.choice([p for p in pod_names if p != writer_pod]) + # Get timeout from config + retry_config = self.config.get('retries', {}) + pvc_bind_timeout = retry_config.get('pvc_bind_timeout', 30) + + self.logger.info(f"[MANY2ONE] PVC {pvc_name} created with storage class {sc_name}") + + # Wait for PVC to be bound with configured timeout + if not self._wait_for_pvc_bound(pvc_name, timeout=pvc_bind_timeout): + self.logger.error(f"[MANY2ONE] FAILED: Timeout waiting for PVC {pvc_name} to be bound after {pvc_bind_timeout}s") + return None + + return pvc_name + + except Exception as e: + self.logger.error(f"[MANY2ONE] FAILED: Error creating PVC: {e}") + return None + + def _create_many_to_one_pods(self, pvc_name): + # Get pod count range from config or use defaults + scenario_config = self.config.get('scenarios', {}).get('many_to_one', {}) + min_pods = scenario_config.get('min_pods', 3) + max_pods = scenario_config.get('max_pods', 5) + num_pods = random.randint(min_pods, max_pods) + + self.logger.info(f"[MANY2ONE] STEP 2: Creating {num_pods} pods for the same PVC {pvc_name}") + pod_names = [] + for i in range(num_pods): + self.logger.info(f"[MANY2ONE] Creating pod {i+1}/{num_pods} for PVC {pvc_name}") + pod_name = self._attach_pod(pvc_name) + if pod_name: + self.logger.info(f"[MANY2ONE] Successfully created and attached pod {pod_name}") + pod_names.append(pod_name) + else: + self.logger.error(f"[MANY2ONE] Failed to create pod {i+1}/{num_pods}") + self.logger.info(f"[MANY2ONE] Created {len(pod_names)}/{num_pods} pods successfully") + return pod_names + + def _test_many_to_one_rw(self, pvc_name, pod_names): + import subprocess + test_file = f"many2one-{uuid.uuid4().hex[:8]}.txt" + test_content = f"Many2One test content: {uuid.uuid4()}" + writer_pod = random.choice(pod_names) + reader_pod = random.choice([p for p in pod_names if p != writer_pod]) + self.logger.info(f"[MANY2ONE] STEP 3: Testing read/write operations") + self.logger.info(f"[MANY2ONE] Writer pod: {writer_pod}, Reader pod: {reader_pod}") + try: + write_cmd = f"kubectl exec -n {self.namespace} {writer_pod} -- /bin/sh -c 'echo \"{test_content}\" > /data/{test_file}'" + self.logger.info(f"[MANY2ONE] Executing write command: {write_cmd}") + subprocess.run(write_cmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + time.sleep(5) + read_cmd = f"kubectl exec -n {self.namespace} {reader_pod} -- cat /data/{test_file}" + self.logger.info(f"[MANY2ONE] Executing read command: {read_cmd}") + read_process = subprocess.run(read_cmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + read_result = read_process.stdout.strip() + self.logger.info(f"[MANY2ONE] Read command result: '{read_result}'") + return test_content in read_result + except Exception as e: + self.logger.error(f"[MANY2ONE] FAILED: Error during read/write test: {e}") + return False + + def _collect_many_to_one_failure_logs(self, pvc_name, pod_names): + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + test_name = f"many2one_failure_{timestamp}" + failed_resources = ( + [{"type": "pod", "name": pod, "namespace": self.namespace} for pod in pod_names] + + [{"type": "pvc", "name": pvc_name, "namespace": self.namespace}] + ) + logs_path = collect_logs_on_test_failure( + test_name, + self.metrics_collector, + self.driver_pod_name, + failed_resources=failed_resources + ) + self.logger.info(f"Collected detailed failure logs to: {logs_path}") + + def _scenario_one_to_one(self): + """ + Test one pod per PVC scenario + 1. Create multiple PVCs + 2. Create one pod per PVC + 3. Verify each pod can write to its own volume using kubectl subprocess + 4. Clean up + """ + self.logger.info("Running scenario: One pod per PVC") + self.scenarios['one_to_one']['runs'] += 1 + + # Get pair count range from config + scenario_config = self.config.get('scenarios', {}).get('one_to_one', {}) + min_pairs = scenario_config.get('min_pairs', 3) + max_pairs = scenario_config.get('max_pairs', 5) + + # Use configured values instead of hardcoded ones + num_pairs = random.randint(min_pairs, max_pairs) + self.logger.info(f"Creating {num_pairs} PVC-pod pairs (range from config: {min_pairs}-{max_pairs})") + pairs = self._create_one_to_one_pairs(num_pairs) + if len(pairs) < 2: + self.logger.warning(f"Failed to create enough PVC-pod pairs, only created {len(pairs)}") + self.scenarios['one_to_one']['fail'] += 1 + return + success = self._test_one_to_one_rw(pairs) + if success: + self.logger.info(f"[ONE2ONE] One-to-one scenario successful with {len(pairs)} PVC-pod pairs") + self.scenarios['one_to_one']['success'] += 1 + else: + self.logger.error("[ONE2ONE] One-to-one scenario failed") + self.scenarios['one_to_one']['fail'] += 1 + self._collect_one_to_one_failure_logs(pairs) + + def _create_one_to_one_pairs(self, num_pairs): + """Create pairs of PVCs and pods for one-to-one scenario""" + # Get the number of pairs to create + num_pairs = self._get_one_to_one_pair_count(num_pairs) + self.logger.info(f"[ONE2ONE] Creating {num_pairs} PVC-pod pairs") + + # Create the pairs + pairs = [] + for i in range(num_pairs): + pair = self._create_one_to_one_pair() + if pair: + pairs.append(pair) + + return pairs + + def _get_one_to_one_pair_count(self, requested_pairs): + """Determine how many PVC-pod pairs to create based on config and request""" + # Get configuration for one-to-one scenario + scenario_config = self.config.get('scenarios', {}).get('one_to_one', {}) + min_pairs = scenario_config.get('min_pairs', 3) + max_pairs = scenario_config.get('max_pairs', 5) + + # If requested_pairs wasn't specified, use configured range + if requested_pairs <= 0: + pairs = random.randint(min_pairs, max_pairs) + self.logger.info(f"[ONE2ONE] Using configured range: creating {pairs} PVC-pod pairs") + return pairs + + return requested_pairs + + def _create_one_to_one_pair(self): + """Create a single PVC-pod pair for one-to-one scenario""" + # Generate PVC name + pvc_name = f"one2one-{uuid.uuid4().hex[:8]}" + + # Create the PVC manifest + pvc_manifest = self._build_one_to_one_pvc_manifest(pvc_name) + + try: + # Create the PVC + self.core_v1.create_namespaced_persistent_volume_claim( + namespace=self.namespace, + body=pvc_manifest + ) + self.pvcs.append(pvc_name) + self.pods[pvc_name] = [] + + # Get timeout from config + retry_config = self.config.get('retries', {}) + pvc_bind_timeout = retry_config.get('pvc_bind_timeout', 30) - self.logger.info(f"[MANY2ONE] STEP 3: Testing read/write operations") - self.logger.info(f"[MANY2ONE] Writer pod: {writer_pod}, Reader pod: {reader_pod}") - self.logger.info(f"[MANY2ONE] Test file: /data/{test_file}") - self.logger.info(f"[MANY2ONE] Test content: {test_content}") + # Wait for PVC to be bound + if not self._wait_for_pvc_bound(pvc_name, timeout=pvc_bind_timeout): + self.logger.warning(f"[ONE2ONE] Timeout waiting for PVC {pvc_name} to be bound after {pvc_bind_timeout}s") + return None + + # Create and attach pod + pod_name = self._attach_pod(pvc_name) + if pod_name: + self.logger.info(f"[ONE2ONE] Successfully created pair: PVC {pvc_name}, Pod {pod_name}") + return (pvc_name, pod_name) + + return None - import subprocess + except Exception as e: + self.logger.error(f"[ONE2ONE] Error creating PVC or pod: {e}") + return None + + def _build_one_to_one_pvc_manifest(self, pvc_name): + """Build PVC manifest for one-to-one scenario""" + scenario_config = self.config.get('scenarios', {}).get('one_to_one', {}) + + # Get storage class name from config + sc_name = self.config.get('storage_class', {}).get('name', 'efs-sc') + + # Create base manifest + pvc_manifest = { + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": {"name": pvc_name}, + "spec": { + "accessModes": ["ReadWriteMany"], + "storageClassName": sc_name, + "resources": {"requests": {"storage": "1Gi"}} + } + } + + # Add any PVC annotations if configured + pvc_annotations = scenario_config.get('pvc_annotations', {}) + if pvc_annotations: + if 'metadata' not in pvc_manifest: + pvc_manifest['metadata'] = {} + pvc_manifest['metadata']['annotations'] = pvc_annotations + return pvc_manifest + + def _test_one_to_one_rw(self, pairs): + import subprocess + for pvc_name, pod_name in pairs: + test_file = f"one2one-{uuid.uuid4().hex[:8]}.txt" + test_content = f"One2One test content for {pvc_name}: {uuid.uuid4()}" try: - # Write with first pod using kubectl subprocess - write_cmd = f"kubectl exec -n {self.namespace} {writer_pod} -- /bin/sh -c 'echo \"{test_content}\" > /data/{test_file}'" - self.logger.info(f"[MANY2ONE] Executing write command: {write_cmd}") - - write_process = subprocess.run( - write_cmd, - shell=True, - check=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True - ) - - # Verify the file exists in writer pod - ls_cmd = f"kubectl exec -n {self.namespace} {writer_pod} -- ls -la /data/{test_file}" - ls_process = subprocess.run( - ls_cmd, + write_cmd = f"kubectl exec -n {self.namespace} {pod_name} -- /bin/sh -c 'echo \"{test_content}\" > /data/{test_file}'" + self.logger.info(f"[ONE2ONE] Executing write command for pod {pod_name}: {write_cmd}") + subprocess.run( + write_cmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True ) - self.logger.info(f"[MANY2ONE] Writer pod ls check: '{ls_process.stdout.strip()}'") - - # Sleep to ensure filesystem sync - self.logger.info(f"[MANY2ONE] Sleeping for 5 seconds to ensure filesystem sync") - time.sleep(5) - - # Read with second pod using kubectl subprocess - read_cmd = f"kubectl exec -n {self.namespace} {reader_pod} -- cat /data/{test_file}" - self.logger.info(f"[MANY2ONE] Executing read command: {read_cmd}") - + read_cmd = f"kubectl exec -n {self.namespace} {pod_name} -- cat /data/{test_file}" + self.logger.info(f"[ONE2ONE] Executing read command for pod {pod_name}: {read_cmd}") read_process = subprocess.run( read_cmd, shell=True, @@ -908,302 +1095,150 @@ def _scenario_many_to_one(self): stderr=subprocess.PIPE, text=True ) - read_result = read_process.stdout.strip() - self.logger.info(f"[MANY2ONE] Read command result: '{read_result}'") - - # Verify content - if test_content in read_result: - self.logger.info(f"[MANY2ONE] SUCCESS: Many-to-one scenario successful with {len(pod_names)} pods") - self.scenarios['many_to_one']['success'] += 1 + self.logger.info(f"[ONE2ONE] Pod {pod_name} read result: '{read_result}'") + if test_content not in read_result: + self.logger.error(f"[ONE2ONE] Pod {pod_name} failed to read its own write. Expected '{test_content}', got '{read_result}'") + return False else: - self.logger.error(f"[MANY2ONE] FAILED: Pods cannot share data - Expected '{test_content}', got '{read_result}'") - - # Check mount info using kubectl subprocess - mount_cmd = f"kubectl exec -n {self.namespace} {writer_pod} -- mount | grep /data" - mount_process = subprocess.run( - mount_cmd, - shell=True, - check=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True - ) - self.logger.info(f"[MANY2ONE] Writer pod mount info: '{mount_process.stdout.strip()}'") - - reader_mount_cmd = f"kubectl exec -n {self.namespace} {reader_pod} -- mount | grep /data" - reader_mount_process = subprocess.run( - reader_mount_cmd, - shell=True, - check=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True - ) - self.logger.info(f"[MANY2ONE] Reader pod mount info: '{reader_mount_process.stdout.strip()}'") - - self.scenarios['many_to_one']['fail'] += 1 - # Collect logs for failure diagnostics with detailed information about failed resources - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - test_name = f"many2one_failure_{timestamp}" - - # Track failed resources for detailed logging - failed_resources = [ - # Add any pods involved in the scenario - {"type": "pod", "name": writer_pod, "namespace": self.namespace}, - {"type": "pod", "name": reader_pod, "namespace": self.namespace}, - # Add the PVC - {"type": "pvc", "name": pvc_name, "namespace": self.namespace} - ] - - logs_path = collect_logs_on_test_failure( - test_name, - self.metrics_collector, - self.driver_pod_name, - failed_resources=failed_resources - ) - self.logger.info(f"Collected detailed failure logs to: {logs_path}") - + self.logger.info(f"[ONE2ONE] Pod {pod_name} successfully wrote and read from its own volume") except subprocess.CalledProcessError as e: - self.logger.error(f"[MANY2ONE] Command execution failed: {e}") - self.logger.error(f"[MANY2ONE] Command stderr: {e.stderr}") - self.scenarios['many_to_one']['fail'] += 1 + self.logger.error(f"[ONE2ONE] Command execution failed for pod {pod_name}: {e}") + self.logger.error(f"[ONE2ONE] Command stderr: {e.stderr}") + return False except Exception as e: - self.logger.error(f"[MANY2ONE] FAILED: Error during read/write test: {e}") - self.scenarios['many_to_one']['fail'] += 1 - - except Exception as e: - self.logger.error(f"[MANY2ONE] FAILED: Unhandled error in many-to-one scenario: {e}") - self.scenarios['many_to_one']['fail'] += 1 - - self.logger.info("+" * 80) - self.logger.info("COMPLETED MANY-TO-ONE SCENARIO DIAGNOSTICS") - self.logger.info("+" * 80) - - # Clean up (will be managed by the orchestrator's cleanup) - - def _scenario_one_to_one(self): - """ - Test one pod per PVC scenario - 1. Create multiple PVCs - 2. Create one pod per PVC - 3. Verify each pod can write to its own volume using kubectl subprocess - 4. Clean up - """ - self.logger.info("Running scenario: One pod per PVC") - self.scenarios['one_to_one']['runs'] += 1 - - num_pairs = random.randint(3, 5) - self.logger.info(f"Creating {num_pairs} PVC-pod pairs") - - pairs = [] - - try: - # Create multiple PVC-pod pairs - for i in range(num_pairs): - # Create PVC - pvc_name = f"one2one-{uuid.uuid4().hex[:8]}" - - # Create PVC manifest - pvc_manifest = { - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "metadata": {"name": pvc_name}, - "spec": { - "accessModes": ["ReadWriteMany"], - "storageClassName": "efs-sc", - "resources": { - "requests": {"storage": "1Gi"} - } - } - } - - # Create PVC - self.core_v1.create_namespaced_persistent_volume_claim( - namespace=self.namespace, - body=pvc_manifest - ) - - # Track the PVC - self.pvcs.append(pvc_name) - self.pods[pvc_name] = [] - - # Wait for PVC to be bound - if not self._wait_for_pvc_bound(pvc_name, timeout=30): - self.logger.warning(f"Timeout waiting for PVC {pvc_name} to be bound") - continue - - # Create a pod for this PVC - pod_name = self._attach_pod(pvc_name) - if pod_name: - pairs.append((pvc_name, pod_name)) - - if len(pairs) < 2: - self.logger.warning(f"Failed to create enough PVC-pod pairs, only created {len(pairs)}") - self.scenarios['one_to_one']['fail'] += 1 - return - - # Test that each pod can write to its own volume using kubectl subprocess - import subprocess - success = True - - for pvc_name, pod_name in pairs: - test_file = f"one2one-{uuid.uuid4().hex[:8]}.txt" - test_content = f"One2One test content for {pvc_name}: {uuid.uuid4()}" - - try: - # Write to file using kubectl - write_cmd = f"kubectl exec -n {self.namespace} {pod_name} -- /bin/sh -c 'echo \"{test_content}\" > /data/{test_file}'" - self.logger.info(f"[ONE2ONE] Executing write command for pod {pod_name}: {write_cmd}") - - write_process = subprocess.run( - write_cmd, - shell=True, - check=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True - ) - - # Read the file back to verify using kubectl - read_cmd = f"kubectl exec -n {self.namespace} {pod_name} -- cat /data/{test_file}" - self.logger.info(f"[ONE2ONE] Executing read command for pod {pod_name}: {read_cmd}") - - read_process = subprocess.run( - read_cmd, - shell=True, - check=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True - ) - - read_result = read_process.stdout.strip() - self.logger.info(f"[ONE2ONE] Pod {pod_name} read result: '{read_result}'") - - if test_content not in read_result: - self.logger.error(f"[ONE2ONE] Pod {pod_name} failed to read its own write. Expected '{test_content}', got '{read_result}'") - success = False - break - else: - self.logger.info(f"[ONE2ONE] Pod {pod_name} successfully wrote and read from its own volume") - - except subprocess.CalledProcessError as e: - self.logger.error(f"[ONE2ONE] Command execution failed for pod {pod_name}: {e}") - self.logger.error(f"[ONE2ONE] Command stderr: {e.stderr}") - success = False - break - except Exception as e: - self.logger.error(f"[ONE2ONE] Error in one-to-one scenario for pod {pod_name}: {e}") - success = False - break - - if success: - self.logger.info(f"[ONE2ONE] One-to-one scenario successful with {len(pairs)} PVC-pod pairs") - self.scenarios['one_to_one']['success'] += 1 - else: - self.logger.error("[ONE2ONE] One-to-one scenario failed") - self.scenarios['one_to_one']['fail'] += 1 - - # Collect logs for failure diagnostics with detailed information - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - test_name = f"one2one_failure_{timestamp}" - - # Track all failed resources for detailed logging - failed_resources = [] - - # Add all PVC-pod pairs - for pvc_name, pod_name in pairs: - failed_resources.append({"type": "pod", "name": pod_name, "namespace": self.namespace}) - failed_resources.append({"type": "pvc", "name": pvc_name, "namespace": self.namespace}) - - logs_path = collect_logs_on_test_failure( - test_name, - self.metrics_collector, - self.driver_pod_name, - failed_resources=failed_resources - ) - self.logger.info(f"Collected detailed failure logs to: {logs_path}") - - except Exception as e: - self.logger.error(f"[ONE2ONE] Unhandled error in one-to-one scenario: {e}") - self.scenarios['one_to_one']['fail'] += 1 - + self.logger.error(f"[ONE2ONE] Error in one-to-one scenario for pod {pod_name}: {e}") + return False + return True + + def _collect_one_to_one_failure_logs(self, pairs): + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + test_name = f"one2one_failure_{timestamp}" + failed_resources = [] + for pvc_name, pod_name in pairs: + failed_resources.append({"type": "pod", "name": pod_name, "namespace": self.namespace}) + failed_resources.append({"type": "pvc", "name": pvc_name, "namespace": self.namespace}) + logs_path = collect_logs_on_test_failure( + test_name, + self.metrics_collector, + self.driver_pod_name, + failed_resources=failed_resources + ) + self.logger.info(f"Collected detailed failure logs to: {logs_path}") + def _scenario_concurrent_pvc(self): """ Test rapid PVC creation and deletion 1. Create multiple PVCs in quick succession 2. Create pods for some of them 3. Delete some PVCs in quick succession - 4. Verify operations complete successfully + 4. Verify operations successfully """ self.logger.info("Running scenario: Rapid PVC operations") self.scenarios['concurrent_pvc']['runs'] += 1 + # Get PVC count range from config + scenario_config = self.config.get('scenarios', {}).get('concurrent_pvc', {}) + min_pvcs = scenario_config.get('min_pvcs', 3) + max_pvcs = scenario_config.get('max_pvcs', 7) + # Number of PVCs to create - num_pvcs = random.randint(3, 7) - self.logger.info(f"Creating {num_pvcs} PVCs in quick succession") + num_pvcs = random.randint(min_pvcs, max_pvcs) + self.logger.info(f"Creating {num_pvcs} PVCs (range from config: {min_pvcs}-{max_pvcs})") pvc_names = [f"concurrent-pvc-{uuid.uuid4().hex[:8]}" for _ in range(num_pvcs)] created_pvcs = [] try: - # Create multiple PVCs in quick succession - for pvc_name in pvc_names: - success = self._create_pvc_for_concurrent(pvc_name) - if success: - created_pvcs.append(pvc_name) + # Step 1: Create multiple PVCs in quick succession + created_pvcs = self._concurrent_create_pvcs(pvc_names) if len(created_pvcs) < 2: - self.logger.warning(f"Failed to create enough PVCs, only created {len(created_pvcs)}") - self.scenarios['concurrent_pvc']['fail'] += 1 + self._mark_concurrent_scenario_failed(f"Failed to create enough PVCs, only created {len(created_pvcs)}") return - self.logger.info(f"Successfully created {len(created_pvcs)} PVCs") - - # Create pods for some of the PVCs - num_pods = min(len(created_pvcs), 3) - pod_pvcs = random.sample(created_pvcs, num_pods) + # Step 2: Create pods for some of the PVCs + self._concurrent_create_pods(created_pvcs) - for pvc_name in pod_pvcs: - self._attach_pod(pvc_name) + # Step 3: Delete some PVCs in quick succession + self._concurrent_delete_pvcs(created_pvcs) - # Delete some PVCs in quick succession - num_to_delete = min(len(created_pvcs), 3) - pvcs_to_delete = random.sample(created_pvcs, num_to_delete) - - for pvc_name in pvcs_to_delete: - self._delete_pvc(pvc_name) - - self.logger.info(f"Rapid PVC scenario completed") + # Mark scenario as successful + self.logger.info("Rapid PVC scenario completed successfully") self.scenarios['concurrent_pvc']['success'] += 1 except Exception as e: - self.logger.error(f"Error in rapid PVC scenario: {e}") - self.scenarios['concurrent_pvc']['fail'] += 1 - - # Collect logs for failure diagnostics with detailed information - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - test_name = f"concurrent_pvc_failure_{timestamp}" - - # Track all resources involved in this scenario - failed_resources = [] - - # Add all created PVCs - for pvc_name in created_pvcs: - failed_resources.append({"type": "pvc", "name": pvc_name, "namespace": self.namespace}) + self._handle_concurrent_scenario_failure(e, created_pvcs) + + def _concurrent_create_pvcs(self, pvc_names): + """Create multiple PVCs in quick succession for the concurrent scenario""" + created_pvcs = [] + self.logger.info(f"Creating {len(pvc_names)} PVCs in quick succession") + + for pvc_name in pvc_names: + success = self._create_pvc_for_concurrent(pvc_name) + if success: + created_pvcs.append(pvc_name) + + self.logger.info(f"Successfully created {len(created_pvcs)} PVCs") + return created_pvcs + + def _concurrent_create_pods(self, created_pvcs): + """Create pods for some of the PVCs in the concurrent scenario""" + num_pods = min(len(created_pvcs), 3) + pod_pvcs = random.sample(created_pvcs, num_pods) + + self.logger.info(f"Creating {num_pods} pods for PVCs in concurrent scenario") + for pvc_name in pod_pvcs: + self._attach_pod(pvc_name) + + def _concurrent_delete_pvcs(self, created_pvcs): + """Delete some PVCs in quick succession""" + num_to_delete = min(len(created_pvcs), 3) + pvcs_to_delete = random.sample(created_pvcs, num_to_delete) + + self.logger.info(f"Deleting {num_to_delete} PVCs in quick succession") + for pvc_name in pvcs_to_delete: + self._delete_pvc(pvc_name) + + def _mark_concurrent_scenario_failed(self, reason): + """Mark concurrent scenario as failed with a reason""" + self.logger.warning(reason) + self.scenarios['concurrent_pvc']['fail'] += 1 + + def _handle_concurrent_scenario_failure(self, e, created_pvcs): + """Handle failure in the concurrent PVC scenario""" + self.logger.error(f"Error in rapid PVC scenario: {e}") + self.scenarios['concurrent_pvc']['fail'] += 1 + + # Collect logs for failure diagnostics with detailed information + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + test_name = f"concurrent_pvc_failure_{timestamp}" + + # Track all resources involved in this scenario + failed_resources = self._collect_concurrent_failure_resources(created_pvcs) + + logs_path = collect_logs_on_test_failure( + test_name, + self.metrics_collector, + self.driver_pod_name, + failed_resources=failed_resources + ) + self.logger.info(f"Collected detailed failure logs to: {logs_path}") + + def _collect_concurrent_failure_resources(self, created_pvcs): + """Collect resources involved in concurrent scenario failure""" + failed_resources = [] + + # Add all created PVCs + for pvc_name in created_pvcs: + failed_resources.append({"type": "pvc", "name": pvc_name, "namespace": self.namespace}) + + # Add pods using those PVCs + for pod_name in self.pods.get(pvc_name, []): + failed_resources.append({"type": "pod", "name": pod_name, "namespace": self.namespace}) - # Add pods using those PVCs - for pod_name in self.pods.get(pvc_name, []): - failed_resources.append({"type": "pod", "name": pod_name, "namespace": self.namespace}) - - logs_path = collect_logs_on_test_failure( - test_name, - self.metrics_collector, - self.driver_pod_name, - failed_resources=failed_resources - ) - self.logger.info(f"Collected detailed failure logs to: {logs_path}") + return failed_resources def _create_pvc_for_concurrent(self, pvc_name): """ @@ -1211,20 +1246,29 @@ def _create_pvc_for_concurrent(self, pvc_name): Returns True if successful, False otherwise """ try: - # Create PVC manifest + # Get configuration values + scenario_config = self.config.get('scenarios', {}).get('concurrent_pvc', {}) + sc_name = self.config.get('storage_class', {}).get('name', 'efs-sc') + + # Create PVC manifest using config values pvc_manifest = { "apiVersion": "v1", "kind": "PersistentVolumeClaim", "metadata": {"name": pvc_name}, "spec": { - "accessModes": ["ReadWriteMany"], - "storageClassName": "efs-sc", + "accessModes": ["ReadWriteMany"], # This is generally fixed for EFS + "storageClassName": sc_name, "resources": { - "requests": {"storage": "1Gi"} + "requests": {"storage": "1Gi"} # Size doesn't matter for EFS but required in PVC spec } } } + # Add annotations if configured + pvc_annotations = scenario_config.get('pvc_annotations', {}) + if pvc_annotations: + pvc_manifest['metadata']['annotations'] = pvc_annotations + # Create PVC self.core_v1.create_namespaced_persistent_volume_claim( namespace=self.namespace, @@ -1237,11 +1281,15 @@ def _create_pvc_for_concurrent(self, pvc_name): # Update results self.results['create_pvc']['success'] += 1 - self.logger.info(f"Created PVC: {pvc_name}") + self.logger.info(f"Created PVC: {pvc_name} with storage class {sc_name}") + + # Get timeout from config + retry_config = self.config.get('retries', {}) + pvc_bind_timeout = retry_config.get('pvc_bind_timeout', 30) # Wait for PVC to be bound - if not self._wait_for_pvc_bound(pvc_name, timeout=30): - self.logger.warning(f"Timeout waiting for PVC {pvc_name} to be bound") + if not self._wait_for_pvc_bound(pvc_name, timeout=pvc_bind_timeout): + self.logger.warning(f"Timeout waiting for PVC {pvc_name} to be bound after {pvc_bind_timeout}s") return False return True @@ -1264,185 +1312,202 @@ def _wait_for_pod_ready(self, pod_name, timeout=60): diagnostic_logged = False while time.time() - start_time < timeout: - try: - pod = self.core_v1.read_namespaced_pod_status( - name=pod_name, - namespace=self.namespace - ) + pod_status = self._check_pod_status(pod_name) + + # Pod not found + if pod_status.get('not_found', False): + return False - # Log phase transitions for better diagnostics - if pod.status.phase != last_phase: - self.logger.info(f"Pod {pod_name} phase: {pod.status.phase}") - last_phase = pod.status.phase + # Update last_phase for tracking phase transitions + if pod_status.get('phase') != last_phase: + self.logger.info(f"Pod {pod_name} phase: {pod_status.get('phase')}") + last_phase = pod_status.get('phase') - # Check if pod is ready - if pod.status.phase == "Running": - if pod.status.conditions: - ready_condition = None - all_conditions = [] - - for condition in pod.status.conditions: - all_conditions.append(f"{condition.type}={condition.status}") - if condition.type == "Ready": - ready_condition = condition - - # Log all conditions for diagnostics - self.logger.info(f"Pod {pod_name} conditions: {', '.join(all_conditions)}") - - if ready_condition and ready_condition.status == "True": - self.logger.info(f"Pod {pod_name} is ready") - return True + # Check if pod is ready + if pod_status.get('ready', False): + self.logger.info(f"Pod {pod_name} is ready") + return True - # Check for failure states - if pod.status.phase in ["Failed", "Unknown"]: - self.logger.error(f"Pod {pod_name} entered {pod.status.phase} state") - self._log_pod_diagnostics(pod_name) - return False - - # Generate diagnostics only once during waiting if we're halfway through the timeout - elapsed = time.time() - start_time - if elapsed > timeout / 2 and not diagnostic_logged: - self.logger.info(f"Pod {pod_name} taking longer than expected to become ready ({elapsed:.1f}s). Collecting diagnostics...") - self._log_pod_diagnostics(pod_name) - diagnostic_logged = True - - # Still waiting - self.logger.debug(f"Pod {pod_name} is in {pod.status.phase} state, waiting...") + # Check for failure states + if self._is_pod_in_failure_state(pod_status): + self._log_pod_diagnostics(pod_name) + return False + + # Check if we should log diagnostics + elapsed = time.time() - start_time + if self._should_log_wait_diagnostics(elapsed, timeout, diagnostic_logged): + self.logger.info(f"Pod {pod_name} taking longer than expected to become ready ({elapsed:.1f}s). Collecting diagnostics...") + self._log_pod_diagnostics(pod_name) + diagnostic_logged = True - except client.exceptions.ApiException as e: - if e.status == 404: - self.logger.warning(f"Pod {pod_name} not found") - return False - self.logger.warning(f"Error checking pod status: {e}") - time.sleep(2) self.logger.warning(f"Timeout waiting for pod {pod_name} to be ready after {timeout}s") - # Collect detailed diagnostics on timeout self._log_pod_diagnostics(pod_name) return False - def _log_pod_diagnostics(self, pod_name): - """ - Collect and log detailed pod diagnostics - This helps diagnose why a pod isn't becoming ready - """ + def _check_pod_status(self, pod_name): + """Check pod status and return information about its current state""" try: - # Get detailed pod information - self.logger.info(f"===== DIAGNOSTICS FOR POD {pod_name} =====") - - # 1. Get full pod details - pod = self.core_v1.read_namespaced_pod( + pod = self.core_v1.read_namespaced_pod_status( name=pod_name, namespace=self.namespace ) - # 2. Check container statuses - if pod.status.container_statuses: - for container in pod.status.container_statuses: - self.logger.info(f"Container {container.name} status:") - self.logger.info(f" - Ready: {container.ready}") - self.logger.info(f" - Started: {container.started}") - self.logger.info(f" - Restart Count: {container.restart_count}") - - if container.state.waiting: - self.logger.info(f" - Waiting: reason={container.state.waiting.reason}, message={container.state.waiting.message}") - elif container.state.running: - self.logger.info(f" - Running: started at {container.state.running.started_at}") - elif container.state.terminated: - self.logger.info(f" - Terminated: reason={container.state.terminated.reason}, exit_code={container.state.terminated.exit_code}") - else: - self.logger.info("No container statuses available") + status = { + 'phase': pod.status.phase, + 'ready': False, + 'conditions': [], + 'all_conditions_text': '' + } - # 3. Check pod events - try: - field_selector = f"involvedObject.name={pod_name}" - events = self.core_v1.list_namespaced_event( - namespace=self.namespace, - field_selector=field_selector - ) + # Extract conditions if available + if pod.status.phase == "Running" and pod.status.conditions: + all_conditions = [] - if events.items: - self.logger.info(f"Pod events:") - for event in events.items: - self.logger.info(f" [{event.last_timestamp}] {event.type}/{event.reason}: {event.message}") - else: - self.logger.info("No events found for pod") - except Exception as e: - self.logger.warning(f"Error retrieving pod events: {e}") - - # 4. Get pod logs if container is running - try: - logs = self.core_v1.read_namespaced_pod_log( - name=pod_name, - namespace=self.namespace, - container="test-container", - tail_lines=20 - ) - if logs: - self.logger.info(f"Container logs (last 20 lines):") - for line in logs.splitlines()[-20:]: - self.logger.info(f" {line}") - else: - self.logger.info("No logs available") - except Exception as e: - self.logger.warning(f"Error retrieving pod logs: {e}") - - # 5. Check volume issues - if pod.spec.volumes: - self.logger.info(f"Pod volumes:") - for volume in pod.spec.volumes: - volume_details = {} - if hasattr(volume, 'persistent_volume_claim') and volume.persistent_volume_claim: - volume_details["type"] = "PVC" - volume_details["claim_name"] = volume.persistent_volume_claim.claim_name - elif hasattr(volume, 'host_path') and volume.host_path: - volume_details["type"] = "HostPath" - volume_details["path"] = volume.host_path.path + for condition in pod.status.conditions: + condition_text = f"{condition.type}={condition.status}" + all_conditions.append(condition_text) - self.logger.info(f" - {volume.name}: {volume_details}") + # Check if the Ready condition is true + if condition.type == "Ready" and condition.status == "True": + status['ready'] = True + + status['conditions'] = all_conditions + status['all_conditions_text'] = ', '.join(all_conditions) + + # Log all conditions for diagnostics if available + if all_conditions: + self.logger.info(f"Pod {pod_name} conditions: {status['all_conditions_text']}") - # 6. Try to run diagnostic command in pod if it's running - if pod.status.phase == "Running": - try: - # Check mount points - mount_cmd = "mount | grep /data" - exec_command = ['/bin/sh', '-c', mount_cmd] - - resp = self.core_v1.connect_get_namespaced_pod_exec( - pod_name, - self.namespace, - command=exec_command, - stdin=False, - stdout=True, - stderr=True, - tty=False - ) - self.logger.info(f"Mount diagnostic output: {resp}") - - # Check if we can write to the volume - touch_cmd = "touch /data/test_write && echo 'Write test successful'" - exec_command = ['/bin/sh', '-c', touch_cmd] - - resp = self.core_v1.connect_get_namespaced_pod_exec( - pod_name, - self.namespace, - command=exec_command, - stdin=False, - stdout=True, - stderr=True, - tty=False - ) - self.logger.info(f"Write test output: {resp}") - - except Exception as e: - self.logger.warning(f"Cannot execute diagnostic commands in pod: {e}") + return status + except client.exceptions.ApiException as e: + if e.status == 404: + self.logger.warning(f"Pod {pod_name} not found") + return {'not_found': True, 'phase': 'NotFound'} + self.logger.warning(f"Error checking pod status: {e}") + return {'error': str(e), 'phase': 'Error'} + + def _is_pod_in_failure_state(self, pod_status): + """Check if pod is in a terminal failure state""" + failure_phases = ["Failed", "Unknown"] + return pod_status.get('phase') in failure_phases + + def _should_log_wait_diagnostics(self, elapsed, timeout, already_logged): + """Determine if diagnostics should be logged during wait operations""" + if already_logged: + return False + return elapsed > timeout / 2 + + def _log_pod_diagnostics(self, pod_name): + """ + Collect and log detailed pod diagnostics + This helps diagnose why a pod isn't becoming ready + """ + try: + self.logger.info(f"===== DIAGNOSTICS FOR POD {pod_name} =====") + pod = self.core_v1.read_namespaced_pod(name=pod_name, namespace=self.namespace) + self._log_container_statuses(pod) + self._log_pod_events(pod_name) + self._log_pod_logs(pod_name) + self._log_pod_volumes(pod) + if pod.status.phase == "Running": + self._run_pod_diagnostics_commands(pod_name) self.logger.info(f"===== END DIAGNOSTICS FOR POD {pod_name} =====") - except Exception as e: self.logger.error(f"Error collecting pod diagnostics: {e}") - + + def _log_container_statuses(self, pod): + if pod.status.container_statuses: + for container in pod.status.container_statuses: + self.logger.info(f"Container {container.name} status:") + self.logger.info(f" - Ready: {container.ready}") + self.logger.info(f" - Started: {container.started}") + self.logger.info(f" - Restart Count: {container.restart_count}") + if container.state.waiting: + self.logger.info(f" - Waiting: reason={container.state.waiting.reason}, message={container.state.waiting.message}") + elif container.state.running: + self.logger.info(f" - Running: started at {container.state.running.started_at}") + elif container.state.terminated: + self.logger.info(f" - Terminated: reason={container.state.terminated.reason}, exit_code={container.state.terminated.exit_code}") + else: + self.logger.info("No container statuses available") + + def _log_pod_events(self, pod_name): + try: + field_selector = f"involvedObject.name={pod_name}" + events = self.core_v1.list_namespaced_event(namespace=self.namespace, field_selector=field_selector) + if events.items: + self.logger.info(f"Pod events:") + for event in events.items: + self.logger.info(f" [{event.last_timestamp}] {event.type}/{event.reason}: {event.message}") + else: + self.logger.info("No events found for pod") + except Exception as e: + self.logger.warning(f"Error retrieving pod events: {e}") + + def _log_pod_logs(self, pod_name): + try: + logs = self.core_v1.read_namespaced_pod_log( + name=pod_name, + namespace=self.namespace, + container="test-container", + tail_lines=20 + ) + if logs: + self.logger.info(f"Container logs (last 20 lines):") + for line in logs.splitlines()[-20:]: + self.logger.info(f" {line}") + else: + self.logger.info("No logs available") + except Exception as e: + self.logger.warning(f"Error retrieving pod logs: {e}") + + def _log_pod_volumes(self, pod): + if pod.spec.volumes: + self.logger.info(f"Pod volumes:") + for volume in pod.spec.volumes: + volume_details = {} + if hasattr(volume, 'persistent_volume_claim') and volume.persistent_volume_claim: + volume_details["type"] = "PVC" + volume_details["claim_name"] = volume.persistent_volume_claim.claim_name + elif hasattr(volume, 'host_path') and volume.host_path: + volume_details["type"] = "HostPath" + volume_details["path"] = volume.host_path.path + self.logger.info(f" - {volume.name}: {volume_details}") + + def _run_pod_diagnostics_commands(self, pod_name): + try: + # Check mount points + mount_cmd = "mount | grep /data" + exec_command = ['/bin/sh', '-c', mount_cmd] + resp = self.core_v1.connect_get_namespaced_pod_exec( + pod_name, + self.namespace, + command=exec_command, + stdin=False, + stdout=True, + stderr=True, + tty=False + ) + self.logger.info(f"Mount diagnostic output: {resp}") + # Check if we can write to the volume + touch_cmd = "touch /data/test_write && echo 'Write test successful'" + exec_command = ['/bin/sh', '-c', touch_cmd] + resp = self.core_v1.connect_get_namespaced_pod_exec( + pod_name, + self.namespace, + command=exec_command, + stdin=False, + stdout=True, + stderr=True, + tty=False + ) + self.logger.info(f"Write test output: {resp}") + except Exception as e: + self.logger.warning(f"Cannot execute diagnostic commands in pod: {e}") + def _wait_for_pod_deleted(self, pod_name, timeout=60): """ Wait for pod to be deleted @@ -1531,6 +1596,29 @@ def _wait_for_pvc_deleted(self, pvc_name, timeout=60): self.logger.warning(f"Timeout waiting for PVC {pvc_name} to be deleted after {timeout}s") return False + + def _log_efs_filesystem_state(self): + """Log the state of the EFS file system after test completion.""" + try: + fs_id = self.config.get('driver', {}).get('filesystem_id') + region = self.config.get('cluster', {}).get('region', 'us-west-1') + if not fs_id: + self.logger.warning("No filesystem_id found in config for EFS state check.") + return None + efs = boto3.client('efs', region_name=region) + response = efs.describe_file_systems(FileSystemId=fs_id) + fs = response['FileSystems'][0] + fs_info = { + "filesystem_id": fs_id, + "state": fs['LifeCycleState'], + "size_bytes": fs['SizeInBytes']['Value'], + "mount_targets": fs['NumberOfMountTargets'] + } + self.logger.info(f"EFS FileSystem {fs_id} state: {fs['LifeCycleState']}, Size: {fs['SizeInBytes']['Value']} bytes, MountTargets: {fs['NumberOfMountTargets']}") + return fs_info + except Exception as e: + self.logger.error(f"Failed to log EFS file system state: {e}") + return None def _ensure_storage_class(self): """Ensure EFS StorageClass exists""" @@ -1581,51 +1669,38 @@ def _cleanup(self): self.logger.info("===== STARTING COMPREHENSIVE CLEANUP =====") cleanup_start_time = time.time() cleanup_timeout = 180 # 3 minutes timeout for entire cleanup - - # Track cleanup failures for reporting cleanup_failures = [] - - # Force delete flag - initially false, but will be set to true for retry attempts force_delete = False - try: - # First attempt - normal deletion - self._perform_cleanup(force_delete, cleanup_failures) - - # Check if there are any remaining resources that weren't cleaned up + self._cleanup_resources(force_delete, cleanup_failures) remaining_resources = self._get_remaining_resources() - - # If resources remain, try force delete if remaining_resources: self.logger.warning(f"First cleanup pass incomplete. Remaining resources: {remaining_resources}") self.logger.info("Attempting force deletion of remaining resources...") force_delete = True - self._perform_cleanup(force_delete, cleanup_failures) - - # Check again for any remaining resources + self._cleanup_resources(force_delete, cleanup_failures) remaining_resources = self._get_remaining_resources() if remaining_resources: self.logger.error(f"Cleanup incomplete. Remaining resources after force deletion: {remaining_resources}") - elapsed = time.time() - cleanup_start_time if cleanup_failures: self.logger.warning(f"Cleanup completed in {elapsed:.2f} seconds with {len(cleanup_failures)} failures") self.logger.warning(f"Failed deletions: {cleanup_failures}") else: self.logger.info(f"Cleanup completed successfully in {elapsed:.2f} seconds") - except Exception as e: self.logger.error(f"Error during cleanup: {e}", exc_info=True) - finally: self.logger.info("===== CLEANUP PROCESS FINISHED =====") - - def _perform_cleanup(self, force=False, failures=None): - """Perform the actual cleanup with optional force deletion""" - if failures is None: - failures = [] - - # Delete all pods first + self._log_efs_filesystem_state() + + def _cleanup_resources(self, force, failures): + """Delete all pods and PVCs with error handling""" + self._cleanup_pods(force, failures) + time.sleep(5) # Allow pod termination before PVC deletion + self._cleanup_pvcs(force, failures) + + def _cleanup_pods(self, force, failures): self.logger.info(f"Deleting {self.current_pod_count} pods (force={force})...") for pvc_name, pod_list in list(self.pods.items()): for pod_name in list(pod_list): @@ -1636,11 +1711,8 @@ def _perform_cleanup(self, force=False, failures=None): except Exception as e: self.logger.error(f"Error deleting pod {pod_name}: {e}") failures.append(f"pod/{pod_name}") - - # Wait briefly to allow pod termination before PVC deletion - time.sleep(5) - - # Then delete all PVCs + + def _cleanup_pvcs(self, force, failures): self.logger.info(f"Deleting {len(self.pvcs)} PVCs (force={force})...") for pvc_name in list(self.pvcs): try: @@ -1650,7 +1722,7 @@ def _perform_cleanup(self, force=False, failures=None): except Exception as e: self.logger.error(f"Error deleting PVC {pvc_name}: {e}") failures.append(f"pvc/{pvc_name}") - + def _get_remaining_resources(self): """Get a list of any resources that weren't cleaned up""" remaining = [] @@ -1682,70 +1754,62 @@ def _get_remaining_resources(self): def _generate_report(self): """Generate test report""" + # Get EFS filesystem state information + fs_info = self._log_efs_filesystem_state() + report = { "test_duration": time.time(), - "operations": { - "create_pvc": { - "success": self.results['create_pvc']['success'], - "fail": self.results['create_pvc']['fail'], - "success_rate": self._calculate_success_rate(self.results['create_pvc']), - }, - "attach_pod": { - "success": self.results['attach_pod']['success'], - "fail": self.results['attach_pod']['fail'], - "success_rate": self._calculate_success_rate(self.results['attach_pod']), - }, - "delete_pod": { - "success": self.results['delete_pod']['success'], - "fail": self.results['delete_pod']['fail'], - "success_rate": self._calculate_success_rate(self.results['delete_pod']), - }, - "delete_pvc": { - "success": self.results['delete_pvc']['success'], - "fail": self.results['delete_pvc']['fail'], - "success_rate": self._calculate_success_rate(self.results['delete_pvc']), - }, - "verify_read_write": { - "write_success": self.results['verify_write']['success'], - "write_fail": self.results['verify_write']['fail'], - "read_success": self.results['verify_read']['success'], - "read_fail": self.results['verify_read']['fail'], - "write_success_rate": self._calculate_success_rate(self.results['verify_write']), - "read_success_rate": self._calculate_success_rate(self.results['verify_read']), - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": self.scenarios['shared_volume_rw']['runs'], - "success": self.scenarios['shared_volume_rw']['success'], - "fail": self.scenarios['shared_volume_rw']['fail'], - "success_rate": self._calculate_scenario_success_rate('shared_volume_rw') - }, - "many_to_one": { - "runs": self.scenarios['many_to_one']['runs'], - "success": self.scenarios['many_to_one']['success'], - "fail": self.scenarios['many_to_one']['fail'], - "success_rate": self._calculate_scenario_success_rate('many_to_one') - }, - "one_to_one": { - "runs": self.scenarios['one_to_one']['runs'], - "success": self.scenarios['one_to_one']['success'], - "fail": self.scenarios['one_to_one']['fail'], - "success_rate": self._calculate_scenario_success_rate('one_to_one') - }, - "concurrent_pvc": { - "runs": self.scenarios['concurrent_pvc']['runs'], - "success": self.scenarios['concurrent_pvc']['success'], - "fail": self.scenarios['concurrent_pvc']['fail'], - "success_rate": self._calculate_scenario_success_rate('concurrent_pvc') - } - } + "operations": self._generate_operations_report(), + "efs_filesystem": fs_info, + "scenarios": self._generate_scenarios_report() } # Print report summary self._print_report_summary(report) return report + + def _generate_operations_report(self): + """Generate the operations section of the report""" + operations_report = {} + + # Standard operations + for op_name in ['create_pvc', 'attach_pod', 'delete_pod', 'delete_pvc']: + operations_report[op_name] = self._get_operation_stats(op_name) + + # Special case for read/write operations + operations_report["verify_read_write"] = { + "write_success": self.results['verify_write']['success'], + "write_fail": self.results['verify_write']['fail'], + "read_success": self.results['verify_read']['success'], + "read_fail": self.results['verify_read']['fail'], + "write_success_rate": self._calculate_success_rate(self.results['verify_write']), + "read_success_rate": self._calculate_success_rate(self.results['verify_read']), + } + + return operations_report + + def _get_operation_stats(self, op_name): + """Get statistics for a specific operation""" + return { + "success": self.results[op_name]['success'], + "fail": self.results[op_name]['fail'], + "success_rate": self._calculate_success_rate(self.results[op_name]), + } + + def _generate_scenarios_report(self): + """Generate the scenarios section of the report""" + scenarios_report = {} + + for scenario_name in self.scenarios: + scenarios_report[scenario_name] = { + "runs": self.scenarios[scenario_name]['runs'], + "success": self.scenarios[scenario_name]['success'], + "fail": self.scenarios[scenario_name]['fail'], + "success_rate": self._calculate_scenario_success_rate(scenario_name) + } + + return scenarios_report def _calculate_success_rate(self, result): """Calculate success rate as percentage""" @@ -1800,7 +1864,6 @@ def main(): # Initialize and run orchestrator orchestrator = EFSCSIOrchestrator(config_file=args.config, namespace=args.namespace) - # Override default test parameters if specified if args.duration: orchestrator.test_duration = args.duration From b725856b9a0c44ce22c7564b427f35e0747b9e8a Mon Sep 17 00:00:00 2001 From: Ishaan Bhardwaj Date: Tue, 17 Jun 2025 18:21:27 +0000 Subject: [PATCH 07/13] Add modularized testing framework with enhanced documentation --- test/stress-scale-tests/README.md | 1 + test/stress-scale-tests/config/orchestrator_config.yaml | 1 + test/stress-scale-tests/run_tests.py | 1 + test/stress-scale-tests/tests/orchestrator.py | 1 + test/stress-scale-tests/utils/log_integration.py | 1 + test/stress-scale-tests/utils/metrics_collector.py | 1 + test/stress-scale-tests/utils/report_generator.py | 1 + 7 files changed, 7 insertions(+) diff --git a/test/stress-scale-tests/README.md b/test/stress-scale-tests/README.md index f007395af..ac352b3e3 100644 --- a/test/stress-scale-tests/README.md +++ b/test/stress-scale-tests/README.md @@ -101,3 +101,4 @@ Test reports are stored in: - The tests use `kubectl` subprocess calls for pod exec operations to avoid WebSocket protocol issues - All tests run in the namespace specified in the config (default: `efs-stress-test`) +# AWS EFS CSI Driver Testing Framework diff --git a/test/stress-scale-tests/config/orchestrator_config.yaml b/test/stress-scale-tests/config/orchestrator_config.yaml index 7a8880132..f9a57ac21 100644 --- a/test/stress-scale-tests/config/orchestrator_config.yaml +++ b/test/stress-scale-tests/config/orchestrator_config.yaml @@ -124,3 +124,4 @@ test: duration: 120 namespace: efs-stress-test operation_interval: 3 +# Updated configuration for EFS test orchestrator diff --git a/test/stress-scale-tests/run_tests.py b/test/stress-scale-tests/run_tests.py index 9baf9a424..22f7a6764 100644 --- a/test/stress-scale-tests/run_tests.py +++ b/test/stress-scale-tests/run_tests.py @@ -415,3 +415,4 @@ def main(): if __name__ == "__main__": main() +# Enhanced modularized implementation diff --git a/test/stress-scale-tests/tests/orchestrator.py b/test/stress-scale-tests/tests/orchestrator.py index 86e9a5bdc..6e0809706 100644 --- a/test/stress-scale-tests/tests/orchestrator.py +++ b/test/stress-scale-tests/tests/orchestrator.py @@ -1875,3 +1875,4 @@ def main(): if __name__ == "__main__": main() +# Enhanced modular implementation for orchestrator diff --git a/test/stress-scale-tests/utils/log_integration.py b/test/stress-scale-tests/utils/log_integration.py index e69d23fdc..7f46d7040 100644 --- a/test/stress-scale-tests/utils/log_integration.py +++ b/test/stress-scale-tests/utils/log_integration.py @@ -364,3 +364,4 @@ def collect_logs_on_test_failure(test_name, metrics_collector=None, driver_pod_n # Example usage tarball_path = collect_logs_for_test("example_test") print(f"Logs collected to: {tarball_path}") +# Enhanced log integration module diff --git a/test/stress-scale-tests/utils/metrics_collector.py b/test/stress-scale-tests/utils/metrics_collector.py index 2f2eb6c7a..419cf48c3 100644 --- a/test/stress-scale-tests/utils/metrics_collector.py +++ b/test/stress-scale-tests/utils/metrics_collector.py @@ -631,3 +631,4 @@ def get_all_metrics(self): "k8s_events": self.k8s_events, "file_performance": file_performance_summary } +# Enhanced modular implementation for metrics collection diff --git a/test/stress-scale-tests/utils/report_generator.py b/test/stress-scale-tests/utils/report_generator.py index b93b098a5..2c07c5915 100644 --- a/test/stress-scale-tests/utils/report_generator.py +++ b/test/stress-scale-tests/utils/report_generator.py @@ -333,3 +333,4 @@ def _write_generic_results(self, file, results): file.write(f"{key}: {value}\n") else: file.write(f"{result}\n") +# Enhanced report generator module From c503170ae100ef81f4604ba54385eaad0ec9bf6a Mon Sep 17 00:00:00 2001 From: Ishaan Bhardwaj Date: Tue, 17 Jun 2025 18:28:01 +0000 Subject: [PATCH 08/13] Remove logs and report files from repository --- .../logs/cleanup_20250609_161210.log | 8 - test/stress-scale-tests/logs/efs_tests.log | 134 --- .../logs/orchestrator_20250605_210820.log | 7 - .../logs/orchestrator_20250605_210842.log | 70 -- .../logs/orchestrator_20250609_142216.log | 12 - .../logs/orchestrator_20250609_142258.log | 12 - .../logs/orchestrator_20250609_142522.log | 1058 ----------------- .../logs/orchestrator_20250609_182011.log | 7 - .../logs/orchestrator_20250609_183658.log | 7 - .../logs/orchestrator_20250609_183729.log | 111 -- ...tests_20250605_201716_20250605_201716.json | 104 -- .../efs_orchestrator_20250605_201716.json | 59 - .../efs_orchestrator_20250609_145633.json | 59 - .../efs_orchestrator_20250609_184030.json | 59 - .../test-reports/sample-test-report.json | 145 --- 15 files changed, 1852 deletions(-) delete mode 100644 test/stress-scale-tests/logs/cleanup_20250609_161210.log delete mode 100644 test/stress-scale-tests/logs/efs_tests.log delete mode 100644 test/stress-scale-tests/logs/orchestrator_20250605_210820.log delete mode 100644 test/stress-scale-tests/logs/orchestrator_20250605_210842.log delete mode 100644 test/stress-scale-tests/logs/orchestrator_20250609_142216.log delete mode 100644 test/stress-scale-tests/logs/orchestrator_20250609_142258.log delete mode 100644 test/stress-scale-tests/logs/orchestrator_20250609_142522.log delete mode 100644 test/stress-scale-tests/logs/orchestrator_20250609_182011.log delete mode 100644 test/stress-scale-tests/logs/orchestrator_20250609_183658.log delete mode 100644 test/stress-scale-tests/logs/orchestrator_20250609_183729.log delete mode 100644 test/stress-scale-tests/reports/general/efs_tests_20250605_201716_20250605_201716.json delete mode 100644 test/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_201716.json delete mode 100644 test/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250609_145633.json delete mode 100644 test/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250609_184030.json delete mode 100644 test/stress-scale-tests/test-reports/sample-test-report.json diff --git a/test/stress-scale-tests/logs/cleanup_20250609_161210.log b/test/stress-scale-tests/logs/cleanup_20250609_161210.log deleted file mode 100644 index 35962ed2c..000000000 --- a/test/stress-scale-tests/logs/cleanup_20250609_161210.log +++ /dev/null @@ -1,8 +0,0 @@ -2025-06-09 16:12:10,963 - INFO - Starting EFS CSI Driver test resource cleanup -2025-06-09 16:12:12,039 - ERROR - Failed to list namespaces: (401) -Reason: Unauthorized -HTTP response headers: HTTPHeaderDict({'Audit-Id': '1a009cf9-0d14-4835-a132-092db20a6853', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Mon, 09 Jun 2025 16:12:12 GMT', 'Content-Length': '129'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} - - -2025-06-09 16:12:12,040 - INFO - Cleanup process completed diff --git a/test/stress-scale-tests/logs/efs_tests.log b/test/stress-scale-tests/logs/efs_tests.log deleted file mode 100644 index c861cbbb6..000000000 --- a/test/stress-scale-tests/logs/efs_tests.log +++ /dev/null @@ -1,134 +0,0 @@ -2025-06-09 14:22:15,661 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-09 14:22:15,661 - __main__ - INFO - Verifying AWS credentials -2025-06-09 14:22:15,663 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-09 14:22:15,663 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-09 14:22:15,664 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-09 14:22:15,664 - __main__ - INFO - Running orchestrator stress test suite -2025-06-09 14:22:15,664 - __main__ - INFO - Starting orchestrator for 300 seconds -2025-06-09 14:22:15,664 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-09 14:22:16,871 - __main__ - INFO - Test duration overridden to 300 seconds -2025-06-09 14:22:16,871 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) -2025-06-09 14:22:57,994 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-09 14:22:57,994 - __main__ - INFO - Verifying AWS credentials -2025-06-09 14:22:57,995 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-09 14:22:57,996 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-09 14:22:57,996 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-09 14:22:57,996 - __main__ - INFO - Running orchestrator stress test suite -2025-06-09 14:22:57,997 - __main__ - INFO - Starting orchestrator for 300 seconds -2025-06-09 14:22:57,997 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-09 14:22:59,034 - __main__ - INFO - Test duration overridden to 300 seconds -2025-06-09 14:22:59,034 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) -2025-06-09 14:25:21,361 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-09 14:25:21,361 - __main__ - INFO - Verifying AWS credentials -2025-06-09 14:25:21,362 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-09 14:25:21,363 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-09 14:25:21,363 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-09 14:25:21,364 - __main__ - INFO - Running orchestrator stress test suite -2025-06-09 14:25:21,364 - __main__ - INFO - Starting orchestrator for None seconds -2025-06-09 14:25:21,364 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-09 14:25:22,404 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) -2025-06-09 14:56:33,409 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250609_145633.json -2025-06-09 18:20:10,833 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-09 18:20:10,833 - __main__ - INFO - Verifying AWS credentials -2025-06-09 18:20:10,834 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-09 18:20:10,834 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-09 18:20:10,835 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-09 18:20:10,835 - __main__ - INFO - Running orchestrator stress test suite -2025-06-09 18:20:10,835 - __main__ - INFO - Starting orchestrator for None seconds -2025-06-09 18:20:10,835 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-09 18:20:11,779 - __main__ - ERROR - Error running tests: (401) -Reason: Unauthorized -HTTP response headers: HTTPHeaderDict({'Audit-Id': 'd5d12f12-977a-4faf-bd61-ad049e1a9ec2', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Mon, 09 Jun 2025 18:20:11 GMT', 'Content-Length': '129'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} - -Traceback (most recent call last): - File "/local/home/ibhard/efs-csi-driver-fork/aws-efs-csi-driver/test/e2e/stress-scale-tests/run_tests.py", line 208, in main - orchestrator = EFSCSIOrchestrator(config_file=orchestrator_config) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/local/home/ibhard/efs-csi-driver-fork/aws-efs-csi-driver/test/e2e/stress-scale-tests/tests/orchestrator.py", line 91, in __init__ - self._ensure_namespace_exists() - File "/local/home/ibhard/efs-csi-driver-fork/aws-efs-csi-driver/test/e2e/stress-scale-tests/tests/orchestrator.py", line 97, in _ensure_namespace_exists - self.core_v1.read_namespace(name=self.namespace) - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api/core_v1_api.py", line 22992, in read_namespace - return self.read_namespace_with_http_info(name, **kwargs) # noqa: E501 - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api/core_v1_api.py", line 23071, in read_namespace_with_http_info - return self.api_client.call_api( - ^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api_client.py", line 348, in call_api - return self.__call_api(resource_path, method, - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api_client.py", line 180, in __call_api - response_data = self.request( - ^^^^^^^^^^^^^ - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api_client.py", line 373, in request - return self.rest_client.GET(url, - ^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/rest.py", line 244, in GET - return self.request("GET", url, - ^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/rest.py", line 238, in request - raise ApiException(http_resp=r) -kubernetes.client.exceptions.ApiException: (401) -Reason: Unauthorized -HTTP response headers: HTTPHeaderDict({'Audit-Id': 'd5d12f12-977a-4faf-bd61-ad049e1a9ec2', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Mon, 09 Jun 2025 18:20:11 GMT', 'Content-Length': '129'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} - - -2025-06-09 18:36:58,115 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-09 18:36:58,115 - __main__ - INFO - Verifying AWS credentials -2025-06-09 18:36:58,116 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-09 18:36:58,116 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-09 18:36:58,117 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-09 18:36:58,117 - __main__ - INFO - Running orchestrator stress test suite -2025-06-09 18:36:58,117 - __main__ - INFO - Starting orchestrator for None seconds -2025-06-09 18:36:58,117 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-09 18:36:59,092 - __main__ - ERROR - Error running tests: (401) -Reason: Unauthorized -HTTP response headers: HTTPHeaderDict({'Audit-Id': 'd3c86ac1-4d01-4ad7-a9bc-d37363c9419a', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Mon, 09 Jun 2025 18:36:59 GMT', 'Content-Length': '129'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} - -Traceback (most recent call last): - File "/local/home/ibhard/efs-csi-driver-fork/aws-efs-csi-driver/test/e2e/stress-scale-tests/run_tests.py", line 208, in main - orchestrator = EFSCSIOrchestrator(config_file=orchestrator_config) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/local/home/ibhard/efs-csi-driver-fork/aws-efs-csi-driver/test/e2e/stress-scale-tests/tests/orchestrator.py", line 91, in __init__ - self._ensure_namespace_exists() - File "/local/home/ibhard/efs-csi-driver-fork/aws-efs-csi-driver/test/e2e/stress-scale-tests/tests/orchestrator.py", line 97, in _ensure_namespace_exists - self.core_v1.read_namespace(name=self.namespace) - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api/core_v1_api.py", line 22992, in read_namespace - return self.read_namespace_with_http_info(name, **kwargs) # noqa: E501 - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api/core_v1_api.py", line 23071, in read_namespace_with_http_info - return self.api_client.call_api( - ^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api_client.py", line 348, in call_api - return self.__call_api(resource_path, method, - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api_client.py", line 180, in __call_api - response_data = self.request( - ^^^^^^^^^^^^^ - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/api_client.py", line 373, in request - return self.rest_client.GET(url, - ^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/rest.py", line 244, in GET - return self.request("GET", url, - ^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/ibhard/.local/share/mise/installs/python/3.12.10/lib/python3.12/site-packages/kubernetes/client/rest.py", line 238, in request - raise ApiException(http_resp=r) -kubernetes.client.exceptions.ApiException: (401) -Reason: Unauthorized -HTTP response headers: HTTPHeaderDict({'Audit-Id': 'd3c86ac1-4d01-4ad7-a9bc-d37363c9419a', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Mon, 09 Jun 2025 18:36:59 GMT', 'Content-Length': '129'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} - - -2025-06-09 18:37:28,448 - __main__ - INFO - Starting EFS CSI tests with configuration from config/orchestrator_config.yaml -2025-06-09 18:37:28,448 - __main__ - INFO - Verifying AWS credentials -2025-06-09 18:37:28,449 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-09 18:37:28,449 - urllib3.connectionpool - WARNING - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-09 18:37:28,449 - urllib3.connectionpool - WARNING - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')': /api/v1/namespaces -2025-06-09 18:37:28,450 - __main__ - INFO - Running orchestrator stress test suite -2025-06-09 18:37:28,450 - __main__ - INFO - Starting orchestrator for None seconds -2025-06-09 18:37:28,450 - __main__ - INFO - Using orchestrator config: config/orchestrator_config.yaml -2025-06-09 18:37:29,435 - __main__ - INFO - Operation interval overridden to 1 seconds (from rate 5/s) -2025-06-09 18:40:30,388 - __main__ - INFO - Orchestrator report generated: reports/orchestrator/efs_orchestrator_20250609_184030.json diff --git a/test/stress-scale-tests/logs/orchestrator_20250605_210820.log b/test/stress-scale-tests/logs/orchestrator_20250605_210820.log deleted file mode 100644 index efa109bb9..000000000 --- a/test/stress-scale-tests/logs/orchestrator_20250605_210820.log +++ /dev/null @@ -1,7 +0,0 @@ -2025-06-05 21:08:20,584 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 21:08:20,807 - tests.orchestrator - ERROR - Error checking namespace: (401) -Reason: Unauthorized -HTTP response headers: HTTPHeaderDict({'Audit-Id': '0bd8e6e9-7167-4275-a18c-06f08e3ee80c', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Jun 2025 21:08:20 GMT', 'Content-Length': '129'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} - - diff --git a/test/stress-scale-tests/logs/orchestrator_20250605_210842.log b/test/stress-scale-tests/logs/orchestrator_20250605_210842.log deleted file mode 100644 index 58576fca5..000000000 --- a/test/stress-scale-tests/logs/orchestrator_20250605_210842.log +++ /dev/null @@ -1,70 +0,0 @@ -2025-06-05 21:08:42,081 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-05 21:08:42,282 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-05 21:08:42,282 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-05 21:08:42,475 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-05 21:08:42,475 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-05 21:08:42,475 - tests.orchestrator - INFO - Creating PVC: test-pvc-dfd12d43 -2025-06-05 21:08:42,554 - tests.orchestrator - INFO - Created PVC: test-pvc-dfd12d43 -2025-06-05 21:08:42,554 - tests.orchestrator - INFO - Waiting for PVC test-pvc-dfd12d43 to be bound -2025-06-05 21:08:44,686 - tests.orchestrator - INFO - PVC test-pvc-dfd12d43 is bound -2025-06-05 21:08:44,687 - tests.orchestrator - INFO - Creating PVC: test-pvc-c542ee58 -2025-06-05 21:08:44,755 - tests.orchestrator - INFO - Created PVC: test-pvc-c542ee58 -2025-06-05 21:08:44,755 - tests.orchestrator - INFO - Waiting for PVC test-pvc-c542ee58 to be bound -2025-06-05 21:08:46,886 - tests.orchestrator - INFO - PVC test-pvc-c542ee58 is bound -2025-06-05 21:08:46,887 - tests.orchestrator - INFO - Creating pod test-pod-62d8b803 on PVC test-pvc-c542ee58 -2025-06-05 21:08:46,966 - tests.orchestrator - INFO - Created pod: test-pod-62d8b803 using PVC: test-pvc-c542ee58 -2025-06-05 21:08:46,966 - tests.orchestrator - INFO - Waiting for pod test-pod-62d8b803 to be ready -2025-06-05 21:08:47,032 - tests.orchestrator - INFO - Pod test-pod-62d8b803 phase: Pending -2025-06-05 21:08:49,100 - tests.orchestrator - INFO - Pod test-pod-62d8b803 phase: Running -2025-06-05 21:08:49,100 - tests.orchestrator - INFO - Pod test-pod-62d8b803 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 21:08:51,168 - tests.orchestrator - INFO - Pod test-pod-62d8b803 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 21:08:53,236 - tests.orchestrator - INFO - Pod test-pod-62d8b803 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 21:08:55,304 - tests.orchestrator - INFO - Pod test-pod-62d8b803 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 21:08:57,372 - tests.orchestrator - INFO - Pod test-pod-62d8b803 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 21:08:59,439 - tests.orchestrator - INFO - Pod test-pod-62d8b803 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 21:09:01,598 - tests.orchestrator - INFO - Pod test-pod-62d8b803 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 21:09:03,669 - tests.orchestrator - INFO - Pod test-pod-62d8b803 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 21:09:05,735 - tests.orchestrator - INFO - Pod test-pod-62d8b803 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 21:09:07,804 - tests.orchestrator - INFO - Pod test-pod-62d8b803 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 21:09:09,873 - tests.orchestrator - INFO - Pod test-pod-62d8b803 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 21:09:09,873 - tests.orchestrator - INFO - Pod test-pod-62d8b803 is ready -2025-06-05 21:09:09,873 - tests.orchestrator - INFO - Creating pod test-pod-87095965 on PVC test-pvc-c542ee58 -2025-06-05 21:09:09,947 - tests.orchestrator - INFO - Created pod: test-pod-87095965 using PVC: test-pvc-c542ee58 -2025-06-05 21:09:09,947 - tests.orchestrator - INFO - Waiting for pod test-pod-87095965 to be ready -2025-06-05 21:09:10,014 - tests.orchestrator - INFO - Pod test-pod-87095965 phase: Pending -2025-06-05 21:09:12,083 - tests.orchestrator - INFO - Pod test-pod-87095965 phase: Running -2025-06-05 21:09:12,083 - tests.orchestrator - INFO - Pod test-pod-87095965 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 21:09:14,152 - tests.orchestrator - INFO - Pod test-pod-87095965 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 21:09:16,219 - tests.orchestrator - INFO - Pod test-pod-87095965 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 21:09:18,288 - tests.orchestrator - INFO - Pod test-pod-87095965 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 21:09:20,355 - tests.orchestrator - INFO - Pod test-pod-87095965 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 21:09:22,423 - tests.orchestrator - INFO - Pod test-pod-87095965 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 21:09:24,491 - tests.orchestrator - INFO - Pod test-pod-87095965 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 21:09:26,557 - tests.orchestrator - INFO - Pod test-pod-87095965 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 21:09:28,625 - tests.orchestrator - INFO - Pod test-pod-87095965 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 21:09:30,692 - tests.orchestrator - INFO - Pod test-pod-87095965 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 21:09:30,692 - tests.orchestrator - INFO - Pod test-pod-87095965 is ready -2025-06-05 21:09:30,693 - tests.orchestrator - INFO - Creating pod test-pod-4044e577 on PVC test-pvc-c542ee58 -2025-06-05 21:09:30,774 - tests.orchestrator - INFO - Created pod: test-pod-4044e577 using PVC: test-pvc-c542ee58 -2025-06-05 21:09:30,774 - tests.orchestrator - INFO - Waiting for pod test-pod-4044e577 to be ready -2025-06-05 21:09:30,840 - tests.orchestrator - INFO - Pod test-pod-4044e577 phase: Pending -2025-06-05 21:09:32,908 - tests.orchestrator - INFO - Pod test-pod-4044e577 phase: Running -2025-06-05 21:09:32,908 - tests.orchestrator - INFO - Pod test-pod-4044e577 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 21:09:34,976 - tests.orchestrator - INFO - Pod test-pod-4044e577 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 21:09:37,045 - tests.orchestrator - INFO - Pod test-pod-4044e577 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 21:09:39,115 - tests.orchestrator - INFO - Pod test-pod-4044e577 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 21:09:41,211 - tests.orchestrator - INFO - Pod test-pod-4044e577 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 21:09:43,278 - tests.orchestrator - INFO - Pod test-pod-4044e577 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 21:09:45,346 - tests.orchestrator - INFO - Pod test-pod-4044e577 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 21:09:47,414 - tests.orchestrator - INFO - Pod test-pod-4044e577 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 21:09:49,483 - tests.orchestrator - INFO - Pod test-pod-4044e577 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-05 21:09:51,551 - tests.orchestrator - INFO - Pod test-pod-4044e577 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-05 21:09:51,551 - tests.orchestrator - INFO - Pod test-pod-4044e577 is ready -2025-06-05 21:09:51,551 - tests.orchestrator - INFO - Testing read/write between pods test-pod-4044e577 and test-pod-87095965 sharing PVC test-pvc-c542ee58 -2025-06-05 21:09:51,551 - tests.orchestrator - INFO - Executing write command: kubectl exec -n efs-stress-test test-pod-4044e577 -- /bin/sh -c 'echo "Test content: 3c9e6d0b-4c30-48ec-81e0-6a10b24ddcba" > /data/test-d1bd009a.txt' -2025-06-05 21:09:52,943 - tests.orchestrator - INFO - Pod test-pod-4044e577 wrote to /data/test-d1bd009a.txt -2025-06-05 21:09:54,943 - tests.orchestrator - INFO - Executing read command: kubectl exec -n efs-stress-test test-pod-87095965 -- cat /data/test-d1bd009a.txt -2025-06-05 21:09:56,145 - tests.orchestrator - INFO - Read result: 'Test content: 3c9e6d0b-4c30-48ec-81e0-6a10b24ddcba' -2025-06-05 21:09:56,145 - tests.orchestrator - INFO - Pod test-pod-87095965 successfully read content written by test-pod-4044e577 -2025-06-05 21:09:56,145 - tests.orchestrator - INFO - Deleting pod: test-pod-62d8b803 from PVC: test-pvc-c542ee58 -2025-06-05 21:09:56,221 - tests.orchestrator - INFO - Waiting for pod test-pod-62d8b803 to be deleted diff --git a/test/stress-scale-tests/logs/orchestrator_20250609_142216.log b/test/stress-scale-tests/logs/orchestrator_20250609_142216.log deleted file mode 100644 index afaa08b91..000000000 --- a/test/stress-scale-tests/logs/orchestrator_20250609_142216.log +++ /dev/null @@ -1,12 +0,0 @@ -2025-06-09 14:22:16,647 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-09 14:22:16,871 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-09 14:22:16,872 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-09 14:22:17,065 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-09 14:22:17,065 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-09 14:22:17,065 - tests.orchestrator - INFO - Creating PVC: test-pvc-4ada8088 -2025-06-09 14:22:17,141 - tests.orchestrator - INFO - Created PVC: test-pvc-4ada8088 -2025-06-09 14:22:17,185 - tests.orchestrator - INFO - Waiting for PVC test-pvc-4ada8088 to be bound -2025-06-09 14:22:19,316 - tests.orchestrator - INFO - PVC test-pvc-4ada8088 is bound -2025-06-09 14:22:19,317 - tests.orchestrator - INFO - Creating PVC: test-pvc-5c13c143 -2025-06-09 14:22:19,386 - tests.orchestrator - INFO - Created PVC: test-pvc-5c13c143 -2025-06-09 14:22:19,386 - tests.orchestrator - INFO - Waiting for PVC test-pvc-5c13c143 to be bound diff --git a/test/stress-scale-tests/logs/orchestrator_20250609_142258.log b/test/stress-scale-tests/logs/orchestrator_20250609_142258.log deleted file mode 100644 index e4a908e42..000000000 --- a/test/stress-scale-tests/logs/orchestrator_20250609_142258.log +++ /dev/null @@ -1,12 +0,0 @@ -2025-06-09 14:22:58,840 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-09 14:22:59,033 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-09 14:22:59,034 - tests.orchestrator - INFO - Starting orchestrator test for 300 seconds -2025-06-09 14:22:59,227 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-09 14:22:59,228 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-09 14:22:59,228 - tests.orchestrator - INFO - Creating PVC: test-pvc-02e00a44 -2025-06-09 14:22:59,305 - tests.orchestrator - INFO - Created PVC: test-pvc-02e00a44 -2025-06-09 14:22:59,305 - tests.orchestrator - INFO - Waiting for PVC test-pvc-02e00a44 to be bound -2025-06-09 14:23:01,462 - tests.orchestrator - INFO - PVC test-pvc-02e00a44 is bound -2025-06-09 14:23:01,462 - tests.orchestrator - INFO - Creating PVC: test-pvc-4b26f6f6 -2025-06-09 14:23:01,585 - tests.orchestrator - INFO - Created PVC: test-pvc-4b26f6f6 -2025-06-09 14:23:01,585 - tests.orchestrator - INFO - Waiting for PVC test-pvc-4b26f6f6 to be bound diff --git a/test/stress-scale-tests/logs/orchestrator_20250609_142522.log b/test/stress-scale-tests/logs/orchestrator_20250609_142522.log deleted file mode 100644 index f1adc59d4..000000000 --- a/test/stress-scale-tests/logs/orchestrator_20250609_142522.log +++ /dev/null @@ -1,1058 +0,0 @@ -2025-06-09 14:25:22,187 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-09 14:25:22,403 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-09 14:25:22,404 - tests.orchestrator - INFO - Starting orchestrator test for 1200 seconds -2025-06-09 14:25:22,592 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-09 14:25:22,592 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-09 14:25:22,592 - tests.orchestrator - INFO - Creating PVC: test-pvc-da1ffeef -2025-06-09 14:25:22,670 - tests.orchestrator - INFO - Created PVC: test-pvc-da1ffeef -2025-06-09 14:25:22,670 - tests.orchestrator - INFO - Waiting for PVC test-pvc-da1ffeef to be bound -2025-06-09 14:25:24,806 - tests.orchestrator - INFO - PVC test-pvc-da1ffeef is bound -2025-06-09 14:25:24,806 - tests.orchestrator - INFO - Creating PVC: test-pvc-add96f02 -2025-06-09 14:25:24,873 - tests.orchestrator - INFO - Created PVC: test-pvc-add96f02 -2025-06-09 14:25:24,873 - tests.orchestrator - INFO - Waiting for PVC test-pvc-add96f02 to be bound -2025-06-09 14:25:27,007 - tests.orchestrator - INFO - PVC test-pvc-add96f02 is bound -2025-06-09 14:25:27,007 - tests.orchestrator - INFO - Creating pod test-pod-74c06ab7 on PVC test-pvc-add96f02 -2025-06-09 14:25:27,091 - tests.orchestrator - INFO - Created pod: test-pod-74c06ab7 using PVC: test-pvc-add96f02 -2025-06-09 14:25:27,091 - tests.orchestrator - INFO - Waiting for pod test-pod-74c06ab7 to be ready -2025-06-09 14:25:27,157 - tests.orchestrator - INFO - Pod test-pod-74c06ab7 phase: Pending -2025-06-09 14:25:29,226 - tests.orchestrator - INFO - Pod test-pod-74c06ab7 phase: Running -2025-06-09 14:25:29,227 - tests.orchestrator - INFO - Pod test-pod-74c06ab7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:25:31,295 - tests.orchestrator - INFO - Pod test-pod-74c06ab7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:25:33,362 - tests.orchestrator - INFO - Pod test-pod-74c06ab7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:25:35,431 - tests.orchestrator - INFO - Pod test-pod-74c06ab7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:25:37,500 - tests.orchestrator - INFO - Pod test-pod-74c06ab7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:25:39,567 - tests.orchestrator - INFO - Pod test-pod-74c06ab7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:25:41,692 - tests.orchestrator - INFO - Pod test-pod-74c06ab7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:25:43,760 - tests.orchestrator - INFO - Pod test-pod-74c06ab7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:25:45,827 - tests.orchestrator - INFO - Pod test-pod-74c06ab7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:25:47,895 - tests.orchestrator - INFO - Pod test-pod-74c06ab7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:25:49,964 - tests.orchestrator - INFO - Pod test-pod-74c06ab7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-09 14:25:49,964 - tests.orchestrator - INFO - Pod test-pod-74c06ab7 is ready -2025-06-09 14:25:49,964 - tests.orchestrator - INFO - Creating pod test-pod-667e05fd on PVC test-pvc-add96f02 -2025-06-09 14:25:50,042 - tests.orchestrator - INFO - Created pod: test-pod-667e05fd using PVC: test-pvc-add96f02 -2025-06-09 14:25:50,043 - tests.orchestrator - INFO - Waiting for pod test-pod-667e05fd to be ready -2025-06-09 14:25:50,111 - tests.orchestrator - INFO - Pod test-pod-667e05fd phase: Pending -2025-06-09 14:25:52,177 - tests.orchestrator - INFO - Pod test-pod-667e05fd phase: Running -2025-06-09 14:25:52,177 - tests.orchestrator - INFO - Pod test-pod-667e05fd conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:25:54,246 - tests.orchestrator - INFO - Pod test-pod-667e05fd conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:25:56,315 - tests.orchestrator - INFO - Pod test-pod-667e05fd conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:25:58,385 - tests.orchestrator - INFO - Pod test-pod-667e05fd conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:26:00,454 - tests.orchestrator - INFO - Pod test-pod-667e05fd conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:26:02,523 - tests.orchestrator - INFO - Pod test-pod-667e05fd conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:26:04,592 - tests.orchestrator - INFO - Pod test-pod-667e05fd conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:26:06,662 - tests.orchestrator - INFO - Pod test-pod-667e05fd conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:26:08,730 - tests.orchestrator - INFO - Pod test-pod-667e05fd conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:26:10,799 - tests.orchestrator - INFO - Pod test-pod-667e05fd conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:26:12,875 - tests.orchestrator - INFO - Pod test-pod-667e05fd conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-09 14:26:12,875 - tests.orchestrator - INFO - Pod test-pod-667e05fd is ready -2025-06-09 14:26:12,875 - tests.orchestrator - INFO - Creating pod test-pod-00c7ec79 on PVC test-pvc-add96f02 -2025-06-09 14:26:12,971 - tests.orchestrator - INFO - Created pod: test-pod-00c7ec79 using PVC: test-pvc-add96f02 -2025-06-09 14:26:12,971 - tests.orchestrator - INFO - Waiting for pod test-pod-00c7ec79 to be ready -2025-06-09 14:26:13,050 - tests.orchestrator - INFO - Pod test-pod-00c7ec79 phase: Pending -2025-06-09 14:26:15,118 - tests.orchestrator - INFO - Pod test-pod-00c7ec79 phase: Running -2025-06-09 14:26:15,118 - tests.orchestrator - INFO - Pod test-pod-00c7ec79 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:26:17,187 - tests.orchestrator - INFO - Pod test-pod-00c7ec79 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:26:19,256 - tests.orchestrator - INFO - Pod test-pod-00c7ec79 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:26:21,322 - tests.orchestrator - INFO - Pod test-pod-00c7ec79 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:26:23,392 - tests.orchestrator - INFO - Pod test-pod-00c7ec79 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:26:25,460 - tests.orchestrator - INFO - Pod test-pod-00c7ec79 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:26:27,527 - tests.orchestrator - INFO - Pod test-pod-00c7ec79 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:26:29,596 - tests.orchestrator - INFO - Pod test-pod-00c7ec79 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:26:31,666 - tests.orchestrator - INFO - Pod test-pod-00c7ec79 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:26:33,733 - tests.orchestrator - INFO - Pod test-pod-00c7ec79 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-09 14:26:33,733 - tests.orchestrator - INFO - Pod test-pod-00c7ec79 is ready -2025-06-09 14:26:33,734 - tests.orchestrator - INFO - Testing read/write between pods test-pod-00c7ec79 and test-pod-667e05fd sharing PVC test-pvc-add96f02 -2025-06-09 14:26:33,734 - tests.orchestrator - INFO - Executing write command: kubectl exec -n efs-stress-test test-pod-00c7ec79 -- /bin/sh -c 'echo "Test content: bf00aef3-7774-4aae-9288-d929351fe781" > /data/test-e4d325ac.txt' -2025-06-09 14:26:35,176 - tests.orchestrator - INFO - Pod test-pod-00c7ec79 wrote to /data/test-e4d325ac.txt -2025-06-09 14:26:37,176 - tests.orchestrator - INFO - Executing read command: kubectl exec -n efs-stress-test test-pod-667e05fd -- cat /data/test-e4d325ac.txt -2025-06-09 14:26:38,472 - tests.orchestrator - INFO - Read result: 'Test content: bf00aef3-7774-4aae-9288-d929351fe781' -2025-06-09 14:26:38,472 - tests.orchestrator - INFO - Pod test-pod-667e05fd successfully read content written by test-pod-00c7ec79 -2025-06-09 14:26:38,472 - tests.orchestrator - INFO - Deleting pod: test-pod-74c06ab7 from PVC: test-pvc-add96f02 -2025-06-09 14:26:38,548 - tests.orchestrator - INFO - Waiting for pod test-pod-74c06ab7 to be deleted -2025-06-09 14:27:11,161 - tests.orchestrator - INFO - Pod test-pod-74c06ab7 has been deleted -2025-06-09 14:27:11,161 - tests.orchestrator - INFO - Deleted pod: test-pod-74c06ab7 -2025-06-09 14:27:11,161 - tests.orchestrator - INFO - Deleting PVC: test-pvc-add96f02 -2025-06-09 14:27:11,161 - tests.orchestrator - INFO - Deleting 2 pods using PVC test-pvc-add96f02 -2025-06-09 14:27:11,161 - tests.orchestrator - INFO - Deleting pod: test-pod-667e05fd from PVC: test-pvc-add96f02 -2025-06-09 14:27:11,238 - tests.orchestrator - INFO - Waiting for pod test-pod-667e05fd to be deleted -2025-06-09 14:27:43,855 - tests.orchestrator - INFO - Pod test-pod-667e05fd has been deleted -2025-06-09 14:27:43,855 - tests.orchestrator - INFO - Deleted pod: test-pod-667e05fd -2025-06-09 14:27:43,855 - tests.orchestrator - INFO - Deleting pod: test-pod-00c7ec79 from PVC: test-pvc-add96f02 -2025-06-09 14:27:43,930 - tests.orchestrator - INFO - Waiting for pod test-pod-00c7ec79 to be deleted -2025-06-09 14:28:16,549 - tests.orchestrator - INFO - Pod test-pod-00c7ec79 has been deleted -2025-06-09 14:28:16,549 - tests.orchestrator - INFO - Deleted pod: test-pod-00c7ec79 -2025-06-09 14:28:16,623 - tests.orchestrator - INFO - Waiting for PVC test-pvc-add96f02 to be deleted -2025-06-09 14:28:16,695 - tests.orchestrator - INFO - PVC test-pvc-add96f02 has been deleted -2025-06-09 14:28:16,695 - tests.orchestrator - INFO - Deleted PVC: test-pvc-add96f02 -2025-06-09 14:28:16,695 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations -2025-06-09 14:28:16,695 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 1 times) -2025-06-09 14:28:16,695 - tests.orchestrator - INFO - Creating PVC: test-pvc-867df8a7 -2025-06-09 14:28:16,776 - tests.orchestrator - INFO - Created PVC: test-pvc-867df8a7 -2025-06-09 14:28:16,776 - tests.orchestrator - INFO - Waiting for PVC test-pvc-867df8a7 to be bound -2025-06-09 14:28:18,914 - tests.orchestrator - INFO - PVC test-pvc-867df8a7 is bound -2025-06-09 14:28:19,915 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 1 times) -2025-06-09 14:28:19,915 - tests.orchestrator - INFO - Creating pod test-pod-6fd060e1 on PVC test-pvc-da1ffeef -2025-06-09 14:28:19,990 - tests.orchestrator - INFO - Created pod: test-pod-6fd060e1 using PVC: test-pvc-da1ffeef -2025-06-09 14:28:19,991 - tests.orchestrator - INFO - Waiting for pod test-pod-6fd060e1 to be ready -2025-06-09 14:28:20,059 - tests.orchestrator - INFO - Pod test-pod-6fd060e1 phase: Pending -2025-06-09 14:28:24,195 - tests.orchestrator - INFO - Pod test-pod-6fd060e1 phase: Running -2025-06-09 14:28:24,196 - tests.orchestrator - INFO - Pod test-pod-6fd060e1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:28:26,264 - tests.orchestrator - INFO - Pod test-pod-6fd060e1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:28:28,330 - tests.orchestrator - INFO - Pod test-pod-6fd060e1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:28:30,399 - tests.orchestrator - INFO - Pod test-pod-6fd060e1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:28:32,467 - tests.orchestrator - INFO - Pod test-pod-6fd060e1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:28:34,534 - tests.orchestrator - INFO - Pod test-pod-6fd060e1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:28:36,603 - tests.orchestrator - INFO - Pod test-pod-6fd060e1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:28:38,672 - tests.orchestrator - INFO - Pod test-pod-6fd060e1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:28:40,738 - tests.orchestrator - INFO - Pod test-pod-6fd060e1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-09 14:28:40,739 - tests.orchestrator - INFO - Pod test-pod-6fd060e1 is ready -2025-06-09 14:28:41,739 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 2 times) -2025-06-09 14:28:41,739 - tests.orchestrator - INFO - Creating PVC: test-pvc-ff5ff1dd -2025-06-09 14:28:41,847 - tests.orchestrator - INFO - Created PVC: test-pvc-ff5ff1dd -2025-06-09 14:28:41,847 - tests.orchestrator - INFO - Waiting for PVC test-pvc-ff5ff1dd to be bound -2025-06-09 14:28:43,985 - tests.orchestrator - INFO - PVC test-pvc-ff5ff1dd is bound -2025-06-09 14:28:44,985 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 2 times) -2025-06-09 14:28:44,985 - tests.orchestrator - INFO - Creating pod test-pod-fe9965de on PVC test-pvc-ff5ff1dd -2025-06-09 14:28:45,063 - tests.orchestrator - INFO - Created pod: test-pod-fe9965de using PVC: test-pvc-ff5ff1dd -2025-06-09 14:28:45,063 - tests.orchestrator - INFO - Waiting for pod test-pod-fe9965de to be ready -2025-06-09 14:28:45,133 - tests.orchestrator - INFO - Pod test-pod-fe9965de phase: Pending -2025-06-09 14:28:49,271 - tests.orchestrator - INFO - Pod test-pod-fe9965de phase: Running -2025-06-09 14:28:49,271 - tests.orchestrator - INFO - Pod test-pod-fe9965de conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:28:51,338 - tests.orchestrator - INFO - Pod test-pod-fe9965de conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:28:53,406 - tests.orchestrator - INFO - Pod test-pod-fe9965de conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:28:55,475 - tests.orchestrator - INFO - Pod test-pod-fe9965de conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:28:57,542 - tests.orchestrator - INFO - Pod test-pod-fe9965de conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:28:59,611 - tests.orchestrator - INFO - Pod test-pod-fe9965de conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:29:01,682 - tests.orchestrator - INFO - Pod test-pod-fe9965de conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:29:03,754 - tests.orchestrator - INFO - Pod test-pod-fe9965de conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:29:05,823 - tests.orchestrator - INFO - Pod test-pod-fe9965de conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:29:07,892 - tests.orchestrator - INFO - Pod test-pod-fe9965de conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-09 14:29:07,892 - tests.orchestrator - INFO - Pod test-pod-fe9965de is ready -2025-06-09 14:29:08,892 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 3 times) -2025-06-09 14:29:08,892 - tests.orchestrator - INFO - Creating PVC: test-pvc-19553927 -2025-06-09 14:29:08,970 - tests.orchestrator - INFO - Created PVC: test-pvc-19553927 -2025-06-09 14:29:08,970 - tests.orchestrator - INFO - Waiting for PVC test-pvc-19553927 to be bound -2025-06-09 14:29:11,105 - tests.orchestrator - INFO - PVC test-pvc-19553927 is bound -2025-06-09 14:29:12,105 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 4 times) -2025-06-09 14:29:12,105 - tests.orchestrator - INFO - Creating PVC: test-pvc-d5437f32 -2025-06-09 14:29:12,179 - tests.orchestrator - INFO - Created PVC: test-pvc-d5437f32 -2025-06-09 14:29:12,179 - tests.orchestrator - INFO - Waiting for PVC test-pvc-d5437f32 to be bound -2025-06-09 14:29:14,315 - tests.orchestrator - INFO - PVC test-pvc-d5437f32 is bound -2025-06-09 14:29:15,315 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario (selected 1 times) -2025-06-09 14:29:15,315 - tests.orchestrator - INFO - ============================================================ -2025-06-09 14:29:15,315 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_many_to_one -2025-06-09 14:29:15,316 - tests.orchestrator - INFO - ============================================================ -2025-06-09 14:29:15,316 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -2025-06-09 14:29:15,316 - tests.orchestrator - INFO - STARTING MANY-TO-ONE SCENARIO DIAGNOSTICS -2025-06-09 14:29:15,316 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -2025-06-09 14:29:15,316 - tests.orchestrator - INFO - [MANY2ONE] STEP 1: Creating dedicated PVC: many2one-654a0929 -2025-06-09 14:29:15,316 - tests.orchestrator - INFO - [MANY2ONE] Creating PVC manifest with storageClassName: efs-sc -2025-06-09 14:29:15,386 - tests.orchestrator - INFO - [MANY2ONE] PVC many2one-654a0929 created successfully -2025-06-09 14:29:15,386 - tests.orchestrator - INFO - [MANY2ONE] Waiting for PVC many2one-654a0929 to be bound... -2025-06-09 14:29:15,386 - tests.orchestrator - INFO - Waiting for PVC many2one-654a0929 to be bound -2025-06-09 14:29:17,517 - tests.orchestrator - INFO - PVC many2one-654a0929 is bound -2025-06-09 14:29:17,584 - tests.orchestrator - INFO - [MANY2ONE] PVC status: phase=Bound, capacity={'storage': '1Gi'} -2025-06-09 14:29:17,585 - tests.orchestrator - INFO - [MANY2ONE] STEP 2: Creating 4 pods for the same PVC many2one-654a0929 -2025-06-09 14:29:17,585 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 1/4 for PVC many2one-654a0929 -2025-06-09 14:29:17,585 - tests.orchestrator - INFO - Creating pod test-pod-5559abbe on PVC many2one-654a0929 -2025-06-09 14:29:17,660 - tests.orchestrator - INFO - Created pod: test-pod-5559abbe using PVC: many2one-654a0929 -2025-06-09 14:29:17,660 - tests.orchestrator - INFO - Waiting for pod test-pod-5559abbe to be ready -2025-06-09 14:29:17,728 - tests.orchestrator - INFO - Pod test-pod-5559abbe phase: Pending -2025-06-09 14:29:19,795 - tests.orchestrator - INFO - Pod test-pod-5559abbe phase: Running -2025-06-09 14:29:19,796 - tests.orchestrator - INFO - Pod test-pod-5559abbe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:29:21,866 - tests.orchestrator - INFO - Pod test-pod-5559abbe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:29:23,932 - tests.orchestrator - INFO - Pod test-pod-5559abbe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:29:26,001 - tests.orchestrator - INFO - Pod test-pod-5559abbe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:29:28,070 - tests.orchestrator - INFO - Pod test-pod-5559abbe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:29:30,137 - tests.orchestrator - INFO - Pod test-pod-5559abbe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:29:32,206 - tests.orchestrator - INFO - Pod test-pod-5559abbe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:29:34,275 - tests.orchestrator - INFO - Pod test-pod-5559abbe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:29:36,342 - tests.orchestrator - INFO - Pod test-pod-5559abbe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:29:38,411 - tests.orchestrator - INFO - Pod test-pod-5559abbe conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-09 14:29:38,411 - tests.orchestrator - INFO - Pod test-pod-5559abbe is ready -2025-06-09 14:29:38,411 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-5559abbe -2025-06-09 14:29:38,411 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 2/4 for PVC many2one-654a0929 -2025-06-09 14:29:38,411 - tests.orchestrator - INFO - Creating pod test-pod-f9432ba6 on PVC many2one-654a0929 -2025-06-09 14:29:38,486 - tests.orchestrator - INFO - Created pod: test-pod-f9432ba6 using PVC: many2one-654a0929 -2025-06-09 14:29:38,486 - tests.orchestrator - INFO - Waiting for pod test-pod-f9432ba6 to be ready -2025-06-09 14:29:38,554 - tests.orchestrator - INFO - Pod test-pod-f9432ba6 phase: Pending -2025-06-09 14:29:40,622 - tests.orchestrator - INFO - Pod test-pod-f9432ba6 phase: Running -2025-06-09 14:29:40,623 - tests.orchestrator - INFO - Pod test-pod-f9432ba6 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:29:42,697 - tests.orchestrator - INFO - Pod test-pod-f9432ba6 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:29:44,767 - tests.orchestrator - INFO - Pod test-pod-f9432ba6 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:29:46,836 - tests.orchestrator - INFO - Pod test-pod-f9432ba6 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:29:48,903 - tests.orchestrator - INFO - Pod test-pod-f9432ba6 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:29:50,971 - tests.orchestrator - INFO - Pod test-pod-f9432ba6 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:29:53,041 - tests.orchestrator - INFO - Pod test-pod-f9432ba6 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:29:55,109 - tests.orchestrator - INFO - Pod test-pod-f9432ba6 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-09 14:29:55,109 - tests.orchestrator - INFO - Pod test-pod-f9432ba6 is ready -2025-06-09 14:29:55,110 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-f9432ba6 -2025-06-09 14:29:55,110 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 3/4 for PVC many2one-654a0929 -2025-06-09 14:29:55,110 - tests.orchestrator - INFO - Creating pod test-pod-8843b6a8 on PVC many2one-654a0929 -2025-06-09 14:29:55,197 - tests.orchestrator - INFO - Created pod: test-pod-8843b6a8 using PVC: many2one-654a0929 -2025-06-09 14:29:55,197 - tests.orchestrator - INFO - Waiting for pod test-pod-8843b6a8 to be ready -2025-06-09 14:29:55,263 - tests.orchestrator - INFO - Pod test-pod-8843b6a8 phase: Pending -2025-06-09 14:29:59,400 - tests.orchestrator - INFO - Pod test-pod-8843b6a8 phase: Running -2025-06-09 14:29:59,400 - tests.orchestrator - INFO - Pod test-pod-8843b6a8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:30:01,467 - tests.orchestrator - INFO - Pod test-pod-8843b6a8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:30:03,536 - tests.orchestrator - INFO - Pod test-pod-8843b6a8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:30:05,605 - tests.orchestrator - INFO - Pod test-pod-8843b6a8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:30:07,674 - tests.orchestrator - INFO - Pod test-pod-8843b6a8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:30:09,742 - tests.orchestrator - INFO - Pod test-pod-8843b6a8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:30:11,950 - tests.orchestrator - INFO - Pod test-pod-8843b6a8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:30:14,021 - tests.orchestrator - INFO - Pod test-pod-8843b6a8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:30:16,090 - tests.orchestrator - INFO - Pod test-pod-8843b6a8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-09 14:30:16,090 - tests.orchestrator - INFO - Pod test-pod-8843b6a8 is ready -2025-06-09 14:30:16,090 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-8843b6a8 -2025-06-09 14:30:16,090 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 4/4 for PVC many2one-654a0929 -2025-06-09 14:30:16,090 - tests.orchestrator - INFO - Creating pod test-pod-5c8cee78 on PVC many2one-654a0929 -2025-06-09 14:30:16,166 - tests.orchestrator - INFO - Created pod: test-pod-5c8cee78 using PVC: many2one-654a0929 -2025-06-09 14:30:16,167 - tests.orchestrator - INFO - Waiting for pod test-pod-5c8cee78 to be ready -2025-06-09 14:30:16,235 - tests.orchestrator - INFO - Pod test-pod-5c8cee78 phase: Pending -2025-06-09 14:30:20,371 - tests.orchestrator - INFO - Pod test-pod-5c8cee78 phase: Running -2025-06-09 14:30:20,371 - tests.orchestrator - INFO - Pod test-pod-5c8cee78 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:30:22,440 - tests.orchestrator - INFO - Pod test-pod-5c8cee78 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:30:24,509 - tests.orchestrator - INFO - Pod test-pod-5c8cee78 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:30:26,589 - tests.orchestrator - INFO - Pod test-pod-5c8cee78 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:30:28,659 - tests.orchestrator - INFO - Pod test-pod-5c8cee78 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:30:30,727 - tests.orchestrator - INFO - Pod test-pod-5c8cee78 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:30:32,794 - tests.orchestrator - INFO - Pod test-pod-5c8cee78 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-09 14:30:32,795 - tests.orchestrator - INFO - Pod test-pod-5c8cee78 is ready -2025-06-09 14:30:32,795 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-5c8cee78 -2025-06-09 14:30:32,795 - tests.orchestrator - INFO - [MANY2ONE] Created 4/4 pods successfully -2025-06-09 14:30:32,795 - tests.orchestrator - INFO - [MANY2ONE] STEP 3: Testing read/write operations -2025-06-09 14:30:32,795 - tests.orchestrator - INFO - [MANY2ONE] Writer pod: test-pod-f9432ba6, Reader pod: test-pod-5559abbe -2025-06-09 14:30:32,795 - tests.orchestrator - INFO - [MANY2ONE] Test file: /data/many2one-a1b870f0.txt -2025-06-09 14:30:32,795 - tests.orchestrator - INFO - [MANY2ONE] Test content: Many2One test content: 9db66010-5289-410b-815e-3c54cfd68970 -2025-06-09 14:30:32,795 - tests.orchestrator - INFO - [MANY2ONE] Executing write command: kubectl exec -n efs-stress-test test-pod-f9432ba6 -- /bin/sh -c 'echo "Many2One test content: 9db66010-5289-410b-815e-3c54cfd68970" > /data/many2one-a1b870f0.txt' -2025-06-09 14:30:35,420 - tests.orchestrator - INFO - [MANY2ONE] Writer pod ls check: '-rw-r--r-- 1 50027 50027 60 Jun 9 14:30 /data/many2one-a1b870f0.txt' -2025-06-09 14:30:35,420 - tests.orchestrator - INFO - [MANY2ONE] Sleeping for 5 seconds to ensure filesystem sync -2025-06-09 14:30:40,420 - tests.orchestrator - INFO - [MANY2ONE] Executing read command: kubectl exec -n efs-stress-test test-pod-5559abbe -- cat /data/many2one-a1b870f0.txt -2025-06-09 14:30:41,956 - tests.orchestrator - INFO - [MANY2ONE] Read command result: 'Many2One test content: 9db66010-5289-410b-815e-3c54cfd68970' -2025-06-09 14:30:41,956 - tests.orchestrator - INFO - [MANY2ONE] SUCCESS: Many-to-one scenario successful with 4 pods -2025-06-09 14:30:41,956 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -2025-06-09 14:30:41,956 - tests.orchestrator - INFO - COMPLETED MANY-TO-ONE SCENARIO DIAGNOSTICS -2025-06-09 14:30:41,956 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -2025-06-09 14:30:41,956 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_many_to_one -2025-06-09 14:30:41,956 - tests.orchestrator - INFO - ------------------------------------------------------------ -2025-06-09 14:30:42,957 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 1 times) -2025-06-09 14:30:42,957 - tests.orchestrator - INFO - Testing read/write between pods test-pod-8843b6a8 and test-pod-f9432ba6 sharing PVC many2one-654a0929 -2025-06-09 14:30:42,957 - tests.orchestrator - INFO - Executing write command: kubectl exec -n efs-stress-test test-pod-8843b6a8 -- /bin/sh -c 'echo "Test content: 9401f428-fb98-4a34-a95d-1a15171af885" > /data/test-a691b284.txt' -2025-06-09 14:30:44,295 - tests.orchestrator - INFO - Pod test-pod-8843b6a8 wrote to /data/test-a691b284.txt -2025-06-09 14:30:46,295 - tests.orchestrator - INFO - Executing read command: kubectl exec -n efs-stress-test test-pod-f9432ba6 -- cat /data/test-a691b284.txt -2025-06-09 14:30:47,601 - tests.orchestrator - INFO - Read result: 'Test content: 9401f428-fb98-4a34-a95d-1a15171af885' -2025-06-09 14:30:47,601 - tests.orchestrator - INFO - Pod test-pod-f9432ba6 successfully read content written by test-pod-8843b6a8 -2025-06-09 14:30:48,601 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 2 times) -2025-06-09 14:30:48,602 - tests.orchestrator - INFO - Testing read/write between pods test-pod-5c8cee78 and test-pod-5559abbe sharing PVC many2one-654a0929 -2025-06-09 14:30:48,602 - tests.orchestrator - INFO - Executing write command: kubectl exec -n efs-stress-test test-pod-5c8cee78 -- /bin/sh -c 'echo "Test content: c297a021-7006-47e4-9a4b-06e221180f62" > /data/test-787c9f72.txt' -2025-06-09 14:30:49,904 - tests.orchestrator - INFO - Pod test-pod-5c8cee78 wrote to /data/test-787c9f72.txt -2025-06-09 14:30:51,905 - tests.orchestrator - INFO - Executing read command: kubectl exec -n efs-stress-test test-pod-5559abbe -- cat /data/test-787c9f72.txt -2025-06-09 14:30:53,226 - tests.orchestrator - INFO - Read result: 'Test content: c297a021-7006-47e4-9a4b-06e221180f62' -2025-06-09 14:30:53,226 - tests.orchestrator - INFO - Pod test-pod-5559abbe successfully read content written by test-pod-5c8cee78 -2025-06-09 14:30:54,226 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario (selected 2 times) -2025-06-09 14:30:54,226 - tests.orchestrator - INFO - ============================================================ -2025-06-09 14:30:54,226 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_many_to_one -2025-06-09 14:30:54,226 - tests.orchestrator - INFO - ============================================================ -2025-06-09 14:30:54,226 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -2025-06-09 14:30:54,226 - tests.orchestrator - INFO - STARTING MANY-TO-ONE SCENARIO DIAGNOSTICS -2025-06-09 14:30:54,227 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -2025-06-09 14:30:54,227 - tests.orchestrator - INFO - [MANY2ONE] STEP 1: Creating dedicated PVC: many2one-f8c35bac -2025-06-09 14:30:54,227 - tests.orchestrator - INFO - [MANY2ONE] Creating PVC manifest with storageClassName: efs-sc -2025-06-09 14:30:54,305 - tests.orchestrator - INFO - [MANY2ONE] PVC many2one-f8c35bac created successfully -2025-06-09 14:30:54,305 - tests.orchestrator - INFO - [MANY2ONE] Waiting for PVC many2one-f8c35bac to be bound... -2025-06-09 14:30:54,305 - tests.orchestrator - INFO - Waiting for PVC many2one-f8c35bac to be bound -2025-06-09 14:30:56,439 - tests.orchestrator - INFO - PVC many2one-f8c35bac is bound -2025-06-09 14:30:56,506 - tests.orchestrator - INFO - [MANY2ONE] PVC status: phase=Bound, capacity={'storage': '1Gi'} -2025-06-09 14:30:56,506 - tests.orchestrator - INFO - [MANY2ONE] STEP 2: Creating 5 pods for the same PVC many2one-f8c35bac -2025-06-09 14:30:56,506 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 1/5 for PVC many2one-f8c35bac -2025-06-09 14:30:56,506 - tests.orchestrator - INFO - Creating pod test-pod-5e4161c8 on PVC many2one-f8c35bac -2025-06-09 14:30:56,582 - tests.orchestrator - INFO - Created pod: test-pod-5e4161c8 using PVC: many2one-f8c35bac -2025-06-09 14:30:56,582 - tests.orchestrator - INFO - Waiting for pod test-pod-5e4161c8 to be ready -2025-06-09 14:30:56,652 - tests.orchestrator - INFO - Pod test-pod-5e4161c8 phase: Pending -2025-06-09 14:30:58,721 - tests.orchestrator - INFO - Pod test-pod-5e4161c8 phase: Running -2025-06-09 14:30:58,721 - tests.orchestrator - INFO - Pod test-pod-5e4161c8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:31:00,788 - tests.orchestrator - INFO - Pod test-pod-5e4161c8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:31:02,857 - tests.orchestrator - INFO - Pod test-pod-5e4161c8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:31:04,927 - tests.orchestrator - INFO - Pod test-pod-5e4161c8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:31:06,993 - tests.orchestrator - INFO - Pod test-pod-5e4161c8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:31:09,062 - tests.orchestrator - INFO - Pod test-pod-5e4161c8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:31:11,131 - tests.orchestrator - INFO - Pod test-pod-5e4161c8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:31:13,205 - tests.orchestrator - INFO - Pod test-pod-5e4161c8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:31:15,274 - tests.orchestrator - INFO - Pod test-pod-5e4161c8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:31:17,343 - tests.orchestrator - INFO - Pod test-pod-5e4161c8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:31:19,410 - tests.orchestrator - INFO - Pod test-pod-5e4161c8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-09 14:31:19,410 - tests.orchestrator - INFO - Pod test-pod-5e4161c8 is ready -2025-06-09 14:31:19,410 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-5e4161c8 -2025-06-09 14:31:19,410 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 2/5 for PVC many2one-f8c35bac -2025-06-09 14:31:19,411 - tests.orchestrator - INFO - Creating pod test-pod-592a8021 on PVC many2one-f8c35bac -2025-06-09 14:31:19,487 - tests.orchestrator - INFO - Created pod: test-pod-592a8021 using PVC: many2one-f8c35bac -2025-06-09 14:31:19,487 - tests.orchestrator - INFO - Waiting for pod test-pod-592a8021 to be ready -2025-06-09 14:31:19,554 - tests.orchestrator - INFO - Pod test-pod-592a8021 phase: Pending -2025-06-09 14:31:21,624 - tests.orchestrator - INFO - Pod test-pod-592a8021 phase: Running -2025-06-09 14:31:21,624 - tests.orchestrator - INFO - Pod test-pod-592a8021 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:31:23,690 - tests.orchestrator - INFO - Pod test-pod-592a8021 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:31:25,759 - tests.orchestrator - INFO - Pod test-pod-592a8021 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:31:27,828 - tests.orchestrator - INFO - Pod test-pod-592a8021 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:31:29,895 - tests.orchestrator - INFO - Pod test-pod-592a8021 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:31:31,964 - tests.orchestrator - INFO - Pod test-pod-592a8021 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:31:34,032 - tests.orchestrator - INFO - Pod test-pod-592a8021 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:31:36,099 - tests.orchestrator - INFO - Pod test-pod-592a8021 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:31:38,168 - tests.orchestrator - INFO - Pod test-pod-592a8021 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:31:40,237 - tests.orchestrator - INFO - Pod test-pod-592a8021 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:31:42,309 - tests.orchestrator - INFO - Pod test-pod-592a8021 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-09 14:31:42,309 - tests.orchestrator - INFO - Pod test-pod-592a8021 is ready -2025-06-09 14:31:42,309 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-592a8021 -2025-06-09 14:31:42,309 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 3/5 for PVC many2one-f8c35bac -2025-06-09 14:31:42,309 - tests.orchestrator - INFO - Creating pod test-pod-953c034c on PVC many2one-f8c35bac -2025-06-09 14:31:42,407 - tests.orchestrator - INFO - Created pod: test-pod-953c034c using PVC: many2one-f8c35bac -2025-06-09 14:31:42,407 - tests.orchestrator - INFO - Waiting for pod test-pod-953c034c to be ready -2025-06-09 14:31:42,476 - tests.orchestrator - INFO - Pod test-pod-953c034c phase: Pending -2025-06-09 14:31:44,545 - tests.orchestrator - INFO - Pod test-pod-953c034c phase: Running -2025-06-09 14:31:44,545 - tests.orchestrator - INFO - Pod test-pod-953c034c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:31:46,614 - tests.orchestrator - INFO - Pod test-pod-953c034c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:31:48,681 - tests.orchestrator - INFO - Pod test-pod-953c034c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:31:50,750 - tests.orchestrator - INFO - Pod test-pod-953c034c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:31:52,819 - tests.orchestrator - INFO - Pod test-pod-953c034c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:31:54,886 - tests.orchestrator - INFO - Pod test-pod-953c034c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:31:56,955 - tests.orchestrator - INFO - Pod test-pod-953c034c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:31:59,024 - tests.orchestrator - INFO - Pod test-pod-953c034c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:32:01,091 - tests.orchestrator - INFO - Pod test-pod-953c034c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:32:03,160 - tests.orchestrator - INFO - Pod test-pod-953c034c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-09 14:32:03,160 - tests.orchestrator - INFO - Pod test-pod-953c034c is ready -2025-06-09 14:32:03,160 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-953c034c -2025-06-09 14:32:03,160 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 4/5 for PVC many2one-f8c35bac -2025-06-09 14:32:03,160 - tests.orchestrator - INFO - Creating pod test-pod-c20fb8e7 on PVC many2one-f8c35bac -2025-06-09 14:32:03,238 - tests.orchestrator - INFO - Created pod: test-pod-c20fb8e7 using PVC: many2one-f8c35bac -2025-06-09 14:32:03,238 - tests.orchestrator - INFO - Waiting for pod test-pod-c20fb8e7 to be ready -2025-06-09 14:32:03,306 - tests.orchestrator - INFO - Pod test-pod-c20fb8e7 phase: Pending -2025-06-09 14:32:05,376 - tests.orchestrator - INFO - Pod test-pod-c20fb8e7 phase: Running -2025-06-09 14:32:05,376 - tests.orchestrator - INFO - Pod test-pod-c20fb8e7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:32:07,443 - tests.orchestrator - INFO - Pod test-pod-c20fb8e7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:32:09,511 - tests.orchestrator - INFO - Pod test-pod-c20fb8e7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:32:11,639 - tests.orchestrator - INFO - Pod test-pod-c20fb8e7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:32:13,706 - tests.orchestrator - INFO - Pod test-pod-c20fb8e7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:32:15,775 - tests.orchestrator - INFO - Pod test-pod-c20fb8e7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:32:17,844 - tests.orchestrator - INFO - Pod test-pod-c20fb8e7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:32:19,911 - tests.orchestrator - INFO - Pod test-pod-c20fb8e7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:32:21,980 - tests.orchestrator - INFO - Pod test-pod-c20fb8e7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:32:24,050 - tests.orchestrator - INFO - Pod test-pod-c20fb8e7 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-09 14:32:24,050 - tests.orchestrator - INFO - Pod test-pod-c20fb8e7 is ready -2025-06-09 14:32:24,050 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-c20fb8e7 -2025-06-09 14:32:24,050 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 5/5 for PVC many2one-f8c35bac -2025-06-09 14:32:24,051 - tests.orchestrator - INFO - Creating pod test-pod-1b84e5d8 on PVC many2one-f8c35bac -2025-06-09 14:32:24,133 - tests.orchestrator - INFO - Created pod: test-pod-1b84e5d8 using PVC: many2one-f8c35bac -2025-06-09 14:32:24,133 - tests.orchestrator - INFO - Waiting for pod test-pod-1b84e5d8 to be ready -2025-06-09 14:32:24,202 - tests.orchestrator - INFO - Pod test-pod-1b84e5d8 phase: Pending -2025-06-09 14:32:26,269 - tests.orchestrator - INFO - Pod test-pod-1b84e5d8 phase: Running -2025-06-09 14:32:26,269 - tests.orchestrator - INFO - Pod test-pod-1b84e5d8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:32:28,355 - tests.orchestrator - INFO - Pod test-pod-1b84e5d8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:32:30,424 - tests.orchestrator - INFO - Pod test-pod-1b84e5d8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:32:32,491 - tests.orchestrator - INFO - Pod test-pod-1b84e5d8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:32:34,559 - tests.orchestrator - INFO - Pod test-pod-1b84e5d8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:32:36,628 - tests.orchestrator - INFO - Pod test-pod-1b84e5d8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:32:38,695 - tests.orchestrator - INFO - Pod test-pod-1b84e5d8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:32:40,764 - tests.orchestrator - INFO - Pod test-pod-1b84e5d8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:32:42,836 - tests.orchestrator - INFO - Pod test-pod-1b84e5d8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:32:44,903 - tests.orchestrator - INFO - Pod test-pod-1b84e5d8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-09 14:32:44,903 - tests.orchestrator - INFO - Pod test-pod-1b84e5d8 is ready -2025-06-09 14:32:44,903 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-1b84e5d8 -2025-06-09 14:32:44,903 - tests.orchestrator - INFO - [MANY2ONE] Created 5/5 pods successfully -2025-06-09 14:32:44,903 - tests.orchestrator - INFO - [MANY2ONE] STEP 3: Testing read/write operations -2025-06-09 14:32:44,903 - tests.orchestrator - INFO - [MANY2ONE] Writer pod: test-pod-1b84e5d8, Reader pod: test-pod-5e4161c8 -2025-06-09 14:32:44,903 - tests.orchestrator - INFO - [MANY2ONE] Test file: /data/many2one-0e18d56e.txt -2025-06-09 14:32:44,903 - tests.orchestrator - INFO - [MANY2ONE] Test content: Many2One test content: 2a157b23-d25d-4ab9-84e0-29eb3fec41a6 -2025-06-09 14:32:44,903 - tests.orchestrator - INFO - [MANY2ONE] Executing write command: kubectl exec -n efs-stress-test test-pod-1b84e5d8 -- /bin/sh -c 'echo "Many2One test content: 2a157b23-d25d-4ab9-84e0-29eb3fec41a6" > /data/many2one-0e18d56e.txt' -2025-06-09 14:32:47,545 - tests.orchestrator - INFO - [MANY2ONE] Writer pod ls check: '-rw-r--r-- 1 50028 50028 60 Jun 9 14:32 /data/many2one-0e18d56e.txt' -2025-06-09 14:32:47,545 - tests.orchestrator - INFO - [MANY2ONE] Sleeping for 5 seconds to ensure filesystem sync -2025-06-09 14:32:52,545 - tests.orchestrator - INFO - [MANY2ONE] Executing read command: kubectl exec -n efs-stress-test test-pod-5e4161c8 -- cat /data/many2one-0e18d56e.txt -2025-06-09 14:32:53,861 - tests.orchestrator - INFO - [MANY2ONE] Read command result: 'Many2One test content: 2a157b23-d25d-4ab9-84e0-29eb3fec41a6' -2025-06-09 14:32:53,861 - tests.orchestrator - INFO - [MANY2ONE] SUCCESS: Many-to-one scenario successful with 5 pods -2025-06-09 14:32:53,861 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -2025-06-09 14:32:53,861 - tests.orchestrator - INFO - COMPLETED MANY-TO-ONE SCENARIO DIAGNOSTICS -2025-06-09 14:32:53,861 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -2025-06-09 14:32:53,862 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_many_to_one -2025-06-09 14:32:53,862 - tests.orchestrator - INFO - ------------------------------------------------------------ -2025-06-09 14:32:54,862 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 1 times) -2025-06-09 14:32:54,862 - tests.orchestrator - INFO - Deleting PVC: test-pvc-da1ffeef -2025-06-09 14:32:54,862 - tests.orchestrator - INFO - Deleting 1 pods using PVC test-pvc-da1ffeef -2025-06-09 14:32:54,862 - tests.orchestrator - INFO - Deleting pod: test-pod-6fd060e1 from PVC: test-pvc-da1ffeef -2025-06-09 14:32:54,938 - tests.orchestrator - INFO - Waiting for pod test-pod-6fd060e1 to be deleted -2025-06-09 14:33:27,550 - tests.orchestrator - INFO - Pod test-pod-6fd060e1 has been deleted -2025-06-09 14:33:27,550 - tests.orchestrator - INFO - Deleted pod: test-pod-6fd060e1 -2025-06-09 14:33:27,625 - tests.orchestrator - INFO - Waiting for PVC test-pvc-da1ffeef to be deleted -2025-06-09 14:33:27,689 - tests.orchestrator - INFO - PVC test-pvc-da1ffeef has been deleted -2025-06-09 14:33:27,690 - tests.orchestrator - INFO - Deleted PVC: test-pvc-da1ffeef -2025-06-09 14:33:28,690 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 3 times) -2025-06-09 14:33:28,690 - tests.orchestrator - INFO - Testing read/write between pods test-pod-8843b6a8 and test-pod-5c8cee78 sharing PVC many2one-654a0929 -2025-06-09 14:33:28,690 - tests.orchestrator - INFO - Executing write command: kubectl exec -n efs-stress-test test-pod-8843b6a8 -- /bin/sh -c 'echo "Test content: dda92edc-ba48-49c3-88a3-1d4d7f4c3d27" > /data/test-be592f5a.txt' -2025-06-09 14:33:30,002 - tests.orchestrator - INFO - Pod test-pod-8843b6a8 wrote to /data/test-be592f5a.txt -2025-06-09 14:33:32,002 - tests.orchestrator - INFO - Executing read command: kubectl exec -n efs-stress-test test-pod-5c8cee78 -- cat /data/test-be592f5a.txt -2025-06-09 14:33:33,296 - tests.orchestrator - INFO - Read result: 'Test content: dda92edc-ba48-49c3-88a3-1d4d7f4c3d27' -2025-06-09 14:33:33,296 - tests.orchestrator - INFO - Pod test-pod-5c8cee78 successfully read content written by test-pod-8843b6a8 -2025-06-09 14:33:34,296 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 4 times) -2025-06-09 14:33:34,296 - tests.orchestrator - INFO - Testing read/write between pods test-pod-592a8021 and test-pod-953c034c sharing PVC many2one-f8c35bac -2025-06-09 14:33:34,296 - tests.orchestrator - INFO - Executing write command: kubectl exec -n efs-stress-test test-pod-592a8021 -- /bin/sh -c 'echo "Test content: 384e4dbc-dfb3-4109-b698-b34081fcf2f1" > /data/test-bddd9de4.txt' -2025-06-09 14:33:35,613 - tests.orchestrator - INFO - Pod test-pod-592a8021 wrote to /data/test-bddd9de4.txt -2025-06-09 14:33:37,613 - tests.orchestrator - INFO - Executing read command: kubectl exec -n efs-stress-test test-pod-953c034c -- cat /data/test-bddd9de4.txt -2025-06-09 14:33:38,931 - tests.orchestrator - INFO - Read result: 'Test content: 384e4dbc-dfb3-4109-b698-b34081fcf2f1' -2025-06-09 14:33:38,931 - tests.orchestrator - INFO - Pod test-pod-953c034c successfully read content written by test-pod-592a8021 -2025-06-09 14:33:39,931 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 1 times) -2025-06-09 14:33:39,931 - tests.orchestrator - INFO - Deleting pod: test-pod-5c8cee78 from PVC: many2one-654a0929 -2025-06-09 14:33:40,009 - tests.orchestrator - INFO - Waiting for pod test-pod-5c8cee78 to be deleted -2025-06-09 14:34:12,623 - tests.orchestrator - INFO - Pod test-pod-5c8cee78 has been deleted -2025-06-09 14:34:12,623 - tests.orchestrator - INFO - Deleted pod: test-pod-5c8cee78 -2025-06-09 14:34:13,624 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario (selected 3 times) -2025-06-09 14:34:13,624 - tests.orchestrator - INFO - ============================================================ -2025-06-09 14:34:13,624 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_many_to_one -2025-06-09 14:34:13,624 - tests.orchestrator - INFO - ============================================================ -2025-06-09 14:34:13,624 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -2025-06-09 14:34:13,624 - tests.orchestrator - INFO - STARTING MANY-TO-ONE SCENARIO DIAGNOSTICS -2025-06-09 14:34:13,624 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -2025-06-09 14:34:13,624 - tests.orchestrator - INFO - [MANY2ONE] STEP 1: Creating dedicated PVC: many2one-3cfcc636 -2025-06-09 14:34:13,624 - tests.orchestrator - INFO - [MANY2ONE] Creating PVC manifest with storageClassName: efs-sc -2025-06-09 14:34:13,703 - tests.orchestrator - INFO - [MANY2ONE] PVC many2one-3cfcc636 created successfully -2025-06-09 14:34:13,703 - tests.orchestrator - INFO - [MANY2ONE] Waiting for PVC many2one-3cfcc636 to be bound... -2025-06-09 14:34:13,703 - tests.orchestrator - INFO - Waiting for PVC many2one-3cfcc636 to be bound -2025-06-09 14:34:15,838 - tests.orchestrator - INFO - PVC many2one-3cfcc636 is bound -2025-06-09 14:34:15,905 - tests.orchestrator - INFO - [MANY2ONE] PVC status: phase=Bound, capacity={'storage': '1Gi'} -2025-06-09 14:34:15,905 - tests.orchestrator - INFO - [MANY2ONE] STEP 2: Creating 5 pods for the same PVC many2one-3cfcc636 -2025-06-09 14:34:15,905 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 1/5 for PVC many2one-3cfcc636 -2025-06-09 14:34:15,905 - tests.orchestrator - INFO - Creating pod test-pod-e2f780f1 on PVC many2one-3cfcc636 -2025-06-09 14:34:15,983 - tests.orchestrator - INFO - Created pod: test-pod-e2f780f1 using PVC: many2one-3cfcc636 -2025-06-09 14:34:15,983 - tests.orchestrator - INFO - Waiting for pod test-pod-e2f780f1 to be ready -2025-06-09 14:34:16,051 - tests.orchestrator - INFO - Pod test-pod-e2f780f1 phase: Pending -2025-06-09 14:34:18,131 - tests.orchestrator - INFO - Pod test-pod-e2f780f1 phase: Running -2025-06-09 14:34:18,131 - tests.orchestrator - INFO - Pod test-pod-e2f780f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:34:20,200 - tests.orchestrator - INFO - Pod test-pod-e2f780f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:34:23,114 - tests.orchestrator - INFO - Pod test-pod-e2f780f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:34:25,181 - tests.orchestrator - INFO - Pod test-pod-e2f780f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:34:27,250 - tests.orchestrator - INFO - Pod test-pod-e2f780f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:34:29,319 - tests.orchestrator - INFO - Pod test-pod-e2f780f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:34:31,386 - tests.orchestrator - INFO - Pod test-pod-e2f780f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:34:33,455 - tests.orchestrator - INFO - Pod test-pod-e2f780f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:34:35,523 - tests.orchestrator - INFO - Pod test-pod-e2f780f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:34:37,590 - tests.orchestrator - INFO - Pod test-pod-e2f780f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-09 14:34:37,590 - tests.orchestrator - INFO - Pod test-pod-e2f780f1 is ready -2025-06-09 14:34:37,590 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-e2f780f1 -2025-06-09 14:34:37,590 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 2/5 for PVC many2one-3cfcc636 -2025-06-09 14:34:37,591 - tests.orchestrator - INFO - Creating pod test-pod-1245ee3c on PVC many2one-3cfcc636 -2025-06-09 14:34:37,667 - tests.orchestrator - INFO - Created pod: test-pod-1245ee3c using PVC: many2one-3cfcc636 -2025-06-09 14:34:37,667 - tests.orchestrator - INFO - Waiting for pod test-pod-1245ee3c to be ready -2025-06-09 14:34:37,733 - tests.orchestrator - INFO - Pod test-pod-1245ee3c phase: Pending -2025-06-09 14:34:41,868 - tests.orchestrator - INFO - Pod test-pod-1245ee3c phase: Running -2025-06-09 14:34:41,868 - tests.orchestrator - INFO - Pod test-pod-1245ee3c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:34:43,937 - tests.orchestrator - INFO - Pod test-pod-1245ee3c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:34:46,006 - tests.orchestrator - INFO - Pod test-pod-1245ee3c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:34:48,073 - tests.orchestrator - INFO - Pod test-pod-1245ee3c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:34:50,142 - tests.orchestrator - INFO - Pod test-pod-1245ee3c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:34:52,211 - tests.orchestrator - INFO - Pod test-pod-1245ee3c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:34:54,278 - tests.orchestrator - INFO - Pod test-pod-1245ee3c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:34:56,347 - tests.orchestrator - INFO - Pod test-pod-1245ee3c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:34:58,416 - tests.orchestrator - INFO - Pod test-pod-1245ee3c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-09 14:34:58,416 - tests.orchestrator - INFO - Pod test-pod-1245ee3c is ready -2025-06-09 14:34:58,416 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-1245ee3c -2025-06-09 14:34:58,416 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 3/5 for PVC many2one-3cfcc636 -2025-06-09 14:34:58,416 - tests.orchestrator - INFO - Creating pod test-pod-b6422666 on PVC many2one-3cfcc636 -2025-06-09 14:34:58,500 - tests.orchestrator - INFO - Created pod: test-pod-b6422666 using PVC: many2one-3cfcc636 -2025-06-09 14:34:58,500 - tests.orchestrator - INFO - Waiting for pod test-pod-b6422666 to be ready -2025-06-09 14:34:58,581 - tests.orchestrator - INFO - Pod test-pod-b6422666 phase: Pending -2025-06-09 14:35:00,651 - tests.orchestrator - INFO - Pod test-pod-b6422666 phase: Running -2025-06-09 14:35:00,651 - tests.orchestrator - INFO - Pod test-pod-b6422666 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:35:02,720 - tests.orchestrator - INFO - Pod test-pod-b6422666 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:35:04,790 - tests.orchestrator - INFO - Pod test-pod-b6422666 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:35:06,857 - tests.orchestrator - INFO - Pod test-pod-b6422666 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:35:08,926 - tests.orchestrator - INFO - Pod test-pod-b6422666 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:35:10,995 - tests.orchestrator - INFO - Pod test-pod-b6422666 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:35:13,064 - tests.orchestrator - INFO - Pod test-pod-b6422666 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:35:15,133 - tests.orchestrator - INFO - Pod test-pod-b6422666 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:35:17,202 - tests.orchestrator - INFO - Pod test-pod-b6422666 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:35:19,268 - tests.orchestrator - INFO - Pod test-pod-b6422666 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:35:21,337 - tests.orchestrator - INFO - Pod test-pod-b6422666 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-09 14:35:21,337 - tests.orchestrator - INFO - Pod test-pod-b6422666 is ready -2025-06-09 14:35:21,338 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-b6422666 -2025-06-09 14:35:21,338 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 4/5 for PVC many2one-3cfcc636 -2025-06-09 14:35:21,338 - tests.orchestrator - INFO - Creating pod test-pod-043741f4 on PVC many2one-3cfcc636 -2025-06-09 14:35:21,414 - tests.orchestrator - INFO - Created pod: test-pod-043741f4 using PVC: many2one-3cfcc636 -2025-06-09 14:35:21,414 - tests.orchestrator - INFO - Waiting for pod test-pod-043741f4 to be ready -2025-06-09 14:35:21,494 - tests.orchestrator - INFO - Pod test-pod-043741f4 phase: Pending -2025-06-09 14:35:23,563 - tests.orchestrator - INFO - Pod test-pod-043741f4 phase: Running -2025-06-09 14:35:23,563 - tests.orchestrator - INFO - Pod test-pod-043741f4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:35:25,631 - tests.orchestrator - INFO - Pod test-pod-043741f4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:35:27,700 - tests.orchestrator - INFO - Pod test-pod-043741f4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:35:29,768 - tests.orchestrator - INFO - Pod test-pod-043741f4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:35:31,835 - tests.orchestrator - INFO - Pod test-pod-043741f4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:35:33,904 - tests.orchestrator - INFO - Pod test-pod-043741f4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:35:35,972 - tests.orchestrator - INFO - Pod test-pod-043741f4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:35:38,040 - tests.orchestrator - INFO - Pod test-pod-043741f4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:35:40,109 - tests.orchestrator - INFO - Pod test-pod-043741f4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:35:42,185 - tests.orchestrator - INFO - Pod test-pod-043741f4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-09 14:35:42,185 - tests.orchestrator - INFO - Pod test-pod-043741f4 is ready -2025-06-09 14:35:42,186 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-043741f4 -2025-06-09 14:35:42,186 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 5/5 for PVC many2one-3cfcc636 -2025-06-09 14:35:42,186 - tests.orchestrator - INFO - Creating pod test-pod-9b313857 on PVC many2one-3cfcc636 -2025-06-09 14:35:42,282 - tests.orchestrator - INFO - Created pod: test-pod-9b313857 using PVC: many2one-3cfcc636 -2025-06-09 14:35:42,282 - tests.orchestrator - INFO - Waiting for pod test-pod-9b313857 to be ready -2025-06-09 14:35:42,354 - tests.orchestrator - INFO - Pod test-pod-9b313857 phase: Pending -2025-06-09 14:35:44,421 - tests.orchestrator - INFO - Pod test-pod-9b313857 phase: Running -2025-06-09 14:35:44,421 - tests.orchestrator - INFO - Pod test-pod-9b313857 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:35:46,492 - tests.orchestrator - INFO - Pod test-pod-9b313857 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:35:48,561 - tests.orchestrator - INFO - Pod test-pod-9b313857 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:35:50,627 - tests.orchestrator - INFO - Pod test-pod-9b313857 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:35:52,696 - tests.orchestrator - INFO - Pod test-pod-9b313857 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:35:54,764 - tests.orchestrator - INFO - Pod test-pod-9b313857 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:35:56,832 - tests.orchestrator - INFO - Pod test-pod-9b313857 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:35:58,901 - tests.orchestrator - INFO - Pod test-pod-9b313857 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:36:00,969 - tests.orchestrator - INFO - Pod test-pod-9b313857 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:36:03,036 - tests.orchestrator - INFO - Pod test-pod-9b313857 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-09 14:36:03,036 - tests.orchestrator - INFO - Pod test-pod-9b313857 is ready -2025-06-09 14:36:03,037 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-9b313857 -2025-06-09 14:36:03,037 - tests.orchestrator - INFO - [MANY2ONE] Created 5/5 pods successfully -2025-06-09 14:36:03,037 - tests.orchestrator - INFO - [MANY2ONE] STEP 3: Testing read/write operations -2025-06-09 14:36:03,037 - tests.orchestrator - INFO - [MANY2ONE] Writer pod: test-pod-b6422666, Reader pod: test-pod-043741f4 -2025-06-09 14:36:03,037 - tests.orchestrator - INFO - [MANY2ONE] Test file: /data/many2one-a602e48d.txt -2025-06-09 14:36:03,037 - tests.orchestrator - INFO - [MANY2ONE] Test content: Many2One test content: 1c0e68f4-7c6d-4048-abc6-aac816db2715 -2025-06-09 14:36:03,037 - tests.orchestrator - INFO - [MANY2ONE] Executing write command: kubectl exec -n efs-stress-test test-pod-b6422666 -- /bin/sh -c 'echo "Many2One test content: 1c0e68f4-7c6d-4048-abc6-aac816db2715" > /data/many2one-a602e48d.txt' -2025-06-09 14:36:05,660 - tests.orchestrator - INFO - [MANY2ONE] Writer pod ls check: '-rw-r--r-- 1 50020 50020 60 Jun 9 14:36 /data/many2one-a602e48d.txt' -2025-06-09 14:36:05,660 - tests.orchestrator - INFO - [MANY2ONE] Sleeping for 5 seconds to ensure filesystem sync -2025-06-09 14:36:10,660 - tests.orchestrator - INFO - [MANY2ONE] Executing read command: kubectl exec -n efs-stress-test test-pod-043741f4 -- cat /data/many2one-a602e48d.txt -2025-06-09 14:36:12,140 - tests.orchestrator - INFO - [MANY2ONE] Read command result: 'Many2One test content: 1c0e68f4-7c6d-4048-abc6-aac816db2715' -2025-06-09 14:36:12,140 - tests.orchestrator - INFO - [MANY2ONE] SUCCESS: Many-to-one scenario successful with 5 pods -2025-06-09 14:36:12,141 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -2025-06-09 14:36:12,141 - tests.orchestrator - INFO - COMPLETED MANY-TO-ONE SCENARIO DIAGNOSTICS -2025-06-09 14:36:12,141 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -2025-06-09 14:36:12,141 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_many_to_one -2025-06-09 14:36:12,141 - tests.orchestrator - INFO - ------------------------------------------------------------ -2025-06-09 14:36:13,141 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 5 times) -2025-06-09 14:36:13,141 - tests.orchestrator - INFO - Testing read/write between pods test-pod-c20fb8e7 and test-pod-953c034c sharing PVC many2one-f8c35bac -2025-06-09 14:36:13,141 - tests.orchestrator - INFO - Executing write command: kubectl exec -n efs-stress-test test-pod-c20fb8e7 -- /bin/sh -c 'echo "Test content: 9424e180-93c5-45ac-9c19-1f641c94e5dd" > /data/test-68c72f8a.txt' -2025-06-09 14:36:14,451 - tests.orchestrator - INFO - Pod test-pod-c20fb8e7 wrote to /data/test-68c72f8a.txt -2025-06-09 14:36:16,451 - tests.orchestrator - INFO - Executing read command: kubectl exec -n efs-stress-test test-pod-953c034c -- cat /data/test-68c72f8a.txt -2025-06-09 14:36:17,756 - tests.orchestrator - INFO - Read result: 'Test content: 9424e180-93c5-45ac-9c19-1f641c94e5dd' -2025-06-09 14:36:17,756 - tests.orchestrator - INFO - Pod test-pod-953c034c successfully read content written by test-pod-c20fb8e7 -2025-06-09 14:36:18,756 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 3 times) -2025-06-09 14:36:18,756 - tests.orchestrator - INFO - Creating pod test-pod-1cd15e4f on PVC test-pvc-ff5ff1dd -2025-06-09 14:36:18,840 - tests.orchestrator - INFO - Created pod: test-pod-1cd15e4f using PVC: test-pvc-ff5ff1dd -2025-06-09 14:36:18,840 - tests.orchestrator - INFO - Waiting for pod test-pod-1cd15e4f to be ready -2025-06-09 14:36:18,909 - tests.orchestrator - INFO - Pod test-pod-1cd15e4f phase: Pending -2025-06-09 14:36:20,975 - tests.orchestrator - INFO - Pod test-pod-1cd15e4f phase: Running -2025-06-09 14:36:20,976 - tests.orchestrator - INFO - Pod test-pod-1cd15e4f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:36:23,044 - tests.orchestrator - INFO - Pod test-pod-1cd15e4f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:36:25,114 - tests.orchestrator - INFO - Pod test-pod-1cd15e4f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:36:27,181 - tests.orchestrator - INFO - Pod test-pod-1cd15e4f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:36:29,249 - tests.orchestrator - INFO - Pod test-pod-1cd15e4f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:36:31,318 - tests.orchestrator - INFO - Pod test-pod-1cd15e4f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:36:33,385 - tests.orchestrator - INFO - Pod test-pod-1cd15e4f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:36:35,454 - tests.orchestrator - INFO - Pod test-pod-1cd15e4f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:36:37,523 - tests.orchestrator - INFO - Pod test-pod-1cd15e4f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:36:39,590 - tests.orchestrator - INFO - Pod test-pod-1cd15e4f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-09 14:36:39,590 - tests.orchestrator - INFO - Pod test-pod-1cd15e4f is ready -2025-06-09 14:36:40,590 - tests.orchestrator - INFO - Selected operation: _attach_pod (selected 4 times) -2025-06-09 14:36:40,590 - tests.orchestrator - INFO - Creating pod test-pod-8d9fce38 on PVC test-pvc-19553927 -2025-06-09 14:36:40,666 - tests.orchestrator - INFO - Created pod: test-pod-8d9fce38 using PVC: test-pvc-19553927 -2025-06-09 14:36:40,666 - tests.orchestrator - INFO - Waiting for pod test-pod-8d9fce38 to be ready -2025-06-09 14:36:40,732 - tests.orchestrator - INFO - Pod test-pod-8d9fce38 phase: Pending -2025-06-09 14:36:42,804 - tests.orchestrator - INFO - Pod test-pod-8d9fce38 phase: Running -2025-06-09 14:36:42,805 - tests.orchestrator - INFO - Pod test-pod-8d9fce38 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:36:44,873 - tests.orchestrator - INFO - Pod test-pod-8d9fce38 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:36:46,942 - tests.orchestrator - INFO - Pod test-pod-8d9fce38 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:36:49,011 - tests.orchestrator - INFO - Pod test-pod-8d9fce38 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:36:51,078 - tests.orchestrator - INFO - Pod test-pod-8d9fce38 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:36:53,147 - tests.orchestrator - INFO - Pod test-pod-8d9fce38 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:36:55,215 - tests.orchestrator - INFO - Pod test-pod-8d9fce38 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:36:57,282 - tests.orchestrator - INFO - Pod test-pod-8d9fce38 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:36:59,351 - tests.orchestrator - INFO - Pod test-pod-8d9fce38 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:37:01,419 - tests.orchestrator - INFO - Pod test-pod-8d9fce38 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-09 14:37:01,420 - tests.orchestrator - INFO - Pod test-pod-8d9fce38 is ready -2025-06-09 14:37:02,420 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 5 times) -2025-06-09 14:37:02,420 - tests.orchestrator - INFO - Creating PVC: test-pvc-7fb8de82 -2025-06-09 14:37:02,503 - tests.orchestrator - INFO - Created PVC: test-pvc-7fb8de82 -2025-06-09 14:37:02,503 - tests.orchestrator - INFO - Waiting for PVC test-pvc-7fb8de82 to be bound -2025-06-09 14:37:04,638 - tests.orchestrator - INFO - PVC test-pvc-7fb8de82 is bound -2025-06-09 14:37:05,638 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 2 times) -2025-06-09 14:37:05,638 - tests.orchestrator - INFO - Deleting PVC: many2one-f8c35bac -2025-06-09 14:37:05,638 - tests.orchestrator - INFO - Deleting 5 pods using PVC many2one-f8c35bac -2025-06-09 14:37:05,638 - tests.orchestrator - INFO - Deleting pod: test-pod-5e4161c8 from PVC: many2one-f8c35bac -2025-06-09 14:37:05,716 - tests.orchestrator - INFO - Waiting for pod test-pod-5e4161c8 to be deleted -2025-06-09 14:37:38,325 - tests.orchestrator - INFO - Pod test-pod-5e4161c8 has been deleted -2025-06-09 14:37:38,325 - tests.orchestrator - INFO - Deleted pod: test-pod-5e4161c8 -2025-06-09 14:37:38,325 - tests.orchestrator - INFO - Deleting pod: test-pod-592a8021 from PVC: many2one-f8c35bac -2025-06-09 14:37:38,399 - tests.orchestrator - INFO - Waiting for pod test-pod-592a8021 to be deleted -2025-06-09 14:38:11,021 - tests.orchestrator - INFO - Pod test-pod-592a8021 has been deleted -2025-06-09 14:38:11,021 - tests.orchestrator - INFO - Deleted pod: test-pod-592a8021 -2025-06-09 14:38:11,021 - tests.orchestrator - INFO - Deleting pod: test-pod-953c034c from PVC: many2one-f8c35bac -2025-06-09 14:38:11,099 - tests.orchestrator - INFO - Waiting for pod test-pod-953c034c to be deleted -2025-06-09 14:38:43,711 - tests.orchestrator - INFO - Pod test-pod-953c034c has been deleted -2025-06-09 14:38:43,711 - tests.orchestrator - INFO - Deleted pod: test-pod-953c034c -2025-06-09 14:38:43,711 - tests.orchestrator - INFO - Deleting pod: test-pod-c20fb8e7 from PVC: many2one-f8c35bac -2025-06-09 14:38:43,788 - tests.orchestrator - INFO - Waiting for pod test-pod-c20fb8e7 to be deleted -2025-06-09 14:39:16,404 - tests.orchestrator - INFO - Pod test-pod-c20fb8e7 has been deleted -2025-06-09 14:39:16,404 - tests.orchestrator - INFO - Deleted pod: test-pod-c20fb8e7 -2025-06-09 14:39:16,405 - tests.orchestrator - INFO - Deleting pod: test-pod-1b84e5d8 from PVC: many2one-f8c35bac -2025-06-09 14:39:16,485 - tests.orchestrator - INFO - Waiting for pod test-pod-1b84e5d8 to be deleted -2025-06-09 14:39:49,101 - tests.orchestrator - INFO - Pod test-pod-1b84e5d8 has been deleted -2025-06-09 14:39:49,101 - tests.orchestrator - INFO - Deleted pod: test-pod-1b84e5d8 -2025-06-09 14:39:49,177 - tests.orchestrator - INFO - Waiting for PVC many2one-f8c35bac to be deleted -2025-06-09 14:39:49,243 - tests.orchestrator - INFO - PVC many2one-f8c35bac has been deleted -2025-06-09 14:39:49,243 - tests.orchestrator - INFO - Deleted PVC: many2one-f8c35bac -2025-06-09 14:39:50,244 - tests.orchestrator - INFO - Selected operation: _delete_pvc (selected 3 times) -2025-06-09 14:39:50,244 - tests.orchestrator - INFO - Deleting PVC: test-pvc-19553927 -2025-06-09 14:39:50,244 - tests.orchestrator - INFO - Deleting 1 pods using PVC test-pvc-19553927 -2025-06-09 14:39:50,244 - tests.orchestrator - INFO - Deleting pod: test-pod-8d9fce38 from PVC: test-pvc-19553927 -2025-06-09 14:39:50,319 - tests.orchestrator - INFO - Waiting for pod test-pod-8d9fce38 to be deleted -2025-06-09 14:40:22,937 - tests.orchestrator - INFO - Pod test-pod-8d9fce38 has been deleted -2025-06-09 14:40:22,937 - tests.orchestrator - INFO - Deleted pod: test-pod-8d9fce38 -2025-06-09 14:40:23,009 - tests.orchestrator - INFO - Waiting for PVC test-pvc-19553927 to be deleted -2025-06-09 14:40:23,075 - tests.orchestrator - INFO - PVC test-pvc-19553927 has been deleted -2025-06-09 14:40:23,075 - tests.orchestrator - INFO - Deleted PVC: test-pvc-19553927 -2025-06-09 14:40:24,075 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 6 times) -2025-06-09 14:40:24,076 - tests.orchestrator - INFO - Creating PVC: test-pvc-fc084aac -2025-06-09 14:40:24,147 - tests.orchestrator - INFO - Created PVC: test-pvc-fc084aac -2025-06-09 14:40:24,148 - tests.orchestrator - INFO - Waiting for PVC test-pvc-fc084aac to be bound -2025-06-09 14:40:26,281 - tests.orchestrator - INFO - PVC test-pvc-fc084aac is bound -2025-06-09 14:40:27,282 - tests.orchestrator - INFO - Selected operation: _verify_readwrite (selected 6 times) -2025-06-09 14:40:27,282 - tests.orchestrator - INFO - Testing read/write between pods test-pod-043741f4 and test-pod-9b313857 sharing PVC many2one-3cfcc636 -2025-06-09 14:40:27,282 - tests.orchestrator - INFO - Executing write command: kubectl exec -n efs-stress-test test-pod-043741f4 -- /bin/sh -c 'echo "Test content: 0283c180-fcf2-42d9-a801-d8be4bba70dc" > /data/test-140b67fc.txt' -2025-06-09 14:40:28,646 - tests.orchestrator - INFO - Pod test-pod-043741f4 wrote to /data/test-140b67fc.txt -2025-06-09 14:40:30,647 - tests.orchestrator - INFO - Executing read command: kubectl exec -n efs-stress-test test-pod-9b313857 -- cat /data/test-140b67fc.txt -2025-06-09 14:40:31,981 - tests.orchestrator - INFO - Read result: 'Test content: 0283c180-fcf2-42d9-a801-d8be4bba70dc' -2025-06-09 14:40:31,981 - tests.orchestrator - INFO - Pod test-pod-9b313857 successfully read content written by test-pod-043741f4 -2025-06-09 14:40:32,981 - tests.orchestrator - INFO - Selected operation: _create_pvc (selected 7 times) -2025-06-09 14:40:32,981 - tests.orchestrator - INFO - Creating PVC: test-pvc-2e916b1d -2025-06-09 14:40:33,050 - tests.orchestrator - INFO - Created PVC: test-pvc-2e916b1d -2025-06-09 14:40:33,050 - tests.orchestrator - INFO - Waiting for PVC test-pvc-2e916b1d to be bound -2025-06-09 14:40:35,185 - tests.orchestrator - INFO - PVC test-pvc-2e916b1d is bound -2025-06-09 14:40:36,185 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario (selected 4 times) -2025-06-09 14:40:36,185 - tests.orchestrator - INFO - ============================================================ -2025-06-09 14:40:36,185 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_many_to_one -2025-06-09 14:40:36,185 - tests.orchestrator - INFO - ============================================================ -2025-06-09 14:40:36,185 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -2025-06-09 14:40:36,185 - tests.orchestrator - INFO - STARTING MANY-TO-ONE SCENARIO DIAGNOSTICS -2025-06-09 14:40:36,185 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -2025-06-09 14:40:36,185 - tests.orchestrator - INFO - [MANY2ONE] STEP 1: Creating dedicated PVC: many2one-7fc11e0a -2025-06-09 14:40:36,185 - tests.orchestrator - INFO - [MANY2ONE] Creating PVC manifest with storageClassName: efs-sc -2025-06-09 14:40:36,254 - tests.orchestrator - INFO - [MANY2ONE] PVC many2one-7fc11e0a created successfully -2025-06-09 14:40:36,254 - tests.orchestrator - INFO - [MANY2ONE] Waiting for PVC many2one-7fc11e0a to be bound... -2025-06-09 14:40:36,254 - tests.orchestrator - INFO - Waiting for PVC many2one-7fc11e0a to be bound -2025-06-09 14:40:38,388 - tests.orchestrator - INFO - PVC many2one-7fc11e0a is bound -2025-06-09 14:40:38,455 - tests.orchestrator - INFO - [MANY2ONE] PVC status: phase=Bound, capacity={'storage': '1Gi'} -2025-06-09 14:40:38,456 - tests.orchestrator - INFO - [MANY2ONE] STEP 2: Creating 3 pods for the same PVC many2one-7fc11e0a -2025-06-09 14:40:38,456 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 1/3 for PVC many2one-7fc11e0a -2025-06-09 14:40:38,456 - tests.orchestrator - INFO - Creating pod test-pod-2c32e834 on PVC many2one-7fc11e0a -2025-06-09 14:40:38,536 - tests.orchestrator - INFO - Created pod: test-pod-2c32e834 using PVC: many2one-7fc11e0a -2025-06-09 14:40:38,536 - tests.orchestrator - INFO - Waiting for pod test-pod-2c32e834 to be ready -2025-06-09 14:40:38,605 - tests.orchestrator - INFO - Pod test-pod-2c32e834 phase: Pending -2025-06-09 14:40:42,749 - tests.orchestrator - INFO - Pod test-pod-2c32e834 phase: Running -2025-06-09 14:40:42,749 - tests.orchestrator - INFO - Pod test-pod-2c32e834 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:40:44,818 - tests.orchestrator - INFO - Pod test-pod-2c32e834 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:40:46,886 - tests.orchestrator - INFO - Pod test-pod-2c32e834 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:40:48,953 - tests.orchestrator - INFO - Pod test-pod-2c32e834 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:40:51,022 - tests.orchestrator - INFO - Pod test-pod-2c32e834 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:40:53,090 - tests.orchestrator - INFO - Pod test-pod-2c32e834 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:40:55,158 - tests.orchestrator - INFO - Pod test-pod-2c32e834 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-09 14:40:55,158 - tests.orchestrator - INFO - Pod test-pod-2c32e834 is ready -2025-06-09 14:40:55,158 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-2c32e834 -2025-06-09 14:40:55,158 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 2/3 for PVC many2one-7fc11e0a -2025-06-09 14:40:55,158 - tests.orchestrator - INFO - Creating pod test-pod-d04ff60a on PVC many2one-7fc11e0a -2025-06-09 14:40:55,244 - tests.orchestrator - INFO - Created pod: test-pod-d04ff60a using PVC: many2one-7fc11e0a -2025-06-09 14:40:55,244 - tests.orchestrator - INFO - Waiting for pod test-pod-d04ff60a to be ready -2025-06-09 14:40:55,310 - tests.orchestrator - INFO - Pod test-pod-d04ff60a phase: Pending -2025-06-09 14:40:57,378 - tests.orchestrator - INFO - Pod test-pod-d04ff60a phase: Running -2025-06-09 14:40:57,379 - tests.orchestrator - INFO - Pod test-pod-d04ff60a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:40:59,447 - tests.orchestrator - INFO - Pod test-pod-d04ff60a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:41:01,571 - tests.orchestrator - INFO - Pod test-pod-d04ff60a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:41:03,640 - tests.orchestrator - INFO - Pod test-pod-d04ff60a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:41:05,709 - tests.orchestrator - INFO - Pod test-pod-d04ff60a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:41:07,776 - tests.orchestrator - INFO - Pod test-pod-d04ff60a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:41:09,844 - tests.orchestrator - INFO - Pod test-pod-d04ff60a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:41:12,112 - tests.orchestrator - INFO - Pod test-pod-d04ff60a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:41:14,179 - tests.orchestrator - INFO - Pod test-pod-d04ff60a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:41:16,247 - tests.orchestrator - INFO - Pod test-pod-d04ff60a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-09 14:41:16,247 - tests.orchestrator - INFO - Pod test-pod-d04ff60a is ready -2025-06-09 14:41:16,248 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-d04ff60a -2025-06-09 14:41:16,248 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 3/3 for PVC many2one-7fc11e0a -2025-06-09 14:41:16,248 - tests.orchestrator - INFO - Creating pod test-pod-723a440a on PVC many2one-7fc11e0a -2025-06-09 14:41:16,323 - tests.orchestrator - INFO - Created pod: test-pod-723a440a using PVC: many2one-7fc11e0a -2025-06-09 14:41:16,323 - tests.orchestrator - INFO - Waiting for pod test-pod-723a440a to be ready -2025-06-09 14:41:16,392 - tests.orchestrator - INFO - Pod test-pod-723a440a phase: Pending -2025-06-09 14:41:18,459 - tests.orchestrator - INFO - Pod test-pod-723a440a phase: Running -2025-06-09 14:41:18,459 - tests.orchestrator - INFO - Pod test-pod-723a440a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:41:20,527 - tests.orchestrator - INFO - Pod test-pod-723a440a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:41:22,596 - tests.orchestrator - INFO - Pod test-pod-723a440a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:41:24,663 - tests.orchestrator - INFO - Pod test-pod-723a440a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:41:26,742 - tests.orchestrator - INFO - Pod test-pod-723a440a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:41:28,811 - tests.orchestrator - INFO - Pod test-pod-723a440a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:41:30,878 - tests.orchestrator - INFO - Pod test-pod-723a440a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:41:32,946 - tests.orchestrator - INFO - Pod test-pod-723a440a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:41:35,015 - tests.orchestrator - INFO - Pod test-pod-723a440a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:41:37,085 - tests.orchestrator - INFO - Pod test-pod-723a440a conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-09 14:41:37,085 - tests.orchestrator - INFO - Pod test-pod-723a440a is ready -2025-06-09 14:41:37,085 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-723a440a -2025-06-09 14:41:37,085 - tests.orchestrator - INFO - [MANY2ONE] Created 3/3 pods successfully -2025-06-09 14:41:37,085 - tests.orchestrator - INFO - [MANY2ONE] STEP 3: Testing read/write operations -2025-06-09 14:41:37,085 - tests.orchestrator - INFO - [MANY2ONE] Writer pod: test-pod-2c32e834, Reader pod: test-pod-723a440a -2025-06-09 14:41:37,085 - tests.orchestrator - INFO - [MANY2ONE] Test file: /data/many2one-a38a9757.txt -2025-06-09 14:41:37,085 - tests.orchestrator - INFO - [MANY2ONE] Test content: Many2One test content: bab6d8de-2f87-4a4c-a89a-55c41d6666a9 -2025-06-09 14:41:37,085 - tests.orchestrator - INFO - [MANY2ONE] Executing write command: kubectl exec -n efs-stress-test test-pod-2c32e834 -- /bin/sh -c 'echo "Many2One test content: bab6d8de-2f87-4a4c-a89a-55c41d6666a9" > /data/many2one-a38a9757.txt' -2025-06-09 14:41:39,710 - tests.orchestrator - INFO - [MANY2ONE] Writer pod ls check: '-rw-r--r-- 1 50030 50030 60 Jun 9 14:41 /data/many2one-a38a9757.txt' -2025-06-09 14:41:39,710 - tests.orchestrator - INFO - [MANY2ONE] Sleeping for 5 seconds to ensure filesystem sync -2025-06-09 14:41:44,710 - tests.orchestrator - INFO - [MANY2ONE] Executing read command: kubectl exec -n efs-stress-test test-pod-723a440a -- cat /data/many2one-a38a9757.txt -2025-06-09 14:41:46,012 - tests.orchestrator - INFO - [MANY2ONE] Read command result: 'Many2One test content: bab6d8de-2f87-4a4c-a89a-55c41d6666a9' -2025-06-09 14:41:46,012 - tests.orchestrator - INFO - [MANY2ONE] SUCCESS: Many-to-one scenario successful with 3 pods -2025-06-09 14:41:46,012 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -2025-06-09 14:41:46,012 - tests.orchestrator - INFO - COMPLETED MANY-TO-ONE SCENARIO DIAGNOSTICS -2025-06-09 14:41:46,012 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -2025-06-09 14:41:46,012 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_many_to_one -2025-06-09 14:41:46,012 - tests.orchestrator - INFO - ------------------------------------------------------------ -2025-06-09 14:41:47,012 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario (selected 5 times) -2025-06-09 14:41:47,013 - tests.orchestrator - INFO - ============================================================ -2025-06-09 14:41:47,013 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_many_to_one -2025-06-09 14:41:47,013 - tests.orchestrator - INFO - ============================================================ -2025-06-09 14:41:47,013 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -2025-06-09 14:41:47,013 - tests.orchestrator - INFO - STARTING MANY-TO-ONE SCENARIO DIAGNOSTICS -2025-06-09 14:41:47,013 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -2025-06-09 14:41:47,013 - tests.orchestrator - INFO - [MANY2ONE] STEP 1: Creating dedicated PVC: many2one-cbec9511 -2025-06-09 14:41:47,013 - tests.orchestrator - INFO - [MANY2ONE] Creating PVC manifest with storageClassName: efs-sc -2025-06-09 14:41:47,092 - tests.orchestrator - INFO - [MANY2ONE] PVC many2one-cbec9511 created successfully -2025-06-09 14:41:47,092 - tests.orchestrator - INFO - [MANY2ONE] Waiting for PVC many2one-cbec9511 to be bound... -2025-06-09 14:41:47,092 - tests.orchestrator - INFO - Waiting for PVC many2one-cbec9511 to be bound -2025-06-09 14:41:49,223 - tests.orchestrator - INFO - PVC many2one-cbec9511 is bound -2025-06-09 14:41:49,290 - tests.orchestrator - INFO - [MANY2ONE] PVC status: phase=Bound, capacity={'storage': '1Gi'} -2025-06-09 14:41:49,290 - tests.orchestrator - INFO - [MANY2ONE] STEP 2: Creating 5 pods for the same PVC many2one-cbec9511 -2025-06-09 14:41:49,290 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 1/5 for PVC many2one-cbec9511 -2025-06-09 14:41:49,290 - tests.orchestrator - INFO - Creating pod test-pod-9ec3d1f1 on PVC many2one-cbec9511 -2025-06-09 14:41:49,367 - tests.orchestrator - INFO - Created pod: test-pod-9ec3d1f1 using PVC: many2one-cbec9511 -2025-06-09 14:41:49,367 - tests.orchestrator - INFO - Waiting for pod test-pod-9ec3d1f1 to be ready -2025-06-09 14:41:49,436 - tests.orchestrator - INFO - Pod test-pod-9ec3d1f1 phase: Pending -2025-06-09 14:41:51,507 - tests.orchestrator - INFO - Pod test-pod-9ec3d1f1 phase: Running -2025-06-09 14:41:51,508 - tests.orchestrator - INFO - Pod test-pod-9ec3d1f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:41:53,574 - tests.orchestrator - INFO - Pod test-pod-9ec3d1f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:41:55,643 - tests.orchestrator - INFO - Pod test-pod-9ec3d1f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:41:57,712 - tests.orchestrator - INFO - Pod test-pod-9ec3d1f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:41:59,778 - tests.orchestrator - INFO - Pod test-pod-9ec3d1f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:42:01,847 - tests.orchestrator - INFO - Pod test-pod-9ec3d1f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:42:03,916 - tests.orchestrator - INFO - Pod test-pod-9ec3d1f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:42:05,983 - tests.orchestrator - INFO - Pod test-pod-9ec3d1f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:42:08,051 - tests.orchestrator - INFO - Pod test-pod-9ec3d1f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:42:10,120 - tests.orchestrator - INFO - Pod test-pod-9ec3d1f1 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-09 14:42:10,120 - tests.orchestrator - INFO - Pod test-pod-9ec3d1f1 is ready -2025-06-09 14:42:10,120 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-9ec3d1f1 -2025-06-09 14:42:10,120 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 2/5 for PVC many2one-cbec9511 -2025-06-09 14:42:10,121 - tests.orchestrator - INFO - Creating pod test-pod-9389b3d8 on PVC many2one-cbec9511 -2025-06-09 14:42:10,194 - tests.orchestrator - INFO - Created pod: test-pod-9389b3d8 using PVC: many2one-cbec9511 -2025-06-09 14:42:10,194 - tests.orchestrator - INFO - Waiting for pod test-pod-9389b3d8 to be ready -2025-06-09 14:42:10,262 - tests.orchestrator - INFO - Pod test-pod-9389b3d8 phase: Pending -2025-06-09 14:42:12,333 - tests.orchestrator - INFO - Pod test-pod-9389b3d8 phase: Running -2025-06-09 14:42:12,334 - tests.orchestrator - INFO - Pod test-pod-9389b3d8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:42:14,402 - tests.orchestrator - INFO - Pod test-pod-9389b3d8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:42:16,471 - tests.orchestrator - INFO - Pod test-pod-9389b3d8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:42:18,538 - tests.orchestrator - INFO - Pod test-pod-9389b3d8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:42:20,606 - tests.orchestrator - INFO - Pod test-pod-9389b3d8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:42:22,675 - tests.orchestrator - INFO - Pod test-pod-9389b3d8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:42:24,742 - tests.orchestrator - INFO - Pod test-pod-9389b3d8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:42:26,821 - tests.orchestrator - INFO - Pod test-pod-9389b3d8 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-09 14:42:26,821 - tests.orchestrator - INFO - Pod test-pod-9389b3d8 is ready -2025-06-09 14:42:26,821 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-9389b3d8 -2025-06-09 14:42:26,821 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 3/5 for PVC many2one-cbec9511 -2025-06-09 14:42:26,821 - tests.orchestrator - INFO - Creating pod test-pod-6b550453 on PVC many2one-cbec9511 -2025-06-09 14:42:26,906 - tests.orchestrator - INFO - Created pod: test-pod-6b550453 using PVC: many2one-cbec9511 -2025-06-09 14:42:26,906 - tests.orchestrator - INFO - Waiting for pod test-pod-6b550453 to be ready -2025-06-09 14:42:26,974 - tests.orchestrator - INFO - Pod test-pod-6b550453 phase: Pending -2025-06-09 14:42:31,111 - tests.orchestrator - INFO - Pod test-pod-6b550453 phase: Running -2025-06-09 14:42:31,111 - tests.orchestrator - INFO - Pod test-pod-6b550453 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:42:33,180 - tests.orchestrator - INFO - Pod test-pod-6b550453 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:42:35,246 - tests.orchestrator - INFO - Pod test-pod-6b550453 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:42:37,315 - tests.orchestrator - INFO - Pod test-pod-6b550453 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:42:39,384 - tests.orchestrator - INFO - Pod test-pod-6b550453 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:42:41,452 - tests.orchestrator - INFO - Pod test-pod-6b550453 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:42:43,520 - tests.orchestrator - INFO - Pod test-pod-6b550453 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:42:45,590 - tests.orchestrator - INFO - Pod test-pod-6b550453 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:42:47,658 - tests.orchestrator - INFO - Pod test-pod-6b550453 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-09 14:42:47,658 - tests.orchestrator - INFO - Pod test-pod-6b550453 is ready -2025-06-09 14:42:47,658 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-6b550453 -2025-06-09 14:42:47,658 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 4/5 for PVC many2one-cbec9511 -2025-06-09 14:42:47,658 - tests.orchestrator - INFO - Creating pod test-pod-9c4bbb5c on PVC many2one-cbec9511 -2025-06-09 14:42:47,735 - tests.orchestrator - INFO - Created pod: test-pod-9c4bbb5c using PVC: many2one-cbec9511 -2025-06-09 14:42:47,735 - tests.orchestrator - INFO - Waiting for pod test-pod-9c4bbb5c to be ready -2025-06-09 14:42:47,805 - tests.orchestrator - INFO - Pod test-pod-9c4bbb5c phase: Pending -2025-06-09 14:42:51,942 - tests.orchestrator - INFO - Pod test-pod-9c4bbb5c phase: Running -2025-06-09 14:42:51,943 - tests.orchestrator - INFO - Pod test-pod-9c4bbb5c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:42:54,010 - tests.orchestrator - INFO - Pod test-pod-9c4bbb5c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:42:56,078 - tests.orchestrator - INFO - Pod test-pod-9c4bbb5c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:42:58,147 - tests.orchestrator - INFO - Pod test-pod-9c4bbb5c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:43:00,214 - tests.orchestrator - INFO - Pod test-pod-9c4bbb5c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:43:02,283 - tests.orchestrator - INFO - Pod test-pod-9c4bbb5c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:43:04,352 - tests.orchestrator - INFO - Pod test-pod-9c4bbb5c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:43:06,419 - tests.orchestrator - INFO - Pod test-pod-9c4bbb5c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:43:08,488 - tests.orchestrator - INFO - Pod test-pod-9c4bbb5c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-09 14:43:08,488 - tests.orchestrator - INFO - Pod test-pod-9c4bbb5c is ready -2025-06-09 14:43:08,488 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-9c4bbb5c -2025-06-09 14:43:08,488 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 5/5 for PVC many2one-cbec9511 -2025-06-09 14:43:08,488 - tests.orchestrator - INFO - Creating pod test-pod-0eeb1063 on PVC many2one-cbec9511 -2025-06-09 14:43:08,574 - tests.orchestrator - INFO - Created pod: test-pod-0eeb1063 using PVC: many2one-cbec9511 -2025-06-09 14:43:08,574 - tests.orchestrator - INFO - Waiting for pod test-pod-0eeb1063 to be ready -2025-06-09 14:43:08,642 - tests.orchestrator - INFO - Pod test-pod-0eeb1063 phase: Pending -2025-06-09 14:43:12,781 - tests.orchestrator - INFO - Pod test-pod-0eeb1063 phase: Running -2025-06-09 14:43:12,781 - tests.orchestrator - INFO - Pod test-pod-0eeb1063 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:43:14,849 - tests.orchestrator - INFO - Pod test-pod-0eeb1063 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:43:16,918 - tests.orchestrator - INFO - Pod test-pod-0eeb1063 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:43:18,985 - tests.orchestrator - INFO - Pod test-pod-0eeb1063 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:43:21,054 - tests.orchestrator - INFO - Pod test-pod-0eeb1063 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:43:23,956 - tests.orchestrator - INFO - Pod test-pod-0eeb1063 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:43:26,023 - tests.orchestrator - INFO - Pod test-pod-0eeb1063 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-09 14:43:26,023 - tests.orchestrator - INFO - Pod test-pod-0eeb1063 is ready -2025-06-09 14:43:26,023 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-0eeb1063 -2025-06-09 14:43:26,023 - tests.orchestrator - INFO - [MANY2ONE] Created 5/5 pods successfully -2025-06-09 14:43:26,023 - tests.orchestrator - INFO - [MANY2ONE] STEP 3: Testing read/write operations -2025-06-09 14:43:26,023 - tests.orchestrator - INFO - [MANY2ONE] Writer pod: test-pod-6b550453, Reader pod: test-pod-9c4bbb5c -2025-06-09 14:43:26,023 - tests.orchestrator - INFO - [MANY2ONE] Test file: /data/many2one-79e58722.txt -2025-06-09 14:43:26,023 - tests.orchestrator - INFO - [MANY2ONE] Test content: Many2One test content: 4c5425ca-afa1-4ae9-99e0-0cae3ac50968 -2025-06-09 14:43:26,023 - tests.orchestrator - INFO - [MANY2ONE] Executing write command: kubectl exec -n efs-stress-test test-pod-6b550453 -- /bin/sh -c 'echo "Many2One test content: 4c5425ca-afa1-4ae9-99e0-0cae3ac50968" > /data/many2one-79e58722.txt' -2025-06-09 14:43:28,655 - tests.orchestrator - INFO - [MANY2ONE] Writer pod ls check: '-rw-r--r-- 1 50031 50031 60 Jun 9 14:43 /data/many2one-79e58722.txt' -2025-06-09 14:43:28,656 - tests.orchestrator - INFO - [MANY2ONE] Sleeping for 5 seconds to ensure filesystem sync -2025-06-09 14:43:33,656 - tests.orchestrator - INFO - [MANY2ONE] Executing read command: kubectl exec -n efs-stress-test test-pod-9c4bbb5c -- cat /data/many2one-79e58722.txt -2025-06-09 14:43:34,981 - tests.orchestrator - INFO - [MANY2ONE] Read command result: 'Many2One test content: 4c5425ca-afa1-4ae9-99e0-0cae3ac50968' -2025-06-09 14:43:34,981 - tests.orchestrator - INFO - [MANY2ONE] SUCCESS: Many-to-one scenario successful with 5 pods -2025-06-09 14:43:34,981 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -2025-06-09 14:43:34,981 - tests.orchestrator - INFO - COMPLETED MANY-TO-ONE SCENARIO DIAGNOSTICS -2025-06-09 14:43:34,981 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -2025-06-09 14:43:34,981 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_many_to_one -2025-06-09 14:43:34,981 - tests.orchestrator - INFO - ------------------------------------------------------------ -2025-06-09 14:43:35,981 - tests.orchestrator - INFO - Selected operation: _delete_pod (selected 2 times) -2025-06-09 14:43:35,982 - tests.orchestrator - INFO - Deleting pod: test-pod-723a440a from PVC: many2one-7fc11e0a -2025-06-09 14:43:36,058 - tests.orchestrator - INFO - Waiting for pod test-pod-723a440a to be deleted -2025-06-09 14:44:08,668 - tests.orchestrator - INFO - Pod test-pod-723a440a has been deleted -2025-06-09 14:44:08,668 - tests.orchestrator - INFO - Deleted pod: test-pod-723a440a -2025-06-09 14:44:09,669 - tests.orchestrator - INFO - Selected operation: _run_specific_scenario (selected 6 times) -2025-06-09 14:44:09,669 - tests.orchestrator - INFO - ============================================================ -2025-06-09 14:44:09,669 - tests.orchestrator - INFO - SELECTED SCENARIO: _scenario_many_to_one -2025-06-09 14:44:09,669 - tests.orchestrator - INFO - ============================================================ -2025-06-09 14:44:09,669 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -2025-06-09 14:44:09,669 - tests.orchestrator - INFO - STARTING MANY-TO-ONE SCENARIO DIAGNOSTICS -2025-06-09 14:44:09,669 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -2025-06-09 14:44:09,669 - tests.orchestrator - INFO - [MANY2ONE] STEP 1: Creating dedicated PVC: many2one-7c4a7e37 -2025-06-09 14:44:09,669 - tests.orchestrator - INFO - [MANY2ONE] Creating PVC manifest with storageClassName: efs-sc -2025-06-09 14:44:09,744 - tests.orchestrator - INFO - [MANY2ONE] PVC many2one-7c4a7e37 created successfully -2025-06-09 14:44:09,744 - tests.orchestrator - INFO - [MANY2ONE] Waiting for PVC many2one-7c4a7e37 to be bound... -2025-06-09 14:44:09,744 - tests.orchestrator - INFO - Waiting for PVC many2one-7c4a7e37 to be bound -2025-06-09 14:44:11,920 - tests.orchestrator - INFO - PVC many2one-7c4a7e37 is bound -2025-06-09 14:44:11,987 - tests.orchestrator - INFO - [MANY2ONE] PVC status: phase=Bound, capacity={'storage': '1Gi'} -2025-06-09 14:44:11,988 - tests.orchestrator - INFO - [MANY2ONE] STEP 2: Creating 3 pods for the same PVC many2one-7c4a7e37 -2025-06-09 14:44:11,988 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 1/3 for PVC many2one-7c4a7e37 -2025-06-09 14:44:11,988 - tests.orchestrator - INFO - Creating pod test-pod-c4f3501e on PVC many2one-7c4a7e37 -2025-06-09 14:44:12,073 - tests.orchestrator - INFO - Created pod: test-pod-c4f3501e using PVC: many2one-7c4a7e37 -2025-06-09 14:44:12,073 - tests.orchestrator - INFO - Waiting for pod test-pod-c4f3501e to be ready -2025-06-09 14:44:12,145 - tests.orchestrator - INFO - Pod test-pod-c4f3501e phase: Pending -2025-06-09 14:44:14,214 - tests.orchestrator - INFO - Pod test-pod-c4f3501e phase: Running -2025-06-09 14:44:14,214 - tests.orchestrator - INFO - Pod test-pod-c4f3501e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:44:16,286 - tests.orchestrator - INFO - Pod test-pod-c4f3501e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:44:18,357 - tests.orchestrator - INFO - Pod test-pod-c4f3501e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:44:20,430 - tests.orchestrator - INFO - Pod test-pod-c4f3501e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:44:22,500 - tests.orchestrator - INFO - Pod test-pod-c4f3501e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:44:24,567 - tests.orchestrator - INFO - Pod test-pod-c4f3501e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:44:26,638 - tests.orchestrator - INFO - Pod test-pod-c4f3501e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:44:28,707 - tests.orchestrator - INFO - Pod test-pod-c4f3501e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:44:30,773 - tests.orchestrator - INFO - Pod test-pod-c4f3501e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:44:32,842 - tests.orchestrator - INFO - Pod test-pod-c4f3501e conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-09 14:44:32,842 - tests.orchestrator - INFO - Pod test-pod-c4f3501e is ready -2025-06-09 14:44:32,842 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-c4f3501e -2025-06-09 14:44:32,842 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 2/3 for PVC many2one-7c4a7e37 -2025-06-09 14:44:32,842 - tests.orchestrator - INFO - Creating pod test-pod-b5fa3100 on PVC many2one-7c4a7e37 -2025-06-09 14:44:32,919 - tests.orchestrator - INFO - Created pod: test-pod-b5fa3100 using PVC: many2one-7c4a7e37 -2025-06-09 14:44:32,919 - tests.orchestrator - INFO - Waiting for pod test-pod-b5fa3100 to be ready -2025-06-09 14:44:32,987 - tests.orchestrator - INFO - Pod test-pod-b5fa3100 phase: Pending -2025-06-09 14:44:37,122 - tests.orchestrator - INFO - Pod test-pod-b5fa3100 phase: Running -2025-06-09 14:44:37,122 - tests.orchestrator - INFO - Pod test-pod-b5fa3100 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:44:39,192 - tests.orchestrator - INFO - Pod test-pod-b5fa3100 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:44:41,262 - tests.orchestrator - INFO - Pod test-pod-b5fa3100 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:44:43,329 - tests.orchestrator - INFO - Pod test-pod-b5fa3100 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:44:45,397 - tests.orchestrator - INFO - Pod test-pod-b5fa3100 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:44:47,466 - tests.orchestrator - INFO - Pod test-pod-b5fa3100 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:44:49,533 - tests.orchestrator - INFO - Pod test-pod-b5fa3100 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:44:51,606 - tests.orchestrator - INFO - Pod test-pod-b5fa3100 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:44:53,675 - tests.orchestrator - INFO - Pod test-pod-b5fa3100 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:44:55,742 - tests.orchestrator - INFO - Pod test-pod-b5fa3100 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-09 14:44:55,742 - tests.orchestrator - INFO - Pod test-pod-b5fa3100 is ready -2025-06-09 14:44:55,742 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-b5fa3100 -2025-06-09 14:44:55,742 - tests.orchestrator - INFO - [MANY2ONE] Creating pod 3/3 for PVC many2one-7c4a7e37 -2025-06-09 14:44:55,742 - tests.orchestrator - INFO - Creating pod test-pod-00d66dda on PVC many2one-7c4a7e37 -2025-06-09 14:44:55,827 - tests.orchestrator - INFO - Created pod: test-pod-00d66dda using PVC: many2one-7c4a7e37 -2025-06-09 14:44:55,827 - tests.orchestrator - INFO - Waiting for pod test-pod-00d66dda to be ready -2025-06-09 14:44:55,894 - tests.orchestrator - INFO - Pod test-pod-00d66dda phase: Pending -2025-06-09 14:44:57,962 - tests.orchestrator - INFO - Pod test-pod-00d66dda phase: Running -2025-06-09 14:44:57,962 - tests.orchestrator - INFO - Pod test-pod-00d66dda conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:45:00,029 - tests.orchestrator - INFO - Pod test-pod-00d66dda conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:45:02,098 - tests.orchestrator - INFO - Pod test-pod-00d66dda conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:45:04,167 - tests.orchestrator - INFO - Pod test-pod-00d66dda conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:45:06,234 - tests.orchestrator - INFO - Pod test-pod-00d66dda conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:45:08,302 - tests.orchestrator - INFO - Pod test-pod-00d66dda conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:45:10,371 - tests.orchestrator - INFO - Pod test-pod-00d66dda conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:45:12,438 - tests.orchestrator - INFO - Pod test-pod-00d66dda conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:45:14,507 - tests.orchestrator - INFO - Pod test-pod-00d66dda conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 14:45:16,578 - tests.orchestrator - INFO - Pod test-pod-00d66dda conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-09 14:45:16,578 - tests.orchestrator - INFO - Pod test-pod-00d66dda is ready -2025-06-09 14:45:16,578 - tests.orchestrator - INFO - [MANY2ONE] Successfully created and attached pod test-pod-00d66dda -2025-06-09 14:45:16,578 - tests.orchestrator - INFO - [MANY2ONE] Created 3/3 pods successfully -2025-06-09 14:45:16,578 - tests.orchestrator - INFO - [MANY2ONE] STEP 3: Testing read/write operations -2025-06-09 14:45:16,578 - tests.orchestrator - INFO - [MANY2ONE] Writer pod: test-pod-00d66dda, Reader pod: test-pod-c4f3501e -2025-06-09 14:45:16,578 - tests.orchestrator - INFO - [MANY2ONE] Test file: /data/many2one-7884b2d1.txt -2025-06-09 14:45:16,578 - tests.orchestrator - INFO - [MANY2ONE] Test content: Many2One test content: c20849b6-5111-4950-bbd6-2142ee259beb -2025-06-09 14:45:16,578 - tests.orchestrator - INFO - [MANY2ONE] Executing write command: kubectl exec -n efs-stress-test test-pod-00d66dda -- /bin/sh -c 'echo "Many2One test content: c20849b6-5111-4950-bbd6-2142ee259beb" > /data/many2one-7884b2d1.txt' -2025-06-09 14:45:19,195 - tests.orchestrator - INFO - [MANY2ONE] Writer pod ls check: '-rw-r--r-- 1 50032 50032 60 Jun 9 14:45 /data/many2one-7884b2d1.txt' -2025-06-09 14:45:19,195 - tests.orchestrator - INFO - [MANY2ONE] Sleeping for 5 seconds to ensure filesystem sync -2025-06-09 14:45:24,195 - tests.orchestrator - INFO - [MANY2ONE] Executing read command: kubectl exec -n efs-stress-test test-pod-c4f3501e -- cat /data/many2one-7884b2d1.txt -2025-06-09 14:45:25,483 - tests.orchestrator - INFO - [MANY2ONE] Read command result: 'Many2One test content: c20849b6-5111-4950-bbd6-2142ee259beb' -2025-06-09 14:45:25,484 - tests.orchestrator - INFO - [MANY2ONE] SUCCESS: Many-to-one scenario successful with 3 pods -2025-06-09 14:45:25,484 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -2025-06-09 14:45:25,484 - tests.orchestrator - INFO - COMPLETED MANY-TO-ONE SCENARIO DIAGNOSTICS -2025-06-09 14:45:25,484 - tests.orchestrator - INFO - ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -2025-06-09 14:45:25,484 - tests.orchestrator - INFO - COMPLETED SCENARIO: _scenario_many_to_one -2025-06-09 14:45:25,484 - tests.orchestrator - INFO - ------------------------------------------------------------ -2025-06-09 14:45:26,484 - tests.orchestrator - INFO - Test completed in 1204.08 seconds -2025-06-09 14:45:26,484 - tests.orchestrator - INFO - ===== STARTING COMPREHENSIVE CLEANUP ===== -2025-06-09 14:45:26,484 - tests.orchestrator - INFO - Deleting 20 pods (force=False)... -2025-06-09 14:45:26,484 - tests.orchestrator - INFO - Deleting pod: test-pod-fe9965de from PVC: test-pvc-ff5ff1dd -2025-06-09 14:45:26,561 - tests.orchestrator - INFO - Waiting for pod test-pod-fe9965de to be deleted -2025-06-09 14:45:59,174 - tests.orchestrator - INFO - Pod test-pod-fe9965de has been deleted -2025-06-09 14:45:59,174 - tests.orchestrator - INFO - Deleted pod: test-pod-fe9965de -2025-06-09 14:45:59,174 - tests.orchestrator - INFO - Deleting pod: test-pod-1cd15e4f from PVC: test-pvc-ff5ff1dd -2025-06-09 14:45:59,249 - tests.orchestrator - INFO - Waiting for pod test-pod-1cd15e4f to be deleted -2025-06-09 14:46:31,992 - tests.orchestrator - INFO - Pod test-pod-1cd15e4f has been deleted -2025-06-09 14:46:31,993 - tests.orchestrator - INFO - Deleted pod: test-pod-1cd15e4f -2025-06-09 14:46:31,993 - tests.orchestrator - INFO - Deleting pod: test-pod-5559abbe from PVC: many2one-654a0929 -2025-06-09 14:46:32,069 - tests.orchestrator - INFO - Waiting for pod test-pod-5559abbe to be deleted -2025-06-09 14:47:04,680 - tests.orchestrator - INFO - Pod test-pod-5559abbe has been deleted -2025-06-09 14:47:04,680 - tests.orchestrator - INFO - Deleted pod: test-pod-5559abbe -2025-06-09 14:47:04,680 - tests.orchestrator - INFO - Deleting pod: test-pod-f9432ba6 from PVC: many2one-654a0929 -2025-06-09 14:47:04,757 - tests.orchestrator - INFO - Waiting for pod test-pod-f9432ba6 to be deleted -2025-06-09 14:47:37,373 - tests.orchestrator - INFO - Pod test-pod-f9432ba6 has been deleted -2025-06-09 14:47:37,373 - tests.orchestrator - INFO - Deleted pod: test-pod-f9432ba6 -2025-06-09 14:47:37,373 - tests.orchestrator - INFO - Deleting pod: test-pod-8843b6a8 from PVC: many2one-654a0929 -2025-06-09 14:47:37,449 - tests.orchestrator - INFO - Waiting for pod test-pod-8843b6a8 to be deleted -2025-06-09 14:48:10,139 - tests.orchestrator - INFO - Pod test-pod-8843b6a8 has been deleted -2025-06-09 14:48:10,139 - tests.orchestrator - INFO - Deleted pod: test-pod-8843b6a8 -2025-06-09 14:48:10,139 - tests.orchestrator - INFO - Deleting pod: test-pod-e2f780f1 from PVC: many2one-3cfcc636 -2025-06-09 14:48:10,214 - tests.orchestrator - INFO - Waiting for pod test-pod-e2f780f1 to be deleted -2025-06-09 14:48:42,843 - tests.orchestrator - INFO - Pod test-pod-e2f780f1 has been deleted -2025-06-09 14:48:42,843 - tests.orchestrator - INFO - Deleted pod: test-pod-e2f780f1 -2025-06-09 14:48:42,843 - tests.orchestrator - INFO - Deleting pod: test-pod-1245ee3c from PVC: many2one-3cfcc636 -2025-06-09 14:48:42,924 - tests.orchestrator - INFO - Waiting for pod test-pod-1245ee3c to be deleted -2025-06-09 14:49:15,536 - tests.orchestrator - INFO - Pod test-pod-1245ee3c has been deleted -2025-06-09 14:49:15,537 - tests.orchestrator - INFO - Deleted pod: test-pod-1245ee3c -2025-06-09 14:49:15,537 - tests.orchestrator - INFO - Deleting pod: test-pod-b6422666 from PVC: many2one-3cfcc636 -2025-06-09 14:49:15,615 - tests.orchestrator - INFO - Waiting for pod test-pod-b6422666 to be deleted -2025-06-09 14:49:48,223 - tests.orchestrator - INFO - Pod test-pod-b6422666 has been deleted -2025-06-09 14:49:48,223 - tests.orchestrator - INFO - Deleted pod: test-pod-b6422666 -2025-06-09 14:49:48,223 - tests.orchestrator - INFO - Deleting pod: test-pod-043741f4 from PVC: many2one-3cfcc636 -2025-06-09 14:49:48,299 - tests.orchestrator - INFO - Waiting for pod test-pod-043741f4 to be deleted -2025-06-09 14:50:20,916 - tests.orchestrator - INFO - Pod test-pod-043741f4 has been deleted -2025-06-09 14:50:20,916 - tests.orchestrator - INFO - Deleted pod: test-pod-043741f4 -2025-06-09 14:50:20,916 - tests.orchestrator - INFO - Deleting pod: test-pod-9b313857 from PVC: many2one-3cfcc636 -2025-06-09 14:50:20,992 - tests.orchestrator - INFO - Waiting for pod test-pod-9b313857 to be deleted -2025-06-09 14:50:53,621 - tests.orchestrator - INFO - Pod test-pod-9b313857 has been deleted -2025-06-09 14:50:53,621 - tests.orchestrator - INFO - Deleted pod: test-pod-9b313857 -2025-06-09 14:50:53,621 - tests.orchestrator - INFO - Deleting pod: test-pod-2c32e834 from PVC: many2one-7fc11e0a -2025-06-09 14:50:53,696 - tests.orchestrator - INFO - Waiting for pod test-pod-2c32e834 to be deleted -2025-06-09 14:51:26,311 - tests.orchestrator - INFO - Pod test-pod-2c32e834 has been deleted -2025-06-09 14:51:26,311 - tests.orchestrator - INFO - Deleted pod: test-pod-2c32e834 -2025-06-09 14:51:26,311 - tests.orchestrator - INFO - Deleting pod: test-pod-d04ff60a from PVC: many2one-7fc11e0a -2025-06-09 14:51:26,388 - tests.orchestrator - INFO - Waiting for pod test-pod-d04ff60a to be deleted -2025-06-09 14:51:59,004 - tests.orchestrator - INFO - Pod test-pod-d04ff60a has been deleted -2025-06-09 14:51:59,004 - tests.orchestrator - INFO - Deleted pod: test-pod-d04ff60a -2025-06-09 14:51:59,004 - tests.orchestrator - INFO - Deleting pod: test-pod-9ec3d1f1 from PVC: many2one-cbec9511 -2025-06-09 14:51:59,082 - tests.orchestrator - INFO - Waiting for pod test-pod-9ec3d1f1 to be deleted -2025-06-09 14:52:32,541 - tests.orchestrator - INFO - Pod test-pod-9ec3d1f1 has been deleted -2025-06-09 14:52:32,541 - tests.orchestrator - INFO - Deleted pod: test-pod-9ec3d1f1 -2025-06-09 14:52:32,541 - tests.orchestrator - INFO - Deleting pod: test-pod-9389b3d8 from PVC: many2one-cbec9511 -2025-06-09 14:52:32,616 - tests.orchestrator - INFO - Waiting for pod test-pod-9389b3d8 to be deleted -2025-06-09 14:53:05,227 - tests.orchestrator - INFO - Pod test-pod-9389b3d8 has been deleted -2025-06-09 14:53:05,228 - tests.orchestrator - INFO - Deleted pod: test-pod-9389b3d8 -2025-06-09 14:53:05,228 - tests.orchestrator - INFO - Deleting pod: test-pod-6b550453 from PVC: many2one-cbec9511 -2025-06-09 14:53:05,303 - tests.orchestrator - INFO - Waiting for pod test-pod-6b550453 to be deleted -2025-06-09 14:53:37,916 - tests.orchestrator - INFO - Pod test-pod-6b550453 has been deleted -2025-06-09 14:53:37,916 - tests.orchestrator - INFO - Deleted pod: test-pod-6b550453 -2025-06-09 14:53:37,916 - tests.orchestrator - INFO - Deleting pod: test-pod-9c4bbb5c from PVC: many2one-cbec9511 -2025-06-09 14:53:37,993 - tests.orchestrator - INFO - Waiting for pod test-pod-9c4bbb5c to be deleted -2025-06-09 14:54:10,610 - tests.orchestrator - INFO - Pod test-pod-9c4bbb5c has been deleted -2025-06-09 14:54:10,610 - tests.orchestrator - INFO - Deleted pod: test-pod-9c4bbb5c -2025-06-09 14:54:10,610 - tests.orchestrator - INFO - Deleting pod: test-pod-0eeb1063 from PVC: many2one-cbec9511 -2025-06-09 14:54:10,691 - tests.orchestrator - INFO - Waiting for pod test-pod-0eeb1063 to be deleted -2025-06-09 14:54:43,304 - tests.orchestrator - INFO - Pod test-pod-0eeb1063 has been deleted -2025-06-09 14:54:43,305 - tests.orchestrator - INFO - Deleted pod: test-pod-0eeb1063 -2025-06-09 14:54:43,305 - tests.orchestrator - INFO - Deleting pod: test-pod-c4f3501e from PVC: many2one-7c4a7e37 -2025-06-09 14:54:43,380 - tests.orchestrator - INFO - Waiting for pod test-pod-c4f3501e to be deleted -2025-06-09 14:55:16,030 - tests.orchestrator - INFO - Pod test-pod-c4f3501e has been deleted -2025-06-09 14:55:16,030 - tests.orchestrator - INFO - Deleted pod: test-pod-c4f3501e -2025-06-09 14:55:16,030 - tests.orchestrator - INFO - Deleting pod: test-pod-b5fa3100 from PVC: many2one-7c4a7e37 -2025-06-09 14:55:16,106 - tests.orchestrator - INFO - Waiting for pod test-pod-b5fa3100 to be deleted -2025-06-09 14:55:48,720 - tests.orchestrator - INFO - Pod test-pod-b5fa3100 has been deleted -2025-06-09 14:55:48,721 - tests.orchestrator - INFO - Deleted pod: test-pod-b5fa3100 -2025-06-09 14:55:48,721 - tests.orchestrator - INFO - Deleting pod: test-pod-00d66dda from PVC: many2one-7c4a7e37 -2025-06-09 14:55:48,795 - tests.orchestrator - INFO - Waiting for pod test-pod-00d66dda to be deleted -2025-06-09 14:56:21,410 - tests.orchestrator - INFO - Pod test-pod-00d66dda has been deleted -2025-06-09 14:56:21,411 - tests.orchestrator - INFO - Deleted pod: test-pod-00d66dda -2025-06-09 14:56:26,411 - tests.orchestrator - INFO - Deleting 11 PVCs (force=False)... -2025-06-09 14:56:26,411 - tests.orchestrator - INFO - Deleting PVC: test-pvc-867df8a7 -2025-06-09 14:56:26,506 - tests.orchestrator - INFO - Waiting for PVC test-pvc-867df8a7 to be deleted -2025-06-09 14:56:26,580 - tests.orchestrator - INFO - PVC test-pvc-867df8a7 has been deleted -2025-06-09 14:56:26,580 - tests.orchestrator - INFO - Deleted PVC: test-pvc-867df8a7 -2025-06-09 14:56:26,580 - tests.orchestrator - INFO - Deleting PVC: test-pvc-ff5ff1dd -2025-06-09 14:56:26,672 - tests.orchestrator - INFO - Waiting for PVC test-pvc-ff5ff1dd to be deleted -2025-06-09 14:56:26,740 - tests.orchestrator - INFO - PVC test-pvc-ff5ff1dd has been deleted -2025-06-09 14:56:26,740 - tests.orchestrator - INFO - Deleted PVC: test-pvc-ff5ff1dd -2025-06-09 14:56:26,740 - tests.orchestrator - INFO - Deleting PVC: test-pvc-d5437f32 -2025-06-09 14:56:26,816 - tests.orchestrator - INFO - Waiting for PVC test-pvc-d5437f32 to be deleted -2025-06-09 14:56:26,886 - tests.orchestrator - INFO - PVC test-pvc-d5437f32 has been deleted -2025-06-09 14:56:26,887 - tests.orchestrator - INFO - Deleted PVC: test-pvc-d5437f32 -2025-06-09 14:56:26,887 - tests.orchestrator - INFO - Deleting PVC: many2one-654a0929 -2025-06-09 14:56:26,958 - tests.orchestrator - INFO - Waiting for PVC many2one-654a0929 to be deleted -2025-06-09 14:56:27,023 - tests.orchestrator - INFO - PVC many2one-654a0929 has been deleted -2025-06-09 14:56:27,023 - tests.orchestrator - INFO - Deleted PVC: many2one-654a0929 -2025-06-09 14:56:27,023 - tests.orchestrator - INFO - Deleting PVC: many2one-3cfcc636 -2025-06-09 14:56:27,098 - tests.orchestrator - INFO - Waiting for PVC many2one-3cfcc636 to be deleted -2025-06-09 14:56:27,164 - tests.orchestrator - INFO - PVC many2one-3cfcc636 has been deleted -2025-06-09 14:56:27,164 - tests.orchestrator - INFO - Deleted PVC: many2one-3cfcc636 -2025-06-09 14:56:27,164 - tests.orchestrator - INFO - Deleting PVC: test-pvc-7fb8de82 -2025-06-09 14:56:27,234 - tests.orchestrator - INFO - Waiting for PVC test-pvc-7fb8de82 to be deleted -2025-06-09 14:56:27,301 - tests.orchestrator - INFO - PVC test-pvc-7fb8de82 has been deleted -2025-06-09 14:56:27,301 - tests.orchestrator - INFO - Deleted PVC: test-pvc-7fb8de82 -2025-06-09 14:56:27,302 - tests.orchestrator - INFO - Deleting PVC: test-pvc-fc084aac -2025-06-09 14:56:27,374 - tests.orchestrator - INFO - Waiting for PVC test-pvc-fc084aac to be deleted -2025-06-09 14:56:27,440 - tests.orchestrator - INFO - PVC test-pvc-fc084aac has been deleted -2025-06-09 14:56:27,441 - tests.orchestrator - INFO - Deleted PVC: test-pvc-fc084aac -2025-06-09 14:56:27,441 - tests.orchestrator - INFO - Deleting PVC: test-pvc-2e916b1d -2025-06-09 14:56:27,511 - tests.orchestrator - INFO - Waiting for PVC test-pvc-2e916b1d to be deleted -2025-06-09 14:56:27,575 - tests.orchestrator - INFO - PVC test-pvc-2e916b1d has been deleted -2025-06-09 14:56:27,576 - tests.orchestrator - INFO - Deleted PVC: test-pvc-2e916b1d -2025-06-09 14:56:27,576 - tests.orchestrator - INFO - Deleting PVC: many2one-7fc11e0a -2025-06-09 14:56:27,649 - tests.orchestrator - INFO - Waiting for PVC many2one-7fc11e0a to be deleted -2025-06-09 14:56:27,717 - tests.orchestrator - INFO - PVC many2one-7fc11e0a has been deleted -2025-06-09 14:56:27,717 - tests.orchestrator - INFO - Deleted PVC: many2one-7fc11e0a -2025-06-09 14:56:27,717 - tests.orchestrator - INFO - Deleting PVC: many2one-cbec9511 -2025-06-09 14:56:27,790 - tests.orchestrator - INFO - Waiting for PVC many2one-cbec9511 to be deleted -2025-06-09 14:56:27,860 - tests.orchestrator - INFO - PVC many2one-cbec9511 has been deleted -2025-06-09 14:56:27,860 - tests.orchestrator - INFO - Deleted PVC: many2one-cbec9511 -2025-06-09 14:56:27,860 - tests.orchestrator - INFO - Deleting PVC: many2one-7c4a7e37 -2025-06-09 14:56:27,931 - tests.orchestrator - INFO - Waiting for PVC many2one-7c4a7e37 to be deleted -2025-06-09 14:56:27,996 - tests.orchestrator - INFO - PVC many2one-7c4a7e37 has been deleted -2025-06-09 14:56:27,996 - tests.orchestrator - INFO - Deleted PVC: many2one-7c4a7e37 -2025-06-09 14:56:28,201 - tests.orchestrator - WARNING - First cleanup pass incomplete. Remaining resources: ['pod/test-pod-0d0fddfe', 'pod/test-pod-158cdfa4', 'pod/test-pod-4044e577', 'pod/test-pod-87095965', 'pvc/test-pvc-02e00a44', 'pvc/test-pvc-0bb0d20c', 'pvc/test-pvc-2127e71a', 'pvc/test-pvc-2cf5b7a2', 'pvc/test-pvc-31fbd501', 'pvc/test-pvc-4ada8088', 'pvc/test-pvc-4b26f6f6', 'pvc/test-pvc-528b4f5c', 'pvc/test-pvc-5c13c143', 'pvc/test-pvc-c542ee58', 'pvc/test-pvc-dfd12d43', 'pvc/test-pvc-ecb12f74'] -2025-06-09 14:56:28,201 - tests.orchestrator - INFO - Attempting force deletion of remaining resources... -2025-06-09 14:56:28,202 - tests.orchestrator - INFO - Deleting 0 pods (force=True)... -2025-06-09 14:56:33,202 - tests.orchestrator - INFO - Deleting 0 PVCs (force=True)... -2025-06-09 14:56:33,407 - tests.orchestrator - ERROR - Cleanup incomplete. Remaining resources after force deletion: ['pod/test-pod-0d0fddfe', 'pod/test-pod-158cdfa4', 'pod/test-pod-4044e577', 'pod/test-pod-87095965', 'pvc/test-pvc-02e00a44', 'pvc/test-pvc-0bb0d20c', 'pvc/test-pvc-2127e71a', 'pvc/test-pvc-2cf5b7a2', 'pvc/test-pvc-31fbd501', 'pvc/test-pvc-4ada8088', 'pvc/test-pvc-4b26f6f6', 'pvc/test-pvc-528b4f5c', 'pvc/test-pvc-5c13c143', 'pvc/test-pvc-c542ee58', 'pvc/test-pvc-dfd12d43', 'pvc/test-pvc-ecb12f74'] -2025-06-09 14:56:33,408 - tests.orchestrator - INFO - Cleanup completed successfully in 666.92 seconds -2025-06-09 14:56:33,408 - tests.orchestrator - INFO - ===== CLEANUP PROCESS FINISHED ===== -2025-06-09 14:56:33,408 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-09 14:56:33,408 - tests.orchestrator - INFO - --- Operations --- -2025-06-09 14:56:33,408 - tests.orchestrator - INFO - create_pvc: 9 succeeded, 0 failed (100.0%) -2025-06-09 14:56:33,408 - tests.orchestrator - INFO - attach_pod: 32 succeeded, 0 failed (100.0%) -2025-06-09 14:56:33,408 - tests.orchestrator - INFO - delete_pod: 32 succeeded, 0 failed (100.0%) -2025-06-09 14:56:33,408 - tests.orchestrator - INFO - delete_pvc: 15 succeeded, 0 failed (100.0%) -2025-06-09 14:56:33,408 - tests.orchestrator - INFO - verify_read_write: Writes 7 succeeded, 0 failed (100.0%) -2025-06-09 14:56:33,408 - tests.orchestrator - INFO - verify_read_write: Reads 7 succeeded, 0 failed (100.0%) -2025-06-09 14:56:33,408 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-09 14:56:33,408 - tests.orchestrator - INFO - shared_volume_rw: 7 succeeded, 0 failed out of 7 runs (100.0%) -2025-06-09 14:56:33,408 - tests.orchestrator - INFO - many_to_one: 6 succeeded, 0 failed out of 6 runs (100.0%) -2025-06-09 14:56:33,408 - tests.orchestrator - INFO - one_to_one: No runs -2025-06-09 14:56:33,408 - tests.orchestrator - INFO - concurrent_pvc: No runs -2025-06-09 14:56:33,408 - tests.orchestrator - INFO - ========================================= diff --git a/test/stress-scale-tests/logs/orchestrator_20250609_182011.log b/test/stress-scale-tests/logs/orchestrator_20250609_182011.log deleted file mode 100644 index e370848f9..000000000 --- a/test/stress-scale-tests/logs/orchestrator_20250609_182011.log +++ /dev/null @@ -1,7 +0,0 @@ -2025-06-09 18:20:11,585 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-09 18:20:11,779 - tests.orchestrator - ERROR - Error checking namespace: (401) -Reason: Unauthorized -HTTP response headers: HTTPHeaderDict({'Audit-Id': 'd5d12f12-977a-4faf-bd61-ad049e1a9ec2', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Mon, 09 Jun 2025 18:20:11 GMT', 'Content-Length': '129'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} - - diff --git a/test/stress-scale-tests/logs/orchestrator_20250609_183658.log b/test/stress-scale-tests/logs/orchestrator_20250609_183658.log deleted file mode 100644 index 8e649feca..000000000 --- a/test/stress-scale-tests/logs/orchestrator_20250609_183658.log +++ /dev/null @@ -1,7 +0,0 @@ -2025-06-09 18:36:58,870 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-09 18:36:59,092 - tests.orchestrator - ERROR - Error checking namespace: (401) -Reason: Unauthorized -HTTP response headers: HTTPHeaderDict({'Audit-Id': 'd3c86ac1-4d01-4ad7-a9bc-d37363c9419a', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Mon, 09 Jun 2025 18:36:59 GMT', 'Content-Length': '129'}) -HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401} - - diff --git a/test/stress-scale-tests/logs/orchestrator_20250609_183729.log b/test/stress-scale-tests/logs/orchestrator_20250609_183729.log deleted file mode 100644 index 584a1fdbf..000000000 --- a/test/stress-scale-tests/logs/orchestrator_20250609_183729.log +++ /dev/null @@ -1,111 +0,0 @@ -2025-06-09 18:37:29,228 - tests.orchestrator - INFO - EFS CSI Orchestrator initialized -2025-06-09 18:37:29,434 - tests.orchestrator - INFO - Namespace 'efs-stress-test' already exists -2025-06-09 18:37:29,435 - tests.orchestrator - INFO - Starting orchestrator test for 120 seconds -2025-06-09 18:37:29,623 - tests.orchestrator - INFO - StorageClass 'efs-sc' already exists -2025-06-09 18:37:29,623 - tests.orchestrator - INFO - Running each operation type once to ensure coverage -2025-06-09 18:37:29,623 - tests.orchestrator - INFO - Creating PVC: test-pvc-c676e061 -2025-06-09 18:37:29,703 - tests.orchestrator - INFO - Created PVC: test-pvc-c676e061 -2025-06-09 18:37:29,703 - tests.orchestrator - INFO - Waiting for PVC test-pvc-c676e061 to be bound -2025-06-09 18:37:31,843 - tests.orchestrator - INFO - PVC test-pvc-c676e061 is bound -2025-06-09 18:37:31,843 - tests.orchestrator - INFO - Creating PVC: test-pvc-8b3b85d1 -2025-06-09 18:37:31,913 - tests.orchestrator - INFO - Created PVC: test-pvc-8b3b85d1 -2025-06-09 18:37:31,913 - tests.orchestrator - INFO - Waiting for PVC test-pvc-8b3b85d1 to be bound -2025-06-09 18:37:34,049 - tests.orchestrator - INFO - PVC test-pvc-8b3b85d1 is bound -2025-06-09 18:37:34,049 - tests.orchestrator - INFO - Creating pod test-pod-b3edaf2c on PVC test-pvc-c676e061 -2025-06-09 18:37:34,132 - tests.orchestrator - INFO - Created pod: test-pod-b3edaf2c using PVC: test-pvc-c676e061 -2025-06-09 18:37:34,132 - tests.orchestrator - INFO - Waiting for pod test-pod-b3edaf2c to be ready -2025-06-09 18:37:34,199 - tests.orchestrator - INFO - Pod test-pod-b3edaf2c phase: Pending -2025-06-09 18:37:36,269 - tests.orchestrator - INFO - Pod test-pod-b3edaf2c phase: Running -2025-06-09 18:37:36,269 - tests.orchestrator - INFO - Pod test-pod-b3edaf2c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 18:37:38,338 - tests.orchestrator - INFO - Pod test-pod-b3edaf2c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 18:37:40,405 - tests.orchestrator - INFO - Pod test-pod-b3edaf2c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 18:37:42,479 - tests.orchestrator - INFO - Pod test-pod-b3edaf2c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 18:37:44,549 - tests.orchestrator - INFO - Pod test-pod-b3edaf2c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 18:37:46,616 - tests.orchestrator - INFO - Pod test-pod-b3edaf2c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 18:37:48,685 - tests.orchestrator - INFO - Pod test-pod-b3edaf2c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 18:37:50,755 - tests.orchestrator - INFO - Pod test-pod-b3edaf2c conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-09 18:37:50,755 - tests.orchestrator - INFO - Pod test-pod-b3edaf2c is ready -2025-06-09 18:37:50,755 - tests.orchestrator - INFO - Creating pod test-pod-486f01f4 on PVC test-pvc-c676e061 -2025-06-09 18:37:50,829 - tests.orchestrator - INFO - Created pod: test-pod-486f01f4 using PVC: test-pvc-c676e061 -2025-06-09 18:37:50,829 - tests.orchestrator - INFO - Waiting for pod test-pod-486f01f4 to be ready -2025-06-09 18:37:50,902 - tests.orchestrator - INFO - Pod test-pod-486f01f4 phase: Pending -2025-06-09 18:37:55,040 - tests.orchestrator - INFO - Pod test-pod-486f01f4 phase: Running -2025-06-09 18:37:55,040 - tests.orchestrator - INFO - Pod test-pod-486f01f4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 18:37:57,109 - tests.orchestrator - INFO - Pod test-pod-486f01f4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 18:37:59,176 - tests.orchestrator - INFO - Pod test-pod-486f01f4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 18:38:01,245 - tests.orchestrator - INFO - Pod test-pod-486f01f4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 18:38:03,314 - tests.orchestrator - INFO - Pod test-pod-486f01f4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 18:38:05,383 - tests.orchestrator - INFO - Pod test-pod-486f01f4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 18:38:07,452 - tests.orchestrator - INFO - Pod test-pod-486f01f4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 18:38:09,521 - tests.orchestrator - INFO - Pod test-pod-486f01f4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 18:38:11,707 - tests.orchestrator - INFO - Pod test-pod-486f01f4 conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-09 18:38:11,707 - tests.orchestrator - INFO - Pod test-pod-486f01f4 is ready -2025-06-09 18:38:11,707 - tests.orchestrator - INFO - Creating pod test-pod-f51dba0f on PVC test-pvc-8b3b85d1 -2025-06-09 18:38:11,942 - tests.orchestrator - INFO - Created pod: test-pod-f51dba0f using PVC: test-pvc-8b3b85d1 -2025-06-09 18:38:11,943 - tests.orchestrator - INFO - Waiting for pod test-pod-f51dba0f to be ready -2025-06-09 18:38:12,057 - tests.orchestrator - INFO - Pod test-pod-f51dba0f phase: Pending -2025-06-09 18:38:14,126 - tests.orchestrator - INFO - Pod test-pod-f51dba0f phase: Running -2025-06-09 18:38:14,126 - tests.orchestrator - INFO - Pod test-pod-f51dba0f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 18:38:16,196 - tests.orchestrator - INFO - Pod test-pod-f51dba0f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 18:38:18,263 - tests.orchestrator - INFO - Pod test-pod-f51dba0f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 18:38:20,336 - tests.orchestrator - INFO - Pod test-pod-f51dba0f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 18:38:22,405 - tests.orchestrator - INFO - Pod test-pod-f51dba0f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 18:38:24,472 - tests.orchestrator - INFO - Pod test-pod-f51dba0f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 18:38:26,544 - tests.orchestrator - INFO - Pod test-pod-f51dba0f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 18:38:28,613 - tests.orchestrator - INFO - Pod test-pod-f51dba0f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 18:38:30,681 - tests.orchestrator - INFO - Pod test-pod-f51dba0f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=False, ContainersReady=False, PodScheduled=True -2025-06-09 18:38:32,751 - tests.orchestrator - INFO - Pod test-pod-f51dba0f conditions: PodReadyToStartContainers=True, Initialized=True, Ready=True, ContainersReady=True, PodScheduled=True -2025-06-09 18:38:32,751 - tests.orchestrator - INFO - Pod test-pod-f51dba0f is ready -2025-06-09 18:38:32,751 - tests.orchestrator - INFO - Testing read/write between pods test-pod-b3edaf2c and test-pod-486f01f4 sharing PVC test-pvc-c676e061 -2025-06-09 18:38:32,751 - tests.orchestrator - INFO - Executing write command: kubectl exec -n efs-stress-test test-pod-b3edaf2c -- /bin/sh -c 'echo "Test content: 101c0ddd-87ba-424d-8a9d-5dbe582e6e47" > /data/test-69dc76bb.txt' -2025-06-09 18:38:34,042 - tests.orchestrator - INFO - Pod test-pod-b3edaf2c wrote to /data/test-69dc76bb.txt -2025-06-09 18:38:36,042 - tests.orchestrator - INFO - Executing read command: kubectl exec -n efs-stress-test test-pod-486f01f4 -- cat /data/test-69dc76bb.txt -2025-06-09 18:38:37,308 - tests.orchestrator - INFO - Read result: 'Test content: 101c0ddd-87ba-424d-8a9d-5dbe582e6e47' -2025-06-09 18:38:37,308 - tests.orchestrator - INFO - Pod test-pod-486f01f4 successfully read content written by test-pod-b3edaf2c -2025-06-09 18:38:37,308 - tests.orchestrator - INFO - Deleting pod: test-pod-486f01f4 from PVC: test-pvc-c676e061 -2025-06-09 18:38:37,387 - tests.orchestrator - INFO - Waiting for pod test-pod-486f01f4 to be deleted -2025-06-09 18:39:10,176 - tests.orchestrator - INFO - Pod test-pod-486f01f4 has been deleted -2025-06-09 18:39:10,177 - tests.orchestrator - INFO - Deleted pod: test-pod-486f01f4 -2025-06-09 18:39:10,177 - tests.orchestrator - INFO - Deleting PVC: test-pvc-8b3b85d1 -2025-06-09 18:39:10,177 - tests.orchestrator - INFO - Deleting 1 pods using PVC test-pvc-8b3b85d1 -2025-06-09 18:39:10,177 - tests.orchestrator - INFO - Deleting pod: test-pod-f51dba0f from PVC: test-pvc-8b3b85d1 -2025-06-09 18:39:10,251 - tests.orchestrator - INFO - Waiting for pod test-pod-f51dba0f to be deleted -2025-06-09 18:39:42,876 - tests.orchestrator - INFO - Pod test-pod-f51dba0f has been deleted -2025-06-09 18:39:42,876 - tests.orchestrator - INFO - Deleted pod: test-pod-f51dba0f -2025-06-09 18:39:42,953 - tests.orchestrator - INFO - Waiting for PVC test-pvc-8b3b85d1 to be deleted -2025-06-09 18:39:47,093 - tests.orchestrator - INFO - PVC test-pvc-8b3b85d1 has been deleted -2025-06-09 18:39:47,093 - tests.orchestrator - INFO - Deleted PVC: test-pvc-8b3b85d1 -2025-06-09 18:39:47,093 - tests.orchestrator - INFO - Completed initial operation sequence, continuing with randomized operations -2025-06-09 18:39:47,093 - tests.orchestrator - INFO - Test completed in 137.66 seconds -2025-06-09 18:39:47,093 - tests.orchestrator - INFO - ===== STARTING COMPREHENSIVE CLEANUP ===== -2025-06-09 18:39:47,093 - tests.orchestrator - INFO - Deleting 1 pods (force=False)... -2025-06-09 18:39:47,093 - tests.orchestrator - INFO - Deleting pod: test-pod-b3edaf2c from PVC: test-pvc-c676e061 -2025-06-09 18:39:47,169 - tests.orchestrator - INFO - Waiting for pod test-pod-b3edaf2c to be deleted -2025-06-09 18:40:19,821 - tests.orchestrator - INFO - Pod test-pod-b3edaf2c has been deleted -2025-06-09 18:40:19,821 - tests.orchestrator - INFO - Deleted pod: test-pod-b3edaf2c -2025-06-09 18:40:24,822 - tests.orchestrator - INFO - Deleting 1 PVCs (force=False)... -2025-06-09 18:40:24,822 - tests.orchestrator - INFO - Deleting PVC: test-pvc-c676e061 -2025-06-09 18:40:24,898 - tests.orchestrator - INFO - Waiting for PVC test-pvc-c676e061 to be deleted -2025-06-09 18:40:24,965 - tests.orchestrator - INFO - PVC test-pvc-c676e061 has been deleted -2025-06-09 18:40:24,965 - tests.orchestrator - INFO - Deleted PVC: test-pvc-c676e061 -2025-06-09 18:40:25,180 - tests.orchestrator - WARNING - First cleanup pass incomplete. Remaining resources: ['pod/test-pod-0d0fddfe', 'pod/test-pod-158cdfa4', 'pod/test-pod-4044e577', 'pod/test-pod-87095965', 'pvc/test-pvc-02e00a44', 'pvc/test-pvc-0bb0d20c', 'pvc/test-pvc-2127e71a', 'pvc/test-pvc-2cf5b7a2', 'pvc/test-pvc-31fbd501', 'pvc/test-pvc-4ada8088', 'pvc/test-pvc-4b26f6f6', 'pvc/test-pvc-528b4f5c', 'pvc/test-pvc-5c13c143', 'pvc/test-pvc-c542ee58', 'pvc/test-pvc-dfd12d43', 'pvc/test-pvc-ecb12f74'] -2025-06-09 18:40:25,180 - tests.orchestrator - INFO - Attempting force deletion of remaining resources... -2025-06-09 18:40:25,180 - tests.orchestrator - INFO - Deleting 0 pods (force=True)... -2025-06-09 18:40:30,180 - tests.orchestrator - INFO - Deleting 0 PVCs (force=True)... -2025-06-09 18:40:30,387 - tests.orchestrator - ERROR - Cleanup incomplete. Remaining resources after force deletion: ['pod/test-pod-0d0fddfe', 'pod/test-pod-158cdfa4', 'pod/test-pod-4044e577', 'pod/test-pod-87095965', 'pvc/test-pvc-02e00a44', 'pvc/test-pvc-0bb0d20c', 'pvc/test-pvc-2127e71a', 'pvc/test-pvc-2cf5b7a2', 'pvc/test-pvc-31fbd501', 'pvc/test-pvc-4ada8088', 'pvc/test-pvc-4b26f6f6', 'pvc/test-pvc-528b4f5c', 'pvc/test-pvc-5c13c143', 'pvc/test-pvc-c542ee58', 'pvc/test-pvc-dfd12d43', 'pvc/test-pvc-ecb12f74'] -2025-06-09 18:40:30,387 - tests.orchestrator - INFO - Cleanup completed successfully in 43.29 seconds -2025-06-09 18:40:30,387 - tests.orchestrator - INFO - ===== CLEANUP PROCESS FINISHED ===== -2025-06-09 18:40:30,387 - tests.orchestrator - INFO - ===== EFS CSI Driver Test Summary ===== -2025-06-09 18:40:30,387 - tests.orchestrator - INFO - --- Operations --- -2025-06-09 18:40:30,387 - tests.orchestrator - INFO - create_pvc: 2 succeeded, 0 failed (100.0%) -2025-06-09 18:40:30,387 - tests.orchestrator - INFO - attach_pod: 3 succeeded, 0 failed (100.0%) -2025-06-09 18:40:30,387 - tests.orchestrator - INFO - delete_pod: 3 succeeded, 0 failed (100.0%) -2025-06-09 18:40:30,387 - tests.orchestrator - INFO - delete_pvc: 2 succeeded, 0 failed (100.0%) -2025-06-09 18:40:30,387 - tests.orchestrator - INFO - verify_read_write: Writes 1 succeeded, 0 failed (100.0%) -2025-06-09 18:40:30,388 - tests.orchestrator - INFO - verify_read_write: Reads 1 succeeded, 0 failed (100.0%) -2025-06-09 18:40:30,388 - tests.orchestrator - INFO - --- Scenarios --- -2025-06-09 18:40:30,388 - tests.orchestrator - INFO - shared_volume_rw: 1 succeeded, 0 failed out of 1 runs (100.0%) -2025-06-09 18:40:30,388 - tests.orchestrator - INFO - many_to_one: No runs -2025-06-09 18:40:30,388 - tests.orchestrator - INFO - one_to_one: No runs -2025-06-09 18:40:30,388 - tests.orchestrator - INFO - concurrent_pvc: No runs -2025-06-09 18:40:30,388 - tests.orchestrator - INFO - ========================================= diff --git a/test/stress-scale-tests/reports/general/efs_tests_20250605_201716_20250605_201716.json b/test/stress-scale-tests/reports/general/efs_tests_20250605_201716_20250605_201716.json deleted file mode 100644 index 1b0bbef9f..000000000 --- a/test/stress-scale-tests/reports/general/efs_tests_20250605_201716_20250605_201716.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "test_name": "efs_tests_20250605_201716", - "test_type": "general", - "timestamp": "20250605_201716", - "system_info": { - "hostname": "dev-dsk-ibhard-1d-b948589c.us-east-1.amazon.com", - "platform": "Linux-5.10.236-209.935.amzn2int.x86_64-x86_64-with-glibc2.26", - "python_version": "3.12.10", - "cpu_count": 8, - "memory_total_gb": 30.88, - "timestamp": "2025-06-05T20:17:16.474900", - "kubernetes_version": "Unknown", - "aws_region": "Unknown", - "efs_filesystem_id": "Unknown" - }, - "results": { - "orchestrator": { - "test_duration": 1749154636.4718866, - "operations": { - "create_pvc": { - "success": 2, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 11, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 11, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 4, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 1, - "write_fail": 0, - "read_success": 1, - "read_fail": 0, - "write_success_rate": 100.0, - "read_success_rate": 100.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 1, - "success": 1, - "fail": 0, - "success_rate": 100.0 - }, - "many_to_one": { - "runs": 2, - "success": 2, - "fail": 0, - "success_rate": 100.0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - } - } - }, - "metrics": { - "operations": {}, - "system": {}, - "kubernetes": {}, - "csi": {}, - "controller": { - "request_latency": {}, - "operation_counts": {}, - "success_rates": {}, - "volume_attach_timing": {} - }, - "node": { - "mount_timing": {}, - "mount_errors": {}, - "resource_utilization": {} - }, - "efs": { - "access_point_timing": {}, - "mount_completion_timing": {}, - "api_throttling_incidents": [] - }, - "k8s_events": { - "volume_events": [], - "binding_times": {}, - "pod_startup_delays": {} - } - } - } -} \ No newline at end of file diff --git a/test/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_201716.json b/test/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_201716.json deleted file mode 100644 index a62039169..000000000 --- a/test/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250605_201716.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "test_duration": 1749154636.4718866, - "operations": { - "create_pvc": { - "success": 2, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 11, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 11, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 4, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 1, - "write_fail": 0, - "read_success": 1, - "read_fail": 0, - "write_success_rate": 100.0, - "read_success_rate": 100.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 1, - "success": 1, - "fail": 0, - "success_rate": 100.0 - }, - "many_to_one": { - "runs": 2, - "success": 2, - "fail": 0, - "success_rate": 100.0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - } - } -} \ No newline at end of file diff --git a/test/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250609_145633.json b/test/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250609_145633.json deleted file mode 100644 index ecaa3f823..000000000 --- a/test/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250609_145633.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "test_duration": 1749480993.4082596, - "operations": { - "create_pvc": { - "success": 9, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 32, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 32, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 15, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 7, - "write_fail": 0, - "read_success": 7, - "read_fail": 0, - "write_success_rate": 100.0, - "read_success_rate": 100.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 7, - "success": 7, - "fail": 0, - "success_rate": 100.0 - }, - "many_to_one": { - "runs": 6, - "success": 6, - "fail": 0, - "success_rate": 100.0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - } - } -} \ No newline at end of file diff --git a/test/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250609_184030.json b/test/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250609_184030.json deleted file mode 100644 index 5472774ff..000000000 --- a/test/stress-scale-tests/reports/orchestrator/efs_orchestrator_20250609_184030.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "test_duration": 1749494430.3875926, - "operations": { - "create_pvc": { - "success": 2, - "fail": 0, - "success_rate": 100.0 - }, - "attach_pod": { - "success": 3, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pod": { - "success": 3, - "fail": 0, - "success_rate": 100.0 - }, - "delete_pvc": { - "success": 2, - "fail": 0, - "success_rate": 100.0 - }, - "verify_read_write": { - "write_success": 1, - "write_fail": 0, - "read_success": 1, - "read_fail": 0, - "write_success_rate": 100.0, - "read_success_rate": 100.0 - } - }, - "scenarios": { - "shared_volume_rw": { - "runs": 1, - "success": 1, - "fail": 0, - "success_rate": 100.0 - }, - "many_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "one_to_one": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - }, - "concurrent_pvc": { - "runs": 0, - "success": 0, - "fail": 0, - "success_rate": 0 - } - } -} \ No newline at end of file diff --git a/test/stress-scale-tests/test-reports/sample-test-report.json b/test/stress-scale-tests/test-reports/sample-test-report.json deleted file mode 100644 index d75b5ff76..000000000 --- a/test/stress-scale-tests/test-reports/sample-test-report.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "test_name": "EFS CSI Stress Test", - "test_type": "stress", - "timestamp": "20250609_180200", - "system_info": { - "hostname": "test-vm", - "platform": "Linux-5.10.0", - "kubernetes_version": "v1.25", - "aws_region": "us-west-2", - "efs_filesystem_id": "fs-1234abcd" - }, - "results": { - "test_duration": 300, - "test_info": { - "test_name": "EFS CSI Stress Test", - "test_type": "stress", - "duration": 300 - }, - "operations": { - "create_pvc_1": { - "start_time": 1717686500.0, - "end_time": 1717686502.4, - "duration": 2.4, - "status": "success" - }, - "create_pvc_2": { - "start_time": 1717686505.0, - "end_time": 1717686508.1, - "duration": 3.1, - "status": "success" - }, - "attach_pod_1": { - "start_time": 1717686510.0, - "end_time": 1717686518.5, - "duration": 8.5, - "status": "success" - }, - "attach_pod_2": { - "start_time": 1717686520.0, - "end_time": 1717686527.2, - "duration": 7.2, - "status": "success" - }, - "verify_readwrite_1": { - "start_time": 1717686530.0, - "end_time": 1717686531.3, - "duration": 1.3, - "status": "success" - }, - "delete_pod_1": { - "start_time": 1717686540.0, - "end_time": 1717686541.9, - "duration": 1.9, - "status": "success" - }, - "delete_pvc_1": { - "start_time": 1717686550.0, - "end_time": 1717686551.7, - "duration": 1.7, - "status": "success" - } - }, - "system": { - "create_pvc_1": { - "start_cpu_percent": 15.2, - "end_cpu_percent": 18.7, - "start_memory_percent": 45.6, - "end_memory_percent": 46.8, - "start_disk_read_bytes": 10240000, - "end_disk_read_bytes": 10241024, - "start_disk_write_bytes": 5120000, - "end_disk_write_bytes": 5125120 - }, - "attach_pod_1": { - "start_cpu_percent": 18.9, - "end_cpu_percent": 25.4, - "start_memory_percent": 47.2, - "end_memory_percent": 48.9, - "start_disk_read_bytes": 10245000, - "end_disk_read_bytes": 10250000, - "start_disk_write_bytes": 5130000, - "end_disk_write_bytes": 5140000 - } - }, - "test_scenarios": { - "many_to_one": { - "success_rate": 95.5, - "duration": 45.2 - }, - "one_to_one": { - "success_rate": 98.2, - "duration": 32.7 - }, - "concurrent_pvc": { - "success_rate": 92.1, - "duration": 28.9 - } - }, - "efs": { - "access_point_timing": { - "fsap-1234abcd": 1.8, - "fsap-5678efgh": 2.1 - }, - "mount_completion_timing": { - "test-pod-1": { - "test-pvc-1": 6.4 - }, - "test-pod-2": { - "test-pvc-2": 5.9 - } - }, - "mount_errors": { - "worker-node-1": 0, - "worker-node-2": 1 - }, - "api_throttling_incidents": [ - { - "timestamp": 1717686525.0, - "operation_type": "CreateAccessPoint", - "error_message": "Rate exceeded" - } - ] - }, - "k8s_events": { - "volume_events": [ - { - "timestamp": 1717686500.0, - "name": "test-pvc-1", - "kind": "PersistentVolumeClaim", - "reason": "ProvisioningSucceeded", - "message": "Successfully provisioned volume", - "count": 1 - } - ], - "binding_times": { - "test-pvc-1-test-pv-1": 0.8, - "test-pvc-2-test-pv-2": 0.9 - }, - "pod_startup_delays": { - "test-pod-1": 7.2, - "test-pod-2": 6.8 - } - } - } -} From 0d42a02a368bbd1d65d6d842f706958f63710429 Mon Sep 17 00:00:00 2001 From: Ishaan Bhardwaj Date: Tue, 17 Jun 2025 18:28:59 +0000 Subject: [PATCH 09/13] Add .gitignore to exclude logs, reports, and other unnecessary files --- test/stress-scale-tests/.gitignore | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 test/stress-scale-tests/.gitignore diff --git a/test/stress-scale-tests/.gitignore b/test/stress-scale-tests/.gitignore new file mode 100644 index 000000000..6e064e180 --- /dev/null +++ b/test/stress-scale-tests/.gitignore @@ -0,0 +1,25 @@ +# Logs +logs/ +*.log + +# Reports +reports/ +test-reports/ + +# Result files +results/ +results.tgz + +# Python cache files +__pycache__/ +*.py[cod] +*.class + +# Helper scripts that shouldn't be committed +count_method_lines.py +count_method_lines_by_category.py +method_stats.txt + +# Other files +monitoring/ +efs-permissions-policy.json From 6f147a289caf74970a890466a0b3ab3d50b4dd97 Mon Sep 17 00:00:00 2001 From: Ishaan Bhardwaj Date: Tue, 17 Jun 2025 18:32:09 +0000 Subject: [PATCH 10/13] Remove Python cache files from repository --- .../__pycache__/cluster_setup.cpython-312.pyc | Bin 31301 -> 0 bytes .../__pycache__/run_tests.cpython-312.pyc | Bin 9544 -> 0 bytes .../tests/__pycache__/__init__.cpython-312.pyc | Bin 196 -> 0 bytes .../tests/__pycache__/base_test.cpython-312.pyc | Bin 8668 -> 0 bytes .../__pycache__/chaos_test.cpython-312.pyc | Bin 28455 -> 0 bytes .../concurrent_api_test.cpython-312.pyc | Bin 21034 -> 0 bytes .../tests/__pycache__/io_tests.cpython-312.pyc | Bin 16448 -> 0 bytes .../__pycache__/mount_tests.cpython-312.pyc | Bin 7627 -> 0 bytes .../__pycache__/orchestrator.cpython-312.pyc | Bin 75792 -> 0 bytes .../run_stress_tests.cpython-312.pyc | Bin 11283 -> 0 bytes .../test_access_point_scaling.cpython-312.pyc | Bin 16665 -> 0 bytes ...asic_operations.cpython-312-pytest-8.3.5.pyc | Bin 10279 -> 0 bytes .../test_scalability.cpython-312.pyc | Bin 21309 -> 0 bytes .../test_statefulset_scaling.cpython-312.pyc | Bin 13283 -> 0 bytes ...ress_operations.cpython-312-pytest-8.3.5.pyc | Bin 20282 -> 0 bytes .../test_stress_operations.cpython-312.pyc | Bin 48424 -> 0 bytes .../utils/__pycache__/__init__.cpython-312.pyc | Bin 196 -> 0 bytes .../__pycache__/log_integration.cpython-312.pyc | Bin 15868 -> 0 bytes .../metrics_collector.cpython-312.pyc | Bin 25216 -> 0 bytes .../report_generator.cpython-312.pyc | Bin 17838 -> 0 bytes 20 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 test/stress-scale-tests/__pycache__/cluster_setup.cpython-312.pyc delete mode 100644 test/stress-scale-tests/__pycache__/run_tests.cpython-312.pyc delete mode 100644 test/stress-scale-tests/tests/__pycache__/__init__.cpython-312.pyc delete mode 100644 test/stress-scale-tests/tests/__pycache__/base_test.cpython-312.pyc delete mode 100644 test/stress-scale-tests/tests/__pycache__/chaos_test.cpython-312.pyc delete mode 100644 test/stress-scale-tests/tests/__pycache__/concurrent_api_test.cpython-312.pyc delete mode 100644 test/stress-scale-tests/tests/__pycache__/io_tests.cpython-312.pyc delete mode 100644 test/stress-scale-tests/tests/__pycache__/mount_tests.cpython-312.pyc delete mode 100644 test/stress-scale-tests/tests/__pycache__/orchestrator.cpython-312.pyc delete mode 100644 test/stress-scale-tests/tests/__pycache__/run_stress_tests.cpython-312.pyc delete mode 100644 test/stress-scale-tests/tests/__pycache__/test_access_point_scaling.cpython-312.pyc delete mode 100644 test/stress-scale-tests/tests/__pycache__/test_basic_operations.cpython-312-pytest-8.3.5.pyc delete mode 100644 test/stress-scale-tests/tests/__pycache__/test_scalability.cpython-312.pyc delete mode 100644 test/stress-scale-tests/tests/__pycache__/test_statefulset_scaling.cpython-312.pyc delete mode 100644 test/stress-scale-tests/tests/__pycache__/test_stress_operations.cpython-312-pytest-8.3.5.pyc delete mode 100644 test/stress-scale-tests/tests/__pycache__/test_stress_operations.cpython-312.pyc delete mode 100644 test/stress-scale-tests/utils/__pycache__/__init__.cpython-312.pyc delete mode 100644 test/stress-scale-tests/utils/__pycache__/log_integration.cpython-312.pyc delete mode 100644 test/stress-scale-tests/utils/__pycache__/metrics_collector.cpython-312.pyc delete mode 100644 test/stress-scale-tests/utils/__pycache__/report_generator.cpython-312.pyc diff --git a/test/stress-scale-tests/__pycache__/cluster_setup.cpython-312.pyc b/test/stress-scale-tests/__pycache__/cluster_setup.cpython-312.pyc deleted file mode 100644 index 8ebc0830c3ba48a034f969d81d6bf419ae60175b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 31301 zcmb__d2kz7dM7~KK!Ybpf~UY!1WAx0MP1Y_Qk1C68cVjvvOOG#4N{Omg3|ycvxgda zj;hc*N?F=XO7zAPb8-!*##PZDo2)jwRjWCz*(9}q2n867va7B5AOAvfxOeE-;i3$F?$(?8LTayaFSPiM_0(=8KkI%nd| zd`ZlF&dh#G&XurV%Q*}CwVt!GU)wnw`?a65vtP$K2m5uNbK=($bIrKVxy@9UHRhS| zob#AXC8mh|qF-rP`$cNKcBfqNHkQ9pxz~Pq`%lnUI#`Zo=~{n|lw1tx@1<*{)O5)Q zj-ObhG`Ux;Pv_>HO8#?|C8i@L-t|2b@BT@Nl=EB_@A;nToG((%bC zl}13`%=rlQi$`gc2*={HkpQ|HotcXZNhF_NV0B~MNu*C+z^@QSLw)X#=@p*}n)BAv z;p7GUI(fh{8J&rULmge=#3Wq?6M;~mqiYsmkS}AA%aNGweunxsji+KwEIys+m*3(< zf+j;Bl7g1JjTOqXOVi#6ft3Wn`micSy%EU z-!o%bTQH$*ycK6VZ^PNa+i`aC4xC-Q6K6Lqf^#0;jWfpI5y@}x2*8YiJK zTGF3(;(W`Llo5oS0pHhq2v}n3$ zFDyrQ7hO-M@s_AbDRB) zE2P1+LERkh)#X-4zd`O&zARa-w4bb1t}fOnSK4ney8I*44^-#_LY$KGOT342-FvVu zS`FVr1z)Kfk5_HQICmBA)8#IGPVVZ0cU<qzEWZ~y=pS!?}qvM*Wa|fYPw<$){0xlXQRnzI2ILwFB!R+3kyI_ zAU}z~ln|c@OvY!YqSNz0aVBb*R&U~qi8Ya;D3voVgy%34b^w^OosIGG8j!BM)1k7bwts5$JUY1n;~hI-AT} z3C-s@1rgH-@*ebFfr&s9U#O!K@jUq|C8yNd#2ko6zFZ-R6Y|rbEpJUkVpFJ!b&8EA zZ=OQGBq~24ch)OGN4`xEE(y@_WU4qIg#sm-~j?rc+Ex~XsV+Fk2f(_u=iM_cV(bQQ?8cV*jmq}z9-0y~zh z51dV(_W)pNZ`+fE2QdEqhB3S4S9aT2xnokLg-9oQL5u6tB~VX2oxG^0rpQ1CG4vl; z6fDQ+o&n7iT7ZdE$|9Advx3E0Ih8sXJ)iAJs+jDeig&e~yj7vftS%LMYB>tpQt_qs zT&O@FtF(LNN-H;e3ORgAy7mh$RO_2cwY2BJ>oR=JrX;Ovc4#8Xcg(u-Mq_grx65dr zS~K!AD0OSUTJQ0^NlRC*wCB(1gDUDU_!s|$@t7%3qCKw`eSkYFkFp+XfF71ENo1G3W*A#aV&PQ~+eAgj@- z1?4G~gTO?@pAjv!UeE{~G$dMPiwHto$h%(-$L1r4*>#OVl_@`wL795{b7O4f>;q@_BTre@ z(|q64oa4%~TxXi=%y3=ox;4$UKH%Cv;ws-c{`T>d|44>A${uv5x$X?t%SyJ~_q3=b zA8=ids%o=U9rvp`9+ubK;%;)Orn8yymmbyxQZ4*iO(azwc_`OczwI`k-F`H^{U``i z%6}}w9e?EHvd*@&vn}K7$kBC6+S!tEwteI*r5>kzyED!`j~WBn#!$L3l&h(~b?N4% zTx(~xbyvD|SFWig+tid!T7$~N?*8+tzR*vsAPCcDf1DaLLp!1u&mGkm|u@I7z7 zYSv^UBb*O;sQ~BmkWR36tyFX2fmBuc(4HVwl^e8F-u|8iGL(adJn~)%@18dEp7%;{ z=5Xgl85PHBWss_J`dVsI#r}M$aSpyzL~RPy+8|Gvh${JV9y~u^g+BQBil~{d=4<%c zX%p}N6Ek1O*Q3w1d;@w}$2am#d~?){Je%ILA~nELTliKjb=tzWy;p)9?eDqpwBtP+ zq)oNs7Y(SA5k*=wqsIo&{v#9LsYyXsEt)K~XyG?2qlJWP)yY-{%C^OlMXQn<>bWlZ z)9#b1BF>1O@WDk(5nZxC+ECxfqUDUKSl+naElV5SV%rGAqJhXIXhUKuzLxN_EMkZX zM;nro&k9=rLa|aR(-)2RUmgg@<}QS{<-IZj7-1}N&>|4kkuW69da+vJV^LF5h{vEJ z2_?c$JS3=uB)wFygcX830zN%47oLoWd&9!)Fu361Xn1CrUChN}(aDAWL`1k8os5J~ zQ>6dkOjwN1LOB7w-6&H{oMyR#B|yZHWEi@iaK1d8Ook^fNKX#Fcq&0pNrCmk_~=N! zOs??YBolcOqIGsUH1IJYx{rw$`}j0YdB=2QHX?*$5G;iG2r95r`=X1+7EL7PBVs9u zP18brevWl5&*@tg-Gf^O215f|LIXSUwu7-){7TR&c1D11lhH{XAsUU4Zt;8sy#diV z9fvHF5IZ)`HwH=Y4Abo;o>kpRlIG5mc!%nmB9nu|YD1U9(OCF=G!{)RoP(MzkvDfk zw`D&(H3fQ;w;yCgC+~bwR?>-WPj^O|Pr5GW-Di>ln1U10OA)b?-r|dqZ_Gn}IUAnt z*926WMw60s(%+r8#jngp_@GsI5das?(CI9J-i3+9Lp+A)YS8v*gt!!a+nm();cv`? zaS%64txFolXEcvj>UddFq2^s@lF%8>L}rsgmozasMn7jKBWEEi=gas=A}U1qkuatz z;FfpHL}$-Hc*jNfO?KhF9FNV~83>jOgH(Jgor*XVon(yO<*iJ#wO+*3 z5{Q=-NVA`}^7Auu!gjjfL8lI!!0Zss&$}fQFafSc=%xfk`lXtLAW}2|m$3%I5Yoif z0>(gQH00MU^Fd)M`&U?MZeEFJahZd23i=5VEwwKtFh*jrX3ZP>y}*&rJsaOo9o?6rJqfem}={8P2Zfp6}uT*8NS_~9UMsy zj-&=g?;cL|A6u(AzT|o6tpM2%x3=@wi>$)BjXuA0GDSx7wOt(P#-g|VbozlT*+a4L{0 zpPkc=njKVgA!SagvgSoqH>owD8Mn-)*)vI1r+87-9BO${zl-WuMhvb->yd%n&$|~( zE~=VdtsdUPa|)*cZpOPou2Oxj9~4Ud23iEZj4z*7wf3ZA|59+SP(}smT^p6r8?mG2 ze;u5Ix)zM!R*4oEq%Rui8I+k>^p>?g8`W5aS#H1s0{T8YQF%S>9pOp!5A+(1XV+h& zK@asAznxm%&)4Cp)`)t=*MqxuD0N(gUS9w1ZkPq94q`^{q){pR+_%?+x{cnRHa?^J zo4;`Vo4#<)z!%Qh@`ZD@YB~9~4c~eD4=h)0NmWm@XyZGS^$5AazG%O=Nqsfy1>XtC zu~~_^u~`R2-O7AQig-3ugNwGJS^R}@m2N&5&G&8%4jatJ9=?|k@qNZH;`@e$wXo!STm?!NUwG971;2>?13AE zg(lJ0*9Z340_>;0Z z2P}Am?+cucMZyVK$}dDFFOfP+kQ&#Ue)^s`m_}zfkhdq2s9q4p(Ez)N&nLwyNykOa zg(ZOoIUQh+NdU+@=4UU>g3T5=$$$?NyFfL)gx_(PS|sUCU}D<=Sv;JLCj(O`4ZA&z z_PDtM`vd_7Dy+GP5L;0DE$f(Mxw20zmutXS$uj{;wrU1}LKI zGhx{Gp+^f0Yzqx;34ptvCl?1;sujH)@(M&x1)8!?N=j#)ibkF$Pms zM3Q!OEsA9fh5;TDVlNcwtdY zgTVN~lK~8U27({Pl@Nyl0eD@Ukp$DgH|FEXZ~|TpkU62$?0WijCLlTnMVN!&WN|8z zB!H53lqvWwL!N+A6B1=Y3q*kX6MHB`()|->C?pvogB485V8VhX zdE~hXVSX0!!ORT&4uoM=CA0*2D-_JI>s?I5XW@AP%V0tvGzm2%M9n#!&_hB^-gZTZ z!g{(B*FlFQ#<;L%C)mr#+t^CYbJ7Hr6JZ}s%*T>>7aKi{kJFNDLkNy+cQ_>JW*ZX3 z<^t*FDRu78ktgx@So@G{%Br$uo9~xxRuz}vwlzlEH<)p5&H4OUUq{;4aliBUn(xGt z>ygKIpQ)uF5dB@7G)vu=8wx?Qmr0RBNxLpsqvUiTY zee}jxe*gGKKL4#_H;*kZtok9ttogPs9aiZ=E_*7&j6@PQ>wT09T;saqW-A?R3o@YucMsaIc5ac_KTDXDZX*&)1fl{b3Ro=qzQYo2y> z1UoX^PGto9SGV8pO84web?-^l?agre)LcUuZku|uC&TSkZ(hi7r_~#2)bgtv&2Wd& z!G_&;hO)a)q<5c44WGPsHsybDY5ZYD^^NFCRi*;q%#~N%a@=$*2Uo^ccc$BiQf=FA zzj1f_y{`1gnbaZVe<@S`GDNk?PEwT9q-fZAr|ym?RYwhcq)dV9#}xqjwr89>(6Gy! zb=IYwbvdr$#gE%}mcJXeowO86d}QTj5?=r^8I$;A*$nCG0eQ?^B#)IadCbDx_xh4V zFWGUC>MJ_*7RYDkgtTPSOH1Gg+`1A*T#M??2rgFZg;7oiYIhpQdMf8_q(_GbkSeP# zxPu;XNMbVJ$p#J170`Y7y~$*N=oUH89ao(>6r~TC4+q1l$(W4)Nbi^s<^U;)*Jzh9 zlgwiPNrynUk%u?}C>5jUW07GYf&AjgNPIrVGpf!Qk_Dv|sX$-gEIjohpuf`>lH`(S zK&1MFZ&1quorn`T6|@WAq8m+@L^xcyMyI!Mg06Qne)1mHCkgi^2;2&KC7dmG7NA$D za|XsRxYL*taTEZWt*Mu$^Zl-owVKf-PtH?wV=?V%Up3$N1Zhz+v(ml{w_m9(Dz)?KN(;S9H1hrxscXtTh;L&|qH`IljKfk7X) z(qW|D(5o$;lM}8+8<-)W@;5U4fTh=o1jwUKN~K4n{NQao9N3KO=AFhh^DZ8aX*@UW zQ6@4|r+C@&EoJJIZrx~%1i3P$B$n8-g^hA8Cl*@;wQ*5Zk7y%N82$nikAXsouUsrC z6m~X*3>?bH0A&mtssm;x5~WK3i+UX_NagWmV9}tB@QcGDS*48K(711rAf?pIMEOR7 z-BmIk8qb6(GO0lb-NXlYGNiTf?b@g|QmbJGb?}}1W{8|YzMF?Q_Y8P#G@l55UbU|f ze7_X@LQ2UmGnbwNKYJ1U9E%RVkHOEmh*3Z_qO)+eJtzMb-B_qOb{L7k z{D78|AH3?~ORu^Y-HWbEz@WmV4c~DGzm*?agwdDZ##XROimrMVJzrpTanI@h_C>C6 zyo(&agAsJ|Y+%vNmy1~KSKXRp&hy5mj>&xe61#pK<9i-<*{Q+ii?65+VOGWO;)nU& z65ash_OSJ?LC*V}x#TUHOQnmYqoy~80Ee>YKy&Y6SrIfhTDu+mKK5SsFP8BK7H!jN z@64v_+pW)|LI-v5diJb4q~&~S*8O-x2zKxztar4|HT7cobKnMN0P2j=B0tL3Uj={o zYUN@DAPBMlYSm&DKSt@8Wl-ZM%-1>o$kj?H(al!VVx`db90(uPsozx9GZbs%Kd>n6 zqOLuc`)s)CR zuPWsNqK9ghhT@@7)U1p74o4-|AP)&&j%#Gp!?ui~^oPoM#_z$7lpzFMI~ zWO5$bgM|>Y=8IfNx?t)Hkt2&gbAe7wheU5EB#mT_*{6dPCZQA$u;L6H6jda7Tm`5o z;3%T&7z0#>!nbMAo3#E)s+FQ@@(yMX9_K|0*0>NCU~MkyXb9DkW3cE}It-sTTthdJ zj8Dd6c}sF~PJ|kUVrcT#!7W3(=x!2S5M5jP*kAt^fMO~kR!W@+(F~D&G{j1jj0h~) z1%~j@3`B~2sq7YUdR~}==1r^!g{=7LT?mI&OFYGg^pGniL?|5J#lI)c{RN=-1g*1SO+MvYL1l=|a zOmci;5^$m1D}0xp(qb3BLnpFt2(36lgBgoN=JJ*p!XF%0;P-^8^G%WlRd$q^Aa^C5 zqi4{^glJ)j%G>eg;opj&iRlShhv~p2^L?`60*~BSW6CopUDHF!Ths%m5Ilks6}~?n z1{*X=)&ehr?=W|ja1eK0nm1}uhkT#+!ca1yzNoxiPNx|` z`qkn?(zouyQ)6!u8i}^%y`&8IbJ)DzGO60HEg9#4rfaPk&^tvOPn|xSJ^fnx^lSI7 zrhIQ?oL|dTRLlB9NlVsiq$SI7)v4NDq%^ayd+hCS_Ow@epi(T?REqa?D#iW`w?)+) zTtEJ(qITJtu4u_tY))5fUaJT$*<~f>jhU6Pm7OVHZ^jvd!mheb(g@26yMY{6zZ_c~ z%J%F@_v}e^?@cxC%W(~78VP(y&?wj4qudckA zZrZX|Ik4o&mDeu2(&ep7wx`vGdvctAxkEnrZcGt;E&&RpHnc-+SZRV5rn)m--6?mw zFUJK^t%Esk(@H%ur~E^=&t|ucrMHcxhK{6aSm&AUQ5t~P(S-pxYCKDB)Wy}@h^0LU zO1bapQnjkZ1E{LKHGFe;`65)^&4X)ITbG(EP zye`!+oGY(i9$RG(tVA-+fd18KE<2JcPQf=c{Yg2otnaI)-3?8wYhBR)MuQjw&B*r zn-^D_VP0t2wN^8HXJ5MJ*wPVzps8i)UWel`DU`XQeN-gzJ3Y>9Fk_8cTxF*{;NU%Mg2GMPKS53 z8f}ao7sU7rjB3oo+{%zsoM7zyMICJ}v*b{3T{1$^Y) zTDWPRqYg+e#pAqWFYc1M4aZZ~=Z!i$90)okwuZ4pe@?Bb9;O2My9hL6n1d+Lfc@2)#vDF}FE;f}*(9rWG!Cd{w7ZM(ADj-Ysgxb=DJoJ)7)CoDZC5Ki`BLDetZ)pZn#D1o@@?Wz9&1VLMUCUyn@yy<{yTDrj5P;pck z@n_(zsX=bRh4{PZbMYCm z=_ScG2xx`1Ah{+!!K;)50p z_cw$)K0t0we1H?QFFSBBJ#a9!<ly5BKJR*g#r9I6no#0CuOC-mx<-8SHZy@ar ztc>0Fb{juAdfy8^46$#4G#7ZlwJN4ZnDKqtT_@7JPNa68Ox2yra4+ciJ5`=x?${Ko z)|Ga4A)fAOE)2!z&qihk z{P*e%KIDsC%9OZevhmQWbkNY(TW1n{Huu7-DqXBUtxf~HfSo4faf4f`3i`D5MXRZ3 z2*3*xT4nMy9#gQ15WXT~5c1!p1qP?oJ}8ch@r)66xhxL#^O3n& zd_kAHKMsN#3oj(Fw@4W7LH&_Qp`^g>pbEt= zffy)1AqIvMG$$n^#9)=2l$^wmOkm`arj=vekTstPqAaAblth)EQjed3GN`7H2xa_N zWMn9#=5=%G^{;1NkELIS3+Bwa3D(Q=CHV18mMl4eQ&ew8Ec~(Mqbq!>_aLIe2qly- zl%Vrs)S-ligLk{`)~D*8))6VLoR3W<-BgL+v&50G<2P{TlpLjGmr|WSp)p<|oWk%* zA1=O(`xCa`vxtYDyDlU~Fd(s`hWkp4NS^OY{2slw}bE4vW z$*|t3xs*;Js#!(Z8Xp*-mKfsCo85T_Hk_f(h@H@MDLh_LW@g@#J1yGBSTG90C*+ik zN`XmNhbPuS&XXurqEFU+^5#AV4u=#{JOmQ_=r=8CTCRVy99Luo)nhAImF>h5!j?YQzEck0GS-6z{#VY zAShxxp6F~O?;;_VMK%-%V*i|m(p8AEHkO?&z(olwCL*+FN{>Z{A5q@rjVtDKdCN@>Muh4;5`7xD?`hnqepxuaF}3{q@+pcS@Ey-M zPoP!Bga)3R#$aJ%a2Je0RSq71fS?1#NqEl#Ztv4LTh-6s1LodPKi{p>E)$k7ItpBW zReXHWXs;GEniiN@*4zWvdx|Q=El;k%PbU{eA|sE1ru?E2hOz2)X@rz+mSd`$n;Ay4P48s<|7*52GMd;_L=%ns-^_1XEYbC! z+)b7KcL595KWAnKSX(Xnwic>&We5~S2j9xdv=x?l$}00WvQ?&C?3Y}K80nO016?@?QB~+D9i}CGBuBaUN72Lz-azU=>p0 zAt~)7T*SmmxwG)}14tz~aSDx}lBo0FQ}>w2r!o>^vnKZ$fmke)2P?m&&0@`F`CSwR zB(eZOPVN9}U-UC(iT2KxlhsO`o25PM67Z5L4vQ(un>TL`dUd8Q0|`=Srh!ohbT%+T z(h>{6s5>VKnXbg7Y^%#Va7pSjIN3~M4^m|hSZOx=Pb&St==4uGvApUIERuZcm1(== z1(YCkPXelPNP_BKq>3#CJb?t|33-%~QLvMkf&U>f0~Eoin9^7vVv%U6sHbZV-1TJ- zo=G1(lR9uV<$EdPe3_BqTgPu6zxBe+7gk=l{oEd#Gqhq~bxJ-b{+9Q> z?|N5a*jm)TW66tB!BDnqINde8KBET}vmI`_fzxCF|SXZ+f>WwRy){-Oi=b-+1bBq1|{2V_r9& zZt8k}=G_^{)@#j&QqG3oU~{^G(WR3Jl1p=acwV)6$wB+3Y+l})@giW5d^Bj^l$n*U zWVp>_zzcv$QH1rC@zpDLtaqkUZDVT{N0#i5JQZ@Fp-x1v+m+#lbu5y?!o*jmS6)r^ zji!8uGtM!zt$B68;bM7qbsF+Nxo?z^`v$Q_b<2%uPvz5Pj!EB&J@qw)*dho=+e26E_&oPxfBN9rF>Rq(ej4H@#Jy47k#yum`h0aHv zl$mZuW&k06J5M#^e}nM8j7p<20L|3GJGVBFywXd(Nn9jo%l9a=-qq zFPeq#BPSV7iVoo#((8*7aSNwpHPH~>^~Uv~WsAP$4I$7VZB}Sp3)mvl#YVLQ>V-B3 zpKCe}7}s13ndgjBw`Ivfb(<;aQ@YZQjl{0nb)}6MJl~?dP>KI4Slg@iZ`rZEd8u@D z#eBu2IK~vz-um3y>o%(W`tSbLb+>F-_q)Y)2iwH)5p2ydOP{n*1b+I01kd)KWBLc3 z*A8h|B#sJpdg(hA`n@L_&cNxF`K7~AJUojk!In{6F3jelpKWJH93rVgbd^e0d)i<@ z_(MAVF`fR1PVdm^+jLSHD5i-K-bY&At8=G@|96dAzuw~+Ufs0TSB})9w?2D;MBYB*BXYr*9m-@u$Q}xqTY35^qpe1Ol;2 zt^mFcw6}Q_KK8uWV%UHNIofqZ9klK}1cWODAG_2r+eSnegTm=^ip_>?OiC^VZ1<4D zHwix}Y4c=0Sf-sKDiFRIT>M0xIk>8w6#TmuF7#z zwzDXedJbldeWphE50v4r==2|P%DW`jYI6QIM-n9Z$y^{~0_np6Z1Rb|LPk(ZjS%nm zb)0m1?17APkhwlIt$CVO+Ow@Y*IL4Uy~caooNMWj zc2&>$>Xx@*JBXaGbJZ&Gb~#`3iXHBqLzU1lwN~Ar&HQ|=E4}He{@VlhtA-vnx4wVz z-HWTu*d-!J>{+&XEZsbI_gkq~UR`TGcjL%EwFcq)+}^#qKi$52+4a!hO#2w5I``ap z_0I8B`_YVlJg4p+*x0hNE!Dk0*Vuo%Zrx_-ssCq_rLF;9p>5rtRhp_Bh)YccPGo#1 zAIke@2!C*D-RD zhxLs&E?-!6<$R5-7VKY70JQ$d^F!LYfjVJDC)`RW3cSVlo=!D>B~$()`Y7-4=j(sq z-1586wwZiQk4<)bX;Ve^y3aZJpg`VMVBGGO|J z=b+_i!1OEbVA*)7>B9ln(I(4>!~Uaw%RRph*Y^UhagXI*&_8as{Mv5A^{-1^FyTINtMYpvSvT~DR23} z`V*ObCcCY z7cGk$^~(7kb|?QR{+O_Ify3I3=!)e>&L28gtNzrxd@bec&p5Yy zRN0uVY)x0T=D4zlRZX|{-`p?%V4u^^T(&Crk^>T3z4UrAzi3kXtLW zo~AT#J@)u%RrKUdhwkpkj=Yo}c`0@1L?E zN^DJ!FIsDCFPR^2DYNxFZmqSAnI8{I=Mvj%=Eq!%ZQtWD^IqFA^QYf3@30MgDwwNn z5*S#6lW0plRFw7C7KXBbxTM3-rOy0;Y9hEwQm;+Pt^G>Qc^}xd@9b@mk8KJ6f&01n zJdl*RFObiRe5FPCR0XuwKLw4^P9jE#RCwjtpUg-i8eBSzHg?zKR9!6wTdP7AZvh(J z0M&lcHmArI>|gf|;~ROU$ua7l^!uY%Dw=6qx-OdHwJr?=)Pd>z($ncWA$f}H@L{K6*o zV*6;Eg}Jh4g|aPuV@%or5f$3Q(uYUvfIUULOG@TYd^WZqcD#VIwA0jr!6#rK3KMCi zn0`KTetuf?9iAlzGe~l{qa6mPLBv4~U<)#dJu;~)+#ORDrV2M31o4Bv75l1{CT|sF z0WJ4Qk{geBeu$my<0MFJJD9}w?vSvMrU>ETDG+L9&?JhNjmSbB_>VYt$vh?bs#WKm zh>3=39{Ws|9u%gr$@^^bG~Ei=@}K7?@RX&opqfO4?h|<%8!zJ?@=kQ)1buEY?~uoz zw~Zb?^wJT5@FSxM%y^LZ$l81Yv_#`}PzFZ))^UZM3Kc<-wv2})$V57c!jVZL=*Zhy zPw`b5)(waJiAt#s3LX|drHZ_A(Fr!zr=>;f3?{VT$NT>);yL~d7my6S)mbmTbk&|3 zcp>9G{npqct2=A0OCgsG zE7;Svf~Oof6==a5*F4Q{je_@Za=#~jN2H+7vbW3LbC6*mj~Xwq#@SZo_>~1EN{8Tu++o z$#5Y>V)Gx)aAUbD9~!Kzz148D;omi>vAn68q1!LtKAQ6F%{cev%F2KDk;nJg4JjYmZAkghW1;IR z8%hzMM)P1|B3LV!F$JVg4qT&I)Jmss(}{GNLK!{%9bLI_1x^?EDi%F2fT=MFB-rFV z=fjDpyqQPdI(};Gg}i4B%Acdqo?$E9ycZvFh=gb4TgD>iUD%4?1p7#m#D)_xXOmL+ zKXmmLooGWijjhW&_elKYJ|Tby_-9J|HBOLd&E`+6ZnO1MzscXznks&4 zD*H#1{~t|Fc)4cZTaK)$Ds8IDxhvl3eY-d3Irzxgu;j=(8`92(RO8T^b6d{YLj>m`f0h?QU4A&ARH-uKMNuYp$NR?4Mg-aGTwqnCSfYyl;=W nVtvtUvT{q}eM?==%Dpx6dq=;0^zF0XfBAvchY4z}XXE<6YR`E% diff --git a/test/stress-scale-tests/__pycache__/run_tests.cpython-312.pyc b/test/stress-scale-tests/__pycache__/run_tests.cpython-312.pyc deleted file mode 100644 index 54121a4350ef4f0012ebe703afc551950573bce8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 9544 zcmb_BTWlLwb~BuzIDAtQ^|T%%%X-i<^>F--EnAUfTa_)#mgCq08g|SXNtF4}-Wgiv zMD%*wEoyHASl%w6ERvw!V1e4O(764n7AUX|P+*I86O1edVkZh3VBL@WX&o0XlAoS? zheJ||)C5H+XzuIWbI(2ZyzaUG?shvU_@!SxHT4%Q6!iz(Fn^X@<*Q#qWq}eYk&aU1 zMoEv;q-`0ukdhf^NXd?~P+Fqan04GrV^}6y60?olV%#`K+HBMwbBsGk-5PbqT;ncM zFNwNio^cPPy&?x+pJ<0~spx=jndpRXx#SmJ*V-uxaTZD{f0QGUrc!i@4!FO> z?U4YjbE2R~N+c!)EP6>e8j<1(l!^F6WD-Wp22YOlj}4te;VDU0kf0QRoFX+m&SlwTM zHET+xSu<%LqXG_97?YIb3?EHQPDbLB{8d4gL_QJc4cSP-!ABBt2M<5;=|hw95mLil z2G9|{p9oS8m=_~qJV`()0;t<^$aQN}x-La^c4*|}Ih~z|L?yKh#^f$(LJlb)d2%N- zFFpium2O02C6u_P?rUimWWWcKEAv6Vr9Cd>s{~z9J0{7pFbRMG203B3ddWlr#RNs? zri8c{m5>~8>CB|0=r;HWih>Y6aNUxSbuK1clf(#;b#_KjrgR(BF>2ijEQy4T9qP`L zK=f$?neHA_kR-&+rX6se0X9lV=RzSoHxvr62#bX7fPIAwZs~ShH#ouJD&~WXMIUqY z>o>ciiLemuno7i^uE^CX0g0GB!LS?&iYRhjLP6NnwJu>+4sPu4!W`@Z^#v{jWxyc? z2~WFFGM?k^OiFjl_`E}~Fg|b81nXi!lz#=EThzLXDyy5XT&TTQ`_9mPv{JhJ)}O5T zs@}8Rul>7*Uo||oEmf{Gomlbp-5SbzYwq5>bMu`Et@X%?_tjgcUzAW*+g}a;WcU}B z1=~H_g7=>HKDXivJRbOzJ^YMyuCi6jY*p4%mGurjbCs{U8kb#-0O1O(;a1bKt7*m6 zyl!R69RF>zSxa7&QPA(KTXxpHcX`FxsQAlTLkYDOBc(&bA@uzFIFs8ekbNCfUEu;*Nr8+lC8uV*U4kYJfExH?3(e z@Z7AKW(^DzubvqdkSu|TiN!$Q_-G_9@dBDm#=v#S#ITUh1rTQxFS1Ewnz#v6ExnM8 z^Wf!paC1CS3vh5wQIf+bGJ~y^$_0alf_~j-lwIK6^6oIFyqzf_7FBCU!9ws#Vxy7d zu_A+ewTaS%kc=uiJ0(SDR9BFwVNgy+6e+;yu41&hH9RGN^oy4Uqm~bdk8MzyTMNK7DiEm?O^R@2;bmAbDx>ubom`E@%1umXUq2*AJQ z_GR7sx0&w4qrTFtyK|fA4z% z`lqDuH9i7ji6jvWa72iLK!Uzq7ori+@e~i?6V^L{mm{&6sKob;4)G9!M8Cul7D$>e zgrpxO!M25PEC+_*qR#B=?$KSZ$Au(V1{6^xapYvciZH8mdp~INrJlZ-i0+BvP!Pvc z@{ACcbo=0qutaRO&Oo#S=8+KqjD`^jp#%y+r;K?G;S>>Ry%g+t_*$rlcJvyI3-B+$ z0UwBaScldB%;R76G%R}>Ry_P|Cd)Zjx!Pr}_Q7i!SDWFwvflFlS#$cIQ&y{g&0qBn z{Wm9X+p|vJ{Lr$q{(i&b!^=%a*UGD(dCR|9cLL1UGLFLkyfr{QwD%NF z8cGt=ZFn057sXjD1buISsV)eLdfsvD2mYx$@f!UX1_uU5E)Mmb8RG{pj}AeTKX-v2 z8oAVWW@tcV-aOQ;j{d+aA5-@_u7E|}yh(maxGsTnn~A{2dGH-rJ0fwSAD!y3V-!p~ z4rhoWrM5zVothMvlQ2)2l2Qc03`&4Ij;8XFh9q{Xy^cQO3iA9W=!zJ=hgVRl;0mVz zA--Jk!lV$1cdF-qXqGr|hQT_6J^w>6J|M@2-nEdfBE16}03sqpL&dU#PQru){L5G= zZ&A-!`|V(ctzKj8n!D$5v*tRGVGlv;cI!{i;!el$410o9E@#*)q%xdg&tQn)r)>4N zjc-0L(F?c#VihZxSun`hWRM?(y4VJ7`hhxHjIR9@nAiU^qp-o`T0mtj=3G$Naue~7 z3zrdgj-9SB`wJzQ=`Cq_mwG=x7p@#z&n;H*Dif!>JVGAW{hIJbYZL=m88JTS$>;*Gq$GIiTG;hu>l%k`MTbgbv^qV!& zDZ0$FNL!U=vwzOI=r-}EOOzI~-`siH7NOFd9bfNo47}+ldahIR7PNPplkl3D$@LJ& zRhxdOdYhTP!c4JLEECJ$v!w0wbo}Ku@P43g_a@}fe3ud{-p}W(Af;HDb|^TYC%@@V zvsNgldkS^4Hs>&@k%QZuQPrFS)~eoCYB$Qq7Fw2eY_qc(jJa6ql~)RME!LWprJY-I z0CjKOKj#e9=J9N;M?1EjxtXT`b-=__DAAa5*ldZc@@k>JHN5JiD045SoC1vv=DcmV zQ#a?FK5D`gN}#cR>vdqH;r*h}0OlVr%rk3w>9>D}wDg%X3gw(tY&3U}b}2z~WExxf zO}{yp$S*dT&|qZ-O@L|a95($-s-}lF4M0=eX>u>^{(sQy5}W7Tz?)s5EjeqS_7r&r zkJw`NV=o}KLTbMy?Va;ZpZzv1bM`q}Y+GzEq-3xsDE0;0oMvFovr*Oo6DKy}15Q~| zmS(DllId9sO;QeGO*I0 z8MVUoU^6vuU)z$FP?~|=Zsl=y8+3sLbeOy)c5*B4(E!ps@@-DbWq^Atud>IK1{z&IH z-_@=$ML-H3Hgg3=T!vF48)xtnC=uh;noAOjOr*AosarP%*(6*n3O{q9r+TR0$fH0? z2j}@Gl5rzBAB`v}J|aW1UKXxKA!H5mI3Y2E*CL?mP%dP}iE&6sL0AfLrV-`}klR7z zzGr4g$QI09c0^v10JrXdtKUpg2|;>Yw;A1#h(N=@pMZs=zz}v|YAYn{ zNG7K^UB(;PZ4%BJIWqP5z=bRPh1W;;v*!i|kMM7hoD*b$AOtUD?@02AM-X8rV)&lP zysrL*`HbS5U;KsXT6%km-( ztT4_|=KwKz`O(>FFjNTNFc7}x=uA8@t8<`D6F3_K!}wrWF&UfDUDynSatsXkbQY%y zb$Ws@37x>ROYrLvJaajWQzE+T%hhuymOy%G%y=t?)Oqf)7%RLCnQ>Y7Z;%YLw|qmd zA;Uz(^r{UbMa$YT{h)n7c=c8=_$?1j1JN?Ru0Td;!_T;;vN15|C_p4HOs<&TOa9;1|E7M)`{agJH=6OChs0{zXzN%?L1H|eFszh2h(OBw$gUzArZ9RB&?wF8Hjn*Mq0N$Hb|zq|6$m5=IIj-6XM zFp4L?lAqlF``P$ZeWd?U&SrR|>^*^iI z^|xn!dFI`b)w)-f>t1<$Zl!Ku-o6Gi4?aBj;n9ajv;K~3W!nS${kI;sX#PWAS(!a$ zny>wZgQ~36YWF_w&r}>*>if*!m<65;-5b&x_CD@g66S|8{u3{)ROMkf7xT=HX}y=U zV{d46movUAU$h>%eQJK>DZIu?H03(*GD(AoFT0z+d4}_!7SIUJt@(lvga>1qrzgYh z&5P_^=g*wqtXl!(>zAh)SVOBg^vw$&<*Im2`9Q*Q(q6Ua!TAUMT1EHLzT2)0cLG@C z@~t}SpF$>8Tx~u0wDlmCzmYp5+K!VM*Qs?2=k~quQ+Ai38F!s`V3{Vs__Z7YrG2~d zY$x?W<=G}?iG$j|G@dRQ*+>1prDmj^`F(E_)IV;rp6#%G+=_=k?zE4zFdrXm9^si! zcq`ODX}6B_*ggr`0P>T4_R%`#lVg1qqg9MnWrYTQ(8ops9`q{=`+OM|MVIgg2i&L| ze~4vu6keG~Cfo2WWhBt+EVfXFRj-7Xxv1Q^*?1Cfp;sA$-?F2`jSx>{yw>0w6~Y%K zV#E**oDsf36{X_1qp^gTj7q)e$1s4SQF#nL>kLiPf1vE2QI5|j=Vz4XGs^opRsWw< z%YRWlpHq83r}jPPD(SZ8-3RH$=ZJRE{^xZZUGv;OV4?l%XIP4L-BzD6HUGhKe|d79 Kq1alo=>Gz2)%~Ra diff --git a/test/stress-scale-tests/tests/__pycache__/__init__.cpython-312.pyc b/test/stress-scale-tests/tests/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index 209026e0da3d68809337344a369e4e076819ca11..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 196 zcmX@j%ge<81hv-g86f&Gh(HIQS%4zb87dhx8U0o=6fpsLpFwJVCF|$pCnx6UXXNLm z>SrcpBo?LUr=}I_CKqSwrW9qCr55R?PsUgy zr!=LrW``YMx?7W$cFA9X37yjaoM~saB)bVao#{w18L2OxDKpI<&3^{76BeekJ?B0> zNj74knan=mzI*RG_ug~QJ?D4NIpW`oiYx@4e<|5L@xPUX{5LlAhtE{j{uL^VL?j{; zB4gT{8DnUh9b@6mhPWwyjGq$51V;OA7&EX$GE5fhljYvfTbgU>HIci)f6QtvhS0Hg z{afxWSw3d8)|izg`-mu$Mb-Fw% zQ|_x?D?mjxOaHFD3sBkCQ(#M{5}7OfC~>o~E&HUXYh0F;3D*;XP+V{vF28G9js)PW z+`MWGL}bZ#s#`T3gHxKonnj65WH?5w8$oUgRNO`xHC!zXMf{@g1PIutDTRz8RLz4h z%bVT((?QiTfb(c|+o(2^)-w0QT(o+ftRd@jhOiZkSaftX8DriP@c;f|^Co2vR$Ra>&EZMn+5;@p&Sx|2@# zva@5Q#+9n+O4f8O*LYX*XzrPXQp+k4%oe#ErVC`%1sbRdROP+|Wm_a>5+{>cwS#Y& zC(gwA8;ow0pmJG-oXnpsLoKp#=I_Ygv%0YZy-}ZOvVii5T#iySiDuCfXM&7qJ)ZlW10nCkrTQA$V&c@{V~SE))bg z{=`l~{3@<$~wu_JE|x@LpX3A8nU9+xD`V0sh&uow!4 zk6&k19)VLeW2sF015&JVND2XHi7x4MP>BK&&e7}o7SH_KpMjpl9DC>_fe9&aBJr0FPdq^LGPqCI+3H7HSkbO!c1>zBhI zH`T2%@jR=VbJGa)2muLE%`;-vp%PeB6xew%CTaU-B-J*3D&U*)hlArNX|zd~_Q+^W zswKk&P~}5XuD1@Tlp;9yGi23dHV-gqnGQm8lX}vc#1t=}MM#-6t$ET2gx9^tscQxBukNS4t}vi!T)?8h6}0dUNkm z)6(lpgNcS$@09lc{^6>C#E&0%NNF_=w6ECg7Y>~}l&Gb7)$_uYXV+sYmz zx&4u{6~yv>Kk08Lw<_!dedJbq{lJUdr^1$j9o(lo1g!U&2RCz{33Y>w+-Hpf)B}jR z`hzDyQCPe9pNhhOt2q>=_?E?Cz>UFqF3uFj;~>#dvEX-3%sOxLd^GDsOqz8Db8kR5 zo}wGK9^EMRaPM(>*kg*h26gP2j))retOp%=3(PFHV?jZ|I1@e*j?9L$xPs#mS8%*s z4|F4LL9HlYED=$D0Y0l7T%d9nP6$v<+yXTXfP#$DQ&W(8kIll7>R-T5kMqT0j(VO7Io{8Ietft&1d)%H}?&Scfj zCBwg&m#Ype?9Zy2=i2FOlS}MU_fm7BdiNdMD^%5tF#eH(7x?@33i5Mvea13Dy1K;^r)?hC3$Q5yHpgraUn-V1}kAvO8vc6~n8;`2>K#F>x;b(_z3 za>h@6W@s6GJ~0x&ZbYbH03dV%M2pvx+=s;=7I^DXUxEru#w(o`7%nKULUES-C*OQm zD7(j+1j`yn4E9y5t(HUWKHDtVGks87J;2aTOQ!Q43pH(WoHqdfvX8FqYiM!n-E{C9 z*t5(S53V^G6TlrOV+IT_#*Cr?O4FRtZB|VK;DH?l?29#H4SaFHNa}#@**oeQ936HI zQ6JAmy|#e99IlV%Nlo-LUHWNmQtrCXyTS{&NV<<4CqA5&J}_qigVQM)0+A3Ks~rvp zqd|WNU|lmP&FOWG!)=DPVWsYH#uKIogJ%a$HH}0R)!PL%bq|##O6E>cjC0 zZrJ70tN^qn2ut~Vu`*q19=+=gpi|sXoFR8jwv@@4G&wKV-Z8bVRJl@B9m%SW<*Kd) z{=6-1vR$yAv;MS5^E~qfiYP+`8iUFr$pt%cM$CGeJ!GDZW(73K1L=k*p7k>GMJclQ zN5?!69otO0iNegX40($%@Mn(Id@nZ_t2zV`1U+(>CX5*wR3F?3Mjipjf(K^_T#7mW zR7f=|{&C5NBBsHIo0Cx#@+cH4Gp-<~03@1KyatU}c~*9L=kllUMM2~{L%u00yV!AA zxl`19_2f#;reAnI@LVffuIX49I)Csxjuf|~ikg!}&5sm3%AeMDbN?*#8Uy(rI?9yF zu>~rNPx0S3C{g7o>VURoDSRp@N5WI!qm8&3wZbQbl^F=Y*7*?tBJ|90;mr14hia@O zyFE><-i1yD(*uZ%#9WjzH6=|=Uzu98^UvoST^7ps-$F0t8#*>VB7@h$JOei(Gnv(c z9F@r4$cr7KS^b#I#v6IGdA|5uIr!WtQu<6HFB_gh@dDhj(Rnt`7K$50;Y2%V!rPD> zFd`N~yn=BIDHumsaaW>^mPJpYDW+dbXjyuN7t5{)^nNc8am2YB_B`PE!%}eMC?O&# zpJ(4==gM<6ttammuZV5hLp8#M7&)qi&(sIkq5?(6$Vwi90<;9#jkTgIYwf zYYxJUDPK@jxt&|LBHqnFhpZZ8={VRj*l{U@J*R>gAAm!FA%$wgfunTbu&5S`L0JmG z{^lTRnNnzK2K2fy~+pyS<1)i!}p7DIv8z--!5IqO#+E|r5Hy;NAC}qs{9caZi z<;=E_wI2f-K&c@nOfcV6th~k~7&f09lOY%6nF8gZ-j))k(@~Q>ETysrTmFiBd24?&#H9 zFIf|HFQrQRlBInQ>PczC<5dn;f280Qc&Y70@=5)mg}YVR-#RFOGcG`5og0pTKq2Wr zKxRKS{@C;kSmFSdxM#uAJRj!^U0g%t-4lfT#F$HsEEvZba9m7rW+sQ~hhFN3)95(S}j{GC!39d6Ekdnho zm0t3Y?qfZ0aZmfBaG!=%4#M+{Lw!bzNX&i5rLc5*THYP&%3fdrZBxGNems3NgRkuN zWXb;m)>vzV;^F_0hi6El^2n{eCF8}}tIZ#Ee%P66+MaCMey3{tt-jx_;br-kO|49< zj7ker40n}zJe26qV)KWf_(^A}Fy%AG*%Hd7A@@}rOv$|xYDY%yD^x3oMG zkB6$Ik9d@h+eS6|f>w(JSzjMY3{ap!QB-$O9;J8-+~m5ZI1Ixo*9 z?d}ERs-ei-Lld8#q{FjRop5-T9fJ!)>8jer*w11YxfMs{GAcC`UpbDfvc`%(Vc?gc zUsyk|CN}R};(lrS#FpAMoZK~>*m)q)a`2AxP}-4n;l8%AfZzUW9%Z?xY0Ba8?ox_FJKEUJ_fB2w`)HS7Q zJCn7YsoEElwJ+W$OlEJk4{V9H-lgro?E9oI_0pl_ONSD@hi|`;IP&^!CDA&XavuHC zc@!K#q<{R)MyHf9_hj>J-kZ!dB~iUSW!v$kZ3hl9_dI@3_s3m1sjvJ<`72O_zwX&< zBDXC4jv){GX*~;#PaAvox027C{f>QB_E#M&G=Al&-zx~ewtDvx;n%$eAo`pL(DS*# zVck@}&%}M+>fFb1UvL7{zp&QtZxX&}_wKJ3ZdVy#hTHW5^xSUZu-;lf+`|2abq+Uh zziAMl4*7ssm6j5rzhGv5VD(@s1WaZh@G0-dVi=19SRBOyGavHnSYWgvBm8bm86L(y zERa^&1O=uHyViJKsJw6EgwFdGPH4SvsSsXX^E3%PT0#&9xQkUbqR3vT%9uz|E%@87 z_T##S1c>3P5r3NjfL5)?{7Nt|NdLcp$`6n1eN8n(j`$G$=|z z$)rOknqw)GMpVX$=p>D)W3^V(H07i{8QGad$+SIqA~ir@HBnoqr_O25p_arGP1^MP z@56go2rZs0Jw1Jt?z`_U|K0EZ-T(gg;s<4ARu0c!|HYOQzyH%5_h0EldGuM!y>1Q1 zUEp}`2*+!9?WpF6hP|~%wCt@rqGNCU5j}ewju_b6c*MxwrXwc2b))7n%Mr_%^@vqN zbo8UPG5ZnwSlN*>_H7t-j5&`uHJp|Un?`&}CUS4Mw&cDP8*jW~`ammxW#hIfez|wp z{Q=5A&k>K78{~NNS2^CI#5+>STffR3sp4%gtHW;I{#qx;3H2=Z-XmV*$UD*}$Imz6 zUj0GVL7}hVCbg4Ve_7~RBxv1!A`}&Uhr?nlGJf25Aj(gShJDAPf^X-pL%!{Y_V{)P zkvGDE)q40uMD&fx{=?DnSST_s`ogDULU=6f6Y&y7UkqeKpLjAn961&l4vmgZ`uMOI zIX)ideX*$TSY({{9Tmc%*C?%%(a3mAv_{5#u@m8<%P3o|bve0*XwBrxjP z9Ee@yBf_7h7!txh5DSmTBGGY5)*3o~oN|P$`$9-`EHvpG4ox6}I4Q=$V+bwALPsN` zk=SIn)%xXugF}0U1~>W+=Mfc8L?=dhUvwO$I2!f|6XS^G3;C$DxBIoN5gM~ zN5`21`yNBdL&xEbR1M5CV?>jVg<`{nokRY*H-q;DF3e%na5Q2tTB>>NS2Y+dI`lUK zuZL;m4KPi-5vG~<@+SCOsFxnG@-^t4nj<#WJMDZeZ^u^|U&oi>?cnQq2i{J;fp_BV z;v0Dv-sOBb;#Tk#Fx|Wxrib^ytmG?UR`FFZt0yh~ri_jCQR<>$A-;0^Xb3%xdgw6g zy3_{LotAf^;q0Jer8L6S;s3oi*i;eA4mOTUV5(4t**b1Y8>3L{8BwMe#R2bxRvAVK zE~tj&we+V@(Qq-75@p1!*m7@WKq8z?aUHQMw%l6@hm=%$c`;nMkcurYhDCA9i{Vk+ za__ttYF=VBihrJOnc|k*S`GISr@`OZ>C?KGxKkRxE?&21 zJQ9n9MkDdC?^Gmqf(8>t2WE!Ra7H5x`JEYk^kjHEqn`|ojbVh42fj2Tn#ZreRovZ>-MP4`K9R5XjC}QT`$y%fQK#Vm5RL zR=5dNzZuh*Joqxkpcsx#ob>B5dNDkDEMt=s3(C1XCQ_n4pYK7WXEcgguIEH_EZh@0 zdLkt7J>g?wU|5UF>E^z2Z`Mz+Y zC$Af6{MvZcA!>(Z)9est%F1APT%15#D=j-r^TK6~Pzy35k@gAHPX+4C@rr^Zo3CJt|03KKv>p)3v@4V7Ro#YDtbmJOwq$@uBbBx#yj<5 z_Kc__T5;hxUZ>107)9<@E~d^MBMbuNLXxf2jM2skZ%%0vnh{4~So-4i3^Igy18?L3 zcOn|z%v(^B67vXerCQ@{%(OowOd0caFw^<2o_D=#n9@F^X57p3lh9r%&z1KvHB*rb z`4Y#wr!;SBCfzxk>~czTip$P4o_OO<);$=m$W8?^=LnyUV30j9(ts3)(UbJVIpb{w zfjVU04qs@Lc=jZK4#oQHm(c;@<=HOR{aU}+9f|hj%{D9vp&g}-J3*owGeI{-e>Zr1 zoTV5#nbE-`Q$}-%j75SGKBHUT+l#qtbb@$X#t6g%Ami6(TqlL-8xav7U~vFv;f$Rk z9%3M751(=H5rN@^$!EjDm?(51WxpX~4V{d<5Edw2Mh~#XXUv$}LVPF|%GgwrNXdFK zB%t(2RHT|X8s#VB4ck#|ncXwY5-p_X*Rj!@(S$QLaX2^@8jl>K8IlNN{*}kK z4|w@;zuwr-1QZfEor;!ooC*DCS~uI?A=#)C+*#f_AR&U74PhOdtcJK z^``ylAKh|Rrkp;>=}S2Sk~46Zb0V74_0H+HPtUCRg|}|EtV`FgOgDGTd6T}rbnDvN zU2D^wT_4&m+pg}EIycTgEp_fouk1;;tayLbd#mOmQcM55S8CaI$6{!$x?{67Rh+Xe zdAah+3(oV-lxwZzT06fg=~}z!+I7oSb-{MtmaN~Da&4Aeo9{PpuBIO^>5zj5Viij9 zmDOvx-)-3L)O}Pw(6imD`$MY%9|f}(%{6S++6xO7N17bw8OcXR@TEtYdP|`mpc`E+ z_f78Cw1+sqK3+Q*9uEuXBfgMtVm$KtMA-N23)_8c99bFP3Y>gnI56}XO<nz&FD^qPYccP$eW}y6~W_Kl0n1_GD?jFZMw{V4@lf=g>i9P$lQw-jl3>F96*0uljR5I2z^`L)$!JM^;5cpR_Nw! ze}^7p=gZ_CGo|Mp34KJ9(D6>jD_onAA@B)8|k@r#7v5C>kWazyaR_9cI$5s!gU%{6WmPvYw0;4PkQ6 zSg;a~P6$Z;vR>!~DPc7kBrX8J#bP~fW3Y8BIx!y0cFcHt5&P|k4!>kh5Q>1>1WXUlcLs$@^g5SiL zid@g-)HFf4XhFGY>@)!>tR`cV=eHmgHlu@=umw~_LT}(e^e`@zu?4f4Ly&lN!OWHO zc#fjpf8c^w_SBM2+fsGMWUQ&U zW3kmcruW~e=UmNc->NG+KHPhG?+5!+U7Mw@&GQH6L&>f$C07k38@HXa-S%vne;H9X zRh+ZW9Fbg|a}APf{gRn;S6}El-<7J^EmiDJ*S4lBo0klndynQGr*k`|x8FB&Wo}}B zGhMT>*<(p>VA0;4c2`aB1t;|0<+K(r@yGpd?@!kCUiDlxUOhTrKCesGY$nd>uKw|o z0YM&!UKH>v1N{S=xQ{kjw)u4*>+80)={{~Vz+SK3m$&NfF2ww)t z%Uag)gqL7fym{8~59bdEUXDlOL^y(=L%aOO>>!Z67*k+F>j?&d@Crx>uTt_dqLTB# zdNMUYUN6p4U*3yz9s**BwEm%9Y_Zf3l0v6Q*|$mdZRyGybS?C&vhAAtoUY9AkX}`- z^r@Oh^(imwQ#B9n5#s#Hx_)lnGO$ATk*02-S@%)10rsI`J-n2Mgea^>^)XhL(4Eoo z+T)rjodt_4UQcZZCP&L3EPak^!IG@JjkmvxFnk&B;GOSkc^6;K-W9OB-!;O=3U;7H zyh;|XimwKf^nyw1AF|k?927l#O+tUAR_U)`Vrs5SE(Gg}`)ihosSCm^6U$L4)nn^f zk8LQy#59bFVGPkbug5mV+qNrAj6lnTs7ThbGF?p9=#xY7=flG%qP~v9U`OC+JmkM2 zfuwL9n|!BEU{?3Vgo*GbA0PFxfNh;vdW8MHty_IPvIZ;%GR0(zlL+ z0#fcnwPkeAMtRmj;|`L31R#7NVKc3ZLZc^PiJN+kM#g)@6LD=|IBun-nmqZ(4UD>9 z9%C5`V`$IP%&mWF?SV)}zq+?~jAXN8;jySNnb8fOoX8kf_w|!sAM+E2C=15X$XFyM zV$9J}E@Oy{L6StWr7M}Vo(}303SY({B(?J2u6F4Cf zE6FDFLcE1UL=A>AAik|)`ssA_c(N)wJ-DRr$}-OeVxBK1>$@SsX1qV|s4-Bz{4xm&B3 zR&vdO+x1x2fs6Ll-Z2=O;bHJqgThnm@2syUe%ws_cxjhL<*I9?inTeex~>FMJv}>; z^sZjC_dS%UdaqjNTjrme-=3`b(xQE!h^c;Uz&Fszebi~$R;BxBb=|gd-N)qy*#Ey} zs?_0#sR}$B2|F0mADRGS`so_UT$fbGX7inkSlrSt?5SXV7ZyL^N&Yt3#EeEiQ^3wX) zyjU3ZJZT0b5DInwW2isAq_!{83Xd%5uE*r1<*_BTKdhu@KQE^3cf>okkA_3zKI{!a zz~WQo8$KD^K*?ZI)(nW(vVeqQm3<^NFk>Bl4O@!I9HcxyFu27=U);TuWCqY;z&eho z6V5utzvuyP3+O!`q28*YgG8OgT%jyy2N1RrinqhQ4sg*X+$AxQj5 z03$Ewex7XVN-xu~EYd?{C1J$N=t;vMV_}gfc{iq;+upB#uRhh>CpGt_SM*@TTC)i}xTY2h)|#eE zwu`ppiuLolZ`;3RPxbGS`u8N)?@hMtyXoD3$H{qHmg@6!GAOwRlLuc;x(65BuY8fL z3`yRhQdxP17SB(bm;3eDzI?~}w)I@!*Bw8)AK>cS?{XTH_?EYkN}TN2IKSrGo4>U= zwQ0Y!X@7F#f$Im8&pvluOtv3Nc@N+49>&fBh5zxVB{?Z+Fm`1p9sfNu91?mOCnhQWaL zyG>ere7E1SYm?!7nx0)x8NSzRM5ymQWgwSL6#aW!ErTm{-#2*&+jZY>H;_GG+1+dS z{!=}>R~fFi8$sdvDg#`u_Y&#rYb<*jb=Nn0_tfbAsKx+$fq;SJ(@enddk{T69!aqU zxc1jnUaldz_mkqOyjaJ9nPRGJ96P6~Od1+12KX9tS{@0F!u^1@wgPRLkY))dqvC^nZA88-3Cl=-Re>9q%AIwDl#pWDU#irB@lo7Fe6DyT( zTfz>Zmi=8L?RgYw;Y?)r#>+xJ%&b;rl4dd-0UO4=Z{XQIzGI`&C{O$hER=CmvSW=u zNrE5f_C>}Z+YgP0qZ1<3>=YuZWdM?qjp`S950iXkqi<+p>}XiVmLi zq0_-r(3k1OXc+(Ds6yiWduFL%pv+ln)wTJ`sjq7*q$N zkYzy*o3{re`Z^&53bR5p-9b_o^s5Bpt%tHQEniXP$m%F;PbY3*rO7y1kRU2C$m;L1 zCS%Jh%g|>$+Av}SkJJ`KbdDS($>>I*%2kHYc?5q6 zLH%Vi!Z5JTA^Zx(E??Gs8GE)!v?nZP%wa`q(sXPh2Gt)iqX`OUSjw^kLm7nc`0}>BaB5DU+KG$T0(x_4){u`0p%|OZh3aC&qSW>lNzJ0#>njz`gwP+qB zl_>k$_LR9zGPfn$pIS7pzg=B3ZHLO0!#B6@W?BFI?sS!Rdhk|d?aaDgdF7lYUDuSV z>yzsG7V6fXGpAkE7aZpuGp8@Tb@8pV$D8uBNuIXZ6Up9zdpfPJ;x9RExf@WWs^z|e ztLvCO^~2hpbH;R4{e`jfW8a9L(`DRMX^$`E>5x1fbGi@hm+h&}4N~WZo1Tq#Ok7>> z=Wfoi;eI3Mt()1rP~ADVPO9#`x<;ygYTB;Cs(|DUB)gwpaDQpqd@GB-s`{?>&u_ol zea&>;o_zkrr1z+xl?9x zS(iAI#rk;wc}SM^KM)Drexw`d+3D1Myru)@cb$fvUQ>bWLfs@f_C&Iazb}r9%>w!} zysMki<#FCT-g-{+D*6|8xAS;!DV;23)-I`&g^h^pF1aD5a@&#|mK*V)wu+vFfwvsf z^45f&R>@av3J(SzcpNoSA}M+jhGQDuzKoU$_<%O>Yk0Nyl@ ztW;cbZ@z-*Cpsv-(zQC?4nI$6zq}d<(c+m!Zg3fQ{&Qfhry#C3F$nKQhQ+Eau5j5*G6^!v$!`L8Y84AsC!M zP_sqsOl8tRq#v3Ljv?+2vf;Qw>{K{Bo(mxo#2AZ=LM9+@Bl$XenGXFpw%zhTPfH%? z$uDv~=CS#5*&^gK%#^XSRU^=rJa)D$Sr7syZ9WD)4`SEC2KdFD!=$z!pm?&D7*m8g z95-!eJC6ZGHUfDahc5OT2rushnourmqz5AZ9auUi^A||E{uxat0`)-QUy;F5X(6n| z^gXELlvu9)SYaTSX+F5z8KJVLF<87A5X@-AxaA;&HmG21qkWSct+`5IxyYr>u!p~< z2w4g5QQm%ojBR9WCj*lvHVtJg3!I%X9%iss##G12)4=eUaF#-UogDNdI5s1z0(1}r zE)WYnM8ZSs4^}M#)?J*iiHujtc%KXg$gHwb%)fy{reb-=%9vzq2gntXAy=7<&4O~~ zGi7WDgtMPaDkNhtl4vpP#f(A#R?u@w0c)TjI#B>x8NyRA<8w)X_V}Zi*{d{ ze6sc>qqA(ECXLwk^Y)Z$hveFku4zs;b^suj?<4_Ic?GsP%PUhZpXBn*dKXwdm;}lvzKsf5GFQdrtCn<9*9hdtv|i z{mIsy*Fx70C7bswdiMV9UoSa{&;yb9)^}<)4H~uIH#U&;r*%L};vXjXDS(@*faf*_ z&65gvNGFKb6)oWPNVDiExoxHj9W)#SIr(5y75hVA0;UG2i7hB%V^an&Z~&>|2N1@n z0GKIk8$o>1fQ>i+lj0{I#+>cZ6Eh?+Q7M{$s0vM%Lvu_O$MBXbRt3yK5M@FBt);>8 zt7};PZQ1S&@@HVJ`*h6yP&emMCK)Cv(Z% zmHosbC#Z|^_FOIIiEhkF-m@a7GA43zbiI*z@_U8fXniPgRejw_ZNucNjR6sL5Z4+GK5d1Z6Ng07YMy_HRniZOV*bY zrKx<^TtXD~QC1XI75~JlFee%4Wi@4S-s1$0ixS!^0DDXjMw=h(RB3@YNhPFOmdnCu z4`rD=jFBrX56hR>ERR@_JOM9ts6#=wtx_84DIm+48X9QItaXO)^N(Cj1r|KSu^h)P#RVMjsjNWc)T6 zIm}ir3(+i;Yu2j@MSNK|6S73|GvV75`wST&7#XYFVC0u4XhU@fG~|RUWV}VjzaZl) zWc(2s|DKFXFhmkDD5FPx%Lva)?W*qg<@cPn?F9d6u!V*0hlToSLV8Ou3XtBG`N;gs z*R9Dz;pD+%$z!pkXJXO(MiJ!OxM=-SSiX|rUd!z2+17J!P47-u zRHZ6fq>7f={<*^o6@Am&GtLTC@OIPHxn$ugRspJ2bxvDBJZ1JuX75bx?DLD}RiNyx zowna9Ynt13v#jT8zbeh!@WaY>i1Qjc<}|ZnvTju=VO~J;1m;f9*A~k1R{R}gdA*S3 z)i=!~7V5g@CZ)Rm`887A)}+~ctF~e0_&2+f{!LQtrukv1c3|3BB;eb=;NCH9R<>8G zHqXC!ZFqj@dUMjVf6;tE75uGE_HDbiCb@d>+UrTro<;Ls)dvThuCGaMeC9elo?A2@ zTn+(O0*tDHKwodVW5ccX&bjK}d?nq~O%Sl$LLxy6M1t!$M{lyMM-~ZwTs_b>*rxl# z%^fhm-)0yL6p93CNkQD}9*05lkB!m5rD%)0=z%Ncs_20$X(Sgta8(bZ!Z(2{Kn~l7sxS-Nho%oOw&+o}4+$Z&CwIUd zm}xy?njRz6Q}GGgNGyRr5{8jxg?jFtQ2Do#7ofKd`RMJDrP9BKb)ax9)}l}xX;W;u z_ee+1uGkXB9o##XVeN5lSo@<2pjWIspGMz6q@;87?qaXsOBrEqZf(-ACtNpWvWoxG53}aM;B5;NUU>;pIqA$ zf=xIcLXYL_A%5&2c5IzO`5~d0cvcxrj}2W(HKBy&t8m((n#8)pxVC}%`6)Uy*QKP$ z6z9tFYO0hF+nOj_Rv#s6P8H$W^M&RR$(Bu(y;%m4>^EdvP9$5rnT;?{Wih-r{-;(2 zeGsgGnDaD_DX226>@^|S&W8Myj#2w)2Rni*S8`!~l6`GuUlo_eP^!u&i4C(dRyA9c zy}a$<$82q9H^9}{@lF-Smo6HiC??DmE~35SW?z;Pm7=fX{-9EKws@rq}ZePJI7b;Hn*ljVy48Zn0Aod-%* zJeDUgicxy=oHA zLwK!(Q|M>viU{N{DKfQ+Cy3_=V9R=RG9!!7Rd_AR;$gOc_zhGWHsV>1aQD!OXat8Z z+4_PlE#~1Ze2c;uf~P`}SVqsz-LQ3rsydikVz3p5-zm$63G>Ta_10YNVOJjcqyCY% zZ20GtfHrzkpiMyGQ~EOFfM}4V#g-FHYtX5*541L@c|;N{DMLB?mX{N?)P;g2dvkon zvIRxSaKBDViVwlC3zigH=TFa%TsI}3efj!S@)bVm2``$D6|E>XFPgUi+m@B5?5&c$ z702E3ES-|Qleu(B_AX{wCD~Uk*^E`zY3q`QtE^6Ys?s&J>DoFR`}bL=^^4|uNUtGo z-Ee-xH#W_BCAS|i91Ds(^#DkptLkPp{jjp*oaR<#-Gx`rzdE~dp|bm&R$Zu&hQOS# z(A0O+wT3hTI`3QQD2o5Q{~N1j%{MAKKW$!_YF;fhuTC|umzvjqG`P_J%149quO|;3 zP91tlI`q=bLofe+`L#i*|CQviNV0kTLi5OZ$2tAE6Q4A;oZGde)mMHAI+Kl;$}X0@ z>x7KkTYq8l{Ny)&X7cFjr)=RHme06T)Lc>!tTF7@VIW9V8Prv`xdv9G87CJWG z^lnuI}($<-3>zCU4=j)`lo$1=fRBg9Z+dY?%YBzrhC5auW`d+EN_iB|?-@oA9 zaBH~}P7}2_?vz<+C1bT%|N3r#+wiL9N6;cA9Z4>Jp;Z;kFd^_6=&t>5X>llfFR zT)t~|?G5O^>#Zhx?|=iZKkhV;8L;f@HU05ZJ^NOfe$Z}&%MVr=;PQiB9og3y_H8M= zo`+x?TU5~$@wn@GaK{W+>WoDGU0S#cMk_QT98oTMc(!L=N*tckUVNCYB(GE9gFYWV zBAA`UN6)nGPizi0iTEVDZ?+0^OUm9O*?X><7wsFEETUdA*U!AXV5W@^b2r{;XT`Mj z5%*jCKi-_1N;N0by_2~m1eZ|WE8W0{iyBz?AJm5bb6FeyDYaooJ{wVZZaHa3yt;)mDNz)93h(j81Rl8YW?88W4b5mI1JsXI6jTk+0&%hZ+S z0=wr)6-4BMVR@q-%BM46Z2afERU5CPyq4Tu92#9lmM$Y9)Ia?)X>+G%@&Q%c970wg4+%*@roZm9*Nj0yNn%7-D zCN*zaaBs~{ARUsUW6pfFbHTCcAEM9E#chx7bHeFI-hr4;@(sbz~XyN7kgtZ;p4%8aTcXlxb+WqaM0?$l#)d<>6u$JERP++{pST z2uGU3V^Q2PdmJCxOTXm3hcj>}WVrtcF7eia$-4CN%8Gs~(zs>=44~cCB-xr~jW=u^ zS9X1P;PQd_wbv@=1B+d|Z#&(2qwiK-^QA2px6FA`{w+J4DcYtZ<&3`9)i=Khov;|Sf_7~0m-?ZYV2qKL+*6(Pc}r1f^56ZdXD|{xCo{X~1;esqlYj*+s@68i+ut zon0$TOKR$k*-b_CHEQh7(CWB<*O&+8a(#`umpr{M2LXQuzrwYo0s@vV!0V}N5iWlO zpu1A_VZ-HydHsC&>eK>ykYiVFa;}%`^)oLn+EDX%vSaHNM_yPb3#(xlcBwljkUg%(wY)Mxm?NtGiR#4hE_N?Z-V~QpfX+>t z$W1w=07+v7-Pw~sO0q2{Y4p3}?t{4Mna-8w)(T?08jehQSmf~;+n)U{s6rAohnFlO zv8ZQ3K5K*{?j=3y5N^)Go{fCD`Zg6j1D)?Ds$~0kSob#3g+5O8Bz+O zyzCZQFlS#m(SY~?;$uKmc%Wye9NG*ln!A_uPRokhtsSY>%~I>;YkkQ>&nH_q zFSNceJ$P;();X9kS9YiT+a&+C>qnFRZ43VArU#RrcFDd1Ufq4Ez+Ne^H~IXRlcVFw zz}|%bZUBMOnPexWvlS~-?K`CQ9oL7G!BDb&$3pv2kUb#TTX0#2t9lyeAuXJ%hHkG~ zp`L1d>gwxPhm$oM7wwyFHDP1;n{Ob_;7xn|ZD-ZA9Uq!Fk=k{TT@KTfs-Ron$aDdz5T!2K+g)5#MF1pC!8qEo%i@7cnd=5YrWCh7>TLaF^vv24$>E z!&)Gt3I9a_##gyN(XYLyw;HPMdUb~OyH<~3%iS)mVa;7LjCjm=PVuXZ)8 z$7E}}q59sCrpsWx_ljn(2Co;jn_(t2d+>4hMXkx;zFWZzJ8bnXL(kndqoL=XtJ+Y_ zm?6u=Fw$2a><-Jb=213lKKe4l{EK?0Xcjc6bX1Fchtw6LqIbqPTB&d*?AV%r1J^0C zxlk_?cxY&QjB?5zx`hgd5FQu9l^3I&uXutO?z|WtzVZoT;GQeqd!?qJtR?3A53)JO z;;S}Z|7=LWF+0fm>7FFtC@yRAL6sYK8{&$jcq67z^`3wEe(d}y78?yVTwpjXMA*%V zadUvKexgghHE6$h6J6v)T29&R1oFuqZDZGmV=#RgdCBNcghx-tjRAIK7lh_^lR@MAV;+!_cd zB6Q(zCtY>%3%Mjk+?e$z_Gi4EZ~sub`+r*gv-*Q%8e_$ z;yur!J@Z8^>?8q;UGn@S-L3^`KJ>K&sv+D(Y(@uw@J$S798id3U4iuy`tbuE2k^3P z86D&}>^@>T#fej-=nh5}PsqC{R`7x-Gqw=Rkx*#5@w_rs)+Uv;C09SUSa$HtuC(5o z(pO9R>KSXYYiGJ{^OBA;c<*x>!`9oXm-)uZmvK-nxpJrKJE+NdTdwG?9!%HmP(63$ zdD^bDUKP@H1FGltJWuOg*!!8gUlP#yA9a^lK^gLxrK0lhp3W#RIac}ps|C?Io614v41 zRu!})L=P>O+TiEU-LdUB~Mwkrl@0>kMuBZHTv z^I>{|T&_Sas};(*cj;X6<~Z8qv_`~+dcLAG@?m>KKG+YtJf^nRBOinxi=NW?$SIKU zWu-&NIaOeI9w689G9WKqW3DHtv6aYaLs2&IiF^=FDSF^qqFr%Ie70|+<|?S>@_KM7 z=BxgY4j8U>q`4>&d@7>`N;}VS98==aK|$)#BgjK%==jQfC1>1B@M!A3EOi6UV?LUC z72STrR~KK^c(z5)ojs)ICcRL`Xi-Xn1<@*~W#uTgD?Yh5UlZ$4zMv?doiPg4Gq4P* zoie>?nyi!Gd5fU>xa$D!T*Rh(Oqi5q?D17wwrs(j@bX2EDj%1B9!S2*k7+`PK7@J89yR})>#66wMoX%!jI|m zJ7hS>?SGN&f5V8^S*cKLT_dY;9F)&_Q5iBCYeus%u7ylO(qS zgHAL3q!0KW&hN49B)>;yR_iHlKa7m&R7jvDBtxp~+IyyyECX;V8A8aHAIPQqm}1h> zEu+JIkn;MCtqgyGJYFH=RWkk-1{9p%zzav`z}Ru_?GNa~LLc(Yjdo@QEP?Ken9;o! zp3E5D2#rpJ*01gR*?mdZ zmPP&6Tlg6I*`cKSrxx|WCB5HJ{%Lvr%;8HfUwnC?+&^th(}Iwgoa>zpZ*Q2f-LSRJ z#@?TNZ}RHS8yy?ct^QQ&R;hJsx~lfV$oY|Uz3-4X+Pe?XmC46p%kT9-&f5Uqlt{QH%u1mMCO11Bj+IOXE8ZI?l zY{(W4BvuR(i4{~RTB~ArJ$s*kr*F}`Cf(8f{?vO@zx@`}4W{>;OVGlV?qzP7d1+?f z?CVLdf6)%z+LfusUa7J7hmC#bt><*-p3nYhz|7HE_pD*|;GALhRIK7Q}{f+BU~^1;Lwt`OakJ_C@oKTTU14LmSHFMey$p z>IXM*-``}}-J`p%Z{EF9cYUP+c3PNHF9`(8TdDmnc*=X-kpi_GS@%D^~ zejS32Ia=u#e1vfb@I>|_5%Nz!Q~>Q~ulMNAUhp6*#nb92Q;oqjDt2doq6Z7^>M>zRxNCrJ0RpQ8|Tlm#{pX`w0)OX}H>flP`SJ$~CGCYUHp z3waa2%9uCH3+=5!FFf#%6Mq>7E0jAU2k`#ZF~OGpwnn^7q_);x;ka4l`OS0{R@`1Gse&L zub4H}cQ`WdCR{E})7`dSP5Bb9;q>6eq26_ATeVqF~_q*quJ3sS!-5j33{hyDW`TTy4`z!j;9&6rn zr_01~*Eo?AO;PTI`Zk>~v2XJUGkaT3SlHWo!ph#Z6E^m?pRlvHYI5QWvjjhuvQFtS?x$vd=MB5Gf+h+AEpI6Z03h!{m z+h_wlCo0X{eol0}#EDKV--#-b7hNxzPWVOlOWcX-aE0hO+r@EG9qM8l>s50aU4&P^ zoe$6`o49Gyv?<{In8J?)Oo~$s#l!K)csO8I?4!|0cp{E>Y~obpG-7>+!*MAxDi6h? z(eP+ICM5zLT_JgtWsrqFp`$ArJ{OKogg`+39uLcM=rq1S(8eMa`>B{T9*U2yuAKUH z=Sz5B z!O2NE9~Od_!uVrYT39q{uv=EY5F^ptQXJ#zDpab2;R$&PbhH$s(U2@F7C9VO_)#g0 zr7Q*~&yC84ub3zYEQ(bQM^7mpwQ|8zk!V;koswymVGZ1q2BNXiP;}r-Y&<*=Ie8`| zi38zNa^I*N=@X^Mxv#dW+1#dJfJS3J{feveH1FQIC-G3l7gYh zNRXoXC#Mx>Fc?9v2ZM>GQp>Yni*7_6=_UIXUZXOTBQjq4|yeSNrEH zmKu6yhhIMUsf8o|$=K%<(&5QAe?=t@2;xNw-;Mm|K3`D*CS5X3T5!OjUul=1ZDy=-cH3d#19`GtYBk)r{#y z({xqAr-qz2o#*l#!JlXzVGE6I5BbG8L~Bh5Ux;98-?P&)kVmmaT50b~w699ksm68+ zp(qxmIGxY+@pmu*5*7Ky+kbYa+#iVz7}h^4LTW=B6XhV$kM-A&HP=5H8&9wjLz9XH z5sH^quskit!{fn-s91Jv+z1X6okD=5*ro7k>{J1(;+vFW=OQvbu&*7CiD8AOjK^5U zCq<=9j7V%-pME+ljmuIuN)Ff*H`eL1VTtl7*0YfbQE`F;g@6=7ibp4jlH8La39Uz= zGWEpCm^hti8dCX?P_jpk1l&>$N>M7QV~Y5gJsLZG8f2}J39uW+LEYgCqv1)m@hUv{ z=!o{I*yVUAJ|(OC*RL|a)${@ui4dA%3M(FYG&mlbh@2w6l;?U9?IuzkoJ4nN4pfTS zZbEwbT{suH6>FvIu^jKaeDu=MWxhYd_b(hs^Zi-=v73D5<%5?Frt9{6z(4*sH_NM+ z%Y{t2uw34kDewD)D@V3+-{lLJF3fHD?XTZjzb&_}JJ;GdUzZj(=i0X3>fM^_>ivP| zEziP8rfb*Y(7J(^7fGvr@-ZR9`E< zTE6Vtn(=L2>`D8!W_|l^`f9Fuu6okzb}#!L&G;U@+r;@=ezjsj4erS_4&U6gmHT1S zP`TyZiot;)x8=uf8$MRS8)74jef=Il4-hsO#B1E^<{9&^!^n~VF*hs)7=wkbGc(Z7 zKc(1O0TyAI7g!oG+@eRcCe5#LqV079BTSl)abSfy1S-YaO0h7o!bfH-Ny}JeQ6cyg z?Pqsj<_Lz`RF_EQV6|~3E$A)lDhA}5vAk%Rc7Xvg-vk&bg%lW&=oUR9pEQHXdQY2x zhs%L-Dzch^TcR%Jr1ggX@3HrZYZpCYHMRG3!*-SL`vxr``m|%l z@_=5ec~Gwv_f?=)V>v=dc>38P6~LVaXqgbey22L{zC1J`2`&Iw5hOSwUc&UGW6- zr6N&+Hu53;NS8kX_8U?_uE5D(!v1`byH(wh3a6e+SM|+0SM0oN9r5@>mkurSPh|Kf za@9?_hPD+O=N&ZN;bRt(0+% zEjfQruD&Hzm#*u+B@ConHs;!UZ~2>3_46axO;#-C=E~a+dv)1um#5A>%da%JU3GeU z_Gb9KXiToUmD)3uZ;x}W_Fzk!>rGuqjiu{0W%+xYwPHh(dot~w-x{nOY~bE)Xxh8c^6rlAy*-xqdTj9j*gjY>T*v*S z&NX7U{G_#U#ANv|CL8>twEJieadNgBonv5H-;FFdCWBvN53qns+OC@!Oa*5>Z5G|& zq2Q;Y_qti*riibAjKGI?75@9L+gTd$FyyXb>1xF~@Y?#%>|Xqy&9_AcyB zf!4%Em8l3JVKOG($8m-6*wh4$qCz|-#Lt9@PZlxZnAj)nMI98&(=otToPz^-R&0-k zPKKj$qKt8CoE_t#$OM7fP;?TWgk#`jWMV)*lQ8#X8?AExHJ{sEr>CJC2sE`fmcA-vqu;pKsykdiU-@jcHro-<3|P$kIeMPFT{5X zax@&C6gF+7d??gScwS|J2^-6%RoBIed^!|Aqgc0ZJseT2n>KD7Ct2EfcswReE0)p8 zDaE#F^G=G}%;KaGYPmfc8IQ!}fL+}oY>{yY07y(Q4u&Qw){t}>$9T2S!_=R$^c3ZD zF&<1CfmVm=0-rW)JRcJ~#Af2w`w$zAN={aq0NKYQ6HKDDU&6kJy(K+v8eabGyDqs9fP{TB=qya!r9-b#19HfMxgB+_u@8 z5Mf(a1qz$D+AHPce*bU4$LZAc zqe1^(H}{^~wRflGy{g8&TP*KwvBAIU@IZ13#;)5LyIvEtmSChLJ%#imFKKT@b-A%OgMa0{IiG*o*P8LQF8jJNzOMPYtPk*{7Ep_UllspB zCv^;*)Ze>9llq&3&4aDnyRELhI1+a^?sZ$lYhj&+oT5nE;XrX=eU1T zQ)tnot}ZdDL9#G`+14_J%HBVFCn=PU4y%MfI4Wp!bL|bIkm64$|F_{3&tArh_1UZP z;(w&cE1gGuRsm%z0vPB0Dm>S?f>vrQ&ruPqKj4)*rxi^J@If5Z2|%!Cj5gS!Cusny z(`M*U9y`V`69ji)ToNEmY+MlY8WbGF84y%Y<%m2T7P`hG(P%_g5l7%OBmczZCyqdD zX;-r=4thtRkhGG%QL!;p7O<)(LObwFa9jq^3SFQ#PNC7zv`oz=ETEnNy+M@}jc;f= zvA*;v(3s}u$S=3SxyXHlgIL+svbl4qb4$Mdg_?|S$E@w9lfUe~V*&@p|;3R$6o z#MVoXi4+ed6bBkj#A6dx>mizd-*n9cCE0;Lf?Nv2TldV=gfJQnhbE>b1prQ<#RybA z(Po=yyOSUMMqD*QzFLR?D-O_r**4bmJa z8r}*LQLvh5$rBq&4{>q24C*C62j{Q3zrje?HeQ>)I{nJmQeVi__RaFSvdWu{EpI&X z>Lai1o;{Q+Z=6g1qP#~Jrf$#rcCa0%Gvn*b`U1I@p7}GGmhD&g5BZ7}FIQgmw+xeh zbFg)=&9q7vq_4^alxif zU38tUR@L zg%9`*D;BPPBiQg~Cb~_Dq?bT&6%*BE-;)?P3>V_nr?j$;fju*mU|L?cY2D1$9SwZZ zQ)(@WhSNM$ZS-fqZUHma2dEUJ)mHF~HEGSSU|T^sFFNA7id}Rj&7v!TNkmLutDy$6 zXY7VtZbL3l(p)mV15>9I?B#EGHQF=IhpT(ib`}fZib-eUod_sonNb)_C_7KvDTty z;xp;iQm_{I*Qlj;?G}J8?SMs3(w=l^vkYVTP_tQ`5t8|e>1Fpr=CCH9>YnjDV4YSc zJz~vkR6TIsA5}W{^MLjAS4GH9*+Ml;LBMoHP*6$F%I; zGD+XU>-awDEQJ#&lCF^Rd*ob&lV}|(QC&#nCh0sbQuo}m^lv_e+UC3I9)A?=R}I5< z32#|oPNx|SxeI+H5i<(-3pj`cDCN*#OYxwdj)5h`0z9i&0EVRBMlP5a01Qb<^4cNn zjEsgAOEf&8?y55gmI%d2beLwFtn^#tqrFt|h9|@V^?+BEsl7o_RvJ<%MP67NQ7st4 zc2W-Jb1}di1jbf0FRJCy_Tkg0!ExkPg>w#o>q$8*vS!J&*U%)>9~Fw%N>dYL9HqfI zZEO?0CAa)FQvY`lmpvF=?E5~L`ds#0@}z5aEIJnw?=-z1fB#ha_?OcDFK3-!$pNVI zm-uC8d&b$G?%0)e?p}dNZ(XjTF;`Q2+hMJ7&${pOTusBZ=dV8h$`@zdxvHj=GgH+y z>x8vWb={o#`cTR`Z=HYPEoVm9w6t#Xl7Gvr>!!bU&T{=&%0F*?vvH}ZZ^_?3>w>XR zZR2uHZ>FYq;bgj|cd6!)Sr4d`S6z-=ihTR*+<~RC_K$eqmBuXJgm7QY<(W$}b7E>_ z?#tQoo*VMQ*5xh3nJvRh10(O7elhUmM|I6_l)YM(dSSkOscztkE!Wtx+}M+8?3o`~ zYTSIq`OB)toWFT4x#aI%FlGFkvVPL~%w1UWcg+uG{C!z}SFWOZxnlhuC^H?=yR`dR+n3aZAO5l z!?r&F(7A?VjReEe36p4^u|9y!Tas4D>#V=V#$XVJhz=v@1Joi~uGcLyHf)N<`e72& zQwjhk4X_br0QBbp0JJ~c1~}tLI`RO}X@GLP0m^v+0L#_{fG$I>azifP8UWBOR?J|_ zlJ=s7C4gW7%9S*vs1&^#D0n-6NCh>I+-Ml3YPGM7u2`j|hEcXVX&0-Yz{7@#I_R*k zkY235QKO}w;UBIalTLz#0HfOr7OP2KqXGbP4<^Kgx>SWl%& zFBqtoGCwi&+6;i0R0XP~pqw}9(Rz2LY@)052qgHJzhYH!q$8!G|Ql4V*7)`E3w)Y6OAto4NKsoFan*nlLehpqGB<)2eJlt5I znDRq_+@AaM6!_XqMbd}kK!w=KK<@g)|8i?kp$O!X@R#XVNxDSPQBSCQ{?V~5Rh87A zQHDZy3D2DgL&Xhauo%>$q_a^Y89-;UP>4y;4)xjqWrpD*%dXhQVSyA$_)B4B7$6Z` zF6grp?%_PJ+$AI`H2B#oJj0;*i_-6+$C-Jz2Co!1L(h!DeefANcO5=Mbj|?qE1)2K zhn#;-&hL|R4bBniU%*W??h8Sg&Q$uU)JKqECXN8q6M`K=l?+cZ{jD@dpRdBnqiyL& z6u`>)_wXg!3_!ckNL;!wKyB$Y%5$BZ*U9+nVb=XiPj#-o zX}La-sSn)P@Pq!h`WGt}hnKnsm+JS_`s)Tzl4;wr zFtWIFscraG=dHHRIp<&glK}lgrXtjTz_cdpe`K-qowmiMch02!N3+hSwF&gFuz&t5 zOO=m)7V5_hwtKZWRN}G@yKR!cebY~ubYLDsOe}I2Gdy5oEGc0AnC3eZ0Dx)z39U5} z1{@ER(|r^&EDQ$0IfLUdaS9whsPMty3sWI>zYSlGU{H*W27{77IZ3Rj@*2iNNW?7_ zcnD)SjFJ3+BK{+sw>WlGVM|zYQz$t#_%9E^d5ODiHrt$coPOK-PkQ~f$3E%pvDJNY z!n6}^+~l>@f6@%+PP4_jsvrp0 zPZX#~UyUuFBj>I~Po+OvFijEu3x2YfS;N;+8M7bx{ zD7RKa`sg`_n)?C0g0mXy6(u~s&is~oYB8Fa2IHt2+4*>;Luhgf|N65{n2)q79(-H_atzYo=v z*swngD@+poO5w?v1R*Y6Kw_d?Cewum5NK9Tcsri&81E2+9S1rNcN{xT7mV4BsbK7^ zLRXrM62OG5e@w;|Bhz^XIg!>*R^AiwBT8*iQh5qik-|Z7kR& zQWXL-i+o}>iX%3unrYfbnawKk`T~Ge!K)9_V>Xvy1i*Sg`U|A`T>_#Xz#+qnIJbA0RIHx6ELr;dMM?Z&si=Gu;{JFY!?_0g1jKDHQH{Nnqz zbnlU+s-qVVedO}aM(2jIE+I8YMjNh6t}C5$n^K?qz}bnbqV86xgpK*r{Uh&0-uYts z>Er3%&o5Oyd-2ds1BIN^b=h&rk@oG$Iv+3faix3i`MHCsc-q&Sb@ts_ANYRXn|-_H4Yg8)dBmq^WXp4n_pYl zwK%oZIg;M6?;V)he=h5L=2lfpYDcE3`$htQsyDOiX!diT&kj7B^H*Qne|3L)9pt_J zx$aF1y_xREm%B$Y-6Mb2z5ms+IY+8GS6Bar<5fqhJ#{`6nU@wi7HSvGX<_#Vb&o>n z-E+VM1X@waieEqX+}x9?XVP_DFlxA!uk!Hk9KPMiwQc^?Lw(l8**r7`>B^2&H1(B* zZHwy`%hJC6S?htEr*>{#+S8G?c6_=GX)`Y2-luMO((uT{x8LolKfqbudvti+0k7qK z&UV14f#P}#dtw>h6I2R_!d8+;JVa^5INqcUsY#a z$EvoMTy67bmwZdWFzJ}#rkoD63i-u`!gl(Al#RGX>?`EdeAoilA*KXCLrfE5{94Mw z8!^qVI|?OFw-~hhrc&Dd8-kX4#vFku8_bSN)<$dvz!-c$OV(?&Fk{SPBr_;_O1GzU zTW+*zaEIAySj2WxO|DV94%BY%18TQnRqX&?il1Mnc4Fs^E^Q7$X<-Az4tz%ax*wu` zJ%6wD>owHR4k4mN?4$Y_tRG;(HYODJ0cL(4o-=6YfK3Xor75Mh5c>cD@f?!))q3GEP%(fE3Yb^yX`L|5Bz#x5t))E+`Z-dzRvs<>g&@vdI z|E48xncfobGL-w&jLT4Nj}}vSi(BJ;+E>9+hI#^XZkskb^wS(7H{*KI^>?Y6N}*s< z-}W1~^E{{jB*69~iB1wvYsgdmS&ZVJi6JaZSK&Rlxj8CD*v0jPr;q&}Mjr_ZVclwh z=P`N;Drw4k32RSPGRu!e(Gp7(_T!lK`0+R4djq z;pk+-+1IB^M5O;jL|W)_61#~-In@}=ESDf0#3Xx=PWm5|*xlEs889&ENTQAbVlB*I zAcDfJ)F4AzG()*Z(Mrf|HprAUHDQilEd2yg<0G8~v_7}kD|fGSKEU$ygw zoN%FP(sUo%O(qNUV=8p9E3xru3APfr>IE~B^RcO@D4YzlOIz$#8ZLL?w->^rQ|yL> zIy$(U6~Ye~#U|7y`*AuDhM##}c;@p*gu_RNM+5{U+|MyuD2*;Wv5czS3a`O^vRh$d zcU8J>l8E=BGyf}fCL6v+%ok=Di%u^#f3Lk3`AnMgQkj@g+VAa)IKRl?BywT zY5CL?*#!IYr?31O2S*(3m!$L%Odqs3Kf##ouQWADRyHT6Ns}U}YKJEE$~7jSrUTN( zGlnyodb}LP56PU01!@&1rW}5eLWVdVdLi7vo`^q9pOV7kvT_iyEQ~1$*l$yYi-6_JLmnyz?X8(ZRzf( zU_WcCyUUquhfKHhXm`4IB-i+;9=E3u#oySnXvsBh)8lp&;+)@=7Y1{UkLYo`3vp!& zu3X~|J?4=@jA#B(uJN!QbCixOeC4udUBib8n$`lRD`$@;YaUDa zzu)|3b6VJlJ8#2(diakIFFyW$Z~D0}E{3 zX4(c<6N2f_ii2z4Yr4Z(+so3tfO~|!Pnc%+r~RE7zDuXKCF9wm(;Lq6Bly9P<@HZw z)<029PXN8PL88|NdM%*WvxmrS$nc%$fDPLhPA|s)^y`28^`(xZ`1zw*{z~*CO(C>B z>)fgJbIpcSGBuI*Z_7HjmkPU8?pyI=q`vd)D__WXnwVL3TZV7@fbYOy^38v9w}RuF zaC_WVp=I~?XPpBW2V8qxJKtRo$5-pwVC0pq-TK30ixvOwS^NM?^<(c_XZfu2@Q0v6 z^JJ~kf3@O4!M{T5*G_X!-hmbKllBqsQ$FsezUD(MmY?>U4z1^Y=5@mVv+A}(4VItn zv?Ao^4K{@Qyv0KP^(}`@Fb0Go0jn+;g!S;8PWsNeXk&Z{L8& zNw+CHO%5G`6*v8;m^1kQ-3eab221HMj;e~T12rv zd1T*F#f8225c_o<=|2-~CIlirp>i?e3yKwzy14Wn1-?X13k5nKRdpeIBnS8g{&DhE zI4c&D$#logncN?8_~&8&ydQF9A9CgJ|2fz67hK1mb6YV2CjUjpGFO%1s&ej%%e|L+ zbDr)0$7V5^u`1k+v$e~vhK#FWZs(G#{i6L->p7dreVZfq&ZMW^wBeIOru`<dCa#s*_|+qk>cLREldA<-paHySZL zF)=^ri;iuWnG0~V`$hbhS~cVcINiF3N#^YjW^D5*MEG!a5o#V z*Zt>SX6XxA#_rqwaaT53S(z{2d-?KZ=F9g|zt`)v1U&!yfA!7$ivfc8pXej`C=!`( zW+1alFa$$-i3#aVPLTLpHX(zz%uCJ6C*&l-P>kHGm{m?FXH^rbS@ndPBxJ-zf>Hd0 zV3e%#b4tQFp~0mpC{?qXRH=4>v}>LskgIl5RJy!w))#zA!nd6g)n}an_68dWiki#a zfk9j+D!u;cX_j+IXr)w^`1r;GnN@-%5U;=|IRV@;vY(I>@&(GS5S2sM5V}weLQK27 z&Om^k@^kc1>owXx$8ye~+wY@;01|K|_{Ny^kfQHzE`aP-i7?^8#Rn6HEQNwaDwt~N zN%EBe_<15fO-zFnA5x3*U{dOYN0|Yb$TC!z{D}Bi_BoOXJS6dI_!kw7d{L=+Md+|1 z1U*|+g;inM-E#!7OmemizSI)!%#=}tWgo!^WA5Z>?leT0^5kVgO8GF!sD7dTJRy-q zbq4laz1D=)9^La;Jo(R;Wl5(n36UEEqfPR_&Ax<(ymYT}4kW6FnjAfcn*3l%5-zM^3K&Cj7J&qmX{;KPj>IMDHp*>6^th)Ixm8Cr;%NLDQO;S4PH}@wm z;3Eeta|G`f1sR+aFh^5V-I?VcTbf2P$26s2$HN7=AUyR+*z}w0dv#b3XnFRtXvol% zTq}2T*l=(x!-jA^({c(B!HUp|`s_KHjTP$h)3YOEhiiHsl4(mrE>j=8lL39mehajZ zCwMrHc!2K{=#BP0$akYUrU!Dp64W?RO9ac4)NKS;yO@{J^ITeN4Cfu}`E3IB;JhE^ zEz3T5U&`Gl?-TD*w~71Y_r`8I1bgJ4jJ@uK*vSumn*)1GQZk;lWG?;obfoji97yoT zFO^_BUaj?*myA^=oR@1yWIFJw*qt#dx$G2a2k88mN^1?%^@=)!ts>^wLzzpK9LobM z(o3{6*=Cvi;AXZA?I%MQM*Kc^Fkyaz^$C{ftkdUW=~*_&xm^MJzB@QW2i>!*e?BOE zu)Y}t!O)D`IqmZYf&eC}Tz-yq+-nDmSsMsCxu65J5%cE!j3a?cRI>NLunvUE#}RQN^%R&UtU}kxZ2P zoIq5A(!iY4#fFMT0bQT3SMuLI1p$p?&@!Jt$WI*AhR7_Y4UF8C62eAVa0Zt*B<^r2~OjkcM@*eN+B#<%orlfu5Ve=Li^+ zX_f)T<#cmD9a1F)Mzyq0iDW?3jL*AVEX%M=s1lf&o%6Cm5N!aJ1_D#_UhhJpiKx8b zbbDDwRDl!&&S^HJzv;W{^WXQ;Eb2-~Lr-As+8` zocCCIl4X4~!(s`7w)uk&3}l!3!=WWzRI@$?&jAh@(o&fwC3oC$yA`GlK2QckIqEv6 zLM`QJcu|KqA9P482A&y5N~IYGFo>ITbqHDo!|x0RIj$JD4KB=qFhB)SnOKsetpoH@ zq`a(8)SQ3U#m=FLz#-Fst3nCduDJ%3ppBO^K_V)hb92xgkhPa(=Qwn>6t!%!aX{1! z%(+vz?1iE{z>I{01F(fH`Dbh!ivt+oWnMz))T~h7q2u;Ck0!cb2q! zO2TYi(mu&A`pRrwb+5R8c6WVPFt;pSj2D>KN_GqA#|HCi{z^VyeJb*LWQafi8vnY3 zf9-9)${96G?#T-a$H=&~I%Yj4SdT@zw-+0o|%Y04O=J;Pt{KZ7< z*m>dDdA{ocPhX50FMU-~xfb{&^dPixDq7P0*i!jP-h({eb}VW+9=A1oHvH*u%+?dN z^{kzHTwe3ZhKFcOtS#*RN0h@?@lWD-^wco1b`#e|v`a_;;_*@ni4s zjocr~1A7!v*Sn{do6eK*s`^+}zfje`b&0>_|^WE(p z-rT-xzPU`r>Dtep`@}>h@9xO z!8@k-X&>+B_^ALN2=erN)Oasm-LgFVq}aULyV4sgt`&-F*MG26+>wRn>h^v9^@;6S z{>~db{btno*54UR<0i}M#L7g>R416~HY_`)_Q%KjBCcP1e(Bk^M~~lFrz93TcIXa% z!o56%a9&xtvfj0!UJq@SMtUMg`RWT%mH%9ee+`EiqJJAR1PI8${Qf37SyFr3d~FS|G57iNtD-pO^~3D zh!-ro(_3UDz}F4NEa!GC=f2ycQ0;g7O2SeNkT9(hL&Miz-X7TQ+yHMQzHKEW~qKU0r(eYJX zo3}QHBa>T&TNGb$E^0U*FE*}ruXL{ketH`5aCPPC`i+go4J+UII$wJ$YP=nc88|0~+53K7s}7uMLB;(OTj^7oWb?Mf{i2 zvzqH9v27cuf!FU;N5_hY-*w1Q{=~TgczxNW8KtQ&`z)hn6mL>OL%vJ_B|J@`e641z zfa04hV_HhkDj+WuX~y~~q0BPYLkXQqz##M}P)R=p`5m(222Jf$4wT<0qjq~q_=plF z$VbZ*kOzmZ`xe*<;CEF!@Mh$2h*}4%zRbLrg}lz;cxT?}McXTUsT>Z*?{YY}%h0Vy z#5a%<+%QTs$pmfD0koI>g!nso)4tqEDoVJqWUUJ_sV8|7{N^|mWqE`(l1G@KenL!u zmlRSZ@>Wl%7$u}?27J^Lngz98E9!@?jlyQma=sg|1%*aX3Wnp<6pMC6>^7d0>I3wA z02$2z_MLgCa)Z&EawsPZc`%lc;~E+7U|=mA^f1G=BYBwL#{32o-d;E>Cwl_r$tO&P zWy!NYLxw;i;B`LKiyY24{Vd4rRH$zpnG@LZcha2G$M|RIq`gbK!3RFW($lah`%vXH zOE=ECyM_JT6Bs>mW87ey<`j!KgbI>qQ3(1d&ucR$>?cVymT|>*JSumVfQao1c zKGeRiT|V{)bt%@5+|K+T!j8X++>@jyC2l~Pco>pN33-tMH#~(7_lt7$X5^4lWE>(< zSF#>(kjnuQJIEQ;#~IGQAkt&}cyb(AA-I4nk*tSNOHPaAu!GBzWHk5yd?qaOAfr0# zVYEzM5*nPFxnXpSJ_UoE|H=@k+oGDuX9}>()DTuf9|}KKgaA8JgwtZ28c{k;E^2)L z3OF%gPC8yd#mtnXsAw7eVjg1wmn=9-80*Ir##V+LfJ}Mfs{)r*;xwDuG*GL;)NZ_h z8qK8Kpo_X-K~gMXohRi!O_o8QYLadNrWQ#!`L(ELY`|w-7N7N)rv{uh;?#~)iQ^+U zpUrW85FHQj!yG*r%=t0NG%?L#1^gqKmWMK?75>_ocBX^roR)!;zl%A>9DgXACYcjV zH*;)S<|dguribZePCk?~rkU_4|m;g#0%$)<2a=62pH}H&k^VKnfo@cl=eeD^Vc_d2=@X6ScD-z71RQT^O zeb3ZkP6n<=*Y|0+Xbu;K&9ENH83(KvC9fpsx5I_7YBgLI-t)g#G*_JWrv0`M_Fp7{%5c6y=$79CqRo94tr$Tfo&_zi7(fBA4F0 zj`ikN*mO`Iz-4@U(ey(TjGJD{9S9j@(m4528gJ#Mk)aPS(YOq5PQQekx|edp=H@0z zUOX5 z2$-H)WCf$=`n->J`tZ2|xwD=>Qw)p_(ct{@22xE@nE?)Y#3ejZicmO^Q?i%70V(aD z!dF}9#`Gb@ai4Ps*`~w^Ao(@xewSsyduBg?4!nF*?rF!IlXK33FCTop@R@Lt)3_A8 z+ME;|K*t9}{s4m1$8|P1&-t>rJ?91wz2E2L77}p}Bmi*tL3APATak2#WOGR8_G9;7 z^V~gGb4ZiO`yN-Z6#5!gtIkZTL zx+(C&CySmUdpvZFbf(q3$;Ow62!<8YH^TYVS2eiOGO$XwDo(%Mb)FTixnD+(yLS_K* zI;Z;pJ%lp=aeX_+bmB19Qhx6P@dnIjs7`8l8IFSJ@S5`D(oQd&{UEj?fmPg}LChe| zPtuH^#;$0(y$yZ%SW%ZaxM6u55a->=r)QAtE=zQ-OXtjgqTC8mxXO(VsWzm^bUPhV z+A&|Dh76H{X5v$AN~(BBZEh^nlDi3+wjW2mzSlPuSn#<{(jhhK5f*Cai+DanprSfC zbwmSp|KqO0*%dT9q&U&m+u0_Py*PNnp`|x;Ng^_WCR}>j|KDg+3&vD41QA8#e88G# zbJx)Qk`}fmdfyU!H;B7e>T(8OP}L-%8iA-{2A&roD#?imMA??JZ-+=PRGf@;;l)o{ zxkTB$c99zOGj^FsK`bPsL6yy@5@J5+L*p3k z$~8AHD%v}`QC){rr{>Ti1D6n?YwlThFkn|nJyp18AtuQAP_N}qSnx%i)Rhs$I(%Wx znV%cw{F4xi1mh7zw*bTS?L2y>RhNUrxAW#T7B|wNQOV|Tq z1+47g#KHF?FLDsw9nI`42uACibH9c#`hBAOt{X(BhUkzJsUd`<=2%z8(hQ*^nnUMu z;04{F6XlbBW+7CN5qm};VnHG%>qxkro;JZ=DXQkBfGB4`O;xaEqEKBrb_RYZsVGz~ zt1?kQCX0 zr-4X;F<`eyzDc~CU@HltotYMnm>&Q+2f-?USTc(2s-d$|OcgG8LhySsfXs!raUO5b zcwXX*$)Xl7x42YP;N>57Nfb+lS?7W&C+c979I(W=r$FdZkQT?Pu_TaIa6>3Lhm!Ls zLB0lb83Nlw2rARk%gjsORGkB60SbVkK@C~6ovG=%c|aBTmoTV5BKFLeC{6y8iuzbZ zi%`+B5!f8ut>|958ZWI|_wjYd_>$vGm*Un&zSO=^%9nI5U4n>;zWBq-?_XYP#xX9_ z=ECOPEfa6K7}Z^RtTV-Qw4kHcjh~f&TE5{Hs=FhNQQaU!?+m4UdCN{gOWaZyv)BcT zeWNwfzH8}QR>Z5CVpVNIRoiCeR?%+Nxn*74V%ty%mgZ%}o{}{6Ju#n(+>Bh@>fS!T zeS*I}!yDbt6f%>C1hl5B_zBL0``*r$ZW*HUvcq9 z29aSgnJPr43XvHm5uIUjPlt%>l|Vcn5id|7;u=ich=_Z33L8^B=Z~CHv+r0otpzf?eX2J6Ci+!%0sv^UBrF9h?!|a%(Nk9&SPfIV`f?qGcAaj zD#T0`hyb-JR71K~gqSVX0JBEKtVxTQ)gfm0%ZZX|zV-9nbVCyA^mi+8%Fch8Ffp=*H0w>7$|YS)Qie`mDCh zX5J&pYD@NdiR$Jj@(ycS{m0G zKD_e&m9W(3Gu*J*`g1Lcj z?1^;lnoloXgy{fR1xg>3epI$J95+=iUHA$v64+PlKRddv6^f2*yuMr1wI{17tce3$ z(}Sk<1;N@Lvz`~M=i?Q1@tUJRwd?}<4MCNemj|K$ma^5*N+?#+Ae1z0^z4*$eN|So z-uYSYr@egZV6^PqcJY_izp=)Krgny=;*CeY&_2?}8qW)j=eL|fXSMDkEs^1{vM_a!*_2^VYBOE;&J31j8orpIdi+7v^ zR+{c0R+{S9&!M5{+9>0@M)`&@{^o6=;dZ?J)Q0kLW6KwLkMcIz$WY`Yf8kBO>#b;` zBi`7&ry?3Bkn&k8KCwQq#!Br%shw{=E0mrE4OvqrLbdC`jY}K7{IQ|!+Aoj%&5_vf zgfKk858dJG-iTV>1U380XjvUz8D4wm=Og&heT#qX_V!hN;tp?l zBdU9IPpQ(MBT@J^ZzYefY}t5s!^4;NL=C-h2wFo(ckQ-dXowm51VdlEv^rkd0Aeca z2R$h)+RJ~nwqt@}EZz1xnP?1gYE5O4N@uawP323OK zMp{_xAtTu|`Tu|>9k2*Z zz76~lwx=Io(%&G6ZNu4`F*EVI&H>q|jrjdB63Tvmd_W6je@mc`za>p5Z?>X*M?a+h zt-bqt1@YyP?(uwruR$=pO*2+Z@y%sp`IMkm0=$r~fD)mYLV2_1x|M>x=emj7DNq9Z zj!6L}J5~ziD>UPI)J}cbxRTl>6;QIPR6xmY9;)5V*Npd3yCr4g-PCReuI*N!l0H-$ zB{VlqQBh6fjT2O~okaN)3P@dG(WV|0!9{NEZ_dJzc#$3+TjHs`h zSd?W19+4H7^`rtOnZIDlg*C%{4dD?KozHq8^hq)b#9=ZJ5SB5DQ}7M&d0D{l$tW3B z67r>?BK7A9nStXqH+2nA*JM%G{4-Hc{SM}Z)boIPUKaH{ME%vIAExjxvE(lqSsgg& z_26vJ&kS*d4lB&I(ticI&5?RJ@@^P}j!Qx|rH3sI6r_;2 z%V^yiRky`!>SHyXLQUu9)vXKL{ldA4-I_aa4J%(zR~^!{DXMOcSJuQTTZPKjO>N}D zRTcz=#9gioducr+l(sMBKTdSX47bU)i}{|B?J?eXEvg=k+d2@hw4koW_r9cS z9XVIeAA+d=xmQnEX;POq0DagCDASUahiWW{_%~0aS;p z7S^3YQ>?H?D6HXYk8R%L`$vTnW4nddmy}NmOiTLz*GIyd8j0tZkn|{g0g~X*io*Rn zl>8MW2L+Aiu7n{WRwHzc-=|Kw0Gm)rblQU*eyJpDO2#?Jw?A$!kdj< z%UwbF5tLj-$u*RqHJ-z({a>P71SS6-CBH(+C`!gqf>w8KJekZwCQt_nx~BO0H*!j0 z{aQOtDe}MeD9aTkU!PJb%wHEl@(oHP45+a^4_}=VW&Qwn6TU_5-T6uI_Ci=KAYJ^| zOv3L?@Mg@Rtp)qyr1b*dk;EJv6`o?~jD$kA(j3 oiHiSDbO=PpzMLV+lBHYw@XLR)Wd90vLr$8PFZ=_6-k8(>1A*9V-2eap diff --git a/test/stress-scale-tests/tests/__pycache__/mount_tests.cpython-312.pyc b/test/stress-scale-tests/tests/__pycache__/mount_tests.cpython-312.pyc deleted file mode 100644 index a6c2b84c4687f9153e535e54f6b38b86fdd190c9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 7627 zcmcIpTWl0pnyzbiRo~ohx9>I-7z0JzHaL(#!U6^|7zhqovl;L}dYe9FyG&o4s%nFK zx@V$Qq-mr`G1`^zD2&FtFYDPvT1&et@dz`cQKWt8mYuk#!fKH=FYJ>Ej5N!>?EjzY z8^jTL*^+(g)VciU_W$4a|EK@m@Aok9{3&!~`rmpO=D+Zx`iPaxlWEA@W(Y&r1T*2h z*$I|@a}ylAxdfjSCIprtJP{IN(ly~qx+mO8X+mNdjyc5;@gs(~G}mXmBXdHgrEVyd zG`UvlnP*k`5jGi9S=$v)=vvBp#KO1A+tTrvsa?=a%a+d`G*8ev+m%R9O=(8l5$mK} z<>N^QWNtGWgL8#>vJ)^n!hgh02=kmO+VZ({CS{?->_L>R#1k>oR3_7glEj*bIjhC> zNjnGT z=`3j?LDEb@cLdTx!lV^?w2^lD?tpyf9XGW0kggoI?WTQuNben zpC&423#nLAgL8Fc2XN~n@pQ_H=_$>yU9rS0WU}s|t9ok4oX&EQc-Es$nvuD5B9qjz zA{|ZTi~ww|Ez^TMAG4-ycV*@_moN zEyKv^i~%}m%b>G*atyYty5v@vuBn?ASk8NNOHN>mN2sP?M4sAKnzq{+!nAuW%}DC0 zn5Cyvr;J!!JKwls*d0%7R(}u0rc<_XO-~V9N@`XN&(#*tO3+#}v-HGWP@*L!opnjG zbMYu1C|#P-gzNKwZbhBrHC<4k(C69AP$C_VC5EQcNo`2KIvq2}5T0w?)FZ^u=QJY% zmU(R`cHN9TUp|C7GNcV_L#72gV@6DnqeZCE(mHd9szH>N49?EmEsmB)aVAwQ&-T*2 zz5ucYH>pOLG!Y&@WFC4$g`*FHV;elz9w?mJ@G!nm;Yhi)z3`(A7t`Lecwym6!Bdt( z>r!V)>RjBpCT)X??yco;=fjSka@UT>BHtO<@G-rSa&LcGQJ41@x9%E|%mwl0vecVAiEQyv(6?05IKZG@R!k#e63J^H&~9=ise zkG`(QUU_@li>HR$&n z#&{iG#sN&^YT|~BiF^1F^GD(MQ{k}aahhR^&Cz==)VzS+65$ZN0mGRIkqD5wkXa_& z#09A|?^b2Iif9*iC@8Gclc2#Q}fLGFBD^%-E2jfgM&lDnnGIIX30&0{n zDbj)YA^r&s3SH+7jM?>D*_Pa>8c443p=r8i&lF zLA>zf_&O^9Z}Hi`7^et|JQY(60Ahe;l5rNI=qbgT)|4qQm=sn{Y0B=To=E7X183?K z6@ToLV`E_0E~mTg#G{6b7Gbau&LqeUN;}d=qTD9k>-(oLib(4Gylfm4dGq#0Qe^6VFZ0!htWP zPO2(#9I*PJKG8kj4ZD4tk(nH`DWk$}BLjcVReg-S04+B;7s7Xf$eG$nR0m=hBIJb6 z#p;>mMGMEK&x};f)!)RGV?Skn#nsP@xN|I#?r`HwA7fE(3tR7CjQx36gDhWNMm0xd zltnx_7f5XM4ZJzlLM)|E&Q%=?`~p7CU1Eqo@A{c*yz;)zUT1Fbmze9U8pv)r;qZHm zGCY#Fp;;LNR7EqvwaHFW^4>L$bOZiP#ZC}7cPTq}DnFFzqdR^&`p)=;v5|9clu`O~zmXK%g?Bia853WD8f*WIpS2=1l06fye0z&{c4OSy~y><^jjE zEpHhHSOl%bnM0ZKjC(_|MGY%d-WY@JGTy=DT}Z}MiQ*Lqq^_iCw>pw3-qG<15`=-r zZ3}v8GHuIm-H2&^8g;Oh$kW47my^0-V`~;@dZrDD|R%wP#z-BcQ#iIGZ z<5-Y4S{WajXSZ%@KlKLZp||-@kbKB&2qF12`>=cKdiSfP?pMpf4*1&;*w!e!!SkJ= z4L8%X{|V#n2^8K2!f0u~J#cGaalRB%*F#53p`%}@#fiyc=;&%_s&L|gFZ9XTn`ajX zmToM~6g&2<`G(6|cCARImP19~p+A4q)bTCD!Eo5_?VE2Gy9bwVEYIBK?(VzWSL{5p z<~v&6(!V_Y`<8p}{;sXmGWw-&^nW)*?D5P*SL<&^nn%LS{jhwznZMtw9{2ITb+M5D ztxtqB-e{OL6aR1M2jK_n7*?K>Uk0(vOe0i4Z+r-c23`kWmT54)%?z2(anBoa?z<-3 z58z#ggUw7>Kq*Ab@z0wu7^y4I|BRn^IqwD|b!P`pRE_lfdnYLWRE!inMg>(&(I*u% z6UQKAGLy&#D}E1}X#_m3h;bU0n(Z2|HA&1@9DAaG$TyVlSPKqhpi+&2J|hW%r${Fe z1$o9_LIJe{;{xXX3KO)UDnp1=(G665V*(Nr_kT0Zo{PUkv*+TI*x!Hz9Ie*=C2i@) z#g<4x+Th%B$2Y##b)Qo5DNEsh?D_kiWxcfR;N9J8zN*?5yY{bKTsc$hIJ)LLR&MDo zyz}P=j>dL;zrsFfX{RdM@yu+8VSjOWL}2a<^2jiM-`_nF;qOO8$j57}4s8&{9rgi* zg*cCyfr1ww!lJ@CYUi*^p0%(SeKzBu&nk6!jtDRH3YbmVN2E&5SJ?`b96Su?nb}M| zT~sj}*b2DHjWY5A#H(Th%hdSiBy%MK^)4b&UgGj@SbuYMl=?g8&hfxal@;g2oa-=D zLfe*ln`*AQa+djcNML^0F6Fo^oI_rE$qMD1L_D7g#QUoTKZ^KrQckd1tK&mFj%WYE zQ=QdG<`X&2O)+uq3PkvMIVTf8J=?&w-Jq=O$G)6=i~G1qfEIbV#>Pz;UB@|=mR@Jn zVD=PsL!CPWh?oKBO^0ETuuL>;w8ISEEGP}WsN;vyDBN+qY_DVSG@`Ma>wYbG=x}ji z+8z`Eo;nzBw&?|QaKIt#U<^^5mJTdHSO+d1PV3q8n35)uY+s#{8r%ZB5SUG97P!XX zdOqU^hbNzHgUQ33U~#(}R?#qj)$V?A&e8ge9il!5hXEd{T=UUwsp{m~0#1?gs#zgT zqWa*zD4m+rr;Ov!lVX^01#{1Xmn~AK*!T%nor1*X6I#l4&BYQBnX4X3=0*}LJT&Nz zqDM@j&GuMnE0&=910FMVm`t11jUUt2l^7HKXV##x4kDoQI1C>!GO31jX1CXOv=*ui zKHmm`&{P0>01ca4Z+GA7Uc6Fjex*2YwA6g;UQelctl)a;V%oZHU%GYaZ+}woK!_Ob zFGvrh=5?v3BtbCx>&w5qysVab_peN^Nk<<@fls_Qy~VaY%LkYD6vGGBq}QNNThHQ| zQtK`VTSMW38zhF>7qVY9Z!e6LJJscF%PqzBNMW?x*}LAUmY`(!%IMwLYUi=SsfCf7 zXCC^3>%J`|-6v2q+M1Mr(YszR zcWf_*w>;?Cx_s)N$9^-mHgI@Vxl|fBy!636_TKLM-Y@1?k6rwS=HkHNl5%OaXX3Fe zVz4U8p8xq)Vzy1NCZgN#_WMsZ1%A&7kZumxmjRCcKL8tGQ=qWp91c<8a#rQIJ=na4AH+UJZ-b z2)!)MN>#qhdL7=T<7Bsu zo$&!}1>;5I=tMvSTe-V64GXFq3iQGbMXS_DV_c^(J40E}xQIghG<1L{+2PJPC^PXo zzRHG;oESvDYut$Jj_+f|##VoaGQapfUffb@KL1@_Z0%lrqZHm*kSkoYpfAabKP~PV zErw68NoO90+n}nsZDIba=54@x?cM9`yGrf5mf6+zJ%!OfHno?7o$EoR6jYWv)`EMU zBRww@+76^AH#y|`%I7CmzMok69oK*Yv?2RiODGZn+ha&4Snf}z>8hqdcxvq z_S*rE*zs+LB6^;jWCyF4)7VAzI#zZJ^0pg)ZUnTkJ=Zcy%YMyxzh?YjGlBnLdjFFdE-}MT bg`=!zL3+x->*;YW%d(zD^-m1G=}P|(tJIa& diff --git a/test/stress-scale-tests/tests/__pycache__/orchestrator.cpython-312.pyc b/test/stress-scale-tests/tests/__pycache__/orchestrator.cpython-312.pyc deleted file mode 100644 index 712a7ae9efdf1e7a412e9cf342fa0491cf405f68..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 75792 zcmd?S33OXmdM1dCBmjaWNP_$JxbLJWQ5&_HBB@=}Mq8U@8sY;bP~7|gN+Jx(l}z;< zI#G|QBppdpt`?oD8Pjde6g^3&Se?v-PF2N9Dye!f;R1trrj=CXobhzd3|)42s^rYc zeE)rT3&2O2v^}T$%z)Bc@ABW}-TU9~U+$06((DHOrgj~?`bW7&!_VkO{#jxdpL`t` zcMZIOHx3&vNN3{(BfB?UFtM}wf|;Ey7cA_YbRh|6({S=g%7v5>>jkTk;+cnSBen}R zqrqf2Xy7ehH}FYso1|PW*!kqI8!n`J9DK^PI)g#5nhc)Qw__w24U@)6W4-kwy6>$w zMUw}Hy`E7&&SRs4-YfOSsFip7J$~+LXV+NmrqmwV^ zod}!BXW}jkcUid0#yJ~j7tSu6bNC#*&E<1(%;PKgJUq+i^Kn$LAN^{xKo2S$oF~jtQI}@4TZ|I0WVpn&SoUb&tS} zje8KoJ2vX;9WZO^$ck&a|Kw&0RgvE)zY7|9que}O4O3>n5^G5A8gfvagXXtmF@uJF zB_wa6KRFen!H+zo-w@@S@Kc-HpDMo^a>!TeSq_Jsl=L()oN7qE8`hM{CNq6cHpWXWEG+`78&Q+TWVUOrD*)}Xyn z&R;zTEppj0fY|io_2COo5|V3V4&d?nZVRB|wR zXp8(*JqMFy3Rxf1Yu~MEX!$Cb92ZNcXX>`h=e1m#rd(70?Q(8IJLD_%%%}5?hZ%A$ zpiim&x9ZE7xG1# z(8^e|M%u-Ei6(Tloc0xrpZzuRotN&E5sELx9j1Tjhl#k9FMHjz?lbp#GU6*E8jDxT zm%ncQoU4l0F(&ml$ocAzGec+P_v)FijO!`q)i?5$+?&C|m?{lz2A}bU$!IulFye2< zeEZd#=JSRd#`>y2%i&S4-|HUs20Wbqs)rlX%<|j~um371@j4UIEFLuqy-^Fz^-+|=IT?=H< zlU8jJ`U7dz=c`Am`Tpud)yJ#P1S~BG1GX(z9jS=zczW+-w`07oA8Zai|7bHkPWO-b-NXIz%a3T9 z6^JPW51Hbd7#Khqqn1Ipcep+|Y9A0hU?M4Y)b95C-2+$q$H(}no%alT=(+Y9VH{wq zyn~bdHw3TW6HSfXAq~FX9JL4&qdtM_h?*L6 z^8&GIQ72vYUz@n>>6bc@z$(?&H70nTYwH~MMpFiG*?+w)YU>;y_r;#MaY@e{XTWKq zD@c#+KIx9$n6G&JLMl}%F*Q*eZjYhkd4#C7zn}V2e}6O??aZEBb&vAH9>Et)8W{Gt zg=h*wqnZ)chiI&u+Qx;karYIpLezF(Oc=pyq?LBYk6Il$B*l+95L|s}ck{gb#x%w{ z#x)P`MK-Ai&`D#jge0QRJUVs*x%q`b`UIkB^!fD5Jy6J@a^%yYkPgHSMJ?3Da753F zW-3iG#-ko5u>MFvx5Ve>LyZsM;}ECWpr%#F1VbnMnwV|^5rBBudW>cG$Z+It?nDX=J@BWR4J{fBYYaoKBVDkrYAo3JpP+* z8K0zfrheGpA1GSAI%$z#W)OuOI^EX{pC%j9vt}Ca`NC0uQeoqC z_wA$qXf~u*eQGt>G9uQpkhLsqE&njDB$8Jf%Bx+@5H0wKuNK*Mw?zMr!tiYW6H#d$e<@=2?VE2N8Ltk=)u)ZtX+sQf?cb zr9QSgUQ2y7HDWCeS&JjqijcJ;Y_0matLXKEvwd$~eBrSX2=yY6%s!h;3az zIP~7JcaMF)H`0AE)O}HW{@POau-G*sj=Utc3QI-4>0`@|{D`AE&YGvAFT z$h@@T!NvO*=W;_On<6Fcp_2Cb3yZF$k^|G0u(fcxqUO!UHyY;#Lls*i6+1%}JH?J( z@#LwciqrJI4DpM~9yH!>e9(HoRczR`aC)imndyw>{Mtx7|I%W4i0fXis-7=j zI29_|w_I5@pSjQ(D%?vEPAwWkrTag&B^6SX^p$jjEluiK`5|k*Sg-3`)gX%TC6 z$Xfk@we}y+n9%J*X_ZfWG?@O&qTW{Xk6V*^ccu&|%W*s>)}99LPb`?i?iy^g5XEic zZ-~Mq{1B36f`s2gV;uXUw{)OkCE>@L^iu;%ZdO@xFg$6M20z1>C;`2N{1E2+#9x&` zn3*yLjYB%ic;3RWp>uo^pUkJ|Q{gr8*4NEbL}OyoxF5=1{V-^HX!}C-VArPr^^mHq zhtV>G)2myWa1wsJBWV1F;hQFvo?i#E$fr#iZy6_nH_`cq@rEJBHl_zQbup%ev5d-8 z!+8feub<<`JigIdKgT8>&OJ=5$>dwcsD%OsQhQ~#k*j^;td-wOp?x3y4tieHL>z$< zPPhna!Q0r~A-jwyXWSBXPn_#Q3p7f9{Dz!>;qRZ z;fp$B?3_YCG|5AG*PDftl#9_5%~IZhMI7}G(mY40V;U8R+ZRsY5NOn!*w&ccFCl{O z?{N5YQ zA$A-QOOAfvIQGjAGjbvsTquK!WHg5|ntx%)u;ngiIA6Q@>dkvwe&ZF4AeFVt#kG&~ zOJ^-_roWNCVm6iMew>nAlO;MUR-A^+oV$)Yj)=24#bc^MOK5!m>EC(nLIm^YWy$dfb42Tu`!_KZ{XU?Z7hRh?z z&sGXR$`c;#;pZVR^{mES&=I?hT;U;e2qCtQST*rX=-QB>j#T(cB z#7x>xOoJ=xd}#6-48w|HNM}Hk-?P=eGGKw`*l$Qd?Bpi~40W2-H4Qd8Ls`(I8&iV@ z4pK~=fo_eM$tsS>VrHrI$X9E`)ZlO|rViq{Mob%UGg>j>$6_jIoAiv1Ys5@Vh)Jw< z!f%b34iz;VG+jvs_FN=K44VHN!}FkL$}(lTVwg(8keD1absH`rb)Z1W!IS`63(t~< z6ckF13raOu+3iA$A(+IcQ#S@aqCZ{eOr?B=IyfU1v9<}nxc>Bc?-fK>O(ZVuub^{76|0NErv(-4ddlHf@GDi) zfnIkxMtd47K{`s{2qp#7s83zPD$-}XofS;iE7Md)Fja4K2xjnU^7qDP29QoLHJGuR z&g<`?S-}i2A=wm@D8aSsn;LA{WC*6KZD=%%8ZFv7twEH8AM#dEIrWv=qU#%fKet7U zxBp@zDNSW=piT;dnS9YVEPUL&f%wF6B>dF6!FwH?UVeWDz1L_MFi+X1Y&Q&Gb{ph` zz{Hla*+hPzIqQBMIci;=H{AZO{7rHMq|`9wT%(+=awzr8=LVg`K=OGH^W}a-cq3mh z<-Fy*JujVA77|n|GBiCVRW7U4cK^9+oYlAKyy1+@02c>t*k#b;^u*|D$qx`127>%0eYw(WDV|Y7j*UW6S}l z*zX(n40s2<1N}1O!~o5Y{u#f(7SzEgm5qp13&{B=$AKu14ggvn<0RG zI1d1S8XW5lWU+JxG)3W_m|Hdv))xvpsb1OA;SklUg@J!LGC(K8nFGUapO4|C!Zr$) z+ONTbQ~Nc5aB9B>5ms*i31{}NYO~ya-A0R=1K#myG9XsZC@+vuEt)LVXf$O4OA%my zQJZHJNHOq!Pc+Fl?D32}Pc-*K5Dg2~`@hmWHj|ii3e z|AV9VkIrS@?_IEqAgZWqCArkr_+eJ=-TFKAzuh=%4`nsX^+lTZg_`%hpSADjC6y0e zxc|aj=Ui~9WcQ36C~?D<`Hr8~?*(u>(|PCCJ=<*Ca$dPuy?wzLs_t0I>sYR6j8wFT zD%$U*EVp+@+7E@=4?W5gPoG(8?-S3S3$>pU^XumFLiugW`Na=X@25uc>q7Z;V&lHW zl!eJhsbU{5dWJ)NBjTA+adbkgzaGxN@z?p~uc!WXe#uIjp<&;rS+tV3-?86oj}&bV z6>SbXw|rVcy>E3Hma_JJ+GNNpy6e5;{q1YBM?$$R(+A^_2C=MdzILH$1px~1#}(b4d$Pr8hT;%?(FKdm!3%YR`o+IGe#R&j9d$HAIFT4NrVM_^2Nx z=N+#}epKF>d%Vf;=R5Y(>93lqPFT!8cA8G47=D~(rRzejcc1yk9ZB?LUlN}D#A3$v zPf|JyPgcfb7CD2U)&WbjusV~wm7}CQUjImV|N%ssO zCekhNb8=9u8TL<^k>=VZE!a>=i6>3?@r6NiJW!1_hH1)t%Z&cY&gvSYz9>){gQ?xz z$>+Kx{?W~(@7(wpc)vi?araH{$ixVH&G-YE;cnu5*SzE7#P>0N44`xG6C2I+&5+GD ze?;GDAe|*oVUTLuAZbxmBYt~r~U%oHicNTu?hc^{66V|yWxh02+ByS z3#!H^4r1pod$`LGM)LyomO$0;PJOe2iiwu!Jza z`xrx+z}b;8?SF0Z)yd!7d#~@orTdrOPi;^pE5hG_xQ88(a%MfwDSh4lAaFnMjbNm# zHB{C*Z<@~(%eIK6?PB)UX{#dSC9dP%-nqKDLb0GdY~8wIskD($udH%8zje8!a`v29 z-14|;i&)vdT({|Qe#Pwh`6RJw8$kLB0_h8~KDMP6r%xYSDW>_Ci3s+G9Q&6gdH8Zp zIT)_AF5{;Lb6Pt1AcQ&m>fwlkNej+}9W~1pRny({Z02gnK>}1Mm3s(KGdsEDXq!J3 za%{snuAuj(=FZJQz_2~+*b$rV8$;QRbB7i#i`k9g>`p~KP#JPoiZy!|ZY~Txa)?#W ze&Bp=#cXix2Q!dta2EW+V6rh81}K;EyuRdUfvnWA1!pBsi4)j1|I++`SlSVB>xRUtbPfcBEJ(-3dX4(!En}1li<3OJI&+?LRJ)j-<30lV_Z207lp~bP5_`pZg zt2P!Q&~Jz|KN5cN)2sgAi>EZSB>b+)6Zv{`QdAUaA`cpWEj>s=h=f$>_PS9hd-^o5 zA5Uf!ri^|CYDP$%X*g)4sZ9=#FgoG`R=#GuTW|R+Y&s%^TAkD|Q6+1EM@$VVeg#k- z(h()daFgBy4pzn*V`YfZSTBXPpb2!?*~F|-CuY)foe5HJy5>_g!X7j4n6mI`e0tEt zXYfuw6AFr1_@5m#>&A8QIWkMb=kj@1OhF6SCn#R>g|C}%RdmI`7yllyQ6;4Ai2vN{ zNw5^C;46`86<^KHHMp*QJ%z=rV@y~*i`~FC@=bg*-}1VJZ&k0anD|X#%G$t`CGnes z=Ic;ug0N!?V{SInaY%=`2%35JPVOU6x^8E2Q}L#qVkv2CjWI?X9wz*Tbl3&7S5-nV z+y;{#ZjkmelLSywx0(`u2dkN#3R@(_QyIYZ z_O=ez+jb@PHssp4I@h3y-~DipTmy^^)XS%NbxeLQ%V%Fa8yGa5AsD(v4$l}tsKh>{ z#LsM~#5!O(-x*BSVFZ(JB{N14XAL8`Kk%D&8M*0X8kU?J93W#?7;C7!Jv_+C908%P z&+)c3*>QE-3o-vK^>K%`?$J5YQ36c zAjZ{>@y)_4nlK>og~ag@EdeVKTq2i?+HoJ#R zALAdJ07TA+ZbfxAQ^u3K2>TyZ`jM3C8A(rgaAFwZ1Bu1t6C+B=QrEtWgsw?64f{L^ zNv&ee6N-xlM&F@2M;X*KLGDE=RI}$LZYy^wflA6XR{r5(IX>!-Ed+quBM;Vu?qR^i zqYS^L#g(R-)to4;O0J8dc%TuqRs`wWll6;u4}sX!J0;y_ZiJL$m7-IcBiGPjxaM)L z)ju-cO1V#RH8pZ0D9_ZEP+|{yl}mx>1^#u z(hxT~st`@`j*t{3%{`1os;2`0Ig)vj68j*Zn3@LycFIbsn?Mrla&n?JR#hk6{;Q;d zJ%VKql*ffX!`CmoM?=8D0gQ18os`?Wx0Ux?ZyiO44n8D zT=7A#o~_o8Qz!CC%3y3nI4O)>_6Yxm0@~QRlUCkxMdqk$Q_W&OP&|qSdkbm43!{*5 z`C7bVtr(A_uRWUFLk0`b-tHaiL2XRDj)8S5=k^ED52nX>)HBm6!+VQ>I3*}@;t;os8X59vTFdP&Td z?70bqhLX)lGf1G44%l$Yhxg={P z%TV67T;BL*=^Lfo0jW0Awompiosl81Yo4R;THx& z(;*Xpm1>etwQpN)Zdu;cw%pSCafY?A5DMFMEz1oQslF64Ke}ndO>ODNskWNJkE@HT za#sw+m3b?>4du;`i)-idk<{kGkCT$h@gj-KLtAotm#dnVE1H*^H$kntIv4qJ<7O z@^e>G?NxanA2zlE+R5o)&`!W;aF#&Gl+m#4EWY<#$XPXeDdgM|adw299Sa?cBcWY= zVdt|@M|YOZHiR;ombZ5JK)W(u0V7KF+mnIFrcG-o}v92t#(qE}5!(JKk6g%&+bDVPkU zwv+)VB|@bQb1ak*lY%LOraPwFRtug@X^cv5*Fp8r!VoK(jT3&l0wK*>ck*+@L8{R- zQ%Zy2XY`r1P$xb0#URQcuStPQRiM7~sMJwRsxIaz-!Dx6>pug^0p~51z&G^4EagW$hYuDq~l=e=-^gE3PzuD}) z&~%>g-EMKv}$i!v+HiAOf6% z@{mxD7;nOX^;aPeSc+;e4@uo90r1eK^Ax~C8%)W!6MS}uaz|{AOa{y_zC#p;KbR_)r1DF>|H2Rt1dirk0wSrh4aNzbG#AlH-PT@hf( z%t=cSZj+L6Zy*zB4HEUKR2=}kLdZxKM;OB(Nx=vnV312XtMC|j!nDr@g)t^ygnW=t zwy1r-d*Ip?fno)+4-j=RN@9S^6PF%AQ9FWqq-YRckx2~WPopkIILXMcb^`YmMV{6LCE^}#(cb#Tz? zB3>JdQJ@Dt;U6d+l|?7UT=)-kAk8*`vC0JHMRTNN;$(a=emk0@5#XUfF*>28hJ_up z+!)(W?6`*MPL-JM|BIe7&O3=UderJ39wrXi7c~Rallbvrth{iG++?+ShR?hlsw`2bFQpt``n#$Uj zL0C6iHG6Y*NZhoaaaSC1SIgP?aSBb6XmNfQiQmM6!(r=@6-$k+;BPZrk8A5hXVr3b z8;pG7>4GuO&G{kmnLe?cN$Zjgbxlf8h4G6x7#EpSA~TZJJC`fBuGkD|;3bnFYjiR` zuQ`<6Ja;vmy{lU)rJ7>?qoHUftLNRQ2Z1?IQL5w&X zLe7S{9bsoX9!H!t?>lSeU|H1ofwM*-X^SgYc0OY;3!eC@(Zl{Q^oZQgZgv-#2HBwWXNPojk)R(ys}f)bml%UXU3(d6d|3{~B29bu(D8}K$h6Z$cI zAU$zIlJs7Clo&g$St8Q5YePual88|l#M||zoaBe5O)U0P(v+ztks@OZH0))Z;B!r? zkxnX0 z=gTsl7KqOr8)Q87XxCu~Vf#YtPP&$M5?|~0D6pJG`fv0EqH)Y7rKrn4tz7{q{6;wu%@RALZ?Pcba5;tYwo6#gUL2>*!=|44^-=T>BLg)*vQ44PyEKARtQHY{i4Njx0Ab>-hZeCP1JtD>v+ z3$kc3SH} zrRq8C3p)5LJ`3-3G}rW>@>vhF-;Pl+MNE|*w&yB9t@0{LmYcvjt+y0;tRC{I2DI3% zpU`{u8{%RYnv+%=^Qm=A?Nh+gPM%8NK;0Gw)7MqE>0hdDGlCgI6`EqjE{AAgnedbQ zJTvTFZ`~G=VK4fX^6jIK*n>{6%HwOaYVlXrs#NnOiOeW+tw|^tZuwG{Ynj3VC-$Ya zhsgVjzLfep;lS5*214uir zAvGeb`<2w8hSli-_x>R>$X;*$mBGw}??8Wi(xbg9m^sw6nuhrF_-cQv97gWrkRZ|G zP_O-@QD=>M9W7iV)_)PL&K6>sw=#kIo2PKTVf1g3OOAH==7!p(Cb3=q|6GrXZ>w6j zQ@fyjjyc<8ifgUR8eU^{4jkQo!{Zs1mT}}~la_Vy3Z;Elp+-uoSvZp7D8bR39F!cc z`bh2}spmkYY80BIo|qXuX_MCaIF_sGD?nN6_Kx`>*Laqx+>oZxfPa|tOPwquM zK8Pe>gye>T+#q*U(L~}pPaY-_fcyHGR}~1I^;|wPHgL`3hZZK}2|P$3pH*tE9}$WX z*a}gIzJHUR@wx&6f&|%3ykUsEqW}bT9e%?BHqzn#uz; zXk2ExCMkr*qQDe?)X(U~1#>t-TIK>xc9QI-GNzD8gsc@OU~SAzp1OYFGQUB=j=FvOa1#dH7_H1RA_dsZGE)W?KLF&6~_DO5>ftyHPAlO>r?k}TQq{Yql1sGSuaW@hlu zYaEAL0Th{>0OOWvR5K5&a7uuVO-j?$P(lIaX_IPgV1yU)=^oa9@H>awsDoX_suR9^ zp^PR8rwFr^B~$8Il2QE{-UX`k?9vjQLe%4H@r5BV0PK-v*t+w(hu=B;-4pMe2)Fly zb73^z^TySs6c$~?2k(CneX-e*2`jh&va@i_uSuR6ecK(^@SbJ!hQ`h zsj}(^BlkzrlwDm>09UfsU=dF+~T{J?p&JPxs=m9ZT~Q*i0?K`++y)2 zv8Zj{CAzl3`DsDzL(6yUZ`t2=%-$07woN)cAeY;PGrdq;1kSe4*=K{|mh<8TUOev+%Ll{G zD@X(`3DWPR!<#_Jxp{uu64cq;A?H4vDYp7%=aO^JVrIw*|ABkRv!IgUfh8x^taAs> z@ln=JEb$E^lH3%|cFz^gImGs};*#U9jnm9ZzUVBM=Nf0Z1A2|KL`s9Skkaynr zt6vlvs*W3fxpEVt!@@{L6Lju=VX#YG^U+7>Fs(!F8FKBX?DH=E`&znlL~ z{=8i*-9=s;GfKZb_^qoksV9{VDtk?H&n-FI;UU7g6LQZri;Y@Dhv|K_S1f9VV&5DL zD5(t`yKtslb>2iA+eF8<&zQ#r+bXKN7CRSVsY+fQ{`1hak`1Ds__{F;{_EW1dCC7a z8HYb@v>z|A{^>Rgjz6e4XgFSB_)%r&7M%X7BI8V<`LEjc?>gCL7TX-S5%=uRK9gf! z%1Oda9IA)GQbP5nF;cIE>RmCy3X-rt_S193pbDRW&zbbl873Ii5vC5M%IT_SEpma? z%6ek~8H`^yZ6L13(E;4JndDU#(>q&l{^l={zeX4}nN*(~xU8_jr{R9ft=ePJbj!&u zs?gnw~efR`eC|B2s z<0p^xBs|^S{3U+9*g_yjamqys<+5*orWh~j-|Rr=Q5AFQCF6I-AHgqu%!Pb3O}>&C z9P0YSYe-ziuj4LIug7JOlOx$2PNCxtu^Qh-9IzT>MlNXwub3@4_G#Ov=>}Kf-92~q z%oZ$VH^RIvyrJWo~K?llb1Sbzj^Y78Z&#w#?g- z*CdUXprY3Kw7FNrf_-7@GYOGCuvQSmZixHr6)^uCIIp!ndp$5e;nSpq6Mnj}v;eHG zE{AR`%}0q(nc6GiC++jXTNJ1&Nei&@$r>MgIv=t8k?Tpy$tR1(ze_^CbgN4-(@oIA z&49U?M;B&VoCY)Fbji4x`FF{JDRFkpzsq`aaKX4Whe3h4bX+yyIaVj6tBkB~%5Hj? zr)p`S&eyF=p#tp8PmB(nAC&#XtR_&uF7uhJz90^mU4P#xWU-49VpEkR{IDNe@k3=t z9bX4Zoe+X<_0)T@s}5;%lqKY`c0W}ixww8hnZk_b^;%BX#|0QK%=U2dk6Qr@l6r1gc=TkMBS; zg~A*m*{_&)tNVXJM`?zM*VdmyeK3Vj3Vvb@ydRx7igufTD5K} z^g{BRu3;t>y4J{1kNP&fbdclbFaI5W6*C6Eg$Y&L*Ejc2J5Slw-2#U;%eASV)q0&u z{k)J6Bs*fpG9*ZLz;AM@B1m>9f@Fu>bC~E-%}Tk_o!z#|u^3ydgjFweV{HqjqII?h z)8fC;HCkL*W~iz1JD70ti}4UUnQ4J0F5khY3>QwNV`kn38gxcdP`jJayoU*%_cFor zJ_w#OHbC<;u*KCeGx9IaV{4!t?#82hr1$I7HA>U>TQpmD7Cn@Ll-YkW(&U#bQ3)}UW+L_879 z+h}Viw%+IVbt-B88YRvDQY9_eKuJ#q3%+bgk=urO@(oZJze<&clDMPptoW=t>U^=h zCR3n2T_tA(;duR;Cb+e7DD})&F@-G6ya>@dbn$OlW|D44q$^bmI;J$?Af*ZPi2Z6h z@)h#Gwr2i%U*M_4{MV=jy||}gD%>sStez7_*)P!QN*x$)8|6BPtK+T>mGO*D88_S$ zH8dTSX_AHc2|ZwA)b)ME)cv=K)YU&1_y@Zz&13R#0`DX-I`hvWX%mouAJTP*yOqFg zEwZm1xQdB;vsrl)FP3Mz2athEK61$2kfeB^O?68P7t?lNN(!Wmpb_K7NuMQ^by@7L zN@o=c+d&7pf`>kV@+@$&@hkoX>d*9bp6)x`dyqTc*?YdZ??m&7-W~;T<+=}d9_&4F zrtfgq8G#nql3e@-uAbPat~aClntO#=JP0&iq$D;Y{w3~AU(ZRd4PK;Bex{WG7hoj8 zmNn{`J0IZWr{=(uP_>Y zNNK!92lBxZ$j~H2@dG)Eb0}>S1&TD`&L((p4P?ZuK$&~gmX;QQfE4DfC9WP0bRIs| z(+x$4H9Wxz{}V+cEz7qgZ$(TxA1KjeO(eqc33f*7;Kt#N%(Ewu3g=L6>>KR$Pwoj+ zYr=MoO$=k_HrUi+i&pB^N=$(OGvP0A2o!4~uy)$4X(xC*V||yjP_%uV3*>8}DfN%A z&=qM7*c9Kkfi_JzT@PGjFD*G=k*s8)yP}DrTFOc;Wr!RukgW;9GDlQc4>MA=jV7dm z)ef|1Ldz|17`wU-V)qR%se7<^Qu;icxz)g#+o%UgLcIvz9ONIHS$Q-cFG)oN2PK|& zocjF<3>!>R5>rfoXkwElS5~iE(C&zqpp>KRmf#EIX=0oupT+`FM*1sF+-VOfL&Wa2 zO~%GK^0C$t(~AidXp&)X6`Gj6w2%g}g>*V((1EnGB*mR7O+;0fhACI5$~1jKCL#nf zN%=0$#!*5QC%?k*OWXVsB|A0goO!mZtLMy_4vw0#*+14ylYlI{(P#-t zBMKC!sJ}?&-P<(D$X)9sX{2GWr|*vGyKv0YUNhGt?}AeM#9G_LUBUa>08^`CvlAEO zu3&EusTt!6s;NyWBsm16QfO;K?IE!_Ne&c0A~2mOJ#8fI_?D=c6!#U&p_bKGC z60XssNw~oN*|+Eo)6@D7bVUkGLLnWB=s?<5LNOg)rnl2{D4|1`bj(NtDL(pMCfeBQCpX{=ky5u#2C2GuR;DJf#P;-175vV3IxAoFXh&yt+9 z5%~t{^yH-S7pfJ}J_NYyE448(sq^zBp9S4vYfpmoGA15Pe>Ht(3Y+@Q4lY@nv9+*u zGtQ8u=9aDBueexrVDaW6?EX)Mt*2KkS53CxCH$SWTk z4CjrVq(5~|QmQKLGQr+4t=mMpCtubdxuP}0EbLyHyE!)`?l>-%^oAX@%?R{fG<0>4 z_prqiVhK}X%g$TV%M|JQh*c-Vljj}*Spo;QACFqdY{*x`cXKVyv3zY;5VEp#rp zzt^>}bii0i8bdx za9vo*FyvN4M=`#|Hvhqv`N`kkOPg(%FCJR#5RdnZB`<~@?w@C5&FuWmTeHUZGs=~6 zNX>RSeC9dv?2Aih+~VoWsJQ{-Vd{OAM_A1rH9fr@4MkPo?tb&g8%Mr%EK=1Ts%oD< zJ?|E)c8HZb7hGcgUT6T8wEschcQ3qi;rE}Phe}n~^f74U6><+s?w7zHenH#xkq-;X z=R4h9YcASdq$zs{I`2f74wTP!K ziYH^np$Q>&EgW9hCGD+~@MgzC@j|-v=4ViMwDcL%c0Dd_fj#T3w{D3odl%e`_JyE$ z^s+cGCSD#F$6rSMOc@VQH3f|a>2%NpPj?1G$zjusO{rgLE8l~UY>O|8NXxFmwUaAtLUQawVjB-@KBVr&3z44ZLOst#dY%vUJipZQg4o?J z_P;2WyTi`Q7!2aSyBE-_KZK?rZFto7JEVq!8sfh*`qrq}yl3ImLSSL|QHyx~vUqMl zEaI6RuJ*H)M$Yn6VdrUh7kqH*{;g&BZ&|YzxE7GjWz9Iy#mgF{E#xDm+e4+>mr8fe zq$X5x_u|FH$-NkMW26{ea*v5Gq7^5^>g!?G4GPqIr+0Qw z$kjGqzvSADUE*DxAGmfWe1?j`s~-)0T^e2{q#=60iRr!8cGD-5(_;b!m0RZ9zq|LH z*r#}8akqHFBXWZ?aGcokX73xl%lS0i(LdD2EB9ZKPbKB(1Jzql`s&=7<8bz2((=6)apz$S8Xb?C#OH>^ z+7Z!JwcNOsHnLvMuUk9YrVeD=lqO(sn74rQz=Iqw+`DV#V>W z6RO8Cg<9#&zD47E)_1K70kPsp*m;!RLSZ&x{)#xahMZeJ`!Ks0J8i5={8{LPi_e}D zE6#_V7oM5`_S#6>Mf+_@TXt+$h9pUy71PiQ;`9AtDGlEA4oau@_Jkbub2ai7A`wSh z$kE0&?b!arcNd)3pSE^q4wM+)Gj(q2FE%{d+>?d7A18HI;q;Sw`-_>@pR`-(xMRN= zk3zeTrN5YK`1i%O7t_uEzPj{9oB6Rh8805&lJMklx|ObTGX@IHkE=Vm%cPc3L8Tp{%!+)+qSfVkx-^|n;*%Bqku|)j=%Vl`OoPX&l|Dgt#Du+&X=%cKsvZZl zM}CHx!_3=M-#U7GgS{P#sV7?|ZLpYLnT7WG!gd!HY*CQ`%`j|v0YlkrrWxjfW>^kw zG67V4y*gOBM%3_SqwI&pr1mhpPyFfWHmS}ojRvOCrDJFqwCqMwpinVGR#OgIv^(W8 z=Zxx0tSvIVKtI0zyze){z(lEy3hLjjZGL z-&+itou`#Id=b+|*}sM?LXX;Fy>!+kiIDnxiGF{dwDYCBeK>n6g)hS{>ZWePjPcU% z#GEErrjoykKFD)SQ>C2Mcg6bpG)iWny~3dhrrY1pa#hR_x0*>1JKblo#?=t!7d%WvE_hdgUURip^ch@UB65q+IAf<;xz{D;gAz&n-zqOy*BU{z}T~ zh3>f07%M@d&WnBt(!Nv)(jk#+Qt6Se^mNkXk!eGkEKRH|&1~=UTCF}8%;Ym*hFrf! zy(oHNGDR?Am(prseWNv&qQ+F4-!x^#zUkx|SB<1x>DIKAVw-lfm@(eI3mrD=np$Og zeQvA%cg)Al4WMi7nQPlxU-BBI(W{^BZv$ST9~-OJ&TGw41M=AvJsmR0XP;YhzjHlG zHphA}?Ru`!tlU6ZwgjC+?NVv9$M~_4S{xEiJME$lzi@veUBT>on;Zv}w{D2?c74j@ zb3z}Hs2fzhAUh$?}v-x>J8E@N^`ia?+^fN%2XDv6R} zW)4ayRfwvHN+EJm?LSmeYC=*<3aY@A?R!qSkeDAls|rxmRl+Jsxb+zodQI6-!$k;k zC1I{Ac99w;W@pV(q55C~B1u|PuNRA;U_~NB!XJthNLPcNr27meE7wSD>JtsG<=()k zXwgVlge|DB1A$E%(GBWEG9M>tBO`!WEXheo?gAjHRSGFfKoZTil~M)@R|l5uxC5=~ zJ4s3cK?nLrqMSobtVNB07Frcfm1#;cC`tHK_4G)Roq1rh8cIj}qC*vI%8L^vDkRDf zwxe2w9d!5ux>a`pq4E541T0J-qRKwj6nk8wq!fvVWJaFJ#2!%_lWow)2B8a`B=LqK z4Uq&LEM+n<5LA(eBHmC%8Nwba2gyu)nK;NJopNTvhp0=jzv@Rt_4~)@6sUcwG(rtH zf_dL(OVSZWhy}aD);$Dckf0$LvR2JLw`6UaH-@ZjIR6~hT!fFb&AOLdbui0Dz|>b$ zrA^rH?VLS7dvxxWWU=+wnKhlXVi_^ow*4H)^Uj~Y6msEe7e{<-Kp>GXGYPN-H zw#^F*MzN+ttllLS?Vhm$C9kYqF5a?Sj%_kTZu8^1?PBeY<)(HRSeD;51n4ldh(2#-cb9 z2Go^b9LZ}9<+aXR7c0cP)^J{rg1gm*T(x2Y^QR2^^1Ab3*9E|Sh*QkP!Q)Ya#|!Qr zyK`)|Ztjp+zV!pwwiS43&#n3N7%YObXSV)k;9lo%+)@x52?)9nz6ex~U%KcKFAR!R zSHjM#GUyVFGxvqrFQg8iK)_K7W zriDzgc9&SQd!bt_?gX4$F3Cam|G@uV;N8IYgRmDbdltx~r8%?kam8jW&c`5kV2m$` zydVlw0196*GGO?Y1gal49U+*w{HO^)pp5cM@_HJ%}~wV=_Ab;d6pCFwvEKjF{E3wlMni4W z$Ajohc}gde@cC2W(}3+RF2SedsY~LeYhw9u*f|o{kIpYSw=ck=es|cpoy^7+19!3F zP}q4mF09tX{Ff_bhWv_1UVA97{TBvHW}#9sl4R$M*!R462ATGYFS^D4%VLcr42lVg zq(J)@+uqypF75YHb3E+oT`sMhIrzW23O@TVj|0BBDoqFzUlh-}MfZTnd&Gf3v4$Fz z#roe63%j;t7Jl|=ehhbH2JIRHdwAFE6Hy~b<$4?I#W+JU<2l4JQ5omcqDGCJjEr23MgoZOTv>-wVAH#Y^S%FLv1@w zx0v5=Ny2p;ut;uBX<7EkA7at-|4m>~A5w%zOlfZ$xU`uveHF0IH)(Gi?1TevnMirc zLOZ+wx`YxigA=n+%LzZ#KrRNOD1W;i5L%v`kP5`{j;BIs1#%4a%%|z4 z6TeJ+>cTsHg9~rH<$s1g4e@>f{R*_G7n)B}Am|hgT&)Y722+% z`_oiNX&DRv2KKKBD^|;))H8OHCO}aEdq0PoHF_0WqNmgpZKHS9`?b(?QY+F=HY zp(9G#Si?)svZdFA06aBaC^pBGq*YBwS{=dx8oYg}3!mXhC5u4K>LvVXrRZuhLL(X@ zVIBVw4g5kEI!K^^md?7wJ;{K|j3Oz!~ zKTxcaE+Z03`FUxfSE4K;)fSe+VzrjNq19TnuwPPjB`;k4Xh(9U6)UWy7RR*HXu%dW zk9bGno6SwFA!)9CjjHcA=s+vAdWW<$lhn@0sn_q~Y1Bq3GcV>*M@+3&b@?SZcI!Y3 zNE^r?tPfjdLT?SrE6Im>G@Tx*_tHAefuxo787`zHTE4nOQ_DfQ1sb2aOp6cv542Qk zhVmHPYsBk~i3Ocu>;BkMP0XteS&6x^5_4nSiZirRuv*)DXYXwJ?5U;fT6o_`SgoC# zJtDRo6blZ8t%rYp@YthsKX_%yb@ts?7RzT|S!*%ze$Jl7SEg;E>n!bhgT+I+t>b6K z)zFhDER7WI3>EHNs9P%Rnn{UfgG=NhE?PRxSwC>Kt)v)AYM{P^RnYFG3~sjVeaTZe zV?w2Mki^1XUOGJ>p5nzSPuMva<4Lj0PRO}wK6lBvW1%zT+=DX~LJ8_L?SF7)p?GQQ z0kQqy;)JxTASxlh==Cl4f=fkBbMB?QP1A?u%`Kmw-?5apd-~ANimT$ht$*NbAYbd% zkZG52-!{)4TjJX0+m?#AP9OiUq>^pBvwNXrspQ~v@6U=G^rALmVHmH^1XoUbVqB;Q zbt*^G-1#L3vEq*1VaN7g{i4%o$T^1{XS$^wXH2$^1UdSi1^2=kv9vSn*dL$Hb4w2L z1m)NnakPt$cE-+jJn@kj@5EKT2$|QNnGfYVRh{t~wa4JpL zY$PUydwJSZhd@Q!U5S3$leQb|64*Qm?6LAxF01h6pwH&5>;f7bUH~JUFygAgV zG%MVotN)P>Iym@&)%$KplM5Ay0Asa0!;{$4u0HK^G{28gMmy=yO9!S1PSc(`zX|^i zw>}b!(ZnMuBp)T-p;>QkS#w$~M+!ayPBFXKv9$ySt}mzt{ACq-r6E(7KCETkj}8AH^2rhU=Pf*n&2_98fRX9RPG=^L8p1)Qt87ZBr??pJA)u_4ZS#rLfz@ zkb;`1^aT}HJ_+rUBD6xFuhW0{G*-vlhWkqV=3^Oa)VI#$%+sC-ht|kXZ^D#HoVorS z&^lRh3MRU4x3sL!q-7)DICd|w!+lzIAk|>F){S1bBbemdwnmBc#u}+auArUI`G$qh z-59NW219uC`U}mj3BTBW&Di-vul4mt57b`1mLg5f09#D_@!4<3Mn~g1M`i6eDVSzV zxBt}|HK&KY6thvfgm$0iOSM!Y4F%6qX^d}UuzDmxWWOOUcA+^9RcLNvmtYAm0v$@# zt3AGeeLGbfpkHeFIHg1|ecf@tS+y;jw$hoVFbN>1hWvXq*JeJl%HRro)$a8|_A zP`$Y9VPbO?gh^x&NvIAXI~>>pMetf1o0J4+w4OC6dQ?t7P8{ zx9BM`yeDeh6HPiz8NqEmxINxWKG1zI7c@0-Q48%y4AW!m55S1&t2afHC9o!%^6cof zQK*jKH833yIVcSzGizv1%tn~=yRQ**JUm8LdbGE|D6AyMu?qmgvI+nw6{`fU?uogj zZ7@*ctA|yjZraKk@2?ulB4#=N7FX{4)vccsqUA|CNtIKD2FQ^Ua$~F8LW(iOi67VO{cAg zm0Mrbrd$iJQ_90Q_-NW9MqU0%4Ez6yr-2$hhMk&&Az)N6mg5QH)OVvmUo!wh$t}El z{?7T|zBp}94crMt@|xbyYg%q-xtFnA-*_+maedQw3g0S>)b9<~@4a_;xws)x+!iWs zn?L{E7v6b6Z109cYH#GgbD;yzi9Kh<&F8|!=aem@^5$J{WB_++*oP7n zlt>a^hIY5m$4_Mf)MUwl5cP0IiD39@O8j|3>4SJyf`PCS^Ih z@ZQ1!q$E`?~ho^L)P-yy0EozIm7kKPxF9Q zMzX6y*;SG3worB(|$M4*wy`E5vK^lB_y$v zZNXgCBk6>Ewqm2ip7=-$^IccB^H{Orht-``IK5}@EW_zh!~Qg!{<65F*Jb|8oh`i? z=D%tx#qq}(Nq7dR+xYmS+ zxP54r;UwmBTxS)gH}}sNh5s8y0_zOT)b9Tds|;F^t|md>DX9fcQg!%ea6UlzVZ5#4dfIzhZ^wQPa z-4Cr|-#`jF(%kqJ8x)HR*GNy{VO1)Hy&EWL>iW~wAeFqL@6;5xU z#)?&HfJbdzo$uSRM|=sNUuZy^a16Uzf18vtGbD|m<#xpeYphH|qiIMnbx>*5b3!YK z8>ofyFOmc50!xM|yVn5s#=B%{fU#7_--*x|v=8l;->7Gy48~ijd=1IUF!6^N z)f8iNTU>6619G7OX+4t|3u~HEzE+c)P6$26h=@_g=v#kF*CT&@{vn5<2O9kL_P+)O zsn5-HuPy=Kz;KUmVLWS`a_~)jbI@_c%v?b0eLV+Pr?A%76HMb<_*VS2JlynljE7@; z$*xX9f5d4^+)I|`mtnGaN}MXny8RrT2QS-+1Ee305PyP15gk6R&V#pHH;Q}}UJubf+KG9%;1!zbGPE-pkwl^lWR~+}ruRYw2tCO6Y z#qjz@YyAN7fMXH9#PAL!uRtN&n@8*6kvnkWGQ%1Pi8ynpdus1>z^!3EizXRsbx=l>9QovfQ$ob;^9U#h`H)njJDPym0n-^6oacGm<<6eCuG zzOap_`#NgTeRPghP z=Q`j+Uq`^qaTf!oOPsKo;L6X?%7HxOCcOr3Isvu2m}yfWroAP2u!o}Ox(w~bQsR1r zA0W0M;DBv}F48rvd>OzcQ69BZv-c09%6)+{wf4~^d|VxYY24;b9D7~g;r)ky{^x%V zm^dzw33^x%wY)KiF@X{agjRA;`6NRqUq>_#$^$+xobQeL`=y8d1H*3bNYo5ryWo*f zLwdjv#;ENoV9)+>xBqI?dI*;%>B>U)0Sh&ja1Jp7S*!ir34cZMsBZD1c{;@-@E6&R4}1NrBTG(Ne@>~Tz{x56g9?H`{egr` z!UP>|&|xbM(K1xXC_d7DcsbRQgsSeRR+Ll~olI3x0;36I7RKo?MhPTKA7eC0Dg

Kqg3lS4dmOg-JdhA1x#$lhZm*P4o2h=d@8A3fhva#++j`4g1qR7L!e3nq@D+sA? zNrtLOMB0BB?2_;AAt59M62WLYWK7V>gF4_s;o>&2{1Q7stB~6nm-@edLYCU2F2<_VOOFpQ^&&2$gb0& zU8lv4Gh%gL*!Aq=JdQT^jO4YvpVy)o{Sc%qHe#>N_cGqih&*#P^vv1FGcSamc|qI< zPgGuU$QK!!3=K{GBc!eHwDhrbVs}TRJ7!KbWIRfsZNo9se}X~Kano`^#e=f@Ws!nS z?-y)J$gFGe@OvlTJrU`?80x+l=^hMq4~kt^#QLj0&G#ztwuB3|B*fdlxbwYz@9v9q zT?%zwigfXzE?(U4AxkgeWf0_($fFaxJ5J>JjK`(Bv(nvJqEaxPW5k|k#9lCcoM))m zw^FD;%FHOK9Q$%Y43VC8*tlFN?Lk_;et zz@E=X`d$q6y(q#p9GMe&Bm7Gt{v~lh5bJ#(D$>hC+}-eG2Z!B9n2SL&60 z=H7G9_HS zQnvsXR0N&<_G@pw7O)lEu@!_jylJ2(VbN6;%rAWZ%(XLt{K|#=$_Y=fc5EMzv#o(g!pcRb!;Rn67p$qRB;Ww3T@p!Nm1_Ju(03Ay$Ja>wpuZTdXhIhiq) zvyi=crUY7mI}N*nRa?{=RkeYtCb_C9Q00-UJW6iZ!5Di*BU7G_c7L#YYAY-dw@J_; z6z{(4=zNf-D_H*k=32_9T_D>EChPCm${*;RDMy9-&Q0V}tYT_p`mnTS`$Xq`Xh0gT zH3o9p<(zi1;e%0J^0nlkvm7R1L4efoFSiih19Lls$2xOv#zZS@_@dt6+8wiw**K}V z9d7_5T6kFa`nzbD*)&rk6*nz7n(=<3LBCsP&deN{y#S|_cdgxDez6DsyqF&_d<&}W z9amQHxT>!9fW706y@UAy+bMkYMUhe-e}K{DBa|C=>>gIW9p$}(E~^lZ#xR`Wzd>%e zs=D)9cR1iUB0G*y6_oml(Ae3ifoVGovg3ZcV5~Qb0JWZgwOzKhQ+bqm;(HHV1k?mw#~$j|R}_Hc?AHRb z3#)*lKrEzpjyb#$n z+^TTf{Da{|FfB*bWt)Pv;07t)aFJNpi*kR(z!&i#E8N37N(6)x~AHq;EL_%)vk+e~mmW&i2wR=1qE<&IruGaToM1nRLrV+u!0+ z+hubU;6fQX8*2N=L&HiS7cWZ8X>%;f;truKtVw&>6wZ7U)XMhSMHzy86x1;n1(=JW zjaR&qM(f%OVpZ*g-6+!5v}3A2PGR6*?2&R$ESP(lZaqyyw+@mtbn6NS2Z%(1Sj<@G z=Q)eA2Ip71=Bnn_NUpAcy<4_-Gv)eT;mKpt>fN$6w&d%6nxEH}Vfg8m%C;24$Cdd= zZ>N}$qdh;L!khg+JU@T>_0G>woEf#$@md?*UL%{SRs(G{v3WEua;6%k?NqH9Dg`er z=J4|R=9SY3wU#AvzK-B28cHy)?N9MR(GkU`cve)p1W);NiD$`DM{&9o6I{w>}U z$3?QX_lrF=<)5I0ah_hBb$OlwAW~zrWkiu?7)X-2qza=DIh_#<#+|IiUqWl2+V0Wb zSzO%VV!Jxx8KN(S8xbR7K@r?>oE!awG8Va!p6bx{jgRap7)w&DcF4HrSs*|dt+IzI zBaSqNrdGm{6cC^$M@osr{q2%7c13Zd%-vX0ao`_T;gMLqSMS6)qgH@_ho$1}3yvM| z7LSG2{~T22)?#$!+Ket1QnnzBCHY=IjSG?S4 zmF=xjlY2@VcZv2pzupkv#aKnTP9nLCUfbh0Av>+kb<%)Nh1iS(qN@qrg#~=L4E&-) zLNlVTm_rk`tN$=!&js}~z;G|oA70D>S)7c7qGkCUNUmY|Q)|tjwL=2Q^{)Wj)r7iw z*WMd)Jw@pB%#D`1O1{ni4QcSMBo}4W)@K)E7P=hD_6;Ajp^!# zhXx1wMoBTuuJ`iGG)1p@a)2V0okrL$P?W+mn|U4)DfXdje+IPw1iFYGW(fIoQ-L~! zcd87O}_{L5uUu6n-xpsU49;J5t|+2 zI3~xZi|8~NmN9lKVu`vmA~6Si#^JA)4uqbMW9ag)gZlg>C`jUcMW){PSn5|c?qdRZ zO#rIg!2A_NHmRIzjuq_ty%yL`CydzCDtI%>7=Z03MsBJt@f2D@1MCf=8Pqx2K2*Di z_|+70k`$_hKvo;_Ra^ z+UJLz!z3OIoCD8huI4I%UOWp}BlR673J9<2rV&J-1tUd15Pf|BgsG~Rp;IXNR2>-i z?)PtF4~br$(Sf3%iEbFZrmOW;^1OsO6a9D)c%JtG!$~kSvbJ_ouw~!_A^rILsodN- zOd{d>ik@z25d#Om=+y2IwVT1!Ov?Bx!-TEB*Ko7T@8aXF4(#6NyD4U_)3dW{=e~}% zRzA{hON4fbluaRf4}gStyt81`Y}`Xx45k$xIN04Pe*xTZm^e9)kNP&w+-A1V2h1C< zO^7!D+6Y?~vZ;BBwRy;{W~k!QxT7I^}Iebb6ioW6MfnB7DnGb@lTt4T3)RLhh(kALctG2mFZ30g+zcBo?OZa|EnBeC_aiMkf+@h&j#yS6M)(ju3%%$KxD z_BHp?O2Qxp?mzgHblNYS`i4|8w%{25R~Ri1yfg5JFH5c(Iiu!s`@IN63@2pEO#5uh z%+|T&g@d5!<@tky(t)$m*$a~E)dl-&Vaj;nIz*b2>UK%4-FNMsVb{o9OlX@uJ=ZsT zocvlB9$l~>13)bb=#kfQ*V_6e1O$$y*d2?dsF{9c8jgb+X7@|hynuC=Y~95G6}ur$ zm^GSu;w#1L{c&q?Tb=H9orSg&1*fOP@JXS`Q>pu;#Kblj4Rl_a*Pa-?%}DH=2^@Yd zaOf|Q>s$RQCPyl(u?hoAf(JOH#z@Ar=vWoCR=BQJ#S4{H!0cs*63+>OOGJy|RWnLd zfW?}aP@};t#!O?N4X%b1!GbyOb-)}X%vv!j$^myGG5V4#>y5fA&2T3hUT11)f4mlz zUM4oF%|-lbGF1HtEpXX^5z^3zsjXsUvF=`We3z+>5h}+B8Q2JEI8jl?N6{%QJ3cV3 zj20lSR2p@oLJTwCu_QCsa6!aHE3cWc^zd87IIi#wf$a;-q(iu3;%AYJKT62hzACzy zu(V>jo*EgIt}A~ROO-pxUyCI&@fUdP=y&$^jhyct46}H~ANi6PxzS9vON_(34O0c6 zDFCBqH@EtYZm@5+AVNIWJu%|edViY278%!v5J2b)hWHnMj=wiFfZfn=1E_PM@98?U zVd}O;@W$P!D#oJ|9)tWhbon|ZZ&E^>kAapWqJaO9@^(^k35miN8J!`s%I|n&a04~V zU>XD!Scb*s7Rglllue3TVw>7a5c{C znJb>ll=kkIvR_y*ANaheWOC%@=#9~vuitomer?@nMfI{Z|8mE~na^{blUd*ELO=?A zTuLnvssL5Wh!gwTOx9ewlwUcY-#yWOFQ*8=#P@C2Y*Wk|fwZoEA*reJgIX=q%Ap83-O0G@QduEGgGv|y__MQcE*XLQeSN9W# z{+oFgX5Ljx<>yML>Si`e&y~(U*EA7-FEbxeDA)quJ5@JTD!Dc~&v$ zk;c$~dXTRMXyAO51&E$~eomP6NhP}%9G$Uk1ROPY95v)gG^O^dFLD_QQ4bQ-#WRmG z%Fw*Csqny2Efuvd*gJyuELw3-x+Y(nd0QRX_1*2-XthL!qIh?!Q)jqn|@}4y|xRp|><(bVJ4o?TaZ|)^VgS^%yEK;nVX+b7pM&o%J zjWJ=L4vZ!Y_rt1j9oq{CuOrnKbvi9ycV2*NqT9t3jScFH#w&lxf1?;jWIygz^h}H-XD@7_u+GEktoPjL;#!VeI$=c#-%n%)$Ce7%#XLXkxAqPXZ>61D zT5^W1zhEKOKNz9ca*^?%1dVF=o@m4HK=rVJlh zKd=VY)ywPZrL_%G;g-AhMu7dZiXK=kkqx)YY3-_2j6vrhbi1B4O96^_v-i_7I1$`i z&U$Q_N*f05roDuK>;Z=Y;cNM*^MlSn`Bu4nt5miP_>8-b9S`HFzho>_u-5zXivnfE zg2zH+=7{Ji9FU&h9S97+)Li8+?5r%uRT`q=1)LJq4`Z3MhzsCI!62 z>M3CLRzN}gGb!LDR!;$|w*m^x&!m8tP$|&*Te3W}4PHWRW95HKtZ(%xR`plNdQ@w| zKYN@v@j6A&)(~%0|L61+%a%~TmZjTJpHNFovWBTizD{l;wOX;UkxW{K5?*3CN_dIo zC0iF7^)(i6f#De^T6iItBA_8y3$)R%NHRhboDMRIrGW zHI%q0DMsRs3nejw=^mwcD1l$nx-w+tD%$gX*rbQwMRvt*q(;?YU4&913$rX}Wu4Sf zbmven_lpCkMj;5m)R{FTuA^clM30dNZDGq-dM5EU1O(AdXILZXcwCpYRMH@JP{y|@ z*+Us|e97F2^H63_U+-Yw`9VCp`eb|{sAR|)c~^;L8&xcn-ZKnU1c@y)&bS8~UvU%? zBE)IDUw_MyAEGoHKexrQqqFl6X=7L=0!SmnjywN$GC%a$?EW36E3yW{& z-pHLgD;GA-9FPlJ;n*@z>XA!5a6376YfRdIQf}{?FZBl96#;jf>~5R&emZz-P}+Aw z_VmuXPd-eFFG+n^mY0+KV27Z?o_*J~E~#uQ#u^>wda#pDo-U=z9dZHDRM1K9B&H0# zl%29`r?l(XyzBTwi>Wvj0w8Z|3Qg6|l*qYFN>d=UpmFA~T+ptx6j4HLXs%-3bqMV^ zBid<{bL(a@9hr9>MO$l9AFj=QF8N`T(2$yNIqrd<+))P1MY6eQvSz_tdf%3Q zIZ3R9k)%o{SvKGVN_v6Ihxlc)D(SY!v7*<&kW#`ZwgQwAdUBxcHTv<-F2 zT`lT&t;VHrvRM(PQ`o%1w>$#;&*C)50w9aTYd)YEL;ZSv=Lg{wVPt3o;Vg(QEM)j1 z<{v@1zf@~$OnW}r4lc`lUSbVn&zKmM?S+$`shS&I*qFI&3MSeon%-`J84yD0@SAP2 z-95EWwr!Bi8=f8~no!s{oy5sfV1_X+TeFmnnqC>TC3FxD6=4&bzJ+Qx^fvtYVz-mp!9vb1TXb&R{f2bGVx_I@hY%8BS zAlo)wHU<-I6IE~5vk`W@oBU4l)p6NgF*P6~iV<)r+h?m~n@2Kxo*rsaa~}QRo#!vwOG1>h7c&ZzpJ-!_bTtg%!DT4_TE&eH(3?=TJ6pv8 zh=AI^?UohOFjoehC6i;ab5p?CAUhjAb0Si3?&bE0gF#!~WTtFe z7uJ0+XB+L4^;6ALuTGE3>zky?=GpSOSEYl;fA+f61#d5Bq=8}SmDeBQBNUp*LrSy0 zjeZ6M5B+%bERJ^3PW^xCbSb;^_qRO{i(Pr!@v8@=;`%#z4N{+XA|cq=E$urZHy*j# zF1c#&Xg7tO9v9lF&kjBF zD>Ei^(p9IPxdty{uEA5_8vLnNJEjlLyeMz(ls4^|J0>0Jl}?_OkDQYZ4@pC>p#i_p zLhZEbS>vNTFn9-#2wt?CJr_E%=?b?#a75|6r0ntqb49Rx~|n)J+1%^`gc-8-Q_OF!oz&Y{Q&048yDQQu_1PmjKtewq!f^i$Mk zpf}_(M4>?@i4?a~*m&2z6`o%zIs`0jqW_*+@0!~fLwD=_2l|bb@cfp^bqFx{9u1iPsrNS^9ppx zp}w9T@es<1-IRQnl5t8dQt~<_mnivtO5UKPj6TeEO7>H-o09*a8>i{)w{$i`30ba) zV|4Z{N=PLu(hj;v>qwErP?1C?kv>fE4kaX1^N*Fb-bG5R`0u-kScj=q)^Qc3AN-#A(+H1%*t$9>4 zu1DG&H!hfJ9$hq4n;IUyB$T7@F`-Q`r9bM@zbcrpQ7#&uGo?SSD?rjL;Pg>lDf&7j zK)onDJ|yIssvm93LE{Zg0`fh2ld0-avdPpgJWkFuc?A9pL^=55;z9d$5W;kiKf@!w z%VHr8>c(lWjB;ZRuVK;FgISVL?U;kdPyEoVqE#TM30e#rA+Rx%@xz;d=!i;xOP04r zluuegd028F7GX3U8hdxD&^NA+2;8mLQt3f~8dj;Pl*2uJINBkUsCq5?6cIMsTKQv! zo-jQqq2bGe)zrvGO{~oL0R=5dB{7HkQ~;{#-Kl>29zZN91dBKdAO++C#(ldO$S#ug zN22ZALMYMO@9Pr>m^p*rTv5TlcYmhx-IFL0`Mvwo$p0W&Tk`@=vaZ0s4SgyhBRR{e_}-ii+(%Y z+t<%J_2=w9f3jaZ2eS~LGkl8(JntjIeh-9ktzlBg+$)}j4T-Nml-wkqK7SU2wU;i% zkhRzA?ZI7^7c!t{W^Ll@r%WH447hy-y%UjM7uy{RnN=D^(sr{63|nmff`O^)O^4&In3D4TQqGXH1!3u%Wgbz&FIZ2g1rH^zg-?wiMN9KUQ`Fc$^m z6IoVirvEO-ms&H~}8Crw73!ze$5dAXc%05?U*Ygd`SQ zkc1Lm4!==urEa8KG!O9?K|{s3OU%R_GB8LgN7cMRG0jOK580ERI>#~$Oq0ibQl AT>t<8 diff --git a/test/stress-scale-tests/tests/__pycache__/run_stress_tests.cpython-312.pyc b/test/stress-scale-tests/tests/__pycache__/run_stress_tests.cpython-312.pyc deleted file mode 100644 index 6ba2710b710b01a2ee10f64359818b64c9b183b5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 11283 zcmcgyYit`=cAgQ3CxqdmY<}B}=mWTFZ7~Wj6%P8A+7+(9R4a zYnE~qG(hAwh2>@eDaZzqgGKGY2I`^<)IU>nf4J=*QlcSZq5x`}Kj|-T5+JSHA3gWZ zgOp5JMjLb_-nox+&pC7NoqN7H=l-+PX(!l{$C-pKyU;{ zMu`#iPL7bcTryIE@1_wGzEdL<+)JY7n0drZsywMyS_!mbAD2xxsb$5kr@9iqJRoZG%+^D z3y>>6nTUszg22b6vx#I}y2OhTR67UHUkFJOFT}M9&skm)B4P1lA{ynxQbGuqsKRh+ zf)xF}|1%JJ%hVz{^rVqGdUQa>fL_ zY%fxzOa;6gdiw;U=koVdNdaa430!#!J*N-)43(n(kNO)jXUxE>nEvJhtzND_m8YCG z`6;;t=65tQmK1mv&3Pm~#!3lxG!o_6P@H4KiMW`E^77%9_K+AxOT;g-0ru^d_DLZT z=0(xZ{SwM8?NR<+J{k{c?JeywXbX)2wd~*6^(Ca|{Ip`#v``$UK!*X)A<7F1E%MSC zBtfyAN(eE$@)QOtjgo+th++?MoZiPdED3xlrWO4bg$C9^^kj+|YpXD*5FFQJxNvFI zVDHwR-5@B{D-0SDtu_{4&l6qIL^u@f8c)RduE@3Vkid2EqhcT|Mgp7=d6yRg(C>Oz zXi5xhtnZTGo$BKE@m(TlEQ$dU`0xQ#E8;e>OGw6pYGV*rcTP?#PI&y}WKbI~p$1fO z!%zGal3C(ktnRhy9jnzHnd*-FhgPZ&%rQB-VwJAX(DjRamTq42va8;XjJG4bYjnjs zHs`|4J2UjorOqt9ca3(f(%ua1UEH0e+kUVRz^rOlrfSzpRbcjfjw)ZJd>P8OcsWZo zt<^NF*7!3u{&dIXm6|JQsuH)e8H!Es+?S>HuTl0@sxm`W&L^Hyt>1%!MA&$Fpn&EM zZx0qo7YbkJB3ncLbtZ`jF+)nm=8%G&m@_>3YuCY)vWCPcXXM=~K2rAN1C3x(t0_ z^9OzKxP#`Uo5-NZm5-5J#Tfap@`1KS;Td7v{Z;agi^(_}Lq<0s@KLbP9INgDEZzg; zreP^0NNC6IJ2lLn93E7OHxMaSPM8kD_Hjzy*L(3Qd-3u)_U!rIz9a0LiDZ;xBk_n7 z2}L6^&w?S?Z0HqJC>nik7eNMWNR#h|b2nt4FO8JJ9Pn=0_wF;E|zRFRpeFz_Ts$!}9_Bo&E|CKT%!52GvziVd*~O4Bgb zXbV&6wYGvGyY0B;NLLJG>A{@Cef!w0V~dxU9gR!ez4+buV`jOvca15# zJ#=d*U2`$Z4ClIzqz6Yc&0}-@^Jg;5&h;1jIF?&a8htKhnajD(!|8#sOw%~@8Okv1 zH@eVMS-L;xu9|a!Qe`bmm+rlN_wAL^y|bs+t&R_FzISu6a@o3Lsrg>#-Ofj6mK#qP z+8)c&$8+^*aMv%8QS;ELYUpF6td2_s<806w+`nT|@ z^plzC|K4Wv|7py0=QdJ`n`s*|m|(UYg=QPdj8 z;Ol-R2vA_iH$TA^d`n-6Z-MdKJQCZCae4kp|KFfR&o(+$6zH@av>+#nnEJwR$M}OS zMpj)3+rf&;4Z0Ko3!7lY_G@H2iB%gUmcwG+bQu!ODf2eKN?kqMNvz%=v2v3*C>jzi zDGNx%Zx}!)WD7Q3V+2?8ImvkeJk@RjPj#{n3@ba>b)MCX;RaMB`(J=gTagNNAoY~I zyBG#208bm-A%YM3Jhwy=0f0~PmLLThH)x2Aq{xc=uae+w0Z zt$N^Q{FvzRgf`GT4|=vA0l6OLQH4^*Qx%Apy?XNtn#omq(?%8iCb?ih1NiA>Z7j8t zI+lS*LUzC$BE1pX6yRHs@G}@h3A-RKAa|P(K#3J4osg(737*SS1&n(z zKR4REVOX_cCK*Jh&!(^!nEI{4J|tj25@3xaFc4DgkkMh31Na&R{4hqzsj|{2MWiW4 zS1haID=bdqDGH~dxgUOF1$>PZfUBQnsiQ9gS8rtLlSRN4z1(_YBT(u-kv<>FbX=S3 zPgk^Nn6}*Bqfd@yT1Nmloy#yyxxk?>lwn;zyfoul*CbWKObvrDI}S$6vk%KofpAlH53$<0j1JM-2>SH{z((^W6K z+YP$jtfvntawOAw1?cPlw!a7n8(3~Ux)Fp``c^9gnaaRQ zW!IcNN4sykZn@^=r*xx^Du1!;@Ear4lVuL&YIm;I9?aApT&X=YH)<;V3@z%1V z#ptm=%N)qnHLlj}&(!T-sq3CQlWX0ljiuh<&1FZk(P2-P*_-n{4%{plRQll@a^ZO2+W3bZyCXl>r8HF%Ya*jJ7z zu+_yW3fxRv@Ni*0+@Oc%@B{Zoab6ynRpi9nbivqf>`hD{lf3ED*Htg^OYndG0@mk@ zWyZRNq|%fHxRw>_B-QH-Q&!0c#OU)PoK0D`k$PB@O3&C*G*^z1boCbUD^fPD@)L@y zD%9Y^#x1C;Q#5E>ld@r*3q1WF{){5xm0R|*ZhposC=2lXho4b|?Zq{@Qy-g&Mm?9m zbKaB*B-N(OpAerGeyf>M=FyU2qKVi`h~!iWNn9mJ_?ss`6mOWW5>w>nmJhqtdVihV z+Rw+4?}rEX%49;2a6m$2!GVr}3Uaz2_krwex!MwI;ess#EoWPXFMR=_EdE9W{27Vs z3IksDU`&ll1!Z&Rgb0o#a_U>OcR>Wp5{Ywwv(Bk>VloyBfu~HSI;9(u+@yQdda>u8 z9mbA%_7Xq=c335~%Z;yXbJ^{(9d`kXeuR}vpv>Sh^0tUjV?m`OtU}r$1ea8OT{6S6 zy;}YV``pyIZ&bDcUIVubC)(u_78pU-a(#q2ZHN#*GmhE#%LKF^E2em23L+>zkz%vR(wnx_`rcii`zpHf{=S5dXlbEjwF=$)fW)IIlI_oJg<+8+PZ?-=&g7(r}v&;DZem# zX03onj<(&l-m<1W2eWifvBG@E;?>2WB`NKJ2uwhunBTo9eRlKXo9P{$_qnuZf0jP5 z-sryn@KWFXvip7admc5W+3u$` z2mfdzn)W?oh|;nJ>mBQ2G5XQ_dlqk7SocC@6amLI!;He+L-|7eFy8ZsjCME(77?|{wp54FA4 zfhyAzn|Ywz`lQM{&|nQiqj4eFfWgS`UIC17(**8nb#Fq-nNZa(>iGb3l#v2Xu{PNjoOaVkHq@QkTA|&Gs zO*4Gk&)5ohpzr%l7uPJI4|IH`Z2G7nuCU@-^e0Eqha+ZeH*M3c>V1lwA~dkmc1~^p zPlXY~Q_l))a906Zad5mC79x`<8YVjes#5~N2=EP{H7OwcDm0@6kSHO%hLU3_IgS!U zBG+C5dNwgy$QvE@0CeIvDc10KA_8G7h#2ytAwX9OHO@yTWjYX0BSLbuddEi+f@;2V zIuwh_4R91_9C4BmK2lpW1cQN_zeJcsTKo!W!2}?hiOh8XK@SKNVbd+w_C^GJZfF`Y z65#-mj{QX09tglGhX4W!av9`Abg)7@2mwXec_uNY9$7&^Mz#e4aN6QpLgc|0h8P7U zA<4^CeeuvW2nlL!`f(RO1RO(fvLz8$^@H9JG7Z3>$UPT?NL*s4#zT7)!R^n-c*U6~)}!_r~RZg&I6}>b&q4)CgBm@-}L-!lFSaQ9$Mc zpLtOz1TymgRCUOqAffO&N=~B$0Yk;scO%T>H%Kw_=&=+NfMdU1v0zmM4YR_E3aaOW z9LSHSyVzhU!Au4*ly<67O|3~dg+%Zu8AEc&;&7FUg(7j`7EpWvKd~9cca~UlR<1gm zGfser7qiac*;6@mRHiyZRWCZfBgj4H$@Lvg>5fy#4^#64QDPn-k(b7y(b&_w`*L+h zpP6tgX+E}Ybay?nqx-J>TcP-&hx%f9d>v&)*&Wo1?RXYm@~+zTcD}kcjX$pljz{=ZdHE zam7>W*cw$j-@FjG6Ik+Qsa@-36|+NYrIicbJKm4#W=}t(>dd8vLk;;DkyH-~;il`4 zXB@}Ze7o-VuK4zX7mKN0Wp)4rUYyA?fdaDg8OQlGZ_5(5;@vfOI!`8NnGS&c{=i?` zzOX%TJetU~pO}L%adU=lf%?{ikFI}tV5MasLpRPb^HJR&SatBxuhS=oGd-8mm)}Ys zyqdlmN>^OV(qUcAs@A2{QaoMJouv;Jt6O(_{^J`*#dk25_|ceKOuG8O?}s0k{_T}V z1L?}wo={Kevuke8e?2n;i3)!IaQI}yfR}jUwGFyWPa2yB?WTXUn<1~z!5~^!!Jxky zzbmk&r$u23Eyj0H@+T-ks8D^kY}Y~}?1%6ktA_y``S`r3#{X%<7mRSBQVrW%RP5Bh z-opCvP644pVFo40X@jHk`F9O3bEF%3SA#)?g0FJv3j&9f)sHv&Xe_}cqx>=9Bj}EN zLt+e)XC{&)zqb&i{VM|g-Cq$7`2U)y|B7%y`ZclpYoh6E;=p=c!_w5f)ZNtggo|wZ zj%_8&zq>#lf%FF1OtyY^hU_LwpT$XnV&>#!lQ%~(vnM|ocyHj=r9Z#&l&XM*Ox0p_ F{s%RGAKw4~ diff --git a/test/stress-scale-tests/tests/__pycache__/test_access_point_scaling.cpython-312.pyc b/test/stress-scale-tests/tests/__pycache__/test_access_point_scaling.cpython-312.pyc deleted file mode 100644 index 8c7a90ccfbac2a7d54a3e44b37699af5d49c2894..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16665 zcmcJ0X>c1?o?kb*ao@y2fF#5gZ%E>;gOX+06ffN)Te3Z-4=jvsiJ(A&>INkd3wl+K zKhTLLCTFKsblJ0}vy(NwcGYroR5+ES*5}x>o2sBqY5?^p6;;Jw^l2%HQ{$CCB>#Um z8fa3WkI8IHeErV74cm3Z5W&)~60H7x=pUb92Wxnh7eSK=XsN0EkCD9G$7x{c{#s23jQ?b{SK-J3sftI z12R=BR|!;YTwt_YGmJu%wX=>QZ5`_zCs`NNc5hL;f+f`2tmjRd_5R#g8tb(~G;!^M z+$W6rP2%S|P1$9dm?EdhdV;HJAo%LxYPp^t!Wynxt+%nwv+&J_tDmc$B9-@L@-lIS zdX~6MhAZ#j_)mp(k|CCebK@y^C&$ml&xc9Lb&^Z*@tDw;OeDBiD#^#R(=L2X;{FKa zUMDyL_tLPAqge7)Lj5+wG^BN!w4SBl+Z%943>#TJgeF)y!>i=58Qw;CTUZn1T3IuM zHr4_mtSoDV633J+?399MIUzMLc&0ZN;{@S!GCrO<6N@I|F`%AR z%^kI=vC>@xwms&T)U9sVav7(K!5FTqVA;yR#KL z7tgHPJC~lg6%4(-?Tu}VvLlRmXa^0(=5&ESp!SPN!G}kV0>jxE^Edk#9HLgxmW+X_6cBH87mC6PmWc?r(z~-xf=brDa;t z35kLN$$l=L;Djk5#f?SctVHeW=z#N@n1l>oGVt8_cyc^UORfn%c_}VH0yg`}B+E%Q zY}zWDru()TE#Q zls5(hQk#K;;*lYeNDj_zEHV}ykDtSXjvJU^7(M`R-UmUtee0847Lst$g};H|DzRp8 znFq7l1 zeIVdOU#Y1j-VPmZr2boP)nSJEn9)Og0|?;S;NShz5L`zHC|=f(QFz$mMGV&{)|Q`g zS2kSlh4yecQDHhA>`QRbab|LYd5Hsz<`^y-8(|QCAK0T48aTd#SHr0=B~<|0LK8q; z0PN%>51fWXr{ZIrq!$t#H}M|qC4ozv6R;!Yqe?afe_^wYB-wP+)?Zg!uM^q})eu}I zZkqr+5rh|;aQMpe{b|GHMQPZ_;*~O5 z2KZU}`_tXX_h0){8dh@48C0s{F1@vv>NK69zhy1;u$KBi(puu3xas-+w13cA>c2nT za%&0Lgr{}WQxXyYaSXtS6j}AricM7^Ia-D-2S$X(OMRBS_LM4{{eg5Sq;u_?KMIxK z{ZXjAsZt3W(=A01Dk3X$Ii3PI9^shtz($WV6Fhe=enm2}+_~svA_Z7$M&3l2h{m}0 zbTae7_rW@*-7+m;91z+klI#IyW7g@X`wlRt!VVq}mE=hP#}g@1Vu8_SC4C$i6oC&z z1(~qqW4Q#d3!Cd163O%DfuEq`<8Xr@Q1NXTv?Jx_hbSX!0-O%;@t2Z`$uW>%64CgW z6fQ%2i>h^z%h7l$0-Z%BF2#fhu2Do;nTSG#4IB7JuzM@csZb`-Fe^c#aBegcAkHUZ;(5H88jpGkrfAX#HHe7eeeb~+Jyr=T|!MTGuPq*mlUXEuy-7B6WGyVC> z>g(yBrDv(UXWK&kySv`mm8$<=6bg$`*zED0;^S12`TP|=w3>?UBZ+N%notoVC zE^&KTzNvjpPXr%XC#Yc6x`_zZzHNQOnr+&*O#Qv>m$qEbQL*P}cHgmV!|@w|6L+0N zpl+>3JCGsKGn9SuxvXbs#q<0R7|AIyaH`x$enkGDp%7+!&eSfN+Lx?1Ob5SSZzXE# za@D)U>Rq|&{bKe0dj#nVNr9R*3f|v*wV6vJ&F-5v7ZjS^->lb^YkTwiw0$5X;nTj- zOAS)QN5yw z$}V0$r3$P;mSw$~evNX!CT+hUNt&nWm+2{s8oM^3ft< zty5>y_0L3k6cZ0HXMq!EndD@OL6F=)+_91PVA#xqLYz>pfM!|Z(_R5vps#*{2PF)@ zA6sq&d%gqm!g?8bl^er|asiA*0v%J7Pem}vcR?)yw_yo%LDQs#bQQ(FRSqoIAcCa7 zgaC9qzTov|=bruZ=VmOo92Kv}UyJ{n3-iZBN7G{EmyV7#oyE@NUEb@qIa|)vFS`2k z!BBo%Gr+8S0Kv^&0XXaSX9F#Z-bF*U?Xj%?@l{vvEofqyv&?t?%)agKK zEpKqXf5E$8094Ob^{jgL->R-%*Aw1)5E!BA0|BMurC#!I4e@bJsIQ6oc>lJ(?bIjR z^$;(QG7OyPe^Hcq?}wnwL+o=IJybvOx;~~m58}@Yu;Zr{Ho6STl+#HmB1uL+r@K~5 z6VnE0Z4g|kqOP_amQCq3<%U#Iz{nU>6a(}#YH>h=Y2~m)W0=v@gfF-G*#iCE(~QR~ zqtFIJ&6G6;n&g;nT7MZd%c&v*U5;%oXM6%2lv&^@XkXAn+tQC93MrVCNFWRe<+;wS z!R#>1F75B#&|2ZgnC7&nnHh;*;x-`G*|ZHiRnaN!SGrZuaufEYJHH#-3L`G|>1^H% zb+!`J@yB4e{2&HJ-j;s~k`k5R#>)X)*r;IINo?~p2G3w{8iOYwknCg8RBQyMBQvA| zS}0_2Q+di0kdZ$4Ej**sSPUldQi_~oG&|D9*aGHcH=k(-qb07%)I zuBAtoTCOGsUuD#{*&D(x)W-+wVxG&q#lkGW`_5Wzqby|yy?sB*oka4m1 z!<`?7zDQ;rFRq#*fQ$cp&qr+k6Y}eI+e7e3+G}oBRs+*#uX!M#D6zDoua)?yD%96P zf7C-`c+dkmAG=y1{G>&XVQZ-0PJL=4A>5!8LIV)qRriPC-jS6;kl6gNq^y3{VK`0d z2vB{OVW3!wrBzmGTDN5_y`~nmWi111EYhOm2zW#}t-@&0WmSJdOM4!P8sp`ecI=NcyOJTYz5 z^j`!{&??PB4<^=~F`>3#+6;3mf^lgc8M6wtB#|+e)A49)q}*!Da@vE8#-^Pyu@-sm zx-?ugYy~oTptCxjF=Y%;0xTpl%8+wbQxkHgEg3^bhb@&_5XW4rW~DqC3#@`SV};PC zU5oD;9ZRQS|CQ!@?WZ&W>u1xnD|4x+SH-kFg*s#T$rLT(u+FWDEq>ZGZD#$2)po#Y zm(gKm9I%>&T@CAPE4$tf%?R6RSi5Ow#s>X>c4r#t;#S)9WE^be2LY8btZHS(3F`;n zd)j#|L=)d{h8a74muXkVm2qZdwj8Jg-=Z>bm4m|pj2A}1;3#CqqiWM2AX&~F3uCML zpjsUpm|*K@*l7hy<}E2TK1@AhE<=eJZGN~WTT@y?&|57-kL6D`-awn4(#f87Yj#~I z;KbT@{~5aI(2gG3SC_3M2`EiKZCY>?^tO6@4k5h^M9^DIg?y%Nc$p%S(9O zMud2pyMvoqvIEZ$or(w?SkXbn<1C0BYDRn)$fS~4kwKyp0$8}CS0WI(gR&|Xa1%1M zwDpW)TSoRF?O-;HGi~leqZUpk?cG?UFih!c0Y$Rt6pCl5%P66N4WEZ5U@c2mY%r&^ zg6gT2$HAvveaSIcm=uU@^1K*{Y)`lBVkV->%As)Bd-^B?gg7P<9|LABmP{tl>;UrN zLv)jD>z)4ubirnUtO{6cOzfmG2+1f+%7%2wqg_`4EICnjN+X!M9~x`AxmRwfd^7ob z;X>d?g3+kskB>8H6I1vMVV`0Pq|Spo0qS)nr}71giz4ay=s2hjW#%gsDcB5QWfYa3 zc8rn%oENxpR$=4zBQ-NO4(dZ(70Fs`3%WgzMwU#ng}EVX&ktd+NoGBcdB~qhb|4=q zjxAiF@O(6uqg=^HF~B92jG&)|?F{<}Mu7WBGUK3S;aMW%l1WWTW@V>BHOZbzrlN_6 zoB<{?GpHez7BVvlZ3=10Dt`@d?3e^GJC6O?@La%h%kr~;Tz`SP)VyTnr4+;uQr24{ z-Hj)+tZz`;mW^J8u7rPvT?$4jx9@uOTy-|oF1kCiokODg$czEp8vM-c)7OXRh8Jv$ z(G_3Ej3pnan>V~|dBd{s*i!FGVDF4QUr{sLbbZI%j%=uHiCU@HJ!8CG;hiz&!H#Ep z4J>;NqNyR<_{gg1AoyDZLi0~A92Y|!OHIo`v8(UHsMt01g+F`xIq~T8xyl!2hHly1 zv&R>V^HYnCY{kA+TMx9dSIi!rKR^F$w&}p~qs#5tqinWEVip3ct{p%Sri+69qpPkk%+lwd zC+7y{hec2GV&{ry*OGhbndOUr_uPN1&2O(;c;X%3;sKG_GkY}OxN9*jHtt{U6&oM@ zaEI7%V)mrWN$*;GcJcVq#pTelGdq;bR!yvWFW&M7t{)Cn3b2#si4rnrKTGJWa!8}an#N3ITcdzK( zn};TKJAtBa-|q;@=a*^UuP9UdnCLxLs(qX+RbTTGc6ZL!E!w)XyN7;bJF?cNBLXjy zYlHBrQN){0v9dGUb!4UT=#1@_$)0uXTedB|{NazYj#I0qAKiBPXB=OzHxhwuxylx? zvL#o!N37ff+Gcm~rZ)hxg**7odd0UDKM)$=2z*{2I@(NsVG13y(7$dD9WxsL!9ZeY z2_4&E{D&|J;RcH>sw`xAV*dg7vW8 zIJzMD-lAClTlBQPLB#bh}m9^_GzH9*qytAWANXk% zY@bidrcdw~i;txnH#v*7D@3nk z){mEYupeU#;8J5fmJJFLJvj+pa(WDROVkK=MdkseaDnu9n=BOOvpU5`V;45^MluO> zl%wFoNOY$r*bn!^4I6H8OLsPP1TknFHWL0)j1V4-9)J_fE`E1DislIIrZwEeo zGQWu1L1oz#V{;O6()+gN%!+(%IiC0?G!}jhA`IG{9p?Rco9p!xubs%*9v5wo=Y7F^ zRXtq1_FlAS*d1`S1C!K0@0~Yfw>_S%=v}oPF19!slZPUISPiw(P%o?MEe93Fm#=Vb1;& za3q8$@SYwyW<`;fu~sE{RU|IZDIzj z)Mr*b#6RyqDYh*!L2kn4CdI%w&Y>-pevrPxZiV5u$G4(xZ+!v-?c z2SF9C&k*6NbVCuGLF6fD2hhU>Atrt<4q`CaAb1o#Bzd>q|m zqlpQKq>b$_#mC!)k+iNgmOjqKMv_c39w!84;ei~0dy?6J?xI}k$bd2H8M8>$WBtHf2 z@e`AhzO$GULCAp-vs ztZ0^fWDup*G5A2S@bF!72iXchdP{Mk;1*EG2~r3g8^_|~^3^c}SjfS36i>2e6r~CO zZ=tDjAA;VNfPN69C0`koO|oHHgNajE&xYcwih2SKSqO9r$f51x+nn=@AeUslfM&wq zLVbiZ87qSMVC{l;-hpyRD@qDXcoFcVBOe;g)?B#F>;jSNuAXZ0uMt#ZB^>wW)_i>! zavG}uzna^WM19p=tGUj9*W+O-)(B6nXKe=&3f~UYFZ>ulrp*r+6oLYMjRz=n?Y_IO z1~A9G9WiHZ5Kg)G^~`IToU>DOcIN%Hd3Ojv#?=KF8g@F6tIIWSQ0m>$Ry*FljG=daTd@0U=)Ca zm5fh~Ul<3q+@S2VvBABf3fph;oCE)Q*<-!~rYq?uz^#MZ%mAUHOZBLjhPcN1_#;Se za--k{$V<0=3p-J=Gpd@gLwMr;JG9$iSp2G-%Y@Y)%ER@Wm`mjRkBa_B^TF+k%S7cN zI0=;%vlN^-Z{_s^a|d#sdeKwAF!iOU1FVL5UjS7^IbWmbYs`iZiM~VmYG#ch{Ew|8 zkMa7k*N)9zymoRuRK|1&`Vg1z>vd}x3t{MHCHjonLk|SBbu8^Y+(mrU9DwlSu25eE z^@*E=5WKhVp!EP0q9gcnN~VbXB_a{Y5{Zl@*~tV4aa$yEaWa}feiBkvfLuXP+`9Q7 zw3C0Cz#}34F$_TBB6yi;MeQMf6$51Fc!WFoOT0(Ou%Lh-n(%lG&;}+v55cQMUhlfA zv+5h}Q-s02hOxCOh~3py=m*I{CPweIP0CJ{fJ0CYT6^I=1mhNS+g7% zsP&+2QOi)yX+y0CKkF+(il#*wS1wJ9B1makR1~LFjA&XE0U}kjjcH1eb&{VH3R3c; z{bdC(Jt+HadP>$B`28<3MrAYrY9DbDe^3QX5S`uz@UekG*`QJv(mh)LU6800ffTu# zqLcCR|hfInxm^lx_Xrk-SSl{kg?)xzB>5osfW^_*9+_v13Om& ztyc$EY2PhhW1%V%vScm&tMovgw&duLNQdT=t8^Q%5$4(jLymb|WFB8(4#Vy5G+Z58 zGZ8k=)l>4ZL3kJgE;ABIJ1mH$%#(`vc0eL65_teD91NO?IEH@(=ECDOH7FU+D5e#D z8Ioz50aJ*qyiZeYUFRfHNShf(RvLgO_6|jR04s(!$hT~R157w1878K1(RnL&)`P*% zAdrmcb`8xzcAXL=3;r0C9|tRvpnPHc;8NveDxMJ90LAdP%3x9`{8(A}ou?TLGbd!< zZ5}TG{s;y#PT|LrKe4g_29cH-2O;Ao|CgAyz)#c0VegH{IEiNCu@wCN2Ui6Wz-IFa zxlK_uAbb7sZy?=|FnAP$0M;=bR1o+O{~Jic4k3g-2m#kg^0$Qjw}e&w@AxfY`QJq2 z{~~t7SF}{mROZZ8qPc3mXT{ub)$n_IKu`Ma5*Xe)W$}@owKjsb&7{Aif{3jHMJK*=6On!UIyPIaWOEPWv0we^whaAjX3@kyh;Ur&9q!`)9%pAs1Ve-~BO{r~^~ diff --git a/test/stress-scale-tests/tests/__pycache__/test_basic_operations.cpython-312-pytest-8.3.5.pyc b/test/stress-scale-tests/tests/__pycache__/test_basic_operations.cpython-312-pytest-8.3.5.pyc deleted file mode 100644 index 857fb2da566c60080a3459f933d5ab70a4147d91..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 10279 zcmeHNTaXi1dhS-MrBaBg7@Ul|1FZ{C&>p|NMO}|M~y#{71j@cz6oZX8P5+Z`vv9-?3mOdo}YY51A{J zL`igtnx@IqmS*BMno8TJS=xBxn0Ami&S@uka?>0jr&y{pNF%7vB7qp}b?O9Od$r zxb7m?y~6d}3wM;l1I6&b-SDP!%mw$oYVJe#w;o_#_KZGOaS1p(#IZc(O zsYSMz(rRj-#7)zXnkjT2rL-^61v*LW+S&$^{p5Bq4fGjMNQqsde@b7NHJ%If0%a)Y z|G_uUG=4L}<=0Q9O8^iKS)T zg;dSO;&S5Gu&Prg@2U7VsQVF@f^)2gBsnE(aP&F{c(No#fd>Xc5Dwas=b}a$DV%3x^=U~6tRXH`Q({gpgDXrKU z!PpS6E9bIOzW15C#$du`>fZwSfU4LzSNOi*zxdYqw@SirQ5Y_Vy36go6_)ahtWu1} zTk+R(OcaF)i(`@|EUN*^<+;e8=P!+Z==pB7oAQTCKC$Q%OTM9^Z|EUqbA`)3|Haei zPhZ{o7e6ceL#qts@BY5Rz>Du!TmMJ=kJVx5`I*lBLF#(vgonBv?4014+gtn-EOUot zA%DlyIcaC^Z1zv=W$x@{A>Vx8a{qh#wr2XFtJNfd3#{MsfADT((EMN6^GEJ4B4v>^ zr9>S0w%U$~&1$k@Z5U-83_)o_vK~nXkg14+Yz3tYYtY0g-AF_v8-PS;r5_$DI#kh6 zRG%h;N60Q@qE%fTc#5tz@C~7yUjRXMb+}r}&s0}$Jx8Ou{wHemcdM=^G>&$){9f=^ z7VMYj$EpZjIn%iZ)Ng+e^$(r(LmQ|96EU7DgF%{CEz|9 z?CA^p3KI&@=wHwXH6#nPEZKmg1|}>vi-X9pUet2Q8pxDiX-1$)wK$ux6IY=Ld&t(F{M$(;p9!Fh5MMJ;g!PQ9%K%~naYTMbEa6u4Ph zB5h`EpI5z`#A&FrBqdRC8#A~=aNvivzSgopi6nOKC=JO0K38(2L7XBRY z$1ORvUbBJZDKNjGB=5(J$qgt&i1_mBaA%7I;)zF&O^R_vCa|5@4qeEdibrczHN&E_ z6WRHUbd!E;8y~+(=YP<@DYv*SqVn%oGvc_2xiPZ}GS#Xr5p~$`uKdz)ZZ4+E{E|3s zuy2p3I*(s87;f5iCbzf+$rzGtNVey_06AqvBOllSEueD8R6bDOc~Q$|Pl|a?RAm6z zk{Ssp2-tNW2v#%;%Ia=oaB>vBst2PBv4j={41&ez3=UEE$Qj9MV1SBYm?LQoq-YKb zs^d_4@Y^gItj_PtEh4Ik#uU9J3K%>URpgu!&!%%RMb-s)9i5p^q_jjv)g1(-t9n~h zoy*RrB*Gd!k;=}*Qsn!(D~e|Xl%RX}lX-z7*8xSzDli-vNjj!Q<=1?2s|zqxFTY^@UgK3$GXj zT}5x#)zf#qaDE^hB~C1I;`O!_ZgaV<;}hW{q0}~3Y#Y0^=Wg4?IS=rJ+pqli@{dcQ zBgN2>a?AR1xEqjG_=of(iV5@Q_zFk)+YRir%#AK1f*L6L1|WZ9BdHj;v9ahIEphn& zE98I*qg10sZuDHF<0qp16lnkvcW6AUa zq)p9VSbUb>1V2V45&JYGi$lE)91U-=n0)EU%0?eqH);<2>-r#A{s26^XtgwRuTy_E0A9*rwJA7~h#MOccoz@6 z>1(KlB7D8h>s;7mM|{+5T~#NNaHztZd0BVtYRBaU5i(r?5)(=d}W%g3g}nJ(jzI1 zky&Y7!BfReUTuddzwVnm%{uE@sA-hhTfj|Dz)LRpurkD7 z29_9WpFk`Z&s30z-KvEQjXI>J2H-hSaxO7HWiHt+TrhH0Tqe?%ZzCuz4TuDA!3hG? z5*ZOA8RW!=2E#h`U{q|3l*8($EoDF?7Oh|-kZNE!~wZ(|*_;*I1b}803L2A@R!=CC`0{li5ub?#q5rtD${Y9!Y(53h8oKQ; z)>3DUF7jk04TSio$oT^#Zz4H_EVPtQZ(8hdPZITyX1}j~%+se8zvum&7ea zam#Y|=yG6eMc8^zXjyLUyMFYu=})Ii{X2{OJD2-*EeCe52;+5#b9*tg9fX#9Ub!`Q zYkIkR9|F(U5O;dd9W=%=vLb9KukZe(<66fh_MSgk#hZjmRza>L*jaZAEw$Vo@bwLs8gJ68gRr`o6EQ@bsNhGNLJMbP;v-lCj$z-x zl-W-v;55yF?3fhOVh|wD$#LSdZdM(MeI_d{!X2g^`fb?n2ssX}4c^y=x+eS%&HAw! z88Zo5q+^-HERGNt4a^QPI?8KMci55CvcFg@a!Y z?C34!zQOygUFEi(>)Srt_35tVk$uJ9{r|YJ*!$)`j~9F21Kyr+g|T-99yqy>_kqXP z#-9@^BIR$rSU6uO`3@9)2jEayZs~;W?LUaN%FkCQ#>Fc~;KOT_51|NAz5-;{Dj%vI zD4XJN?7nIFM>ipMiQcG3kKu%+7di@mul?&FXTX&ga(pf; z_H6=9kR#%rJ>m%J1i5n@)=q1?L~@NHzJk0NInSfk8futl2`hjPveYi(jE=;#xd?02 zu!%HkGhBqGK?Nk;9#c-Jx?r?Ei~vg2UF6CJ^_4`qOkp9o;jG^6)a@g<%t6O(j1KmT zAxTwKm<9uLorWC9@Z-FMny746YEVyfW=+-eIEZ+@dS<1oz^7gWQdQ6Ob?V7eokQi$ z{<0Xk`u_d4j&j?v3d?kcD?HURRPG;qz%m12P_zDZ4_X8?E zI~;z%vYqgP?Fc^TY(>?5iKy;tHPs!3z=ox|V~v`NnrbEt6A;bq04Q94-=Az^^h`Jm zVQi+E0fYnK3m&;88uZrod9^2Sgatr?A#RC51Kc!3-sI-LK;r>Wr7wuJmQ6zB&BEhw z-D%;!{_L7D17DJ%kS21jP@0UeW+s{y%i@F z?)opv4pFxQpvC^srFYN&tmK2}+QyP^u;?4SWxwm&Rt~mZZM(d;6pR#uky3D^7#z8^ z{cdn)xixg<;N^qA{*&Lo)Lm%-A(a4#Fih!ZfC z#;|e11jGlIBnyZ;byuxQcUJ2ZJg({vqlto+SLwwX9)H`UWB`Av#v$B}wd>|JJnX}K z;LqLQr%%=2yOAF`5i>}JseFub5IY$MOm4R^y?d0uf;x=EsGEVn%`r`Xjg#?n&%aP^ d`2RK4|F6`xuc^TzHTbRlO@vI>{|&*^-A4cb diff --git a/test/stress-scale-tests/tests/__pycache__/test_scalability.cpython-312.pyc b/test/stress-scale-tests/tests/__pycache__/test_scalability.cpython-312.pyc deleted file mode 100644 index 28de1b2c7f9998fdfcf53a6718f46ae1e65f5250..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 21309 zcmcJ1dsH0fm1kAIyXhD73mO`Vm!Lrn0wDiC5XP zv-ehY^-C=ePR^9XS6_XP`#tKr_q*Tk-s)diEG7c3O55HuhmI1&|H235vZNvp)<}Z5 zOwa^PMu=hgP7agOvuaoccU43^t{K*hYlpQY=Bp#Ras9BKBvixzL2KS1XzjZyncA>{ z*1bUt8yOR=KifzU>=h%FVvO&m8XGQxCo^q$7uv*&B$%SHaygzID&_YRFHIA~i$HzY zI&72b8X4<%3EK2Mb%q9{+Zh{Etn`9C#@PM^sy!*HCBqJhY7tWcRLzWI%%9y4IZj(> zYnrx-woQ<<9ZDBJp>zpND5Yt~cQv&0d-_aY=l5!e`Lj}+F2?p<;(MyBybBs)hMXa5 z3C2}Nu;s&UsT_tL#;uggt@Av*w_*1S?io_PUm!0K7uC-b7f8S73+(=oUnS~7kuWn6 zh5O{hX!x|B6zxZuC>suOeUp(06N*lM*M4p+pmp4LKWpzW|8$YCR`hbI$lfUt-*LTIK<5L)OW2(5H6ZH7`dx`ejC z-A-E}wRpzncZem=GFoq>aY$%HT*p|FRkKeI*%r4mGTyL z5VK?yOG($5Mu}zaaZ;JxI5Cng32PnxDO5-zs#j`^8I)M&uFMORD^gO&%t|bCSIR+4 zX?j^T?3t2EEUSi5Nz1C?RMIl{tQzUQM9Y-?tXxG35OHG6r{rhuN=cHKA!>;>f+H`e zNaA_mTauWcORY2Y&?Aq$uv(+=wnpT5%S>=QB9g4%CTb?9m{ z)Jkrdnh}dQCOSPeB6TNL`S@yY$;!t?$NdomuM?kJiusb}<>PCXwv}(Lm-*hQyi&PL zu9h{;_01ppjhe8wKhzOMTf$H!7^>C`)$8txgnNtN-m>a$oKw#iJ@#Byl_=XPlxW614!S0PRW*%_WC0at1I+`iybT=kw^kv+&+ zbnz_t8aY#(iODHha=j#0c|cms6bMEt#Q=S{o+;*H7?#^39WG5S1ae+85gcc}_zt{= zSaE9Ux187$c0Bbu?|ajD`#TC~z(R@!xD zjxp>w$JRm1eywN%&W0yw(E$7#q=V6*Sd=CSEt#f*EQ|{p<#3FzPSP_& zeglhwP%L2?5GXlpE{hZsb1}qBMS(acnnAD}P##4c7Y#hzZ=ZET5Nj;J!Ubgymgas4!Ry3^&Ti}{lV;mnLvIcx%mKk1 zSn1@=f!pSuFV}6ZgpCqxRKgYzY=QfP6^q*JZ(V%z;^Ou{{np)@9m$HOWOc)GDNnU0 zYdh|4?MOB@zi+y3TG=5q?!M6~G#*MewI*xouQgq5T7F)r*>%Gp)b!pnYHQv1ii)aA z7K%1XiDKtv>m_T#zFn|yzfsQHwccLi_G@BgeGw)& zK}}CluQDKv8Hx#p&QK_5ADJ>j_Y2$rhWu)=1cWP;2O$e$d2*T!F`TH0hQ}FE%S9Mw z>N>0-j){zN*oyorMYAk*VTqlYq+?YD-f4D$W~j)a7W+DJ*8s8-W$u!B%@9~OnBOwJ zXR4%d@WD}7|CzC~JAurQpJY4X&UQnPJs?|@fvs%v z!o)~wOf?0@lvidO(m-`2XIQrl0d_AWh7=_c;snmM^vZb+j^%e1_#ik{g_Zyu111P! zRvjnf>dkHf=VAQw6&W>&R&Ty;u8RExVV+id zdh)L9ZE{u<*L=mSX<^p1zw@kVH(&R$St~Ral$3eb=3Dq$vsU0(nKNs;xb6wF_9w8` z^e}7suQ6-h&6kdosDi-i+>y09b(^ohsd+A-%x}Jhym@{TS0`{mRo;t($IQVAgtH!$ zHLFr)tiVBLtVGGp+(B8>pyIPY&i|oubp1W)n0`+>YV6Paj6%h4h&86AZh8-)E`+17 z>zrYz({L=9pr%-6G<;Fi)68gaIueCl#fZ8WHx&#qG1q=3!bCwCqG)C+GC4C2r|~Z8 zx=PZy*!?gYF>AWW)FjlU>njXne6t!Vc%s93Q%1M+( z?Gzh^bAJVm%`=5-N!k(Higi%pv-sXbM}8~NDbe*f8BA_?tSstqpb}0*N6t@1rpMuE z7YT;PMSp?%PblghxeyFTM__v&nK~chV30Tf8k9UhZV0gm$3K#q@}kTPaH1*mAZnm@ zoT$b=YLE&vRy9Z-~0!`6pdo5@(5^j5yJ^~%Adsn^B%^glw`V*C{Yn81Frlg~CsVPyjL#Wx2sOb`Fx^6kT)}3X1McYbKqWyr- zejw3)NN7LAw;kraN7kH2lg`r1M=l*nIQIz7JxO0f!naNEZAi}?cu$9*PP!-H@{PG?o9fqcV4{mV#3!V_*znp z?@}5st;ubCKyVE3&%D4p238#}KDl-1;tKbp_y_UCu0dhf;LXFUyH0MRJBI}4p+em` zOlJ3oFj^9ZTES4ebojPm+m{db5O#O2{!AK+MYDZF4fo$Z^!^UncT%+>+-t>vh5HBVVt2u$uU|y4MA_Pda-m4r+-{UA->2{cMkKuu^@~M`Bp*JE&6soFp;S z`VQ8se_lsI7)mR`KsgOyU8>r% zF*7!Zdm6SGbITRw22y2OT3Xm$NVx4}kVPz@i+UCgUNkj19i>pfE44E{PPb`mNOd+v ztKV;CcVkcSeX&O{7{DOC*|CH0D5@jO1lxrf$V-`R&99es4)$p*q{rhQ^g#r`Fvvzn z@R5BM%b&n(%XlyvIs<)^_8tyV+MNAj2ohrcLixOw{}(UJ znbxf(Z-w6s|C_UmhXm`EWzQ|^whfihQh`D{ekq>t92Pu>`R7J>&*4?iD+`+BwvHc} z{)1_i8u`Gq?6`6E=JC&7{P~NCqc00bU%s{X<-c;Qm=-KNHFC#OCv7k7E8Qyr{u|Hm z-s5Y|XOrbs>vq?o`TNJ0eXI4m__|KMbBK38y=FhQZg*ZbT{11UeaG^!n6No=dbMiZ z_OO&NT6uf#hs_`QZcg&nm)8sJpAN!O?b~AT1Sl zJmnx_EftSAR4i7S`U1pHy}rIi%}+Zu81^_IzI7`xM34zNNrl|_?EG!|62TG#Q5nwEh%V*ljMj@hU6h2PU z8f+8pTB!#*w3xt?9-rP#^%k&|x zjjQ4sg(~1CX+bxZR=YE$711=S18T*YbR`DVG<4B>W`*9YeslEVI@~&D5i@ltv@-X& zRuQ?g#<(G_j~i*r+i>)bYfy2E>lakqOxzgP;Ks9{V!5mtvBr%sBQ|MX?B7e%A-~qa zI6Z1p+%#62>6a1%C;_(Z*`j#Sm{&>5+{Y?1aV1vh*aH@;66N?lYt9;j1L#sp&CGqQ zCKFd;Fji;W4DVZ`%ox~Ip8iT8ph2mdxzirHG_J)}^IloHN7lHt@Yvt7rMBftm;+nf zR&Wj)Gt`wB?M)q7eQ_IX%oJ?QcBpS(AlX098hv@RS*a0EquMaeitJS$x6^Lg=OutE z$d}GRKZ>&`RK|#7 z#9&UUA*Vp!A*Ut^vO>P%C3G!ar*J5YA+6BBTG^!`qTACY3(f>xudE4x8*V&dBzUID zyP))En=cLH{!N5-R`mUq!x zGqPhrJ>Ma<1Q2ZgHFh9B1?*bsy>QlpQBAE=Xn^`zHfzbM)D06IWjRkUZBVn?D-AB;?sU^?t4tb_xSM%?8mt zCF;595LhVy%)lnVY!ReqVznoGpBXwhG|)vUEkwfOpzzaxi-Z6gp@LB=W|0Sr>NN~J zW};Z;+%$}gq1k7!qx$KIvlCzih-rtwDo4c}px1L)hYAG&r~>Z_K%UADuqk1N(liHM&`?x>dZdDwX)E;%6XYf*x+pX-Vl1W} zYV8$Ez|9~U3{NmD82iwC2WB0uL{rqKjutTwlIc^*DJKq~s)qDfdG0uI7GN@i;Ry7A zMTYqT^c*5qiJikYC@mmdW?&Ybl8#O|oJR(VW1ufI=dYD5h(#kQ6C$h{7ENZNO+IFT z8${|2YuVt$X+}P=fPG4~IA)fHbkGrhrfxCuuCXEQXEYN9h!jHrjo!Dy5f^-u~U z>;x2KCo!19U>xg#vjxY2FNTCuvIj98g{)|S1u>EyyWb@r#866!2ACoVl@#>=tO9Jx zi3VU?X;`8yGa*tL_Ia$Ll?Ki-NdB~xfD06QiZ^xylW|j&*E=~Sn!)!3*P}dD$FYVn zJwQ*cLbKd0SnFU)D|THjzf{iq zT7}|mD?Xw48*@7Fv2fPS8Iou`EENo;i=}IZn!6>=IXxu3WquQq>jXpH(o1WG?cmo@ zR=GI3^rBGNzS4iAOX%poSuAuMy4lYke_1#(k|=#;ZgAaPyl{BQuozo*@g=Qm=55eI z^{$nRD`R|R@7&W#vpr$170k6uPp_Q1adFK&2vh;dEW1+n-SW9Z00AbP0l^tqp1g69 z|K_v2GqCDBF?V1cj;1r0X8z(2mWBmS>)ap^^;RazT7|OK729grQ*(zP#aEv!r;?S8 z_w<@->)gP?pkVghw-SIi4lg&Z`r6?b%9nQEICP`u!&&~sOZ>Ah^Dm#_-QhL+*oH<| z+?}kT-Z^sRNTQ-$sAylU=vX+Ar0TB~eZOc?ldNsH)_t`*QM*g1-Ib{A7HYe1^nZA8 zwRULHuwLq0(k{{0#;=YiwsZezaCqzzVXeUoci&p zpS=9zmv7Fj?m6{Wz00wMBYf@iqNf&{)qS5^?pbZ<CI&M{UB-=WE#XzO2V>t8y!Uguxlzf$(2$`2~}ZBKou`fzum@3_!+oZt5> zOyb%IjADvsV;i7}k^58+2;vp8PX$jS9~1Q}vnQ=LP_I zZ$I@S9}FdeVIde!1kbGo&#gBG_-zAnfyCe`VenL9a6}j!;RjyfUkUN_Xo8LibRuCzi1P1*!c+e%>7^VFPq-C*JEEz9xci5r8w z^}w2;|E|qFXZ`ZwH;B?|04a*QC7fi(N^s>k@7#{NMX#qHY+ljnY)n zJyOwKH^>!^_jV_oJ%Y1GD!S(p7Y5$`xTkmL&>q#_ceFg+p!&3?^647g&qxi1+P*SK z`X5zp40qN;_=^gR|6)6a|6uAZf!jasF+N?T{Y+A4jKW_1D%uL@ST4Z#Co$-x{UK9z#Hlm=plh0v^ECBAXS$K zdXO(Y1&dLgflFw8Tn);lAzQgTjj%`>aEYhgdf`yvhuT`&C?9Vn7^5h^*U)a(kqFS} zLa<3ziTP#?0WEG4>X3S55qg-KZxR5I zf;}`UwBcCSsv+p2SyNmO?SKtiqKjHY-UYyl`8|t584g08xC!0_jDOZN?}MY^rof@N zK>~9C4vCxMl4~4Lu>xSDimPS726{nZFC-g50W5k9UwYmJeY1gETYA*col+}fl6h>Y*<@%Z^hG5AU9!>4zM`qU;HPmN<-jR%PQL_ z#6?Srimb%S1_)HRS%8CxSoL#~(Ks`d$8>HM^(ntYQnw`yn~h@aX$;T`z;Y0XIyk0) z?k6kLMh5gsRKLN4Pf@r*ixV9=N_MJ4cBF9_13aE&j0o&eeC)#Dk09_{GO8^5$B+Pa zX;5BMgAGtwWy#*CG6lXbVsH}xLBn4A$ zD6NQYd6FWUTkh|WZY#WlKRk&FLQ3JNfz@YlfYmB7!30UbYFn1h0Z4sTkc%5cRWUp>Hu=(6?2DwRZ$OZJ3I+Tv*2i69=s9a9nH5L`;yM8r8dD? zcj+*w9KJ1}ZGd8tr}CnLyJfjqDBqE+-U?c79csCU1vB-)3RM;i8!BCKS)LWWKF4-0 z(SA@;ptNk{R!XN#+j3V*vK5~&NmPuH8gh{qK5Z*oS1Lh$w+^lu4yH7p zTB5Wv;b{~+jS0^V!LtLkpt3v8QdD}%e*3WGtFj+)b+B+gY4i2(*8IKOH&CJZbhmGy zO#ib|62l7LK#%@s-6VuLo`GPrk2m%e#Rmcs!kZ}k0ktWGo5=B+sMQhK3(UmREnlGJXgVx7Yxd0`=bcR{g zYpVHk@;w96!G2=ylVAxdh+tVpMgv&$Tptyyp@I=fWTkx0x5{>5#7NKz9R|w|i~C>9 zA)_pzGfW6vx=$l9ij6?dkf;I|bQgGlr<{dku@AoJ;c?k}$?^;!Unf~`YC0O6m|-iR zl4wXFS{$Hc^2Z*cx@{l~_*qd<0?p9foC9o6&XhdchXC;tjcL!V>lzjf1T3hRviTVN zf2Wubcw_Cy6wvJmzexsF8gPi&Zc~8Z6mo5ncO7YWK;v-!wjp6L{uIoU<+}9*YJQEp zub>Z~1ZDt;0XRw8>s$lrxGBiqOSsn+{Bk6{&7BVsz|b;)>iEj1_>$f=^S*SAeF8kBYxK!AHcTL$66OZM+_0=!GY3%2 zRwbRy$+D^?=c4s4)yh|GOV(|@>#STVTRs4u>KkfRWyw9g&Skx4EGjk4nc?7PcfJ*W zGoG*=5NrpM?y6+54@956pL{^5?WPSvZ8UxPu=X*J9Tf~G;}0*v3(d>>mv^p|tn{vI z=iR&3?4AGo<%Sl@J>t+J_-$}pFCsoF^7WOdKWe1ERZC_+bW2eQ7*gXAFGD3BNa1m*!7UBC_#qSj*wjFd)Lnw|@04h7PqcDp9;pJMN8K0Bz}Fg7l$~a}DSDD(LT4tax<>d8gYi>) z_foC+MMf)t%q`K2(QZoeqNjFXQJ`ePZ%<^6nR0U!jpGPh9t%d#h?<=pN5i6~ zZQHi->+pHvI5R%U&cL@=Q`4fht-TZT+NC`92iSj|1a))Qbt$jL@HoJX_+i*M@B}#b z1=-U8ZuaBh2??4X zV8NRcBvyEmJ}!g4?0ZmEmX3b|F+jypB8#p9%sxryF!mc*&WsA+6Bzpn6!>b!o}}xM zvEPS+$k-Ckm8EzNp}0U2loiUq+=1NZ8|EusxJzwK)^EM1Rc~01E9L_jXkv zBUej|+)t)CxlOROWjMJ#!^rX{1SrUq3y-T3!>vFho8jVWWI1F#U?E*P248I<^8p(c zWtb0Q$bJ&*(GKYjOh>yUHZ;?2i48px8b{82keU z$Oh~`LIBIS3FH4B<6xRnx+)sdb{+$`8c#()Bs|UmB~VMOc$@<=6XVxr?QqBl>>aG* z$dt%11!{e`+oiU<^YHitXI)8O$?}JO8kyr)P+n$^x}=j*Hl&KWyUyyR1Irytdtg1o zQZ^A~)k`~;>y`rWT9V~viFv)7u*>?2W1lx`*ZV0A36#+7;d-;yGz~>vPCV9N?OOKApF<# ze+DO}Q(wmeI&UpM!3TN_Pri9q>f_^I0#03E1s6K|A*Zmtz%6MsH$zh4$ywdz);oOD zL=^lg$C06_#Ng{E_##PhJObQWvsvF{c69S)0TG5X6QJy6>2?jk$0?Ztn6^D`DC8ss zIld>11&XF$|=1uTmEdx>kK^pp5Emu zK($Xv-&#=$?Xemqmbs7BW#URqL7!*zY24Tt^JXNTJ1bT3bo{R>a z^L5X&_&X#0;61_6qW*;JX3{SSMZYuSxGj3~Wg;rTvIB=C4B1)etfYK^hX<5JHw4)* z9}SLAiAHqQhT`K>vS)=S3L5!Y*&dIs-0+nz_=^W;B+Z58k=7q#@HqscTP`deLqL*B zP8{fOW0H;uvX242{Vc#2v0lGX7RB#i7CP(5qW3t&#G;gzgWVFXsfUz5h-i}Kx|F9& z9VZZJl4Ui7@&W$eK~k&`c$*`xWoY^@QChzO!B!``7FMq+j}{3DUxvS5!`!{ zW!1^b22dY7dqHA*N*2^0v7MgFU6;BNj#|M{yEJplvF$E=CoX*`)GO5VE*!q&sk!U# zNci^({=KXIrxva2_5N$mUw!_sUsyCHODoc>ejXxP&D}5CX9;M}% zFJ8Ks@HE`=G$gn9ubsJiCb4Db+LoP*{p(W!%kFCNmuFR;Y-5_SH0k>=b>+| zz2I{Ic5U}J9wHEt-feox{Ox89W@bOARjw}|78USY_ucVSpx?SrMms;~GxZz6xNC;+ z6XSLa_mo2TsnOT(SAW__LYRYgYT#8$dlstVzYp5^q%WZh$)bxS2VaxWvXfqV*V*h2 zla>kUv`o;%TKe!5o##g%I%QAQ1ttUmCq1?oTSKVg6x4tp1aT-E}EP+Gx})fmiA zGsQ)Ch}9L+Ve;Ld{tj8zD93!{fl~vfpAtTA1vSpbGICwE9ZD~=i|S%KpI_l z98OpeM%N<_ci?aD+)e!0*ITCkX(;0+?Fwck?}Y3m-S30?69#sGEG72i{pXaQxD z6ES+fQLS~~H>tEO_YFA@Ij z)4CqiY=PhyY1GOZozw%IoAi}d>#VTNgIbr)DD-PEii(r1(kp4R*#?39UwgtqOIM+d zIs@&)Z!l#U6=*fBR+JBDu_#Rg!N9yr=>?Qat7e6iE2SK?lr}PC)o|vKg~zNK=^kX& zDE-PB79}^DR)n(XrF$B!R?1}`n?R+iRnjwe_;oknFuXtwB*yCHcy^d+VXPrLU5+cc zwC-&ciZwKN*ZbEr zu60*^swfgN@uvN2ngdCVDWUNR8sB1MP1B6qJs1^R1W(JVXIs)sCA=Miw`0}26HpPG z?)3rCGR%(GhkjEJ-QkS^`In>KE${1Af84FzuhFBh90y0!xwNSDm!F?CD(s7P|)O-xCaC-C;ImQ7GF z&8WdXzP!39f2pXOn!(p3uYuh_R5anQlCcxuU(Ly@0C#uMGaUsF;TDkF`1^UFMy7s1 zPhJJmg2Fe)KEU9QF_0F;0M<2P3)0SjzRWmC(F8w}Gyw~BBE*OqIvk3E6Ek{NzypBn z>>nV3I0oC1fEqf;{stfSVt~gc(ahy>xitz+%T8 z@c0~DI9y_?*Teh^AwEF!^cg-po(P{4!smE4&cF68n6WXk z7kw4?sSe;ROoF!}ylm9~-pU8jfzfqPbx%#ywcpbb#qLGp(w3!C-raQD-h9tO)U<6F z361wl02ItFux-P5&Ar&|WBl4sd$ zY71tW0%oQLQXv&ndkLAOc0D^=wM^|ikn7gY?n4VZE79YvWoze!eX@b6VaUVo|DV>S zW#kN#ZCj_$^*`tI`Okm(&iPlH%}l}b`~UgYi9`=Y{Wm6Zk1-p0P_ChwI20r6PPX87)ZsN_H z;bc2S2{%npi#L6!6jS3CNLe}KhqRJGQ@mxeCR?a_G#llt9F?p24)w04h>n%Bt<%cRIliV9^k1^C zvO-^6`8!(9wPxH_&b>~{ibBicQC|2ynVpAstjYh0mYSt!=_ZPIHB&;(xSQ150AU61 z&dsv8%~4qOardk4Sz39YrcYC6n4{Eb+E;lGzt@;gBkMzvFdvP>I~JV?ANSF+^8g1NY23i+ zAT+{C>EEKqO`HKzX3hwqg|l%c$g^^0NZDpBKD%7?5--L_Mh_i|2jl$2Oym$BKNJc^ z!qMZQESr(h=^?Ia>D;GGQh|I;*cR}Y zqe4@0L$1YSZlmQg;GO#jP|uRfp0ws-g?FwV^pvNkYT+!@%*9kKOu0N&3wJKB@K&|R zk0oA}D_51;a(PK=(wmDH-np7IHA^*7y_86w*3i^ZiiW=n=dw)2j3qWGT;z|#q6L(A z&unliBGZD;A!}pPd{oww43ij~-~*9ZkP~nnWG6-gCuffF*>(bce%Uk}6Zn^V2dBfb zAq3IDsb1MMI6W<9^MVk?JnJD~)Zp=KRkNt1phVCi8N-yq({e?CkBT!uCs!jB35p`@ z111y^WqLwH{$|<7r~Q!_u$livY>M}XkDUk#oS&Z%yF+5Qn-jvPc%geDCYZAO?gqVB)p2h4!oJ@zl)dkvo-#So#`=`8 ze#zLd?5RzAT2r3ZMNj)V=7QykbX9%2sw-91wOG{y<@V)#%@v?yO&c3h#)dDAO~0!E z(vq$Bv50K;gPI{f^NC+K)Nd%-q^PfORUaHAo76-Bvq5t0GnwCbg%XbbNF~c>%bKWo ze)~@1=3n!a+L$0W!*g_!o+Ns+xBt7vN<>%ewi1rovb|>j&iaQ zwsVjR#)GmYPZD}EPX`5<7j!D(9FN7g*)g9{Z~+;)T;M^z6zzIN!kGCpA$~dz#6{T( za%Lo#l=ZkpXGBFlxkEr8zE*B9f(MDyJnY+0U@905PvAaB24-1SsD!sr0YSpQ{!T6_ zk&vwTQwYvdD|)ACl+IWk7st+zrLEm5YxhFGWbOXay7SSpqay8KQw}!m=uSDh?^AZH z>Tq5>bN-Y-mf%ts4bfsU{p z0`+tpaUK*}>~u7cosn<-8Lg?h5lPu0P)+JfV}r00vWjM+g$y#2m%o9q=QJNNDtil; zjy#;UAQXXf@>WZ(9K^DUBZz{*v95V0%hqH-679rw`qUNroYjNk$*Y@EN~V-@0mZ|h z5Wg^nOmk{Nf8irN5MOe4`$RFM}>g+u(>N@G5gfL~dz!yWp zG)_dZ0A4I5__38Ya5@-{2Ve*1YeVILLW>hnVain{C@X_m%kN|pB)Zn;@h2t1X6Qlu z2m+M922)wa>b$uB{Qk6cTgtjEQ_+&CZd%b%w*H3{W3xXrQnqrbe8-Y?=Zcv!+0w?2 zl(9qV+_7ZbS=_oWW$i1p-lT55?MrL_qlZ5X^2VD=cO(h7`j#DYiEGL%dn1X}# zb@h@=0&K0M<|K$U;dQbdQNXQvCiC07P>=+l6)e7NZMDnAF-aOQ0c6Fys~-fanl!GFF1T$8R@-T-TSNwwlmCAc}iJ~lO=>uGj)j;I_I=W?PLj8kJF<) z3N@Unz73>rqq=X-G^cyIInC*xu3l1)YsQ($n*Cc+C;p18TX}ANh4&;X{521(nkDzO z=L|{1BpNbno;}n>V@PwH3Ta*kS91=KfTVE}O{F!@Wc8X1NS*>ccA++DysWuUuce~8 z5V)Zs&7AQxHD{VLCQTy4+2(Z7Y1LP=$MX(t(v%w;aF9d&9aZ1v4{%Vl<(sXxfcP4W z?I;y($O&*&uTM@j4oZU-%XH0nRz?_q4ONT)B1wbV-%jU%DAPhnnf*?_etxy$ULzOOI6J^MBJ{#Xb3KgSLo)%al3{o~HltoS6^qPF@h>H;>K+qq?7^3ZMRr1nO%-*ojJnYG zSfcYtP(XKLfPD!}IF60Y#97oGB?oUW9=y*XY{Rh=MO8~2gmu)EFoN~^F+gEi&{P82 zLyA6Xi-41*)^pS7vw#Z}jR%xjqE{cDl67J5lSILXEz!A>nFt^CnT3ITh5^k{n28KP zMo@?cFe&t5RqOPrkeJhhvK>;2rX=-bEtbj*gt8IbkN_q{luY1^WFN&R8HK#Wwy#l$ z_zBljzXzbP_z?(Fz#A&6t{lC5^hd9rGcViAFNV*D|KjA;eJOkEjSaW$8&@FvdG-ufW=)SUY(41u+sdD3jdqFR4-79&XU2;CR3@yx;%~yMW zYLmNYCF2Vjhx^e(Kjp4YyP8w3=CrFfbeeZPMu&1iG-Ymaez2oEkD;jfEW2UL?{hil$rkjqWnvO`vPD)Kj7Mmhh_hh#1 z_{YTGCl=d||17b<+|IZCS2unBV^ou7y{A?p<(5yC$T@<4d(CmTTGf_PxDt{*8AJJX)@AN!M>o)o)DK ze=Al0t^1S)#?k4&y8ljX!-@_Pj~}|In)Al~&04yqSj5&s(}7{zW;)zi`!$ zZqa?wr@{DU8sl5)A^y*-4#*YR{&_U^&w)#b7XzKIeK?wcBN_}5mf;u@Ly!db*qSG) zf`6a}dKYV+$vlc$C;;p?tMEbf=P}b#c6-6J0qT|lg3iO%4bdxCy&iyjYVSh zbU1#3phk+E5@jdW3LXnb!tvQa9K2`Q1rmk;@Mwo;X~;%_pN@n>K~dNZ-|5~fV6K61 z|2D)Et!h<}>k}!lfF4bwOTe+=D4Q^{A`d_@C;GI)QD`U|xfvmdaExpaXF}kdi-7CI zI1#+{AU6wc1unhp3gq#a{ICQBz-5ObG68B<$Pf`P*OYcB2u<0>N4Y?u0m^#03}EOP zkldt300-bRC=wd`CqoquV(c{xaB~4jpuo69T#Lw!YMCon;_QpunQZ+;-}-`hy*6(E zb@5*y05`m>;!4fs8p-QVm2H%IM^j~c&*?L+s;eC-SIapg0=3|guqh)uU%6!LAnA+N z^8iRRr;N=~%fOOx2cW{9>YOCclzYUU0Hi2c{!Qh(3{%OEA{PN-0$1@%Ikzq(7oxYGmFx$WjAJWCs;VJV z$!0dxWvc7a)f-dQFw()r>LI|HoaPluXEG}Yg4#Vm-pAecpbNq))E@^Iuj|d{_XWKk z{9HwcKXXVhfsxKyyufOqM!K<}^ud;j0{i)2VObiV1B1&TC3KV!D%HBMSC- z-bv;3NzD@nt%ff#TfPcf$)T}!i7a0xxCLRfpzQF9)t?kvfHPJE*f-BCm z#YX>HIZwp4P%!&ca*jL=@sX9%6(Ljctu zSF4!;c1+g5@q*z{7N#M_uXZdJVVHD%8do;wJ3 zE#OIbDh^nF_{3 zC-w)A@o@a72bG6!UG#W3dM05duYq7>IvnL;NH)+@@gT&7gSoK@`+$C;!Vjkeei04{ zc|U36KM|jbz!1m;O;e~y4hG{VWbM|?2g0(pcjLw>pr@bWr((h^96e6Y$hzLXek|)# z%0RQeF$4Mn2Nj+Q!#s5gHC=cLjyZ)G^ebBxb$tNoi?S(?`rSj5A38AS2Q127K!&Hn zQKDJ*3V`>)5lcA69m*BU#&vplOc?;H4Z0MrQ@Q!6D3HfgyW*A~U^v!>}xt zAkoiwl7l9La^^}7jm7{~6#f)Pp~DkR0uhqDqev`@#EjqsO-F!7Ww$a7Xc(S)DVr$@ zN3iBASks2bI&c>uLe%i6Y&@7v3uL%1P|h$=_$pK2Y8{M$K{D&J6Q@pi9*Gd&Z4buK z?OES}tIC=bY9BDmsl?`Bjssp@z|W%>E};}WH(qSc~dHXW`y#E z*b9n>_8mwjy4N=hO4vU?fF|N|pg|F{w3`O$k!`{Z33Wa zD@$8@Qq~^HziY|5d&Le8N5<{VRMkrjy;5c0U2jvy+bA_}mTI?Tnp$qmq^h?_Ra;jL z+6w!s$y95WjFl@MgUOTchoGRsPFle@mvb7jn8Ap}v2slIUz)wU|2UR?EuUt5>M)YhNr+I+XR^+wggh}1p+owru68uUM-4jlitX}q-$3u zXbNEC9p`sQ8+<7X0K&Udmcg`TBxM=76_bvFp88A~ekA9P2NZ*b5@RwydKlC^0RwAbxeLdB%gzd^a`VCq3&WD9f62N1 ze;tbiVYO#2g#H7ygYVrd|mHHJpEvG67|9R*;m0P;0q zZ8b)h6UBHIMSL^R$r%Gv%&NyIR0oOuIIxT$^tiZ@YG9YU+Q3Ql#s~tLg4tsqS6r?y*$&*zN8^(u;?r zu9u{j4ofe8M>-sk0#V5mTXIh0DOlOXedqUG5-yBA^p*hbR#?he@#vvt4NP~RqHO~x zvON9&?%WP1UABS8;s%)B!q#C8^>JGVgr92Khr5|iyJ!eQY9|qe4G}OmA@avLiN$A3 z#N<|sE}(}`DZ!B4#0+sa^5+Gmj($s>2IY=}XPNxQ(I))EXvMMd6FeK8nK}l?_^}CP zi?9ejAXZXj%NI5d*&zTB z>-Hxvwa})DJ1#dKfmS>g5fylI>vrlF-oYy7<4PJrxaV;13}gfLG>$L3Yz&Z19tgY&ge!x)rvK5T2R6}f4Q?47OvzZvZ-_Gbf_stqz z<9*`>UD<=aMqS1IXX$F4`F{T~T4#RnEaTQyJm@m%DwH0vj}ZLw4=ytP9-L8hY`&lr zj+(3Kttj=#qe4qd9EPe|t>#m+k*FTf0 ze`c|MHyGs({n@=BTdc0LV`RUeESLhXJrGFP%_z^<7nR8l$h29e2NG5mauh8O<_0=$ z2@A^}Qm|*pL4bD@YwgJb(~yUFZ-VLRn~-TSVQ1NcD576Pg9Zv;$3hdU*e3&Q!k0Bv zW}=5d6(qb zz2qFkBFp^f`}?l%yD@vSWwCj;)HEo0hL)VePqw-#O3lN!dL_@ul5-S`?enMJPh3wd zblsvC+lHi8@T&HF=^XhlC^F5r&R0sFjxU{^t7TMU?}~}iRzCtsZ><2c1}B7I`1W5k z-3&;5ZK9n{Nr;NjJDu<=2wxRv^}RkugaeA-#Ihh{#5LOt_;nQ;gy diff --git a/test/stress-scale-tests/tests/__pycache__/test_stress_operations.cpython-312-pytest-8.3.5.pyc b/test/stress-scale-tests/tests/__pycache__/test_stress_operations.cpython-312-pytest-8.3.5.pyc deleted file mode 100644 index c44fc824ffc4d7253f5740321d4f0f018a7a650f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 20282 zcmch932+{bc1KNt? zZ0JN;QCcB~6nq6imRZDMD;|{^Rk#+H7VDuD98(gJ18Xs9)lPdKvRK557QC)HRBu zI66)Z(BxM$tc_}DYFIbG(8?SAfS$ZD3>e6-ali<_nz(7$JYXKS3|NM(16ER28@CPH z2khit7k3Oh2b?sep$<|U^9sf3-_$7m4!AhOE7X9Sui%X5*He@b*HFCs&HSJSJn-b@ zOmD*2@S-W+Gt{cwm!@j@Uh0LXDe5$|J>VY*kh*5x{~e06d{#kq*l(wnO~s?Y@7w2 z?3@)+2WNxS$=M-wac<5D@7?2$P=)M2!Hdbh1ILdi1zr@N9p!~cGL{$-qxo-0My`ic z3;#cuCA%(Jc*oJ4MqU3c)C8TxQsf%asdvTSG_8KsG!-tCgdGY0)K+LJX;5nnsk>dx zfM0cipqxd`9Ho=_^Xzrsd1{F>ZC;aB2fPGKyfs6<@HP2=c$WyNZX(CD;u^w z2_HGE+_#NKhT}3Vgj}*NG0Km~I+9@$BWL+=JQ3jp{Bp7rZ^P%u&hYtq0)8;r)RPeS zlb!oUW3nL%x8Vz&vUxvJgOX`HgZG$gJq{Z`axP!kEGkd%M!@eO8-^7ckxlzXN5y=L z5xB*=-Y`ERjzQ~ct!O+ViXn}xjmAZpJ}V*(vFxM0?eRo35^o<&4D;==GlLO!nF8WEO-1hS^s*;zkb2r4A0g(!Mbd4gB0Ab5Nv~I`<-IR6*cb~Em`9l$++gWvHmw+ z=qO|Bd?ex?{AN{;QTvXO>2Vp#Q*a7C^Z`1{xOMT(x(pQV9?8L>}w534*uq^|^49YKL2iz9ed zKSbnYF$eF;@d6ktQ{*qm-&dKZe|cN@v}~SCn)!rz8cODAfO#4}!#oSzqd1aR4WE6U zSN@NVN>-_D4VB?JrP6xbP){Z8h0NECQ`Q*J`k^Vq^e!7|7TG?yJoC@~D`N#-6Hp-si&jJT! z`ABq-MQ#;h9$6G7MVQwkLoA`(ct!q8V}L(mk$CGxA(rHoY?lxj;S$58S+?QWC7x@| zSCkDLe>O4}PrgCRrjf`nFOEi{{4&0Pl#RIRK|dsaN7(8xyM^ARqRjL1410e~q@TZc@K%G%KcCt~+3 zds)DZDVuhl<45>Qqr&dgwom57(B@!*YuibE5R zMW#DvwBNVBZq2ft65F|0)%jM}Zy%tP`^wrz`VEcH0=GgFBp-bnkWK&puLTkSKvO~p zl8{!|fcJ=%VPP&rmo1FY4%wf1N$~3m9hh`N@)5GkkVA06k6aOt0W2tN3@!|xL}p-r z4U4iiGBPfEIX=!O`4vTkW^9ct8h$_7%wLM~qXad|RzL=Q>Z7a|lab_@D6f00BVG`p zZ`jk}3yJvHFo2ABBsMIE%5=C&QC%`N%*%`z=lRi)QPzok{H$yojU$M*9r1627 z3b2z0mo#;@%DeumYw4@$tbd#2-}aMG#=mXBe|SoltE#)%eWQD-=Z?Q}0rw~*xBb0K z8bh!tSIyo$eEo2?dbd=)JJ-1W`=?(&oo(DIHEzu{Z(3rgnmvC{X=`ei3{-W~Qh*9n zD;+!|`Jc(vu3OSlfy49zN)vET>6Y{qH1>NtIwn;gE7#F+n)Gy!^t28p_3!{qnQX6_ zUo}r{{HFcmhn*DLl&$NK>N>J@yQI2Z_bCmGzmC0m>H4MFP5=6(9NYN!T8eG|-I5kw z{O)1x?=Szbw7ik>RAwt0q>6@YMWedI!iuNL+}Cpa;iB z&K*f`yllmC$4R-TIoZy|1ftlFALWH%F~rE`$Y|^&a;gLwBN{}LCS;FH4{I03i)mU3uy{i|hh}R*xOfBghrUP`&>UgdH)&8ut zRkF6ub!Durx2-!rzT@y_9jxSFvyN8D(R!b$lIhOY0Bin%nOgboIV_7IU3@%Ch96T%K$8tM;sO zljPhqUzKrg$~yZbXWzph<*Wn3$4(xJxP~uR*HJfX_ItEH-e=nH(EiN9z8J@Ukc<`- zpm(5MT~RS8N4lyaG=b`cMQOP-El;RJ1)yFm%V{|6E!~@W76ZM%487~qdXB-*{-iTV z>yTZ*tl2`%CsYLh?n6xhOc7TmK6PEg1?+PmiQkf3z@l_u)H8izOFp%9@%KB_cXdSu z1Y>zeM_qXjV5u3+qfyO-<{}lcq}Cn*@&>B}8)1pjf=!%d5i78O%2@@tC`hLPn*L}5 zY?+XTphVe$&?7OH42wM2Q~+J*6K95iAIOGdV5DSMUjXnWqA5nfKni=1-Qi(n zRS*k?$-1PPSLGIul|2RRiT?|d%hdnexP8HM@(1f?H~sjzpB#U``Sz~QO&z~>`s(Sd zw`sxKG`DWjl<}PWWnj(ZQ+J%+YnH2)Z`mgI<${$r1J?sNr{|jWsx|BEkenU4%K98z z4}hm~?LSbuO8XSEWTrg9>A=-J*@_maq9t3=AystDJ+)A=Gv}|KK6Uk_tiM(Aw`TpD zB>$$l@ellaKMd65g0<6^u7ioCT?)2mgIlEFmdy744}$%70yQ(bnNu@|W@Gcr{NP)T zY*)Y3)t~J;DRrI9YKX9}!Y4omO&J(zH`0C-TXRGAdnyX=Np1yuMThk`hwB^>efX3zB26~X!2Mowl z3xGuJq(^e}ydTRrdKMg~K1C;oB+sF8ojilume&W`mZ$dPhZZVOs~Cr~{wnxF0N?DcBi*;v!-X5Mwtn0jk^w=nN+rXgLX zz@R=%7*_u$Ff5KHih+itT1+Rvpi9()GOR3JP-}pZ(FzJdrp4(=O$L~v4w_hT4HXIhVPGhyM(_)pEJQPAMxuDD*Zs5O8+|0!Y5aXt(@C*>$HoVG#+*c~I65W_$*#U&-iQoMlQ?~!v^ z*#y@L+L#54UrY$+K{@M}t5$XvF6pYir!YvCj6~tEpjyJjl03)8M%YJ2HjfR#)`kl$ zTf%vz4F)IRrbIJ>oa802n(0#m~2HR zUl@jw&-=QC7?w4YRR|*}7Lk1NfKvHUENg4<8=@mI|8?`9uyk3s;|3|?h;IA6RbPQV z#dSmm+;w@cRb8!`enE1zWH#)PTzh{~ExGz9^o}rV}oRDSTr`>@m5ZEP7lpC|H8XvvM*QRo$SS| zE#Ir3;b(`Wrmp$sg|&UZ@E(|=b9MFEx-O}%E9YyP`O-`@<7v)$svj^~);(oiGEy); zK)Tb1Z#zRv7Rq;!e(0p!4Rhw(t{wB2?o`6KZlKd=+w5cNCXU z?f0DvfsyU<=RI2b9)hU!t@&FsK?SXAmSk6!uiz zI6mY0&dE$g!)_M_XQwUskLfWfhNdavUT_ z{?kD>g=+{m#EPp#8W&_WNOc_w9#zhOOg#aTQo}K64N6Zy0Mt-Hb{88=7v0Dub`ljM zl(g0Yts!Et6zi&YfCQiqkkbt6{?KB69-uIkmFQ``+Ac`LRvmTaB$0^aW?x3i7hR@+ zGzt=)Gp7wOhGn5k~h2*;!OAfMJ?Ce<{WgUt>QAomUgfU2Dt12c@wB;ok z_%2e*1=odWE8)vx7)S$43gcLrZk20RGTLxygQ>bBsuV_%i|iHTZi(;`($G$S$V4Os zkyQeQDhV%PGJ(mLF+oZY{sksPx)_VnMff`Aehm`@{W3j7s7baF0t6GOA}bJZsL26&F5lFJ=^@D<`h}}`a^Kxx zXm-tPMaJJc`BcvB%evP}?zJ;r3+@e*JrM8VtD4?A($eA*#Ib;lIHf2ot0yw44TzH#G!2VdYh%KnVZS5=uhDf+hv@06UubE_041 zlvfVn1GN<~IH5P)%J#wXSl|-HcO2LyZx!$?!L8-EgN`!-a?VrcaS9yhYY06RUE(Zp ziL*ciV7LQ}66GIR@(yrBg2Ec;Gu6ZaE<3xco#QXG0}f1pu@4@=Ev>}vcF(wizSeL)uF3eHSsq9JWBE5;WM)B40yWFtB(q0 zoyTS%25T6m%n|x}T)Fmz2|*RtDooZY)wUW5l`sDebS2`jfodvPKv5OVlSY6}CR^4R zkc@%pz@l+2fD^}-xfkZ1U83k-+6&@sW|!pc$ap#-V&RS$~J* z@0bhBcm3qK1^=c~)PYCu4mS{_nblNqWwJy}PK!WKZ9W{+xobPaHBr>M@X+xMUy_73RYgKAUj8*p>Aj=v`kw%(j#-= ze0b72CSvCANfwl45PmQZ&XO#M%kr&pR+W^6>hdMZS;z4d7p5`ghFu76D`0up?^_-{8OwgXbz;#mA_JcOEib-)cdNF=jrxm zOGN~#5*VV*5+s)A)n*lq6mIqj=fv*96TU`87!?R`;Goe0d`8JZV$MLVB<~MdRw3VZ zH+v*z8Hxm(N>T*6tL^_!CKvHX9o*5fHC0Rp+ZyOm4+BvG#Frga4e%j zo)4k5fG1mhNLA%$jCK_)%CX?7Vk2k4?aZcJh2GG~1#PgxpTpJw#};~*Wg|DH#KtP} z3=PDx>G)VQ%JUr031N5^BAA@PWDt^+E$`oAiLNE!HU{sP344a&4vz|FA?G91AyVEx zRL9gs!tvn%Qkrg7*o5yJ*pWmMyl-P8+y)lbW;im=4n{73VCMM|R*4MaAdoGQ43k3r zvJtHu*p=|_paJ18A(2f5EKBATvI~3h9b(jc)Dv;FFINH;=Q<8m_!cHnOmKd}%b4uL z1cge`5S|y*0*Ya>h6+$7;~+L~qB+@jAu)BZdsCaGs)$woq@$`V?SU0=w z14k!#*)oAma|h?PXLcTWfAIZ*jPKN<^Lemw&-*3+zKnC~u5t`20qjgq%PB z^G;>`3^&^~+dTL5;)%~^PKUFn&q}AyW(L2sctV(uzCHBTPui;rt#{Tmen0qn@Gq*+xIS=wAY0Qa)wJGY zbnWhkl+INFD+Ihx=)&uKs51dh9#rCk<0($yk-EZJZgBYCGo+ z&L5UGKP}Zht-h#U2X{uv7`*H7O@gZKe=IhMvDK0#Zk-*R9hej5*UVRBe7hE%yGtUL z?tt5}cP21n&NTODDxY2S9L+U#{y6#e_*>&Y`s4Qpq&?3`Ri|!yPTc|jCmHHB+g01| zmMqZrBXI+4gP+!I?b}HG%(uS{e%{+yb8x5j=Z<~GgWI%!?Wf`4uWP(`zm0+WWv~b- zkieqPt`TrdUL(-|Gid|{&{QMuK>f%|3J z#Q8HtZ9@1y7XJn$k82WcMKe&QpVuN*j|%|Fl?<1bg~Mc&_!_ia&>))1X%HHl=dNeX z)Sj8p?BV$}Z#GT_jOfisdn>MGIyX4I0X&x+lK>T!aLE4v-tC*%Op4Ad6tA(82B;&S=S z4jzH9HW#RFOGgJ%oJunvA#hGT135*)frozrK^DnQ z_)j=>Bs&O6)?#=LBaX&nNyP?m8DD-GlY${YKtKRNfcxPG9Y7F8O<+usDrGCdD_|pt zf~l~CMp0SIV~c$7j$I*AvS{I0La{uSvy?3_7eeAh%UFRp*G?LLf1so1U0S)VL zr8i3m}kLOU;F^@1-)dRK1>Tw zMD@NED9jT$G|bbxVxB=T9DKTYf>|e2nW`fEvuOS&x@rF40&xS#V8B$vXv7qok17P+ zO+5ybM>!~G+uGV>XYq|9n*|i~!oP(CRJ+1`s`Yre7D}xDL>mV91BW(1DM=F>-XNWE!gug21()+L57c`VvP-3^G7(=f8x}P2hErEx0b~gH;)NK44Q1Wfgv|I76H`Ltx{@jj;X% zX+!4Fxhu$7}Yvlb? zndiQ^usf33btYqvE*iPKw7>(f8Ft%z&7$$(9We9a36jn$w)_}2EE-#LzRIkxRr0mM z7-sAcpg9N09vH$*tyI}DS0`2OLoKuowNUe<1x8Tc1`3%S8YTnSKHu_o`&;erug{!# zVWImAneE|>{fmpn$g%-!ntNf<*z@=RLW^K<_E%;7&62-4=MGF)O74bC<2K3Ng+T!r z9*|kzEqQmq5CSbQgn$_Yp^>U<$!mvX2wUgU*=+}$q&8=2|`T_-%jJ{4Ba^hmV%QSa7rVF&!!t%&U`}&vx2fKud;yD z0_QLaVuHXnPu`1Obf1I&PuAdh>!XE3Rwv1EEI2bh^iYF3PJQA1Xi`|#0+`2;@v3(*A9(Eyaj zlMi5c$$46Ee|o0vciHw#T}Q5A!(C79OmOxaKV8KkJEyG0Cz-wcbtPm z1_&QG%MA_e8S4#j)xonZt${Ljk_Q%R&Ur%zJHVCdd~64x1x{Tg#TFVJgO4_h%n%(* zsT~&w(KnV9#`qm9mry)l4eL>94zas;vx=GuCdRhpr6f4R@Hig31&faGLIx%vYmY)~ z27Z#19r;u%IH!{l(3u5UKXwN6*s+wM{Y-46T^vklTB9j5p7c=;1*RC%I&1{;MNK5m z=x8K47-EzhCN>O_?*tRex(L{6Wvf#2h!P=YisC-Tji1y%x;xHQ0JJt+|@U92=V5jJvM7{w}*A*SO&xqiyoRE^74OvzS=- z5@o8ZfB@B+(A_}&%qeKB&3BJsYTyM^U2)Iv!ky{Eedo*XyAT~a1ssVSM1{WvVhj`0 zb~r4X!{DFf#vr&ZEL+3jFO5awXq12_3oNN9DuFmc4UArR3KK+6B=Ss=i~kZIzlzCs zF+r(GG@YWD6#fV1Atfpr(Xa6ioq5V~qM|Ecf=p7x(1=&49OJyFu`mq}w3Obpgm?EV zTNwY+01eOgG%g0{N_mC1_nVwd?fr%T6TH8pc^Ph??B%-m%35F(OAUo7AR%KwT4p5VlUx*QW|zn)K-3Am;F6U>Xum48rvOWaa{)Ox_PaK;FPp-6yi-Vaq&2@21N&7eVjVRRNJ)GFgS1ZwEO6Miq!(!GW`@xh7!i0=a3(!_z zG2f>6g#fE#&nW|`+0DxOa6W#|L`YC^;SkQU0(%oyLi{%0hW0{w*+u}dx`s-Wsvh&8 z!CojLn_->uuv;LGn#aJmFgBw8#Tcw83crPVuOrKT781Y%M#{Se&N}D(L2$v400r;b zT-}HF9KeuQ-3F0Ih?2v*T z3&D+(hjQ)PCl5|toW5}NrHpUgjF|DXNDxw5(sXPtxMLB z#}92JSW##4-m$u-j?Ao^ZF;>uvu5Mmx*yliM}E{aAIP}(8X_;#jiE}} zcxEgHVd?Nk1FU;T$HRrEV0d&KzkM4BJRZm*G14aRM7@+vg)BU8r{tNTG8PX<0NE(F z0&+lMFC?-V|CEw20s-Qp;z`uuE9BX-yGU$pXUCH8caKCR!rMQVjK#&ayqg!or}KYK zjQj;9g(h%U3LjyDs-{9BUD&9J3`LQUOxDt&Xh4%E$XUsH4QgkxcC=|9TY;qM}`m|>pY zdb9g__sq~-<3d$ergA%MANzjL*6^i$`ohiB_0(+JeB(mh-c0SjpSUvip4-OWdj_hu M70%`8DhYJ>f9_6}v;Y7A diff --git a/test/stress-scale-tests/tests/__pycache__/test_stress_operations.cpython-312.pyc b/test/stress-scale-tests/tests/__pycache__/test_stress_operations.cpython-312.pyc deleted file mode 100644 index 244096b4ba86f728de5a80a4d79ef0c770c077f9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 48424 zcmeFad3ambl_!Xu00@8}0FvMi?gIBs5=DuUC~99QS(L5CmKll0XuDFW zF1kiFrjkmFt|()=Hq&kQ7v`&;DW)pbq3tB5opj|r5eiT!^?d3n_sn>DdOph5lpT4d zYkucGJm3)mwK(aSf96TN@8P|>FXx{7JLjJJbF0;?!gcZIZRaxHQ>p%ue#nXdX>pHj2mno}CwHN)Bw-6`FO{*->iaLS-2Kic865#uQ%d)5t`M$D(o zBk8BoM=Yl-?45qtI$}FzW6$d0j8hpbKKm&YL!N{N5!RmQ^gtI)=0TI z<={-;RGo58sJ-S-=vB8@8#VL~5Bf&OaUUBU7(5%*U+x?4Kj&3P9mme`zCP~A*x2x{ zE581V<6}I$>*#kh?Fc{C@ALbkX@~ko`_B4!__rVOjq`*3{;sj%VP8M_R!5Co-?(pl zaKzUy5U{L17q;5s-vTqlbGMdgR0)RPqZoi$+4(a<7sk`A*1}1xXaZCf9dku zA&dN!xXb>KQanCI2z$b({GKj+ICsmi#UTydN9I4YcjH8izrRVyI zzLDXmn)hZ#bz>KNqfs4uVezjD2{d zuvxvnQU686E{EzL?(_S-ny9va*dJ97_-TZBJomRZ43G8q4L6({8}T&^o;laYa}B-$ ze_g+Su#SrCX9bsB#6>;47iErh@%jrB(R9CW{NjaP7Jr~@jj>R#@MJ^E{%devR()*E z6dW~kJt1q;{8P7c3tumKt!PfYoLfEBb#?#eT9vK&oayz>qoLwnq zS1xC-o6=rQznxPU&RHkstXs~h#&65*+~RO4j9^ZC zRx_zFsnFNpE%_RXkJQ8$PWO&p9?O&3pf=TL->Eu%UZwg2Lt;En83xoKX~v*>XPgYm zvG~GnV@2bA1k>gRXRf_nCO2qg0DWJUZSq39)M5i+0 z-x~E5rvQnRR@Ik2s9&SLei!wXrl_wp)R%3I`pSJJ?Q!cTsJR3RakX(({boFV_{XZ> z{6uNxr^inrseXTt>Q~Vh9EsK)FO*{+Dw3ZPcg~scIbkVNjmPQykR#zsei}+a)5}%- zwn<%3Hw^9RiVE%^Ja8_svs2f$}s2mLz@E@o04RKKE0IdjO2aAF9lE~);d`nno4v`7cu zJ1C159#0~v!X@xpq+vytyuRnU$gB-Pc-qP^R_{7)2`Q(ql*WP#T;`uS& z!zsLZ?rYjT-S=J6m*GLb9NypV`G2c^`3A~nBOI@mr#9qU;h{i z5raCa_%@0rQCUZx>1y{pP&at%Nv;;WL$w6F$NL6ZHDOG6z_N{9G^m+vVhUqdueR=;s_Y$;l9C< zs5eD2Yk1YsQZlOd5Bq!I7nBI) z8yB=cw*1f%Zrdle?GrZcUn*UCYN<;oJrXK->YsnTqNgxlVp(C&4rh483{NR)8xiZu82QS^WX9@1wxu@p3gzUzUy@`c1HGg@p{Qvt*-1q+r^sKAf_Bd7V zT05P)S~c%^H2Cped1ubPcGdeGozA`anxEP=`0>-$!aa8V&oW#0r0YM>8{q4MbUogD zVAs;Kvv99l`$1#jUZ-}+smJqDe&N22`lYtkea-q0YYm9u!)86+e7I3d&+Uc#w`o7h zDcs+#{it1!=T#(_78^9re0BuH{QBDLJ^4wxIQ2UUCTb!RtXmxZ7$Wc_=}HA`fbkrp zsWcT+&uP!E!{9!`={P-S;M9X^E{!u{`H=$n;7qjs;LOZTU&D`ueYG;zc0zJ@SU3eW@7Px{v6++V7u6G)U>{>o(mlbFx~TmEKXz%*j~|%g4~=oYsD%Rd zuz>rxsEr%s8SFQ4#K({L`99?DEq&D7cVX}uBIjd3QqB*Ka%_T$rpIGLBIXNyJa`(U z=Ld(SJ2S>jbbF0_9%6_(2!mXWbFdkR-;1yOK63Vx!$28VKYWiT!y;O~n0)lW0iMy{ zJJL5gI6&x!#5H-?6u{GJDNwf-6!YQPSh|oLlfTN6K37)V4;})qLOlmU(`?xNg%z zgSc*gq`DzeTJdJ}8`X0s#nO!nMzOT>u1Q~(eK$S5*fE{H;#Ot4uG_BJ!uA%?-m;J{ z*jvK(U7~&0y?m9u_{$Y7rF!3=h3tN#pjh>K;f@OJk2>>qc(m_%^my)9GNS}Du(AGM z*jOh(;#Ez*=B%n;bGia7ttJ6+1~r^o1}CwIwg)vs%9s!lU8xrI*ECnFbgD@$MwS*} z$oq&_mss4Vq5(3jH%Sly!8u7+>d`zzm{ih5pw2tGw_~&lsec2h8-fN-PhB7;tCLS0HjTj*r^cx**#%mMg)?V*XiO;M11Eq&B=z&jq`aBH2^o$JkPjh+N_rC2A3M^o~e%LTq#| ztCc|Yn#AJ)zvmF2|9^(_vg$uHZ(4Re^Mk6nmLDBo>{+V5)$#On&-Lf7Jr~ZZT+XVT zubMImu4n$yT{5*7Y|HiZYw6#%PIX3d^Imtq=8o81*Dcp9VSA%!Zv=-I@szEksq#wy zNu|rPP3u?8Dp&4|``Wg!vqp5*gq@9|vvGd!vU5u$r(ov9wU@#&4vl!lv$>Sj;O?w=c6&@Y_(VMe&ETWsqN zw>=}aJtJ&9E)<;zxlba7yIDy6E*R3%Uw^7sWgS=hiFf$D%&vCbbvK;;*s#M(mCA@e z`u*8QXwa|DRy+h{9V5aLR0FemOjd&puL2-!aT3F6A3yYCp@DIU2UUS|J$?#cr*tyO z#h`uRp=fKINXY4c`BAWAMD$V-XOKjyx3r)pnD!ggo9XdGKUTfvBuXJaJ$?!g)*B`( zqW4J`mo77J{N`1?z!KDaN#RJNL;Z&{OLok*KV1B)D=wuT;aQUQzHIH^~}tbhDI zh3Uf()UTcpz^R;3Yef?AEDZ#LQ#r);2%Ab*?s8|{(QXKp(rMS9A%#Yt&9#JFH zzoq;&FlL5$-K2}NF*H3}(Vk}NDQmaWM4DWG6KTJt+__h^SH@cH^veTi|r_9S`_pfEmA?-s=dR4ODjm{}}-o zg?NC1NpO;f5VXsKgyl(5by=?EU)KcKPfH@TpS*rhDlbvmFfJgG4qC{5^#2c+fAR_Wu#sb6T)iP4$n$ry|ax*|Ill->40j zwuq%IH=QjjnzXEL!l(pK(|p-q)&8J1+}tHLcL_~9g`!;{_wFYGsS5HGc_5HVIQp#M z+_UU_P63krCYg8yAhp7vGPlZ}AI>NfGsf5edp;|JYk#D0U8J-IkXzOP^*xm~EBl^Nm6azHY?%N0JpAvx<=VB9p|aP0 zanJhzprn@TuVudLuMYm;V7PUc*t$z-*}ZsXDRW6H6deq^58ZSh!itT&e))0o>QQZ~ zzir7BTSIx5(YJYCF9VpcPyikwlep-ge z7r{-8m;D;!WfP0!R132GK_1O5P@aVE}uR>P&ARdW{3%Gqvc@RY&XxlDX_a8RAX-G#rb z8wTblJE#Hlmc#tHx!fCCF7Jj89HxFKQ%)&y=kkMwcM4=ab~5cNm98+kbmLArg;dLK zxn3rXT+yWI@%#pj0W4pFrt_ei{C#LO=!a79)Il|u|4wnDG~hE*EVzfkeBisnxHPIr8#t>;tBFnc}c~e_ziMW_4sL_R~qDAhg`2i zk9_<(^WMOUELOg1MLVYAqiPbVD115+Eg(P1y<=62tFPYw4eFy|Rm(IcwM;d*Xp$qb z3*7szN%L1qtCsbGx}bS=ohnNEhI)_}df(Du8 zcH&wRy;siJM0Ik`ysd%i<751JAMXK}OiUede-Z-hfxO=nLtQ+5#99VS&&G6CJ%NJL z0NDF@S=KCt0YJ_0tf&Scy7tHzN9uG?xQiMtvqWO@?5JV5?~HHQf8R!2Z5>dp@xH-P zLdyGwFW}*RTEm&a(FXsy`>tF@6FngBLDF^-axf^sQO?X5>kY z0th)uJ2)~p?)Mrbk+6Pn1P~gjHjV&A=c2kk{;WT0ky1WHh4V*Ean>|yViMkNu-#zA!k(^+<@~ zTRQ$M0`r8ZNumWENrBlSBZflq!}u1ko%VDw=>SLF!Sl%#3cywy7wNO%v4`iM8NV*OE z=Y0JTe=y|;kF>T4m^}oKavrpBG%bddDnd|hK zrX>mN)Qq9&&ickLkU$66Xf!J(8Cfm==MN!7)QN;*;3w&|FwFodnHMQS(rAgA;sSne zI$O{10L-z}p>+(!DFLVu@A$4n(i9%`k6KLIPU^CF!3!++jzl=qO!wluxC7 zY_v|BZj`?6ea-vbnwdhOa>ruZVzpp96f$3uiOygvBa;CIi@ zoSv&+a4(nzM_0(Q6P1xy^m^`Vx!=v7+8@czo!axUBm27dn)ll^vu4pzJ$L-3qjg1N z%`A=Nm4x%wi+Sr8+Lk7Sy!Fd@C#Sn2IeFIu*8<_3Dlw;OZqv=2R>F%-@05Pu`0nUK3bvSsJe389Y*_njB}&WHP65b?i>k0()n z8;iL8*Y?l6@JEO4<*1;Xp)B39qZPUdR>!LPshoGun-{7D$F7iNccgLC;ySV7z|?^m zlV~Z8INe{|%TT#XXn|CU1>?-VKRfg0;2VQ~dVcn_(7b2KxEK(g>Jg4TE%ZDiVmeHPpg%z60oK}9Xx zC5LIB2l@b;_vLG85mcu7Fkg>1A9}R(TxQx|r~R;Y-g$)shQUY8{iJ(-JA^3dTV|0QqW33P+$lXJ8@qNJEzt1V4x3TNe`FxHOrK zz``oNV3JuoY{D=kmQEx$3+fe8KzIY>Z5T=c&&x4P8qcb*bV!#|45leqn6#jtCWc@f zo|hK{fYZAWw;3`zV$w8|>ErPw5z3Z=84acbdN&7+1G;z2ax9aUpeg0z!;&a-&@z+) zB$sm!altMf7Bk2X(nNfgQ~z;=(;*me&`U1}}i`!gS&XlL)4TW?4UGF$Nl5^fMq4OqqxD60n(RSY|MNekc$~+a!qli)Mmf z>A%SHq|?SC@?%~@=wTmepz&T51B#n{S4c>RPtx8G|6sru%});5s|bhsRo%C*(nuZS z>WF8~kg+b3lBb!q8skFzByLgg9&z@g zgqoT*`K)efMS}%4t;K~rGU$InqHHhO%c?t>S=aNg<4KjG0T`;p>%&M9)N3!x~4vJacDf1oZ)LCA!gpK8*v0SLw z5i)jRVz*^oJvmeLabf9f)9lcE^{)y$rcHP9N?xyht@gY1(`k`{qG=P&|HU&C@03Bj zTP@b_TpU`ie(G1bN2fa@B~9}s^G>07{q$b=%JpEj4i|3~i?>FKD(8&&c3ddzxTn|V zW>0sn017P%yUImZ`Rr36S5?H7b$!pZJ>S{4;#B4Js6TV7^2!h;-tNA(`#XD~lbBbA zNHeo%3^NmR)nak${0m}nn^3e#*tG8#WlLv&Ub|Ez9Dh#8K7Gsn{M{N=(bmuPDqGG< zvno4Jl9M-xSq(ztq2;XZDf4Zk6`C*e`{%a^+c?4IyJZ~sd_BBRoxJZSs{XEf$Ch12 z?N5#RUDyIdBTc?&4MH>1XWxYLq=M)miBw~;^*QhOuJuNQ$Y-^{q#u7c|Hvfy!SZ*F3EYy1hC{e6dJx-XrJ7 zaIi>WW9Vn}f-&1v*my5bjA^Wu*WBn8ANW`vgYNigF#=hF&S0vD%M#X$&h?~5u?Lkwike9s+rp$Bit`Y1255fgH1BBT_Ev7S zfUYhEx@w2w z^L8TVfP)S1WXU-PucHH(p(dCx!O=WH8M2ehFkx2q{Zt!>Qo*7aBioo5G!;CW#>AkN zUnzgGUK9zo74?TPXC}e+)jyZ+i7}T1$gN)`{(Stl&57w_C8f(jfGgFuoP!B-at^%i zK&b>&c={MWg88`*(=+iR<6udo={T^OJ-F0gCsBb-XvzD=IMU^A_w1zY7T^k)7AbKy zlCblK@W>>dfueTL@jjlEvD%Zd3X-A5>;7)uyEwWQb0KQtnl1v;o0Yfyxx%OYybQOmgWff z<-JxmV`)s452U+w;mX3$l4hxCsZ4Nmhb)I9B^8v%Us;N1VQ;Fu?!;{Pf+&<72xV}{~QtVE#RhDpPVh6#goPs|_Q zGzIkrZ{|r!y%ikWC&=cSG!79!XC?aq8pqyu1Bs>nkqiJY(z_;dn#rLxm4e<8KNwSi z>>3xu!nR>IBbG6ktWMylct2{J|ay-*&j}tK#5G%5l60sUMA>4 zqf8Ers!H#h&%N>qr_2Fn2@GF>KAP_O>l4Hcq2K}pP{iqe}Z2)C^4y`b$2WOo2vid62SFuaMf?Wzm zdZ(y0a{`B2g?ptdYmgy@g>u3Pf5<&}3^}D$^~`o2DeAeEp@vpcyyI*PJJe0muB8+x z84$y?IA>2tVXB(8C}Kj}$SI*-H|SJX52Cgn-g~SllTA?%dPO~$v6nQc;n2COA2xg*Rn$RyDlbPcc za+qM;5X7W&yh?sMv`&6X+~wFOUE?+K+o5XtDRGzmaSptv3i zG&#W>kuEkCT>^} z-zS)ERe_S&?lQ^&JSDVoAQ^xm0ZR7;@|Ujg7jR1PB{rauH=<-bPkyu5wy=bHg(MS^ zL&3_#Fl0IqF(6HToI+hBXM&vno}5eM5XIwPf^***+nVNKN*IBXP6|%t>4R00n2v}C zd*R|jl|Z4S^}$_)O$Dh~F>NIL1WR9hNESj7TtQNTL|Ce%Kz>3XL+U~CT0=W~y`>W0 zPw*c9D{^Sr#z)9u+FLBcu;AGIq(wf1Tb?Z8Ero z{~z(>%~ObV?%;)_7cXzj`=7}3KaxX)mH{achM+Je z36ApMhgM0cFKO*mwuU5V)kxgD7R>V)fq7b6DUMjIukLwe&mZo? z?nFn;XBL$spS)OK-Tlh$KO`BDBOh|3=8kaF4zX!RsA*@&by{rN`9{rb$?SV*7yGEc(##fm1+wDA%9Qa_jWrQWlsP>N00Hhp{ZY|c&Ry4#+z z*@MfTrs;ib+eOp7cRpX(a#+Ya5^_ED_paPK`JP$#Yum#4En|3*-%_p{B1s|@d# z=-@6@q#dr)E!FAZex%u1eYjcmcl9>7zic)gacY0L*>=RH6>NGuuM(Bg+@i(bXSCOw zEG3e4#JJ#03-${ildqI^60!9^AxLAE*+9aQFD%S9f#ug@X-c3}(gmQ9@{@FdeOm)U z#1}FjKzlWkF0g6}J^HJgNoger!7NxjQVw7jSiD>Sikb3nfPw@Qu4eJl2ppu4*oB8D z1(RnCl<)D4d!$|E*b_I5$(JL!Zl6asUPPQlz%@7u{ntxA?l9XgUr$nq4#wa>7nRmi zQQN?vZy1&_Al@NkyG%@TU+bv@UMTS%P!wT`EmB&vjou__V`e2=W@0`hzJPdzs10UE zB-Ko?U?Ug!1-w*D-Na+a5aKt;zd=fZx1%>i&Fn`nrNL&;HNeD-UgVE$o|gg#s@9tP zl04p|iS93uEKPJyXh&#Frp8Ed%lzK?cA;?VRCfepsNl7NaPDR?cQc`f#dQ#4G8Zuu5Cq`*r57adJ?{m56PFjEdjo2!ZV$i3O7_1(g3al+<44g?u(D;*z^x|6+fYzi8 zmrB+TvIR7K(x`|hJ!njYlcJ_9_>!VT_+m|L;|Ll#8#L&sm*7Il6zJF0jf`BSuH6)f zDq*^c+BGF|IUYw;gKQN3B^DzS4u1)vb8dp;K`=kfm!m_YG~2WQ*!6_Nq176MZ}R znzROi!!m7zd`0eZ+18dkhO6c?jl=@jVJZBsjw||Jy#5H~!vvPP@1#hJD`sd=)&uE~ zr5P!1#Z)z|86ACsxJy{v!AFjT()d`&UL*E`3A zrq`@SXqo zzBw=9m6L>GdmfBSkw-gME$4@~z<23`zR{Lid?|nM3s-YP8_eKpx8T#NIP2nZzN>sc zRGA`<`ncb>kxB|5sgP^_JCE--6{1bKrW)cJSl$!;wDC8K7xXawQ6)}Q9xgG;JWr5= z)FZEH^%w(vY^JiNMBpcAj%x`RP)lA$E7oDGK8mtyxz;t)GJv|Tzmb|F zX_V7W+_?>jyiMz_?t9D;tL3&(o1)ZPC?~jg(4B4H)pA?u8@CN_w@bblV^4xzb}`1@ zte{&?NmDpRS{TI^&%~(M$YwW-Bza8KMbMmzTjAPRZ>d*=m#F_HC6dmyKW2L}4wKt_1D5CH zy6;hPr;q44au`Sp%988G+Zos$!;?Mx8V&;t=7A+cIji4C%C>ZliG&%)w&h7be52>c z2j}4i`_7Jz0Y~olL$CG&)Sm=7#)9%Lu+V{m9#U;5{5OI9vMr`q;CZ?O`3gKcz7NpT zH-PPqo6C>Szl}Su&t9E>;mIxl0XK12k0OB-Ol4Q z7|EVWz`PlYPTHRcdndH43uwC=ItTAjcFh!OcUsqqxR4GNF_$+soUxaq{+3Vt#2BLsWO_pEt$K=(3zRlJKl}>KOyFV^-X{X48Ny-%D+Qp z&yreB(Nn3-Bn{DW)&~ZVC$82r2nki+gy&q}B_9k<`$lCWIv_KRGXw!(<1!6q{~*;sUAx{gJEzMsuk0 zk_=Sike0Jdz99)3vZ;tn?odKcW#mw0^X24Jl2b(v134J{p(Qie-IOko!GvC@H#4#G zRgwn*QXXRc$dF|x4pPe!Teb3Ir?h~8iW9-Tp zyf*ml^RxTK?E0xak&Nt_oaGD;v$@y`^QPw)QwoI;k#=8(H3Qc*LvW3KB@pPktyWl%qRdA9Br`$Fv^x74xJAe{V~(05K492UMdB8-d*d1E2hg-A*L^!_`} z?Cb5<+QZH=(OEY8^_$LSMGSkFa)hUjEm?)9W1Cq4TSv0o*H2wL70#*PP zSXhT%T;!NG-?OP)`7r)M{X%UW-iwF=MnM!d~o?{?7(xsa|QdwT!O*F{$iQgCA* z)m&95r}^JpUYPjtOFw)GjTPE>O6WZo?j03-N5j3BZuVY^>^$_r;dc**cRnlbd^WuE zjJWg6jCppm=*DKI?V`JrItz-u@2?L3;BdJ0fY^E<+H&>#{A=Ok1LE<4@bOXc_-Od}rJKhuA(mmy z6^iV*dVc+kZIIQ$e%6+-yW^(2 zgJd3ATWNPSZO~SO$S_i}bMdLgE+KDk*tPGbYae;S7t-^VV%ja)Hru|KxrnHChTXev zx_7atcEi_B^#l6ldTSA*Y;|+z=1$G?3ndFqA-f}F-j;E#KN|n> z#1AL_=IcwR#BImL{1dmZJ6x`v>(*}pKTb#7_x;}m^?$c+M}3c5^^<~aM_W`s&EJ6M zpSe4?9V=BW<#xJp`>?sDr$hBo;huH)akItLlcl}6#nxljE}ISTylmIw&2pBOp53ON zcI|SBt!KSSoXj`Z8nRVJdP%+#Hm6|yM|)NccubW_Mnragf_gIT zKB#`j@OBL5gA69F8p~lDQxXUimlLI2ro=ZYGsrRppp$9?IGI;YY{h0F<5HnS*|>CC za&ig-@+oa!5sXWmATFqys#2;p>Vj!7BOmkCqw;13+G$qD%0NEj#>39IS+#g!`a=Fe zuva(;11lA5v*pKdZVq3Kq%cS zYXj%I)G!M3iJoAPnXVy2HsfO#u@zaeS3`01l4*GQAf>~yP0TD74OJf5uuTe&Jt;F3 z=N3ImPM&2?ivIx41EIx6oL~0z4G&)OVG)zc%ubT+924cEj`&J!s*0yCofQOQ``1uq zf%3SHRKiEB@+mYegs`UhAysh=Ie$scM{uGzM@TmGNSmGF@%OT(lvKtb6)<* zzfa-5LC&9(^D}ZNLB(MxvYvl-FKe0Pyawuzp{Oa2!mLm2{}_6%|KA}A2|b_P zW2+IiBw52ow`g?FxI@O$6{EpgyP`D`gSnE86G_53jbct?#NuNAoe@{DP|_keTgg;o z(`PEZ+dO5tXJ?#f>--7mW!ScbjN9dtM4GNz<4P{oqOm%XT^z|Qg!s=?gH22(GjPOABnwb&NQ9E~9bZi#3bc>F|VMmYXfT<3jaCUInF@(%O^}-d-s1Y-uF0gP`$f#M) z*ew`$e})37{(quJwUD^}s}j8p)a^Xd30_Z3@T!o>z#mIg;wU==OgU^i7;Gh7LR_h3?M&@g@z25`V4l}@5Ld^^38 zj8zd)lfEgly(YEcHzQ~?7Mp^?@Ev&1ZzX3NoEX92JIS|_PMkmUBy_?dje9A7Qfdz| zM$2f5(l0namH+2xpE!M}Ohq4b)~zdA1EUXM;=I*l$1{ z7C;>igdK-LBZMP;Lced>F|amu;K3oTK^^`8UvLm~j3r7Spg7ScjIauPAfPhcXA?>J;% zV5XeXNp2o?3?N~mT!X|+k%2Kf+gCz-*W{$uA?B#Moxyp5Uq}Sz5FIJV;AHCTtNAF& z$3C2+fwKucL`&9Sg!=o&*Jh*+(J_dzz$w|NhlgIBzlV|}qkKN1M2-sM43Jb|gqgEK zK7tlW(Ew~>oKAucc`bY+Q6Ie@jOzH~@;v-@gpqh26T{-j#0-FAhAJjV7bqaXg22NF zrA}seNLe}#r%5oHO`>G1Xvk}zlBS%sIUU*GpA!Z9M?@E=V5L*Wk8|>7*1x{_wav5s z<@}ntPBFiJIj3RD@*wAWh2H@zT)~yK&$>M>_<-6?&6-%*fFP7s_{WF5q88NomCu+kcy7I z9V-_}P(fuWgu+tTBSL7g9F6ateuDhMn$~KGtz>jLh^21A>@t_ZEPL2*Xkl?%S#c)q74`3zUGhVN3YOpbpIAByHz zTSYGBPSfZ`O$Wi{vJs@I6MhO3p+MT~W^80%SF4a{w1ZgT9q5qlGhy=9G8Ic!hLs11pVfyiE4ZF?nu6UG*Dk^td+Y@EMS&O)%x% zj=#;^7OsQadR7BnuWj6Ru9F$`-obTo+s~4L?_y|b?c{dhT)f@f9&Rr}?Bn*c`vE*3 zypaZ*G~2jCtQU8)u!p%LvEFO>O5^0It5_j}9i?XjN`mR0UOMa3^{G^pT z3XP?&GJboK^YwTs2CYmd!FC>f5!yW>8_~2>NxHzPDMs{UMld6l0r8YtCC6C*+WD1A z^)!`=dqzn`x#Le%GMIelPTU|~li3!U$bEvGpTus&lfOZ}Pd$EGpuGwWO72;V0{!EM z=AKiQFtHysbqw5Tc|`MbseKI9CN)2P^51%VztZTRv@3FA=bn$JI--1^YWFSI8~1x0 znn?8z<|fttT)y)wwbj=iKZR6#Z&%NnioTg@6>(UmkC+qViuk zo>u8BwaRS_y`NpsRQ@V7u#ZuT2U1ql^ds6t&g0czJ|c&mUonRqg2muUNBqS49UPd|>b_fM=YmiZjD$IYZz$ z-o-R+*oIOl8Ni~|s(=d9-LSiL4*Oe4ai^b@bNcuR+NeVIx_oq+6CIuy56gdonhc~P zjI8s7{lHX$fMsy3jvnZ!raJ!rB5wxiMIfU+fpRfZV^{H&|KG?V#Wnsvk#n4!KOyIT zC+9zt^M8=@19ED}*#sw$liJi-43%#uFIjOzIB zds_t=KAvzSP|3j#&_D;1oCJLQ7+M!U$hJRKj3g^L`7HCR1Z2a2y579LJY&AZrWL8T)T>V{kY>O;6t<=iB7`5jo!>=UwFzW{;lsX(*eMa%1Fk$YB5IH6@Etn`LUwL!nB)JLA|zQme}kOY z$(beR4LEP5Nd)Q7=mnFX{RKU}2`6e{t$*>t5q|7U%y{8j^zmPjL#lQBzlO7l)&Vo~ zNHz^J+dT>`Q1<>J`8Kq((EBfD>$+z(miAL z1tq1n5(*PnbR#O*l|oq%^fR>z;9pbBiW4qa%tY!X@0jxLr!i6w3FzNY%0Hq21u1Ql zF^T5ilBa)5&Ovexk;4R>d+BK(IjpQuW>HCqA3=ruK{!$KGh~c_rOYqV_jk!jC<(4E zt5p8?$jgu6#1#0TuSsN=dSBGKYNk}!L3^J9eS;j5_(|G(tmO|;Ub@L)GSm;~D^-!y zDvwgwbM?Yp1nJ(fEX0fw1Zs~VLn-mb6uy56V&u;=FjYOr+6EqJJG$|PLpso5#yb1# zeC~Yu!g|5h88Yrz(Y>TG+3w_4%nss%`rnlPmG=kUzpR-r6mqsr?Zu|anf%$^?=*jZ z(_5R~-aJ<)9-SDZLJ;!aU%#QkI@9}7&@6HjJua_8cS#r;C|;gJ1E zTvZ14CM_zfD{ScyEgf-Nan`L6_gZcEWXCan^*1x>aR5hF$DQmg3#Y>!-C{@g(o4eE z`j$J+2wVCg)Dc_*clDYqoX}xVTJKs7S*?(8=R2;Y ztvF!lU}p4YW-Vo>@lJNf!bo`QVR0*DOT9v0|MFH&={n9~TPTSwFi3#JGSVn96HClRi4EP%To-Xygxy}z?G>uGESz0-@1EA* zE-sy2|K{d5HqZH&%Ub3;#j*{{#T%!AQky7(bc*0!o+_tADBCbUF@Iht*|D76CD^-= z4f)WMJH=Izih3McQk;EPZ@`%+j@4(LL_Ae@${SaobYfGLR6|{|YHk~BfL0Wtc&+Or zb@h?P=DXH3Z{CXQ;U}R~6wpa11vm+1%R^5>*{806ZPBbovMqXVK>d)88?iXU7O!aW zCi+e_>pM_)O+F4K*}`D^LYsUvNWw^hbW{os6Is!eDvnBt6jn$_M&^@6?ELKME;OS% zKgE$LkW+m>_pRKw^XIJd&n)a+XkUC*D0(X7K6w>?r;ga3C; z`sTdY87Jrtxy5T7r1D_ACzt%c415>}AMOsm?lX5P>!pYzu(CMx#!$F& zgIKxYfmA>apytT*Z-ts|p`7iDJs+HU_f&Y-nVY-L#C32x#q!R@60v;ejOliM>FlPt zv!VPAKiass>4RkvY`2!1=7hhQH z7aaRTmIJWfe>)e(RlbX{rb;Gf=})z*exli3h+iLh zIvaZORKKuyHsbbo>Mkd4qAL%d#6}aj?S(x#+M7A+dmP$jive$z9eTW3&e76yp6S>+ z?Q+@XV`bVjJcyJTnv2e^=dOTGL)ahH8yD|}q=4PkJKP1V@ch>eW|mBT!xHgPI9%wj$&BpJ)p zrwFNkWJm(o*R3u-T^V0h7(0Ez&+q*2`_U^N+#guep$YbssIgc1V4wOA}E z{lyLCO`3v6zLsr5cFDDyxTjPhFmjf6ta6T48U430Xi2fXF=#@LFRmFo?HftDAZl4F zc1Q9S=c|8$LwVBW2$Y-X6FRX8g^+#?mr4H;Ie%c|V?KmTiTE<8G@Lzo%OAk7Om;@( zm~>wSB)a9?07xuY6`PZCzjkccoCl)<6t6PACiINU8nx;q^dNDlPiE{0pRB+h;*^0% z$@cKr-aY9Y?X~RUNs1hjp)JX1F|?{AV|k=5#S`Yki^xBNHW(TZ*c^wkaQ;sm-dknA zE@p)E38G(%1VJXE@Bpih$rdZC^P%apnWs%MHvo+ynde300 zK>P!PuD0VtpqR|jDJp_3#{7x>Z#X@NaB78>IHHMsFJ*u&$o_$T-i8x358_x6ENHRG zQCfL1ozp$^ovos1!6qL7$l|<1pI(L&btsU^*t#Zai|KpGzpawyMC>PjnsUI_SZuY` zgJ}HJ4LylV($BE7)sjM!Hn4836;ra${|+;VzZpv=G8dCLU}x@-nAtvM023n}jdLe% z{%jB4m3kvN`7uq@>~(XMLQ|Jmy>s!bkkyS-Zc1%}rRW~E_f#l+mCp{$Jug;*zZSA~ z!&8xMsvCX^!F$AG^2}Dw<%y-O^YF73ehO_4hb?8WC>F9*Fki7R%vTXCe%`lSyk&ZC%9nqiQ@o-v=I&G9 zDe%00@U?^Cf~{h~)<{KNq`C#lwT0WjF&7qL^9>FdI>^-SN;iiJwtQ+-xjjh%kBS9H z69IeZY=PvEw+gnb)TxTP)!3Cmb<+~Ew4y{V_e|QiJ3gOO<20XFPJVIklv?HSVrK`k z1)dS73M?4s1B;Hu&czm?bWgZouUN2`c5dMNm-n(C;rC|T@BPnTpcLr5pkegI?{!t} zwP}B%$JyiBpX64<{i&{I_cq;68#MIXQnRO0_p_oLJpXK)X-`?&2gb}jMQI=8>hb1- zB0aq+)6#RLX>WSk2Mw8f4QWd%1KzMR$FbhBzc**~aYQsm8Mkx^qvXksBLb(P zFxiipD$Xqt=K}$eVx~&)Eh#xAg~1Lw4IBr+(IGPy2>5XxIGqK~R8uk`+;>1VWi88J zPLw6-g0(PhLRp2)NM_kfDrK8giiMbKsu?$!Lb%UKhFUTeWd{d_nJA466DbXmU|uuQ z0Zdn4knV|j$m(4zeLntv$R>3 zp=gYdC!jRyIfl^$M0l7!G5HD+Tn2CXiIx-5W#pD=<2}Y1P?!4uk%;TRM_8Nz zWiXj`s3R8ps|Q~>7`E&XEjuDvxmZ+#K3KbGm|JZi5E##{>bX{sR7Ra{!~gzQ_D{cX z^$?b>`Na`eO(eH?)-_|h<7p6zv6r*}k9RdQ6k_oSGbgGyMufiQ|Fy7_a(V~5?MUUt1 z{`lZRJrK*z2JIXku*A{6jPs!DKv zeO=NFgGP>)1|er*S~WBT)0svN1|nEI3_;GxxwtHZ$mVj`-Hqp5rjcXj@`9wjlh49J z_X>ly=!Oouu*HEb4|H?lJLI!!Bo3?Nxetwsn8|17;^ZAqHB?U;&M2cQ@ z)Nt_vq;ozTdYu?6?^8Ss0KSGNX_K~8d_SOAHhARWqeAo#fpnSiSR4?kuwJXd_>&b$ z(;ZXW?${5Zv**~R_5dWZIHtBnV32h;=Kh?5nPbzZao(TN5jN&xC*(|J$Vf~94V~OO z?t(~OIq-m7LOu)Y2y^my=UQnX6_nla)J7_52`vDGrme^UU|3i=r<*%F`w|kvxx@xl zBhDqRj?~rzi>t^67FSglDf1#k1sz&k1z3oFl;z=6VjSO7QQ&a^iz{{li!1crapSZf z#8VGELJurZr7v(I6U{AmfyH6OX4KJmtTUOJio91ncmh0`SJU0?bw`c8Yzu#HZ`9n2 z4ei`T?5pUFT6%k5xY##LmJjhO9Y=}#{g7Cn!-03D)EG~b!z>#79zFd5IZNdHM{@ob za_*7yPvqFB$(bnt+A+m5ICGk$mu`> z^o+XT>8xf_1sbKWkM(ve_@pKQ#1I{lmy!Tuw229Jhz3H^l~{%bHN7;;unRN{|KwP} zpb#{~u2dHMZxGLf$mpGo>t}i@$;m}Q8{u{5zA3!QjRtzWd}dy9MM@Zn5h%BT=hM6?$sLk z{EBsV^*^(-S>ioAksq#x@+?_3gN9Ux{3X?i3$P$r_fFjI8b(xS-1iRoDRGxeIJ7JA zPJZGVnFEp^>;NKm zH|au67bLyH&mo$5I{IJ^hGFyPaIes(hp_)7BtjW4(AA>}GC%j1V1(TDW?Eoi3i#f^-t;Y8VkZkn1HN5t^HE z>OEARR9O5>Ws4AtO&>E*z3xH(oRw)?ACQ*BTspXHaO3EesYa`lmD9!CtTp^(v#&YK zb%U$Vy`jgNA@!_K^I(30dT7DNX9X;9&6Bzpbtr$P^psfQq9)qp@F?LHh(Hn)dYyu% zdB+KaWEW1ikHx)uA)U&{D!N!L(M3gmjz5^6q&8Du{h8cmtG`T`W1w(nU>&JpO2?(g zu+OAZQ!d01;;4Ce@a(y9ScvN5m~PG|vyF3n8CyJmv?44ByPe(-XA@ZNl(tHp$ zmi+qRH()um1NPfs$bnUYmUhNjC8eL(xhDLx@D?@n`p9?{9g}fkj9HF|n#V4VU${8l z3)>OLu%I$BbfkCee85_9vSOry>#f*Zaj2pvYVi&DW45mRy#amw5Nw(8zf0x%AskwY z(Q&FUg~mMkFJkG*PxJV=joHkUtODtWp)$s-(!si(8lHB>RA=EyTi-koFv%wR%-$Mlg^)=ZcaTGX zS4ssahNtJySCW|UG{E^ha)|%riIL=qPVe$hx(jH_vQV3I^IRKqhpt&MvRsLvRa3OQ2NHtN#MVZYCo#uE{eMtTn~UtpB~obi@N-f zuJCqNzGN6{IjidB-QVoKom~*lt`f7Wmb1P1efYM!NIH*V*ju>sysy}S36^*s?>jYzM$hi5la@hNtN~7~y%rhHjH@vn@$g7#_5Nw-5#&)dg z=?vf!(O5EDC*W)qV;75tz3|SRMtFG-Y4cUviZ0uf^)VeF(FXHlokzt@Jwn?t;n+#R zaVlhaHezvvEjZ1jHDuXz&TXFBA8FV$wP*VB%%y8D3E5S%e!*2ET56SX56%~cEZgEy z|6%t^6NkPkt;CBv8r_XY?bT!CBX6|*%HOsec zVSANmubRtPwzs{!@3u4R`o?P;uWz}w1-9FQ3;Px}FZzVKgUilCFCRoBPYus>g-o8= zPO2EwE2inn>-E>_!;VJL(YWkrp6^`fyJg%?4PJr_n5xwR^chgf=?a$Snl~3hqf3dDp zjoZI#(I4ta`#V+P5smim3On--cW8gvp~nwFZ+gnC$N4%>>Gh&sk7v=Ue<~+U%+o(r zmgaRu4HqVe5=xT-(dwxA{6(kCB6`S=k`o8HqQ)%} z^tP4%m-vAHDnBiPAVE|AKV!VjzX^QELjB;Sp0k+1Z4*&oF diff --git a/test/stress-scale-tests/utils/__pycache__/__init__.cpython-312.pyc b/test/stress-scale-tests/utils/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index 2406b81bb653bd4b00865bcfafbc1f905e48b451..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 196 zcmX@j%ge<81hv-g86f&Gh(HIQS%4zb87dhx8U0o=6fpsLpFwJVCF|$pCnx6UXXNLm z>SrcpBo?LUr=}I_CKqSwrW9qCr55R? C?=-gn diff --git a/test/stress-scale-tests/utils/__pycache__/log_integration.cpython-312.pyc b/test/stress-scale-tests/utils/__pycache__/log_integration.cpython-312.pyc deleted file mode 100644 index eef6a24a82835b6f15a28fa591c051a8410db46d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 15868 zcmcJ0Yj6`;mS(2jvfi>STk?bS17u-a=KV0(F2GRW7Z_a7yh<&UvXPM`SEdYDNHy$6 z?>1}@fxXiaHZxs;E2bw)z{pHx0AFk@^ZWlE@lSZbjR=Fc4cKI{^+y%Q#xc=-p zS4v41Hq;|J!8h;Y+;h)8_mTH}_vkB~u84x?=9ed~HtnaV|APtfk!2z;{x?LXD3)Sr zKXoDfrZ3PWEx8~eZ|MaodCM-y$Xk9vPTqg1Vq) z4XhkeMXUl+8rH}vA*E$atP0*bubx$3YoI7@hlKJ)7l{XO-OM9ihBaeAC~nExG#`W$ph#kxb@kZ;86k*4dBPzfQX zUuc2b6cx(wq^JmJCJY4;nw313ewwL6QxOU02xU18i~1zY(gDwop~OCak5=WMW+bNb`pVLUy&P9Kl;N#MZ+bmr zAur=*Ji(C>cYtN^?*`|Cyx>@9bS%V#g3O@L?@iG~0}e?_M&j~8{}_MOAxp{mt6sl9 zB@20PhEfVX#Ckc7yrHr~%3-fK>=37cpwFR5$wxU~Ae1WN$F7WWL64W`Q&Mg$kW!o( z=sfw}05bG@Z^CQT?+bk*O(`>Saj0TW1p$vk!!XS1aEm|aar;}Y21mRtzAIPV9NXd@ zdHGNaAL5|-W*&HWn=yxP84LORJg^;dfso!I z&K>dv1J2R$l);-;luHz+6ok;CH19)jn|dx&-?2}0-gQogV!NKnw*5(|T~?Xz>+k9( zZp^Upl7AXrYCpNqep2|^rLR?&pQnp{H8yo?^484WL|J3J{-4529j6vLP6_8PKU2NG z4E*XR24+gWmTh?fEl_KwG)^nj!izl+z!Ik;)ZfFDjL;BDKa#UDoYqlk-c*-HB#-5} zV=sR2)TU$FSn|@Hg*Nd{0wS? z32$MV0}N(0^W%KTJHj-37*zDAe{2ZqHa7!jm`y1fNSj0Bqh4l^X(m$0yr8;}EZX@O zDXreMBBkUtFIAMhM5x?18~R$-$yuPDC%ZL3BD(=C!?@*dK*RY|K!{H$vv{fOl8;Iv zl40ag@T5mQe>;rOx2tm-$)_3mC@g*63_NfN7Cfv}+*n{Um4tL+@vvTu<=!H{VN)(o zjIlD2PgKgv$!t_aq_A1bqOuTTt-6n- z5#?jOC_il3ISK=zFcwnar-zl~)yTDG+ORb<;Z^(tc$F0JsuLxL4@oJ=Y{`8v-yl)h z8@QBzZ!XpXE;$|9NWUtuUof(j8_PuIWEIXywi;%oDzsVb&w59ch>G1LY8b{%9Z?V4 zMf%*ERkP~Hj7SH^q=J#zVBOaIm^mKFu93Azpr6L;Ewfr7u4Jj$p7;O`Ts-A|NecrL1pW}0X6;9Qydd5+F*CA zjTE7_VP@6kM)^F9bF}EgqH!2)eBYo&zK(i#E1WKi@MH5H?lzeA{95Xl; z@Zgcc4Tqa?JXvLm7$D;rb_~u&Lku28$QKq)Bf$XSMuJ1!5Z_8-*Z@+IPhcz42cJYH zI7r$fl^fvPBOc$fpL0jHYQfiZErX-S&M-&LoM6&K4198#2Hzml7Yultm>WL7pK*ua zTR)mP7{d1*N^-nOBd5KgF)olN>?wEXDt0%0oFLlpvguBT+}stn-%sj=ZAa3oLNyST zU`Mb^AY*u2UyxZxy1^iPyqg%0I}iwl)+lsUxB2|$kTCOmQunoaLga#!^N3MOGZJyz+f;%-w2nA#~#+pdpO?}FVRZaP;mU=@B^}H z6p4qtp)_4t0UgsEWX9bi{@2<=@p5M@(*LlnuyXEz6}E`-LY|8pU0qlyT-1O|ByzwIdn5z@6PbB2b(tFjxdLtz*WrJM8M(JpN7$*mZe{xF~; zqkeA)rVBsjL3?3v%Wx{1gISyt=W(A2erj<#Ltp`rEp5# zQYqTY;|UP=j?9xj5SL~&m>w2ba4r;v>jBy-_~ZW(PN}!4mljH2x}>XK&{hBXW-YJsWsp4x!WUHyBD=R z$>Oq?3d&@i(oSmsR=;GlFBt7HL&CW4_K9SPb=omqF>`3fH*--iZ@=BWEK^_u6Pg+J znXD1&S<0u*O`e;&G;0bl>wkl+N!fW zohwFaQ^WMdm?AbFyEa=QRJ7meSvHkVRZdn;`(wuzO*`X!(pWsDoK#M$WBVtRi^kn3 z%q|#3UGe?yyWN8A&=<9TuzzO%L(^wX^S6XcR}+VQ!l7ZI{Mw?{zg%28wRdvw)WOMv zGn&|?L~$$VO=q}&{O<9IOV6~muZC#o{(@=8>MPJ!Is&@bvtZi2`U?7cKrpqEh*K~% zuf8%v%Ytd=chJi`Cf_~RlzdA`FKS94{>5(diLKJ#DKY-L3d4!L(%)?*@peO( zO!|8r#(!T^++~#he%BU=|G`$$RVn?0^N6caV6L#g7wJOZnDnGGH`DF`< z@0EAa$}bOM{E1B7rBgmpD=}`AcUhHBN=dv@-nCizq=v*B%DOtFPdcQ)^DBwEyHWa; z#?rk-`qdUW#68f7?1P{O?m;hp0q2S-G%etre%3P{SwOPzAD~$oGcpP&*LnyRXInpi zJwsD8g3i3Hb=hCH^>KL}6+#4vJaBN^9i>CKrIBYC(U^iKlr>{N&FF1i<5hu1M7|*n zM?86y{*WG*r{8%;JZ0E%3_%TdV7;@5h?oF67V>gocfnbJf9;bAr>W7=HvFa|YK2G( zhbJHo&j`bOFo0i|Hv26n;Bh1T#ch|JBi@i3U{1FaKbBGZz$a1yily`(_b42?ysk8A zN07Jh#UI#yAjo>ZZ_9mQTlxcxpHBSx<_?Ulj6=;KmcXI@3G@emUWoIJa)&Tu8wT4k z*nz=r2mo1Bq(5+6HzavPMly&Ew+o3dh+xTM2mqFm**^_lWj}ITNaA9B#IjZZ3BJ-D%16RKcwEj8H)d7IkOop zs|-sj>w?Pq>xP7?Zb{`>P&tIAV{;$Od4#Q9i>mItT?s#;|GE7zO?^gV_&KdU+#&tE zsPS;C^z&9Z#MeN32$qr+|33jc{Sjc1G$LJZ4X=d=BNCYJdB$B1G+GaR7T5o979Ii_ zGfy7yBg+Cm2tz6&GHe3?8G}eWY!qX;cjOI#Om6kA2OJHT<+zBkNE&)l!&=TjYiyAi z12bL`%ECvih5!N?5GJjUh$0_|fm;svC)NgFCJSTzXt|8vLoQRHT&(#cYR#7%$h8w^Tx8)$75E?Y(8D0mH zV=IWfIwE@|fZ_lULH(-imr)r}t_S^ak@fS3YsA)a@9fH`%jJnNu`H^Js5+_3CjpRG z|Fk*8S|e%#Ur9tAY&c?@^7_tTt&bFM9<_V@GU^BDvt}KA7DbBI)#v5)^WUV;sOBeX zs#csg8)~R7qQN=!X07~0DeB)UMgAP!{I-=fK?kZ}b#2M4uClj`mpyL|G^{gT+K6`D z@%o4L^WQjLx`>YZ~V&QEX5mxp!xS68hf%TEFDp z&o{{52sWPEfc_U^eyCpDsR;zE2mP!E@$ZP}Q9q#Do#Kcf0L~hs`VaNva6-U$o_(;( z-43Qk5h5%ZER$j3$PO2h8N3tk*AaB$M#$wd5r|6cj z9`B^HO*0$=V;I&WNsB1Ai>rmq0S-UOVWTSz*%F-18M+zbwqUkNw2QKUE{7+>ung@p z0^|0Di(H7&Itfs#73mSv4Xd&MZjtx87w}MI9Jb`sr1Oy*UXnKt`n-O&nfKytJM_dC zxE}Oh_p&Fi3_}L7k#ZcTo88x z)wTW3>XKNIVGZlF}2~6S_kl0YuDSuj?WLX|Na@7Z$)=Q-#ATaOV@P%r?g-ZG#Na3f$010Waud}BeBf3lRg}w0 zsk8BvhRnhYjhZCV4ZvV$a+MIpyfmVKW+-lFc~1fUofRDS6V46ve+Pg30zheif0u8F ze{Y$I&IIHA@n}3a_r5T2Svd2)V010Ye!el(T|3h@(-Jqu+u|*Arn&udje@za5b$1+ z4bp&d=uE$=ow85brU^b%qNBZfkoYW!ubo!+KQyDG1eRFn5~)Z zo$Z*fpC6yUCX`=DmRbPN*69J()t{wbYALTPS(_d;;lANyp5#= zs|Or~YMTJsuGsw0`M_Bqvs*A;$(Ol8f2SKou2?eEJT=rLEA2~_&V@?nQsu6N%3X=d zJrnBXQtPj4AKD+-A2vN`nzXu^4f>`2ZaOR5sYkpXRP`=C$KZtCBvqthD|WKOSY{GwyjIHZ40(- z3EPed<)4ae1xlaQC903jOBby@!l{0u=+@c2vksxYGheg4^vfd3R-Hduv-(8!yYt11 z)?T6iG^w~dJ`_JcdsV1^H@|A%>$(uOy_2vUn%lW(dUt+1QO1t={&*wI*Y>kGYtI3) z4zIHkFTT8zFutm8Neb>=j^k`r8}4iFY6MHioMtvEl=LpD`jQ5lpt8L>M3&-$?!DDl zpvY77g6`z%D_BYIK&)f+)i#_j3s%qSt7;suh0-o81lb9>pTgVN92cSHQQ zJInjk(l0bSApS*LX}?AKJM$44B<3w;{dMyB=0mNJcw#s8Z<9XRhg4r}E$wfUe${bU z0*Qa2O{dk;e=%dhzsk&~OQrv6!MLEZoUV}y)m0cjDDPJ&7ibm47u53pV&#IK#4YlE zM!8Ty;&t-=X7xe?iEoqlw<;HQlX#o_v_kn*qJ+9n)$-G3tZUqO;i%qVgT+z)fd;8g0%!EUMXnM#pBI?4sHnauO+uXd>Fj z1`$z<0+PYg_qyMr2gLLXo`~*KF#Pa=`qws?8Afy{VoiuW0fHbGYXG{8Hznlx-dMMa z@G2%amQ)<&sQxV@pkF%zdDQE8>gW0d2<8NglHo_lsf%P$^r-Pi)Ys?LH@&6)Iv8!l zC~BFrhR`TXYq?&G<=zov#01(Y5$6I;-Mf`n11-mb2WyC$2kQSWL#H^O41Q6-J%YR1 zA3mlEwp8R7HRp|uv7li{M$82x`NLZ?vzBaY#UY2-Mx>Z66Z7HLq=YSJt^U$zX{2pU551ruBEw%1ChmDy2vx-_f!V+8N{-Mys{p ziIlBz+$@ij52ImI@I(}E85=Z#3Z6*zzHE)l=lAPRO{A=#KV`3@Z1;Yd#iq82jRQ7S z9kqa#&{`{a*s6$SEmy#$tWnE{mV49bcMg{-bkV%`Ub-zut$U&nuOf)cjjP@O<4*e` z4g!3|H<4e4_yQjP7+ey=H8kh*@CbHhU9eN2?Ov2<4GDF=*|~HSai|WlbV)gu`-1J<<3YM@Yp0Xos>&J72l|jwjs>oPS6d(@BL?z$*fo+P;6s5GpIcn*b1#u&FmQ!+19!W#~H^wxod$$Us~$!w2Avtx%I| z#Gna+u;U1L402wWzdQ()6_V&Ur*ec&992II6G4}_943;I0zgDyf|P=EmIq%$m^A2* zC-?FAQt(&NEiX|&j7q!;0Ei;bSnp`a2?p{ zI}oH)SKPc82h3sR4r7KXL!DbTX(vxFR>|Tk=&k4r5YIsIhcu`Rl95Gg(n*!L5(=h3 z$|M$~y#%py=q*cDjSFTjijSExbX0+UlY*2YU6Rs>aw3UVOrS3$fql3!l(!pul=WLg z#}}>(vUofd;u>G~Nkcf<-~W()1p~x`QCt+*yQ*>!yE%7$F>2IGqGjbI;|1v z4hqJ$MOpiX=#9xTF+Oo^#v0onYZMH7ZXXw&5LGkq(=YTdDQhAwcV$A;p}CHP<;?Aq z--2Ug`Et4Kq4I(9q4t4x=4V37p+x!NJ391H1XFRMH`e(~wf%W{!=2;F%KC>r4|-yj zxHVCE;7;#r>2ikfxEOaN$`9T-@toNWo`Cq@R`*gVBZ$YhdO$pWbNbP_kIy}N|Ks7kjC8CU#Jd?fA?Ixh*8mluuiFK=mhH1zS%Bmc+#c-!1%@P|Az zAiQ@)81M*(*~AvFV60mp_=b-}iOp{)x_INS=Fe?%|~e zmxQecW(N{ghbNA$$!wbqC8|0h6MQspOx~D^PDW?WE|xVdmsdYDJTSx*am5__b$R!5 zNTZg{#?C$~-}SAIs%l-?Oj#X&*+Z$UD-w-*@3O5ji+0Z|A(Q9#96Jwug zcRsJEdbsz&-iHSt9E@p%-NzFZC(!?~bizM#WYN?RJM@jQB55p{QctP{TjP@Lz=G{S z!qztX!Mtfgy=d%yC8x}e74T@xxI#9@dSdNDdF!I~z)BHSwrS$RbVw+!zumK}Hr}ti zTM2xREvg%1d{U*kue__AQ2nO%k^N))Qtgg~+8yzIiQ0pAl#8l1Kw^rTz{OIoNxRP; znC+Qu7mQttvhJj&Opukms)B|EQ^R)vZWyiKQnY%n=tNtwJzf(p7OcCMw0jn`d%jzN z-=OV9FrcVOP&cmfZ7|j69f#_A<fR>lAN3`@cIhAOa*Vggd-o~PHJa=euJEqS4Q?KOtpM8%K;X>g2D0;{qkg#H@aLrg z)8*YmKnQ+Za#EO5q_4PATJKG8U-o;+nhY2B;B`*HHcTIL(D-XVA4ue;+@h?khVB7S0jsydPzM+)tMBlNK z+<(DBEd-xX2Wz3tq6D1se>;_pHLN~Y--3W_fJhJyd5Zz%mYl=d4+ z2a!KhRWD>p+WxIsK_8`G7%$Uw$+sU$D(R9J2g4A5aZ}nu)8#AgODLK4PWY+R`fswb z+wU&P$`)j0Nqy;k|6PC5a1BhiP89FE-Sa1zTF`Eq(N2FT6gi*CT2_iF JnT?40{{fQGZXN&t diff --git a/test/stress-scale-tests/utils/__pycache__/metrics_collector.cpython-312.pyc b/test/stress-scale-tests/utils/__pycache__/metrics_collector.cpython-312.pyc deleted file mode 100644 index ba51e3a5f99500639667e7e0aceb72b330fa3b85..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 25216 zcmdsfX>c1?npop#pmC4@3GfEVmPm<&cuCYv>#$Bz7bQ!wWKko-5F4UEfdr=;luRt? znQToGI`P&>>m;JaazyV=#dM^dFi9n)sY-S^J3C_~o2qmR@&eP=X6Rb~aeicVcxtCS zRY|_@^?|ENQ@gXNnY5^f*WK^x_rCl4-uv^C5+?=M>_0km`R3~s^;dYKT~;;oJO-JY z6i0EU7&V~$ng&ed-aKF?zm@?D`LzyM;ny6aN9X~1#5Q0vVR=i;KEezzBaQ(Fxwpog zBd!6Li852Y6i0uD;%s-#N^b-12~*Ji1>W=r%`(G<6Olx8BqB4ICgY)mEwJ{|@eNQV|Nr4P!RJc+KCTV6*|w`W@2% zjiVT_aW+WpTs>!pXAHb!;Mc)9;Hi^q;GA&h;#_d&hC4Uhm2f3+SIU(_>ft<)dO0to zWn3dy2Jg$ca=7zxK1ltXAJPi00@6yZ64ENJ3esw>8qylB2GUxt7ScMd4${DcCCJL& zW03?O9TfJ(W3k9!BF-nf)GQkw?4 z2m^Tw{D1yWU}|nsQEHM3>95J1UYgu#6R4ELn&gU735S+X5D>z{c zi%F^U%c2wD%Un^NWzQhZ?<7~Gr z*>8KXy35u3yR^&MZ<}-4US=(pTc_f8UpFR>xlO~`xU{)V)M&ZkS}m9TO$4;Owgynw zqdl8&1f9viq0wj}8jeMiS$H1QfIbhY5ej|q|G66yAWSCsXTEJdMFlPWJOV6`2)i&o zI2aKG*?uV;jg9k>k0`zr>%f2Nt5{boJ{XR5U5<}Lx}p~^hk33ma!KeI6rvp*AH5pk zJ1)ifD_!AhLP!4dF2t=aA;Cl29Rjow>A)vK7ob{9=u#0eqz}AvY(i#s#KI#Nx$v%J zTOl%chC)&J>`>k15>b-Us z;rTcZb>I`@#BmlwOd_$KqT_NQC*s_*jUG#KIH8M;6)2 zN5UKsemIY<72S$+VnWF|M8*AW8(=P;YRk2M8X2V z#0c~ox(2<*uL=D@uk6(4QIJc82_X>~38^b8yRK{#vNr%I+6|B7Nw&ab$mN6a(FBh$ z%WH3C_h_7p=yxuFCjG{VO{yPnjYWlo?7b9?MMAKc0DU9j(ZL9OM0{unCPj7(MG{9L zAK_&t6dDb~YKCMRzK!s}0j)wLc1b|`%ko|v5EN1%D_OH7(3Ls~@v%_x7dO^^faYrQoqteSfBT?NbNcat{dzMt79pBik_2f;>6ED@gHq#Q!w624P0rIdK9yYL<`o_%aW_VZ!58v^8Sx(h7 ze}8|fI`|lVPp4|uVSCn8paY6jlz!=6=5@8f>!-A(uJlVM*4me1nWH6&YLomz`8qL%Om>s%*K(q$)Q|TYt|*(64_`=ZG+QC?XMV`UA+^ zq$W*onuIzKaZCx76H>rmhjVK{IlMF~$eELH_a61WdCHui$ty^PHPK6Z0Ka$DJn%ou zZL@K!lja>z3f}1)GuyzVS?kk8O(LzdCS#`%yI2KNL7HH7@OQH6rDyi=Lqd;!hi?%C zd)R(R+4vAJBzQAb#>g*(VO%NR@br^|;0EEpFi;VKE zP(-#O4UnreIvUDH#-N4YN=o4vWEvWdassXc3`>E2kJ5oYq9j{{1O+9ILUrL)NB|ey zR9VGC@3xty+k>;sA9j7vm1^joe?@B8BGzvetG7+lpVv0r3Em36*OsZO0T5daK=WPC z+nyO(Vj2KZ93^R{Sz?-JUwy>1Wjy|AH-Viz(80wfpfdn-f}jJ-29FA_t0tZ`UhH8V zT(WgI)1>J=5VPL5+*JY4L`_%{IyemLqF1(#23W@ZPnwVqaaI+uveopQRszc{+Xjs# zk|3ZgTgs;Pq>FEQM=(d6{T`!f3IN82Gp3HmhP?rpJCgtQ0LaM(Ku+XmK9U&cM^SJl z{7nZ}JyQ($1l)Sq<2tk-+Rw&$f)9EvL=gSvBuj>i-%Nz?X8$T{dIA^y*rnqdshNjx^Hz~*e#C5E^!UE}t+&55GcI}7%nnPQ4I;Ba zK|(}2?b4A@geSidNN5J_#ykaDj}b9YL7O1bun~gfp?%Pn{1_X*>S6bT29A`UpFpmlJ|;bLoDSdo0hn(7AS9^cJBYlO04^Zg)e8Jhc$W*u{9&wt zFf7}`V`B*A)+`wF{p5wpNRW9V{wbWm8RkHYBrGObR|L}8mt#;%2txw!T)I+tc9(x? zqktFG+^Tu6E>qP2yrA4UZ6gqw1=0zL38tC#60?53@)KrTF&OGf`LMzZ%sf~qMnS2W zvSbAbBQb)r+_T zZ|(5bHVo9g@FH&+@)q^Af-7pBvJ>8i%2~k$yw8J1h+%_wlO~l<7`{W=KU`MO9NbRY zwb=r`h)U;8&6c7F7#hIpkg8|?_G|!PwTC@>IRX+A4`NbOVBrtNC3t~2VkmdXRwTBv zO%YOtzIO|+AMOSU&w!#4;je~cvTJZ`9Cd|3G=-IpL`LHLgnsYgqQaFBDode@6Q}}u z$pfM{DNp)?4%s#)pmN8bfw_@OuSP-F3dcY{!KTz2(YT^hMnEOBYrQ#h8g}Kh$p^6^ z+xQro?Sf1Wh8`j>#`wsk=$lG+Hnkf&8CWzh%h#`nz>1D!JQa{172RuQuYUN}2X86&anZeY&Yo`Vky?Am6R~ow z=x&*-N(XmH!5vQ=w9g4GDU>8#-6p}Gt1T03o8GT-Mp|;OU1Z#8rcPq&fbB7>(@YR3 z#azoLOn0%oh~%Egi*EpKBdZcj6Cmv&#VEWubClAgB*XU>&Z@otJG4S`HqJg|)>@h} zJE(-tT$fW}fW29aR1}Wk9QT}hd)7&-5w#HlKoJb+%rI)R1sCVa!FQ8!dz>4o#S{&+ z&488+dkdPxvbIU{e(J)1hMe6∾1SJEj=&>zs0sU)Pj#k{QN^3ofpNE1k4W+JUlq zFn!;`c{ygt2Bi(mX2wtNmEBe6dCFy`CY{59f==L;E6;v8tV^R-!bq&(d}KU+7)!;y zN`3y_lkQF#s}c->g-c32W!);<+HzBHThOEq1mosq!I;}#+^W_cMZ1{ zyo6gaG8NWM)H{D-r6x>GRH8vEIlNZOWq+F~zHO@H1)!yd%tP&D39O6GS54G8Q0S*h z-YR+Lb>;V(=^FK>@9bStjE^7^f0+dLAl9JVc z`sH#}6x8TERFmzI(eTAsggYpEb1ZFRaXukLuLFojZ$OfCT^YX^=};JYvJ#}lotg97JjLRtXX~(FkQ8ziF*|lNn206aI0Up9 zgR;fRrkl z3^l)a0QWMDt%s7$2f>gX0ZldzV;os$nGJi!qOuR|l*IfLDqBw+I&nazg;*pqCNrvSTaaBK`y|E%Sbr`8 z_Mpg{2_ET!>^-SizYn}Q7$H_BP#_{a7_>p;kgeCk(S+>K-vvt*OOXwh5#UjoOK2Nc zE~I)9t+=joX?CH>H!)~_CiW$G&nNh6_(^YKLbMRO_E9ikVJto>L}XJ$Kn6lY7d=sw z5TCkYp&Yw$vbjj6F(~yOv?y!_$N_oA4Pr^gUB2k4_-^v;q*&WIZ%%nOrafCF&(`~n zh0TvVhw!zx@}~2KQ>^QnFHd2G!kq4Z3`n3n&NO=dQ=_j`Jj)SJhwQRa} zom9InRogy&B;#vH`?@7xcc!)_)425sP5C!Ir!4-`>HSZfl)w6B&+qqSy#AZ68?Lms zN%A&{%?BTOdo$k3#rjou25t?^nQmQ}?$1=MNmq4BRh{XojZ)Rd>AvSSs;1$(^@+K@ z?7Zo({A)h-H-GAH1R1L_m~QBl8agvIt7rS?`xg8QwuSHm>%vuWSHF1Tw0LGvtmZO} z!8_l&^)2=1iJfZfdQMpzE3fy0Y*o>CegDnFHxAEy^Pe4iVxueGFg!tK9rf~a(=!JXI)6+I0FP#|&&d8+_Uk+mFZ{;lNNA_8Yv z?lCkad=ruvvLseaA6hK0yxDc5Yj%xP-kL7&kjgt!nfB-!RPyHR2{ z&L5E2p6TA}eUiI=(Nj5Hs%TlDPw5xtiI%kqWX+pYj-^B|lFew4GigSv_LP;tj{)>@ z#D)&mjPfCb&LvCeHjQOU5X%NK<$`N6M_yf`4>Q&U7nS)+PIYodyq+fb(#+kUSY$zqLa$ib@0QoP75@BqBTk?e!4?+LlFVo5>4s)s$L`Mru=SNgz69VLb! z*L;)f z@4hr!EBV@{8Q{|H+F5^k)i!Ar$PkC6RYxS(k<6hZXvJuf+#myNmfTzB-;&%1Gib|b zl$b`5-5@dD^SdNwZ<;wIF^3+sK4MNmQ|J~`3Uo`%#`*pfvv1*>64M92PeB}5d!uG< z`?O7OXV!xj(*}uod7(1J^gY-kF~=S;eZ}TeL|B5fXvTU$qy>Xv|-Wr^&B zb;25?lYgT?5($~fA!{pzBN9-8nn}L*9Uto;f$OU&F!{ycXQ51b<`oY6K*t0=+M88pT>f?Fj-1iL;))lm{g*VbRudXc^H-}L&%+g9sqL73n6#W zOp>(8I{}}b5Y5?ZE?>bgzy@PshR0S`z;u8H_g8$ zF}ptVfV$?*E1z<2yT3#(q5oy)Vgx>l8@HE3WmEY+nK(3`=ibmKpCmf~2ZiVyeX&pjBP-o`=lK#WYA? z-#>tlc|m5p0xf7o%oqa2m(!H8%$NvgKT=>tVOQBG)D?rt%6R42)j?ts%qK@D&g|1+ z;p@R?DUQmc`=_73gWy9ej#FfSGNW~x13V}=VNa?ESE)dc!UVG)pKMtIa0(0VMwmGV zZ5pWni!R^HrgUAKRM#f9ACT$}O0I*M{Rfe@1SEGrtluKJx87$Y_nyVWC$rE+B>X;! z*?Ir+L#DS_X4lzV4EhB&XK1uh7}Rhs>4DOCt&laDz%*0Ln*PH{h_~W7cF&1J;E2~k zlOYgG2~E})Q_mE0CSBAZ`kIlFnz$TJu-C%q>L)$qXopC4vBqsfSgJ@#-y0 z?D-fX03*}~!PmieYpmy8$jc>pjUW?3;*-OKlrTvi3-kX9lKdFS3Z6_uC_h5mKz1@L z*N;Hc4ULj`8C2~Qva%7RpEUE5#JrT{C=KnIdJO#CnafaerExHyC=7*xr#C5ZSVHT{ zWkdut!wZ<~%tofcNz*;kU9|<`Y({7+xb{;QW{A z9|V3q?+FNo0XG;Z|7Xa)3b~^2$-Y=PI>O_mfMP?0#XXRhtyf^Pfouc2m4NO{72jo= z?Al1GR6z9tq2EsVZ1o#~>jb^TA;%}Mv5G=h92qr#72kQHqfqy11OhrQ4ug-9M=;~p zV}f=kMO48d@%J$KBTRmP$?sya1(U;&s9G8MaQ+{{1KFnTSD=;7-hpSuN}1-zQiiW? zaWEh(hULx!;tHQYyFggn9!FoHo2r6MS8S0gwk*_$6J;+zwo)GrsDy zuT}E3rhOffuS4AM`a|FA=wUg2e@XJc^mtA1!*d^;`@{1Yc1@byEU}x@ zY!8@*3vXAqeAxa$d#0)N!<`@O%&c1bVciFHPfP4OD}O=RE2^H-RPEYde(6^|GJS;} znY{}e7Meu%VA|a)xqFF6rVl(ah!nrOr(~a(`jL0fZut4J*RkJZ`EgCdz8#hy@1P++ zXoWJ`g}k1~+DL1GQ7#c0s_AkV{XJ>MxDwQyVF#f#`JWWo=){>0kW^T8lg-K{qZP?K zV^<9YBOQ5=|J14E+!l>wu}>q{b|RU9xqqZ?0$lmHBJ)Q8D=PLuSu1 zLbSecnwtPcTWGfM(w2}c99kUzTv<4fO%;`7e2gXgp!8YQGsVPiWdf~gL07EeWKphN zhBZOa1A9gBMJbYRGeR%^MD9}N)qS`DUS=F+-ne&pjb(+rf>{eD+tZtW>jyaH8 zw<>FzE2rpEE4G5WXVGaW0K%QZ5FyJ_;dpnUG+rQcfP4>*BCC;1t4RX*o-czk`0WK3 zr<+-L(>pCW8#|bSf^Z9@LJ#x9@HUWLUWOnyS8+)Q2UJH0p>4(`Xsrb7a5BRVC)<=g zQKGf9L>)cYR;xS3K(lML)L&5;(~`M|2|UUDRVaw5zN z5%vVczF=(axRCX^VmJ)Ka5i+wi3A8QJyVP_&QC4|_6v&8TgBskfnLR^Wgg z!_jiu81}qRrDVtN`DL;07{orwSno< z&pdS*-zw3)3cOA}pxgFEKb!WimHcaG6Mx}vpWdJGRo(2l(ev((>HR5joSo%T{hffP3p!lcu{y6(H2E0 z;4FC?!+_3k^p*XvIuPnVX-xuUn4(9`sted!*e)216&G-Uk7F|5k!-kok+UlX8kV|MuC1sVUGlKLpJBJN z(h=)R&JKNbM)|+M25?YC>Qjwk&LdmF^=}X1miFYjV+DtSbS_!;kB}(-HMCENqPtdH zwM%mCDu^2ExxY*7IV>LLMOJv^PC)b^di(=3;fJoqjIT*tyG8PCd3*n&&ksR>iUYr1 zV(P^P;yeHg@}7mZ6w@!ByeKgjA(K;P=90v$0UF3`p1+cf1c0r1$8!pa4QMW!KMxJ1 z8L$@i<=Kqb^qI0?3UPxY*@jYxlCDT1`P;}9Ftdf1A-XXH-5i9|I}Omy72cq8`s@)Szqv@4 z&*RCqzz~4#sBj4!2@wcO$})NKlFfOk%PA|~Ga?NYFr|4S&O87#ASTUPX)g!Rgv?23g(E{+uk z*EJeqbhcJ<{mPI!TVoON`VU}~I!o@BT-{5s#dgLC;XUUE0=Ql}>TkkhZjl0CX zC#8myZy#T5Xq?XFsWUN&X`AZ@2h9B=z){o8afvxDo&e;3!w~r|EbJEdo)S-85MPgq z7ly^!E1xhi1^H3s&@NO*hzhbR7x^(>TPufvYZz7N9(4KY_u%whR!i#aP}m8=Uw)9n*@cWSvw9ly#CQF3RYF&Tu_+u`e7O97mF`g{Xjrpn$T<2f$dMA5)aw8l_|6 zZ17tZlVdUW9lr2ydlI@!@**>H!)Z*@; zOXT`ryN7Ey^6yXk`y_wggF5ldYbpP^X~$=t>cuKJ#Q^_6 z@b;HJp{ymP5Yib~JsV8bwTn#kW4A~2vMKkfY18NB71zTvO*e+#ZJ)M3CO6GDF2CDR za09}=zj?M-^0k9Lk@pK-c%9IN!?_Hx`BV7R|Cb{Bib#!V!kV=+WC=_TdC|ptlxE-% z8e6zK58=ab!W{VhiGHcd3&L1rFnTFE2zq@N`f2cR$E>W2ny-! zR^Jcjd9u+J)=BWiM z<>#LA>o27|jo)vYy^>npEqS^{?Q{S`?)?7^!r%s+eH!UxHH66uQ@>qklP`Mf2yDnK zHB`y@R&b0GId>%}qf6KPX8@M{itPLhvc-5&mhHkAI|9W(pRHn`)h<`O8!-h>+0j|o z3Z!3O)nb562R2B74fC*1bN5226gZv^oRR{k#M2=$JeUe_B2)93hgEPz^fjkEYwlUZ z_MNHLU6N;)$m~+s3xJb;IUz@6**Qq@08uzV)-;uKXrE>%o`Uc(aN4~AvY-V*jdC$g zZ4BgJBk7d{15GL+K?e38t^K>|sK5&JBCRX3@e8#6&P8;a!I%6W;(nxz)LnE2XlFmb zSK)ah1s9ai&uLz=gk5U`1Mi@eE^}uaE9l9%gx(W60HGaSN?)=a;}Uv1OSVxsVx#qR zpfq`kWoefWYB^hET**zPG8FC%o2EtQ33@Qe1rS81B9rOr4hxTu2+Llk_mN3p(1=V!*CQcB8$0In5@Qx&{-emfUqHP zB_)Zaf><5#6(|?2SmL{o$WAOF?oUsNU_E(In_^Fcno?5^&Ihcv6Se`}zw%&9 zDsYli+a=ZSy8k;5dQOH$xiI`E1Vctt#UK@7c-3WP~h-BNwGxbc8ge{kWN zRDV2Oe@d!XIN`-q{UB+5GsLrrTlysS(1Udn`)Zm!C$Z9@ilMPbs z28gRuPD#jpMJlt~p8A}q+c{O;E@6ymo*Ies4|M=*H>lb131Jxp$3^3O4u!2}P?;NQpO15Cb; z$%mNydrW?a$sc3#5R*?a`8g(^V?xj=i(NE8DmgTqNDw4mfk(DT5|kl?--qNo)X%Kj zpILX(-Ot}Lx#@~$0YBaStldLzf4;h!-uQgeZu+3<*IH)1l0$$!BFg&E$8 z$*(X$1$GmG0(!Nunf5qVCo~+0^MvJ|y{&E$Y53$@WQ~=;RjuD-Wf?FyLWg>%eMd0tU(t~L%9NeD!5m? z^bn*c;PTFSNFv~&0|dp>(S+AWTy;@i{~}FH`KftVfOboABB_xEmGJkhACg;G2oGUS zxDt*4D&db1{sAsr&8Wq!loN@Q_=gHfPdK)uw5x)2pU)fcy5_t=k0=arMUvV;h=+X9 zNnN4aZfKzcl!vC9XyD;#LOFEHV!%5` zw+Q{?lvUEau*B({PA@3U=?q%8b#t>+k_}?fwUB%ldt$iE z2NTk~6A`x&6Oa=ADWU#uPxF=LMq-!HO}o$)^pxwecg`DF+GUqHq5&!2jClf!lyBQ` zMb-cf%ygp5H76)7D8R1a6n>ghEkG4cHRJdCC}iUJAxR-k*_D(DPMlsAo63TF8jV>H zghGRW%Ym^2qg#GUf|WbQ%vORT#G(U|?_yu(VTIOP=Wd?6b@ArKNbN@TLALtfnm%4| za>bB#FeP;Dn>oddu9z|dgDpb}bvzR6Ss^4~gYgi46{FOofP*;93n1lsnXn{M;>1nqI9^!?*Gx;g^G*sek@LkNV5IW4n zcbF)bXhMYEvN*v!3kq<`y&A0D(Pe3<(A!RzLISV61v6P!J6Pkc=)hVB+t{<2U-d+T znJa{D5#Kf-#V{TG@!fJ6Qh3wlvUG3y2YeL_3&?N$j7rQ(Ii;aWV+&*|*$&L`6L<5re5*vO$47iC{*)l5#L?rXYDOUCG?hJF@MN z%1Gr;HOsgj3lvKPGbPeh@_YpV!{6cotCTZSCS4`ZAF3ruXDXAq(iN%sKxg{GE*;ICC?$H{47;SHPe(Vk*-Li7nAs< zEjN)msT}0VLh@wbl>HJ;Ia@fH5hpSi{Yd%InP3iZ&x&&} zrx~B<8aXq;X-JRTPkJ~tnoAbAoN>8=gQENtm)i|Z8TJRdkpM!L3T7D}r{dM2r(IJX z=bYa~a!P0ir*~X;dlt|PWLFEbj8W193@|Q7>>pX-_!OrR)Jr344tPOA3>Hv3KSG@; z4ARU&X{ZhWC;BhJE&+DP#)>O9to5w5K5lJ%WFvK6iIibIcp zrr6FF+vCLzk1A{5JG)_PV{L75TgQg&Bx^gleucFSGoz>Dm1B>Jt2T=3+2Z<*;%2tE zIbOVHqqvPNZi^Rpe37LsGOQS%Sk(HqM-{sT?hg*0d(`gOXdh(T2ftL}mBaYI3{U)e ziaB%fH~uF|EUWDCp5q&P9PA!Pd{58H0Ap=rO-)Y>TK$nnxdlvNODsQ;+q#zfsJQ%2 z#qEkH9xtvBYc?DAebj%qKen`99dC4OH1@HLefM`W6K}>FFNKY(gRI%Ux&O$=ZTH&N zJnLie{kM?WiQ%s>hvblTj%D}3LHQ6>>4x=g! z4@`}Zb4q~q6Frnar@NqAKB?=}JL<4s?lE;2VZZF?B)SdCUzO?L=2vwU-CE_R8XTnt zb+<+PX^s-|Keg~_QGR!`^3x_9Qa3^h(t+l59y$Cq^!FAYL83@G&!pBA^QHMIy{jnC zI7;v(5FFliBwfvA6;Q}1B1slf$vo5{#ZO@74=Ev4zN1>!1Z1o+gCO*_VMj?!a<|?m`gt)-4-9#NmbP!9zI33vNDP-rVa+HTU zi$a)@LS0K)P%?m$laPR`29M=bk9}v)JoQm3nljtN!T5zH#!Tg8FcN*w)ROb-SUqS)~+JY67Z&+-s#kOIwvle^Y z(y(DUz*-K(oN4A9Z!OfzY zFN~Tz!;0ohGnSnf-o1J%kwrv0ADPTQ>ic0|cxu%Z#UGmLzfxk^y!T>V$pO15Uq7eO zoBK&g@yVmgPmZcjX|?Ii)4#*?_W#a2z0fdiGf8sf5}BlsKAq!3s=;t--jM?}V0zNi zfnl6^fpf%jk)4T*T+q{WukbYe2Qc-ZBO@onE6QyQ=|~gu*j`b>ETGSR4f-6QH@^mb zF3{(_27NxzTSWSTcV4dHLZB~t4gOZ3FMbXB5}+@A4f-;mFBj=`uONTL*XVf_Svk8K zOmIF-+vri-j*vbXp2RWVWJpC4GChi`Y&*8mp9K0UnI26O+m3DY^*~=O)2DIpx6$W; z)W}EKc5J1}@K(Ya!YhvJnv_uQwkM(67s9xjofRY~^omd4VP(i#^z+Jq#}@o=8SuE0 z3mwyo^DbW#<#Ca&%X1!rUYwstF~DsILWUCmcR>&}1YJN4LlQJO#b!1Kt_P^oC~NfS zH+*ooAz0?<9w$zX_YqxVePQ&@v#0E>ifQ;91brd2XB-fQUC?A5iks9iRv3aGdMAB;EXA&3@C^KXca;nE;ouQI2T|+ z0G6+e5P=DJrWWUDPav4%815oQy5$ZSw&x{LEHw^~cohJvzD9Wi5SUn8fS@*4;^c3g zDY5|M!_w4Q)I0=md2F8< zJ{QTrSbRm`F+76aHO8Ms6XK_uGZ0>fXV55$_Kkz+K_>r5dJz4%uwdvV=TQ1Q=eGM z)05$mgsB#QepWW(!C2}OriN64Eeyjmfb%EWs>2L%WG#=ebRVwfkuY_q=%zLS5tbdwM(ZlYt3wZR?3-rGh8dzeEipXS@z8u? zGq+@Q`ay0TpO1n&(Y?{Rhvs&a?|+cn#OLpUXizLSHu%uoiSp+k)PXWM>pzv*t(v0T_0oXzhBK* zV2BJ$)X72f)ew$+t5F;?BS-6W>fLi1Ijf&o`g8+z%HPzf2lj0Bwd6&lY~I&`wLQgE zER0i!L->q{(4oG3z!FRnx2LB-fg05o4*>Q@AkYR&6=FC9LNdyLN@6Nu1nTpE94rbc z(}jg3Owxt@Naa>sfmO;kp>@lufKlR=3>@85+{jS@#j+eq>cZaqd!SZB@JG$dqX}td zkao5mq#71zz(k+}!bRz6w8>!3vMvJ+?DmlP076ZxPeTYHC*u~}oSxDY(ozaC%ZHQM zyoOUfh4eLe&*dj7GWQj@=Y{k;xI(~cwhQbfe&d}C(&s~HO~)_Inc+E>)aTszScUlt zGSn;J!+C|N^kk7Z@{VMvWBpnqkN1?84Jjkf@RE^d_#=)y=Ph@gf*xrXQ98I1#2Tcq0vPCI_) z)G5dKIH&3E>>G4+JvR|Vm&ghLzp{DX)N?&S!_t-yss^(OVib4)E(Xp}4);8FVhlhl ze~%u~BG4NQ&$awQ7r^0*NUH3s{<+0@kF1Gu!`YF+GeeH_%HXmDahn{ogp#EAI~ zI4aa>)F`UU_nZWe@mW~iL&^j;u7Hc00}5&$B|bzeHBTg`f{@7vHIo5fG2~0Rp(t3t z9iK|*JGmE0Ld+i{Rx3P2eQdO@_B=4wY(iYM|A+ldQES3K1OEFu#;&8ZUv$E#S_1o8@hFE93@?f}kv�q^<8VU zaSe}GwT1g1TdcSGZ}vxuq6JZH^cqunC}BCgDQ1@ZL($#7^=YPIfgD~}~C$2W^ABUOJD+O(G4G2AvpF2x*cyYCIJ?`QV+ zFg@p(nl~O=&wo*bRrKReO0e95TZWs4)pL;xk)fD{DLt4lA9`#oUA^|eSSRc4aKdyX zjo!*@8Cxq`esEJHWQ~Cl%Mv)((7DK8O@&kJ{MM zJ+Gj++THK>zS|phuDRm1C&EKp6&GYG+Y**`Ne{29IHqb}>}afIJ&&pEN?05()Z8Rf zbK#-&BGTN@7R_B^Y7fM^n9?H;&23NYm}%GVpH^bFKK%Dj8!_7;&X4O~6y^0D)qQHo z>sRQu1nUv>La^@HBo81Vh@1vCCV^X6@#ZZIy9mR3SqZCuc)G9@fKX++l|_jQ7;r_Z z!Oy@20Z}{^h^D|xRR95B{*WMASV0{iVUG-KZToo!j_MQ$q9RrNRIbiIs|l%RWLPRa z3--ID07qWFtuRgo$SF1S88{h8qa-_UGNo#S%nGTY9b*|}%TBfpKUL0%`!6+ZFdl?Sp zK_lR(F^2#*sabeQIWz5fYY~>nVbi_fQ-YvoJMFJ zXGDTZ`6fXk5dx-J?WaiK8?>s{|2P=JpuJq?S8-f-SYPNES$?Y`B{)vKFj#GU@z{n66c z>DXKM#sSn>*X#bV{ulN4&oO76@sr8SN$*!%q`WCvuu8Y+yowF zR_~5hu+@jd1Co3t(SeUf?vAYWu6yH+qfEnT=JYsIIgzlOktoWdvmg2I`qyUe+v3fm zOcPLCU@9*rEN==eFtSDaceJ2=ZJFIa^qGD5Yqns7Z$U+*FrL3lZoz?srFFB$9<66< zTEl~qd?nGrjYin--Ldw@`ck~{G}AD~j7>0=XA+jP5=~ijZln1a+k9*-c)vB?e41$j znu|>3n+eM$p%tch(SaSU=vdEU_YZ$&ANd-scnbY6idrGV?6Z;H=t-uuViE$4e+QVe)~RVd~p(xnnkj?b!=bK!W-{B7PesKR}5b3gPsw1yS~No82f6 zPzM__On<2gke-wW{Yv*j@?Gq)u_|)zfw56`;f3WTJ`R!^`Y0xtszYq?;gx>m!q>k? zCUP2(3qSd>^PV%&+8wX$VOzVC4tzIT+w)m@FLK~*5f5A1_zDg@QTx8`T^(aTwvNY% zli|KC9z4@>oT)sKuyjgR`t@D-!)wKB#&rmO@DY(01wX#U)VLm6FC!PeZ;J~*85xYJ zn9|mV=7Y$CZ~FbyY|Pe+(`ctJxb6;YtJAKAY-F*Y?b&Lfb2GeKe@qMg3vl-3bsvyP zJhrUDz=qwlG%1c9TGvTS0~>KAH+fl;fyR(RgIwKo_QlA9b5*LFoq^I6Qor^}4Y<*- zyi%hDCt<4`8d8H+YN(P75`l{y(xhLfK{hf&tqc~`Zk6KVD71|BFN~uPI?=8O5euh_ zi{~RxLfq^iM}?~m;}e||j_xyq#SVWaT(5HTM}Ho!N)qDiCf zpj_Ev^1)*o;Ug(d%x?>(rXVMCj5tMcqJry@p{x`QB@uxbF)aOyG2){M)3Kdi!DD17 z535kWH4bKl1hL)mLK^}Y?3KG zk$}Z>vBJe@eY7k##*`jTn2&7cm#(^g+#wlN6gIxjHXmPCGEH4f7Xro)EoZ??un?)> zG}R2+FjM1BY>M18-tv73+Jmp*UlOi-6a2D2I9Q2qvl*#a8N@*ci9T|8jbJ791Na0F zTCn|)LZlG91IWNju7*!0WcwP@>#JY9J@h7od+4}2%(gend`=XKp;h9*9Mye z6e#XgdYR=C02~UR1Qig@P!I&1iDCFuD-3z*VhwivHj5y9Kk#E9=ajGv#i_5l<`z9@ zN$;`~-Rsc>pGTvQVI0Hef4NC@f#^bh?QDgc5K!U#WmaTwt;Q?xf> z+PBkSh+%{Tgb}KDzu)+7cc7Cb7dir7B1B#N4)1sZqcOtdD_p0KuT?mP5x z!@Y)|H?d{St38qPlFy$P?TH?Vos2ETW?`vfSJ%Tb#}`U${|V$-!be(Yl?O4Yj-HA3 z#P%^Y2OnAweUXLL?1c}lM9*_-^^M5b>d}X0`xm8>*NOf~2Yd&Dc7W0FpEKZ}E1>T& zoKDW*bk6(9#W{3says9F&;P^{tR-A*Tkz7rC@2I0R889=w^Q8JAZH%ds9AP~#j zXTBdOw40c>DuUrgK#D?%5&q~-NZ!W&N!9*LrB!!4%`>V?o|a^*E1nWn>e8p(c%|C< z^dx>1((mAn>h`A>@dowYr;~W0`q0xo7WJWLjXCOifji=9H&PRD5Fapza-pj)O71)RJ+fl+xwN+A1*CA-+81*5Lpx2mggap1%!|~r@#@}N4-(orZ bZx-DCGiLoQ1{b+ssk+p7{ZkC3yk!3aUZwx< From a2e87b569fbe5493f368b14d5d5053012c5fc681 Mon Sep 17 00:00:00 2001 From: Ishaan Bhardwaj Date: Tue, 17 Jun 2025 18:41:53 +0000 Subject: [PATCH 11/13] Add comprehensive documentation to orchestrator_config.yaml --- .../config/orchestrator_config.yaml | 123 +++++++++++++++++- 1 file changed, 116 insertions(+), 7 deletions(-) diff --git a/test/stress-scale-tests/config/orchestrator_config.yaml b/test/stress-scale-tests/config/orchestrator_config.yaml index f9a57ac21..67502337d 100644 --- a/test/stress-scale-tests/config/orchestrator_config.yaml +++ b/test/stress-scale-tests/config/orchestrator_config.yaml @@ -1,36 +1,67 @@ +# AWS EFS CSI Driver Orchestrator Configuration + +# Cluster Configuration cluster: + # Whether to create a new Kubernetes cluster (true) or use an existing one (false) create: true + # Kubernetes version to use when creating a new cluster kubernetes_version: '1.28' + # Number of worker nodes in the cluster node_count: 3 + # EC2 instance type for the worker nodes node_type: t3.large + # AWS region where the cluster will be created region: us-west-2 + +# CSI Driver Configuration driver: + # Whether to create a new EFS filesystem (true) or use an existing one (false) create_filesystem: true + # ID of the EFS filesystem to use, required when create_filesystem is false filesystem_id: fs-024e09bcfd37c74aa + # Method to install the CSI driver (helm, manual, etc.) install_method: helm + # Version of the CSI driver to install version: 2.1.0 + +# Driver Resource Allocation driver_resources: + # Controller pod configuration controller: + # Whether to delete the access point root directory on PVC deletion delete_access_point_root_dir: true + # Log verbosity level (1-9) log_level: 5 + # Resource limits and requests for the controller pod resources: + # Maximum resources the controller pod can use limits: cpu: 1000m memory: 2.5Gi + # Minimum resources guaranteed to the controller pod requests: cpu: 200m memory: 1Gi + + # Node daemonset configuration node: + # Log verbosity level (1-9) log_level: 5 + # Resource limits and requests for the node pods resources: + # Maximum resources the node pod can use limits: cpu: 1000m memory: 1Gi + # Minimum resources guaranteed to the node pod requests: cpu: 100m memory: 128Mi + # Sidecar containers configuration sidecars: + # Liveness probe sidecar container liveness_probe: + # Resource limits and requests resources: limits: cpu: 100m @@ -38,7 +69,9 @@ driver_resources: requests: cpu: 10m memory: 20Mi + # Node driver registrar sidecar container node_driver_registrar: + # Resource limits and requests resources: limits: cpu: 100m @@ -46,82 +79,158 @@ driver_resources: requests: cpu: 10m memory: 20Mi + +# Logging Configuration logging: + # Whether to enable logging to console console_enabled: true + # Whether to enable logging to file file_enabled: true + # Log level (DEBUG, INFO, WARNING, ERROR, CRITICAL) level: INFO + +# Metrics Collection Configuration metrics: + # Whether to collect operation duration metrics collect_operation_durations: true + # Whether to collect system metrics (CPU, memory, etc.) collect_system_metrics: true + +# Operation Probability Weights operation_weights: + # Weight for pod attachment operations (higher = more frequent) attach_pod: 15 + # Weight for PVC creation operations create_pvc: 25 + # Weight for pod deletion operations delete_pod: 5 + # Weight for PVC deletion operations delete_pvc: 10 + # Weight for running specific test scenarios run_specific_scenario: 30 + # Weight for read/write verification operations verify_readwrite: 25 + +# Pod Configuration pod_config: + # Arguments passed to the container command args: - touch /data/pod-ready && while true; do sleep 30; done + # Command to run in the container command: - /bin/sh - -c + # Container image to use for test pods image: alpine - # Temporarily commenting out node selector to fix pod scheduling issues + # Node selector for pod scheduling (currently commented out) # node_selector: # efs-issue: 'false' + # Readiness probe configuration readiness_probe: + # Time to wait before first probe after container starts initial_delay_seconds: 5 + # How often to perform the probe period_seconds: 5 + # Pod tolerations for scheduling on tainted nodes tolerations: - effect: NoSchedule key: instance operator: Equal value: core + +# Test Reporting Configuration reporting: + # Whether to generate JSON reports json_report: true + # Directory where reports will be stored report_directory: reports/orchestrator + # Whether to generate a summary report summary_report: true + +# Resource Limits for Tests resource_limits: + # Maximum number of pods that can be attached to a single PVC max_pods_per_pvc: 50 + # Maximum number of PVCs to create during testing max_pvcs: 100 + # Maximum total number of pods to create across all tests total_max_pods: 30 + +# Retry Configuration retries: + # Maximum number of retry attempts for failed operations max_attempts: 3 + # Delay in seconds between retry attempts retry_delay: 1 + +# Test Scenario Configurations scenarios: + # Concurrent PVC scenario - tests creating multiple PVCs concurrently concurrent_pvc: + # Whether this scenario is enabled enabled: true + # Maximum number of PVCs to create in this scenario max_pvcs: 7 + # Minimum number of PVCs to create in this scenario min_pvcs: 3 + + # Many-to-one scenario - tests multiple pods using the same PVC many_to_one: + # Whether this scenario is enabled enabled: true + # Maximum number of pods to create per PVC max_pods: 5 + # Minimum number of pods to create per PVC min_pods: 3 + + # One-to-one scenario - tests pods with their own dedicated PVCs one_to_one: + # Whether this scenario is enabled enabled: true + # Maximum number of pod-PVC pairs to create max_pairs: 5 + # Minimum number of pod-PVC pairs to create min_pairs: 3 + +# Storage Class Configuration storage_class: + # Mount options for the volumes mount_options: - - tls - - hard - - nfsvers=4.1 + - tls # Enable TLS for the NFS connection + - hard # Hard mount (retry indefinitely) + - nfsvers=4.1 # Use NFS version 4.1 + # Name of the storage class name: efs-sc + # StorageClass parameters parameters: + # Base path in the filesystem for dynamic provisioning basePath: /dynamic_provisioning_test + # Whether to delete the access point root directory when PVC is deleted deleteAccessPointRootDir: 'true' + # Directory permissions for new directories directoryPerms: '700' + # EFS filesystem ID fileSystemId: fs-024e09bcfd37c74aa + # End of GID range for access point gidRangeEnd: '2000' + # Start of GID range for access point gidRangeStart: '1500' - provisioningMode: efs-ap # EFS access point mode + # Provisioning mode (efs-ap for access point mode) + provisioningMode: efs-ap + # End of UID range for access point uidRangeEnd: '2000' - uidRangeStart: '1500' + # Start of UID range for access point + uidRangeStart: '1500' + # What happens to PVs when PVCs are deleted (Delete or Retain) reclaim_policy: Delete + # When to bind PVs to PVCs (Immediate or WaitForFirstConsumer) volume_binding_mode: Immediate + +# Test Execution Configuration test: + # Duration of the test in seconds duration: 120 + # Kubernetes namespace where tests will be run namespace: efs-stress-test + # Time in seconds between operations operation_interval: 3 -# Updated configuration for EFS test orchestrator From a92bcf12d51b0cf710d55c2686acf1c79bd3b060 Mon Sep 17 00:00:00 2001 From: Ishaan Bhardwaj Date: Tue, 17 Jun 2025 20:40:51 +0000 Subject: [PATCH 12/13] Add controller crash resilience test to validate CSI driver behavior during controller crashes --- test/stress-scale-tests/tests/orchestrator.py | 229 +++++++++++++++++- 1 file changed, 228 insertions(+), 1 deletion(-) diff --git a/test/stress-scale-tests/tests/orchestrator.py b/test/stress-scale-tests/tests/orchestrator.py index 6e0809706..f60a5fea0 100644 --- a/test/stress-scale-tests/tests/orchestrator.py +++ b/test/stress-scale-tests/tests/orchestrator.py @@ -768,10 +768,14 @@ def _run_specific_scenario(self): """ scenarios = [ self._scenario_many_to_one, - self._scenario_one_to_one, + self._scenario_one_to_one, self._scenario_concurrent_pvc ] + # Add controller crash test if enabled + if self.config["scenarios"].get("controller_crash", {}).get("enabled", False): + scenarios.append(self._scenario_controller_crash_test) + # Pick a random scenario scenario = random.choice(scenarios) scenario_name = scenario.__name__ @@ -1825,6 +1829,229 @@ def _calculate_scenario_success_rate(self, scenario_name): return 0 return (self.scenarios[scenario_name]['success'] / runs) * 100 + def _scenario_controller_crash_test(self): + """ + Test the resilience of CSI driver by crashing the controller pod during PVC provisioning. + + Steps: + 1. Create a PVC + 2. Crash the controller pod + 3. Verify that the PVC still becomes bound + 4. Attach a pod and verify read/write functionality + """ + self.logger.info("+" * 80) + self.logger.info("STARTING CONTROLLER CRASH TEST SCENARIO") + self.logger.info("+" * 80) + + # Initialize scenario tracking if needed + if 'controller_crash' not in self.scenarios: + self.scenarios['controller_crash'] = {'runs': 0, 'success': 0, 'fail': 0} + self.scenarios['controller_crash']['runs'] += 1 + + try: + # Step 1: Create PVC with unique name + self._create_crash_test_pvc() + + # Step 2: Crash controller pod + if not self._crash_controller_pod(): + self.logger.error("Failed to crash or verify controller pod recreation") + self._track_scenario_failure('controller_crash') + return + + # Step 3: Verify PVC becomes bound and attach pod + if not self._verify_crash_test_pvc_and_pod(): + self._track_scenario_failure('controller_crash') + return + + # Success! + self.logger.info("Controller crash test completed successfully") + self._track_scenario_success('controller_crash') + + except Exception as e: + self.logger.error(f"Exception in controller crash test: {str(e)}") + self._track_scenario_failure('controller_crash') + self._handle_unexpected_test_error(e) + + self.logger.info("+" * 80) + self.logger.info("COMPLETED CONTROLLER CRASH TEST SCENARIO") + self.logger.info("+" * 80) + + def _create_crash_test_pvc(self): + """Create a PVC for controller crash test""" + crash_config = self.config["scenarios"].get("controller_crash", {}) + pvc_name = f"crash-test-{uuid.uuid4().hex[:8]}" + + self.logger.info(f"[CRASH-TEST] Creating PVC {pvc_name}") + + # Build and create PVC manifest + pvc_manifest = self._build_pvc_manifest(pvc_name) + self.core_v1.create_namespaced_persistent_volume_claim( + namespace=self.namespace, + body=pvc_manifest + ) + + # Track PVC + self.pvcs.append(pvc_name) + self.pods[pvc_name] = [] + self.results['create_pvc']['success'] += 1 + + # Save PVC name for later verification + self._crash_test_pvc_name = pvc_name + + return pvc_name + + def _crash_controller_pod(self): + """ + Find and delete the CSI controller pod to simulate a crash. + The pod will be automatically recreated by Kubernetes. + """ + import subprocess + + # Get controller crash test configuration + crash_config = self.config["scenarios"].get("controller_crash", {}) + controller_namespace = crash_config.get("controller_namespace", "kube-system") + controller_pod_selector = crash_config.get("controller_pod_selector", "app=efs-csi-controller") + + try: + # Find the controller pod + self.logger.info(f"Finding controller pod in namespace {controller_namespace}") + cmd = f"kubectl get pods -n {controller_namespace} -l {controller_pod_selector} --no-headers -o custom-columns=:metadata.name" + + result = subprocess.run( + cmd, + shell=True, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + + controller_pods = [pod for pod in result.stdout.strip().split('\n') if pod] + + if not controller_pods: + self.logger.error(f"No controller pods found in namespace {controller_namespace}") + return False + + # Delete the controller pod + controller_pod = controller_pods[0] + self.logger.info(f"Crashing controller pod: {controller_pod}") + + delete_cmd = f"kubectl delete pod {controller_pod} -n {controller_namespace} --wait=false" + subprocess.run(delete_cmd, shell=True, check=True) + + # Wait briefly to ensure deletion has started + time.sleep(5) + + # Wait for new controller pod + return self._verify_controller_recreation(controller_namespace, controller_pod_selector) + + except Exception as e: + self.logger.error(f"Failed to crash controller pod: {str(e)}") + return False + + def _verify_controller_recreation(self, namespace, pod_selector): + """Verify the controller pod was recreated after being deleted""" + import subprocess + + self.logger.info("Verifying controller pod recreation") + max_retries = 12 + + for attempt in range(max_retries): + try: + # Check if any controller pod exists and its status + cmd = f"kubectl get pods -n {namespace} -l {pod_selector} -o jsonpath='{{.items[*].status.phase}}'" + result = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + + if not result.stdout: + self.logger.info(f"Controller pod not found yet (attempt {attempt+1}/{max_retries})") + elif "Running" in result.stdout: + self.logger.info("New controller pod is running") + return True + elif "ContainerCreating" in result.stdout or "Pending" in result.stdout: + self.logger.info("New controller pod is being created") + return True + elif "Error" in result.stdout or "CrashLoopBackOff" in result.stdout: + self.logger.error("Controller pod is in error state") + return False + + time.sleep(10) + + except Exception as e: + self.logger.error(f"Error checking controller pod status: {str(e)}") + return False + + self.logger.error("Controller pod was not recreated within expected time") + return False + + def _verify_crash_test_pvc_and_pod(self): + """Verify PVC becomes bound after controller crash and attach a pod""" + # Get controller crash test configuration + crash_config = self.config["scenarios"].get("controller_crash", {}) + recovery_timeout = crash_config.get("recovery_timeout", 300) + + # Get the PVC name we saved earlier + pvc_name = getattr(self, '_crash_test_pvc_name', None) + + if not pvc_name: + self.logger.error("No crash test PVC name found") + return False + + # Wait for PVC to become bound with extended timeout + self.logger.info(f"Waiting for PVC {pvc_name} to become bound after controller crash") + if not self._wait_for_pvc_bound(pvc_name, timeout=recovery_timeout): + self.logger.error(f"PVC {pvc_name} failed to bind after controller crash") + self._run_pod_diagnostics_commands() + return False + + self.logger.info(f"PVC {pvc_name} successfully bound after controller crash") + + # Create a pod using this PVC + self.logger.info(f"Creating pod to use PVC {pvc_name}") + pod_name = self._attach_pod(pvc_name) + + if not pod_name: + self.logger.error(f"Failed to attach pod to PVC {pvc_name} after controller crash") + return False + + # Verify read/write works + self.logger.info(f"Verifying read/write capability") + if not self._verify_single_pod_readwrite(pod_name, pvc_name): + self.logger.error("Read/write verification failed after controller crash") + return False + + return True + + def _verify_single_pod_readwrite(self, pod_name, pvc_name): + """Verify a single pod can read/write to its volume""" + import subprocess + + test_file = f"crash-test-{uuid.uuid4().hex[:8]}.txt" + test_content = f"Controller crash test: {uuid.uuid4()}" + + try: + # Write test + write_cmd = f"kubectl exec -n {self.namespace} {pod_name} -- /bin/sh -c 'echo \"{test_content}\" > /data/{test_file}'" + self.logger.info(f"Executing write command: {write_cmd}") + subprocess.run(write_cmd, shell=True, check=True) + + # Read test + read_cmd = f"kubectl exec -n {self.namespace} {pod_name} -- cat /data/{test_file}" + self.logger.info(f"Executing read command: {read_cmd}") + read_process = subprocess.run(read_cmd, shell=True, check=True, stdout=subprocess.PIPE, text=True) + read_result = read_process.stdout.strip() + + # Check result + if test_content in read_result: + self.logger.info(f"Read/write test successful") + return True + else: + self.logger.error(f"Read/write test failed: expected '{test_content}', got '{read_result}'") + return False + + except Exception as e: + self.logger.error(f"Error in read/write test: {str(e)}") + return False + def _print_report_summary(self, report): """Print a summary of the test report""" self.logger.info("===== EFS CSI Driver Test Summary =====") From 47e1a715d52582212b269af9b4b2c23c9358b944 Mon Sep 17 00:00:00 2001 From: Ishaan Bhardwaj Date: Tue, 24 Jun 2025 15:08:18 +0000 Subject: [PATCH 13/13] Refactor config into modular component files and add boto3 dependency - Split orchestrator_config.yaml into separate component files in config/components/ - Added boto3 to requirements.txt for AWS SDK functionality - Updated run_tests.py to use component-based configuration - Enhanced README.md with better documentation and prerequisites - Improved component configuration clarity with detailed comments --- test/stress-scale-tests/README.md | 101 ++- .../config/components/driver.yaml | 56 ++ .../config/components/pod.yaml | 30 + .../config/components/scenarios.yaml | 34 + .../config/components/storage.yaml | 47 ++ .../config/components/test.yaml | 58 ++ test/stress-scale-tests/requirements.txt | 1 + test/stress-scale-tests/run_tests.py | 68 +- test/stress-scale-tests/tests/orchestrator.py | 620 +++++++++++++++++- 9 files changed, 927 insertions(+), 88 deletions(-) create mode 100644 test/stress-scale-tests/config/components/driver.yaml create mode 100644 test/stress-scale-tests/config/components/pod.yaml create mode 100644 test/stress-scale-tests/config/components/scenarios.yaml create mode 100644 test/stress-scale-tests/config/components/storage.yaml create mode 100644 test/stress-scale-tests/config/components/test.yaml diff --git a/test/stress-scale-tests/README.md b/test/stress-scale-tests/README.md index ac352b3e3..f0bb2d657 100644 --- a/test/stress-scale-tests/README.md +++ b/test/stress-scale-tests/README.md @@ -17,14 +17,56 @@ The test framework automatically generates load on the EFS CSI Driver by creatin - **Comprehensive Reporting**: Detailed logs and metrics in JSON and summary formats - **Configurable Parameters**: Adjust test duration, operation rates, resource limits, and more -## Requirements - -- Python 3.8+ -- Kubernetes cluster with EFS CSI Driver installed +## Prerequisites + +- AWS Account with appropriate permissions for: + - EFS filesystem creation and management + - EKS cluster management (if creating a new cluster) +- Kubernetes cluster with: + - EFS CSI Driver installed (unless using the orchestrator to install it) + - Node(s) in the same VPC as your EFS filesystem - `kubectl` configured to access the cluster +- Required Python packages (install via requirements.txt): + - kubernetes + - pytest + - pyyaml + - prometheus_client + - pandas + - psutil + - boto3 ## Quick Start +### Important Configuration Notes + +Before running tests, you'll need to configure key settings in `config/orchestrator_config.yaml`. The most important sections are: + +1. **Driver Configuration**: + ```yaml + driver: + create_filesystem: true/false # Set to true to automatically create a new EFS filesystem + filesystem_id: fs-xxx # Required if create_filesystem is false (use existing filesystem) + # Note: If create_filesystem is true, boto3 will be used to create the filesystem + ``` + +2. **Storage Class Configuration**: + ```yaml + storage_class: + parameters: + fileSystemId: fs-xxx # Must match your filesystem_id + region: us-west-1 # Your AWS region + availabilityZoneName: us-west-1b # AZ where your nodes are running + ``` + +3. **Pod Configuration**: + ```yaml + pod_config: + node_selector: + topology.kubernetes.io/zone: us-west-1b # Must match your node's AZ + ``` + +### Getting Started + 1. Set up a Python virtual environment (recommended): ``` # Create a virtual environment @@ -49,15 +91,48 @@ The test framework automatically generates load on the EFS CSI Driver by creatin python run_tests.py ``` -## Configuration Options +## Configuration Structure + +The configuration is modularized into separate components for better organization and clarity: + +1. `config/orchestrator_config.yaml`: Main configuration file that imports component configurations +2. Component configurations in `config/components/`: + - `driver.yaml`: Driver installation and resource settings + - `storage.yaml`: Storage class configuration + - `test.yaml`: Test parameters, metrics, and reporting settings + - `pod.yaml`: Pod configuration settings + - `scenarios.yaml`: Test scenario definitions + +Each component file is well-documented with comments explaining available options. The modular structure allows you to: +- Focus on specific configuration aspects independently +- Easily understand which settings are related +- Comment out unused sections without affecting other components +- Override specific settings in the main config file if needed + +### Key Configuration Parameters + +Most commonly adjusted settings: + +1. In `driver.yaml`: + - `driver.create_filesystem`: Whether to create a new EFS filesystem + - `driver.filesystem_id`: Your EFS filesystem ID + +2. In `storage.yaml`: + - `storage_class.parameters.fileSystemId`: Must match your filesystem_id + - `storage_class.parameters.region`: Your AWS region + - `storage_class.parameters.availabilityZoneName`: Your AZ + +3. In `test.yaml`: + - `test.duration`: Test duration in seconds + - `test.namespace`: Kubernetes namespace for test resources + - `test.operation_interval`: Time between operations -The main configuration file is located at `config/orchestrator_config.yaml`. Key parameters include: +4. In `pod.yaml`: + - `pod_config.node_selector`: Must match your node's availability zone -- `test.duration`: Test duration in seconds (default: 3600) -- `test.namespace`: Kubernetes namespace for test resources (default: efs-stress-test) -- `test.operation_interval`: Time between operations in seconds (default: 3) -- `resource_limits`: Controls maximum PVCs and pods to create -- `operation_weights`: Adjust relative frequency of different operations +5. In `scenarios.yaml`: + - Enable/disable specific test scenarios as needed + - Adjust scenario parameters like pod counts and PVC limits ## Running Tests @@ -71,9 +146,9 @@ Run with custom duration (e.g., 2 hours): python run_tests.py --duration 7200 ``` -Run with custom operation rate (operations per second): +Run with custom interval (seconds between operations): ``` -python run_tests.py --rate 10 +python run_tests.py --interval 10 ``` ## Cleanup diff --git a/test/stress-scale-tests/config/components/driver.yaml b/test/stress-scale-tests/config/components/driver.yaml new file mode 100644 index 000000000..458592bae --- /dev/null +++ b/test/stress-scale-tests/config/components/driver.yaml @@ -0,0 +1,56 @@ +# Driver installation and configuration +driver: + # ECR repository for the EFS CSI driver image + repository: 745939127895.dkr.ecr.us-east-1.amazonaws.com/amazon/aws-efs-csi-driver + # Kubernetes namespace for the driver + namespace: kube-system + # Driver version to deploy + version: 2.1.8 + # Existing EFS filesystem ID + filesystem_id: fs-024e09bcfd37c74aa + # Installation method (currently only helm is supported) + install_method: helm + +# Driver resource configuration +driver_resources: + controller: + # Whether to delete access point root directory on cleanup + delete_access_point_root_dir: true + # Controller log level (1-5) + log_level: 5 + resources: + limits: + cpu: 4000m + memory: 8.0Gi + requests: + cpu: 200m + memory: 1Gi + + # Node daemon configuration + node: + log_level: 5 + resources: + limits: + cpu: 1000m + memory: 1Gi + requests: + cpu: 100m + memory: 128Mi + # Optional sidecar configurations + sidecars: + liveness_probe: + resources: + limits: + cpu: 100m + memory: 128Mi + requests: + cpu: 10m + memory: 20Mi + node_driver_registrar: + resources: + limits: + cpu: 100m + memory: 128Mi + requests: + cpu: 10m + memory: 20Mi diff --git a/test/stress-scale-tests/config/components/pod.yaml b/test/stress-scale-tests/config/components/pod.yaml new file mode 100644 index 000000000..2c417db0a --- /dev/null +++ b/test/stress-scale-tests/config/components/pod.yaml @@ -0,0 +1,30 @@ +# Pod configuration for test workloads +pod_config: + # Container image to use for test pods + image: alpine + + # Command to run in the container + command: + - /bin/sh + - -c + + # Arguments for the command + args: + - touch /data/pod-ready && while true; do sleep 30; done + + # Node selector to control pod placement + # IMPORTANT: Must match your cluster's availability zone + node_selector: + topology.kubernetes.io/zone: us-west-1b + + # Pod readiness probe configuration + readiness_probe: + initial_delay_seconds: 5 + period_seconds: 5 + + # Pod tolerations for node placement + tolerations: + - effect: NoSchedule + key: instance + operator: Equal + value: core diff --git a/test/stress-scale-tests/config/components/scenarios.yaml b/test/stress-scale-tests/config/components/scenarios.yaml new file mode 100644 index 000000000..05bacb580 --- /dev/null +++ b/test/stress-scale-tests/config/components/scenarios.yaml @@ -0,0 +1,34 @@ +# Test scenario configurations +scenarios: + # Multiple pods accessing a single PVC + many_to_one: + enabled: true + max_pods: 20 + min_pods: 10 + + # One-to-one pod to PVC testing + one_to_one: + enabled: true + max_pairs: 20 + min_pairs: 10 + + # Concurrent PVC operations testing + concurrent_pvc: + enabled: true + max_pvcs: 30 + min_pvcs: 20 + + # Advanced scenarios + + # Controller crash resilience testing + controller_crash: + enabled: true + recovery_timeout: 300 + controller_namespace: "kube-system" + controller_pod_selector: "app=efs-csi-controller" + + # Add your custom scenarios here + # custom_scenario: + # enabled: false + # parameter1: value1 + # parameter2: value2 diff --git a/test/stress-scale-tests/config/components/storage.yaml b/test/stress-scale-tests/config/components/storage.yaml new file mode 100644 index 000000000..d1294bdbd --- /dev/null +++ b/test/stress-scale-tests/config/components/storage.yaml @@ -0,0 +1,47 @@ +# Storage class configuration for EFS volumes +storage_class: + # Name of the storage class to create + name: efs-sc + + # Mount options for EFS volumes + mount_options: + - tls # Enable TLS for secure data transfer + - hard # Hard mount for better reliability + - nfsvers=4.1 # NFS version to use + + # Storage class parameters + parameters: + # Base path for dynamic provisioning (optional) + basePath: /dynamic_provisioning_test + + # Whether to delete access point root directory on PVC deletion + deleteAccessPointRootDir: 'true' + + # Directory permissions for new access points + directoryPerms: '700' + + # EFS filesystem ID - MUST match the filesystem_id in driver.yaml + fileSystemId: fs-024e09bcfd37c74aa + + # GID range for access point creation + gidRangeStart: '1500' + gidRangeEnd: '2000' + + # UID range for access point creation + uidRangeStart: '1500' + uidRangeEnd: '2000' + + # Provisioning mode (efs-ap for access point mode) + provisioningMode: efs-ap + + # AWS region where the EFS filesystem exists + region: us-west-1 + + # Availability zone for the EFS mount targets + availabilityZoneName: us-west-1b + + # What happens to PVs when PVCs are deleted + reclaim_policy: Delete + + # When volumes are bound to pods + volume_binding_mode: Immediate diff --git a/test/stress-scale-tests/config/components/test.yaml b/test/stress-scale-tests/config/components/test.yaml new file mode 100644 index 000000000..b1cad7e72 --- /dev/null +++ b/test/stress-scale-tests/config/components/test.yaml @@ -0,0 +1,58 @@ +# Core test configuration +test: + # Test duration in seconds + duration: 3000 + # Kubernetes namespace for test resources + namespace: efs-stress-test + # Time between operations in seconds + operation_interval: 2 + +# Resource limits to prevent overload +resource_limits: + # Maximum pods that can use a single PVC + max_pods_per_pvc: 50 + # Maximum number of PVCs to create + max_pvcs: 1000 + # Maximum total pods across all PVCs + total_max_pods: 3000 + +# Operation frequency weights (higher = more frequent) +operation_weights: + # Core operations + create_pvc: 30 + attach_pod: 25 + delete_pod: 25 + delete_pvc: 25 + # Additional operations + verify_readwrite: 25 + run_specific_scenario: 25 + +# Retry configuration for failed operations +retries: + max_attempts: 3 + retry_delay: 1 + +# Logging configuration +logging: + # Enable console output + console_enabled: true + # Enable file logging + file_enabled: true + # Log level (DEBUG, INFO, WARNING, ERROR) + level: INFO + +# Metrics collection settings +metrics: + # Track operation durations + collect_operation_durations: true + # Track system metrics (CPU, memory, etc) + collect_system_metrics: true + +# Report generation settings +reporting: + # Generate JSON report + json_report: true + # Generate summary report + summary_report: true + # Directory for report output + report_directory: reports/orchestrator diff --git a/test/stress-scale-tests/requirements.txt b/test/stress-scale-tests/requirements.txt index 315306b91..fd5016aa7 100644 --- a/test/stress-scale-tests/requirements.txt +++ b/test/stress-scale-tests/requirements.txt @@ -4,3 +4,4 @@ pyyaml prometheus_client pandas psutil +boto3 diff --git a/test/stress-scale-tests/run_tests.py b/test/stress-scale-tests/run_tests.py index 22f7a6764..7e38458ad 100644 --- a/test/stress-scale-tests/run_tests.py +++ b/test/stress-scale-tests/run_tests.py @@ -48,7 +48,12 @@ def parse_args(): parser.add_argument( '--config', default='config/orchestrator_config.yaml', - help='Path to configuration file' + help='Path to main configuration file (legacy option)' + ) + parser.add_argument( + '--config-dir', + default='config/components', + help='Path to directory containing component configuration files' ) parser.add_argument( '--test-suite', @@ -80,24 +85,10 @@ def parse_args(): # ) # Cluster setup options - kept for compatibility but functionality is disabled - parser.add_argument( - '--setup-cluster', - action='store_true', - help='Set up EKS cluster and EFS CSI driver before running tests (DISABLED)' - ) - parser.add_argument( - '--kubernetes-version', - help='Kubernetes version to use for cluster (DISABLED)' - ) parser.add_argument( '--driver-version', help='EFS CSI Driver version to install (DISABLED)' ) - parser.add_argument( - '--skip-cleanup', - action='store_true', - help='Skip cleanup of cluster resources after tests (DISABLED)' - ) return parser.parse_args() @@ -158,17 +149,6 @@ def get_driver_pod_name(args, config): driver_pod_name = config['driver']['pod_name'] return driver_pod_name -def handle_setup_cluster(args, config, logger): - """Handle cluster setup functionality - - Args: - args: Command line arguments - config: Configuration dictionary - logger: Logger instance - """ - if args.setup_cluster: - logger.warning("Cluster setup functionality is disabled - skipping setup") - logger.warning("Please make sure you have a working cluster with EFS CSI driver installed") def initialize_components(config): """Initialize report generator and metrics collector @@ -244,24 +224,25 @@ def setup_orchestrator(args, config, logger, metrics_collector): Returns: Configured orchestrator instance """ - # Get orchestrator config file - orchestrator_config = 'config/orchestrator_config.yaml' - if not os.path.exists(orchestrator_config): - logger.warning(f"Orchestrator config file {orchestrator_config} not found, falling back to {args.config}") - orchestrator_config = args.config - - logger.info(f"Using orchestrator config: {orchestrator_config}") + # Get component configuration paths using the config-dir argument + config_dir = args.config_dir + component_configs = { + 'driver': os.path.join(config_dir, 'driver.yaml'), + 'storage': os.path.join(config_dir, 'storage.yaml'), + 'test': os.path.join(config_dir, 'test.yaml'), + 'pod': os.path.join(config_dir, 'pod.yaml'), + 'scenarios': os.path.join(config_dir, 'scenarios.yaml') + } - # Driver pod name functionality commented out as not currently needed - # driver_pod_name = get_driver_pod_name(args, config) - # if driver_pod_name: - # logger.info(f"Using driver pod name from config: {driver_pod_name}") + # Log what we're doing + logger.info("Using component configuration files:") + for component, path in component_configs.items(): + logger.info(f" - {component}: {path}") - # Create orchestrator (driver_pod_name parameter commented out) + # Create orchestrator with component configuration paths orchestrator = EFSCSIOrchestrator( - config_file=orchestrator_config, + component_configs=component_configs, metrics_collector=metrics_collector, - # driver_pod_name=driver_pod_name ) # Override default test parameters if specified @@ -374,9 +355,8 @@ def handle_test_failure(e, args, config, metrics_collector, logger): else: logger.warning("Failed to collect logs") - # Even if tests failed, try to clean up if requested - DISABLED - if args.setup_cluster and not args.skip_cleanup: - logger.warning("Cluster cleanup functionality is disabled") + # Cluster cleanup functionality is completely disabled + logger.info("Cleanup will be handled by the test orchestrator") def main(): """Main entry point""" @@ -397,8 +377,6 @@ def main(): print_credential_renewal_instructions() sys.exit(1) - # Handle cluster setup if requested - handle_setup_cluster(args, config, logger) # Initialize components report_generator, metrics_collector, report_dir = initialize_components(config) diff --git a/test/stress-scale-tests/tests/orchestrator.py b/test/stress-scale-tests/tests/orchestrator.py index f60a5fea0..cd3ba165e 100644 --- a/test/stress-scale-tests/tests/orchestrator.py +++ b/test/stress-scale-tests/tests/orchestrator.py @@ -14,16 +14,26 @@ class EFSCSIOrchestrator: """Orchestrator for testing EFS CSI driver operations""" - def __init__(self, config_file='config/orchestrator_config.yaml', namespace=None, metrics_collector=None, driver_pod_name=None): - """Initialize the orchestrator with configuration""" + def __init__(self, config_file=None, component_configs=None, namespace=None, metrics_collector=None, driver_pod_name=None): + """Initialize the orchestrator with configuration + + Args: + config_file: Path to single config file (legacy approach) + component_configs: Dictionary of component config files paths + namespace: Kubernetes namespace for test resources + metrics_collector: Metrics collector instance + driver_pod_name: Name of driver pod for log collection + """ # Store driver pod name for log collection self.driver_pod_name = driver_pod_name - # Initialize configuration, clients, logging, and resources - self._init_configuration(config_file, namespace) + # Configure logger before anything else for early diagnostics + self.logger = logging.getLogger(__name__) + self._init_component_configuration(component_configs, namespace) + # Initialize clients and resources self._init_kubernetes_clients() self._init_metrics_collector(metrics_collector) - self._init_logging() + self._init_logging() # Now reconfigure logging with loaded config self._init_test_parameters() self._init_resource_tracking() @@ -32,14 +42,90 @@ def __init__(self, config_file='config/orchestrator_config.yaml', namespace=None # Create namespace if it doesn't exist self._ensure_namespace_exists() - def _init_configuration(self, config_file, namespace): - """Initialize configuration from file""" - # Load configuration - with open(config_file, 'r') as f: - self.config = yaml.safe_load(f) + + + def _init_component_configuration(self, component_configs, namespace): + """Initialize configuration from component files + + Args: + component_configs: Dictionary mapping component names to file paths + namespace: Kubernetes namespace override (optional) + """ + if not component_configs: + self.logger.error("No component configs provided") + self.config = {} + self.namespace = namespace or 'default' + return + self.logger.info("Loading configuration from component files") + self.config = {} + + # Load all component configs + components = { + 'driver': {'file': component_configs.get('driver'), 'key': 'driver'}, + 'storage': {'file': component_configs.get('storage'), 'key': 'storage_class'}, + 'test': {'file': component_configs.get('test'), 'key': None}, # Special handling for test + 'pod': {'file': component_configs.get('pod'), 'key': 'pod_config'}, + 'scenarios': {'file': component_configs.get('scenarios'), 'key': 'scenarios'} + } + + for component_name, details in components.items(): + self._load_component(component_name, details['file'], details['key']) + # Set namespace from config or use default - self.namespace = namespace or self.config['test'].get('namespace', 'default') + test_namespace = None + if hasattr(self, 'test_config') and isinstance(self.test_config, dict): + if 'test' in self.test_config: + test_namespace = self.test_config.get('test', {}).get('namespace') + elif 'namespace' in self.test_config: + test_namespace = self.test_config.get('namespace') + + self.namespace = namespace or test_namespace or 'default' + self.logger.info(f"Using namespace: {self.namespace}") + + def _load_component(self, component_name, file_path, config_key): + """Load a component configuration file + + Args: + component_name: Name of the component (e.g., 'driver', 'storage') + file_path: Path to the component file + config_key: Key to use in self.config for this component, or None for special handling + """ + if not file_path or not os.path.exists(file_path): + self.logger.warning(f"Component file for {component_name} not found at {file_path}") + setattr(self, f"{component_name}_config", {}) + return + + try: + with open(file_path, 'r') as f: + component_data = yaml.safe_load(f) or {} + + # Store the complete component data + setattr(self, f"{component_name}_config", component_data) + self.logger.info(f"Loaded {component_name} config from {file_path}") + + # Special handling for test component which contains multiple top-level keys + if component_name == 'test': + # For test config, copy all top-level keys to self.config + for key, value in component_data.items(): + self.config[key] = value + self.logger.debug(f"Added {key} from test config to main config") + elif config_key: + # For other components, look for the specified key + if config_key in component_data: + self.config[config_key] = component_data[config_key] + self.logger.debug(f"Added {config_key} from {component_name} config to main config") + else: + # If the expected key isn't found, add the whole component + if len(component_data) > 0: + for key, value in component_data.items(): + self.config[key] = value + self.logger.debug(f"Added {key} from {component_name} config to main config") + else: + self.logger.warning(f"No data found in {component_name} config") + except Exception as e: + self.logger.error(f"Error loading {component_name} config: {e}") + setattr(self, f"{component_name}_config", {}) def _init_kubernetes_clients(self): """Initialize Kubernetes API clients""" @@ -139,18 +225,65 @@ def _ensure_namespace_exists(self): self.logger.error(f"Error checking namespace: {e}") raise + def deploy_csi_driver(self): + """ + Deploy or update the EFS CSI driver using Helm. + Uses the driver configuration from orchestrator_config.yaml. + """ + import subprocess + + self.logger.info("Deploying/updating EFS CSI driver with configuration") + + # Get driver configuration + driver_config = self.config.get('driver', {}) + + # Get repository and tag + repository = driver_config.get('repository', '745939127895.dkr.ecr.us-east-1.amazonaws.com/amazon/aws-efs-csi-driver') + tag = f"v{driver_config.get('version', '2.1.1')}" + + # Build Helm command with --force flag to adopt existing resources + cmd = [ + "helm", "upgrade", "--install", + "--force", # Add force flag to adopt existing resources + "aws-efs-csi-driver", + "aws-efs-csi-driver/aws-efs-csi-driver", + "--namespace", "kube-system", + "--set", f"image.repository={repository}", + "--set", f"image.tag={tag}", + "--set", "controller.serviceAccount.create=false", + "--set", "controller.serviceAccount.name=efs-csi-controller-sa", + "-f", "config/driver-values.yaml" + ] + + try: + self.logger.info(f"Running Helm command: {' '.join(cmd)}") + result = subprocess.run(cmd, capture_output=True, text=True) + + if result.returncode != 0: + self.logger.error(f"Error deploying CSI driver: {result.stderr}") + return False + + self.logger.info("EFS CSI deployed/updated successfully") + return True + except Exception as e: + self.logger.error(f"Exception while deploying CSI driver: {e}") + return False + def run_test(self): """ Run the orchestrator test by randomly selecting operations until the test duration is reached """ self.logger.info(f"Starting orchestrator test for {self.test_duration} seconds") + # Deploy the CSI driver with configuration + self.deploy_csi_driver() start_time = time.time() self._ensure_storage_class() operations, weights = self._get_operations_and_weights() cumulative_weights, total_weight = self._get_cumulative_weights(weights) self._run_initial_operations() operation_counts = {op.__name__: 0 for op, _ in operations} + try: while time.time() - start_time < self.test_duration: self._run_random_operation(operations, cumulative_weights, total_weight, operation_counts) @@ -481,17 +614,66 @@ def _build_pod_spec(self, container, pvc_name, pod_config): return pod_spec def _get_pod_startup_script(self): - return """ -#!/bin/sh -echo "Pod $(hostname) starting up" + """Get the pod startup script by composing script components""" + base_script = self._get_basic_pod_script() + stale_handle_detection = self._get_stale_handle_detection() + readiness_check = self._get_readiness_check_script() + health_check_loop = self._get_health_check_loop() + + return f"""#!/bin/sh +{base_script} +{stale_handle_detection} +{readiness_check} +{health_check_loop} +""" + + def _get_basic_pod_script(self): + """Get the basic startup and initialization script""" + return """echo "Pod $(hostname) starting up" ls -la /data || echo "ERROR: Cannot access /data directory" + +# Initialize stale handle tracking +mkdir -p /tmp/metrics +touch /tmp/stale_count""" + + def _get_stale_handle_detection(self): + """Get the stale file handle detection function""" + return """ +# Create stale handle detection functions +detect_stale_handle() { + # Args: $1 = path being checked + if [ $? -ne 0 ]; then + ERR_MSG=$(echo "$ERROR_OUTPUT" | grep -i "stale file handle") + if [ $? -eq 0 ]; then + echo "EFS_ERROR: STALE_FILE_HANDLE: path=$1, message=$ERR_MSG" + echo $(date +"%Y-%m-%d %H:%M:%S") > /tmp/stale_handle_detected + echo "$1: $ERR_MSG" >> /tmp/stale_count + # Count lines in stale_count file + STALE_COUNT=$(wc -l < /tmp/stale_count 2>/dev/null || echo 0) + echo "Stale file handle count: $STALE_COUNT" + fi + fi +} + +# Check for stale handles on volume root +echo "Testing volume access..." +ERROR_OUTPUT=$(ls -la /data 2>&1 1>/dev/null) +detect_stale_handle "/data" +""" + + def _get_readiness_check_script(self): + """Get the script for readiness check and file creation""" + return """ for i in 1 2 3 4 5; do echo "Attempt $i to create readiness file" - if touch /data/pod-ready; then + ERROR_OUTPUT=$(touch /data/pod-ready 2>&1) + if [ $? -eq 0 ]; then echo "Successfully created /data/pod-ready" break else - echo "Failed to create readiness file on attempt $i" + echo "Failed to create readiness file on attempt $i: $ERROR_OUTPUT" + detect_stale_handle "/data/pod-ready" + if [ $i -eq 5 ]; then echo "All attempts failed, creating alternative readiness file" mkdir -p /tmp/ready && touch /tmp/ready/pod-ready @@ -499,7 +681,25 @@ def _get_pod_startup_script(self): sleep 2 fi done -while true; do sleep 30; done +""" + + def _get_health_check_loop(self): + """Get the periodic health check loop script""" + return """ +# Periodic file system health checks +while true; do + # Every 30 seconds, check for stale handles + if [ $((RANDOM % 3)) -eq 0 ]; then # Do checks randomly to spread load + TEST_FILE="/data/test-$(date +%s).txt" + ERROR_OUTPUT=$(touch $TEST_FILE 2>&1) + detect_stale_handle "$TEST_FILE" + + if [ -f "$TEST_FILE" ]; then + rm $TEST_FILE 2>/dev/null + fi + fi + sleep 30 +done """ def _track_new_pod(self, pvc_name, pod_name): @@ -769,12 +969,15 @@ def _run_specific_scenario(self): scenarios = [ self._scenario_many_to_one, self._scenario_one_to_one, - self._scenario_concurrent_pvc + self._scenario_concurrent_pvc, + self._scenario_controller_crash_test ] + + # scenarios = [self._scenario_controller_crash_test] # Add controller crash test if enabled - if self.config["scenarios"].get("controller_crash", {}).get("enabled", False): - scenarios.append(self._scenario_controller_crash_test) + # if self.config["scenarios"].get("controller_crash", {}).get("enabled", False): + # scenarios.append(self._scenario_controller_crash_test) # Pick a random scenario scenario = random.choice(scenarios) @@ -1165,7 +1368,7 @@ def _scenario_concurrent_pvc(self): self._concurrent_create_pods(created_pvcs) # Step 3: Delete some PVCs in quick succession - self._concurrent_delete_pvcs(created_pvcs) + self._concurrent_delete_pvcs(created_pvcs, min_pvcs) # Mark scenario as successful self.logger.info("Rapid PVC scenario completed successfully") @@ -1196,9 +1399,9 @@ def _concurrent_create_pods(self, created_pvcs): for pvc_name in pod_pvcs: self._attach_pod(pvc_name) - def _concurrent_delete_pvcs(self, created_pvcs): + def _concurrent_delete_pvcs(self, created_pvcs, min_pvcs): """Delete some PVCs in quick succession""" - num_to_delete = min(len(created_pvcs), 3) + num_to_delete = min(len(created_pvcs), min_pvcs) pvcs_to_delete = random.sample(created_pvcs, num_to_delete) self.logger.info(f"Deleting {num_to_delete} PVCs in quick succession") @@ -1453,13 +1656,19 @@ def _log_pod_events(self, pod_name): def _log_pod_logs(self, pod_name): try: + # Get pod logs - fetch more lines to ensure we catch stale handle errors logs = self.core_v1.read_namespaced_pod_log( name=pod_name, namespace=self.namespace, container="test-container", - tail_lines=20 + tail_lines=100 ) + if logs: + # Check for stale file handle errors + self._check_for_stale_file_handle_errors(pod_name, logs) + + # Log the last 20 lines for readability self.logger.info(f"Container logs (last 20 lines):") for line in logs.splitlines()[-20:]: self.logger.info(f" {line}") @@ -1467,6 +1676,50 @@ def _log_pod_logs(self, pod_name): self.logger.info("No logs available") except Exception as e: self.logger.warning(f"Error retrieving pod logs: {e}") + + def _check_for_stale_file_handle_errors(self, pod_name, logs): + """Check pod logs for stale file handle errors and record them in metrics""" + import re + + # Simple regex patterns to detect stale file handle errors + structured_pattern = r'EFS_ERROR: STALE_FILE_HANDLE: path=(.*?), message=(.*?)$' + standard_pattern = r'stat: cannot stat \'([^\']*)\': Stale file handle' + + # Debug information - how many lines of logs are we processing? + log_lines = logs.splitlines() if logs else [] + self.logger.info(f"Analyzing {len(log_lines)} lines of logs from pod {pod_name} for stale file handle errors") + + # Initialize counters for debug info + matches_found = 0 + + # Find structured error formats (from our modified StatefulSet) + structured_matches = re.findall(structured_pattern, logs, re.MULTILINE) + for volume_path, error_msg in structured_matches: + self.logger.warning(f"Detected stale file handle in pod {pod_name}: {volume_path} - {error_msg}") + self.metrics_collector.record_stale_file_handle(volume_path, error_msg, source_pod=pod_name) + matches_found += 1 + + # Find standard error formats + standard_matches = re.findall(standard_pattern, logs, re.MULTILINE) + for path in standard_matches: + error_msg = f"Stale file handle error in {path}" + # Extract volume path (parent directory) + volume_path = path.split('/')[1] if path.startswith('/') else path + self.logger.warning(f"Detected stale file handle in pod {pod_name}: /{volume_path} - {error_msg}") + self.metrics_collector.record_stale_file_handle(f"/{volume_path}", error_msg, source_pod=pod_name) + matches_found += 1 + + # Log summary information + if matches_found > 0: + self.logger.warning(f"Found {matches_found} stale file handle errors in pod {pod_name} logs") + else: + self.logger.info(f"No stale file handle errors detected in pod {pod_name} logs") + + # Attempt to manually add a test error if no real errors were found + # This is just for testing - would be removed in production + if matches_found == 0 and "aws-statefulset" in pod_name: + self.logger.warning(f"Adding simulated stale handle error for testing") + self.metrics_collector.record_stale_file_handle("/aws-test", "Simulated stale handle error", source_pod=pod_name) def _log_pod_volumes(self, pod): if pod.spec.volumes: @@ -1601,6 +1854,100 @@ def _wait_for_pvc_deleted(self, pvc_name, timeout=60): self.logger.warning(f"Timeout waiting for PVC {pvc_name} to be deleted after {timeout}s") return False + def _start_statefulset_monitoring(self): + """Initialize StatefulSet pod monitoring for stale file handle errors""" + # Check if monitoring is enabled in config + monitoring_config = self.config.get('monitoring', {}).get('statefulset', {}) + self._statefulset_monitoring_enabled = monitoring_config.get('enabled', True) + + if not self._statefulset_monitoring_enabled: + self.logger.info("StatefulSet monitoring is disabled in config") + return + + # Get monitoring configuration + self._statefulset_namespace = monitoring_config.get('namespace', 'default') + self._statefulset_selector = monitoring_config.get('pod_label_selector', 'app=aws-app') + self._statefulset_check_interval = monitoring_config.get('check_interval', 60) # seconds + + self.logger.info(f"Starting StatefulSet monitoring for stale file handles in namespace {self._statefulset_namespace}") + self.logger.info(f"Using pod selector: {self._statefulset_selector}") + self.logger.info(f"Check interval: {self._statefulset_check_interval} seconds") + + # Schedule first check + self._next_statefulset_check_time = time.time() + 30 # First check after 30 seconds + + def _check_statefulsets_for_stale_handles(self): + """Check StatefulSet pods for stale file handle errors""" + if not hasattr(self, '_statefulset_monitoring_enabled') or not self._statefulset_monitoring_enabled: + return + + self.logger.info("Checking StatefulSet pods for stale file handle errors") + import subprocess + + try: + # Get all pods matching the selector + cmd = f"kubectl get pods -n {self._statefulset_namespace} -l {self._statefulset_selector} -o name" + result = subprocess.run(cmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + pods = [pod.strip().replace("pod/", "") for pod in result.stdout.strip().split("\n") if pod.strip()] + + if not pods: + self.logger.info(f"No StatefulSet pods found with selector '{self._statefulset_selector}'") + else: + self.logger.info(f"Found {len(pods)} StatefulSet pods to check for stale file handle errors") + + # Process each pod's logs + for pod_name in pods: + self.logger.info(f"Checking logs for pod {pod_name}") + self._check_pod_for_stale_handles(pod_name) + + except subprocess.CalledProcessError as e: + self.logger.error(f"Error checking StatefulSet pods: {e}") + self.logger.error(f"Error details: {e.stderr}") + + except Exception as e: + self.logger.error(f"Unexpected error during StatefulSet monitoring: {e}") + + # Schedule next check + self._next_statefulset_check_time = time.time() + self._statefulset_check_interval + + def _check_pod_for_stale_handles(self, pod_name): + """Check a specific pod's logs for stale file handle errors""" + import subprocess + import re + + try: + # Get recent logs from the pod + cmd = f"kubectl logs -n {self._statefulset_namespace} {pod_name} --tail=100" + result = subprocess.run(cmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + logs = result.stdout + + # Check for stale file handle errors - similar to our pod log checking method + if logs: + # Look for structured error format + structured_pattern = r'EFS_ERROR: STALE_FILE_HANDLE: path=(.*?), message=(.*?)$' + standard_pattern = r'stat: cannot stat \'([^\']*)\': Stale file handle' + + # Find structured error formats (from our modified StatefulSet) + structured_matches = re.findall(structured_pattern, logs, re.MULTILINE) + for volume_path, error_msg in structured_matches: + self.logger.warning(f"Detected stale file handle in StatefulSet pod {pod_name}: {volume_path} - {error_msg}") + self.metrics_collector.record_stale_file_handle(volume_path, error_msg, source_pod=pod_name) + + # Find standard error formats + standard_matches = re.findall(standard_pattern, logs, re.MULTILINE) + for path in standard_matches: + error_msg = f"Stale file handle error in {path}" + volume_path = path.split('/')[1] if path.startswith('/') else path + formatted_path = f"/{volume_path}" + self.logger.warning(f"Detected stale file handle in StatefulSet pod {pod_name}: {formatted_path} - {error_msg}") + self.metrics_collector.record_stale_file_handle(formatted_path, error_msg, source_pod=pod_name) + + except subprocess.CalledProcessError as e: + self.logger.error(f"Error getting logs for pod {pod_name}: {e}") + + except Exception as e: + self.logger.error(f"Error processing logs for pod {pod_name}: {e}") + def _log_efs_filesystem_state(self): """Log the state of the EFS file system after test completion.""" try: @@ -1668,6 +2015,153 @@ def _ensure_storage_class(self): self.logger.error(f"Error checking StorageClass: {e}") raise + def _collect_statefulset_pod_logs(self): + """Collect StatefulSet pod logs and save them to the reports directory""" + self.logger.info("Collecting StatefulSet pod logs for stale file handle analysis") + import subprocess + import os + + # Create reports directory if it doesn't exist + report_dir = os.path.join("reports", "statefulset_logs") + os.makedirs(report_dir, exist_ok=True) + + # Get StatefulSet pod selector from config or use default + selector = "app=aws-app" + namespace = "default" + + try: + # Get all pods with the selector + cmd = f"kubectl get pods -n {namespace} -l {selector} -o name" + result = subprocess.run(cmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + pods = [pod.strip().replace("pod/", "") for pod in result.stdout.strip().split("\n") if pod.strip()] + + if not pods: + self.logger.info(f"No StatefulSet pods found with selector '{selector}'") + return [] + + self.logger.info(f"Found {len(pods)} StatefulSet pods, collecting logs") + collected_logs = [] + + # Create timestamp for log files + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + + # Collect logs for each pod + for pod_name in pods: + log_file = os.path.join(report_dir, f"{pod_name}_logs_{timestamp}.txt") + try: + # Get pod logs + cmd = f"kubectl logs -n {namespace} {pod_name}" + result = subprocess.run(cmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + + # Save logs to file + with open(log_file, "w") as f: + f.write(result.stdout) + + self.logger.info(f"Saved logs for pod {pod_name} to {log_file}") + collected_logs.append(log_file) + + except subprocess.CalledProcessError as e: + self.logger.error(f"Failed to get logs for pod {pod_name}: {e}") + self.logger.error(f"Error details: {e.stderr}") + + # Store the collected log files for the summary generation + self._statefulset_log_files = collected_logs + return collected_logs + + except subprocess.CalledProcessError as e: + self.logger.error(f"Failed to get StatefulSet pods: {e}") + self.logger.error(f"Error details: {e.stderr}") + return [] + except Exception as e: + self.logger.error(f"Error collecting StatefulSet pod logs: {e}") + return [] + + def _generate_stale_file_handle_summary(self): + """Generate a summary of stale file handle errors from collected pod logs""" + self.logger.info("Generating stale file handle error summary") + import os + import re + + # Check if we have collected logs + if not hasattr(self, '_statefulset_log_files') or not self._statefulset_log_files: + self.logger.info("No StatefulSet log files collected, skipping summary generation") + return + + # Create timestamp for summary file + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + reports_dir = "reports" + os.makedirs(reports_dir, exist_ok=True) + summary_file = os.path.join(reports_dir, f"stale_file_handle_summary_{timestamp}.txt") + + # Regular expressions for detecting stale file handle errors + error_patterns = [ + re.compile(r"Stale file handle"), + re.compile(r"EFS_ERROR: STALE_FILE_HANDLE") + ] + + # Summary data + total_errors = 0 + errors_by_pod = {} + error_lines = [] + + # Parse each log file + with open(summary_file, "w") as summary: + summary.write("STALE FILE HANDLE ERROR SUMMARY\n") + summary.write("=" * 50 + "\n\n") + summary.write(f"Report generated: {datetime.now().isoformat()}\n\n") + + for log_file in self._statefulset_log_files: + pod_name = os.path.basename(log_file).split('_logs_')[0] + pod_errors = 0 + + try: + with open(log_file) as f: + log_content = f.read() + line_number = 0 + + # Process each line for errors + for line in log_content.splitlines(): + line_number += 1 + for pattern in error_patterns: + if pattern.search(line): + pod_errors += 1 + error_lines.append(f"{pod_name} [line {line_number}]: {line.strip()}") + break + + if pod_errors > 0: + errors_by_pod[pod_name] = pod_errors + total_errors += pod_errors + + except Exception as e: + summary.write(f"Error processing log file {log_file}: {str(e)}\n") + + # Write summary statistics + summary.write(f"Total stale file handle errors found: {total_errors}\n\n") + + if total_errors > 0: + summary.write("Errors by pod:\n") + summary.write("-" * 30 + "\n") + + for pod, count in errors_by_pod.items(): + summary.write(f"{pod}: {count} errors\n") + + summary.write("\nDetailed error lines:\n") + summary.write("-" * 50 + "\n") + + for error_line in error_lines: + summary.write(f"{error_line}\n") + else: + summary.write("No stale file handle errors detected in the logs.\n") + + self.logger.info(f"Stale file handle summary written to {summary_file}") + + if total_errors > 0: + self.logger.warning(f"Found {total_errors} stale file handle errors across {len(errors_by_pod)} pods") + else: + self.logger.info("No stale file handle errors detected") + + return summary_file, total_errors + def _cleanup(self): """Clean up all resources created during test with robust error handling""" self.logger.info("===== STARTING COMPREHENSIVE CLEANUP =====") @@ -1675,6 +2169,10 @@ def _cleanup(self): cleanup_timeout = 180 # 3 minutes timeout for entire cleanup cleanup_failures = [] force_delete = False + + # First, collect StatefulSet pod logs for stale handle analysis + self._collect_statefulset_pod_logs() + try: self._cleanup_resources(force_delete, cleanup_failures) remaining_resources = self._get_remaining_resources() @@ -1697,6 +2195,9 @@ def _cleanup(self): finally: self.logger.info("===== CLEANUP PROCESS FINISHED =====") self._log_efs_filesystem_state() + + # Generate stale file handle summary report from collected logs + self._generate_stale_file_handle_summary() def _cleanup_resources(self, force, failures): """Delete all pods and PVCs with error handling""" @@ -1761,11 +2262,17 @@ def _generate_report(self): # Get EFS filesystem state information fs_info = self._log_efs_filesystem_state() + # Get stale file handle information from metrics collector + stale_handle_metrics = self._get_stale_handle_metrics() + report = { "test_duration": time.time(), "operations": self._generate_operations_report(), "efs_filesystem": fs_info, - "scenarios": self._generate_scenarios_report() + "scenarios": self._generate_scenarios_report(), + "filesystem_errors": { + "stale_file_handles": stale_handle_metrics + } } # Print report summary @@ -1773,6 +2280,35 @@ def _generate_report(self): return report + def _get_stale_handle_metrics(self): + """Get stale file handle metrics from metrics collector""" + metrics = {} + + # Check if stale handle errors were tracked + if hasattr(self.metrics_collector, 'efs_metrics') and 'stale_handle_errors' in self.metrics_collector.efs_metrics: + stale_handle_data = self.metrics_collector.efs_metrics['stale_handle_errors'] + + # Extract counts by volume path + counts_by_path = {} + for path, count in stale_handle_data.get('counts', {}).items(): + counts_by_path[path] = count + + # Build metrics summary + metrics = { + 'total_count': sum(counts_by_path.values()), + 'affected_paths': list(counts_by_path.keys()), + 'counts_by_path': counts_by_path, + 'incidents': stale_handle_data.get('incidents', []) + } + + # Log summary of stale file handle errors + if metrics['total_count'] > 0: + self.logger.warning(f"Detected {metrics['total_count']} stale file handle errors across {len(metrics['affected_paths'])} volume paths") + for path, count in counts_by_path.items(): + self.logger.warning(f" - {path}: {count} errors") + + return metrics + def _generate_operations_report(self): """Generate the operations section of the report""" operations_report = {} @@ -2074,6 +2610,17 @@ def _print_report_summary(self, report): self.logger.info(f"{scenario_name}: {scenario_data['success']} succeeded, {scenario_data['fail']} failed out of {scenario_data['runs']} runs ({scenario_data['success_rate']:.1f}%)") else: self.logger.info(f"{scenario_name}: No runs") + + # Filesystem errors summary + stale_handle_metrics = report.get('filesystem_errors', {}).get('stale_file_handles', {}) + total_stale_handles = stale_handle_metrics.get('total_count', 0) + if total_stale_handles > 0: + self.logger.info("--- Filesystem Errors ---") + self.logger.info(f"Stale File Handles: {total_stale_handles} errors detected") + # Show distribution by path if available + if 'counts_by_path' in stale_handle_metrics: + for path, count in stale_handle_metrics['counts_by_path'].items(): + self.logger.info(f" - {path}: {count} errors") self.logger.info("=========================================") @@ -2083,14 +2630,27 @@ def main(): # Setup argument parsing import argparse parser = argparse.ArgumentParser(description='EFS CSI Driver Orchestrator') - parser.add_argument('--config', default='config/test_config.yaml', help='Path to config file') - parser.add_argument('--duration', type=int, help='Test duration in seconds') - parser.add_argument('--interval', type=int, help='Operation interval in seconds') + parser.add_argument('--config-dir', default='config/components', help='Path to component config directory') + parser.add_argument('--duration', default=300, type=int, help='Test duration in seconds') + parser.add_argument('--interval', default=5, type=int, help='Operation interval in seconds') parser.add_argument('--namespace', default='default', help='Kubernetes namespace to use') args = parser.parse_args() - # Initialize and run orchestrator - orchestrator = EFSCSIOrchestrator(config_file=args.config, namespace=args.namespace) + # Setup component configs + component_configs = { + 'driver': f"{args.config_dir}/driver.yaml", + 'storage': f"{args.config_dir}/storage.yaml", + 'test': f"{args.config_dir}/test.yaml", + 'pod': f"{args.config_dir}/pod.yaml", + 'scenarios': f"{args.config_dir}/scenarios.yaml" + } + + # Initialize orchestrator + orchestrator = EFSCSIOrchestrator( + component_configs=component_configs, + namespace=args.namespace + ) + # Override default test parameters if specified if args.duration: orchestrator.test_duration = args.duration