Skip to content

Commit 8617a06

Browse files
hehe7318hehe7318
authored
[E2E Test] Use isolated venv for test_on_demand_capacity_reservation to avoid affecting parallel tests (#7187)
* Use isolated venv for test_odcr to avoid affecting parallel tests Previously, the test reinstalled pcluster CLI globally with a patched version, which could interfere with other tests running in parallel. Now the patched CLI is installed into an isolated virtual environment in /tmp, leaving the system pcluster unchanged.
1 parent 6bd8e1d commit 8617a06

File tree

1 file changed

+133
-55
lines changed

1 file changed

+133
-55
lines changed

‎tests/integration-tests/tests/capacity_reservations/test_on_demand_capacity_reservation.py‎

Lines changed: 133 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -9,20 +9,62 @@
99
# or in the "LICENSE.txt" file accompanying this file.
1010
# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
1111
# See the License for the specific language governing permissions and limitations under the License.
12+
import json
1213
import logging
1314
import os
15+
import shutil
1416
import subprocess
17+
import tempfile
18+
import venv
1519

1620
import boto3
1721
import pytest
1822
from assertpy import assert_that
19-
from utils import describe_cluster_instances, retrieve_cfn_resources, wait_for_computefleet_changed
23+
from clusters_factory import Cluster
24+
from utils import describe_cluster_instances, random_alphanumeric, retrieve_cfn_resources, wait_for_computefleet_changed
25+
26+
27+
def _run_pcluster_command_in_venv(pcluster_path, args, timeout=7200):
28+
"""Run pcluster command using the specified pcluster executable."""
29+
command = [pcluster_path] + args
30+
logging.info("Executing command: %s", " ".join(command))
31+
result = subprocess.run(
32+
command,
33+
capture_output=True,
34+
universal_newlines=True,
35+
encoding="utf-8",
36+
timeout=timeout,
37+
)
38+
if result.returncode != 0:
39+
logging.error("Command failed with error:\n%s\nand output:\n%s", result.stderr, result.stdout)
40+
return result
41+
42+
43+
def _create_cluster_with_custom_pcluster(pcluster_path, cluster_name, config_file, region):
44+
"""Create a cluster using a custom pcluster executable."""
45+
logging.info("Creating cluster %s with config %s using custom pcluster", cluster_name, config_file)
46+
args = [
47+
"create-cluster",
48+
"--rollback-on-failure",
49+
"false",
50+
"--cluster-configuration",
51+
config_file,
52+
"--cluster-name",
53+
cluster_name,
54+
"--region",
55+
region,
56+
"--wait",
57+
]
58+
result = _run_pcluster_command_in_venv(pcluster_path, args)
59+
response = json.loads(result.stdout)
60+
if response.get("cloudFormationStackStatus") != "CREATE_COMPLETE":
61+
raise Exception(f"Cluster creation failed for {cluster_name}: {result.stdout}")
62+
logging.info("Cluster %s created successfully", cluster_name)
63+
return response
2064

2165

2266
@pytest.mark.usefixtures("os", "region")
23-
def test_on_demand_capacity_reservation(
24-
region, pcluster_config_reader, placement_group_stack, odcr_stack, clusters_factory
25-
):
67+
def test_on_demand_capacity_reservation(region, pcluster_config_reader, placement_group_stack, odcr_stack, request):
2668
"""Verify open, targeted and pg odcrs can be created and instances can be launched into them."""
2769

2870
"""This test is only for slurm."""
@@ -43,22 +85,31 @@ def test_on_demand_capacity_reservation(
4385
pg_capacity_reservation_arn=resource_group_arn,
4486
)
4587

46-
# Apply patch to the repo
47-
logging.info("Applying patch to the repository")
88+
# Create an isolated virtual environment for the patched CLI
89+
# This avoids affecting other parallel tests that share the same Python environment
4890
repo_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../.."))
4991
s3_bucket_file = os.path.join(repo_root, "cli/src/pcluster/models/s3_bucket.py")
5092

5193
# Backup the original file
5294
with open(s3_bucket_file, "r") as f:
5395
original_content = f.read()
5496

97+
venv_dir = tempfile.mkdtemp(prefix="pcluster_test_odcr_")
98+
cluster_name = f"integ-tests-{random_alphanumeric()}"
99+
cluster = None
100+
pcluster_path = None
101+
55102
try:
56-
# Apply the patch - inject the bug that replaces capacity reservation IDs
57-
with open(s3_bucket_file, "r") as f:
58-
content = f.read()
103+
# Create isolated virtual environment
104+
logging.info("Creating isolated virtual environment at %s", venv_dir)
105+
venv.create(venv_dir, with_pip=True)
59106

60-
# Add the bug injection line after the upload_config method definition
61-
modified_content = content.replace(
107+
pip_path = os.path.join(venv_dir, "bin", "pip")
108+
pcluster_path = os.path.join(venv_dir, "bin", "pcluster")
109+
110+
# Apply the patch - inject the bug that replaces capacity reservation IDs
111+
logging.info("Applying patch to the repository")
112+
modified_content = original_content.replace(
62113
" def upload_config(self, config, config_name, format=S3FileFormat.YAML):\n"
63114
' """Upload config file to S3 bucket."""',
64115
" def upload_config(self, config, config_name, format=S3FileFormat.YAML):\n"
@@ -71,57 +122,84 @@ def test_on_demand_capacity_reservation(
71122
with open(s3_bucket_file, "w") as f:
72123
f.write(modified_content)
73124

74-
# Install the CLI
75-
logging.info("Installing CLI from local repository")
76-
subprocess.run(["pip", "install", "./cli"], cwd=repo_root, check=True)
125+
# Install the patched CLI into the isolated environment
126+
logging.info("Installing patched CLI into isolated virtual environment")
127+
subprocess.run([pip_path, "install", "./cli"], cwd=repo_root, check=True)
77128

78-
# Create the cluster
79-
cluster = clusters_factory(cluster_config)
80-
finally:
81129
# Revert the patch by restoring the original file
82130
logging.info("Reverting patch from the repository")
83131
with open(s3_bucket_file, "w") as f:
84132
f.write(original_content)
85133

86-
# Reinstall the CLI
87-
logging.info("Reinstalling CLI from local repository")
88-
subprocess.run(["pip", "install", "./cli"], cwd=repo_root, check=True)
134+
# Create the cluster using the patched pcluster from isolated environment
135+
_create_cluster_with_custom_pcluster(pcluster_path, cluster_name, str(cluster_config), region)
89136

90-
_assert_instance_in_capacity_reservation(cluster, region, "open-odcr-id-cr", odcr_resources["integTestsOpenOdcr"])
91-
_assert_instance_in_capacity_reservation(cluster, region, "open-odcr-arn-cr", odcr_resources["integTestsOpenOdcr"])
92-
_assert_instance_in_capacity_reservation(
93-
cluster, region, "open-odcr-id-pg-cr", odcr_resources["integTestsOpenOdcr"]
94-
)
95-
_assert_instance_in_capacity_reservation(
96-
cluster, region, "open-odcr-arn-pg-cr", odcr_resources["integTestsOpenOdcr"]
97-
)
98-
_assert_instance_in_capacity_reservation(
99-
cluster, region, "target-odcr-id-cr", odcr_resources["integTestsTargetOdcr"]
100-
)
101-
_assert_instance_in_capacity_reservation(
102-
cluster, region, "target-odcr-arn-cr", odcr_resources["integTestsTargetOdcr"]
103-
)
104-
_assert_instance_in_capacity_reservation(
105-
cluster, region, "target-odcr-id-pg-cr", odcr_resources["integTestsTargetOdcr"]
106-
)
107-
_assert_instance_in_capacity_reservation(
108-
cluster, region, "target-odcr-arn-pg-cr", odcr_resources["integTestsTargetOdcr"]
109-
)
110-
_assert_instance_in_capacity_reservation(cluster, region, "pg-odcr-id-cr", odcr_resources["integTestsPgOdcr"])
111-
_assert_instance_in_capacity_reservation(cluster, region, "pg-odcr-arn-cr", odcr_resources["integTestsPgOdcr"])
112-
cluster.stop()
113-
wait_for_computefleet_changed(cluster, "STOPPED")
114-
updated_config_file = pcluster_config_reader(
115-
config_file="pcluster.config.update.yaml",
116-
placement_group=placement_group_stack.cfn_resources["PlacementGroup"],
117-
open_capacity_reservation_id=odcr_resources["integTestsOpenOdcr"],
118-
open_capacity_reservation_arn=resource_group_arn,
119-
target_capacity_reservation_id=odcr_resources["integTestsTargetOdcr"],
120-
target_capacity_reservation_arn=resource_group_arn,
121-
pg_capacity_reservation_id=odcr_resources["integTestsPgOdcr"],
122-
pg_capacity_reservation_arn=resource_group_arn,
123-
)
124-
cluster.update(str(updated_config_file))
137+
# Create a Cluster object for the rest of the test (uses system pcluster for other operations)
138+
cluster = Cluster(
139+
name=cluster_name,
140+
config_file=str(cluster_config),
141+
ssh_key=request.config.getoption("key_path"),
142+
region=region,
143+
)
144+
cluster.mark_as_created()
145+
146+
_assert_instance_in_capacity_reservation(
147+
cluster, region, "open-odcr-id-cr", odcr_resources["integTestsOpenOdcr"]
148+
)
149+
_assert_instance_in_capacity_reservation(
150+
cluster, region, "open-odcr-arn-cr", odcr_resources["integTestsOpenOdcr"]
151+
)
152+
_assert_instance_in_capacity_reservation(
153+
cluster, region, "open-odcr-id-pg-cr", odcr_resources["integTestsOpenOdcr"]
154+
)
155+
_assert_instance_in_capacity_reservation(
156+
cluster, region, "open-odcr-arn-pg-cr", odcr_resources["integTestsOpenOdcr"]
157+
)
158+
_assert_instance_in_capacity_reservation(
159+
cluster, region, "target-odcr-id-cr", odcr_resources["integTestsTargetOdcr"]
160+
)
161+
_assert_instance_in_capacity_reservation(
162+
cluster, region, "target-odcr-arn-cr", odcr_resources["integTestsTargetOdcr"]
163+
)
164+
_assert_instance_in_capacity_reservation(
165+
cluster, region, "target-odcr-id-pg-cr", odcr_resources["integTestsTargetOdcr"]
166+
)
167+
_assert_instance_in_capacity_reservation(
168+
cluster, region, "target-odcr-arn-pg-cr", odcr_resources["integTestsTargetOdcr"]
169+
)
170+
_assert_instance_in_capacity_reservation(cluster, region, "pg-odcr-id-cr", odcr_resources["integTestsPgOdcr"])
171+
_assert_instance_in_capacity_reservation(cluster, region, "pg-odcr-arn-cr", odcr_resources["integTestsPgOdcr"])
172+
173+
cluster.stop()
174+
wait_for_computefleet_changed(cluster, "STOPPED")
175+
176+
updated_config_file = pcluster_config_reader(
177+
config_file="pcluster.config.update.yaml",
178+
placement_group=placement_group_stack.cfn_resources["PlacementGroup"],
179+
open_capacity_reservation_id=odcr_resources["integTestsOpenOdcr"],
180+
open_capacity_reservation_arn=resource_group_arn,
181+
target_capacity_reservation_id=odcr_resources["integTestsTargetOdcr"],
182+
target_capacity_reservation_arn=resource_group_arn,
183+
pg_capacity_reservation_id=odcr_resources["integTestsPgOdcr"],
184+
pg_capacity_reservation_arn=resource_group_arn,
185+
)
186+
cluster.update(str(updated_config_file))
187+
188+
finally:
189+
# Ensure original file is restored
190+
with open(s3_bucket_file, "w") as f:
191+
f.write(original_content)
192+
193+
# Clean up the cluster
194+
if cluster and not request.config.getoption("no_delete"):
195+
try:
196+
cluster.delete()
197+
except Exception as e:
198+
logging.error("Failed to delete cluster: %s", e)
199+
200+
# Clean up the isolated virtual environment
201+
logging.info("Cleaning up isolated virtual environment")
202+
shutil.rmtree(venv_dir, ignore_errors=True)
125203

126204

127205
def _assert_instance_in_capacity_reservation(cluster, region, compute_resource_name, expected_reservation):

0 commit comments

Comments
 (0)