Revert "Do not fail check if PyTorch HUD API is down (#37)" #39
Conversation
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
|
OpenTofu plan for prod Plan: 0 to add, 7 to change, 0 to destroy.OpenTofu used the selected providers to generate the following execution
plan. Resource actions are indicated with the following symbols:
!~ update in-place
OpenTofu will perform the following actions:
# datadog_synthetics_test.pytorch-gha-runners-queue-check-amd will be updated in-place
!~ resource "datadog_synthetics_test" "pytorch-gha-runners-queue-check-amd" {
id = "yt8-7zy-xpj"
name = "GHA Runner Queue Check - AMD Runners"
tags = [
"env:project",
"project:pytorch",
"service:gha-runners",
]
# (10 unchanged attributes hidden)
!~ assertion {
!~ code = <<-EOT
- if (dd.response.statusCode !== 200) {
- // We do not want to fail due to hud.pytorch.org API failure.
- console.log('Status code is not 200, stopping execution');
- dd.expect(true).to.equal(true);
- }
- else {
- const MACHINE_TYPE_FILTER = '.rocm.';
- const jsonData = dd.response.body;
- const parsedData = JSON.parse(jsonData);
+ dd.expect(dd.response.statusCode).to.equal(200);
- const highQueueItems = parsedData
- .filter(item => item.machine_type.includes(MACHINE_TYPE_FILTER) && item.avg_queue_s > 14400)
- .map(item => ({ machine_type: item.machine_type, avg_queue_s: item.avg_queue_s }));
+ const MACHINE_TYPE_FILTER = '.rocm.';
+ const jsonData = dd.response.body;
+ const parsedData = JSON.parse(jsonData);
- if (highQueueItems.length > 0) {
- const machineDetails = highQueueItems
- .map(item => `${item.machine_type} (${item.avg_queue_s}s)`)
- .join(', ');
- const message = `High queue detected for machine types containing ${MACHINE_TYPE_FILTER}: ${machineDetails}`;
- console.error(message);
- }
+ const highQueueItems = parsedData
+ .filter(item => item.machine_type.includes(MACHINE_TYPE_FILTER) && item.avg_queue_s > 14400)
+ .map(item => ({ machine_type: item.machine_type, avg_queue_s: item.avg_queue_s }));
- dd.expect(highQueueItems.length > 0).to.be.false;
+ if (highQueueItems.length > 0) {
+ const machineDetails = highQueueItems
+ .map(item => `${item.machine_type} (${item.avg_queue_s}s)`)
+ .join(', ');
+ const message = `High queue detected for machine types containing ${MACHINE_TYPE_FILTER}: ${machineDetails}`;
+ console.error(message);
}
+
+ dd.expect(highQueueItems.length > 0).to.be.false;
EOT
# (1 unchanged attribute hidden)
}
# (2 unchanged blocks hidden)
}
# datadog_synthetics_test.pytorch-gha-runners-queue-check-ibm will be updated in-place
!~ resource "datadog_synthetics_test" "pytorch-gha-runners-queue-check-ibm" {
id = "sc6-zip-2n9"
name = "GHA Runner Queue Check - IBM Runners"
tags = [
"env:project",
"project:pytorch",
"service:gha-runners",
]
# (10 unchanged attributes hidden)
!~ assertion {
!~ code = <<-EOT
- if (dd.response.statusCode !== 200) {
- // We do not want to fail due to hud.pytorch.org API failure.
- console.log('Status code is not 200, stopping execution');
- dd.expect(true).to.equal(true);
- }
- else {
- const MACHINE_TYPE_FILTER = '.s390x';
- const jsonData = dd.response.body;
- const parsedData = JSON.parse(jsonData);
+ dd.expect(dd.response.statusCode).to.equal(200);
- const highQueueItems = parsedData
- .filter(item => item.machine_type.includes(MACHINE_TYPE_FILTER) && item.avg_queue_s > 10800)
- .map(item => ({ machine_type: item.machine_type, avg_queue_s: item.avg_queue_s }));
+ const MACHINE_TYPE_FILTER = '.s390x';
+ const jsonData = dd.response.body;
+ const parsedData = JSON.parse(jsonData);
- if (highQueueItems.length > 0) {
- const machineDetails = highQueueItems
- .map(item => `${item.machine_type} (${item.avg_queue_s}s)`)
- .join(', ');
- const message = `High queue detected for machine types containing ${MACHINE_TYPE_FILTER}: ${machineDetails}`;
- console.error(message);
- }
+ const highQueueItems = parsedData
+ .filter(item => item.machine_type.includes(MACHINE_TYPE_FILTER) && item.avg_queue_s > 10800)
+ .map(item => ({ machine_type: item.machine_type, avg_queue_s: item.avg_queue_s }));
- dd.expect(highQueueItems.length > 0).to.be.false;
+ if (highQueueItems.length > 0) {
+ const machineDetails = highQueueItems
+ .map(item => `${item.machine_type} (${item.avg_queue_s}s)`)
+ .join(', ');
+ const message = `High queue detected for machine types containing ${MACHINE_TYPE_FILTER}: ${machineDetails}`;
+ console.error(message);
}
+
+ dd.expect(highQueueItems.length > 0).to.be.false;
EOT
# (1 unchanged attribute hidden)
}
# (2 unchanged blocks hidden)
}
# datadog_synthetics_test.pytorch-gha-runners-queue-check-intel will be updated in-place
!~ resource "datadog_synthetics_test" "pytorch-gha-runners-queue-check-intel" {
id = "67g-icy-6mh"
name = "GHA Runner Queue Check - Intel Runners"
tags = [
"env:project",
"project:pytorch",
"service:gha-runners",
]
# (10 unchanged attributes hidden)
!~ assertion {
!~ code = <<-EOT
- if (dd.response.statusCode !== 200) {
- // We do not want to fail due to hud.pytorch.org API failure.
- console.log('Status code is not 200, stopping execution');
- dd.expect(true).to.equal(true);
- }
- else {
- const MACHINE_TYPE_FILTER = '.idc.';
- const jsonData = dd.response.body;
- const parsedData = JSON.parse(jsonData);
+ dd.expect(dd.response.statusCode).to.equal(200);
- const highQueueItems = parsedData
- .filter(item => item.machine_type.includes(MACHINE_TYPE_FILTER) && item.avg_queue_s > 10800)
- .map(item => ({ machine_type: item.machine_type, avg_queue_s: item.avg_queue_s }));
+ const MACHINE_TYPE_FILTER = '.idc.';
+ const jsonData = dd.response.body;
+ const parsedData = JSON.parse(jsonData);
- if (highQueueItems.length > 0) {
- const machineDetails = highQueueItems
- .map(item => `${item.machine_type} (${item.avg_queue_s}s)`)
- .join(', ');
- const message = `High queue detected for machine types containing ${MACHINE_TYPE_FILTER}: ${machineDetails}`;
- console.error(message);
- }
+ const highQueueItems = parsedData
+ .filter(item => item.machine_type.includes(MACHINE_TYPE_FILTER) && item.avg_queue_s > 10800)
+ .map(item => ({ machine_type: item.machine_type, avg_queue_s: item.avg_queue_s }));
- dd.expect(highQueueItems.length > 0).to.be.false;
+ if (highQueueItems.length > 0) {
+ const machineDetails = highQueueItems
+ .map(item => `${item.machine_type} (${item.avg_queue_s}s)`)
+ .join(', ');
+ const message = `High queue detected for machine types containing ${MACHINE_TYPE_FILTER}: ${machineDetails}`;
+ console.error(message);
}
+
+ dd.expect(highQueueItems.length > 0).to.be.false;
EOT
# (1 unchanged attribute hidden)
}
# (2 unchanged blocks hidden)
}
# datadog_synthetics_test.pytorch-gha-runners-queue-check-lf will be updated in-place
!~ resource "datadog_synthetics_test" "pytorch-gha-runners-queue-check-lf" {
id = "p69-6vj-54b"
name = "GHA Runner Queue Check - Linux Foundation Runners"
tags = [
"env:project",
"project:pytorch",
"service:gha-runners",
]
# (10 unchanged attributes hidden)
!~ assertion {
!~ code = <<-EOT
- if (dd.response.statusCode !== 200) {
- // We do not want to fail due to hud.pytorch.org API failure.
- console.log('Status code is not 200, stopping execution');
- dd.expect(true).to.equal(true);
- }
- else {
- const MACHINE_TYPE_FILTER = 'lf.';
- const jsonData = dd.response.body;
- const parsedData = JSON.parse(jsonData);
+ dd.expect(dd.response.statusCode).to.equal(200);
- const highQueueItems = parsedData
- .filter(item => item.machine_type.startsWith(MACHINE_TYPE_FILTER) && item.avg_queue_s > 10800)
- .map(item => ({ machine_type: item.machine_type, avg_queue_s: item.avg_queue_s }));
+ const MACHINE_TYPE_FILTER = 'lf.';
+ const jsonData = dd.response.body;
+ const parsedData = JSON.parse(jsonData);
- if (highQueueItems.length > 0) {
- const machineDetails = highQueueItems
- .map(item => `${item.machine_type} (${item.avg_queue_s}s)`)
- .join(', ');
- const message = `High queue detected for machine types containing ${MACHINE_TYPE_FILTER}: ${machineDetails}`;
- console.error(message);
- }
+ const highQueueItems = parsedData
+ .filter(item => item.machine_type.startsWith(MACHINE_TYPE_FILTER) && item.avg_queue_s > 10800)
+ .map(item => ({ machine_type: item.machine_type, avg_queue_s: item.avg_queue_s }));
- dd.expect(highQueueItems.length > 0).to.be.false;
+ if (highQueueItems.length > 0) {
+ const machineDetails = highQueueItems
+ .map(item => `${item.machine_type} (${item.avg_queue_s}s)`)
+ .join(', ');
+ const message = `High queue detected for machine types containing ${MACHINE_TYPE_FILTER}: ${machineDetails}`;
+ console.error(message);
}
+
+ dd.expect(highQueueItems.length > 0).to.be.false;
EOT
# (1 unchanged attribute hidden)
}
# (2 unchanged blocks hidden)
}
# datadog_synthetics_test.pytorch-gha-runners-queue-check-meta will be updated in-place
!~ resource "datadog_synthetics_test" "pytorch-gha-runners-queue-check-meta" {
id = "nnz-icu-8qk"
name = "GHA Runner Queue Check - Meta Runners"
tags = [
"env:project",
"project:pytorch",
"service:gha-runners",
]
# (10 unchanged attributes hidden)
!~ assertion {
!~ code = <<-EOT
- if (dd.response.statusCode !== 200) {
- // We do not want to fail due to hud.pytorch.org API failure.
- console.log('Status code is not 200, stopping execution');
- dd.expect(true).to.equal(true);
+ dd.expect(dd.response.statusCode).to.equal(200);
+ const EXCLUDED_MACHINE_PATTERNS = ['.dgx.', '.idc.', '.rocm.', '.s390x', '^lf\\.', '^linux.aws.h100'];
+ const jsonData = dd.response.body;
+ const parsedData = JSON.parse(jsonData);
+ const highQueueItems = parsedData
+ .filter(item => {
+ const machineType = item.machine_type;
+ return !EXCLUDED_MACHINE_PATTERNS.some(pattern =>
+ pattern.startsWith('^') ?
+ new RegExp(pattern).test(machineType) :
+ machineType.includes(pattern)
+ ) && item.avg_queue_s > 10800;
+ })
+ .map(item => ({ machine_type: item.machine_type, avg_queue_s: item.avg_queue_s }));
+ if (highQueueItems.length > 0) {
+ const machineDetails = highQueueItems
+ .map(item => `${item.machine_type} (${item.avg_queue_s}s)`)
+ .join(', ');
+ const message = `High queue detected for machine types: ${machineDetails}`;
+ console.error(message);
}
- else {
- const EXCLUDED_MACHINE_PATTERNS = ['.dgx.', '.idc.', '.rocm.', '.s390x', '^lf\\.', '^linux.aws.h100'];
- const jsonData = dd.response.body;
- const parsedData = JSON.parse(jsonData);
- const highQueueItems = parsedData
- .filter(item => {
- const machineType = item.machine_type;
- return !EXCLUDED_MACHINE_PATTERNS.some(pattern =>
- pattern.startsWith('^') ?
- new RegExp(pattern).test(machineType) :
- machineType.includes(pattern)
- ) && item.avg_queue_s > 10800;
- })
- .map(item => ({ machine_type: item.machine_type, avg_queue_s: item.avg_queue_s }));
- if (highQueueItems.length > 0) {
- const machineDetails = highQueueItems
- .map(item => `${item.machine_type} (${item.avg_queue_s}s)`)
- .join(', ');
- const message = `High queue detected for machine types: ${machineDetails}`;
- console.error(message);
- }
- dd.expect(highQueueItems.length > 0).to.be.false;
- }
+ dd.expect(highQueueItems.length > 0).to.be.false;
EOT
# (1 unchanged attribute hidden)
}
# (2 unchanged blocks hidden)
}
# datadog_synthetics_test.pytorch-gha-runners-queue-check-meta-h100 will be updated in-place
!~ resource "datadog_synthetics_test" "pytorch-gha-runners-queue-check-meta-h100" {
id = "hpi-psi-z8i"
name = "GHA Runner Queue Check - Meta Runners - AWS H100"
tags = [
"env:project",
"project:pytorch",
"service:gha-runners",
]
# (10 unchanged attributes hidden)
!~ assertion {
!~ code = <<-EOT
- if (dd.response.statusCode !== 200) {
- // We do not want to fail due to hud.pytorch.org API failure.
- console.log('Status code is not 200, stopping execution');
- dd.expect(true).to.equal(true);
- }
- else {
- const MACHINE_TYPE_FILTER = 'linux.aws.h100';
- const jsonData = dd.response.body;
- const parsedData = JSON.parse(jsonData);
+ dd.expect(dd.response.statusCode).to.equal(200);
- const highQueueItems = parsedData
- .filter(item => item.machine_type === MACHINE_TYPE_FILTER && item.avg_queue_s > 21600)
- .map(item => ({ machine_type: item.machine_type, avg_queue_s: item.avg_queue_s }));
+ const MACHINE_TYPE_FILTER = 'linux.aws.h100';
+ const jsonData = dd.response.body;
+ const parsedData = JSON.parse(jsonData);
- if (highQueueItems.length > 0) {
- const machineDetails = highQueueItems
- .map(item => `${item.machine_type} (${item.avg_queue_s}s)`)
- .join(', ');
- const message = `High queue detected for machine type ${MACHINE_TYPE_FILTER}: ${machineDetails}`;
- console.error(message);
- }
+ const highQueueItems = parsedData
+ .filter(item => item.machine_type === MACHINE_TYPE_FILTER && item.avg_queue_s > 21600)
+ .map(item => ({ machine_type: item.machine_type, avg_queue_s: item.avg_queue_s }));
- dd.expect(highQueueItems.length > 0).to.be.false;
+ if (highQueueItems.length > 0) {
+ const machineDetails = highQueueItems
+ .map(item => `${item.machine_type} (${item.avg_queue_s}s)`)
+ .join(', ');
+ const message = `High queue detected for machine type ${MACHINE_TYPE_FILTER}: ${machineDetails}`;
+ console.error(message);
}
+
+ dd.expect(highQueueItems.length > 0).to.be.false;
EOT
# (1 unchanged attribute hidden)
}
# (2 unchanged blocks hidden)
}
# datadog_synthetics_test.pytorch-gha-runners-queue-check-nvidia will be updated in-place
!~ resource "datadog_synthetics_test" "pytorch-gha-runners-queue-check-nvidia" {
id = "sxd-d72-36u"
name = "GHA Runner Queue Check - Nvidia Runners"
tags = [
"env:project",
"project:pytorch",
"service:gha-runners",
]
# (10 unchanged attributes hidden)
!~ assertion {
!~ code = <<-EOT
- if (dd.response.statusCode !== 200) {
- // We do not want to fail due to hud.pytorch.org API failure.
- console.log('Status code is not 200, stopping execution');
- dd.expect(true).to.equal(true);
- }
- else {
- const MACHINE_TYPE_FILTER = '.dgx.';
- const jsonData = dd.response.body;
- const parsedData = JSON.parse(jsonData);
+ dd.expect(dd.response.statusCode).to.equal(200);
- const highQueueItems = parsedData
- .filter(item => item.machine_type.includes(MACHINE_TYPE_FILTER) && item.avg_queue_s > 10800)
- .map(item => ({ machine_type: item.machine_type, avg_queue_s: item.avg_queue_s }));
+ const MACHINE_TYPE_FILTER = '.dgx.';
+ const jsonData = dd.response.body;
+ const parsedData = JSON.parse(jsonData);
- if (highQueueItems.length > 0) {
- const machineDetails = highQueueItems
- .map(item => `${item.machine_type} (${item.avg_queue_s}s)`)
- .join(', ');
- const message = `High queue detected for machine types containing ${MACHINE_TYPE_FILTER}: ${machineDetails}`;
- console.error(message);
- }
+ const highQueueItems = parsedData
+ .filter(item => item.machine_type.includes(MACHINE_TYPE_FILTER) && item.avg_queue_s > 10800)
+ .map(item => ({ machine_type: item.machine_type, avg_queue_s: item.avg_queue_s }));
- dd.expect(highQueueItems.length > 0).to.be.false;
+ if (highQueueItems.length > 0) {
+ const machineDetails = highQueueItems
+ .map(item => `${item.machine_type} (${item.avg_queue_s}s)`)
+ .join(', ');
+ const message = `High queue detected for machine types containing ${MACHINE_TYPE_FILTER}: ${machineDetails}`;
+ console.error(message);
}
+
+ dd.expect(highQueueItems.length > 0).to.be.false;
EOT
# (1 unchanged attribute hidden)
}
# (2 unchanged blocks hidden)
}
Plan: 0 to add, 7 to change, 0 to destroy.✅ Plan applied in Tofu Apply #38 |
jordanconway
approved these changes
Sep 4, 2025
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
2 participants
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
This change causes more flapping than it was intended to prevent. I think the change 7392723 does a better job of handling the flapping so lets revert this one.
This reverts commit a928f56.