Chore bump 3h queue threshholds to 4h

[jordanconway]

No description provided.

[github-actions]

OpenTofu plan for prod

Plan: 0 to add, 4 to change, 0 to destroy.

OpenTofu used the selected providers to generate the following execution
plan. Resource actions are indicated with the following symbols:
!~  update in-place

OpenTofu will perform the following actions:

  # datadog_synthetics_test.pytorch-gha-runners-queue-check-intel will be updated in-place
!~  resource "datadog_synthetics_test" "pytorch-gha-runners-queue-check-intel" {
        id               = "67g-icy-6mh"
        name             = "GHA Runner Queue Check - Intel Runners"
        tags             = [
            "env:project",
            "project:pytorch",
            "service:gha-runners",
        ]
#        (10 unchanged attributes hidden)

!~      assertion {
!~          code = <<-EOT
                dd.expect(dd.response.statusCode).to.equal(200);
                
                const MACHINE_TYPE_FILTER = '.idc.';
                const jsonData = dd.response.body;
                const parsedData = JSON.parse(jsonData);
                
                const highQueueItems = parsedData
              -   .filter(item => item.machine_type.includes(MACHINE_TYPE_FILTER) && item.avg_queue_s > 10800)
              +   .filter(item => item.machine_type.includes(MACHINE_TYPE_FILTER) && item.avg_queue_s > 14400)
                  .map(item => ({ machine_type: item.machine_type, avg_queue_s: item.avg_queue_s }));
                
                if (highQueueItems.length > 0) {
                  const machineDetails = highQueueItems
                    .map(item => `${item.machine_type} (${item.avg_queue_s}s)`)
                    .join(', ');
                  const message = `High queue detected for machine types containing ${MACHINE_TYPE_FILTER}: ${machineDetails}`;
                  console.error(message);
                }
                
                dd.expect(highQueueItems.length > 0).to.be.false;
            EOT
#            (1 unchanged attribute hidden)
        }

#        (2 unchanged blocks hidden)
    }

  # datadog_synthetics_test.pytorch-gha-runners-queue-check-lf will be updated in-place
!~  resource "datadog_synthetics_test" "pytorch-gha-runners-queue-check-lf" {
        id               = "p69-6vj-54b"
        name             = "GHA Runner Queue Check - Linux Foundation Runners"
        tags             = [
            "env:project",
            "project:pytorch",
            "service:gha-runners",
        ]
#        (10 unchanged attributes hidden)

!~      assertion {
!~          code = <<-EOT
                dd.expect(dd.response.statusCode).to.equal(200);
                
                const MACHINE_TYPE_FILTER = 'lf.';
                const jsonData = dd.response.body;
                const parsedData = JSON.parse(jsonData);
                
                const highQueueItems = parsedData
              -   .filter(item => item.machine_type.startsWith(MACHINE_TYPE_FILTER) && item.avg_queue_s > 10800)
              +   .filter(item => item.machine_type.startsWith(MACHINE_TYPE_FILTER) && item.avg_queue_s > 14400)
                  .map(item => ({ machine_type: item.machine_type, avg_queue_s: item.avg_queue_s }));
                
                if (highQueueItems.length > 0) {
                  const machineDetails = highQueueItems
                    .map(item => `${item.machine_type} (${item.avg_queue_s}s)`)
                    .join(', ');
                  const message = `High queue detected for machine types containing ${MACHINE_TYPE_FILTER}: ${machineDetails}`;
                  console.error(message);
                }
                
                dd.expect(highQueueItems.length > 0).to.be.false;
            EOT
#            (1 unchanged attribute hidden)
        }

#        (2 unchanged blocks hidden)
    }

  # datadog_synthetics_test.pytorch-gha-runners-queue-check-meta will be updated in-place
!~  resource "datadog_synthetics_test" "pytorch-gha-runners-queue-check-meta" {
        id               = "nnz-icu-8qk"
        name             = "GHA Runner Queue Check - Meta Runners"
        tags             = [
            "env:project",
            "project:pytorch",
            "service:gha-runners",
        ]
#        (10 unchanged attributes hidden)

!~      assertion {
!~          code = <<-EOT
                dd.expect(dd.response.statusCode).to.equal(200);
                const EXCLUDED_MACHINE_PATTERNS = ['.dgx.', '.idc.', '.rocm.', '.s390x', '^lf\\.', '^linux.aws.h100'];
                const jsonData = dd.response.body;
                const parsedData = JSON.parse(jsonData);
                const highQueueItems = parsedData
                  .filter(item => {
                    const machineType = item.machine_type;
                    return !EXCLUDED_MACHINE_PATTERNS.some(pattern =>
                      pattern.startsWith('^') ?
                        new RegExp(pattern).test(machineType) :
                        machineType.includes(pattern)
              -     ) && item.avg_queue_s > 10800;
              +     ) && item.avg_queue_s > 14400;
                  })
                  .map(item => ({ machine_type: item.machine_type, avg_queue_s: item.avg_queue_s }));
                if (highQueueItems.length > 0) {
                  const machineDetails = highQueueItems
                    .map(item => `${item.machine_type} (${item.avg_queue_s}s)`)
                    .join(', ');
                  const message = `High queue detected for machine types: ${machineDetails}`;
                  console.error(message);
                }
                dd.expect(highQueueItems.length > 0).to.be.false;
            EOT
#            (1 unchanged attribute hidden)
        }

#        (2 unchanged blocks hidden)
    }

  # datadog_synthetics_test.pytorch-gha-runners-queue-check-nvidia will be updated in-place
!~  resource "datadog_synthetics_test" "pytorch-gha-runners-queue-check-nvidia" {
        id               = "sxd-d72-36u"
        name             = "GHA Runner Queue Check - Nvidia Runners"
        tags             = [
            "env:project",
            "project:pytorch",
            "service:gha-runners",
        ]
#        (10 unchanged attributes hidden)

!~      assertion {
!~          code = <<-EOT
                dd.expect(dd.response.statusCode).to.equal(200);
                
                const MACHINE_TYPE_FILTER = '.dgx.';
                const jsonData = dd.response.body;
                const parsedData = JSON.parse(jsonData);
                
                const highQueueItems = parsedData
              -   .filter(item => item.machine_type.includes(MACHINE_TYPE_FILTER) && item.avg_queue_s > 10800)
              +   .filter(item => item.machine_type.includes(MACHINE_TYPE_FILTER) && item.avg_queue_s > 14400)
                  .map(item => ({ machine_type: item.machine_type, avg_queue_s: item.avg_queue_s }));
                
                if (highQueueItems.length > 0) {
                  const machineDetails = highQueueItems
                    .map(item => `${item.machine_type} (${item.avg_queue_s}s)`)
                    .join(', ');
                  const message = `High queue detected for machine types containing ${MACHINE_TYPE_FILTER}: ${machineDetails}`;
                  console.error(message);
                }
                
                dd.expect(highQueueItems.length > 0).to.be.false;
            EOT
#            (1 unchanged attribute hidden)
        }

#        (2 unchanged blocks hidden)
    }

Plan: 0 to add, 4 to change, 0 to destroy.

✅ Plan applied in Tofu Apply #39