merge main into amd-staging

Author: ronlieb

.ci/metrics/metrics.py

@@ -1,12 +1,9 @@
 import collections
 import datetime
-import dateutil
 import github
-import json
 import logging
 import os
 import requests
-import sys
 import time
 
 from dataclasses import dataclass
@@ -55,18 +52,6 @@
 # by trial and error).
 GRAFANA_METRIC_MAX_AGE_MN = 120
 
-# Lists the BuildKite jobs we want to track. Maps the BuildKite job name to
-# the metric name in Grafana. This is important not to lose metrics history
-# if the workflow name changes.
-BUILDKITE_WORKFLOW_TO_TRACK = {
-    ":linux: Linux x64": "buildkite_linux",
-    ":windows: Windows x64": "buildkite_windows",
-}
-
-# Number of builds to fetch per page. Since we scrape regularly, this can
-# remain small.
-BUILDKITE_GRAPHQL_BUILDS_PER_PAGE = 50
-
 
 @dataclass
 class JobMetrics:
@@ -86,181 +71,6 @@ class GaugeMetric:
     time_ns: int
 
 
-def buildkite_fetch_page_build_list(
-    buildkite_token: str, after_cursor: str = None
-) -> list[dict[str, str]]:
-    """Fetches a page of the build list using the GraphQL BuildKite API.
-
-    Returns the BUILDKITE_GRAPHQL_BUILDS_PER_PAGE last running/queued builds,
-    or the BUILDKITE_GRAPHQL_BUILDS_PER_PAGE running/queued builds
-    older than the one pointer by |after_cursor| if provided.
-    The |after_cursor| value is taken from the previous page returned by the
-    API.
-
-    Args:
-      buildkite_token: the secret token to authenticate GraphQL requests.
-      after_cursor: cursor after which to start the page fetch.
-
-    Returns:
-      The most recent builds after cursor (if set) with the following format:
-      [
-        {
-            "cursor": <value>,
-            "number": <build-number>,
-        }
-      ]
-    """
-
-    BUILDKITE_GRAPHQL_QUERY = """
-    query OrganizationShowQuery {{
-      organization(slug: "llvm-project") {{
-        pipelines(search: "Github pull requests", first: 1) {{
-          edges {{
-            node {{
-              builds (state: [CANCELING, CREATING, FAILING, RUNNING], first: {PAGE_SIZE}, after: {AFTER}) {{
-                edges {{
-                  cursor
-                  node {{
-                    number
-                  }}
-                }}
-              }}
-            }}
-          }}
-        }}
-      }}
-    }}
-    """
-    query = BUILDKITE_GRAPHQL_QUERY.format(
-        PAGE_SIZE=BUILDKITE_GRAPHQL_BUILDS_PER_PAGE,
-        AFTER="null" if after_cursor is None else '"{}"'.format(after_cursor),
-    )
-    query = json.dumps({"query": query})
-    url = "https://graphql.buildkite.com/v1"
-    headers = {
-        "Authorization": "Bearer " + buildkite_token,
-        "Content-Type": "application/json",
-    }
-    data = requests.post(url, data=query, headers=headers).json()
-    # De-nest the build list.
-    if "errors" in data:
-        logging.info("Failed to fetch BuildKite jobs: {}".format(data["errors"]))
-        return []
-    builds = data["data"]["organization"]["pipelines"]["edges"][0]["node"]["builds"][
-        "edges"
-    ]
-    # Fold cursor info into the node dictionnary.
-    return [{**x["node"], "cursor": x["cursor"]} for x in builds]
-
-
-def buildkite_get_build_info(build_number: str) -> dict:
-    """Returns all the info associated with the provided build number.
-
-    Note: for unknown reasons, graphql returns no jobs for a given build,
-    while this endpoint does, hence why this uses this API instead of graphql.
-
-    Args:
-      build_number: which build number to fetch info for.
-
-    Returns:
-      The info for the target build, a JSON dictionnary.
-    """
-
-    URL = "https://buildkite.com/llvm-project/github-pull-requests/builds/{}.json"
-    return requests.get(URL.format(build_number)).json()
-
-
-def buildkite_get_incomplete_tasks(buildkite_token: str) -> list:
-    """Returns all the running/pending BuildKite builds.
-
-    Args:
-     buildkite_token: the secret token to authenticate GraphQL requests.
-     last_cursor: the cursor to stop at if set. If None, a full page is fetched.
-    """
-    output = []
-    cursor = None
-    while True:
-        page = buildkite_fetch_page_build_list(buildkite_token, cursor)
-        if len(page) == 0:
-            break
-        cursor = page[-1]["cursor"]
-        output += page
-    return output
-
-
-def buildkite_get_metrics(
-    buildkite_token: str, previously_incomplete: set[int]
-) -> (list[JobMetrics], set[int]):
-    """Returns a tuple with:
-
-    - the metrics recorded for newly completed workflow jobs.
-    - the set of workflow still running now.
-
-    Args:
-      buildkite_token: the secret token to authenticate GraphQL requests.
-        previously_incomplete: the set of running workflows the last time this
-        function was called.
-    """
-
-    running_builds = buildkite_get_incomplete_tasks(buildkite_token)
-    incomplete_now = set([x["number"] for x in running_builds])
-    output = []
-
-    for build_id in previously_incomplete:
-        if build_id in incomplete_now:
-            continue
-
-        info = buildkite_get_build_info(build_id)
-        metric_timestamp = dateutil.parser.isoparse(info["finished_at"])
-        for job in info["jobs"]:
-            # This workflow is not interesting to us.
-            if job["name"] not in BUILDKITE_WORKFLOW_TO_TRACK:
-                continue
-
-            # Don't count canceled jobs.
-            if job["canceled_at"]:
-                continue
-
-            created_at = dateutil.parser.isoparse(job["created_at"])
-            scheduled_at = dateutil.parser.isoparse(job["scheduled_at"])
-            started_at = dateutil.parser.isoparse(job["started_at"])
-            finished_at = dateutil.parser.isoparse(job["finished_at"])
-
-            job_name = BUILDKITE_WORKFLOW_TO_TRACK[job["name"]]
-            queue_time = (started_at - scheduled_at).seconds
-            run_time = (finished_at - started_at).seconds
-            status = bool(job["passed"])
-
-            # Grafana will refuse to ingest metrics older than ~2 hours, so we
-            # should avoid sending historical data.
-            metric_age_mn = (
-                datetime.datetime.now(datetime.timezone.utc) - metric_timestamp
-            ).total_seconds() / 60
-            if metric_age_mn > GRAFANA_METRIC_MAX_AGE_MN:
-                logging.warning(
-                    f"Job {job['name']} from workflow {build_id} dropped due"
-                    + f" to staleness: {metric_age_mn}mn old."
-                )
-                continue
-
-            metric_timestamp_ns = int(metric_timestamp.timestamp()) * 10**9
-            workflow_id = build_id
-            workflow_name = "Github pull requests"
-            output.append(
-                JobMetrics(
-                    job_name,
-                    queue_time,
-                    run_time,
-                    status,
-                    metric_timestamp_ns,
-                    workflow_id,
-                    workflow_name,
-                )
-            )
-
-    return output, incomplete_now
-
-
 def github_get_metrics(
     github_repo: github.Repository, last_workflows_seen_as_completed: set[int]
 ) -> tuple[list[JobMetrics], int]:
@@ -478,17 +288,13 @@ def upload_metrics(workflow_metrics, metrics_userid, api_key):
 def main():
     # Authenticate with Github
     github_auth = Auth.Token(os.environ["GITHUB_TOKEN"])
-    buildkite_token = os.environ["BUILDKITE_TOKEN"]
     grafana_api_key = os.environ["GRAFANA_API_KEY"]
     grafana_metrics_userid = os.environ["GRAFANA_METRICS_USERID"]
 
     # The last workflow this script processed.
     # Because the Github queries are broken, we'll simply log a 'processed'
     # bit for the last COUNT_TO_PROCESS workflows.
     gh_last_workflows_seen_as_completed = set()
-    # Stores the list of pending/running builds in BuildKite we need to check
-    # at the next iteration.
-    bk_incomplete = set()
 
     # Enter the main loop. Every five minutes we wake up and dump metrics for
     # the relevant jobs.
@@ -500,13 +306,8 @@ def main():
             github_repo, gh_last_workflows_seen_as_completed
         )
 
-        bk_metrics, bk_incomplete = buildkite_get_metrics(
-            buildkite_token, bk_incomplete
-        )
-
-        metrics = gh_metrics + bk_metrics
-        upload_metrics(metrics, grafana_metrics_userid, grafana_api_key)
-        logging.info(f"Uploaded {len(metrics)} metrics")
+        upload_metrics(gh_metrics, grafana_metrics_userid, grafana_api_key)
+        logging.info(f"Uploaded {len(gh_metrics)} metrics")
 
         time.sleep(SCRAPE_INTERVAL_SECONDS)
 

clang/include/clang/CIR/CIRGenerator.h

@@ -78,6 +78,7 @@ class CIRGenerator : public clang::ASTConsumer {
   bool HandleTopLevelDecl(clang::DeclGroupRef group) override;
   void HandleTranslationUnit(clang::ASTContext &astContext) override;
   void HandleInlineFunctionDefinition(clang::FunctionDecl *d) override;
+  void HandleTagDeclDefinition(clang::TagDecl *d) override;
   void CompleteTentativeDefinition(clang::VarDecl *d) override;
 
   mlir::ModuleOp getModule() const;

clang/include/clang/CIR/MissingFeatures.h

@@ -199,7 +199,6 @@ struct MissingFeatures {
   static bool msabi() { return false; }
   static bool typeChecks() { return false; }
   static bool lambdaFieldToName() { return false; }
-  static bool updateCompletedType() { return false; }
   static bool moduleNameHash() { return false; }
   static bool constantFoldSwitchStatement() { return false; }
   static bool cudaSupport() { return false; }

clang/lib/Driver/ToolChains/CommonArgs.h renamed to clang/include/clang/Driver/CommonArgs.h

@@ -291,6 +291,12 @@ void handleVectorizeLoopsArgs(const llvm::opt::ArgList &Args,
 /// Enable -fslp-vectorize based on the optimization level selected.
 void handleVectorizeSLPArgs(const llvm::opt::ArgList &Args,
                             llvm::opt::ArgStringList &CmdArgs);
+
+// Parse -mprefer-vector-width=. Return the Value string if well-formed.
+// Otherwise, return an empty string and issue a diagnosic message if needed.
+StringRef ParseMPreferVectorWidthOption(clang::DiagnosticsEngine &Diags,
+                                        const llvm::opt::ArgList &Args);
+
 } // end namespace tools
 } // end namespace driver
 } // end namespace clang

clang/include/clang/Driver/Options.td

@@ -6044,7 +6044,7 @@ def resource_dir : Separate<["-"], "resource-dir">,
   HelpText<"The directory which holds the compiler resource files">,
   MarshallingInfoString<HeaderSearchOpts<"ResourceDir">>;
 def resource_dir_EQ : Joined<["-"], "resource-dir=">, Flags<[NoXarchOption]>,
-  Visibility<[ClangOption, CC1Option, CLOption, DXCOption, FlangOption]>,
+  Visibility<[ClangOption, CLOption, DXCOption, FlangOption]>,
   Alias<resource_dir>;
 def rpath : Separate<["-"], "rpath">, Flags<[LinkerInput]>, Group<Link_Group>,
   Visibility<[ClangOption, FlangOption]>;

clang/lib/CIR/CodeGen/CIRGenModule.cpp

@@ -838,6 +838,11 @@ void CIRGenModule::maybeSetTrivialComdat(const Decl &d, mlir::Operation *op) {
   assert(!cir::MissingFeatures::opFuncSetComdat());
 }
 
+void CIRGenModule::updateCompletedType(const TagDecl *td) {
+  // Make sure that this type is translated.
+  genTypes.updateCompletedType(td);
+}
+
 // TODO(CIR): this could be a common method between LLVM codegen.
 static bool isVarDeclStrongDefinition(const ASTContext &astContext,
                                       CIRGenModule &cgm, const VarDecl *vd,
@@ -1145,6 +1150,7 @@ void CIRGenModule::emitTopLevelDecl(Decl *decl) {
 
   // No code generation needed.
   case Decl::UsingShadow:
+  case Decl::Empty:
     break;
 
   // C++ Decls

clang/lib/CIR/CodeGen/CIRGenModule.h

@@ -241,6 +241,9 @@ class CIRGenModule : public CIRGenTypeCache {
 
   void emitTentativeDefinition(const VarDecl *d);
 
+  // Make sure that this type is translated.
+  void updateCompletedType(const clang::TagDecl *td);
+
   bool supportsCOMDAT() const;
   void maybeSetTrivialComdat(const clang::Decl &d, mlir::Operation *op);
 

clang/lib/CIR/CodeGen/CIRGenOpenACCClause.cpp

@@ -305,9 +305,19 @@ class OpenACCClauseCIREmitter final
     {
       mlir::OpBuilder::InsertionGuard guardCase(builder);
       builder.setInsertionPointAfter(operation);
-      afterOp = builder.create<AfterOpTy>(opInfo.beginLoc, beforeOp.getResult(),
-                                          opInfo.varValue, structured, implicit,
-                                          opInfo.name, opInfo.bounds);
+
+      if constexpr (std::is_same_v<AfterOpTy, mlir::acc::DeleteOp> ||
+                    std::is_same_v<AfterOpTy, mlir::acc::DetachOp>) {
+        // Detach/Delete ops don't have the variable reference here, so they
+        // take 1 fewer argument to their build function.
+        afterOp = builder.create<AfterOpTy>(
+            opInfo.beginLoc, beforeOp.getResult(), structured, implicit,
+            opInfo.name, opInfo.bounds);
+      } else {
+        afterOp = builder.create<AfterOpTy>(
+            opInfo.beginLoc, beforeOp.getResult(), opInfo.varValue, structured,
+            implicit, opInfo.name, opInfo.bounds);
+      }
     }
 
     // Set the 'rest' of the info for both operations.
@@ -845,6 +855,54 @@ class OpenACCClauseCIREmitter final
       return clauseNotImplemented(clause);
     }
   }
+
+  void VisitNoCreateClause(const OpenACCNoCreateClause &clause) {
+    if constexpr (isOneOfTypes<OpTy, mlir::acc::ParallelOp, mlir::acc::SerialOp,
+                               mlir::acc::KernelsOp>) {
+      for (auto var : clause.getVarList())
+        addDataOperand<mlir::acc::NoCreateOp, mlir::acc::DeleteOp>(
+            var, mlir::acc::DataClause::acc_no_create, /*structured=*/true,
+            /*implicit=*/false);
+    } else if constexpr (isCombinedType<OpTy>) {
+      applyToComputeOp(clause);
+    } else {
+      // TODO: When we've implemented this for everything, switch this to an
+      // unreachable. data remains.
+      return clauseNotImplemented(clause);
+    }
+  }
+
+  void VisitPresentClause(const OpenACCPresentClause &clause) {
+    if constexpr (isOneOfTypes<OpTy, mlir::acc::ParallelOp, mlir::acc::SerialOp,
+                               mlir::acc::KernelsOp>) {
+      for (auto var : clause.getVarList())
+        addDataOperand<mlir::acc::PresentOp, mlir::acc::DeleteOp>(
+            var, mlir::acc::DataClause::acc_present, /*structured=*/true,
+            /*implicit=*/false);
+    } else if constexpr (isCombinedType<OpTy>) {
+      applyToComputeOp(clause);
+    } else {
+      // TODO: When we've implemented this for everything, switch this to an
+      // unreachable. data & declare remain.
+      return clauseNotImplemented(clause);
+    }
+  }
+
+  void VisitAttachClause(const OpenACCAttachClause &clause) {
+    if constexpr (isOneOfTypes<OpTy, mlir::acc::ParallelOp, mlir::acc::SerialOp,
+                               mlir::acc::KernelsOp>) {
+      for (auto var : clause.getVarList())
+        addDataOperand<mlir::acc::AttachOp, mlir::acc::DetachOp>(
+            var, mlir::acc::DataClause::acc_attach, /*structured=*/true,
+            /*implicit=*/false);
+    } else if constexpr (isCombinedType<OpTy>) {
+      applyToComputeOp(clause);
+    } else {
+      // TODO: When we've implemented this for everything, switch this to an
+      // unreachable. data, enter data remain.
+      return clauseNotImplemented(clause);
+    }
+  }
 };
 
 template <typename OpTy>