Support marking unstable jobs from other repos (#5122)

Fixes pytorch/executorch#1759

Before this, a job can only be seen as unstable when it has the unstable
keyword in its name. The trick of renaming the job can be done with
PyTorch test jobs by manipulating the test matrix. However, that doesn't
work for PyTorch build job or Nova workflow without moving the filter
script to a separate job to run before them. That approach doesn't
scale.

So, this PR implements a different approach to query the list of open
unstable issues from Rockset from HUD, then group all jobs with an open
unstable issues to the unstable group.

The PR is big because the following top level pages need to query the
list and pass it around to different React components. The main update
is in the function `isUnstableJob` which calls `hasOpenUnstableIssue`.
The important call paths are as follows:

* HUD main page → `getGroupingData` → `classifyGroup` →
`hasOpenUnstableIssue`
* HUD commit/PR pages → `CommitStatus` → `FilteredJobList` →
`FailedJobInfo` → `JobSummary` → `isUnstableJob`
* HUD commit/PR pages → `CommitStatus` → `WorkflowsContainer` ->
`WorkflowBox` -> `WorkflowJobSummary` -> `JobSummary` → `isUnstableJob`

There are some misc pages like HUD flaky test pages that also needs to
be updated, but they don't need to have the list of unstable issues, so
I will just pass an empty array there.

The second part of this change is on Dr.CI.

### Testing

`trunk / test-coreml-delegate / macos-job` from ET is marked as unstable
by pytorch/executorch#3264, and it seems to
show up correctly
https://torchci-git-support-unstable-job-other-repos-fbopensource.vercel.app/hud/pytorch/executorch/main.

An UX caveat is that all the invocations of the job will appear in the
unstable group because there is no `unstable` keyword to distinguish
them.

Author: huydhn

torchci/components/CommitStatus.tsx

@@ -1,6 +1,6 @@
 import FilteredJobList from "./FilteredJobList";
 import VersionControlLinks from "./VersionControlLinks";
-import { CommitData, JobData } from "lib/types";
+import { CommitData, JobData, IssueData } from "lib/types";
 import WorkflowBox from "./WorkflowBox";
 import styles from "components/commit.module.css";
 import _ from "lodash";
@@ -15,7 +15,13 @@ import useScrollTo from "lib/useScrollTo";
 import WorkflowDispatcher from "./WorkflowDispatcher";
 import { useSession } from "next-auth/react";
 
-function WorkflowsContainer({ jobs }: { jobs: JobData[] }) {
+function WorkflowsContainer({
+  jobs,
+  unstableIssues,
+}: {
+  jobs: JobData[];
+  unstableIssues: IssueData[];
+}) {
   useScrollTo();
 
   if (jobs.length === 0) {
@@ -44,6 +50,7 @@ function WorkflowsContainer({ jobs }: { jobs: JobData[] }) {
               key={workflowName}
               workflowName={workflowName}
               jobs={jobs}
+              unstableIssues={unstableIssues}
             />
           );
         })}
@@ -58,12 +65,14 @@ export default function CommitStatus({
   commit,
   jobs,
   isCommitPage,
+  unstableIssues,
 }: {
   repoOwner: string;
   repoName: string;
   commit: CommitData;
   jobs: JobData[];
   isCommitPage: boolean;
+  unstableIssues: IssueData[];
 }) {
   const session = useSession();
   const isAuthenticated = session.status === "authenticated";
@@ -90,26 +99,30 @@ export default function CommitStatus({
         pred={(job) =>
           isFailedJob(job) &&
           !isRerunDisabledTestsJob(job) &&
-          !isUnstableJob(job)
+          !isUnstableJob(job, unstableIssues)
         }
         showClassification
+        unstableIssues={unstableIssues}
       />
       <FilteredJobList
         filterName="Failed unstable jobs"
         jobs={jobs}
-        pred={(job) => isFailedJob(job) && isUnstableJob(job)}
+        pred={(job) => isFailedJob(job) && isUnstableJob(job, unstableIssues)}
+        unstableIssues={unstableIssues}
       />
       <FilteredJobList
         filterName="Daily rerunning disabled jobs"
         jobs={jobs}
         pred={(job) => isFailedJob(job) && isRerunDisabledTestsJob(job)}
+        unstableIssues={unstableIssues}
       />
       <FilteredJobList
         filterName="Pending jobs"
         jobs={jobs}
         pred={(job) => job.conclusion === "pending"}
+        unstableIssues={unstableIssues}
       />
-      <WorkflowsContainer jobs={jobs} />
+      <WorkflowsContainer jobs={jobs} unstableIssues={unstableIssues} />
       {isAuthenticated && isCommitPage && (
         <WorkflowDispatcher
           repoOwner={repoOwner}

torchci/components/FilteredJobList.tsx

@@ -6,24 +6,26 @@ import JobAnnotationToggle from "./JobAnnotationToggle";
 import JobLinks from "./JobLinks";
 import JobSummary from "./JobSummary";
 import LogViewer from "./LogViewer";
-import { JobAnnotation } from "lib/types";
+import { JobAnnotation, IssueData } from "lib/types";
 import useScrollTo from "lib/useScrollTo";
 
 function FailedJobInfo({
   job,
   showClassification,
   annotation,
+  unstableIssues,
 }: {
   job: JobData;
   showClassification: boolean;
   annotation: JobAnnotation;
+  unstableIssues: IssueData[];
 }) {
   const router = useRouter();
   useScrollTo();
   const { repoOwner, repoName } = router.query;
   return (
     <li key={job.id} id={job.id}>
-      <JobSummary job={job} />
+      <JobSummary job={job} unstableIssues={unstableIssues} />
       <div>
         <JobLinks job={job} />
       </div>
@@ -44,11 +46,13 @@ export default function FilteredJobList({
   jobs,
   pred,
   showClassification = false,
+  unstableIssues,
 }: {
   filterName: string;
   jobs: JobData[];
   pred: (job: JobData) => boolean;
   showClassification?: boolean;
+  unstableIssues: IssueData[];
 }) {
   const router = useRouter();
   const { repoOwner, repoName } = router.query;
@@ -84,6 +88,7 @@ export default function FilteredJobList({
                 data[job?.id ?? ""]["annotation"]) ??
               JobAnnotation.NULL
             }
+            unstableIssues={unstableIssues}
           />
         ))}
       </ul>

torchci/components/GroupJobConclusion.tsx

@@ -1,5 +1,5 @@
 import { getGroupConclusionChar } from "lib/JobClassifierUtil";
-import { GroupData, JobData } from "lib/types";
+import { GroupData, JobData, IssueData } from "lib/types";
 import styles from "./JobConclusion.module.css";
 import hudStyles from "./hud.module.css";
 import TooltipTarget from "components/TooltipTarget";
@@ -39,11 +39,13 @@ export default function HudGroupedCell({
   groupData,
   isExpanded,
   isClassified,
+  unstableIssues,
 }: {
   sha: string;
   groupData: GroupData;
   isExpanded: boolean;
   isClassified: boolean;
+  unstableIssues: IssueData[];
 }) {
   const [pinnedId, setPinnedId] = useContext(PinnedTooltipContext);
   const style = pinnedId.name == groupData.groupName ? hudStyles.highlight : "";
@@ -55,7 +57,7 @@ export default function HudGroupedCell({
   const failedPreviousRunJobs = [];
   for (const job of groupData.jobs) {
     if (isFailedJob(job)) {
-      if (isRerunDisabledTestsJob(job) || isUnstableJob(job)) {
+      if (isRerunDisabledTestsJob(job) || isUnstableJob(job, unstableIssues)) {
         warningOnlyJobs.push(job);
       } else {
         erroredJobs.push(job);

torchci/components/JobLinks.tsx

@@ -7,6 +7,7 @@ import TestInsightsLink from "./TestInsights";
 import ReproductionCommand from "./ReproductionCommand";
 import { useSession } from "next-auth/react";
 import { isFailure } from "../lib/JobClassifierUtil";
+import { transformJobName } from "../lib/jobUtils";
 
 export default function JobLinks({
   job,
@@ -163,9 +164,11 @@ function DisableTest({ job, label }: { job: JobData; label: string }) {
 
   const issues: IssueData[] = data.issues;
   const matchingIssues = issues.filter((issue) => issue.title === issueTitle);
+  const repo = job.repo ?? "pytorch/pytorch";
 
   return (
     <DisableIssue
+      repo={repo}
       matchingIssues={matchingIssues}
       issueTitle={issueTitle}
       issueBody={issueBody}
@@ -174,21 +177,6 @@ function DisableTest({ job, label }: { job: JobData; label: string }) {
   );
 }
 
-const jobNameRe = /^(.*) \(([^,]*),.*\)/;
-function transformJobName(jobName?: string) {
-  if (jobName == undefined) {
-    return null;
-  }
-
-  // We want to have the job name in the following format WORKFLOW / JOB (CONFIG)
-  const jobNameMatch = jobName.match(jobNameRe);
-  if (jobNameMatch !== null) {
-    return `${jobNameMatch[1]} (${jobNameMatch[2]})`;
-  }
-
-  return jobName;
-}
-
 function formatUnstableJobBody() {
   return encodeURIComponent(
     "> Please provide a brief reason on why you need to mark this job as unstable."
@@ -229,9 +217,11 @@ function UnstableJob({ job, label }: { job: JobData; label: string }) {
   const matchingIssues = issues.filter((issue) =>
     issueTitle.includes(issue.title)
   );
+  const repo = job.repo ?? "pytorch/pytorch";
 
   return (
     <DisableIssue
+      repo={repo}
       matchingIssues={matchingIssues}
       issueTitle={issueTitle}
       issueBody={issueBody}
@@ -241,17 +231,19 @@ function UnstableJob({ job, label }: { job: JobData; label: string }) {
 }
 
 function DisableIssue({
+  repo,
   matchingIssues,
   issueTitle,
   issueBody,
   isDisabledTest,
 }: {
+  repo: string;
   matchingIssues: IssueData[];
   issueTitle: string;
   issueBody: string;
   isDisabledTest: boolean;
 }) {
-  let issueLink = `https://github.com/pytorch/pytorch/issues/new?title=${issueTitle}&body=${issueBody}`;
+  let issueLink = `https://github.com/${repo}/issues/new?title=${issueTitle}&body=${issueBody}`;
   let linkText = isDisabledTest
     ? "Disable test"
     : issueTitle.includes("UNSTABLE")

torchci/components/JobSummary.tsx

@@ -1,4 +1,4 @@
-import { JobData } from "lib/types";
+import { JobData, IssueData } from "lib/types";
 import JobConclusion from "./JobConclusion";
 import {
   isFailedJob,
@@ -26,17 +26,19 @@ function BranchName({
 export default function JobSummary({
   job,
   highlight,
+  unstableIssues,
 }: {
   job: JobData;
   highlight: boolean;
+  unstableIssues: IssueData[];
 }) {
   return (
     <>
       <JobConclusion
         conclusion={job.conclusion}
         warningOnly={
           isFailedJob(job) &&
-          (isRerunDisabledTestsJob(job) || isUnstableJob(job))
+          (isRerunDisabledTestsJob(job) || isUnstableJob(job, unstableIssues))
         }
       />
       <a href={job.htmlUrl}> {job.jobName} </a>

torchci/components/WorkflowBox.tsx

@@ -1,7 +1,7 @@
 import styles from "components/commit.module.css";
 import { fetcher } from "lib/GeneralUtils";
 import { isFailedJob } from "lib/jobUtils";
-import { Artifact, JobData } from "lib/types";
+import { Artifact, JobData, IssueData } from "lib/types";
 import useSWR from "swr";
 import JobArtifact from "./JobArtifact";
 import JobSummary from "./JobSummary";
@@ -29,11 +29,13 @@ function WorkflowJobSummary({
   artifacts,
   artifactsToShow,
   setArtifactsToShow,
+  unstableIssues,
 }: {
   job: JobData;
   artifacts?: Artifact[];
   artifactsToShow: Set<string>;
   setArtifactsToShow: any;
+  unstableIssues: IssueData[];
 }) {
   var queueTimeInfo = null;
   if (job.queueTimeS != null) {
@@ -72,7 +74,7 @@ function WorkflowJobSummary({
 
   return (
     <>
-      <JobSummary job={job} />
+      <JobSummary job={job} unstableIssues={unstableIssues} />
       <br />
       <small>
         &nbsp;&nbsp;&nbsp;&nbsp;
@@ -99,9 +101,11 @@ function WorkflowJobSummary({
 export default function WorkflowBox({
   workflowName,
   jobs,
+  unstableIssues,
 }: {
   workflowName: string;
   jobs: JobData[];
+  unstableIssues: IssueData[];
 }) {
   const isFailed = jobs.some(isFailedJob) !== false;
   const workflowClass = isFailed
@@ -171,6 +175,7 @@ export default function WorkflowBox({
               artifacts={groupedArtifacts?.get(job.id)}
               artifactsToShow={artifactsToShow}
               setArtifactsToShow={setArtifactsToShow}
+              unstableIssues={unstableIssues}
             />
             {(searchString && (
               <SearchLogViewer

torchci/lib/JobClassifierUtil.ts

@@ -1,5 +1,6 @@
 import { GroupedJobStatus, JobStatus } from "components/GroupJobConclusion";
-import { GroupData, RowData } from "./types";
+import { GroupData, RowData, IssueData } from "./types";
+import { hasOpenUnstableIssue } from "lib/jobUtils";
 
 const GROUP_MEMORY_LEAK_CHECK = "Memory Leak Check";
 const GROUP_RERUN_DISABLED_TESTS = "Rerun Disabled Tests";
@@ -179,7 +180,16 @@ export function sortGroupNamesForHUD(groupNames: string[]): string[] {
   return result;
 }
 
-export function classifyGroup(jobName: string): string {
+export function classifyGroup(
+  jobName: string,
+  unstableIssues?: IssueData[]
+): string {
+  // Double check first if the job has been marked as unstable but doesn't include
+  // the unstable keyword
+  if (hasOpenUnstableIssue(jobName, unstableIssues)) {
+    return GROUP_UNSTABLE;
+  }
+
   for (const group of groups) {
     if (jobName.match(group.regex)) {
       return group.name;
@@ -276,12 +286,16 @@ export function getConclusionSeverityForSorting(conclusion?: string): number {
   }
 }
 
-export function getGroupingData(shaGrid: RowData[], jobNames: string[]) {
+export function getGroupingData(
+  shaGrid: RowData[],
+  jobNames: string[],
+  unstableIssues?: IssueData[]
+) {
   // Construct Job Groupping Mapping
   const groupNameMapping = new Map<string, Array<string>>(); // group -> [jobs]
   const jobToGroupName = new Map<string, string>(); // job -> group
   for (const name of jobNames) {
-    const groupName = classifyGroup(name);
+    const groupName = classifyGroup(name, unstableIssues);
     const jobsInGroup = groupNameMapping.get(groupName) ?? [];
     jobsInGroup.push(name);
     groupNameMapping.set(groupName, jobsInGroup);
@@ -311,6 +325,9 @@ export function isPersistentGroup(name: string) {
   );
 }
 
-export function isUnstableGroup(name: string) {
-  return name.toLocaleLowerCase().includes("unstable");
+export function isUnstableGroup(name: string, unstableIssues?: IssueData[]) {
+  return (
+    name.toLocaleLowerCase().includes("unstable") ||
+    hasOpenUnstableIssue(name, unstableIssues)
+  );
 }

torchci/lib/fetchHud.ts

@@ -5,6 +5,7 @@ import { HudParams, JobData, RowData } from "./types";
 import rocksetVersions from "rockset/prodVersions.json";
 import { isFailure } from "./JobClassifierUtil";
 import { isRerunDisabledTestsJob, isUnstableJob } from "./jobUtils";
+import fetchIssuesByLabel from "lib/fetchIssuesByLabel";
 
 export default async function fetchHud(params: HudParams): Promise<{
   shaGrid: RowData[];
@@ -93,7 +94,10 @@ export default async function fetchHud(params: HudParams): Promise<{
     results = results?.filter((job: JobData) => !isRerunDisabledTestsJob(job));
   }
   if (params.filter_unstable) {
-    results = results?.filter((job: JobData) => !isUnstableJob(job));
+    const unstableIssues = await fetchIssuesByLabel("unstable");
+    results = results?.filter(
+      (job: JobData) => !isUnstableJob(job, unstableIssues ?? [])
+    );
   }
 
   const namesSet: Set<string> = new Set();