Bump min DVC version to 2.55.0 (live metrics for experiments running outside the workspace) (#3665)

* wrap all loose test data in test data generator

* add new type

* duplicate required functions but use new data

* update test fixtures

* deduplicate functions

* remove checkpoints model and file system watcher (#3684)

* extend timeout of run experiment test (e2e) (#3713)

* prevent plotting of running experiments (#3712)

* trigger plot updates whenever commit data changes (#3715)

* update demo project and min required version of DVC

* fix experiment id for commits (shown in plots) (#3724)

Author: mattseddon

demo

@@ -1,7 +1,7 @@
 import { Plot } from '../../plots/webview/contract'
 
-export const MIN_CLI_VERSION = '2.52.0'
-export const LATEST_TESTED_CLI_VERSION = '2.54.0'
+export const MIN_CLI_VERSION = '2.55.0'
+export const LATEST_TESTED_CLI_VERSION = '2.55.0'
 export const MAX_CLI_VERSION = '3'
 
 type ErrorContents = { type: string; msg: string }
@@ -26,76 +26,92 @@ export type DataStatusOutput = {
 type SingleValue = string | number | boolean | null
 export type Value = SingleValue | SingleValue[]
 
-export interface ValueTreeOrError {
-  data?: ValueTree
-  error?: ErrorContents
-}
-
 type RelPathObject<T> = {
   [relPath: string]: T
 }
 
-export type ValueTreeRoot = RelPathObject<ValueTreeOrError>
-
-export interface ValueTreeNode {
+export type ValueTree = {
   [key: string]: Value | ValueTree
 }
 
-export type ValueTree = ValueTreeRoot | ValueTreeNode
-
 export const isValueTree = (
   value: Value | ValueTree
 ): value is NonNullable<ValueTree> =>
   !!(value && !Array.isArray(value) && typeof value === 'object')
 
 export enum ExperimentStatus {
-  FAILED = 'Failed',
-  QUEUED = 'Queued',
-  RUNNING = 'Running',
-  SUCCESS = 'Success'
+  FAILED = 'failed',
+  QUEUED = 'queued',
+  RUNNING = 'running',
+  SUCCESS = 'success'
 }
 
-export const EXPERIMENT_WORKSPACE_ID = 'workspace'
-
-export interface BaseExperimentFields {
-  name?: string
-  timestamp?: string | null
-  status?: ExperimentStatus
-  executor?: string | null
-  checkpoint_tip?: string
-  checkpoint_parent?: string
-}
+export const EXPERIMENT_WORKSPACE_ID = 'workspace' as const
 
 type Dep = { hash: null | string; size: null | number; nfiles: null | number }
 type Out = Dep & { use_cache: boolean; is_data_source: boolean }
 
+type Outs = RelPathObject<Out>
 export type Deps = RelPathObject<Dep>
 
-export interface ExperimentFields extends BaseExperimentFields {
-  params?: ValueTreeRoot
-  metrics?: ValueTreeRoot
-  deps?: Deps
-  outs?: RelPathObject<Out>
-  error?: ErrorContents
+export type FileDataOrError = { data: ValueTree } | DvcError
+export type MetricsOrParams = RelPathObject<FileDataOrError>
+
+export const fileHasError = (file: FileDataOrError): file is DvcError =>
+  !!(file as DvcError).error
+
+export type ExpData = {
+  rev: string
+  timestamp: string | null
+  params: MetricsOrParams | null
+  metrics: MetricsOrParams | null
+  deps: Deps | null
+  outs: Outs | null
+  meta: { has_checkpoints: boolean }
 }
 
-export interface ExperimentFieldsOrError {
-  data?: ExperimentFields
-  error?: ErrorContents
+export enum Executor {
+  DVC_TASK = 'dvc-task',
+  WORKSPACE = 'workspace'
 }
 
-export interface ExperimentsCommitOutput {
-  [sha: string]: ExperimentFieldsOrError
-  baseline: ExperimentFieldsOrError
+export type ExecutorState = {
+  state: ExperimentStatus
+  name: Executor | null
+  local: {
+    root: string | null
+    log: string | null
+    pid: number | null
+    task_id?: string
+    returncode: null | number
+  } | null
+} | null
+
+export type ExpWithError = {
+  rev: string
+  name?: string
+} & DvcError
+
+type ExpWithData = {
+  rev: string
+  name?: string
+  data: ExpData
 }
 
-export interface ExperimentsOutput {
-  [name: string]: ExperimentsCommitOutput
-  [EXPERIMENT_WORKSPACE_ID]: {
-    baseline: ExperimentFieldsOrError
-  }
+export type ExpState = ExpWithData | ExpWithError
+
+export type ExpRange = {
+  revs: ExpState[]
+  name?: string
+  executor: ExecutorState
 }
 
+export const experimentHasError = (
+  expState: ExpState
+): expState is ExpWithError => !!(expState as { error?: unknown }).error
+
+export type ExpShowOutput = (ExpState & { experiments?: ExpRange[] | null })[]
+
 export interface PlotsData {
   [path: string]: Plot[]
 }

extension/src/cli/dvc/contract.ts

@@ -94,23 +94,22 @@ describe('CliReader', () => {
       })
 
       const cliOutput = await dvcReader.expShow(cwd)
-      expect(cliOutput).toStrictEqual({
-        [EXPERIMENT_WORKSPACE_ID]: {
-          baseline: { error: { msg: unexpectedStderr, type: 'Caught error' } }
+      expect(cliOutput).toStrictEqual([
+        {
+          error: { msg: unexpectedStderr, type: 'Caught error' },
+          rev: EXPERIMENT_WORKSPACE_ID
         }
-      })
+      ])
     })
 
-    it('should return the default output if the cli returns an empty object (no commits)', async () => {
+    it('should return the default output if the cli returns an empty array (no commits)', async () => {
       const cwd = __dirname
       mockedCreateProcess.mockReturnValueOnce(
-        getMockedProcess(JSON.stringify({}))
+        getMockedProcess(JSON.stringify([]))
       )
 
       const cliOutput = await dvcReader.expShow(cwd)
-      expect(cliOutput).toStrictEqual({
-        [EXPERIMENT_WORKSPACE_ID]: { baseline: {} }
-      })
+      expect(cliOutput).toStrictEqual([{ rev: EXPERIMENT_WORKSPACE_ID }])
     })
   })
 

extension/src/cli/dvc/reader.test.ts

@@ -11,11 +11,11 @@ import {
 import {
   DataStatusOutput,
   DvcError,
-  ExperimentsOutput,
   EXPERIMENT_WORKSPACE_ID,
   PlotsOutput,
   PlotsOutputOrError,
-  RawPlotsOutput
+  RawPlotsOutput,
+  ExpShowOutput
 } from './contract'
 import { getOptions } from './options'
 import { typeCheckCommands } from '..'
@@ -24,12 +24,8 @@ import { Logger } from '../../common/logger'
 import { parseNonStandardJson } from '../../util/json'
 import { definedAndNonEmpty } from '../../util/array'
 
-const defaultExperimentsOutput: ExperimentsOutput = {
-  [EXPERIMENT_WORKSPACE_ID]: { baseline: {} }
-}
-
 export const isDvcError = <
-  T extends ExperimentsOutput | DataStatusOutput | RawPlotsOutput
+  T extends ExpShowOutput | DataStatusOutput | RawPlotsOutput
 >(
   dataOrError: T | DvcError
 ): dataOrError is DvcError =>
@@ -68,19 +64,17 @@ export class DvcReader extends DvcCli {
   public async expShow(
     cwd: string,
     ...flags: (ExperimentFlag | string)[]
-  ): Promise<ExperimentsOutput> {
-    const output = await this.readProcess<ExperimentsOutput>(
+  ): Promise<ExpShowOutput> {
+    const output = await this.readProcess<ExpShowOutput>(
       cwd,
-      JSON.stringify(defaultExperimentsOutput),
+      JSON.stringify([]),
       Command.EXPERIMENT,
       SubCommand.SHOW,
       ...flags,
       Flag.JSON
     )
     if (isDvcError(output) || isEmpty(output)) {
-      return {
-        [EXPERIMENT_WORKSPACE_ID]: { baseline: output as DvcError | {} }
-      }
+      return [{ rev: EXPERIMENT_WORKSPACE_ID, ...(output as DvcError) }]
     }
     return output
   }
@@ -150,7 +144,7 @@ export class DvcReader extends DvcCli {
   }
 
   private async readProcess<
-    T extends DataStatusOutput | ExperimentsOutput | RawPlotsOutput
+    T extends DataStatusOutput | ExpShowOutput | RawPlotsOutput
   >(cwd: string, defaultValue: string, ...args: Args): Promise<T | DvcError> {
     try {
       const output =

extension/src/cli/dvc/reader.ts

@@ -51,13 +51,13 @@ export class DvcViewer extends Disposable implements ICli {
     return this.createProcess(name, cwd, args)
   }
 
-  public queueLogs(cwd: string, expName: string) {
+  public queueLogs(cwd: string, id: string) {
     return this.run(
-      `${expName} logs`,
+      `${id} logs`,
       cwd,
       Command.QUEUE,
       QueueSubCommand.LOGS,
-      expName,
+      id,
       Flag.FOLLOW
     )
   }

extension/src/cli/dvc/viewer.ts

@@ -4,15 +4,16 @@ import { getRelativePattern } from '../fileSystem/relativePattern'
 import { createFileSystemWatcher } from '../fileSystem/watcher'
 import { ProcessManager } from '../process/manager'
 import { InternalCommands } from '../commands/internal'
-import { ExperimentsOutput, PlotsOutputOrError } from '../cli/dvc/contract'
+import { ExpShowOutput, PlotsOutputOrError } from '../cli/dvc/contract'
 import { uniqueValues } from '../util/array'
 import { DeferredDisposable } from '../class/deferred'
 import { isSameOrChild } from '../fileSystem'
 
 export abstract class BaseData<
-  T extends { data: PlotsOutputOrError; revs: string[] } | ExperimentsOutput
+  T extends { data: PlotsOutputOrError; revs: string[] } | ExpShowOutput
 > extends DeferredDisposable {
   public readonly onDidUpdate: Event<T>
+  public readonly onDidChangeDvcYaml: Event<void>
 
   protected readonly dvcRoot: string
   protected readonly processManager: ProcessManager
@@ -25,6 +26,10 @@ export abstract class BaseData<
     new EventEmitter()
   )
 
+  private readonly dvcYamlChanged: EventEmitter<void> = this.dispose.track(
+    new EventEmitter<void>()
+  )
+
   constructor(
     dvcRoot: string,
     internalCommands: InternalCommands,
@@ -42,6 +47,8 @@ export abstract class BaseData<
     this.onDidUpdate = this.updated.event
     this.staticFiles = staticFiles
 
+    this.onDidChangeDvcYaml = this.dvcYamlChanged.event
+
     this.watchFiles()
 
     this.waitForInitialData()
@@ -80,6 +87,10 @@ export abstract class BaseData<
         ) {
           void this.managedUpdate(path)
         }
+
+        if (path.endsWith('dvc.yaml')) {
+          this.dvcYamlChanged.fire()
+        }
       }
     )
   }