experiment-action/action.yml at main · langfuse/experiment-action · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
name: "Langfuse Experiment"
description: "Run a Langfuse experiment in CI, gate on regressions, and comment results on the PR."
author: "Langfuse"
branding:
  icon: "shield"
  color: "purple"

inputs:
  langfuse_public_key:
    description: "Langfuse public API key."
    required: true
  langfuse_secret_key:
    description: "Langfuse secret API key."
    required: true
  langfuse_base_url:
    description: "Langfuse base URL."
    required: false
    default: "https://cloud.langfuse.com"

  experiment_path:
    description: "Path to an experiment script (.py/.ts/.js/.mjs), a directory, or a glob pattern."
    required: true
  dataset_name:
    description: "Dataset to run the experiment against. If omitted, the user script is expected to select its own dataset."
    required: false
  dataset_version:
    description: "Pin experiment to a specific dataset version."
    required: false

  experiment_metadata:
    description: "Additional metadata to attach to the experiment run. Multiline, one `key=value` per line. Shown under the Metadata column in the Langfuse UI."
    required: false

  should_fail_on_regression:
    description: "Fail the CI job when an experiment raises `RegressionError`."
    required: false
    default: "true"
  should_fail_on_script_error:
    description: "Fail the CI job when an experiment script crashes or raises a non-regression error."
    required: false
    default: "true"
  should_comment_on_pr:
    description: "Post the experiment report as a PR comment."
    required: false
    default: "true"

  python_sdk_version:
    description: "Version of the Langfuse Python SDK to install via pip (for `.py` experiments). Use a specific version string or `latest`."
    required: false
    default: "4.6.0b1"
  js_sdk_version:
    description: "Version of the Langfuse JS SDK (`@langfuse/client`) to install via npm (for `.ts` / `.js` / `.mjs` experiments). Use a specific version string or `latest`."
    required: false
    default: "5.3.0-beta.0"
  should_skip_sdk_installation:
    description: "Skip installing the Langfuse SDK entirely. Use this when you manage the Python/Node environment yourself (e.g. with your own `pip install` / `npm install` step before this action) — the action then trusts the ambient environment to provide `langfuse` / `@langfuse/client` + `tsx`."
    required: false
    default: "false"

  github_token:
    description: >-
      Workflow GitHub token. Used to (1) post the PR comment — needs
      `pull-requests: write`, and (2) resolve the current job via the
      REST API so the comment links to this specific job and parallel
      matrix legs get their own comment sections — needs
      `actions: read`, otherwise falls back to the workflow-run URL
      and the job key from `$GITHUB_JOB`. Leave blank to skip the PR
      comment and the job lookup. See the README for a usage example.
    required: false
    default: ""

outputs:
  result_json:
    description: "Versioned JSON envelope with action metadata and normalized experiment results. See README for the schema."
  failed:
    description: "'true' if any experiment script errored or raised a regression, else 'false'."

runs:
  using: "node24"
  main: "dist/index.js"