Environment variable based importing and provisioning for OpenAI Evals

Author: Arun

plugins/evals/api_key.go

@@ -0,0 +1,41 @@
+package evals
+
+import (
+	"github.com/1Password/shell-plugins/sdk"
+	"github.com/1Password/shell-plugins/sdk/importer"
+	"github.com/1Password/shell-plugins/sdk/provision"
+	"github.com/1Password/shell-plugins/sdk/schema"
+	"github.com/1Password/shell-plugins/sdk/schema/credname"
+	"github.com/1Password/shell-plugins/sdk/schema/fieldname"
+)
+
+func APIKey() schema.CredentialType {
+	return schema.CredentialType{
+		Name:          credname.APIKey,
+		DocsURL:       sdk.URL("https://help.openai.com/en/articles/4936850-where-do-i-find-my-secret-api-key"),
+		ManagementURL: sdk.URL("https://platform.openai.com/account/api-keys"),
+		Fields: []schema.CredentialField{
+			{
+				Name:                fieldname.APIKey,
+				MarkdownDescription: "API Key used to authenticate to OpenAI platform.",
+				Secret:              true,
+				Composition: &schema.ValueComposition{
+					Length: 51,
+					Prefix: "sk-",
+					Charset: schema.Charset{
+						Uppercase: true,
+						Lowercase: true,
+						Digits:    true,
+					},
+				},
+			},
+		},
+		DefaultProvisioner: provision.EnvVars(defaultEnvVarMapping),
+		Importer: importer.TryAll(
+			importer.TryEnvVarPair(defaultEnvVarMapping),
+		)}
+}
+
+var defaultEnvVarMapping = map[string]sdk.FieldName{
+	"OPENAI_API_KEY": fieldname.APIKey,
+}

plugins/evals/api_key_test.go

@@ -0,0 +1,41 @@
+package evals
+
+import (
+	"testing"
+
+	"github.com/1Password/shell-plugins/sdk"
+	"github.com/1Password/shell-plugins/sdk/plugintest"
+	"github.com/1Password/shell-plugins/sdk/schema/fieldname"
+)
+
+func TestAPIKeyProvisioner(t *testing.T) {
+	plugintest.TestProvisioner(t, APIKey().DefaultProvisioner, map[string]plugintest.ProvisionCase{
+		"default": {
+			ItemFields: map[sdk.FieldName]string{
+				fieldname.APIKey: "sk-ZJipjm9euld2QcpLIX3NCDUiLgexQkvdn0WGlKayAEXAMPLE",
+			},
+			ExpectedOutput: sdk.ProvisionOutput{
+				Environment: map[string]string{
+					"OPENAI_API_KEY": "sk-ZJipjm9euld2QcpLIX3NCDUiLgexQkvdn0WGlKayAEXAMPLE",
+				},
+			},
+		},
+	})
+}
+
+func TestAPIKeyImporter(t *testing.T) {
+	plugintest.TestImporter(t, APIKey().Importer, map[string]plugintest.ImportCase{
+		"environment": {
+			Environment: map[string]string{
+				"OPENAI_API_KEY": "sk-ZJipjm9euld2QcpLIX3NCDUiLgexQkvdn0WGlKayAEXAMPLE",
+			},
+			ExpectedCandidates: []sdk.ImportCandidate{
+				{
+					Fields: map[sdk.FieldName]string{
+						fieldname.APIKey: "sk-ZJipjm9euld2QcpLIX3NCDUiLgexQkvdn0WGlKayAEXAMPLE",
+					},
+				},
+			},
+		},
+	})
+}

plugins/evals/oaieval.go

@@ -0,0 +1,25 @@
+package evals
+
+import (
+	"github.com/1Password/shell-plugins/sdk"
+	"github.com/1Password/shell-plugins/sdk/needsauth"
+	"github.com/1Password/shell-plugins/sdk/schema"
+	"github.com/1Password/shell-plugins/sdk/schema/credname"
+)
+
+func OpenAIEvalsCLI() schema.Executable {
+	return schema.Executable{
+		Name:    "OpenAI Evals CLI",
+		Runs:    []string{"oaieval"},
+		DocsURL: sdk.URL("https://github.com/openai/evals/blob/main/README.md"),
+		NeedsAuth: needsauth.IfAll(
+			needsauth.NotForHelpOrVersion(),
+			needsauth.NotWithoutArgs(),
+		),
+		Uses: []schema.CredentialUsage{
+			{
+				Name: credname.APIKey,
+			},
+		},
+	}
+}

plugins/evals/plugin.go

@@ -0,0 +1,22 @@
+package evals
+
+import (
+	"github.com/1Password/shell-plugins/sdk"
+	"github.com/1Password/shell-plugins/sdk/schema"
+)
+
+func New() schema.Plugin {
+	return schema.Plugin{
+		Name: "evals",
+		Platform: schema.PlatformInfo{
+			Name:     "OpenAI Evals",
+			Homepage: sdk.URL("https://github.com/openai/evals"),
+		},
+		Credentials: []schema.CredentialType{
+			APIKey(),
+		},
+		Executables: []schema.Executable{
+			OpenAIEvalsCLI(),
+		},
+	}
+}