Skip to content

Commit 1966052

Browse files
alexluongalexluong
authored
feat: Destination metadata (loader, validation, API) (#124)
* feat: Destination type metadata loader * feat: Load metadata in destregistry * feat: Provider schema API * feat: Destination validation schema * chore: core.json * test: Remove registry top-level validation tests * test: Update expected validation error in e2e tests * test: Add providers api e2e test
1 parent 62042b0 commit 1966052

33 files changed

+1095
-314
lines changed

‎cmd/e2e/api_test.go‎

Lines changed: 101 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -411,7 +411,10 @@ func (suite *basicSuite) TestDestinationsAPI() {
411411
Match: &httpclient.Response{
412412
StatusCode: http.StatusUnprocessableEntity,
413413
Body: map[string]interface{}{
414-
"message": "validation failed: url is required for webhook destination config",
414+
"message": "validation error",
415+
"data": map[string]interface{}{
416+
"config.url": "required",
417+
},
415418
},
416419
},
417420
},
@@ -566,7 +569,10 @@ func (suite *basicSuite) TestDestinationsAPI() {
566569
Match: &httpclient.Response{
567570
StatusCode: http.StatusUnprocessableEntity,
568571
Body: map[string]interface{}{
569-
"message": "validation failed: url is required for webhook destination config",
572+
"message": "validation error",
573+
"data": map[string]interface{}{
574+
"config.url": "required",
575+
},
570576
},
571577
},
572578
},
@@ -946,6 +952,99 @@ func (suite *basicSuite) TestTopicsAPI() {
946952
suite.RunAPITests(suite.T(), tests)
947953
}
948954

955+
func (suite *basicSuite) TestProvidersAPI() {
956+
providerFieldSchema := map[string]interface{}{
957+
"type": "object",
958+
"required": []interface{}{"key", "type", "label", "description", "required"},
959+
"properties": map[string]interface{}{
960+
"key": map[string]interface{}{"type": "string"},
961+
"type": map[string]interface{}{"type": "string"},
962+
"label": map[string]interface{}{"type": "string"},
963+
"description": map[string]interface{}{"type": "string"},
964+
"required": map[string]interface{}{"type": "boolean"},
965+
},
966+
}
967+
968+
providerSchema := map[string]interface{}{
969+
"type": "object",
970+
"required": []interface{}{"type", "label", "description", "icon", "config_fields", "credential_fields"},
971+
"properties": map[string]interface{}{
972+
"type": map[string]interface{}{"type": "string"},
973+
"label": map[string]interface{}{"type": "string"},
974+
"description": map[string]interface{}{"type": "string"},
975+
"icon": map[string]interface{}{"type": "string"},
976+
"instructions": map[string]interface{}{"type": "string"},
977+
"config_fields": map[string]interface{}{
978+
"type": "array",
979+
"items": providerFieldSchema,
980+
},
981+
"credential_fields": map[string]interface{}{
982+
"type": "array",
983+
"items": providerFieldSchema,
984+
},
985+
"validation": map[string]interface{}{
986+
"type": "object",
987+
},
988+
},
989+
}
990+
991+
tests := []APITest{
992+
{
993+
Name: "GET /providers",
994+
Request: suite.AuthRequest(httpclient.Request{
995+
Method: httpclient.MethodGET,
996+
Path: "/providers",
997+
}),
998+
Expected: APITestExpectation{
999+
Validate: map[string]any{
1000+
"type": "object",
1001+
"properties": map[string]any{
1002+
"statusCode": map[string]any{"const": 200},
1003+
"body": map[string]interface{}{
1004+
"type": "object",
1005+
"required": []interface{}{"webhook", "aws", "rabbitmq"},
1006+
"properties": map[string]interface{}{
1007+
"webhook": providerSchema,
1008+
"aws": providerSchema,
1009+
"rabbitmq": providerSchema,
1010+
},
1011+
},
1012+
},
1013+
},
1014+
},
1015+
},
1016+
{
1017+
Name: "GET /providers/webhook",
1018+
Request: suite.AuthRequest(httpclient.Request{
1019+
Method: httpclient.MethodGET,
1020+
Path: "/providers/webhook",
1021+
}),
1022+
Expected: APITestExpectation{
1023+
Validate: map[string]any{
1024+
"type": "object",
1025+
"properties": map[string]any{
1026+
"statusCode": map[string]any{"const": 200},
1027+
"body": providerSchema,
1028+
},
1029+
},
1030+
},
1031+
},
1032+
{
1033+
Name: "GET /providers/invalid",
1034+
Request: suite.AuthRequest(httpclient.Request{
1035+
Method: httpclient.MethodGET,
1036+
Path: "/providers/invalid",
1037+
}),
1038+
Expected: APITestExpectation{
1039+
Match: &httpclient.Response{
1040+
StatusCode: http.StatusNotFound,
1041+
},
1042+
},
1043+
},
1044+
}
1045+
suite.RunAPITests(suite.T(), tests)
1046+
}
1047+
9491048
func makeDestinationListValidator(length int) map[string]any {
9501049
return map[string]any{
9511050
"type": "object",

‎cmd/tools/genprovider/main.go‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
// cmd/tools/genprovider/main.go
2+
package main
3+
4+
import (
5+
"encoding/json"
6+
"fmt"
7+
"os"
8+
"path/filepath"
9+
"strings"
10+
)
11+
12+
type CoreMetadata struct {
13+
Type string `json:"type"`
14+
ConfigFields []FieldSchema `json:"config_fields"`
15+
CredentialFields []FieldSchema `json:"credential_fields"`
16+
}
17+
18+
type UIMetadata struct {
19+
Label string `json:"label"`
20+
Description string `json:"description"`
21+
Icon string `json:"icon"`
22+
RemoteSetupURL string `json:"remote_setup_url,omitempty"`
23+
}
24+
25+
type FieldSchema struct {
26+
Type string `json:"type"`
27+
Label string `json:"label"`
28+
Description string `json:"description"`
29+
Key string `json:"key"`
30+
Required bool `json:"required"`
31+
}
32+
33+
type ValidationSchema struct {
34+
Schema string `json:"$schema"`
35+
Type string `json:"type"`
36+
Properties map[string]interface{} `json:"properties"`
37+
}
38+
39+
func main() {
40+
if len(os.Args) != 2 {
41+
fmt.Println("Usage: genprovider <provider-name>")
42+
os.Exit(1)
43+
}
44+
45+
provider := os.Args[1]
46+
baseDir := "internal/destregistry/metadata/providers"
47+
providerDir := filepath.Join(baseDir, provider)
48+
49+
// Create provider directory
50+
if err := os.MkdirAll(providerDir, 0755); err != nil {
51+
fmt.Printf("Error creating provider directory: %v\n", err)
52+
os.Exit(1)
53+
}
54+
55+
// Generate core.json
56+
core := CoreMetadata{
57+
Type: provider,
58+
ConfigFields: []FieldSchema{},
59+
CredentialFields: []FieldSchema{},
60+
}
61+
writeJSON(filepath.Join(providerDir, "core.json"), core)
62+
63+
// Generate ui.json
64+
ui := UIMetadata{
65+
Label: strings.ToTitle(provider),
66+
Description: fmt.Sprintf("Send events to %s", provider),
67+
Icon: "",
68+
}
69+
writeJSON(filepath.Join(providerDir, "ui.json"), ui)
70+
71+
// Generate validation.json
72+
validation := ValidationSchema{
73+
Schema: "http://json-schema.org/draft-07/schema#",
74+
Type: "object",
75+
Properties: map[string]interface{}{
76+
"config": map[string]interface{}{
77+
"type": "object",
78+
},
79+
"credentials": map[string]interface{}{
80+
"type": "object",
81+
},
82+
},
83+
}
84+
writeJSON(filepath.Join(providerDir, "validation.json"), validation)
85+
86+
// Generate instructions.md
87+
instructions := fmt.Sprintf("# %s Setup Instructions\n\nBasic setup instructions for %s destination.",
88+
strings.ToTitle(provider), provider)
89+
writeFile(filepath.Join(providerDir, "instructions.md"), instructions)
90+
91+
fmt.Printf("Created provider files for %s in %s\n", provider, providerDir)
92+
}
93+
94+
func writeJSON(path string, v interface{}) {
95+
data, err := json.MarshalIndent(v, "", " ")
96+
if err != nil {
97+
fmt.Printf("Error marshaling JSON for %s: %v\n", path, err)
98+
return
99+
}
100+
writeFile(path, string(data))
101+
}
102+
103+
func writeFile(path string, content string) {
104+
if err := os.WriteFile(path, []byte(content), 0644); err != nil {
105+
fmt.Printf("Error writing file %s: %v\n", path, err)
106+
}
107+
}

‎internal/destregistry/baseprovider.go‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
package destregistry
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"strings"
7+
8+
"github.com/hookdeck/outpost/internal/destregistry/metadata"
9+
"github.com/hookdeck/outpost/internal/models"
10+
"github.com/santhosh-tekuri/jsonschema/v6"
11+
"github.com/santhosh-tekuri/jsonschema/v6/kind"
12+
)
13+
14+
// BaseProvider provides common functionality for all destination providers
15+
type BaseProvider struct {
16+
metadata *metadata.ProviderMetadata
17+
}
18+
19+
// NewBaseProvider creates a new base provider with loaded metadata
20+
func NewBaseProvider(providerType string) (*BaseProvider, error) {
21+
loader := metadata.NewMetadataLoader("")
22+
meta, err := loader.Load(providerType)
23+
if err != nil {
24+
return nil, fmt.Errorf("loading provider metadata: %w", err)
25+
}
26+
27+
return &BaseProvider{
28+
metadata: meta,
29+
}, nil
30+
}
31+
32+
// Metadata returns the provider metadata
33+
func (p *BaseProvider) Metadata() *metadata.ProviderMetadata {
34+
return p.metadata
35+
}
36+
37+
// Validate performs schema validation using the provider's metadata
38+
func (p *BaseProvider) Validate(ctx context.Context, destination *models.Destination) error {
39+
if destination.Type != p.metadata.Type {
40+
return NewErrDestinationValidation([]ValidationErrorDetail{{
41+
Field: "type",
42+
Type: "invalid_type",
43+
}})
44+
}
45+
46+
// Convert the config and credentials to map[string]interface{} for JSON schema validation
47+
validationData := map[string]interface{}{
48+
"config": map[string]interface{}{},
49+
"credentials": map[string]interface{}{},
50+
}
51+
52+
// Copy config values
53+
for k, v := range destination.Config {
54+
validationData["config"].(map[string]interface{})[k] = v
55+
}
56+
57+
// Copy credentials values
58+
for k, v := range destination.Credentials {
59+
validationData["credentials"].(map[string]interface{})[k] = v
60+
}
61+
62+
// Validate using JSON schema
63+
if err := p.metadata.Validation.Validate(validationData); err != nil {
64+
if validationErr, ok := err.(*jsonschema.ValidationError); ok {
65+
errors := formatJSONSchemaErrors(validationErr)
66+
if len(errors) == 0 {
67+
return NewErrDestinationValidation([]ValidationErrorDetail{{
68+
Field: "root",
69+
Type: "unknown",
70+
}})
71+
}
72+
return NewErrDestinationValidation(errors)
73+
}
74+
}
75+
76+
return nil
77+
}
78+
79+
func formatPropertyPath(pathParts []string) string {
80+
var parts []string
81+
for _, part := range pathParts {
82+
if part != "" {
83+
parts = append(parts, part)
84+
}
85+
}
86+
if len(parts) == 0 {
87+
return "root"
88+
}
89+
return strings.Join(parts, ".")
90+
}
91+
92+
func formatJSONSchemaErrors(validationErr *jsonschema.ValidationError) []ValidationErrorDetail {
93+
var errors []ValidationErrorDetail
94+
95+
var processError func(*jsonschema.ValidationError)
96+
processError = func(verr *jsonschema.ValidationError) {
97+
if verr.InstanceLocation != nil {
98+
propertyPath := formatPropertyPath(verr.InstanceLocation)
99+
if errorKind, ok := verr.ErrorKind.(interface{ KeywordPath() []string }); ok {
100+
keywordPath := errorKind.KeywordPath()
101+
errorType := keywordPath[len(keywordPath)-1]
102+
103+
// Handle required field errors specially
104+
if errorType == "required" {
105+
if required, ok := verr.ErrorKind.(*kind.Required); ok {
106+
for _, missingField := range required.Missing {
107+
fullPath := propertyPath
108+
if fullPath != "root" {
109+
fullPath = fullPath + "." + missingField
110+
}
111+
errors = append(errors, ValidationErrorDetail{
112+
Field: fullPath,
113+
Type: "required",
114+
})
115+
}
116+
return
117+
}
118+
}
119+
120+
errors = append(errors, ValidationErrorDetail{
121+
Field: propertyPath,
122+
Type: errorType,
123+
})
124+
}
125+
}
126+
127+
for _, cause := range verr.Causes {
128+
processError(cause)
129+
}
130+
}
131+
132+
processError(validationErr)
133+
return errors
134+
}

‎internal/destregistry/error.go‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,21 @@ package destregistry
22

33
import "fmt"
44

5-
func NewErrDestinationValidation(err error) error {
6-
return fmt.Errorf("validation failed: %w", err)
5+
type ErrDestinationValidation struct {
6+
Errors []ValidationErrorDetail `json:"errors"`
7+
}
8+
9+
type ValidationErrorDetail struct {
10+
Field string `json:"field"`
11+
Type string `json:"type"`
12+
}
13+
14+
func (e *ErrDestinationValidation) Error() string {
15+
return fmt.Sprintf("validation failed")
16+
}
17+
18+
func NewErrDestinationValidation(errors []ValidationErrorDetail) error {
19+
return &ErrDestinationValidation{Errors: errors}
720
}
821

922
type ErrDestinationPublish struct {

0 commit comments

Comments
 (0)