Add ability to dynamically add filter JSON fields (and fix test coverage)

Signed-off-by: Peter Broadhurst <[email protected]>

Author: peterbroadhurst

pkg/ffapi/apiserver.go

@@ -202,7 +202,7 @@ func (as *apiServer[T]) Serve(ctx context.Context) (err error) {
 		if err != nil {
 			return err
 		}
-		as.monitoringPublicURL = buildPublicURL(as.MonitoringConfig, apiHTTPServer.Addr())
+		as.monitoringPublicURL = buildPublicURL(as.MonitoringConfig, monitoringHTTPServer.Addr())
 		go monitoringHTTPServer.ServeHTTP(ctx)
 	}
 
@@ -219,6 +219,10 @@ func (as *apiServer[T]) APIPublicURL() string {
 	return as.apiPublicURL
 }
 
+func (as *apiServer[T]) MonitoringPublicURL() string {
+	return as.monitoringPublicURL
+}
+
 func (as *apiServer[T]) waitForServerStop(httpErrChan, monitoringErrChan chan error) error {
 	select {
 	case err := <-httpErrChan:

pkg/ffapi/apiserver_test.go

@@ -176,6 +176,26 @@ func TestAPIServerInvokeAPIRouteStream(t *testing.T) {
 	assert.Equal(t, "a stream!", string(res.Body()))
 }
 
+func TestAPIServerInvokeAPIRouteLiveness(t *testing.T) {
+	_, as, done := newTestAPIServer(t, true)
+	defer done()
+
+	<-as.Started()
+
+	res, err := resty.New().R().Get(fmt.Sprintf("%s/livez", as.MonitoringPublicURL()))
+	assert.NoError(t, err)
+	assert.Equal(t, 200, res.StatusCode()) // note this should really be 204, but not changing as would change behavior
+}
+
+func TestAPIServerPanicsMisConfig(t *testing.T) {
+	assert.Panics(t, func() {
+		_ = NewAPIServer(context.Background(), APIServerOptions[any]{})
+	})
+	assert.Panics(t, func() {
+		_ = NewAPIServer(context.Background(), APIServerOptions[any]{APIConfig: config.RootSection("any")})
+	})
+}
+
 func TestAPIServerInvokeAPIRouteJSON(t *testing.T) {
 	um, as, done := newTestAPIServer(t, true)
 	defer done()
@@ -342,6 +362,19 @@ func TestAPIServerInvokeEnrichFailForm(t *testing.T) {
 	assert.Equal(t, 500, res.StatusCode())
 }
 
+func TestAPIServerInvokeEnrichFailStream(t *testing.T) {
+	um, as, done := newTestAPIServer(t, true)
+	defer done()
+
+	um.mockEnrichErr = fmt.Errorf("pop")
+	<-as.Started()
+
+	res, err := resty.New().R().
+		Get(fmt.Sprintf("%s/api/v1/ut/utresource/id12345/getit", as.APIPublicURL()))
+	assert.NoError(t, err)
+	assert.Equal(t, 500, res.StatusCode())
+}
+
 func TestAPIServer404(t *testing.T) {
 	_, as, done := newTestAPIServer(t, true)
 	defer done()
@@ -397,6 +430,45 @@ func TestAPIServerFailServeMonitoring(t *testing.T) {
 
 }
 
+func TestAPIServerFailServeMonitoringBadRouteSlash(t *testing.T) {
+	_, as, done := newTestAPIServer(t, false)
+	defer done()
+
+	as.MonitoringConfig.Set(httpserver.HTTPConfAddress, "127.0.0.1:0")
+	as.MonitoringRoutes = []*Route{
+		{Path: "right", Extensions: &APIServerRouteExt[*utManager]{}},
+		{Path: "/wrong"},
+	}
+	err := as.Serve(context.Background())
+	assert.Regexp(t, "FF00255", err)
+
+	// Check we still closed the started channel
+	<-as.Started()
+
+}
+
+func TestAPIServerFailServeBadRouteSlash(t *testing.T) {
+	_, as, done := newTestAPIServer(t, false)
+	defer done()
+
+	as.Routes = []*Route{
+		{
+			Path: "/wrong",
+			Extensions: &APIServerRouteExt[*utManager]{
+				JSONHandler: func(a *APIRequest, um *utManager) (output interface{}, err error) {
+					return nil, nil
+				},
+			},
+		},
+	}
+	err := as.Serve(context.Background())
+	assert.Regexp(t, "FF00255", err)
+
+	// Check we still closed the started channel
+	<-as.Started()
+
+}
+
 func TestWaitForServerStop(t *testing.T) {
 	_, as, done := newTestAPIServer(t, false)
 	defer done()
@@ -611,6 +683,7 @@ func TestVersionedAPIInitErrors(t *testing.T) {
 	err = as.Serve(ctx)
 	assert.Error(t, err)
 	assert.Regexp(t, "FF00253", err)
+
 	as = NewAPIServer(ctx, APIServerOptions[*utManager]{
 		MetricsRegistry: metric.NewPrometheusMetricsRegistry("ut"),
 		VersionedAPIs: &VersionedAPIs{
@@ -633,4 +706,32 @@ func TestVersionedAPIInitErrors(t *testing.T) {
 	assert.Error(t, err)
 	assert.Regexp(t, "FF00254", err)
 
+	as = NewAPIServer(ctx, APIServerOptions[*utManager]{
+		MetricsRegistry: metric.NewPrometheusMetricsRegistry("ut"),
+		VersionedAPIs: &VersionedAPIs{
+			DefaultVersion: "unknown",
+			APIVersions: map[string]*APIVersion{
+				"v1": {
+					Routes: []*Route{
+						{
+							Path: "/wrong",
+							Extensions: &APIServerRouteExt[*utManager]{
+								JSONHandler: func(r *APIRequest, um *utManager) (output interface{}, err error) {
+									return nil, nil
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		Description:      "unit testing",
+		APIConfig:        apiConfig,
+		MonitoringConfig: monitoringConfig,
+	})
+
+	err = as.Serve(ctx)
+	assert.Error(t, err)
+	assert.Regexp(t, "FF00255", err)
+
 }

pkg/ffapi/handler_test.go

@@ -28,6 +28,7 @@ import (
 	"net/url"
 	"strings"
 	"testing"
+	"testing/iotest"
 	"time"
 
 	"github.com/getkin/kin-openapi/openapi3"
@@ -727,3 +728,23 @@ func TestPOSTFormParamsMultiValueUnsupported(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, 400, res.StatusCode)
 }
+
+func TestGetFormParamsFail(t *testing.T) {
+
+	hs := newTestHandlerFactory("", nil)
+	req := httptest.NewRequest(http.MethodPost, "http://localhost:12345/anything", iotest.ErrReader(fmt.Errorf("pop")))
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	_, err := hs.getFormParams(req)
+	require.Regexp(t, "FF00250.*pop", err)
+}
+
+func TestGetFormEmptyValue(t *testing.T) {
+
+	hs := newTestHandlerFactory("", nil)
+	req := httptest.NewRequest(http.MethodPost, "http://localhost:12345/anything", nil)
+	req.Form = url.Values{
+		"nothing": []string{},
+	}
+	_, err := hs.getFormParams(req)
+	require.NoError(t, err)
+}

pkg/ffapi/openapi3_test.go

@@ -178,6 +178,21 @@ var testRoutes = []*Route{
 		JSONOutputValue: func() interface{} { return &TestStruct1{} },
 		JSONOutputCodes: []int{http.StatusOK},
 	},
+	{
+		Name:            "op7",
+		Path:            "example7",
+		Method:          http.MethodPut,
+		PathParams:      nil,
+		QueryParams:     nil,
+		Description:     ExampleDesc,
+		JSONInputValue:  func() interface{} { return &TestStruct1{} },
+		JSONOutputValue: func() interface{} { return nil },
+		JSONOutputCodes: []int{http.StatusNoContent},
+		FormParams: []*FormParam{
+			{Name: "metadata", Description: ExampleDesc},
+		},
+		Tag: example2TagName,
+	},
 }
 
 type TestInOutType struct {

pkg/ffapi/openapihandler.go

@@ -95,10 +95,6 @@ func (ohf *OpenAPIHandlerFactory) getPublicURL(req *http.Request, relativePath s
 	if publicURL == "" {
 		publicURL = ohf.StaticPublicURL
 	}
-	if publicURL == "" {
-		// Do not recommend this fallback - StaticPublicURL and/or DynamicPublicURLHeader should be set
-		publicURL = fmt.Sprintf("https://%s", req.Host)
-	}
 	publicURL = strings.TrimSuffix(publicURL, "/")
 	if len(relativePath) > 0 {
 		publicURL = publicURL + "/" + strings.TrimPrefix(relativePath, "/")

pkg/ffapi/openapihandler_test.go

@@ -24,7 +24,9 @@ import (
 	"testing"
 
 	"github.com/go-resty/resty/v2"
+	"github.com/gorilla/mux"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestOpenAPI3SwaggerUI(t *testing.T) {
@@ -98,3 +100,28 @@ func TestOpenAPI3SwaggerUIDynamicPublicURL(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Contains(t, string(b), "https://example.host.com:12345/api/myswagger.json")
 }
+
+func TestOpenAPIHandlerNonVersioned(t *testing.T) {
+	mux := mux.NewRouter()
+	hf := HandlerFactory{}
+	oah := &OpenAPIHandlerFactory{
+		BaseSwaggerGenOptions: SwaggerGenOptions{
+			Title:                 "FireFly Transaction Manager API",
+			Version:               "1.0",
+			SupportFieldRedaction: true,
+		},
+	}
+	mux.Path("/api/spec.json").Methods(http.MethodGet).Handler(hf.APIWrapper(oah.OpenAPIHandler("", OpenAPIFormatJSON, []*Route{})))
+	mux.Path("/api/spec.yaml").Methods(http.MethodGet).Handler(hf.APIWrapper(oah.OpenAPIHandler("", OpenAPIFormatYAML, []*Route{})))
+
+	ts := httptest.NewServer(mux)
+	defer ts.Close()
+
+	res, err := resty.New().R().Get(fmt.Sprintf("%s/api/spec.json", ts.URL))
+	require.NoError(t, err)
+	require.True(t, res.IsSuccess())
+
+	res, err = resty.New().R().Get(fmt.Sprintf("%s/api/spec.yaml", ts.URL))
+	require.NoError(t, err)
+	require.True(t, res.IsSuccess())
+}

pkg/ffapi/query_fields.go

@@ -38,6 +38,12 @@ type QueryFactory interface {
 	NewFilter(ctx context.Context) FilterBuilder
 	NewFilterLimit(ctx context.Context, defLimit uint64) FilterBuilder
 	NewUpdate(ctx context.Context) UpdateBuilder
+	Clone() ClonedQueryFactory
+}
+
+type ClonedQueryFactory interface {
+	QueryFactory
+	AddField(n string, f Field)
 }
 
 type FieldMod int
@@ -53,23 +59,35 @@ type HasFieldMods interface {
 
 type QueryFields map[string]Field
 
-func (qf *QueryFields) NewFilterLimit(ctx context.Context, defLimit uint64) FilterBuilder {
+func (qf QueryFields) NewFilterLimit(ctx context.Context, defLimit uint64) FilterBuilder {
 	return &filterBuilder{
 		ctx:         ctx,
-		queryFields: *qf,
+		queryFields: qf,
 		limit:       defLimit,
 	}
 }
 
-func (qf *QueryFields) NewFilter(ctx context.Context) FilterBuilder {
+func (qf QueryFields) NewFilter(ctx context.Context) FilterBuilder {
 	return qf.NewFilterLimit(ctx, 0)
 }
 
-func (qf *QueryFields) NewUpdate(ctx context.Context) UpdateBuilder {
+func (qf QueryFields) NewUpdate(ctx context.Context) UpdateBuilder {
 	return &updateBuilder{
 		ctx:         ctx,
-		queryFields: *qf,
+		queryFields: qf,
+	}
+}
+
+func (qf QueryFields) AddField(n string, f Field) {
+	qf[n] = f
+}
+
+func (qf QueryFields) Clone() ClonedQueryFactory {
+	qf2 := make(QueryFields, len(qf))
+	for n, f := range qf {
+		qf2[n] = f
 	}
+	return qf2
 }
 
 // FieldSerialization - we stand on the shoulders of the well adopted SQL serialization interface here to help us define what
@@ -284,17 +302,11 @@ func (f *bigIntField) Scan(src interface{}) (err error) {
 	case int64:
 		f.i = fftypes.NewFFBigInt(tv)
 	case uint:
-		if tv > math.MaxInt64 {
-			return i18n.NewError(context.Background(), i18n.MsgTypeRestoreFailed, src, f.i)
-		}
-		f.i = fftypes.NewFFBigInt(int64(tv))
+		f.i = (*fftypes.FFBigInt)(new(big.Int).SetUint64(uint64(tv)))
 	case uint32:
 		f.i = fftypes.NewFFBigInt(int64(tv))
 	case uint64:
-		if tv > math.MaxInt64 {
-			return i18n.NewError(context.Background(), i18n.MsgTypeRestoreFailed, src, f.i)
-		}
-		f.i = fftypes.NewFFBigInt(int64(tv))
+		f.i = (*fftypes.FFBigInt)(new(big.Int).SetUint64(tv))
 	case fftypes.FFBigInt:
 		i := tv
 		f.i = &i

pkg/ffapi/query_fields_test.go

@@ -17,6 +17,8 @@
 package ffapi
 
 import (
+	"fmt"
+	"math"
 	"testing"
 	"time"
 
@@ -138,6 +140,15 @@ func TestInt64Field(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, int64(0), v)
 
+	err = f.Scan(uint64(math.MaxInt64 + 1))
+	assert.Regexp(t, "FF00105", err)
+
+	err = f.Scan(uint(math.MaxInt64 + 1))
+	assert.Regexp(t, "FF00105", err)
+
+	err = f.Scan(fmt.Sprintf("%d", uint64(math.MaxInt64+1)))
+	assert.Regexp(t, "FF00105", err)
+
 }
 
 func TestBigIntField(t *testing.T) {