CLOUDP-302671: Add filter command that removes x-xgen extensions from OAS

tools/cli/internal/cli/filter/filter.go

@@ -0,0 +1,113 @@
+// Copyright 2024 MongoDB Inc
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package filter
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/getkin/kin-openapi/openapi3"
+	"github.com/mongodb/openapi/tools/cli/internal/apiversion"
+	"github.com/mongodb/openapi/tools/cli/internal/cli/flag"
+	"github.com/mongodb/openapi/tools/cli/internal/cli/usage"
+	"github.com/mongodb/openapi/tools/cli/internal/openapi"
+	"github.com/mongodb/openapi/tools/cli/internal/openapi/filter"
+	"github.com/spf13/afero"
+	"github.com/spf13/cobra"
+)
+
+type Opts struct {
+	fs         afero.Fs
+	basePath   string
+	outputPath string
+	env        string
+	version    string
+	format     string
+}
+
+func (o *Opts) Run() error {
+	loader := openapi.NewOpenAPI3()
+	specInfo, err := loader.CreateOpenAPISpecFromPath(o.basePath)
+	if err != nil {
+		return err
+	}
+
+	var filteredOAS *openapi3.T
+	// If a version is provided, versioning filters will also be applied.
+	if o.version != "" {
+		filteredOAS, err = ByVersion(specInfo.Spec, o.version, o.env)
+	} else {
+		filters := filter.FiltersWithoutVersioning
+		metadata := filter.NewMetadata(nil, o.env)
+		filteredOAS, err = filter.ApplyFilters(specInfo.Spec, metadata, filters)
+	}
+
+	if err != nil {
+		return err
+	}
+
+	return openapi.Save(o.outputPath, filteredOAS, o.format, o.fs)
+}
+
+func ByVersion(oas *openapi3.T, version, env string) (result *openapi3.T, err error) {
+	log.Printf("Filtering OpenAPI document by version %q", version)
+	apiVersion, err := apiversion.New(apiversion.WithVersion(version))
+	if err != nil {
+		return nil, err
+	}
+
+	return filter.ApplyFilters(oas, filter.NewMetadata(apiVersion, env), filter.DefaultFilters)
+}
+
+func (o *Opts) PreRunE(_ []string) error {
+	if o.basePath == "" {
+		return fmt.Errorf("no OAS detected. Please, use the flag %s to include the base OAS", flag.Base)
+	}
+
+	return openapi.ValidateFormatAndOutput(o.format, o.outputPath)
+}
+
+// Builder builds the filter command with the following signature:
+// filter -s oas -o output-oas.json.
+func Builder() *cobra.Command {
+	opts := &Opts{
+		fs: afero.NewOsFs(),
+	}
+
+	cmd := &cobra.Command{
+		Use: "filter -s spec ",
+		Short: `Filter Open API specification removing hidden endpoints and extension metadata. 
+If a version is provided, versioning filters will also be applied.`,
+		Args: cobra.NoArgs,
+		PreRunE: func(_ *cobra.Command, args []string) error {
+			return opts.PreRunE(args)
+		},
+		RunE: func(_ *cobra.Command, _ []string) error {
+			return opts.Run()
+		},
+	}
+
+	cmd.Flags().StringVarP(&opts.basePath, flag.Spec, flag.SpecShort, "-", usage.Spec)
+	cmd.Flags().StringVar(&opts.env, flag.Environment, "", usage.Environment)
+	cmd.Flags().StringVarP(&opts.outputPath, flag.Output, flag.OutputShort, "", usage.Output)
+	cmd.Flags().StringVar(&opts.version, flag.Version, "", usage.Version)
+	cmd.Flags().StringVarP(&opts.format, flag.Format, flag.FormatShort, openapi.ALL, usage.Format)
+
+	// Required flags
+	_ = cmd.MarkFlagRequired(flag.Output)
+	_ = cmd.MarkFlagRequired(flag.Spec)
+	_ = cmd.MarkFlagRequired(flag.Environment)
+	return cmd
+}

tools/cli/internal/cli/filter/filter_test.go

@@ -0,0 +1,143 @@
+// Copyright 2024 MongoDB Inc
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package filter
+
+import (
+	"net/url"
+	"testing"
+
+	"github.com/getkin/kin-openapi/openapi3"
+	"github.com/mongodb/openapi/tools/cli/internal/openapi"
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/require"
+	"github.com/tufin/oasdiff/load"
+)
+
+func TestSuccessfulFilter_Run(t *testing.T) {
+	fs := afero.NewMemMapFs()
+	t.Parallel()
+
+	opts := &Opts{
+		basePath:   "../../../test/data/base_spec.json",
+		outputPath: "filtered-oas.yaml",
+		fs:         fs,
+		env:        "dev",
+	}
+
+	if err := opts.Run(); err != nil {
+		t.Fatalf("Run() unexpected error: %v", err)
+	}
+
+	newSpec, err := loadRunResultOas(fs, opts.outputPath)
+	require.NoError(t, err)
+
+	// check all paths are kept
+	for _, pathItem := range newSpec.Spec.Paths.Map() {
+		for _, operation := range pathItem.Operations() {
+			// check extensions are removed at the operation level
+			require.Nil(t, operation.Extensions)
+		}
+	}
+}
+
+func TestSuccessfulFilterWithVersion_Run(t *testing.T) {
+	fs := afero.NewMemMapFs()
+	t.Parallel()
+
+	opts := &Opts{
+		basePath:   "../../../test/data/base_spec.json",
+		outputPath: "filtered-oas.yaml",
+		fs:         fs,
+		env:        "dev",
+		version:    "2023-01-01",
+	}
+
+	if err := opts.Run(); err != nil {
+		t.Fatalf("Run() unexpected error: %v", err)
+	}
+
+	s, err := loadRunResultOas(fs, opts.outputPath)
+	require.NoError(t, err)
+	// assert /api/atlas/v2/groups/{groupId}:migrate does not exist in filtered spec as it is from a newer version
+	paths := s.Spec.Paths.Map()
+	require.Contains(t, paths, "/api/atlas/v2/groups")
+	require.NotContains(t, paths, "/api/atlas/v2/groups/{groupId}:migrate")
+}
+
+func TestOpts_PreRunE(t *testing.T) {
+	testCases := []struct {
+		wantErr  require.ErrorAssertionFunc
+		basePath string
+		name     string
+	}{
+		{
+			wantErr: require.Error,
+			name:    "NoBasePath",
+		},
+		{
+			wantErr:  require.NoError,
+			basePath: "../../../test/data/base_spec.json",
+			name:     "Successful",
+		},
+	}
+
+	for _, tt := range testCases {
+		t.Run(tt.name, func(t *testing.T) {
+			o := &Opts{
+				basePath: tt.basePath,
+				format:   "json",
+			}
+			tt.wantErr(t, o.PreRunE(nil))
+		})
+	}
+}
+
+func TestInvalidFormat_PreRun(t *testing.T) {
+	opts := &Opts{
+		outputPath: "foas.json",
+		basePath:   "base.json",
+		format:     "html",
+	}
+
+	err := opts.PreRunE(nil)
+	require.Error(t, err)
+	require.EqualError(t, err, "format must be either 'json', 'yaml' or 'all', got 'html'")
+}
+
+func TestInvalidPath_PreRun(t *testing.T) {
+	opts := &Opts{
+		outputPath: "foas.html",
+		basePath:   "base.json",
+		format:     "all",
+	}
+
+	err := opts.PreRunE(nil)
+	require.Error(t, err)
+	require.EqualError(t, err, "output file must be either a JSON or YAML file, got foas.html")
+}
+
+func loadRunResultOas(fs afero.Fs, fileName string) (*load.SpecInfo, error) {
+	oas := openapi.NewOpenAPI3()
+	oas.Loader.ReadFromURIFunc = func(_ *openapi3.Loader, _ *url.URL) ([]byte, error) {
+		f, err := fs.OpenFile(fileName, 0, 0)
+		if err != nil {
+			return nil, err
+		}
+		defer f.Close()
+		return afero.ReadAll(f)
+	}
+
+	return oas.CreateOpenAPISpecFromPath(fileName)
+}

tools/cli/internal/cli/flag/flag.go

@@ -49,4 +49,6 @@ const (
 	To                       = "to"
 	StabilityLevel           = "stability-level"
 	StabilityLevelShort      = "l"
+	Version                  = "version"
+	VersionShort             = "v"
 )

tools/cli/internal/cli/merge/merge.go

@@ -17,7 +17,6 @@ package merge
 import (
 	"encoding/json"
 	"fmt"
-	"strings"
 
 	"github.com/mongodb/openapi/tools/cli/internal/cli/flag"
 	"github.com/mongodb/openapi/tools/cli/internal/cli/usage"
@@ -71,11 +70,7 @@ func (o *Opts) PreRunE(_ []string) error {
 		return fmt.Errorf("no external OAS detected. Please, use the flag %s to include at least one OAS", flag.External)
 	}
 
-	if o.outputPath != "" && !strings.Contains(o.outputPath, ".json") && !strings.Contains(o.outputPath, ".yaml") {
-		return fmt.Errorf("output file must be either a JSON or YAML file, got %s", o.outputPath)
-	}
-
-	if err := openapi.ValidateFormat(o.format); err != nil {
+	if err := openapi.ValidateFormatAndOutput(o.format, o.outputPath); err != nil {
 		return err
 	}
 

tools/cli/internal/cli/merge/merge_test.go

@@ -159,6 +159,7 @@ func TestInvalidPath_PreRun(t *testing.T) {
 		outputPath:    "foas.html",
 		externalPaths: externalPaths,
 		basePath:      "base.json",
+		format:        "json",
 	}
 
 	err := opts.PreRunE(nil)

tools/cli/internal/cli/root/openapi/builder.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/mongodb/openapi/tools/cli/internal/cli/breakingchanges"
 	"github.com/mongodb/openapi/tools/cli/internal/cli/changelog"
+	"github.com/mongodb/openapi/tools/cli/internal/cli/filter"
 	"github.com/mongodb/openapi/tools/cli/internal/cli/merge"
 	"github.com/mongodb/openapi/tools/cli/internal/cli/split"
 	"github.com/mongodb/openapi/tools/cli/internal/cli/sunset"
@@ -61,6 +62,7 @@ func Builder() *cobra.Command {
 		changelog.Builder(),
 		breakingchanges.Builder(),
 		sunset.Builder(),
+		filter.Builder(),
 	)
 	return rootCmd
 }

tools/cli/internal/cli/split/split.go

@@ -20,11 +20,10 @@ import (
 	"strings"
 
 	"github.com/getkin/kin-openapi/openapi3"
-	"github.com/mongodb/openapi/tools/cli/internal/apiversion"
+	"github.com/mongodb/openapi/tools/cli/internal/cli/filter"
 	"github.com/mongodb/openapi/tools/cli/internal/cli/flag"
 	"github.com/mongodb/openapi/tools/cli/internal/cli/usage"
 	"github.com/mongodb/openapi/tools/cli/internal/openapi"
-	"github.com/mongodb/openapi/tools/cli/internal/openapi/filter"
 	"github.com/spf13/afero"
 	"github.com/spf13/cobra"
 )
@@ -51,7 +50,7 @@ func (o *Opts) Run() error {
 	}
 
 	for _, version := range versions {
-		filteredOAS, err := o.filter(specInfo.Spec, version)
+		filteredOAS, err := filter.ByVersion(specInfo.Spec, version, o.env)
 		if err != nil {
 			return err
 		}
@@ -74,16 +73,6 @@ func (o *Opts) Run() error {
 	return nil
 }
 
-func (o *Opts) filter(oas *openapi3.T, version string) (result *openapi3.T, err error) {
-	log.Printf("Filtering OpenAPI document by version %q", version)
-	apiVersion, err := apiversion.New(apiversion.WithVersion(version))
-	if err != nil {
-		return nil, err
-	}
-
-	return filter.ApplyFilters(oas, filter.NewMetadata(apiVersion, o.env), filter.DefaultFilters)
-}
-
 func (o *Opts) saveVersionedOas(oas *openapi3.T, version string) error {
 	path := o.basePath
 	if o.outputPath != "" {
@@ -109,11 +98,7 @@ func (o *Opts) PreRunE(_ []string) error {
 		return fmt.Errorf("no OAS detected. Please, use the flag %s to include the base OAS", flag.Base)
 	}
 
-	if o.outputPath != "" && !strings.Contains(o.outputPath, openapi.DotJSON) && !strings.Contains(o.outputPath, openapi.DotYAML) {
-		return fmt.Errorf("output file must be either a JSON or YAML file, got %s", o.outputPath)
-	}
-
-	return openapi.ValidateFormat(o.format)
+	return openapi.ValidateFormatAndOutput(o.format, o.outputPath)
 }
 
 // Builder builds the split command with the following signature: