Make all file operations through the filesystem interface (#283)

Author: jsoriano

code/go/internal/fspath/fspath.go

@@ -0,0 +1,38 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package fspath
+
+import (
+	"io/fs"
+	"os"
+	"path/filepath"
+)
+
+// FS implements the fs interface and can also show a path where the fs is located.
+// This is useful to report error messages relative to the location of the file system.
+type FS interface {
+	fs.FS
+
+	Path(name ...string) string
+}
+
+type fsDir struct {
+	fs.FS
+
+	path string
+}
+
+// Path returns a path for the given names, based on the location of the file system.
+func (fs *fsDir) Path(names ...string) string {
+	return filepath.Join(append([]string{fs.path}, names...)...)
+}
+
+// DirFS returns a file system for a directory, it keeps the path to implement the FS interface.
+func DirFS(path string) FS {
+	return &fsDir{
+		FS:   os.DirFS(path),
+		path: path,
+	}
+}

code/go/internal/pkgpath/files.go

@@ -7,7 +7,7 @@ package pkgpath
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"strings"
@@ -16,31 +16,34 @@ import (
 	"github.com/joeshaw/multierror"
 	"github.com/pkg/errors"
 	"gopkg.in/yaml.v3"
+
+	"github.com/elastic/package-spec/code/go/internal/fspath"
 )
 
 // File represents a file in the package.
 type File struct {
+	fsys fspath.FS
 	path string
 	os.FileInfo
 }
 
 // Files finds files for the given glob
-func Files(glob string) ([]File, error) {
-	paths, err := filepath.Glob(glob)
+func Files(fsys fspath.FS, glob string) ([]File, error) {
+	paths, err := fs.Glob(fsys, glob)
 	if err != nil {
 		return nil, err
 	}
 
 	var errs multierror.Errors
 	var files = make([]File, 0)
 	for _, path := range paths {
-		info, err := os.Stat(path)
+		info, err := fs.Stat(fsys, path)
 		if err != nil {
 			errs = append(errs, err)
 			continue
 		}
 
-		file := File{path, info}
+		file := File{fsys, path, info}
 		files = append(files, file)
 	}
 
@@ -58,19 +61,19 @@ func (f File) Values(path string) (interface{}, error) {
 		return nil, fmt.Errorf("cannot extract values from file type = %s", fileExt)
 	}
 
-	contents, err := ioutil.ReadFile(f.path)
+	contents, err := fs.ReadFile(f.fsys, f.path)
 	if err != nil {
 		return nil, errors.Wrap(err, "reading file content failed")
 	}
 
 	var v interface{}
 	if fileExt == "yaml" || fileExt == "yml" {
 		if err := yaml.Unmarshal(contents, &v); err != nil {
-			return nil, errors.Wrapf(err, "unmarshalling YAML file failed (path: %s)", fileName)
+			return nil, errors.Wrapf(err, "unmarshalling YAML file failed (path: %s)", f.fsys.Path(fileName))
 		}
 	} else if fileExt == "json" {
 		if err := json.Unmarshal(contents, &v); err != nil {
-			return nil, errors.Wrapf(err, "unmarshalling JSON file failed (path: %s)", fileName)
+			return nil, errors.Wrapf(err, "unmarshalling JSON file failed (path: %s)", f.fsys.Path(fileName))
 		}
 	}
 

code/go/internal/validator/folder_item_content.go

@@ -8,15 +8,15 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io/fs"
 	"mime"
 
 	"github.com/pkg/errors"
 	"gopkg.in/yaml.v3"
 )
 
-func loadItemContent(itemPath, mediaType string) ([]byte, error) {
-	itemData, err := ioutil.ReadFile(itemPath)
+func loadItemContent(fsys fs.FS, itemPath, mediaType string) ([]byte, error) {
+	itemData, err := fs.ReadFile(fsys, itemPath)
 	if err != nil {
 		return nil, errors.Wrap(err, "reading item file failed")
 	}

code/go/internal/validator/folder_item_spec.go

@@ -7,7 +7,6 @@ package validator
 import (
 	"fmt"
 	"io/fs"
-	"os"
 	"path/filepath"
 	"regexp"
 	"sync"
@@ -35,7 +34,7 @@ type folderItemSpec struct {
 
 var formatCheckersMutex sync.Mutex
 
-func (s *folderItemSpec) matchingFileExists(files []os.FileInfo) (bool, error) {
+func (s *folderItemSpec) matchingFileExists(files []fs.DirEntry) (bool, error) {
 	if s.Name != "" {
 		for _, file := range files {
 			if file.Name() == s.Name {
@@ -57,7 +56,13 @@ func (s *folderItemSpec) matchingFileExists(files []os.FileInfo) (bool, error) {
 	return false, nil
 }
 
-func (s *folderItemSpec) isSameType(file os.FileInfo) bool {
+// sameFileChecker is the interface that parameters of isSameType should implement,
+// this is intended to accept both fs.DirEntry and fs.FileInfo.
+type sameFileChecker interface {
+	IsDir() bool
+}
+
+func (s *folderItemSpec) isSameType(file sameFileChecker) bool {
 	switch s.ItemType {
 	case itemTypeFile:
 		return !file.IsDir()
@@ -68,17 +73,17 @@ func (s *folderItemSpec) isSameType(file os.FileInfo) bool {
 	return false
 }
 
-func (s *folderItemSpec) validate(fs fs.FS, folderSpecPath string, itemPath string) ve.ValidationErrors {
+func (s *folderItemSpec) validate(schemaFS fs.FS, fsys fs.FS, folderSpecPath string, itemPath string) ve.ValidationErrors {
 	// loading item content
-	itemData, err := loadItemContent(itemPath, s.ContentMediaType)
+	itemData, err := loadItemContent(fsys, itemPath, s.ContentMediaType)
 	if err != nil {
 		return ve.ValidationErrors{err}
 	}
 
 	var schemaLoader gojsonschema.JSONLoader
 	if s.Ref != "" {
 		schemaPath := filepath.Join(filepath.Dir(folderSpecPath), s.Ref)
-		schemaLoader = yamlschema.NewReferenceLoaderFileSystem("file:///"+schemaPath, fs)
+		schemaLoader = yamlschema.NewReferenceLoaderFileSystem("file:///"+schemaPath, schemaFS)
 	} else {
 		return nil // item's schema is not defined
 	}
@@ -93,8 +98,8 @@ func (s *folderItemSpec) validate(fs fs.FS, folderSpecPath string, itemPath stri
 		formatCheckersMutex.Unlock()
 	}()
 
-	semantic.LoadRelativePathFormatChecker(filepath.Dir(itemPath))
-	semantic.LoadDataStreamNameFormatChecker(filepath.Dir(itemPath))
+	semantic.LoadRelativePathFormatChecker(fsys, filepath.Dir(itemPath))
+	semantic.LoadDataStreamNameFormatChecker(fsys, filepath.Dir(itemPath))
 	result, err := gojsonschema.Validate(schemaLoader, documentLoader)
 	if err != nil {
 		return ve.ValidationErrors{err}

code/go/internal/validator/folder_item_spec_format.go

@@ -8,15 +8,15 @@ import (
 	"fmt"
 	"io/fs"
 	"io/ioutil"
-	"path"
 	"path/filepath"
 	"regexp"
 	"strings"
 
-	ve "github.com/elastic/package-spec/code/go/internal/errors"
-
 	"github.com/pkg/errors"
 	"gopkg.in/yaml.v3"
+
+	ve "github.com/elastic/package-spec/code/go/internal/errors"
+	"github.com/elastic/package-spec/code/go/internal/fspath"
 )
 
 const (
@@ -66,11 +66,11 @@ func newFolderSpec(fs fs.FS, specPath string) (*folderSpec, error) {
 	return &spec, nil
 }
 
-func (s *folderSpec) validate(packageName string, folderPath string) ve.ValidationErrors {
+func (s *folderSpec) validate(packageName string, fsys fspath.FS, path string) ve.ValidationErrors {
 	var errs ve.ValidationErrors
-	files, err := ioutil.ReadDir(folderPath)
+	files, err := fs.ReadDir(fsys, path)
 	if err != nil {
-		errs = append(errs, errors.Wrapf(err, "could not read folder [%s]", folderPath))
+		errs = append(errs, errors.Wrapf(err, "could not read folder [%s]", fsys.Path(path)))
 		return errs
 	}
 
@@ -87,16 +87,16 @@ func (s *folderSpec) validate(packageName string, folderPath string) ve.Validati
 			if file.IsDir() {
 				if !s.DevelopmentFolder && strings.Contains(fileName, "-") {
 					errs = append(errs,
-						fmt.Errorf(`file "%s/%s" is invalid: directory name inside package %s contains -: %s`,
-							folderPath, fileName, packageName, fileName))
+						fmt.Errorf(`file "%s" is invalid: directory name inside package %s contains -: %s`,
+							fsys.Path(path, fileName), packageName, fileName))
 				}
 			}
 			continue
 		}
 
 		if itemSpec == nil && !s.AdditionalContents {
 			// No spec found for current folder item and we do not allow additional contents in folder.
-			errs = append(errs, fmt.Errorf("item [%s] is not allowed in folder [%s]", fileName, folderPath))
+			errs = append(errs, fmt.Errorf("item [%s] is not allowed in folder [%s]", fileName, fsys.Path(path)))
 			continue
 		}
 
@@ -118,7 +118,7 @@ func (s *folderSpec) validate(packageName string, folderPath string) ve.Validati
 
 			var subFolderSpec *folderSpec
 			if itemSpec.Ref != "" {
-				subFolderSpecPath := path.Join(filepath.Dir(s.specPath), itemSpec.Ref)
+				subFolderSpecPath := filepath.Join(filepath.Dir(s.specPath), itemSpec.Ref)
 				subFolderSpec, err = newFolderSpec(s.fs, subFolderSpecPath)
 				if err != nil {
 					errs = append(errs, err)
@@ -140,23 +140,23 @@ func (s *folderSpec) validate(packageName string, folderPath string) ve.Validati
 				subFolderSpec.DevelopmentFolder = true
 			}
 
-			subFolderPath := path.Join(folderPath, fileName)
-			subErrs := subFolderSpec.validate(packageName, subFolderPath)
+			subFolderPath := filepath.Join(path, fileName)
+			subErrs := subFolderSpec.validate(packageName, fsys, subFolderPath)
 			if len(subErrs) > 0 {
 				errs = append(errs, subErrs...)
 			}
 
 		} else {
 			if !itemSpec.isSameType(file) {
-				errs = append(errs, fmt.Errorf("[%s] is a file but is expected to be a folder", fileName))
+				errs = append(errs, fmt.Errorf("[%s] is a file but is expected to be a folder", fsys.Path(fileName)))
 				continue
 			}
 
-			itemPath := filepath.Join(folderPath, file.Name())
-			itemValidationErrs := itemSpec.validate(s.fs, s.specPath, itemPath)
+			itemPath := filepath.Join(path, file.Name())
+			itemValidationErrs := itemSpec.validate(s.fs, fsys, s.specPath, itemPath)
 			if itemValidationErrs != nil {
 				for _, ive := range itemValidationErrs {
-					errs = append(errs, errors.Wrapf(ive, "file \"%s\" is invalid", itemPath))
+					errs = append(errs, errors.Wrapf(ive, "file \"%s\" is invalid", fsys.Path(itemPath)))
 				}
 			}
 		}
@@ -177,9 +177,9 @@ func (s *folderSpec) validate(packageName string, folderPath string) ve.Validati
 		if !fileFound {
 			var err error
 			if itemSpec.Name != "" {
-				err = fmt.Errorf("expecting to find [%s] %s in folder [%s]", itemSpec.Name, itemSpec.ItemType, folderPath)
+				err = fmt.Errorf("expecting to find [%s] %s in folder [%s]", itemSpec.Name, itemSpec.ItemType, fsys.Path(path))
 			} else if itemSpec.Pattern != "" {
-				err = fmt.Errorf("expecting to find %s matching pattern [%s] in folder [%s]", itemSpec.ItemType, itemSpec.Pattern, folderPath)
+				err = fmt.Errorf("expecting to find %s matching pattern [%s] in folder [%s]", itemSpec.ItemType, itemSpec.Pattern, fsys.Path(path))
 			}
 			errs = append(errs, err)
 		}

code/go/internal/validator/folder_spec.go

@@ -6,9 +6,9 @@ package validator
 
 import (
 	"fmt"
-	"io/ioutil"
+	"io/fs"
 	"os"
-	"path"
+	"path/filepath"
 
 	"github.com/Masterminds/semver/v3"
 	"github.com/pkg/errors"
@@ -19,7 +19,19 @@ import (
 type Package struct {
 	Name        string
 	SpecVersion *semver.Version
-	RootPath    string
+
+	fs       fs.FS
+	location string
+}
+
+// Open opens a file in the package filesystem.
+func (p *Package) Open(name string) (fs.File, error) {
+	return p.fs.Open(name)
+}
+
+// Path returns a path meaningful for the user.
+func (p *Package) Path(names ...string) string {
+	return filepath.Join(append([]string{p.location}, names...)...)
 }
 
 // NewPackage creates a new Package from a path to the package's root folder
@@ -33,13 +45,19 @@ func NewPackage(pkgRootPath string) (*Package, error) {
 		return nil, fmt.Errorf("no package folder found at path [%v]", pkgRootPath)
 	}
 
-	pkgManifestPath := path.Join(pkgRootPath, "manifest.yml")
-	info, err = os.Stat(pkgManifestPath)
+	return NewPackageFromFS(pkgRootPath, os.DirFS(pkgRootPath))
+}
+
+// NewPackageFromFS creates a new package from a given filesystem. A root path can be indicated
+// to help building paths meaningful for the users.
+func NewPackageFromFS(location string, fsys fs.FS) (*Package, error) {
+	pkgManifestPath := "manifest.yml"
+	_, err := fs.Stat(fsys, pkgManifestPath)
 	if os.IsNotExist(err) {
 		return nil, errors.Wrapf(err, "no package manifest file found at path [%v]", pkgManifestPath)
 	}
 
-	data, err := ioutil.ReadFile(pkgManifestPath)
+	data, err := fs.ReadFile(fsys, pkgManifestPath)
 	if err != nil {
 		return nil, fmt.Errorf("could not read package manifest file [%v]", pkgManifestPath)
 	}
@@ -60,8 +78,10 @@ func NewPackage(pkgRootPath string) (*Package, error) {
 	// Instantiate Package object and return it
 	p := Package{
 		Name:        manifest.Name,
-		RootPath:    pkgRootPath,
 		SpecVersion: specVersion,
+		fs:          fsys,
+
+		location: location,
 	}
 
 	return &p, nil