Merge pull request #98 from planetlabs/urls

Accept URLs for describe, validate, and convert input

Author: tschaub

cmd/gpq/command/command.go

@@ -1,8 +1,13 @@
 package command
 
 import (
+	"bytes"
 	"fmt"
 	"io"
+	"os"
+	"strings"
+
+	"github.com/planetlabs/gpq/internal/storage"
 )
 
 var CLI struct {
@@ -12,12 +17,6 @@ var CLI struct {
 	Version  VersionCmd  `cmd:"" help:"Print the version of this program."`
 }
 
-type ReaderAtSeeker interface {
-	io.Reader
-	io.ReaderAt
-	io.Seeker
-}
-
 type CommandError struct {
 	err error
 }
@@ -33,3 +32,19 @@ func (e *CommandError) Error() string {
 func (e *CommandError) Unwrap() error {
 	return e.err
 }
+
+func readerFromInput(input string) (storage.ReaderAtSeeker, error) {
+	if input == "" {
+		data, err := io.ReadAll(os.Stdin)
+		if err != nil {
+			return nil, fmt.Errorf("trouble reading from stdin: %w", err)
+		}
+		return bytes.NewReader(data), nil
+	}
+
+	if strings.HasPrefix(input, "http://") || strings.HasPrefix(input, "https://") {
+		return storage.NewHttpReader(input)
+	}
+
+	return os.Open(input)
+}

cmd/gpq/command/command_test.go

@@ -2,6 +2,8 @@ package command_test
 
 import (
 	"io"
+	"net/http"
+	"net/http/httptest"
 	"os"
 	"testing"
 
@@ -14,6 +16,7 @@ type Suite struct {
 	mockStdin      *os.File
 	originalStdout *os.File
 	mockStdout     *os.File
+	server         *httptest.Server
 }
 
 func (s *Suite) SetupTest() {
@@ -28,6 +31,9 @@ func (s *Suite) SetupTest() {
 	s.originalStdout = os.Stdout
 	s.mockStdout = stdout
 	os.Stdout = stdout
+
+	handler := http.FileServer(http.Dir("../../../internal"))
+	s.server = httptest.NewServer(handler)
 }
 
 func (s *Suite) writeStdin(data []byte) {
@@ -58,6 +64,8 @@ func (s *Suite) TearDownTest() {
 
 	_ = s.mockStdout.Close()
 	s.NoError(os.Remove(s.mockStdout.Name()))
+
+	s.server.Close()
 }
 
 func TestSuite(t *testing.T) {

cmd/gpq/command/convert.go

@@ -15,8 +15,7 @@
 package command
 
 import (
-	"bytes"
-	"io"
+	"net/url"
 	"os"
 	"strings"
 
@@ -25,7 +24,7 @@ import (
 )
 
 type ConvertCmd struct {
-	Input              string `arg:"" optional:"" name:"input" help:"Input file.  If not provided, input is read from stdin." type:"path"`
+	Input              string `arg:"" optional:"" name:"input" help:"Input file path or URL.  If not provided, input is read from stdin."`
 	From               string `help:"Input file format.  Possible values: ${enum}." enum:"auto, geojson, geoparquet, parquet" default:"auto"`
 	Output             string `arg:"" optional:"" name:"output" help:"Output file.  If not provided, output is written to stdout." type:"path"`
 	To                 string `help:"Output file format.  Possible values: ${enum}." enum:"auto, geojson, geoparquet" default:"auto"`
@@ -64,14 +63,17 @@ func parseFormatType(format string) FormatType {
 	return ft
 }
 
-func getFormatType(filename string) FormatType {
-	if strings.HasSuffix(filename, ".json") || strings.HasSuffix(filename, ".geojson") {
+func getFormatType(resource string) FormatType {
+	if u, err := url.Parse(resource); err == nil {
+		resource = u.Path
+	}
+	if strings.HasSuffix(resource, ".json") || strings.HasSuffix(resource, ".geojson") {
 		return GeoJSONType
 	}
-	if strings.HasSuffix(filename, ".gpq") || strings.HasSuffix(filename, ".geoparquet") {
+	if strings.HasSuffix(resource, ".gpq") || strings.HasSuffix(resource, ".geoparquet") {
 		return GeoParquetType
 	}
-	if strings.HasSuffix(filename, ".pq") || strings.HasSuffix(filename, ".parquet") {
+	if strings.HasSuffix(resource, ".pq") || strings.HasSuffix(resource, ".parquet") {
 		return ParquetType
 	}
 	return UnknownType
@@ -116,20 +118,9 @@ func (c *ConvertCmd) Run() error {
 		return NewCommandError("could not determine input format for %s", inputSource)
 	}
 
-	var input ReaderAtSeeker
-	if inputSource == "" {
-		data, err := io.ReadAll(os.Stdin)
-		if err != nil {
-			return NewCommandError("trouble reading from stdin: %w", err)
-		}
-		input = bytes.NewReader(data)
-	} else {
-		i, readErr := os.Open(inputSource)
-		if readErr != nil {
-			return NewCommandError("failed to read from %q: %w", inputSource, readErr)
-		}
-		defer i.Close()
-		input = i
+	input, inputErr := readerFromInput(inputSource)
+	if inputErr != nil {
+		return NewCommandError("trouble getting a reader from %q: %w", c.Input, inputErr)
 	}
 
 	var output *os.File

cmd/gpq/command/convert_test.go

@@ -120,3 +120,17 @@ func (s *Suite) TestConvertUnknownStdinToGeoParquetStdout() {
 
 	s.ErrorContains(cmd.Run(), "when reading from stdin, the --from option must be provided")
 }
+
+func (s *Suite) TestConvertGeoParquetUrlToGeoJSONStdout() {
+	cmd := &command.ConvertCmd{
+		Input: s.server.URL + "/testdata/cases/example-v1.0.0.parquet",
+		To:    "geojson",
+	}
+
+	s.Require().NoError(cmd.Run())
+	data := s.readStdout()
+
+	collection := &geo.FeatureCollection{}
+	s.Require().NoError(json.Unmarshal(data, collection))
+	s.Len(collection.Features, 5)
+}

cmd/gpq/command/describe.go

@@ -15,11 +15,9 @@
 package command
 
 import (
-	"bytes"
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io"
 	"os"
 	"strconv"
 	"strings"
@@ -34,7 +32,7 @@ import (
 )
 
 type DescribeCmd struct {
-	Input        string `arg:"" optional:"" name:"input" help:"Path to a GeoParquet file.  If not provided, input is read from stdin." type:"existingfile"`
+	Input        string `arg:"" optional:"" name:"input" help:"Path or URL for a GeoParquet file.  If not provided, input is read from stdin."`
 	Format       string `help:"Report format.  Possible values: ${enum}." enum:"text, json" default:"text"`
 	MetadataOnly bool   `help:"Print the unformatted geo metadata only (other arguments will be ignored)."`
 	Unpretty     bool   `help:"No newlines or indentation in the JSON output."`
@@ -53,20 +51,9 @@ const (
 )
 
 func (c *DescribeCmd) Run() error {
-	var input ReaderAtSeeker
-	if c.Input == "" {
-		data, err := io.ReadAll(os.Stdin)
-		if err != nil {
-			return NewCommandError("trouble reading from stdin: %w", err)
-		}
-		input = bytes.NewReader(data)
-	} else {
-		i, readErr := os.Open(c.Input)
-		if readErr != nil {
-			return NewCommandError("failed to read from %q: %w", c.Input, readErr)
-		}
-		defer i.Close()
-		input = i
+	input, inputErr := readerFromInput(c.Input)
+	if inputErr != nil {
+		return NewCommandError("trouble getting a reader from %q: %w", c.Input, inputErr)
 	}
 
 	fileReader, fileErr := file.NewParquetReader(input)

cmd/gpq/command/describe_test.go

@@ -172,3 +172,28 @@ func (s *Suite) TestDescribeMissingMetadata() {
 	s.Require().Len(info.Issues, 1)
 	s.Contains(info.Issues[0], "Not a valid GeoParquet file (missing the \"geo\" metadata key).")
 }
+
+func (s *Suite) TestDescribeFromUrl() {
+	cmd := &command.DescribeCmd{
+		Format: "json",
+		Input:  s.server.URL + "/testdata/cases/example-v1.0.0.parquet",
+	}
+
+	s.Require().NoError(cmd.Run())
+
+	output := s.readStdout()
+	info := &command.DescribeInfo{}
+	err := json.Unmarshal(output, info)
+	s.Require().NoError(err)
+
+	s.Equal(int64(5), info.NumRows)
+	s.Equal(int64(1), info.NumRowGroups)
+	s.Require().Len(info.Schema.Fields, 6)
+
+	s.Equal("geometry", info.Schema.Fields[0].Name)
+	s.Equal("binary", info.Schema.Fields[0].Type)
+	s.Equal("gzip", info.Schema.Fields[0].Compression)
+	s.True(info.Schema.Fields[0].Optional)
+
+	s.Len(info.Issues, 0)
+}

cmd/gpq/command/validate.go

@@ -15,11 +15,9 @@
 package command
 
 import (
-	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
-	"io"
 	"os"
 	"strings"
 
@@ -29,34 +27,22 @@ import (
 )
 
 type ValidateCmd struct {
-	Input        string `arg:"" optional:"" name:"input" help:"Path to a GeoParquet file.  If not provided, input is read from stdin." type:"existingfile"`
+	Input        string `arg:"" optional:"" name:"input" help:"Path or URL for a GeoParquet file.  If not provided, input is read from stdin."`
 	MetadataOnly bool   `help:"Only run rules that apply to file metadata and schema (no data will be scanned)."`
 	Unpretty     bool   `help:"No colors in text output, no newlines and indentation in JSON output."`
 	Format       string `help:"Report format.  Possible values: ${enum}." enum:"text, json" default:"text"`
 }
 
 func (c *ValidateCmd) Run(ctx *kong.Context) error {
+	input, inputErr := readerFromInput(c.Input)
+	if inputErr != nil {
+		return NewCommandError("trouble getting a reader from %q: %w", c.Input, inputErr)
+	}
+
 	inputName := c.Input
-	var input ReaderAtSeeker
-	if c.Input == "" {
-		if !hasStdin() {
-			return NewCommandError("input argument must be provided if there is no stdin data")
-		}
-		data, err := io.ReadAll(os.Stdin)
-		if err != nil {
-			return NewCommandError("trouble reading from stdin: %w", err)
-		}
-		input = bytes.NewReader(data)
+	if inputName == "" {
 		inputName = "<stdin>"
-	} else {
-		i, readErr := os.Open(c.Input)
-		if readErr != nil {
-			return NewCommandError("failed to read from %q: %w", c.Input, readErr)
-		}
-		defer i.Close()
-		input = i
 	}
-
 	v := validator.New(c.MetadataOnly)
 	report, err := v.Validate(context.Background(), input, inputName)
 	if err != nil {