Discovery downloads JSON schemas (#174)

* Discovery downloads JSON schemas

* Configurable list of URLs

Author: mikhailshilkov

Makefile

@@ -29,7 +29,7 @@ update_submodules:: init_submodules
 	done
 
 discovery:: update_submodules codegen
-	$(WORKING_DIR)/bin/$(CODEGEN) discovery $(CFN_SCHEMA_DIR) ${VERSION}
+	$(WORKING_DIR)/bin/$(CODEGEN) discovery $(CFN_SCHEMA_DIR) ${VERSION} https://schema.cloudformation.us-east-1.amazonaws.com/CloudformationSchema.zip,https://schema.cloudformation.us-west-2.amazonaws.com/CloudformationSchema.zip
 
 ensure:: init_submodules
 	@echo "GO111MODULE=on go mod tidy"

provider/cmd/pulumi-gen-aws-native/main.go

@@ -3,13 +3,16 @@
 package main
 
 import (
+	"archive/zip"
 	"bytes"
 	"compress/gzip"
 	ctx "context"
 	"encoding/json"
 	"flag"
 	"fmt"
+	"io"
 	"io/ioutil"
+	"net/http"
 	"os"
 	"path/filepath"
 	"sort"
@@ -32,7 +35,7 @@ import (
 
 func main() {
 	flag.Usage = func() {
-		const usageFormat = "Usage: %s <languages> <schema-file> <root-dir>"
+		const usageFormat = "Usage: %s <languages> <schema-folder> <version> (<schema urls>)"
 		_, err := fmt.Fprintf(flag.CommandLine.Output(), usageFormat, os.Args[0])
 		contract.IgnoreError(err)
 		flag.PrintDefaults()
@@ -52,9 +55,18 @@ func main() {
 	genDir := filepath.Join(".", "provider", "cmd", "pulumi-gen-aws-native")
 
 	if languages == "discovery" {
+		if len(args) < 4 {
+			flag.Usage()
+			return
+		}
+		jsonSchemaUrls := strings.Split(args[3], ",")
+
 		if err := writeSupportedResourceTypes(genDir); err != nil {
 			panic(err)
 		}
+		if err := downloadCloudFormationSchemas(jsonSchemaUrls, filepath.Join(".", schemaFolder)); err != nil {
+			panic(err)
+		}
 		return
 	}
 
@@ -181,6 +193,51 @@ func writeSupportedResourceTypes(outDir string) error {
 	return emitFile(outDir, supportedResourcesFile, []byte(val))
 }
 
+func downloadCloudFormationSchemas(urls []string, outDir string) error {
+	for _, url := range urls {
+		resp, err := http.Get(url)
+		if err != nil {
+			return err
+		}
+		defer resp.Body.Close()
+
+		body, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return err
+		}
+
+		zipReader, err := zip.NewReader(bytes.NewReader(body), int64(len(body)))
+		if err != nil {
+			return err
+		}
+
+		// Read all the files from zip archive
+		for _, f := range zipReader.File {
+			outPath := filepath.Join(outDir, f.Name)
+
+			outFile, err := os.OpenFile(outPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
+			if err != nil {
+				return err
+			}
+
+			rc, err := f.Open()
+			if err != nil {
+				return err
+			}
+
+			_, err = io.Copy(outFile, rc)
+			if err != nil {
+				return err
+			}
+
+			outFile.Close()
+			rc.Close()
+		}
+	}
+
+	return nil
+}
+
 func generate(ppkg *pschema.Package, language string) (map[string][]byte, error) {
 	toolDescription := "the Pulumi SDK Generator"
 	extraFiles := map[string][]byte{}