Merge branch 'hashicorp:main' into shallow-tag-clone

Author: zachwhaley

.circleci/config.yml

@@ -15,19 +15,22 @@ commands:
         type: string
       platform:
         type: string
+      govet:
+        type: string
+        default: ""
     steps:
       - run:
           name: "Run go tests"
           command: |
             PACKAGE_NAMES=$(go list ./... | circleci tests split --split-by=timings --timings-type=classname)
             echo "Running $(echo $PACKAGE_NAMES | wc -w) packages"
             echo $PACKAGE_NAMES
-            << parameters.cmd >> --format=short-verbose --junitfile $TEST_RESULTS_PATH/go-getter/gotestsum-report.xml -- -p 2 -cover -coverprofile=<< parameters.platform >>_cov_$CIRCLE_NODE_INDEX.part $PACKAGE_NAMES
+            << parameters.cmd >> --format=short-verbose --junitfile $TEST_RESULTS_PATH/go-getter/gotestsum-report.xml -- -p 2 -cover -race -vet=<< parameters.govet >> -coverprofile=<< parameters.platform >>_cov_$CIRCLE_NODE_INDEX.part $PACKAGE_NAMES
         
 jobs:
   linux-tests:
     docker:
-      - image: circleci/golang:<< parameters.go-version >>
+      - image: docker.mirror.hashicorp.services/circleci/golang:<< parameters.go-version >>
     parameters:
       go-version:
         type: string
@@ -140,6 +143,9 @@ jobs:
       - run-gotests:
           cmd: "./gotestsum.exe"
           platform: "win"
+          # Otherwise gcc is required for race detector
+          # See https://github.com/golang/go/issues/27089
+          govet: "off"
 
       # Save coverage report parts
       - persist_to_workspace:

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,50 @@
+name: "Code scanning - scheduled (weekly) or on-demand"
+
+on:
+  schedule:
+    - cron: '0 15 * * 0'
+  workflow_dispatch:
+
+jobs:
+  CodeQL-Build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+      with:
+        # We must fetch at least the immediate parents so that if this is
+        # a pull request then we can checkout the head.
+        fetch-depth: 2
+
+    # If this run was triggered by a pull request event, then checkout
+    # the head of the pull request instead of the merge commit.
+    - run: git checkout HEAD^2
+      if: ${{ github.event_name == 'pull_request' }}
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      # Override language selection by uncommenting this and choosing your languages
+      with:
+        languages: go
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    # - name: Autobuild
+    #  uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/release.yml

@@ -0,0 +1,76 @@
+### This builds, packages, signs, performs AV and malware scanning, and
+### creates a new GitHub release for the newest version of go-getter. 
+### The GitHub release step performs the actions outlined in 
+### release.goreleaser.yml. A release is triggered when a new tag 
+### is pushed in the format vX.X.X
+
+name: Release
+
+on:
+  push:
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+*'
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Setup go
+        uses: actions/setup-go@v2
+        with:
+          go-version: '^1.15'
+      - name: Install hc-codesign
+        id: codesign
+        run: |
+          docker login docker.pkg.github.com -u docker -p $GITHUB_TOKEN && \
+          docker pull docker.pkg.github.com/hashicorp/hc-codesign/hc-codesign:$VERSION && \
+          echo "::set-output name=image::docker.pkg.github.com/hashicorp/hc-codesign/hc-codesign:$VERSION"
+        env:
+          VERSION: v0
+          GITHUB_TOKEN: ${{ secrets.CODESIGN_GITHUB_TOKEN }}
+      - name: Install wget & clamAV antivirus scanner
+        run : |
+           sudo apt-get -qq install -y ca-certificates wget clamav
+           wget --version
+      - name: Install maldet malware scanner
+        run: |
+          wget --no-verbose -O maldet-$VERSION.tar.gz https://github.com/rfxn/linux-malware-detect/archive/$VERSION.tar.gz
+          sha256sum -c - <<< "$SHA256SUM  maldet-$VERSION.tar.gz"
+          sudo mkdir -p maldet-$VERSION
+          sudo tar -xzf maldet-$VERSION.tar.gz --strip-components=1 -C maldet-$VERSION
+          cd maldet-$VERSION
+          sudo ./install.sh
+          sudo maldet -u
+        env: 
+          VERSION: 1.6.4
+          SHA256SUM: 3ad66eebd443d32dd6c811dcf2d264b78678c75ed1d40c15434180d4453e60d2
+      - name: Import PGP key for archive signing
+        run: echo -e $PGP_KEY | base64 -di | gpg --import --batch
+        env:
+          GPG_TTY: $(tty)
+          PGP_KEY: ${{ secrets.PGP_SIGNING_KEY }}
+      - name: GitHub Release
+        uses: goreleaser/goreleaser-action@v1
+        with:
+          version: latest
+          args: release --skip-validate --timeout "60m"
+        env:
+          PGP_KEY_ID: ${{ secrets.PGP_KEY_ID }}
+          CODESIGN_IMAGE: ${{ steps.codesign.outputs.image }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          ARTIFACTORY_TOKEN: ${{ secrets.ARTIFACTORY_TOKEN }}
+          ARTIFACTORY_USER: ${{ secrets.ARTIFACTORY_USER }}
+          CIRCLE_TOKEN: ${{ secrets.CIRCLE_TOKEN }}
+      - name: Run clamAV antivirus scanner
+        run: sudo clamscan /home/runner/work/$REPO/$REPO/dist/
+        env:
+          REPO: ${{ github.event.repository.name }}
+      - name: Run maldet malware scanner
+        run: sudo maldet -a /home/runner/work/$REPO/$REPO/dist/
+        env:
+          REPO: ${{ github.event.repository.name }}
+

.goreleaser.yml

@@ -0,0 +1,50 @@
+env:
+  - GOPRIVATE=github.com/hashicorp
+
+builds:
+  - id: signable
+    mod_timestamp: '{{ .CommitTimestamp }}'
+    targets:
+      - darwin_amd64
+      - windows_386
+      - windows_amd64
+    hooks:
+      post: |
+        docker run
+          -e ARTIFACTORY_TOKEN={{ .Env.ARTIFACTORY_TOKEN }}
+          -e ARTIFACTORY_USER={{ .Env.ARTIFACTORY_USER }}
+          -e CIRCLE_TOKEN={{ .Env.CIRCLE_TOKEN }}
+          -v {{ dir .Path }}:/workdir
+          {{ .Env.CODESIGN_IMAGE }}
+          sign -product-name={{ .ProjectName }} {{ .Name }}
+    dir: ./cmd/go-getter/
+    flags:
+      - -trimpath
+    ldflags:
+      - -X main.GitCommit={{ .Commit }}
+  - mod_timestamp: '{{ .CommitTimestamp }}'
+    targets:
+      - linux_386
+      - linux_amd64
+    dir: ./cmd/go-getter/
+    flags:
+      - -trimpath
+    ldflags:
+      - -X main.GitCommit={{ .Commit }}
+
+archives:
+  - format: zip
+    name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}"
+    files: 
+      - none*
+
+checksum:
+  name_template: '{{ .ProjectName }}_{{ .Version }}_SHA256SUMS'
+  algorithm: sha256
+
+signs:
+  - args: ["-u", "{{ .Env.PGP_KEY_ID }}", "--output", "${signature}", "--detach-sign", "${artifact}"]
+    artifacts: checksum
+
+changelog:
+  skip: true

README.md

@@ -1,9 +1,9 @@
 # go-getter
 
-[![CircleCI](https://circleci.com/gh/hashicorp/go-getter/tree/master.svg?style=svg)][circleci]
+[![CircleCI](https://circleci.com/gh/hashicorp/go-getter/tree/main.svg?style=svg)][circleci]
 [![Go Documentation](http://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)][godocs]
 
-[circleci]: https://circleci.com/gh/hashicorp/go-getter/tree/master
+[circleci]: https://circleci.com/gh/hashicorp/go-getter/tree/main
 [godocs]: http://godoc.org/github.com/hashicorp/go-getter
 
 go-getter is a library for Go (golang) for downloading files or directories
@@ -80,6 +80,8 @@ is built-in by default:
     file URLs.
   * GitHub URLs, such as "github.com/mitchellh/vagrant" are automatically
     changed to Git protocol over HTTP.
+  * GitLab URLs, such as "gitlab.com/inkscape/inkscape" are automatically 
+    changed to Git protocol over HTTP.
   * BitBucket URLs, such as "bitbucket.org/mitchellh/vagrant" are automatically
     changed to a Git or mercurial protocol using the BitBucket API.
 
@@ -316,6 +318,7 @@ are also supported. If the query parameters are present, these take priority.
   * `aws_access_key_id` - AWS access key.
   * `aws_access_key_secret` - AWS access key secret.
   * `aws_access_token` - AWS access token if this is being used.
+  * `aws_profile` - Use this profile from local ~/.aws/ config. Takes priority over the other three.
 
 #### Using IAM Instance Profiles with S3
 

checksum.go

@@ -259,7 +259,10 @@ func (c *Client) ChecksumFromFile(checksumFile string, src *url.URL) (*FileCheck
 				return nil, fmt.Errorf(
 					"Error reading checksum file: %s", err)
 			}
-			break
+			if line == "" {
+				break
+			}
+			// parse the line, if we hit EOF, but the line is not empty
 		}
 		checksum, err := parseChecksumLine(line)
 		if err != nil || checksum == nil {

client.go

@@ -39,6 +39,10 @@ type Client struct {
 	// for documentation.
 	Mode ClientMode
 
+	// Umask is used to mask file permissions when storing local files or decompressing
+	// an archive
+	Umask os.FileMode
+
 	// Detectors is the list of detectors that are tried on the source.
 	// If this is nil, then the default Detectors will be used.
 	Detectors []Detector
@@ -63,9 +67,32 @@ type Client struct {
 	// By default a no op progress listener is used.
 	ProgressListener ProgressTracker
 
+	// Insecure controls whether a client verifies the server's
+	// certificate chain and host name. If Insecure is true, crypto/tls
+	// accepts any certificate presented by the server and any host name in that
+	// certificate. In this mode, TLS is susceptible to machine-in-the-middle
+	// attacks unless custom verification is used. This should be used only for
+	// testing or in combination with VerifyConnection or VerifyPeerCertificate.
+	// This is identical to tls.Config.InsecureSkipVerify.
+	Insecure bool
+
 	Options []ClientOption
 }
 
+// umask returns the effective umask for the Client, defaulting to the process umask
+func (c *Client) umask() os.FileMode {
+	if c == nil {
+		return 0
+	}
+	return c.Umask
+}
+
+// mode returns file mode umasked by the Client umask
+func (c *Client) mode(mode os.FileMode) os.FileMode {
+	m := mode & ^c.umask()
+	return m
+}
+
 // Get downloads the configured source to the destination.
 func (c *Client) Get() error {
 	if err := c.Configure(c.Options...); err != nil {
@@ -233,7 +260,7 @@ func (c *Client) Get() error {
 		if decompressor != nil {
 			// We have a decompressor, so decompress the current destination
 			// into the final destination with the proper mode.
-			err := decompressor.Decompress(decompressDst, dst, decompressDir)
+			err := decompressor.Decompress(decompressDst, dst, decompressDir, c.umask())
 			if err != nil {
 				return err
 			}
@@ -271,7 +298,7 @@ func (c *Client) Get() error {
 		// if we're specifying a subdir.
 		err := g.Get(dst, u)
 		if err != nil {
-			err = fmt.Errorf("error downloading '%s': %s", src, err)
+			err = fmt.Errorf("error downloading '%s': %s", u.Redacted(), err)
 			return err
 		}
 	}
@@ -281,7 +308,7 @@ func (c *Client) Get() error {
 		if err := os.RemoveAll(realDst); err != nil {
 			return err
 		}
-		if err := os.MkdirAll(realDst, 0755); err != nil {
+		if err := os.MkdirAll(realDst, c.mode(0755)); err != nil {
 			return err
 		}
 
@@ -291,7 +318,7 @@ func (c *Client) Get() error {
 			return err
 		}
 
-		return copyDir(c.Ctx, realDst, subDir, false)
+		return copyDir(c.Ctx, realDst, subDir, false, c.umask())
 	}
 
 	return nil

client_option_insecure.go

@@ -0,0 +1,14 @@
+package getter
+
+// WithInsecure allows for a user to avoid
+// checking certificates (not recommended).
+// For example, when connecting on HTTPS where an
+// invalid certificate is presented.
+// User assumes all risk.
+// Not all getters have support for insecure mode yet.
+func WithInsecure() func(*Client) error {
+	return func(c *Client) error {
+		c.Insecure = true
+		return nil
+	}
+}

cmd/go-getter/main.go

@@ -14,6 +14,7 @@ import (
 func main() {
 	modeRaw := flag.String("mode", "any", "get mode (any, file, dir)")
 	progress := flag.Bool("progress", false, "display terminal progress")
+	insecure := flag.Bool("insecure", false, "do not verify server's certificate chain (not recommended)")
 	flag.Parse()
 	args := flag.Args()
 	if len(args) < 2 {
@@ -46,6 +47,11 @@ func main() {
 		opts = append(opts, getter.WithProgress(defaultProgressBar))
 	}
 
+	if *insecure {
+		log.Println("WARNING: Using Insecure TLS transport!")
+		opts = append(opts, getter.WithInsecure())
+	}
+
 	ctx, cancel := context.WithCancel(context.Background())
 	// Build the client
 	client := &getter.Client{