tests: backward-compatibility with v0.8.2

We recently introduced bad-looking regressions in v0.9.0. These were not
caught by our test suite. This commit adds a new
`backward_compatibility` suite to make sure it doesn't happen again.

Signed-off-by: Albin Kerouanton <[email protected]>

Author: akerouanton

.github/workflows/build.yml

@@ -50,10 +50,16 @@ jobs:
           - macOS-14
           - macOS-13
           - windows-2022
+    env:
+      TEST_BC_BASE_VERSION: 0.8.2
+      TEST_BC_SKIP_VERSION_COMPARE: "true"
     steps:
       -
         name: Checkout
         uses: actions/checkout@v4
+        with:
+          # fetch-tags is required for backward compatibility tests
+          fetch-tags: true
       -
         name: Set up Go
         uses: actions/setup-go@v5
@@ -106,6 +112,70 @@ jobs:
         run: |
           make test COVERAGEDIR=${{ env.DESTDIR }}
         shell: bash
+      -
+        name: Test backward compatibility (pass)
+        if: startsWith(matrix.os, 'ubuntu-') || startsWith(matrix.os, 'macOS-')
+        run: |
+          export TMPDIR=${TMPDIR:-/tmp}
+          export helper=pass
+
+          wget -O ${TMPDIR}/docker-credential-${helper} https://github.com/docker/docker-credential-helpers/releases/download/v${TEST_BC_BASE_VERSION}/docker-credential-${helper}-v${TEST_BC_BASE_VERSION}.$(go env GOOS)-$(go env GOARCH)
+          chmod +x ${TMPDIR}/docker-credential-${helper}
+
+          make build-${helper}
+
+          export TEST_BC_PREVIOUS_COMMAND=${TMPDIR}/docker-credential-${helper}
+          export TEST_BC_COMMAND=$(realpath ${DESTDIR}/docker-credential-${helper})
+
+          make test-backward-compatibility
+        shell: bash
+      -
+        name: Test backward compatibility (secretservice)
+        if: startsWith(matrix.os, 'ubuntu-')
+        run: |
+          export TMPDIR=${TMPDIR:-/tmp}
+          export helper=secretservice
+
+          wget -O ${TMPDIR}/docker-credential-${helper} https://github.com/docker/docker-credential-helpers/releases/download/v${TEST_BC_BASE_VERSION}/docker-credential-${helper}-v${TEST_BC_BASE_VERSION}.$(go env GOOS)-$(go env GOARCH)
+          chmod +x ${TMPDIR}/docker-credential-${helper}
+
+          make build-${helper}
+
+          export TEST_BC_PREVIOUS_COMMAND=${TMPDIR}/docker-credential-${helper}
+          export TEST_BC_COMMAND=$(realpath ${DESTDIR}/docker-credential-${helper})
+
+          make test-backward-compatibility
+        shell: bash
+      -
+        name: Test backward compatibility (osxkeychain)
+        if: startsWith(matrix.os, 'macOS-')
+        run: |
+          export TMPDIR=${TMPDIR:-/tmp}
+          export helper=osxkeychain
+
+          wget -O ${TMPDIR}/docker-credential-${helper} https://github.com/docker/docker-credential-helpers/releases/download/v${TEST_BC_BASE_VERSION}/docker-credential-${helper}-v${TEST_BC_BASE_VERSION}.$(go env GOOS)-$(go env GOARCH)
+          chmod +x ${TMPDIR}/docker-credential-${helper}
+
+          make build-${helper}
+
+          export TEST_BC_PREVIOUS_COMMAND=${TMPDIR}/docker-credential-${helper}
+          export TEST_BC_COMMAND=$(realpath ${DESTDIR}/docker-credential-${helper})
+
+          make test-backward-compatibility
+        shell: bash
+      -
+        name: Test backward compatibility (wincred)
+        if: startsWith(matrix.os, 'windows-')
+        run: |
+          export helper=wincred
+
+          Invoke-WebRequest -OutFile ${TMPDIR}/docker-credential-${helper}.exe https://github.com/docker/docker-credential-helpers/releases/download/v${TEST_BC_BASE_VERSION}/docker-credential-${helper}-v${TEST_BC_BASE_VERSION}.$(go env GOOS)-$(go env GOARCH).exe
+
+          export TEST_BC_PREVIOUS_COMMAND=${TMPDIR}/docker-credential-${helper}.exe
+          export TEST_BC_COMMAND=$(Resolve-Path ${DESTDIR}/docker-credential-${helper}.exe)
+
+          make test-backward-compatibility
+        shell: bash
       -
         name: Upload coverage
         uses: codecov/codecov-action@v5

Makefile

@@ -62,6 +62,10 @@ test:
 	go test -short -v -coverprofile=$(COVERAGEDIR)/coverage.txt -covermode=atomic ./...
 	go tool cover -func=$(COVERAGEDIR)/coverage.txt
 
+.PHONY: test-backward-compatibility
+test-backward-compatibility:
+	go test -short -v ./tests/backward_compatibility
+
 .PHONY: lint
 lint:
 	$(BUILDX_CMD) bake lint

go.mod

@@ -7,6 +7,10 @@ retract v0.9.0 // osxkeychain: a regression caused backward-incompatibility with
 require (
 	github.com/danieljoos/wincred v1.2.2
 	github.com/keybase/go-keychain v0.0.1
+	gotest.tools/v3 v3.5.2
 )
 
-require golang.org/x/sys v0.20.0 // indirect
+require (
+	github.com/google/go-cmp v0.5.9 // indirect
+	golang.org/x/sys v0.20.0 // indirect
+)

go.sum

@@ -2,6 +2,8 @@ github.com/danieljoos/wincred v1.2.2 h1:774zMFJrqaeYCK2W57BgAem/MLi6mtSE47MB6BOJ
 github.com/danieljoos/wincred v1.2.2/go.mod h1:w7w4Utbrz8lqeMbDAK0lkNJUv5sAOkFi7nd/ogr0Uh8=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU=
 github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -14,3 +16,5 @@ golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
 golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q=
+gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA=