Skip to content

[DNM] tests: backward-compatibility with v0.8.2 #365

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
akerouanton wants to merge 1 commit into from
Closed

[DNM] tests: backward-compatibility with v0.8.2 #365

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 71 additions & 0 deletions .github/workflows/build.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,15 @@ jobs:
- macOS-14
- macOS-13
- windows-2022
env:
TEST_BC_BASE_VERSION: 0.8.2
steps:
-
name: Checkout
uses: actions/checkout@v4
with:
# fetch-tags is required for backward compatibility tests
fetch-tags: true
-
name: Set up Go
uses: actions/setup-go@v5
Expand Down Expand Up @@ -106,6 +111,72 @@ jobs:
run: |
make test COVERAGEDIR=${{ env.DESTDIR }}
shell: bash
-
name: Test backward compatibility (pass)
if: always() && (startsWith(matrix.os, 'ubuntu-') || startsWith(matrix.os, 'macOS-'))
run: |
export TMPDIR=${TMPDIR:-/tmp}
export helper=pass

wget -O ${TMPDIR}/docker-credential-${helper} https://github.com/docker/docker-credential-helpers/releases/download/v${TEST_BC_BASE_VERSION}/docker-credential-${helper}-v${TEST_BC_BASE_VERSION}.$(go env GOOS)-$(go env GOARCH)
chmod +x ${TMPDIR}/docker-credential-${helper}

make build-${helper}

export TEST_BC_PREVIOUS_COMMAND=${TMPDIR}/docker-credential-${helper}
export TEST_BC_COMMAND=$(realpath ${DESTDIR}/docker-credential-${helper})

make test-backward-compatibility
shell: bash
-
name: Test backward compatibility (secretservice)
if: always() && startsWith(matrix.os, 'ubuntu-')
run: |
export TMPDIR=${TMPDIR:-/tmp}
export helper=secretservice

wget -O ${TMPDIR}/docker-credential-${helper} https://github.com/docker/docker-credential-helpers/releases/download/v${TEST_BC_BASE_VERSION}/docker-credential-${helper}-v${TEST_BC_BASE_VERSION}.$(go env GOOS)-$(go env GOARCH)
chmod +x ${TMPDIR}/docker-credential-${helper}

make build-${helper}

export TEST_BC_PREVIOUS_COMMAND=${TMPDIR}/docker-credential-${helper}
export TEST_BC_COMMAND=$(realpath ${DESTDIR}/docker-credential-${helper})

make test-backward-compatibility
shell: bash
-
name: Test backward compatibility (osxkeychain)
if: always() && startsWith(matrix.os, 'macOS-')
run: |
export TMPDIR=${TMPDIR:-/tmp}
export helper=osxkeychain

wget -O ${TMPDIR}/docker-credential-${helper} https://github.com/docker/docker-credential-helpers/releases/download/v${TEST_BC_BASE_VERSION}/docker-credential-${helper}-v${TEST_BC_BASE_VERSION}.$(go env GOOS)-$(go env GOARCH)
chmod +x ${TMPDIR}/docker-credential-${helper}

make build-${helper}

export TEST_BC_PREVIOUS_COMMAND=${TMPDIR}/docker-credential-${helper}
export TEST_BC_COMMAND=$(realpath ${DESTDIR}/docker-credential-${helper})

# security -v unlock-keychain ~/Library/Keychains/login.keychain-db
sudo security authorizationdb write com.apple.trust-settings.admin allow

make test-backward-compatibility
shell: bash
-
name: Test backward compatibility (wincred)
if: always() && startsWith(matrix.os, 'windows-')
run: |
export helper=wincred

Invoke-WebRequest -OutFile ${TMPDIR}/docker-credential-${helper}.exe https://github.com/docker/docker-credential-helpers/releases/download/v${TEST_BC_BASE_VERSION}/docker-credential-${helper}-v${TEST_BC_BASE_VERSION}.$(go env GOOS)-$(go env GOARCH).exe

export TEST_BC_PREVIOUS_COMMAND=${TMPDIR}/docker-credential-${helper}.exe
export TEST_BC_COMMAND=$(Resolve-Path ${DESTDIR}/docker-credential-${helper}.exe)

make test-backward-compatibility
-
name: Upload coverage
uses: codecov/codecov-action@v5
Expand Down
4 changes: 4 additions & 0 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,10 @@ test:
go test -short -v -coverprofile=$(COVERAGEDIR)/coverage.txt -covermode=atomic ./...
go tool cover -func=$(COVERAGEDIR)/coverage.txt

.PHONY: test-backward-compatibility
test-backward-compatibility:
go test -short -v ./tests/backward_compatibility

.PHONY: lint
lint:
$(BUILDX_CMD) bake lint
Expand Down
6 changes: 5 additions & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@ retract v0.9.0 // osxkeychain: a regression caused backward-incompatibility with
require (
github.com/danieljoos/wincred v1.2.2
github.com/keybase/go-keychain v0.0.1
gotest.tools/v3 v3.5.2
)

require golang.org/x/sys v0.20.0 // indirect
require (
github.com/google/go-cmp v0.5.9 // indirect
golang.org/x/sys v0.20.0 // indirect
)
4 changes: 4 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ github.com/danieljoos/wincred v1.2.2 h1:774zMFJrqaeYCK2W57BgAem/MLi6mtSE47MB6BOJ
github.com/danieljoos/wincred v1.2.2/go.mod h1:w7w4Utbrz8lqeMbDAK0lkNJUv5sAOkFi7nd/ogr0Uh8=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU=
github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
Expand All @@ -14,3 +16,5 @@ golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q=
gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA=
249 changes: 249 additions & 0 deletions tests/backward_compatibility/backward_compatibility_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,249 @@
package client

import (
"fmt"
"os"
"strings"
"testing"

"github.com/docker/docker-credential-helpers/client"
"github.com/docker/docker-credential-helpers/credentials"
"gotest.tools/v3/assert"
is "gotest.tools/v3/assert/cmp"
"gotest.tools/v3/skip"
)

func newShellCommands(t *testing.T) (string, client.ProgramFunc, client.ProgramFunc) {
cmd := os.Getenv("TEST_BC_COMMAND")
previousCmd := os.Getenv("TEST_BC_PREVIOUS_COMMAND")
skip.If(t, cmd == "", "TEST_BC_COMMAND not set, skipping backward compatibility test")
skip.If(t, previousCmd == "", "TEST_BC_PREVIOUS_COMMAND not set, skipping backward compatibility test")

oldP := client.NewShellProgramFunc(previousCmd)
newP := client.NewShellProgramFunc(cmd)

oldName, oldErr := version(oldP)
newName, newErr := version(newP)

assert.NilError(t, oldErr)
assert.NilError(t, newErr)
assert.Equal(t, oldName, newName)

return oldName, oldP, newP
}

func version(program client.ProgramFunc) (string, error) {
cmd := program(credentials.ActionVersion)
out, err := cmd.Output()
t := strings.TrimSpace(string(out))
if err != nil {
return "", fmt.Errorf("error getting version - err: %v, out: `%s`", err, t)
}
parts := strings.Split(t, " ")
return parts[0], nil
}

// TestGetFromNewVersion tests that a new version of the helper can read
// credentials stored by an older version of the helper.
func TestGetFromNewVersion(t *testing.T) {
helperName, oldP, newP := newShellCommands(t)

skip.If(t, helperName == "docker-credential-secretservice", "test requires gnome-keyring but CI doesn't have it")

testcases := []struct {
name string
serverURL string
}{
{
name: "with no path",
serverURL: "https://registry.example.com/",
},
{
name: "with path",
serverURL: "https://registry.example.com/v1/",
},
{
name: "with port",
serverURL: "https://registry.example.com:5000/",
},
{
name: "with path and port",
serverURL: "https://registry.example.com:5000/v1/",
},
}

for _, tc := range testcases {
t.Run(tc.name, func(t *testing.T) {
err := client.Store(oldP, &credentials.Credentials{
ServerURL: tc.serverURL,
Username: "testuser",
Secret: "testsecret",
})
defer client.Erase(newP, tc.serverURL)
assert.NilError(t, err)

creds, err := client.Get(newP, tc.serverURL)
assert.NilError(t, err)

assert.Check(t, is.Equal(creds.Username, "testuser"))
assert.Check(t, is.Equal(creds.Secret, "testsecret"))
})
}
}

// TestListFromNewVersion tests that a new version of the helper can list
// credentials stored by an older version of the helper.
func TestListFromNewVersion(t *testing.T) {
helperName, oldP, newP := newShellCommands(t)

skip.If(t, helperName == "docker-credential-secretservice", "test requires gnome-keyring but CI doesn't have it")

priorCreds, err := client.List(newP)
assert.NilError(t, err)
// Capture the number of credentials before we store any new ones. When
// tests are run on a developer's machine, they may have credentials
// already stored in the helper, and we don't want to count those
// against the helper's ability to list credentials.
priorList := len(priorCreds)

testCreds := []struct {
credentials.Credentials
skip bool
skipReason string
}{
{
Credentials: credentials.Credentials{
ServerURL: "https://registry.example.com/",
Username: "testuser1",
Secret: "testsecret1",
},
},
{
Credentials: credentials.Credentials{
ServerURL: "https://registry.example.com/v1/",
Username: "testuser1",
Secret: "testsecret1",
},
},
{
Credentials: credentials.Credentials{
ServerURL: "https://registry.example.com/v2/",
Username: "testuser2",
Secret: "testsecret2",
},
},
{
Credentials: credentials.Credentials{
ServerURL: "https://registry.example.com:5000",
Username: "testuser1",
Secret: "testsecret1",
},
},
{
Credentials: credentials.Credentials{
ServerURL: "https://registry.example.com:5000/v1/",
Username: "testuser1",
Secret: "testsecret1",
},
skip: helperName == "docker-credential-osxkeychain",
skipReason: "docker-credential-osxkeychain returns malformed URI when a port is specified",
},
}
for i := range testCreds {
creds := testCreds[i]

err = client.Store(oldP, &creds.Credentials)
defer client.Erase(oldP, creds.ServerURL)
assert.NilError(t, err)
}

oldCreds, err := client.List(oldP)
assert.NilError(t, err)
t.Logf("credentials found by old version: %+v", oldCreds)
assert.Check(t, is.Equal(len(oldCreds), priorList+len(testCreds)))

newCreds, err := client.List(newP)
assert.NilError(t, err)
t.Logf("credentials found by new version: %+v", newCreds)
assert.Check(t, is.Equal(len(newCreds), priorList+len(testCreds)))

for _, tc := range testCreds {
t.Run(tc.ServerURL, func(t *testing.T) {
skip.If(t, tc.skip, tc.skipReason)
if _, ok := oldCreds[tc.ServerURL]; !ok {
t.Errorf("ServerURL=%q: no credentials stored in oldCreds, want one", tc.ServerURL)
return
}
if _, ok := newCreds[tc.ServerURL]; !ok {
t.Errorf("ServerURL=%q: no credentials stored in newCreds, want one", tc.ServerURL)
return
}
assert.Check(t, is.Equal(newCreds[tc.ServerURL], tc.Username), "ServerURL=%q: got username=%q, want username %q", tc.ServerURL, newCreds[tc.ServerURL], tc.Username)
assert.Check(t, is.Equal(newCreds[tc.ServerURL], oldCreds[tc.ServerURL]), "ServerURL=%q: got username=%q, want username %q", tc.ServerURL, newCreds[tc.ServerURL], tc.Username)
})
}
}

// TestReplaceFromNewVersion tests that a new version of the helper can
// credentials stored by an older version of the helper, and that both the old
// and new helpers can read the credentials.
func TestReplaceFromNewVersion(t *testing.T) {
helperName, oldP, newP := newShellCommands(t)

skip.If(t, helperName == "docker-credential-secretservice", "test requires gnome-keyring but CI doesn't have it")

const serverURL = "https://registry.example.com/"
err := client.Store(oldP, &credentials.Credentials{
ServerURL: serverURL,
Username: "testuser1",
Secret: "testsecret1",
})
// defer client.Erase(oldP, serverURL)
assert.NilError(t, err)

err = client.Store(newP, &credentials.Credentials{
ServerURL: serverURL,
Username: "testuser2",
Secret: "testsecret2",
})
// defer client.Erase(newP, serverURL)
assert.NilError(t, err)

creds, err := client.Get(oldP, serverURL)
assert.NilError(t, err)
assert.Check(t, is.Equal(creds.Username, "testuser2"))
assert.Check(t, is.Equal(creds.Secret, "testsecret2"))

creds, err = client.Get(newP, serverURL)
assert.NilError(t, err)
assert.Check(t, is.Equal(creds.Username, "testuser2"))
assert.Check(t, is.Equal(creds.Secret, "testsecret2"))
}

// TestEraseFromNewVersion tests that a new version of the helper can erase
// credentials stored by an older version of the helper.
func TestEraseFromNewVersion(t *testing.T) {
helperName, oldP, newP := newShellCommands(t)

skip.If(t, helperName == "docker-credential-secretservice", "test requires gnome-keyring but CI doesn't have it")

const serverURL = "https://registry.example.com/"
err := client.Store(oldP, &credentials.Credentials{
ServerURL: serverURL,
Username: "testuser1",
Secret: "testsecret1",
})
defer client.Erase(oldP, serverURL)
assert.NilError(t, err)

err = client.Erase(newP, serverURL)
assert.NilError(t, err)

creds, err := client.Get(oldP, serverURL)
assert.ErrorIs(t, err, credentials.NewErrCredentialsNotFound(), "expected err %q; got err = nil, creds = %+v", credentials.NewErrCredentialsNotFound(), creds)
assert.Check(t, is.Nil(creds))

creds, err = client.Get(newP, serverURL)
assert.ErrorIs(t, err, credentials.NewErrCredentialsNotFound(), "expected err %q; got err = nil, creds = %+v", credentials.NewErrCredentialsNotFound(), creds)
assert.Check(t, is.Nil(creds))
}
Loading
Loading