Skip to content

Commit 300620e

Browse files
markusthoemmesjdolitsky
markusthoemmes
and
jdolitsky
authored
Bring back the FetchPackage function on APK (#2049)
Turns out, there are a couple of downstream consumers of this that we don't want to break right now. --------- Signed-off-by: Josh Dolitsky <josh@dolit.ski> Co-authored-by: Josh Dolitsky <josh@dolit.ski>
1 parent aee88fb commit 300620e

File tree

2 files changed

+291
-0
lines changed

2 files changed

+291
-0
lines changed

‎pkg/apk/apk/implementation.go‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1146,6 +1146,23 @@ func packageAsURL(pkg LocatablePackage) (*url.URL, error) {
11461146
return url.Parse(string(asURI))
11471147
}
11481148

1149+
// FetchPackage fetches the given package and returns a ReadCloser for its contents.
1150+
// This is only kept for backwards compatibility, prefer using packageGetter.GetPackage instead.
1151+
func (a *APK) FetchPackage(ctx context.Context, pkg FetchablePackage) (io.ReadCloser, error) {
1152+
// To keep existing behavior, this always uses the default package getter.
1153+
var getterOpts []packageGetterOption
1154+
if a.sizeLimits != nil {
1155+
if a.sizeLimits.APKControlMaxSize != 0 {
1156+
getterOpts = append(getterOpts, withAPKControlMaxSize(a.sizeLimits.APKControlMaxSize))
1157+
}
1158+
if a.sizeLimits.APKDataMaxSize != 0 {
1159+
getterOpts = append(getterOpts, withAPKDataMaxSize(a.sizeLimits.APKDataMaxSize))
1160+
}
1161+
}
1162+
getter := newDefaultPackageGetter(a.client, a.cache, a.auth, getterOpts...)
1163+
return getter.fetchPackage(ctx, pkg)
1164+
}
1165+
11491166
type WriteHeaderer interface {
11501167
WriteHeader(hdr tar.Header, tfs fs.FS, pkg *Package) (bool, error)
11511168
}

‎pkg/apk/apk/implementation_test.go‎

Lines changed: 274 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"io/fs"
2121
"net/http"
2222
"net/http/httptest"
23+
"net/url"
2324
"os"
2425
"path/filepath"
2526
"runtime"
@@ -656,3 +657,276 @@ func TestLoadSystemKeyring(t *testing.T) {
656657
})
657658
}
658659
}
660+
661+
// TestFetchPackage_original is the original TestFetchPackage added back to support FetchPackage method
662+
func TestFetchPackage_original(t *testing.T) {
663+
var (
664+
repo = Repository{URI: fmt.Sprintf("%s/%s", testAlpineRepos, testArch)}
665+
packages = []*Package{&testPkg}
666+
repoWithIndex = repo.WithIndex(&APKIndex{
667+
Packages: packages,
668+
})
669+
testEtag = "testetag"
670+
pkg = NewRepositoryPackage(&testPkg, repoWithIndex)
671+
ctx = context.Background()
672+
)
673+
prepLayout := func(t *testing.T, tr http.RoundTripper, cache string) *APK {
674+
src := apkfs.NewMemFS()
675+
err := src.MkdirAll("usr/lib/apk/db", 0o755)
676+
require.NoError(t, err, "unable to mkdir /usr/lib/apk/db")
677+
678+
opts := []Option{WithFS(src), WithIgnoreMknodErrors(ignoreMknodErrors), WithTransport(tr)}
679+
if cache != "" {
680+
opts = append(opts, WithCache(cache, false, NewCache(false)))
681+
}
682+
a, err := New(ctx, opts...)
683+
require.NoError(t, err, "unable to create APK")
684+
err = a.InitDB(ctx)
685+
require.NoError(t, err)
686+
687+
// set a client so we use local testdata instead of heading out to the Internet each time
688+
return a
689+
}
690+
t.Run("no cache", func(t *testing.T) {
691+
a := prepLayout(t, &testLocalTransport{root: testPrimaryPkgDir, basenameOnly: true}, "")
692+
_, err := a.FetchPackage(ctx, pkg)
693+
require.NoErrorf(t, err, "unable to install package")
694+
})
695+
t.Run("cache miss no network", func(t *testing.T) {
696+
// we use a transport that always returns a 404 so we know we're not hitting the network
697+
// it should fail for a cache hit
698+
tmpDir := t.TempDir()
699+
a := prepLayout(t, &testLocalTransport{fail: true}, tmpDir)
700+
_, err := a.FetchPackage(ctx, pkg)
701+
require.Error(t, err, "should fail when no cache and no network")
702+
})
703+
/*
704+
These tests commented out since a.expandPackage is no longer available
705+
t.Run("cache miss network should fill cache", func(t *testing.T) {
706+
tmpDir := t.TempDir()
707+
a := prepLayout(t, &testLocalTransport{root: testPrimaryPkgDir, basenameOnly: true}, tmpDir)
708+
// fill the cache
709+
repoDir := filepath.Join(tmpDir, url.QueryEscape(testAlpineRepos), testArch)
710+
err := os.MkdirAll(repoDir, 0o755)
711+
require.NoError(t, err, "unable to mkdir cache")
712+
713+
cacheApkFile := filepath.Join(repoDir, testPkgFilename)
714+
cacheApkDir := strings.TrimSuffix(cacheApkFile, ".apk")
715+
716+
_, err = a.expandPackage(ctx, pkg)
717+
require.NoErrorf(t, err, "unable to install pkg")
718+
// check that the package file is in place
719+
_, err = os.Stat(cacheApkDir)
720+
require.NoError(t, err, "apk file not found in cache")
721+
// check that the contents are the same
722+
exp, err := a.cachedPackage(ctx, pkg, cacheApkDir)
723+
if err != nil {
724+
t.Logf("did not find cachedPackage(%q) in %s: %v", pkg.Name, cacheApkDir, err)
725+
files, err := os.ReadDir(cacheApkDir)
726+
require.NoError(t, err, "listing "+cacheApkDir)
727+
for _, f := range files {
728+
t.Logf(" found %q", f.Name())
729+
}
730+
}
731+
require.NoError(t, err, "unable to read cache apk file")
732+
f, err := exp.APK()
733+
require.NoError(t, err, "unable to read cached files as apk")
734+
defer f.Close()
735+
736+
apk1, err := io.ReadAll(f)
737+
require.NoError(t, err, "unable to read cached apk bytes")
738+
739+
apk2, err := os.ReadFile(filepath.Join(testPrimaryPkgDir, testPkgFilename))
740+
require.NoError(t, err, "unable to read previous apk file")
741+
require.Equal(t, apk1, apk2, "apk files do not match")
742+
})
743+
t.Run("handle missing cache files when expanding APK", func(t *testing.T) {
744+
tmpDir := t.TempDir()
745+
a := prepLayout(t, http.DefaultTransport, tmpDir)
746+
747+
// Fill the cache
748+
exp, err := a.expandPackage(ctx, pkg)
749+
require.NoError(t, err, "unable to expand package")
750+
_, err = os.Stat(exp.TarFile)
751+
require.NoError(t, err, "unable to stat cached tar file")
752+
753+
// Delete the tar file from the cache
754+
require.NoError(t, os.Remove(exp.TarFile), "unable to delete cached tar file")
755+
_, err = os.Stat(exp.TarFile)
756+
require.ErrorIs(t, err, os.ErrNotExist, "unexpectedly able to stat cached tar file that should have been deleted")
757+
758+
// Expand the package again, this should re-populate the cache.
759+
exp2, err := a.expandPackage(ctx, pkg)
760+
require.NoError(t, err, "unable to expandPackage after deleting cached tar file")
761+
_, err = os.Stat(exp2.TarFile)
762+
require.NoError(t, err, "unable to stat cached tar file")
763+
764+
// Delete and recreate the tar file from the cache (changing its inodes)
765+
bs, err := os.ReadFile(exp2.TarFile)
766+
require.NoError(t, err, "unable to read cached tar file")
767+
require.NoError(t, os.Remove(exp2.TarFile), "unable to delete cached tar file")
768+
require.NoError(t, os.WriteFile(exp2.TarFile, bs, 0o644), "unable to recreate cached tar file")
769+
770+
// Ensure that the underlying reader is different (i.e. we re-read the file)
771+
exp3, err := a.expandPackage(ctx, pkg)
772+
require.NoError(t, err, "unable to expandPackage after deleting and recreating cached tar file")
773+
require.NotEqual(t, exp2.TarFS.UnderlyingReader(), exp3.TarFS.UnderlyingReader())
774+
775+
// We should be able to read the APK contents
776+
rc, err := exp3.APK()
777+
require.NoError(t, err, "unable to get reader for APK()")
778+
_, err = io.ReadAll(rc)
779+
require.NoError(t, err, "unable to read APK contents")
780+
})
781+
*/
782+
t.Run("cache hit no etag", func(t *testing.T) {
783+
tmpDir := t.TempDir()
784+
a := prepLayout(t,
785+
&testLocalTransport{root: testAlternatePkgDir, basenameOnly: true, headers: map[string][]string{http.CanonicalHeaderKey("etag"): {testEtag}}},
786+
tmpDir)
787+
// fill the cache
788+
repoDir := filepath.Join(tmpDir, url.QueryEscape(testAlpineRepos), testArch)
789+
err := os.MkdirAll(repoDir, 0o755)
790+
require.NoError(t, err, "unable to mkdir cache")
791+
792+
contents, err := os.ReadFile(filepath.Join(testPrimaryPkgDir, testPkgFilename))
793+
require.NoError(t, err, "unable to read apk file")
794+
cacheApkFile := filepath.Join(repoDir, testPkgFilename)
795+
err = os.WriteFile(cacheApkFile, contents, 0o644) //nolint:gosec // we're writing a test file
796+
require.NoError(t, err, "unable to write cache apk file")
797+
798+
_, err = a.FetchPackage(ctx, pkg)
799+
require.NoErrorf(t, err, "unable to install pkg")
800+
// check that the package file is in place
801+
_, err = os.Stat(cacheApkFile)
802+
require.NoError(t, err, "apk file not found in cache")
803+
// check that the contents are the same as the original
804+
apk1, err := os.ReadFile(cacheApkFile)
805+
require.NoError(t, err, "unable to read cache apk file")
806+
require.Equal(t, apk1, contents, "apk files do not match")
807+
})
808+
t.Run("cache hit etag match", func(t *testing.T) {
809+
tmpDir := t.TempDir()
810+
a := prepLayout(t,
811+
&testLocalTransport{root: testAlternatePkgDir, basenameOnly: true, headers: map[string][]string{http.CanonicalHeaderKey("etag"): {testEtag}}},
812+
tmpDir)
813+
// fill the cache
814+
repoDir := filepath.Join(tmpDir, url.QueryEscape(testAlpineRepos), testArch)
815+
err := os.MkdirAll(repoDir, 0o755)
816+
require.NoError(t, err, "unable to mkdir cache")
817+
818+
contents, err := os.ReadFile(filepath.Join(testPrimaryPkgDir, testPkgFilename))
819+
require.NoError(t, err, "unable to read apk file")
820+
cacheApkFile := filepath.Join(repoDir, testPkgFilename)
821+
err = os.WriteFile(cacheApkFile, contents, 0o644) //nolint:gosec // we're writing a test file
822+
require.NoError(t, err, "unable to write cache apk file")
823+
err = os.WriteFile(cacheApkFile+".etag", []byte(testEtag), 0o644) //nolint:gosec // we're writing a test file
824+
require.NoError(t, err, "unable to write etag")
825+
826+
_, err = a.FetchPackage(ctx, pkg)
827+
require.NoErrorf(t, err, "unable to install pkg")
828+
// check that the package file is in place
829+
_, err = os.Stat(cacheApkFile)
830+
require.NoError(t, err, "apk file not found in cache")
831+
// check that the contents are the same as the original
832+
apk1, err := os.ReadFile(cacheApkFile)
833+
require.NoError(t, err, "unable to read cache apk file")
834+
require.Equal(t, apk1, contents, "apk files do not match")
835+
})
836+
t.Run("cache hit etag miss", func(t *testing.T) {
837+
tmpDir := t.TempDir()
838+
a := prepLayout(t,
839+
&testLocalTransport{root: testAlternatePkgDir, basenameOnly: true, headers: map[string][]string{http.CanonicalHeaderKey("etag"): {testEtag + "abcdefg"}}},
840+
tmpDir)
841+
// fill the cache
842+
repoDir := filepath.Join(tmpDir, url.QueryEscape(testAlpineRepos), testArch)
843+
err := os.MkdirAll(repoDir, 0o755)
844+
require.NoError(t, err, "unable to mkdir cache")
845+
846+
contents, err := os.ReadFile(filepath.Join(testPrimaryPkgDir, testPkgFilename))
847+
require.NoError(t, err, "unable to read apk file")
848+
cacheApkFile := filepath.Join(repoDir, testPkgFilename)
849+
err = os.WriteFile(cacheApkFile, contents, 0o644) //nolint:gosec // we're writing a test file
850+
require.NoError(t, err, "unable to write cache apk file")
851+
err = os.WriteFile(cacheApkFile+".etag", []byte(testEtag), 0o644) //nolint:gosec // we're writing a test file
852+
require.NoError(t, err, "unable to write etag")
853+
854+
_, err = a.FetchPackage(ctx, pkg)
855+
require.NoErrorf(t, err, "unable to install pkg")
856+
// check that the package file is in place
857+
_, err = os.Stat(cacheApkFile)
858+
require.NoError(t, err, "apk file not found in cache")
859+
// check that the contents are the same as the original
860+
apk1, err := os.ReadFile(cacheApkFile)
861+
require.NoError(t, err, "unable to read cache apk file")
862+
apk2, err := os.ReadFile(filepath.Join(testAlternatePkgDir, testPkgFilename))
863+
require.NoError(t, err, "unable to read testdata apk file")
864+
require.Equal(t, apk1, apk2, "apk files do not match")
865+
})
866+
}
867+
868+
// TestAuth_good_original is the original TestAuth_good added back to support FetchPackage method
869+
func TestAuth_good_original(t *testing.T) {
870+
called := false
871+
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
872+
called = true
873+
if gotuser, gotpass, ok := r.BasicAuth(); !ok || gotuser != testUser || gotpass != testPass {
874+
w.WriteHeader(http.StatusForbidden)
875+
return
876+
}
877+
http.FileServer(http.Dir(testPrimaryPkgDir)).ServeHTTP(w, r)
878+
}))
879+
defer s.Close()
880+
host := strings.TrimPrefix(s.URL, "http://")
881+
882+
repo := Repository{URI: s.URL}
883+
repoWithIndex := repo.WithIndex(&APKIndex{Packages: []*Package{&testPkg}})
884+
pkg := NewRepositoryPackage(&testPkg, repoWithIndex)
885+
ctx := context.Background()
886+
887+
src := apkfs.NewMemFS()
888+
err := src.MkdirAll("usr/lib/apk/db", 0o755)
889+
require.NoError(t, err, "unable to mkdir /usr/lib/apk/db")
890+
891+
a, err := New(ctx, WithFS(src), WithAuthenticator(auth.StaticAuth(host, testUser, testPass)))
892+
require.NoError(t, err, "unable to create APK")
893+
err = a.InitDB(ctx)
894+
require.NoError(t, err)
895+
896+
_, err = a.FetchPackage(ctx, pkg)
897+
require.NoErrorf(t, err, "unable to install package")
898+
require.True(t, called, "did not make request")
899+
}
900+
901+
// TestAuth_bad_original is the original TestAuth_bad added back to support FetchPackage method
902+
func TestAuth_bad_original(t *testing.T) {
903+
called := false
904+
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
905+
called = true
906+
if gotuser, gotpass, ok := r.BasicAuth(); !ok || gotuser != testUser || gotpass != testPass {
907+
w.WriteHeader(http.StatusForbidden)
908+
return
909+
}
910+
http.FileServer(http.Dir(testPrimaryPkgDir)).ServeHTTP(w, r)
911+
}))
912+
defer s.Close()
913+
host := strings.TrimPrefix(s.URL, "http://")
914+
915+
repo := Repository{URI: s.URL}
916+
repoWithIndex := repo.WithIndex(&APKIndex{Packages: []*Package{&testPkg}})
917+
pkg := NewRepositoryPackage(&testPkg, repoWithIndex)
918+
ctx := context.Background()
919+
920+
src := apkfs.NewMemFS()
921+
err := src.MkdirAll("usr/lib/apk/db", 0o755)
922+
require.NoError(t, err, "unable to mkdir /usr/lib/apk/db")
923+
924+
a, err := New(ctx, WithFS(src), WithAuthenticator(auth.StaticAuth(host, "baduser", "badpass")))
925+
require.NoError(t, err, "unable to create APK")
926+
err = a.InitDB(ctx)
927+
require.NoError(t, err)
928+
929+
_, err = a.FetchPackage(ctx, pkg)
930+
require.Error(t, err, "should fail with bad auth")
931+
require.True(t, called, "did not make request")
932+
}

0 commit comments

Comments
 (0)