Skip to content
This repository was archived by the owner on Mar 27, 2024. It is now read-only.

Commit 07a2e60

Browse files
cftorrescftorres
authored
Used env vars to search for packages in PYTHONPATH
2 parents 3a83f11 + fa1fd61 commit 07a2e60

File tree

60 files changed

+154
-56
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

60 files changed

+154
-56
lines changed

‎.gofmt.sh‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
21
#!/bin/bash
32

43
# Copyright 2017 Google Inc. All rights reserved.

‎differs/aptDiff.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,8 @@ func (d AptDiffer) Diff(image1, image2 utils.Image) (utils.DiffResult, error) {
1919
return diff, err
2020
}
2121

22-
func (d AptDiffer) getPackages(path string) (map[string]utils.PackageInfo, error) {
22+
func (d AptDiffer) getPackages(image utils.Image) (map[string]utils.PackageInfo, error) {
23+
path := image.FSPath
2324
packages := make(map[string]utils.PackageInfo)
2425
if _, err := os.Stat(path); err != nil {
2526
// invalid image directory path

‎differs/aptDiff_test.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,8 @@ func TestGetAptPackages(t *testing.T) {
105105
}
106106
for _, test := range testCases {
107107
d := AptDiffer{}
108-
packages, err := d.getPackages(test.path)
108+
image := utils.Image{FSPath: test.path}
109+
packages, err := d.getPackages(image)
109110
if err != nil && !test.err {
110111
t.Errorf("Got unexpected error: %s", err)
111112
}

‎differs/nodeDiff.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,8 @@ func buildNodePaths(path string) ([]string, error) {
2727
return []string{globalPaths, localPath}, nil
2828
}
2929

30-
func (d NodeDiffer) getPackages(path string) (map[string]map[string]utils.PackageInfo, error) {
30+
func (d NodeDiffer) getPackages(image utils.Image) (map[string]map[string]utils.PackageInfo, error) {
31+
path := image.FSPath
3132
packages := make(map[string]map[string]utils.PackageInfo)
3233
if _, err := os.Stat(path); err != nil {
3334
// path provided invalid

‎differs/nodeDiff_test.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,8 +44,9 @@ func TestGetNodePackages(t *testing.T) {
4444
}
4545

4646
for _, test := range testCases {
47+
image := utils.Image{FSPath: test.path}
4748
d := NodeDiffer{}
48-
packages, err := d.getPackages(test.path)
49+
packages, err := d.getPackages(image)
4950
if err != nil && !test.err {
5051
t.Errorf("Got unexpected error: %s", err)
5152
}

‎differs/package_differs.go‎

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -7,22 +7,19 @@ import (
77
)
88

99
type MultiVersionPackageDiffer interface {
10-
getPackages(path string) (map[string]map[string]utils.PackageInfo, error)
10+
getPackages(image utils.Image) (map[string]map[string]utils.PackageInfo, error)
1111
}
1212

1313
type SingleVersionPackageDiffer interface {
14-
getPackages(path string) (map[string]utils.PackageInfo, error)
14+
getPackages(image utils.Image) (map[string]utils.PackageInfo, error)
1515
}
1616

1717
func multiVersionDiff(image1, image2 utils.Image, differ MultiVersionPackageDiffer) (utils.DiffResult, error) {
18-
img1FS := image1.FSPath
19-
img2FS := image2.FSPath
20-
21-
pack1, err := differ.getPackages(img1FS)
18+
pack1, err := differ.getPackages(image1)
2219
if err != nil {
2320
return &utils.MultiVersionPackageDiffResult{}, err
2421
}
25-
pack2, err := differ.getPackages(img2FS)
22+
pack2, err := differ.getPackages(image2)
2623
if err != nil {
2724
return &utils.MultiVersionPackageDiffResult{}, err
2825
}
@@ -33,14 +30,11 @@ func multiVersionDiff(image1, image2 utils.Image, differ MultiVersionPackageDiff
3330
}
3431

3532
func singleVersionDiff(image1, image2 utils.Image, differ SingleVersionPackageDiffer) (utils.DiffResult, error) {
36-
img1FS := image1.FSPath
37-
img2FS := image2.FSPath
38-
39-
pack1, err := differ.getPackages(img1FS)
33+
pack1, err := differ.getPackages(image1)
4034
if err != nil {
4135
return &utils.PackageDiffResult{}, err
4236
}
43-
pack2, err := differ.getPackages(img2FS)
37+
pack2, err := differ.getPackages(image2)
4438
if err != nil {
4539
return &utils.PackageDiffResult{}, err
4640
}

‎differs/pipDiff.go‎

Lines changed: 38 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,10 @@ package differs
33
import (
44
"io/ioutil"
55
"path/filepath"
6+
"reflect"
67
"regexp"
78
"strconv"
9+
"strings"
810

911
"github.com/GoogleCloudPlatform/container-diff/utils"
1012
"github.com/golang/glog"
@@ -37,18 +39,43 @@ func getPythonVersion(pathToLayer string) ([]string, error) {
3739
return matches, nil
3840
}
3941

40-
func (d PipDiffer) getPackages(path string) (map[string]map[string]utils.PackageInfo, error) {
41-
packages := make(map[string]map[string]utils.PackageInfo)
42+
func getPythonPaths(vars []string) []string {
43+
paths := []string{}
44+
for _, envVar := range vars {
45+
pythonPathPattern := regexp.MustCompile("^PYTHONPATH=(.*)")
46+
match := pythonPathPattern.FindStringSubmatch(envVar)
47+
if len(match) != 0 {
48+
pythonPath := match[1]
49+
paths = strings.Split(pythonPath, ":")
50+
break
51+
}
52+
}
53+
return paths
54+
}
4255

56+
func (d PipDiffer) getPackages(image utils.Image) (map[string]map[string]utils.PackageInfo, error) {
57+
path := image.FSPath
58+
packages := make(map[string]map[string]utils.PackageInfo)
59+
pythonPaths := []string{}
60+
if !reflect.DeepEqual(utils.ConfigSchema{}, image.Config) {
61+
paths := getPythonPaths(image.Config.Config.Env)
62+
for _, p := range paths {
63+
pythonPaths = append(pythonPaths, p)
64+
}
65+
}
4366
pythonVersions, err := getPythonVersion(path)
4467
if err != nil {
45-
// layer doesn't have a Python version installed
68+
// Image doesn't have Python installed
4669
return packages, nil
4770
}
4871

49-
for _, pyVersion := range pythonVersions {
50-
packagesPath := filepath.Join(path, "usr/local/lib", pyVersion, "site-packages")
51-
contents, err := ioutil.ReadDir(packagesPath)
72+
for _, pythonVersion := range pythonVersions {
73+
packagesPath := filepath.Join(path, "usr/local/lib", pythonVersion, "site-packages")
74+
pythonPaths = append(pythonPaths, packagesPath)
75+
}
76+
77+
for _, pythonPath := range pythonPaths {
78+
contents, err := ioutil.ReadDir(pythonPath)
5279
if err != nil {
5380
// python version folder doesn't have a site-packages folder
5481
continue
@@ -57,7 +84,6 @@ func (d PipDiffer) getPackages(path string) (map[string]map[string]utils.Package
5784
for i := 0; i < len(contents); i++ {
5885
c := contents[i]
5986
fileName := c.Name()
60-
6187
// check if package
6288
packageDir := regexp.MustCompile("^([a-z|A-Z|0-9|_]+)-(([0-9]+?\\.){2,3})dist-info$")
6389
packageMatch := packageDir.FindStringSubmatch(fileName)
@@ -69,7 +95,7 @@ func (d PipDiffer) getPackages(path string) (map[string]map[string]utils.Package
6995
// by taking the file entry alphabetically before it (for a package) or after it (for a script)
7096
var size string
7197
if i-1 >= 0 && contents[i-1].Name() == packageName {
72-
packagePath := filepath.Join(packagesPath, packageName)
98+
packagePath := filepath.Join(pythonPath, packageName)
7399
intSize, err := utils.GetDirectorySize(packagePath)
74100
if err != nil {
75101
glog.Errorf("Could not obtain size for package %s", packagePath)
@@ -85,21 +111,21 @@ func (d PipDiffer) getPackages(path string) (map[string]map[string]utils.Package
85111
continue
86112
}
87113
currPackage := utils.PackageInfo{Version: version, Size: size}
88-
addToMap(packages, packageName, pyVersion, currPackage)
114+
addToMap(packages, packageName, pythonPath, currPackage)
89115
}
90116
}
91117
}
92118

93119
return packages, nil
94120
}
95121

96-
func addToMap(packages map[string]map[string]utils.PackageInfo, pack string, pyVersion string, packInfo utils.PackageInfo) {
122+
func addToMap(packages map[string]map[string]utils.PackageInfo, pack string, path string, packInfo utils.PackageInfo) {
97123
if _, ok := packages[pack]; !ok {
98124
// package not yet seen
99125
infoMap := make(map[string]utils.PackageInfo)
100-
infoMap[pyVersion] = packInfo
126+
infoMap[path] = packInfo
101127
packages[pack] = infoMap
102128
return
103129
}
104-
packages[pack][pyVersion] = packInfo
130+
packages[pack][path] = packInfo
105131
}

‎differs/pipDiff_test.go‎

Lines changed: 62 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -53,46 +53,91 @@ func TestGetPythonVersion(t *testing.T) {
5353
t.Error("Expected error but got none.")
5454
}
5555
if !reflect.DeepEqual(version, test.expectedVersions) {
56-
t.Errorf("Expected: %s. Got: %s", test.expectedVersions, version)
56+
t.Errorf("\nExpected: %s\nGot: %s", test.expectedVersions, version)
5757
}
5858
}
5959
}
6060

6161
func TestGetPythonPackages(t *testing.T) {
6262
testCases := []struct {
63-
path string
63+
descrip string
64+
image utils.Image
6465
expectedPackages map[string]map[string]utils.PackageInfo
6566
}{
6667
{
67-
path: "testDirs/pipTests/noPackagesTest",
68+
descrip: "noPackagesTest",
69+
image: utils.Image{
70+
FSPath: "testDirs/pipTests/noPackagesTest",
71+
},
6872
expectedPackages: map[string]map[string]utils.PackageInfo{},
6973
},
7074
{
71-
path: "testDirs/pipTests/packagesOneLayer",
75+
descrip: "packagesMultiVersion, no PYTHONPATH",
76+
image: utils.Image{
77+
FSPath: "testDirs/pipTests/packagesMultiVersion",
78+
},
79+
expectedPackages: map[string]map[string]utils.PackageInfo{
80+
"packageone": {
81+
"testDirs/pipTests/packagesMultiVersion/usr/local/lib/python3.6/site-packages": {Version: "3.6.9", Size: "0"},
82+
"testDirs/pipTests/packagesMultiVersion/usr/local/lib/python2.7/site-packages": {Version: "0.1.1", Size: "0"},
83+
},
84+
"packagetwo": {"testDirs/pipTests/packagesMultiVersion/usr/local/lib/python3.6/site-packages": {Version: "4.6.2", Size: "0"}},
85+
"script1": {"testDirs/pipTests/packagesMultiVersion/usr/local/lib/python3.6/site-packages": {Version: "1.0", Size: "0"}},
86+
"script2": {"testDirs/pipTests/packagesMultiVersion/usr/local/lib/python3.6/site-packages": {Version: "2.0", Size: "0"}},
87+
"script3": {"testDirs/pipTests/packagesMultiVersion/usr/local/lib/python2.7/site-packages": {Version: "3.0", Size: "0"}},
88+
},
89+
},
90+
{
91+
descrip: "packagesSingleVersion, no PYTHONPATH",
92+
image: utils.Image{
93+
FSPath: "testDirs/pipTests/packagesSingleVersion",
94+
},
7295
expectedPackages: map[string]map[string]utils.PackageInfo{
73-
"packageone": {"python3.6": {Version: "3.6.9", Size: "0"}},
74-
"packagetwo": {"python3.6": {Version: "4.6.2", Size: "0"}},
75-
"script1": {"python3.6": {Version: "1.0", Size: "0"}},
76-
"script2": {"python3.6": {Version: "2.0", Size: "0"}},
96+
"packageone": {"testDirs/pipTests/packagesSingleVersion/usr/local/lib/python3.6/site-packages": {Version: "3.6.9", Size: "0"}},
97+
"packagetwo": {"testDirs/pipTests/packagesSingleVersion/usr/local/lib/python3.6/site-packages": {Version: "4.6.2", Size: "0"}},
98+
"script1": {"testDirs/pipTests/packagesSingleVersion/usr/local/lib/python3.6/site-packages": {Version: "1.0", Size: "0"}},
99+
"script2": {"testDirs/pipTests/packagesSingleVersion/usr/local/lib/python3.6/site-packages": {Version: "2.0", Size: "0"}},
77100
},
78101
},
79102
{
80-
path: "testDirs/pipTests/packagesMultiVersion",
103+
descrip: "pythonPathTests, PYTHONPATH",
104+
image: utils.Image{
105+
FSPath: "testDirs/pipTests/pythonPathTests",
106+
Config: utils.ConfigSchema{
107+
Config: utils.ConfigObject{
108+
Env: []string{"PYTHONPATH=testDirs/pipTests/pythonPathTests/pythonPath1:testDirs/pipTests/pythonPathTests/pythonPath2/subdir", "ENVVAR2=something"},
109+
},
110+
},
111+
},
112+
expectedPackages: map[string]map[string]utils.PackageInfo{
113+
"packageone": {"testDirs/pipTests/pythonPathTests/usr/local/lib/python3.6/site-packages": {Version: "3.6.9", Size: "0"}},
114+
"packagetwo": {"testDirs/pipTests/pythonPathTests/usr/local/lib/python3.6/site-packages": {Version: "4.6.2", Size: "0"}},
115+
"packagefive": {"testDirs/pipTests/pythonPathTests/pythonPath2/subdir": {Version: "3.6.9", Size: "0"}},
116+
"packagesix": {"testDirs/pipTests/pythonPathTests/pythonPath1": {Version: "3.6.9", Size: "0"}},
117+
"packageseven": {"testDirs/pipTests/pythonPathTests/pythonPath1": {Version: "4.6.2", Size: "0"}},
118+
},
119+
},
120+
{
121+
descrip: "pythonPathTests, no PYTHONPATH",
122+
image: utils.Image{
123+
FSPath: "testDirs/pipTests/pythonPathTests",
124+
Config: utils.ConfigSchema{
125+
Config: utils.ConfigObject{
126+
Env: []string{"ENVVAR=something"},
127+
},
128+
},
129+
},
81130
expectedPackages: map[string]map[string]utils.PackageInfo{
82-
"packageone": {"python3.6": {Version: "3.6.9", Size: "0"},
83-
"python2.7": {Version: "0.1.1", Size: "0"}},
84-
"packagetwo": {"python3.6": {Version: "4.6.2", Size: "0"}},
85-
"script1": {"python3.6": {Version: "1.0", Size: "0"}},
86-
"script2": {"python3.6": {Version: "2.0", Size: "0"}},
87-
"script3": {"python2.7": {Version: "3.0", Size: "0"}},
131+
"packageone": {"testDirs/pipTests/pythonPathTests/usr/local/lib/python3.6/site-packages": {Version: "3.6.9", Size: "0"}},
132+
"packagetwo": {"testDirs/pipTests/pythonPathTests/usr/local/lib/python3.6/site-packages": {Version: "4.6.2", Size: "0"}},
88133
},
89134
},
90135
}
91136
for _, test := range testCases {
92137
d := PipDiffer{}
93-
packages, _ := d.getPackages(test.path)
138+
packages, _ := d.getPackages(test.image)
94139
if !reflect.DeepEqual(packages, test.expectedPackages) {
95-
t.Errorf("Expected: %s but got: %s", test.expectedPackages, packages)
140+
t.Errorf("%s\nExpected: %s\nGot: %s", test.descrip, test.expectedPackages, packages)
96141
}
97142
}
98143
}

differs/testDirs/pipTests/noPackagesTest/noPackagesInstalled/usr/local/lib/python3.6/site-packages/file renamed to differs/testDirs/pipTests/noPackagesTest/noPackagesInstalled/layer/usr/local/lib/python3.6/site-packages/file

File renamed without changes.

differs/testDirs/pipTests/noPackagesTest/noSitePackagesFolder/usr/local/lib/python3.6/notSitePackages/file renamed to differs/testDirs/pipTests/noPackagesTest/noSitePackagesFolder/layer/usr/local/lib/python3.6/notSitePackages/file

File renamed without changes.

0 commit comments

Comments
 (0)