Remove the usage of pip-pop library (#468)

* Remove the usage of pip-grep tool

* Add some fixes to address PR comments

* Handle recursive requirements.txt files

Co-authored-by: Andrew Meyer <[email protected]>

Author: brayanhenao

manifest.yml

@@ -53,14 +53,6 @@ dependencies:
   - cflinuxfs3
   source: https://files.pythonhosted.org/packages/88/d9/761f0b1e0551a3559afe4d34bd9bf68fc8de3292363b3775dda39b62ce84/pip-22.0.3.tar.gz
   source_sha256: f29d589df8c8ab99c060e68ad294c4a9ed896624f6368c5349d70aa581b333d0
-- name: pip-pop
-  version: 0.1.5
-  cf_stacks:
-  - cflinuxfs3
-  uri: https://buildpacks.cloudfoundry.org/dependencies/manual-binaries/pip-pop/pip-pop-0.1.5-b32efe86.tar.gz
-  sha256: b32efe865fc6d956e3345690215bc92a7038acf728db228243097b254403048f
-  source: https://github.com/cloudfoundry/pip-pop/archive/v0.1.5.tar.gz
-  source_sha256: 879221acbf123fb24e4a91b3d02e400031f89799bb70316fd97f36d2307a8230
 - name: pipenv
   version: 2022.1.8
   uri: https://buildpacks.cloudfoundry.org/dependencies/pipenv/pipenv_2022.1.8_linux_noarch_cflinuxfs3_119108d6.tgz

src/python/finalize/cli/main.go

@@ -4,10 +4,12 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
+	"time"
+
 	"github.com/cloudfoundry/python-buildpack/src/python/finalize"
 	_ "github.com/cloudfoundry/python-buildpack/src/python/hooks"
 	"github.com/cloudfoundry/python-buildpack/src/python/pyfinder"
-	"time"
+	"github.com/cloudfoundry/python-buildpack/src/python/requirements"
 
 	"github.com/cloudfoundry/libbuildpack"
 )
@@ -55,6 +57,7 @@ func main() {
 		Logfile:        logfile,
 		Command:        &libbuildpack.Command{},
 		ManagePyFinder: pyfinder.ManagePyFinder{},
+		Requirements:   requirements.Reqs{},
 	}
 
 	if err := finalize.Run(&f); err != nil {

src/python/finalize/finalize.go

@@ -10,8 +10,9 @@ import (
 	"regexp"
 	"strings"
 
-	"github.com/cloudfoundry/libbuildpack"
 	"github.com/kr/text"
+
+	"github.com/cloudfoundry/libbuildpack"
 )
 
 type Manifest interface {
@@ -34,13 +35,19 @@ type ManagePyFinder interface {
 	FindManagePy(dir string) (string, error)
 }
 
+type Reqs interface {
+	FindAnyPackage(buildDir string, searchedPackages ...string) (bool, error)
+	FindStalePackages(oldRequirementsPath, newRequirementsPath string, excludedPackages ...string) ([]string, error)
+}
+
 type Finalizer struct {
 	Stager         Stager
 	Log            *libbuildpack.Logger
 	Logfile        *os.File
 	Manifest       Manifest
 	Command        Command
 	ManagePyFinder ManagePyFinder
+	Requirements   Reqs
 }
 
 func Run(f *Finalizer) error {
@@ -67,7 +74,13 @@ func (f *Finalizer) HandleCollectstatic() error {
 	if len(os.Getenv("DISABLE_COLLECTSTATIC")) > 0 {
 		return nil
 	}
-	if err := f.Command.Execute(f.Stager.BuildDir(), os.Stdout, os.Stderr, "pip-grep", "-s", "requirements.txt", "django", "Django"); err != nil {
+
+	exists, err := f.Requirements.FindAnyPackage(f.Stager.BuildDir(), "django", "Django")
+	if err != nil {
+		return err
+	}
+
+	if !exists {
 		return nil
 	}
 

src/python/finalize/finalize_test.go

@@ -33,6 +33,7 @@ var _ = Describe("Finalize", func() {
 		mockManifest       *MockManifest
 		mockCommand        *MockCommand
 		mockManagePyFinder *MockManagePyFinder
+		mockRequirements   *MockReqs
 	)
 
 	BeforeEach(func() {
@@ -53,6 +54,7 @@ var _ = Describe("Finalize", func() {
 		mockManifest = NewMockManifest(mockCtrl)
 		mockCommand = NewMockCommand(mockCtrl)
 		mockManagePyFinder = NewMockManagePyFinder(mockCtrl)
+		mockRequirements = NewMockReqs(mockCtrl)
 
 		args := []string{buildDir, "", depsDir, depsIdx}
 		stager := libbuildpack.NewStager(args, logger, &libbuildpack.Manifest{})
@@ -63,6 +65,7 @@ var _ = Describe("Finalize", func() {
 			Log:            logger,
 			Command:        mockCommand,
 			ManagePyFinder: mockManagePyFinder,
+			Requirements:   mockRequirements,
 		}
 	})
 
@@ -93,7 +96,7 @@ var _ = Describe("Finalize", func() {
 			})
 			Context("app uses Django", func() {
 				BeforeEach(func() {
-					mockCommand.EXPECT().Execute(buildDir, gomock.Any(), gomock.Any(), "pip-grep", "-s", "requirements.txt", "django", "Django").Return(nil)
+					mockRequirements.EXPECT().FindAnyPackage(buildDir, "django", "Django").Return(true, nil)
 					mockManagePyFinder.EXPECT().FindManagePy(buildDir).Return("/foo/bar/manage.py", nil)
 				})
 
@@ -118,7 +121,7 @@ var _ = Describe("Finalize", func() {
 
 			Context("app does not use Django", func() {
 				BeforeEach(func() {
-					mockCommand.EXPECT().Execute(buildDir, gomock.Any(), gomock.Any(), "pip-grep", "-s", "requirements.txt", "django", "Django").Return(fmt.Errorf("Not found"))
+					mockRequirements.EXPECT().FindAnyPackage(buildDir, "django", "Django").Return(false, nil)
 				})
 
 				It("does not run anything", func() {

src/python/finalize/mocks_test.go

@@ -0,0 +1,106 @@
+package requirements
+
+import (
+	"io/ioutil"
+	"path/filepath"
+	"regexp"
+	"strings"
+)
+
+const requirementsParserRegex = `(?m)^[\w\-\w\[\]]+`
+
+type Reqs struct{}
+
+func (m Reqs) FindAnyPackage(buildDir string, searchedPackages ...string) (bool, error) {
+	reqPackages, err := parseRequirementsWithoutVersion(filepath.Join(buildDir, "requirements.txt"))
+	if err != nil {
+		return false, err
+	}
+
+	for _, searchedPackage := range searchedPackages {
+		if containsPackage(reqPackages, searchedPackage) {
+			return true, nil
+		}
+	}
+
+	return false, nil
+}
+
+func (m Reqs) FindStalePackages(oldRequirementsPath, newRequirementsPath string, excludedPackages ...string) ([]string, error) {
+	var stalePackages []string
+
+	oldPkgs, err := parseRequirements(oldRequirementsPath)
+	if err != nil {
+		return nil, err
+	}
+
+	newPkgs, err := parseRequirements(newRequirementsPath)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, oldPkg := range oldPkgs {
+		if !containsPackage(newPkgs, oldPkg) && !packageIsExcluded(excludedPackages, oldPkg) {
+			stalePackages = append(stalePackages, oldPkg)
+		}
+	}
+
+	return stalePackages, nil
+}
+
+func containsPackage(packages []string, searchedPackage string) bool {
+	for _, pkg := range packages {
+		if pkg == searchedPackage {
+			return true
+		}
+	}
+	return false
+}
+
+func packageIsExcluded(excludedPackages []string, packageFullName string) bool {
+
+	regex := regexp.MustCompile(requirementsParserRegex)
+
+	packageWithoutVersion := regex.FindString(packageFullName)
+
+	for _, excludedPackage := range excludedPackages {
+		if packageWithoutVersion == excludedPackage {
+			return true
+		}
+	}
+
+	return false
+}
+
+func parseRequirements(requirementsPath string) ([]string, error) {
+	content, err := ioutil.ReadFile(requirementsPath)
+	if err != nil {
+		return nil, err
+	}
+
+	requirements := strings.Split(string(content), "\n")
+
+	for _, requirement := range requirements {
+		if strings.HasPrefix(requirement, "-r ") {
+			requirement = strings.TrimPrefix(requirement, "-r ")
+			nestedReqs, err := parseRequirements(strings.ReplaceAll(requirementsPath, filepath.Base(requirementsPath), requirement))
+			if err != nil {
+				return nil, err
+			}
+
+			requirements = append(requirements, nestedReqs...)
+		}
+	}
+
+	return requirements, nil
+}
+
+func parseRequirementsWithoutVersion(requirementsPath string) ([]string, error) {
+	parsedRequirements, err := parseRequirements(requirementsPath)
+	if err != nil {
+		return nil, err
+	}
+
+	regex := regexp.MustCompile(`(?m)^[\w\-\w\[\]]+`)
+	return regex.FindAllString(strings.Join(parsedRequirements, "\n"), -1), nil
+}

src/python/requirements/requirements.go

@@ -0,0 +1,13 @@
+package requirements_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestRequirements(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Reqs Suite")
+}