Add allowed repositories list

This in addition to ignored repositories list allows to specify
a list of repositories that will be upgraded. The list should be
placed inside project.yml and look for example like this:

project.repositories.allowed:
    - apache-mynewt-core
    - apache-mynewt-nimble
    - apache-mynewt-mcumgr
    - mcuboot
    - arm-CMSIS_5
    - nordic-nrfx
    - mbedtls

If the same repository will be placed on both lists, it will be
ignored by newt.

IMPORTANT NOTE: Ignored repositories will be ignored even if
they were previously downloaded. So if repo was downloaded
by newt upgrade and then added to ignored list,
targets that depend on this repo won't build.

Author: m-gorecki

newt/deprepo/deprepo.go

@@ -136,6 +136,10 @@ func BuildDepGraph(repos RepoMap, rootReqs RequirementMap) (DepGraph, error) {
 			reqMap := RequirementMap{}
 			for _, d := range deps {
 				depRepo := repos[d.Name]
+				// This might be nil if d.Name is not on allowed repositories list
+				if depRepo == nil {
+					continue
+				}
 				verReqs, err := depRepo.NormalizeVerReq(d.VerReqs)
 				if err != nil {
 					return nil, err

newt/install/install.go

@@ -199,6 +199,10 @@ func (inst *Installer) ensureDepsInList(repos []*repo.Repo,
 		}
 		for _, d := range deps {
 			depRepo := inst.repos[d.Name]
+			// This might be nil if d.Name is not on allowed repositories list
+			if depRepo == nil {
+				continue
+			}
 			result = append(result, recurse(depRepo)...)
 		}
 

newt/project/project.go

@@ -70,7 +70,12 @@ type Project struct {
 	//                read.
 	repos deprepo.RepoMap
 
-	// Contains names of repositories that will not be upgraded/downloaded.
+	// Contains names of repositories that will be upgraded.
+	// If it's empty all repos are allowed.
+	reposAllowed []string
+
+	// Contains names of repositories that will be excluded from upgrade.
+	// Can override repositories from reposAllowed.
 	reposIgnored []string
 
 	// The local repository at the top-level of the project.  This repo is
@@ -176,9 +181,16 @@ func NewProject(dir string, download bool) (*Project, error) {
 	return proj, nil
 }
 
-func (proj *Project) isRepoAdded(r *repo.Repo) bool {
+func (proj *Project) isRepoAllowed(repoName string) bool {
+	if (len(proj.reposAllowed) == 0) || (util.SliceContains(proj.reposAllowed, repoName)) {
+		return !util.SliceContains(proj.reposIgnored, repoName)
+	}
+	return false
+}
+
+func (proj *Project) isRepoAdded(repoName string) bool {
 	for _, pr := range proj.repos {
-		if pr.Name() == r.Name() {
+		if pr.Name() == repoName {
 			return true
 		}
 	}
@@ -191,7 +203,7 @@ func (proj *Project) GetPkgRepos() error {
 			if pkg.PkgConfig().HasKey("repository") {
 				for k, _ := range pkg.PkgConfig().AllSettings() {
 					repoName := strings.TrimPrefix(k, "repository.")
-					if repoName != k && !util.SliceContains(proj.reposIgnored, repoName) {
+					if repoName != k {
 						fields, err := pkg.PkgConfig().GetValStringMapString(k, nil)
 						util.OneTimeWarningError(err)
 
@@ -203,6 +215,10 @@ func (proj *Project) GetPkgRepos() error {
 								return err
 							}
 						}
+						if r == nil {
+							continue
+						}
+
 						verReq, err := newtutil.ParseRepoVersion(fields["vers"])
 						if err != nil {
 							return util.FmtNewtError(
@@ -212,7 +228,7 @@ func (proj *Project) GetPkgRepos() error {
 						}
 						r.SetPkgName(pkg.Name())
 
-						if !proj.isRepoAdded(r) {
+						if !proj.isRepoAdded(r.Name()) {
 							if err := proj.addRepo(r, true); err != nil {
 								return err
 							}
@@ -405,7 +421,7 @@ func (proj *Project) InfoIf(predicate func(r *repo.Repo) bool,
 // @param name                  The name of the repo to read.
 // @param fields                Fields containing the basic repo description.
 //
-// @return *Repo                The fully-read repo on success; nil on failure.
+// @return *Repo                The fully-read repo on success; nil on failure or when repo is not allowed
 // @return error                Error on failure.
 func (proj *Project) loadRepo(name string, fields map[string]string) (
 	*repo.Repo, error) {
@@ -427,12 +443,16 @@ func (proj *Project) loadRepo(name string, fields map[string]string) (
 		return nil, err
 	}
 
+	if !proj.isRepoAllowed(r.Name()) {
+		return nil, nil
+	}
+
 	for _, ignDir := range ignoreSearchDirs {
 		r.AddIgnoreDir(ignDir)
 	}
 
 	// Read the full repo definition from its `repository.yml` file.
-	if err := r.Read(proj.reposIgnored); err != nil {
+	if err := r.Read(); err != nil {
 		return r, err
 	}
 
@@ -500,14 +520,17 @@ func (proj *Project) loadRepoDeps(download bool) error {
 								return nil, err
 							}
 						}
+						if depRepo == nil {
+							continue
+						}
 						if err := proj.addRepo(depRepo, download); err != nil {
 							return nil, err
 						}
 					}
 					newRepos = append(newRepos, depRepo)
 
 					if download {
-						if _, err := depRepo.UpdateDesc(proj.reposIgnored); err != nil {
+						if _, err := depRepo.UpdateDesc(); err != nil {
 							return nil, err
 						}
 					}
@@ -542,7 +565,7 @@ func (proj *Project) downloadRepositoryYmlFiles() error {
 	// specified in the `project.yml` file).
 	for _, r := range proj.repos.Sorted() {
 		if !r.IsLocal() && !r.IsExternal(r.Path()) {
-			if _, err := r.UpdateDesc(proj.reposIgnored); err != nil {
+			if _, err := r.UpdateDesc(); err != nil {
 				return err
 			}
 		}
@@ -631,14 +654,18 @@ func (proj *Project) loadConfig(download bool) error {
 	proj.name, err = yc.GetValString("project.name", nil)
 	util.OneTimeWarningError(err)
 
+	proj.reposAllowed = make([]string, 0)
+	proj.reposAllowed, err = yc.GetValStringSlice("project.repositories.allowed", nil)
+	util.OneTimeWarningError(err)
+
 	proj.reposIgnored = make([]string, 0)
 	proj.reposIgnored, err = yc.GetValStringSlice("project.repositories.ignored", nil)
 	util.OneTimeWarningError(err)
 	proj.reposIgnored = append(proj.reposIgnored, newtutil.NewtIgnore...)
 
-	if util.SliceContains(proj.reposIgnored, "apache-mynewt-core") {
-		return util.NewNewtError("apache-mynewt-core repository can't be ignored. " +
-			"Please remove it from the ignored repositories list.")
+	if !proj.isRepoAllowed("apache-mynewt-core") {
+		return util.NewNewtError("apache-mynewt-core repository must be allowed. " +
+			"Please add it to the allowed list and/or remove it from the ignored list.")
 	}
 
 	// Local repository always included in initialization
@@ -669,6 +696,10 @@ func (proj *Project) loadConfig(download bool) error {
 					return err
 				}
 			}
+			if r == nil {
+				continue
+			}
+
 			verReq, err := newtutil.ParseRepoVersion(fields["vers"])
 			if err != nil {
 				return util.FmtNewtError(

newt/repo/repo.go

@@ -397,7 +397,7 @@ func (r *Repo) Upgrade(ver newtutil.RepoVersion) error {
 // from master.  The repo object is then populated with the contents of the
 // downladed file.  If this repo has already had its descriptor updated, this
 // function is a no-op.
-func (r *Repo) UpdateDesc(reposIgnored []string) (bool, error) {
+func (r *Repo) UpdateDesc() (bool, error) {
 	if r.updated {
 		return false, nil
 	}
@@ -417,7 +417,7 @@ func (r *Repo) UpdateDesc(reposIgnored []string) (bool, error) {
 	}
 
 	// Read `repository.yml` and populate this repo object.
-	if err := r.Read(reposIgnored); err != nil {
+	if err := r.Read(); err != nil {
 		return false, err
 	}
 
@@ -555,14 +555,11 @@ func parseRepoDepMap(depName string,
 	return result, nil
 }
 
-func (r *Repo) readDepRepos(yc ycfg.YCfg, reposIgnored []string) error {
+func (r *Repo) readDepRepos(yc ycfg.YCfg) error {
 	depMap, err := yc.GetValStringMap("repo.deps", nil)
 	util.OneTimeWarningError(err)
 
 	for depName, repoMapYml := range depMap {
-		if util.SliceContains(reposIgnored, depName) {
-			continue
-		}
 		rdm, err := parseRepoDepMap(depName, repoMapYml)
 		if err != nil {
 			return util.FmtNewtError(
@@ -580,7 +577,7 @@ func (r *Repo) readDepRepos(yc ycfg.YCfg, reposIgnored []string) error {
 
 // Reads a `repository.yml` file and populates the receiver repo with its
 // contents.
-func (r *Repo) Read(reposIgnored []string) error {
+func (r *Repo) Read() error {
 	r.Init(r.Name(), r.downloader)
 
 	yc, err := config.ReadFile(r.repoFilePath() + "/" + REPO_FILE_NAME)
@@ -607,7 +604,7 @@ func (r *Repo) Read(reposIgnored []string) error {
 		r.vers[vers] = commit
 	}
 
-	if err := r.readDepRepos(yc, reposIgnored); err != nil {
+	if err := r.readDepRepos(yc); err != nil {
 		return err
 	}