Merge pull request #649 from jkloetzke/ls-recipes

Add new 'ls-recipes' command

Author: jkloetzke

contrib/bash-completion/bob

@@ -30,7 +30,8 @@ __bob_complete_dir()
 }
 
 __bob_commands="build dev clean graph help init jenkins ls project status \
-                query-scm query-recipe query-path query-meta show layers"
+                query-scm query-recipe query-path query-meta show layers \
+                ls-recipes"
 
 # Complete a Bob path
 #
@@ -433,6 +434,12 @@ __bob_show()
     fi
 }
 
+__bob_ls_recipes()
+{
+    __bob_complete_path "-c -D --sandbox --no-sandbox --slim-sandbox --dev-sandbox --strict-sandbox
+        --all --used --orphaned --sources"
+}
+
 __bob_subcommands()
 {
    local i c command completion_func

doc/conf.py

@@ -269,6 +269,7 @@ def __getattr__(cls, name):
     ('manpages/bob-jenkins', 'bob-jenkins', 'Configure Jenkins server', ['Jan Klötzke'], 1),
     ('manpages/bob-layers', 'bob-layers', 'Manage Layers', ['BobBuildTool Team'], 1),
     ('manpages/bob-ls', 'bob-ls', 'List package hierarchy', ['Jan Klötzke'], 1),
+    ('manpages/bob-ls-recipes', 'bob-ls-recipes', 'List all known recipes', ['Jan Klötzke'], 1),
     ('manpages/bobpaths', 'bobpaths', 'Specifying paths to Bob packages', ['Jan Klötzke'], 7),
     ('manpages/bob-project', 'bob-project', 'Create IDE project files', ['Jan Klötzke'], 1),
     ('manpages/bob-query-meta', 'bob-query-meta', 'Query metaEnvironment variables', ['Ralf Hubert'], 1),

doc/manpages/bob-ls-recipes.rst

@@ -0,0 +1,75 @@
+.. _manpage-ls-recipes:
+
+bob-ls-recipes
+==============
+
+.. only:: not man
+
+   Name
+   ----
+
+   bob-ls-recipes - List all known recipes
+
+Synopsis
+--------
+
+::
+
+    bob ls-recipes [-h] [-D DEFINES] [-c CONFIGFILE]
+                   [--sandbox | --slim-sandbox | --dev-sandbox | --strict-sandbox | --no-sandbox]
+                   [--all | --used | --orphaned] [--sources]
+
+
+Description
+-----------
+
+List known recipes. In contrast to the :ref:`manpage-bob-ls` command, which works
+on packages, this command works on recipes.
+
+By default, all found recipes are printed (``--all``). Because the recipe YAML
+files can declare more than one recipe (see
+:ref:`configuration-recipes-multipackage`), the number of listed recipes is
+usually bigger than the number of YAML files. Add the ``--sources`` option too
+see which file declared what recipe.
+
+Note that listing used or orphaned recipes very much depends on the project
+configuration. Especially when using layers, there may be a large number of
+unused recipes.
+
+Options
+-------
+
+``--all``
+    List all recipes (default).
+
+``-c CONFIGFILE``
+    Use config File
+
+``--dev-sandbox``
+    Enable development sandboxing.
+
+``-D DEFINES``
+    Override default environment variable
+
+``--no-sandbox``
+   Disable sandboxing.
+
+``--orphaned``
+   List recipes that are unused.
+
+``--sandbox``
+   Enable partial sandboxing.
+
+``--slim-sandbox``
+    Enable slim sandboxing.
+
+``--sources``
+    Print source YAML file names. This includes the recipe and all inherited
+    classes. The file names are separated by TAB character.
+
+``--strict-sandbox``
+    Enable strict sandboxing.
+
+``--used``
+    List all used recipes. These are recipes that are referenced directly or
+    indirectly by a root package.

doc/manpages/index.rst

@@ -16,6 +16,7 @@ Contents:
    bob-jenkins
    bob-layers
    bob-ls
+   bob-ls-recipes
    bobpaths
    bob-project
    bob-query-meta

pym/bob/cmds/misc.py

@@ -340,3 +340,66 @@ def doInit(argv, bobRoot):
             f.write(os.path.abspath(args.project))
     except OSError as e:
         raise ParseError("Cannot create project link: " + str(e))
+
+def doLsRecipes(argv, bobRoot):
+    parser = argparse.ArgumentParser(prog="bob ls-recipes", description="List all known recipes.")
+    parser.add_argument('-D', default=[], action='append', dest="defines",
+        help="Override default environment variable")
+    parser.add_argument('-c', dest="configFile", default=[], action='append',
+        help="Use config File")
+
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument('--sandbox', action='store_true', default=False,
+        help="Enable sandboxing")
+    group.add_argument('--slim-sandbox', action='store_false', dest='sandbox',
+        help="Enable slim sandboxing")
+    group.add_argument('--dev-sandbox', action='store_true', dest='sandbox',
+        help="Enable development sandboxing")
+    group.add_argument('--strict-sandbox', action='store_true', dest='sandbox',
+        help="Enable strict sandboxing")
+    group.add_argument('--no-sandbox', action='store_false', dest='sandbox',
+        help="Disable sandboxing (default)")
+
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument('--all', action='store_const', dest='mode', const='all', default='all',
+                       help="List all recipes (default)")
+    group.add_argument('--used', action='store_const', dest='mode', const='used',
+                       help="List recipes used for packages")
+    group.add_argument('--orphaned', action='store_const', dest='mode', const='orphaned',
+                       help="List unused recipes")
+
+    parser.add_argument('--sources', action='store_true', help="Show recipe source files")
+
+    args = parser.parse_args(argv)
+
+    defines = processDefines(args.defines)
+
+    recipes = RecipeSet()
+    recipes.setConfigFiles(args.configFile)
+    recipes.parse(defines)
+
+    if args.mode in ('used', 'orphaned'):
+        packages = recipes.generatePackages(lambda s,m: "unused", args.sandbox)
+        used = set(p.getName() for p in packages.queryPackagePath("//*"))
+
+    if args.mode in ('all', 'orphaned'):
+        # filter virtual root recipe
+        available = set(r for r in recipes.getRecipes() if r != "")
+
+    if args.sources:
+        def show(p):
+            print(p, *recipes.getRecipe(p).getSources(), sep="\t")
+    else:
+        def show(p):
+            print(p)
+
+    if args.mode == 'all':
+        for i in sorted(available):
+            show(i)
+    elif args.mode == 'used':
+        for i in sorted(used):
+            show(i)
+    else:
+        assert args.mode == 'orphaned'
+        for i in sorted(available - used):
+            show(i)

pym/bob/input.py

@@ -2981,17 +2981,18 @@ def loadFromFile(recipeSet, rootDir, fileName):
 
         alias = recipeSet.loadYaml(fileName, (AliasPackage.SCHEMA, b''))
         if isinstance(alias, str):
-            return [ AliasPackage(recipeSet, alias, baseName) ]
+            return [ AliasPackage(recipeSet, alias, baseName, fileName) ]
         else:
             return [
-                AliasPackage(recipeSet, subAlias, baseName + ("-"+subName if subName else ""))
+                AliasPackage(recipeSet, subAlias, baseName + ("-"+subName if subName else ""), fileName)
                 for (subName, subAlias) in alias["multiPackage"].items()
             ]
 
-    def __init__(self, recipeSet, target, packageName):
+    def __init__(self, recipeSet, target, packageName, fileName):
         self.__recipeSet = recipeSet
         self.__target = target
         self.__packageName = packageName
+        self.__source = fileName
 
     def prepare(self, env, sandboxEnabled, states, sandbox, tools, stack, packageName=None):
         target = env.substitute(self.__target, "alias package")
@@ -3010,6 +3011,12 @@ def resolveClasses(self, env):
     def isRoot(self):
         return False
 
+    def getPrimarySource(self):
+        return self.__source
+
+    def getSources(self):
+        return [self.__source]
+
 
 class PackageMatcher:
     __slots__ = ( 'corePackage', 'env', 'tools', 'states', 'sandbox',
@@ -4115,6 +4122,9 @@ def __createSchemas(self):
         self.__userConfigSchema = (schema.Schema(userConfigSchemaSpec), self.__pluginSettingsDeps)
 
 
+    def getRecipes(self):
+        return self.__recipes.keys()
+
     def getRecipe(self, packageName):
         if packageName not in self.__recipes:
             raise ParseError("Package {} requested but not found.".format(packageName))

pym/bob/scripts.py

@@ -107,6 +107,11 @@ def __jenkinsExecute(*args, **kwargs):
     from .cmds.jenkins.exec import doJenkinsExecute
     return doJenkinsExecute(*args, **kwargs)
 
+def __lsrecipes(*args, **kwargs):
+     from .cmds.misc import doLsRecipes
+     doLsRecipes(*args, **kwargs)
+     return 0
+
 availableCommands = {
     "archive"       : ('hl', __archive, "Manage binary artifact archives"),
     "build"         : ('hl', __build, "Build (sub-)packages in release mode"),
@@ -122,6 +127,7 @@ def __jenkinsExecute(*args, **kwargs):
     "show"          : ('hl', __show, "Show properties of a package"),
     "status"        : ('hl', __status, "Show SCM status"),
 
+    "ls-recipes"    : ('ll', __lsrecipes, "List all known recipes"),
     "query-scm"     : ('ll', __queryscm, "Query SCM information"),
     "query-recipe"  : ('ll', __queryrecipe, "Query package sources"),
     "query-path"    : ('ll', __querypath, "Query path information"),

test/black-box/commands/run.sh

@@ -20,7 +20,7 @@ for c in $cmds; do
 	case "$c" in
 		archive | init | jenkins | layers | help | _*)
 			;;
-		clean)
+		clean | ls-recipes)
 			run_bob $c -DBAR=1 -c testconfig
 			;;
 		project)

test/black-box/ls-recipes/config.yaml

@@ -0,0 +1 @@
+bobMinimumVersion: "1.0"

test/black-box/ls-recipes/recipes/lib.yaml

@@ -0,0 +1,2 @@
+packageScript: |
+    echo lib