Merge branch 't/30865/sage_bootstrap__update_extend_system_package_tools' into t/30947/src_doc_bootstrap__simplify_by_using_new_options_of__sage__package_list_

Author: Matthias Koeppe

Makefile

@@ -64,11 +64,11 @@ build/make/Makefile: configure $(SPKG_COLLECT_FILES) $(CONFIG_FILES:%=%.in)
 buildbot-python3:
 	$(MAKE)
 
-# Preemptively download all standard upstream source tarballs.
+# Preemptively download all source tarballs of normal packages.
 download:
 	export SAGE_ROOT=$$(pwd) && \
-	export PATH=$$SAGE_ROOT/src/bin:$$PATH && \
-	./src/bin/sage-download-upstream
+	export PATH=$$SAGE_ROOT/build/bin:$$PATH && \
+	sage-package download :all:
 
 dist: build/make/Makefile
 	./sage --sdist

build/pkgs/sagelib/src/setup.py

@@ -142,7 +142,6 @@
                  'bin/sage-cleaner',
                  ## Only makes sense in sage-the-distribution. TODO: Move to another installation script.
                  'bin/sage-list-packages',
-                 'bin/sage-download-upstream',
                  'bin/sage-location',
                  ## Uncategorized scripts in alphabetical order
                  'bin/math-readline',

build/sage_bootstrap/app.py

@@ -43,7 +43,7 @@ def config(self):
         from sage_bootstrap.config import Configuration
         print(Configuration())
 
-    def list_cls(self, package_class):
+    def list_cls(self, *package_classes, **filters):
         """
         Print a list of all available packages
 
@@ -54,9 +54,19 @@ def list_cls(self, package_class):
         autotools
         [...]
         zn_poly
+
+        $ sage -package list --has-file=spkg-configure.m4 :experimental:
+        perl_term_readline_gnu
+
+        $ sage -package list --has-file=spkg-configure.m4 --has-file=distros/debian.txt | sort
+        arb
+        boost_cropped
+        brial
+        [...]
+        zn_poly
         """
         log.debug('Listing packages')
-        pc = PackageClass(package_class)
+        pc = PackageClass(*package_classes, **filters)
         for pkg_name in pc.names:
             print(pkg_name)
 
@@ -157,8 +167,13 @@ def download(self, package_name, allow_upstream=False):
         package.tarball.download(allow_upstream=allow_upstream)
         print(package.tarball.upstream_fqn)
 
-    def download_cls(self, package_name_or_class):
-        pc = PackageClass(package_name_or_class)
+    def download_cls(self, package_name_or_class, allow_upstream=False):
+        """
+        Download a package or a class of packages
+        """
+        pc = PackageClass(package_name_or_class, has_files=['checksums.ini'])
+        def download_with_args(package):
+            return self.download(package, allow_upstream=allow_upstream)
         pc.apply(self.download)
 
     def upload(self, package_name):

build/sage_bootstrap/cmdline.py

@@ -200,8 +200,11 @@ def make_parser():
         help='Print a list of all available packages')
     parser_list.add_argument(
         'package_class',
-        type=str, default=':all:', nargs='?',
+        type=str, default=[':all:'], nargs='*',
         help='Package class like :all: (default) or :standard:')
+    parser_list.add_argument(
+        '--has-file', action='append', default=[], metavar='FILENAME', dest='has_files',
+        help='Only include packages that have this file')
 
     parser_name = subparsers.add_parser(
         'name', epilog=epilog_name,
@@ -300,7 +303,7 @@ def run():
     if args.subcommand == 'config':
         app.config()
     elif args.subcommand == 'list':
-        app.list_cls(args.package_class)
+        app.list_cls(*args.package_class, has_files=args.has_files)
     elif args.subcommand == 'name':
         app.name(args.tarball_filename)
     elif args.subcommand == 'tarball':

build/sage_bootstrap/expand_class.py

@@ -25,39 +25,46 @@
 
 class PackageClass(object):
 
-    def __init__(self, package_name_or_class):
-        if package_name_or_class == ':all:':
-            self._init_all()
-        elif package_name_or_class == ':standard:':
-            self._init_standard()
-        elif package_name_or_class == ':optional:':
-            self._init_optional()
-        elif package_name_or_class == ':experimental:':
-            self._init_experimental()
-        elif package_name_or_class == ':huge:':
-            self._init_huge()
-        else:
-            if package_name_or_class.startswith(':'):
-                raise ValueError('Package name cannot start with ":", got %s', package_name_or_class)
-            if package_name_or_class.endswith(':'):
-                raise ValueError('Package name cannot end with ":", got %s', package_name_or_class)
-            self.names = [package_name_or_class]
-
-    def _init_all(self):
-        self.names = [pkg.name for pkg in Package.all()]
-            
-    def _init_standard(self):
-        self.names = [pkg.name for pkg in Package.all() if pkg.type == 'standard']
-            
-    def _init_optional(self):
-        self.names = [pkg.name for pkg in Package.all() if pkg.type == 'optional']
-            
-    def _init_experimental(self):
-        self.names = [pkg.name for pkg in Package.all() if pkg.type == 'experimental']
-            
-    def _init_huge(self):
-        self.names = [pkg.name for pkg in Package.all() if pkg.type == 'huge']
-            
+    def __init__(self, *package_names_or_classes, **filters):
+        self.names = []
+        filenames = filters.pop('has_files', [])
+        if filters:
+            raise ValueError('filter not supported')
+        def included_in_filter(pkg):
+            return all(pkg.has_file(filename) for filename in filenames)
+        for package_name_or_class in package_names_or_classes:
+            if package_name_or_class == ':all:':
+                self._init_all(predicate=included_in_filter)
+            elif package_name_or_class == ':standard:':
+                self._init_standard(predicate=included_in_filter)
+            elif package_name_or_class == ':optional:':
+                self._init_optional(predicate=included_in_filter)
+            elif package_name_or_class == ':experimental:':
+                self._init_experimental(predicate=included_in_filter)
+            elif package_name_or_class == ':huge:':
+                self._init_huge(predicate=included_in_filter)
+            else:
+                if package_name_or_class.startswith(':'):
+                    raise ValueError('Package name cannot start with ":", got %s', package_name_or_class)
+                if package_name_or_class.endswith(':'):
+                    raise ValueError('Package name cannot end with ":", got %s', package_name_or_class)
+                self.names.append(package_name_or_class)
+
+    def _init_all(self, predicate):
+        self.names.extend(pkg.name for pkg in Package.all() if predicate(pkg))
+
+    def _init_standard(self, predicate):
+        self.names.extend(pkg.name for pkg in Package.all() if pkg.type == 'standard' and predicate(pkg))
+
+    def _init_optional(self, predicate):
+        self.names.extend(pkg.name for pkg in Package.all() if pkg.type == 'optional' and predicate(pkg))
+
+    def _init_experimental(self, predicate):
+        self.names.extend(pkg.name for pkg in Package.all() if pkg.type == 'experimental' and predicate(pkg))
+
+    def _init_huge(self, predicate):
+        self.names.extend(pkg.name for pkg in Package.all() if pkg.type == 'huge' and predicate(pkg))
+
     def apply(self, function, *args, **kwds):
         for package_name in self.names:
             function(package_name, *args, **kwds)

build/sage_bootstrap/package.py

@@ -146,7 +146,11 @@ def tarball_filename(self):
 
         String. The full-qualified tarball filename.
         """
-        return self.tarball_pattern.replace('VERSION', self.version)
+        pattern = self.tarball_pattern
+        if pattern:
+            return self.tarball_pattern.replace('VERSION', self.version)
+        else:
+            return None
 
     @property
     def tarball_upstream_url_pattern(self):
@@ -236,8 +240,8 @@ def all(cls):
         base = os.path.join(SAGE_ROOT, 'build', 'pkgs')
         for subdir in os.listdir(base):
             path = os.path.join(base, subdir)
-            if not os.path.isfile(os.path.join(path, "checksums.ini")):
-                log.debug('%s has no checksums.ini', subdir)
+            if not os.path.isfile(os.path.join(path, "type")):
+                log.debug('%s has no type', subdir)
                 continue
             try:
                 yield cls(subdir)
@@ -251,46 +255,60 @@ def path(self):
         Return the package directory
         """
         return os.path.join(SAGE_ROOT, 'build', 'pkgs', self.name)
-            
+
+    def has_file(self, filename):
+        """
+        Return whether the file exists in the package directory
+        """
+        return os.path.exists(os.path.join(self.path, filename))
+
     def _init_checksum(self):
         """
         Load the checksums from the appropriate ``checksums.ini`` file
         """
         checksums_ini = os.path.join(self.path, 'checksums.ini')
         assignment = re.compile('(?P<var>[a-zA-Z0-9_]*)=(?P<value>.*)')
         result = dict()
-        with open(checksums_ini, 'rt') as f:
-            for line in f.readlines():
-                match = assignment.match(line)
-                if match is None:
-                    continue
-                var, value = match.groups()
-                result[var] = value
+        try:
+            with open(checksums_ini, 'rt') as f:
+                for line in f.readlines():
+                    match = assignment.match(line)
+                    if match is None:
+                        continue
+                    var, value = match.groups()
+                    result[var] = value
+        except IOError:
+            pass
         self.__md5 = result.get('md5', None)
         self.__sha1 = result.get('sha1', None)
         self.__cksum = result.get('cksum', None)
-        self.__tarball_pattern = result['tarball']
+        self.__tarball_pattern = result.get('tarball', None)
         self.__tarball_upstream_url_pattern = result.get('upstream_url', None)
         # Name of the directory containing the checksums.ini file
         self.__tarball_package_name = os.path.realpath(checksums_ini).split(os.sep)[-2]
 
     VERSION_PATCHLEVEL = re.compile('(?P<version>.*)\.p(?P<patchlevel>[0-9]+)')
 
     def _init_version(self):
-        with open(os.path.join(self.path, 'package-version.txt')) as f:
-            package_version = f.read().strip()
-        match = self.VERSION_PATCHLEVEL.match(package_version)
-        if match is None:
-            self.__version = package_version
-            self.__patchlevel = -1
+        try:
+            with open(os.path.join(self.path, 'package-version.txt')) as f:
+                package_version = f.read().strip()
+        except IOError:
+            self.__version = None
+            self.__patchlevel = None
         else:
-            self.__version = match.group('version')
-            self.__patchlevel = int(match.group('patchlevel'))
-        
+            match = self.VERSION_PATCHLEVEL.match(package_version)
+            if match is None:
+                self.__version = package_version
+                self.__patchlevel = -1
+            else:
+                self.__version = match.group('version')
+                self.__patchlevel = int(match.group('patchlevel'))
+
     def _init_type(self):
         with open(os.path.join(self.path, 'type')) as f:
             package_type = f.read().strip()
         assert package_type in [
-            'base', 'standard', 'optional', 'experimental', 'script', 'pip'
+            'base', 'standard', 'optional', 'experimental'
         ]
         self.__type = package_type

build/sage_bootstrap/tarball.py

@@ -140,6 +140,8 @@ def download(self, allow_upstream=False):
         on the sage mirrors, fall back to downloading it from
         the upstream URL if the package has one.
         """
+        if not self.filename:
+            raise ValueError('non-normal package does define a tarball, so cannot download')
         destination = self.upstream_fqn
         if os.path.isfile(destination):
             if self.checksum_verifies():

src/bin/sage-download-upstream

@@ -1096,15 +1096,7 @@ you. This is influenced by the following environment variable:
 
   - ``SAGE_SERVER/spkg/upstream``
 
-  for clean upstream tarballs, and it searches the directories
-
-  - ``SAGE_SERVER/spkg/standard/``,
-  - ``SAGE_SERVER/spkg/optional/``,
-  - ``SAGE_SERVER/spkg/experimental/``,
-  - ``SAGE_SERVER/spkg/archive/``
-
-  for old-style Sage packages.
-
+  for upstream tarballs.
 
 Here are some of the more commonly used variables affecting the build process:
 

src/doc/en/installation/source.rst

@@ -142,7 +142,6 @@
                  'bin/sage-cleaner',
                  ## Only makes sense in sage-the-distribution. TODO: Move to another installation script.
                  'bin/sage-list-packages',
-                 'bin/sage-download-upstream',
                  'bin/sage-location',
                  ## Uncategorized scripts in alphabetical order
                  'bin/math-readline',