diff --git a/gitman/cli.py b/gitman/cli.py index 25497a71..136a30c9 100644 --- a/gitman/cli.py +++ b/gitman/cli.py @@ -336,7 +336,10 @@ def _run_command(function, args, kwargs): exit_message = "Run again with '--force' to ignore script errors" except exceptions.InvalidConfig as exception: _show_error(exception) - exit_message = "Adapt config and run again" + exit_message = ( + "Check the gitman config of the already imported " + "repositories and the repository that raises the conflict error" + ) finally: if exit_message: common.show(exit_message, color='message') diff --git a/gitman/models/config.py b/gitman/models/config.py index 49c53211..e7858f2f 100644 --- a/gitman/models/config.py +++ b/gitman/models/config.py @@ -8,17 +8,33 @@ from .. import common, exceptions, shell from ..decorators import preserve_cwd from .group import Group -from .source import Source +from .source import Source, create_sym_link + + +# TODO remove the next pylint statement and adapt code + +# disable because of the dynamics setattr in __post_init__ to avoid +# that datafile persist this values -> is there another way to tell +# datafiles to ignore specified fiels +# pylint: disable=attribute-defined-outside-init, access-member-before-definition + +# due to the protected access of _get_sources -> make them public in the next step +# pylint: disable=protected-access @datafile("{self.root}/{self.filename}", defaults=True, manual=True) class Config: + RESOLVER_RECURSIVE_NESTED = "recursive-nested" + RESOLVER_RECURSIVE_FLAT = "recursive-flat" + RESOLVER_RECURSIVE_FLAT_NESTED_LINKS = "recursive-flat-nested-links" + RESOLVER_FLAT = "flat" + """Specifies all dependencies for a project.""" root: Optional[str] = None filename: str = "gitman.yml" - location: str = "gitman_sources" + resolver: str = RESOLVER_RECURSIVE_NESTED sources: List[Source] = field(default_factory=list) sources_locked: List[Source] = field(default_factory=list) default_group: str = field(default_factory=str) @@ -28,11 +44,38 @@ def __post_init__(self): if self.root is None: self.root = os.getcwd() + # used only internally and should not be serialized + location_path = os.path.normpath(os.path.join(self.root, self.location)) + setattr(self, 'location_path', location_path) + processed_sources: List[Source] = [] + setattr(self, 'processed_sources', processed_sources) + registered_sources: List[Source] = [] + setattr(self, 'registered_sources', registered_sources) + + # check if any of the valid resolver values is set + # if not then set RESOLVER_RECURSIVE_NESTED as default + if ( + self.resolver != Config.RESOLVER_RECURSIVE_NESTED + and self.resolver != Config.RESOLVER_RECURSIVE_FLAT + and self.resolver != Config.RESOLVER_RECURSIVE_FLAT_NESTED_LINKS + and self.resolver != Config.RESOLVER_FLAT + ): + msg = "Unknown resolver name \"{}\"".format(self.resolver) + raise exceptions.InvalidConfig(msg) + + def __post_load__(self): + # update location path because default location may different then loaded value + self.location_path = os.path.normpath( + os.path.join(self.root, self.location) # type: ignore[arg-type] + ) + @property def config_path(self) -> str: """Get the full path to the config file.""" assert self.root - return os.path.normpath(os.path.join(self.root, self.filename)) + return os.path.normpath( + os.path.join(self.root, self.filename) # type: ignore[arg-type] + ) path = config_path @@ -41,12 +84,6 @@ def log_path(self) -> str: """Get the full path to the log file.""" return os.path.normpath(os.path.join(self.location_path, "gitman.log")) - @property - def location_path(self) -> str: - """Get the full path to the dependency storage location.""" - assert self.root - return os.path.normpath(os.path.join(self.root, self.location)) - def validate(self): """Check for conflicts between source names and group names.""" for source in self.sources: @@ -81,16 +118,89 @@ def install_dependencies( clean=True, skip_changes=False, skip_default_group=False, - ): # pylint: disable=too-many-locals + ): # pylint: disable=too-many-locals, too-many-statements """Download or update the specified dependencies.""" if depth == 0: log.info("Skipped directory: %s", self.location_path) return 0 - sources = self._get_sources(use_locked=False if update else None) - sources_filter = self._get_sources_filter( - *names, sources=sources, skip_default_group=skip_default_group - ) + sources = None + sources_filter = None + + if ( # pylint: disable=too-many-nested-blocks + self.resolver == Config.RESOLVER_RECURSIVE_FLAT + or self.resolver == Config.RESOLVER_RECURSIVE_FLAT_NESTED_LINKS + ): + sources = self._get_sources( + use_locked=False if update else None, use_extra=False + ) + sources_filter = self._get_sources_filter( + *names, sources=sources, skip_default_group=skip_default_group + ) + + # gather flat sources and check for rev conflicts + new_sources: List[Source] = [] + for source in sources: + add_source: bool = True + for registered_source in self.registered_sources: # type: ignore + if ( + source.name == registered_source.name + ): # check if current source was already processed + if ( + source.rev != registered_source.rev + or source.repo != registered_source.repo + ): + # we skip the detected recursed branch rev if we install + # locked sources always the toplevel rev is leading and + # to ensure creation order we process the locked version + # next instead of the noted rev + if not update: + # check if we have already processed the matched source + # pylint: disable=line-too-long + if not registered_source in self.processed_sources: # type: ignore + new_sources.append(registered_source) + else: + # already processed therefore we don't care anymore + sources_filter.remove(source.name) + + add_source = False + continue + + error_msg = ( + "Repo/rev conflict encountered in " + "flat hierarchy while updating {}\n" + "Details: {} conflict with {}" + ).format(self.root, str(registered_source), str(source)) + raise exceptions.InvalidConfig(error_msg) + + # new source name detected -> store new source name + # to list (cache) used to check for rev conflicts + if add_source: + self.registered_sources.append(source) # type: ignore + new_sources.append(source) + + # assign filtered and collected sources + sources = new_sources + + else: + sources = self._get_sources(use_locked=False if update else None) + sources_filter = self._get_sources_filter( + *names, sources=sources, skip_default_group=skip_default_group + ) + + for source in sources: + for registered_source in self.registered_sources: # type: ignore + if ( + source.name == registered_source.name + ): # check if current source was already processed + error_msg = ( + "Repo conflict encountered " + "while updating {}\n" + "Details: {} conflict with {}" + ).format(self.root, str(registered_source), str(source)) + raise exceptions.InvalidConfig(error_msg) + + self.registered_sources.append(source) # type: ignore if not os.path.isdir(self.location_path): shell.mkdir(self.location_path) @@ -106,6 +216,10 @@ def install_dependencies( log.info("Skipped dependency: %s", source.name) continue + # check if source has not already been processed + if source in self.processed_sources: # type: ignore + continue + source.update_files( force=force, force_interactive=force_interactive, @@ -114,12 +228,37 @@ def install_dependencies( skip_changes=skip_changes, ) source.create_links(self.root, force=force) + + # store processed source + self.processed_sources.append(source) # type: ignore + common.newline() count += 1 + if self.resolver == Config.RESOLVER_FLAT: + # don't process nested configs if flat resolver is active + continue + config = load_config(search=False) if config: common.indent() + + if ( + self.resolver == Config.RESOLVER_RECURSIVE_FLAT + or self.resolver == Config.RESOLVER_RECURSIVE_FLAT_NESTED_LINKS + ): + # Top level preference for flat hierarchy should + # forward / propagate resolver settings + config.resolver = self.resolver + # forward / override default location -> always use root location + # to install dependencies all into the same folder + org_location_path = config.location_path + config.location_path = self.location_path + # forward registered and processed sources list to + # check for global conflicts + config.registered_sources = self.registered_sources # type: ignore + config.processed_sources = self.processed_sources # type: ignore + count += config.install_dependencies( depth=None if depth is None else max(0, depth - 1), update=update and recurse, @@ -130,6 +269,14 @@ def install_dependencies( skip_changes=skip_changes, skip_default_group=skip_default_group, ) + + # create nested symlinks + if self.resolver == Config.RESOLVER_RECURSIVE_FLAT_NESTED_LINKS: + for src in config.sources: + link_src = os.path.join(self.location_path, src.name) + link_target = os.path.join(org_location_path, src.name) + create_sym_link(link_src, link_target, True) + common.dedent() shell.cd(self.location_path, _show=False) @@ -162,12 +309,31 @@ def run_scripts(self, *names, depth=None, force=False, show_shell_stdout=False): if source.name in sources_filter: shell.cd(source.name) + if self.resolver == Config.RESOLVER_FLAT: + # don't process nested configs if flat resolver is active + continue + config = load_config(search=False) if config: common.indent() remaining_depth = None if depth is None else max(0, depth - 1) if remaining_depth: common.newline() + + if ( + self.resolver == Config.RESOLVER_RECURSIVE_FLAT + or self.resolver == Config.RESOLVER_RECURSIVE_FLAT_NESTED_LINKS + ): + # Top level preference for flat hierarchy should + # always propagate resolver settings + config.resolver = self.resolver + # override default location -> always use root location + # to install dependencies all into the same folder + config.location_path = self.location_path + # forward processed sources list to check for global conflicts + # pylint: disable=line-too-long + config.processed_sources = self.processed_sources # type: ignore + count += config.run_scripts(depth=remaining_depth, force=force) common.dedent() @@ -182,16 +348,26 @@ def run_scripts(self, *names, depth=None, force=False, show_shell_stdout=False): def lock_dependencies(self, *names, obey_existing=True, skip_changes=False): """Lock down the immediate dependency versions.""" - sources = self._get_sources(use_locked=obey_existing).copy() + recursive = ( + self.resolver == Config.RESOLVER_RECURSIVE_FLAT + or self.resolver == Config.RESOLVER_RECURSIVE_FLAT_NESTED_LINKS + ) + sources = self._get_sources( + use_locked=obey_existing, recursive=recursive + ).copy() sources_filter = self._get_sources_filter( *names, sources=sources, skip_default_group=False ) + if not os.path.isdir(self.location_path): + raise exceptions.InvalidRepository("No dependencies resolved") + shell.cd(self.location_path) common.newline() common.indent() count = 0 + for source in sources: if source.name not in sources_filter: log.info("Skipped dependency: %s", source.name) @@ -290,7 +466,7 @@ def log(self, message="", *args): with open(self.log_path, 'a') as outfile: outfile.write(message.format(*args) + '\n') - def _get_sources(self, *, use_locked=None): + def _get_sources(self, *, use_locked=None, use_extra=True, recursive=False): """Merge source lists using the requested section as the base.""" if use_locked is True: if self.sources_locked: @@ -310,10 +486,29 @@ def _get_sources(self, *, use_locked=None): sources = self.sources extras = [] - for source in self.sources + self.sources_locked: - if source not in sources: - log.info("Source %r missing from selected section", source.name) - extras.append(source) + + if recursive: + recursive_sources = sources + for source in sources: + config_path = os.path.join( + self.location_path, source.name # type: ignore[arg-type] + ) + config = load_config(start=config_path, search=False) + if config: + recursive_sources = recursive_sources + config._get_sources( + use_locked=use_locked, use_extra=False, recursive=True + ) + + for source in recursive_sources: + if source not in sources: + extras.append(source) + + if use_extra: + all_sources = self.sources + self.sources_locked + for source in all_sources: + if source not in sources: + log.info("Source %r missing from selected section", source.name) + extras.append(source) return sources + extras @@ -353,6 +548,7 @@ def load_config(start=None, *, search=True): for filename in os.listdir(path): if _valid_filename(filename): config = Config(path, filename) + config.__post_load__() config.validate() log.debug("Found config: %s", config.path) return config diff --git a/tests/test_api.py b/tests/test_api.py index 3f0768ef..dd247e92 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -23,6 +23,7 @@ CONFIG = """ location: deps +resolver: recursive-nested sources: - repo: https://github.com/jacebrowning/gitman-demo name: gitman_1 @@ -93,6 +94,7 @@ def it_creates_a_new_config_file(tmpdir): expect(Config().datafile.text) == strip( """ location: gitman_sources + resolver: recursive-nested sources: - repo: https://github.com/githubtraining/hellogitworld name: sample_dependency @@ -144,6 +146,7 @@ def it_merges_sources(config): config.datafile.text = strip( """ location: deps + resolver: recursive-nested sources: - repo: https://github.com/jacebrowning/gitman-demo name: gitman_1 @@ -182,6 +185,7 @@ def it_can_handle_missing_locked_sources(config): config.datafile.text = strip( """ location: deps + resolver: recursive-nested sources: - repo: https://github.com/jacebrowning/gitman-demo name: gitman_1 @@ -225,6 +229,7 @@ def config_with_link(config): config.datafile.text = strip( """ location: deps + resolver: recursive-nested sources: - name: gitman_1 repo: https://github.com/jacebrowning/gitman-demo @@ -268,6 +273,7 @@ def config_with_links(config): config.datafile.text = strip( """ location: deps + resolver: recursive-nested sources: - name: gitman_1 repo: https://github.com/jacebrowning/gitman-demo @@ -314,6 +320,7 @@ def config_with_scripts(config): config.datafile.text = strip( """ location: deps + resolver: recursive-nested sources: - name: gitman_1 type: git @@ -342,6 +349,7 @@ def config_with_scripts(config): config.datafile.text = strip( """ location: deps + resolver: recursive-nested sources: - name: gitman_1 type: git @@ -376,6 +384,7 @@ def config_with_group(config): config.datafile.text = strip( """ location: deps + resolver: recursive-nested sources: - name: gitman_1 type: git @@ -438,6 +447,7 @@ def config_with_default_group(config): config.datafile.text = strip( """ location: deps + resolver: recursive-nested sources: - repo: https://github.com/jacebrowning/gitman-demo name: gitman_1 @@ -478,6 +488,7 @@ def config_without_default_group(config): config.datafile.text = strip( """ location: deps + resolver: recursive-nested sources: - repo: https://github.com/jacebrowning/gitman-demo name: gitman_1 @@ -617,6 +628,7 @@ def it_locks_previously_unlocked_dependencies(config): config.datafile.text = strip( """ location: deps + resolver: recursive-nested sources: - repo: https://github.com/jacebrowning/gitman-demo name: gitman_1 @@ -661,6 +673,7 @@ def it_locks_previously_unlocked_dependencies(config): expect(config.datafile.text) == strip( """ location: deps + resolver: recursive-nested sources: - repo: https://github.com/jacebrowning/gitman-demo name: gitman_1 @@ -703,6 +716,7 @@ def it_should_not_lock_dependencies_when_disabled(config): config.datafile.text = strip( """ location: deps + resolver: recursive-nested sources: - repo: https://github.com/jacebrowning/gitman-demo name: gitman_1 @@ -746,6 +760,7 @@ def it_should_not_lock_dependencies_when_disabled(config): expect(config.datafile.text) == strip( """ location: deps + resolver: recursive-nested sources: - repo: https://github.com/jacebrowning/gitman-demo name: gitman_1 @@ -788,6 +803,7 @@ def it_should_not_allow_source_and_group_name_conflicts(config): config.datafile.text = strip( """ location: deps + resolver: recursive-nested sources: - name: gitman_1 type: git @@ -813,6 +829,7 @@ def it_locks_previously_locked_dependencies_by_group_name(config): config.datafile.text = strip( """ location: deps + resolver: recursive-nested sources: - repo: https://github.com/jacebrowning/gitman-demo name: gitman_1 @@ -880,6 +897,7 @@ def it_locks_previously_locked_dependencies_by_group_name(config): expect(config.datafile.text) == strip( """ location: deps + resolver: recursive-nested sources: - repo: https://github.com/jacebrowning/gitman-demo name: gitman_1 @@ -961,6 +979,7 @@ def git_changes( config.datafile.text = strip( """ location: deps + resolver: recursive-nested sources: - repo: https://github.com/jacebrowning/gitman-demo name: gitman_2 @@ -994,6 +1013,7 @@ def git_changes( expect(config.datafile.text) == strip( """ location: deps + resolver: recursive-nested sources: - repo: https://github.com/jacebrowning/gitman-demo name: gitman_2 @@ -1049,6 +1069,7 @@ def git_changes( config.datafile.text = strip( """ location: deps + resolver: recursive-nested sources: - repo: https://github.com/jacebrowning/gitman-demo name: gitman_2 @@ -1083,6 +1104,7 @@ def git_changes( expect(config.datafile.text) == strip( """ location: deps + resolver: recursive-nested sources: - repo: https://github.com/jacebrowning/gitman-demo name: gitman_2 @@ -1115,6 +1137,7 @@ def it_merges_sources(config): config.datafile.text = strip( """ location: deps + resolver: recursive-nested sources: - repo: https://github.com/jacebrowning/gitman-demo name: gitman_1