From b5504968ab81bd3da107486c127b6c934a003d72 Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Mon, 17 Feb 2025 22:06:59 +0000 Subject: [PATCH 1/8] Move the template-instanciating code out of the seed command. We plan to reuse the code that copies/instantiates the template-derived files for the upcoming `update` command, so we first need to isolate it in a separate function. --- odk/odk.py | 73 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 41 insertions(+), 32 deletions(-) diff --git a/odk/odk.py b/odk/odk.py index 3bafd489..45bc1af7 100755 --- a/odk/odk.py +++ b/odk/odk.py @@ -907,6 +907,46 @@ def unpack_files(basedir, txt): f.close() return tgts +def install_template_files(generator, templatedir, targetdir): + """ + Installs all template-derived files into a target directory. + """ + tgts = [] + for root, subdirs, files in os.walk(templatedir): + tdir = root.replace(templatedir,targetdir+"/") + os.makedirs(tdir, exist_ok=True) + + # first copy plain files... + for f in files: + srcf = os.path.join(root, f) + tgtf = os.path.join(tdir, f) + logging.info(' Copying: {} -> {}'.format(srcf, tgtf)) + if not tgtf.endswith(TEMPLATE_SUFFIX): + # copy file directly, no template expansions + copy(srcf, tgtf) + tgts.append(tgtf) + logging.info('Applying templates') + # ...then apply templates + for f in files: + srcf = os.path.join(root, f) + tgtf = os.path.join(tdir, f) + if srcf.endswith(TEMPLATE_SUFFIX): + derived_file = tgtf.replace(TEMPLATE_SUFFIX, "") + with open(derived_file,"w") as s: + if f.startswith("_dynamic"): + logging.info(' Unpacking: {}'.format(derived_file)) + tgts += unpack_files(tdir, generator.generate(srcf)) + s.close() + os.remove(derived_file) + else: + logging.info(' Compiling: {} -> {}'.format(srcf, derived_file)) + s.write(generator.generate(srcf)) + tgts.append(derived_file) + if not f.startswith("_dynamic"): + copymode(srcf, derived_file) + return tgts + + ## ======================================== ## Command Line Wrapper ## ======================================== @@ -1039,38 +1079,7 @@ def seed(config, clean, outdir, templatedir, dependencies, title, user, source, if not os.path.exists(templatedir) and templatedir == "/tools/templates/": logging.info("No templates folder in /tools/; assume not in docker context") templatedir = "./template" - for root, subdirs, files in os.walk(templatedir): - tdir = root.replace(templatedir,outdir+"/") - os.makedirs(tdir, exist_ok=True) - - # first copy plain files... - for f in files: - srcf = os.path.join(root, f) - tgtf = os.path.join(tdir, f) - logging.info(' Copying: {} -> {}'.format(srcf, tgtf)) - if not tgtf.endswith(TEMPLATE_SUFFIX): - # copy file directly, no template expansions - copy(srcf, tgtf) - tgts.append(tgtf) - logging.info('Applying templates') - # ...then apply templates - for f in files: - srcf = os.path.join(root, f) - tgtf = os.path.join(tdir, f) - if srcf.endswith(TEMPLATE_SUFFIX): - derived_file = tgtf.replace(TEMPLATE_SUFFIX, "") - with open(derived_file,"w") as s: - if f.startswith("_dynamic"): - logging.info(' Unpacking: {}'.format(derived_file)) - tgts += unpack_files(tdir, mg.generate(srcf)) - s.close() - os.remove(derived_file) - else: - logging.info(' Compiling: {} -> {}'.format(srcf, derived_file)) - s.write(mg.generate(srcf)) - tgts.append(derived_file) - if not f.startswith("_dynamic"): - copymode(srcf, derived_file) + tgts += install_template_files(mg, templatedir, outdir) tgt_project_file = "{}/project.yaml".format(outdir) if project.export_project_yaml: From 6be15133c6036036768b4bb295c61d3b255c475c Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Tue, 18 Feb 2025 20:14:42 +0000 Subject: [PATCH 2/8] Simplify the install_templates function. Use list comprehensions to iterate over only the plain files first, then over only the templates. This is no more efficient (we still need to iterate over all the files twice), but is more readable. When applying the templates, check whether the template is a "dynamic pack" first, and only open the "derived_file" if it is not. A further simplification would be to loop over all the files only once and test for the template suffix once inside the loop: for f in files: if not f.endswith(TEMPLATE_SUFFIX): # Copy plain file else: # Apply template (single file or pack file) but this would not allow to preserve the current behaviour of first copying all the plain files before instantiating the templates (plain files and templates would be processed in the order they are listed). --- odk/odk.py | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/odk/odk.py b/odk/odk.py index 45bc1af7..0f183437 100755 --- a/odk/odk.py +++ b/odk/odk.py @@ -917,32 +917,27 @@ def install_template_files(generator, templatedir, targetdir): os.makedirs(tdir, exist_ok=True) # first copy plain files... - for f in files: + for f in [f for f in files if not f.endswith(TEMPLATE_SUFFIX)]: srcf = os.path.join(root, f) tgtf = os.path.join(tdir, f) logging.info(' Copying: {} -> {}'.format(srcf, tgtf)) - if not tgtf.endswith(TEMPLATE_SUFFIX): - # copy file directly, no template expansions - copy(srcf, tgtf) - tgts.append(tgtf) + # copy file directly, no template expansions + copy(srcf, tgtf) + tgts.append(tgtf) logging.info('Applying templates') # ...then apply templates - for f in files: + for f in [f for f in files if f.endswith(TEMPLATE_SUFFIX)]: srcf = os.path.join(root, f) tgtf = os.path.join(tdir, f) - if srcf.endswith(TEMPLATE_SUFFIX): - derived_file = tgtf.replace(TEMPLATE_SUFFIX, "") + derived_file = tgtf.replace(TEMPLATE_SUFFIX, "") + if f.startswith("_dynamic"): + logging.info(' Unpacking: {}'.format(derived_file)) + tgts += unpack_files(tdir, generator.generate(srcf)) + else: with open(derived_file,"w") as s: - if f.startswith("_dynamic"): - logging.info(' Unpacking: {}'.format(derived_file)) - tgts += unpack_files(tdir, generator.generate(srcf)) - s.close() - os.remove(derived_file) - else: - logging.info(' Compiling: {} -> {}'.format(srcf, derived_file)) - s.write(generator.generate(srcf)) - tgts.append(derived_file) - if not f.startswith("_dynamic"): + logging.info(' Compiling: {} -> {}'.format(srcf, derived_file)) + s.write(generator.generate(srcf)) + tgts.append(derived_file) copymode(srcf, derived_file) return tgts From 90e6a73ae626ad3c8fc08c32e7fbab469cf44f58 Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Tue, 18 Feb 2025 21:03:57 +0000 Subject: [PATCH 3/8] Allow to have a per-file installation policy. Make the install_template_files function accept an optional `policies` dictionary. A key is that dictionary is the pathname of a template file (relative to the top level of a ODK repository); the value should be one of: * IF_MISSING: install the corresponding file into the target repository, only if it does not already exist; * ALWAYS: always install the corresponding file; * NEVER: never install the corresponding file; The policy dictionary is queried for every single file to install (whether it is a plain file or a file from a Jinja2 template), and the file is installed or not based on the corresponding policy. If the dictionary does not contain an entry for a file, the default policy is IF_MISSING. In the `seed` command, there is no policy dictionary and so all files are installed (unless they happen to exist already, in the case the `seed` command was called without the `--clean` option) and the command behaves as before. But the policy dictionary will make it possible to implement an `update` command where we can selectively decide which files must be overwritten when we update an existing repository. --- odk/odk.py | 78 ++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 61 insertions(+), 17 deletions(-) diff --git a/odk/odk.py b/odk/odk.py index 0f183437..4678e9fc 100755 --- a/odk/odk.py +++ b/odk/odk.py @@ -876,7 +876,7 @@ def save_project_yaml(project : OntologyProject, path : str): with open(path, "w") as f: f.write(yaml.dump(json_obj, default_flow_style=False)) -def unpack_files(basedir, txt): +def unpack_files(basedir, txt, policies={}): """ This unpacks a custom tar-like format in which multiple file paths can be specified, separated by ^^^s @@ -887,16 +887,21 @@ def unpack_files(basedir, txt): lines = txt.split("\n") f = None tgts = [] + ignore = False for line in lines: if line.startswith(MARKER): - path = os.path.join(basedir, line.replace(MARKER, "")) - os.makedirs(os.path.dirname(path), exist_ok=True) + # Close previous file, if any if f != None: f.close() - f = open(path,"w") - tgts.append(path) - logging.info(' Unpacking into: {}'.format(path)) - else: + filename = line.replace(MARKER, "") + path = os.path.join(basedir, filename) + ignore = not must_install_file(filename, path, policies) + if not ignore: + os.makedirs(os.path.dirname(path), exist_ok=True) + f = open(path,"w") + tgts.append(path) + logging.info(' Unpacking into: {}'.format(path)) + elif not ignore: if f is None: if line == "": continue @@ -907,7 +912,45 @@ def unpack_files(basedir, txt): f.close() return tgts -def install_template_files(generator, templatedir, targetdir): +def get_template_name(templatedir, pathname): + """ + Helper function to get the user-visible name of a template file + from its complete pathname in the template directory. + + For example, if the pathname is + "/tools/template/src/ontology/run.sh.jinja2", this will return + "src/ontology/run.sh". + """ + name = pathname.replace(templatedir, "") + if len(name) > 0 and name[0] == '/': + name = name[1:] + if name.endswith(TEMPLATE_SUFFIX): + name = name.replace(TEMPLATE_SUFFIX, "") + return name + +# Available policies for installing a file +IF_MISSING, ALWAYS, NEVER = range(3) + +def must_install_file(templatefile, targetfile, policies): + """ + Given a template filename, indicate whether the file should be + installed according to any per-file policy. + + policies is a dictionary associating a template filename to one + of the following three values: + * IF_MISSING (default): install the file if it does not already exist + * ALWAYS: always install the file, overwrite any existing file + * NEVER: never install the file + """ + policy = policies.get(templatefile, IF_MISSING) + if policy == ALWAYS: + return True + elif policy == NEVER: + return False + else: + return not os.path.exists(targetfile) + +def install_template_files(generator, templatedir, targetdir, policies={}): """ Installs all template-derived files into a target directory. """ @@ -920,10 +963,11 @@ def install_template_files(generator, templatedir, targetdir): for f in [f for f in files if not f.endswith(TEMPLATE_SUFFIX)]: srcf = os.path.join(root, f) tgtf = os.path.join(tdir, f) - logging.info(' Copying: {} -> {}'.format(srcf, tgtf)) - # copy file directly, no template expansions - copy(srcf, tgtf) - tgts.append(tgtf) + if must_install_file(get_template_name(templatedir, srcf), tgtf, policies): + logging.info(' Copying: {} -> {}'.format(srcf, tgtf)) + # copy file directly, no template expansions + copy(srcf, tgtf) + tgts.append(tgtf) logging.info('Applying templates') # ...then apply templates for f in [f for f in files if f.endswith(TEMPLATE_SUFFIX)]: @@ -932,13 +976,13 @@ def install_template_files(generator, templatedir, targetdir): derived_file = tgtf.replace(TEMPLATE_SUFFIX, "") if f.startswith("_dynamic"): logging.info(' Unpacking: {}'.format(derived_file)) - tgts += unpack_files(tdir, generator.generate(srcf)) - else: + tgts += unpack_files(tdir, generator.generate(srcf), policies) + elif must_install_file(get_template_name(templatedir, srcf), derived_file, policies): + logging.info(' Compiling: {} -> {}'.format(srcf, derived_file)) with open(derived_file,"w") as s: - logging.info(' Compiling: {} -> {}'.format(srcf, derived_file)) s.write(generator.generate(srcf)) - tgts.append(derived_file) - copymode(srcf, derived_file) + tgts.append(derived_file) + copymode(srcf, derived_file) return tgts From 5c7dfed8567162e47ced150ff64d94f6578e12ad Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Tue, 18 Feb 2025 21:51:00 +0000 Subject: [PATCH 4/8] Add a `update` command to odk.py. Now that we can selectively install instantiated template files in a directory depending on a per-file policy and on whether the target file already exists or not, we can implement a repository update method directly in the main odk.py script. The command simply: 1. finds the '*-odk.yaml' configuration file, aborting if it can't find it (this also acts as a check that we are within a ODK repository); 2. loads the configuration; 3. declares which files must be ignored or overwritten; 4. instantiates the templates. This is a much more straightforward process than the original `update_repo.sh` script, which involved seeding a brand new repository in a staging directory and then selectively copying files from the staging directory into the real repository. Also, because the update code is within the odk.py script, this means it is within the ODK image. Therefore, when using that command, it is always the latest available update code from the latest ODK version that is immediately used. With the previous method, the `update_repo.sh` script that was executed was the one that was installed with the previous version of the ODK, leading to the recommendation of running the script twice so that the second invocation would use the updated code. --- odk/odk.py | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/odk/odk.py b/odk/odk.py index 4678e9fc..bd7a647a 100755 --- a/odk/odk.py +++ b/odk/odk.py @@ -18,6 +18,7 @@ from dacite import from_dict import yaml import os +import glob import subprocess import shutil from shutil import copy, copymode @@ -1059,6 +1060,69 @@ def dump_schema(class_name): clazz = globals()[class_name] # Get the class object from the globals dictionary print(json.dumps(clazz.json_schema(), sort_keys=True, indent=4)) +@cli.command() +@click.option('-T', '--templatedir', default='/tools/templates/') +def update(templatedir): + """ + Updates a pre-existing repository. This command is expected to be + run from within the src/ontology directory (the directory + containing the configuration file). + """ + config_matches = list(glob.glob('*-odk.yaml')) + if len(config_matches) == 0: + raise click.ClickException("No ODK configuration file found") + elif len(config_matches) > 1: + raise click.ClickException("More than ODK configuration file found") + config = config_matches[0] + mg = Generator() + mg.load_config(config) + project = mg.context.project + + # When updating, for most files, we only install them if + # they do not already exist in the repository (typically + # because they are new files that didn't exist in the + # templates of the previous version of the ODK). But a + # handful of files are not reinstalled even if they are + # missing (e.g. DOSDP example files) or on the contrary + # always reinstalled to overwrite any local changes (e.g. + # the main Makefile). We declare the corresponding policies. + policies = {} + policies['CODE_OF_CONDUCT.md'] = NEVER + policies['CONTRIBUTING.md'] = NEVER + policies['issue_template.md'] = NEVER + policies['README.md'] = NEVER + policies['src/patterns/data/default/example.tsv'] = NEVER + policies['src/patterns/dosdp-patterns/example.yaml'] = NEVER + policies['src/ontology/Makefile'] = ALWAYS + policies['src/ontology/run.sh'] = ALWAYS + policies['docs/odk-workflows'] = ALWAYS + if 'github_actions' in project.ci: + for workflow in ['qc', 'diff', 'release-diff']: + if workflow in project.workflows: + policies['.github/workflows/' + workflow + '.yml'] = ALWAYS + if project.documentation is not None and 'docs' in project.workflows: + policies['github/workflows/docs.yml'] = ALWAYS + # HACK: For the odk-workflows documentation directory, + # we want to clean up that directory completely. The + # policy dictionary only works on files, so to force + # reinstalling the entire directory (also removing any + # non-standard file along the way), we forcefully + # remove the directory. + if project.documentation is not None: + shutil.rmtree('../../docs/odk-workflows') + + # Proceed with template instantiation, using the policies + # declared above. We instantiate directly at the root of + # the repository -- no need for a staging directory. + install_template_files(mg, templatedir, '../..', policies) + + print("WARNING: These files should be manually migrated:") + print(" mkdocs.yaml, .gitignore, src/ontology/catalog.xml") + print(" (if you added a new import or component)") + if 'github_actions' in project.ci and 'qc' not in project.workflows: + print("WARNING: Your QC workflows have not been updated automatically.") + print(" Please update the ODK version number in .github/workflows/qc.yml") + print("Ontology repository update successfully completed.") @cli.command() @click.option('-C', '--config', type=click.Path(exists=True), From a3a41a4d684a1069cb1b458e48e1b9019a86da16 Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Tue, 18 Feb 2025 22:21:30 +0000 Subject: [PATCH 5/8] Remove the old update_repo.sh script. That script will not be used anymore. We keep the `update_repo` target in the Makefile, but make it invoke the update command of the odk.py script. Using that target is not recommended, it is kept for backwards compatibility only. (Its problem is that the Makefile is overwritten during the update process.) --- template/src/ontology/Makefile.jinja2 | 3 +- template/src/scripts/update_repo.sh.jinja2 | 44 ---------------------- 2 files changed, 2 insertions(+), 45 deletions(-) delete mode 100644 template/src/scripts/update_repo.sh.jinja2 diff --git a/template/src/ontology/Makefile.jinja2 b/template/src/ontology/Makefile.jinja2 index 9c62559e..9a5cbe58 100644 --- a/template/src/ontology/Makefile.jinja2 +++ b/template/src/ontology/Makefile.jinja2 @@ -1391,9 +1391,10 @@ normalize_src: $(SRC) validate_idranges: dicer-cli {{ project.id }}-idranges.owl +# Deprecated: Use 'sh run.sh odk.py update' without using the Makefile. .PHONY: update_repo update_repo: - sh $(SCRIPTSDIR)/update_repo.sh + odk.py update {% if project.documentation is not none %} update_docs: diff --git a/template/src/scripts/update_repo.sh.jinja2 b/template/src/scripts/update_repo.sh.jinja2 deleted file mode 100644 index 65411ad4..00000000 --- a/template/src/scripts/update_repo.sh.jinja2 +++ /dev/null @@ -1,44 +0,0 @@ -echo "This (experimental) update script will create a new repo according to your config file. It will:" -echo "(1) overwrite your repositories Makefile, ODK sparql queries (your custom queries wont be touched) and docker wrapper (run.sh)." -echo "(2) and add missing files, if any." - -set -e - -OID={{project.id}} -ROOTDIR=../.. -SRCDIR=.. -CONFIG=$OID"-odk.yaml" - -rm -rf target -mkdir target -/tools/odk.py seed -c -g -C $CONFIG -ls -l target/$OID/src -ls -l $SRCDIR/ -cp target/$OID/src/scripts/update_repo.sh $SRCDIR/scripts/ -rsync -r -u --ignore-existing --exclude 'patterns/data/default/example.tsv' --exclude 'patterns/dosdp-patterns/example.yaml' target/$OID/src/ $SRCDIR/ -{% if project.documentation is not none %} -rsync -r -u --ignore-existing target/$OID/docs/ $ROOTDIR/docs -mkdir -p $ROOTDIR/docs/odk-workflows -rm -rf $ROOTDIR/docs/odk-workflows/* -cp -r target/$OID/docs/odk-workflows/* $ROOTDIR/docs/odk-workflows -{% endif -%} -cp target/$OID/src/ontology/Makefile $SRCDIR/ontology/ -cp target/$OID/src/ontology/run.sh $SRCDIR/ontology/ -cp -r target/$OID/src/sparql/* $SRCDIR/sparql/ -{%- if project.ci is defined -%}{% if 'travis' in project.ci %} -cp -n target/$OID/.travis.yml $ROOTDIR/ -{%- endif -%}{% if 'github_actions' in project.ci %} -mkdir -p $ROOTDIR/.github -mkdir -p $ROOTDIR/.github/workflows -{% if 'qc' in project.workflows %}cp target/$OID/.github/workflows/qc.yml $ROOTDIR/.github/workflows/qc.yml{% endif %} -{% if 'diff' in project.workflows %}cp target/$OID/.github/workflows/diff.yml $ROOTDIR/.github/workflows/diff.yml{% endif %} -{% if 'release-diff' in project.workflows %}cp target/$OID/.github/workflows/release-diff.yml $ROOTDIR/.github/workflows/release-diff.yml{% endif %} -{% if project.documentation is not none %} -{% if 'docs' in project.workflows %}cp target/$OID/.github/workflows/docs.yml $ROOTDIR/.github/workflows/docs.yml{% endif %} -{% endif %}{% endif %}{% endif %} -{% if project.documentation is not none %} -cp -n target/$OID/mkdocs.yaml $ROOTDIR/ -{% endif %} -echo "WARNING: These files should be manually migrated: mkdocs.yaml, .gitignore, src/ontology/catalog.xml (if you added a new import or component)" -{% if 'github_actions' in project.ci and 'qc' not in project.workflows %}echo "WARNING: Your QC workflows have not been updated automatically. Please update the ODK version number(s) in .github/workflows/qc.yml."{% endif %} -echo "Ontology repository update successfully completed." From 114c1f156bab7e6cc2626598ca090a5899a1c8f8 Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Tue, 18 Feb 2025 22:51:38 +0000 Subject: [PATCH 6/8] Install a helper update_repo script, update documentation. Updating a repo can now be done by invoking `sh run.sh odk.py update`. But for convenience, we install a dead simple script in /tools/update_repo that merely call the above command, so that an update can be triggered using `sh run.sh update_repo`. This is because until now, most people never had to deal with the `odk.py` script directly, since it was typically only used for seeding a repository (which is only ever done once). It was also indirectly used for updates (since an update involved seeding a brand new repository and then copying its files to the current repository), but this fact was hidden from the users. So, I believe suddenly introducing the need to call that odk.py script to update a repo could cause needless confusion in users' mind, so we hide it behing a single `update_repo` command. This also has the benefit of shielding us from any future potential change to the interface of the odk.py update command. --- docker/odklite/Dockerfile | 3 +++ docs/FrequentlyUsedODKCommands.md | 2 +- scripts/update_repo.sh | 3 +++ template/src/ontology/Makefile.jinja2 | 7 +++++-- 4 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 scripts/update_repo.sh diff --git a/docker/odklite/Dockerfile b/docker/odklite/Dockerfile index 4732a9ee..024fcf46 100644 --- a/docker/odklite/Dockerfile +++ b/docker/odklite/Dockerfile @@ -96,6 +96,9 @@ RUN sed -i '/secure_path/d' /etc/sudoers && \ COPY --chmod=755 scripts/odk-info.sh /tools/odk-info RUN sed -i s/@@ODK_IMAGE_VERSION@@/$ODK_VERSION/ /tools/odk-info +# Install a helper script to launch a repository update +COPY --chmod=755 scripts/update_repo.sh /tools/update_repo + # Install the ODK itself. COPY --chmod=755 odk/make-release-assets.py odk/odk.py /tools COPY template/ /tools/templates/ diff --git a/docs/FrequentlyUsedODKCommands.md b/docs/FrequentlyUsedODKCommands.md index 235f720b..b430aeb7 100644 --- a/docs/FrequentlyUsedODKCommands.md +++ b/docs/FrequentlyUsedODKCommands.md @@ -3,7 +3,7 @@ ## Updates the Makefile to the latest ODK ``` -sh run.sh make update_repo +sh run.sh update_repo ``` ## Recreates and deploys the automated documentation diff --git a/scripts/update_repo.sh b/scripts/update_repo.sh new file mode 100644 index 00000000..20db4cf1 --- /dev/null +++ b/scripts/update_repo.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +exec /tools/odk.py update diff --git a/template/src/ontology/Makefile.jinja2 b/template/src/ontology/Makefile.jinja2 index 9a5cbe58..11949e1d 100644 --- a/template/src/ontology/Makefile.jinja2 +++ b/template/src/ontology/Makefile.jinja2 @@ -1428,7 +1428,7 @@ Usage: [IMAGE=(odklite|odkfull)] [ODK_DEBUG=yes] sh run.sh make [(IMP|MIR|IMP_LA Core commands: * prepare_release: Run the entire release pipeline. Use make IMP=false prepare_release to avoid rerunning the imports * prepare_release_fast: Run the entire release pipeline without refreshing imports, recreating components or recompiling patterns. -* update_repo: Update the ODK repository setup using the config file {{ project.id }}-odk.yaml +* update_repo: Update the ODK repository setup using the config file {{ project.id }}-odk.yaml (DEPRECATED) * test: Running all validation tests * test_fast: Runs the test suite, but without updating imports or components * odkversion: Show the current version of the ODK Makefile and ROBOT. @@ -1480,7 +1480,6 @@ Additional QC commands (advanced users) Examples: * sh run.sh make IMP=false prepare_release -* sh run.sh make update_repo * sh run.sh make test Tricks: @@ -1488,6 +1487,10 @@ Tricks: * Use the IMAGE parameter to the run.sh script to use a different image like odklite * Use ODK_DEBUG=yes sh run.sh make ... to print information about timing and debugging +Updating the repository: +(to apply changes to the ODK configuration or switch to a newer ODK version) +* sh run.sh update_repo + endef export data From d899404d72cc23038292981128b2e09c9f4eea8b Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Wed, 19 Feb 2025 00:19:59 +0000 Subject: [PATCH 7/8] Allow overwriting run.sh without causing a failure. When updating, the shell executing the run.sh script (through which the update command is executing, as any other ODK process) could fail because the run.sh script is overwritten during the update. To avoid that, we detect at the very beginning of the script if the user is invoking the update command, and if it is, we make a copy of the script in the tmp/ directory and immediately execute that copy, so that any overwrite of the original script has no effect on the executing shell. --- template/src/ontology/run.sh.jinja2 | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/template/src/ontology/run.sh.jinja2 b/template/src/ontology/run.sh.jinja2 index 83959d2e..dea05ad7 100755 --- a/template/src/ontology/run.sh.jinja2 +++ b/template/src/ontology/run.sh.jinja2 @@ -14,6 +14,16 @@ # # See README-editors.md for more details. +case "$@" in +*update_repo*) + # Execute a copy of this script so that it can be overwritten + # during the update process + mkdir -p tmp + sed -n '/^set -e/,$p' $0 > tmp/$0 + exec $SHELL tmp/$0 "$@" + ;; +esac + set -e # Check for spaces in the current directory path From 8e232350ce7584482113008846845daea3e51710 Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Wed, 19 Feb 2025 10:33:07 +0000 Subject: [PATCH 8/8] Associate updating policies to shell-like globbing patterns. Change the type of the policies argument to `must_install_file` from a dictionary to a list of tuples, where the first item in each tuple is a shell-like globbing pattern. This is slightly less efficient as it requires iterating over the entire list to find a matching policy (instead of the on-average constant access time of a hashtable), but is more convenient as it allows to have a single policy for an entire subtree. Notably, it allows to do policies.add(('src/sparql/*', ALWAYS)) to apply a ALWAYS policy to all files under `src/sparql`, without having to enumerate them all. --- odk/odk.py | 59 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 33 insertions(+), 26 deletions(-) diff --git a/odk/odk.py b/odk/odk.py index bd7a647a..151bddfb 100755 --- a/odk/odk.py +++ b/odk/odk.py @@ -19,6 +19,7 @@ import yaml import os import glob +import fnmatch import subprocess import shutil from shutil import copy, copymode @@ -877,7 +878,7 @@ def save_project_yaml(project : OntologyProject, path : str): with open(path, "w") as f: f.write(yaml.dump(json_obj, default_flow_style=False)) -def unpack_files(basedir, txt, policies={}): +def unpack_files(basedir, txt, policies=[]): """ This unpacks a custom tar-like format in which multiple file paths can be specified, separated by ^^^s @@ -937,13 +938,25 @@ def must_install_file(templatefile, targetfile, policies): Given a template filename, indicate whether the file should be installed according to any per-file policy. - policies is a dictionary associating a template filename to one - of the following three values: - * IF_MISSING (default): install the file if it does not already exist + policies is a list of (PATTERN,POLICY) tuples where PATTERN is + a shell-like globbing pattern and POLICY is the update policy + that should be applied to any template whose pathname matches + the pattern. + + Patterns are tested in the order they are found in the list, + and the first match takes precedence over any subsequent match. + If there is no match, the default policy is IF_MISSING. + + Valid policies are: + * IF_MISSING: install the file if it does not already exist * ALWAYS: always install the file, overwrite any existing file * NEVER: never install the file """ - policy = policies.get(templatefile, IF_MISSING) + policy = IF_MISSING + for pattern, pattern_policy in policies: + if fnmatch.fnmatch(templatefile, pattern): + policy = pattern_policy + break if policy == ALWAYS: return True elif policy == NEVER: @@ -951,7 +964,7 @@ def must_install_file(templatefile, targetfile, policies): else: return not os.path.exists(targetfile) -def install_template_files(generator, templatedir, targetdir, policies={}): +def install_template_files(generator, templatedir, targetdir, policies=[]): """ Installs all template-derived files into a target directory. """ @@ -1086,30 +1099,24 @@ def update(templatedir): # missing (e.g. DOSDP example files) or on the contrary # always reinstalled to overwrite any local changes (e.g. # the main Makefile). We declare the corresponding policies. - policies = {} - policies['CODE_OF_CONDUCT.md'] = NEVER - policies['CONTRIBUTING.md'] = NEVER - policies['issue_template.md'] = NEVER - policies['README.md'] = NEVER - policies['src/patterns/data/default/example.tsv'] = NEVER - policies['src/patterns/dosdp-patterns/example.yaml'] = NEVER - policies['src/ontology/Makefile'] = ALWAYS - policies['src/ontology/run.sh'] = ALWAYS - policies['docs/odk-workflows'] = ALWAYS + policies = [ + ('CODE_OF_CONDUCT.md', NEVER), + ('CONTRIBUTING.md', NEVER), + ('issue_template.md', NEVER), + ('README.md', NEVER), + ('src/patterns/data/default/example.tsv', NEVER), + ('src/patterns/dosdp-patterns/example.yaml', NEVER), + ('src/ontology/Makefile', ALWAYS), + ('src/ontology/run.sh', ALWAYS), + ('src/sparql/*', ALWAYS), + ('docs/odk-workflows/*', ALWAYS) + ] if 'github_actions' in project.ci: for workflow in ['qc', 'diff', 'release-diff']: if workflow in project.workflows: - policies['.github/workflows/' + workflow + '.yml'] = ALWAYS + policies.append(('.github/workflows/' + workflow + '.yml', ALWAYS)) if project.documentation is not None and 'docs' in project.workflows: - policies['github/workflows/docs.yml'] = ALWAYS - # HACK: For the odk-workflows documentation directory, - # we want to clean up that directory completely. The - # policy dictionary only works on files, so to force - # reinstalling the entire directory (also removing any - # non-standard file along the way), we forcefully - # remove the directory. - if project.documentation is not None: - shutil.rmtree('../../docs/odk-workflows') + policies.append(('.github/workflows/docs.yml', ALWAYS)) # Proceed with template instantiation, using the policies # declared above. We instantiate directly at the root of