diff --git a/docker/odklite/Dockerfile b/docker/odklite/Dockerfile index 4732a9ee..024fcf46 100644 --- a/docker/odklite/Dockerfile +++ b/docker/odklite/Dockerfile @@ -96,6 +96,9 @@ RUN sed -i '/secure_path/d' /etc/sudoers && \ COPY --chmod=755 scripts/odk-info.sh /tools/odk-info RUN sed -i s/@@ODK_IMAGE_VERSION@@/$ODK_VERSION/ /tools/odk-info +# Install a helper script to launch a repository update +COPY --chmod=755 scripts/update_repo.sh /tools/update_repo + # Install the ODK itself. COPY --chmod=755 odk/make-release-assets.py odk/odk.py /tools COPY template/ /tools/templates/ diff --git a/docs/FrequentlyUsedODKCommands.md b/docs/FrequentlyUsedODKCommands.md index 235f720b..b430aeb7 100644 --- a/docs/FrequentlyUsedODKCommands.md +++ b/docs/FrequentlyUsedODKCommands.md @@ -3,7 +3,7 @@ ## Updates the Makefile to the latest ODK ``` -sh run.sh make update_repo +sh run.sh update_repo ``` ## Recreates and deploys the automated documentation diff --git a/odk/odk.py b/odk/odk.py index 3bafd489..151bddfb 100755 --- a/odk/odk.py +++ b/odk/odk.py @@ -18,6 +18,8 @@ from dacite import from_dict import yaml import os +import glob +import fnmatch import subprocess import shutil from shutil import copy, copymode @@ -876,7 +878,7 @@ def save_project_yaml(project : OntologyProject, path : str): with open(path, "w") as f: f.write(yaml.dump(json_obj, default_flow_style=False)) -def unpack_files(basedir, txt): +def unpack_files(basedir, txt, policies=[]): """ This unpacks a custom tar-like format in which multiple file paths can be specified, separated by ^^^s @@ -887,16 +889,21 @@ def unpack_files(basedir, txt): lines = txt.split("\n") f = None tgts = [] + ignore = False for line in lines: if line.startswith(MARKER): - path = os.path.join(basedir, line.replace(MARKER, "")) - os.makedirs(os.path.dirname(path), exist_ok=True) + # Close previous file, if any if f != None: f.close() - f = open(path,"w") - tgts.append(path) - logging.info(' Unpacking into: {}'.format(path)) - else: + filename = line.replace(MARKER, "") + path = os.path.join(basedir, filename) + ignore = not must_install_file(filename, path, policies) + if not ignore: + os.makedirs(os.path.dirname(path), exist_ok=True) + f = open(path,"w") + tgts.append(path) + logging.info(' Unpacking into: {}'.format(path)) + elif not ignore: if f is None: if line == "": continue @@ -907,6 +914,92 @@ def unpack_files(basedir, txt): f.close() return tgts +def get_template_name(templatedir, pathname): + """ + Helper function to get the user-visible name of a template file + from its complete pathname in the template directory. + + For example, if the pathname is + "/tools/template/src/ontology/run.sh.jinja2", this will return + "src/ontology/run.sh". + """ + name = pathname.replace(templatedir, "") + if len(name) > 0 and name[0] == '/': + name = name[1:] + if name.endswith(TEMPLATE_SUFFIX): + name = name.replace(TEMPLATE_SUFFIX, "") + return name + +# Available policies for installing a file +IF_MISSING, ALWAYS, NEVER = range(3) + +def must_install_file(templatefile, targetfile, policies): + """ + Given a template filename, indicate whether the file should be + installed according to any per-file policy. + + policies is a list of (PATTERN,POLICY) tuples where PATTERN is + a shell-like globbing pattern and POLICY is the update policy + that should be applied to any template whose pathname matches + the pattern. + + Patterns are tested in the order they are found in the list, + and the first match takes precedence over any subsequent match. + If there is no match, the default policy is IF_MISSING. + + Valid policies are: + * IF_MISSING: install the file if it does not already exist + * ALWAYS: always install the file, overwrite any existing file + * NEVER: never install the file + """ + policy = IF_MISSING + for pattern, pattern_policy in policies: + if fnmatch.fnmatch(templatefile, pattern): + policy = pattern_policy + break + if policy == ALWAYS: + return True + elif policy == NEVER: + return False + else: + return not os.path.exists(targetfile) + +def install_template_files(generator, templatedir, targetdir, policies=[]): + """ + Installs all template-derived files into a target directory. + """ + tgts = [] + for root, subdirs, files in os.walk(templatedir): + tdir = root.replace(templatedir,targetdir+"/") + os.makedirs(tdir, exist_ok=True) + + # first copy plain files... + for f in [f for f in files if not f.endswith(TEMPLATE_SUFFIX)]: + srcf = os.path.join(root, f) + tgtf = os.path.join(tdir, f) + if must_install_file(get_template_name(templatedir, srcf), tgtf, policies): + logging.info(' Copying: {} -> {}'.format(srcf, tgtf)) + # copy file directly, no template expansions + copy(srcf, tgtf) + tgts.append(tgtf) + logging.info('Applying templates') + # ...then apply templates + for f in [f for f in files if f.endswith(TEMPLATE_SUFFIX)]: + srcf = os.path.join(root, f) + tgtf = os.path.join(tdir, f) + derived_file = tgtf.replace(TEMPLATE_SUFFIX, "") + if f.startswith("_dynamic"): + logging.info(' Unpacking: {}'.format(derived_file)) + tgts += unpack_files(tdir, generator.generate(srcf), policies) + elif must_install_file(get_template_name(templatedir, srcf), derived_file, policies): + logging.info(' Compiling: {} -> {}'.format(srcf, derived_file)) + with open(derived_file,"w") as s: + s.write(generator.generate(srcf)) + tgts.append(derived_file) + copymode(srcf, derived_file) + return tgts + + ## ======================================== ## Command Line Wrapper ## ======================================== @@ -980,6 +1073,63 @@ def dump_schema(class_name): clazz = globals()[class_name] # Get the class object from the globals dictionary print(json.dumps(clazz.json_schema(), sort_keys=True, indent=4)) +@cli.command() +@click.option('-T', '--templatedir', default='/tools/templates/') +def update(templatedir): + """ + Updates a pre-existing repository. This command is expected to be + run from within the src/ontology directory (the directory + containing the configuration file). + """ + config_matches = list(glob.glob('*-odk.yaml')) + if len(config_matches) == 0: + raise click.ClickException("No ODK configuration file found") + elif len(config_matches) > 1: + raise click.ClickException("More than ODK configuration file found") + config = config_matches[0] + mg = Generator() + mg.load_config(config) + project = mg.context.project + + # When updating, for most files, we only install them if + # they do not already exist in the repository (typically + # because they are new files that didn't exist in the + # templates of the previous version of the ODK). But a + # handful of files are not reinstalled even if they are + # missing (e.g. DOSDP example files) or on the contrary + # always reinstalled to overwrite any local changes (e.g. + # the main Makefile). We declare the corresponding policies. + policies = [ + ('CODE_OF_CONDUCT.md', NEVER), + ('CONTRIBUTING.md', NEVER), + ('issue_template.md', NEVER), + ('README.md', NEVER), + ('src/patterns/data/default/example.tsv', NEVER), + ('src/patterns/dosdp-patterns/example.yaml', NEVER), + ('src/ontology/Makefile', ALWAYS), + ('src/ontology/run.sh', ALWAYS), + ('src/sparql/*', ALWAYS), + ('docs/odk-workflows/*', ALWAYS) + ] + if 'github_actions' in project.ci: + for workflow in ['qc', 'diff', 'release-diff']: + if workflow in project.workflows: + policies.append(('.github/workflows/' + workflow + '.yml', ALWAYS)) + if project.documentation is not None and 'docs' in project.workflows: + policies.append(('.github/workflows/docs.yml', ALWAYS)) + + # Proceed with template instantiation, using the policies + # declared above. We instantiate directly at the root of + # the repository -- no need for a staging directory. + install_template_files(mg, templatedir, '../..', policies) + + print("WARNING: These files should be manually migrated:") + print(" mkdocs.yaml, .gitignore, src/ontology/catalog.xml") + print(" (if you added a new import or component)") + if 'github_actions' in project.ci and 'qc' not in project.workflows: + print("WARNING: Your QC workflows have not been updated automatically.") + print(" Please update the ODK version number in .github/workflows/qc.yml") + print("Ontology repository update successfully completed.") @cli.command() @click.option('-C', '--config', type=click.Path(exists=True), @@ -1039,38 +1189,7 @@ def seed(config, clean, outdir, templatedir, dependencies, title, user, source, if not os.path.exists(templatedir) and templatedir == "/tools/templates/": logging.info("No templates folder in /tools/; assume not in docker context") templatedir = "./template" - for root, subdirs, files in os.walk(templatedir): - tdir = root.replace(templatedir,outdir+"/") - os.makedirs(tdir, exist_ok=True) - - # first copy plain files... - for f in files: - srcf = os.path.join(root, f) - tgtf = os.path.join(tdir, f) - logging.info(' Copying: {} -> {}'.format(srcf, tgtf)) - if not tgtf.endswith(TEMPLATE_SUFFIX): - # copy file directly, no template expansions - copy(srcf, tgtf) - tgts.append(tgtf) - logging.info('Applying templates') - # ...then apply templates - for f in files: - srcf = os.path.join(root, f) - tgtf = os.path.join(tdir, f) - if srcf.endswith(TEMPLATE_SUFFIX): - derived_file = tgtf.replace(TEMPLATE_SUFFIX, "") - with open(derived_file,"w") as s: - if f.startswith("_dynamic"): - logging.info(' Unpacking: {}'.format(derived_file)) - tgts += unpack_files(tdir, mg.generate(srcf)) - s.close() - os.remove(derived_file) - else: - logging.info(' Compiling: {} -> {}'.format(srcf, derived_file)) - s.write(mg.generate(srcf)) - tgts.append(derived_file) - if not f.startswith("_dynamic"): - copymode(srcf, derived_file) + tgts += install_template_files(mg, templatedir, outdir) tgt_project_file = "{}/project.yaml".format(outdir) if project.export_project_yaml: diff --git a/scripts/update_repo.sh b/scripts/update_repo.sh new file mode 100644 index 00000000..20db4cf1 --- /dev/null +++ b/scripts/update_repo.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +exec /tools/odk.py update diff --git a/template/src/ontology/Makefile.jinja2 b/template/src/ontology/Makefile.jinja2 index 9c62559e..11949e1d 100644 --- a/template/src/ontology/Makefile.jinja2 +++ b/template/src/ontology/Makefile.jinja2 @@ -1391,9 +1391,10 @@ normalize_src: $(SRC) validate_idranges: dicer-cli {{ project.id }}-idranges.owl +# Deprecated: Use 'sh run.sh odk.py update' without using the Makefile. .PHONY: update_repo update_repo: - sh $(SCRIPTSDIR)/update_repo.sh + odk.py update {% if project.documentation is not none %} update_docs: @@ -1427,7 +1428,7 @@ Usage: [IMAGE=(odklite|odkfull)] [ODK_DEBUG=yes] sh run.sh make [(IMP|MIR|IMP_LA Core commands: * prepare_release: Run the entire release pipeline. Use make IMP=false prepare_release to avoid rerunning the imports * prepare_release_fast: Run the entire release pipeline without refreshing imports, recreating components or recompiling patterns. -* update_repo: Update the ODK repository setup using the config file {{ project.id }}-odk.yaml +* update_repo: Update the ODK repository setup using the config file {{ project.id }}-odk.yaml (DEPRECATED) * test: Running all validation tests * test_fast: Runs the test suite, but without updating imports or components * odkversion: Show the current version of the ODK Makefile and ROBOT. @@ -1479,7 +1480,6 @@ Additional QC commands (advanced users) Examples: * sh run.sh make IMP=false prepare_release -* sh run.sh make update_repo * sh run.sh make test Tricks: @@ -1487,6 +1487,10 @@ Tricks: * Use the IMAGE parameter to the run.sh script to use a different image like odklite * Use ODK_DEBUG=yes sh run.sh make ... to print information about timing and debugging +Updating the repository: +(to apply changes to the ODK configuration or switch to a newer ODK version) +* sh run.sh update_repo + endef export data diff --git a/template/src/ontology/run.sh.jinja2 b/template/src/ontology/run.sh.jinja2 index 83959d2e..dea05ad7 100755 --- a/template/src/ontology/run.sh.jinja2 +++ b/template/src/ontology/run.sh.jinja2 @@ -14,6 +14,16 @@ # # See README-editors.md for more details. +case "$@" in +*update_repo*) + # Execute a copy of this script so that it can be overwritten + # during the update process + mkdir -p tmp + sed -n '/^set -e/,$p' $0 > tmp/$0 + exec $SHELL tmp/$0 "$@" + ;; +esac + set -e # Check for spaces in the current directory path diff --git a/template/src/scripts/update_repo.sh.jinja2 b/template/src/scripts/update_repo.sh.jinja2 deleted file mode 100644 index 65411ad4..00000000 --- a/template/src/scripts/update_repo.sh.jinja2 +++ /dev/null @@ -1,44 +0,0 @@ -echo "This (experimental) update script will create a new repo according to your config file. It will:" -echo "(1) overwrite your repositories Makefile, ODK sparql queries (your custom queries wont be touched) and docker wrapper (run.sh)." -echo "(2) and add missing files, if any." - -set -e - -OID={{project.id}} -ROOTDIR=../.. -SRCDIR=.. -CONFIG=$OID"-odk.yaml" - -rm -rf target -mkdir target -/tools/odk.py seed -c -g -C $CONFIG -ls -l target/$OID/src -ls -l $SRCDIR/ -cp target/$OID/src/scripts/update_repo.sh $SRCDIR/scripts/ -rsync -r -u --ignore-existing --exclude 'patterns/data/default/example.tsv' --exclude 'patterns/dosdp-patterns/example.yaml' target/$OID/src/ $SRCDIR/ -{% if project.documentation is not none %} -rsync -r -u --ignore-existing target/$OID/docs/ $ROOTDIR/docs -mkdir -p $ROOTDIR/docs/odk-workflows -rm -rf $ROOTDIR/docs/odk-workflows/* -cp -r target/$OID/docs/odk-workflows/* $ROOTDIR/docs/odk-workflows -{% endif -%} -cp target/$OID/src/ontology/Makefile $SRCDIR/ontology/ -cp target/$OID/src/ontology/run.sh $SRCDIR/ontology/ -cp -r target/$OID/src/sparql/* $SRCDIR/sparql/ -{%- if project.ci is defined -%}{% if 'travis' in project.ci %} -cp -n target/$OID/.travis.yml $ROOTDIR/ -{%- endif -%}{% if 'github_actions' in project.ci %} -mkdir -p $ROOTDIR/.github -mkdir -p $ROOTDIR/.github/workflows -{% if 'qc' in project.workflows %}cp target/$OID/.github/workflows/qc.yml $ROOTDIR/.github/workflows/qc.yml{% endif %} -{% if 'diff' in project.workflows %}cp target/$OID/.github/workflows/diff.yml $ROOTDIR/.github/workflows/diff.yml{% endif %} -{% if 'release-diff' in project.workflows %}cp target/$OID/.github/workflows/release-diff.yml $ROOTDIR/.github/workflows/release-diff.yml{% endif %} -{% if project.documentation is not none %} -{% if 'docs' in project.workflows %}cp target/$OID/.github/workflows/docs.yml $ROOTDIR/.github/workflows/docs.yml{% endif %} -{% endif %}{% endif %}{% endif %} -{% if project.documentation is not none %} -cp -n target/$OID/mkdocs.yaml $ROOTDIR/ -{% endif %} -echo "WARNING: These files should be manually migrated: mkdocs.yaml, .gitignore, src/ontology/catalog.xml (if you added a new import or component)" -{% if 'github_actions' in project.ci and 'qc' not in project.workflows %}echo "WARNING: Your QC workflows have not been updated automatically. Please update the ODK version number(s) in .github/workflows/qc.yml."{% endif %} -echo "Ontology repository update successfully completed."