removing mireot adding new import params

allowing is_large, use_base, use_gzipped on imports

fixes #396

Author: matentzn

odk/odk.py

@@ -104,6 +104,15 @@ class ImportProduct(Product):
 
     mirror_from: Optional[Url] = None
     """if specified this URL is used rather than the default OBO PURL for the main OWL product"""
+    
+    is_large: bool = False
+    """if large, ODK may take measures to reduce the memory footprint of the import."""
+    
+    use_base: bool = False
+    """if use_base is true, try use the base IRI instead of normal one to mirror from."""
+    
+    use_gzipped: bool = False
+    """if use_gzipped is true, try use the base IRI instead of normal one to mirror from."""
 
 @dataclass_json
 @dataclass
@@ -228,7 +237,7 @@ class ImportGroup(ProductGroup):
     """all import products"""
 
     module_type : str = "slme"
-    """Module type. Supported: slme, mireot, minimal, custom"""
+    """Module type. Supported: slme, minimal, custom"""
 
     module_type_slme : str = "BOT"
     """SLME module type. Supported: BOT, TOP, STAR"""

template/src/ontology/Makefile.jinja2

@@ -254,12 +254,6 @@ imports/%_import.owl: mirror/%.owl imports/%_terms_combined.txt
 		query --update ../sparql/inject-subset-declaration.ru --update ../sparql/postprocess-module.ru \
 		remove --term rdfs:label -T imports/$*_terms_combined.txt --select complement --select "classes individuals annotation-properties" \
 		annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) --output $@.tmp.owl && mv $@.tmp.owl $@; fi
-{% elif 'mireot' == project.import_group.module_type %}
-imports/%_import.owl: mirror/%.owl imports/%_terms_combined.txt
-	if [ $(IMP) = true ]; then $(ROBOT) query  -i $< --update ../sparql/preprocess-module.ru \
-		extract -L imports/$*_terms_combined.txt -U imports/$*_terms_combined.txt --force true --copy-ontology-annotations true --method MIREOT \
-		query --update ../sparql/inject-subset-declaration.ru --update ../sparql/postprocess-module.ru \
-		annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) --output $@.tmp.owl && mv $@.tmp.owl $@; fi
 {% elif 'custom' == project.import_group.module_type %}
 imports/%_import.owl: mirror/%.owl
 	echo "ERROR: You have configured your default module type to be custom; this behavior needs to be overwritten in {{ project.id }}.Makefile!" && false
@@ -272,7 +266,29 @@ imports/%_import.owl: mirror/%.owl
 imports/%_import.obo: imports/%_import.owl
 	if [ $(IMP) = true ]; then $(ROBOT) convert --check false -i $< -f obo -o $@.tmp.obo && mv $@.tmp.obo $@; fi
 {% endif -%}
-{% endif %}
+
+{%- for ont in project.import_group.products -%}
+{% if ont.is_large  -%}
+## Special treatment for large ontology: {{ ont.id }}
+{% if 'slme' == project.import_group.module_type %}
+imports/{{ ont.id }}_import.owl: mirror/{{ ont.id }}.owl imports/{{ ont.id }}_terms_combined.txt
+	if [ $(IMP) = true ]; then $(ROBOT) extract -i $< -T imports/{{ ont.id }}_terms_combined.txt --force true --method {{ project.import_group.module_type_slme }} \
+		query --update ../sparql/inject-subset-declaration.ru \
+		annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) --output $@.tmp.owl && mv $@.tmp.owl $@; fi
+{% elif 'minimal' == project.import_group.module_type %}
+imports/{{ ont.id }}_import.owl: mirror/{{ ont.id }}.owl imports/{{ ont.id }}_terms_combined.txt
+	if [ $(IMP) = true ]; then $(ROBOT) extract  -i $< -T imports/{{ ont.id }}_terms_combined.txt --force true --method BOT \
+		remove --base-iri $(URIBASE)"/$(shell echo {{ ont.id }} | tr a-z A-Z)_" --axioms external --preserve-structure false --trim false \
+		query --update ../sparql/inject-subset-declaration.ru \
+		remove --term rdfs:label -T imports/{{ ont.id }}_terms_combined.txt --select complement --select "classes individuals annotation-properties" \
+		annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) --output $@.tmp.owl && mv $@.tmp.owl $@; fi
+{% elif 'custom' == project.import_group.module_type %}
+imports/{{ ont.id }}_import.owl: mirror/{{ ont.id }}.owl
+	echo "ERROR: You have configured your default module type to be custom; this behavior needs to be overwritten in {{ project.id }}.Makefile!" && false
+{%- endif %}
+{%- endif %}
+{%- endfor %}
+{%- endif %}
 
 {% if project.components is not none %}
 # ----------------------------------------
@@ -319,13 +335,27 @@ mirror/{{ ont.id }}.trigger: $(SRC)
 mirror/{{ ont.id }}.trigger:
 	touch $@
 {% endif -%}
-{% if ont.mirror_from %}
+
+{%- if ont.mirror_from %}
 mirror/{{ ont.id }}.owl: mirror/{{ ont.id }}.trigger
 	if [ $(MIR) = true ] && [ $(IMP) = true ]; then $(ROBOT) convert -I {{ ont.mirror_from }} -o $@.tmp.owl && mv $@.tmp.owl $@; fi
-{% else %}
+{%- elif ont.use_base %}
+{%- if ont.use_gzipped %}
+mirror/{{ ont.id }}.owl: mirror/{{ ont.id }}.trigger
+	if [ $(MIR) = true ] && [ $(IMP) = true ]; then $(ROBOT) convert -I $(URIBASE)/{{ ont.id }}/{{ ont.id }}-base.owl.gz -o $@.tmp.owl && mv $@.tmp.owl $@; fi
+{%- else %}
+mirror/{{ ont.id }}.owl: mirror/{{ ont.id }}.trigger
+	if [ $(MIR) = true ] && [ $(IMP) = true ]; then $(ROBOT) convert -I $(URIBASE)/{{ ont.id }}/{{ ont.id }}-base.owl -o $@.tmp.owl && mv $@.tmp.owl $@; fi
+{%- endif %}
+{%- else %}
+{%- if ont.use_gzipped %}
+mirror/{{ ont.id }}.owl: mirror/{{ ont.id }}.trigger
+	if [ $(MIR) = true ] && [ $(IMP) = true ]; then $(ROBOT) convert -I $(URIBASE)/{{ ont.id }}.owl.gz -o $@.tmp.owl && mv $@.tmp.owl $@; fi
+{%- else %}
 mirror/{{ ont.id }}.owl: mirror/{{ ont.id }}.trigger
 	if [ $(MIR) = true ] && [ $(IMP) = true ]; then $(ROBOT) convert -I $(URIBASE)/{{ ont.id }}.owl -o $@.tmp.owl && mv $@.tmp.owl $@; fi
-{% endif %}
+{%- endif %}
+{%- endif %}
 .PRECIOUS: mirror/%.owl
 {% endfor -%}
 {% endif %}

tests/test-module-minimal-large.yaml

@@ -0,0 +1,17 @@
+id: modmini
+title: "Test ROBOT extract minimal with base and large option"
+github_org: INCATools
+repo: ontology-development-kit
+edit_format: obo
+export_formats:
+  - owl
+  - obo
+  - json
+import_group:
+  products:
+    - id: ro
+      use_base: TRUE
+    - id: pato
+      is_large: TRUE
+  module_type: minimal
+