dryad deposit: remove unneeded tarballs, trim down otts

jar398 · jar398 · commit e375305af1c1 · 2017-04-13T11:09:16.000-04:00
diff --git a/doc/Interim-taxonomy-file-format.md b/doc/Interim-taxonomy-file-format.md
@@ -43,7 +43,7 @@ Example (from NCBI):
 * _uniqueName_: a human-readable string that is unique to this taxon, typically the taxon name if it is unique, or taxon name followed by "([rank] in [ancestor])" where rank is the taxon's rank and ancestor is an ancestor that is unique to this taxon (among the taxa that have the same name).
 * _flags_: a comma-separated list of flags or markers.  Usually these are generated by taxonomy synthesis and are used to decide whether a taxon is 'hidden' or not.  For example, if there's an 'extinct' flag then it may be desirable to suppress the taxon in an application.  See [here](https://github.com/OpenTreeOfLife/taxomachine/blob/master/src/main/java/org/opentree/taxonomy/OTTFlag.java).
 
-### Synonyms
+### File `synonyms.tsv`
 
 Usually there are synonyms.  These go into a second file, `synonyms.tsv`.  This file must have a header row
 
@@ -60,11 +60,26 @@ Example from NCBI:
 
         89373	|	Flexibacteraceae	|	synonym	|	|	
 
-### Metadata
+### File `forwards.tsv`
 
-Overall metadata for the taxonomy is placed in a separate file.  The metadata format is currently under development. `Smasher` generates this in JSON format as `about.json`, but this file is currently not used programmatically, and is in the process of being overhauled. When generating a taxonomy according to this format in external tools, for now it is best to simply write a markdown or plain text file called `about.md` (in the same directory as `taxonomy.tsv` and `synonyms.tsv`).
+This file provides aliases, resulting from a situation where one taxon
+id has been discovered to be equivalent to another.  This can be due
+to changes in the the way the taxonomy is processed, discovery of new
+synonyms, or due to merge events ("lumping").  For example:
 
-The metadata provided in the file should include the source of the taxonomy (article or database) as a URL and any other descriptive information that's available.  The purpose of the metadata is not just explanatory but also to explain how to check the correctness of the taxonomy against its source and make corrections and other improvements should the source be updated. When using information from changing sources (databases) the date or dates of retrieval should be recorded.
+    id      replacement
+    3434315 3434301
+    5255304 828663
+
+The second line says that older id 3434315 (occurring in one or more
+previous versions) should be replaced by newer 3434301 (defined in
+this version).
+
+
+### File `version.txt`
+
+When OTT is generated, the version number is placed in this file,
+e.g. `ott3.0draft6`.
 
 ***
 
diff --git a/doc/method/data-package/Makefile b/doc/method/data-package/Makefile
@@ -1,29 +1,74 @@
 # Creates a bunch of .tgz files, to be uploaded to Dryad one at a time.
 # Be sure to upload README as well.
 
-# This Makefile needs to be run in a directory that has just successfully built OTT 3.0 draft 3.
+# This Makefile needs to be run in a directory that has successfully
+# built OTT 3.0, so that all the processed source taxonomies are available.
 
 F=files
 REF=../../..
 DRAFT=
 
-all: $F/ott3.0.tgz $F/separation.tgz $F/silva.tgz $F/worms.tgz $F/fung.tgz $F/ncbi.tgz $F/gbif.tgz $F/irmng.tgz \
-  $F/accessions.tgz $F/amendments.tgz $F/ott2.10.tgz $F/by_qid.tgz $F/amendments.tgz $F/README
+all: $F/README $F/ott3.0.tgz $F/separation.tgz \
+  $F/accessions.tgz $F/amendments.tgz $F/ott2.10.tgz $F/by_qid.tgz 
+
+# Decided not to include these
+# $F/silva.tgz $F/worms.tgz $F/fung.tgz $F/ncbi.tgz $F/gbif.tgz $F/irmng.tgz
 
 $F/README: README
+	mkdir -p $F
 	cp -p $< $@
 
-$F/ott3.0.tgz:
-	wget -O $@ http://files.opentreeoflife.org/ott/ott3.0/ott3.0.tgz
-
-$F/ott2.10.tgz:
-	wget -O $@ http://files.opentreeoflife.org/ott/ott2.10/ott2.10.tgz
-
+$F/ott3.0.tgz: cache/ott3.0.tgz readme-ott-3.0.txt
+	rm -rf ott tax/ott3.0
+	mkdir -p tax $F
+	tar xzf cache/ott3.0.tgz \
+	    ott/taxonomy.tsv ott/synonyms.tsv ott/forwards.tsv ott/version.txt
+	mv ott tax/ott3.0
+	cp -p readme-ott-3.0.txt tax/ott3.0/README
+	tar czf $@ tax/ott3.0
+
+# For assigning identifiers only
+
+$F/ott2.10.tgz: cache/ott2.10.tgz readme-ott-2.10.txt
+	rm -rf ott tax/ott2.10
+	mkdir -p tax $F
+	tar xzf cache/ott2.10.tgz \
+	    ott/taxonomy.tsv ott/synonyms.tsv ott/forwards.tsv ott/version.txt
+	mv ott tax/ott2.10
+	cp -p readme-ott-3.0.txt tax/ott2.10/README
+	tar czf $@ tax/ott2.10
+
+cache/ott3.0.tgz:
+	mkdir -p cache
+	wget -O $@.new http://files.opentreeoflife.org/ott/ott3.0/ott3.0.tgz
+	mv $@.new $@
+
+cache/ott2.10.tgz:
+	mkdir -p cache
+	wget -O $@.new $@ http://files.opentreeoflife.org/ott/ott2.10/ott2.10.tgz
+	mv $@.new $@
+
+# The file is misnamed on the server.  It's really a .tar.gz file, not a .csv.gz file.
 $F/by_qid.tgz:
-	wget -O $@ http://files.opentreeoflife.org/idlist/idlist-20161118/by_qid.csv.gz
+	wget -O $@.new http://files.opentreeoflife.org/idlist/idlist-20161118/by_qid.csv.gz
+	mv $@.new $@
 
+# Copy the separation taxonomy from the git repo.
 $F/separation.tgz: $(REF)/tax/separation/taxonomy.tsv
-	tar czf $@ -C $(REF) tax/separation
+	tar czf $@ -C $(REF) --exclude "*~" tax/separation
+
+# Genbank accessions
+$F/accessions.tgz: $(REF)/feed/silva/work/accessions.tsv
+	tar czf $@  -C $(REF) feed/silva/work/accessions.tsv
+
+# Curator additions
+$F/amendments.tgz: $(REF)/feed/amendments/amendments-1
+	tar czf $@ -C $(REF) --exclude "\\.git" --exclude LICENSE feed/amendments
+
+clean:
+	rm -rf files tax
+
+# Don't put these in the data package.
 
 $F/silva.tgz: $(REF)/tax/silva/taxonomy.tsv
 	tar czf $@ -C $(REF) tax/silva
@@ -43,15 +88,3 @@ $F/gbif.tgz: $(REF)/tax/gbif/taxonomy.tsv
 $F/irmng.tgz: $(REF)/tax/irmng/taxonomy.tsv
 	tar czf $@ -C $(REF) tax/irmng
 
-# genbank accessions
-$F/accessions.tgz: $(REF)/feed/silva/work/accessions.tsv
-	tar czf $@  -C $(REF) feed/silva/work/accessions.tsv
-
-# amendments
-$F/amendments.tgz: $(REF)/feed/amendments/amendments-1
-	tar czf $@ -C $(REF) --exclude "\\.git" --exclude LICENSE feed/amendments
-
-# /amendments-1 
-# --exclude "\\.git" --exclude LICENSE 
-
-clean: rm *.tgz
diff --git a/doc/method/data-package/README b/doc/method/data-package/README
@@ -17,62 +17,20 @@ not the original sources.  Information about sources is provided.
 "Source date" refers to the write date of the source file.  "Access
 date" is the date on which the source file was retrieved.
 
-All files are GNU 'tar' archives, compressed using gzip compression.
+Except where noted, all files are GNU 'tar' archives, compressed using
+gzip compression.
 
 -----
 
 File:          ott3.0.tgz
-Description:   Open Tree Taxonomy version 3.0
+Description:   Open Tree Taxonomy version 3.0 (draft 6)
+Created:       26 February 2017
+Note:          See https://github.com/OpenTreeOfLife/reference-taxonomy/blob/master/doc/Interim-taxonomy-file-format.md
 
 File:          separation.tgz
 Description:   Separation taxonomy
 Note:          Same as what's found in the code repository.
 
-File:          silva.tgz
-Description:   SILVA Taxonomy version 115
-Source URL:    ftp://ftp.arb-silva.de/release_115/Exports/SSURef_NR99_115_tax_silva.fasta.tgz
-Source date:   7 September 2013
-Access date:   1 November 2013
-Source length: 816923384 bytes
-
-File:          fung.tgz
-Description:   Index Fungorum
-Source URL:    derived from database query result files provided by Paul Kirk
-Access date:   7 April 2014
-
-File:          worms.tgz
-Description:   WoRMS taxonomy
-Source URL:    accessed via web service described at http://www.marinespecies.org/aphia.php?p=webservice
-Access date:   1 October 2015
-
-File:          ncbi.tgz
-Description:   NCBI Taxonomy
-Source URL:    ftp://ftp.ncbi.nih.gov/pub/taxonomy/taxdump.tar.gz
-Source date:   9 November 2016
-Access date:   9 November 2016
-Source length: 37595767 bytes
-
-File:          gbif.tgz
-Description:   GBIF backbone taxonomy
-Source URL:    http://rs.gbif.org/datasets/backbone/backbone.zip
-Source date:   29 July 2016
-Access date:   11 November 2016
-Source length: 347031944 bytes
-
-File:          irmng.tgz
-Description:   IRMNG
-Source URL:    http://www.cmar.csiro.au/datacentre/downloads/IRMNG_DWC.zip
-Source date:   31 January 2014
-Access date:   7 June 2014
-Note:          We obtained a separate extinct/extant annotations file from Tony Rees of CSIRO
-Source length: IRMNG_DWC.zip 111780936, IRMNG_DWC_SP_PROFILE.csv 62845750 bytes
-
-File:          prev_ott.tgz
-Description:   OTT version 2.10 (used only for id assignment)
-
-File:          by_qid.csv.gz
-Description:   Cumulative source id to OTT id mappings (used only for id assignment)
-
 File:          amendments.tgz
 Description:   Taxonomy additions submitted using Open Tree curation tool
 Source URL:    https://github.com/opentreeoflife/additions-1
@@ -86,7 +44,10 @@ Source URL:    ftp://ftp.ncbi.nlm.nih.gov/genbank/
 Access date:   28 June 2016
 Note:          see script in source code repository
 
-"The selected license applies to all of your files displayed in the
-top of the form. If you want to upload some files under a different
-license, please do so in two separate uploads."
+File:          ott2.10.tgz
+Description:   OTT version 2.10 (used by version 3.0 only for id assignment)
+Created:       1 October 2016
 
+File:          by_qid.tgz
+Description:   Cumulative source id to OTT id mappings (used only for id assignment)
+Created:       18 November 2016
diff --git a/doc/method/data-package/more-README b/doc/method/data-package/more-README
@@ -0,0 +1,41 @@
+File:          silva.tgz
+Description:   SILVA Taxonomy version 115
+Source URL:    ftp://ftp.arb-silva.de/release_115/Exports/SSURef_NR99_115_tax_silva.fasta.tgz
+Source date:   7 September 2013
+Access date:   1 November 2013
+Source length: 816923384 bytes
+Note:          File ncbi_to_silva.tsv is a mapping from NCBI ids to SILVA 
+               reference sequence genbank ids.
+
+File:          fung.tgz
+Description:   Index Fungorum
+Source URL:    derived from database query result files provided by Paul Kirk
+Access date:   7 April 2014
+
+File:          worms.tgz
+Description:   WoRMS taxonomy
+Source URL:    accessed via web service described at http://www.marinespecies.org/aphia.php?p=webservice
+Access date:   1 October 2015
+
+File:          ncbi.tgz
+Description:   NCBI Taxonomy
+Source URL:    ftp://ftp.ncbi.nih.gov/pub/taxonomy/taxdump.tar.gz
+Source date:   9 November 2016
+Access date:   9 November 2016
+Source length: 37595767 bytes
+
+File:          gbif.tgz
+Description:   GBIF backbone taxonomy
+Source URL:    http://rs.gbif.org/datasets/backbone/backbone.zip
+Source date:   29 July 2016
+Access date:   11 November 2016
+Source length: 347031944 bytes
+
+File:          irmng.tgz
+Description:   IRMNG
+Source URL:    http://www.cmar.csiro.au/datacentre/downloads/IRMNG_DWC.zip
+Source date:   31 January 2014
+Access date:   7 June 2014
+Note:          We obtained a separate extinct/extant annotations file from Tony Rees of CSIRO
+Source length: IRMNG_DWC.zip 111780936, IRMNG_DWC_SP_PROFILE.csv 62845750 bytes
+
diff --git a/doc/method/data-package/readme-ott-2.10.txt b/doc/method/data-package/readme-ott-2.10.txt
@@ -0,0 +1,25 @@
+Open Tree Taxonomy 2.10
+
+Release notes for this version of the taxonomy:
+  https://github.com/OpenTreeOfLife/reference-taxonomy/blob/master/doc/ott2.10.md
+
+Files:
+
+  taxonomy.tsv
+    The taxonomy itself, with taxon id, parent, rank, source links, and other 
+    information
+
+  synonyms.tsv
+    Synonyms
+
+  forwards.tsv
+    List of id aliases
+
+  version.txt
+    Version number for this OTT version
+
+For descriptions of the file formats, see
+
+  https://github.com/OpenTreeOfLife/reference-taxonomy/blob/master/doc/Interim-taxonomy-file-format.md
+  https://github.com/OpenTreeOfLife/reference-taxonomy/blob/master/doc/taxon-flags.md
+
diff --git a/doc/method/data-package/readme-ott-3.0.txt b/doc/method/data-package/readme-ott-3.0.txt
@@ -0,0 +1,25 @@
+Open Tree Taxonomy 3.0
+
+Release notes for this version of the taxonomy:
+  https://github.com/OpenTreeOfLife/reference-taxonomy/blob/master/doc/ott3.0.md
+
+Files:
+
+  taxonomy.tsv
+    The taxonomy itself, with taxon id, parent, rank, source links, and other 
+    information
+
+  synonyms.tsv
+    Synonyms
+
+  forwards.tsv
+    List of id aliases
+
+  version.txt
+    Version number for this OTT version
+
+For descriptions of the file formats, see
+
+  https://github.com/OpenTreeOfLife/reference-taxonomy/blob/master/doc/Interim-taxonomy-file-format.md
+  https://github.com/OpenTreeOfLife/reference-taxonomy/blob/master/doc/taxon-flags.md
+
