2.6 final

* Add report on conflicts ('tattered')
* Add Cyphellopsis
* Fix Epiphloea alignment
* Diadasia is not extinct
* Beginnings of better flag input processing (not yet deployed)

Author: jar398

Makefile

@@ -5,7 +5,7 @@
 # Get it from http://files.opentreeoflife.org/ott/
 # and if there's a file "taxonomy" change that to "taxonomy.tsv".
 
-WHICH=2.6draft5
+WHICH=2.6
 PREV_WHICH=2.5
 
 #  $^ = all prerequisites

make-ott.py

@@ -61,6 +61,18 @@
 
 # Index Fungorum
 fung  = Taxonomy.getTaxonomy('tax/if/', 'if')
+
+# JAR 2014-04-11 Missing in earlier IF, mistake in later IF -
+# extraneous authority string
+cyph = fung.taxon('Cyphellopsis')
+if cyph == None:
+	cyph = fung.taxon('Cyphellopsis Donk 1931')
+	if cyph != None:
+		cyph.rename('Cyphellopsis')
+	else:
+		cyph = fung.newTaxon('Cyphellopsis', 'genus', 'if:17439')
+fung.taxon('Niaceae').take(cyph)
+
 # smush will fold sibling taxa that have the same name.
 fung.smush()
 
@@ -116,6 +128,10 @@
 ott.notSame(ncbi.taxon('Burkea'), fung.taxon('Burkea'))
 ott.notSame(ncbi.taxon('Coscinium'), fung.taxon('Coscinium'))
 ott.notSame(ncbi.taxon('Perezia'), fung.taxon('Perezia'))
+
+# JAR 2014-04-11 Discovered during regression testing
+ott.notSame(ncbi.taxon('Epiphloea', 'Rhodophyta'), fung.taxon('Epiphloea', 'Lichinaceae'))
+
 # analyzeOTUs sets flags on questionable taxa ("unclassified",
 #  hybrids, and so on) to allow the option of suppression downstream
 ncbi.analyzeOTUs()
@@ -410,15 +426,15 @@
 # TBD: Change species names from Norops X to Anolis X for all X
 ott.taxon('Anolis').absorb(ott.taxon('Norops', 'Iguanidae'))
 
-# JAR 2014-4-08 - these are in study OTUs - see IRMNG
+# JAR 2014-04-08 - these are in study OTUs - see IRMNG
 ott.taxon('Inseliellum').extant()
 ott.taxon('Conus', 'Gastropoda').extant()
 ott.taxon('Patelloida').extant()
 ott.taxon('Phyllanthus', 'Phyllanthaceae').extant()
 ott.taxon('Stelis','Orchidaceae').extant()
 ott.taxon('Chloris', 'Poaceae').extant()
 ott.taxon('Acropora', 'Acroporidae').extant()
-
+ott.taxon('Diadasia').extant()
 
 # -----------------------------------------------------------------------------
 # Finish up

org/opentreeoflife/smasher/Flag.java

@@ -0,0 +1,65 @@
+package org.opentreeoflife.smasher;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public enum Flag {
+
+	// NCBI - individually troublesome - not sticky - combine using &
+	NOT_OTU			     ("not_otu", "not_otu", Taxonomy.NOT_OTU),
+	VIRAL				 ("viral", "viral", Taxonomy.VIRAL),
+	HYBRID				 ("hybrid", "hybrid", Taxonomy.HYBRID),
+
+	// Final analysis...
+	// Containers - unconditionally so.
+	INCERTAE_SEDIS		 ("incertae_sedis_direct", "incertae_sedis_inherited", Taxonomy.INCERTAE_SEDIS),
+	UNCLASSIFIED		 ("unclassified_direct", "unclassified_inherited", Taxonomy.UNCLASSIFIED),
+	ENVIRONMENTAL		 ("environmental", "environmental_inherited", Taxonomy.ENVIRONMENTAL),
+
+	// Set during assembly
+	HIDDEN				 ("hidden", "hidden_inherited", Taxonomy.HIDDEN),	  // combine using &
+	MAJOR_RANK_CONFLICT  ("major_rank_conflict_direct", "major_rank_conflict_inherited", Taxonomy.MAJOR_RANK_CONFLICT),     // Parent-dependent.  Retain value
+
+	// Australopithecus
+	SIBLING_HIGHER		 ("sibling_higher", null, Taxonomy.SIBLING_HIGHER),
+	SIBLING_LOWER		 ("sibling_lower", null, Taxonomy.SIBLING_LOWER),
+
+	TATTERED			 ("tattered", "tattered_inherited", Taxonomy.TATTERED),  // combine using |
+	EDITED				 ("edited", null, Taxonomy.EDITED),	  				  // combine using |
+	FORCED_VISIBLE		 ("forced_visible", null, Taxonomy.FORCED_VISIBLE),	  		  // combine using |
+	EXTINCT			 	 ("extinct_direct", "extinct_inherited", Taxonomy.EXTINCT),	  // combine using |
+
+	// Is 'species' or lower rank ('infraspecific' when inherited)
+	// Unconditional ?
+	SPECIFIC			 (null, "infraspecific", Taxonomy.SPECIFIC),
+
+	// Opposite of 'barren' - propagated upward
+	ANYSPECIES			 (null, null, Taxonomy.ANYSPECIES);
+
+	String name, inheritedName;
+	int bit;
+
+	Flag(String name, String inheritedName, int bit) {
+		this.name = name;
+		this.inheritedName = inheritedName;
+		this.bit = bit;
+	}
+
+	static final Map<String, Flag> lookupTable = new HashMap<String, Flag>();
+	static final Map<String, Flag> lookupInheritedTable = new HashMap<String, Flag>();
+	static {
+		for (Flag flag : Flag.values())
+			lookupTable.put(flag.name, flag);
+		for (Flag flag : Flag.values())
+			lookupInheritedTable.put(flag.inheritedName, flag);
+	}
+
+	static Flag lookup(String name) {
+		return lookupTable.get(name);
+	}
+
+	static Flag lookupInherited(String name) {
+		return lookupInheritedTable.get(name);
+	}
+
+}

org/opentreeoflife/smasher/Taxon.java

@@ -309,12 +309,13 @@ QualifiedId getQualifiedId() {
 			return new QualifiedId(this.taxonomy.getTag(), this.id);
 		else {
 			// What if from a Newick string?
-			System.err.println("!? [getQualifiedId] Taxon has no id: " + this.name);
-			return new QualifiedId("?", this.name);
+			System.err.println("** [getQualifiedId] Taxon has no id, using name: " + this.name);
+			return new QualifiedId(this.taxonomy.getTag(), this.name);
 		}
 	}
 
-	// Method on Node, called for every node in the source taxonomy
+	// Method on Node, called for every node in the source taxonomy.
+	// Input is node in source taxonomy.  Returns node in union taxonomy.
 	Taxon augment(UnionTaxonomy union) {
 
 		Taxon newnode = null;
@@ -380,7 +381,6 @@ Taxon augment(UnionTaxonomy union) {
 
 			} else if (this.refinementp(oldChildren, newChildren)) {
 
-
 					// Move the new internal node over to union taxonomy.
 					// It will end up becoming a descendent of oldParent.
 					newnode = new Taxon(union);
@@ -408,14 +408,7 @@ Taxon augment(UnionTaxonomy union) {
 				for (Taxon augChild: newChildren)
 					newnode.addChild(augChild);
 
-				if (this.taxonomy.tag.equals("if")) {
-					boolean javasux = false;
-					for (Taxon p = this.parent; p != null; p = p.parent)
-						if (p.name.equals("Fungi")) { javasux = true; break; }
-					if (javasux)
-						for (Taxon o : oldChildren)
-							System.err.format("** %s losing %s to %s\n", this, o, o.parent);
-				}
+				union.reportConflict(oldChildren.get(0), this);
 
 				newflags |= Taxonomy.TATTERED;
 				union.logAndMark(Answer.yes(this, null, "new/tattered", null));
@@ -913,6 +906,7 @@ Taxon scan(Taxonomy other) {
 		return up;
 	}
 
+	// Use getDepth() only after the tree is in its final form
 	int depth = -1;
 	int getDepth() {
 		if (this.depth < 0) {
@@ -923,12 +917,12 @@ int getDepth() {
 		}
 		return this.depth;
 	}
+	// Does not cache - depths may change during merge
 	int measureDepth() {		// Robust in presence of insertions
 		if (this.parent == null)
-			this.depth = 0;
+			return 0;
 		else
-			this.depth = this.parent.measureDepth() + 1;
-		return this.depth;
+			return this.parent.measureDepth() + 1;
 	}
 
 	Taxon mrca(Taxon b) {
@@ -1267,8 +1261,16 @@ public void extinct() {
 	}
 
 	public void extant() {
+		boolean wasExtant = true;
 		for (Taxon node = this; node != null; node = node.parent)
-			this.properFlags &= ~Taxonomy.EXTINCT;
+			if ((this.properFlags & Taxonomy.EXTINCT) != 0) {
+				this.properFlags &= ~Taxonomy.EXTINCT;
+				if (node != this)
+					System.err.format("** Ancestor %s of %s was marked extinct\n", node, this);
+				wasExtant = false;
+			}
+		if (wasExtant)
+			System.err.format("** Note: %s wasn't marked extinct\n", this);
 	}
 
 	// add a tree to the forest?

org/opentreeoflife/smasher/Taxonomy.java

@@ -161,13 +161,13 @@ void setTag() {
 		}
 	}
 
-	Taxon highest(String name) {
+	Taxon highest(String name) { // See pin()
 		Taxon best = null;
 		List<Taxon> l = this.lookup(name);
 		if (l != null) {
 			int depth = 1 << 30;
 			for (Taxon node : l)
-				if (node.getDepth() < depth) {
+				if (node.measureDepth() < depth) {
 					depth = node.getDepth();
 					best = node;
 				}
@@ -323,6 +323,8 @@ void loadMetadata(String filename) throws IOException {
 
 	abstract void dumpMetadata(String filename) throws IOException;
 
+	static Pattern commaPattern = Pattern.compile(",");
+
 	// load | dump taxonomy proper
 
 	void loadTaxonomyProper(String filename) throws IOException {
@@ -401,11 +403,8 @@ void loadTaxonomyProper(String filename) throws IOException {
 					addSynonym(rawname, node);
 					++normalizations;
 				}
-				if (this.flagscolumn != null) {
-					if (parts[this.flagscolumn].contains("extinct"))
-						// kludge. could be _direct or _inherited
-						node.properFlags |= Taxonomy.EXTINCT;
-				}
+				if (this.flagscolumn != null && parts[this.flagscolumn].length() > 0)
+					this.parseFlags(parts[this.flagscolumn], node);
 			}
 			++row;
 			if (row % 500000 == 0)
@@ -724,14 +723,17 @@ void dumpSynonyms(String filename, String sep) throws IOException {
 
 	void dumpHidden(String filename) throws IOException {
 		PrintStream out = Taxonomy.openw(filename);
+		int count = 0;
 		for (Taxon node : this) {
 			if (node.isHidden()) {
+				++count;
 				out.format("%s\t%s\t%s\t%s\t", node.id, node.name, node.getSourceIdsString(), node.division);
 				this.printFlags(node, out);
 				out.println();
 			}
 		}
 		out.close();
+		System.out.format("| %s hidden taxa\n", count);
 	}
 
 	/*
@@ -852,6 +854,13 @@ void analyze() {
 	// Opposite of 'barren' - propagated upward
 	static final int ANYSPECIES			 =	 8 * 1024;
 
+	void parseFlags(String flags, Taxon node) {
+		// String[] tags = commaPattern.split(flags);
+		if (flags.contains("extinct"))
+			// kludge. could be _direct or _inherited
+			node.properFlags |= Taxonomy.EXTINCT;
+	}
+
 	// Returns the node's rank (as an int).	 In general the return
 	// value should be >= parentRank, but conceivably funny things
 	// could happen when combinings taxonomies.
@@ -2365,6 +2374,7 @@ public void dump(String outprefix, String sep) throws IOException {
 		this.dumpNodes(this.roots, outprefix, sep);
 		this.dumpSynonyms(outprefix + "synonyms.tsv", sep);
 		this.dumpHidden(outprefix + "hidden.tsv");
+		this.dumpConflicts(outprefix + "conflicts.tsv");
 	}
 
 	// Overrides method in Taxonomy class
@@ -2506,6 +2516,52 @@ void logAndReport(Answer answer) {
 		answer.x.report(answer.reason, answer.y, answer.witness);
 	}
 
+	// 3799 conflicts as of 2014-04-12
+	// unode.comapped.parent == fromparent
+	void reportConflict(Taxon unode, Taxon fromparent) {
+		conflicts.add(new Conflict(unode, fromparent));
+	}
+
+	List<Conflict> conflicts = new ArrayList<Conflict>();
+	void dumpConflicts(String filename) throws IOException {
+		PrintStream out = Taxonomy.openw(filename);
+		for (Conflict conflict : this.conflicts)
+			if (!conflict.unode.isHidden())
+				out.println(conflict.toString());
+		out.close();
+	}
+}
+
+class Conflict {
+	Taxon unode;					// in source taxonomy
+	Taxon fromParent;				// in union taxonomy
+	Conflict(Taxon unode, Taxon fromParent) {
+		this.unode = unode; this.fromParent = fromParent;
+	}
+	public String toString() {
+		// cf. Taxon.mrca
+		Taxon b = fromParent;
+		while (b != null && b.mapped == null)
+			b = b.parent;
+		b = b.mapped;
+		Taxon a = unode;
+		int da = a.measureDepth();
+		int db = b.measureDepth();
+		while (db > da) {
+			b = b.parent;
+			--db;
+		}
+		while (da > db) {
+			a = a.parent;
+			--da;
+		}
+		while (a != null && a.parent != b.parent) {
+			a = a.parent;
+			b = b.parent;
+			--da;
+		}
+		return (da + " " + fromParent + " in " + b + " lost child " + unode.comapped + " to " + unode.parent + " in " + a);
+	}
 }
 
 // For each source node, consider all possible union nodes it might map to