experiment with getting all the instant range info in analysis

zebdelk · zebdelk · commit a07ea3de0aec · 2020-06-26T14:10:56.000-06:00
note: I just made a weird expansion to the temporal schema for this to see if this works best for handling these dates - need to think out the best place to actually incorporate into the schema if so.
diff --git a/schemas-analyze/src/main/java/org/cedar/schemas/analyze/DateInfo.java b/schemas-analyze/src/main/java/org/cedar/schemas/analyze/DateInfo.java
@@ -7,6 +7,8 @@
 import java.time.temporal.ChronoField;
 import java.time.temporal.ChronoUnit;
 import java.time.temporal.TemporalAccessor;
+import java.time.Year;
+import java.time.YearMonth;
 import java.time.DateTimeException;
 import java.time.ZonedDateTime;
 import java.util.ArrayList;
@@ -25,9 +27,14 @@ public class DateInfo implements Comparable<DateInfo> {
   public final String zoneSpecified;
   public final String utcDateTimeString;
   public final Long year;
-  public final Integer dayOfYear; // values 1 - 366
-  public final Integer dayOfMonth; // values 1 - 31
-  public final Integer month; // values 1 - 12
+  public Integer dayOfYear; // values 1 - 366 // TODO temp remove final from these fields, just to sanity check things
+  public Integer dayOfMonth; // values 1 - 31
+  public Integer month; // values 1 - 12
+
+  // special stuff for dealing with how we interpret year or month precision instants:
+  public Integer endDayOfYear;
+  public Integer endDayOfMonth;
+  public Integer endMonth;
 
   public DateInfo(String dateString, boolean start) {
     if (dateString == null || dateString.length() == 0) {
@@ -55,6 +62,27 @@ public DateInfo(String dateString, boolean start) {
     dayOfYear = extractField(parsedDate, ChronoField.DAY_OF_YEAR);
     dayOfMonth = extractField(parsedDate, ChronoField.DAY_OF_MONTH);
     month = extractField(parsedDate, ChronoField.MONTH_OF_YEAR);
+    if (parsedDate != null && dayOfMonth == null && month != null) { // TODO or parsedDate instanceof YearMonth
+      endMonth = month;
+      dayOfMonth = 1;
+      endDayOfMonth = ((YearMonth)parsedDate).lengthOfMonth();
+      dayOfYear = ((YearMonth)parsedDate).atDay(1).getDayOfYear();
+      endDayOfYear = ((YearMonth)parsedDate).atEndOfMonth().getDayOfYear();
+      System.out.println("year month - setting endDayOf* fields..." + endDayOfMonth);
+      System.out.println("year month - setting endDayOf* fields..." + dayOfYear);
+      System.out.println("year month - setting endDayOf* fields..." + endDayOfYear);
+    } else if (parsedDate != null && month == null && year != null) { // TODO or instanceof Year?
+      dayOfMonth = 1;
+      dayOfYear = 1;
+      endDayOfYear = ((Year)parsedDate).length(); // number of days in the year, including leap years
+      endDayOfMonth = 31;
+      month = 1;
+      endMonth = 12;
+    } else {
+      endMonth = month;
+      endDayOfMonth = dayOfMonth;
+      endDayOfYear = dayOfYear;
+    }
 
     if (longDate != null && !indexable(longDate)) {
       descriptor = ValidDescriptor.VALID;
diff --git a/schemas-analyze/src/main/java/org/cedar/schemas/analyze/Temporal.java b/schemas-analyze/src/main/java/org/cedar/schemas/analyze/Temporal.java
@@ -111,6 +111,9 @@ public static TemporalBoundingAnalysis analyzeBounding(Discovery metadata) {
       builder.setInstantDayOfYear(instantInfo.dayOfYear);
       builder.setInstantDayOfMonth(instantInfo.dayOfMonth);
       builder.setInstantMonth(instantInfo.month);
+      builder.setInstantEndDayOfYear(instantInfo.endDayOfYear);
+      builder.setInstantEndDayOfMonth(instantInfo.endDayOfMonth);
+      builder.setInstantEndMonth(instantInfo.endMonth);
 
       builder.setRangeDescriptor(rangeDescriptor);
     }
diff --git a/schemas-analyze/src/test/groovy/org/cedar/schemas/analyze/AnalyzersSpec.groovy b/schemas-analyze/src/test/groovy/org/cedar/schemas/analyze/AnalyzersSpec.groovy
@@ -97,6 +97,9 @@ class AnalyzersSpec extends Specification {
             instantDayOfMonth       : null,
             instantYear             : null,
             instantMonth            : null,
+            instantEndDayOfYear     : null,
+            instantEndDayOfMonth    : null,
+            instantEndMonth         : null,
             rangeDescriptor         : BOUNDED,
         ],
         spatialBounding : [
diff --git a/schemas-analyze/src/test/groovy/org/cedar/schemas/analyze/DateInfoSpec.groovy b/schemas-analyze/src/test/groovy/org/cedar/schemas/analyze/DateInfoSpec.groovy
@@ -31,17 +31,18 @@ class DateInfoSpec extends Specification {
     where:
     input | year | dayOfYear | dayOfMonth | month
     '1999-12-31' | 1999 | 365 | 31 | 12
-    '1999' | 1999 | null | null | null
+    '1999' | 1999 | 1 | 1 | 1
+    '1999-06' | 1999 | 152 | 1 | 6
     '1999-01-01T00:00:00Z' | 1999 | 1 | 1 | 1
     '-1000000000' | -1_000_000_000L | null | null | null // too big paleo number is parsed as a year value, although it cannot be a datetime string
-    '-35000000' | -35_000_000L | null | null | null // this one can be a datetime string
+    '-35000000' | -35_000_000L | 1 | 1 | 1 // this one can be a datetime string
     '2008-04-01T00:00:00Z' | 2008 | 92 | 1 | 4
     '1975-06-15T12:30:00Z' | 1975 | 166 | 15 | 6
     null | null | null | null | null
     'not a date' | null | null | null | null
     '2020-02-29' | 2020 | 60 | 29 | 2 // leap day
     '2020-12-31' | 2020 | 366 | 31 | 12 // last day of leap year
-    '20191025' | 20191025 | null | null | null // string datetime from the bug
+    '20191025' | 20191025 | 1 | 1 | 1 // string datetime from the bug
   }
 
   def 'extracts date info from date string: #input'() {
diff --git a/schemas-analyze/src/test/groovy/org/cedar/schemas/analyze/TemporalSpec.groovy b/schemas-analyze/src/test/groovy/org/cedar/schemas/analyze/TemporalSpec.groovy
@@ -17,6 +17,35 @@ class TemporalSpec extends Specification {
 
   final String analysisAvro = ClassLoader.systemClassLoader.getResourceAsStream('avro/psi/analysis.avsc').text
 
+  def "instant #description (#instant) correctly extrapolates date range"() {
+    given:
+    def bounding = TemporalBounding.newBuilder()
+        .setInstant(instant)
+        .build()
+    def discovery = Discovery.newBuilder().setTemporalBounding(bounding).build()
+
+    when:
+    def result = Temporal.analyzeBounding(discovery)
+
+    then:
+    println("????"+result)
+    result.instantMonth == month
+    result.instantEndMonth == endMonth
+    result.instantDayOfMonth == dayOfMonth
+    result.instantEndDayOfMonth == endDayOfMonth
+    result.instantDayOfYear == dayOfYear
+    result.instantEndDayOfYear == endDayOfYear
+
+    where:
+
+    description | instant | year | dayOfYear | dayOfMonth | month | endYear | endDayOfYear | endDayOfMonth | endMonth
+    "instant with month precision" | '2003-02' | 2003 | 32 | 1 | 2 | 2003 | 59 | 28 | 2
+    "instant on leapyear with month precision" | '2004-02' | 2004 | 32 | 1 | 2 | 2004 | 60 | 29 | 2
+    "instant with day precision" | '2001-06-22' | 2001 | 173 | 22 | 6 | 2001 | 173 | 22 | 6
+    "instant with day precision on leapyear" | '2020-06-22' | 2020 | 174 | 22 | 6 | 2020 | 174 | 22 | 6
+
+  }
+
   def "#descriptor date range correctly identified when #situation"() {
     given:
     def bounding = TemporalBounding.newBuilder()
diff --git a/schemas-core/src/main/resources/avro/psi/analysis.avsc b/schemas-core/src/main/resources/avro/psi/analysis.avsc
@@ -297,7 +297,7 @@
                 "string"
               ],
               "default": null,
-              "doc": "The precision of the beginning of the analyzed bounding, e.g. 'Years', 'Seconds', etc. See java.time.temporal.ChronoUnit"
+              "doc": "The precision of the beginning of the analyzed bounding, e.g. 'Years', 'Seconds', etc. See java.time.temporal.ChronoUnit. Note that this may sometimes not exactly match the input data, since it reflects the Java class the date is able to be parsed into"
             },
             {
               "name": "beginIndexable",
@@ -306,7 +306,7 @@
                 "boolean"
               ],
               "default": null,
-              "doc": "Indicates whether or not the beginning can be indexed for searching"
+              "doc": "Indicates whether or not beginUtcDateTimeString can be indexed for searching"
             },
             {
               "name": "beginZoneSpecified",
@@ -342,7 +342,7 @@
                 "int"
               ],
               "default": null,
-              "doc": "If possible, an integer indicating the day of year for the beginning value"
+              "doc": "If possible, an integer indicating the day of year for the beginning value. This may be populated by extrapolating into a DateTime. See beginPrecision for the best indicator of if this value is extrapolated or provided in the original date format"
             },
             {
               "name": "beginDayOfMonth",
@@ -351,7 +351,7 @@
                 "int"
               ],
               "default": null,
-              "doc": "If possible, an integer indicating the day of month for the beginning value"
+              "doc": "If possible, an integer indicating the day of month for the beginning value. This may be populated by extrapolating into a DateTime. See beginPrecision for the best indicator of if this value is extrapolated or provided in the original date format"
             },
             {
               "name": "beginMonth",
@@ -360,7 +360,7 @@
                 "int"
               ],
               "default": null,
-              "doc": "If possible, an integer indicating the month for the beginning value"
+              "doc": "If possible, an integer indicating the month for the beginning value. This may be populated by extrapolating into a DateTime. See beginPrecision for the best indicator of if this value is extrapolated or provided in the original date format"
             },
             {
               "name": "endDescriptor",
@@ -378,7 +378,7 @@
                 "string"
               ],
               "default": null,
-              "doc": "The precision of the end of the analyzed bounding, e.g. 'Years', 'Seconds', etc. See java.time.temporal.ChronoUnit"
+              "doc": "The precision of the end of the analyzed bounding, e.g. 'Years', 'Seconds', etc. See java.time.temporal.ChronoUnit. Note that this may sometimes not exactly match the input data, since it reflects the Java class the date is able to be parsed into"
             },
             {
               "name": "endIndexable",
@@ -387,7 +387,7 @@
                 "boolean"
               ],
               "default": null,
-              "doc": "Indicates whether or not the ending can be indexed for searching"
+              "doc": "Indicates whether or not endUtcDateTimeString can be indexed for searching"
             },
             {
               "name": "endZoneSpecified",
@@ -423,7 +423,7 @@
                 "int"
               ],
               "default": null,
-              "doc": "If possible, an integer indicating the day of year for the end value"
+              "doc": "If possible, an integer indicating the day of year for the end value. This may be populated by extrapolating into a DateTime. See endPrecision for the best indicator of if this value is extrapolated or provided in the original date format"
             },
             {
               "name": "endDayOfMonth",
@@ -432,7 +432,7 @@
                 "int"
               ],
               "default": null,
-              "doc": "If possible, an integer indicating the day of month for the end value"
+              "doc": "If possible, an integer indicating the day of month for the end value. This may be populated by extrapolating into a DateTime. See endPrecision for the best indicator of if this value is extrapolated or provided in the original date format"
             },
             {
               "name": "endMonth",
@@ -441,7 +441,7 @@
                 "int"
               ],
               "default": null,
-              "doc": "If possible, an integer indicating the month for the end value"
+              "doc": "If possible, an integer indicating the month for the end value. This may be populated by extrapolating into a DateTime. See endPrecision for the best indicator of if this value is extrapolated or provided in the original date format"
             },
             {
               "name": "instantDescriptor",
@@ -459,7 +459,7 @@
                 "string"
               ],
               "default": null,
-              "doc": "The precision of the instant of the analyzed bounding, e.g. 'Years', 'Seconds', etc. See java.time.temporal.ChronoUnit"
+              "doc": "The precision of the instant of the analyzed bounding, e.g. 'Years', 'Seconds', etc. See java.time.temporal.ChronoUnit. Note that this may sometimes not exactly match the input data, since it reflects the Java class the date is able to be parsed into"
             },
             {
               "name": "instantIndexable",
@@ -468,7 +468,7 @@
                 "boolean"
               ],
               "default": null,
-              "doc": "Indicates whether or not the instant can be indexed for searching"
+              "doc": "Indicates whether or not instantUtcDateTimeString can be indexed for searching"
             },
             {
               "name": "instantZoneSpecified",
@@ -504,7 +504,16 @@
                 "int"
               ],
               "default": null,
-              "doc": "If possible, an integer indicating the day of year for the instant value"
+              "doc": "If possible, an integer indicating the day of year for the instant value. This may be populated by extrapolating into a DateTime. See instantPrecision for the best indicator of if this value is extrapolated or provided in the original date format"
+            },
+            {
+              "name": "instantEndDayOfYear",
+              "type": [
+                "null",
+                "int"
+              ],
+              "default": null,
+              "doc": "If instantPrecision does not include day, an integer indicating the assumed day of year the instant ends, e.g. the last day of a month or year"
             },
             {
               "name": "instantDayOfMonth",
@@ -513,7 +522,16 @@
                 "int"
               ],
               "default": null,
-              "doc": "If possible, an integer indicating the day of month for the instant value"
+              "doc": "If possible, an integer indicating the day of month for the instant value. This may be populated by extrapolating into a DateTime. See instantPrecision for the best indicator of if this value is extrapolated or provided in the original date format"
+            },
+            {
+              "name": "instantEndDayOfMonth",
+              "type": [
+                "null",
+                "int"
+              ],
+              "default": null,
+              "doc": "If instantPrecision does not include day, an integer indicating the assumed day of month the instant ends, e.g. the last day of a month or year"
             },
             {
               "name": "instantMonth",
@@ -522,7 +540,16 @@
                 "int"
               ],
               "default": null,
-              "doc": "If possible, an integer indicating the month for the instant value"
+              "doc": "If possible, an integer indicating the month for the instant value. This may be populated by extrapolating into a DateTime. See instantPrecision for the best indicator of if this value is extrapolated or provided in the original date format"
+            },
+            {
+              "name": "instantEndMonth",
+              "type": [
+                "null",
+                "int"
+              ],
+              "default": null,
+              "doc": "If instantPrecision does not include day, an integer indicating the assumed month the instant ends, e.g. the last month of the year"
             },
             {
               "name": "rangeDescriptor",
diff --git a/schemas-core/src/test/groovy/org/cedar/schemas/avro/util/TemporalTestData.groovy b/schemas-core/src/test/groovy/org/cedar/schemas/avro/util/TemporalTestData.groovy
diff --git a/schemas-core/src/test/groovy/org/cedar/schemas/avro/util/TemporalTestDataSanitySpec.groovy b/schemas-core/src/test/groovy/org/cedar/schemas/avro/util/TemporalTestDataSanitySpec.groovy
diff --git a/schemas-core/src/test/resources/example-record-avro.json b/schemas-core/src/test/resources/example-record-avro.json

Original file line number	Diff line number	Diff line change
`@@ -111,6 +111,9 @@ public static TemporalBoundingAnalysis analyzeBounding(Discovery metadata) {`
`111`	`111`	`builder.setInstantDayOfYear(instantInfo.dayOfYear);`
`112`	`112`	`builder.setInstantDayOfMonth(instantInfo.dayOfMonth);`
`113`	`113`	`builder.setInstantMonth(instantInfo.month);`
	`114`	`+ builder.setInstantEndDayOfYear(instantInfo.endDayOfYear);`
	`115`	`+ builder.setInstantEndDayOfMonth(instantInfo.endDayOfMonth);`
	`116`	`+ builder.setInstantEndMonth(instantInfo.endMonth);`
`114`	`117`
`115`	`118`	`builder.setRangeDescriptor(rangeDescriptor);`
`116`	`119`	`}`