Replace SimpleDateFormat with DateTimeFormatter (#21)

* Change java target version from 1.7 to 1.8

* Refactoring parseDate

* Modify parsers

* Remove openjdk7

* Modify Makefile, README and pom

* Add ./dev/get-app-version.sh

* Modify Makefile

Author: yu-iskw

.travis.yml

@@ -4,7 +4,6 @@ sudo: false # faster builds
 jdk:
   - oraclejdk8
   - oraclejdk9
-  - openjdk7
   - openjdk8
 
 script:

Makefile

@@ -3,9 +3,10 @@
 #
 
 DOCKER_TAG := yuiskw/bigquery-to-datastore:latest
-DOCKER_TAG_VERSION := yuiskw/bigquery-to-datastore:beam-2.1
+DOCKER_TAG_VERSION := yuiskw/bigquery-to-datastore:$(shell bash ./dev/get-app-version.sh)-beam-2.1
 
-.PHONY: test checkstyle clean
+.PHONY: test checkstyle clean \
+		build-docker test-docker push-docker push-docker-latest push-docker-version
 
 all: checkstyle test package
 
@@ -23,8 +24,8 @@ clean:
 	  mvn clean
 
 build-docker: package
-	docker build -t $(DOCKER_TAG) .
-	docker build -t $(DOCKER_TAG_VERSION) .
+	docker build --rm -t $(DOCKER_TAG) .
+	docker build --rm -t $(DOCKER_TAG_VERSION) .
 
 test-docker: build-docker
 	docker run --rm --entrypoint "bash" $(DOCKER_TAG) ./dev/test.sh

README.md

@@ -13,7 +13,7 @@ Apache Beam's `DatastoreIO` doesn't allow us to write same key at once.
 ## Requirements
 
 - Maven
-- Java 1.7+
+- Java 1.8+
 - Google Cloud Platform account
 
 ## Usage
@@ -46,7 +46,7 @@ Apache Beam's `DatastoreIO` doesn't allow us to write same key at once.
 mvn clean package
 
 # Run bigquery-to-datastore via the compiled JAR file
-java -cp $(pwd)/target/bigquery-to-datastore-bundled-0.3.jar \
+java -cp $(pwd)/target/bigquery-to-datastore-bundled-0.5.0.jar \
   com.github.yuiskw.beam.BigQuery2Datastore \
   --project=your-gcp-project \
   --runner=DataflowRunner \
@@ -71,7 +71,7 @@ mvn clean package
 make package
 
 # run
-java -cp $(pwd)/target/bigquery-to-datastore-bundled-0.3.jar --help
+java -cp $(pwd)/target/bigquery-to-datastore-bundled-0.5.0.jar --help
 # or
 ./bin/bigquery-to-datastore --help
 ```
@@ -80,7 +80,7 @@ java -cp $(pwd)/target/bigquery-to-datastore-bundled-0.3.jar --help
 We also offers docker images for this project in [yuiskw/bigquery\-to\-datastore \- Docker Hub](https://hub.docker.com/r/yuiskw/bigquery-to-datastore/).
 We have several docker images based on Apache Beam versions.
 ```
-docker run yuiskw/bigquery-to-datastore:latest --help
+docker run yuiskw/bigquery-to-datastore:0.5.0-beam-2.1 --help
 ```
 
 ### How to install it with homebrew
@@ -90,7 +90,7 @@ You can install it with homebrew from [yu-iskw/homebrew-bigquery-to-datastore](h
 brew install yu-iskw/bigquery-to-datastore/bigquery-to-datastore
 
 # show help
-bigquery-to-datastore --help
+./bin/bigquery-to-datastore --help
 ```
 
 ### Type conversions between BigQuery and Google Datastore

dev/get-app-version.sh

@@ -0,0 +1,11 @@
+#
+# Copyright (c) 2017 Yu Ishikawa.
+#
+
+#
+# This script is used for extracting the application version from pom.xml.
+#
+PROJECT_DIR=$(dirname $(dirname $(readlink -f $0)))
+XML_TAG=$(grep -e '<version>[0-9\.]*</version>' "${PROJECT_DIR}/pom.xml")
+VERSION=$(echo "$XML_TAG" | sed -e 's/<version>//' -e 's/<\/version>//' | tr -d '[:space:]')
+echo "$VERSION"

pom.xml

@@ -22,7 +22,7 @@
 
   <groupId>com.github.yuiskw</groupId>
   <artifactId>bigquery-to-datastore</artifactId>
-  <version>0.4.1</version>
+  <version>0.5.0</version>
 
   <packaging>jar</packaging>
 
@@ -67,8 +67,8 @@
         <artifactId>maven-compiler-plugin</artifactId>
         <version>${maven-compiler-plugin.version}</version>
         <configuration>
-          <source>1.7</source>
-          <target>1.7</target>
+          <source>1.8</source>
+          <target>1.8</target>
         </configuration>
       </plugin>
 
@@ -173,7 +173,6 @@
           <groupId>org.apache.beam</groupId>
           <artifactId>beam-runners-google-cloud-dataflow-java</artifactId>
           <version>${beam.version}</version>
-          <scope>runtime</scope>
         </dependency>
       </dependencies>
     </profile>

src/main/java/com/github/yuiskw/beam/TableRow2EntityFn.java

@@ -3,9 +3,12 @@
  */
 package com.github.yuiskw.beam;
 
-import java.text.DateFormat;
 import java.text.ParseException;
-import java.text.SimpleDateFormat;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeParseException;
+import java.time.format.ResolverStyle;
 import java.util.*;
 
 import com.google.api.services.bigquery.model.TableRow;
@@ -61,6 +64,14 @@ public static Timestamp toTimestamp(Date date) {
     return timestamp;
   }
 
+  public static Timestamp toTimestamp(Instant instant) {
+    long second = instant.getEpochSecond();
+    Timestamp timestamp = Timestamp.newBuilder()
+        .setSeconds(second)
+        .build();
+    return timestamp;
+  }
+
   /**
    * Convert TableRow to Entity
    */
@@ -120,8 +131,8 @@ else if (value instanceof org.joda.time.LocalDateTime) {
           .setExcludeFromIndexes(isExcluded).build();
     }
     else if (value instanceof String && parseTimestamp((String) value) != null) {
-      Date date = parseTimestamp((String) value);
-      Timestamp timestamp = toTimestamp(date);
+      Instant instant = parseTimestamp((String) value);
+      Timestamp timestamp = toTimestamp(instant);
       v = Value.newBuilder().setTimestampValue(timestamp)
           .setExcludeFromIndexes(isExcluded).build();
     }
@@ -131,18 +142,11 @@ else if (value instanceof org.joda.time.LocalDate) {
       v = Value.newBuilder().setTimestampValue(timestamp)
           .setExcludeFromIndexes(isExcluded).build();
     } else if (value instanceof String && parseDate((String) value) != null) {
-      Date date = parseDate((String) value);
-      Timestamp timestamp = toTimestamp(date);
+      Instant instant = parseDate((String) value);
+      Timestamp timestamp = toTimestamp(instant);
       v = Value.newBuilder().setTimestampValue(timestamp)
           .setExcludeFromIndexes(isExcluded).build();
     }
-    // TIME
-    // NOTE: Datastore doesn't have any data type to time.
-    else if (value instanceof org.joda.time.LocalTime) {
-      ;
-    } else if (value instanceof String && parseTime((String) value) != null) {
-      ;
-    }
     // STRING
     else if (value instanceof String) {
       v = Value.newBuilder().setStringValue((String) value)
@@ -254,63 +258,51 @@ public static Integer parseInteger(String value) {
    *
    * Note: SimpledateFormat("yyyy-MM-dd") can't help parseing "yyyy-MM-dd HH:mm:ss" as well.
    */
-  public static Date parseDate(String value) {
-    Date date = null;
+  public static Instant parseDate(String value) {
+    Instant instant = null;
     try {
-      DateFormat sourceFormat = new SimpleDateFormat("yyyy-MM-dd");
-      date = sourceFormat.parse(value);
-    } catch (ParseException e) {
+      DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-M-d")
+          .withResolverStyle(ResolverStyle.SMART);
+      java.time.LocalDate localDate = java.time.LocalDate.parse(value, formatter);
+      instant = localDate.atStartOfDay().atZone(ZoneId.systemDefault()).toInstant();
+    } catch (DateTimeParseException e) {
       // Do nothing.
       ;
     }
-    return date;
-  }
-
-  /**
-   * Parse string on time format
-   */
-  public static Date parseTime(String value) {
-    Date date = null;
-    List<String> patterns = Arrays.asList(
-        "HH:mm:ss.SSS z",
-        "HH:mm:ss.SSS",
-        "HH:mm:ss"
-    );
-    for (String pattern : patterns) {
-      try {
-        DateFormat sourceFormat = new SimpleDateFormat(pattern);
-        date = sourceFormat.parse(value);
-        return date;
-      } catch (ParseException e) {
-        // Do nothing.
-        ;
-      }
-    }
-    return date;
+    return instant;
   }
 
   /**
    * Parse string on timestamp format
    */
-  public static Date parseTimestamp(String value) {
-    Date date = null;
+  public static Instant parseTimestamp(String value) {
+    Instant instant = null;
     List<String> patterns = Arrays.asList(
-        "yyyy-MM-dd HH:mm:ss.SSS z",
-        "yyyy-MM-dd HH:mm:ss.SSS",
-        "yyyy-MM-dd HH:mm:ss",
-        "yyyy-MM-dd'T'HH:mm:ss"
+        "yyyy-M-d H:m:s.SSS z",
+        "yyyy-M-d H:m:s.SS z",
+        "yyyy-M-d H:m:s.S z",
+        "yyyy-M-d H:m:s.SSS",
+        "yyyy-M-d H:m:s.SS",
+        "yyyy-M-d H:m:s.S",
+        "yyyy-M-d H:m:s",
+        "yyyy-M-d'T'H:m:s.SSS",
+        "yyyy-M-d'T'H:m:s.SS",
+        "yyyy-M-d'T'H:m:s.S",
+        "yyyy-M-d'T'H:m:s"
     );
     for (String pattern : patterns) {
       try {
-        DateFormat sourceFormat = new SimpleDateFormat(pattern);
-        date = sourceFormat.parse(value);
-        return date;
-      } catch (ParseException e) {
+        DateTimeFormatter formatter = DateTimeFormatter.ofPattern(pattern)
+            .withResolverStyle(ResolverStyle.SMART);
+        java.time.LocalDateTime localDateTime = java.time.LocalDateTime.parse(value, formatter);
+        instant = localDateTime.atZone(ZoneId.systemDefault()).toInstant();
+        return instant;
+      } catch (DateTimeParseException e) {
         // Do nothing.
         ;
       }
     }
-    return date;
+    return instant;
   }
 
   /**

src/test/java/com/github/yuiskw/beam/TableRow2EntityFnTest.java

@@ -12,6 +12,7 @@
 import com.google.datastore.v1.Entity;
 import com.google.datastore.v1.Key;
 import com.google.datastore.v1.Value;
+import com.google.protobuf.Timestamp;
 import org.joda.time.DateTime;
 import org.joda.time.Instant;
 
@@ -94,25 +95,30 @@ public void testConvert2() {
   public void testIsDate() {
     assertNotNull(TableRow2EntityFn.parseDate("2017-01-01"));
     assertNotNull(TableRow2EntityFn.parseDate("2017-1-1"));
+    assertNotNull(TableRow2EntityFn.parseDate("2017-01-1"));
+    assertNotNull(TableRow2EntityFn.parseDate("2017-1-01"));
+    assertNotNull(TableRow2EntityFn.parseDate("2017-02-30"));
     assertNull(TableRow2EntityFn.parseDate("hoge"));
   }
 
-  @Test
-  public void testIsTime() {
-    assertNotNull(TableRow2EntityFn.parseTime("04:14:37.844024"));
-    assertNotNull(TableRow2EntityFn.parseTime("4:4:7.4"));
-    assertNotNull(TableRow2EntityFn.parseTime("04:14:37"));
-    assertNull(TableRow2EntityFn.parseTime("hoge"));
-  }
-
   @Test
   public void testIsTimestamp() {
-    assertNotNull(TableRow2EntityFn.parseTimestamp("2017-09-16 04:14:37.844024 UTC"));
-    assertNotNull(TableRow2EntityFn.parseTimestamp("2017-09-16 04:14:37.844024 PST"));
-    assertNotNull(TableRow2EntityFn.parseTimestamp("2017-09-16 04:14:37.844024 JST"));
-    assertNotNull(TableRow2EntityFn.parseTimestamp("2017-9-16 4:14:37.844024 UTC"));
-    assertNotNull(TableRow2EntityFn.parseTimestamp("2017-09-16T04:14:37.844024"));
+    assertNotNull(TableRow2EntityFn.parseTimestamp("2017-09-16 04:14:37.844 UTC"));
+    assertNotNull(TableRow2EntityFn.parseTimestamp("2017-9-1 4:1:1.1 UTC"));
+    assertNotNull(TableRow2EntityFn.parseTimestamp("2017-9-1 4:1:1.12 UTC"));
+    assertNotNull(TableRow2EntityFn.parseTimestamp("2017-9-1 4:1:1.001 UTC"));
+
+    assertNotNull(TableRow2EntityFn.parseTimestamp("2017-09-16 04:14:37.844 PST"));
+    assertNotNull(TableRow2EntityFn.parseTimestamp("2017-09-16 04:14:37.844 JST"));
+
+    assertNotNull(TableRow2EntityFn.parseTimestamp("2017-09-16T04:14:37.844"));
+    assertNotNull(TableRow2EntityFn.parseTimestamp("2017-9-01T4:1:1.1"));
+    assertNotNull(TableRow2EntityFn.parseTimestamp("2017-9-01T4:1:1.12"));
+    assertNotNull(TableRow2EntityFn.parseTimestamp("2017-9-01T4:1:1.001"));
     assertNotNull(TableRow2EntityFn.parseTimestamp("2017-09-16 04:14:37"));
+    assertNotNull(TableRow2EntityFn.parseTimestamp("2017-9-01 4:02:03"));
+
+    assertNull(TableRow2EntityFn.parseDate("98-9-12.com"));
     assertNull(TableRow2EntityFn.parseTimestamp("hoge"));
   }
 
@@ -125,4 +131,11 @@ public void testIsExlucedFromIndex() {
 
     assertTrue(TableRow2EntityFn.isExcludedFromIndex(null, indexedColumns));
   }
+
+  @Test
+  public void testToTimestamp() {
+    java.time.Instant instant = java.time.Instant.now();
+    Timestamp timestamp = TableRow2EntityFn.toTimestamp(instant);
+    assertEquals(instant.getEpochSecond(), timestamp.getSeconds());
+  }
 }