Merge pull request #940 from eraly/eraly

Word2vec model automatic download

Author: treo

dl4j-examples/src/main/java/org/deeplearning4j/examples/recurrent/word2vecsentiment/Word2VecSentimentRNN.java

@@ -37,49 +37,53 @@
 import org.nd4j.linalg.indexing.NDArrayIndex;
 import org.nd4j.linalg.learning.config.Adam;
 import org.nd4j.linalg.lossfunctions.LossFunctions;
+import org.nd4j.resources.Downloader;
 
 import java.io.File;
+import java.io.IOException;
 import java.net.URL;
 import java.nio.charset.Charset;
+import java.util.Scanner;
 
-/**Example: Given a movie review (raw text), classify that movie review as either positive or negative based on the words it contains.
+/**
+ * Example: Given a movie review (raw text), classify that movie review as either positive or negative based on the words it contains.
  * This is done by combining Word2Vec vectors and a recurrent neural network model. Each word in a review is vectorized
  * (using the Word2Vec model) and fed into a recurrent neural network.
  * Training data is the "Large Movie Review Dataset" from http://ai.stanford.edu/~amaas/data/sentiment/
  * This data set contains 25,000 training reviews + 25,000 testing reviews
- *
+ * <p>
  * Process:
+ * 0. If path to the wordvectors is not set and a download not found previously in the default location you will be prompted if you want to download it.
  * 1. Automatic on first run of example: Download data (movie reviews) + extract
- * 2. Load existing Word2Vec model (for example: Google News word vectors. You will have to download this MANUALLY)
+ * 2. Load existing Word2Vec model (for example: Google News word vectors.)
  * 3. Load each each review. Convert words to vectors + reviews to sequences of vectors
  * 4. Train network
- *
+ * <p>
  * With the current configuration, gives approx. 83% accuracy after 1 epoch. Better performance may be possible with
  * additional tuning.
  *
- * NOTE / INSTRUCTIONS:
- * You will have to download the Google News word vector model manually. ~1.5GB
- * The Google News vector model available here: https://code.google.com/p/word2vec/
- * Download the GoogleNews-vectors-negative300.bin.gz file
- * Then: set the WORD_VECTORS_PATH field to point to this location.
- *
  * @author Alex Black
  */
 public class Word2VecSentimentRNN {
 
-    /** Data URL for downloading */
+    /**
+     * Data URL for downloading
+     */
     public static final String DATA_URL = "http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz";
-    /** Location to save and extract the training/testing data */
+    /**
+     * Location to save and extract the training/testing data
+     */
     public static final String DATA_PATH = FilenameUtils.concat(System.getProperty("java.io.tmpdir"), "dl4j_w2vSentiment/");
-    /** Location (local file system) for the Google News vectors. Set this manually. */
-    public static final String WORD_VECTORS_PATH = "/PATH/TO/YOUR/VECTORS/GoogleNews-vectors-negative300.bin.gz";
-
+    /**
+     * Location (local file system) for the Google News vectors. Set this manually.
+     */
+    public static String wordVectorsPath = "/PATH/TO/YOUR/VECTORS/GoogleNews-vectors-negative300.bin.gz";
 
     public static void main(String[] args) throws Exception {
-        if(WORD_VECTORS_PATH.startsWith("/PATH/TO/YOUR/VECTORS/")){
-            throw new RuntimeException("Please set the WORD_VECTORS_PATH before running this example");
+        if (wordVectorsPath.startsWith("/PATH/TO/YOUR/VECTORS/")) {
+            System.out.println("wordVectorsPath has not been set. Checking default location in ~/dl4j-examples-data for download...");
+            checkDownloadW2VECModel();
         }
-
         //Download and extract data
         downloadData();
 
@@ -109,7 +113,7 @@ public static void main(String[] args) throws Exception {
         net.init();
 
         //DataSetIterators for training and testing respectively
-        WordVectors wordVectors = WordVectorSerializer.loadStaticModel(new File(WORD_VECTORS_PATH));
+        WordVectors wordVectors = WordVectorSerializer.loadStaticModel(new File(wordVectorsPath));
         SentimentExampleIterator train = new SentimentExampleIterator(DATA_PATH, wordVectors, batchSize, truncateReviewsToLength, true);
         SentimentExampleIterator test = new SentimentExampleIterator(DATA_PATH, wordVectors, batchSize, truncateReviewsToLength, false);
 
@@ -119,7 +123,7 @@ public static void main(String[] args) throws Exception {
 
         //After training: load a single example and generate predictions
         File shortNegativeReviewFile = new File(FilenameUtils.concat(DATA_PATH, "aclImdb/test/neg/12100_1.txt"));
-        String shortNegativeReview = FileUtils.readFileToString(shortNegativeReviewFile, (Charset)null);
+        String shortNegativeReview = FileUtils.readFileToString(shortNegativeReviewFile, (Charset) null);
 
         INDArray features = test.loadFeaturesFromString(shortNegativeReview, truncateReviewsToLength);
         INDArray networkOutput = net.output(features);
@@ -138,15 +142,15 @@ public static void main(String[] args) throws Exception {
     public static void downloadData() throws Exception {
         //Create directory if required
         File directory = new File(DATA_PATH);
-        if(!directory.exists()) directory.mkdir();
+        if (!directory.exists()) directory.mkdir();
 
         //Download file:
         String archizePath = DATA_PATH + "aclImdb_v1.tar.gz";
         File archiveFile = new File(archizePath);
         String extractedPath = DATA_PATH + "aclImdb";
         File extractedFile = new File(extractedPath);
 
-        if( !archiveFile.exists() ){
+        if (!archiveFile.exists()) {
             System.out.println("Starting data download (80MB)...");
             FileUtils.copyURLToFile(new URL(DATA_URL), archiveFile);
             System.out.println("Data (.tar.gz file) downloaded to " + archiveFile.getAbsolutePath());
@@ -155,14 +159,39 @@ public static void downloadData() throws Exception {
         } else {
             //Assume if archive (.tar.gz) exists, then data has already been extracted
             System.out.println("Data (.tar.gz file) already exists at " + archiveFile.getAbsolutePath());
-            if( !extractedFile.exists()){
-            	//Extract tar.gz file to output directory
-            	DataUtilities.extractTarGz(archizePath, DATA_PATH);
+            if (!extractedFile.exists()) {
+                //Extract tar.gz file to output directory
+                DataUtilities.extractTarGz(archizePath, DATA_PATH);
             } else {
-            	System.out.println("Data (extracted) already exists at " + extractedFile.getAbsolutePath());
+                System.out.println("Data (extracted) already exists at " + extractedFile.getAbsolutePath());
             }
         }
     }
 
-
+    public static void checkDownloadW2VECModel() throws IOException {
+        String defaultwordVectorsPath = FilenameUtils.concat(System.getProperty("user.home"), "dl4j-examples-data/w2vec300");
+        String md5w2vec = "1c892c4707a8a1a508b01a01735c0339";
+        wordVectorsPath = new File(defaultwordVectorsPath, "GoogleNews-vectors-negative300.bin.gz").getAbsolutePath();
+        if (new File(wordVectorsPath).exists()) {
+            System.out.println("\n\tGoogleNews-vectors-negative300.bin.gz file found at path: " + defaultwordVectorsPath);
+            System.out.println("\tChecking md5 of existing file..");
+            if (Downloader.checkMD5OfFile(md5w2vec, new File(wordVectorsPath))) {
+                System.out.println("\tExisting file hash matches.");
+                return;
+            } else {
+                System.out.println("\tExisting file hash doesn't match. Retrying download...");
+            }
+        } else {
+            System.out.println("\n\tNo previous download of GoogleNews-vectors-negative300.bin.gz found at path: " + defaultwordVectorsPath);
+        }
+        System.out.println("\tWARNING: GoogleNews-vectors-negative300.bin.gz is a 1.5GB file.");
+        System.out.println("\tPress \"ENTER\" to start a download of GoogleNews-vectors-negative300.bin.gz to " + defaultwordVectorsPath);
+        Scanner scanner = new Scanner(System.in);
+        scanner.nextLine();
+        System.out.println("Starting model download (1.5GB!)...");
+        Downloader.download("Word2Vec", new URL("https://dl4jdata.blob.core.windows.net/resources/wordvectors/GoogleNews-vectors-negative300.bin.gz"), new File(wordVectorsPath), md5w2vec, 5);
+        System.out.println("Successfully downloaded word2vec model to " + wordVectorsPath);
+    }
 }
+
+