Merge pull request #1070 from DonaldAlan/meldoy4j-review-971

Meldoy4j review 971

Author: agibsonccc

dl4j-examples/src/main/java/org/deeplearning4j/examples/wip/advanced/modelling/melodl4j/MelodyModelingExample.java renamed to dl4j-examples/src/main/java/org/deeplearning4j/examples/advanced/modelling/charmodelling/melodl4j/MelodyModelingExample.java

@@ -1,6 +1,4 @@
 /*******************************************************************************
- *
- *
  *
  * This program and the accompanying materials are made available under the
  * terms of the Apache License, Version 2.0 which is available at
@@ -17,7 +15,7 @@
  * SPDX-License-Identifier: Apache-2.0
  ******************************************************************************/
 
-package org.deeplearning4j.examples.wip.advanced.modelling.melodl4j;
+package org.deeplearning4j.examples.advanced.modelling.charmodelling.melodl4j;
 
 import org.apache.commons.io.FileUtils;
 import org.deeplearning4j.examples.advanced.modelling.charmodelling.utils.CharacterIterator;
@@ -31,39 +29,56 @@
 import org.deeplearning4j.nn.weights.WeightInit;
 import org.deeplearning4j.optimize.listeners.ScoreIterationListener;
 import org.deeplearning4j.util.ModelSerializer;
+import org.nd4j.common.util.ArchiveUtils;
 import org.nd4j.linalg.activations.Activation;
 import org.nd4j.linalg.api.ndarray.INDArray;
 import org.nd4j.linalg.dataset.DataSet;
 import org.nd4j.linalg.factory.Nd4j;
+import org.nd4j.linalg.learning.config.Adam;
 import org.nd4j.linalg.learning.config.RmsProp;
 import org.nd4j.linalg.lossfunctions.LossFunctions;
 
+import javax.sound.midi.InvalidMidiDataException;
 import java.io.*;
 import java.net.URL;
 import java.nio.charset.Charset;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.text.NumberFormat;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Random;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
 
 /**
  * LSTM  Symbolic melody modelling example, to compose music from symbolic melodies extracted from MIDI.
- * Based closely on LSTMCharModellingExample.java.
+ * LSTM logic is based closely on LSTMCharModellingExample.java.
  * See the README file in this directory for documentation.
  *
  * @author Alex Black, Donald A. Smith.
  */
 public class MelodyModelingExample {
-    final static String inputSymbolicMelodiesFilename = "bach-melodies-input.txt";
-    // Examples:  bach-melodies-input.txt, beatles-melodies-input.txt ,  pop-melodies-input.txt (large)
+    // If you want to change the MIDI files used in learning, create a zip file containing your MIDI
+    // files and replace the following path.  For example, you might use something like:
+    //final static String midiFileZipFileUrlPath = "file:d:/music/midi/classical-midi.zip";
+    final static String midiFileZipFileUrlPath = "http://waliberals.org/truthsite/music/bach-midi.zip";
 
-    final static String tmpDir = System.getProperty("java.io.tmpdir");
+    // For example "bach-midi.txt"
+    final static String inputSymbolicMelodiesFilename = getMelodiesFileNameFromURLPath(midiFileZipFileUrlPath);
 
-    final static String symbolicMelodiesInputFilePath = tmpDir + "/" + inputSymbolicMelodiesFilename;  // Point to melodies created by MidiMelodyExtractor.java
+    // Examples:  bach-melodies-input.txt, beatles-melodies-input.txt ,  pop-melodies-input.txt (large)
+    final static String tmpDir = System.getProperty("java.io.tmpdir");
+    final static String inputSymbolicMelodiesFilePath = tmpDir + "/" + inputSymbolicMelodiesFilename;  // Point to melodies created by MidiMelodyExtractor.java
     final static String composedMelodiesOutputFilePath = tmpDir + "/composition.txt"; // You can listen to these melodies by running PlayMelodyStrings.java against this file.
 
     //final static String symbolicMelodiesInputFilePath = "D:/tmp/bach-melodies.txt";
     //final static String composedMelodiesOutputFilePath = tmpDir + "/bach-composition.txt"; // You can listen to these melodies by running PlayMelodyStrings.java against this file.
-
+    final static NumberFormat numberFormat = NumberFormat.getNumberInstance();
+    static {
+        numberFormat.setMinimumFractionDigits(1);
+        numberFormat.setMaximumFractionDigits(1);
+    }
     //....
     public static void main(String[] args) throws Exception {
         String loadNetworkPath = null; //"/tmp/MelodyModel-bach.zip"; //null;
@@ -73,6 +88,8 @@ public static void main(String[] args) throws Exception {
             generationInitialization = args[1];
         }
 
+        makeMidiStringFileIfNecessary();
+
         int lstmLayerSize = 200;                    //Number of units in each LSTM layer
         int miniBatchSize = 32;                     //Size of mini batch to use when training
         int exampleLength = 500; //1000; 		    //Length of each training example sequence to use.
@@ -107,9 +124,10 @@ public static void main(String[] args) throws Exception {
 
         //Set up network configuration:
         MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
-            .updater(new RmsProp(0.1))
-            .seed(12345)
-            .l2(0.001)
+            //.updater(new RmsProp(0.1))
+            .updater(new Adam(0.005))
+            .seed(System.currentTimeMillis()) // So each run generates new melodies
+            .l2(0.0001)
             .weightInit(WeightInit.XAVIER)
             .list()
             .layer(0, new LSTM.Builder().nIn(iter.inputColumns()).nOut(lstmLayerSize)
@@ -123,7 +141,6 @@ public static void main(String[] args) throws Exception {
             .backpropType(BackpropType.TruncatedBPTT).tBPTTForwardLength(tbpttLength).tBPTTBackwardLength(tbpttLength)
             .build();
 
-
         learn(miniBatchSize, exampleLength, numEpochs, generateSamplesEveryNMinibatches, nSamplesToGenerate, nCharactersToSample, generationInitialization, rng, startTime, iter, conf);
     }
 
@@ -154,6 +171,7 @@ private static void learn(int miniBatchSize, int exampleLength, int numEpochs, i
         // order, so that the best melodies are at the start of the file.
         //Do training, and then generate and print samples from network
         int miniBatchNumber = 0;
+        long lastTime = System.currentTimeMillis();
         for (int epoch = 0; epoch < numEpochs; epoch++) {
             System.out.println("Starting epoch " + epoch);
             while (iter.hasNext()) {
@@ -176,12 +194,19 @@ private static void learn(int miniBatchSize, int exampleLength, int numEpochs, i
                 }
             }
             iter.reset();    //Reset iterator for another epoch
+            final double secondsForEpoch = 0.001 * (System.currentTimeMillis() - startTime);
+            final long now = System.currentTimeMillis();
             if (melodies.size() > 0) {
                 String melody = melodies.get(melodies.size() - 1);
                 int seconds = 25;
                 System.out.println("\nFirst " + seconds + " seconds of " + melody);
                 PlayMelodyStrings.playMelody(melody, seconds);
             }
+            double seconds = 0.001*(now - lastTime);
+            lastTime = now;
+            System.out.println("\nEpoch " + epoch + " time in seconds: " + numberFormat.format(seconds));
+            // 531.9 for GPU GTX 1070
+            // 821.4 for CPU i7-6700K @ 4GHZ
         }
         int indexOfLastPeriod = inputSymbolicMelodiesFilename.lastIndexOf('.');
         String saveFileName = inputSymbolicMelodiesFilename.substring(0, indexOfLastPeriod > 0 ? indexOfLastPeriod : inputSymbolicMelodiesFilename.length());
@@ -193,42 +218,82 @@ private static void learn(int miniBatchSize, int exampleLength, int numEpochs, i
             printWriter.println(melodies.get(i));
         }
         printWriter.close();
-        double seconds = 0.001 * (System.currentTimeMillis() - startTime);
 
-        System.out.println("\n\nExample complete in " + seconds + " seconds");
         System.exit(0);
     }
 
-    public static void makeSureFileIsInTmpDir(String filename) {
+    public static File makeSureFileIsInTmpDir(String urlString) throws IOException {
+        final URL url = new URL(urlString);
+        final String filename = urlString.substring(1+urlString.lastIndexOf("/"));
         final File f = new File(tmpDir + "/" + filename);
-        if (!f.exists()) {
-            URL url = null;
-            try {
-                url = new URL("http://truthsite.org/music/" + filename);
-                FileUtils.copyURLToFile(url, f);
-            } catch (Exception exc) {
-                System.err.println("Error copying " + url + " to " + f);
-                throw new RuntimeException(exc);
-            }
+        if (f.exists()) {
+            System.out.println("Using existing " + f.getAbsolutePath());
+        } else {
+            FileUtils.copyURLToFile(url, f);
             if (!f.exists()) {
                 throw new RuntimeException(f.getAbsolutePath() + " does not exist");
             }
             System.out.println("File downloaded to " + f.getAbsolutePath());
-        } else {
-            System.out.println("Using existing text file at " + f.getAbsolutePath());
         }
+        return f;
     }
 
+    //https://stackoverflow.com/questions/10633595/java-zip-how-to-unzip-folder
+    public static void unzip(File zipFile, File targetDirFile) throws IOException {
+        InputStream is =  new FileInputStream(zipFile);
+        Path targetDir = targetDirFile.toPath();
+        targetDir = targetDir.toAbsolutePath();
+        try (ZipInputStream zipIn = new ZipInputStream(is)) {
+            for (ZipEntry ze; (ze = zipIn.getNextEntry()) != null; ) {
+                Path resolvedPath = targetDir.resolve(ze.getName()).normalize();
+                if (!resolvedPath.startsWith(targetDir)) {
+                    // see: https://snyk.io/research/zip-slip-vulnerability
+                    throw new RuntimeException("Entry with an illegal path: "
+                            + ze.getName());
+                }
+                if (ze.isDirectory()) {
+                    Files.createDirectories(resolvedPath);
+                } else {
+                    Files.createDirectories(resolvedPath.getParent());
+                    Files.copy(zipIn, resolvedPath);
+                }
+            }
+        }
+        is.close();
+    }
+    private static void makeMidiStringFileIfNecessary() throws IOException, InvalidMidiDataException {
+        final File inputMelodiesFile = new File(inputSymbolicMelodiesFilePath);
+        if (inputMelodiesFile.exists() && inputMelodiesFile.length()>1000) {
+            System.out.println("Using existing " + inputSymbolicMelodiesFilePath);
+            return;
+        }
+        final File midiZipFile = makeSureFileIsInTmpDir(midiFileZipFileUrlPath);
+        final String midiZipFileName = midiZipFile.getName();
+        final String midiZipFileNameWithoutSuffix = midiZipFileName.substring(0,midiZipFileName.lastIndexOf("."));
+        final File outputDirectoryFile  = new File(tmpDir,midiZipFileNameWithoutSuffix);
+        final String outputDirectoryPath = outputDirectoryFile.getAbsolutePath();
+        if (!outputDirectoryFile.exists()) {
+            outputDirectoryFile.mkdir();
+        }
+        if (!outputDirectoryFile.exists() || !outputDirectoryFile.isDirectory()) {
+            throw new IllegalStateException(outputDirectoryFile + " is not a directory or can't be created");
+        }
+        final PrintStream printStream = new PrintStream(inputSymbolicMelodiesFilePath);
+        System.out.println("Unzipping "+ midiZipFile.getAbsolutePath() + " to " + outputDirectoryPath);
+        unzip(midiZipFile, outputDirectoryFile);
+        System.out.println("Extracted " + midiZipFile.getAbsolutePath() + " to " + outputDirectoryPath);
+        MidiMelodyExtractor.processDirectoryAndWriteMelodyFile(outputDirectoryFile,inputMelodiesFile);
+        printStream.close();
+    }
     /**
      * Sets up and return a simple DataSetIterator that does vectorization based on the melody sample.
      *
      * @param miniBatchSize  Number of text segments in each training mini-batch
      * @param sequenceLength Number of characters in each text segment.
      */
     public static CharacterIterator getMidiIterator(int miniBatchSize, int sequenceLength) throws Exception {
-        makeSureFileIsInTmpDir(inputSymbolicMelodiesFilename);
         final char[] validCharacters = MelodyStrings.allValidCharacters.toCharArray(); //Which characters are allowed? Others will be removed
-        return new CharacterIterator(symbolicMelodiesInputFilePath, Charset.forName("UTF-8"),
+        return new CharacterIterator(inputSymbolicMelodiesFilePath, Charset.forName("UTF-8"),
             miniBatchSize, sequenceLength, validCharacters, new Random(12345), MelodyStrings.COMMENT_STRING);
     }
 
@@ -312,5 +377,13 @@ public static int sampleFromDistribution(double[] distribution, Random rng) {
         //Should be extremely unlikely to happen if distribution is a valid probability distribution
         throw new IllegalArgumentException("Distribution is invalid? d=" + d + ", sum=" + sum);
     }
+    private static String getMelodiesFileNameFromURLPath(String midiFileZipFileUrlPath) {
+        if (!(midiFileZipFileUrlPath.endsWith(".zip") || midiFileZipFileUrlPath.endsWith(".ZIP"))) {
+            throw new IllegalStateException("zipFilePath must end with .zip");
+        }
+        midiFileZipFileUrlPath = midiFileZipFileUrlPath.replace('\\','/');
+        String fileName = midiFileZipFileUrlPath.substring(midiFileZipFileUrlPath.lastIndexOf("/") + 1);
+        return fileName + ".txt";
+    }
 }