Merge pull request #128 from SeqWare/feature/messaging

Feature/messaging

Former-commit-id: ce492bb

Author: denis-yuen

seqware-common/src/main/java/net/sourceforge/seqware/common/hibernate/WorkflowRunReport.java

@@ -185,6 +185,9 @@ protected Collection<Processing> collectProcessings(WorkflowRun wr) {
      * @return a {@link java.lang.String} object.
      */
     public String calculateTotalTime(Collection<Processing> processings) {
+      if (processings.isEmpty())
+        return "";
+      
         Date earlyDate = new Date(Long.MAX_VALUE), lateDate = new Date(0);
 
         for (Processing p : processings) {

seqware-common/src/main/java/net/sourceforge/seqware/common/util/configtools/ConfigTools.java

@@ -24,17 +24,13 @@ public class ConfigTools {
   /** Constant <code>SEQWARE_SETTINGS_PROPERTY="SEQWARE_SETTINGS"</code> */
   public static final String SEQWARE_SETTINGS_PROPERTY = "SEQWARE_SETTINGS";
 
-  public static Map<String, String> getSettings() {
-      return getSettings(false);
-  }
-  
   /**
    * The output keys are always uppercase!
    *
    * @throws java.lang.Exception if any.
    * @return a {@link java.util.Map} object.
    */
-  public static Map<String, String> getSettings(boolean ignorePermissions) {
+  public static Map<String, String> getSettings() {
 
     // first, try to figure out the location of the settings file
     String settings = getSettingsFilePath();
@@ -48,23 +44,6 @@ public static Map<String, String> getSettings(boolean ignorePermissions) {
       throw new RuntimeException("The settings file " + settings + " is not a file!");
     }
 
-    //SEQWARE-1595 : it seems that the Java 6 File API cannot retrieve permissions separated by owner and group
-    // will use Linux command until Java 7 NIO (hopefully)
-    String settingPerms = FileTools.determineFilePermissions(settings);
-    if (!ignorePermissions && !settingPerms.equals("-rw-------") && !settingPerms.equals("-rwx------")){  
-        String bigWarning = "*** SECURITY WARNING ***\nSeqWare settings file has incorrect file permissions. It should only be readable and writeable by the owner.\n In other words, run \"chmod 600 ~/.seqware/settings\"";
-        Log.fatal(bigWarning);
-        Log.stderr(bigWarning);
-    }
-    
-    // else it should be OK
-    // make sure the permissions are OK
-    //settingsFile.setExecutable(false, false);
-    //settingsFile.setWritable(false, false);
-    //settingsFile.setWritable(true, true);
-    //settingsFile.setReadable(false, false);
-    //settingsFile.setReadable(true, true);
-
     HashMap<String, String> hm = new HashMap<String, String>();
 
     // now read back the values from the settings file

seqware-common/src/test/java/net/sourceforge/seqware/common/util/configtools/ConfigToolsTest.java

@@ -82,7 +82,7 @@ public void tearDown() {
   public void testGetSettings() throws Exception {
     System.setProperty(SEQWARE_SETTINGS_PROPERTY, getClass().getResource("ConfigToolsTest_settings").getPath());
 
-    Map<String, String> settings = ConfigTools.getSettings(true);
+    Map<String, String> settings = ConfigTools.getSettings();
 
     assertEquals(settings.get("SETTING1"), "one");
     assertEquals(settings.get("SETTING2"), "TWO");

seqware-distribution/docs/README

@@ -77,6 +77,10 @@ In order to publish to our unstable docs supporting the development of 1.0.0, ch
     cd seqware-distribution 
     mvn site-deploy
 
+If doing this in succession, you will see a git checkout error.  Until we fix the maven pom, you will need to:
+
+    rm -Rf target/scmpublish-checkout/
+
 Then you would look at the site at: http://seqware.github.io/unstable.seqware.github.com/
 
 

seqware-distribution/docs/site/content/docs/17-plugins.md

@@ -0,0 +1,117 @@
+---
+
+title:                 "Modules"
+toc_includes_sections: true
+markdown:              advanced
+
+---
+
+
+##  GenericCommandRunner
+net.sourceforge.seqware.pipeline.modules.GenericCommandRunner
+This is a simple command runner.
+
+| Command-line option | Description |
+|--------------------|--------------|
+|--gcr-algorithm|You can pass in an algorithm name that will be recorded in the metadb if you are writing back to the metadb, otherwise GenericCommandRunner is used.|
+|--gcr-check-output-file|Specify the path to the file.|
+|--gcr-command|The command being executed (quote as needed).|
+|--gcr-output-file|Specify this option one or more times for each output file created by the command called by this module. The argument is a '::' delimited list of type, meta_type, and file_path.|
+|--gcr-skip-if-missing|If the registered output files don't exist don't worry about it. Useful for workflows that can produce variable file outputs but also potentially dangerous.|
+|--gcr-skip-if-output-exists|If the registered output files exist then this step won't be run again. This only works if gcr-output-file is defined too since we need to be able to check the output files to see if they exist. If this step produces no output files then it's hard to say if it was run successfully before.|
+|--gcr-stderr|Optional: Returns stderr (stderr of the command called is normally suppressed, except in case of failure)|
+|--gcr-stdout|Optional: Reports stdout (stdout of the command called is normally suppressed, except in case of failure)|
+
+
+##  GenericMetadataSaver
+net.sourceforge.seqware.pipeline.modules.GenericMetadataSaver
+This is a simple metadata saver.
+
+| Command-line option | Description |
+|--------------------|--------------|
+|--gms-algorithm|You can pass in an algorithm name that will be recorded in the metadb if you are writing back to the metadb.|
+|--gms-output-file|Specify this option one or more times for each output file created by the command called by this module. The argument is a '::' delimited list of type, meta_type, and file_path.|
+|--gms-suppress-output-file-check|If provided, this will suppress checking that the gms-output-file options contain valid file paths. Useful if these are remote resources like HTTP or S3 file URLs.|
+
+
+##  ProvisionDependenciesBundle
+net.sourceforge.seqware.pipeline.modules.utilities.ProvisionDependenciesBundle
+
+
+| Command-line option | Description |
+|--------------------|--------------|
+|--i, --input-file|Required: input file, multiple should be specified seperately|
+|--o, --output-dir|Required: output file location|
+
+
+##  ProvisionFiles
+net.sourceforge.seqware.pipeline.modules.utilities.ProvisionFiles
+
+
+| Command-line option | Description |
+|--------------------|--------------|
+|--a, --algorithm|Optional: by default the algorithm is 'ProvisionFiles' but you can override here if you like.|
+|--d, --decrypt-key|Optional: if specified this key will be used to decrypt data when reading from its source.|
+|--decrypt-key-from-settings, --dkfs|Optional: if flag is specified then the key will be read from the SW_DECRYPT_KEY field in your SeqWare settings file and used to decrypt data as its pulled from the source.  If this option is specified along with --decrypt-key the key provided by the latter will be used.|
+|--e, --encrypt-key|Optional: if specified this key will be used to encrypt data before writing to its destination.|
+|--ekfs, --encrypt-key-from-settings|Optional: if flag is specified then the key will be read from the SW_ENCRYPT_KEY field in your SeqWare settings file and used to encrypt data before writing to its destination.  If this option is specified along with --encrypt-key the key provided by the latter will be used.|
+|--force-copy|Optional: if this is specified local to local file transfers are done with a copy rather than symlink. This is useful if you're writing to a temp area that will be deleted so you have to move the file essentially.|
+|--i, --input-file|Required: use this or --input-file-metadata, this is the input file, multiple should be specified seperately|
+|--im, --input-file-metadata|Required: use this or --input-file, this is the input file, multiple should be specified seperately|
+|--o, --output-dir|Required: output file location|
+|--of, --output-file|Optional: output file path, if this is provided than the program accepts exactly one --input-file and one --output file. If an --output-dir is also specified an error will be thrown.|
+|--r, --recursive|Optional: if the input-file points to a local directory then this option will cause the program to recursively copy the directory and its contents to the destination. An actual copy will be done for local to local copies rather than symlinks.|
+|--s3-connection-timeout|Optional: Sets the amount of time to wait (in milliseconds) when initially establishing a connection before giving up and timing out. Default is 50000|
+|--s3-max-connections|Optional: Sets the maximum number of allowed open HTTPS connections. Default is 50|
+|--s3-max-error-retries|Optional: Sets the maximum number of retry attempts for failed retryable requests (ex: 5xx error responses from services). Default is 3|
+|--s3-max-socket-timeout|Optional: Sets the amount of time to wait (in milliseconds) for data to be transfered over an established, open connection before the connection times out and is closed. A value of 0 means infinity, and isn't recommended. Default is 50000|
+|--s3-no-server-side-encryption|Optional: If specified, do not use S3 server-side encryption. Default is to use S3 server-side encryption for S3 destinations.|
+|--skip-if-missing|Optional: useful for workflows with variable output files, this will silently skip any missing inputs (this is a little dangerous).|
+|--skip-record-file|Optional: If specified, do not record new entries in the file table.|
+|--v, --verbose|Optional: verbose causes the S3 transfer status to display.|
+
+
+##  S3CreateFileURLs
+net.sourceforge.seqware.pipeline.modules.utilities.S3CreateFileURLs
+
+
+| Command-line option | Description |
+|--------------------|--------------|
+|--a, --all-files|Optional: if specified, the --s3-url should take the form s3://<bucket>. This option indicates all files in that bucket should have URLs created.|
+|--l, --lifetime|How long (in minutes) should this URL be valid for (129600 = 90 days, 86400 = 60 days, 43200 = 30 days, 10080 = 7 days, 1440 = 1 day).|
+|--u, --s3-url|A URL of the form s3://<bucket>/<path>/<file> or s3://<bucket> if using the --all-files option|
+
+
+##  S3DeleteFiles
+net.sourceforge.seqware.pipeline.modules.utilities.S3DeleteFiles
+
+
+| Command-line option | Description |
+|--------------------|--------------|
+|--f, --s3-url-file|Optional: a file containing one URL per line of the form s3://<bucket>/<path>/<file>|
+|--u, --s3-url|Optional: a URL of the form s3://<bucket>/<path>/<file>|
+
+
+##  S3ListFiles
+net.sourceforge.seqware.pipeline.modules.utilities.S3ListFiles
+
+
+| Command-line option | Description |
+|--------------------|--------------|
+|--in-bytes|Optional: flag, if set values print in bytes rather than human friendsly|
+|--l, --list-buckets|Optional: list all the buckets you own.|
+|--reset-owner-permissions|Optional: this will give the bucket owner full read/write permissions, useful if many different people have been writing to the same bucket.|
+|--s, --search-local-dir|Optional: attempt to match files in S3 with files in this local directory.|
+|--u, --s3-url|Optional: a URL of the form s3://<bucket>/<path>/<file>|
+|--t, --tab-output-file|Optional: tab-formated output file.|
+
+
+##  S3UploadDirectory
+net.sourceforge.seqware.pipeline.modules.utilities.S3UploadDirectory
+
+
+| Command-line option | Description |
+|--------------------|--------------|
+|--b, --output-bucket|Required: the output bucket name in S3|
+|--i, --input-dir|Required: the directory to copy recursively|
+|--p, --output-prefix|Required: the prefix to add after the bucket name.|

seqware-distribution/docs/site/content/docs/17a-modules.md

@@ -36,70 +36,35 @@ By the end of these tutorials you will:
 ## How to Install a Workflow
 
 <!-- make this install from a zip for the admin guide --> 
-When provided with a tested workflow from a workflow developer, the next step
+When provided with a tested workflow bundle from a workflow developer, the next step
 is to install it, this means it will be inserted into the MetaDB via a running
-web service.  During this process it will copy the bundle and put it into your
-released-bundles directory and provision it into your provisioned bundles
+web service.  During this process it will copy the bundle into your
+released-bundles directory and provision it into your provisioned-bundles
 directory. The provisioned bundles directory is where running workflows will
 access their files.
 
 Here is an example showing how this process works on the VM and what is
 happening in the database and your released-bundles directory as you do this.
-You may want to delete the zip file that is in the released-bundles directory
-before you do this step below (or back it up somewhere first).
 
 See the [Developer Tutorial](/docs/3-getting-started/developer-tutorial/) for
 how to make the zipped workflow bundle. After the zip bundle is created, the
 bundle can be provided to the admin for install as below.
 
 	$ seqware bundle install --zip ~/packaged-bundles/Workflow_Bundle_MyHelloWorld_1.0-SNAPSHOT_SeqWare_<%= seqware_release_version %>.zip 
-	Installing Bundle
-	Bundle: /home/seqware/packaged-bundles/Workflow_Bundle_MyHelloWorld_1.0-SNAPSHOT_SeqWare_<%= seqware_release_version %>.zip
-	Now packaging /home/seqware/packaged-bundles/Workflow_Bundle_MyHelloWorld_1.0-SNAPSHOT_SeqWare_<%= seqware_release_version %>.zip to a zip file and transferring to the directory: /home/seqware/released-bundles Please be aware, this process can take hours if the bundle is many GB in size.
-	  PROCESSING INPUT: /home/seqware/packaged-bundles/Workflow_Bundle_MyHelloWorld_1.0-SNAPSHOT_SeqWare_<%= seqware_release_version %>.zip OUTPUT: /home/seqware/released-bundles OUTPUT FILE: null
+	Now transferring /home/seqware/packaged-bundles/Workflow_Bundle_MyHelloWorld_1.0-SNAPSHOT_SeqWare_<%= seqware_release_version %>.zip to the directory: /home/seqware/released-bundles Please be aware, this process can take hours if the bundle is many GB in size.
+Processing input: /home/seqware/packaged-bundles/Workflow_Bundle_MyHelloWorld_1.0-SNAPSHOT_SeqWare_<%= seqware_release_version %>.zip
+      output-dir: /home/seqware/released-bundles
+
+WORKFLOW_ACCESSION: 16
+Bundle Has Been Installed to the MetaDB and Provisioned to /home/seqware/packaged-bundles/Workflow_Bundle_MyHelloWorld_1.0-SNAPSHOT_SeqWare_<%= seqware_release_version %>.zip!
 
-	WORKFLOW_ACCESSION: 17
-	Bundle Has Been Installed to the MetaDB and Provisioned to /home/seqware/packaged-bundles/Workflow_Bundle_MyHelloWorld_1.0-SNAPSHOT_SeqWare_<%= seqware_release_version %>.zip!
 
 What happens here is the <code>Workflow_Bundle_MyHelloWorld_1.0-SNAPSHOT_SeqWare_<%= seqware_release_version %>.zip</code> copied to your released-bundles directory and unzip'd into your provisioned-bundles directory. The metadata about the workflow is then saved to the database.
 
 <%= render '/includes/monitor_workflows/' %>
 
 For more information see the [Monitor Configuration](/docs/6-pipeline/monitor_configuration/) documentation.
 
-## How to Cancel Workflows
-
-After launching a workflow, you can cancel it in order to stop further execution. This will set the status of the workflow run to `submitted_cancel`, and after the next status propagation, will set the status to `cancelled`.
-
-    $ seqware workflow-run cancel --accession 28
-    $ seqware workflow-run report --accession 28
-	-[ RECORD 0 ]------------------+-------------------------------------
-	Workflow                       | HelloWorld 1.0-SNAPSHOT              
-	Workflow Run SWID              | 28                                   
-	Workflow Run Status            | cancelled                            
-	Workflow Run Create Timestamp  | 2013-08-23 14:58:35.374              
-	Workflow Run Host              | master                               
-	Workflow Run Status Command    | 0000000-130823144527997-oozie-oozi-W 
-	Library Sample Names           |                                      
-	Library Sample SWIDs           |                                      
-	Identity Sample Names          |                                      
-	Identity Sample SWIDs          |                                      
-	Input File Meta-Types          |                                      
-	Input File SWIDs               |                                      
-	Input File Paths               |                                      
-	Immediate Input File Meta-Types|                                      
-	Immediate Input File SWIDs     |                                      
-	Immediate Input File Paths     |                                      
-	Output File Meta-Types         |                                      
-	Output File SWIDs              |                                      
-	Output File Paths              |                                      
-	Workflow Run Time              | 33.0s
-
-
-## How to Retry Failed and Cancelled Workflows
-
-<%= render '/includes/debug/oozie_restart/' %>
-
 ## See Also
 
 <p class="warning"><strong>Note:</strong>