Made several updates associated with real-time SIFT classification,

including some bug fixes and optimizations.

Author: chkothe

.gitignore

@@ -17,3 +17,4 @@ userdata/
 LBPA40-atlas-2011-04-28/*
 LBPA40-atlas-2011-04-28/PDFGM/*
 
+/.project

RELEASE NOTES.TXT

@@ -237,6 +237,20 @@ Minor New Features
 - added the ability to read private plugins from ~/.bcilab/code
 - added robust re-referencing
 - flt_standarize now uses regularization for sphering
+- vis_artifacts: can now also display event markers
+- eeg_checkset: optimized chanlocs processing 
+- par_getworkers_ssh: added sanity checks, made shell command editable, added ability to auto-generate MATLAB command (default)
+  and added ability to bring up its own GUI
+- env_startup now creates a few more directories in ~/.bcilab (including code/scripts and logs/workers)
+- env_acquire_cluster/env_release_cluster (and "Request Cluster Availability" button) now by default override BCILAB's
+  current global setting of whether to run on the cluster and which worker pool to use, making routine cluster acquisition 
+  with the click of a button more convenient (given that the cluster acquire options are set up correctly in the config file 
+  or settings GUI)
+- flt_selchans: can now select the closest channels to the selected ones, if the desired channels are not in the data set
+- flt_iir: now does not need the Sigproc toolbox for online processing and runs with less overhead (removed dfilt dependency)
+- bci_batchtrain: now returns error messages produced during a cluster run, and allows more flexible wildcard syntax when
+  listing data sets to process (using rdir() rather than dir()), such as /data/mystudy/**/*.set
+- hlp_serialize: now serializes gpuArray data (as double arrays)
 
 Major Changes:
 - arg_tovals: now by default sets the arg_direct flag to false rather than true, meaning that some 
@@ -281,7 +295,11 @@ Minor Changes:
 - flt_ica: if TransformData is checked, by default the .icaweights field is cleared afterwards (can be disabled by 
   setting the ClearAfterTransform flag to false)
 - ParadigmSpectralmeans: now uses a FIR filter by default instead of IIR
-
+- arg_guidialog: now by default hides arguments marked "guru" (can be toggled with the ShowGuru option)
+- flt_fir: normalize_response now off by default
+- bci_batchtrain: was previously running locally by default; now the default matches the current global setting
+- fit_eeg_distribution: now by default fits an ensemble of truncated distributions and returns a quantile thereof 
+- flt_clean_windows: updated defaults in accordance with fit_eeg_distribution 
 
 Serious Fixes:
 - ml_calcloss: fixed calculation of FP and FN (Dan Roberts)
@@ -309,4 +327,7 @@ Minor Fixes:
 - ParadigmSpectralmeans: fixed an off-by-one error in frequency indexing (Dan Roberts)
 - flt_eog: fixed bug which disabled regression when removing EOG channels
 - utl_check_dataset: fixed a warning message
-
+- onl_append: changed the order of stream field updates to work with new MATLAB versions 
+- flt_movavg: fixed buffering bug when data had 0 samples
+- ParadigmFBCSP: used missing function when robust flag was set (cov_robust, now cov_blockgeom)
+- ParadigmSIFT: feature extraction now works for online processing

bcilab_config.m

@@ -48,7 +48,7 @@
 % But to automatically acquire machines, you need to have a Linux .ssh identity set up so that 
 % you can ssh into any of these machines without needing to type a password. Also, make sure that 
 % these machines/ports are never accessible from the internet (except perhaps via SSH tunnels).
-parallel = {'engine', 'local', 'pool', {'localhost:23547', 'localhost:23548', 'localhost:23549', 'localhost:23550', 'localhost:23551', 'localhost:23552', 'localhost:23553', 'localhost:23554'}};
+parallel = {'engine','local', 'pool', {'localhost:23547', 'localhost:23548', 'localhost:23549', 'localhost:23550', 'localhost:23551', 'localhost:23552', 'localhost:23553', 'localhost:23554'}};
 
 % this controls how likely BCILAB is to cache things on disk; since disk access may be slower than
 % recomputing the results again, this is the expected number of times you believe you'll need a
@@ -70,14 +70,9 @@
 % whether to show guru-level options in the GUI by default (this can be toggled while editing)
 show_guru = false;
 
-% custom options to control how workers are acquired; parameters to par_getworkers_* (rarely used)
-acquire_options = {};
-
-
-
-
-
-
+% custom options to control how workers are acquired; parameters to par_getworkers_*
+% these options are editable in the cluster settings dialog and are used when calling env_acquire_cluster (or clicking the "request cluster availability" button)
+acquire_options = {'Hostnames',{'localhost'},'ShutdownTimeout',300};
 
 
 

code/arguments/arg_guidialog.m

@@ -6,6 +6,12 @@
 % function must declare its arguments via arg_define. In addition, only a Subset of the function's
 % specified arguments can be displayed.
 %
+% If this function is called with no arguments from within a function that uses arg_define it brings
+% up a dialog for the function's defined arguments, with values assigned based on the function's
+% inputs (contents of in varargin) and returns an argument struct if the dialog was confirmed with a
+% click on OK, and an empty value otherwise. The calling function can exit on empty, or otherwise
+% resume with entered parameters.
+%
 % In:
 %   Function : the function for which to display arguments
 %
@@ -22,11 +28,15 @@
 %
 %                 'Invoke' : whether to invoke the function directly; in this case, the output
 %                            arguments are those of the function (default: true, unless called in 
-%                            the form g = arg_guidialog; e.g., from within some function)
+%                            the form g = arg_guidialog; from within some function)
+%
+%                 'ShowGuru' : whether to show parameters marked as guru-level. (default: according
+%                              to env_startup setting)
 %
 % Out:
-%   Parameters : either a struct that is a valid input to the Function or the outputs of the function
-%                when Invoke is set to true (possibly multiple outputs)
+%   Parameters : If Invoke was set to true, this is either the results of the function call or empty
+%                (if the dialog was cancelled). If Invoke was set to false, this is either a struct 
+%                that is a valid input to the Function, or an empty value if cancel was clicked. 
 %
 % Examples:
 %   % bring up a configuration dialog for the given function
@@ -41,6 +51,15 @@
 %   % bring up a dialog, and invoke the function with the selected settings after the user clicks OK
 %   settings = arg_guidialog(@myfunction,'Invoke',true)
 %
+%   % from within a function
+%   args = arg_define(varargin, ...);
+%   if nargin < 1
+%       % optionally allow the user to enter arguments via a GUI
+%       args = arg_guidialog;
+%       % user clicked cancel?
+%       if isempty(args), return; end
+%   end
+%
 % See also:
 %   arg_guidialog_ex, arg_guipanel, arg_define
 %
@@ -61,14 +80,22 @@
 % write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
 % USA
 
+global tracking;
+
 if ~exist('func','var')
-    % called with no arguments, from inside a function: open function dialog
+    % called with no arguments, from inside a function: open function dialog    
     func = hlp_getcaller;
     varargin = {'Parameters',evalin('caller','varargin'),'Invoke',nargout==0};
 end
 
 % parse arguments...
-hlp_varargin2struct(varargin,{'params','Parameters','parameters'},{}, {'subset','Subset'},{}, {'dialogtitle','title','Title'}, [char(func) '()'], {'buttons','Buttons'},[],{'invoke','Invoke'},true);
+hlp_varargin2struct(varargin, ...
+    {'params','Parameters','parameters'},{}, ...
+    {'subset','Subset'},{}, ...
+    {'dialogtitle','title','Title'}, [char(func) '()'], ...
+    {'buttons','Buttons'},[], ...
+    {'invoke','Invoke'},true, ...
+    {'show_guru','ShowGuru'},tracking.gui.show_guru);
 oldparams = params;
 
 % obtain the argument specification for the function
@@ -83,7 +110,7 @@
     subset(1) = [];
     subset = allnames(~ismember(allnames,[subset 'arg_direct']));
 end
-[spec,subset] = obtain_items(rawspec,subset);
+[spec,subset] = obtain_items(rawspec,subset,'',show_guru);
 
 % create an inputgui() dialog...
 geometry = repmat({[0.6 0.35]},1,length(spec)+length(buttons)/2);
@@ -95,7 +122,7 @@
     s = spec{k};
     if isempty(s)
         uilist(end+1:end+2) = {{} {}};
-    else        
+    else
         if isempty(s.help)
             error(['Cannot display the argument ' subset{k} ' because it contains no description.']);
         else
@@ -138,6 +165,7 @@
     end
 end
 
+
 % invoke the GUI, obtaining a list of output values...
 helptopic = char(func);
 try
@@ -149,6 +177,14 @@
 catch
     disp('Cannot deduce help topic.');
 end
+
+% % strip geometry for guru arguments
+% if ~show_guru
+%     tmpspec = [spec{:}];
+%     geometry([tmpspec.guru]) = [];
+%     geomvert([tmpspec.guru]) = [];
+% end
+
 [outs,dummy,okpressed] = inputgui('geometry',geometry, 'uilist',uilist,'helpcom',['env_doc ' helptopic], 'title',dialogtitle,'geomvert',geomvert); %#ok<ASGLU>
 
 if ~isempty(okpressed)
@@ -204,7 +240,7 @@
 end
 
 % obtain a cell array of spec entries by name from the given specification
-function [items,ids] = obtain_items(rawspec,requested,prefix)
+function [items,ids] = obtain_items(rawspec,requested,prefix,show_guru)
 if ~exist('prefix','var')
     prefix = ''; end
 items = {}; 
@@ -241,7 +277,7 @@
     % special case: switch arguments are not spliced, but instead the argument that defines the
     % option popupmenu will be retained
     if ~isempty(items{k}) && ~isempty(items{k}.children) && (~iscellstr(items{k}.range) || isempty(requested))
-        [subitems, subids] = obtain_items(items{k}.children,{},[ids{k} '.']);
+        [subitems, subids] = obtain_items(items{k}.children,{},[ids{k} '.'],show_guru);
         if ~isempty(subitems)
             % and introduce blank rows around them
             items = [items(1:k-1) {{}} subitems {{}} items(k+1:end)];
@@ -251,7 +287,7 @@
 end
 
 % remove items that cannot be displayed
-retain = cellfun(@(x)isempty(x)||x.displayable,items);
+retain = cellfun(@(x)isempty(x)||x.displayable&&(show_guru||~x.guru),items);
 items = items(retain);
 ids = ids(retain);
 

code/environment/env_acquire_cluster.m

@@ -31,7 +31,9 @@
     % invoke, but also impose default arguments (if unspecified)
     % by default, workers do not recruit (but only list) other workers, preventing a cascading effect
     try
-        par_getworkers_ssh(arguments{:});
+        tracking.parallel.pool = par_getworkers_ssh(arguments{:});
+        tracking.parallel.engine = 'BLS';
+        disp('Set default compute scheduler to BLS (parallel).');
     catch e
         disp('Could not acquire worker machines; traceback: ');
         env_handleerror(e);

code/environment/env_release_cluster.m

@@ -11,3 +11,5 @@
 global tracking;
 % this will lead to the deletion of any involved heartbeat timer.
 tracking.cluster_requested = [];
+tracking.parallel.engine = 'local';
+disp('Set default compute scheduling to local.');

code/environment/env_startup.m

@@ -435,8 +435,8 @@ function env_startup(varargin)
         % create directories in the user's .bcilab folder...
         home_basedir = [hlp_homedir filesep '.bcilab' filesep];
         home_codedirs = {['code' filesep 'filters'],['code' filesep 'dataset_editing'], ...
-            ['code' filesep 'machine_learning'], ['code' filesep 'paradigms']};
-        home_miscdirs = {'models','approaches','code',['code' filesep 'dependencies'],'logs'};
+            ['code' filesep 'machine_learning'], ['code' filesep 'paradigms'], ['code' filesep 'scripts']};
+        home_miscdirs = {'models','approaches','code',['code' filesep 'dependencies'],'logs',['logs' filesep 'workers']};
         for d = [home_codedirs home_miscdirs]
             try
                 subdir = [home_basedir d{1}];

code/filters/flt_clean_windows.m

@@ -96,7 +96,7 @@
 %                                Christian Kothe, Swartz Center for Computational Neuroscience, UCSD
 %                                2010-07-06
 
-% flt_clean_windows_version<1.0> -- for the cache
+% flt_clean_windows_version<1.01> -- for the cache
 
 if ~exp_beginfun('editing') return; end;
 
@@ -111,8 +111,8 @@
     arg_sub({'fit_params','ParameterFitting','parameter_fitting'}, {}, { ...
         arg({'max_dropout_fraction','MaxDropoutFraction'}, 0.1, [], 'Maximum fraction that can have dropouts. This is the maximum fraction of time windows that may have arbitrarily low amplitude (e.g., due to the sensors being unplugged).'), ...
         arg({'min_clean_fraction','MinCleanFraction'}, 0.25, [], 'Minimum fraction that needs to be clean. This is the minimum fraction of time windows that need to contain essentially uncontaminated EEG.'), ...
-        arg({'fit_quantiles','FitQuantiles'}, [0.022 0.6], [], 'Truncated Gaussian quantile. Quantile range [upper,lower] of the truncated Gaussian distribution that shall be fit to the EEG contents.','guru',true), ...
-        arg({'step_sizes','StepSizes'}, [0.001 0.01], [], 'Grid search stepping. Step size of the grid search, in quantiles; separately for [lower,upper] edge of the truncated Gaussian. The lower edge has finer stepping because the clean data density is assumed to be lower there, so small changes in quantile amount to large changes in data space.','guru',true), ...
+        arg({'fit_quantiles','FitQuantiles'}, [0.022*ones(21,1) (0.5:0.01:0.7)'], [], 'Truncated Gaussian quantile. Quantile range [upper,lower] of the truncated Gaussian distribution that shall be fit to the EEG contents.','guru',true), ...
+        arg({'step_sizes','StepSizes'}, [0.01 0.01], [], 'Grid search stepping. Step size of the grid search, in quantiles; separately for [lower,upper] edge of the truncated Gaussian. The lower edge has finer stepping because the clean data density is assumed to be lower there, so small changes in quantile amount to large changes in data space.','guru',true), ...
         arg_deprecated({'num_bins','NumBins'},50,[],'This parameter is now auto-determined.') ...
     }, 'Parameter fitting details. Group of sub-arguments that govern how EEG distribution parameters should be fit.'), ...
     arg({'keep_metadata','KeepMetadata'}, true, [], 'Retain metadata of EEG set. Retaining meta data (events, ICA decomposition, etc.) is quite slow.'), ...

code/filters/flt_fir.m

@@ -106,7 +106,7 @@
 %                                Christian Kothe, Swartz Center for Computational Neuroscience, UCSD
 %                                2010-04-17  
 
-% flt_fir_version<1.03> -- for the cache
+% flt_fir_version<1.04> -- for the cache
 
 if ~exp_beginfun('filter') return; end
 
@@ -122,7 +122,7 @@
     arg({'stopripple','StopbandRipple'}, -40, [-180 1], 'Maximum relative ripple amplitude in stop-band. Relative to nominal pass-band gain. Affects the filter length (i.e., delay). Assumed to be in db if negative, otherwise taken as a ratio.'), ...
     arg({'designrule','DesignRule'}, 'Frequency Sampling', {'Parks-McClellan','Window Method','Frequency Sampling'}, 'Filter design rule. Parks-McClellan minimizes the maximum error, the Window Method minimizes the square error, and Frequency Sampling constructs the filter via the Fourier transform without tuning (the latter requires no sigproc toolbox).'), ...
     arg({'chunk_length','ChunkLength'},50000,[], 'Maximum chunk length. Process the data in chunks of no larger than this (to avoid memory limitations).','guru',true), ...
-    arg({'normalize_amplitude','NormalizeAmplitude'},true,[], 'Normalize amplitude. Normalizes the amplitude such that the maximum gain is as desired. This helps with the occasional erratic filter design result.','guru',true), ...
+    arg({'normalize_amplitude','NormalizeAmplitude'},false,[], 'Normalize amplitude. Normalizes the amplitude such that the maximum gain is as desired. This helps with the occasional erratic filter design result.','guru',true), ...
     arg_nogui({'state','State'}));
 
 if isempty(state)
@@ -227,7 +227,7 @@
 
 [b,n] = deal(state.b,length(state.b));
 % process each known time series field
-for fld = {'data','srcpot','icaact'}
+for fld = utl_timeseries_fields(signal)
     field = fld{1};
     if isfield(signal,field) && ~isempty(signal.(field)) && ~isequal(signal.(field),1)
         if ~isfield(state,field)
@@ -277,9 +277,10 @@
     signal.etc.filter_delay = 0; end
 
 if strcmp(ftype,'linear-phase')
-    signal.etc.filter_delay = signal.etc.filter_delay + length(b)/2/signal.srate;
+    signal.etc.filter_delay = signal.etc.filter_delay + (length(b)/2-1)/signal.srate;
 elseif strcmp(ftype,'minimum-phase')
-    signal.etc.filter_delay = signal.etc.filter_delay + hlp_getresult(2,@max,b)/signal.srate;
+    [dummy,maxidx] = max(b); %#ok<ASGLU>
+    signal.etc.filter_delay = signal.etc.filter_delay + (maxidx-1)/signal.srate;
 end    
 
 exp_endfun;