Skip to content

Commit c44f44e

Browse files
t-bt-b
authored
Merge pull request #2433 from AllenInstitute/bugfix/2433-find-indizes-make-it-faster
Make FindIndizes faster
2 parents 73bcf57 + 065216b commit c44f44e

File tree

5 files changed

+148
-33
lines changed

5 files changed

+148
-33
lines changed

‎Packages/MIES/MIES_Cache.ipf‎

Lines changed: 27 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -298,6 +298,21 @@ threadsafe static Function CA_WaveScalingCRC(variable crc, WAVE wv, [variable di
298298
return crc
299299
End
300300

301+
/// @brief Calculate CRC values of the wave `dims` giving the dimensions of a wave
302+
threadsafe static Function CA_WaveSizeCRC(WAVE dims)
303+
304+
variable numRows, crc, i
305+
306+
numRows = DimSize(dims, ROWS)
307+
ASSERT_TS(numRows > 0 && numRows <= MAX_DIMENSION_COUNT && DimSize(dims, COLS) <= 1, "Invalid dims dimensions")
308+
309+
for(i = 0; i < numRows; i += 1)
310+
crc = StringCRC(crc, num2istr(dims[i]))
311+
endfor
312+
313+
return crc
314+
End
315+
301316
/// @brief Calculate all CRC values of the waves referenced in waveRefs
302317
///
303318
/// @param waveRefs wave reference wave
@@ -361,18 +376,23 @@ End
361376
/// Only the size is relevant, the rest is undefined.
362377
threadsafe Function/S CA_TemporaryWaveKey(WAVE dims)
363378

364-
variable numRows, crc, i
365-
366-
numRows = DimSize(dims, ROWS)
367-
ASSERT_TS(numRows > 0 && numRows <= MAX_DIMENSION_COUNT && DimSize(dims, COLS) <= 1, "Invalid dims dimensions")
379+
variable crc
368380

369-
for(i = 0; i < numRows; i += 1)
370-
crc = StringCRC(crc, num2istr(dims[i]))
371-
endfor
381+
crc = CA_WaveSizeCRC(dims)
372382

373383
return num2istr(crc) + "Temporary waves Version 2"
374384
End
375385

386+
/// @brief Key generator for FindIndizes
387+
threadsafe Function/S CA_FindIndizesKey(WAVE dims)
388+
389+
variable crc
390+
391+
crc = CA_WaveSizeCRC(dims)
392+
393+
return num2istr(crc) + "FindIndizes Version 1"
394+
End
395+
376396
/// @brief Calculate the cache key for the hardware device info wave
377397
Function/S CA_HWDeviceInfoKey(string device, variable hardwareType, variable deviceID)
378398

‎Packages/MIES/MIES_Constants.ipf‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1982,7 +1982,8 @@ Constant TP_GETVALUES_LATEST_AUTOTPCYCLE = 0x1
19821982
/// Possible names for TSDS_Read*/TSDS_Write
19831983
/// @anchor ThreadsafeDataExchangeNames
19841984
///@{
1985-
StrConstant TSDS_BUGCOUNT = "BugCount"
1985+
StrConstant TSDS_BUGCOUNT = "BugCount"
1986+
StrConstant TSDS_PROCCOUNT = "ProcessorCount"
19861987
///@}
19871988

19881989
/// Headstage contingency modes

‎Packages/MIES/MIES_IgorHooks.ipf‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -335,6 +335,8 @@ static Function AfterCompiledHook()
335335

336336
MultiThreadingControl setmode=4
337337

338+
TSDS_WriteVar(TSDS_PROCCOUNT, ThreadprocessorCount)
339+
338340
LOG_AddEntry(PACKAGE_MIES, "end")
339341

340342
if(!modifiedBefore)

‎Packages/MIES/MIES_MiesUtilities_Algorithm.ipf‎

Lines changed: 71 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -372,6 +372,48 @@ threadsafe static Function DecimateMinMax(WAVE input, WAVE output, variable idx,
372372
output[targetLast][colOut] = V_max
373373
End
374374

375+
/// @brief Return the number of threads for `Multithread` given a problem with dimension `dims`
376+
///
377+
/// The algorithm was developed for "easy" problems on the RHS of MultiThread.
378+
///
379+
/// Example:
380+
///
381+
/// \rst
382+
/// .. code-block:: igorpro
383+
///
384+
/// Make/FREE/N=(1025, 10) data
385+
/// WAVE dims = GetWaveDimensions(data)
386+
/// variable numThreads = GetNumberOfUsefulThreads(dims)
387+
/// MultiThread/Y=(numThreads) data = ...
388+
///
389+
/// \endrst
390+
threadsafe Function GetNumberOfUsefulThreads(WAVE dims)
391+
392+
variable pointsPerThread, numCores, numPoints, numThreads, numRows, numCols, i
393+
394+
ASSERT_TS(WaveExists(dims) && IsNumericWave(dims), "Needs a numeric wave")
395+
396+
numRows = DimSize(dims, ROWS)
397+
ASSERT_TS(numRows <= MAX_DIMENSION_COUNT, "Expected at most MAX_DIMENSION_COUNT rows")
398+
399+
numCols = DimSize(dims, COLS)
400+
ASSERT_TS(numCols <= 1, "Expected a 1D wave")
401+
402+
pointsPerThread = 4096
403+
numCores = TSDS_ReadVar(TSDS_PROCCOUNT)
404+
405+
numPoints = 1
406+
for(i = 0; i < numRows; i += 1)
407+
numPoints *= max(1, dims[i])
408+
endfor
409+
410+
numThreads = min(ceil(numPoints / pointsPerThread), numCores)
411+
412+
ASSERT_TS(IsInteger(numThreads) && numThreads > 0, "Invalid thread count")
413+
414+
return numThreads
415+
End
416+
375417
/// @brief Extended version of `FindValue`
376418
///
377419
/// Allows to search only the specified column for a value
@@ -404,7 +446,7 @@ End
404446
/// value could not be found.
405447
threadsafe Function/WAVE FindIndizes(WAVE numericOrTextWave, [variable col, string colLabel, variable var, string str, variable prop, variable startRow, variable endRow, variable startLayer, variable endLayer])
406448

407-
variable numCols, numRows, numLayers, maskedProp
449+
variable numCols, numRows, numLayers, maskedProp, numThreads, numRowsEffective, numLayersEffective
408450
string key
409451

410452
ASSERT_TS((ParamIsDefault(prop) && (ParamIsDefault(var) + ParamIsDefault(str)) == 1) \
@@ -478,12 +520,13 @@ threadsafe Function/WAVE FindIndizes(WAVE numericOrTextWave, [variable col, stri
478520
endif
479521

480522
if(ParamIsDefault(endRow))
481-
endRow = Inf
523+
endRow = numRows - 1
482524
else
483525
ASSERT_TS(endRow >= 0 && endRow < numRows, "Invalid endRow")
484526
endif
485527

486528
ASSERT_TS(startRow <= endRow, "endRow must be larger than startRow")
529+
numRowsEffective = (endRow - startRow) + 1
487530

488531
if(ParamIsDefault(startLayer))
489532
startLayer = 0
@@ -499,94 +542,97 @@ threadsafe Function/WAVE FindIndizes(WAVE numericOrTextWave, [variable col, stri
499542
endif
500543

501544
ASSERT_TS(startLayer <= endLayer, "endLayer must be larger than startLayer")
545+
numLayersEffective = (endLayer - startLayer) + 1
502546

503547
// Algorithm:
504548
// * The matches wave has the same size as one column of the input wave
505549
// * -1 means no match, every value larger or equal than zero is the row index of the match
506550
// * There is no distinction between different layers matching
507551
// * After the matches have been calculated we take the maximum of the transposed matches
508-
// wave in each colum transpose back and replace -1 with NaN
552+
// wave in each colum transpose back and replace -1 with NaN. This multiple layer matching algorithm
553+
// using maxCols is also the reason why we can't start with NaN on no match but have to use -1
509554
// * This gives a 1D wave with NaN in the rows with no match, and the row index of the match otherwise
510555
// * Delete all NaNs in the wave and return it
511556

512-
key = CA_TemporaryWaveKey({numRows, numLayers})
557+
key = CA_FindIndizesKey({numRows, numLayers})
513558
WAVE/Z/D matches = CA_TryFetchingEntryFromCache(key, options = CA_OPTS_NO_DUPLICATE)
514559

515560
if(!WaveExists(matches))
516561
Make/N=(numRows, numLayers)/FREE/D matches
517562
CA_StoreEntryIntoCache(key, matches, options = CA_OPTS_NO_DUPLICATE)
518563
endif
519564

565+
numThreads = GetNumberOfUsefulThreads({numRowsEffective, numLayersEffective})
566+
520567
FastOp matches = -1
521568

522569
if(WaveExists(wv))
523570
if(!ParamIsDefault(prop))
524571
if(prop & PROP_EMPTY)
525572
if(prop & PROP_NOT)
526-
MultiThread matches[startRow, endRow][startLayer, endLayer] = (numtype(wv[p][col][q]) != 2) ? p : -1
573+
MultiThread/NT=(numThreads) matches[startRow, endRow][startLayer, endLayer] = (numtype(wv[p][col][q]) != 2) ? p : -1
527574
else
528-
MultiThread matches[startRow, endRow][startLayer, endLayer] = (numtype(wv[p][col][q]) == 2) ? p : -1
575+
MultiThread/NT=(numThreads) matches[startRow, endRow][startLayer, endLayer] = (numtype(wv[p][col][q]) == 2) ? p : -1
529576
endif
530577
elseif(prop & PROP_MATCHES_VAR_BIT_MASK)
531578
if(prop & PROP_NOT)
532-
MultiThread matches[startRow, endRow][startLayer, endLayer] = !(wv[p][col][q] & var) ? p : -1
579+
MultiThread/NT=(numThreads) matches[startRow, endRow][startLayer, endLayer] = !(wv[p][col][q] & var) ? p : -1
533580
else
534-
MultiThread matches[startRow, endRow][startLayer, endLayer] = (wv[p][col][q] & var) ? p : -1
581+
MultiThread/NT=(numThreads) matches[startRow, endRow][startLayer, endLayer] = (wv[p][col][q] & var) ? p : -1
535582
endif
536583
elseif(prop & PROP_GREP)
537584
if(prop & PROP_NOT)
538-
MultiThread matches[startRow, endRow][startLayer, endLayer] = !GrepString(num2strHighPrec(wv[p][col][q]), str) ? p : -1
585+
MultiThread/NT=(numThreads) matches[startRow, endRow][startLayer, endLayer] = !GrepString(num2strHighPrec(wv[p][col][q]), str) ? p : -1
539586
else
540-
MultiThread matches[startRow, endRow][startLayer, endLayer] = GrepString(num2strHighPrec(wv[p][col][q]), str) ? p : -1
587+
MultiThread/NT=(numThreads) matches[startRow, endRow][startLayer, endLayer] = GrepString(num2strHighPrec(wv[p][col][q]), str) ? p : -1
541588
endif
542589
elseif(prop & PROP_WILDCARD)
543590
if(prop & PROP_NOT)
544-
MultiThread matches[startRow, endRow][startLayer, endLayer] = !StringMatch(num2strHighPrec(wv[p][col][q]), str) ? p : -1
591+
MultiThread/NT=(numThreads) matches[startRow, endRow][startLayer, endLayer] = !StringMatch(num2strHighPrec(wv[p][col][q]), str) ? p : -1
545592
else
546-
MultiThread matches[startRow, endRow][startLayer, endLayer] = StringMatch(num2strHighPrec(wv[p][col][q]), str) ? p : -1
593+
MultiThread/NT=(numThreads) matches[startRow, endRow][startLayer, endLayer] = StringMatch(num2strHighPrec(wv[p][col][q]), str) ? p : -1
547594
endif
548595
elseif(prop & PROP_NOT)
549-
MultiThread matches[startRow, endRow][startLayer, endLayer] = (wv[p][col][q] != var) ? p : -1
596+
MultiThread/NT=(numThreads) matches[startRow, endRow][startLayer, endLayer] = (wv[p][col][q] != var) ? p : -1
550597
endif
551598
else
552599
ASSERT_TS(!IsNaN(var), "Use PROP_EMPTY to search for NaN")
553-
MultiThread matches[startRow, endRow][startLayer, endLayer] = (wv[p][col][q] == var) ? p : -1
600+
MultiThread/NT=(numThreads) matches[startRow, endRow][startLayer, endLayer] = (wv[p][col][q] == var) ? p : -1
554601
endif
555602
else
556603
if(!ParamIsDefault(prop))
557604
if(prop & PROP_EMPTY)
558605
if(prop & PROP_NOT)
559-
MultiThread matches[startRow, endRow][startLayer, endLayer] = strlen(wvText[p][col][q]) ? p : -1
606+
MultiThread/NT=(numThreads) matches[startRow, endRow][startLayer, endLayer] = strlen(wvText[p][col][q]) ? p : -1
560607
else
561-
MultiThread matches[startRow, endRow][startLayer, endLayer] = !strlen(wvText[p][col][q]) ? p : -1
608+
MultiThread/NT=(numThreads) matches[startRow, endRow][startLayer, endLayer] = !strlen(wvText[p][col][q]) ? p : -1
562609
endif
563610
elseif(prop & PROP_MATCHES_VAR_BIT_MASK)
564611
if(prop & PROP_NOT)
565-
MultiThread matches[startRow, endRow][startLayer, endLayer] = !(str2num(wvText[p][col][q]) & var) ? p : -1
612+
MultiThread/NT=(numThreads) matches[startRow, endRow][startLayer, endLayer] = !(str2num(wvText[p][col][q]) & var) ? p : -1
566613
else
567-
MultiThread matches[startRow, endRow][startLayer, endLayer] = (str2num(wvText[p][col][q]) & var) ? p : -1
614+
MultiThread/NT=(numThreads) matches[startRow, endRow][startLayer, endLayer] = (str2num(wvText[p][col][q]) & var) ? p : -1
568615
endif
569616
elseif(prop & PROP_GREP)
570617
if(prop & PROP_NOT)
571-
MultiThread matches[startRow, endRow][startLayer, endLayer] = !GrepString(wvText[p][col][q], str) ? p : -1
618+
MultiThread/NT=(numThreads) matches[startRow, endRow][startLayer, endLayer] = !GrepString(wvText[p][col][q], str) ? p : -1
572619
else
573-
MultiThread matches[startRow, endRow][startLayer, endLayer] = GrepString(wvText[p][col][q], str) ? p : -1
620+
MultiThread/NT=(numThreads) matches[startRow, endRow][startLayer, endLayer] = GrepString(wvText[p][col][q], str) ? p : -1
574621
endif
575622
elseif(prop & PROP_WILDCARD)
576623
if(prop & PROP_NOT)
577-
MultiThread matches[startRow, endRow][startLayer, endLayer] = !StringMatch(wvText[p][col][q], str) ? p : -1
624+
MultiThread/NT=(numThreads) matches[startRow, endRow][startLayer, endLayer] = !StringMatch(wvText[p][col][q], str) ? p : -1
578625
else
579-
MultiThread matches[startRow, endRow][startLayer, endLayer] = StringMatch(wvText[p][col][q], str) ? p : -1
626+
MultiThread/NT=(numThreads) matches[startRow, endRow][startLayer, endLayer] = StringMatch(wvText[p][col][q], str) ? p : -1
580627
endif
581628
elseif(prop & PROP_NOT)
582-
MultiThread matches[startRow, endRow][startLayer, endLayer] = CmpStr(wvText[p][col][q], str) ? p : -1
629+
MultiThread/NT=(numThreads) matches[startRow, endRow][startLayer, endLayer] = CmpStr(wvText[p][col][q], str) ? p : -1
583630
endif
584631
else
585-
MultiThread matches[startRow, endRow][startLayer, endLayer] = !CmpStr(wvText[p][col][q], str) ? p : -1
632+
MultiThread/NT=(numThreads) matches[startRow, endRow][startLayer, endLayer] = !CmpStr(wvText[p][col][q], str) ? p : -1
586633
endif
587634
endif
588635

589-
endRow = numRows - 1
590636
MatrixOp/FREE result = zapNans(replace(maxCols(subRange(matches, startRow, endRow, startLayer, endLayer)^t)^t, -1, NaN))
591637

592638
if(DimSize(result, ROWS) == 0)

‎Packages/tests/Basic/UTF_Utils_Mies_Algorithm.ipf‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -855,3 +855,49 @@ Function FI_AbortsWithInvalidRegExp()
855855
endtry
856856
End
857857
/// @}
858+
859+
/// GetNumberOfUsefulThreads
860+
/// @{
861+
862+
Function GNOT_Works()
863+
864+
REQUIRE_GE_VAR(ThreadProcessorCount, 2)
865+
866+
Make/FREE/N=(0) empty
867+
CHECK_EQUAL_VAR(GetNumberOfUsefulThreads(empty), 1)
868+
CHECK_EQUAL_VAR(GetNumberOfUsefulThreads({0}), 1)
869+
CHECK_EQUAL_VAR(GetNumberOfUsefulThreads({1000}), 1)
870+
CHECK_EQUAL_VAR(GetNumberOfUsefulThreads({1000, 8}), 2)
871+
872+
// limited to ThreadProcessorCount
873+
CHECK_EQUAL_VAR(GetNumberOfUsefulThreads({1000, 1000, 1000, 1000}), ThreadProcessorCount)
874+
End
875+
876+
Function GNOT_ChecksParameters()
877+
878+
try
879+
Make/FREE/N=(2, 2) dims
880+
GetNumberOfUsefulThreads(dims)
881+
FAIL()
882+
catch
883+
CHECK_NO_RTE()
884+
endtry
885+
886+
try
887+
Make/FREE/T dimsText
888+
GetNumberOfUsefulThreads(dimsText)
889+
FAIL()
890+
catch
891+
CHECK_NO_RTE()
892+
endtry
893+
894+
try
895+
Make/FREE dimsTooLarge
896+
GetNumberOfUsefulThreads(dimsTooLarge)
897+
FAIL()
898+
catch
899+
CHECK_NO_RTE()
900+
endtry
901+
End
902+
903+
/// @}

0 commit comments

Comments
 (0)