incremental h5 on folder import, new processBatch

ctc/goImport.m

@@ -11,7 +11,7 @@ end
 dlist=dir(fullfile(SRC,'**'));
 idx=find([dlist.isdir]);
 dlist=dlist(idx);
-parfor dd=1:length(dlist)
+for dd=1:length(dlist)
     if strcmp(dlist(dd).name,'.') || strcmp(dlist(dd).name,'..')
         continue
     end

ctc/importExport/exportAll.m

 % exportAll.m
 % ROOT='g:\leverjs\ctc2018_finalResults\';
-ROOT='\\bioimage32.coe.drexel.edu\f\leverjs\ctc2020rc';
+ROOT='g:\leverjs\ctc2020_final';
 gtPath='\\bioimage32.coe.drexel.edu\g\gRaw\ctc2020\';
-outfolder='g:\leverjs\ctc2020_finalResults\results';
+outfolder='g:\leverjs\ctc2020_sw';
 flist=dir(fullfile(ROOT,'*\*challenge*.LEVER'));
 
 for ff=1:length(flist)

ctc/importExport/generatedBats/Fluo-N3DL-DRO_01.bat

-processCSB.bat "..\leverData\Fluo-N3DL-DRO_01.LEVER"

ctc/importExport/generatedBats/Fluo-N3DL-DRO_02.bat

-processCSB.bat "..\leverData\Fluo-N3DL-DRO_02.LEVER"

ctc/importExport/generatedBats/Fluo-N3DL-TRIC_01.bat

-processCSB.bat "..\leverData\Fluo-N3DL-TRIC_01.LEVER"

ctc/importExport/generatedBats/Fluo-N3DL-TRIC_02.bat

-processCSB.bat "..\leverData\Fluo-N3DL-TRIC_02.LEVER"

ctc/importExport/processBatch.bat

-REM @echo off
+@echo off
 REM processBatch
 REM main worker batch file to run lever.js for cell tracking challenge 2019 datasets
-REM usage : processBatch imageFolder, leverFile, groundTruthFolder
-REM     assumes tif images are ...
+REM usage : processBatch imageFolder, leverFile, groundTruthFolder, nProcessors
+REM     note that nProcessors used for TRIF images to limit RAM
+REM     assumes tif images are ../
 REM     generates lever .h5 images in ./temporaryImages
 REM     
 echo.
-echo *** IMPORT processing %1 (tiffs), lever file=%2, groundTruth=%3
+echo *** IMPORT processing %1 (tiffs), lever file=%2, groundTruth=%3 nProcessors=%4
 .\ctcImportExport\importExport.exe %1 .\temporaryImages
-echo *** resegAndTrack -- leverjs running now
+echo *** batch segmentation starting
+.\prebuilt\matlabPollDB.exe batchSegment %2 %4
+echo *** resegAndTrack starting
 .\node.exe .\leverjs\leverjs\resegAndTrackDataset.js %2  resegAll 1 "{}" 
 echo *** exporting results
-.\ctcImportExport\importExport.exe %2 ..\
+.\ctcImportExport\importExport.exe %2 .\ %3
 echo.
 
 

ctc/importFolder.m

@@ -27,22 +27,46 @@ for t=1:length(flist)
         catch
             imd1.PixelPhysicalSize=[1,1,1];
         end
+        if contains(infolder,'TRIF')
+            imd1.Dimensions(3)=imd1.NumberOfFrames;
+            imd1.NumberOfFrames=length(dir(fullfile(infolder,'*.tif')));
         end
-    for z=1:length(info)
-        im(:,:,z,1,t)=imread(fullfile(infolder,fname),z);
+        h5file=fullfile(outfolder,[expname '.h5']);
+        prepareH5(h5file,imd1);
     end
+    for z=1:length(info)
+        im1=imread(fullfile(infolder,fname),z);        
+        appendH5Frame(h5file,im1,t,z);
     end
-im=mat2gray(im);
-writeH5(im,imd1,expname,outfolder);
     
-function writeH5(im,imd1,expName,outfolder)
-% make sure leverjsUtilities project is on your path!
-% e.g. path(path,'pathTo\git\utilities\src\MATLAB')
-imd=MicroscopeData.MakeMetadataFromImage(im);
-imd.DatasetName=expName;
-imd.PixelPhysicalSize=imd1.PixelPhysicalSize;
-if (size(imd.Dimensions,2)~=3)
-    imd.Dimensions=[imd.Dimensions 1];
 end
 
-MicroscopeData.WriterH5(im,'imageData',imd,'path',outfolder);
+function appendH5Frame(h5File,im,t,z)
+
+szChunk=size(im);
+szChunk(3)=1;
+szChunk(4)=1;
+szChunk(5)=1;
+h5write(h5File, '/Images/Original',im,[1,1,z,1,t],szChunk);
+
+% create an h5 file for incremental writing. this is for custom imports, so
+% we can write one frame at a time, without having to read the whole image
+% stack into ram
+%
+% imageData=MicroscopeData.MakeMetadataFromImage(imx);
+%       remember to set imageData.NumberOfFrames and imageData.
+function prepareH5(h5File,imageData)
+
+szImage=[imageData.Dimensions(2) imageData.Dimensions(1) imageData.Dimensions(3)];
+szImage(4)=imageData.NumberOfChannels;
+szImage(5)=Inf;
+szChunk=szImage;
+szChunk(3)=1;
+szChunk(5)=1;
+
+
+h5create(h5File,'/Images/Original',szImage,'chunksize',szChunk, ...
+    'Datatype','single','Deflate',2);
+h5writeatt(h5File,'/','Metadata',Utils.CreateJSON(imageData,false));
+outfolder=fileparts(h5File);
+MicroscopeData.CreateMetadata(outfolder,imageData,false);