batch merging and parameter testing code, including results tabulation

src/+Smooth/mergeProcess.m

+function mergeProcess(inputDir,dataset)
+
+dirList = Helpers.listDirs(inputDir);
+
+% for running only specific datasets
+if exist('dataset','var')
+    dirList = dirList(contains({dirList(:).name},dataset));
+end
+
+dbList = [];
+for i = 1:length(dirList)
+
+    currentDir = fullfile(inputDir,dirList(i).name);
+    subDirList = Helpers.listDirs(currentDir);
+
+    for j = 1:length(subDirList)
+
+        currentSubDir = fullfile(subDirList(j).folder,subDirList(j).name);
+        tmpInfo = dir(fullfile(currentSubDir,'*.LEVER'));
+
+        dbList = [dbList,tmpInfo];
+
+    end
+
+end
+
+for i = 1:length(dbList)
+
+    strDB = fullfile(dbList(i).folder,dbList(i).name);
+    [conn,CONSTANTS,segParams] = openDB(strDB);
+    nProcessors = ljsNProcs(CONSTANTS);
+
+    tx = tic();
+    nMerged = Smooth.Merge.goMerge(conn,CONSTANTS,segParams,nProcessors);
+    % nMerged = Smooth.Merge.goMerge2(conn, CONSTANTS, segParams);
+    CONSTANTS.processInfo.tMerge = getElapsed(tx);
+    CONSTANTS.processInfo.nMerged = nMerged;
+
+    tx = tic();
+    Batch.batchTrack(strDB);
+    CONSTANTS.processInfo.tTrack1 = getElapsed(tx);
+
+    tx = tic();
+    Smooth.Patch.patch11(conn);
+    Smooth.Patch.patch12(conn);
+    Smooth.Patch.patch21(conn);
+    Smooth.Patch.patchOcclusion(conn);
+    CONSTANTS.processInfo.tPatch = getElapsed(tx);
+
+    tx = tic();
+    Smooth.Classify.goClassifyMitosis(conn,CONSTANTS,nProcessors);
+    Smooth.extFamily(conn);
+    CONSTANTS.processInfo.tClassify = getElapsed(tx);
+    
+    CONSTANTS.processInfo.processDate = datetime('now');
+
+end
+
+end
+
+function tElapsed = getElapsed(timerID)
+    tElapsed = toc(timerID);
+    tElapsed = round(tElapsed,2);
+end
\ No newline at end of file

src/+Smooth/mergeSetup.m

+inputDir = '/g/leverjs/layton/parameterTesting';
+outputDir = '/g/leverjs/layton/merging/';
+
+outputDirList = Helpers.listDirs(outputDir);
+
+for i = 1:length(outputDirList)
+
+    dstFolder = fullfile(outputDirList(i).folder,outputDirList(i).name);
+
+    copyfile(srcFolder,dstFolder)
+
+end
\ No newline at end of file

src/+Smooth/mergeTest.m

+function mergeTest(inputFolder)
+
+tmp = dir(inputFolder);
+folderList = tmp([tmp.isdir]);
+folderList = folderList(3:end);
+
+for i = 1:length(folderList)
+    flist = dir(fullfile(folderList(i).folder,folderList(i).name));
+    flist = flist(3:end);
+
+    for j = 1:length(flist)
+        dbList = dir(fullfile(flist(j).folder,flist(j).name,'*.LEVER'));
+
+        for k = 1:length(dbList)
+
+            strTarget = fullfile(dbList(k).folder,dbList(k).name);
+
+            eraseAll=ljsPath('eraseAll.js');
+            if isempty(eraseAll)
+                ctcEraseAll(strTarget);
+            else
+                system(['/home/layton/.nvm/versions/node/v18.17.1/bin/node ' eraseAll ' ' strTarget]);
+            end
+
+
+            Batch.processMovie(strTarget,[],16);
+
+        end
+    end
+end
+
+4;
+end
\ No newline at end of file

src/+Smooth/parameterTest.m

+% running nonMerged processing testing various radii
+% dsName = Fluo-N2DH-SIM+ (example)
+
+% inputFolder = '/g/leverjs/layton/ctcDBs';
+inputFolder = '/g/leverjs/ctc2024/2d/';
+testFolder = '/g/leverjs/layton/parameterTesting';
+resultsFolder = '/g/leverjs/layton/parameterTestingScores';
+gtRoot = '/g/gRaw/ctc2021';
+
+dbList = dir(fullfile(inputFolder,'*training*.LEVER'));
+dbList = dbList(11:end);
+
+for i = 1:length(dbList)
+
+    strDB = fullfile(dbList(i).folder,dbList(i).name);
+    [~,~,segParams] = openDB(strDB);
+
+    if segParams.isPhase
+        continue;
+    end
+
+    dbRadius = segParams.minimumRadius_um;
+    if dbRadius(1)-1 < 0
+        radiiRange = 1:0.5:dbRadius(3)+1;
+    else
+        radiiRange = dbRadius-1:0.5:dbRadius(3)+1;
+    end
+
+    % dsName = extractBefore(dbList(i).name,'_');
+    dbName = dbList(i).name;
+
+    Smooth.parameterTestSetup(inputFolder,testFolder,radiiRange,dbName);
+
+end
+
+4;
+
+tmp = dir(testFolder);
+folderList = tmp([tmp.isdir]);
+folderList = folderList(3:end);
+
+for j = 1:length(folderList)
+    fileList = dir(fullfile(folderList(j).folder,folderList(j).name,'*.LEVER'));
+
+    for k = 1:length(fileList)
+
+        Batch.processMovie((fullfile(fileList(k).folder,fileList(k).name)),[],16);
+
+    end
+end
+
+4;
+
+results = Smooth.parameterTestScoring(testFolder,resultsFolder,gtRoot);
+% writetable(results,fullfile(resultsFolder,[dsName,'_','results.xlsx']));
+writetable(results,fullfile(resultsFolder,'results.xlsx'));
\ No newline at end of file

src/+Smooth/parameterTestProcess.m

+function parameterTestProcess(inputDir,dataset)
+
+dirList = Helpers.listDirs(inputDir);
+
+% for running only specific datasets
+if exist('dataset','var')
+    dirList = dirList(contains({dirList(:).name},dataset));
+end
+
+dbList = [];
+for i = 1:length(dirList)
+
+    currentDir = fullfile(inputDir,dirList(i).name);
+    subDirList = Helpers.listDirs(currentDir);
+
+    for j = 1:length(subDirList)
+
+        currentSubDir = fullfile(subDirList(j).folder,subDirList(j).name);
+        tmpInfo = dir(fullfile(currentSubDir,'*.LEVER'));
+        
+        dbList = [dbList,tmpInfo];
+
+    end
+
+end
+
+for i = 1:length(dbList)
+    
+    strDB = fullfile(dbList(i).folder,dbList(i).name);
+    Helpers.eraseAll(strDB); % DBs should be empty, but make sure
+    Batch.processMovie(strDB,[])
+
+end
+
+end
\ No newline at end of file

src/+Smooth/parameterTestScoring.m

+function results = parameterTestScoring(inputDir,resultsDir,gtRoot,dataset)
+
+dirList = Helpers.listDirs(inputDir);
+
+% for running only specific datasets
+if exist('dataset','var')
+    dirList = dirList(contains({dirList(:).name},dataset));
+end
+
+dbList = [];
+for i = 1:length(dirList)
+
+    currentDir = fullfile(inputDir,dirList(i).name);
+    subDirList = Helpers.listDirs(currentDir);
+
+    for j = 1:length(subDirList)
+
+        currentSubDir = fullfile(subDirList(j).folder,subDirList(j).name);
+        tmpInfo = dir(fullfile(currentSubDir,'*.LEVER'));
+
+        dbList = [dbList,tmpInfo];
+
+    end
+
+end
+
+% results = table('Size',size(dbList));
+results = [];
+for i = 1:length(dbList)
+
+    dsName = extractBefore(dbList(i).name,'_');
+    gtFolder = [gtRoot,'/2d_',dsName,'_training/',dsName,'/'];
+    
+    if contains(dbList(i).name,'01')
+        res = ctcEval(fullfile(dbList(i).folder,dbList(i).name),[],resultsDir, ...
+            [gtFolder,'01_GT'],[]);
+        results = [results;res];
+
+    else
+        res = ctcEval(fullfile(dbList(i).folder,dbList(i).name),[],resultsDir, ...
+            [gtFolder,'02_GT'],[]);
+        results = [results;res];
+
+    end
+
+end
+
+4;
+
+if exist('dataset','var')
+    writetable(results,fullfile(resultsDir,[dataset,'_','results.xlsx']));
+else
+    writetable(results,fullfile(resultsDir,'results.xlsx'));
+end
+
+end
\ No newline at end of file

src/+Smooth/parameterTestSetup.m

+function parameterTestSetup(inputDir,dataset)
+
+dirList = Helpers.listDirs(inputDir);
+
+% for running only specific datasets
+if exist('dataset','var')
+    dirList = dirList(contains({dirList(:).name},dataset));
+end
+
+for i = 1:length(dirList)
+
+    tmp = dir(fullfile(inputDir,dirList(i).name,'*.LEVER'));
+    strDB = fullfile(tmp.folder,tmp.name);
+    [~,~,segParams] = openDB(strDB);
+    [~,fname,~] = fileparts(strDB);
+
+    radiusMin = segParams.minimumRadius_um(1) * .9; % drop min radius by 10%
+    radiusMin = round(radiusMin * 2) / 2; % round to nearest .5
+
+    radiusMax = segParams.minimumRadius_um(3) * 1.10; % raise max by 10%
+    radiusMax = round(radiusMax * 2) / 2; % round to nearest .5
+
+    radiiPairs = nchoosek(radiusMin:0.5:radiusMax,2);
+    radiusStep = segParams.minimumRadius_um(2);
+
+    for j = 1:length(radiiPairs)
+
+        strRadiusMin = strrep(num2str(radiiPairs(j,1)),'.','-');
+        strRadiusMax = strrep(num2str(radiiPairs(j,2)),'.','-');
+        folderName = [strRadiusMin,'_',strRadiusMax];
+        tmpDir = fullfile(inputDir,dirList(i).name,folderName);
+
+        segParams.minimumRadius_um = [radiiPairs(j,1),radiusStep,radiiPairs(j,2)];
+
+        Helpers.buildWrangleFile(strDB,tmpDir,fname,segParams);
+
+    end
+
+end
\ No newline at end of file

src/+Smooth/scratch.m

+inputDir = '/g/leverjs/layton/merging/merge_v01';
+
+dirList = Helpers.listDirs(inputDir);
+
+% for running only specific datasets
+if exist('dataset','var')
+    dirList = dirList(contains({dirList(:).name},dataset));
+end
+
+dbList = [];
+for i = 1:length(dirList)
+
+    currentDir = fullfile(inputDir,dirList(i).name);
+    subDirList = Helpers.listDirs(currentDir);
+
+    for j = 1:length(subDirList)
+
+        currentSubDir = fullfile(subDirList(j).folder,subDirList(j).name);
+        tmpInfo = dir(fullfile(currentSubDir,'*.LEVER'));
+
+        dbList = [dbList,tmpInfo];
+
+    end
+
+end
+
+for i = 1:length(dbList)
+
+    strDB = fullfile(dbList(i).folder,dbList(i).name);
+    [conn,CONSTANTS,segParams] = openDB(strDB);
+    nProcessors = ljsNProcs(CONSTANTS);
+
+    tx = tic();
+    nMerged = Smooth.Merge.goMerge(conn,CONSTANTS,segParams,nProcessors);
+    % nMerged = Smooth.Merge.goMerge2(conn, CONSTANTS, segParams);
+    CONSTANTS.processInfo.tMerge = getElapsed(tx);
+    CONSTANTS.processInfo.nMerged = nMerged;
+
+    tx = tic();
+    Batch.batchTrack(strDB);
+    CONSTANTS.processInfo.tTrack1 = getElapsed(tx);
+
+    tx = tic();
+    Smooth.Patch.patch11(conn);
+    Smooth.Patch.patch12(conn);
+    Smooth.Patch.patch21(conn);
+    Smooth.Patch.patchOcclusion(conn);
+    CONSANTS.processInfo.tPatch = getElapsed(tx);
+
+    tx = tic();
+    Smooth.Classify.goClassifyMitosis(conn,CONSTANTS,nProcessors);
+    Smooth.extFamily(conn);
+    CONSTANTS.processInfo.tClassify = getElapsed(tx);
+    
+    CONSTANTS.processInfo.processDate=datetime('now');
+
+end
+
+function tElapsed = getElapsed(timerID)
+    tElapsed = toc(timerID);
+    tElapsed = round(tElapsed,2);
+end
\ No newline at end of file

src/+Smooth/tabulateResults.m

+inputDir = '/g/leverjs/layton/mergeScoring/mergeTest/';
+
+dirList = Helpers.listDirs(inputDir);
+
+for x = 1:length(dirList)
+
+    tmpDir = fullfile(dirList(x).folder,dirList(x).name);
+    resultsList = dir(fullfile(tmpDir,'*.xlsx'));
+
+    results = array2table(zeros(0,5),'VariableNames',{'dataset','TRA','SEG','DET','minimumRadius'});
+    % why is this how empty tables are made? that is idiotic
+
+    if isempty(resultsList)
+        writetable(results,fullfile(inputDir,'tabulatedResults.xlsx'),'Sheet',dirList(x).name);
+        continue;
+    end
+
+    for i = 1:length(resultsList)
+        tmp = readtable(fullfile(resultsList(i).folder,resultsList(i).name));
+
+        for j = 1:size(tmp,1)
+            tmpParams = tmp.params{j};
+            minRadius = extractBetween(tmpParams,'[',']');
+
+            resultRow = {tmp.name{j},tmp.TSD_1(j),tmp.TSD_2(j),tmp.TSD_3(j),minRadius};
+            results = [results;resultRow];
+
+        end
+
+    end
+
+    writetable(results,fullfile(inputDir,'tabulatedResults.xlsx'),'Sheet',dirList(x).name);
+
+end

src/+Smooth/temp.m

+% inputDir = '/g/leverjs/layton/parameterTesting';
+inputDir = '/g/leverjs/layton/merging/merge_v02/';
+resultsDir = '/g/leverjs/layton/mergeScoring/merge_v02/';
+gtRoot = '/g/gRaw/ctc2021';
+
+% Smooth.parameterTestSetup(inputDir,'hela');
+% Smooth.parameterTestProcess(inputDir,'hela');
+Smooth.parameterTestScoring(inputDir,resultsDir,gtRoot,'gowt');
+
+% inputDir = '/g/leverjs/layton/merging/merge_v01';
+% Smooth.mergeProcess(inputDir,'gowt');
\ No newline at end of file