Commit 1aaf1351 authored by actb's avatar actb

add eval software, goFullEval, new download (from 32)

parent 7b8b37c8
ROOT='G:\gRaw\ctc2020';
zlist=dir(fullfile(ROOT,'*.zip'));
parfor z=1:length(zlist)
name=zlist(z).name(1:end-4);
if name(end)=='_'
name(end)='';
end
cmd=['"c:\Program Files\7-Zip"\7z.exe x ' fullfile(zlist(z).folder,zlist(z).name) ...
' -o' fullfile(zlist(z).folder,name)];
system(cmd);
end
......@@ -5,20 +5,31 @@
load creds.mat
options=weboptions('username',uu,'password',pp);
outfolder='R:\Images\ctc2018\training\2d';
outfolder='g:\gRaw\ctc2020';
if ~exist(outfolder,'dir')
mkdir(outfolder);
end
page=webread('http://celltrackingchallenge.net/2d-datasets/');
match=regexp(page,'href="\S+zip"','match');
for i=1:length(match)
targetURL=match{i}(7:end-1);
[folder,fname,ext]=fileparts(targetURL);
if isempty(strfind(targetURL,'training-datasets'))
continue
match=regexp(page,'href="(\S+zip)"','tokens');
tblMatch=cell2table(match','VariableNames',{'URL'});
tblMatch.xd=repmat('2d',size(tblMatch,1),1);
page=webread('http://celltrackingchallenge.net/3d-datasets/');
match3=regexp(page,'href="(\S+zip)"','tokens');
tblMatch3=cell2table(match3','VariableNames',{'URL'});
tblMatch3.xd=repmat('3d',size(tblMatch,1),1);
tblMatch=[tblMatch;tblMatch3];
for i=1:length(tblMatch.URL)
[folder,fname,ext]=fileparts(tblMatch.URL{i});
targetName=[tblMatch.xd(i,:) '_' fname];
if strfind(tblMatch.URL{i},'training')
targetName=[targetName '_training_' ext];
else
targetName=[targetName '_challenge_' ext];
end
localname=fullfile(outfolder,[fname ext]);
fprintf(1,'%s : %s\n',localname,targetURL);
websave(localname,targetURL,options);
localname=fullfile(outfolder,targetName);
fprintf(1,'%s : %s\n',localname,tblMatch.URL{i});
websave(localname,tblMatch.URL{i},options);
end
\ No newline at end of file
% goImport
SRC='R:\Images\ctc2018\training\';
DST='f:\leverjs\ctc2019_training';
SRC='G:\gRaw\ctc2020';
DST='g:\leverjs\ctc2020';
if ~exist(DST,'dir')
mkdir(DST);
mkdir(fullfile(DST,'2d'));
mkdir(fullfile(DST,'3d'));
end
dlist=dir(fullfile(SRC,'**'));
idx=find([dlist.isdir]);
dlist=dlist(idx);
for dd=1:length(dlist)
parfor dd=1:length(dlist)
if strcmp(dlist(dd).name,'.') || strcmp(dlist(dd).name,'..')
continue
end
if isnan(str2double(dlist(dd).name))
continue
end
[~,expname]=fileparts(dlist(dd).folder);
if exist(fullfile(DST,[expname '_' dlist(dd).name '.h5']),'file')
folderparts=split(dlist(dd).folder,filesep);
expname=[folderparts{end-1} '_' dlist(dd).name];
targetFolder=fullfile(DST,expname(1:2));
expname=expname(4:end); % remove 2d_ and 3d_ prefix
if exist(fullfile(DST,[expname '.h5']),'file')
continue
end
% if exist(fullfile(dlist(dd).folder,[dlist(dd).name
importFolder(fullfile(dlist(dd).folder,dlist(dd).name),DST);
importFolder(fullfile(dlist(dd).folder,dlist(dd).name),targetFolder,expname);
end
......
<<<<<<< HEAD
% importExport.m
% mcc -m importExport.m -a D:\Andy\git\leverjs\matlab\sqlite-jdbc-3.21.0.jar -a ./assignment
function importExport(filename,outpath, gtFile)
tic
if ~exist('gtFile','var')
gtFile=[];
end
[~,~,ext]=fileparts(filename);
if strcmp(ext,'.LEVER')
export(filename,outpath,gtFile);
elseif exist(filename,'dir')
importFolder(filename,outpath);
else
fprintf(2,'error - argument unknown (tif or .lever): %s\n',filename);
end
y=toc;
if ~isempty(gtFile)
fprintf(1,'importExport elapsed time=%3.1f :: %s,%s,%s\n',y,filename,outpath,gtFile);
else
fprintf(1,'importExport elapsed time=%3.1f :: %s,%s\n',y,filename,outpath);
end
4;
function importFolder(folder,outfolder)
parts=strsplit(folder,filesep);
expname=[parts{end-1} '_' parts{end}];
if exist(fullfile(outfolder,[expname '.h5']),'file')
return
end
fprintf(1,'importing folder %s -> %s\n',folder, fullfile(outfolder,[expname '.h5']));
flist=dir(fullfile(folder,'*.tif'));
im=[];
imd1=[];
for t=1:length(flist)
fname=['t' num2str(t-1,'%03d') '.tif'];
info=imfinfo(fullfile(folder,fname));
if 1==t
try
imd1=MicroscopeData.Original.ReadMetadata(folder,fname);
catch
imd1.PixelPhysicalSize=[1,1,1];
end
end
for z=1:length(info)
im(:,:,z,1,t)=imread(fullfile(folder,fname),z);
end
end
writeH5(im,imd1,expname,outfolder);
function writeH5(im,imd1,expName,outfolder)
% make sure leverjsUtilities project is on your path!
% e.g. path(path,'pathTo\git\utilities\src\MATLAB')
imd=MicroscopeData.MakeMetadataFromImage(im);
imd.DatasetName=expName;
imd.PixelPhysicalSize=imd1.PixelPhysicalSize;
if (size(imd.Dimensions,2)~=3)
imd.Dimensions=[imd.Dimensions 1];
end
MicroscopeData.WriterH5(im,'imageData',imd,'path',outfolder);
function export(filename,outpath,gtFile)
[~,exp,~]=fileparts(filename);
tokens=regexp(exp,'(\D+)_(\d+)','tokens');
dsName=tokens{1}{1};
dsID=tokens{1}{2};
% strip off the _01 or _02 of the exp
if strcmp(dsID,'01')
subdir='01_RES';
else
subdir='02_RES';
end
% exp=exp(1:end-3);
outpath=fullfile(outpath,exp,subdir);
if exist(outpath,'dir')
rmdir(outpath,'s');
end
mkdir(outpath);
fprintf(1,'exporting to folder %s\n',outpath);
conn = database(filename, '','', 'org.sqlite.JDBC', 'jdbc:sqlite:');
CONSTANTS=Read.getConstants(conn);
if isempty(CONSTANTS)
fprintf(2,'bad lever file? %s\n',filename);
return
end
szImage=CONSTANTS.imageData.Dimensions;
szImage=[szImage(2),szImage(1),szImage(3)];
if ~isempty(gtFile)
% the datasets that have a 'ground truth' specify
tids=setExtFamilyGT(conn,CONSTANTS,gtFile);
else
tids=Read.getExtFamilies(conn);% which cells to track
end
if 0==length(tids)
fprintf(2,'no ext families -- no export?\n');
return
end
% generate the label,tBegin,tEnd,parent info
% lbgep=[label, begin, end, parent]
lbep=[];
for i=1:length(tids)
label=tids(i);
cmd=['select min(time),max(time) from tblCells where trackID=' num2str(tids(i))];
q=ljsFetch(conn,cmd);
if isempty(q)
continue
end
tBegin=q{1};
tEnd=q{2};
if strcmp(tBegin,'null') || strcmp(tEnd,'null') || isempty(tBegin) || isempty(tEnd)
continue
end
% find our parent
% note - cellID==trackID for first cell on track
cmd=['select cellID,trackID,time from tblCells inner join tblFamilies on '...
' cellID=cellID_parent where (cellID_child1=' num2str(tids(i)) ...
' or cellID_child2=' num2str(tids(i)) ') and cellID_child2 is not null'];
q=ljsFetch(conn,cmd);
if isempty(q)
parent=0;
else
parent=q{2};
cmd=['select max(time) from tblCells where trackID=' num2str(parent)];
q=ljsFetch(conn,cmd);
if ~isempty(q)
if q{1}>tBegin
fprintf(2,'found bad mitosis -- skipping\n');
parent=0;
end
end
end
% find our children
cmd=['select cellID,trackID,time from tblCells inner join tblFamilies on '...
' (cellID=cellID_child2 or cellID=cellID_child1) AND cellID_child2 is not null where cellID_parent IN '...
' (select cellID from tblCells where trackID=' num2str(tids(i)) ')'];
q=ljsFetch(conn,cmd);
if isempty(q)
childIDs=[0,0];
elseif 2==size(q,1)
childIDs=[q{1,1},q{2,1}];
else
% mitosis corrupted? not sure why. remove the children, mark this
% one as a bad parent, and keep going
fprintf(2,' bad mitotic event detected: corrupt db? --> %s\n',filename);
childIDs=[0,0];
end
% skip track fragments
if tEnd-tBegin<5 && childIDs(1)==0 && parent==0
continue
end
% skip tracks with no child / parent and that never make it out of the
% restricted ROI
if childIDs(1)==0 && parent==0 && roiClipTrack(conn,tids(i),50,szImage)
continue
end
lbep=[lbep; label, tBegin-1, tEnd-1, parent,tids(i),childIDs];
end
if 0==size(lbep,1)
fprintf(2,'no tracks to export?\n');
return;
end
if any(lbep(:,2)>lbep(:,3))
fprintf(2,'bad begin end timing detected ACK\n');
end
lbepOut=lbep(:,1:4);
dlmwrite(fullfile(outpath,'res_track.txt'),lbepOut,'newline','pc','delimiter',' ');
tids=[lbep(:,5),lbep(:,1:3)]; % [trackID,proxy,tBegin,tEnd]
if size(tids,1)>1
tidString=jsonencode(tids(:,1));
tidString(1)='(';
tidString(end)=')';
else
tidString=['(' jsonencode(tids(1)) ')'];
end
for t=1:CONSTANTS.imageData.NumberOfFrames
tImageLabels=[0]; % make sure no one gets over written
% NOTE -- add segCC asc as there is an issue with segCC's of -1 (from
% splits via the tblEnsemble) covering up other segmentations
cmd=['select cellID,trackID from tblCells where time=' num2str(t)...
' and trackID IN ' tidString 'order by segCC asc'];
Q=ljsFetch(conn,cmd);
ctID=cell2mat(Q);
im=zeros(szImage);
for i=1:size(ctID,1)
iCell=Read.getCell(conn,ctID(i,1));
if szImage(3)>1
idx=sub2ind(szImage,iCell.pts(:,2),iCell.pts(:,1),iCell.pts(:,3));
else
idx=sub2ind(szImage,iCell.pts(:,2),iCell.pts(:,1));
end
im(idx)=iCell.trackID;
end
im=setMissingTrackLabels(conn,im,tids,t);
fname=['mask' num2str(t-1,'%03d') '.tif'];
im=uint16(im);
if size(im,3)>1
imwrite(im(:,:,1),fullfile(outpath,fname));
for z=2:size(im,3)
imwrite(im(:,:,z),fullfile(outpath,fname),'WriteMode','append');
end
else
imwrite(im,fullfile(outpath,fname));
end
end
close(conn);
% tids -- trackid,proxy,tbegin,tend
function im=setMissingTrackLabels(conn,im,tids,t)
idsIm=unique(im(:));
idsIm(idsIm==0)=[];
tctc=t-1;
idx=find(tctc>=tids(:,3) & tctc<=tids(:,4));
idsNeeded=tids(idx,1);
idsMissing=setdiff(idsNeeded,idsIm);
if isempty(idsMissing)
return
end
for i=1:length(idsMissing)
% get the centroids
cmd=['select time,centroid from tblCells where trackID=' num2str(idsMissing(i))];
q=ljsFetch(conn,cmd);
trackTime=cell2mat(q(:,1));
cx=cellfun(@jsondecode,q(:,2),'uniformoutput',false);
idx0=find(trackTime==t-1);
idx1=find(trackTime==t+1);
if isempty(idx0) && isempty(idx1)
fprintf(2,'neither t-1 nor t+1 available for interp! failing...\n');
continue
end
if isempty(idx0)
idx0=idx1;
end
if isempty(idx1)
idx1=idx0;
end
cxInterp=(cx{idx0}+cx{idx1}) ./ 2;
cxInterp=round(cxInterp);
if length(size(im))<3
cxInterp=cxInterp(1:2);
end
cxInterp=min(cxInterp,size(im)');
cxInterp=max(cxInterp,ones(length(cxInterp),1));
im(cxInterp(2),cxInterp(1))=idsMissing(i);
end
=======
% importExport.m
% mcc -m importExport.m -a D:\Andy\git\leverjs\matlab\sqlite-jdbc-3.21.0.jar -a ./assignment
function importExport(filename,outpath, gtFile)
......@@ -268,3 +540,4 @@ end
>>>>>>> update from bioimage32
function importFolder(folder,outfolder)
function importFolder(infolder,outfolder,expname)
parts=strsplit(folder,filesep);
expname=[parts{end-1} '_' parts{end}];
if exist(fullfile(outfolder,[expname '.h5']),'file')
return
end
flist=dir(fullfile(folder,'*.tif'));
flist=dir(fullfile(infolder,'*.tif'));
im=[];
imd1=[];
if exist(fullfile(infolder,'t000.tif'),'file')
tFormat='%03d';
else
if exist(fullfile(infolder,'t0000.tif'),'file')
tFormat='%04d';
else
fprintf(2,'importFolder : %s : could not read tif format! no t000 or t0000\n',infolder);
return;
end
end
for t=1:length(flist)
fname=['t' num2str(t-1,'%03d') '.tif'];
info=imfinfo(fullfile(folder,fname));
fname=['t' num2str(t-1,tFormat) '.tif'];
info=imfinfo(fullfile(infolder,fname));
if 1==t
try
imd1=MicroscopeData.Original.ReadMetadata(folder,fname);
imd1=MicroscopeData.Original.ReadMetadata(infolder,fname);
catch
imd1.PixelPhysicalSize=[1,1,1];
end
end
for z=1:length(info)
im(:,:,z,1,t)=imread(fullfile(folder,fname),z);
im(:,:,z,1,t)=imread(fullfile(infolder,fname),z);
end
end
im=mat2gray(im);
......
function buildWrangleFile(strDB,outfolder,dsName,segParams)
strTarget=fullfile(outfolder,[dsName '.LEVER']);
if exist(strTarget,'file')
delete(strTarget);
end
copyfile(strDB,strTarget);
eraseAll=getLeverjsPath('eraseAll.js');
system(['node ' eraseAll ' ' strTarget]);
conn=database(strTarget, '','', 'org.sqlite.JDBC', 'jdbc:sqlite:');
Write.setSegParams(conn,segParams);
close(conn);
ROOT='g:\leverjs\ctc2020\2d';
DS='Fluo-N2DL-HeLa_training_01';
EXT='_param_1';
DEST='f:\autowrangler';
if ~exist(DEST,'dir')
mkdir(DEST);
end
strDB=fullfile(ROOT,[DS '.LEVER']);
strTarget=fullfile(DEST,[DS EXT '.LEVER']);
if exist(strTarget,'file')
delete(strTarget);
end
copyfile(strDB,strTarget);
eraseAll=getLeverjsPath('eraseAll.js');
system(['node ' eraseAll ' ' strTarget]);
conn=database(strTarget, '','', 'org.sqlite.JDBC', 'jdbc:sqlite:');
segParams=Read.getSegmentationParams(conn);
segParams.alpha=[0.9,0.1];
Write.setSegParams(conn,segParams);
close(conn);
% run it
Batch.batchSegment(strTarget,[],true);
% score it...
ROOT='F:\leverjs\ctc2019_training\';
DS='Fluo-N2DL-HeLa_01';
strDB=fullfile(ROOT,[DS '.LEVER']);
%conn = database(strDB, '','', 'org.sqlite.JDBC', 'jdbc:sqlite:');
%CONSTANTS=Read.getConstants(conn);
segParams=Segment.getDefaultSegParams();
segParams.useCuda=true;
segParams.minimumRadius_um=[3.5,0.2,5];
segParams.denoise=true;
segParams.NLM=[0.065,6,5,false];
segParams.alpha=0.1;
strTarget=fullfile(ROOT,[DS '_param_X.LEVER']);
if exist(strTarget,'file')
delete(strTarget);
end
copyfile(strDB,strTarget);
conn=database(strTarget, '','', 'org.sqlite.JDBC', 'jdbc:sqlite:');
algorithm=[];
algorithm.name='optimization search';
algorithm.type='segment';
algorithm.params=segParams;
algorithm.function='+Segment.FrameSegment_texture.m';
algorithm.commandHost='matlab';
strAlg=jsonencode(algorithm);
exec(conn,'delete from tblAlgorithms where type="segment"');
cmd=['insert into tblAlgorithms (fqn,type,jsAlgorithmInfo) '...
' values("optimization search","segment", '' ' strAlg ' '')'];
exec(conn,cmd);
close(conn);
% run it
system(['node.exe C:\Users\ac\Documents\git\leverjs\leverjs\resegAndTrackDataset.js "' ...
strTarget '" resegAll 1 "{}" ' ]);
% score it
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment