added mat files to lfs, pushed supervised mat results

.gitattributes

+*.mat filter=lfs diff=lfs merge=lfs -text

readme.md

@@ -11,4 +11,8 @@ A.R. Cohen and P.M.B. Vitányi, The Cluster Structure Function, in preparation.
 
 A.R. Cohen and P.M.B. Vitányi, Web Similarity in Sets of Search Terms Using Database Queries, SN Computer Science, 1, 161 (2020). https://doi.org/10.1007/s42979-020-00148-5.  Also arXiv 1502.05957 (pdf).
 
-A.R.Cohen and P.M.B. Vitányi, Normalized Compression Distance of Multisets with Applications, IEEE Transactions on Pattern Analysis and Machine Intelligence. 2015 Aug;37(8):1602-14,
+A.R.Cohen and P.M.B. Vitányi, Normalized Compression Distance of Multisets with Applications, IEEE Transactions on Pattern Analysis and Machine Intelligence. 2015 Aug;37(8):1602-14.
+
+A.R. Cohen, F. Gomes, B. Roysam, and M. Cayouette, Computational prediction of neural progenitor cell fates. Nature Methods, 2010. 7(3): p. 213 – 218.
+
+A.R. Cohen, C. Bjornsson, S. Temple, G. Banker and B. Roysam, Automatic Summarization of Changes in Biological Image Sequences using Algorithmic Information Theory. IEEE Transactions on Pattern Analysis and Machine Intelligence, 2009. 31(8): pp. 1386-1403.

src/MATLAB/apps/nist digits/MNISTdistance.m

@@ -3,6 +3,10 @@ function [d,c]=MNISTdistance(Training)
 d=zeros(length(Training),length(Training));
 c=d;
 N=length(Training);
+
+% ACKACK NEED TO MAKE THIS DETECT IF WE WANT PARFOR
+% what's the best way to do that? can't f=@if (keyword) ? @parfor
+
 parfor i=1:N
 % for i=1:N
     im1={Training(i).im};

src/MATLAB/apps/nist digits/goNISTclassify.m

 target=[0:9];
 K=length(target);
-cardinality=[10];
+cardinality=[5];
 tblResults=table();
-for n=1:1
+clusterConfusionSpectral={};
+clusterConfusionOD={};
+startParallel(64);
+for n=1:1000
     
     xDigits=getNISTdigits(target,cardinality); 
-    rgAssign=zeros(length(xDigits),length(xDigits));
-    for niter=1:52
-        % pick subsets randomly of length=50% to 90% of original
-        iterCardinality=round(cardinality*length(target)*0.1*(5+mod(niter-1,4)));
-        idxPerm=randperm(length(xDigits));
-        trialsDigits=xDigits(idxPerm(1:iterCardinality));
-        dDigits=MNISTdistance(trialsDigits);    
+    dDigits=MNISTdistance(xDigits);
     idxSpectral=Cluster.SpectralCluster(dDigits,K);
-        for ik=1:max(idxSpectral)
-            idxk=find(idxSpectral==ik);
-            for ii=1:length(idxk)
-                for jj=1:length(idxk)
-                    mi=idxPerm(idxk(ii));
-                    mj=idxPerm(idxk(jj));
-                    [mi,mj];
-                    rgAssign(mi,mj)=rgAssign(mi,mj)+1;
-                end
-            end
-        end
-    end
+    clusterConfusionSpectral = updateConfusion(xDigits,clusterConfusionSpectral,K,K,idxSpectral);
+
+    idxOD=deltaEM(xDigits,K);
+    clusterConfusionOD = updateConfusion(xDigits,clusterConfusionOD,K,K,idxOD);
 end
+   
\ No newline at end of file

src/MATLAB/apps/nist digits/goNISTk.m

@@ -13,7 +13,7 @@ rgResults=[]; % maxDigit,trial,iteration
 kMax=12;
 NUMBER_OF_SAMPLES=2000;
 NUMBER_OF_TRIALS=100;
-p=startParallel(256);
+p=startParallel(128);
 
 outname=['getK5k' datestr(now,'mm-dd-yyyy') '.mat']
 NDIGITS=length(maxDigits);
@@ -28,7 +28,7 @@ for nTrial=1:NUMBER_OF_TRIALS
         [kpred,mx,sd,clusterConfusion]=getK(xDigits,kMax,NUMBER_OF_SAMPLES,clusterConfusion);
         rgResults(d,nTrial)=kpred;
         kConfusion(K,kpred)=kConfusion(K,kpred)+1;
-        drawResults(kpred,K,mx,sd,outfolder,nTrial);
+%         drawResults(kpred,K,mx,sd,outfolder,nTrial);
         5;
     end
     if 0==mod(nTrial,20)

src/MATLAB/apps/nist digits/supervised/getConfidenceIntervals.m

+A=load("testrun second round nov 17.mat");
+B=load("testrun nov 16 final.mat");
+
+kConfusionNN=A.kConfusionNN+B.kConfusionNN;
+save('finalABrunSupervised Nov 2021.mat');
+
+% k prediction
+acc=sum(diag(kConfusionNN))
+N=sum(kConfusionNN(:));
+
+idx=[ones(1,acc),zeros(1,N-acc)];
+sci=bootci(20000,@mean,idx)
+
+% clustering
+clusterConfusion=A.clusterConfusion{10}+B.clusterConfusion{10};
+sum(diag(clusterConfusion)) / sum(clusterConfusion(:))
+
+acc=sum(diag(clusterConfusion));
+N=sum(clusterConfusion(:));
+
+idxCluster=[ones(1,acc),zeros(1,N-acc)];
+sciCluster=bootci(20000,@mean,idxCluster)
+