Commit 0f0db00f authored by Mark Winter's avatar Mark Winter

Adding external utilities dependencies, including jar files.

parent 8c1fdaf6
function graphStruct = BuildDependencyGraph(chkPath, bRecurseExternal)
if ( ~exist('chkPath','var') || isempty(chkPath) )
chkPath = pwd();
end
if ( ~exist('bRecurseExternal','var'))
bRecurseExternal = true;
end
%% Make sure we get back to our current dir even on error.
oldDir = cd(chkPath);
cleanupObj = onCleanup(@()(cd(oldDir)));
%%
filenames = getAllFiles(chkPath);
%% Initialize sparse matrix assuming a fanout of about 10x
n = length(filenames);
graphStruct = struct('nodes',{filenames}, 'graph',{sparse([],[],[],n,n,10*n)});
graphStruct = recursiveGetDeps(chkPath, graphStruct, filenames, bRecurseExternal);
graphStruct = sortNodes(chkPath,graphStruct);
end
function graphStruct = sortNodes(localPath, graphStruct)
bLocal = strncmp(localPath,graphStruct.nodes, length(localPath));
localIdx = find(bLocal);
externalIdx = find(~bLocal);
%% Sort lexicographically, but all local functions are first.
[~,localSrt] = sort(graphStruct.nodes(bLocal));
[~,externalSrt] = sort(graphStruct.nodes(~bLocal));
srtIdx = [localIdx(localSrt); externalIdx(externalSrt)];
graphStruct.nodes = graphStruct.nodes(srtIdx);
graphStruct.graph = graphStruct.graph(srtIdx,:);
graphStruct.graph = graphStruct.graph(:,srtIdx);
end
function graphStruct = recursiveGetDeps(localPath,graphStruct, checkNames, bRecurseExternal)
if ( isempty(checkNames) )
return;
end
newEntries = {};
matRoot = matlabroot();
% Get single-link dependencies
for i=1:length(checkNames)
[fList,pList] = matlab.codetools.requiredFilesAndProducts(checkNames{i}, 'toponly');
toolboxes = arrayfun(@(x)(fullfile(matRoot,'toolbox',x.Name)),pList, 'UniformOutput',false);
selfIdx = find(strcmp(checkNames{i},fList));
if ( isempty(selfIdx) )
selfIdx = 1;
fList = [checkNames(i) fList];
end
newNodes = [fList.'; toolboxes.'];
newGraph = createCallGraph(selfIdx, newNodes);
newStruct = struct('nodes',{newNodes},'graph',{newGraph});
[graphStruct,addedNodes] = Dev.MergeGraphStruct(graphStruct, newStruct);
newEntries = [newEntries; addedNodes];
end
% Don't recurse through external dependencies
bMatlab = strncmp(matRoot,newEntries, length(matRoot));
newEntries = newEntries(~bMatlab);
if ( ~bRecurseExternal )
bNewLocal = strncmp(localPath,newEntries, length(localPath));
newEntries = newEntries(bNewLocal);
end
% Add java jar/class dependencies
% IMPORTANT: This assumes that ALL files in the same folder as the
% JAR or CLASS file depend upon it and that these are the ONLY direct
% dependencies on the java classes!
graphStruct = checkJavaDeps(graphStruct,newEntries);
graphStruct = recursiveGetDeps(localPath,graphStruct, newEntries, bRecurseExternal);
end
function graphStruct = checkJavaDeps(graphStruct,checkNodes)
[checkDirs,~,ic] = unique(cellfun(@(x)(fileparts(x)),checkNodes, 'UniformOutput',false));
for i=1:length(checkDirs)
jarList = dir(fullfile(checkDirs{i},'*.jar'));
classList = dir(fullfile(checkDirs{i},'*.class'));
javaNodes = arrayfun(@(x)(fullfile(checkDirs{i},x.name)), [jarList;classList], 'UniformOutput',false);
if ( isempty(javaNodes) )
continue;
end
depNodes = checkNodes(ic==i);
[javaGraph,mergeNodes] = createCompleteCallGraph(depNodes,javaNodes);
newStruct = struct('nodes',{mergeNodes},'graph',{javaGraph});
graphStruct = Dev.MergeGraphStruct(graphStruct,newStruct);
end
end
% This creates a completely connected caller->callee graph, there cannot be
% any overlap in caller/callee nodes.
function [callGraph,mergeNodes] = createCompleteCallGraph(callerNodes,callNodes)
[iIdx,jIdx] = ndgrid(1:length(callerNodes),length(callerNodes)+(1:length(callNodes)));
numEdges = numel(jIdx);
mergeNodes = [callerNodes;callNodes];
callGraph = sparse(iIdx(:),jIdx(:), ones(numEdges,1), length(mergeNodes),length(mergeNodes));
end
% This uses a pre-merged node entry list with a single caller and connects
% the caller to all the other nodes in the graph.
function callGraph = createCallGraph(callerIdx,newNodes)
jIdx = setdiff(1:length(newNodes),callerIdx);
iIdx = repmat(callerIdx,1,length(jIdx));
callGraph = sparse(iIdx,jIdx, ones(1,length(jIdx)), length(newNodes),length(newNodes));
end
function fullNames = getAllFiles(dirName)
matlabFiles = what(dirName);
funcFileNames = vertcat(matlabFiles.m);
funcFileNames = [funcFileNames; vertcat(matlabFiles.mex)];
fullNames = cellfun(@(x)(fullfile(dirName,x)), funcFileNames, 'UniformOutput',false);
for i=1:length(matlabFiles.packages)
pkgFullNames = getAllFiles(fullfile(dirName, ['+' matlabFiles.packages{i}]));
fullNames = [fullNames; pkgFullNames];
end
for i=1:length(matlabFiles.classes)
classDir = fullfile(dirName, ['@' matlabFiles.classes{i}]);
if ( ~exist(classDir,'dir') )
continue;
end
classFullNames = getAllFiles(classDir);
fullNames = [fullNames; classFullNames];
end
end
function toolboxMap = BuildToolboxMap()
toolboxRoot = toolboxdir('');
toolboxMap = containers.Map('keyType','char', 'valueType','any');
% Ignore fixPoint, because
toolboxList = dir(toolboxRoot);
bInvalidName = arrayfun(@(x)(strcmp(x.name,'.') || strcmp(x.name,'..') || strcmp(x.name,'fixpoint')), toolboxList);
bValidDir = ~bInvalidName & (vertcat(toolboxList.isdir) > 0);
toolboxList = toolboxList(bValidDir);
% Always add local/shared directory to matlab
toolboxMap('MATLAB') = {fullfile(toolboxRoot,'local');fullfile(toolboxRoot,'shared')};
for i=1:length(toolboxList)
verStruct = ver(toolboxList(i).name);
if ( isempty(verStruct) )
continue;
end
if ( isKey(toolboxMap,verStruct.Name) )
toolboxMap(verStruct.Name) = [toolboxMap(verStruct.Name); {fullfile(toolboxRoot,toolboxList(i).name)}];
else
toolboxMap(verStruct.Name) = {fullfile(toolboxRoot,toolboxList(i).name)};
end
end
end
% [repoName,commitHash] = GetCommitInfo(inPath)
%
function [repoName,commitHash] = GetCommitInfo(inPath)
remoteUrl = Dev.GitRemote(inPath,'origin');
repoName = '';
tokMatch = regexp(remoteUrl,'\w+@[\w\-.]+:[\w\-.]+/([\w\-]+\.git)', 'tokens','once');
if ( ~isempty(tokMatch) )
repoName = tokMatch{1};
end
commitHash = Dev.GitCommitHash(inPath);
end
% version = GetVersion(command)
% Get a version string or number depending on the command argument. This
% m-file serves as a single repository for all version-related information.
%
% Note: Uses autogenerated version information in +Dev\VersionInfo.m
function versionStr = GetVersion(command)
versionStr = [];
verInfo = Dev.LoadVersion();
if ( ~exist('command','var') )
command = 'string';
end
% Get rid of slashes in branch names
cleanBranch = strrep(verInfo.branchName, '/', '_');
cleanBranch = strrep(cleanBranch, '\', '_');
primaryHash = '';
matchTok = regexp(verInfo.commitHash{1},'.*:\s*(\w+)','tokens','once');
if ( ~isempty(matchTok) )
primaryHash = matchTok{1};
end
if ( strcmpi(command, 'string') || strcmpi(command, 'versionString') )
versionStr = [num2str(verInfo.majorVersion) '.' num2str(verInfo.minorVersion) ' ' cleanBranch];
return;
end
if ( strcmpi(command, 'buildString') )
versionStr = [verInfo.buildNumber '/' verInfo.buildMachine];
return;
end
if ( strcmpi(command, 'primaryHash') )
versionStr = primaryHash;
return
end
if ( strcmpi(command, 'buildHashes') )
versionStr = verInfo.commitHash;
return
end
if ( strcmpi(command, 'fullString') )
versionStr = [verInfo.name ' v' num2str(verInfo.majorVersion) '.' num2str(verInfo.minorVersion) ' ' verInfo.buildNumber '/' verInfo.buildMachine ' ' cleanBranch ' ' primaryHash];
return;
end
if ( strcmpi(command, 'file') )
minorStr = num2str(verInfo.minorVersion);
minorStr = strrep(minorStr, '.', '_');
versionStr = [num2str(verInfo.majorVersion) minorStr '_' cleanBranch];
return;
end
end
function commitHash = GitCommitHash(inPath)
bGit = Dev.SetupGit();
if ( ~bGit )
return;
end
oldDir = cd(inPath);
cleanupObj = onCleanup(@()(cd(oldDir)));
[status,hashOut] = system('git rev-parse HEAD');
if ( status ~= 0 )
return;
end
commitHash = strtrim(hashOut);
end
function remoteURL = GitRemote(inPath,remoteName)
remoteURL = '';
bGit = Dev.SetupGit();
if ( ~bGit )
return;
end
oldDir = cd(inPath);
cleanupObj = onCleanup(@()(cd(oldDir)));
[status,urlOut] = system(['git remote get-url ' remoteName]);
if ( status ~= 0 )
return;
end
remoteURL = strtrim(urlOut);
end
function changeLines = GitStatus(chkPath,chkFiles)
changeLines = {};
if ( ~exist('chkFiles','var') )
chkFiles = {};
end
bGit = Dev.SetupGit();
if ( ~bGit )
return;
end
oldDir = cd(chkPath);
cleanupObj = onCleanup(@()(cd(oldDir)));
[status,res] = system('git status -uall --porcelain');
if ( status ~= 0 )
return;
end
statusLines = strsplit(res,'\n').';
bValid = cellfun(@(x)(~isempty(x)),statusLines);
statusLines = statusLines(bValid);
changeTypes = cellfun(@(x)(8*changeMap(x(1)) + changeMap(x(2))), statusLines);
changeFiles = cellfun(@(x)(x(4:end)), statusLines, 'UniformOutput',false);
if ( isempty(chkFiles) )
changeLines = changeFiles(changeTypes > 0);
return;
end
changeLines = {};
regexpFiles = regexptranslate('escape', strrep(chkFiles, '\','/'));
for i=1:length(changeFiles)
matchStarts = regexp(changeFiles{i},regexpFiles, 'start','once');
bMatched = cellfun(@(x)(~isempty(x)), matchStarts);
if ( any(bMatched) )
changeLines = [changeLines; changeFiles(i)];
end
end
extCell = arrayfun(@(x)([ '.\' x.ext '|']), mexext('all'), 'UniformOutput',false);
extStr = [extCell{:}];
bMexFiles = cellfun(@(x)(~isempty(x)), regexp(chkFiles,extStr(1:end-1),'once'));
if ( any(bMexFiles) )
matchStarts = regexp(changeFiles,'^src/c/', 'start','once');
bMatched = cellfun(@(x)(~isempty(x)), matchStarts);
cFiles = changeFiles(bMatched);
for i=1:length(cFiles)
changeLines = [changeLines; {'Possible MEX dependency - ' cFiles{i}}];
end
end
end
function change = changeMap(c)
changeList = ['?','M','A','D'];
change = find(c == changeList);
if ( isempty(change) )
change = 0;
end
end
function initStruct = InitCompiler(productName,forceVersion)
if ( ~exist('forceVersion','var') )
forceVersion = '';
end
rootDir = pwd();
initStruct = [];
%% Build dependency graph for current directory
depGraph = Dev.BuildDependencyGraph(rootDir,true);
%% Get list of matlab toolbox dependencies
toolboxRoot = toolboxdir('');
bMatlab = strncmp(toolboxRoot,depGraph.nodes, length(toolboxRoot));
toolboxNodes = depGraph.nodes(bMatlab);
toolboxList = Dev.LoadToolboxes(toolboxNodes);
%% Remove matlab paths/dependencies
depGraph.nodes = depGraph.nodes(~bMatlab);
depGraph.graph = depGraph.graph(~bMatlab,~bMatlab);
%% Get rootPaths and local subdirectories for dependencies
[rootPaths,rootNames,localPaths] = Dev.SplitDependencyNames(depGraph.nodes);
%% Get a localized list of java dependencies
bJava = cellfun(@(x)(~isempty(regexpi(x,'(\.jar|\.class)$','once'))), localPaths);
javaList = localPaths(bJava);
%% Verify no uncommited changes on dependencies
changeString = {};
[chkPaths,ia,ic] = unique(rootPaths);
chkNames = rootNames(ia);
for i=1:length(chkPaths)
bInPath = (ic == i);
changeLines = Dev.GitStatus(chkPaths{i},localPaths(bInPath));
if ( ~isempty(changeLines) )
changeString = [changeString; {[chkNames{i} ' (' chkPaths{i} '):']}];
for j=1:length(changeLines)
changeString = [changeString; {[' ' changeLines{j}]}];
end
end
end
if ( ~isempty(changeString) )
message = sprintf('The following dependencies have uncommitted changes, are you sure you wish to continue?\n\n');
for i=1:length(changeString)
message = [message sprintf('%s\n',changeString{i})];
end
answer = questdlg(message, 'Uncommitted changes!', 'Continue','Cancel', 'Cancel');
if ( strcmpi(answer,'Cancel') )
initStruct = [];
return;
end
end
%% Make full version string and fallback version file
Dev.MakeVersion(productName,forceVersion,chkPaths);
%% Copy the external dependencies to local paths
bExternal = ~strncmp(rootDir,rootPaths,length(rootDir));
externalPaths = rootPaths(bExternal);
externalDeps = localPaths(bExternal);
copyPaths = {};
for i=1:length(externalPaths)
localDir = fileparts(externalDeps{i});
if ( ~exist(fullfile(rootDir,localDir),'dir') )
mkdir(fullfile(rootDir,localDir));
end
copyPaths = [copyPaths; fullfile(rootDir,externalDeps{i})];
copyfile(fullfile(externalPaths{i},externalDeps{i}), fullfile(rootDir,externalDeps{i}));
end
initStruct.javaList = javaList;
initStruct.toolboxList = toolboxList;
initStruct.cleanupObj = onCleanup(@()(compilerCleanup(copyPaths)));
% temporarily remove any startup scripts that would normally be run by matlabrc
enableStartupScripts(false);
end
function compilerCleanup(copyPaths)
% Remove all copied dependencies
for i=1:length(copyPaths)
if ( exist(copyPaths{i},'file') )
delete(copyPaths{i});
end
end
% Re-enable any disabled startup scripts
enableStartupScripts(true);
end
function enableStartupScripts(bEnable)
searchPrefix = '';
renamePrefix = 'disabled_';
if ( bEnable )
searchPrefix = 'disabled_';
renamePrefix = '';
end
searchName = [searchPrefix 'startup.m'];
newName = [renamePrefix 'startup.m'];
startupScripts = findFilesInPath(searchName, userpath);
for i=1:length(startupScripts)
scriptPath = fileparts(startupScripts{i});
movefile(startupScripts{i}, fullfile(scriptPath,newName));
end
end
function fullNames = findFilesInPath(filename, searchPaths)
fullNames = {};
chkPaths = [];
while ( ~isempty(searchPaths) )
[newPath,remainder] = strtok(searchPaths, pathsep);
if ( isempty(newPath) )
searchPaths = remainder;
continue;
end
chkPaths = [chkPaths; {newPath}];
searchPaths = remainder;
end
for i=1:length(chkPaths)
chkFullPath = fullfile(chkPaths{i}, filename);
if ( exist(chkFullPath, 'file') )
fullNames = [fullNames; {chkFullPath}];
end
end
end
function toolboxPaths = LoadToolboxes(toolboxNodes)
toolboxMap = Dev.BuildToolboxMap();
toolboxRoot = toolboxdir('');
toolboxNames = cellfun(@(x)(x(length(toolboxRoot)+2:end)), toolboxNodes, 'UniformOutput',false);
toolboxPaths = {};
for i=1:length(toolboxNames)
if ( ~isKey(toolboxMap,toolboxNames{i}) )
continue;
end
toolboxPaths = [toolboxPaths; toolboxMap(toolboxNames{i})];
end
end
function verInfo = LoadVersion()
%% Use compiled VersionInfo function when deployed
if ( isdeployed )
verInfo = Dev.VersionInfo();
return;
end
%% Try to load version tag and branch using git
bFoundGit = Dev.SetupGit();
verInfo = [];
if ( bFoundGit )
chkVerInfo = gitVersionInfo();
end
%% Use fallback file for name (set by Dev.MakeVersion)
fallbackFile = 'version.json';
fallbackVerInfo = loadFallbackInfo(fallbackFile);
if ( isempty(fallbackVerInfo) )
fprintf('WARNING: Invalid fallback file, unable to load version information.\n');
return;
end
if ( isempty(chkVerInfo) )
fprintf('WARNING: Could not find git directory, using fallback %s\n', fallbackFile);
chkVerInfo = fallbackVerInfo;
end
% Always use the name from fallback file
chkVerInfo.name = fallbackVerInfo.name;
verInfo = chkVerInfo;
end
%% Read fallback json version file
function verInfo = loadFallbackInfo(fallbackFile)
verInfo = [];
fid = fopen(fallbackFile);
if ( fid <= 0 )
return;
end
jsonVer = fread(fid, '*char').';
fclose(fid);
verInfo = Utils.ParseJSON(jsonVer);
end
%% Use git tags to get version information
function verInfo = gitVersionInfo()
verInfo = [];
[verStatus,verString] = system('git describe --tags --match v[0-9]*.[0-9]* --abbrev=0');
[branchStatus,branchString] = system('git rev-parse --abbrev-ref HEAD');
[majorVer,minorVer] = Dev.ParseVerTag(verString);
if ( verStatus ~= 0 || isempty(majorVer) || isempty(minorVer) )
fprintf('WARNING: There was an error retrieving tag from git:\n %s\n', verString);
return;
end
branchName = strtrim(branchString);
if ( branchStatus ~= 0 )
fprintf('WARNING: There was an error retrieving branch name from git:\n %s\n', branchName);
return;
end
[repoName,commitHash] = Dev.GetCommitInfo(pwd());
commitString = [repoName ' : ' commitHash];
%% VersionInfo default structure
verInfo = struct(...
'name',{repoName},...
'majorVersion',{majorVer},...
'minorVersion',{minorVer+1},...
'branchName',{branchName},...
'buildNumber',{'UNKNOWN'},...
'buildMachine',{'UNKNOWN'},...
'commitHash',{{commitString}});
end
function verInfo = MakeVersion(productName, forceVersion, dependencyPaths)
%% Template for the Dev.VersionInfo command
funcString = {
'%% versionInfo = VersionInfo()'
'%% Return the version info structure'
'%%'
'%% Note: This file is autogenerated by build script DO NOT MODIFY!!'
''
'function versionInfo = VersionInfo()'
' jsonVer = ''%s'';'
' '
' versionInfo = Utils.ParseJSON(jsonVer);'
'end'};
if ( ~exist('forceVersion', 'var') )
forceVersion = '';
end
if ( ~exist('dependencyPaths', 'var') )
dependencyPaths = {};
end
fallbackFile = 'version.json';
%% Load version info from git tags
verInfo = Dev.LoadVersion();
if ( isempty(verInfo) )
error('Unable to load git version information');
end
% Force name to be the passed in product name
verInfo.name = productName;
%% If we are forcing a specific version number
if ( ~isempty(forceVersion) )
verString = ['v' forceVersion];
[majorVer,minorVer] = Dev.ParseVerTag(verString);
if ( isempty(majorVer) )
error('Invalid version string %s', verString);
end
verInfo.majorVersion = majorVer;
verInfo.minorVersion = minorVer;
end
%% Get a timestamp build-number
c = clock();
verInfo.buildNumber = sprintf('%d.%02d.%02d.%02d', c(1), c(2), c(3), c(4));
%% Get machine ID
[status,buildMachine] = system('hostname');
if ( status ~= 0 )
fprintf('WARNING: There was an error retrieving hostname:\n %s\n', buildMachine);
else
verInfo.buildMachine = strtrim(buildMachine);
end
%% Make sure local path is not in the list
localPath = pwd();
bHasLocal = strcmp(localPath,dependencyPaths);
dependencyPaths = dependencyPaths(~bHasLocal);
%% Add all other dependent commit hashes to list
hashStrings = cell(length(dependencyPaths),1);
for i=1:length(dependencyPaths)
[repoName,commitHash] = Dev.GetCommitInfo(dependencyPaths{i});
hashStrings{i} = [repoName ' : ' commitHash];
end
verInfo.commitHash = [verInfo.commitHash; hashStrings];
%% Create +Dev/VersionInfo.m for use in compiled files
% Concatenate the template function lines into one giant string
templateString = [];
for i=1:length(funcString)
templateString = [templateString funcString{i} '\n'];
end
% Now insert all our arguments into the template and write to a file.
if ( ~exist('+Dev','dir') )
mkdir('+Dev');
end
fid = fopen(fullfile('+Dev','VersionInfo.m'), 'wt');
if ( fid <= 0 )
error('Unable to open +Dev/VersionInfo.m for writing');
end
jsonVer = Utils.CreateJSON(verInfo,false);
fprintf(fid, templateString, jsonVer);
fclose(fid);
%% Update fallback file if we used git to retrieve version info.
jsonStr = Utils.CreateJSON(verInfo);
fid = fopen(fallbackFile, 'wt');
if ( fid <= 0 )
return;
end
fprintf(fid, '%s\n', jsonStr);
fclose(fid);
end
function [graphStruct,newNodes] = MergeGraphStruct(graphStruct, newStruct)