feat: add processinfo index, add externalId (#1055)

If a NYC_PROCESSINFO_EXTERNAL_ID environment variable is set, then it is
saved in the processinfo as `externalId`.
Furthermore, when this file is generated, some additional helpful
metadata is memoized to the processinfo json files, to minimize the cost
of repeated generation. (This isn't necessarily a breaking change, but
it is an update to the de facto schema for those files.)
As soon as possible, index generation and process tree display should be
migrated out to a new 'istanbul-lib-processinfo' library.
This opens the door to add features in the v14 release family to improve
support for partial/resumed test runs and file watching.
- When a process is run with --clean=false and a previously seen
externalId, clear away all the coverage files in the set for that
externalId.
- When a file is changed, a test runner can use the index to determine
which tests (by externalId) ought to be re-run.
- Adds a NYC_PROCESS_ID to environment
- Adds `parent` to processInfo object, a uuid referring to parent.
- Rebase onto processinfo-numeric-pids branch
- Avoid re-writing the processinfo/{uuid}.json files
- Update process tree output to rely on process index instead of
duplicating effort.
BREAKING CHANGE: This adds a file named 'index.json' to the
.nyc_output/processinfo directory, which has a different format from the
other files in this dir.

Author: isaacs

bin/nyc.js

@@ -65,6 +65,10 @@ if ([
 ),function(done){
 varmainChildExitCode=process.exitCode
 
+if(argv.showProcessTree||argv.buildProcessTree){
+nyc.writeProcessIndex()
+}
+
 if(argv.checkCoverage){
 nyc.checkCoverage({
 lines: argv.lines,

bin/wrap.js

@@ -8,8 +8,13 @@ config.isChildProcess = true
 config._processInfo={
 pid: process.pid,
 ppid: process.ppid,
+parent: process.env.NYC_PROCESS_ID||null,
 root: process.env.NYC_ROOT_ID
 }
+if(process.env.NYC_PROCESSINFO_EXTERNAL_ID){
+config._processInfo.externalId=process.env.NYC_PROCESSINFO_EXTERNAL_ID
+deleteprocess.env.NYC_PROCESSINFO_EXTERNAL_ID
+}
 
 ;(newNYC(config)).wrap()
 

index.js

@@ -312,6 +312,7 @@ NYC.prototype._wrapExit = function () {
 }
 
 NYC.prototype.wrap=function(bin){
+process.env.NYC_PROCESS_ID=this.processInfo.uuid
 this._addRequireHooks()
 this._wrapExit()
 this._loadAdditionalModules()
@@ -341,7 +342,7 @@ NYC.prototype.writeCoverageFile = function () {
 coverage=this.sourceMaps.remapCoverage(coverage)
 }
 
-varid=this.generateUniqueID()
+varid=this.processInfo.uuid
 varcoverageFilename=path.resolve(this.tempDirectory(),id+'.json')
 
 fs.writeFileSync(
@@ -355,6 +356,7 @@ NYC.prototype.writeCoverageFile = function () {
 }
 
 this.processInfo.coverageFilename=coverageFilename
+this.processInfo.files=Object.keys(coverage)
 
 fs.writeFileSync(
 path.resolve(this.processInfoDirectory(),id+'.json'),
@@ -412,6 +414,80 @@ NYC.prototype.report = function () {
 }
 }
 
+// XXX(@isaacs) Index generation should move to istanbul-lib-processinfo
+NYC.prototype.writeProcessIndex=function(){
+constdir=this.processInfoDirectory()
+constpidToUid=newMap()
+constinfoByUid=newMap()
+consteidToUid=newMap()
+constinfos=fs.readdirSync(dir).filter(f=>f!=='index.json').map(f=>{
+try{
+constinfo=JSON.parse(fs.readFileSync(path.resolve(dir,f),'utf-8'))
+info.children=[]
+pidToUid.set(info.uuid,info.pid)
+pidToUid.set(info.pid,info.uuid)
+infoByUid.set(info.uuid,info)
+if(info.externalId){
+eidToUid.set(info.externalId,info.uuid)
+}
+returninfo
+}catch(er){
+returnnull
+}
+}).filter(Boolean)
+
+// create all the parent-child links and write back the updated info
+infos.forEach(info=>{
+if(info.parent){
+constparentInfo=infoByUid.get(info.parent)
+if(parentInfo.children.indexOf(info.uuid)===-1){
+parentInfo.children.push(info.uuid)
+}
+}
+})
+
+// figure out which files were touched by each process.
+constfiles=infos.reduce((files,info)=>{
+info.files.forEach(f=>{
+files[f]=files[f]||[]
+files[f].push(info.uuid)
+})
+returnfiles
+},{})
+
+// build the actual index!
+constindex=infos.reduce((index,info)=>{
+index.processes[info.uuid]={}
+index.processes[info.uuid].parent=info.parent
+if(info.externalId){
+if(index.externalIds[info.externalId]){
+thrownewError(`External ID ${info.externalId} used by multiple processes`)
+}
+index.processes[info.uuid].externalId=info.externalId
+index.externalIds[info.externalId]={
+root: info.uuid,
+children: info.children
+}
+}
+index.processes[info.uuid].children=Array.from(info.children)
+returnindex
+},{processes: {},files: files,externalIds: {}})
+
+// flatten the descendant sets of all the externalId procs
+Object.keys(index.externalIds).forEach(eid=>{
+const{ children }=index.externalIds[eid]
+// push the next generation onto the list so we accumulate them all
+for(leti=0;i<children.length;i++){
+constnextGen=index.processes[children[i]].children
+if(nextGen&&nextGen.length){
+children.push(...nextGen.filter(uuid=>children.indexOf(uuid)===-1))
+}
+}
+})
+
+fs.writeFileSync(path.resolve(dir,'index.json'),JSON.stringify(index))
+}
+
 NYC.prototype.showProcessTree=function(){
 varprocessTree=ProcessInfo.buildProcessTree(this._loadProcessInfos())
 
@@ -448,19 +524,25 @@ NYC.prototype._checkCoverage = function (summary, thresholds, file) {
 }
 
 NYC.prototype._loadProcessInfos=function(){
-var_this=this
-varfiles=fs.readdirSync(this.processInfoDirectory())
-
-returnfiles.map(function(f){
+returnfs.readdirSync(this.processInfoDirectory()).map(f=>{
+letdata
 try{
-returnnewProcessInfo(JSON.parse(fs.readFileSync(
-path.resolve(_this.processInfoDirectory(),f),
+data=JSON.parse(fs.readFileSync(
+path.resolve(this.processInfoDirectory(),f),
 'utf-8'
-)))
+))
 }catch(e){// handle corrupt JSON output.
-return{}
+returnnull
 }
-})
+if(f!=='index.json'){
+data.nodes=[]
+data=newProcessInfo(data)
+}
+return{file: path.basename(f,'.json'),data: data}
+}).filter(Boolean).reduce((infos,info)=>{
+infos[info.file]=info.data
+returninfos
+},{})
 }
 
 NYC.prototype.eachReport=function(filenames,iterator,baseDirectory){

lib/process.js

@@ -1,9 +1,12 @@
 constarchy=require('archy')
 constlibCoverage=require('istanbul-lib-coverage')
+constuuid=require('uuid/v4')
 
 functionProcessInfo(defaults){
 defaults=defaults||{}
 
+this.uuid=null
+this.parent=null
 this.pid=String(process.pid)
 this.argv=process.argv
 this.execArgv=process.execArgv
@@ -12,13 +15,14 @@ function ProcessInfo (defaults) {
 this.ppid=null
 this.root=null
 this.coverageFilename=null
-this.nodes=[]// list of children, filled by buildProcessTree()
-
-this._coverageMap=null
 
 for(varkeyindefaults){
 this[key]=defaults[key]
 }
+
+if(!this.uuid){
+this.uuid=uuid()
+}
 }
 
 Object.defineProperty(ProcessInfo.prototype,'label',{
@@ -36,29 +40,19 @@ Object.defineProperty(ProcessInfo.prototype, 'label', {
 })
 
 ProcessInfo.buildProcessTree=function(infos){
-vartreeRoot=newProcessInfo({_label: 'nyc'})
-varnodes={}
-
-infos=infos.sort(function(a,b){
-returna.time-b.time
-})
-
-infos.forEach(function(p){
-nodes[p.root+':'+p.pid]=p
-})
-
-infos.forEach(function(p){
-if(!p.ppid){
-return
+consttreeRoot=newProcessInfo({_label: 'nyc',nodes: []})
+constindex=infos.index
+for(constidinindex.processes){
+constnode=infos[id]
+if(!node){
+thrownewError(`Invalid entry in processinfo index: ${id}`)
 }
-
-varparent=nodes[p.root+':'+p.ppid]
-if(!parent){
-parent=treeRoot
+constidx=index.processes[id]
+node.nodes=idx.children.map(id=>infos[id]).sort((a,b)=>a.time-b.time)
+if(!node.parent){
+treeRoot.nodes.push(node)
 }
-
-parent.nodes.push(p)
-})
+}
 
 returntreeRoot
 }

test/processinfo.js

@@ -0,0 +1,95 @@
+const{ resolve }=require('path')
+constbin=resolve(__dirname,'../self-coverage/bin/nyc')
+const{ spawn }=require('child_process')
+constt=require('tap')
+constrimraf=require('rimraf')
+constnode=process.execPath
+constfixturesCLI=resolve(__dirname,'./fixtures/cli')
+consttmp='processinfo-test'
+constfs=require('fs')
+constresolvedJS=resolve(fixturesCLI,'selfspawn-fibonacci.js')
+
+rimraf.sync(resolve(fixturesCLI,tmp))
+t.teardown(()=>rimraf.sync(resolve(fixturesCLI,tmp)))
+
+t.test('build some processinfo',t=>{
+varargs=[
+bin,'-t',tmp,'--build-process-tree',
+node,'selfspawn-fibonacci.js','5'
+]
+varproc=spawn(process.execPath,args,{
+cwd: fixturesCLI,
+env: {
+PATH: process.env.PATH,
+NYC_PROCESSINFO_EXTERNAL_ID: 'blorp'
+}
+})
+// don't actually care about the output for this test, just the data
+proc.stderr.resume()
+proc.stdout.resume()
+proc.on('close',(code,signal)=>{
+t.equal(code,0)
+t.equal(signal,null)
+t.end()
+})
+})
+
+t.test('validate the created processinfo data',t=>{
+constcovs=fs.readdirSync(resolve(fixturesCLI,tmp))
+.filter(f=>f!=='processinfo')
+t.plan(covs.length*2)
+
+covs.forEach(f=>{
+fs.readFile(resolve(fixturesCLI,tmp,f),'utf8',(er,covjson)=>{
+if(er){
+thrower
+}
+constcovdata=JSON.parse(covjson)
+t.same(Object.keys(covdata),[resolvedJS])
+// should have matching processinfo for each cov json
+constprocInfoFile=resolve(fixturesCLI,tmp,'processinfo',f)
+fs.readFile(procInfoFile,'utf8',(er,procInfoJson)=>{
+if(er){
+thrower
+}
+constprocInfoData=JSON.parse(procInfoJson)
+t.match(procInfoData,{
+pid: Number,
+ppid: Number,
+uuid: f.replace(/\.json$/,''),
+argv: [
+node,
+resolvedJS,
+/[1-5]/
+],
+execArgv: [],
+cwd: fixturesCLI,
+time: Number,
+root: /^[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}$/,
+coverageFilename: resolve(fixturesCLI,tmp,f),
+files: [resolvedJS]
+})
+})
+})
+})
+})
+
+t.test('check out the index',t=>{
+constindexFile=resolve(fixturesCLI,tmp,'processinfo','index.json')
+constindexJson=fs.readFileSync(indexFile,'utf-8')
+constindex=JSON.parse(indexJson)
+constu=/^[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}$/
+t.match(index,{
+processes: {},
+files: {
+[resolvedJS]: [u,u,u,u,u,u,u,u,u]
+},
+externalIds: {
+blorp: {
+root: u,
+children: [u,u,u,u,u,u,u,u]
+}
+}
+})
+t.end()
+})