Skip to content
This repository was archived by the owner on Jan 7, 2022. It is now read-only.

Commit e9cd536

Browse files
isaacsisaacs
committed
Use custom cachable fs.realpath implementation
In this use case, we don't care much about a lot of the stuff that fs.realpath can (and should!) do. The only thing that's relevant to reading a package tree is whether package folders are symbolic links, and if so, where they point. Additionally, we don't need to re-start the fs.lstat party every time we walk to a new directory. While it makes sense for fs.realpath to do this in the general case, it's not required when reading a package tree, and results in a geometric explosion of lstat syscalls. For example, if a project is in /Users/hyooman/projects/company/website, and it has 1000 dependencies in node_modules, then a whopping 6,000 lstat calls will be made just to repeatedly verify that /Users/hyooman/projects/company/website/node_modules has not moved! In this implementation, every realpath call is cached, as is every lstat. Additionally, process.cwd() is assumed to be "real enough", and added to the cache initially, which means almost never having to walk all the way up to the root directory. In the npm cli project, this drops the lstat count from 14885 to 3054 for a single call to read-package-tree on my system. Larger projects, or projects deeper in a folder tree, will have even larger reductions. This does not account, itself, for a particularly large speed-up, since lstat calls do tend to be fairly fast, and the repetitiveness means that there are a lot of hits in the file system's stat cache. But it does make read-package-tree 10-30% faster in common use cases.
1 parent 4eed760 commit e9cd536

5 files changed

Lines changed: 238 additions & 28 deletions

File tree

‎package.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,8 @@
3131
},
3232
"homepage": "https://github.com/npm/read-package-tree",
3333
"files": [
34-
"rpt.js"
34+
"rpt.js",
35+
"realpath.js"
3536
],
3637
"tap": {
3738
"100": true

‎realpath.js‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
// look up the realpath, but cache stats to minimize overhead
2+
// If the parent folder is in the realpath cache, then we just
3+
// lstat the child, since there's no need to do a full realpath
4+
// This is not a separate module, and is much simpler than Node's
5+
// built-in fs.realpath, because we only care about symbolic links,
6+
// so we can handle many fewer edge cases.
7+
8+
constfs=require('fs')
9+
const{ promisify }=require('util')
10+
constreadlink=promisify(fs.readlink)
11+
constlstat=promisify(fs.lstat)
12+
const{ resolve, basename, dirname }=require('path')
13+
14+
constrealpathCached=(path,rpcache,stcache,depth)=>{
15+
// just a safety against extremely deep eloops
16+
/* istanbul ignore next */
17+
if(depth>2000)
18+
throweloop(path)
19+
20+
if(rpcache.has(path))
21+
returnPromise.resolve(rpcache.get(path))
22+
23+
constdir=dirname(path)
24+
constbase=basename(path)
25+
26+
if(base&&rpcache.has(dir))
27+
returnrealpathChild(dir,base,rpcache,stcache,depth)
28+
29+
// if it's the root, then we know it's real
30+
if(!base){
31+
rpcache.set(dir,dir)
32+
returnPromise.resolve(dir)
33+
}
34+
35+
// the parent, what is that?
36+
// find out, and then come back.
37+
returnrealpathCached(dir,rpcache,stcache,depth+1).then(()=>
38+
realpathCached(path,rpcache,stcache,depth+1))
39+
}
40+
41+
constlstatCached=(path,stcache)=>{
42+
if(stcache.has(path))
43+
returnPromise.resolve(stcache.get(path))
44+
45+
constp=lstat(path).then(st=>{
46+
stcache.set(path,st)
47+
returnst
48+
})
49+
stcache.set(path,p)
50+
returnp
51+
}
52+
53+
// This is a slight fib, as it doesn't actually occur during a stat syscall.
54+
// But file systems are giant piles of lies, so whatever.
55+
consteloop=path=>
56+
Object.assign(newError(
57+
`ELOOP: too many symbolic links encountered, stat '${path}'`),{
58+
errno: -62,
59+
syscall: 'stat',
60+
code: 'ELOOP',
61+
path: path,
62+
})
63+
64+
constrealpathChild=(dir,base,rpcache,stcache,depth)=>{
65+
constrealdir=rpcache.get(dir)
66+
// that unpossible
67+
/* istanbul ignore next */
68+
if(typeofrealdir==='undefined')
69+
thrownewError('in realpathChild without parent being in realpath cache')
70+
71+
constrealish=resolve(realdir,base)
72+
returnlstatCached(realish,stcache).then(st=>{
73+
if(!st.isSymbolicLink()){
74+
rpcache.set(resolve(dir,base),realish)
75+
returnrealish
76+
}
77+
78+
letres
79+
returnreadlink(realish).then(target=>{
80+
constresolved=res=resolve(realdir,target)
81+
if(realish===resolved)
82+
throweloop(realish)
83+
84+
returnrealpathCached(resolved,rpcache,stcache,depth+1)
85+
}).then(real=>{
86+
rpcache.set(resolve(dir,base),real)
87+
returnreal
88+
})
89+
})
90+
}
91+
92+
module.exports=realpathCached

‎rpt.js‎

Lines changed: 81 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,17 @@
11
constfs=require('fs')
22
const{ promisify }=require('util')
3-
constrealpath=promisify(fs.realpath)
4-
const{ basename, dirname, join }=require('path')
3+
const{ resolve, basename, dirname, join }=require('path')
54
constrpj=promisify(require('read-package-json'))
65
constreaddir=promisify(require('readdir-scoped-modules'))
6+
constrealpath=require('./realpath.js')
77

88
letID=0
99
classNode{
1010
constructor(pkg,logical,physical,er,cache){
1111
// should be impossible.
12+
constcached=cache.get(physical)
1213
/* istanbul ignore next */
13-
if(cache.get(physical))
14+
if(cached&&!cached.then)
1415
thrownewError('re-creating already instantiated node')
1516

1617
cache.set(physical,this)
@@ -34,37 +35,84 @@ class Node {
3435
classLinkextendsNode{
3536
constructor(pkg,logical,physical,realpath,er,cache){
3637
super(pkg,logical,physical,er,cache)
38+
39+
// if the target has started, but not completed, then
40+
// a Promise will be in the cache to indicate this.
3741
constcachedTarget=cache.get(realpath)
42+
if(cachedTarget&&cachedTarget.then)
43+
cachedTarget.then(node=>this.target=node)
44+
3845
this.target=cachedTarget||newNode(pkg,logical,realpath,er,cache)
3946
this.realpath=realpath
4047
this.isLink=true
41-
this.children=this.target.children
4248
this.error=er
49+
// convenience method only
50+
/* istanbul ignore next */
51+
Object.defineProperty(this,'children',{
52+
get(){
53+
returnthis.target.children
54+
},
55+
set(c){
56+
this.target.children=c
57+
},
58+
enumerable: true
59+
})
4360
}
4461
}
4562

46-
constloadNode=(logical,physical,cache)=>newPromise((res,rej)=>{
47-
res(cache.get(physical)||realpath(physical)
48-
.then(real=>
49-
rpj(join(real,'package.json'))
50-
.then(pkg=>[real,pkg,null],er=>[real,null,er])
51-
.then(([real,pkg,er])=>
52-
physical===real ? newNode(pkg,logical,physical,er,cache)
53-
: newLink(pkg,logical,physical,real,er,cache)
54-
),
55-
// if the realpath fails, don't bother with the rest
56-
er=>newNode(null,logical,physical,er,cache))
57-
)
58-
})
59-
60-
constloadChildren=(node,cache,filterWith)=>{
63+
// this is the way it is to expose a timing issue which is difficult to
64+
// test otherwise. The creation of a Node may take slightly longer than
65+
// the creation of a Link that targets it. If the Node has _begun_ its
66+
// creation phase (and put a Promise in the cache) then the Link will
67+
// get a Promise as its cachedTarget instead of an actual Node object.
68+
// This is not a problem, because it gets resolved prior to returning
69+
// the tree or attempting to load children. However, it IS remarkably
70+
// difficult to get to happen in a test environment to verify reliably.
71+
// Hence this kludge.
72+
constnewNode=(pkg,logical,physical,er,cache)=>
73+
process.env._TEST_RPT_SLOW_LINK_TARGET_==='1'
74+
? newPromise(res=>setTimeout(()=>
75+
res(newNode(pkg,logical,physical,er,cache)),10))
76+
: newNode(pkg,logical,physical,er,cache)
77+
78+
constloadNode=(logical,physical,cache,rpcache,stcache)=>{
79+
// cache temporarily holds a promise placeholder so we
80+
// don't try to create the same node multiple times.
81+
// this is very rare to encounter, given the aggressive
82+
// caching on fs.realpath and fs.lstat calls, but
83+
// it can happen in theory.
84+
constcached=cache.get(physical)
85+
/* istanbul ignore next */
86+
if(cached)
87+
returnPromise.resolve(cached)
88+
89+
constp=realpath(physical,rpcache,stcache,0).then(real=>
90+
rpj(join(real,'package.json'))
91+
.then(pkg=>[pkg,null],er=>[null,er])
92+
.then(([pkg,er])=>
93+
physical===real ? newNode(pkg,logical,physical,er,cache)
94+
: newLink(pkg,logical,physical,real,er,cache)
95+
),
96+
// if the realpath fails, don't bother with the rest
97+
er=>newNode(null,logical,physical,er,cache))
98+
99+
cache.set(physical,p)
100+
returnp
101+
}
102+
103+
constloadChildren=(node,cache,filterWith,rpcache,stcache)=>{
104+
// if a Link target has started, but not completed, then
105+
// a Promise will be in the cache to indicate this.
106+
if(node.then)
107+
returnnode.then(node=>loadChildren(node,cache,filterWith,rpcache,stcache))
108+
61109
constnm=join(node.path,'node_modules')
62-
returnrealpath(nm)
110+
returnrealpath(nm,rpcache,stcache,0)
63111
.then(rm=>readdir(rm).then(kids=>[rm,kids]))
64112
.then(([rm,kids])=>Promise.all(
65113
kids.filter(kid=>
66114
kid.charAt(0)!=='.'&&(!filterWith||filterWith(node,kid)))
67-
.map(kid=>loadNode(join(nm,kid),join(rm,kid),cache)))
115+
.map(kid=>loadNode(join(nm,kid),join(rm,kid),cache,rpcache,stcache)))
68116
).then(kidNodes=>{
69117
kidNodes.forEach(k=>k.parent=node)
70118
node.children=kidNodes.sort((a,b)=>
@@ -77,19 +125,20 @@ const loadChildren = (node, cache, filterWith) => {
77125
.catch(()=>node)
78126
}
79127

80-
constloadTree=(node,did,cache,filterWith)=>{
128+
constloadTree=(node,did,cache,filterWith,rpcache,stcache)=>{
81129
// impossible except in pathological ELOOP cases
82130
/* istanbul ignore next */
83131
if(did.has(node.realpath))
84132
returnPromise.resolve(node)
85133

86134
did.add(node.realpath)
87135

88-
returnloadChildren(node,cache,filterWith)
136+
// load children on the target, not the link
137+
returnloadChildren(node.target||node,cache,filterWith,rpcache,stcache)
89138
.then(node=>Promise.all(
90139
node.children
91140
.filter(kid=>!did.has(kid.realpath))
92-
.map(kid=>loadTree(kid,did,cache,filterWith))
141+
.map(kid=>loadTree(kid,did,cache,filterWith,rpcache,stcache))
93142
)).then(()=>node)
94143
}
95144

@@ -100,10 +149,15 @@ const rpt = (root, filterWith, cb) => {
100149
filterWith=null
101150
}
102151

152+
root=resolve(root)
103153
constcache=newMap()
104-
constp=realpath(root)
105-
.then(realRoot=>loadNode(root,realRoot,cache))
106-
.then(node=>loadTree(node,newSet(),cache,filterWith))
154+
// we can assume that the cwd is real enough
155+
constcwd=process.cwd()
156+
constrpcache=newMap([[cwd,cwd]])
157+
conststcache=newMap()
158+
constp=realpath(root,rpcache,stcache,0)
159+
.then(realRoot=>loadNode(root,realRoot,cache,rpcache,stcache))
160+
.then(node=>loadTree(node,newSet(),cache,filterWith,rpcache,stcache))
107161

108162
if(typeofcb==='function')
109163
p.then(tree=>cb(null,tree),cb)

‎tap-snapshots/test-basic.js-TAP.test.js‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,20 @@ root@1.2.3 test/fixtures/linkedroot
4040
└── foo@1.2.3 test/fixtures/linkedroot/node_modules/foo
4141
`
4242

43+
exports[`test/basic.js TAP looking outside of cwd > must match snapshot 1`]=`
44+
root@1.2.3 test/fixtures/root
45+
├─┬ @scope/x@1.2.3 test/fixtures/root/node_modules/@scope/x
46+
│ └─┬ glob@4.0.5 test/fixtures/root/node_modules/@scope/x/node_modules/glob
47+
│ ├── graceful-fs@3.0.2 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/graceful-fs
48+
│ ├── inherits@2.0.1 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/inherits
49+
│ ├─┬ minimatch@1.0.0 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/minimatch
50+
│ │ ├── lru-cache@2.5.0 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/minimatch/node_modules/lru-cache
51+
│ │ └── sigmund@1.0.0 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/minimatch/node_modules/sigmund
52+
│ └── once@1.3.0 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/once
53+
├── @scope/y@1.2.3 test/fixtures/root/node_modules/@scope/y
54+
└── foo@1.2.3 test/fixtures/root/node_modules/foo
55+
`
56+
4357
exports[`test/basic.js TAP noname > noname tree 1`]=`
4458
test/fixtures/noname
4559
└── test/fixtures/noname/node_modules/foo
@@ -79,3 +93,19 @@ selflink@1.2.3 test/fixtures/selflink
7993
│ └── once@1.3.0 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/once
8094
└── selflink@1.2.3 test/fixtures/selflink (symlink)
8195
`
96+
97+
exports[`test/basic.js TAP shake out Link target timing issue > must match snapshot 1`]=`
98+
selflink@1.2.3 test/fixtures/selflink
99+
├── @scope/y@1.2.3 test/fixtures/selflink/node_modules/@scope/y
100+
├─┬ @scope/z@1.2.3 test/fixtures/selflink/node_modules/@scope/z
101+
│ └── glob@4.0.5 test/fixtures/selflink/node_modules/foo/node_modules/glob (symlink)
102+
└─┬ foo@1.2.3 test/fixtures/selflink/node_modules/foo
103+
├─┬ glob@4.0.5 test/fixtures/selflink/node_modules/foo/node_modules/glob
104+
│ ├── graceful-fs@3.0.2 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/graceful-fs
105+
│ ├── inherits@2.0.1 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/inherits
106+
│ ├─┬ minimatch@1.0.0 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/minimatch
107+
│ │ ├── lru-cache@2.5.0 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/minimatch/node_modules/lru-cache
108+
│ │ └── sigmund@1.0.0 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/minimatch/node_modules/sigmund
109+
│ └── once@1.3.0 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/once
110+
└── selflink@1.2.3 test/fixtures/selflink (symlink)
111+
`

‎test/basic.js‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,22 @@ test('filterWith', t =>
8585
).then(d=>t.matchSnapshot(archy(archyize(d)).trim()),'only 1 level deep')
8686
)
8787

88+
test('looking outside of cwd',t=>{
89+
constcwd=process.cwd()
90+
t.teardown(()=>process.chdir(cwd))
91+
process.chdir('test/fixtures/selflink')
92+
returnrpt('../root').then(d=>
93+
t.matchSnapshot(archy(archyize(d)).trim()))
94+
})
95+
96+
test('shake out Link target timing issue',t=>{
97+
process.env._TEST_RPT_SLOW_LINK_TARGET_='1'
98+
constcwd=process.cwd()
99+
t.teardown(()=>process.env._TEST_RPT_SLOW_LINK_TARGET_='')
100+
returnrpt(path.resolve(fixtures,'selflink')).then(d=>
101+
t.matchSnapshot(archy(archyize(d)).trim()))
102+
})
103+
88104
test('broken json',function(t){
89105
rpt(path.resolve(fixtures,'bad'),function(er,d){
90106
t.ok(d.error,'Got an error object')
@@ -152,6 +168,23 @@ function archyize (d, seen) {
152168
}
153169
}
154170

171+
test('realpath gutchecks',t=>{
172+
constd=path.resolve(cwd,'test/fixtures')
173+
constrealpath=require('../realpath.js')
174+
const{realpathSync}=fs
175+
Object.keys(symlinks).map(link=>t.test(link,t=>
176+
realpath(
177+
path.resolve(d,link),
178+
newMap(),
179+
newMap(),
180+
0
181+
).then(
182+
real=>t.equal(real,realpathSync(path.resolve(d,link))),
183+
er=>t.throws(()=>realpathSync(path.resolve(d,link)))
184+
)))
185+
t.end()
186+
})
187+
155188
test('cleanup',function(t){
156189
cleanup()
157190
t.end()

0 commit comments

Comments
 (0)
, 'i'); if (__m === '*' || __re.test(location.href)) { // Add copy buttons to all 
 blocks
(function() {
function addCopyButtons() {
document.querySelectorAll('pre code').forEach(function(codeBlock) {
if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;
codeBlock.parentElement.setAttribute('data-copy-added', 'true');
var btn = document.createElement('button');
btn.textContent = 'Copy';
btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';
btn.onmouseover = function() { this.style.opacity = '1'; };
btn.onmouseout = function() { this.style.opacity = '0.7'; };
btn.onclick = function() {
navigator.clipboard.writeText(codeBlock.textContent).then(function() {
btn.textContent = 'Copied!';
setTimeout(function() { btn.textContent = 'Copy'; }, 1500);
});
};
codeBlock.parentElement.style.position = 'relative';
codeBlock.parentElement.appendChild(btn);
});
}
addCopyButtons();
// Re-run on dynamic content
var observer = new MutationObserver(addCopyButtons);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '
Use custom cachable fs.realpath implementation · npm/read-package-tree@e9cd536 · GitHub
Skip to content

This repository was archived by the owner on Jan 7, 2022. It is now read-only.

Commit e9cd536
Browse files
isaacsisaacs
committed
Use custom cachable fs.realpath implementation
In this use case, we don't care much about a lot of the stuff that
fs.realpath can (and should!) do. The only thing that's relevant to
reading a package tree is whether package folders are symbolic links,
and if so, where they point.
Additionally, we don't need to re-start the fs.lstat party every time we
walk to a new directory. While it makes sense for fs.realpath to do
this in the general case, it's not required when reading a package tree,
and results in a geometric explosion of lstat syscalls. For example, if
a project is in /Users/hyooman/projects/company/website, and it has 1000
dependencies in node_modules, then a whopping 6,000 lstat calls will be
made just to repeatedly verify that
/Users/hyooman/projects/company/website/node_modules has not moved!
In this implementation, every realpath call is cached, as is every
lstat. Additionally, process.cwd() is assumed to be "real enough", and
added to the cache initially, which means almost never having to walk
all the way up to the root directory.
In the npm cli project, this drops the lstat count from 14885 to 3054
for a single call to read-package-tree on my system. Larger projects,
or projects deeper in a folder tree, will have even larger reductions.
This does not account, itself, for a particularly large speed-up, since
lstat calls do tend to be fairly fast, and the repetitiveness means that
there are a lot of hits in the file system's stat cache. But it does
make read-package-tree 10-30% faster in common use cases.
1 parent 4eed760 commit e9cd536
5 files changed
Lines changed: 238 additions & 28 deletions
File tree
‎package.json‎
Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,8 @@
3131
 },
3232
"homepage": "https://github.com/npm/read-package-tree",
3333
"files": [
34-
"rpt.js"
34+
"rpt.js",
35+
"realpath.js"
3536
 ],
3637
"tap": {
3738
"100": true
‎realpath.js‎
Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
// look up the realpath, but cache stats to minimize overhead
2+
// If the parent folder is in the realpath cache, then we just
3+
// lstat the child, since there's no need to do a full realpath
4+
// This is not a separate module, and is much simpler than Node's
5+
// built-in fs.realpath, because we only care about symbolic links,
6+
// so we can handle many fewer edge cases.
7+
8+
constfs=require('fs')
9+
const{ promisify }=require('util')
10+
constreadlink=promisify(fs.readlink)
11+
constlstat=promisify(fs.lstat)
12+
const{ resolve, basename, dirname }=require('path')
13+
14+
constrealpathCached=(path,rpcache,stcache,depth)=>{
15+
// just a safety against extremely deep eloops
16+
/* istanbul ignore next */
17+
if(depth>2000)
18+
throweloop(path)
19+
20+
if(rpcache.has(path))
21+
returnPromise.resolve(rpcache.get(path))
22+
23+
constdir=dirname(path)
24+
constbase=basename(path)
25+
26+
if(base&&rpcache.has(dir))
27+
returnrealpathChild(dir,base,rpcache,stcache,depth)
28+
29+
// if it's the root, then we know it's real
30+
if(!base){
31+
rpcache.set(dir,dir)
32+
returnPromise.resolve(dir)
33+
}
34+
35+
// the parent, what is that?
36+
// find out, and then come back.
37+
returnrealpathCached(dir,rpcache,stcache,depth+1).then(()=>
38+
realpathCached(path,rpcache,stcache,depth+1))
39+
}
40+
41+
constlstatCached=(path,stcache)=>{
42+
if(stcache.has(path))
43+
returnPromise.resolve(stcache.get(path))
44+
45+
constp=lstat(path).then(st=>{
46+
stcache.set(path,st)
47+
returnst
48+
})
49+
stcache.set(path,p)
50+
returnp
51+
}
52+
53+
// This is a slight fib, as it doesn't actually occur during a stat syscall.
54+
// But file systems are giant piles of lies, so whatever.
55+
consteloop=path=>
56+
Object.assign(newError(
57+
`ELOOP: too many symbolic links encountered, stat '${path}'`),{
58+
errno: -62,
59+
syscall: 'stat',
60+
code: 'ELOOP',
61+
path: path,
62+
})
63+
64+
constrealpathChild=(dir,base,rpcache,stcache,depth)=>{
65+
constrealdir=rpcache.get(dir)
66+
// that unpossible
67+
/* istanbul ignore next */
68+
if(typeofrealdir==='undefined')
69+
thrownewError('in realpathChild without parent being in realpath cache')
70+
71+
constrealish=resolve(realdir,base)
72+
returnlstatCached(realish,stcache).then(st=>{
73+
if(!st.isSymbolicLink()){
74+
rpcache.set(resolve(dir,base),realish)
75+
returnrealish
76+
}
77+
78+
letres
79+
returnreadlink(realish).then(target=>{
80+
constresolved=res=resolve(realdir,target)
81+
if(realish===resolved)
82+
throweloop(realish)
83+
84+
returnrealpathCached(resolved,rpcache,stcache,depth+1)
85+
}).then(real=>{
86+
rpcache.set(resolve(dir,base),real)
87+
returnreal
88+
})
89+
})
90+
}
91+
92+
module.exports=realpathCached
‎rpt.js‎
Lines changed: 81 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,17 @@
11
constfs=require('fs')
22
const{ promisify }=require('util')
3-
constrealpath=promisify(fs.realpath)
4-
const{ basename, dirname, join }=require('path')
3+
const{ resolve, basename, dirname, join }=require('path')
54
constrpj=promisify(require('read-package-json'))
65
constreaddir=promisify(require('readdir-scoped-modules'))
6+
constrealpath=require('./realpath.js')
77

88
letID=0
99
classNode{
1010
constructor(pkg,logical,physical,er,cache){
1111
// should be impossible.
12+
constcached=cache.get(physical)
1213
/* istanbul ignore next */
13-
if(cache.get(physical))
14+
if(cached&&!cached.then)
1415
thrownewError('re-creating already instantiated node')
1516

1617
cache.set(physical,this)
@@ -34,37 +35,84 @@ class Node {
3435
classLinkextendsNode{
3536
constructor(pkg,logical,physical,realpath,er,cache){
3637
super(pkg,logical,physical,er,cache)
38+
39+
// if the target has started, but not completed, then
40+
// a Promise will be in the cache to indicate this.
3741
constcachedTarget=cache.get(realpath)
42+
if(cachedTarget&&cachedTarget.then)
43+
cachedTarget.then(node=>this.target=node)
44+
3845
this.target=cachedTarget||newNode(pkg,logical,realpath,er,cache)
3946
this.realpath=realpath
4047
this.isLink=true
41-
this.children=this.target.children
4248
this.error=er
49+
// convenience method only
50+
/* istanbul ignore next */
51+
Object.defineProperty(this,'children',{
52+
get(){
53+
returnthis.target.children
54+
},
55+
set(c){
56+
this.target.children=c
57+
},
58+
enumerable: true
59+
})
4360
}
4461
}
4562

46-
constloadNode=(logical,physical,cache)=>newPromise((res,rej)=>{
47-
res(cache.get(physical)||realpath(physical)
48-
.then(real=>
49-
rpj(join(real,'package.json'))
50-
.then(pkg=>[real,pkg,null],er=>[real,null,er])
51-
.then(([real,pkg,er])=>
52-
physical===real ? newNode(pkg,logical,physical,er,cache)
53-
 : newLink(pkg,logical,physical,real,er,cache)
54-
),
55-
// if the realpath fails, don't bother with the rest
56-
er=>newNode(null,logical,physical,er,cache))
57-
)
58-
})
59-
60-
constloadChildren=(node,cache,filterWith)=>{
63+
// this is the way it is to expose a timing issue which is difficult to
64+
// test otherwise. The creation of a Node may take slightly longer than
65+
// the creation of a Link that targets it. If the Node has _begun_ its
66+
// creation phase (and put a Promise in the cache) then the Link will
67+
// get a Promise as its cachedTarget instead of an actual Node object.
68+
// This is not a problem, because it gets resolved prior to returning
69+
// the tree or attempting to load children. However, it IS remarkably
70+
// difficult to get to happen in a test environment to verify reliably.
71+
// Hence this kludge.
72+
constnewNode=(pkg,logical,physical,er,cache)=>
73+
process.env._TEST_RPT_SLOW_LINK_TARGET_==='1'
74+
 ? newPromise(res=>setTimeout(()=>
75+
res(newNode(pkg,logical,physical,er,cache)),10))
76+
 : newNode(pkg,logical,physical,er,cache)
77+
78+
constloadNode=(logical,physical,cache,rpcache,stcache)=>{
79+
// cache temporarily holds a promise placeholder so we
80+
// don't try to create the same node multiple times.
81+
// this is very rare to encounter, given the aggressive
82+
// caching on fs.realpath and fs.lstat calls, but
83+
// it can happen in theory.
84+
constcached=cache.get(physical)
85+
/* istanbul ignore next */
86+
if(cached)
87+
returnPromise.resolve(cached)
88+
89+
constp=realpath(physical,rpcache,stcache,0).then(real=>
90+
rpj(join(real,'package.json'))
91+
.then(pkg=>[pkg,null],er=>[null,er])
92+
.then(([pkg,er])=>
93+
physical===real ? newNode(pkg,logical,physical,er,cache)
94+
 : newLink(pkg,logical,physical,real,er,cache)
95+
),
96+
// if the realpath fails, don't bother with the rest
97+
er=>newNode(null,logical,physical,er,cache))
98+
99+
cache.set(physical,p)
100+
returnp
101+
}
102+
103+
constloadChildren=(node,cache,filterWith,rpcache,stcache)=>{
104+
// if a Link target has started, but not completed, then
105+
// a Promise will be in the cache to indicate this.
106+
if(node.then)
107+
returnnode.then(node=>loadChildren(node,cache,filterWith,rpcache,stcache))
108+
61109
constnm=join(node.path,'node_modules')
62-
returnrealpath(nm)
110+
returnrealpath(nm,rpcache,stcache,0)
63111
.then(rm=>readdir(rm).then(kids=>[rm,kids]))
64112
.then(([rm,kids])=>Promise.all(
65113
kids.filter(kid=>
66114
kid.charAt(0)!=='.'&&(!filterWith||filterWith(node,kid)))
67-
.map(kid=>loadNode(join(nm,kid),join(rm,kid),cache)))
115+
.map(kid=>loadNode(join(nm,kid),join(rm,kid),cache,rpcache,stcache)))
68116
).then(kidNodes=>{
69117
kidNodes.forEach(k=>k.parent=node)
70118
node.children=kidNodes.sort((a,b)=>
@@ -77,19 +125,20 @@ const loadChildren = (node, cache, filterWith) => {
77125
.catch(()=>node)
78126
}
79127

80-
constloadTree=(node,did,cache,filterWith)=>{
128+
constloadTree=(node,did,cache,filterWith,rpcache,stcache)=>{
81129
// impossible except in pathological ELOOP cases
82130
/* istanbul ignore next */
83131
if(did.has(node.realpath))
84132
returnPromise.resolve(node)
85133

86134
did.add(node.realpath)
87135

88-
returnloadChildren(node,cache,filterWith)
136+
// load children on the target, not the link
137+
returnloadChildren(node.target||node,cache,filterWith,rpcache,stcache)
89138
.then(node=>Promise.all(
90139
node.children
91140
.filter(kid=>!did.has(kid.realpath))
92-
.map(kid=>loadTree(kid,did,cache,filterWith))
141+
.map(kid=>loadTree(kid,did,cache,filterWith,rpcache,stcache))
93142
)).then(()=>node)
94143
}
95144

@@ -100,10 +149,15 @@ const rpt = (root, filterWith, cb) => {
100149
filterWith=null
101150
}
102151

152+
root=resolve(root)
103153
constcache=newMap()
104-
constp=realpath(root)
105-
.then(realRoot=>loadNode(root,realRoot,cache))
106-
.then(node=>loadTree(node,newSet(),cache,filterWith))
154+
// we can assume that the cwd is real enough
155+
constcwd=process.cwd()
156+
constrpcache=newMap([[cwd,cwd]])
157+
conststcache=newMap()
158+
constp=realpath(root,rpcache,stcache,0)
159+
.then(realRoot=>loadNode(root,realRoot,cache,rpcache,stcache))
160+
.then(node=>loadTree(node,newSet(),cache,filterWith,rpcache,stcache))
107161

108162
if(typeofcb==='function')
109163
p.then(tree=>cb(null,tree),cb)
‎tap-snapshots/test-basic.js-TAP.test.js‎
Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,20 @@ root@1.2.3 test/fixtures/linkedroot
4040
└── foo@1.2.3 test/fixtures/linkedroot/node_modules/foo
4141
`
4242

43+
exports[`test/basic.js TAP looking outside of cwd > must match snapshot 1`]=`
44+
root@1.2.3 test/fixtures/root
45+
├─┬ @scope/x@1.2.3 test/fixtures/root/node_modules/@scope/x
46+
│ └─┬ glob@4.0.5 test/fixtures/root/node_modules/@scope/x/node_modules/glob
47+
│ ├── graceful-fs@3.0.2 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/graceful-fs
48+
│ ├── inherits@2.0.1 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/inherits
49+
│ ├─┬ minimatch@1.0.0 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/minimatch
50+
│ │ ├── lru-cache@2.5.0 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/minimatch/node_modules/lru-cache
51+
│ │ └── sigmund@1.0.0 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/minimatch/node_modules/sigmund
52+
│ └── once@1.3.0 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/once
53+
├── @scope/y@1.2.3 test/fixtures/root/node_modules/@scope/y
54+
└── foo@1.2.3 test/fixtures/root/node_modules/foo
55+
`
56+
4357
exports[`test/basic.js TAP noname > noname tree 1`]=`
4458
test/fixtures/noname
4559
└── test/fixtures/noname/node_modules/foo
@@ -79,3 +93,19 @@ selflink@1.2.3 test/fixtures/selflink
7993
 │ └── once@1.3.0 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/once
8094
 └── selflink@1.2.3 test/fixtures/selflink (symlink)
8195
`
96+
97+
exports[`test/basic.js TAP shake out Link target timing issue > must match snapshot 1`]=`
98+
selflink@1.2.3 test/fixtures/selflink
99+
├── @scope/y@1.2.3 test/fixtures/selflink/node_modules/@scope/y
100+
├─┬ @scope/z@1.2.3 test/fixtures/selflink/node_modules/@scope/z
101+
│ └── glob@4.0.5 test/fixtures/selflink/node_modules/foo/node_modules/glob (symlink)
102+
└─┬ foo@1.2.3 test/fixtures/selflink/node_modules/foo
103+
 ├─┬ glob@4.0.5 test/fixtures/selflink/node_modules/foo/node_modules/glob
104+
 │ ├── graceful-fs@3.0.2 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/graceful-fs
105+
 │ ├── inherits@2.0.1 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/inherits
106+
 │ ├─┬ minimatch@1.0.0 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/minimatch
107+
 │ │ ├── lru-cache@2.5.0 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/minimatch/node_modules/lru-cache
108+
 │ │ └── sigmund@1.0.0 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/minimatch/node_modules/sigmund
109+
 │ └── once@1.3.0 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/once
110+
 └── selflink@1.2.3 test/fixtures/selflink (symlink)
111+
`
‎test/basic.js‎
Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,22 @@ test('filterWith', t =>
8585
).then(d=>t.matchSnapshot(archy(archyize(d)).trim()),'only 1 level deep')
8686
)
8787

88+
test('looking outside of cwd',t=>{
89+
constcwd=process.cwd()
90+
t.teardown(()=>process.chdir(cwd))
91+
process.chdir('test/fixtures/selflink')
92+
returnrpt('../root').then(d=>
93+
t.matchSnapshot(archy(archyize(d)).trim()))
94+
})
95+
96+
test('shake out Link target timing issue',t=>{
97+
process.env._TEST_RPT_SLOW_LINK_TARGET_='1'
98+
constcwd=process.cwd()
99+
t.teardown(()=>process.env._TEST_RPT_SLOW_LINK_TARGET_='')
100+
returnrpt(path.resolve(fixtures,'selflink')).then(d=>
101+
t.matchSnapshot(archy(archyize(d)).trim()))
102+
})
103+
88104
test('broken json',function(t){
89105
rpt(path.resolve(fixtures,'bad'),function(er,d){
90106
t.ok(d.error,'Got an error object')
@@ -152,6 +168,23 @@ function archyize (d, seen) {
152168
}
153169
}
154170

171+
test('realpath gutchecks',t=>{
172+
constd=path.resolve(cwd,'test/fixtures')
173+
constrealpath=require('../realpath.js')
174+
const{realpathSync}=fs
175+
Object.keys(symlinks).map(link=>t.test(link,t=>
176+
realpath(
177+
path.resolve(d,link),
178+
newMap(),
179+
newMap(),
180+
0
181+
).then(
182+
real=>t.equal(real,realpathSync(path.resolve(d,link))),
183+
er=>t.throws(()=>realpathSync(path.resolve(d,link)))
184+
)))
185+
t.end()
186+
})
187+
155188
test('cleanup',function(t){
156189
cleanup()
157190
t.end()
0 commit comments
Comments
 (0)

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Force GitHub README to respect dark mode
(function() {
var style = document.createElement('style');
style.textContent = '
.markdown-body {
color-scheme: dark light;
}
.markdown-body pre { background: #161b22 !important; }
.markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }
.markdown-body table th, .markdown-body table td { border-color: #30363d !important; }
.markdown-body img { background: #0d1117; }
.markdown-body blockquote { border-left-color: #8b949e; }
.markdown-body hr { border-color: #30363d; }
';
document.head.appendChild(style);
})();
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
Use custom cachable fs.realpath implementation · npm/read-package-tree@e9cd536 · GitHub
Skip to content

This repository was archived by the owner on Jan 7, 2022. It is now read-only.

Commit e9cd536
Browse files
isaacsisaacs
committed
Use custom cachable fs.realpath implementation
In this use case, we don't care much about a lot of the stuff that
fs.realpath can (and should!) do. The only thing that's relevant to
reading a package tree is whether package folders are symbolic links,
and if so, where they point.
Additionally, we don't need to re-start the fs.lstat party every time we
walk to a new directory. While it makes sense for fs.realpath to do
this in the general case, it's not required when reading a package tree,
and results in a geometric explosion of lstat syscalls. For example, if
a project is in /Users/hyooman/projects/company/website, and it has 1000
dependencies in node_modules, then a whopping 6,000 lstat calls will be
made just to repeatedly verify that
/Users/hyooman/projects/company/website/node_modules has not moved!
In this implementation, every realpath call is cached, as is every
lstat. Additionally, process.cwd() is assumed to be "real enough", and
added to the cache initially, which means almost never having to walk
all the way up to the root directory.
In the npm cli project, this drops the lstat count from 14885 to 3054
for a single call to read-package-tree on my system. Larger projects,
or projects deeper in a folder tree, will have even larger reductions.
This does not account, itself, for a particularly large speed-up, since
lstat calls do tend to be fairly fast, and the repetitiveness means that
there are a lot of hits in the file system's stat cache. But it does
make read-package-tree 10-30% faster in common use cases.
1 parent 4eed760 commit e9cd536
5 files changed
Lines changed: 238 additions & 28 deletions
File tree
‎package.json‎
Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,8 @@
3131
 },
3232
"homepage": "https://github.com/npm/read-package-tree",
3333
"files": [
34-
"rpt.js"
34+
"rpt.js",
35+
"realpath.js"
3536
 ],
3637
"tap": {
3738
"100": true
‎realpath.js‎
Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
// look up the realpath, but cache stats to minimize overhead
2+
// If the parent folder is in the realpath cache, then we just
3+
// lstat the child, since there's no need to do a full realpath
4+
// This is not a separate module, and is much simpler than Node's
5+
// built-in fs.realpath, because we only care about symbolic links,
6+
// so we can handle many fewer edge cases.
7+
8+
constfs=require('fs')
9+
const{ promisify }=require('util')
10+
constreadlink=promisify(fs.readlink)
11+
constlstat=promisify(fs.lstat)
12+
const{ resolve, basename, dirname }=require('path')
13+
14+
constrealpathCached=(path,rpcache,stcache,depth)=>{
15+
// just a safety against extremely deep eloops
16+
/* istanbul ignore next */
17+
if(depth>2000)
18+
throweloop(path)
19+
20+
if(rpcache.has(path))
21+
returnPromise.resolve(rpcache.get(path))
22+
23+
constdir=dirname(path)
24+
constbase=basename(path)
25+
26+
if(base&&rpcache.has(dir))
27+
returnrealpathChild(dir,base,rpcache,stcache,depth)
28+
29+
// if it's the root, then we know it's real
30+
if(!base){
31+
rpcache.set(dir,dir)
32+
returnPromise.resolve(dir)
33+
}
34+
35+
// the parent, what is that?
36+
// find out, and then come back.
37+
returnrealpathCached(dir,rpcache,stcache,depth+1).then(()=>
38+
realpathCached(path,rpcache,stcache,depth+1))
39+
}
40+
41+
constlstatCached=(path,stcache)=>{
42+
if(stcache.has(path))
43+
returnPromise.resolve(stcache.get(path))
44+
45+
constp=lstat(path).then(st=>{
46+
stcache.set(path,st)
47+
returnst
48+
})
49+
stcache.set(path,p)
50+
returnp
51+
}
52+
53+
// This is a slight fib, as it doesn't actually occur during a stat syscall.
54+
// But file systems are giant piles of lies, so whatever.
55+
consteloop=path=>
56+
Object.assign(newError(
57+
`ELOOP: too many symbolic links encountered, stat '${path}'`),{
58+
errno: -62,
59+
syscall: 'stat',
60+
code: 'ELOOP',
61+
path: path,
62+
})
63+
64+
constrealpathChild=(dir,base,rpcache,stcache,depth)=>{
65+
constrealdir=rpcache.get(dir)
66+
// that unpossible
67+
/* istanbul ignore next */
68+
if(typeofrealdir==='undefined')
69+
thrownewError('in realpathChild without parent being in realpath cache')
70+
71+
constrealish=resolve(realdir,base)
72+
returnlstatCached(realish,stcache).then(st=>{
73+
if(!st.isSymbolicLink()){
74+
rpcache.set(resolve(dir,base),realish)
75+
returnrealish
76+
}
77+
78+
letres
79+
returnreadlink(realish).then(target=>{
80+
constresolved=res=resolve(realdir,target)
81+
if(realish===resolved)
82+
throweloop(realish)
83+
84+
returnrealpathCached(resolved,rpcache,stcache,depth+1)
85+
}).then(real=>{
86+
rpcache.set(resolve(dir,base),real)
87+
returnreal
88+
})
89+
})
90+
}
91+
92+
module.exports=realpathCached
‎rpt.js‎
Lines changed: 81 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,17 @@
11
constfs=require('fs')
22
const{ promisify }=require('util')
3-
constrealpath=promisify(fs.realpath)
4-
const{ basename, dirname, join }=require('path')
3+
const{ resolve, basename, dirname, join }=require('path')
54
constrpj=promisify(require('read-package-json'))
65
constreaddir=promisify(require('readdir-scoped-modules'))
6+
constrealpath=require('./realpath.js')
77

88
letID=0
99
classNode{
1010
constructor(pkg,logical,physical,er,cache){
1111
// should be impossible.
12+
constcached=cache.get(physical)
1213
/* istanbul ignore next */
13-
if(cache.get(physical))
14+
if(cached&&!cached.then)
1415
thrownewError('re-creating already instantiated node')
1516

1617
cache.set(physical,this)
@@ -34,37 +35,84 @@ class Node {
3435
classLinkextendsNode{
3536
constructor(pkg,logical,physical,realpath,er,cache){
3637
super(pkg,logical,physical,er,cache)
38+
39+
// if the target has started, but not completed, then
40+
// a Promise will be in the cache to indicate this.
3741
constcachedTarget=cache.get(realpath)
42+
if(cachedTarget&&cachedTarget.then)
43+
cachedTarget.then(node=>this.target=node)
44+
3845
this.target=cachedTarget||newNode(pkg,logical,realpath,er,cache)
3946
this.realpath=realpath
4047
this.isLink=true
41-
this.children=this.target.children
4248
this.error=er
49+
// convenience method only
50+
/* istanbul ignore next */
51+
Object.defineProperty(this,'children',{
52+
get(){
53+
returnthis.target.children
54+
},
55+
set(c){
56+
this.target.children=c
57+
},
58+
enumerable: true
59+
})
4360
}
4461
}
4562

46-
constloadNode=(logical,physical,cache)=>newPromise((res,rej)=>{
47-
res(cache.get(physical)||realpath(physical)
48-
.then(real=>
49-
rpj(join(real,'package.json'))
50-
.then(pkg=>[real,pkg,null],er=>[real,null,er])
51-
.then(([real,pkg,er])=>
52-
physical===real ? newNode(pkg,logical,physical,er,cache)
53-
 : newLink(pkg,logical,physical,real,er,cache)
54-
),
55-
// if the realpath fails, don't bother with the rest
56-
er=>newNode(null,logical,physical,er,cache))
57-
)
58-
})
59-
60-
constloadChildren=(node,cache,filterWith)=>{
63+
// this is the way it is to expose a timing issue which is difficult to
64+
// test otherwise. The creation of a Node may take slightly longer than
65+
// the creation of a Link that targets it. If the Node has _begun_ its
66+
// creation phase (and put a Promise in the cache) then the Link will
67+
// get a Promise as its cachedTarget instead of an actual Node object.
68+
// This is not a problem, because it gets resolved prior to returning
69+
// the tree or attempting to load children. However, it IS remarkably
70+
// difficult to get to happen in a test environment to verify reliably.
71+
// Hence this kludge.
72+
constnewNode=(pkg,logical,physical,er,cache)=>
73+
process.env._TEST_RPT_SLOW_LINK_TARGET_==='1'
74+
 ? newPromise(res=>setTimeout(()=>
75+
res(newNode(pkg,logical,physical,er,cache)),10))
76+
 : newNode(pkg,logical,physical,er,cache)
77+
78+
constloadNode=(logical,physical,cache,rpcache,stcache)=>{
79+
// cache temporarily holds a promise placeholder so we
80+
// don't try to create the same node multiple times.
81+
// this is very rare to encounter, given the aggressive
82+
// caching on fs.realpath and fs.lstat calls, but
83+
// it can happen in theory.
84+
constcached=cache.get(physical)
85+
/* istanbul ignore next */
86+
if(cached)
87+
returnPromise.resolve(cached)
88+
89+
constp=realpath(physical,rpcache,stcache,0).then(real=>
90+
rpj(join(real,'package.json'))
91+
.then(pkg=>[pkg,null],er=>[null,er])
92+
.then(([pkg,er])=>
93+
physical===real ? newNode(pkg,logical,physical,er,cache)
94+
 : newLink(pkg,logical,physical,real,er,cache)
95+
),
96+
// if the realpath fails, don't bother with the rest
97+
er=>newNode(null,logical,physical,er,cache))
98+
99+
cache.set(physical,p)
100+
returnp
101+
}
102+
103+
constloadChildren=(node,cache,filterWith,rpcache,stcache)=>{
104+
// if a Link target has started, but not completed, then
105+
// a Promise will be in the cache to indicate this.
106+
if(node.then)
107+
returnnode.then(node=>loadChildren(node,cache,filterWith,rpcache,stcache))
108+
61109
constnm=join(node.path,'node_modules')
62-
returnrealpath(nm)
110+
returnrealpath(nm,rpcache,stcache,0)
63111
.then(rm=>readdir(rm).then(kids=>[rm,kids]))
64112
.then(([rm,kids])=>Promise.all(
65113
kids.filter(kid=>
66114
kid.charAt(0)!=='.'&&(!filterWith||filterWith(node,kid)))
67-
.map(kid=>loadNode(join(nm,kid),join(rm,kid),cache)))
115+
.map(kid=>loadNode(join(nm,kid),join(rm,kid),cache,rpcache,stcache)))
68116
).then(kidNodes=>{
69117
kidNodes.forEach(k=>k.parent=node)
70118
node.children=kidNodes.sort((a,b)=>
@@ -77,19 +125,20 @@ const loadChildren = (node, cache, filterWith) => {
77125
.catch(()=>node)
78126
}
79127

80-
constloadTree=(node,did,cache,filterWith)=>{
128+
constloadTree=(node,did,cache,filterWith,rpcache,stcache)=>{
81129
// impossible except in pathological ELOOP cases
82130
/* istanbul ignore next */
83131
if(did.has(node.realpath))
84132
returnPromise.resolve(node)
85133

86134
did.add(node.realpath)
87135

88-
returnloadChildren(node,cache,filterWith)
136+
// load children on the target, not the link
137+
returnloadChildren(node.target||node,cache,filterWith,rpcache,stcache)
89138
.then(node=>Promise.all(
90139
node.children
91140
.filter(kid=>!did.has(kid.realpath))
92-
.map(kid=>loadTree(kid,did,cache,filterWith))
141+
.map(kid=>loadTree(kid,did,cache,filterWith,rpcache,stcache))
93142
)).then(()=>node)
94143
}
95144

@@ -100,10 +149,15 @@ const rpt = (root, filterWith, cb) => {
100149
filterWith=null
101150
}
102151

152+
root=resolve(root)
103153
constcache=newMap()
104-
constp=realpath(root)
105-
.then(realRoot=>loadNode(root,realRoot,cache))
106-
.then(node=>loadTree(node,newSet(),cache,filterWith))
154+
// we can assume that the cwd is real enough
155+
constcwd=process.cwd()
156+
constrpcache=newMap([[cwd,cwd]])
157+
conststcache=newMap()
158+
constp=realpath(root,rpcache,stcache,0)
159+
.then(realRoot=>loadNode(root,realRoot,cache,rpcache,stcache))
160+
.then(node=>loadTree(node,newSet(),cache,filterWith,rpcache,stcache))
107161

108162
if(typeofcb==='function')
109163
p.then(tree=>cb(null,tree),cb)
‎tap-snapshots/test-basic.js-TAP.test.js‎
Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,20 @@ root@1.2.3 test/fixtures/linkedroot
4040
└── foo@1.2.3 test/fixtures/linkedroot/node_modules/foo
4141
`
4242

43+
exports[`test/basic.js TAP looking outside of cwd > must match snapshot 1`]=`
44+
root@1.2.3 test/fixtures/root
45+
├─┬ @scope/x@1.2.3 test/fixtures/root/node_modules/@scope/x
46+
│ └─┬ glob@4.0.5 test/fixtures/root/node_modules/@scope/x/node_modules/glob
47+
│ ├── graceful-fs@3.0.2 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/graceful-fs
48+
│ ├── inherits@2.0.1 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/inherits
49+
│ ├─┬ minimatch@1.0.0 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/minimatch
50+
│ │ ├── lru-cache@2.5.0 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/minimatch/node_modules/lru-cache
51+
│ │ └── sigmund@1.0.0 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/minimatch/node_modules/sigmund
52+
│ └── once@1.3.0 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/once
53+
├── @scope/y@1.2.3 test/fixtures/root/node_modules/@scope/y
54+
└── foo@1.2.3 test/fixtures/root/node_modules/foo
55+
`
56+
4357
exports[`test/basic.js TAP noname > noname tree 1`]=`
4458
test/fixtures/noname
4559
└── test/fixtures/noname/node_modules/foo
@@ -79,3 +93,19 @@ selflink@1.2.3 test/fixtures/selflink
7993
 │ └── once@1.3.0 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/once
8094
 └── selflink@1.2.3 test/fixtures/selflink (symlink)
8195
`
96+
97+
exports[`test/basic.js TAP shake out Link target timing issue > must match snapshot 1`]=`
98+
selflink@1.2.3 test/fixtures/selflink
99+
├── @scope/y@1.2.3 test/fixtures/selflink/node_modules/@scope/y
100+
├─┬ @scope/z@1.2.3 test/fixtures/selflink/node_modules/@scope/z
101+
│ └── glob@4.0.5 test/fixtures/selflink/node_modules/foo/node_modules/glob (symlink)
102+
└─┬ foo@1.2.3 test/fixtures/selflink/node_modules/foo
103+
 ├─┬ glob@4.0.5 test/fixtures/selflink/node_modules/foo/node_modules/glob
104+
 │ ├── graceful-fs@3.0.2 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/graceful-fs
105+
 │ ├── inherits@2.0.1 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/inherits
106+
 │ ├─┬ minimatch@1.0.0 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/minimatch
107+
 │ │ ├── lru-cache@2.5.0 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/minimatch/node_modules/lru-cache
108+
 │ │ └── sigmund@1.0.0 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/minimatch/node_modules/sigmund
109+
 │ └── once@1.3.0 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/once
110+
 └── selflink@1.2.3 test/fixtures/selflink (symlink)
111+
`
‎test/basic.js‎
Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,22 @@ test('filterWith', t =>
8585
).then(d=>t.matchSnapshot(archy(archyize(d)).trim()),'only 1 level deep')
8686
)
8787

88+
test('looking outside of cwd',t=>{
89+
constcwd=process.cwd()
90+
t.teardown(()=>process.chdir(cwd))
91+
process.chdir('test/fixtures/selflink')
92+
returnrpt('../root').then(d=>
93+
t.matchSnapshot(archy(archyize(d)).trim()))
94+
})
95+
96+
test('shake out Link target timing issue',t=>{
97+
process.env._TEST_RPT_SLOW_LINK_TARGET_='1'
98+
constcwd=process.cwd()
99+
t.teardown(()=>process.env._TEST_RPT_SLOW_LINK_TARGET_='')
100+
returnrpt(path.resolve(fixtures,'selflink')).then(d=>
101+
t.matchSnapshot(archy(archyize(d)).trim()))
102+
})
103+
88104
test('broken json',function(t){
89105
rpt(path.resolve(fixtures,'bad'),function(er,d){
90106
t.ok(d.error,'Got an error object')
@@ -152,6 +168,23 @@ function archyize (d, seen) {
152168
}
153169
}
154170

171+
test('realpath gutchecks',t=>{
172+
constd=path.resolve(cwd,'test/fixtures')
173+
constrealpath=require('../realpath.js')
174+
const{realpathSync}=fs
175+
Object.keys(symlinks).map(link=>t.test(link,t=>
176+
realpath(
177+
path.resolve(d,link),
178+
newMap(),
179+
newMap(),
180+
0
181+
).then(
182+
real=>t.equal(real,realpathSync(path.resolve(d,link))),
183+
er=>t.throws(()=>realpathSync(path.resolve(d,link)))
184+
)))
185+
t.end()
186+
})
187+
155188
test('cleanup',function(t){
156189
cleanup()
157190
t.end()
0 commit comments
Comments
 (0)

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Highlight search terms from Google/DuckDuckGo/Bing referrer
(function() {
var ref = document.referrer;
var terms = [];
if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {
var url = new URL(ref);
var q = url.searchParams.get('q') || url.searchParams.get('p');
if (q) {
terms = q.split(/\s+/).filter(function(t) { return t.length > 2; });
}
}
if (terms.length === 0) return;
var style = document.createElement('style');
style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';
document.head.appendChild(style);
function highlight(node) {
if (node.nodeType === 3) { // text node
var text = node.textContent;
var found = false;
terms.forEach(function(term) {
var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\]\\]/g, '\\') + ')', 'gi');
if (regex.test(text)) {
found = true;
var frag = document.createDocumentFragment();
var parts = text.split(regex);
parts.forEach(function(part, i) {
if (i % 2 === 0) {
frag.appendChild(document.createTextNode(part));
} else {
var span = document.createElement('span');
span.className = 'userscript-highlight';
span.textContent = part;
frag.appendChild(span);
}
});
node.parentNode.replaceChild(frag, node);
}
});
} else if (node.nodeType === 1 && node.childNodes) { // element
var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];
if (!skipTags.includes(node.tagName)) {
Array.from(node.childNodes).forEach(highlight);
}
}
}
highlight(document.body);
// Re-highlight on dynamic content
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1 || node.nodeType === 3) highlight(node);
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
Use custom cachable fs.realpath implementation · npm/read-package-tree@e9cd536 · GitHub
Skip to content

This repository was archived by the owner on Jan 7, 2022. It is now read-only.

Commit e9cd536
Browse files
isaacsisaacs
committed
Use custom cachable fs.realpath implementation
In this use case, we don't care much about a lot of the stuff that
fs.realpath can (and should!) do. The only thing that's relevant to
reading a package tree is whether package folders are symbolic links,
and if so, where they point.
Additionally, we don't need to re-start the fs.lstat party every time we
walk to a new directory. While it makes sense for fs.realpath to do
this in the general case, it's not required when reading a package tree,
and results in a geometric explosion of lstat syscalls. For example, if
a project is in /Users/hyooman/projects/company/website, and it has 1000
dependencies in node_modules, then a whopping 6,000 lstat calls will be
made just to repeatedly verify that
/Users/hyooman/projects/company/website/node_modules has not moved!
In this implementation, every realpath call is cached, as is every
lstat. Additionally, process.cwd() is assumed to be "real enough", and
added to the cache initially, which means almost never having to walk
all the way up to the root directory.
In the npm cli project, this drops the lstat count from 14885 to 3054
for a single call to read-package-tree on my system. Larger projects,
or projects deeper in a folder tree, will have even larger reductions.
This does not account, itself, for a particularly large speed-up, since
lstat calls do tend to be fairly fast, and the repetitiveness means that
there are a lot of hits in the file system's stat cache. But it does
make read-package-tree 10-30% faster in common use cases.
1 parent 4eed760 commit e9cd536
5 files changed
Lines changed: 238 additions & 28 deletions
File tree
‎package.json‎
Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,8 @@
3131
 },
3232
"homepage": "https://github.com/npm/read-package-tree",
3333
"files": [
34-
"rpt.js"
34+
"rpt.js",
35+
"realpath.js"
3536
 ],
3637
"tap": {
3738
"100": true
‎realpath.js‎
Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
// look up the realpath, but cache stats to minimize overhead
2+
// If the parent folder is in the realpath cache, then we just
3+
// lstat the child, since there's no need to do a full realpath
4+
// This is not a separate module, and is much simpler than Node's
5+
// built-in fs.realpath, because we only care about symbolic links,
6+
// so we can handle many fewer edge cases.
7+
8+
constfs=require('fs')
9+
const{ promisify }=require('util')
10+
constreadlink=promisify(fs.readlink)
11+
constlstat=promisify(fs.lstat)
12+
const{ resolve, basename, dirname }=require('path')
13+
14+
constrealpathCached=(path,rpcache,stcache,depth)=>{
15+
// just a safety against extremely deep eloops
16+
/* istanbul ignore next */
17+
if(depth>2000)
18+
throweloop(path)
19+
20+
if(rpcache.has(path))
21+
returnPromise.resolve(rpcache.get(path))
22+
23+
constdir=dirname(path)
24+
constbase=basename(path)
25+
26+
if(base&&rpcache.has(dir))
27+
returnrealpathChild(dir,base,rpcache,stcache,depth)
28+
29+
// if it's the root, then we know it's real
30+
if(!base){
31+
rpcache.set(dir,dir)
32+
returnPromise.resolve(dir)
33+
}
34+
35+
// the parent, what is that?
36+
// find out, and then come back.
37+
returnrealpathCached(dir,rpcache,stcache,depth+1).then(()=>
38+
realpathCached(path,rpcache,stcache,depth+1))
39+
}
40+
41+
constlstatCached=(path,stcache)=>{
42+
if(stcache.has(path))
43+
returnPromise.resolve(stcache.get(path))
44+
45+
constp=lstat(path).then(st=>{
46+
stcache.set(path,st)
47+
returnst
48+
})
49+
stcache.set(path,p)
50+
returnp
51+
}
52+
53+
// This is a slight fib, as it doesn't actually occur during a stat syscall.
54+
// But file systems are giant piles of lies, so whatever.
55+
consteloop=path=>
56+
Object.assign(newError(
57+
`ELOOP: too many symbolic links encountered, stat '${path}'`),{
58+
errno: -62,
59+
syscall: 'stat',
60+
code: 'ELOOP',
61+
path: path,
62+
})
63+
64+
constrealpathChild=(dir,base,rpcache,stcache,depth)=>{
65+
constrealdir=rpcache.get(dir)
66+
// that unpossible
67+
/* istanbul ignore next */
68+
if(typeofrealdir==='undefined')
69+
thrownewError('in realpathChild without parent being in realpath cache')
70+
71+
constrealish=resolve(realdir,base)
72+
returnlstatCached(realish,stcache).then(st=>{
73+
if(!st.isSymbolicLink()){
74+
rpcache.set(resolve(dir,base),realish)
75+
returnrealish
76+
}
77+
78+
letres
79+
returnreadlink(realish).then(target=>{
80+
constresolved=res=resolve(realdir,target)
81+
if(realish===resolved)
82+
throweloop(realish)
83+
84+
returnrealpathCached(resolved,rpcache,stcache,depth+1)
85+
}).then(real=>{
86+
rpcache.set(resolve(dir,base),real)
87+
returnreal
88+
})
89+
})
90+
}
91+
92+
module.exports=realpathCached
‎rpt.js‎
Lines changed: 81 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,17 @@
11
constfs=require('fs')
22
const{ promisify }=require('util')
3-
constrealpath=promisify(fs.realpath)
4-
const{ basename, dirname, join }=require('path')
3+
const{ resolve, basename, dirname, join }=require('path')
54
constrpj=promisify(require('read-package-json'))
65
constreaddir=promisify(require('readdir-scoped-modules'))
6+
constrealpath=require('./realpath.js')
77

88
letID=0
99
classNode{
1010
constructor(pkg,logical,physical,er,cache){
1111
// should be impossible.
12+
constcached=cache.get(physical)
1213
/* istanbul ignore next */
13-
if(cache.get(physical))
14+
if(cached&&!cached.then)
1415
thrownewError('re-creating already instantiated node')
1516

1617
cache.set(physical,this)
@@ -34,37 +35,84 @@ class Node {
3435
classLinkextendsNode{
3536
constructor(pkg,logical,physical,realpath,er,cache){
3637
super(pkg,logical,physical,er,cache)
38+
39+
// if the target has started, but not completed, then
40+
// a Promise will be in the cache to indicate this.
3741
constcachedTarget=cache.get(realpath)
42+
if(cachedTarget&&cachedTarget.then)
43+
cachedTarget.then(node=>this.target=node)
44+
3845
this.target=cachedTarget||newNode(pkg,logical,realpath,er,cache)
3946
this.realpath=realpath
4047
this.isLink=true
41-
this.children=this.target.children
4248
this.error=er
49+
// convenience method only
50+
/* istanbul ignore next */
51+
Object.defineProperty(this,'children',{
52+
get(){
53+
returnthis.target.children
54+
},
55+
set(c){
56+
this.target.children=c
57+
},
58+
enumerable: true
59+
})
4360
}
4461
}
4562

46-
constloadNode=(logical,physical,cache)=>newPromise((res,rej)=>{
47-
res(cache.get(physical)||realpath(physical)
48-
.then(real=>
49-
rpj(join(real,'package.json'))
50-
.then(pkg=>[real,pkg,null],er=>[real,null,er])
51-
.then(([real,pkg,er])=>
52-
physical===real ? newNode(pkg,logical,physical,er,cache)
53-
 : newLink(pkg,logical,physical,real,er,cache)
54-
),
55-
// if the realpath fails, don't bother with the rest
56-
er=>newNode(null,logical,physical,er,cache))
57-
)
58-
})
59-
60-
constloadChildren=(node,cache,filterWith)=>{
63+
// this is the way it is to expose a timing issue which is difficult to
64+
// test otherwise. The creation of a Node may take slightly longer than
65+
// the creation of a Link that targets it. If the Node has _begun_ its
66+
// creation phase (and put a Promise in the cache) then the Link will
67+
// get a Promise as its cachedTarget instead of an actual Node object.
68+
// This is not a problem, because it gets resolved prior to returning
69+
// the tree or attempting to load children. However, it IS remarkably
70+
// difficult to get to happen in a test environment to verify reliably.
71+
// Hence this kludge.
72+
constnewNode=(pkg,logical,physical,er,cache)=>
73+
process.env._TEST_RPT_SLOW_LINK_TARGET_==='1'
74+
 ? newPromise(res=>setTimeout(()=>
75+
res(newNode(pkg,logical,physical,er,cache)),10))
76+
 : newNode(pkg,logical,physical,er,cache)
77+
78+
constloadNode=(logical,physical,cache,rpcache,stcache)=>{
79+
// cache temporarily holds a promise placeholder so we
80+
// don't try to create the same node multiple times.
81+
// this is very rare to encounter, given the aggressive
82+
// caching on fs.realpath and fs.lstat calls, but
83+
// it can happen in theory.
84+
constcached=cache.get(physical)
85+
/* istanbul ignore next */
86+
if(cached)
87+
returnPromise.resolve(cached)
88+
89+
constp=realpath(physical,rpcache,stcache,0).then(real=>
90+
rpj(join(real,'package.json'))
91+
.then(pkg=>[pkg,null],er=>[null,er])
92+
.then(([pkg,er])=>
93+
physical===real ? newNode(pkg,logical,physical,er,cache)
94+
 : newLink(pkg,logical,physical,real,er,cache)
95+
),
96+
// if the realpath fails, don't bother with the rest
97+
er=>newNode(null,logical,physical,er,cache))
98+
99+
cache.set(physical,p)
100+
returnp
101+
}
102+
103+
constloadChildren=(node,cache,filterWith,rpcache,stcache)=>{
104+
// if a Link target has started, but not completed, then
105+
// a Promise will be in the cache to indicate this.
106+
if(node.then)
107+
returnnode.then(node=>loadChildren(node,cache,filterWith,rpcache,stcache))
108+
61109
constnm=join(node.path,'node_modules')
62-
returnrealpath(nm)
110+
returnrealpath(nm,rpcache,stcache,0)
63111
.then(rm=>readdir(rm).then(kids=>[rm,kids]))
64112
.then(([rm,kids])=>Promise.all(
65113
kids.filter(kid=>
66114
kid.charAt(0)!=='.'&&(!filterWith||filterWith(node,kid)))
67-
.map(kid=>loadNode(join(nm,kid),join(rm,kid),cache)))
115+
.map(kid=>loadNode(join(nm,kid),join(rm,kid),cache,rpcache,stcache)))
68116
).then(kidNodes=>{
69117
kidNodes.forEach(k=>k.parent=node)
70118
node.children=kidNodes.sort((a,b)=>
@@ -77,19 +125,20 @@ const loadChildren = (node, cache, filterWith) => {
77125
.catch(()=>node)
78126
}
79127

80-
constloadTree=(node,did,cache,filterWith)=>{
128+
constloadTree=(node,did,cache,filterWith,rpcache,stcache)=>{
81129
// impossible except in pathological ELOOP cases
82130
/* istanbul ignore next */
83131
if(did.has(node.realpath))
84132
returnPromise.resolve(node)
85133

86134
did.add(node.realpath)
87135

88-
returnloadChildren(node,cache,filterWith)
136+
// load children on the target, not the link
137+
returnloadChildren(node.target||node,cache,filterWith,rpcache,stcache)
89138
.then(node=>Promise.all(
90139
node.children
91140
.filter(kid=>!did.has(kid.realpath))
92-
.map(kid=>loadTree(kid,did,cache,filterWith))
141+
.map(kid=>loadTree(kid,did,cache,filterWith,rpcache,stcache))
93142
)).then(()=>node)
94143
}
95144

@@ -100,10 +149,15 @@ const rpt = (root, filterWith, cb) => {
100149
filterWith=null
101150
}
102151

152+
root=resolve(root)
103153
constcache=newMap()
104-
constp=realpath(root)
105-
.then(realRoot=>loadNode(root,realRoot,cache))
106-
.then(node=>loadTree(node,newSet(),cache,filterWith))
154+
// we can assume that the cwd is real enough
155+
constcwd=process.cwd()
156+
constrpcache=newMap([[cwd,cwd]])
157+
conststcache=newMap()
158+
constp=realpath(root,rpcache,stcache,0)
159+
.then(realRoot=>loadNode(root,realRoot,cache,rpcache,stcache))
160+
.then(node=>loadTree(node,newSet(),cache,filterWith,rpcache,stcache))
107161

108162
if(typeofcb==='function')
109163
p.then(tree=>cb(null,tree),cb)
‎tap-snapshots/test-basic.js-TAP.test.js‎
Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,20 @@ root@1.2.3 test/fixtures/linkedroot
4040
└── foo@1.2.3 test/fixtures/linkedroot/node_modules/foo
4141
`
4242

43+
exports[`test/basic.js TAP looking outside of cwd > must match snapshot 1`]=`
44+
root@1.2.3 test/fixtures/root
45+
├─┬ @scope/x@1.2.3 test/fixtures/root/node_modules/@scope/x
46+
│ └─┬ glob@4.0.5 test/fixtures/root/node_modules/@scope/x/node_modules/glob
47+
│ ├── graceful-fs@3.0.2 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/graceful-fs
48+
│ ├── inherits@2.0.1 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/inherits
49+
│ ├─┬ minimatch@1.0.0 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/minimatch
50+
│ │ ├── lru-cache@2.5.0 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/minimatch/node_modules/lru-cache
51+
│ │ └── sigmund@1.0.0 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/minimatch/node_modules/sigmund
52+
│ └── once@1.3.0 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/once
53+
├── @scope/y@1.2.3 test/fixtures/root/node_modules/@scope/y
54+
└── foo@1.2.3 test/fixtures/root/node_modules/foo
55+
`
56+
4357
exports[`test/basic.js TAP noname > noname tree 1`]=`
4458
test/fixtures/noname
4559
└── test/fixtures/noname/node_modules/foo
@@ -79,3 +93,19 @@ selflink@1.2.3 test/fixtures/selflink
7993
 │ └── once@1.3.0 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/once
8094
 └── selflink@1.2.3 test/fixtures/selflink (symlink)
8195
`
96+
97+
exports[`test/basic.js TAP shake out Link target timing issue > must match snapshot 1`]=`
98+
selflink@1.2.3 test/fixtures/selflink
99+
├── @scope/y@1.2.3 test/fixtures/selflink/node_modules/@scope/y
100+
├─┬ @scope/z@1.2.3 test/fixtures/selflink/node_modules/@scope/z
101+
│ └── glob@4.0.5 test/fixtures/selflink/node_modules/foo/node_modules/glob (symlink)
102+
└─┬ foo@1.2.3 test/fixtures/selflink/node_modules/foo
103+
 ├─┬ glob@4.0.5 test/fixtures/selflink/node_modules/foo/node_modules/glob
104+
 │ ├── graceful-fs@3.0.2 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/graceful-fs
105+
 │ ├── inherits@2.0.1 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/inherits
106+
 │ ├─┬ minimatch@1.0.0 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/minimatch
107+
 │ │ ├── lru-cache@2.5.0 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/minimatch/node_modules/lru-cache
108+
 │ │ └── sigmund@1.0.0 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/minimatch/node_modules/sigmund
109+
 │ └── once@1.3.0 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/once
110+
 └── selflink@1.2.3 test/fixtures/selflink (symlink)
111+
`
‎test/basic.js‎
Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,22 @@ test('filterWith', t =>
8585
).then(d=>t.matchSnapshot(archy(archyize(d)).trim()),'only 1 level deep')
8686
)
8787

88+
test('looking outside of cwd',t=>{
89+
constcwd=process.cwd()
90+
t.teardown(()=>process.chdir(cwd))
91+
process.chdir('test/fixtures/selflink')
92+
returnrpt('../root').then(d=>
93+
t.matchSnapshot(archy(archyize(d)).trim()))
94+
})
95+
96+
test('shake out Link target timing issue',t=>{
97+
process.env._TEST_RPT_SLOW_LINK_TARGET_='1'
98+
constcwd=process.cwd()
99+
t.teardown(()=>process.env._TEST_RPT_SLOW_LINK_TARGET_='')
100+
returnrpt(path.resolve(fixtures,'selflink')).then(d=>
101+
t.matchSnapshot(archy(archyize(d)).trim()))
102+
})
103+
88104
test('broken json',function(t){
89105
rpt(path.resolve(fixtures,'bad'),function(er,d){
90106
t.ok(d.error,'Got an error object')
@@ -152,6 +168,23 @@ function archyize (d, seen) {
152168
}
153169
}
154170

171+
test('realpath gutchecks',t=>{
172+
constd=path.resolve(cwd,'test/fixtures')
173+
constrealpath=require('../realpath.js')
174+
const{realpathSync}=fs
175+
Object.keys(symlinks).map(link=>t.test(link,t=>
176+
realpath(
177+
path.resolve(d,link),
178+
newMap(),
179+
newMap(),
180+
0
181+
).then(
182+
real=>t.equal(real,realpathSync(path.resolve(d,link))),
183+
er=>t.throws(()=>realpathSync(path.resolve(d,link)))
184+
)))
185+
t.end()
186+
})
187+
155188
test('cleanup',function(t){
156189
cleanup()
157190
t.end()
0 commit comments
Comments
 (0)

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Strip utm_, fbclid, gclid, etc. from all links on page
(function() {
var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',
'ref', 'ref_src', 'source', 'medium', 'campaign'];
function cleanUrl(url) {
try {
var u = new URL(url, window.location.origin);
var changed = false;
trackingParams.forEach(function(p) {
if (u.searchParams.has(p)) {
u.searchParams.delete(p);
changed = true;
}
});
return changed ? u.toString() : url;
} catch (e) {
return url;
}
}
function cleanLinks() {
document.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
cleanLinks();
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1) {
if (node.tagName === 'A') cleanLinks();
node.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '
Use custom cachable fs.realpath implementation · npm/read-package-tree@e9cd536 · GitHub
Skip to content

This repository was archived by the owner on Jan 7, 2022. It is now read-only.

Commit e9cd536
Browse files
isaacsisaacs
committed
Use custom cachable fs.realpath implementation
In this use case, we don't care much about a lot of the stuff that
fs.realpath can (and should!) do. The only thing that's relevant to
reading a package tree is whether package folders are symbolic links,
and if so, where they point.
Additionally, we don't need to re-start the fs.lstat party every time we
walk to a new directory. While it makes sense for fs.realpath to do
this in the general case, it's not required when reading a package tree,
and results in a geometric explosion of lstat syscalls. For example, if
a project is in /Users/hyooman/projects/company/website, and it has 1000
dependencies in node_modules, then a whopping 6,000 lstat calls will be
made just to repeatedly verify that
/Users/hyooman/projects/company/website/node_modules has not moved!
In this implementation, every realpath call is cached, as is every
lstat. Additionally, process.cwd() is assumed to be "real enough", and
added to the cache initially, which means almost never having to walk
all the way up to the root directory.
In the npm cli project, this drops the lstat count from 14885 to 3054
for a single call to read-package-tree on my system. Larger projects,
or projects deeper in a folder tree, will have even larger reductions.
This does not account, itself, for a particularly large speed-up, since
lstat calls do tend to be fairly fast, and the repetitiveness means that
there are a lot of hits in the file system's stat cache. But it does
make read-package-tree 10-30% faster in common use cases.
1 parent 4eed760 commit e9cd536
5 files changed
Lines changed: 238 additions & 28 deletions
File tree
‎package.json‎
Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,8 @@
3131
 },
3232
"homepage": "https://github.com/npm/read-package-tree",
3333
"files": [
34-
"rpt.js"
34+
"rpt.js",
35+
"realpath.js"
3536
 ],
3637
"tap": {
3738
"100": true
‎realpath.js‎
Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
// look up the realpath, but cache stats to minimize overhead
2+
// If the parent folder is in the realpath cache, then we just
3+
// lstat the child, since there's no need to do a full realpath
4+
// This is not a separate module, and is much simpler than Node's
5+
// built-in fs.realpath, because we only care about symbolic links,
6+
// so we can handle many fewer edge cases.
7+
8+
constfs=require('fs')
9+
const{ promisify }=require('util')
10+
constreadlink=promisify(fs.readlink)
11+
constlstat=promisify(fs.lstat)
12+
const{ resolve, basename, dirname }=require('path')
13+
14+
constrealpathCached=(path,rpcache,stcache,depth)=>{
15+
// just a safety against extremely deep eloops
16+
/* istanbul ignore next */
17+
if(depth>2000)
18+
throweloop(path)
19+
20+
if(rpcache.has(path))
21+
returnPromise.resolve(rpcache.get(path))
22+
23+
constdir=dirname(path)
24+
constbase=basename(path)
25+
26+
if(base&&rpcache.has(dir))
27+
returnrealpathChild(dir,base,rpcache,stcache,depth)
28+
29+
// if it's the root, then we know it's real
30+
if(!base){
31+
rpcache.set(dir,dir)
32+
returnPromise.resolve(dir)
33+
}
34+
35+
// the parent, what is that?
36+
// find out, and then come back.
37+
returnrealpathCached(dir,rpcache,stcache,depth+1).then(()=>
38+
realpathCached(path,rpcache,stcache,depth+1))
39+
}
40+
41+
constlstatCached=(path,stcache)=>{
42+
if(stcache.has(path))
43+
returnPromise.resolve(stcache.get(path))
44+
45+
constp=lstat(path).then(st=>{
46+
stcache.set(path,st)
47+
returnst
48+
})
49+
stcache.set(path,p)
50+
returnp
51+
}
52+
53+
// This is a slight fib, as it doesn't actually occur during a stat syscall.
54+
// But file systems are giant piles of lies, so whatever.
55+
consteloop=path=>
56+
Object.assign(newError(
57+
`ELOOP: too many symbolic links encountered, stat '${path}'`),{
58+
errno: -62,
59+
syscall: 'stat',
60+
code: 'ELOOP',
61+
path: path,
62+
})
63+
64+
constrealpathChild=(dir,base,rpcache,stcache,depth)=>{
65+
constrealdir=rpcache.get(dir)
66+
// that unpossible
67+
/* istanbul ignore next */
68+
if(typeofrealdir==='undefined')
69+
thrownewError('in realpathChild without parent being in realpath cache')
70+
71+
constrealish=resolve(realdir,base)
72+
returnlstatCached(realish,stcache).then(st=>{
73+
if(!st.isSymbolicLink()){
74+
rpcache.set(resolve(dir,base),realish)
75+
returnrealish
76+
}
77+
78+
letres
79+
returnreadlink(realish).then(target=>{
80+
constresolved=res=resolve(realdir,target)
81+
if(realish===resolved)
82+
throweloop(realish)
83+
84+
returnrealpathCached(resolved,rpcache,stcache,depth+1)
85+
}).then(real=>{
86+
rpcache.set(resolve(dir,base),real)
87+
returnreal
88+
})
89+
})
90+
}
91+
92+
module.exports=realpathCached
‎rpt.js‎
Lines changed: 81 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,17 @@
11
constfs=require('fs')
22
const{ promisify }=require('util')
3-
constrealpath=promisify(fs.realpath)
4-
const{ basename, dirname, join }=require('path')
3+
const{ resolve, basename, dirname, join }=require('path')
54
constrpj=promisify(require('read-package-json'))
65
constreaddir=promisify(require('readdir-scoped-modules'))
6+
constrealpath=require('./realpath.js')
77

88
letID=0
99
classNode{
1010
constructor(pkg,logical,physical,er,cache){
1111
// should be impossible.
12+
constcached=cache.get(physical)
1213
/* istanbul ignore next */
13-
if(cache.get(physical))
14+
if(cached&&!cached.then)
1415
thrownewError('re-creating already instantiated node')
1516

1617
cache.set(physical,this)
@@ -34,37 +35,84 @@ class Node {
3435
classLinkextendsNode{
3536
constructor(pkg,logical,physical,realpath,er,cache){
3637
super(pkg,logical,physical,er,cache)
38+
39+
// if the target has started, but not completed, then
40+
// a Promise will be in the cache to indicate this.
3741
constcachedTarget=cache.get(realpath)
42+
if(cachedTarget&&cachedTarget.then)
43+
cachedTarget.then(node=>this.target=node)
44+
3845
this.target=cachedTarget||newNode(pkg,logical,realpath,er,cache)
3946
this.realpath=realpath
4047
this.isLink=true
41-
this.children=this.target.children
4248
this.error=er
49+
// convenience method only
50+
/* istanbul ignore next */
51+
Object.defineProperty(this,'children',{
52+
get(){
53+
returnthis.target.children
54+
},
55+
set(c){
56+
this.target.children=c
57+
},
58+
enumerable: true
59+
})
4360
}
4461
}
4562

46-
constloadNode=(logical,physical,cache)=>newPromise((res,rej)=>{
47-
res(cache.get(physical)||realpath(physical)
48-
.then(real=>
49-
rpj(join(real,'package.json'))
50-
.then(pkg=>[real,pkg,null],er=>[real,null,er])
51-
.then(([real,pkg,er])=>
52-
physical===real ? newNode(pkg,logical,physical,er,cache)
53-
 : newLink(pkg,logical,physical,real,er,cache)
54-
),
55-
// if the realpath fails, don't bother with the rest
56-
er=>newNode(null,logical,physical,er,cache))
57-
)
58-
})
59-
60-
constloadChildren=(node,cache,filterWith)=>{
63+
// this is the way it is to expose a timing issue which is difficult to
64+
// test otherwise. The creation of a Node may take slightly longer than
65+
// the creation of a Link that targets it. If the Node has _begun_ its
66+
// creation phase (and put a Promise in the cache) then the Link will
67+
// get a Promise as its cachedTarget instead of an actual Node object.
68+
// This is not a problem, because it gets resolved prior to returning
69+
// the tree or attempting to load children. However, it IS remarkably
70+
// difficult to get to happen in a test environment to verify reliably.
71+
// Hence this kludge.
72+
constnewNode=(pkg,logical,physical,er,cache)=>
73+
process.env._TEST_RPT_SLOW_LINK_TARGET_==='1'
74+
 ? newPromise(res=>setTimeout(()=>
75+
res(newNode(pkg,logical,physical,er,cache)),10))
76+
 : newNode(pkg,logical,physical,er,cache)
77+
78+
constloadNode=(logical,physical,cache,rpcache,stcache)=>{
79+
// cache temporarily holds a promise placeholder so we
80+
// don't try to create the same node multiple times.
81+
// this is very rare to encounter, given the aggressive
82+
// caching on fs.realpath and fs.lstat calls, but
83+
// it can happen in theory.
84+
constcached=cache.get(physical)
85+
/* istanbul ignore next */
86+
if(cached)
87+
returnPromise.resolve(cached)
88+
89+
constp=realpath(physical,rpcache,stcache,0).then(real=>
90+
rpj(join(real,'package.json'))
91+
.then(pkg=>[pkg,null],er=>[null,er])
92+
.then(([pkg,er])=>
93+
physical===real ? newNode(pkg,logical,physical,er,cache)
94+
 : newLink(pkg,logical,physical,real,er,cache)
95+
),
96+
// if the realpath fails, don't bother with the rest
97+
er=>newNode(null,logical,physical,er,cache))
98+
99+
cache.set(physical,p)
100+
returnp
101+
}
102+
103+
constloadChildren=(node,cache,filterWith,rpcache,stcache)=>{
104+
// if a Link target has started, but not completed, then
105+
// a Promise will be in the cache to indicate this.
106+
if(node.then)
107+
returnnode.then(node=>loadChildren(node,cache,filterWith,rpcache,stcache))
108+
61109
constnm=join(node.path,'node_modules')
62-
returnrealpath(nm)
110+
returnrealpath(nm,rpcache,stcache,0)
63111
.then(rm=>readdir(rm).then(kids=>[rm,kids]))
64112
.then(([rm,kids])=>Promise.all(
65113
kids.filter(kid=>
66114
kid.charAt(0)!=='.'&&(!filterWith||filterWith(node,kid)))
67-
.map(kid=>loadNode(join(nm,kid),join(rm,kid),cache)))
115+
.map(kid=>loadNode(join(nm,kid),join(rm,kid),cache,rpcache,stcache)))
68116
).then(kidNodes=>{
69117
kidNodes.forEach(k=>k.parent=node)
70118
node.children=kidNodes.sort((a,b)=>
@@ -77,19 +125,20 @@ const loadChildren = (node, cache, filterWith) => {
77125
.catch(()=>node)
78126
}
79127

80-
constloadTree=(node,did,cache,filterWith)=>{
128+
constloadTree=(node,did,cache,filterWith,rpcache,stcache)=>{
81129
// impossible except in pathological ELOOP cases
82130
/* istanbul ignore next */
83131
if(did.has(node.realpath))
84132
returnPromise.resolve(node)
85133

86134
did.add(node.realpath)
87135

88-
returnloadChildren(node,cache,filterWith)
136+
// load children on the target, not the link
137+
returnloadChildren(node.target||node,cache,filterWith,rpcache,stcache)
89138
.then(node=>Promise.all(
90139
node.children
91140
.filter(kid=>!did.has(kid.realpath))
92-
.map(kid=>loadTree(kid,did,cache,filterWith))
141+
.map(kid=>loadTree(kid,did,cache,filterWith,rpcache,stcache))
93142
)).then(()=>node)
94143
}
95144

@@ -100,10 +149,15 @@ const rpt = (root, filterWith, cb) => {
100149
filterWith=null
101150
}
102151

152+
root=resolve(root)
103153
constcache=newMap()
104-
constp=realpath(root)
105-
.then(realRoot=>loadNode(root,realRoot,cache))
106-
.then(node=>loadTree(node,newSet(),cache,filterWith))
154+
// we can assume that the cwd is real enough
155+
constcwd=process.cwd()
156+
constrpcache=newMap([[cwd,cwd]])
157+
conststcache=newMap()
158+
constp=realpath(root,rpcache,stcache,0)
159+
.then(realRoot=>loadNode(root,realRoot,cache,rpcache,stcache))
160+
.then(node=>loadTree(node,newSet(),cache,filterWith,rpcache,stcache))
107161

108162
if(typeofcb==='function')
109163
p.then(tree=>cb(null,tree),cb)
‎tap-snapshots/test-basic.js-TAP.test.js‎
Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,20 @@ root@1.2.3 test/fixtures/linkedroot
4040
└── foo@1.2.3 test/fixtures/linkedroot/node_modules/foo
4141
`
4242

43+
exports[`test/basic.js TAP looking outside of cwd > must match snapshot 1`]=`
44+
root@1.2.3 test/fixtures/root
45+
├─┬ @scope/x@1.2.3 test/fixtures/root/node_modules/@scope/x
46+
│ └─┬ glob@4.0.5 test/fixtures/root/node_modules/@scope/x/node_modules/glob
47+
│ ├── graceful-fs@3.0.2 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/graceful-fs
48+
│ ├── inherits@2.0.1 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/inherits
49+
│ ├─┬ minimatch@1.0.0 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/minimatch
50+
│ │ ├── lru-cache@2.5.0 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/minimatch/node_modules/lru-cache
51+
│ │ └── sigmund@1.0.0 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/minimatch/node_modules/sigmund
52+
│ └── once@1.3.0 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/once
53+
├── @scope/y@1.2.3 test/fixtures/root/node_modules/@scope/y
54+
└── foo@1.2.3 test/fixtures/root/node_modules/foo
55+
`
56+
4357
exports[`test/basic.js TAP noname > noname tree 1`]=`
4458
test/fixtures/noname
4559
└── test/fixtures/noname/node_modules/foo
@@ -79,3 +93,19 @@ selflink@1.2.3 test/fixtures/selflink
7993
 │ └── once@1.3.0 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/once
8094
 └── selflink@1.2.3 test/fixtures/selflink (symlink)
8195
`
96+
97+
exports[`test/basic.js TAP shake out Link target timing issue > must match snapshot 1`]=`
98+
selflink@1.2.3 test/fixtures/selflink
99+
├── @scope/y@1.2.3 test/fixtures/selflink/node_modules/@scope/y
100+
├─┬ @scope/z@1.2.3 test/fixtures/selflink/node_modules/@scope/z
101+
│ └── glob@4.0.5 test/fixtures/selflink/node_modules/foo/node_modules/glob (symlink)
102+
└─┬ foo@1.2.3 test/fixtures/selflink/node_modules/foo
103+
 ├─┬ glob@4.0.5 test/fixtures/selflink/node_modules/foo/node_modules/glob
104+
 │ ├── graceful-fs@3.0.2 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/graceful-fs
105+
 │ ├── inherits@2.0.1 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/inherits
106+
 │ ├─┬ minimatch@1.0.0 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/minimatch
107+
 │ │ ├── lru-cache@2.5.0 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/minimatch/node_modules/lru-cache
108+
 │ │ └── sigmund@1.0.0 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/minimatch/node_modules/sigmund
109+
 │ └── once@1.3.0 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/once
110+
 └── selflink@1.2.3 test/fixtures/selflink (symlink)
111+
`
‎test/basic.js‎
Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,22 @@ test('filterWith', t =>
8585
).then(d=>t.matchSnapshot(archy(archyize(d)).trim()),'only 1 level deep')
8686
)
8787

88+
test('looking outside of cwd',t=>{
89+
constcwd=process.cwd()
90+
t.teardown(()=>process.chdir(cwd))
91+
process.chdir('test/fixtures/selflink')
92+
returnrpt('../root').then(d=>
93+
t.matchSnapshot(archy(archyize(d)).trim()))
94+
})
95+
96+
test('shake out Link target timing issue',t=>{
97+
process.env._TEST_RPT_SLOW_LINK_TARGET_='1'
98+
constcwd=process.cwd()
99+
t.teardown(()=>process.env._TEST_RPT_SLOW_LINK_TARGET_='')
100+
returnrpt(path.resolve(fixtures,'selflink')).then(d=>
101+
t.matchSnapshot(archy(archyize(d)).trim()))
102+
})
103+
88104
test('broken json',function(t){
89105
rpt(path.resolve(fixtures,'bad'),function(er,d){
90106
t.ok(d.error,'Got an error object')
@@ -152,6 +168,23 @@ function archyize (d, seen) {
152168
}
153169
}
154170

171+
test('realpath gutchecks',t=>{
172+
constd=path.resolve(cwd,'test/fixtures')
173+
constrealpath=require('../realpath.js')
174+
const{realpathSync}=fs
175+
Object.keys(symlinks).map(link=>t.test(link,t=>
176+
realpath(
177+
path.resolve(d,link),
178+
newMap(),
179+
newMap(),
180+
0
181+
).then(
182+
real=>t.equal(real,realpathSync(path.resolve(d,link))),
183+
er=>t.throws(()=>realpathSync(path.resolve(d,link)))
184+
)))
185+
t.end()
186+
})
187+
155188
test('cleanup',function(t){
156189
cleanup()
157190
t.end()
0 commit comments
Comments
 (0)

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Auto-enable theater mode on YouTube
(function() {
function tryTheater() {
var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]');
if (btn && !btn.classList.contains('activated')) {
btn.click();
}
}
// Try immediately
tryTheater();
// Try after navigation (SPA)
var lastUrl = location.href;
setInterval(function() {
if (location.href !== lastUrl) {
lastUrl = location.href;
setTimeout(tryTheater, 500);
}
}, 1000);
// Also try on player load
var observer = new MutationObserver(tryTheater);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
Use custom cachable fs.realpath implementation · npm/read-package-tree@e9cd536 · GitHub
Skip to content

This repository was archived by the owner on Jan 7, 2022. It is now read-only.

Commit e9cd536
Browse files
isaacsisaacs
committed
Use custom cachable fs.realpath implementation
In this use case, we don't care much about a lot of the stuff that
fs.realpath can (and should!) do. The only thing that's relevant to
reading a package tree is whether package folders are symbolic links,
and if so, where they point.
Additionally, we don't need to re-start the fs.lstat party every time we
walk to a new directory. While it makes sense for fs.realpath to do
this in the general case, it's not required when reading a package tree,
and results in a geometric explosion of lstat syscalls. For example, if
a project is in /Users/hyooman/projects/company/website, and it has 1000
dependencies in node_modules, then a whopping 6,000 lstat calls will be
made just to repeatedly verify that
/Users/hyooman/projects/company/website/node_modules has not moved!
In this implementation, every realpath call is cached, as is every
lstat. Additionally, process.cwd() is assumed to be "real enough", and
added to the cache initially, which means almost never having to walk
all the way up to the root directory.
In the npm cli project, this drops the lstat count from 14885 to 3054
for a single call to read-package-tree on my system. Larger projects,
or projects deeper in a folder tree, will have even larger reductions.
This does not account, itself, for a particularly large speed-up, since
lstat calls do tend to be fairly fast, and the repetitiveness means that
there are a lot of hits in the file system's stat cache. But it does
make read-package-tree 10-30% faster in common use cases.
1 parent 4eed760 commit e9cd536
5 files changed
Lines changed: 238 additions & 28 deletions
File tree
‎package.json‎
Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,8 @@
3131
 },
3232
"homepage": "https://github.com/npm/read-package-tree",
3333
"files": [
34-
"rpt.js"
34+
"rpt.js",
35+
"realpath.js"
3536
 ],
3637
"tap": {
3738
"100": true
‎realpath.js‎
Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
// look up the realpath, but cache stats to minimize overhead
2+
// If the parent folder is in the realpath cache, then we just
3+
// lstat the child, since there's no need to do a full realpath
4+
// This is not a separate module, and is much simpler than Node's
5+
// built-in fs.realpath, because we only care about symbolic links,
6+
// so we can handle many fewer edge cases.
7+
8+
constfs=require('fs')
9+
const{ promisify }=require('util')
10+
constreadlink=promisify(fs.readlink)
11+
constlstat=promisify(fs.lstat)
12+
const{ resolve, basename, dirname }=require('path')
13+
14+
constrealpathCached=(path,rpcache,stcache,depth)=>{
15+
// just a safety against extremely deep eloops
16+
/* istanbul ignore next */
17+
if(depth>2000)
18+
throweloop(path)
19+
20+
if(rpcache.has(path))
21+
returnPromise.resolve(rpcache.get(path))
22+
23+
constdir=dirname(path)
24+
constbase=basename(path)
25+
26+
if(base&&rpcache.has(dir))
27+
returnrealpathChild(dir,base,rpcache,stcache,depth)
28+
29+
// if it's the root, then we know it's real
30+
if(!base){
31+
rpcache.set(dir,dir)
32+
returnPromise.resolve(dir)
33+
}
34+
35+
// the parent, what is that?
36+
// find out, and then come back.
37+
returnrealpathCached(dir,rpcache,stcache,depth+1).then(()=>
38+
realpathCached(path,rpcache,stcache,depth+1))
39+
}
40+
41+
constlstatCached=(path,stcache)=>{
42+
if(stcache.has(path))
43+
returnPromise.resolve(stcache.get(path))
44+
45+
constp=lstat(path).then(st=>{
46+
stcache.set(path,st)
47+
returnst
48+
})
49+
stcache.set(path,p)
50+
returnp
51+
}
52+
53+
// This is a slight fib, as it doesn't actually occur during a stat syscall.
54+
// But file systems are giant piles of lies, so whatever.
55+
consteloop=path=>
56+
Object.assign(newError(
57+
`ELOOP: too many symbolic links encountered, stat '${path}'`),{
58+
errno: -62,
59+
syscall: 'stat',
60+
code: 'ELOOP',
61+
path: path,
62+
})
63+
64+
constrealpathChild=(dir,base,rpcache,stcache,depth)=>{
65+
constrealdir=rpcache.get(dir)
66+
// that unpossible
67+
/* istanbul ignore next */
68+
if(typeofrealdir==='undefined')
69+
thrownewError('in realpathChild without parent being in realpath cache')
70+
71+
constrealish=resolve(realdir,base)
72+
returnlstatCached(realish,stcache).then(st=>{
73+
if(!st.isSymbolicLink()){
74+
rpcache.set(resolve(dir,base),realish)
75+
returnrealish
76+
}
77+
78+
letres
79+
returnreadlink(realish).then(target=>{
80+
constresolved=res=resolve(realdir,target)
81+
if(realish===resolved)
82+
throweloop(realish)
83+
84+
returnrealpathCached(resolved,rpcache,stcache,depth+1)
85+
}).then(real=>{
86+
rpcache.set(resolve(dir,base),real)
87+
returnreal
88+
})
89+
})
90+
}
91+
92+
module.exports=realpathCached
‎rpt.js‎
Lines changed: 81 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,17 @@
11
constfs=require('fs')
22
const{ promisify }=require('util')
3-
constrealpath=promisify(fs.realpath)
4-
const{ basename, dirname, join }=require('path')
3+
const{ resolve, basename, dirname, join }=require('path')
54
constrpj=promisify(require('read-package-json'))
65
constreaddir=promisify(require('readdir-scoped-modules'))
6+
constrealpath=require('./realpath.js')
77

88
letID=0
99
classNode{
1010
constructor(pkg,logical,physical,er,cache){
1111
// should be impossible.
12+
constcached=cache.get(physical)
1213
/* istanbul ignore next */
13-
if(cache.get(physical))
14+
if(cached&&!cached.then)
1415
thrownewError('re-creating already instantiated node')
1516

1617
cache.set(physical,this)
@@ -34,37 +35,84 @@ class Node {
3435
classLinkextendsNode{
3536
constructor(pkg,logical,physical,realpath,er,cache){
3637
super(pkg,logical,physical,er,cache)
38+
39+
// if the target has started, but not completed, then
40+
// a Promise will be in the cache to indicate this.
3741
constcachedTarget=cache.get(realpath)
42+
if(cachedTarget&&cachedTarget.then)
43+
cachedTarget.then(node=>this.target=node)
44+
3845
this.target=cachedTarget||newNode(pkg,logical,realpath,er,cache)
3946
this.realpath=realpath
4047
this.isLink=true
41-
this.children=this.target.children
4248
this.error=er
49+
// convenience method only
50+
/* istanbul ignore next */
51+
Object.defineProperty(this,'children',{
52+
get(){
53+
returnthis.target.children
54+
},
55+
set(c){
56+
this.target.children=c
57+
},
58+
enumerable: true
59+
})
4360
}
4461
}
4562

46-
constloadNode=(logical,physical,cache)=>newPromise((res,rej)=>{
47-
res(cache.get(physical)||realpath(physical)
48-
.then(real=>
49-
rpj(join(real,'package.json'))
50-
.then(pkg=>[real,pkg,null],er=>[real,null,er])
51-
.then(([real,pkg,er])=>
52-
physical===real ? newNode(pkg,logical,physical,er,cache)
53-
 : newLink(pkg,logical,physical,real,er,cache)
54-
),
55-
// if the realpath fails, don't bother with the rest
56-
er=>newNode(null,logical,physical,er,cache))
57-
)
58-
})
59-
60-
constloadChildren=(node,cache,filterWith)=>{
63+
// this is the way it is to expose a timing issue which is difficult to
64+
// test otherwise. The creation of a Node may take slightly longer than
65+
// the creation of a Link that targets it. If the Node has _begun_ its
66+
// creation phase (and put a Promise in the cache) then the Link will
67+
// get a Promise as its cachedTarget instead of an actual Node object.
68+
// This is not a problem, because it gets resolved prior to returning
69+
// the tree or attempting to load children. However, it IS remarkably
70+
// difficult to get to happen in a test environment to verify reliably.
71+
// Hence this kludge.
72+
constnewNode=(pkg,logical,physical,er,cache)=>
73+
process.env._TEST_RPT_SLOW_LINK_TARGET_==='1'
74+
 ? newPromise(res=>setTimeout(()=>
75+
res(newNode(pkg,logical,physical,er,cache)),10))
76+
 : newNode(pkg,logical,physical,er,cache)
77+
78+
constloadNode=(logical,physical,cache,rpcache,stcache)=>{
79+
// cache temporarily holds a promise placeholder so we
80+
// don't try to create the same node multiple times.
81+
// this is very rare to encounter, given the aggressive
82+
// caching on fs.realpath and fs.lstat calls, but
83+
// it can happen in theory.
84+
constcached=cache.get(physical)
85+
/* istanbul ignore next */
86+
if(cached)
87+
returnPromise.resolve(cached)
88+
89+
constp=realpath(physical,rpcache,stcache,0).then(real=>
90+
rpj(join(real,'package.json'))
91+
.then(pkg=>[pkg,null],er=>[null,er])
92+
.then(([pkg,er])=>
93+
physical===real ? newNode(pkg,logical,physical,er,cache)
94+
 : newLink(pkg,logical,physical,real,er,cache)
95+
),
96+
// if the realpath fails, don't bother with the rest
97+
er=>newNode(null,logical,physical,er,cache))
98+
99+
cache.set(physical,p)
100+
returnp
101+
}
102+
103+
constloadChildren=(node,cache,filterWith,rpcache,stcache)=>{
104+
// if a Link target has started, but not completed, then
105+
// a Promise will be in the cache to indicate this.
106+
if(node.then)
107+
returnnode.then(node=>loadChildren(node,cache,filterWith,rpcache,stcache))
108+
61109
constnm=join(node.path,'node_modules')
62-
returnrealpath(nm)
110+
returnrealpath(nm,rpcache,stcache,0)
63111
.then(rm=>readdir(rm).then(kids=>[rm,kids]))
64112
.then(([rm,kids])=>Promise.all(
65113
kids.filter(kid=>
66114
kid.charAt(0)!=='.'&&(!filterWith||filterWith(node,kid)))
67-
.map(kid=>loadNode(join(nm,kid),join(rm,kid),cache)))
115+
.map(kid=>loadNode(join(nm,kid),join(rm,kid),cache,rpcache,stcache)))
68116
).then(kidNodes=>{
69117
kidNodes.forEach(k=>k.parent=node)
70118
node.children=kidNodes.sort((a,b)=>
@@ -77,19 +125,20 @@ const loadChildren = (node, cache, filterWith) => {
77125
.catch(()=>node)
78126
}
79127

80-
constloadTree=(node,did,cache,filterWith)=>{
128+
constloadTree=(node,did,cache,filterWith,rpcache,stcache)=>{
81129
// impossible except in pathological ELOOP cases
82130
/* istanbul ignore next */
83131
if(did.has(node.realpath))
84132
returnPromise.resolve(node)
85133

86134
did.add(node.realpath)
87135

88-
returnloadChildren(node,cache,filterWith)
136+
// load children on the target, not the link
137+
returnloadChildren(node.target||node,cache,filterWith,rpcache,stcache)
89138
.then(node=>Promise.all(
90139
node.children
91140
.filter(kid=>!did.has(kid.realpath))
92-
.map(kid=>loadTree(kid,did,cache,filterWith))
141+
.map(kid=>loadTree(kid,did,cache,filterWith,rpcache,stcache))
93142
)).then(()=>node)
94143
}
95144

@@ -100,10 +149,15 @@ const rpt = (root, filterWith, cb) => {
100149
filterWith=null
101150
}
102151

152+
root=resolve(root)
103153
constcache=newMap()
104-
constp=realpath(root)
105-
.then(realRoot=>loadNode(root,realRoot,cache))
106-
.then(node=>loadTree(node,newSet(),cache,filterWith))
154+
// we can assume that the cwd is real enough
155+
constcwd=process.cwd()
156+
constrpcache=newMap([[cwd,cwd]])
157+
conststcache=newMap()
158+
constp=realpath(root,rpcache,stcache,0)
159+
.then(realRoot=>loadNode(root,realRoot,cache,rpcache,stcache))
160+
.then(node=>loadTree(node,newSet(),cache,filterWith,rpcache,stcache))
107161

108162
if(typeofcb==='function')
109163
p.then(tree=>cb(null,tree),cb)
‎tap-snapshots/test-basic.js-TAP.test.js‎
Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,20 @@ root@1.2.3 test/fixtures/linkedroot
4040
└── foo@1.2.3 test/fixtures/linkedroot/node_modules/foo
4141
`
4242

43+
exports[`test/basic.js TAP looking outside of cwd > must match snapshot 1`]=`
44+
root@1.2.3 test/fixtures/root
45+
├─┬ @scope/x@1.2.3 test/fixtures/root/node_modules/@scope/x
46+
│ └─┬ glob@4.0.5 test/fixtures/root/node_modules/@scope/x/node_modules/glob
47+
│ ├── graceful-fs@3.0.2 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/graceful-fs
48+
│ ├── inherits@2.0.1 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/inherits
49+
│ ├─┬ minimatch@1.0.0 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/minimatch
50+
│ │ ├── lru-cache@2.5.0 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/minimatch/node_modules/lru-cache
51+
│ │ └── sigmund@1.0.0 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/minimatch/node_modules/sigmund
52+
│ └── once@1.3.0 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/once
53+
├── @scope/y@1.2.3 test/fixtures/root/node_modules/@scope/y
54+
└── foo@1.2.3 test/fixtures/root/node_modules/foo
55+
`
56+
4357
exports[`test/basic.js TAP noname > noname tree 1`]=`
4458
test/fixtures/noname
4559
└── test/fixtures/noname/node_modules/foo
@@ -79,3 +93,19 @@ selflink@1.2.3 test/fixtures/selflink
7993
 │ └── once@1.3.0 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/once
8094
 └── selflink@1.2.3 test/fixtures/selflink (symlink)
8195
`
96+
97+
exports[`test/basic.js TAP shake out Link target timing issue > must match snapshot 1`]=`
98+
selflink@1.2.3 test/fixtures/selflink
99+
├── @scope/y@1.2.3 test/fixtures/selflink/node_modules/@scope/y
100+
├─┬ @scope/z@1.2.3 test/fixtures/selflink/node_modules/@scope/z
101+
│ └── glob@4.0.5 test/fixtures/selflink/node_modules/foo/node_modules/glob (symlink)
102+
└─┬ foo@1.2.3 test/fixtures/selflink/node_modules/foo
103+
 ├─┬ glob@4.0.5 test/fixtures/selflink/node_modules/foo/node_modules/glob
104+
 │ ├── graceful-fs@3.0.2 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/graceful-fs
105+
 │ ├── inherits@2.0.1 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/inherits
106+
 │ ├─┬ minimatch@1.0.0 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/minimatch
107+
 │ │ ├── lru-cache@2.5.0 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/minimatch/node_modules/lru-cache
108+
 │ │ └── sigmund@1.0.0 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/minimatch/node_modules/sigmund
109+
 │ └── once@1.3.0 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/once
110+
 └── selflink@1.2.3 test/fixtures/selflink (symlink)
111+
`
‎test/basic.js‎
Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,22 @@ test('filterWith', t =>
8585
).then(d=>t.matchSnapshot(archy(archyize(d)).trim()),'only 1 level deep')
8686
)
8787

88+
test('looking outside of cwd',t=>{
89+
constcwd=process.cwd()
90+
t.teardown(()=>process.chdir(cwd))
91+
process.chdir('test/fixtures/selflink')
92+
returnrpt('../root').then(d=>
93+
t.matchSnapshot(archy(archyize(d)).trim()))
94+
})
95+
96+
test('shake out Link target timing issue',t=>{
97+
process.env._TEST_RPT_SLOW_LINK_TARGET_='1'
98+
constcwd=process.cwd()
99+
t.teardown(()=>process.env._TEST_RPT_SLOW_LINK_TARGET_='')
100+
returnrpt(path.resolve(fixtures,'selflink')).then(d=>
101+
t.matchSnapshot(archy(archyize(d)).trim()))
102+
})
103+
88104
test('broken json',function(t){
89105
rpt(path.resolve(fixtures,'bad'),function(er,d){
90106
t.ok(d.error,'Got an error object')
@@ -152,6 +168,23 @@ function archyize (d, seen) {
152168
}
153169
}
154170

171+
test('realpath gutchecks',t=>{
172+
constd=path.resolve(cwd,'test/fixtures')
173+
constrealpath=require('../realpath.js')
174+
const{realpathSync}=fs
175+
Object.keys(symlinks).map(link=>t.test(link,t=>
176+
realpath(
177+
path.resolve(d,link),
178+
newMap(),
179+
newMap(),
180+
0
181+
).then(
182+
real=>t.equal(real,realpathSync(path.resolve(d,link))),
183+
er=>t.throws(()=>realpathSync(path.resolve(d,link)))
184+
)))
185+
t.end()
186+
})
187+
155188
test('cleanup',function(t){
156189
cleanup()
157190
t.end()
0 commit comments
Comments
 (0)

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Remove or un-stick sticky/fixed headers that block content
(function() {
function unstick() {
document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) {
if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {
el.style.position = 'static';
el.style.top = 'auto';
el.style.zIndex = 'auto';
}
});
}
unstick();
var observer = new MutationObserver(unstick);
observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });
})();
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
Use custom cachable fs.realpath implementation · npm/read-package-tree@e9cd536 · GitHub
Skip to content

This repository was archived by the owner on Jan 7, 2022. It is now read-only.

Commit e9cd536
Browse files
isaacsisaacs
committed
Use custom cachable fs.realpath implementation
In this use case, we don't care much about a lot of the stuff that
fs.realpath can (and should!) do. The only thing that's relevant to
reading a package tree is whether package folders are symbolic links,
and if so, where they point.
Additionally, we don't need to re-start the fs.lstat party every time we
walk to a new directory. While it makes sense for fs.realpath to do
this in the general case, it's not required when reading a package tree,
and results in a geometric explosion of lstat syscalls. For example, if
a project is in /Users/hyooman/projects/company/website, and it has 1000
dependencies in node_modules, then a whopping 6,000 lstat calls will be
made just to repeatedly verify that
/Users/hyooman/projects/company/website/node_modules has not moved!
In this implementation, every realpath call is cached, as is every
lstat. Additionally, process.cwd() is assumed to be "real enough", and
added to the cache initially, which means almost never having to walk
all the way up to the root directory.
In the npm cli project, this drops the lstat count from 14885 to 3054
for a single call to read-package-tree on my system. Larger projects,
or projects deeper in a folder tree, will have even larger reductions.
This does not account, itself, for a particularly large speed-up, since
lstat calls do tend to be fairly fast, and the repetitiveness means that
there are a lot of hits in the file system's stat cache. But it does
make read-package-tree 10-30% faster in common use cases.
1 parent 4eed760 commit e9cd536
5 files changed
Lines changed: 238 additions & 28 deletions
File tree
‎package.json‎
Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,8 @@
3131
 },
3232
"homepage": "https://github.com/npm/read-package-tree",
3333
"files": [
34-
"rpt.js"
34+
"rpt.js",
35+
"realpath.js"
3536
 ],
3637
"tap": {
3738
"100": true
‎realpath.js‎
Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
// look up the realpath, but cache stats to minimize overhead
2+
// If the parent folder is in the realpath cache, then we just
3+
// lstat the child, since there's no need to do a full realpath
4+
// This is not a separate module, and is much simpler than Node's
5+
// built-in fs.realpath, because we only care about symbolic links,
6+
// so we can handle many fewer edge cases.
7+
8+
constfs=require('fs')
9+
const{ promisify }=require('util')
10+
constreadlink=promisify(fs.readlink)
11+
constlstat=promisify(fs.lstat)
12+
const{ resolve, basename, dirname }=require('path')
13+
14+
constrealpathCached=(path,rpcache,stcache,depth)=>{
15+
// just a safety against extremely deep eloops
16+
/* istanbul ignore next */
17+
if(depth>2000)
18+
throweloop(path)
19+
20+
if(rpcache.has(path))
21+
returnPromise.resolve(rpcache.get(path))
22+
23+
constdir=dirname(path)
24+
constbase=basename(path)
25+
26+
if(base&&rpcache.has(dir))
27+
returnrealpathChild(dir,base,rpcache,stcache,depth)
28+
29+
// if it's the root, then we know it's real
30+
if(!base){
31+
rpcache.set(dir,dir)
32+
returnPromise.resolve(dir)
33+
}
34+
35+
// the parent, what is that?
36+
// find out, and then come back.
37+
returnrealpathCached(dir,rpcache,stcache,depth+1).then(()=>
38+
realpathCached(path,rpcache,stcache,depth+1))
39+
}
40+
41+
constlstatCached=(path,stcache)=>{
42+
if(stcache.has(path))
43+
returnPromise.resolve(stcache.get(path))
44+
45+
constp=lstat(path).then(st=>{
46+
stcache.set(path,st)
47+
returnst
48+
})
49+
stcache.set(path,p)
50+
returnp
51+
}
52+
53+
// This is a slight fib, as it doesn't actually occur during a stat syscall.
54+
// But file systems are giant piles of lies, so whatever.
55+
consteloop=path=>
56+
Object.assign(newError(
57+
`ELOOP: too many symbolic links encountered, stat '${path}'`),{
58+
errno: -62,
59+
syscall: 'stat',
60+
code: 'ELOOP',
61+
path: path,
62+
})
63+
64+
constrealpathChild=(dir,base,rpcache,stcache,depth)=>{
65+
constrealdir=rpcache.get(dir)
66+
// that unpossible
67+
/* istanbul ignore next */
68+
if(typeofrealdir==='undefined')
69+
thrownewError('in realpathChild without parent being in realpath cache')
70+
71+
constrealish=resolve(realdir,base)
72+
returnlstatCached(realish,stcache).then(st=>{
73+
if(!st.isSymbolicLink()){
74+
rpcache.set(resolve(dir,base),realish)
75+
returnrealish
76+
}
77+
78+
letres
79+
returnreadlink(realish).then(target=>{
80+
constresolved=res=resolve(realdir,target)
81+
if(realish===resolved)
82+
throweloop(realish)
83+
84+
returnrealpathCached(resolved,rpcache,stcache,depth+1)
85+
}).then(real=>{
86+
rpcache.set(resolve(dir,base),real)
87+
returnreal
88+
})
89+
})
90+
}
91+
92+
module.exports=realpathCached
‎rpt.js‎
Lines changed: 81 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,17 @@
11
constfs=require('fs')
22
const{ promisify }=require('util')
3-
constrealpath=promisify(fs.realpath)
4-
const{ basename, dirname, join }=require('path')
3+
const{ resolve, basename, dirname, join }=require('path')
54
constrpj=promisify(require('read-package-json'))
65
constreaddir=promisify(require('readdir-scoped-modules'))
6+
constrealpath=require('./realpath.js')
77

88
letID=0
99
classNode{
1010
constructor(pkg,logical,physical,er,cache){
1111
// should be impossible.
12+
constcached=cache.get(physical)
1213
/* istanbul ignore next */
13-
if(cache.get(physical))
14+
if(cached&&!cached.then)
1415
thrownewError('re-creating already instantiated node')
1516

1617
cache.set(physical,this)
@@ -34,37 +35,84 @@ class Node {
3435
classLinkextendsNode{
3536
constructor(pkg,logical,physical,realpath,er,cache){
3637
super(pkg,logical,physical,er,cache)
38+
39+
// if the target has started, but not completed, then
40+
// a Promise will be in the cache to indicate this.
3741
constcachedTarget=cache.get(realpath)
42+
if(cachedTarget&&cachedTarget.then)
43+
cachedTarget.then(node=>this.target=node)
44+
3845
this.target=cachedTarget||newNode(pkg,logical,realpath,er,cache)
3946
this.realpath=realpath
4047
this.isLink=true
41-
this.children=this.target.children
4248
this.error=er
49+
// convenience method only
50+
/* istanbul ignore next */
51+
Object.defineProperty(this,'children',{
52+
get(){
53+
returnthis.target.children
54+
},
55+
set(c){
56+
this.target.children=c
57+
},
58+
enumerable: true
59+
})
4360
}
4461
}
4562

46-
constloadNode=(logical,physical,cache)=>newPromise((res,rej)=>{
47-
res(cache.get(physical)||realpath(physical)
48-
.then(real=>
49-
rpj(join(real,'package.json'))
50-
.then(pkg=>[real,pkg,null],er=>[real,null,er])
51-
.then(([real,pkg,er])=>
52-
physical===real ? newNode(pkg,logical,physical,er,cache)
53-
 : newLink(pkg,logical,physical,real,er,cache)
54-
),
55-
// if the realpath fails, don't bother with the rest
56-
er=>newNode(null,logical,physical,er,cache))
57-
)
58-
})
59-
60-
constloadChildren=(node,cache,filterWith)=>{
63+
// this is the way it is to expose a timing issue which is difficult to
64+
// test otherwise. The creation of a Node may take slightly longer than
65+
// the creation of a Link that targets it. If the Node has _begun_ its
66+
// creation phase (and put a Promise in the cache) then the Link will
67+
// get a Promise as its cachedTarget instead of an actual Node object.
68+
// This is not a problem, because it gets resolved prior to returning
69+
// the tree or attempting to load children. However, it IS remarkably
70+
// difficult to get to happen in a test environment to verify reliably.
71+
// Hence this kludge.
72+
constnewNode=(pkg,logical,physical,er,cache)=>
73+
process.env._TEST_RPT_SLOW_LINK_TARGET_==='1'
74+
 ? newPromise(res=>setTimeout(()=>
75+
res(newNode(pkg,logical,physical,er,cache)),10))
76+
 : newNode(pkg,logical,physical,er,cache)
77+
78+
constloadNode=(logical,physical,cache,rpcache,stcache)=>{
79+
// cache temporarily holds a promise placeholder so we
80+
// don't try to create the same node multiple times.
81+
// this is very rare to encounter, given the aggressive
82+
// caching on fs.realpath and fs.lstat calls, but
83+
// it can happen in theory.
84+
constcached=cache.get(physical)
85+
/* istanbul ignore next */
86+
if(cached)
87+
returnPromise.resolve(cached)
88+
89+
constp=realpath(physical,rpcache,stcache,0).then(real=>
90+
rpj(join(real,'package.json'))
91+
.then(pkg=>[pkg,null],er=>[null,er])
92+
.then(([pkg,er])=>
93+
physical===real ? newNode(pkg,logical,physical,er,cache)
94+
 : newLink(pkg,logical,physical,real,er,cache)
95+
),
96+
// if the realpath fails, don't bother with the rest
97+
er=>newNode(null,logical,physical,er,cache))
98+
99+
cache.set(physical,p)
100+
returnp
101+
}
102+
103+
constloadChildren=(node,cache,filterWith,rpcache,stcache)=>{
104+
// if a Link target has started, but not completed, then
105+
// a Promise will be in the cache to indicate this.
106+
if(node.then)
107+
returnnode.then(node=>loadChildren(node,cache,filterWith,rpcache,stcache))
108+
61109
constnm=join(node.path,'node_modules')
62-
returnrealpath(nm)
110+
returnrealpath(nm,rpcache,stcache,0)
63111
.then(rm=>readdir(rm).then(kids=>[rm,kids]))
64112
.then(([rm,kids])=>Promise.all(
65113
kids.filter(kid=>
66114
kid.charAt(0)!=='.'&&(!filterWith||filterWith(node,kid)))
67-
.map(kid=>loadNode(join(nm,kid),join(rm,kid),cache)))
115+
.map(kid=>loadNode(join(nm,kid),join(rm,kid),cache,rpcache,stcache)))
68116
).then(kidNodes=>{
69117
kidNodes.forEach(k=>k.parent=node)
70118
node.children=kidNodes.sort((a,b)=>
@@ -77,19 +125,20 @@ const loadChildren = (node, cache, filterWith) => {
77125
.catch(()=>node)
78126
}
79127

80-
constloadTree=(node,did,cache,filterWith)=>{
128+
constloadTree=(node,did,cache,filterWith,rpcache,stcache)=>{
81129
// impossible except in pathological ELOOP cases
82130
/* istanbul ignore next */
83131
if(did.has(node.realpath))
84132
returnPromise.resolve(node)
85133

86134
did.add(node.realpath)
87135

88-
returnloadChildren(node,cache,filterWith)
136+
// load children on the target, not the link
137+
returnloadChildren(node.target||node,cache,filterWith,rpcache,stcache)
89138
.then(node=>Promise.all(
90139
node.children
91140
.filter(kid=>!did.has(kid.realpath))
92-
.map(kid=>loadTree(kid,did,cache,filterWith))
141+
.map(kid=>loadTree(kid,did,cache,filterWith,rpcache,stcache))
93142
)).then(()=>node)
94143
}
95144

@@ -100,10 +149,15 @@ const rpt = (root, filterWith, cb) => {
100149
filterWith=null
101150
}
102151

152+
root=resolve(root)
103153
constcache=newMap()
104-
constp=realpath(root)
105-
.then(realRoot=>loadNode(root,realRoot,cache))
106-
.then(node=>loadTree(node,newSet(),cache,filterWith))
154+
// we can assume that the cwd is real enough
155+
constcwd=process.cwd()
156+
constrpcache=newMap([[cwd,cwd]])
157+
conststcache=newMap()
158+
constp=realpath(root,rpcache,stcache,0)
159+
.then(realRoot=>loadNode(root,realRoot,cache,rpcache,stcache))
160+
.then(node=>loadTree(node,newSet(),cache,filterWith,rpcache,stcache))
107161

108162
if(typeofcb==='function')
109163
p.then(tree=>cb(null,tree),cb)
‎tap-snapshots/test-basic.js-TAP.test.js‎
Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,20 @@ root@1.2.3 test/fixtures/linkedroot
4040
└── foo@1.2.3 test/fixtures/linkedroot/node_modules/foo
4141
`
4242

43+
exports[`test/basic.js TAP looking outside of cwd > must match snapshot 1`]=`
44+
root@1.2.3 test/fixtures/root
45+
├─┬ @scope/x@1.2.3 test/fixtures/root/node_modules/@scope/x
46+
│ └─┬ glob@4.0.5 test/fixtures/root/node_modules/@scope/x/node_modules/glob
47+
│ ├── graceful-fs@3.0.2 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/graceful-fs
48+
│ ├── inherits@2.0.1 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/inherits
49+
│ ├─┬ minimatch@1.0.0 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/minimatch
50+
│ │ ├── lru-cache@2.5.0 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/minimatch/node_modules/lru-cache
51+
│ │ └── sigmund@1.0.0 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/minimatch/node_modules/sigmund
52+
│ └── once@1.3.0 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/once
53+
├── @scope/y@1.2.3 test/fixtures/root/node_modules/@scope/y
54+
└── foo@1.2.3 test/fixtures/root/node_modules/foo
55+
`
56+
4357
exports[`test/basic.js TAP noname > noname tree 1`]=`
4458
test/fixtures/noname
4559
└── test/fixtures/noname/node_modules/foo
@@ -79,3 +93,19 @@ selflink@1.2.3 test/fixtures/selflink
7993
 │ └── once@1.3.0 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/once
8094
 └── selflink@1.2.3 test/fixtures/selflink (symlink)
8195
`
96+
97+
exports[`test/basic.js TAP shake out Link target timing issue > must match snapshot 1`]=`
98+
selflink@1.2.3 test/fixtures/selflink
99+
├── @scope/y@1.2.3 test/fixtures/selflink/node_modules/@scope/y
100+
├─┬ @scope/z@1.2.3 test/fixtures/selflink/node_modules/@scope/z
101+
│ └── glob@4.0.5 test/fixtures/selflink/node_modules/foo/node_modules/glob (symlink)
102+
└─┬ foo@1.2.3 test/fixtures/selflink/node_modules/foo
103+
 ├─┬ glob@4.0.5 test/fixtures/selflink/node_modules/foo/node_modules/glob
104+
 │ ├── graceful-fs@3.0.2 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/graceful-fs
105+
 │ ├── inherits@2.0.1 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/inherits
106+
 │ ├─┬ minimatch@1.0.0 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/minimatch
107+
 │ │ ├── lru-cache@2.5.0 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/minimatch/node_modules/lru-cache
108+
 │ │ └── sigmund@1.0.0 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/minimatch/node_modules/sigmund
109+
 │ └── once@1.3.0 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/once
110+
 └── selflink@1.2.3 test/fixtures/selflink (symlink)
111+
`
‎test/basic.js‎
Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,22 @@ test('filterWith', t =>
8585
).then(d=>t.matchSnapshot(archy(archyize(d)).trim()),'only 1 level deep')
8686
)
8787

88+
test('looking outside of cwd',t=>{
89+
constcwd=process.cwd()
90+
t.teardown(()=>process.chdir(cwd))
91+
process.chdir('test/fixtures/selflink')
92+
returnrpt('../root').then(d=>
93+
t.matchSnapshot(archy(archyize(d)).trim()))
94+
})
95+
96+
test('shake out Link target timing issue',t=>{
97+
process.env._TEST_RPT_SLOW_LINK_TARGET_='1'
98+
constcwd=process.cwd()
99+
t.teardown(()=>process.env._TEST_RPT_SLOW_LINK_TARGET_='')
100+
returnrpt(path.resolve(fixtures,'selflink')).then(d=>
101+
t.matchSnapshot(archy(archyize(d)).trim()))
102+
})
103+
88104
test('broken json',function(t){
89105
rpt(path.resolve(fixtures,'bad'),function(er,d){
90106
t.ok(d.error,'Got an error object')
@@ -152,6 +168,23 @@ function archyize (d, seen) {
152168
}
153169
}
154170

171+
test('realpath gutchecks',t=>{
172+
constd=path.resolve(cwd,'test/fixtures')
173+
constrealpath=require('../realpath.js')
174+
const{realpathSync}=fs
175+
Object.keys(symlinks).map(link=>t.test(link,t=>
176+
realpath(
177+
path.resolve(d,link),
178+
newMap(),
179+
newMap(),
180+
0
181+
).then(
182+
real=>t.equal(real,realpathSync(path.resolve(d,link))),
183+
er=>t.throws(()=>realpathSync(path.resolve(d,link)))
184+
)))
185+
t.end()
186+
})
187+
155188
test('cleanup',function(t){
156189
cleanup()
157190
t.end()
0 commit comments
Comments
 (0)

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Universal Dark Mode - works on any site
(function() {
var enabled = true;
function applyDarkMode() {
if (!enabled) return;
// Create style element if it doesn't exist
var style = document.getElementById('universal-dark-mode-style');
if (!style) {
style = document.createElement('style');
style.id = 'universal-dark-mode-style';
document.head.appendChild(style);
}
// Dark mode CSS - inverts colors but preserves images/video
style.textContent = '
/* Invert everything except media */
html {
filter: invert(1) hue-rotate(180deg) !important;
background: #1a1a2e !important;
}
/* Restore images, videos, iframes, canvas */
img, video, iframe, canvas, svg, picture, [style*="background-image"] {
filter: invert(1) hue-rotate(180deg) !important;
}
/* Preserve specific elements that should not be inverted */
.no-dark-mode, .no-dark-mode *,
[data-theme="light"], [data-theme="light"],
.ace_editor, .ace_editor *,
.CodeMirror, .CodeMirror *,
.monaco-editor, .monaco-editor *,
.markdown-body pre, .markdown-body pre *,
.highlight, .highlight *,
pre code, pre code * {
filter: none !important;
}
/* Fix common UI elements */
.modal, .popup, .dropdown-menu, .tooltip, .popover {
filter: invert(1) hue-rotate(180deg) !important;
background: #2d2d44 !important;
border-color: #444 !important;
}
/* Scrollbars */
::-webkit-scrollbar { background: #1a1a2e !important; }
::-webkit-scrollbar-thumb { background: #444 !important; }
::-webkit-scrollbar-thumb:hover { background: #555 !important; }
/* Selection */
::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
';
}
function removeDarkMode() {
var style = document.getElementById('universal-dark-mode-style');
if (style) style.remove();
}
// Toggle with Alt+Shift+D
document.addEventListener('keydown', function(e) {
if (e.altKey && e.shiftKey && e.key === 'D') {
e.preventDefault();
enabled = !enabled;
if (enabled) {
applyDarkMode();
console.log('[Universal Dark Mode] Enabled');
} else {
removeDarkMode();
console.log('[Universal Dark Mode] Disabled');
}
}
});
// Apply on load
applyDarkMode();
// Re-apply on dynamic content
var observer = new MutationObserver(function(mutations) {
if (enabled && !document.getElementById('universal-dark-mode-style')) {
applyDarkMode();
}
});
observer.observe(document.head, { childList: true });
console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');
})();
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();
Use custom cachable fs.realpath implementation · npm/read-package-tree@e9cd536 · GitHub
Skip to content

This repository was archived by the owner on Jan 7, 2022. It is now read-only.

Commit e9cd536
Browse files
isaacsisaacs
committed
Use custom cachable fs.realpath implementation
In this use case, we don't care much about a lot of the stuff that
fs.realpath can (and should!) do. The only thing that's relevant to
reading a package tree is whether package folders are symbolic links,
and if so, where they point.
Additionally, we don't need to re-start the fs.lstat party every time we
walk to a new directory. While it makes sense for fs.realpath to do
this in the general case, it's not required when reading a package tree,
and results in a geometric explosion of lstat syscalls. For example, if
a project is in /Users/hyooman/projects/company/website, and it has 1000
dependencies in node_modules, then a whopping 6,000 lstat calls will be
made just to repeatedly verify that
/Users/hyooman/projects/company/website/node_modules has not moved!
In this implementation, every realpath call is cached, as is every
lstat. Additionally, process.cwd() is assumed to be "real enough", and
added to the cache initially, which means almost never having to walk
all the way up to the root directory.
In the npm cli project, this drops the lstat count from 14885 to 3054
for a single call to read-package-tree on my system. Larger projects,
or projects deeper in a folder tree, will have even larger reductions.
This does not account, itself, for a particularly large speed-up, since
lstat calls do tend to be fairly fast, and the repetitiveness means that
there are a lot of hits in the file system's stat cache. But it does
make read-package-tree 10-30% faster in common use cases.
1 parent 4eed760 commit e9cd536
5 files changed
Lines changed: 238 additions & 28 deletions
File tree
‎package.json‎
Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,8 @@
3131
 },
3232
"homepage": "https://github.com/npm/read-package-tree",
3333
"files": [
34-
"rpt.js"
34+
"rpt.js",
35+
"realpath.js"
3536
 ],
3637
"tap": {
3738
"100": true
‎realpath.js‎
Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
// look up the realpath, but cache stats to minimize overhead
2+
// If the parent folder is in the realpath cache, then we just
3+
// lstat the child, since there's no need to do a full realpath
4+
// This is not a separate module, and is much simpler than Node's
5+
// built-in fs.realpath, because we only care about symbolic links,
6+
// so we can handle many fewer edge cases.
7+
8+
constfs=require('fs')
9+
const{ promisify }=require('util')
10+
constreadlink=promisify(fs.readlink)
11+
constlstat=promisify(fs.lstat)
12+
const{ resolve, basename, dirname }=require('path')
13+
14+
constrealpathCached=(path,rpcache,stcache,depth)=>{
15+
// just a safety against extremely deep eloops
16+
/* istanbul ignore next */
17+
if(depth>2000)
18+
throweloop(path)
19+
20+
if(rpcache.has(path))
21+
returnPromise.resolve(rpcache.get(path))
22+
23+
constdir=dirname(path)
24+
constbase=basename(path)
25+
26+
if(base&&rpcache.has(dir))
27+
returnrealpathChild(dir,base,rpcache,stcache,depth)
28+
29+
// if it's the root, then we know it's real
30+
if(!base){
31+
rpcache.set(dir,dir)
32+
returnPromise.resolve(dir)
33+
}
34+
35+
// the parent, what is that?
36+
// find out, and then come back.
37+
returnrealpathCached(dir,rpcache,stcache,depth+1).then(()=>
38+
realpathCached(path,rpcache,stcache,depth+1))
39+
}
40+
41+
constlstatCached=(path,stcache)=>{
42+
if(stcache.has(path))
43+
returnPromise.resolve(stcache.get(path))
44+
45+
constp=lstat(path).then(st=>{
46+
stcache.set(path,st)
47+
returnst
48+
})
49+
stcache.set(path,p)
50+
returnp
51+
}
52+
53+
// This is a slight fib, as it doesn't actually occur during a stat syscall.
54+
// But file systems are giant piles of lies, so whatever.
55+
consteloop=path=>
56+
Object.assign(newError(
57+
`ELOOP: too many symbolic links encountered, stat '${path}'`),{
58+
errno: -62,
59+
syscall: 'stat',
60+
code: 'ELOOP',
61+
path: path,
62+
})
63+
64+
constrealpathChild=(dir,base,rpcache,stcache,depth)=>{
65+
constrealdir=rpcache.get(dir)
66+
// that unpossible
67+
/* istanbul ignore next */
68+
if(typeofrealdir==='undefined')
69+
thrownewError('in realpathChild without parent being in realpath cache')
70+
71+
constrealish=resolve(realdir,base)
72+
returnlstatCached(realish,stcache).then(st=>{
73+
if(!st.isSymbolicLink()){
74+
rpcache.set(resolve(dir,base),realish)
75+
returnrealish
76+
}
77+
78+
letres
79+
returnreadlink(realish).then(target=>{
80+
constresolved=res=resolve(realdir,target)
81+
if(realish===resolved)
82+
throweloop(realish)
83+
84+
returnrealpathCached(resolved,rpcache,stcache,depth+1)
85+
}).then(real=>{
86+
rpcache.set(resolve(dir,base),real)
87+
returnreal
88+
})
89+
})
90+
}
91+
92+
module.exports=realpathCached
‎rpt.js‎
Lines changed: 81 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,17 @@
11
constfs=require('fs')
22
const{ promisify }=require('util')
3-
constrealpath=promisify(fs.realpath)
4-
const{ basename, dirname, join }=require('path')
3+
const{ resolve, basename, dirname, join }=require('path')
54
constrpj=promisify(require('read-package-json'))
65
constreaddir=promisify(require('readdir-scoped-modules'))
6+
constrealpath=require('./realpath.js')
77

88
letID=0
99
classNode{
1010
constructor(pkg,logical,physical,er,cache){
1111
// should be impossible.
12+
constcached=cache.get(physical)
1213
/* istanbul ignore next */
13-
if(cache.get(physical))
14+
if(cached&&!cached.then)
1415
thrownewError('re-creating already instantiated node')
1516

1617
cache.set(physical,this)
@@ -34,37 +35,84 @@ class Node {
3435
classLinkextendsNode{
3536
constructor(pkg,logical,physical,realpath,er,cache){
3637
super(pkg,logical,physical,er,cache)
38+
39+
// if the target has started, but not completed, then
40+
// a Promise will be in the cache to indicate this.
3741
constcachedTarget=cache.get(realpath)
42+
if(cachedTarget&&cachedTarget.then)
43+
cachedTarget.then(node=>this.target=node)
44+
3845
this.target=cachedTarget||newNode(pkg,logical,realpath,er,cache)
3946
this.realpath=realpath
4047
this.isLink=true
41-
this.children=this.target.children
4248
this.error=er
49+
// convenience method only
50+
/* istanbul ignore next */
51+
Object.defineProperty(this,'children',{
52+
get(){
53+
returnthis.target.children
54+
},
55+
set(c){
56+
this.target.children=c
57+
},
58+
enumerable: true
59+
})
4360
}
4461
}
4562

46-
constloadNode=(logical,physical,cache)=>newPromise((res,rej)=>{
47-
res(cache.get(physical)||realpath(physical)
48-
.then(real=>
49-
rpj(join(real,'package.json'))
50-
.then(pkg=>[real,pkg,null],er=>[real,null,er])
51-
.then(([real,pkg,er])=>
52-
physical===real ? newNode(pkg,logical,physical,er,cache)
53-
 : newLink(pkg,logical,physical,real,er,cache)
54-
),
55-
// if the realpath fails, don't bother with the rest
56-
er=>newNode(null,logical,physical,er,cache))
57-
)
58-
})
59-
60-
constloadChildren=(node,cache,filterWith)=>{
63+
// this is the way it is to expose a timing issue which is difficult to
64+
// test otherwise. The creation of a Node may take slightly longer than
65+
// the creation of a Link that targets it. If the Node has _begun_ its
66+
// creation phase (and put a Promise in the cache) then the Link will
67+
// get a Promise as its cachedTarget instead of an actual Node object.
68+
// This is not a problem, because it gets resolved prior to returning
69+
// the tree or attempting to load children. However, it IS remarkably
70+
// difficult to get to happen in a test environment to verify reliably.
71+
// Hence this kludge.
72+
constnewNode=(pkg,logical,physical,er,cache)=>
73+
process.env._TEST_RPT_SLOW_LINK_TARGET_==='1'
74+
 ? newPromise(res=>setTimeout(()=>
75+
res(newNode(pkg,logical,physical,er,cache)),10))
76+
 : newNode(pkg,logical,physical,er,cache)
77+
78+
constloadNode=(logical,physical,cache,rpcache,stcache)=>{
79+
// cache temporarily holds a promise placeholder so we
80+
// don't try to create the same node multiple times.
81+
// this is very rare to encounter, given the aggressive
82+
// caching on fs.realpath and fs.lstat calls, but
83+
// it can happen in theory.
84+
constcached=cache.get(physical)
85+
/* istanbul ignore next */
86+
if(cached)
87+
returnPromise.resolve(cached)
88+
89+
constp=realpath(physical,rpcache,stcache,0).then(real=>
90+
rpj(join(real,'package.json'))
91+
.then(pkg=>[pkg,null],er=>[null,er])
92+
.then(([pkg,er])=>
93+
physical===real ? newNode(pkg,logical,physical,er,cache)
94+
 : newLink(pkg,logical,physical,real,er,cache)
95+
),
96+
// if the realpath fails, don't bother with the rest
97+
er=>newNode(null,logical,physical,er,cache))
98+
99+
cache.set(physical,p)
100+
returnp
101+
}
102+
103+
constloadChildren=(node,cache,filterWith,rpcache,stcache)=>{
104+
// if a Link target has started, but not completed, then
105+
// a Promise will be in the cache to indicate this.
106+
if(node.then)
107+
returnnode.then(node=>loadChildren(node,cache,filterWith,rpcache,stcache))
108+
61109
constnm=join(node.path,'node_modules')
62-
returnrealpath(nm)
110+
returnrealpath(nm,rpcache,stcache,0)
63111
.then(rm=>readdir(rm).then(kids=>[rm,kids]))
64112
.then(([rm,kids])=>Promise.all(
65113
kids.filter(kid=>
66114
kid.charAt(0)!=='.'&&(!filterWith||filterWith(node,kid)))
67-
.map(kid=>loadNode(join(nm,kid),join(rm,kid),cache)))
115+
.map(kid=>loadNode(join(nm,kid),join(rm,kid),cache,rpcache,stcache)))
68116
).then(kidNodes=>{
69117
kidNodes.forEach(k=>k.parent=node)
70118
node.children=kidNodes.sort((a,b)=>
@@ -77,19 +125,20 @@ const loadChildren = (node, cache, filterWith) => {
77125
.catch(()=>node)
78126
}
79127

80-
constloadTree=(node,did,cache,filterWith)=>{
128+
constloadTree=(node,did,cache,filterWith,rpcache,stcache)=>{
81129
// impossible except in pathological ELOOP cases
82130
/* istanbul ignore next */
83131
if(did.has(node.realpath))
84132
returnPromise.resolve(node)
85133

86134
did.add(node.realpath)
87135

88-
returnloadChildren(node,cache,filterWith)
136+
// load children on the target, not the link
137+
returnloadChildren(node.target||node,cache,filterWith,rpcache,stcache)
89138
.then(node=>Promise.all(
90139
node.children
91140
.filter(kid=>!did.has(kid.realpath))
92-
.map(kid=>loadTree(kid,did,cache,filterWith))
141+
.map(kid=>loadTree(kid,did,cache,filterWith,rpcache,stcache))
93142
)).then(()=>node)
94143
}
95144

@@ -100,10 +149,15 @@ const rpt = (root, filterWith, cb) => {
100149
filterWith=null
101150
}
102151

152+
root=resolve(root)
103153
constcache=newMap()
104-
constp=realpath(root)
105-
.then(realRoot=>loadNode(root,realRoot,cache))
106-
.then(node=>loadTree(node,newSet(),cache,filterWith))
154+
// we can assume that the cwd is real enough
155+
constcwd=process.cwd()
156+
constrpcache=newMap([[cwd,cwd]])
157+
conststcache=newMap()
158+
constp=realpath(root,rpcache,stcache,0)
159+
.then(realRoot=>loadNode(root,realRoot,cache,rpcache,stcache))
160+
.then(node=>loadTree(node,newSet(),cache,filterWith,rpcache,stcache))
107161

108162
if(typeofcb==='function')
109163
p.then(tree=>cb(null,tree),cb)
‎tap-snapshots/test-basic.js-TAP.test.js‎
Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,20 @@ root@1.2.3 test/fixtures/linkedroot
4040
└── foo@1.2.3 test/fixtures/linkedroot/node_modules/foo
4141
`
4242

43+
exports[`test/basic.js TAP looking outside of cwd > must match snapshot 1`]=`
44+
root@1.2.3 test/fixtures/root
45+
├─┬ @scope/x@1.2.3 test/fixtures/root/node_modules/@scope/x
46+
│ └─┬ glob@4.0.5 test/fixtures/root/node_modules/@scope/x/node_modules/glob
47+
│ ├── graceful-fs@3.0.2 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/graceful-fs
48+
│ ├── inherits@2.0.1 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/inherits
49+
│ ├─┬ minimatch@1.0.0 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/minimatch
50+
│ │ ├── lru-cache@2.5.0 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/minimatch/node_modules/lru-cache
51+
│ │ └── sigmund@1.0.0 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/minimatch/node_modules/sigmund
52+
│ └── once@1.3.0 test/fixtures/root/node_modules/@scope/x/node_modules/glob/node_modules/once
53+
├── @scope/y@1.2.3 test/fixtures/root/node_modules/@scope/y
54+
└── foo@1.2.3 test/fixtures/root/node_modules/foo
55+
`
56+
4357
exports[`test/basic.js TAP noname > noname tree 1`]=`
4458
test/fixtures/noname
4559
└── test/fixtures/noname/node_modules/foo
@@ -79,3 +93,19 @@ selflink@1.2.3 test/fixtures/selflink
7993
 │ └── once@1.3.0 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/once
8094
 └── selflink@1.2.3 test/fixtures/selflink (symlink)
8195
`
96+
97+
exports[`test/basic.js TAP shake out Link target timing issue > must match snapshot 1`]=`
98+
selflink@1.2.3 test/fixtures/selflink
99+
├── @scope/y@1.2.3 test/fixtures/selflink/node_modules/@scope/y
100+
├─┬ @scope/z@1.2.3 test/fixtures/selflink/node_modules/@scope/z
101+
│ └── glob@4.0.5 test/fixtures/selflink/node_modules/foo/node_modules/glob (symlink)
102+
└─┬ foo@1.2.3 test/fixtures/selflink/node_modules/foo
103+
 ├─┬ glob@4.0.5 test/fixtures/selflink/node_modules/foo/node_modules/glob
104+
 │ ├── graceful-fs@3.0.2 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/graceful-fs
105+
 │ ├── inherits@2.0.1 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/inherits
106+
 │ ├─┬ minimatch@1.0.0 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/minimatch
107+
 │ │ ├── lru-cache@2.5.0 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/minimatch/node_modules/lru-cache
108+
 │ │ └── sigmund@1.0.0 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/minimatch/node_modules/sigmund
109+
 │ └── once@1.3.0 test/fixtures/selflink/node_modules/foo/node_modules/glob/node_modules/once
110+
 └── selflink@1.2.3 test/fixtures/selflink (symlink)
111+
`
‎test/basic.js‎
Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,22 @@ test('filterWith', t =>
8585
).then(d=>t.matchSnapshot(archy(archyize(d)).trim()),'only 1 level deep')
8686
)
8787

88+
test('looking outside of cwd',t=>{
89+
constcwd=process.cwd()
90+
t.teardown(()=>process.chdir(cwd))
91+
process.chdir('test/fixtures/selflink')
92+
returnrpt('../root').then(d=>
93+
t.matchSnapshot(archy(archyize(d)).trim()))
94+
})
95+
96+
test('shake out Link target timing issue',t=>{
97+
process.env._TEST_RPT_SLOW_LINK_TARGET_='1'
98+
constcwd=process.cwd()
99+
t.teardown(()=>process.env._TEST_RPT_SLOW_LINK_TARGET_='')
100+
returnrpt(path.resolve(fixtures,'selflink')).then(d=>
101+
t.matchSnapshot(archy(archyize(d)).trim()))
102+
})
103+
88104
test('broken json',function(t){
89105
rpt(path.resolve(fixtures,'bad'),function(er,d){
90106
t.ok(d.error,'Got an error object')
@@ -152,6 +168,23 @@ function archyize (d, seen) {
152168
}
153169
}
154170

171+
test('realpath gutchecks',t=>{
172+
constd=path.resolve(cwd,'test/fixtures')
173+
constrealpath=require('../realpath.js')
174+
const{realpathSync}=fs
175+
Object.keys(symlinks).map(link=>t.test(link,t=>
176+
realpath(
177+
path.resolve(d,link),
178+
newMap(),
179+
newMap(),
180+
0
181+
).then(
182+
real=>t.equal(real,realpathSync(path.resolve(d,link))),
183+
er=>t.throws(()=>realpathSync(path.resolve(d,link)))
184+
)))
185+
t.end()
186+
})
187+
155188
test('cleanup',function(t){
156189
cleanup()
157190
t.end()
0 commit comments
Comments
 (0)