win,install: only download target_arch node.lib#2857

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
StefanStojanovic merged 1 commit into from
Jun 9, 2023
Merged

win,install: only download target_arch node.lib #2857

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
169 changes: 93 additions & 76 deletions lib/install.js
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,10 @@ const streamPipeline = util.promisify(stream.pipeline)

async function install (fs, gyp, argv) {
const release = processRelease(argv, gyp, process.version, process.release)
// Detecting target_arch based on logic from create-cnfig-gyp.js. Used on Windows only.
Comment thread
StefanStojanovic marked this conversation as resolved.
const arch = win ? (gyp.opts.target_arch || gyp.opts.arch || process.arch || 'ia32') : ''
// Used to prevent downloading tarball if only new node.lib is required on Windows.
let shouldDownloadTarball = true

// Determine which node dev files version we are installing
log.verbose('install', 'input version string %j', release.version)
Expand DownExpand Up@@ -92,6 +96,26 @@ async function install (fs, gyp, argv) {
}
}
log.verbose('install', 'version is good')
if (win) {
log.verbose('on Windows; need to check node.lib')
const nodeLibPath = path.resolve(devDir, arch, 'node.lib')
try {
await fs.promises.stat(nodeLibPath)
} catch (err) {
if (err.code === 'ENOENT') {
log.verbose('install', `version not already installed for ${arch}, continuing with install`, release.version)
Comment thread
StefanStojanovic marked this conversation as resolved.
try {
shouldDownloadTarball = false
return await go()
} catch (err) {
return rollback(err)
}
} else if (err.code === 'EACCES') {
return eaccesFallback(err)
}
throw err
}
}
} else {
try {
return await go()
Expand DownExpand Up@@ -179,66 +203,69 @@ async function install (fs, gyp, argv) {
}

// download the tarball and extract!
// Ommited on Windows if only new node.lib is required

// on Windows there can be file errors from tar if parallel installs
// are happening (not uncommon with multiple native modules) so
// extract the tarball to a temp directory first and then copy over
const tarExtractDir = win ? await fs.promises.mkdtemp(path.join(os.tmpdir(), 'node-gyp-tmp-')) : devDir

try {
if (tarPath) {
await tar.extract({
file: tarPath,
strip: 1,
filter: isValid,
onwarn,
cwd: tarExtractDir
})
} else {
try {
const res = await download(gyp, release.tarballUrl)
if (shouldDownloadTarball) {
if (tarPath) {
await tar.extract({
file: tarPath,
strip: 1,
filter: isValid,
onwarn,
cwd: tarExtractDir
})
} else {
try {
const res = await download(gyp, release.tarballUrl)

if (res.status !== 200) {
throw new Error(`${res.status} response downloading ${release.tarballUrl}`)
}
if (res.status !== 200) {
throw new Error(`${res.status} response downloading ${release.tarballUrl}`)
}

await streamPipeline(
res.body,
// content checksum
new ShaSum((_, checksum) => {
const filename = path.basename(release.tarballUrl).trim()
contentShasums[filename] = checksum
log.verbose('content checksum', filename, checksum)
}),
tar.extract({
strip: 1,
cwd: tarExtractDir,
filter: isValid,
onwarn
})
)
} catch (err) {
await streamPipeline(
res.body,
// content checksum
new ShaSum((_, checksum) => {
const filename = path.basename(release.tarballUrl).trim()
contentShasums[filename] = checksum
log.verbose('content checksum', filename, checksum)
}),
tar.extract({
strip: 1,
cwd: tarExtractDir,
filter: isValid,
onwarn
})
)
} catch (err) {
// something went wrong downloading the tarball?
if (err.code === 'ENOTFOUND') {
throw new Error('This is most likely not a problem with node-gyp or the package itself and\n' +
if (err.code === 'ENOTFOUND') {
throw new Error('This is most likely not a problem with node-gyp or the package itself and\n' +
'is related to network connectivity. In most cases you are behind a proxy or have bad \n' +
'network settings.')
}
throw err
}
throw err
}
}

// invoked after the tarball has finished being extracted
if (extractErrors || extractCount === 0) {
throw new Error('There was a fatal problem while downloading/extracting the tarball')
}
// invoked after the tarball has finished being extracted
if (extractErrors || extractCount === 0) {
throw new Error('There was a fatal problem while downloading/extracting the tarball')
}

log.verbose('tarball', 'done parsing tarball')
log.verbose('tarball', 'done parsing tarball')
}

const installVersionPath = path.resolve(tarExtractDir, 'installVersion')
await Promise.all([
// need to download node.lib
...(win ? downloadNodeLib() : []),
// need to download node.lib
...(win ? [downloadNodeLib()] : []),
// write the "installVersion" file
fs.promises.writeFile(installVersionPath, gyp.package.installVersion + '\n'),
// Only download SHASUMS.txt if we downloaded something in need of SHA verification
Expand DownExpand Up@@ -293,43 +320,33 @@ async function install (fs, gyp, argv) {
log.verbose('checksum data', JSON.stringify(expectShasums))
}

function downloadNodeLib () {
async function downloadNodeLib () {
log.verbose('on Windows; need to download `' + release.name + '.lib`...')
const archs = ['ia32', 'x64', 'arm64']
return archs.map(async (arch) => {
const dir = path.resolve(tarExtractDir, arch)
const targetLibPath = path.resolve(dir, release.name + '.lib')
const { libUrl, libPath } = release[arch]
const name = `${arch} ${release.name}.lib`
log.verbose(name, 'dir', dir)
log.verbose(name, 'url', libUrl)

await fs.promises.mkdir(dir, { recursive: true })
log.verbose('streaming', name, 'to:', targetLibPath)

const res = await download(gyp, libUrl)

if (res.status === 403 || res.status === 404) {
if (arch === 'arm64') {
// Arm64 is a newer platform on Windows and not all node distributions provide it.
log.verbose(`${name} was not found in ${libUrl}`)
} else {
log.warn(`${name} was not found in ${libUrl}`)
}
return
} else if (res.status !== 200) {
throw new Error(`${res.status} status code downloading ${name}`)
}
const dir = path.resolve(tarExtractDir, arch)
const targetLibPath = path.resolve(dir, release.name + '.lib')
const { libUrl, libPath } = release[arch]
const name = `${arch} ${release.name}.lib`
log.verbose(name, 'dir', dir)
log.verbose(name, 'url', libUrl)

await fs.promises.mkdir(dir, { recursive: true })
log.verbose('streaming', name, 'to:', targetLibPath)

const res = await download(gyp, libUrl)

// Since only required node.lib is downloaded throw error if it is not fetched
if (res.status !== 200) {
throw new Error(`${res.status} status code downloading ${name}`)
}

return streamPipeline(
res.body,
new ShaSum((_, checksum) => {
contentShasums[libPath] = checksum
log.verbose('content checksum', libPath, checksum)
}),
fs.createWriteStream(targetLibPath)
)
})
return streamPipeline(
res.body,
new ShaSum((_, checksum) => {
contentShasums[libPath] = checksum
log.verbose('content checksum', libPath, checksum)
}),
fs.createWriteStream(targetLibPath)
)
} // downloadNodeLib()
} // go()

Expand Down
2 changes: 1 addition & 1 deletion package.json
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -12,7 +12,7 @@
"gyp"
],
"version": "9.3.1",
"installVersion": 10,

@StefanStojanovicStefanStojanovicMay 26, 2023

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A question for people with more experience in node-gyp: Since a PR that changed installVersion from 9 to 10 landed yesterday in aaa117c do I need to increase it to 11 now, or not, because there was no release with installVersion: 10?

@dsanders11dsanders11Jun 7, 2023

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A question for people with more experience in node-gyp

I'm not more experienced, but my 2 cents: it's probably unnecessary for end users (since as you stated, no release went out), but AFAIK there's no harm in bumping it again, and it will prevent a situation where anyone who tested main (so folks working on node-gyp like you and I, or maybe even someone some where consuming node-gyp@main for whatever reason) and got their cache populated for installVersion: 10 won't end up in a broken situation.

So to me bumping it to 11 seems reasonable to prevent any problems.

"installVersion": 11,
"author": "Nathan Rajlich <nathan@tootallnate.net> (http://tootallnate.net)",
"repository": {
"type": "git",
Expand Down
2 changes: 1 addition & 1 deletion test/test-download.js
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -180,7 +180,7 @@ describe('download', function () {
await util.promisify(install)(prog, [])

const data = await fs.promises.readFile(path.join(expectedDir, 'installVersion'), 'utf8')
assert.strictEqual(data, '10\n', 'correct installVersion')
assert.strictEqual(data, '11\n', 'correct installVersion')

const list = await fs.promises.readdir(path.join(expectedDir, 'include/node'))
assert.ok(list.includes('common.gypi'))
Expand Down
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
win,install: only download target_arch node.lib#2857

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

StefanStojanovic
merged 1 commit into

from

Jun 9, 2023
 Merged


win,install: only download target_arch node.lib
#2857
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


169 changes: 93 additions & 76 deletions
lib/install.js

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,10 @@ const streamPipeline = util.promisify(stream.pipeline)

async function install (fs, gyp, argv) {
 const release = processRelease(argv, gyp, process.version, process.release)
 // Detecting target_arch based on logic from create-cnfig-gyp.js. Used on Windows only.
Comment thread
StefanStojanovic marked this conversation as resolved.

 const arch = win ? (gyp.opts.target_arch || gyp.opts.arch || process.arch || 'ia32') : ''
 // Used to prevent downloading tarball if only new node.lib is required on Windows.
 let shouldDownloadTarball = true

 // Determine which node dev files version we are installing
 log.verbose('install', 'input version string %j', release.version)
Expand DownExpand Up@@ -92,6 +96,26 @@ async function install (fs, gyp, argv) {
 }
 }
 log.verbose('install', 'version is good')
 if (win) {
 log.verbose('on Windows; need to check node.lib')
 const nodeLibPath = path.resolve(devDir, arch, 'node.lib')
 try {
 await fs.promises.stat(nodeLibPath)
 } catch (err) {
 if (err.code === 'ENOENT') {
 log.verbose('install', `version not already installed for ${arch}, continuing with install`, release.version)
Comment thread
StefanStojanovic marked this conversation as resolved.

 try {
 shouldDownloadTarball = false
 return await go()
 } catch (err) {
 return rollback(err)
 }
 } else if (err.code === 'EACCES') {
 return eaccesFallback(err)
 }
 throw err
 }
 }
 } else {
 try {
 return await go()
Expand DownExpand Up@@ -179,66 +203,69 @@ async function install (fs, gyp, argv) {
 }

 // download the tarball and extract!
 // Ommited on Windows if only new node.lib is required

 // on Windows there can be file errors from tar if parallel installs
 // are happening (not uncommon with multiple native modules) so
 // extract the tarball to a temp directory first and then copy over
 const tarExtractDir = win ? await fs.promises.mkdtemp(path.join(os.tmpdir(), 'node-gyp-tmp-')) : devDir

 try {
 if (tarPath) {
 await tar.extract({
 file: tarPath,
 strip: 1,
 filter: isValid,
 onwarn,
 cwd: tarExtractDir
 })
 } else {
 try {
 const res = await download(gyp, release.tarballUrl)
 if (shouldDownloadTarball) {
 if (tarPath) {
 await tar.extract({
 file: tarPath,
 strip: 1,
 filter: isValid,
 onwarn,
 cwd: tarExtractDir
 })
 } else {
 try {
 const res = await download(gyp, release.tarballUrl)

 if (res.status !== 200) {
 throw new Error(`${res.status} response downloading ${release.tarballUrl}`)
 }
if (res.status !== 200) {
throw new Error(`${res.status} response downloading ${release.tarballUrl}`)
}

 await streamPipeline(
 res.body,
 // content checksum
 new ShaSum((_, checksum) => {
 const filename = path.basename(release.tarballUrl).trim()
 contentShasums[filename] = checksum
 log.verbose('content checksum', filename, checksum)
 }),
 tar.extract({
 strip: 1,
 cwd: tarExtractDir,
 filter: isValid,
 onwarn
 })
 )
 } catch (err) {
await streamPipeline(
res.body,
// content checksum
new ShaSum((_, checksum) => {
const filename = path.basename(release.tarballUrl).trim()
contentShasums[filename] = checksum
log.verbose('content checksum', filename, checksum)
}),
tar.extract({
strip: 1,
cwd: tarExtractDir,
filter: isValid,
onwarn
})
)
} catch (err) {
 // something went wrong downloading the tarball?
 if (err.code === 'ENOTFOUND') {
 throw new Error('This is most likely not a problem with node-gyp or the package itself and\n' +
if (err.code === 'ENOTFOUND') {
throw new Error('This is most likely not a problem with node-gyp or the package itself and\n' +
 'is related to network connectivity. In most cases you are behind a proxy or have bad \n' +
 'network settings.')
 }
 throw err
 }
 throw err
 }
 }

 // invoked after the tarball has finished being extracted
 if (extractErrors || extractCount === 0) {
 throw new Error('There was a fatal problem while downloading/extracting the tarball')
 }
// invoked after the tarball has finished being extracted
if (extractErrors || extractCount === 0) {
throw new Error('There was a fatal problem while downloading/extracting the tarball')
}

 log.verbose('tarball', 'done parsing tarball')
 log.verbose('tarball', 'done parsing tarball')
 }

 const installVersionPath = path.resolve(tarExtractDir, 'installVersion')
 await Promise.all([
// need to download node.lib
 ...(win ? downloadNodeLib() : []),
 // need to download node.lib
 ...(win ? [downloadNodeLib()] : []),
 // write the "installVersion" file
 fs.promises.writeFile(installVersionPath, gyp.package.installVersion + '\n'),
 // Only download SHASUMS.txt if we downloaded something in need of SHA verification
Expand DownExpand Up@@ -293,43 +320,33 @@ async function install (fs, gyp, argv) {
 log.verbose('checksum data', JSON.stringify(expectShasums))
 }

 function downloadNodeLib () {
async function downloadNodeLib () {
 log.verbose('on Windows; need to download `' + release.name + '.lib`...')
 const archs = ['ia32', 'x64', 'arm64']
 return archs.map(async (arch) => {
 const dir = path.resolve(tarExtractDir, arch)
 const targetLibPath = path.resolve(dir, release.name + '.lib')
 const { libUrl, libPath } = release[arch]
 const name = `${arch} ${release.name}.lib`
 log.verbose(name, 'dir', dir)
 log.verbose(name, 'url', libUrl)

 await fs.promises.mkdir(dir, { recursive: true })
 log.verbose('streaming', name, 'to:', targetLibPath)

 const res = await download(gyp, libUrl)

 if (res.status === 403 || res.status === 404) {
 if (arch === 'arm64') {
 // Arm64 is a newer platform on Windows and not all node distributions provide it.
 log.verbose(`${name} was not found in ${libUrl}`)
 } else {
 log.warn(`${name} was not found in ${libUrl}`)
 }
 return
 } else if (res.status !== 200) {
 throw new Error(`${res.status} status code downloading ${name}`)
 }
 const dir = path.resolve(tarExtractDir, arch)
 const targetLibPath = path.resolve(dir, release.name + '.lib')
 const { libUrl, libPath } = release[arch]
 const name = `${arch} ${release.name}.lib`
 log.verbose(name, 'dir', dir)
 log.verbose(name, 'url', libUrl)

 await fs.promises.mkdir(dir, { recursive: true })
 log.verbose('streaming', name, 'to:', targetLibPath)

 const res = await download(gyp, libUrl)

 // Since only required node.lib is downloaded throw error if it is not fetched
 if (res.status !== 200) {
 throw new Error(`${res.status} status code downloading ${name}`)
 }

 return streamPipeline(
 res.body,
 new ShaSum((_, checksum) => {
 contentShasums[libPath] = checksum
 log.verbose('content checksum', libPath, checksum)
 }),
 fs.createWriteStream(targetLibPath)
 )
 })
 return streamPipeline(
 res.body,
 new ShaSum((_, checksum) => {
 contentShasums[libPath] = checksum
 log.verbose('content checksum', libPath, checksum)
 }),
 fs.createWriteStream(targetLibPath)
 )
 } // downloadNodeLib()
 } // go()

Expand Down

2 changes: 1 addition & 1 deletion
package.json

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -12,7 +12,7 @@
 "gyp"
 ],
 "version": "9.3.1",
 "installVersion": 10,
@StefanStojanovicStefanStojanovicMay 26, 2023

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A question for people with more experience in node-gyp: Since a PR that changed installVersion from 9 to 10 landed yesterday in aaa117c do I need to increase it to 11 now, or not, because there was no release with installVersion: 10?
@dsanders11dsanders11Jun 7, 2023

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A question for people with more experience in node-gyp
I'm not more experienced, but my 2 cents: it's probably unnecessary for end users (since as you stated, no release went out), but AFAIK there's no harm in bumping it again, and it will prevent a situation where anyone who tested main (so folks working on node-gyp like you and I, or maybe even someone some where consuming node-gyp@main for whatever reason) and got their cache populated for installVersion: 10 won't end up in a broken situation.
So to me bumping it to 11 seems reasonable to prevent any problems.
 "installVersion": 11,
 "author": "Nathan Rajlich <nathan@tootallnate.net> (http://tootallnate.net)",
 "repository": {
 "type": "git",
Expand Down

2 changes: 1 addition & 1 deletion
test/test-download.js

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -180,7 +180,7 @@ describe('download', function () {
 await util.promisify(install)(prog, [])

 const data = await fs.promises.readFile(path.join(expectedDir, 'installVersion'), 'utf8')
 assert.strictEqual(data, '10\n', 'correct installVersion')
 assert.strictEqual(data, '11\n', 'correct installVersion')

 const list = await fs.promises.readdir(path.join(expectedDir, 'include/node'))
 assert.ok(list.includes('common.gypi'))
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
win,install: only download target_arch node.lib#2857

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

StefanStojanovic
merged 1 commit into

from

Jun 9, 2023
 Merged


win,install: only download target_arch node.lib
#2857
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


169 changes: 93 additions & 76 deletions
lib/install.js

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,10 @@ const streamPipeline = util.promisify(stream.pipeline)

async function install (fs, gyp, argv) {
 const release = processRelease(argv, gyp, process.version, process.release)
 // Detecting target_arch based on logic from create-cnfig-gyp.js. Used on Windows only.
Comment thread
StefanStojanovic marked this conversation as resolved.

 const arch = win ? (gyp.opts.target_arch || gyp.opts.arch || process.arch || 'ia32') : ''
 // Used to prevent downloading tarball if only new node.lib is required on Windows.
 let shouldDownloadTarball = true

 // Determine which node dev files version we are installing
 log.verbose('install', 'input version string %j', release.version)
Expand DownExpand Up@@ -92,6 +96,26 @@ async function install (fs, gyp, argv) {
 }
 }
 log.verbose('install', 'version is good')
 if (win) {
 log.verbose('on Windows; need to check node.lib')
 const nodeLibPath = path.resolve(devDir, arch, 'node.lib')
 try {
 await fs.promises.stat(nodeLibPath)
 } catch (err) {
 if (err.code === 'ENOENT') {
 log.verbose('install', `version not already installed for ${arch}, continuing with install`, release.version)
Comment thread
StefanStojanovic marked this conversation as resolved.

 try {
 shouldDownloadTarball = false
 return await go()
 } catch (err) {
 return rollback(err)
 }
 } else if (err.code === 'EACCES') {
 return eaccesFallback(err)
 }
 throw err
 }
 }
 } else {
 try {
 return await go()
Expand DownExpand Up@@ -179,66 +203,69 @@ async function install (fs, gyp, argv) {
 }

 // download the tarball and extract!
 // Ommited on Windows if only new node.lib is required

 // on Windows there can be file errors from tar if parallel installs
 // are happening (not uncommon with multiple native modules) so
 // extract the tarball to a temp directory first and then copy over
 const tarExtractDir = win ? await fs.promises.mkdtemp(path.join(os.tmpdir(), 'node-gyp-tmp-')) : devDir

 try {
 if (tarPath) {
 await tar.extract({
 file: tarPath,
 strip: 1,
 filter: isValid,
 onwarn,
 cwd: tarExtractDir
 })
 } else {
 try {
 const res = await download(gyp, release.tarballUrl)
 if (shouldDownloadTarball) {
 if (tarPath) {
 await tar.extract({
 file: tarPath,
 strip: 1,
 filter: isValid,
 onwarn,
 cwd: tarExtractDir
 })
 } else {
 try {
 const res = await download(gyp, release.tarballUrl)

 if (res.status !== 200) {
 throw new Error(`${res.status} response downloading ${release.tarballUrl}`)
 }
if (res.status !== 200) {
throw new Error(`${res.status} response downloading ${release.tarballUrl}`)
}

 await streamPipeline(
 res.body,
 // content checksum
 new ShaSum((_, checksum) => {
 const filename = path.basename(release.tarballUrl).trim()
 contentShasums[filename] = checksum
 log.verbose('content checksum', filename, checksum)
 }),
 tar.extract({
 strip: 1,
 cwd: tarExtractDir,
 filter: isValid,
 onwarn
 })
 )
 } catch (err) {
await streamPipeline(
res.body,
// content checksum
new ShaSum((_, checksum) => {
const filename = path.basename(release.tarballUrl).trim()
contentShasums[filename] = checksum
log.verbose('content checksum', filename, checksum)
}),
tar.extract({
strip: 1,
cwd: tarExtractDir,
filter: isValid,
onwarn
})
)
} catch (err) {
 // something went wrong downloading the tarball?
 if (err.code === 'ENOTFOUND') {
 throw new Error('This is most likely not a problem with node-gyp or the package itself and\n' +
if (err.code === 'ENOTFOUND') {
throw new Error('This is most likely not a problem with node-gyp or the package itself and\n' +
 'is related to network connectivity. In most cases you are behind a proxy or have bad \n' +
 'network settings.')
 }
 throw err
 }
 throw err
 }
 }

 // invoked after the tarball has finished being extracted
 if (extractErrors || extractCount === 0) {
 throw new Error('There was a fatal problem while downloading/extracting the tarball')
 }
// invoked after the tarball has finished being extracted
if (extractErrors || extractCount === 0) {
throw new Error('There was a fatal problem while downloading/extracting the tarball')
}

 log.verbose('tarball', 'done parsing tarball')
 log.verbose('tarball', 'done parsing tarball')
 }

 const installVersionPath = path.resolve(tarExtractDir, 'installVersion')
 await Promise.all([
// need to download node.lib
 ...(win ? downloadNodeLib() : []),
 // need to download node.lib
 ...(win ? [downloadNodeLib()] : []),
 // write the "installVersion" file
 fs.promises.writeFile(installVersionPath, gyp.package.installVersion + '\n'),
 // Only download SHASUMS.txt if we downloaded something in need of SHA verification
Expand DownExpand Up@@ -293,43 +320,33 @@ async function install (fs, gyp, argv) {
 log.verbose('checksum data', JSON.stringify(expectShasums))
 }

 function downloadNodeLib () {
async function downloadNodeLib () {
 log.verbose('on Windows; need to download `' + release.name + '.lib`...')
 const archs = ['ia32', 'x64', 'arm64']
 return archs.map(async (arch) => {
 const dir = path.resolve(tarExtractDir, arch)
 const targetLibPath = path.resolve(dir, release.name + '.lib')
 const { libUrl, libPath } = release[arch]
 const name = `${arch} ${release.name}.lib`
 log.verbose(name, 'dir', dir)
 log.verbose(name, 'url', libUrl)

 await fs.promises.mkdir(dir, { recursive: true })
 log.verbose('streaming', name, 'to:', targetLibPath)

 const res = await download(gyp, libUrl)

 if (res.status === 403 || res.status === 404) {
 if (arch === 'arm64') {
 // Arm64 is a newer platform on Windows and not all node distributions provide it.
 log.verbose(`${name} was not found in ${libUrl}`)
 } else {
 log.warn(`${name} was not found in ${libUrl}`)
 }
 return
 } else if (res.status !== 200) {
 throw new Error(`${res.status} status code downloading ${name}`)
 }
 const dir = path.resolve(tarExtractDir, arch)
 const targetLibPath = path.resolve(dir, release.name + '.lib')
 const { libUrl, libPath } = release[arch]
 const name = `${arch} ${release.name}.lib`
 log.verbose(name, 'dir', dir)
 log.verbose(name, 'url', libUrl)

 await fs.promises.mkdir(dir, { recursive: true })
 log.verbose('streaming', name, 'to:', targetLibPath)

 const res = await download(gyp, libUrl)

 // Since only required node.lib is downloaded throw error if it is not fetched
 if (res.status !== 200) {
 throw new Error(`${res.status} status code downloading ${name}`)
 }

 return streamPipeline(
 res.body,
 new ShaSum((_, checksum) => {
 contentShasums[libPath] = checksum
 log.verbose('content checksum', libPath, checksum)
 }),
 fs.createWriteStream(targetLibPath)
 )
 })
 return streamPipeline(
 res.body,
 new ShaSum((_, checksum) => {
 contentShasums[libPath] = checksum
 log.verbose('content checksum', libPath, checksum)
 }),
 fs.createWriteStream(targetLibPath)
 )
 } // downloadNodeLib()
 } // go()

Expand Down

2 changes: 1 addition & 1 deletion
package.json

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -12,7 +12,7 @@
 "gyp"
 ],
 "version": "9.3.1",
 "installVersion": 10,
@StefanStojanovicStefanStojanovicMay 26, 2023

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A question for people with more experience in node-gyp: Since a PR that changed installVersion from 9 to 10 landed yesterday in aaa117c do I need to increase it to 11 now, or not, because there was no release with installVersion: 10?
@dsanders11dsanders11Jun 7, 2023

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A question for people with more experience in node-gyp
I'm not more experienced, but my 2 cents: it's probably unnecessary for end users (since as you stated, no release went out), but AFAIK there's no harm in bumping it again, and it will prevent a situation where anyone who tested main (so folks working on node-gyp like you and I, or maybe even someone some where consuming node-gyp@main for whatever reason) and got their cache populated for installVersion: 10 won't end up in a broken situation.
So to me bumping it to 11 seems reasonable to prevent any problems.
 "installVersion": 11,
 "author": "Nathan Rajlich <nathan@tootallnate.net> (http://tootallnate.net)",
 "repository": {
 "type": "git",
Expand Down

2 changes: 1 addition & 1 deletion
test/test-download.js

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -180,7 +180,7 @@ describe('download', function () {
 await util.promisify(install)(prog, [])

 const data = await fs.promises.readFile(path.join(expectedDir, 'installVersion'), 'utf8')
 assert.strictEqual(data, '10\n', 'correct installVersion')
 assert.strictEqual(data, '11\n', 'correct installVersion')

 const list = await fs.promises.readdir(path.join(expectedDir, 'include/node'))
 assert.ok(list.includes('common.gypi'))
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
win,install: only download target_arch node.lib#2857

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

StefanStojanovic
merged 1 commit into

from

Jun 9, 2023
 Merged


win,install: only download target_arch node.lib
#2857
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


169 changes: 93 additions & 76 deletions
lib/install.js

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,10 @@ const streamPipeline = util.promisify(stream.pipeline)

async function install (fs, gyp, argv) {
 const release = processRelease(argv, gyp, process.version, process.release)
 // Detecting target_arch based on logic from create-cnfig-gyp.js. Used on Windows only.
Comment thread
StefanStojanovic marked this conversation as resolved.

 const arch = win ? (gyp.opts.target_arch || gyp.opts.arch || process.arch || 'ia32') : ''
 // Used to prevent downloading tarball if only new node.lib is required on Windows.
 let shouldDownloadTarball = true

 // Determine which node dev files version we are installing
 log.verbose('install', 'input version string %j', release.version)
Expand DownExpand Up@@ -92,6 +96,26 @@ async function install (fs, gyp, argv) {
 }
 }
 log.verbose('install', 'version is good')
 if (win) {
 log.verbose('on Windows; need to check node.lib')
 const nodeLibPath = path.resolve(devDir, arch, 'node.lib')
 try {
 await fs.promises.stat(nodeLibPath)
 } catch (err) {
 if (err.code === 'ENOENT') {
 log.verbose('install', `version not already installed for ${arch}, continuing with install`, release.version)
Comment thread
StefanStojanovic marked this conversation as resolved.

 try {
 shouldDownloadTarball = false
 return await go()
 } catch (err) {
 return rollback(err)
 }
 } else if (err.code === 'EACCES') {
 return eaccesFallback(err)
 }
 throw err
 }
 }
 } else {
 try {
 return await go()
Expand DownExpand Up@@ -179,66 +203,69 @@ async function install (fs, gyp, argv) {
 }

 // download the tarball and extract!
 // Ommited on Windows if only new node.lib is required

 // on Windows there can be file errors from tar if parallel installs
 // are happening (not uncommon with multiple native modules) so
 // extract the tarball to a temp directory first and then copy over
 const tarExtractDir = win ? await fs.promises.mkdtemp(path.join(os.tmpdir(), 'node-gyp-tmp-')) : devDir

 try {
 if (tarPath) {
 await tar.extract({
 file: tarPath,
 strip: 1,
 filter: isValid,
 onwarn,
 cwd: tarExtractDir
 })
 } else {
 try {
 const res = await download(gyp, release.tarballUrl)
 if (shouldDownloadTarball) {
 if (tarPath) {
 await tar.extract({
 file: tarPath,
 strip: 1,
 filter: isValid,
 onwarn,
 cwd: tarExtractDir
 })
 } else {
 try {
 const res = await download(gyp, release.tarballUrl)

 if (res.status !== 200) {
 throw new Error(`${res.status} response downloading ${release.tarballUrl}`)
 }
if (res.status !== 200) {
throw new Error(`${res.status} response downloading ${release.tarballUrl}`)
}

 await streamPipeline(
 res.body,
 // content checksum
 new ShaSum((_, checksum) => {
 const filename = path.basename(release.tarballUrl).trim()
 contentShasums[filename] = checksum
 log.verbose('content checksum', filename, checksum)
 }),
 tar.extract({
 strip: 1,
 cwd: tarExtractDir,
 filter: isValid,
 onwarn
 })
 )
 } catch (err) {
await streamPipeline(
res.body,
// content checksum
new ShaSum((_, checksum) => {
const filename = path.basename(release.tarballUrl).trim()
contentShasums[filename] = checksum
log.verbose('content checksum', filename, checksum)
}),
tar.extract({
strip: 1,
cwd: tarExtractDir,
filter: isValid,
onwarn
})
)
} catch (err) {
 // something went wrong downloading the tarball?
 if (err.code === 'ENOTFOUND') {
 throw new Error('This is most likely not a problem with node-gyp or the package itself and\n' +
if (err.code === 'ENOTFOUND') {
throw new Error('This is most likely not a problem with node-gyp or the package itself and\n' +
 'is related to network connectivity. In most cases you are behind a proxy or have bad \n' +
 'network settings.')
 }
 throw err
 }
 throw err
 }
 }

 // invoked after the tarball has finished being extracted
 if (extractErrors || extractCount === 0) {
 throw new Error('There was a fatal problem while downloading/extracting the tarball')
 }
// invoked after the tarball has finished being extracted
if (extractErrors || extractCount === 0) {
throw new Error('There was a fatal problem while downloading/extracting the tarball')
}

 log.verbose('tarball', 'done parsing tarball')
 log.verbose('tarball', 'done parsing tarball')
 }

 const installVersionPath = path.resolve(tarExtractDir, 'installVersion')
 await Promise.all([
// need to download node.lib
 ...(win ? downloadNodeLib() : []),
 // need to download node.lib
 ...(win ? [downloadNodeLib()] : []),
 // write the "installVersion" file
 fs.promises.writeFile(installVersionPath, gyp.package.installVersion + '\n'),
 // Only download SHASUMS.txt if we downloaded something in need of SHA verification
Expand DownExpand Up@@ -293,43 +320,33 @@ async function install (fs, gyp, argv) {
 log.verbose('checksum data', JSON.stringify(expectShasums))
 }

 function downloadNodeLib () {
async function downloadNodeLib () {
 log.verbose('on Windows; need to download `' + release.name + '.lib`...')
 const archs = ['ia32', 'x64', 'arm64']
 return archs.map(async (arch) => {
 const dir = path.resolve(tarExtractDir, arch)
 const targetLibPath = path.resolve(dir, release.name + '.lib')
 const { libUrl, libPath } = release[arch]
 const name = `${arch} ${release.name}.lib`
 log.verbose(name, 'dir', dir)
 log.verbose(name, 'url', libUrl)

 await fs.promises.mkdir(dir, { recursive: true })
 log.verbose('streaming', name, 'to:', targetLibPath)

 const res = await download(gyp, libUrl)

 if (res.status === 403 || res.status === 404) {
 if (arch === 'arm64') {
 // Arm64 is a newer platform on Windows and not all node distributions provide it.
 log.verbose(`${name} was not found in ${libUrl}`)
 } else {
 log.warn(`${name} was not found in ${libUrl}`)
 }
 return
 } else if (res.status !== 200) {
 throw new Error(`${res.status} status code downloading ${name}`)
 }
 const dir = path.resolve(tarExtractDir, arch)
 const targetLibPath = path.resolve(dir, release.name + '.lib')
 const { libUrl, libPath } = release[arch]
 const name = `${arch} ${release.name}.lib`
 log.verbose(name, 'dir', dir)
 log.verbose(name, 'url', libUrl)

 await fs.promises.mkdir(dir, { recursive: true })
 log.verbose('streaming', name, 'to:', targetLibPath)

 const res = await download(gyp, libUrl)

 // Since only required node.lib is downloaded throw error if it is not fetched
 if (res.status !== 200) {
 throw new Error(`${res.status} status code downloading ${name}`)
 }

 return streamPipeline(
 res.body,
 new ShaSum((_, checksum) => {
 contentShasums[libPath] = checksum
 log.verbose('content checksum', libPath, checksum)
 }),
 fs.createWriteStream(targetLibPath)
 )
 })
 return streamPipeline(
 res.body,
 new ShaSum((_, checksum) => {
 contentShasums[libPath] = checksum
 log.verbose('content checksum', libPath, checksum)
 }),
 fs.createWriteStream(targetLibPath)
 )
 } // downloadNodeLib()
 } // go()

Expand Down

2 changes: 1 addition & 1 deletion
package.json

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -12,7 +12,7 @@
 "gyp"
 ],
 "version": "9.3.1",
 "installVersion": 10,
@StefanStojanovicStefanStojanovicMay 26, 2023

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A question for people with more experience in node-gyp: Since a PR that changed installVersion from 9 to 10 landed yesterday in aaa117c do I need to increase it to 11 now, or not, because there was no release with installVersion: 10?
@dsanders11dsanders11Jun 7, 2023

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A question for people with more experience in node-gyp
I'm not more experienced, but my 2 cents: it's probably unnecessary for end users (since as you stated, no release went out), but AFAIK there's no harm in bumping it again, and it will prevent a situation where anyone who tested main (so folks working on node-gyp like you and I, or maybe even someone some where consuming node-gyp@main for whatever reason) and got their cache populated for installVersion: 10 won't end up in a broken situation.
So to me bumping it to 11 seems reasonable to prevent any problems.
 "installVersion": 11,
 "author": "Nathan Rajlich <nathan@tootallnate.net> (http://tootallnate.net)",
 "repository": {
 "type": "git",
Expand Down

2 changes: 1 addition & 1 deletion
test/test-download.js

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -180,7 +180,7 @@ describe('download', function () {
 await util.promisify(install)(prog, [])

 const data = await fs.promises.readFile(path.join(expectedDir, 'installVersion'), 'utf8')
 assert.strictEqual(data, '10\n', 'correct installVersion')
 assert.strictEqual(data, '11\n', 'correct installVersion')

 const list = await fs.promises.readdir(path.join(expectedDir, 'include/node'))
 assert.ok(list.includes('common.gypi'))
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
win,install: only download target_arch node.lib#2857

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

StefanStojanovic
merged 1 commit into

from

Jun 9, 2023
 Merged


win,install: only download target_arch node.lib
#2857
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


169 changes: 93 additions & 76 deletions
lib/install.js

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,10 @@ const streamPipeline = util.promisify(stream.pipeline)

async function install (fs, gyp, argv) {
 const release = processRelease(argv, gyp, process.version, process.release)
 // Detecting target_arch based on logic from create-cnfig-gyp.js. Used on Windows only.
Comment thread
StefanStojanovic marked this conversation as resolved.

 const arch = win ? (gyp.opts.target_arch || gyp.opts.arch || process.arch || 'ia32') : ''
 // Used to prevent downloading tarball if only new node.lib is required on Windows.
 let shouldDownloadTarball = true

 // Determine which node dev files version we are installing
 log.verbose('install', 'input version string %j', release.version)
Expand DownExpand Up@@ -92,6 +96,26 @@ async function install (fs, gyp, argv) {
 }
 }
 log.verbose('install', 'version is good')
 if (win) {
 log.verbose('on Windows; need to check node.lib')
 const nodeLibPath = path.resolve(devDir, arch, 'node.lib')
 try {
 await fs.promises.stat(nodeLibPath)
 } catch (err) {
 if (err.code === 'ENOENT') {
 log.verbose('install', `version not already installed for ${arch}, continuing with install`, release.version)
Comment thread
StefanStojanovic marked this conversation as resolved.

 try {
 shouldDownloadTarball = false
 return await go()
 } catch (err) {
 return rollback(err)
 }
 } else if (err.code === 'EACCES') {
 return eaccesFallback(err)
 }
 throw err
 }
 }
 } else {
 try {
 return await go()
Expand DownExpand Up@@ -179,66 +203,69 @@ async function install (fs, gyp, argv) {
 }

 // download the tarball and extract!
 // Ommited on Windows if only new node.lib is required

 // on Windows there can be file errors from tar if parallel installs
 // are happening (not uncommon with multiple native modules) so
 // extract the tarball to a temp directory first and then copy over
 const tarExtractDir = win ? await fs.promises.mkdtemp(path.join(os.tmpdir(), 'node-gyp-tmp-')) : devDir

 try {
 if (tarPath) {
 await tar.extract({
 file: tarPath,
 strip: 1,
 filter: isValid,
 onwarn,
 cwd: tarExtractDir
 })
 } else {
 try {
 const res = await download(gyp, release.tarballUrl)
 if (shouldDownloadTarball) {
 if (tarPath) {
 await tar.extract({
 file: tarPath,
 strip: 1,
 filter: isValid,
 onwarn,
 cwd: tarExtractDir
 })
 } else {
 try {
 const res = await download(gyp, release.tarballUrl)

 if (res.status !== 200) {
 throw new Error(`${res.status} response downloading ${release.tarballUrl}`)
 }
if (res.status !== 200) {
throw new Error(`${res.status} response downloading ${release.tarballUrl}`)
}

 await streamPipeline(
 res.body,
 // content checksum
 new ShaSum((_, checksum) => {
 const filename = path.basename(release.tarballUrl).trim()
 contentShasums[filename] = checksum
 log.verbose('content checksum', filename, checksum)
 }),
 tar.extract({
 strip: 1,
 cwd: tarExtractDir,
 filter: isValid,
 onwarn
 })
 )
 } catch (err) {
await streamPipeline(
res.body,
// content checksum
new ShaSum((_, checksum) => {
const filename = path.basename(release.tarballUrl).trim()
contentShasums[filename] = checksum
log.verbose('content checksum', filename, checksum)
}),
tar.extract({
strip: 1,
cwd: tarExtractDir,
filter: isValid,
onwarn
})
)
} catch (err) {
 // something went wrong downloading the tarball?
 if (err.code === 'ENOTFOUND') {
 throw new Error('This is most likely not a problem with node-gyp or the package itself and\n' +
if (err.code === 'ENOTFOUND') {
throw new Error('This is most likely not a problem with node-gyp or the package itself and\n' +
 'is related to network connectivity. In most cases you are behind a proxy or have bad \n' +
 'network settings.')
 }
 throw err
 }
 throw err
 }
 }

 // invoked after the tarball has finished being extracted
 if (extractErrors || extractCount === 0) {
 throw new Error('There was a fatal problem while downloading/extracting the tarball')
 }
// invoked after the tarball has finished being extracted
if (extractErrors || extractCount === 0) {
throw new Error('There was a fatal problem while downloading/extracting the tarball')
}

 log.verbose('tarball', 'done parsing tarball')
 log.verbose('tarball', 'done parsing tarball')
 }

 const installVersionPath = path.resolve(tarExtractDir, 'installVersion')
 await Promise.all([
// need to download node.lib
 ...(win ? downloadNodeLib() : []),
 // need to download node.lib
 ...(win ? [downloadNodeLib()] : []),
 // write the "installVersion" file
 fs.promises.writeFile(installVersionPath, gyp.package.installVersion + '\n'),
 // Only download SHASUMS.txt if we downloaded something in need of SHA verification
Expand DownExpand Up@@ -293,43 +320,33 @@ async function install (fs, gyp, argv) {
 log.verbose('checksum data', JSON.stringify(expectShasums))
 }

 function downloadNodeLib () {
async function downloadNodeLib () {
 log.verbose('on Windows; need to download `' + release.name + '.lib`...')
 const archs = ['ia32', 'x64', 'arm64']
 return archs.map(async (arch) => {
 const dir = path.resolve(tarExtractDir, arch)
 const targetLibPath = path.resolve(dir, release.name + '.lib')
 const { libUrl, libPath } = release[arch]
 const name = `${arch} ${release.name}.lib`
 log.verbose(name, 'dir', dir)
 log.verbose(name, 'url', libUrl)

 await fs.promises.mkdir(dir, { recursive: true })
 log.verbose('streaming', name, 'to:', targetLibPath)

 const res = await download(gyp, libUrl)

 if (res.status === 403 || res.status === 404) {
 if (arch === 'arm64') {
 // Arm64 is a newer platform on Windows and not all node distributions provide it.
 log.verbose(`${name} was not found in ${libUrl}`)
 } else {
 log.warn(`${name} was not found in ${libUrl}`)
 }
 return
 } else if (res.status !== 200) {
 throw new Error(`${res.status} status code downloading ${name}`)
 }
 const dir = path.resolve(tarExtractDir, arch)
 const targetLibPath = path.resolve(dir, release.name + '.lib')
 const { libUrl, libPath } = release[arch]
 const name = `${arch} ${release.name}.lib`
 log.verbose(name, 'dir', dir)
 log.verbose(name, 'url', libUrl)

 await fs.promises.mkdir(dir, { recursive: true })
 log.verbose('streaming', name, 'to:', targetLibPath)

 const res = await download(gyp, libUrl)

 // Since only required node.lib is downloaded throw error if it is not fetched
 if (res.status !== 200) {
 throw new Error(`${res.status} status code downloading ${name}`)
 }

 return streamPipeline(
 res.body,
 new ShaSum((_, checksum) => {
 contentShasums[libPath] = checksum
 log.verbose('content checksum', libPath, checksum)
 }),
 fs.createWriteStream(targetLibPath)
 )
 })
 return streamPipeline(
 res.body,
 new ShaSum((_, checksum) => {
 contentShasums[libPath] = checksum
 log.verbose('content checksum', libPath, checksum)
 }),
 fs.createWriteStream(targetLibPath)
 )
 } // downloadNodeLib()
 } // go()

Expand Down

2 changes: 1 addition & 1 deletion
package.json

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -12,7 +12,7 @@
 "gyp"
 ],
 "version": "9.3.1",
 "installVersion": 10,
@StefanStojanovicStefanStojanovicMay 26, 2023

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A question for people with more experience in node-gyp: Since a PR that changed installVersion from 9 to 10 landed yesterday in aaa117c do I need to increase it to 11 now, or not, because there was no release with installVersion: 10?
@dsanders11dsanders11Jun 7, 2023

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A question for people with more experience in node-gyp
I'm not more experienced, but my 2 cents: it's probably unnecessary for end users (since as you stated, no release went out), but AFAIK there's no harm in bumping it again, and it will prevent a situation where anyone who tested main (so folks working on node-gyp like you and I, or maybe even someone some where consuming node-gyp@main for whatever reason) and got their cache populated for installVersion: 10 won't end up in a broken situation.
So to me bumping it to 11 seems reasonable to prevent any problems.
 "installVersion": 11,
 "author": "Nathan Rajlich <nathan@tootallnate.net> (http://tootallnate.net)",
 "repository": {
 "type": "git",
Expand Down

2 changes: 1 addition & 1 deletion
test/test-download.js

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -180,7 +180,7 @@ describe('download', function () {
 await util.promisify(install)(prog, [])

 const data = await fs.promises.readFile(path.join(expectedDir, 'installVersion'), 'utf8')
 assert.strictEqual(data, '10\n', 'correct installVersion')
 assert.strictEqual(data, '11\n', 'correct installVersion')

 const list = await fs.promises.readdir(path.join(expectedDir, 'include/node'))
 assert.ok(list.includes('common.gypi'))
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
win,install: only download target_arch node.lib#2857

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

StefanStojanovic
merged 1 commit into

from

Jun 9, 2023
 Merged


win,install: only download target_arch node.lib
#2857
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


169 changes: 93 additions & 76 deletions
lib/install.js

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,10 @@ const streamPipeline = util.promisify(stream.pipeline)

async function install (fs, gyp, argv) {
 const release = processRelease(argv, gyp, process.version, process.release)
 // Detecting target_arch based on logic from create-cnfig-gyp.js. Used on Windows only.
Comment thread
StefanStojanovic marked this conversation as resolved.

 const arch = win ? (gyp.opts.target_arch || gyp.opts.arch || process.arch || 'ia32') : ''
 // Used to prevent downloading tarball if only new node.lib is required on Windows.
 let shouldDownloadTarball = true

 // Determine which node dev files version we are installing
 log.verbose('install', 'input version string %j', release.version)
Expand DownExpand Up@@ -92,6 +96,26 @@ async function install (fs, gyp, argv) {
 }
 }
 log.verbose('install', 'version is good')
 if (win) {
 log.verbose('on Windows; need to check node.lib')
 const nodeLibPath = path.resolve(devDir, arch, 'node.lib')
 try {
 await fs.promises.stat(nodeLibPath)
 } catch (err) {
 if (err.code === 'ENOENT') {
 log.verbose('install', `version not already installed for ${arch}, continuing with install`, release.version)
Comment thread
StefanStojanovic marked this conversation as resolved.

 try {
 shouldDownloadTarball = false
 return await go()
 } catch (err) {
 return rollback(err)
 }
 } else if (err.code === 'EACCES') {
 return eaccesFallback(err)
 }
 throw err
 }
 }
 } else {
 try {
 return await go()
Expand DownExpand Up@@ -179,66 +203,69 @@ async function install (fs, gyp, argv) {
 }

 // download the tarball and extract!
 // Ommited on Windows if only new node.lib is required

 // on Windows there can be file errors from tar if parallel installs
 // are happening (not uncommon with multiple native modules) so
 // extract the tarball to a temp directory first and then copy over
 const tarExtractDir = win ? await fs.promises.mkdtemp(path.join(os.tmpdir(), 'node-gyp-tmp-')) : devDir

 try {
 if (tarPath) {
 await tar.extract({
 file: tarPath,
 strip: 1,
 filter: isValid,
 onwarn,
 cwd: tarExtractDir
 })
 } else {
 try {
 const res = await download(gyp, release.tarballUrl)
 if (shouldDownloadTarball) {
 if (tarPath) {
 await tar.extract({
 file: tarPath,
 strip: 1,
 filter: isValid,
 onwarn,
 cwd: tarExtractDir
 })
 } else {
 try {
 const res = await download(gyp, release.tarballUrl)

 if (res.status !== 200) {
 throw new Error(`${res.status} response downloading ${release.tarballUrl}`)
 }
if (res.status !== 200) {
throw new Error(`${res.status} response downloading ${release.tarballUrl}`)
}

 await streamPipeline(
 res.body,
 // content checksum
 new ShaSum((_, checksum) => {
 const filename = path.basename(release.tarballUrl).trim()
 contentShasums[filename] = checksum
 log.verbose('content checksum', filename, checksum)
 }),
 tar.extract({
 strip: 1,
 cwd: tarExtractDir,
 filter: isValid,
 onwarn
 })
 )
 } catch (err) {
await streamPipeline(
res.body,
// content checksum
new ShaSum((_, checksum) => {
const filename = path.basename(release.tarballUrl).trim()
contentShasums[filename] = checksum
log.verbose('content checksum', filename, checksum)
}),
tar.extract({
strip: 1,
cwd: tarExtractDir,
filter: isValid,
onwarn
})
)
} catch (err) {
 // something went wrong downloading the tarball?
 if (err.code === 'ENOTFOUND') {
 throw new Error('This is most likely not a problem with node-gyp or the package itself and\n' +
if (err.code === 'ENOTFOUND') {
throw new Error('This is most likely not a problem with node-gyp or the package itself and\n' +
 'is related to network connectivity. In most cases you are behind a proxy or have bad \n' +
 'network settings.')
 }
 throw err
 }
 throw err
 }
 }

 // invoked after the tarball has finished being extracted
 if (extractErrors || extractCount === 0) {
 throw new Error('There was a fatal problem while downloading/extracting the tarball')
 }
// invoked after the tarball has finished being extracted
if (extractErrors || extractCount === 0) {
throw new Error('There was a fatal problem while downloading/extracting the tarball')
}

 log.verbose('tarball', 'done parsing tarball')
 log.verbose('tarball', 'done parsing tarball')
 }

 const installVersionPath = path.resolve(tarExtractDir, 'installVersion')
 await Promise.all([
// need to download node.lib
 ...(win ? downloadNodeLib() : []),
 // need to download node.lib
 ...(win ? [downloadNodeLib()] : []),
 // write the "installVersion" file
 fs.promises.writeFile(installVersionPath, gyp.package.installVersion + '\n'),
 // Only download SHASUMS.txt if we downloaded something in need of SHA verification
Expand DownExpand Up@@ -293,43 +320,33 @@ async function install (fs, gyp, argv) {
 log.verbose('checksum data', JSON.stringify(expectShasums))
 }

 function downloadNodeLib () {
async function downloadNodeLib () {
 log.verbose('on Windows; need to download `' + release.name + '.lib`...')
 const archs = ['ia32', 'x64', 'arm64']
 return archs.map(async (arch) => {
 const dir = path.resolve(tarExtractDir, arch)
 const targetLibPath = path.resolve(dir, release.name + '.lib')
 const { libUrl, libPath } = release[arch]
 const name = `${arch} ${release.name}.lib`
 log.verbose(name, 'dir', dir)
 log.verbose(name, 'url', libUrl)

 await fs.promises.mkdir(dir, { recursive: true })
 log.verbose('streaming', name, 'to:', targetLibPath)

 const res = await download(gyp, libUrl)

 if (res.status === 403 || res.status === 404) {
 if (arch === 'arm64') {
 // Arm64 is a newer platform on Windows and not all node distributions provide it.
 log.verbose(`${name} was not found in ${libUrl}`)
 } else {
 log.warn(`${name} was not found in ${libUrl}`)
 }
 return
 } else if (res.status !== 200) {
 throw new Error(`${res.status} status code downloading ${name}`)
 }
 const dir = path.resolve(tarExtractDir, arch)
 const targetLibPath = path.resolve(dir, release.name + '.lib')
 const { libUrl, libPath } = release[arch]
 const name = `${arch} ${release.name}.lib`
 log.verbose(name, 'dir', dir)
 log.verbose(name, 'url', libUrl)

 await fs.promises.mkdir(dir, { recursive: true })
 log.verbose('streaming', name, 'to:', targetLibPath)

 const res = await download(gyp, libUrl)

 // Since only required node.lib is downloaded throw error if it is not fetched
 if (res.status !== 200) {
 throw new Error(`${res.status} status code downloading ${name}`)
 }

 return streamPipeline(
 res.body,
 new ShaSum((_, checksum) => {
 contentShasums[libPath] = checksum
 log.verbose('content checksum', libPath, checksum)
 }),
 fs.createWriteStream(targetLibPath)
 )
 })
 return streamPipeline(
 res.body,
 new ShaSum((_, checksum) => {
 contentShasums[libPath] = checksum
 log.verbose('content checksum', libPath, checksum)
 }),
 fs.createWriteStream(targetLibPath)
 )
 } // downloadNodeLib()
 } // go()

Expand Down

2 changes: 1 addition & 1 deletion
package.json

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -12,7 +12,7 @@
 "gyp"
 ],
 "version": "9.3.1",
 "installVersion": 10,
@StefanStojanovicStefanStojanovicMay 26, 2023

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A question for people with more experience in node-gyp: Since a PR that changed installVersion from 9 to 10 landed yesterday in aaa117c do I need to increase it to 11 now, or not, because there was no release with installVersion: 10?
@dsanders11dsanders11Jun 7, 2023

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A question for people with more experience in node-gyp
I'm not more experienced, but my 2 cents: it's probably unnecessary for end users (since as you stated, no release went out), but AFAIK there's no harm in bumping it again, and it will prevent a situation where anyone who tested main (so folks working on node-gyp like you and I, or maybe even someone some where consuming node-gyp@main for whatever reason) and got their cache populated for installVersion: 10 won't end up in a broken situation.
So to me bumping it to 11 seems reasonable to prevent any problems.
 "installVersion": 11,
 "author": "Nathan Rajlich <nathan@tootallnate.net> (http://tootallnate.net)",
 "repository": {
 "type": "git",
Expand Down

2 changes: 1 addition & 1 deletion
test/test-download.js

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -180,7 +180,7 @@ describe('download', function () {
 await util.promisify(install)(prog, [])

 const data = await fs.promises.readFile(path.join(expectedDir, 'installVersion'), 'utf8')
 assert.strictEqual(data, '10\n', 'correct installVersion')
 assert.strictEqual(data, '11\n', 'correct installVersion')

 const list = await fs.promises.readdir(path.join(expectedDir, 'include/node'))
 assert.ok(list.includes('common.gypi'))
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
win,install: only download target_arch node.lib#2857

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

StefanStojanovic
merged 1 commit into

from

Jun 9, 2023
 Merged


win,install: only download target_arch node.lib
#2857
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


169 changes: 93 additions & 76 deletions
lib/install.js

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,10 @@ const streamPipeline = util.promisify(stream.pipeline)

async function install (fs, gyp, argv) {
 const release = processRelease(argv, gyp, process.version, process.release)
 // Detecting target_arch based on logic from create-cnfig-gyp.js. Used on Windows only.
Comment thread
StefanStojanovic marked this conversation as resolved.

 const arch = win ? (gyp.opts.target_arch || gyp.opts.arch || process.arch || 'ia32') : ''
 // Used to prevent downloading tarball if only new node.lib is required on Windows.
 let shouldDownloadTarball = true

 // Determine which node dev files version we are installing
 log.verbose('install', 'input version string %j', release.version)
Expand DownExpand Up@@ -92,6 +96,26 @@ async function install (fs, gyp, argv) {
 }
 }
 log.verbose('install', 'version is good')
 if (win) {
 log.verbose('on Windows; need to check node.lib')
 const nodeLibPath = path.resolve(devDir, arch, 'node.lib')
 try {
 await fs.promises.stat(nodeLibPath)
 } catch (err) {
 if (err.code === 'ENOENT') {
 log.verbose('install', `version not already installed for ${arch}, continuing with install`, release.version)
Comment thread
StefanStojanovic marked this conversation as resolved.

 try {
 shouldDownloadTarball = false
 return await go()
 } catch (err) {
 return rollback(err)
 }
 } else if (err.code === 'EACCES') {
 return eaccesFallback(err)
 }
 throw err
 }
 }
 } else {
 try {
 return await go()
Expand DownExpand Up@@ -179,66 +203,69 @@ async function install (fs, gyp, argv) {
 }

 // download the tarball and extract!
 // Ommited on Windows if only new node.lib is required

 // on Windows there can be file errors from tar if parallel installs
 // are happening (not uncommon with multiple native modules) so
 // extract the tarball to a temp directory first and then copy over
 const tarExtractDir = win ? await fs.promises.mkdtemp(path.join(os.tmpdir(), 'node-gyp-tmp-')) : devDir

 try {
 if (tarPath) {
 await tar.extract({
 file: tarPath,
 strip: 1,
 filter: isValid,
 onwarn,
 cwd: tarExtractDir
 })
 } else {
 try {
 const res = await download(gyp, release.tarballUrl)
 if (shouldDownloadTarball) {
 if (tarPath) {
 await tar.extract({
 file: tarPath,
 strip: 1,
 filter: isValid,
 onwarn,
 cwd: tarExtractDir
 })
 } else {
 try {
 const res = await download(gyp, release.tarballUrl)

 if (res.status !== 200) {
 throw new Error(`${res.status} response downloading ${release.tarballUrl}`)
 }
if (res.status !== 200) {
throw new Error(`${res.status} response downloading ${release.tarballUrl}`)
}

 await streamPipeline(
 res.body,
 // content checksum
 new ShaSum((_, checksum) => {
 const filename = path.basename(release.tarballUrl).trim()
 contentShasums[filename] = checksum
 log.verbose('content checksum', filename, checksum)
 }),
 tar.extract({
 strip: 1,
 cwd: tarExtractDir,
 filter: isValid,
 onwarn
 })
 )
 } catch (err) {
await streamPipeline(
res.body,
// content checksum
new ShaSum((_, checksum) => {
const filename = path.basename(release.tarballUrl).trim()
contentShasums[filename] = checksum
log.verbose('content checksum', filename, checksum)
}),
tar.extract({
strip: 1,
cwd: tarExtractDir,
filter: isValid,
onwarn
})
)
} catch (err) {
 // something went wrong downloading the tarball?
 if (err.code === 'ENOTFOUND') {
 throw new Error('This is most likely not a problem with node-gyp or the package itself and\n' +
if (err.code === 'ENOTFOUND') {
throw new Error('This is most likely not a problem with node-gyp or the package itself and\n' +
 'is related to network connectivity. In most cases you are behind a proxy or have bad \n' +
 'network settings.')
 }
 throw err
 }
 throw err
 }
 }

 // invoked after the tarball has finished being extracted
 if (extractErrors || extractCount === 0) {
 throw new Error('There was a fatal problem while downloading/extracting the tarball')
 }
// invoked after the tarball has finished being extracted
if (extractErrors || extractCount === 0) {
throw new Error('There was a fatal problem while downloading/extracting the tarball')
}

 log.verbose('tarball', 'done parsing tarball')
 log.verbose('tarball', 'done parsing tarball')
 }

 const installVersionPath = path.resolve(tarExtractDir, 'installVersion')
 await Promise.all([
// need to download node.lib
 ...(win ? downloadNodeLib() : []),
 // need to download node.lib
 ...(win ? [downloadNodeLib()] : []),
 // write the "installVersion" file
 fs.promises.writeFile(installVersionPath, gyp.package.installVersion + '\n'),
 // Only download SHASUMS.txt if we downloaded something in need of SHA verification
Expand DownExpand Up@@ -293,43 +320,33 @@ async function install (fs, gyp, argv) {
 log.verbose('checksum data', JSON.stringify(expectShasums))
 }

 function downloadNodeLib () {
async function downloadNodeLib () {
 log.verbose('on Windows; need to download `' + release.name + '.lib`...')
 const archs = ['ia32', 'x64', 'arm64']
 return archs.map(async (arch) => {
 const dir = path.resolve(tarExtractDir, arch)
 const targetLibPath = path.resolve(dir, release.name + '.lib')
 const { libUrl, libPath } = release[arch]
 const name = `${arch} ${release.name}.lib`
 log.verbose(name, 'dir', dir)
 log.verbose(name, 'url', libUrl)

 await fs.promises.mkdir(dir, { recursive: true })
 log.verbose('streaming', name, 'to:', targetLibPath)

 const res = await download(gyp, libUrl)

 if (res.status === 403 || res.status === 404) {
 if (arch === 'arm64') {
 // Arm64 is a newer platform on Windows and not all node distributions provide it.
 log.verbose(`${name} was not found in ${libUrl}`)
 } else {
 log.warn(`${name} was not found in ${libUrl}`)
 }
 return
 } else if (res.status !== 200) {
 throw new Error(`${res.status} status code downloading ${name}`)
 }
 const dir = path.resolve(tarExtractDir, arch)
 const targetLibPath = path.resolve(dir, release.name + '.lib')
 const { libUrl, libPath } = release[arch]
 const name = `${arch} ${release.name}.lib`
 log.verbose(name, 'dir', dir)
 log.verbose(name, 'url', libUrl)

 await fs.promises.mkdir(dir, { recursive: true })
 log.verbose('streaming', name, 'to:', targetLibPath)

 const res = await download(gyp, libUrl)

 // Since only required node.lib is downloaded throw error if it is not fetched
 if (res.status !== 200) {
 throw new Error(`${res.status} status code downloading ${name}`)
 }

 return streamPipeline(
 res.body,
 new ShaSum((_, checksum) => {
 contentShasums[libPath] = checksum
 log.verbose('content checksum', libPath, checksum)
 }),
 fs.createWriteStream(targetLibPath)
 )
 })
 return streamPipeline(
 res.body,
 new ShaSum((_, checksum) => {
 contentShasums[libPath] = checksum
 log.verbose('content checksum', libPath, checksum)
 }),
 fs.createWriteStream(targetLibPath)
 )
 } // downloadNodeLib()
 } // go()

Expand Down

2 changes: 1 addition & 1 deletion
package.json

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -12,7 +12,7 @@
 "gyp"
 ],
 "version": "9.3.1",
 "installVersion": 10,
@StefanStojanovicStefanStojanovicMay 26, 2023

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A question for people with more experience in node-gyp: Since a PR that changed installVersion from 9 to 10 landed yesterday in aaa117c do I need to increase it to 11 now, or not, because there was no release with installVersion: 10?
@dsanders11dsanders11Jun 7, 2023

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A question for people with more experience in node-gyp
I'm not more experienced, but my 2 cents: it's probably unnecessary for end users (since as you stated, no release went out), but AFAIK there's no harm in bumping it again, and it will prevent a situation where anyone who tested main (so folks working on node-gyp like you and I, or maybe even someone some where consuming node-gyp@main for whatever reason) and got their cache populated for installVersion: 10 won't end up in a broken situation.
So to me bumping it to 11 seems reasonable to prevent any problems.
 "installVersion": 11,
 "author": "Nathan Rajlich <nathan@tootallnate.net> (http://tootallnate.net)",
 "repository": {
 "type": "git",
Expand Down

2 changes: 1 addition & 1 deletion
test/test-download.js

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -180,7 +180,7 @@ describe('download', function () {
 await util.promisify(install)(prog, [])

 const data = await fs.promises.readFile(path.join(expectedDir, 'installVersion'), 'utf8')
 assert.strictEqual(data, '10\n', 'correct installVersion')
 assert.strictEqual(data, '11\n', 'correct installVersion')

 const list = await fs.promises.readdir(path.join(expectedDir, 'include/node'))
 assert.ok(list.includes('common.gypi'))
Expand Down

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
win,install: only download target_arch node.lib#2857

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

StefanStojanovic
merged 1 commit into

from

Jun 9, 2023
 Merged


win,install: only download target_arch node.lib
#2857
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


169 changes: 93 additions & 76 deletions
lib/install.js

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -22,6 +22,10 @@ const streamPipeline = util.promisify(stream.pipeline)

async function install (fs, gyp, argv) {
 const release = processRelease(argv, gyp, process.version, process.release)
 // Detecting target_arch based on logic from create-cnfig-gyp.js. Used on Windows only.
Comment thread
StefanStojanovic marked this conversation as resolved.

 const arch = win ? (gyp.opts.target_arch || gyp.opts.arch || process.arch || 'ia32') : ''
 // Used to prevent downloading tarball if only new node.lib is required on Windows.
 let shouldDownloadTarball = true

 // Determine which node dev files version we are installing
 log.verbose('install', 'input version string %j', release.version)
Expand DownExpand Up@@ -92,6 +96,26 @@ async function install (fs, gyp, argv) {
 }
 }
 log.verbose('install', 'version is good')
 if (win) {
 log.verbose('on Windows; need to check node.lib')
 const nodeLibPath = path.resolve(devDir, arch, 'node.lib')
 try {
 await fs.promises.stat(nodeLibPath)
 } catch (err) {
 if (err.code === 'ENOENT') {
 log.verbose('install', `version not already installed for ${arch}, continuing with install`, release.version)
Comment thread
StefanStojanovic marked this conversation as resolved.

 try {
 shouldDownloadTarball = false
 return await go()
 } catch (err) {
 return rollback(err)
 }
 } else if (err.code === 'EACCES') {
 return eaccesFallback(err)
 }
 throw err
 }
 }
 } else {
 try {
 return await go()
Expand DownExpand Up@@ -179,66 +203,69 @@ async function install (fs, gyp, argv) {
 }

 // download the tarball and extract!
 // Ommited on Windows if only new node.lib is required

 // on Windows there can be file errors from tar if parallel installs
 // are happening (not uncommon with multiple native modules) so
 // extract the tarball to a temp directory first and then copy over
 const tarExtractDir = win ? await fs.promises.mkdtemp(path.join(os.tmpdir(), 'node-gyp-tmp-')) : devDir

 try {
 if (tarPath) {
 await tar.extract({
 file: tarPath,
 strip: 1,
 filter: isValid,
 onwarn,
 cwd: tarExtractDir
 })
 } else {
 try {
 const res = await download(gyp, release.tarballUrl)
 if (shouldDownloadTarball) {
 if (tarPath) {
 await tar.extract({
 file: tarPath,
 strip: 1,
 filter: isValid,
 onwarn,
 cwd: tarExtractDir
 })
 } else {
 try {
 const res = await download(gyp, release.tarballUrl)

 if (res.status !== 200) {
 throw new Error(`${res.status} response downloading ${release.tarballUrl}`)
 }
if (res.status !== 200) {
throw new Error(`${res.status} response downloading ${release.tarballUrl}`)
}

 await streamPipeline(
 res.body,
 // content checksum
 new ShaSum((_, checksum) => {
 const filename = path.basename(release.tarballUrl).trim()
 contentShasums[filename] = checksum
 log.verbose('content checksum', filename, checksum)
 }),
 tar.extract({
 strip: 1,
 cwd: tarExtractDir,
 filter: isValid,
 onwarn
 })
 )
 } catch (err) {
await streamPipeline(
res.body,
// content checksum
new ShaSum((_, checksum) => {
const filename = path.basename(release.tarballUrl).trim()
contentShasums[filename] = checksum
log.verbose('content checksum', filename, checksum)
}),
tar.extract({
strip: 1,
cwd: tarExtractDir,
filter: isValid,
onwarn
})
)
} catch (err) {
 // something went wrong downloading the tarball?
 if (err.code === 'ENOTFOUND') {
 throw new Error('This is most likely not a problem with node-gyp or the package itself and\n' +
if (err.code === 'ENOTFOUND') {
throw new Error('This is most likely not a problem with node-gyp or the package itself and\n' +
 'is related to network connectivity. In most cases you are behind a proxy or have bad \n' +
 'network settings.')
 }
 throw err
 }
 throw err
 }
 }

 // invoked after the tarball has finished being extracted
 if (extractErrors || extractCount === 0) {
 throw new Error('There was a fatal problem while downloading/extracting the tarball')
 }
// invoked after the tarball has finished being extracted
if (extractErrors || extractCount === 0) {
throw new Error('There was a fatal problem while downloading/extracting the tarball')
}

 log.verbose('tarball', 'done parsing tarball')
 log.verbose('tarball', 'done parsing tarball')
 }

 const installVersionPath = path.resolve(tarExtractDir, 'installVersion')
 await Promise.all([
// need to download node.lib
 ...(win ? downloadNodeLib() : []),
 // need to download node.lib
 ...(win ? [downloadNodeLib()] : []),
 // write the "installVersion" file
 fs.promises.writeFile(installVersionPath, gyp.package.installVersion + '\n'),
 // Only download SHASUMS.txt if we downloaded something in need of SHA verification
Expand DownExpand Up@@ -293,43 +320,33 @@ async function install (fs, gyp, argv) {
 log.verbose('checksum data', JSON.stringify(expectShasums))
 }

 function downloadNodeLib () {
async function downloadNodeLib () {
 log.verbose('on Windows; need to download `' + release.name + '.lib`...')
 const archs = ['ia32', 'x64', 'arm64']
 return archs.map(async (arch) => {
 const dir = path.resolve(tarExtractDir, arch)
 const targetLibPath = path.resolve(dir, release.name + '.lib')
 const { libUrl, libPath } = release[arch]
 const name = `${arch} ${release.name}.lib`
 log.verbose(name, 'dir', dir)
 log.verbose(name, 'url', libUrl)

 await fs.promises.mkdir(dir, { recursive: true })
 log.verbose('streaming', name, 'to:', targetLibPath)

 const res = await download(gyp, libUrl)

 if (res.status === 403 || res.status === 404) {
 if (arch === 'arm64') {
 // Arm64 is a newer platform on Windows and not all node distributions provide it.
 log.verbose(`${name} was not found in ${libUrl}`)
 } else {
 log.warn(`${name} was not found in ${libUrl}`)
 }
 return
 } else if (res.status !== 200) {
 throw new Error(`${res.status} status code downloading ${name}`)
 }
 const dir = path.resolve(tarExtractDir, arch)
 const targetLibPath = path.resolve(dir, release.name + '.lib')
 const { libUrl, libPath } = release[arch]
 const name = `${arch} ${release.name}.lib`
 log.verbose(name, 'dir', dir)
 log.verbose(name, 'url', libUrl)

 await fs.promises.mkdir(dir, { recursive: true })
 log.verbose('streaming', name, 'to:', targetLibPath)

 const res = await download(gyp, libUrl)

 // Since only required node.lib is downloaded throw error if it is not fetched
 if (res.status !== 200) {
 throw new Error(`${res.status} status code downloading ${name}`)
 }

 return streamPipeline(
 res.body,
 new ShaSum((_, checksum) => {
 contentShasums[libPath] = checksum
 log.verbose('content checksum', libPath, checksum)
 }),
 fs.createWriteStream(targetLibPath)
 )
 })
 return streamPipeline(
 res.body,
 new ShaSum((_, checksum) => {
 contentShasums[libPath] = checksum
 log.verbose('content checksum', libPath, checksum)
 }),
 fs.createWriteStream(targetLibPath)
 )
 } // downloadNodeLib()
 } // go()

Expand Down

2 changes: 1 addition & 1 deletion
package.json

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -12,7 +12,7 @@
 "gyp"
 ],
 "version": "9.3.1",
 "installVersion": 10,
@StefanStojanovicStefanStojanovicMay 26, 2023

Copy link


Copy Markdown

ContributorAuthor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A question for people with more experience in node-gyp: Since a PR that changed installVersion from 9 to 10 landed yesterday in aaa117c do I need to increase it to 11 now, or not, because there was no release with installVersion: 10?
@dsanders11dsanders11Jun 7, 2023

Copy link


Copy Markdown

Contributor
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A question for people with more experience in node-gyp
I'm not more experienced, but my 2 cents: it's probably unnecessary for end users (since as you stated, no release went out), but AFAIK there's no harm in bumping it again, and it will prevent a situation where anyone who tested main (so folks working on node-gyp like you and I, or maybe even someone some where consuming node-gyp@main for whatever reason) and got their cache populated for installVersion: 10 won't end up in a broken situation.
So to me bumping it to 11 seems reasonable to prevent any problems.
 "installVersion": 11,
 "author": "Nathan Rajlich <nathan@tootallnate.net> (http://tootallnate.net)",
 "repository": {
 "type": "git",
Expand Down

2 changes: 1 addition & 1 deletion
test/test-download.js

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -180,7 +180,7 @@ describe('download', function () {
 await util.promisify(install)(prog, [])

 const data = await fs.promises.readFile(path.join(expectedDir, 'installVersion'), 'utf8')
 assert.strictEqual(data, '10\n', 'correct installVersion')
 assert.strictEqual(data, '11\n', 'correct installVersion')

 const list = await fs.promises.readdir(path.join(expectedDir, 'include/node'))
 assert.ok(list.includes('common.gypi'))
Expand Down