tools: log and verify sha256sum

fasenderos · MoLow · commit 24abe07dda9c · 2023-07-07T00:06:17.000+03:00
PR-URL: #48088 Refs: nodejs/security-wg#973 Reviewed-By: Rafael Gonzaga <rafael.nunu@hotmail.com> Reviewed-By: Marco Ippolito <marcoippolito54@gmail.com>
diff --git a/tools/dep_updaters/update-ada.sh b/tools/dep_updaters/update-ada.sh
@@ -6,6 +6,9 @@ BASE_DIR=$(cd "$(dirname "$0")/../.." && pwd)
 DEPS_DIR="$BASE_DIR/deps"
 ADA_VERSION=$1
 
+# shellcheck disable=SC1091
+. "$BASE_DIR/tools/dep_updaters/utils.sh"
+
 if [ "$#" -le 0 ]; then
 echo "Error: please provide an ada version to update to"
 echo "	e.g. $0 1.0.0"
@@ -25,13 +28,14 @@ cleanup () {
 trap cleanup INT TERM EXIT
 
 ADA_REF="v$ADA_VERSION"
-ADA_ZIP="ada-$ADA_VERSION.zip"
+ADA_ZIP="ada-$ADA_REF.zip"
 ADA_LICENSE="LICENSE-MIT"
 
 cd "$WORKSPACE"
 
 echo "Fetching ada source archive..."
 curl -sL -o "$ADA_ZIP" "https://github.com/ada-url/ada/releases/download/$ADA_REF/singleheader.zip"
+log_and_verify_sha256sum "ada" "$ADA_ZIP"
 unzip "$ADA_ZIP"
 rm "$ADA_ZIP"
 
diff --git a/tools/dep_updaters/update-base64.sh b/tools/dep_updaters/update-base64.sh
@@ -8,6 +8,9 @@ DEPS_DIR="$BASE_DIR/deps"
 [ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node"
 [ -x "$NODE" ] || NODE=$(command -v node)
 
+# shellcheck disable=SC1091
+. "$BASE_DIR/tools/dep_updaters/utils.sh"
+
 NEW_VERSION="$("$NODE" --input-type=module <<'EOF'
 const res = await fetch('https://api.github.com/repos/aklomp/base64/releases/latest');
 if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res });
@@ -39,8 +42,13 @@ trap cleanup INT TERM EXIT
 
 cd "$WORKSPACE"
 
+BASE64_TARBALL="base64-v$NEW_VERSION.tar.gz"
+
 echo "Fetching base64 source archive"
-curl -sL "https://api.github.com/repos/aklomp/base64/tarball/v$NEW_VERSION" | tar xzf -
+curl -sL -o "$BASE64_TARBALL" "https://api.github.com/repos/aklomp/base64/tarball/v$NEW_VERSION"
+log_and_verify_sha256sum "base64" "$BASE64_TARBALL"
+gzip -dc "$BASE64_TARBALL" | tar xf -
+rm "$BASE64_TARBALL"
 mv aklomp-base64-* base64
 
 echo "Replacing existing base64"
diff --git a/tools/dep_updaters/update-brotli.sh b/tools/dep_updaters/update-brotli.sh
@@ -8,6 +8,9 @@ DEPS_DIR="$BASE_DIR/deps"
 [ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node"
 [ -x "$NODE" ] || NODE=$(command -v node)
 
+# shellcheck disable=SC1091
+. "$BASE_DIR/tools/dep_updaters/utils.sh"
+
 NEW_VERSION="$("$NODE" --input-type=module <<'EOF'
 const res = await fetch('https://api.github.com/repos/google/brotli/releases/latest');
 if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res });
@@ -44,10 +47,11 @@ trap cleanup INT TERM EXIT
 
 cd "$WORKSPACE"
 
-BROTLI_TARBALL="v$NEW_VERSION.tar.gz"
+BROTLI_TARBALL="brotli-v$NEW_VERSION.tar.gz"
 
 echo "Fetching brotli source archive"
-curl -sL -o "$BROTLI_TARBALL" "https://github.com/google/brotli/archive/$BROTLI_TARBALL"
+curl -sL -o "$BROTLI_TARBALL" "https://github.com/google/brotli/archive/v$NEW_VERSION.tar.gz"
+log_and_verify_sha256sum "brotli" "$BROTLI_TARBALL"
 gzip -dc "$BROTLI_TARBALL" | tar xf -
 rm "$BROTLI_TARBALL"
 mv "brotli-$NEW_VERSION" "brotli"
diff --git a/tools/dep_updaters/update-c-ares.sh b/tools/dep_updaters/update-c-ares.sh
@@ -8,6 +8,9 @@ DEPS_DIR="$BASE_DIR/deps"
 [ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node"
 [ -x "$NODE" ] || NODE=$(command -v node)
 
+# shellcheck disable=SC1091
+. "$BASE_DIR/tools/dep_updaters/utils.sh"
+
 NEW_VERSION="$("$NODE" --input-type=module <<'EOF'
 const res = await fetch('https://api.github.com/repos/c-ares/c-ares/releases/latest');
 if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res });
@@ -43,7 +46,10 @@ ARES_TARBALL="c-ares-$NEW_VERSION.tar.gz"
 cd "$WORKSPACE"
 
 echo "Fetching c-ares source archive"
-curl -sL "https://github.com/c-ares/c-ares/releases/download/$ARES_REF/$ARES_TARBALL" | tar xz
+curl -sL -o "$ARES_TARBALL" "https://github.com/c-ares/c-ares/releases/download/$ARES_REF/$ARES_TARBALL"
+log_and_verify_sha256sum "c-ares" "$ARES_TARBALL"
+gzip -dc "$ARES_TARBALL" | tar xf -
+rm "$ARES_TARBALL"
 mv "c-ares-$NEW_VERSION" cares
 
 echo "Removing tests"
diff --git a/tools/dep_updaters/update-libuv.sh b/tools/dep_updaters/update-libuv.sh
@@ -7,6 +7,9 @@ DEPS_DIR="$BASE_DIR/deps"
 [ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node"
 [ -x "$NODE" ] || NODE=$(command -v node)
 
+# shellcheck disable=SC1091
+. "$BASE_DIR/tools/dep_updaters/utils.sh"
+
 NEW_VERSION="$("$NODE" --input-type=module <<'EOF'
 const res = await fetch('https://api.github.com/repos/libuv/libuv/releases/latest');
 if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res });
@@ -45,8 +48,13 @@ trap cleanup INT TERM EXIT
 
 cd "$WORKSPACE"
 
+LIBUV_TARBALL="libuv-v$NEW_VERSION.tar.gz"
+
 echo "Fetching libuv source archive..."
-curl -sL "https://api.github.com/repos/libuv/libuv/tarball/v$NEW_VERSION" | tar xzf -
+curl -sL -o "$LIBUV_TARBALL" "https://api.github.com/repos/libuv/libuv/tarball/v$NEW_VERSION"
+log_and_verify_sha256sum "libuv" "$LIBUV_TARBALL"
+gzip -dc "$LIBUV_TARBALL" | tar xf -
+rm "$LIBUV_TARBALL"
 mv libuv-libuv-* uv
 
 echo "Replacing existing libuv (except GYP build files)"
diff --git a/tools/dep_updaters/update-llhttp.sh b/tools/dep_updaters/update-llhttp.sh
@@ -9,6 +9,9 @@ DEPS_DIR="${BASE_DIR}/deps"
 [ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node"
 [ -x "$NODE" ] || NODE=$(command -v node)
 
+# shellcheck disable=SC1091
+. "$BASE_DIR/tools/dep_updaters/utils.sh"
+
 NEW_VERSION="$("$NODE" --input-type=module <<'EOF'
 const res = await fetch('https://api.github.com/repos/nodejs/llhttp/releases/latest');
 if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res });
@@ -52,19 +55,20 @@ if echo "$NEW_VERSION" | grep -qs "/" ; then # Download a release
 echo "Checking out branch $BRANCH ..."
 git checkout "$BRANCH"
 
- echo "Building llhtttp ..."
+ echo "Building llhttp ..."
 npm install
 make release
 
- echo "Copying llhtttp release ..."
+ echo "Copying llhttp release ..."
 rm -rf "$DEPS_DIR/llhttp"
 cp -a release "$DEPS_DIR/llhttp"
 else
 echo "Download llhttp release $NEW_VERSION ..."
- curl -sL -o llhttp.tar.gz "https://github.com/nodejs/llhttp/archive/refs/tags/release/v$NEW_VERSION.tar.gz"
- gzip -dc llhttp.tar.gz | tar xf -
+ LLHTTP_TARBALL="llhttp-v$NEW_VERSION.tar.gz"
+ curl -sL -o "$LLHTTP_TARBALL" "https://github.com/nodejs/llhttp/archive/refs/tags/release/v$NEW_VERSION.tar.gz"
+ gzip -dc "$LLHTTP_TARBALL" | tar xf -
 
- echo "Copying llhtttp release ..."
+ echo "Copying llhttp release ..."
 rm -rf "$DEPS_DIR/llhttp"
 cp -a "llhttp-release-v$NEW_VERSION" "$DEPS_DIR/llhttp"
 fi
diff --git a/tools/dep_updaters/update-nghttp2.sh b/tools/dep_updaters/update-nghttp2.sh
@@ -8,6 +8,9 @@ DEPS_DIR="$BASE_DIR/deps"
 [ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node"
 [ -x "$NODE" ] || NODE=$(command -v node)
 
+# shellcheck disable=SC1091
+. "$BASE_DIR/tools/dep_updaters/utils.sh"
+
 NEW_VERSION="$("$NODE" --input-type=module <<'EOF'
 const res = await fetch('https://api.github.com/repos/nghttp2/nghttp2/releases/latest');
 if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res });
@@ -44,6 +47,11 @@ cd "$WORKSPACE"
 
 echo "Fetching nghttp2 source archive"
 curl -sL -o "$NGHTTP2_TARBALL" "https://github.com/nghttp2/nghttp2/releases/download/$NGHTTP2_REF/$NGHTTP2_TARBALL"
+
+DEPOSITED_CHECKSUM=$(curl -sL "https://github.com/nghttp2/nghttp2/releases/download/$NGHTTP2_REF/checksums.txt" | grep "$NGHTTP2_TARBALL")
+
+log_and_verify_sha256sum "nghttp2" "$NGHTTP2_TARBALL" "$DEPOSITED_CHECKSUM"
+
 gzip -dc "$NGHTTP2_TARBALL" | tar xf -
 rm "$NGHTTP2_TARBALL"
 mv "nghttp2-$NEW_VERSION" nghttp2
diff --git a/tools/dep_updaters/update-nghttp3.sh b/tools/dep_updaters/update-nghttp3.sh
@@ -7,6 +7,9 @@ DEPS_DIR="$BASE_DIR/deps"
 [ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node"
 [ -x "$NODE" ] || NODE=$(command -v node)
 
+# shellcheck disable=SC1091
+. "$BASE_DIR/tools/dep_updaters/utils.sh"
+
 NEW_VERSION="$("$NODE" --input-type=module <<'EOF'
 const res = await fetch('https://api.github.com/repos/ngtcp2/nghttp3/releases');
 if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res });
@@ -44,6 +47,7 @@ cd "$WORKSPACE"
 
 echo "Fetching nghttp3 source archive..."
 curl -sL -o "$NGHTTP3_ZIP.zip" "https://github.com/ngtcp2/nghttp3/archive/refs/tags/$NGHTTP3_REF.zip"
+log_and_verify_sha256sum "nghttp3" "$NGHTTP3_ZIP.zip"
 unzip "$NGHTTP3_ZIP.zip"
 rm "$NGHTTP3_ZIP.zip"
 mv "$NGHTTP3_ZIP" nghttp3
diff --git a/tools/dep_updaters/update-ngtcp2.sh b/tools/dep_updaters/update-ngtcp2.sh
@@ -7,6 +7,9 @@ DEPS_DIR="$BASE_DIR/deps"
 [ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node"
 [ -x "$NODE" ] || NODE=$(command -v node)
 
+# shellcheck disable=SC1091
+. "$BASE_DIR/tools/dep_updaters/utils.sh"
+
 NEW_VERSION="$("$NODE" --input-type=module <<'EOF'
 const res = await fetch('https://api.github.com/repos/ngtcp2/ngtcp2/releases');
 if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res });
@@ -44,6 +47,7 @@ cd "$WORKSPACE"
 
 echo "Fetching ngtcp2 source archive..."
 curl -sL -o "$NGTCP2_ZIP.zip" "https://github.com/ngtcp2/ngtcp2/archive/refs/tags/$NGTCP2_REF.zip"
+log_and_verify_sha256sum "ngtcp2" "$NGTCP2_ZIP.zip"
 unzip "$NGTCP2_ZIP.zip"
 rm "$NGTCP2_ZIP.zip"
 mv "$NGTCP2_ZIP" ngtcp2
diff --git a/tools/dep_updaters/update-npm.sh b/tools/dep_updaters/update-npm.sh
@@ -7,6 +7,9 @@ DEPS_DIR="$BASE_DIR/deps"
 [ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node"
 [ -x "$NODE" ] || NODE=$(command -v node)
 
+# shellcheck disable=SC1091
+. "$BASE_DIR/tools/dep_updaters/utils.sh"
+
 NPM="$DEPS_DIR/npm/bin/npm-cli.js"
 
 NPM_VERSION=$1
@@ -30,12 +33,14 @@ trap cleanup INT TERM EXIT
 
 cd "$WORKSPACE"
 
-NPM_TGZ=npm.tgz
+NPM_TGZ="npm-v$NPM_VERSION.tar.gz"
 
 NPM_TARBALL="$($NODE "$NPM" view npm@"$NPM_VERSION" dist.tarball)"
 
 curl -s "$NPM_TARBALL" > "$NPM_TGZ"
 
+log_and_verify_sha256sum "npm" "$NPM_TGZ"
+
 rm -rf "$DEPS_DIR/npm"
 
 mkdir "$DEPS_DIR/npm"
diff --git a/tools/dep_updaters/update-openssl.sh b/tools/dep_updaters/update-openssl.sh
@@ -0,0 +1,105 @@
+#!/bin/sh
+set -e
+# Shell script to update OpenSSL in the source tree to a specific version
+# Based on https://github.com/nodejs/node/blob/main/doc/contributing/maintaining-openssl.md
+
+cleanup() {
+ EXIT_CODE=$?
+ [ -d "$WORKSPACE" ] && rm -rf "$WORKSPACE"
+ exit $EXIT_CODE
+}
+
+download() {
+ if [ -z "$1" ]; then
+ echo "Error: please provide an OpenSSL version to update to"
+ echo "	e.g. ./$0 download 3.0.7+quic1"
+ exit 1
+ fi
+
+ OPENSSL_VERSION=$1
+ echo "Making temporary workspace..."
+ WORKSPACE=$(mktemp -d 2> /dev/null || mktemp -d -t 'tmp')
+
+ # shellcheck disable=SC1091
+ . "$BASE_DIR/tools/dep_updaters/utils.sh"
+
+ cd "$WORKSPACE"
+
+ echo "Fetching OpenSSL source archive..."
+ OPENSSL_TARBALL="openssl-v$OPENSSL_VERSION.tar.gz"
+ curl -sL -o "$OPENSSL_TARBALL" "https://api.github.com/repos/quictls/openssl/tarball/openssl-$OPENSSL_VERSION"
+ log_and_verify_sha256sum "openssl" "$OPENSSL_TARBALL"
+ gzip -dc "$OPENSSL_TARBALL" | tar xf -
+ rm "$OPENSSL_TARBALL"
+ mv quictls-openssl-* openssl
+
+ echo "Replacing existing OpenSSL..."
+ rm -rf "$DEPS_DIR/openssl/openssl"
+ mv "$WORKSPACE/openssl" "$DEPS_DIR/openssl/"
+
+ echo "All done!"
+ echo ""
+ echo "Please git add openssl, and commit the new version:"
+ echo ""
+ echo "$ git add -A deps/openssl/openssl"
+ echo "$ git commit -m \"deps: upgrade openssl sources to quictls/openssl-$OPENSSL_VERSION\""
+ echo ""
+}
+
+regenerate() {
+ command -v perl >/dev/null 2>&1 || { echo >&2 "Error: 'Perl' required but not installed."; exit 1; }
+ command -v nasm >/dev/null 2>&1 || { echo >&2 "Error: 'nasm' required but not installed."; exit 1; }
+ command -v as >/dev/null 2>&1 || { echo >&2 "Error: 'GNU as' required but not installed."; exit 1; }
+ perl -e "use Text::Template">/dev/null 2>&1 || { echo >&2 "Error: 'Text::Template' Perl module required but not installed."; exit 1; }
+
+ echo "Regenerating platform-dependent files..."
+
+ make -C "$DEPS_DIR/openssl/config" clean
+ # Needed for compatibility with nasm on 32-bit Windows
+ # See https://github.com/nodejs/node/blob/main/doc/contributing/maintaining-openssl.md#2-execute-make-in-depsopensslconfig-directory
+ sed -i 's/#ifdef/%ifdef/g' "$DEPS_DIR/openssl/openssl/crypto/perlasm/x86asm.pl"
+ sed -i 's/#endif/%endif/g' "$DEPS_DIR/openssl/openssl/crypto/perlasm/x86asm.pl"
+ make -C "$DEPS_DIR/openssl/config"
+
+ echo "All done!"
+ echo ""
+ echo "Please commit the regenerated files:"
+ echo ""
+ echo "$ git add -A deps/openssl/config/archs deps/openssl/openssl"
+ echo "$ git commit -m \"deps: update archs files for openssl\""
+ echo ""
+}
+
+help() {
+ echo "Shell script to update OpenSSL in the source tree to a specific version"
+ echo "Sub-commands:"
+ printf "%-23s %s\n" "help" "show help menu and commands"
+ printf "%-23s %s\n" "download" "download and replace OpenSSL source code with new version"
+ printf "%-23s %s\n" "regenerate" "regenerate platform-specific files"
+ echo ""
+ exit "${1:-0}"
+}
+
+main() {
+ if [ ${#} -eq 0 ]; then
+ help 0
+ fi
+
+ trap cleanup INT TERM EXIT
+
+ BASE_DIR=$(cd "$(dirname "$0")/../.." && pwd)
+ DEPS_DIR="$BASE_DIR/deps"
+
+ case ${1} in
+ help | download | regenerate )
+ $1 "${2}"
+ ;;
+ * )
+ echo "unknown command: $1"
+ help 1
+ exit 1
+ ;;
+ esac
+}
+
+main "$@"
diff --git a/tools/dep_updaters/update-simdutf.sh b/tools/dep_updaters/update-simdutf.sh
@@ -7,6 +7,9 @@ DEPS_DIR="$BASE_DIR/deps"
 [ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node"
 [ -x "$NODE" ] || NODE=$(command -v node)
 
+# shellcheck disable=SC1091
+. "$BASE_DIR/tools/dep_updaters/utils.sh"
+
 NEW_VERSION="$("$NODE" --input-type=module <<'EOF'
 const res = await fetch('https://api.github.com/repos/simdutf/simdutf/releases/latest');
 if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res });
@@ -36,13 +39,14 @@ cleanup () {
 trap cleanup INT TERM EXIT
 
 SIMDUTF_REF="v$NEW_VERSION"
-SIMDUTF_ZIP="simdutf-$NEW_VERSION.zip"
+SIMDUTF_ZIP="simdutf-$SIMDUTF_REF.zip"
 SIMDUTF_LICENSE="LICENSE-MIT"
 
 cd "$WORKSPACE"
 
 echo "Fetching simdutf source archive..."
 curl -sL -o "$SIMDUTF_ZIP" "https://github.com/simdutf/simdutf/releases/download/$SIMDUTF_REF/singleheader.zip"
+log_and_verify_sha256sum "simdutf" "$SIMDUTF_ZIP"
 unzip "$SIMDUTF_ZIP"
 rm "$SIMDUTF_ZIP"
 rm ./*_demo.cpp
diff --git a/tools/dep_updaters/update-uvwasi.sh b/tools/dep_updaters/update-uvwasi.sh
@@ -8,6 +8,9 @@ DEPS_DIR="$BASE_DIR/deps"
 [ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node"
 [ -x "$NODE" ] || NODE=$(command -v node)
 
+# shellcheck disable=SC1091
+. "$BASE_DIR/tools/dep_updaters/utils.sh"
+
 NEW_VERSION="$("$NODE" --input-type=module <<'EOF'
 const res = await fetch('https://api.github.com/repos/nodejs/uvwasi/releases/latest');
 if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res });
@@ -46,6 +49,8 @@ cd "$WORKSPACE"
 echo "Fetching UVWASI source archive..."
 curl -sL -o "$UVWASI_ZIP.zip" "https://github.com/nodejs/uvwasi/archive/refs/tags/v$NEW_VERSION.zip"
 
+log_and_verify_sha256sum "uvwasi" "$UVWASI_ZIP.zip"
+
 echo "Moving existing GYP build file"
 mv "$DEPS_DIR/uvwasi/"*.gyp "$WORKSPACE/"
 rm -rf "$DEPS_DIR/uvwasi/"
diff --git a/tools/dep_updaters/update-zlib.sh b/tools/dep_updaters/update-zlib.sh
diff --git a/tools/dep_updaters/utils.sh b/tools/dep_updaters/utils.sh

-Original file line number
+Diff line change
 [ -z"$NODE" ] && NODE="$BASE_DIR/out/Release/node"
 [ -x"$NODE" ] || NODE=$(command -v node)
 +# shellcheck disable=SC1091
 +."$BASE_DIR/tools/dep_updaters/utils.sh"
++
 NEW_VERSION="$("$NODE" --input-type=module <<'EOF'
 const res = await fetch('https://api.github.com/repos/aklomp/base64/releases/latest');
 if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res });
 cd"$WORKSPACE"
 +BASE64_TARBALL="base64-v$NEW_VERSION.tar.gz"
++
 echo"Fetching base64 source archive"
 -curl -sL "https://api.github.com/repos/aklomp/base64/tarball/v$NEW_VERSION"| tar xzf -
 +curl -sL -o "$BASE64_TARBALL""https://api.github.com/repos/aklomp/base64/tarball/v$NEW_VERSION"
 +log_and_verify_sha256sum "base64""$BASE64_TARBALL"
 +gzip -dc "$BASE64_TARBALL"| tar xf -
 +rm "$BASE64_TARBALL"
 mv aklomp-base64-* base64
 echo"Replacing existing base64"