HBASE-24297 release scripts should be able to use an existing project clone

dev-support/create-release/do-release-docker.sh

@@ -53,6 +53,7 @@ export PROJECT="${PROJECT:-hbase}"
 SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # shellcheck source=SCRIPTDIR/release-util.sh
 . "$SELF/release-util.sh"
+ORIG_PWD="$(pwd)"
 
 function usage {
  local NAME
@@ -71,6 +72,7 @@ Options:
  -j [path] path to local JDK installation to use building. By default the script will
  use openjdk8 installed in the docker image.
  -p [project] project to build, such as 'hbase' or 'hbase-thirdparty'; defaults to $PROJECT env var
+ -r [repo] git repo to use for remote git operations. defaults to ASF gitbox for project.
  -s [step] runs a single step of the process; valid steps are: tag|publish-dist|publish-release.
  If none specified, runs tag, then publish-dist, and then publish-release.
  'publish-snapshot' is also an allowed, less used, option.
@@ -81,13 +83,15 @@ WORKDIR=
 IMGTAG=latest
 JAVA=
 RELEASE_STEP=
-while getopts "d:fhj:p:s:t:" opt; do
+GIT_REPO=
+while getopts "d:fhj:p:r:s:t:" opt; do
  case $opt in
  d) WORKDIR="$OPTARG" ;;
  f) DRY_RUN=0 ;;
  t) IMGTAG="$OPTARG" ;;
  j) JAVA="$OPTARG" ;;
  p) PROJECT="$OPTARG" ;;
+ r) GIT_REPO="$OPTARG" ;;
  s) RELEASE_STEP="$OPTARG" ;;
  h) usage ;;
  ?) error "Invalid option. Run with -h for help." ;;
@@ -168,11 +172,65 @@ if [ -n "$JAVA" ]; then
  JAVA_VOL=(--volume "$JAVA:/opt/hbase-java")
 fi
 
+#TODO some debug output would be good here
+GIT_REPO_MOUNT=()
+if [ -n "${GIT_REPO}" ]; then
+ case "${GIT_REPO}" in
+ # skip the easy to identify remote protocols
+ ssh://*|git://*|http://*|https://*|ftp://*|ftps://*) ;;
+ # for sure local
+ /*)
+ GIT_REPO_MOUNT=(--mount "type=bind,src=${GIT_REPO},dst=/opt/hbase-repo")
+ echo "HOST_GIT_REPO=${GIT_REPO}" >> "${ENVFILE}"
+ GIT_REPO="/opt/hbase-repo"
+ ;;
+ # on the host but normally git wouldn't use the local optimization
+ file://*)
+ echo "[INFO] converted file:// git repo to a local path, which changes git to assume --local."
+ GIT_REPO_MOUNT=(--mount "type=bind,src=${GIT_REPO#file://},dst=/opt/hbase-repo")
+ echo "HOST_GIT_REPO=${GIT_REPO}" >> "${ENVFILE}"
+ GIT_REPO="/opt/hbase-repo"
+ ;;
+ # have to decide if it's a local path or the "scp-ish" remote
+ *)
+ declare colon_remove_prefix;
+ declare slash_remove_prefix;
+ declare local_path;
+ colon_remove_prefix="${GIT_REPO#*:}"
+ slash_remove_prefix="${GIT_REPO#*/}"
+ if [ "${GIT_REPO}" = "${colon_remove_prefix}" ]; then
+ # if there was no colon at all, we assume this must be a local path
+ local_path="no colon at all"
+ elif [ "${GIT_REPO}" != "${slash_remove_prefix}" ]; then
+ # if there was a colon and there is no slash, then we assume it must be scp-style host
+ # and a relative path
+
+ if [ "${#colon_remove_prefix}" -lt "${#slash_remove_prefix}" ]; then
+ # Given the substrings made by removing everything up to the first colon and slash
+ # we can determine which comes first based on the longer substring length.
+ # if the slash is first, then we assume the colon is part of a path name and if the colon
+ # is first then it is the seperator between a scp-style host name and the path.
+ local_path="slash happened before a colon"
+ fi
+ fi
+ if [ -n "${local_path}" ]; then
+ # convert to an absolute path
+ GIT_REPO="$(cd "$(dirname "${ORIG_PWD}/${GIT_REPO}")"; pwd)/$(basename "${ORIG_PWD}/${GIT_REPO}")"
+ GIT_REPO_MOUNT=(--mount "type=bind,src=${GIT_REPO},dst=/opt/hbase-repo")
+ echo "HOST_GIT_REPO=${GIT_REPO}" >> "${ENVFILE}"
+ GIT_REPO="/opt/hbase-repo"
+ fi
+ ;;
+ esac
+ echo "GIT_REPO=${GIT_REPO}" >> "${ENVFILE}"
+fi
+
 echo "Building $RELEASE_TAG; output will be at $WORKDIR/output"
 cmd=(docker run -ti \
  --env-file "$ENVFILE" \
  --volume "$WORKDIR:/opt/hbase-rm" \
  "${JAVA_VOL[@]}" \
+ "${GIT_REPO_MOUNT[@]}" \
  "hbase-rm:$IMGTAG")
 echo "${cmd[*]}"
 "${cmd[@]}"

dev-support/create-release/release-build.sh

@@ -109,12 +109,11 @@ if [[ "$1" == "tag" ]]; then
  set -o pipefail
  set -x # detailed logging during action
  check_get_passwords ASF_PASSWORD
- check_needed_vars PROJECT ASF_USERNAME ASF_PASSWORD RELEASE_VERSION RELEASE_TAG NEXT_VERSION \
- GIT_EMAIL GIT_NAME GIT_BRANCH
- ASF_REPO="gitbox.apache.org/repos/asf/${PROJECT}.git"
- encoded_username="$(python -c "import urllib; print urllib.quote('''$ASF_USERNAME''')")"
- encoded_password="$(python -c "import urllib; print urllib.quote('''$ASF_PASSWORD''')")"
- git clone "https://$encoded_username:$encoded_password@$ASF_REPO" -b "$GIT_BRANCH" "${PROJECT}"
+ check_needed_vars PROJECT RELEASE_VERSION RELEASE_TAG NEXT_VERSION GIT_EMAIL GIT_NAME GIT_BRANCH
+ if [ -z "${GIT_REPO}" ]; then
+ check_needed_vars ASF_USERNAME ASF_PASSWORD
+ fi
+ git_clone_overwrite
 
  # 'update_releasenotes' searches the project's Jira for issues where 'Fix Version' matches specified
  # $jira_fix_version. For most projects this is same as ${RELEASE_VERSION}. However, all the 'hbase-*'
@@ -184,8 +183,7 @@ fi
 if is_dry_run && [[ "${TAG_SAME_DRY_RUN:-}" == "true" && -d "${PROJECT}.tag" ]]; then
  ln -s "${PROJECT}.tag" "${PROJECT}"
 else
- ASF_REPO="${ASF_REPO:-https://gitbox.apache.org/repos/asf/${PROJECT}.git}"
- git clone "$ASF_REPO" "${PROJECT}"
+ git_clone_overwrite
 fi
 cd "${PROJECT}"
 git checkout "$GIT_REF"

dev-support/create-release/release-util.sh

@@ -389,6 +389,43 @@ function configure_maven {
 EOF
 }
 
+# clone of the repo, deleting anything that exists in the working directory named after the project.
+# optionally with auth details for pushing.
+function git_clone_overwrite {
+ local asf_repo
+ if [ -z "${PROJECT}" ] || [ "${PROJECT}" != "${PROJECT#/}" ]; then
+ error "Project name must be defined and not start with a '/'. PROJECT='${PROJECT}'"
+ fi
+ rm -rf "${PROJECT}"
+
+ if [[ -z "${GIT_REPO}" ]]; then
+ asf_repo="gitbox.apache.org/repos/asf/${PROJECT}.git"
+ echo "[INFO] clone will be of the gitbox repo for ${PROJECT}."
+ if [ -n "${ASF_USERNAME}" ] && [ -n "${ASF_PASSWORD}" ]; then
+ # Ugly!
+ encoded_username=$(python -c "import urllib; print urllib.quote('''$ASF_USERNAME''')")
+ encoded_password=$(python -c "import urllib; print urllib.quote('''$ASF_PASSWORD''')")
+ GIT_REPO="https://$encoded_username:$encoded_password@${asf_repo}"
+ else
+ GIT_REPO="https://${asf_repo}"
+ fi
+ else
+ echo "[INFO] clone will be of provided git repo."
+ fi
+ # N.B. we use the shared flag because the clone is short lived and if a local repo repo was
+ # given this will let us refer to objects there directly instead of hardlinks or copying.
+ # The option is silently ignored for non-local repositories. see the note on git help clone
+ # for the --shared option for details.
+ git clone --shared -b "${GIT_BRANCH}" -- "${GIT_REPO}" "${PROJECT}"
+ # If this was a host local git repo then add in an alternates and remote that will
+ # work back on the host if the RM needs to do any post-processing steps, i.e. pushing the git tag
+ # for more info see 'git help remote' and 'git help repository-layout'.
+ if [ -n "$HOST_GIT_REPO" ]; then
+ echo "${HOST_GIT_REPO}/objects" >> "${PROJECT}/.git/objects/info/alternates"
+ (cd "${PROJECT}"; git remote add host "${HOST_GIT_REPO}")
+ fi
+}
+
 # Writes report into cwd!
 function generate_api_report {
  local project="$1"