From b63b9baad8f41274e844c11bceb38efd20dfb3fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucio=20Andr=C3=A9s=20Illanes=20Albornoz?= Date: Sat, 23 Jan 2021 10:49:42 +0000 Subject: Implements setting up & fetching from mirrors. etc/pkgtool.usage: updated. etc/README.md: updated. pkgtool.sh:pkgtoolp_mirror{,_fetch}(): initial implementation. subr/pkg_fetch_download.subr: default to empty ${PKG_MIRRORS}. subr/pkg_fetch_download.subr: factor out ${ARG_FETCH_FORCE} check and ${BUILD_DLCACHEDIR}. subr/pkgtool_init.subr: updated. subr.rtl/rtl_fetch.subr:rtl_fetch_urls_{git,wget}(): factor out ${ARG_FETCH_FORCE} check and ${BUILD_DLCACHEDIR}; minor cleanup. subr.rtl/rtl_fetch.subr:rtl_fetch_url_wget(): return vs. exit on fatal failure. subr.rtl/rtl_fetch.subr:rtlp_fetch_url_git(): copy only if ${_cache_dname} != ${_tgtdir}. subr.rtl/rtl_fetch.subr:rtlp_fetch_url_git(): return if unable to git-clone(1). subr.rtl/rtl_fetch.subr:{rtlp_fetch_url_git,rtl_fetch_urls_git}(): implement Git repository mirroring. vars/{gcc,python[23]{,_host}}.vars: updated. --- subr.rtl/rtl_fetch.subr | 159 ++++++++++++++++++++++++------------------------ 1 file changed, 79 insertions(+), 80 deletions(-) (limited to 'subr.rtl') diff --git a/subr.rtl/rtl_fetch.subr b/subr.rtl/rtl_fetch.subr index 64eac921..7f44346c 100644 --- a/subr.rtl/rtl_fetch.subr +++ b/subr.rtl/rtl_fetch.subr @@ -3,29 +3,38 @@ # rtlp_fetch_url_git() { - local _tgtdir="${1}" _subdir="${2}" _url="${3}" _branch="${4}" \ - _oldpwd=""; + local _cache_dname="${1}" _git_branch="${2}" _mirrors="${3}" _subdir="${4}" _tgtdir="${5}" _url="${6}"\ + _clonefl=0 _oldpwd="" _url_base=""; (set -o errexit -o noglob -o nounset; rtl_flock_acquire 4 || exit "${?}"; - trap "rm -f \"${BUILD_DLCACHEDIR}/${_subdir%%[/]}.fetching\"" EXIT; - if [ -e "${BUILD_DLCACHEDIR}/${_subdir}" ]; then - (rtl_fileop cd "${BUILD_DLCACHEDIR}/${_subdir}" &&\ - git pull ${DEFAULT_GIT_ARGS} origin "${_branch:-main}"); - (rtl_fileop cd "${BUILD_DLCACHEDIR}/${_subdir}" &&\ + trap "rm -f \"${_cache_dname}/${_subdir%%[/]}.fetching\"" EXIT; + if [ -e "${_cache_dname}/${_subdir}" ]; then + (rtl_fileop cd "${_cache_dname}/${_subdir}" &&\ + git pull ${DEFAULT_GIT_ARGS} origin "${_git_branch:-main}"); + (rtl_fileop cd "${_cache_dname}/${_subdir}" &&\ git submodule update); else - git clone ${DEFAULT_GIT_ARGS} "${_url}" "${BUILD_DLCACHEDIR}/${_subdir}"; - if [ -n "${_branch}" ]; then - (rtl_fileop cd "${BUILD_DLCACHEDIR}/${_subdir}" &&\ - git checkout "${_branch}"); + for _url_base in ${_url%/*} ${_mirrors}; do + if git clone ${DEFAULT_GIT_ARGS} "${_url_base}/${_url##*/}" "${_cache_dname}/${_subdir}"; then + _clonefl=1; break; + fi; + done; + if [ "${_clonefl}" -eq 0 ]; then + return 1; + else if [ -n "${_git_branch}" ]; then + (rtl_fileop cd "${_cache_dname}/${_subdir}" &&\ + git checkout "${_git_branch}"); + fi; + (rtl_fileop cd "${_cache_dname}/${_subdir}" &&\ + git submodule update --init); fi; - (rtl_fileop cd "${BUILD_DLCACHEDIR}/${_subdir}" &&\ - git submodule update --init); fi; - _oldpwd="${PWD}"; rtl_fileop cd "${PKG_BASE_DIR}"; - rtl_fileop rm "${_tgtdir}/${_subdir}"; - rtl_fileop cp "${BUILD_DLCACHEDIR}/${_subdir}" "${_tgtdir}"; - rtl_fileop cd "${_oldpwd}";) 4<>"${BUILD_DLCACHEDIR}/${_subdir%%[/]}.fetching"; + if [ "${_cache_dname}" != "${_tgtdir}" ]; then + _oldpwd="${PWD}"; rtl_fileop cd "${_tgtdir}"; + rtl_fileop rm "${_tgtdir}/${_subdir}"; + rtl_fileop cp "${_cache_dname}/${_subdir}" "${_tgtdir}"; + rtl_fileop cd "${_oldpwd}"; + fi) 4<>"${_cache_dname}/${_subdir%%[/]}.fetching"; if [ "${?}" -eq 0 ]; then cd "$(pwd)"; else @@ -34,81 +43,71 @@ rtlp_fetch_url_git() { }; rtl_fetch_urls_git() { - local _tgtdir="" _url_spec="" _subdir="" _url="" _git_branch=""; - _tgtdir="${1}"; shift; - if [ "${ARG_FETCH_FORCE}" = "offline" ]; then - return 0; - fi; + local _cache_dname="${1}" _tgtdir="${2}" _mirrors="${3}" _git_branch="" _subdir="" _url="" _url_spec=""; shift 3; for _url_spec in "${@}"; do - _subdir="${_url_spec%=*}"; - _url="${_url_spec#*=}"; - _url="${_url%@*}"; + _subdir="${_url_spec%=*}"; _url="${_url_spec#*=}"; _url="${_url%@*}"; if [ "${_url_spec#*@}" != "${_url_spec}" ]; then _git_branch=${_url_spec#*@}; fi; - rtlp_fetch_url_git "${_tgtdir}" "${_subdir}" \ - "${_url}" "${_git_branch}"; + rtlp_fetch_url_git "${_cache_dname}" "${_git_branch}" "${_mirrors}" "${_subdir}" "${_tgtdir}" "${_url}"; done; }; # N.B. URLs ($1) may contain `?' or '&' characters. rtl_fetch_url_wget() { - local _urls="${1}" _sha256sum_src="${2}" _target_fname="${3}" _target_name="${4}" _mirrors="${5:-}" \ + local _urls="${1}" _sha256sum_src="${2}" _target_dname="${3}" _target_fname="${4}" _target_name="${5}" _mirrors="${6:-}" \ _rc=0 _target_fname_full="" _url="" _url_base="" _urls_count=0 _urls_full=""; - if [ "${ARG_FETCH_FORCE}" = "offline" ]; then - return 0; - else _urls_full="${_urls}"; - for _url_base in ${_mirrors}; do - _urls_full="$(rtl_lconcat "${_urls_full}" "${_url_base%/}/${_target_name}/${_target_fname}")"; - done; - _urls_count="$(rtl_llength "${_urls_full}")"; - for _url in ${_urls_full}; do - if [ -z "${_target_fname}" ]; then - _target_fname="$(rtl_basename "${_url}")"; + _urls_full="${_urls}"; + for _url_base in ${_mirrors}; do + _urls_full="$(rtl_lconcat "${_urls_full}" "${_url_base%/}/${_target_name}/${_target_fname}")"; + done; + _urls_count="$(rtl_llength "${_urls_full}")"; + for _url in ${_urls_full}; do + if [ -z "${_target_fname}" ]; then + _target_fname="$(rtl_basename "${_url}")"; + fi; + _target_fname_full="${_target_dname}/${_target_fname}"; + (set +o errexit -o noglob -o nounset; + rtl_flock_acquire 4 || exit 1; + trap "_rc=\"\${?}\"; rm -f \"${_target_fname_full}.fetching\"; exit \"\${_rc}\";" EXIT; + if [ -z "${_sha256sum_src}" ]\ + || ! rtl_check_digest_file "${_target_fname_full}" "${_sha256sum_src}" "${_target_fname_full}.fetched"; then + wget ${DEFAULT_WGET_ARGS} -O "${_target_fname_full}" "${_url}"; _rc="${?}"; + if [ "${_rc}" -ne 0 ]; then + exit $((${_rc}+2)); + elif [ -n "${_sha256sum_src}" ]\ + && ! rtl_check_digest "${_target_fname_full}" "${_sha256sum_src}"; then + exit 2; + else + printf "%s" "${RTL_CHECK_DIGEST_DIGEST}" > "${_target_fname_full}.fetched"; exit 0; fi; - _target_fname_full="${BUILD_DLCACHEDIR}/${_target_fname}"; - (set +o errexit -o noglob -o nounset; - rtl_flock_acquire 4 || exit 1; - trap "_rc=\"\${?}\"; rm -f \"${_target_fname_full}.fetching\"; exit \"\${_rc}\";" EXIT; - if [ -z "${_sha256sum_src}" ]\ - || ! rtl_check_digest_file "${_target_fname_full}" "${_sha256sum_src}" "${_target_fname_full}.fetched"; then - wget ${DEFAULT_WGET_ARGS} -O "${_target_fname_full}" "${_url}"; _rc="${?}"; - if [ "${_rc}" -ne 0 ]; then - exit $((${_rc}+2)); - elif [ -n "${_sha256sum_src}" ]\ - && ! rtl_check_digest "${_target_fname_full}" "${_sha256sum_src}"; then - exit 2; - else - printf "%s" "${RTL_CHECK_DIGEST_DIGEST}" > "${_target_fname_full}.fetched"; exit 0; - fi; + else + exit 0; + fi;) 4<>"${_target_fname_full}.fetching"; _rc="${?}"; : $((_urls_count-=1)); + case "${_rc}" in + 0) break; ;; + 1) if [ "${_urls_count}" -ge 1 ]; then + rtl_log_msg warning "Warning: failed to acquire fetching lock for URL \`%s', retrying with alternative URL..." "${_url}"; else - exit 0; - fi;) 4<>"${_target_fname_full}.fetching"; _rc="${?}"; : $((_urls_count-=1)); - case "${_rc}" in - 0) break; ;; - 1) if [ "${_urls_count}" -ge 1 ]; then - rtl_log_msg warning "Warning: failed to acquire fetching lock for URL \`%s', retrying with alternative URL..." "${_url}"; - else - rtl_log_msg fatalexit "Error: failed to acquire fetching lock for URL \`%s'." "${_url}"; - fi; ;; - 2) if [ "${_urls_count}" -ge 1 ]; then - rtl_log_msg warning "Warning: hash mismatch for URL \`%s', retrying with alternative URL... (from build variables: %s.)"\ - "${_url}" "${_sha256sum_src}"; - else - rtl_log_msg fatalexit "Error: hash mismatch for URL \`%s' (from build variables: %s.)"\ - "${_url}" "${_sha256sum_src}"; - fi; ;; - *) if [ "${_urls_count}" -ge 1 ]; then - rtl_log_msg warning "Warning: failed to fetch URL \`%s', retrying with alternative URL... (wget(1) exit status: %s)"\ - "${_url}" "$((${_rc}-2))"; - else - rtl_log_msg fatalexit "Error: failed to fetch URL \`%s' (wget(1) exit status: %s)"\ - "${_url}" "$((${_rc}-2))"; - fi; ;; - esac; - done; - return "${_rc}"; - fi; + rtl_log_msg fatal "Error: failed to acquire fetching lock for URL \`%s'." "${_url}"; break; + fi; ;; + 2) if [ "${_urls_count}" -ge 1 ]; then + rtl_log_msg warning "Warning: hash mismatch for URL \`%s', retrying with alternative URL... (from build variables: %s.)"\ + "${_url}" "${_sha256sum_src}"; + else + rtl_log_msg fatal "Error: hash mismatch for URL \`%s' (from build variables: %s.)"\ + "${_url}" "${_sha256sum_src}"; break; + fi; ;; + *) if [ "${_urls_count}" -ge 1 ]; then + rtl_log_msg warning "Warning: failed to fetch URL \`%s', retrying with alternative URL... (wget(1) exit status: %s)"\ + "${_url}" "$((${_rc}-2))"; + else + rtl_log_msg fatal "Error: failed to fetch URL \`%s' (wget(1) exit status: %s)"\ + "${_url}" "$((${_rc}-2))"; break; + fi; ;; + esac; + done; + return "${_rc}"; }; # vim:filetype=sh -- cgit v1.2.3