From 3689626622516dec6f76b9b4dab05a7544ab583d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucio=20Andr=C3=A9s=20Illanes=20Albornoz?= Date: Sun, 24 Jan 2021 11:33:40 +0000 Subject: Reimplements Git repository mirroring because Git is a special snowflake. etc/{README.md,pkgtool.usage}: updated. groups/221.native_packages_dev.group:cparser:${PKG_DISABLED}: disabled due to repository unavailability. groups/251.native_packages_lib.group:libfirm:${PKG_DISABLED}: disabled due to repository unavailability. midipix.env:${DEFAULT_BUILD_VARS}: adds MIRRORS_GIT. midipix.env:${DEFAULT_MIRRORS{,_GIT}}: split; mirror Git repositories via https://midipix.lucioillanes.de/repos_git/. pkgtool.sh:pkgtoolp_mirror_fetch(): call rtl_fetch_mirror_urls_git() vs. rtl_fetch_urls_git(). pkgtool.sh:pkgtoolp_mirror_fetch(): symlink instead of downloading given package w/ parent package. pkgtool.sh:pkgtoolp_mirror{,_fetch}(): split archives vs. Git repositories directory name. subr.rtl/rtl_fetch.subr: factor out ${DEFAULT_GIT_ARGS}. subr.rtl/rtl_fetch.subr:rtl_fetch_mirror_urls_git(): initial implementation. subr.rtl/rtl_fetch.subr:rtlp_fetch_url_git(): always return on failure. subr.rtl/rtl_fetch.subr:rtlp_fetch_url_git(): correctly attempt to git-clone(1) from ${PKG_URL} and then via ${PKG_MIRRORS}. subr.rtl/rtl_fetch.subr:{rtlp_fetch_url_git,rtl_fetch_urls_git}(): cleanup. {subr/pkg_fetch_download.subr,vars/{gcc,python[23]{,_host}}.vars}: call rtl_fetch_urls_git() w/ ${PKG_NAME} and ${PKG_MIRRORS_GIT:-}. --- subr.rtl/rtl_fetch.subr | 66 +++++++++++++++++++++++++++++++++++-------------- 1 file changed, 47 insertions(+), 19 deletions(-) (limited to 'subr.rtl') diff --git a/subr.rtl/rtl_fetch.subr b/subr.rtl/rtl_fetch.subr index 5d21820f..6cb49a94 100644 --- a/subr.rtl/rtl_fetch.subr +++ b/subr.rtl/rtl_fetch.subr @@ -2,39 +2,62 @@ # set +o errexit -o noglob -o nounset is assumed. # +rtl_fetch_mirror_urls_git() { + local _git_args="${1}" _tgtdir="${2}" _rc=0 _repo_dname="" _subdir="" _url="" _url_spec=""; shift 2; + + for _url_spec in "${@}"; do + _subdir="${_url_spec%=*}"; _url="${_url_spec#*=}"; _url="${_url%@*}"; + _repo_dname="${_url##*/}"; [ "${_repo_dname%.git}" = "${_repo_dname}" ] && _repo_dname="${_repo_dname}.git"; + (set -o errexit -o noglob -o nounset; + rtl_flock_acquire 4 || exit "${?}"; + trap "rm -f \"${_tgtdir}/.fetching\"" EXIT; + if [ -e "${_tgtdir}/${_repo_dname}" ]; then + (rtl_fileop cd "${_tgtdir}/${_repo_dname}" && git fetch ${_git_args} --all) || return 1; + else (rtl_fileop cd "${_tgtdir}" && git clone ${_git_args} --mirror "${_url}") || return 1; + fi) 4<>"${_tgtdir}/.fetching"; + if [ "${?}" -ne 0 ]; then + _rc=1; + fi; + done; return "${_rc}"; +}; + rtlp_fetch_url_git() { - local _cache_dname="${1}" _git_branch="${2}" _mirrors="${3}" _pkg_name="${4}" _subdir="${5}" _tgtdir="${6}" _url="${7}"\ - _clonefl=0 _oldpwd="" _url_base=""; + local _cache_dname="${1}" _git_args="${2}" _git_branch="${3}" _mirrors="${4}" _pkg_name="${5}"\ + _subdir="${6}" _tgtdir="${7}" _url="${8}" _clonefl=0 _oldpwd="" _url_base=""; + (set -o errexit -o noglob -o nounset; rtl_flock_acquire 4 || exit "${?}"; trap "rm -f \"${_cache_dname}/${_subdir%%[/]}.fetching\"" EXIT; if [ -e "${_cache_dname}/${_subdir}" ]; then (rtl_fileop cd "${_cache_dname}/${_subdir}" &&\ - git pull ${DEFAULT_GIT_ARGS} origin "${_git_branch:-main}"); + git pull ${_git_args} origin "${_git_branch:-main}") || return 1; (rtl_fileop cd "${_cache_dname}/${_subdir}" &&\ - git submodule update); - else - for _url_base in ${_url%/*} ${_mirrors}; do - if git clone ${DEFAULT_GIT_ARGS} "${_url_base}/${_pkg_name}/${_url##*/}" "${_cache_dname}/${_subdir}"; then - _clonefl=1; break; - fi; - done; + git submodule update) || return 1; + else if git clone ${_git_args} "${_url}" "${_cache_dname}/${_subdir}"; then + _clonefl=1; + else for _url_base in ${_mirrors}; do + if git clone ${_git_args} "${_url_base}/${_pkg_name}/${_url##*/}" "${_cache_dname}/${_subdir}"; then + _clonefl=1; break; + fi; + done; + fi; if [ "${_clonefl}" -eq 0 ]; then return 1; else if [ -n "${_git_branch}" ]; then (rtl_fileop cd "${_cache_dname}/${_subdir}" &&\ - git checkout "${_git_branch}"); + git checkout "${_git_branch}") || return 1; fi; (rtl_fileop cd "${_cache_dname}/${_subdir}" &&\ - git submodule update --init); + git submodule update --init) || return 1; fi; fi; if [ "${_cache_dname}" != "${_tgtdir}" ]; then - _oldpwd="${PWD}"; rtl_fileop cd "${_tgtdir}"; - rtl_fileop rm "${_tgtdir}/${_subdir}"; - rtl_fileop cp "${_cache_dname}/${_subdir}" "${_tgtdir}"; - rtl_fileop cd "${_oldpwd}"; + _oldpwd="${PWD}"; rtl_fileop cd "${_tgtdir}" || return 1; + rtl_fileop rm "${_tgtdir}/${_subdir}" || return 1; + rtl_fileop cp "${_cache_dname}/${_subdir}" "${_tgtdir}" || return 1; + rtl_fileop cd "${_oldpwd}" || return 1; fi) 4<>"${_cache_dname}/${_subdir%%[/]}.fetching"; + if [ "${?}" -eq 0 ]; then cd "$(pwd)"; else @@ -43,13 +66,17 @@ rtlp_fetch_url_git() { }; rtl_fetch_urls_git() { - local _cache_dname="${1}" _tgtdir="${2}" _pkg_name="${3}" _mirrors="${4}" _git_branch="" _subdir="" _url="" _url_spec=""; shift 4; + local _cache_dname="${1}" _git_args="${2}" _tgtdir="${3}" _pkg_name="${4}" _mirrors="${5}"\ + _git_branch="" _subdir="" _url="" _url_spec=""; shift 5; + for _url_spec in "${@}"; do _subdir="${_url_spec%=*}"; _url="${_url_spec#*=}"; _url="${_url%@*}"; if [ "${_url_spec#*@}" != "${_url_spec}" ]; then _git_branch=${_url_spec#*@}; fi; - if ! rtlp_fetch_url_git "${_cache_dname}" "${_git_branch}" "${_mirrors}" "${_pkg_name}" "${_subdir}" "${_tgtdir}" "${_url}"; then + if ! rtlp_fetch_url_git \ + "${_cache_dname}" "${_git_args}" "${_git_branch}" "${_mirrors}"\ + "${_pkg_name}" "${_subdir}" "${_tgtdir}" "${_url}"; then return 1; fi; done; @@ -57,8 +84,9 @@ rtl_fetch_urls_git() { # N.B. URLs ($1) may contain `?' or '&' characters. rtl_fetch_url_wget() { - local _urls="${1}" _sha256sum_src="${2}" _target_dname="${3}" _target_fname="${4}" _target_name="${5}" _mirrors="${6:-}" \ + local _urls="${1}" _sha256sum_src="${2}" _target_dname="${3}" _target_fname="${4}" _target_name="${5}" _mirrors="${6:-}"\ _rc=0 _target_fname_full="" _url="" _url_base="" _urls_count=0 _urls_full=""; + _urls_full="${_urls}"; for _url_base in ${_mirrors}; do _urls_full="$(rtl_lconcat "${_urls_full}" "${_url_base%/}/${_target_name}/${_target_fname}")"; -- cgit v1.2.3