From 3689626622516dec6f76b9b4dab05a7544ab583d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucio=20Andr=C3=A9s=20Illanes=20Albornoz?= Date: Sun, 24 Jan 2021 11:33:40 +0000 Subject: Reimplements Git repository mirroring because Git is a special snowflake. etc/{README.md,pkgtool.usage}: updated. groups/221.native_packages_dev.group:cparser:${PKG_DISABLED}: disabled due to repository unavailability. groups/251.native_packages_lib.group:libfirm:${PKG_DISABLED}: disabled due to repository unavailability. midipix.env:${DEFAULT_BUILD_VARS}: adds MIRRORS_GIT. midipix.env:${DEFAULT_MIRRORS{,_GIT}}: split; mirror Git repositories via https://midipix.lucioillanes.de/repos_git/. pkgtool.sh:pkgtoolp_mirror_fetch(): call rtl_fetch_mirror_urls_git() vs. rtl_fetch_urls_git(). pkgtool.sh:pkgtoolp_mirror_fetch(): symlink instead of downloading given package w/ parent package. pkgtool.sh:pkgtoolp_mirror{,_fetch}(): split archives vs. Git repositories directory name. subr.rtl/rtl_fetch.subr: factor out ${DEFAULT_GIT_ARGS}. subr.rtl/rtl_fetch.subr:rtl_fetch_mirror_urls_git(): initial implementation. subr.rtl/rtl_fetch.subr:rtlp_fetch_url_git(): always return on failure. subr.rtl/rtl_fetch.subr:rtlp_fetch_url_git(): correctly attempt to git-clone(1) from ${PKG_URL} and then via ${PKG_MIRRORS}. subr.rtl/rtl_fetch.subr:{rtlp_fetch_url_git,rtl_fetch_urls_git}(): cleanup. {subr/pkg_fetch_download.subr,vars/{gcc,python[23]{,_host}}.vars}: call rtl_fetch_urls_git() w/ ${PKG_NAME} and ${PKG_MIRRORS_GIT:-}. --- etc/README.md | 35 +++++++++---------- etc/pkgtool.usage | 30 ++++++++-------- groups/221.native_packages_dev.group | 1 + groups/251.native_packages_lib.group | 1 + midipix.env | 30 +++++++++------- pkgtool.sh | 51 +++++++++++++++++++--------- subr.rtl/rtl_fetch.subr | 66 +++++++++++++++++++++++++----------- subr/pkg_fetch_download.subr | 8 +++-- vars/gcc.vars | 11 ++++-- vars/python2.vars | 2 +- vars/python2_host.vars | 2 +- vars/python3.vars | 2 +- vars/python3_host.vars | 2 +- 13 files changed, 153 insertions(+), 88 deletions(-) diff --git a/etc/README.md b/etc/README.md index 914a953a..080f29fa 100644 --- a/etc/README.md +++ b/etc/README.md @@ -379,24 +379,24 @@ for a list of package build steps and how they are overriden. ## 3.5. ``pkgtool.sh`` ``` -usage: ./pkgtool.sh [-a nt32|nt64] [-b debug|release] [-i|-m |-r|-s|-t] - [=[ ..]] name - - -a nt32|nt64 Selects 32-bit or 64-bit architecture; defaults to nt64. - -b debug|release Selects debug or release build kind; defaults to debug. - -i List package variables and dependencies of single named package. - -m Setup package archives mirror in - -r List reverse dependencies of single named package. - -s Enter interactive package build shell environment for single - named package; requires a package dump file. If the package - has not been built yet or built successfully, it will be rebuilt - at build steps up until, by default, the `build' build step and - forcibly aborted and dumped prior to enterting the shell. - -t Produce tarball of package build root directory and build log - file for the purpose of distribution given build failure. +usage: ./pkgtool.sh [-a nt32|nt64] [-b debug|release] [-i|-m ,|-r|-s|-t] + [=[ ..]] name + + -a nt32|nt64 Selects 32-bit or 64-bit architecture; defaults to nt64. + -b debug|release Selects debug or release build kind; defaults to debug. + -i List package variables and dependencies of single named package. + -m , Setup package archives mirror in and Git repositories mirror in + -r List reverse dependencies of single named package. + -s Enter interactive package build shell environment for single + named package; requires a package dump file. If the package + has not been built yet or built successfully, it will be rebuilt + at build steps up until, by default, the `build' build step and + forcibly aborted and dumped prior to enterting the shell. + -t Produce tarball of package build root directory and build log + file for the purpose of distribution given build failure. =[ ..] - Override build variable. + Override build variable. ``` > N.B. When using ``pkgtool.sh`` on a build w/ build variables (see section [4](#4-build-variables)) @@ -581,7 +581,8 @@ VERSION`` and/or ``URLS_GIT``, respectively. | MAKEFLAGS_INSTALL_EXTRA | Additional list of ``make(1)`` flags during package ``make(1)`` installation | | MAKEFLAGS_VERBOSITY | Variable-value pair to pass to ``make(1)`` in order to force echo-back of command lines prior to execution | | MAKE_INSTALL_VNAME | Variable name of ``make(1)`` installation destination directory variable during package ``make(1)`` installation | -| MIRRORS | List of package archive/repository mirror base URLs to attempt downloading from; cf. ``pkgtool.sh -m `` | +| MIRRORS | List of package archive mirror base URLs to attempt downloading from; cf. ``pkgtool.sh -m `` | +| MIRRORS_GIT | List of package Git repository mirror base URLs to attempt cloning from; cf. ``pkgtool.sh -m `` | | NO_CLEAN | Inhibit cleaning of package build directory beneath ``${PKG_BASE_DIR}`` pre-finish | | NO_CLEAN_BASE_DIR | Inhibit cleaning of package build root directory beneath ``${BUILD_WORKDIR}`` | | NO_LOG_VARS | Inhibit logging of build & package variables pre-package build | diff --git a/etc/pkgtool.usage b/etc/pkgtool.usage index 3031180d..0ebc5acf 100644 --- a/etc/pkgtool.usage +++ b/etc/pkgtool.usage @@ -1,18 +1,18 @@ -usage: ./pkgtool.sh [-a nt32|nt64] [-b debug|release] [-i|-m |-r|-s|-t] - [=[ ..]] name +usage: ./pkgtool.sh [-a nt32|nt64] [-b debug|release] [-i|-m ,|-r|-s|-t] + [=[ ..]] name - -a nt32|nt64 Selects 32-bit or 64-bit architecture; defaults to nt64. - -b debug|release Selects debug or release build kind; defaults to debug. - -i List package variables and dependencies of single named package. - -m Setup package archives mirror in - -r List reverse dependencies of single named package. - -s Enter interactive package build shell environment for single - named package; requires a package dump file. If the package - has not been built yet or built successfully, it will be rebuilt - at build steps up until, by default, the `build' build step and - forcibly aborted and dumped prior to enterting the shell. - -t Produce tarball of package build root directory and build log - file for the purpose of distribution given build failure. + -a nt32|nt64 Selects 32-bit or 64-bit architecture; defaults to nt64. + -b debug|release Selects debug or release build kind; defaults to debug. + -i List package variables and dependencies of single named package. + -m , Setup package archives mirror in and Git repositories mirror in + -r List reverse dependencies of single named package. + -s Enter interactive package build shell environment for single + named package; requires a package dump file. If the package + has not been built yet or built successfully, it will be rebuilt + at build steps up until, by default, the `build' build step and + forcibly aborted and dumped prior to enterting the shell. + -t Produce tarball of package build root directory and build log + file for the purpose of distribution given build failure. =[ ..] - Override build variable. + Override build variable. diff --git a/groups/221.native_packages_dev.group b/groups/221.native_packages_dev.group index 3d2db8bf..c69cd602 100644 --- a/groups/221.native_packages_dev.group +++ b/groups/221.native_packages_dev.group @@ -23,6 +23,7 @@ patch patchutils perl python2 python3 ruby sbsigntools smallbasic tk")"; : ${PKG_CPARSER_URLS_GIT:="cparser=http://pp.info.uni-karlsruhe.de/git/cparser.git@master bcparser=${DEFAULT_GITROOT}/compilers/bcparser@main"}; : ${PKG_CPARSER_BASE_DIR:=${BUILD_WORKDIR}/cparser+libfirm-native-${DEFAULT_TARGET}}; : ${PKG_CPARSER_BUILD_DIR:=build/cparser}; +: ${PKG_CPARSER_DISABLED:=1}; : ${PKG_CPARSER_CONFIGURE:=bcparser/configure}; : ${PKG_CPARSER_CONFIGURE_ARGS:="--source-dir=../../cparser --prefix="}; : ${PKG_CPARSER_NO_CLEAN_BASE_DIR:=1}; diff --git a/groups/251.native_packages_lib.group b/groups/251.native_packages_lib.group index 1ed6188e..32e867ba 100644 --- a/groups/251.native_packages_lib.group +++ b/groups/251.native_packages_lib.group @@ -81,6 +81,7 @@ libxslt libz libzip lzo sdl1 sdl1_net sdl1_image sdl1_ttf sdl2 sdl2_image sdl2_t : ${PKG_LIBFIRM_URLS_GIT:="libfirm=http://pp.info.uni-karlsruhe.de/git/libfirm.git@master bfirm=${DEFAULT_GITROOT}/compilers/bfirm@main"}; : ${PKG_LIBFIRM_BASE_DIR:=${BUILD_WORKDIR}/cparser+libfirm-native-${DEFAULT_TARGET}}; : ${PKG_LIBFIRM_BUILD_DIR:=build/libfirm}; +: ${PKG_LIBFIRM_DISABLED:=1}; : ${PKG_LIBFIRM_CONFIGURE:=bfirm/configure}; : ${PKG_LIBFIRM_CONFIGURE_ARGS:="--source-dir=../../libfirm --prefix="}; : ${PKG_LIBFLAC_DEPENDS:="libogg"}; diff --git a/midipix.env b/midipix.env index bf26893f..c594b27a 100644 --- a/midipix.env +++ b/midipix.env @@ -15,20 +15,21 @@ DEFAULT_BUILD_STEPS=" build \ install_subdirs install_make install_files install_libs install install_rpm clean"; DEFAULT_BUILD_VARS=" - AR BASE_DIR BUILD_DIR BUILD_STEPS_DISABLE BUILD_TYPE CC \ - CFLAGS_BUILD CFLAGS_BUILD_EXTRA CFLAGS_CONFIGURE CFLAGS_CONFIGURE_EXTRA \ - CONFIG_CACHE CONFIG_CACHE_EXTRA CONFIG_CACHE_LOCAL CONFIGURE \ - CONFIGURE_ARGS CONFIGURE_ARGS_EXTRA CXX CXXFLAGS_CONFIGURE \ - CXXFLAGS_CONFIGURE_EXTRA DEPENDS DESTDIR DESTDIR_HOST DISABLED \ - ENV_VARS_EXTRA FNAME FORCE_AUTORECONF GITROOT INHERIT_FROM \ - INSTALL_FILES INSTALL_FILES_DESTDIR INSTALL_FILES_DESTDIR_EXTRA \ - INSTALL_TARGET INSTALL_TARGET_EXTRA IN_TREE LDFLAGS_BUILD_EXTRA \ - LDFLAGS_CONFIGURE LDFLAGS_CONFIGURE_EXTRA LIBTOOL MAKE \ - MAKE_INSTALL_VNAME MAKE_SUBDIRS MAKEFLAGS_BUILD MAKEFLAGS_BUILD_EXTRA \ + AR BASE_DIR BUILD_DIR BUILD_STEPS_DISABLE BUILD_TYPE \ + CC CFLAGS_BUILD CFLAGS_BUILD_EXTRA CFLAGS_CONFIGURE \ + CFLAGS_CONFIGURE_EXTRA CONFIG_CACHE CONFIG_CACHE_EXTRA \ + CONFIG_CACHE_LOCAL CONFIGURE CONFIGURE_ARGS CONFIGURE_ARGS_EXTRA \ + CXX CXXFLAGS_CONFIGURE CXXFLAGS_CONFIGURE_EXTRA \ + DEPENDS DESTDIR DESTDIR_HOST DISABLED ENV_VARS_EXTRA \ + FNAME FORCE_AUTORECONF GITROOT INHERIT_FROM INSTALL_FILES \ + INSTALL_FILES_DESTDIR INSTALL_FILES_DESTDIR_EXTRA INSTALL_TARGET \ + INSTALL_TARGET_EXTRA IN_TREE LDFLAGS_BUILD_EXTRA LDFLAGS_CONFIGURE \ + LDFLAGS_CONFIGURE_EXTRA LIBTOOL MAKE MAKE_INSTALL_VNAME \ + MAKE_SUBDIRS MAKEFLAGS_BUILD MAKEFLAGS_BUILD_EXTRA \ MAKEFLAGS_INSTALL MAKEFLAGS_INSTALL_EXTRA MAKEFLAGS_VERBOSITY \ - MIRRORS NO_CLEAN NO_CLEAN_BASE_DIR NO_LOG_VARS PYTHON PATCHES_EXTRA \ - PKG_CONFIG PKG_CONFIG_LIBDIR PKGLIST_DISABLE PREFIX RANLIB \ - RPM_DISABLE SHA256SUM SUBDIR TARGET URL URLS_GIT VERSION"; + MIRRORS MIRRORS_GIT NO_CLEAN NO_CLEAN_BASE_DIR NO_LOG_VARS PYTHON \ + PATCHES_EXTRA PKG_CONFIG PKG_CONFIG_LIBDIR PKGLIST_DISABLE PREFIX \ + RANLIB RPM_DISABLE SHA256SUM SUBDIR TARGET URL URLS_GIT VERSION"; # Path names : ${PREFIX_ROOT:="${HOME}/midipix"}; @@ -69,6 +70,9 @@ DEFAULT_BUILD_VARS=" https://midipix.org/mirror/ https://midipix.lucioillanes.de/archives/ "}; +: ${DEFAULT_MIRRORS_GIT:=" +https://midipix.lucioillanes.de/repos_git/ +"}; # Default flags & variables : ${DEFAULT_BUILD_CPUS:=}; diff --git a/pkgtool.sh b/pkgtool.sh index f04875c1..e2328244 100755 --- a/pkgtool.sh +++ b/pkgtool.sh @@ -51,50 +51,69 @@ pkgtoolp_info() { }; pkgtoolp_mirror() { - local _mirror_dname="${1}" _group_name="" _pkg_name="" _pkg_parent="" _rc=0; - umask 022; + local _mirror_dnames="${1}" _group_name="" _mirror_dname="" _mirror_dname_git=""\ + _pkg_name="" _pkg_parent="" _rc=0; + + rtl_lassign "_mirror_dname _mirror_dname_git" "," "${_mirror_dnames}"; umask 022; if ! ex_pkg_load_groups; then _rc=1; _status="Error: failed to load build groups."; + elif ! rtl_fileop mkdir "${_mirror_dname}"; then + _rc=1; _status="Error: failed to create \`${_mirror_dname}'."; + elif ! rtl_fileop mkdir "${_mirror_dname_git}"; then + _rc=1; _status="Error: failed to create \`${_mirror_dname_git}'."; else for _group_name in ${EX_PKG_BUILD_GROUPS}; do for _pkg_name in $(ex_pkg_get_packages "${_group_name}"); do _pkg_parent="$(rtl_get_var_unsafe -u "PKG_${_pkg_name}_INHERIT_FROM")"; - pkgtoolp_mirror_fetch "${_mirror_dname}" "${_pkg_name}" "${_pkg_parent:-${_pkg_name}}"; _rc="${?}"; + if ! pkgtoolp_mirror_fetch \ + "${_mirror_dname}" "${_mirror_dname_git}" "${_pkg_name}"\ + "${_pkg_parent:-${_pkg_name}}"; then + _rc=1; _status="Warning: failed to mirror one or more packages."; + fi; done; done; fi; return "${_rc}"; }; pkgtoolp_mirror_fetch() { - local _mirror_dname="${1}" _pkg_name="${2}" _pkg_name_real="${3}" _pkg_disabled=0 _pkg_sha256sum="" _pkg_url=""; + local _mirror_dname="${1}" _mirror_dname_git="${2}" _pkg_name="${3}" _pkg_name_real="${4}"\ + _pkg_disabled=0 _pkg_sha256sum="" _pkg_url="" _rc=0; + if _pkg_disabled="$(rtl_get_var_unsafe -u "PKG_${_pkg_name_real}_DISABLED")"\ && [ "${_pkg_disabled:-0}" -eq 1 ]; then rtl_log_msg info "Package \`%s' (parent package: \`%s') disabled, skipping." "${_pkg_name}" "${_pkg_name_real}"; - else - if _pkg_url="$(rtl_get_var_unsafe -u "PKG_${_pkg_name_real}_URL")"\ + else if _pkg_url="$(rtl_get_var_unsafe -u "PKG_${_pkg_name_real}_URL")"\ && _pkg_sha256sum="$(rtl_get_var_unsafe -u "PKG_${_pkg_name_real}_SHA256SUM")"; then if [ "${_pkg_name}" != "${_pkg_name_real}" ]; then rtl_log_msg info "Mirroring package \`%s' (parent package: \`%s'), archive URL(s): \`%s'..." "${_pkg_name}" "${_pkg_name_real}" "${_pkg_url}"; + if ! rtl_fileop ln_symbolic "${_pkg_name_real}" "${_mirror_dname}/${_pkg_name}"; then + _rc=1; rtl_log_msg warning "Failed to create symbolic link \`%s' for package \`%s' w/ parent package \`%s'."\ + "${_mirror_dname}/${_pkg_name}" "${_pkg_name}" "${_pkg_name_real}"; + fi; else rtl_log_msg info "Mirroring package \`%s', archive URL(s): \`%s'..." "${_pkg_name}" "${_pkg_url}"; - fi; - if ! rtl_fileop mkdir "${_mirror_dname}/${_pkg_name}"\ - || ! rtl_fetch_url_wget "${_pkg_url}" "${_pkg_sha256sum}" "${_mirror_dname}/${_pkg_name}" "${_pkg_url##*/}" "${_pkg_name_real}" ""; then - _rc="${?}"; rtl_log_msg warning "Failed to mirror package \`%s', skipping." "${_pkg_name}"; + if ! rtl_fileop mkdir "${_mirror_dname}/${_pkg_name}"\ + || ! rtl_fetch_url_wget "${_pkg_url}" "${_pkg_sha256sum}" "${_mirror_dname}/${_pkg_name}" "${_pkg_url##*/}" "${_pkg_name_real}" ""; then + _rc=1; rtl_log_msg warning "Failed to mirror package \`%s', skipping." "${_pkg_name}"; + fi; fi; elif _pkg_url="$(rtl_get_var_unsafe -u "PKG_${_pkg_name_real}_URLS_GIT")"; then if [ "${_pkg_name}" != "${_pkg_name_real}" ]; then rtl_log_msg info "Mirroring package \`%s' (parent package: \`%s'), Git URL(s): \`%s'..." "${_pkg_name}" "${_pkg_name_real}" "${_pkg_url}"; + if ! rtl_fileop ln_symbolic "${_pkg_name_real}" "${_mirror_dname_git}/${_pkg_name}"; then + _rc=1; rtl_log_msg warning "Failed to create symbolic link \`%s' for package \`%s' w/ parent package \`%s'."\ + "${_mirror_dname_git}/${_pkg_name}" "${_pkg_name}" "${_pkg_name_real}"; + fi; else rtl_log_msg info "Mirroring package \`%s', Git URL(s): \`%s'..." "${_pkg_name}" "${_pkg_url}"; - fi; - if ! rtl_fileop mkdir "${_mirror_dname}/${_pkg_name}"\ - || ! rtl_fetch_urls_git --mirror "${_mirror_dname}/${_pkg_name}" "${_mirror_dname}/${_pkg_name}" "${_pkg_name}" "" ${_pkg_url}; then - _rc="${?}"; rtl_log_msg warning "Failed to mirror package \`%s', skipping." "${_pkg_name}"; + if ! rtl_fileop mkdir "${_mirror_dname_git}/${_pkg_name}"\ + || ! rtl_fetch_mirror_urls_git "${DEFAULT_GIT_ARGS}" "${_mirror_dname_git}/${_pkg_name}" ${_pkg_url}; then + _rc=1; rtl_log_msg warning "Failed to mirror package \`%s', skipping." "${_pkg_name}"; + fi; fi; else - rtl_log_msg warning "Package \`%s' has neither archive nor Git URL(s), skipping." "${_pkg_name}"; + _rc=1; rtl_log_msg warning "Package \`%s' has neither archive nor Git URL(s), skipping." "${_pkg_name}"; fi; - fi; + fi; return "${_rc}"; }; pkgtoolp_restart_at() { diff --git a/subr.rtl/rtl_fetch.subr b/subr.rtl/rtl_fetch.subr index 5d21820f..6cb49a94 100644 --- a/subr.rtl/rtl_fetch.subr +++ b/subr.rtl/rtl_fetch.subr @@ -2,39 +2,62 @@ # set +o errexit -o noglob -o nounset is assumed. # +rtl_fetch_mirror_urls_git() { + local _git_args="${1}" _tgtdir="${2}" _rc=0 _repo_dname="" _subdir="" _url="" _url_spec=""; shift 2; + + for _url_spec in "${@}"; do + _subdir="${_url_spec%=*}"; _url="${_url_spec#*=}"; _url="${_url%@*}"; + _repo_dname="${_url##*/}"; [ "${_repo_dname%.git}" = "${_repo_dname}" ] && _repo_dname="${_repo_dname}.git"; + (set -o errexit -o noglob -o nounset; + rtl_flock_acquire 4 || exit "${?}"; + trap "rm -f \"${_tgtdir}/.fetching\"" EXIT; + if [ -e "${_tgtdir}/${_repo_dname}" ]; then + (rtl_fileop cd "${_tgtdir}/${_repo_dname}" && git fetch ${_git_args} --all) || return 1; + else (rtl_fileop cd "${_tgtdir}" && git clone ${_git_args} --mirror "${_url}") || return 1; + fi) 4<>"${_tgtdir}/.fetching"; + if [ "${?}" -ne 0 ]; then + _rc=1; + fi; + done; return "${_rc}"; +}; + rtlp_fetch_url_git() { - local _cache_dname="${1}" _git_branch="${2}" _mirrors="${3}" _pkg_name="${4}" _subdir="${5}" _tgtdir="${6}" _url="${7}"\ - _clonefl=0 _oldpwd="" _url_base=""; + local _cache_dname="${1}" _git_args="${2}" _git_branch="${3}" _mirrors="${4}" _pkg_name="${5}"\ + _subdir="${6}" _tgtdir="${7}" _url="${8}" _clonefl=0 _oldpwd="" _url_base=""; + (set -o errexit -o noglob -o nounset; rtl_flock_acquire 4 || exit "${?}"; trap "rm -f \"${_cache_dname}/${_subdir%%[/]}.fetching\"" EXIT; if [ -e "${_cache_dname}/${_subdir}" ]; then (rtl_fileop cd "${_cache_dname}/${_subdir}" &&\ - git pull ${DEFAULT_GIT_ARGS} origin "${_git_branch:-main}"); + git pull ${_git_args} origin "${_git_branch:-main}") || return 1; (rtl_fileop cd "${_cache_dname}/${_subdir}" &&\ - git submodule update); - else - for _url_base in ${_url%/*} ${_mirrors}; do - if git clone ${DEFAULT_GIT_ARGS} "${_url_base}/${_pkg_name}/${_url##*/}" "${_cache_dname}/${_subdir}"; then - _clonefl=1; break; - fi; - done; + git submodule update) || return 1; + else if git clone ${_git_args} "${_url}" "${_cache_dname}/${_subdir}"; then + _clonefl=1; + else for _url_base in ${_mirrors}; do + if git clone ${_git_args} "${_url_base}/${_pkg_name}/${_url##*/}" "${_cache_dname}/${_subdir}"; then + _clonefl=1; break; + fi; + done; + fi; if [ "${_clonefl}" -eq 0 ]; then return 1; else if [ -n "${_git_branch}" ]; then (rtl_fileop cd "${_cache_dname}/${_subdir}" &&\ - git checkout "${_git_branch}"); + git checkout "${_git_branch}") || return 1; fi; (rtl_fileop cd "${_cache_dname}/${_subdir}" &&\ - git submodule update --init); + git submodule update --init) || return 1; fi; fi; if [ "${_cache_dname}" != "${_tgtdir}" ]; then - _oldpwd="${PWD}"; rtl_fileop cd "${_tgtdir}"; - rtl_fileop rm "${_tgtdir}/${_subdir}"; - rtl_fileop cp "${_cache_dname}/${_subdir}" "${_tgtdir}"; - rtl_fileop cd "${_oldpwd}"; + _oldpwd="${PWD}"; rtl_fileop cd "${_tgtdir}" || return 1; + rtl_fileop rm "${_tgtdir}/${_subdir}" || return 1; + rtl_fileop cp "${_cache_dname}/${_subdir}" "${_tgtdir}" || return 1; + rtl_fileop cd "${_oldpwd}" || return 1; fi) 4<>"${_cache_dname}/${_subdir%%[/]}.fetching"; + if [ "${?}" -eq 0 ]; then cd "$(pwd)"; else @@ -43,13 +66,17 @@ rtlp_fetch_url_git() { }; rtl_fetch_urls_git() { - local _cache_dname="${1}" _tgtdir="${2}" _pkg_name="${3}" _mirrors="${4}" _git_branch="" _subdir="" _url="" _url_spec=""; shift 4; + local _cache_dname="${1}" _git_args="${2}" _tgtdir="${3}" _pkg_name="${4}" _mirrors="${5}"\ + _git_branch="" _subdir="" _url="" _url_spec=""; shift 5; + for _url_spec in "${@}"; do _subdir="${_url_spec%=*}"; _url="${_url_spec#*=}"; _url="${_url%@*}"; if [ "${_url_spec#*@}" != "${_url_spec}" ]; then _git_branch=${_url_spec#*@}; fi; - if ! rtlp_fetch_url_git "${_cache_dname}" "${_git_branch}" "${_mirrors}" "${_pkg_name}" "${_subdir}" "${_tgtdir}" "${_url}"; then + if ! rtlp_fetch_url_git \ + "${_cache_dname}" "${_git_args}" "${_git_branch}" "${_mirrors}"\ + "${_pkg_name}" "${_subdir}" "${_tgtdir}" "${_url}"; then return 1; fi; done; @@ -57,8 +84,9 @@ rtl_fetch_urls_git() { # N.B. URLs ($1) may contain `?' or '&' characters. rtl_fetch_url_wget() { - local _urls="${1}" _sha256sum_src="${2}" _target_dname="${3}" _target_fname="${4}" _target_name="${5}" _mirrors="${6:-}" \ + local _urls="${1}" _sha256sum_src="${2}" _target_dname="${3}" _target_fname="${4}" _target_name="${5}" _mirrors="${6:-}"\ _rc=0 _target_fname_full="" _url="" _url_base="" _urls_count=0 _urls_full=""; + _urls_full="${_urls}"; for _url_base in ${_mirrors}; do _urls_full="$(rtl_lconcat "${_urls_full}" "${_url_base%/}/${_target_name}/${_target_fname}")"; diff --git a/subr/pkg_fetch_download.subr b/subr/pkg_fetch_download.subr index 93183f8f..43f4388c 100644 --- a/subr/pkg_fetch_download.subr +++ b/subr/pkg_fetch_download.subr @@ -5,12 +5,16 @@ pkg_fetch_download() { if [ "${ARG_FETCH_FORCE:-}" != "offline" ]; then if [ -n "${PKG_URL:-}" ]; then - if ! rtl_fetch_url_wget "${PKG_URL}" "${PKG_SHA256SUM}" "${BUILD_DLCACHEDIR}" "${PKG_FNAME}" "${PKG_NAME}" "${PKG_MIRRORS:-}"; then + if ! rtl_fetch_url_wget \ + "${PKG_URL}" "${PKG_SHA256SUM}" "${BUILD_DLCACHEDIR}"\ + "${PKG_FNAME}" "${PKG_NAME}" "${PKG_MIRRORS:-}"; then return 1; fi; fi; if [ -n "${PKG_URLS_GIT:-}" ]; then - if ! rtl_fetch_urls_git "${BUILD_DLCACHEDIR}" "${PKG_BASE_DIR}" "${PKG_NAME}" "${PKG_MIRRORS:-}" ${PKG_URLS_GIT}; then + if ! rtl_fetch_urls_git \ + "${BUILD_DLCACHEDIR}" "${DEFAULT_GIT_ARGS}" "${PKG_BASE_DIR}"\ + "${PKG_NAME}" "${PKG_MIRRORS_GIT:-}" ${PKG_URLS_GIT}; then return 1; fi; fi; diff --git a/vars/gcc.vars b/vars/gcc.vars index 92a56202..c7e094a9 100644 --- a/vars/gcc.vars +++ b/vars/gcc.vars @@ -122,7 +122,10 @@ pkgp_gcc_stage1_all() { return 1; fi; if ! ex_pkg_state_test "${PKG_NAME}" fetch "${_restart_at}"; then - rtl_fetch_urls_git "${BUILD_DLCACHEDIR}" "${PKG_BASE_DIR}" "${PKG_MIRRORS:-}" "cbb-gcc-${PKG_VERSION}=${PKG_GITROOT}/cbb/cbb-gcc-${PKG_VERSION}" || return 1; + rtl_fetch_urls_git \ + "${BUILD_DLCACHEDIR}" "${DEFAULT_GIT_ARGS}" "${PKG_BASE_DIR}" \ + "${PKG_NAME}" "${PKG_MIRRORS_GIT:-}" \ + "cbb-gcc-${PKG_VERSION}=${PKG_GITROOT}/cbb/cbb-gcc-${PKG_VERSION}" || return 1; pkgp_gcc_state_set "fetch" -"patch"; fi; if ! ex_pkg_state_test "${PKG_NAME}" patch "${_restart_at}"; then @@ -210,7 +213,11 @@ pkg_gcc_full_all() { export MAKE="make LIBTOOL=slibtool ${_vis_hide}"; pkgp_gcc_build_dir "cbb-gcc-${PKG_VERSION}" native; if ! ex_pkg_state_test "${PKG_NAME}" fetch "${_restart_at}"; then - rtl_fetch_urls_git "${BUILD_DLCACHEDIR}" "${PKG_BASE_DIR}" "${PKG_MIRRORS:-}" "cbb-gcc-${PKG_VERSION}=${PKG_GITROOT}/cbb/cbb-gcc-${PKG_VERSION}" || return 1; + rtl_fetch_urls_git \ + "${BUILD_DLCACHEDIR}" "${DEFAULT_GIT_ARGS}" \ + "${PKG_BASE_DIR}" "${PKG_NAME}" \ + "${PKG_MIRRORS_GIT:-}" \ + "cbb-gcc-${PKG_VERSION}=${PKG_GITROOT}/cbb/cbb-gcc-${PKG_VERSION}" || return 1; pkgp_gcc_state_set "fetch" -"patch"; fi; if ! ex_pkg_state_test "${PKG_NAME}" patch "${_restart_at}"; then diff --git a/vars/python2.vars b/vars/python2.vars index 047b0bbc..7bdcd205 100644 --- a/vars/python2.vars +++ b/vars/python2.vars @@ -5,7 +5,7 @@ pkg_python2_fetch_download() { if ! pkg_fetch_download\ || ! rtl_fileop rm "${PKG_BASE_DIR}/sbpython2"\ - || ! rtl_fetch_urls_git "${BUILD_DLCACHEDIR}" "${PKG_BASE_DIR}/build" "${PKG_MIRRORS:-}" "sbpython2=${DEFAULT_GITROOT}/python/sbpython2@main"\ + || ! rtl_fetch_urls_git "${BUILD_DLCACHEDIR}" "${DEFAULT_GIT_ARGS}" "${PKG_BASE_DIR}/build" "${PKG_NAME}" "${PKG_MIRRORS_GIT:-}" "sbpython2=${DEFAULT_GITROOT}/python/sbpython2@main"\ || ! ex_pkg_state_set "${PKG_NAME}" "fetch_download" "-fetch_extract"; then return 1; fi; diff --git a/vars/python2_host.vars b/vars/python2_host.vars index b4fa01bd..0cb83e2a 100644 --- a/vars/python2_host.vars +++ b/vars/python2_host.vars @@ -5,7 +5,7 @@ pkg_python2_host_fetch_download() { if ! pkg_fetch_download\ || ! rtl_fileop rm "${PKG_BASE_DIR}/sbpython2"\ - || ! rtl_fetch_urls_git "${BUILD_DLCACHEDIR}" "${PKG_BASE_DIR}/build" "${PKG_MIRRORS:-}" "sbpython2=${DEFAULT_GITROOT}/python/sbpython2@main"\ + || ! rtl_fetch_urls_git "${BUILD_DLCACHEDIR}" "${DEFAULT_GIT_ARGS}" "${PKG_BASE_DIR}/build" "${PKG_NAME}" "${PKG_MIRRORS_GIT:-}" "sbpython2=${DEFAULT_GITROOT}/python/sbpython2@main"\ || ! rtl_fileop cp "../sbpython2/project/config/cfgdefs.sh" "../sbpython2/project/config/cfgdefs.sh.orig"\ || ! sed -i"" -e"s/-ltinfo/&w/" -e"s/mb_ncurses_libs='\([^']*\)'/mb_ncurses_libs='\1 -ltinfow'/" ../sbpython2/project/config/cfgdefs.sh\ || ! sed -i"" -e'108a\ diff --git a/vars/python3.vars b/vars/python3.vars index c21d3a01..a58325c1 100644 --- a/vars/python3.vars +++ b/vars/python3.vars @@ -5,7 +5,7 @@ pkg_python3_fetch_download() { if ! pkg_fetch_download\ || ! rtl_fileop rm "${PKG_BASE_DIR}/sbpython3"\ - || ! rtl_fetch_urls_git "${BUILD_DLCACHEDIR}" "${PKG_BASE_DIR}/build" "${PKG_MIRRORS:-}" "sbpython3=${DEFAULT_GITROOT}/python/sbpython3@main"\ + || ! rtl_fetch_urls_git "${BUILD_DLCACHEDIR}" "${DEFAULT_GIT_ARGS}" "${PKG_BASE_DIR}/build" "${PKG_NAME}" "${PKG_MIRRORS_GIT:-}" "sbpython3=${DEFAULT_GITROOT}/python/sbpython3@main"\ || ! ex_pkg_state_set "${PKG_NAME}" "fetch_download" "-fetch_extract"; then return 1; fi; diff --git a/vars/python3_host.vars b/vars/python3_host.vars index dac0f5de..05b58e2b 100644 --- a/vars/python3_host.vars +++ b/vars/python3_host.vars @@ -5,7 +5,7 @@ pkg_python3_host_fetch_download() { if ! pkg_fetch_download\ || ! rtl_fileop rm "${PKG_BASE_DIR}/sbpython3"\ - || ! rtl_fetch_urls_git "${BUILD_DLCACHEDIR}" "${PKG_BASE_DIR}/build" "${PKG_MIRRORS:-}" "sbpython3=${DEFAULT_GITROOT}/python/sbpython3@main"\ + || ! rtl_fetch_urls_git "${BUILD_DLCACHEDIR}" "${DEFAULT_GIT_ARGS}" "${PKG_BASE_DIR}/build" "${PKG_NAME}" "${PKG_MIRRORS_GIT:-}" "sbpython3=${DEFAULT_GITROOT}/python/sbpython3@main"\ || ! rtl_fileop cp "../sbpython3/project/config/cfgdefs.sh" "../sbpython3/project/config/cfgdefs.sh.orig"\ || ! sed -i"" -e"s/-ltinfo/&w/" -e"s/mb_ncurses_libs='\([^']*\)'/mb_ncurses_libs='\1 -ltinfow'/" "../sbpython3/project/config/cfgdefs.sh"\ || ! sed -i"" -e'108a\ -- cgit v1.2.3