From c6d6e08feab839a9dc5792071fb803494cc51a69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucio=20Andr=C3=A9s=20Illanes=20Albornoz?= Date: Wed, 26 Feb 2020 21:54:12 +0000 Subject: General cleanup, pt. II. --- subr/ex_pkg_dispatch.subr | 294 +++++++++++++++++++++++++--------------------- 1 file changed, 162 insertions(+), 132 deletions(-) (limited to 'subr/ex_pkg_dispatch.subr') diff --git a/subr/ex_pkg_dispatch.subr b/subr/ex_pkg_dispatch.subr index 2d34a900..bd152f28 100644 --- a/subr/ex_pkg_dispatch.subr +++ b/subr/ex_pkg_dispatch.subr @@ -3,49 +3,106 @@ # # -# exp_pkg_check_depends() - check single named package for unsatisfied dependencies -# @_pkg_complete: list of completed packages -# @_pkg_name: single package name -# @_pkg_wait: list of in-progress packages -# @_restart_recursive: optional flag specifiying either no dependency expansion (0,) dependency expansion (1,) dependency expansion and forcibly rebuild (2.) -# -# Return: zero (0) given no outstanding dependencies, non-zero (>0) otherwise -# -exp_pkg_check_depends() { - local _pkg_complete="${1}" _pkg_name="${2}" _pkg_wait="${3}" _restart_recursive="${4}" \ - _pkg_depends="" _pkg_name_depend="" _dependfl=0; - if _pkg_depends="$(ex_rtl_lunfold_depends 'PKG_${_name}_DEPENDS' $(ex_rtl_get_var_unsafe -u "PKG_"${_pkg_name}"_DEPENDS"))"\ - && [ -n "${_pkg_depends}" ]\ - && ! [ -n "${_restart}" ] || [ "${_restart_recursive:-0}" -ge 1 ]; then - for _pkg_name_depend in $(ex_rtl_uniq ${_pkg_depends}); do - if ! ex_rtl_lmatch "${_pkg_complete}" "${_pkg_name_depend}"\ - || ex_rtl_lmatch "${_pkg_wait}" "${_pkg_name_depend}"; then - _dependfl=1; break; +# exp_pkg_dispatch_complete() - XXX +# @_dispatch_fn: top-level dispatch function name +# @_group_names: build group name(s) +# @_pkg_disabled: list of disabled packages +# @_pkg_finished: list of finished packages +# +# Return: zero (0) on success, non-zero (>0) on failure. +# +exp_pkg_dispatch_complete() { + local _dispatch_fn="${1}" _group_name="${2}" _pkg_disabled="${3}" _pkg_finished="${4}" _pkg_name=""; + for _pkg_name in ${_pkg_disabled}; do + "${_dispatch_fn}" disabled_pkg "${_group_name}" "${_pkg_name}"; + done; + for _pkg_name in ${_pkg_finished}; do + "${_dispatch_fn}" skipped_pkg "${_group_name}" "${_pkg_name}"; + done; +}; + +# +# exp_pkg_dispatch_group() - dispatch a single build group +# @_build_steps_default: list of default build steps +# @_build_vars_default: list of default build variables +# @_dispatch_fn: top-level dispatch function name +# @_group_name: build group name +# @_njobs_max: maximum count of simultaneous jobs +# @_pipe_path: pathname to build FIFO +# @_restart_at: optional comma-separated list of build steps at which to rebuild or ALL +# @_restart_recursive: optional flag specifiying either no dependency expansion (0,) dependency expansion (1,) dependency expansion and forcibly rebuild (2.) +# @_workdir: pathname to build-specific temporary directory +# +# Return: zero (0) on success, non-zero (>0) on failure. +# +exp_pkg_dispatch_group() { + local _build_steps_default="${1}" _build_vars_default="${2}" _dispatch_fn="${3}" \ + _group_name="${4}" _njobs_max="${5}" _pipe_path="${6}" _restart_at="${7}" \ + _restart_recursive="${8}" _workdir="${9}" _pipe_msg="" _pkg_name="" _rc=0; + rtl_fileop mkfifo "${_pipe_path}"; + while true; do + while [ "${EXP_PKG_DISPATCH_NJOBS:-0}" -gt 0 ] && read _pipe_msg; do + case "${_pipe_msg%% *}" in + done) : $((EXP_PKG_DISPATCH_NJOBS-=1)); _pkg_name="${_pipe_msg#done * }"; + "${_dispatch_fn}" finish_pkg ${_pipe_msg#done }; + EXP_PKG_COMPLETE="$(rtl_lconcat "${EXP_PKG_COMPLETE}" "${_pkg_name}")"; + EXP_PKG_NAMES="$(rtl_lfilter "${EXP_PKG_NAMES}" "${_pkg_name}")"; + EXP_PKG_DISPATCH_WAIT="$(rtl_lfilter "${EXP_PKG_DISPATCH_WAIT}" "${_pkg_name}")"; + if [ -n "${EXP_PKG_NAMES}" ] && [ "${_rc}" -eq 0 ]; then + if [ "${EXP_PKG_DISPATCH_NJOBS}" -ne "${_njobs_max}" ]; then + exp_pkg_dispatch_packages "${_build_steps_default}" \ + "${_build_vars_default}" "${_dispatch_fn}" \ + "${_group_name}" "${_njobs_max}" \ + "${_pipe_path}" "${EXP_PKG_COMPLETE}" \ + "${_restart_at}" "${_restart_recursive}" "${_workdir}"; + fi; + elif [ "${EXP_PKG_DISPATCH_NJOBS:-0}" -eq 0 ]; then + break; + fi; ;; + fail) : $((EXP_PKG_DISPATCH_NJOBS-=1)); _rc=1; + "${_dispatch_fn}" fail_pkg ${_pipe_msg#fail }; ;; + step) "${_dispatch_fn}" step_pkg ${_pipe_msg#step }; ;; + esac; done <>"${_pipe_path}"; + if [ -n "${EXP_PKG_NAMES}" ] && [ "${_rc}" -eq 0 ]; then + if [ "${EXP_PKG_DISPATCH_NJOBS}" -ne "${_njobs_max}" ]; then + exp_pkg_dispatch_packages "${_build_steps_default}" \ + "${_build_vars_default}" "${_dispatch_fn}" \ + "${_group_name}" "${_njobs_max}" "${_pipe_path}" \ + "${EXP_PKG_COMPLETE}" "${_restart_at}" \ + "${_restart_recursive}" "${_workdir}"; fi; - done; - fi; - return "${_dependfl}"; + elif [ "${EXP_PKG_DISPATCH_NJOBS:-0}" -eq 0 ]; then + break; + fi; + done; + rtl_fileop rm "${_pipe_path}"; + return "${_rc}"; }; # # exp_pkg_dispatch_package() - dispatch single named packages -# @_dispatch_fn: top-level dispatch function name -# @_group_name: build group name -# @_pkg_name: single package name -# @_restart_at: optional comma-separated list of build steps at which to rebuild or ALL +# @_build_steps_default: list of default build steps +# @_build_vars_default: list of default build variables +# @_dispatch_fn: top-level dispatch function name +# @_group_name: build group name +# @_pkg_name: single package name +# @_restart_at: optional comma-separated list of build steps at which to rebuild or ALL +# @_workdir: pathname to build-specific temporary directory # -# Return: zero (0) on success, non-zero (>0) on failure, ${NJOBS}, ${PKG_COUNT}, ${PKG_NAMES}, and ${PKG_WAIT} may be mutated post-return. +# Return: zero (0) on success, non-zero (>0) on failure, ${EXP_PKG_DISPATCH_NJOBS}, ${EXP_PKG_DISPATCH_COUNT}, ${EXP_PKG_NAMES}, and ${EXP_PKG_DISPATCH_WAIT} may be mutated post-return. # exp_pkg_dispatch_package() { - local _dispatch_fn="${1}" _group_name="${2}" _pkg_name="${3}" _restart_at="${4}"; - if "${_dispatch_fn}" start_pkg "${_group_name}" "${_pkg_name}" "$((${PKG_COUNT}+1))" "${PKG_COUNT_MAX}"; then - : $((NJOBS+=1)); : $((PKG_COUNT+=1)); PKG_WAIT="$(ex_rtl_lconcat "${PKG_WAIT}" "${_pkg_name}")"; + local _build_steps_default="${1}" _build_vars_default="${2}" _dispatch_fn="${3}" \ + _group_name="${4}" _pkg_name="${5}" _restart_at="${6}" _workdir="${7}"; + if "${_dispatch_fn}" start_pkg "${_group_name}" "${_pkg_name}" "$((${EXP_PKG_DISPATCH_COUNT}+1))" "${EXP_PKG_DISPATCH_COUNT_MAX}"; then + : $((EXP_PKG_DISPATCH_NJOBS+=1)); : $((EXP_PKG_DISPATCH_COUNT+=1)); EXP_PKG_DISPATCH_WAIT="$(rtl_lconcat "${EXP_PKG_DISPATCH_WAIT}" "${_pkg_name}")"; (set +o errexit -o noglob; BUILD_IS_PARENT=0; - if ex_pkg_env "${_group_name}" "${_pkg_name}" "${_restart_at}"; then + if ex_pkg_env "${_build_steps_default}" "${_build_vars_default}" \ + "${_group_name}" "${_pkg_name}" "${_restart_at}" "${_workdir}"; then ex_pkg_exec "${_dispatch_fn}" "${_group_name}" "${_pkg_name}" "${_restart_at}"; else return "${?}"; - fi;) 1>"${BUILD_WORKDIR}/${_pkg_name}_stderrout.log" 2>&1 3>"${_pipe_path}" & + fi;) 1>"${_workdir}/${_pkg_name}_stderrout.log" 2>&1 3>"${_pipe_path}" & else return "${?}"; fi; @@ -53,123 +110,96 @@ exp_pkg_dispatch_package() { # # exp_pkg_dispatch_packages() - dispatch set of packages -# @_dispatch_fn: top-level dispatch function name -# @_group_name: build group name -# @_njobs_max: maximum count of simultaneous jobs -# @_pipe_path: pathname to parent-child process FIFO -# @_pkg_complete: list of completed packages -# @_restart_at: optional comma-separated list of build steps at which to rebuild or ALL -# @_restart_recursive: optional flag specifiying either no dependency expansion (0,) dependency expansion (1,) dependency expansion and forcibly rebuild (2.) -# -# Return: zero (0) on success, non-zero (>0) on failure, ${NJOBS}, ${PKG_COUNT}, ${PKG_NAMES}, and ${PKG_WAIT} may be mutated post-return. +# @_build_steps_default: list of default build steps +# @_build_vars_default: list of default build variables +# @_dispatch_fn: top-level dispatch function name +# @_group_name: build group name +# @_njobs_max: maximum count of simultaneous jobs +# @_pipe_path: pathname to parent-child process FIFO +# @_pkg_complete: list of completed packages +# @_restart_at: optional comma-separated list of build steps at which to rebuild or ALL +# @_restart_recursive: optional flag specifiying either no dependency expansion (0,) dependency expansion (1,) dependency expansion and forcibly rebuild (2.) +# @_workdir: pathname to build-specific temporary directory +# +# Return: zero (0) on success, non-zero (>0) on failure, ${EXP_PKG_DISPATCH_NJOBS}, ${EXP_PKG_DISPATCH_COUNT}, ${EXP_PKG_NAMES}, and ${EXP_PKG_DISPATCH_WAIT} may be mutated post-return. # exp_pkg_dispatch_packages() { - local _dispatch_fn="${1}" _group_name="${2}" _njobs_max="${3}" _pipe_path="${4}" \ - _pkg_complete="${5}" _restart_at="${6}" _restart_recursive="${7}" \ + local _build_steps_default="${1}" _build_vars_default="${2}" _dispatch_fn="${3}" \ + _group_name="${4}" _njobs_max="${5}" _pipe_path="${6}" _pkg_complete="${7}" \ + _restart_at="${8}" _restart_recursive="${9}" _workdir="${10}" \ _njob=0 _pkg_depends="" _pkg_name=""; - for _njob in $(seq 1 $((${_njobs_max}-${NJOBS}))); do - for _pkg_name in ${PKG_NAMES}; do - if ! ex_rtl_lmatch "${_pkg_complete}" "${_pkg_name}"\ - && ! ex_rtl_lmatch "${PKG_WAIT}" "${_pkg_name}"\ - && exp_pkg_check_depends "${_pkg_complete}" "${_pkg_name}" "${PKG_WAIT}" "${_restart_recursive}"; then - exp_pkg_dispatch_package "${_dispatch_fn}" "${_group_name}" "${_pkg_name}" "${_restart_at}"; break; + for _njob in $(seq 1 $((${_njobs_max}-${EXP_PKG_DISPATCH_NJOBS}))); do + for _pkg_name in ${EXP_PKG_NAMES}; do + if ! rtl_lmatch "${_pkg_complete}" "${_pkg_name}"\ + && ! rtl_lmatch "${EXP_PKG_DISPATCH_WAIT}" "${_pkg_name}"\ + && ex_pkg_check_depends "${_pkg_complete}" "${_pkg_name}" "${EXP_PKG_DISPATCH_WAIT}" "${_restart_recursive}"; then + exp_pkg_dispatch_package "${_build_steps_default}" \ + "${_build_vars_default}" "${_dispatch_fn}" \ + "${_group_name}" "${_pkg_name}" "${_restart_at}" \ + "${_workdir}"; break; fi; done; done; }; # -# exp_pkg_expand_packages() - expand build group name to list of packages ordered and filtered according to dependency and restart constraints -# @_group_name: build group name -# @_restart: optional comma-separated list of package names to rebuild -# @_restart_recursive: optional flag specifiying either no dependency expansion (0,) dependency expansion (1,) dependency expansion and forcibly rebuild (2.) +# ex_pkg_dispatch() - dispatch a set of build group +# @_build_steps_default: list of default build steps +# @_build_vars_default: list of default build variables +# @_dispatch_fn: top-level dispatch function name +# @_group_names: build group name(s) +# @_njobs_max: maximum count of simultaneous jobs +# @_pipe_path: pathname to build FIFO +# @_restart: optional comma-separated list of package names to rebuild +# @_restart_at: optional comma-separated list of build steps at which to rebuild or ALL +# @_restart_recursive: optional flag specifiying either no dependency expansion (0,) dependency expansion (1,) dependency expansion and forcibly rebuild (2.) +# @_workdir: pathname to build-specific temporary directory # -# Return: zero (0) on success, non-zero (>0) on failure, ${EX_PKG_NAMES} and ${PKG_COMPLETE} set post-return. +# Return: zero (0) on success, non-zero (>0) on failure, ${EX_PKG_DISPATCH_UNKNOWN} mutated post-return. # -exp_pkg_expand_packages() { - local _group_name="${1}" _restart="${2}" _restart_recursive="${3}" _pkg_name="" _pkg_names=""; - if _pkg_names="$(ex_rtl_get_var_unsafe -u "${_group_name}_PACKAGES")"\ - && [ -n "${_pkg_names}" ]; then - if [ -n "${_restart}" ] && ! ex_rtl_lmatch "${_restart}" "ALL LAST"; then - _pkg_names="$(ex_rtl_lsearch "${_pkg_names}" "${_restart}")"; - fi; - if ! [ -n "${_restart}" ] || [ "${_restart_recursive:-0}" -ge 1 ]; then - _pkg_names="$(ex_rtl_uniq $(ex_rtl_lunfold_depends 'PKG_${_name}_DEPENDS' ${_pkg_names}))"; +ex_pkg_dispatch() { + local _build_steps_default="${1}" _build_vars_default="${2}" _dispatch_fn="${3}" \ + _group_names="${4}" _njobs_max="${5}" _pipe_path="${6}" _restart="${7}" \ + _restart_at="${8}" _restart_recursive="${9}" _workdir="${10}" \ + _pkg_name="" _pkg_names="" _rc=0 \ + EXP_PKG_COMPLETE EXP_PKG_DISABLED EXP_PKG_FINISHED EXP_PKG_DISPATCH_COUNT \ + EXP_PKG_DISPATCH_COUNT_MAX EXP_PKG_DISPATCH_NJOBS EXP_PKG_NAMES EXP_PKG_DISPATCH_WAIT; + EX_PKG_DISPATCH_UNKNOWN=""; + for _group_name in ${_group_names}; do + EXP_PKG_COMPLETE="" EXP_PKG_DISABLED="" EXP_PKG_FINISHED=""; + EXP_PKG_DISPATCH_COUNT=0 EXP_PKG_DISPATCH_COUNT_MAX=0 EXP_PKG_DISPATCH_NJOBS=0; + EXP_PKG_NAMES="" EXP_PKG_DISPATCH_WAIT=""; + if "${_dispatch_fn}" start_group "${_group_name}" ""; then + if rtl_fileop mkdir "${_workdir}"\ + && rtl_log_msg vnfo "Resolving \`${_group_name}' dependencies..."\ + && ex_pkg_expand_packages "${_group_name}" "${_restart}" "${_restart_recursive}"\ + && exp_pkg_dispatch_complete "${_dispatch_fn}" "${_group_name}" "${EXP_PKG_DISABLED}" "${EXP_PKG_FINISHED}"\ + && rtl_log_msg vnfo "Resolved \`${_group_name}' dependencies."\ + && EXP_PKG_DISPATCH_COUNT_MAX="$(rtl_llength "${EXP_PKG_NAMES}")"\ + && [ "${EXP_PKG_DISPATCH_COUNT_MAX}" -gt 0 ]; then + _pkg_names="$(rtl_lconcat "${_pkg_names}" "${EXP_PKG_NAMES}")"; + exp_pkg_dispatch_group "${_build_steps_default}" \ + "${_build_vars_default}" "${_dispatch_fn}" "${_group_name}" \ + "${_njobs_max}" "${_pipe_path}" "${_restart_at}" \ + "${_restart_recursive}" "${_workdir}"; _rc="${?}"; + fi; + "${_dispatch_fn}" finish_group "${_group_name}" ""; + if [ "${_rc}" -ne 0 ]; then + break; + fi; fi; - for _pkg_name in ${_pkg_names}; do - if [ -n "$(ex_rtl_get_var_unsafe -u "PKG_${_pkg_name}_DISABLED")" ]; then - PKG_COMPLETE="$(ex_rtl_lconcat "${PKG_COMPLETE}" "${_pkg_name}")"; - _pkg_names="$(ex_rtl_lfilter "${_pkg_names}" "${_pkg_name}")"; - "${_dispatch_fn}" disabled_pkg "${_group_name}" "${_pkg_name}"; - elif ex_pkg_state_test "${_pkg_name}" finish\ - && ! ex_rtl_lmatch "${_restart}" "${_pkg_name}"\ - && [ "${_restart_recursive:-0}" -ne 2 ]; then - PKG_COMPLETE="$(ex_rtl_lconcat "${PKG_COMPLETE}" "${_pkg_name}")"; - _pkg_names="$(ex_rtl_lfilter "${_pkg_names}" "${_pkg_name}")"; - "${_dispatch_fn}" skipped_pkg "${_group_name}" "${_pkg_name}"; + done; + if ! rtl_lmatch "${_restart}" "ALL LAST"; then + for _pkg_name in ${_restart}; do + if ! rtl_lmatch "${_pkg_names}" "${_restart}"; then + EX_PKG_DISPATCH_UNKNOWN="$(rtl_lconcat "${EX_PKG_DISPATCH_UNKNOWN}" "${_pkg_name}")"; fi; done; - EX_PKG_NAMES="${_pkg_names}"; return 0; fi; -}; - -# -# ex_pkg_dispatch() - dispatch a single build group -# @_dispatch_fn: top-level dispatch function name -# @_group_name: build group name -# @_njobs_max: maximum count of simultaneous jobs -# @_restart: optional comma-separated list of package names to rebuild -# @_restart_at: optional comma-separated list of build steps at which to rebuild or ALL -# @_restart_recursive: optional flag specifiying either no dependency expansion (0,) dependency expansion (1,) dependency expansion and forcibly rebuild (2.) -# -# Return: zero (0) on success, non-zero (>0) on failure, ${EX_PKG_NAMES} set post-return. -# -ex_pkg_dispatch() { - local _dispatch_fn="${1}" _group_name="${2}" _njobs_max="${3}" _restart="${4}" _restart_at="${5}" \ - _restart_recursive="${6}" _pipe_msg="" _pipe_path="${BUILD_WORKDIR}/build.fifo" _pkg_name="" \ - _script_rc=0 NJOBS=0 PKG_COMPLETE="" PKG_COUNT=0 PKG_COUNT_MAX=0 PKG_NAMES="" PKG_WAIT=""; EX_PKG_NAMES=""; - if "${_dispatch_fn}" start_group "${_group_name}" ""; then - if ex_rtl_fileop mkdir "${BUILD_WORKDIR}"\ - && ex_rtl_log_msg vnfo "Resolving \`${_group_name}' dependencies..."\ - && exp_pkg_expand_packages "${_group_name}" "${_restart}" "${_restart_recursive}"\ - && ex_rtl_log_msg vnfo "Resolved \`${_group_name}' dependencies."\ - && PKG_COUNT_MAX="$(ex_rtl_llength "${EX_PKG_NAMES}")"\ - && [ "${PKG_COUNT_MAX}" -gt 0 ]; then - PKG_NAMES="${EX_PKG_NAMES}"; ex_rtl_fileop mkfifo "${_pipe_path}"; - while true; do - while [ "${NJOBS:-0}" -gt 0 ] && read _pipe_msg; do - case "${_pipe_msg%% *}" in - done) : $((NJOBS-=1)); _pkg_name="${_pipe_msg#done * }"; - "${_dispatch_fn}" finish_pkg ${_pipe_msg#done }; - PKG_COMPLETE="$(ex_rtl_lconcat "${PKG_COMPLETE}" "${_pkg_name}")"; - PKG_NAMES="$(ex_rtl_lfilter "${PKG_NAMES}" "${_pkg_name}")"; - PKG_WAIT="$(ex_rtl_lfilter "${PKG_WAIT}" "${_pkg_name}")"; - if [ -n "${PKG_NAMES}" ] && [ "${_script_rc}" -eq 0 ]; then - if [ "${NJOBS}" -ne "${_njobs_max}" ]; then - exp_pkg_dispatch_packages "${_dispatch_fn}" "${_group_name}" "${_njobs_max}" \ - "${_pipe_path}" "${PKG_COMPLETE}" "${_restart_at}" "${_restart_recursive}"; - fi; - elif [ "${NJOBS:-0}" -eq 0 ]; then - break; - fi; ;; - fail) : $((NJOBS-=1)); _script_rc=1; - "${_dispatch_fn}" fail_pkg ${_pipe_msg#fail }; ;; - step) "${_dispatch_fn}" step_pkg ${_pipe_msg#step }; ;; - esac; done <>"${_pipe_path}"; - if [ -n "${PKG_NAMES}" ] && [ "${_script_rc}" -eq 0 ]; then - if [ "${NJOBS}" -ne "${_njobs_max}" ]; then - exp_pkg_dispatch_packages "${_dispatch_fn}" "${_group_name}" "${_njobs_max}" \ - "${_pipe_path}" "${PKG_COMPLETE}" "${_restart_at}" "${_restart_recursive}"; - fi; - elif [ "${NJOBS:-0}" -eq 0 ]; then - break; - fi; - done; - ex_rtl_fileop rm "${_pipe_path}"; - fi; - "${_dispatch_fn}" finish_group "${_group_name}" ""; + if [ -n "${EX_PKG_DISPATCH_UNKNOWN}" ]; then + _rc=1; fi; - return "${_script_rc}"; + return "${_rc}"; }; # vim:filetype=sh textwidth=0 -- cgit v1.2.3