#!/usr/bin/env bash # Shared bounded-batch concurrent job runner. Sourced by # scripts/sync-plugin-content.sh, tests/run-tests.sh, and tests/run-bats.sh so # their concurrency-cap and per-item log/status handling can't silently # diverge -- previously the same batching logic (core-count cap, per-item # log/status files, batched `wait`) was hand-implemented independently in all # three places. # # Batches (not a rolling pool) because a bounded rolling pool needs `wait -n`, # which is bash 4.3+ -- all three callers are explicitly bash-3.2-safe. # `getconf` over `nproc` for the same reason: `nproc` doesn't exist on macOS. # # Not meant to be executed directly -- source it. # batch_jobs_limit # Prints the concurrency cap to use for batching. batch_jobs_limit() { getconf _NPROCESSORS_ONLN 2>/dev/null || echo 4 } # batch_run [ ...] # # For each key/cmd pair, backgrounds `eval "$cmd"` with its combined # stdout+stderr redirected to "/.log", bounded to at most # batch_jobs_limit concurrent jobs (waiting out the current batch before # starting the next). # # Each owns writing its own result to "/.status" -- # this helper only owns dispatch/throttling and log capture, not status # semantics. Callers differ on how they do that (capturing $? of an external # command with `|| rc=$?`, or a sync function writing its own status flag # directly) -- both patterns are preserved as-is by callers, not standardized # here, so existing error-handling behavior (including how each pattern # interacts with `set -e` in the caller) is unchanged by this extraction. # # batch_run waits ONLY on the PIDs it started, never with a bare `wait`. A bare # `wait` blocks on every background job of the calling shell, so a caller that # backgrounds anything of its own would (a) have batch_run block until that # unrelated job finished and (b) have that job reaped here, with its exit status # consumed by the wrong `wait` -- leaving the caller's later `wait $pid` to fail # with "not a child of this shell". None of the three current callers backgrounds # anything else, so this was latent rather than live, but it was an undocumented # constraint on every future caller. Recording each `$!` and waiting on it by PID # removes the constraint instead of documenting it. # batch_wait_pids [ ...] # Reaps exactly the given PIDs and always returns 0. # # The `|| true` is load-bearing, not defensive noise: unlike a bare `wait` # (which is unconditionally 0), `wait ` returns that job's exit status, so # without it a single failing job would make batch_run return nonzero and abort # its `set -e` caller at the call site -- before the caller could read the # .status files and print its own summary. Status semantics stay entirely in # the .status files, exactly as before. # # `${@+"$@"}` rather than a bare `"$@"`, for the same reason every `${arr[@]}` # in this repo carries the `${arr[@]+...}` guard. Bash 4.4 is what relaxed # `set -u` for an all-empty `@`/`*` expansion (CHANGES, 4.4 "New Features in # Bash" 3a); 3.2 predates that relaxation, and no bash on a modern machine can # reproduce the abort, so the guarded spelling is asserted rather than tested. # Zero args is a normal path here, not an edge case: the trailing call receives # an empty list whenever the job count divides evenly into the concurrency cap. batch_wait_pids() { local pid for pid in ${@+"$@"}; do wait "$pid" || true done } batch_run() { local scratch_dir="$1" shift local jobs_limit running key cmd pids jobs_limit="$(batch_jobs_limit)" running=0 pids=() while [[ $# -gt 0 ]]; do key="$1" cmd="$2" shift 2 (eval "$cmd") >"$scratch_dir/$key.log" 2>&1 & pids+=("$!") running=$((running + 1)) if [[ $running -ge $jobs_limit ]]; then batch_wait_pids ${pids[@]+"${pids[@]}"} pids=() running=0 fi done batch_wait_pids ${pids[@]+"${pids[@]}"} }