holocron/scripts/lib/batch-run.sh

#!/usr/bin/env bash
# Shared bounded-batch concurrent job runner. Sourced by
# scripts/sync-plugin-content.sh, tests/run-tests.sh, and tests/run-bats.sh so
# their concurrency-cap and per-item log/status handling can't silently
# diverge -- previously the same batching logic (core-count cap, per-item
# log/status files, batched `wait`) was hand-implemented independently in all
# three places.
#
# Batches (not a rolling pool) because a bounded rolling pool needs `wait -n`,
# which is bash 4.3+ -- all three callers are explicitly bash-3.2-safe.
# `getconf` over `nproc` for the same reason: `nproc` doesn't exist on macOS.
#
# Not meant to be executed directly -- source it.

# batch_jobs_limit
#   Prints the concurrency cap to use for batching.
batch_jobs_limit() {
  getconf _NPROCESSORS_ONLN 2>/dev/null || echo 4
}

# batch_run <scratch_dir> <key1> <cmd1> [<key2> <cmd2> ...]
#
# For each key/cmd pair, backgrounds `eval "$cmd"` with its combined
# stdout+stderr redirected to "<scratch_dir>/<key>.log", bounded to at most
# batch_jobs_limit concurrent jobs (waiting out the current batch before
# starting the next).
#
# Each <cmd> owns writing its own result to "<scratch_dir>/<key>.status" --
# this helper only owns dispatch/throttling and log capture, not status
# semantics. Callers differ on how they do that (capturing $? of an external
# command with `|| rc=$?`, or a sync function writing its own status flag
# directly) -- both patterns are preserved as-is by callers, not standardized
# here, so existing error-handling behavior (including how each pattern
# interacts with `set -e` in the caller) is unchanged by this extraction.
#
# batch_run waits ONLY on the PIDs it started, never with a bare `wait`. A bare
# `wait` blocks on every background job of the calling shell, so a caller that
# backgrounds anything of its own would (a) have batch_run block until that
# unrelated job finished and (b) have that job reaped here, with its exit status
# consumed by the wrong `wait` -- leaving the caller's later `wait $pid` to fail
# with "not a child of this shell". None of the three current callers backgrounds
# anything else, so this was latent rather than live, but it was an undocumented
# constraint on every future caller. Recording each `$!` and waiting on it by PID
# removes the constraint instead of documenting it.

# batch_wait_pids [<pid> ...]
#   Reaps exactly the given PIDs and always returns 0.
#
#   The `|| true` is load-bearing, not defensive noise: unlike a bare `wait`
#   (which is unconditionally 0), `wait <pid>` returns that job's exit status, so
#   without it a single failing job would make batch_run return nonzero and abort
#   its `set -e` caller at the call site -- before the caller could read the
#   .status files and print its own summary. Status semantics stay entirely in
#   the .status files, exactly as before.
#
#   `${@+"$@"}` rather than a bare `"$@"`, for the same reason every `${arr[@]}`
#   in this repo carries the `${arr[@]+...}` guard. Bash 4.4 is what relaxed
#   `set -u` for an all-empty `@`/`*` expansion (CHANGES, 4.4 "New Features in
#   Bash" 3a); 3.2 predates that relaxation, and no bash on a modern machine can
#   reproduce the abort, so the guarded spelling is asserted rather than tested.
#   Zero args is a normal path here, not an edge case: the trailing call receives
#   an empty list whenever the job count divides evenly into the concurrency cap.
batch_wait_pids() {
  local pid
  for pid in ${@+"$@"}; do
    wait "$pid" || true
  done
}

batch_run() {
  local scratch_dir="$1"
  shift
  local jobs_limit running key cmd pids
  jobs_limit="$(batch_jobs_limit)"
  running=0
  pids=()

  while [[ $# -gt 0 ]]; do
    key="$1" cmd="$2"
    shift 2
    (eval "$cmd") >"$scratch_dir/$key.log" 2>&1 &
    pids+=("$!")
    running=$((running + 1))
    if [[ $running -ge $jobs_limit ]]; then
      batch_wait_pids ${pids[@]+"${pids[@]}"}
      pids=()
      running=0
    fi
  done
  batch_wait_pids ${pids[@]+"${pids[@]}"}
}