holocron/scripts/sync-plugin-content.sh

#!/usr/bin/env bash
set -euo pipefail

# Mirrors each plugin's .apm/{agents,skills,prompts,commands,instructions,extensions,hooks}
# into flat plugin-root directories (agents/, skills/, commands/, instructions/,
# extensions/, hooks/hooks.json) -- Claude Code's and GitHub Copilot's plugin loaders
# convention-scan those flat paths at the plugin root; neither has any awareness of
# apm's .apm/ nesting (confirmed via `strings` on the installed claude binary and a
# live `claude --plugin-dir <bundle> -p ...` discoverability test -- see issue #90).
# `apm pack --format plugin` already implements the correct .apm/ -> plugin-convention
# mapping (it's built for distributable bundles under build/, a directory nothing in
# marketplace.json points at); this script reuses that mapping and copies the relevant
# subset back into the plugin root as compiled output -- same status as
# .claude-plugin/plugin.json, never hand-edited when .apm/ is present.
#
# plugin.json, apm.lock.yaml, and .mcp.json from the bundle are deliberately NOT
# copied: .claude-plugin/plugin.json + .github/plugin/plugin.json are already
# generated in-place at the plugin root by a separate apm code path
# (core/plugin_manifest.py, run as part of the same `apm pack` invocation, keyed off
# cwd rather than -o), and .mcp.json is hand-authored at the plugin root per ADR-0015
# (it is not an .apm/ primitive). Both real and check-mode syncs pass --force so that
# path actually refreshes both files from current apm.yml/.apm/ content -- apm pack
# silently skips regenerating an existing plugin.json otherwise ("already exists;
# skipping plugin.json generation"), which would let them go stale after a
# name/version/description edit (real mode) or let --check compare a copy against
# itself and never see the drift (check mode; see sync_plugin_manifest below).
#
# apm's Copilot-ecosystem plugin.json builder omits mcpServers entirely -- its own
# docstring calls it out-of-schema for Copilot, but this repo's researched Copilot
# plugin schema docs (plugins/kyberforge/docs/research/docs/github-copilot-plugins/
# configuration.md) document mcpServers as valid there. Both modes re-inject it into
# .github/plugin/plugin.json from the plugin's own .mcp.json after apm pack runs (see
# reinject_mcp_servers below) -- real mode into the plugin root directly, check mode
# into the throwaway copy first so the manifest diff below sees the same content a
# real sync would actually produce.
#
# apm pack also writes .claude-plugin/plugin.json and .github/plugin/plugin.json into
# cwd whenever those files don't already exist yet -- regardless of --force -- so
# --check (which must never mutate the real plugin root) never cds into plugin_dir
# directly. It packs a throwaway copy instead (see sync_one's pack_cwd), forces
# regeneration of both manifest files inside that copy, then diffs them
# (sync_plugin_manifest) against the real plugin root's committed manifests to catch
# drift in name/version/description/mcpServers -- only the copy's manifest files,
# never the real ones, can get created as a first-write.
#
# The merged hooks file is mirrored like the other MIRROR_DIRS content: synced when
# .apm/hooks/ produces one, and removed (real mode) / flagged as drift (--check) when
# it no longer does but a mirrored copy is still sitting there from a prior sync.
# It lands at hooks/hooks.json, not at the plugin root: Claude Code convention-scans
# `hooks/hooks.json` "at the plugin root, not inside .claude-plugin/"
# (plugins/kyberforge/docs/research/docs/claude-code-plugins/configuration.md's "Plugin
# Directory Layout" table; quoted verbatim in ADR-0017's own root-cause analysis), and
# the compiled plugin.json carries no `hooks` pointer to override that -- apm's
# build_plugin_manifest strips pointer fields unconditionally, and re-injecting one is
# the option ADR-0017 explicitly rejected. A root-level hooks.json (this script's own
# pre-fix output shape) is therefore scanned by nothing at all, and is deleted as stale
# by a real sync / reported as drift by --check.
#
# tests/ subdirectories (e.g. .apm/skills/<name>/tests/*.bats) are excluded from the
# mirror -- they are dev-time fixtures a plugin host never needs to discover, and several
# reference their own repo root via a hardcoded relative walk-up (e.g.
# `../../../../../../`) sized for the .apm/-nested depth. Mirroring them verbatim would
# duplicate each file one directory level shallower than that walk-up expects, breaking
# the duplicate and double-running the original under any repo-wide bats/test discovery.
# The exclusion is depth-scoped to <category>/<name>/tests, because a skill may legitimately
# ship a directory literally named tests as a template asset it scaffolds FROM
# (skills/skill-author/assets/templates/tests) -- stripping that breaks the shipped
# scaffolder, which sed's its way through the template tree file by file.
#
# --check does NOT reimplement any of the above. It runs the real sync functions against
# a throwaway copy of the plugin root and diffs the result against the live mirror with
# no exclusions, so the tests/ scoping and the hooks path exist in exactly one place and
# check mode is structurally incapable of disagreeing with what a real sync produces.

usage() {
  echo "Usage: $0 [--check] (--all | <plugin-dir> [<plugin-dir> ...])" >&2
  exit 1
}

CHECK=0
if [[ "${1:-}" == "--check" ]]; then
  CHECK=1
  shift
fi

ALL=0
if [[ "${1:-}" == "--all" ]]; then
  ALL=1
  shift
fi

if [[ "$ALL" -eq 1 ]]; then
  [[ $# -eq 0 ]] || usage
else
  [[ $# -ge 1 ]] || usage
fi

if ! command -v apm &>/dev/null; then
  echo "Error: apm is required but not installed (see kyberforge:apm-install)" >&2
  exit 1
fi

if ! command -v jq &>/dev/null; then
  echo "Error: jq is required but not installed" >&2
  exit 1
fi

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=lib/marketplace-plugins.sh
source "$SCRIPT_DIR/lib/marketplace-plugins.sh"
# shellcheck source=lib/batch-run.sh
source "$SCRIPT_DIR/lib/batch-run.sh"

# Convention subdirectories apm's plugin exporter can populate from .apm/.
MIRROR_DIRS=(agents skills commands instructions extensions)

# Where the merged hooks file lands (Claude Code's convention-scanned path), and
# the pre-fix root-level path a real sync now cleans up as stale.
HOOKS_REL="hooks/hooks.json"
LEGACY_HOOKS_REL="hooks.json"

FAIL=0
SCRATCH_ROOT="$(mktemp -d)"
trap 'rm -rf "$SCRATCH_ROOT"' EXIT

if [[ "$ALL" -eq 1 ]]; then
  # Derives the plugin list from marketplace.json via the shared
  # list_marketplace_local_plugins helper (scripts/lib/marketplace-plugins.sh),
  # the same one scripts/check-manifests.sh uses, instead of hand-maintaining a
  # duplicate walk at every call site (see .pre-commit-config.yaml's
  # check-plugin-content-sync).
  REPO_ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)"
  MARKETPLACE="$REPO_ROOT/.claude-plugin/marketplace.json"
  if [[ ! -f "$MARKETPLACE" ]]; then
    echo "Error: --all requires $MARKETPLACE" >&2
    exit 1
  fi
  declare -a plugin_dirs=()
  while IFS=$'\t' read -r _name plugin_dir; do
    plugin_dirs+=("$plugin_dir")
  done < <(list_marketplace_local_plugins "$REPO_ROOT" "$MARKETPLACE")
else
  declare -a plugin_dirs=("$@")
fi

# Fail fast on a basename collision rather than letting two plugin_dir arguments
# silently share (and corrupt) the same $name.log/$name.status/$name.checkcopy
# scratch paths below.
declare -a seen_names=()
for plugin_dir in ${plugin_dirs[@]+"${plugin_dirs[@]}"}; do
  name="$(basename "${plugin_dir%/}")"
  for seen in ${seen_names[@]+"${seen_names[@]}"}; do
    if [[ "$seen" == "$name" ]]; then
      echo "Error: duplicate plugin basename '$name' among arguments -- scratch paths would collide" >&2
      exit 1
    fi
  done
  seen_names+=("$name")
done

normalize_trailing_newline() {
  # apm's bundle exporter writes hooks.json without a trailing newline, which
  # end-of-file-fixer (pre-commit) would flag on every regeneration -- normalize
  # instead of fighting that hook on every sync.
  printf '%s\n' "$(cat "$1")" >"$2"
}

# Real-mode mirror write. There is no --check branch here on purpose: check mode
# calls this same function against a throwaway copy of the plugin root and diffs
# the result (see sync_one), so the tests/ exclusion below is the only copy of
# that rule anywhere in this script.
sync_dir() {
  local target_dir="$1" bundle_dir="$2" d="$3"
  local src="$bundle_dir/$d" dst="$target_dir/$d"

  if [[ -d "$src" ]]; then
    rm -rf "$dst"
    mkdir -p "$dst"
    cp -a "$src/." "$dst/"
    # Depth-scoped to <category>/<name>/tests (depth 2 relative to $dst), which is
    # where every dev-time fixture lives. A depth-agnostic `-name tests` also matched
    # template assets a skill ships for its own scaffolder to copy FROM -- e.g.
    # skills/skill-author/assets/templates/tests at depth 4, whose removal made the
    # mirrored new-skill.sh die on `sed: can't read .../tests/README.md` midway
    # through writing a scaffold.
    find "$dst" -mindepth 2 -maxdepth 2 -type d -name tests -exec rm -rf {} +
  elif [[ -d "$dst" ]]; then
    rm -rf "$dst"
  fi
}

# Real-mode merged-hooks write; same single-implementation contract as sync_dir.
sync_hooks_json() {
  local target_dir="$1" bundle_dir="$2"
  local src="$bundle_dir/hooks.json"
  local dst="$target_dir/$HOOKS_REL" legacy="$target_dir/$LEGACY_HOOKS_REL"

  # Earlier revisions of this script wrote the merged hooks file to the plugin
  # root. Nothing scans it there (see the header comment), so it is stale output
  # regardless of whether .apm/hooks/ still produces one -- clean it up first.
  if [[ -f "$legacy" ]]; then
    rm -f "$legacy"
  fi

  if [[ -f "$src" ]]; then
    mkdir -p "$target_dir/hooks"
    normalize_trailing_newline "$src" "$dst"
  elif [[ -f "$dst" ]]; then
    # .apm/hooks/ no longer produces a hooks.json, but one is still sitting at
    # $dst from a prior sync -- that's stale mirrored output, not "no .apm/hooks/
    # content" (which would mean $dst never existed in the first place).
    rm -f "$dst"
    rmdir "$target_dir/hooks" 2>/dev/null || true
  fi
}

# --- check mode: compare the real plugin root against a synced throwaway copy ---
#
# Every function below is pure comparison. None of them knows what the mirror
# rules are; $expected_dir is the output of the real sync functions above.

check_dir() {
  local plugin_dir="$1" expected_dir="$2" d="$3"
  local expected="$expected_dir/$d" actual="$plugin_dir/$d"

  if [[ -d "$expected" ]]; then
    if [[ ! -d "$actual" ]]; then
      echo "DRIFT $actual: missing (would be created from .apm/)" >&2
      FAIL=1
    elif ! diff -rq "$expected" "$actual" >/dev/null 2>&1; then
      echo "DRIFT $actual: out of sync with .apm/" >&2
      # `|| true`: pipefail (set -o at the top) turns diff's exit 1 into a failed
      # pipeline, and set -e would abort sync_one right here -- reporting only the
      # first drifted directory per plugin and turning one push into N fix cycles.
      diff -rq "$expected" "$actual" 2>&1 | sed 's/^/    /' >&2 || true
      FAIL=1
    fi
  elif [[ -d "$actual" ]]; then
    echo "DRIFT $actual: stale, no longer produced from .apm/" >&2
    FAIL=1
  fi
}

check_file() {
  local plugin_dir="$1" expected_dir="$2" rel="$3" desc="$4"
  local expected="$expected_dir/$rel" actual="$plugin_dir/$rel"

  if [[ -f "$expected" ]]; then
    if [[ ! -f "$actual" ]]; then
      echo "DRIFT $actual: missing (would be created from $desc)" >&2
      FAIL=1
    elif ! diff -q "$expected" "$actual" >/dev/null 2>&1; then
      echo "DRIFT $actual: out of sync with $desc" >&2
      FAIL=1
    fi
  elif [[ -f "$actual" ]]; then
    echo "DRIFT $actual: stale, no longer produced from $desc" >&2
    FAIL=1
  fi
}

# Prints "<kind> <relative-path>" for every entry under the given relative paths.
# `find` is used rather than a stat(1) call because stat's flags for mode
# formatting are incompatible between GNU and BSD/macOS.
path_manifest() {
  local root="$1"
  shift
  local rel f kind
  for rel in "$@"; do
    if [[ ! -e "$root/$rel" ]] && [[ ! -L "$root/$rel" ]]; then
      continue
    fi
    find "$root/$rel" -print 2>/dev/null | LC_ALL=C sort | while IFS= read -r f; do
      if [[ -L "$f" ]]; then
        kind="symlink"
      elif [[ -d "$f" ]]; then
        kind="dir"
      elif [[ -x "$f" ]]; then
        kind="exec"
      else
        kind="file"
      fi
      printf '%s %s\n' "$kind" "${f#"$root"/}"
    done || true
  done
}

# `diff -r` compares content only: it dereferences symlinks and ignores file modes
# entirely. So `chmod -x` on a mirrored script, or swapping a mirrored file for a
# symlink to identical content, both leave --check at exit 0 while a real sync
# silently repairs them -- check and sync disagreeing, which is the one thing this
# gate exists to prevent. Compare an explicit type+exec-bit manifest as well.
check_path_modes() {
  local plugin_dir="$1" expected_dir="$2"
  shift 2
  local expected actual
  expected="$(mktemp)"
  actual="$(mktemp)"
  path_manifest "$expected_dir" "$@" >"$expected"
  path_manifest "$plugin_dir" "$@" >"$actual"
  if ! diff -q "$expected" "$actual" >/dev/null 2>&1; then
    echo "DRIFT $plugin_dir: mirrored file types/modes differ from a fresh sync" >&2
    diff "$expected" "$actual" 2>&1 | sed 's/^/    /' >&2 || true
    FAIL=1
  fi
  rm -f "$expected" "$actual"
}

reinject_mcp_servers() {
  local plugin_dir="$1" target_dir="$2"
  local mcp_src="$plugin_dir/.mcp.json" dst="$target_dir/.github/plugin/plugin.json"
  [[ -f "$mcp_src" ]] || return 0
  [[ -f "$dst" ]] || return 0

  # Match apm's own Claude-ecosystem plugin.json builder: mcpServers is omitted
  # entirely when the plugin declares none, not written out as an empty object.
  local count
  count="$(jq '(.mcpServers // {}) | length' "$mcp_src")"
  [[ "$count" -gt 0 ]] || return 0

  local tmp
  tmp="$(mktemp)"
  jq --slurpfile mcp "$mcp_src" '.mcpServers = $mcp[0].mcpServers' "$dst" >"$tmp"
  mv "$tmp" "$dst"
}

# --check-only: diffs a freshly-regenerated manifest file (in the throwaway
# pack_cwd copy, produced by a --force'd apm pack) against the one actually
# committed at the real plugin root. Real-mode syncs never need this -- there
# pack_cwd IS plugin_dir, so --force already refreshed the real file in place.
sync_plugin_manifest() {
  local plugin_dir="$1" pack_cwd="$2" rel="$3"
  local src="$pack_cwd/$rel" dst="$plugin_dir/$rel"

  if [[ -f "$src" ]]; then
    if [[ ! -f "$dst" ]]; then
      echo "DRIFT $dst: missing (would be created by apm pack from apm.yml/.mcp.json)" >&2
      FAIL=1
    elif ! diff -q "$src" "$dst" >/dev/null 2>&1; then
      echo "DRIFT $dst: out of sync with apm.yml/.mcp.json" >&2
      # `|| true` for the same pipefail/set -e reason as check_dir's diff above.
      diff "$src" "$dst" 2>&1 | sed 's/^/    /' >&2 || true
      FAIL=1
    fi
  elif [[ -f "$dst" ]]; then
    echo "DRIFT $dst: stale, no longer produced by apm pack" >&2
    FAIL=1
  fi
}

# Runs entirely inside a backgrounded subshell (see the dispatch loop below), so
# FAIL here is that subshell's own copy -- it never touches the parent's FAIL
# and must be handed back via status_file instead.
sync_one() {
  local plugin_dir="${1%/}" status_file="$2"
  local apm_dir="$plugin_dir/.apm"
  FAIL=0

  if [[ ! -d "$plugin_dir" ]]; then
    echo "FAIL $plugin_dir: plugin directory does not exist" >&2
    FAIL=1
    echo "$FAIL" >"$status_file"
    return 0
  fi

  if [[ ! -d "$apm_dir" ]]; then
    echo "SKIP $plugin_dir: no .apm/ directory" >&2
    echo "$FAIL" >"$status_file"
    return 0
  fi

  local name scratch bundle_dir pack_log pack_cwd
  name="$(basename "$plugin_dir")"
  scratch="$SCRATCH_ROOT/$name"
  mkdir -p "$scratch"
  pack_log="$(mktemp)"

  # --force always: in real mode it refreshes the real plugin.json in place
  # (pack_cwd IS plugin_dir there); in check mode it forces regeneration inside
  # the throwaway pack_cwd copy so sync_plugin_manifest below has a genuinely
  # fresh manifest to diff against the real one -- without --force, apm pack
  # would silently skip regenerating a plugin.json that already exists in the
  # copy (it was seeded from the real plugin_dir), so --check would always
  # compare the copy against itself and never see drift.
  local force_flag=(--force)
  if [[ "$CHECK" -eq 0 ]]; then
    pack_cwd="$plugin_dir"
  else
    pack_cwd="$SCRATCH_ROOT/$name.checkcopy"
    mkdir -p "$pack_cwd"
    cp -a "$plugin_dir/." "$pack_cwd/"
  fi

  if ! (cd "$pack_cwd" && apm pack --format plugin "${force_flag[@]+"${force_flag[@]}"}" -o "$scratch") >"$pack_log" 2>&1; then
    echo "FAIL $plugin_dir: apm pack failed:" >&2
    sed 's/^/    /' "$pack_log" >&2
    rm -f "$pack_log"
    FAIL=1
    echo "$FAIL" >"$status_file"
    return 0
  fi
  rm -f "$pack_log"

  bundle_dir="$(find "$scratch" -mindepth 1 -maxdepth 1 -type d | head -1)"
  if [[ -z "$bundle_dir" ]]; then
    echo "FAIL $plugin_dir: apm pack produced no bundle directory under $scratch" >&2
    FAIL=1
    echo "$FAIL" >"$status_file"
    return 0
  fi

  # Both modes run the identical real-mode mirror write; only the target differs.
  # In real mode that target IS the plugin root. In check mode it's the throwaway
  # pack_cwd copy, which then gets diffed against the plugin root below -- so
  # --check can only ever report what a real sync would actually change.
  local d
  for d in "${MIRROR_DIRS[@]}"; do
    sync_dir "$pack_cwd" "$bundle_dir" "$d"
  done
  sync_hooks_json "$pack_cwd" "$bundle_dir"
  if [[ "$CHECK" -eq 0 ]]; then
    reinject_mcp_servers "$plugin_dir" "$plugin_dir"
  else
    # Reinject into the throwaway copy too, so the manifest diff below compares
    # against what a real sync would actually produce (mcpServers included),
    # not apm's own Copilot-ecosystem output (which omits it).
    reinject_mcp_servers "$plugin_dir" "$pack_cwd"
    local -a checked_paths
    checked_paths=("${MIRROR_DIRS[@]}" "$HOOKS_REL" "$LEGACY_HOOKS_REL")
    for d in "${MIRROR_DIRS[@]}"; do
      check_dir "$plugin_dir" "$pack_cwd" "$d"
    done
    check_file "$plugin_dir" "$pack_cwd" "$HOOKS_REL" ".apm/hooks/"
    check_file "$plugin_dir" "$pack_cwd" "$LEGACY_HOOKS_REL" \
      ".apm/hooks/ (Claude Code convention-scans $HOOKS_REL, not the plugin root)"
    check_path_modes "$plugin_dir" "$pack_cwd" "${checked_paths[@]}"
    sync_plugin_manifest "$plugin_dir" "$pack_cwd" ".claude-plugin/plugin.json"
    sync_plugin_manifest "$plugin_dir" "$pack_cwd" ".github/plugin/plugin.json"
  fi
  echo "$FAIL" >"$status_file"
}

# Each plugin's `apm pack` is an independent CLI invocation dominated by fixed
# process-startup cost, not by per-plugin work -- run them concurrently rather
# than paying that startup cost N times serially. Output is buffered per plugin
# (not streamed) so concurrent DRIFT/FAIL messages from different plugins never
# interleave; it's flushed in stable $@ order once every job has finished.
# Dispatch/throttling itself is scripts/lib/batch-run.sh's batch_run (shared
# with tests/run-tests.sh and tests/run-bats.sh) -- see that file for why this
# is batched rather than a rolling `wait -n` pool.
declare -a batch_args=()
for plugin_dir in ${plugin_dirs[@]+"${plugin_dirs[@]}"}; do
  name="$(basename "${plugin_dir%/}")"
  cmd="$(printf 'sync_one %q %q' "$plugin_dir" "$SCRATCH_ROOT/$name.status")"
  batch_args+=("$name" "$cmd")
done
batch_run "$SCRATCH_ROOT" ${batch_args[@]+"${batch_args[@]}"}

for plugin_dir in ${plugin_dirs[@]+"${plugin_dirs[@]}"}; do
  name="$(basename "${plugin_dir%/}")"
  cat "$SCRATCH_ROOT/$name.log" >&2
  status="$(cat "$SCRATCH_ROOT/$name.status" 2>/dev/null || echo 1)"
  [[ "$status" -ne 0 ]] && FAIL=1
done

if [[ "$FAIL" -ne 0 ]]; then
  if [[ "$CHECK" -eq 1 ]]; then
    if [[ "$ALL" -eq 1 ]]; then
      echo "Plugin content mirror is out of sync with .apm/. Fix: bash scripts/sync-plugin-content.sh --all" >&2
    else
      echo "Plugin content mirror is out of sync with .apm/. Fix: bash scripts/sync-plugin-content.sh $*" >&2
    fi
  fi
  exit 1
fi