#!/usr/bin/env bash # Preflight the recorded team configuration, then launch tasks sequentially. # lib_launch.sh runs each inference phase and overlaps its evaluation with the next. set -u ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" cd "$ROOT" || { echo "FATAL: cannot cd $ROOT" >&2; exit 1; } harness_env=$(python3 "$ROOT/agent/harness_config.py" shell "$@") || exit $? eval "$harness_env" REQUESTED_AGENT_HARNESS="$AGENT_HARNESS" REQUESTED_HARNESS_VERSION="$HARNESS_VERSION" MODE=single-node TASKS="${TASKS:-jgm__pandoc.5caad90 gromacs__gromacs.665ea4c universal-ctags__ctags.243595e php__php-src.c891263 ffmpeg__ffmpeg.360a402}" HOURS="${HOURS:-6}" WORKERS="${WORKERS:-8}" source "$ROOT/tasks/programbench/scripts/lib_launch.sh" source "$ROOT/tasks/programbench/scripts/lib_images.sh" mkdir -p "$TRAJ_ROOT" "$ROOT/out" pv() { awk -F= -v k="$2" '$1==k{print substr($0, length(k) + 2); exit}' "$ROOT/.env.$1"; } n_tasks=$(wc -w <<<"$TASKS") log "preflight over $n_tasks tasks, run $SUFFIX" for task in $TASKS; do profile_file="$ROOT/.env.$(task_profile "$task")" [ -f "$profile_file" ] || { log "FATAL: $profile_file missing — generate profiles first"; exit 1; } python3 "$ROOT/agent/harness_config.py" --check-profile "$profile_file" || exit 1 done for task in $TASKS; do scrub_shadowed_env "$ROOT/.env.$(task_profile "$task")" done fail=0 # Check every selected profile before starting task stacks; image probes use containers. refuse_double_launch || fail=1 for task in $TASKS; do p="$(task_profile "$task")" [ -f "$ROOT/.env.$p" ] || { log "FATAL: .env.$p missing — run tasks/programbench/scripts/gen_env.sh"; fail=1; continue; } o="$(pv "$p" AGENSH_ORCHESTRATOR)" [ "${o:-compose}" = "compose" ] || { log "FATAL: .env.$p supports single-node Compose only"; fail=1; } [ "${AGENSH_ORCHESTRATOR:-compose}" = "compose" ] || { log "FATAL: AGENSH_ORCHESTRATOR supports single-node Compose only"; fail=1; } k="$(pv "$p" AGENT_HARNESS)" case "$k" in pi) prefix=PI ;; copilot) prefix=COPILOT ;; codex) prefix=CODEX ;; claude) prefix=CLAUDE ;; *) log "FATAL: .env.$p AGENT_HARNESS=$k must be pi, copilot, codex or claude"; fail=1; continue ;; esac if [ "$k" != "$REQUESTED_AGENT_HARNESS" ]; then log "FATAL: .env.$p selects $k, but launch requested $REQUESTED_AGENT_HARNESS; regenerate profiles with --agent-harness $REQUESTED_AGENT_HARNESS" fail=1 continue fi v="$(pv "$p" HARNESS_VERSION)" if [ "$v" != "$REQUESTED_HARNESS_VERSION" ]; then log "FATAL: .env.$p selects $k $v, but launch requested $REQUESTED_HARNESS_VERSION; regenerate profiles with --harness-version $REQUESTED_HARNESS_VERSION" fail=1 continue fi img="$(pv "$p" AGENT_IMAGE)" [ -n "$img" ] || { log "FATAL: .env.$p AGENT_IMAGE missing"; fail=1; continue; } sudo docker image inspect "$img" >/dev/null 2>&1 || { log "FATAL: image missing: $img"; fail=1; } runtime_env=() for key in AGENT_HARNESS HARNESS_VERSION LLM_MODEL REASONING_EFFORT PROMPT_TOKENS OUTPUT_TOKENS TOOL_OUTPUT_BYTES COMPACT_FRACTION; do runtime_env+=(-e "$key=$(pv "$p" "$key")") done if ! runtime=$(sudo docker run --rm --network none "${runtime_env[@]}" --entrypoint python3 "$img" \ /opt/omcgitea/check_harness.py "$k" "$v") \ || ! python3 -c 'import json,sys;sys.path.insert(0,"agent");from harness_config import PROFILE;assert json.loads(sys.argv[1]).get("limits_profile")==PROFILE' "$runtime"; then log "FATAL: $img needs a rebuild for the selected $k harness/resource policy"; fail=1 fi m="$(pv "$p" ANTHROPIC_MODEL)" [ "$m" = "$LLM_MODEL" ] || { log "FATAL: .env.$p ANTHROPIC_MODEL=$m expected $LLM_MODEL"; fail=1; } w="$(pv "$p" "${prefix}_PROVIDER_WIRE_API")" expected_wire=responses [ "$k" = claude ] && expected_wire=messages [ "$w" = "$expected_wire" ] || { log "FATAL: .env.$p ${prefix}_PROVIDER_WIRE_API='$w' expected $expected_wire"; fail=1; } gw="$(pv "$p" AGENSH_WORKERS)" [ "$gw" = "$WORKERS" ] || { log "FATAL: .env.$p AGENSH_WORKERS=$gw expected $WORKERS"; fail=1; } sn="$(pv "$p" AGENSH_SUBNET_DEFAULT)" python3 -c " import ipaddress,sys n=ipaddress.ip_network('$sn'); need=2*int('${WORKERS:-0}' or 0)+8 sys.exit(0 if n.num_addresses-2 >= need else 1) " 2>/dev/null || { log "FATAL: .env.$p AGENSH_SUBNET_DEFAULT=$sn too small for $WORKERS workers (need $((2*WORKERS+8)) IPs)"; fail=1; } cleanroom_tag="$(pbench_cleanroom_tag "$ROOT/.env.$p")" || { fail=1; continue; } ci="${PROGRAMBENCH_DOCKER_ORG:-programbench}/${task/__/_1776_}:$cleanroom_tag" sudo docker image inspect "$ci" >/dev/null 2>&1 || { log "FATAL: cleanroom missing: $ci"; fail=1; } dd="$(pv "$p" DATA_ROOT)"; dd="${dd:-$ROOT/.claude-team-data}/$p" if [ -d "$dd" ] && [ -n "$(sudo ls -A "$dd" 2>/dev/null)" ]; then log "FATAL: $dd is not empty — once its trajectory is exported, run make clean PROFILE=$p"; fail=1 fi left="$(unfinished_runs "$ROOT/out/$p" | tr '\n' ' ')" [ -z "$left" ] || { log "FATAL: out/$p holds unfinished run(s): $left— archive them first"; fail=1; } done stale_img() { # Detect source edits newer than the image, excluding installed dependencies. local img="$1" label="$2"; shift 2 sudo docker image inspect "$img" >/dev/null 2>&1 || { log "FATAL: image missing: $img"; return 1; } local built newest built=$(sudo docker image inspect "$img" --format '{{.Created}}') built=$(date -d "$built" +%s 2>/dev/null) || return 0 newest=$(find "$@" \( -name node_modules -o -name __pycache__ \) -prune -o -type f -printf '%T@\n' 2>/dev/null \ | cut -d. -f1 | sort -rn | head -1) [ -n "$newest" ] || return 0 if [ "$newest" -gt "$built" ]; then log "FATAL: $img built $(date -d @$built -u +%m-%d\ %H:%M) but $label changed $(date -d @$newest -u +%m-%d\ %H:%M) — rebuild it" return 1 fi log "image OK: $img is newer than $label" } BS="$ROOT/board-service"; MR="$ROOT/mini-router" for task in $TASKS; do p="$(task_profile "$task")" board_img="$(pv "$p" BOARD_IMAGE)" router_img="$(pv "$p" MINI_ROUTER_IMAGE)" stale_img "${board_img:-omcgitea-board:local}" board-service/ \ "$BS/Dockerfile" "$BS/requirements.txt" "$BS"/*.py || fail=1 stale_img "${router_img:-omcgitea-mini-router:local}" mini-router/ \ "$MR/Dockerfile" "$MR/package.json" "$MR/package-lock.json" "$MR/tsconfig.json" \ "$MR/src" || fail=1 done for task in $TASKS; do p="$(task_profile "$task")"; img="$(pv "$p" AGENT_IMAGE 2>/dev/null)" if image_file_matches "$img" /opt/omcgitea/entrypoint.sh "$ROOT/agent/entrypoint.sh"; then log "image OK: $img bakes this checkout's agent/entrypoint.sh" else log "FATAL: ${img:-$p has no AGENT_IMAGE, which} does not bake this checkout's agent/entrypoint.sh -- rebuild it: TASKS=$task bash tasks/programbench/scripts/build_images.sh" fail=1 fi done sm="$(systemctl show agensh.slice -p MemoryMax --value 2>/dev/null)" case "$sm" in ''|infinity) if [ "${AGENSH_SKIP_HOST_GUARDS:-0}" = "1" ]; then log "WARN: agensh.slice has no MemoryMax — skipped by AGENSH_SKIP_HOST_GUARDS=1" else log "FATAL: agensh.slice has no MemoryMax"; fail=1 fi ;; *) log "agensh.slice MemoryMax=$sm ($(( sm / 1073741824 )) GiB)" ;; esac if [ "$(systemctl is-enabled agensh-reboot-guard.service 2>/dev/null)" = "enabled" ]; then log "reboot guard enabled — a maintenance reboot will stop stacks cleanly" else if [ "${AGENSH_SKIP_HOST_GUARDS:-0}" = "1" ]; then log "WARN: agensh-reboot-guard.service not enabled — skipped by AGENSH_SKIP_HOST_GUARDS=1 (a maintenance reboot will kill stacks mid-run)" else log "FATAL: agensh-reboot-guard.service not enabled (see docker/agensh_reboot_guard.service)"; fail=1 fi fi # Probe the selected API family with tool declarations and configured reasoning. probed="" for task in $TASKS; do p="$(task_profile "$task")" [ -f "$ROOT/.env.$p" ] || continue U="$(pv "$p" AGENSH_LLM_UPSTREAM)" K="$(pv "$p" UPSTREAM_AUTH_TOKEN)" M="$(pv "$p" ANTHROPIC_MODEL)" case "$(pv "$p" AGENT_HARNESS)" in pi) prefix=PI ;; codex) prefix=CODEX ;; claude) prefix=CLAUDE ;; *) prefix=COPILOT ;; esac R="$(pv "$p" REASONING_EFFORT)" wire_api="$(pv "$p" "${prefix}_PROVIDER_WIRE_API")" probe_output=2000 output_limit="$(pv "$p" OUTPUT_TOKENS)" if [ -n "$output_limit" ] && [ "$output_limit" -gt 0 ] && [ "$output_limit" -lt "$probe_output" ]; then probe_output="$output_limit" fi case "$probed" in *"|$U/$wire_api|"*) continue;; esac probed="$probed|$U/$wire_api|" probe_path=responses [ "$wire_api" = messages ] && probe_path=messages probe_body=$(python3 - "$M" "$R" "$probe_output" "$wire_api" <<'PY' import json,sys model,effort,limit,wire=sys.argv[1:] schema={'type':'object','properties':{'p':{'type':'string'}}} if wire=='messages': data={'model':model,'messages':[{'role':'user','content':'ok'}], 'output_config':{'effort':effort},'max_tokens':int(limit), 'tools':[{'name':'read_file','description':'r','input_schema':schema}]} else: data={'model':model,'input':'ok','reasoning':{'effort':effort},'max_output_tokens':int(limit), 'tools':[{'type':'function','name':'read_file','description':'r','parameters':schema}]} print(json.dumps(data)) PY ) code=$(upstream_auth_header "$(pv "$p" UPSTREAM_AUTH_STYLE)" "$K" \ | curl -sS -o /dev/null -w '%{http_code}' --max-time 90 \ -H @- -H 'content-type: application/json' \ -X POST "$U/v1/$probe_path" \ -d "$probe_body" 2>/dev/null) host="${U#*://}" [ "$code" = "200" ] && log "upstream OK [$p] ${host%%/*} $M effort=$R -> HTTP 200" \ || { log "FATAL: upstream [$p] ${host%%/*} $M effort=$R -> HTTP $code"; fail=1; } done first_p="$(task_profile "${TASKS%% *}")" PSRC="$ROOT/prompts/agents.txt" want=$(wc -c < "$PSRC" 2>/dev/null) img="$(pv "$first_p" AGENT_IMAGE 2>/dev/null)" got=$(sudo docker run --rm --network none -v "$PSRC:/opt/omcgitea/prompts/agents.md:ro" \ --entrypoint sh "$img" -c \ "wc -c < /opt/omcgitea/prompts/agents.md" 2>/dev/null | tr -d ' ') if [ -n "$want" ] && [ "$got" = "$want" ]; then log "prompt OK: prompts/agents.txt = $want bytes (host file reaches the container)" else log "FATAL: prompts/agents.txt reads '$got' bytes in-container, host has $want -- the mount is not effective" fail=1 fi # The task pack's fragments are mounted as a directory; check one through it. TSRC="$ROOT/tasks/programbench/prompts" want=$(wc -c < "$TSRC/objective.txt" 2>/dev/null) got=$(sudo docker run --rm --network none -v "$TSRC:/opt/omcgitea/task-prompts:ro" \ --entrypoint sh "$img" -c \ "wc -c < /opt/omcgitea/task-prompts/objective.txt" 2>/dev/null | tr -d ' ') if [ -n "$want" ] && [ "$got" = "$want" ]; then log "prompt OK: tasks/programbench/prompts/objective.txt = $want bytes (task prompts reach the container)" else log "FATAL: task prompt objective.txt reads '$got' bytes in-container, host has $want -- the mount is not effective" fail=1 fi for task in $TASKS; do p="$(task_profile "$task")"; ef="$ROOT/.env.$p" [ -f "$ef" ] || continue # Render the same Compose inputs that Make will use, then verify worker counts. make -s render-base PROFILE="$p" >/dev/null 2>&1 || true make -n up PROFILE="$p" >/dev/null 2>&1 || { log "FATAL: make -n up PROFILE=$p fails"; fail=1; continue; } ovl=$(make -n up PROFILE="$p" 2>/dev/null | grep -oE '\-f docker/docker_compose[^ ]*' | tr '\n' ' ') if ! rendered=$(env -u ANTHROPIC_BASE_URL -u ANTHROPIC_MODEL -u ANTHROPIC_API_KEY -u AGENT_HARNESS \ -u COPILOT_PROVIDER_TYPE sudo docker compose -f "$ROOT/docker/_gen_prod.$p.yml" $ovl \ --project-name "omcgitea-$p" --env-file "$ef" config 2>/dev/null); then log "FATAL: $p compose config FAILED to render -- tasks would exit in 60s scoring 0.00%" fail=1; continue fi na=$(grep -cE '^ agent-worker[0-9]+:' <<<"$rendered") nl=$(grep -oE 'AGENTS: .*' <<<"$rendered" | head -1 | tr ',' '\n' | grep -cE -- '-worker[0-9]+') if [ "$na" = "$WORKERS" ] && [ "$nl" = "$WORKERS" ]; then log "compose OK: $p renders $na agent services and a $nl-name AGENTS list" else log "FATAL: $p renders $na agent services / $nl AGENTS names, expected $WORKERS each" fail=1 fi done [ "$fail" = "0" ] || { log "preflight FAILED — nothing launched"; exit 1; } log "preflight OK: $n_tasks profiles, $REQUESTED_AGENT_HARNESS, host guards and configured model API probe" if [ "${PREFLIGHT_ONLY:-0}" = "1" ]; then log "PREFLIGHT_ONLY=1 -- not launching" exit 0 fi eta_h=$(python3 -c "print(f'{$n_tasks * float(\"$HOURS\"):.1f}')") log "=== launching $n_tasks tasks x ${HOURS}h (~${eta_h}h), run $SUFFIX ===" run_chain log "=== single-node launch $SUFFIX complete ==="