# ViBench through an EnvironmentServer: the resources server starts the build sandbox with the PRD # staged and lends it, OpenCode builds the app there through an agent session, then the resources # server copies the app out, stops the sandbox and grades the copy in fresh compose stacks. # configs/vibench_legacy_agent.yaml keeps the deprecated legacy relay path through vibench_agent; the two paths are # configured differently, so do not compare their scores as the same measurement. # With Docker, pair this config with resources_servers/vibench/configs/docker.yaml, not # responses_api_agents/vibench_agent/configs/docker.yaml: that overlay is for the legacy config only, and with it the # sandbox gets a loopback model URL, so OpenCode cannot reach the model. config_paths: # opencode_agent's command deny-list, which also denies the web and skill tools. The agent below adds `task: deny` # and sets its own fields. - responses_api_agents/opencode_agent/configs/permissions/default.yaml - environment_servers/single_agent_turn_legacy/configs/single_agent_turn_legacy.yaml vibench_resources_server: resources_servers: vibench: entrypoint: app.py domain: coding verified: false description: PRD-to-web-app benchmark; apps are graded by seeding and driving them in a browser. value: Eval whether a model can ship a working web app from a spec, judged through the running UI rather than the diff. allowed_agents: [opencode_agent] # Absolute path to a ViBench checkout. Grading shells into # _harness/runner/scripts/run-seed.py then run-evaluate-post-seeding.py inside it. # (Not run-seed-then-evaluate.py -- see the comment above SEED_SCRIPT in app.py.) # Defaulted so the config resolves without the variable set: data tooling and CI load # this file without ever starting the server, and a bare ${oc.env:...} makes that fail # with a config interpolation error. Set VIBENCH_REPO_ROOT to a real checkout to run. vibench_repo_root: ${oc.env:VIBENCH_REPO_ROOT,/nonexistent/set-VIBENCH_REPO_ROOT} # Names a top-level provider block, e.g. nemo_gym/sandbox/providers/opensandbox/configs/opensandbox.yaml # or resources_servers/vibench/configs/docker.yaml. The agent borrows the build sandbox through it too. sandbox_provider: sandbox sandbox_config: # The same base image ViBench's own pipeline builds and reuses (see # build_base_image_if_needed in _harness/runner/scripts/common.py): # docker build -f _harness/runner/docker/Dockerfile.base -t app-bench-base:latest . # Its WORKDIR is /app, which is where the builder lands. image: ${oc.env:VIBENCH_BUILD_IMAGE,app-bench-base:latest} # Without a TTL the OpenSandbox SDK expires a sandbox 10 minutes after creation. ttl_s: 14400 ready_timeout_s: 600 resources: cpu: 4 memory_mib: 8192 disk_gib: 20 metadata: harness: opencode app_workdir: /app harvest_timeout_s: 900 # Supplies AGENT_SEEDING_LLM_* / AGENT_EVALUATION_LLM_* for the grader agents. # These are the verifier's models, deliberately not the policy model under test. vibench_env_file: ${oc.env:VIBENCH_ENV_FILE,null} evaluation_timeout_s: 5400 # Each test plan stands up postgres + the app + playwright. Two at a time per rollout # is already heavy; raise only on a big host. max_concurrent_test_plans: 2 # Set VIBENCH_KEEP_ARTIFACTS=true to retain per-test-plan traces, screenshots, # DB dumps and container logs -- the only way to diagnose a grading failure # after the fact, since the work dir is otherwise removed. keep_evaluation_artifacts: ${oc.env:VIBENCH_KEEP_ARTIFACTS,false} # OpenCode builds the app in the lent sandbox, with opencode_agent's command deny-list and without the web, subagent # and skill tools, all of which the legacy recipe allows. The other settings keep vibench_agent's # (configs/vibench_legacy_agent.yaml) except where noted. vibench_opencode_agent: _inherit_from: opencode_agent responses_api_agents: opencode_agent: entrypoint: app.py num_workers: 1 model: ${policy_model_name} model_server: type: responses_api_models name: policy_model resources_server: type: resources_servers name: vibench_resources_server opencode_version: 1.17.11 # OpenCode asks for max_output_tokens on every call, and an OpenAI-compatible server refuses a request whose # prompt plus output budget exceeds the served context, so use the served context length minus the output # budget (262,144 - 32,768). The legacy recipe asks for the whole window, which vLLM refuses outright. context_window: 229376 max_output_tokens: 32768 # The build budget per rollout, as in the legacy recipe. timeout: 5400 sandbox_install_timeout_seconds: 600 session_close_timeout_seconds: 60 session_close_retry_window_seconds: 300 # Sessions are process-local to this one worker; grading, not the agent, bounds throughput. concurrency: 32 opencode_config: # These follow the deny-list's "*": allow, which they override. The legacy recipe allows all four tools. permission: webfetch: deny websearch: deny task: deny skill: deny datasets: - name: example type: example jsonl_fpath: resources_servers/vibench/data/example.jsonl vibench_environment_server: _inherit_from: single_agent_turn_legacy environment_servers: single_agent_turn_legacy: resources_server: type: resources_servers name: vibench_resources_server agent_server: type: responses_api_agents name: vibench_opencode_agent # The build (up to 5400 s), then up to max_concurrent_test_plans grading stacks at a # time, each bounded by evaluation_timeout_s. default_episode_timeout_seconds: 21600