# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # Multi-turn visual coding: OpenCode with a Python image toolkit solves an image question, # and a GPT judge grades the final \boxed{} answer against the gold answer. Nothing here is # dataset-specific; benchmarks (e.g. benchmarks/babyvision_opencode) and training sets only # supply rows with an image, the prompt, `question` and `expected_answer`. # # opencode_agent runs each task in its own sandbox, through the Environment Server, as an agent session. The resources # server lends that sandbox: each episode's seed starts one through the `sandbox` provider, and the episode's close # stops it. The prompt image goes into the sandbox and is attached with `opencode run --file`. Gym's token capture # records the model calls, so the same config serves evaluation and RL training. # # Needs a sandbox image built from resources_servers/visual_coding_opencode/sandbox_image/Dockerfile # (VISUAL_CODING_OPENCODE_SANDBOX_IMAGE) and the OpenSandbox provider settings (OPENSANDBOX_DOMAIN, # OPENSANDBOX_API_KEY). The sandbox reaches only the policy model server's host, by address. config_paths: - nemo_gym/sandbox/providers/opensandbox/configs/opensandbox.yaml # The sandbox calls the policy model server by address, and its egress rule opens every port of that host. So every # other server sets host 127.0.0.1, which a launcher's default_host (NeMo RL sets one) cannot change: the judge (it # spends the judge's API key for any caller), the agent (a request without a session runs OpenCode on this host), and # the resources and Environment Servers (they start sandboxes and episodes). All their callers run on this host. # Under NeMo RL, Gym's head server takes the node address, so the sandbox can reach it (see the README). use_absolute_ip: true absolute_ip_server_types: [responses_api_models] visual_coding_opencode: resources_servers: visual_coding_opencode: entrypoint: app.py host: 127.0.0.1 domain: other verified: false description: Image questions solved by OpenCode in a sandbox session with a Python image toolkit, GPT-judged final answers. value: Train and evaluate tool-assisted fine-grained visual reasoning, with Gym's token capture for RL. judge_model_server: type: responses_api_models name: visual_coding_opencode_judge_model sandbox_provider: sandbox sandbox_config: image: ${oc.env:VISUAL_CODING_OPENCODE_SANDBOX_IMAGE,null} # Without ttl_s, OpenSandbox ends the sandbox after 10 minutes. ttl_s: 7200 ready_timeout_s: 1200 resources: {cpu: 2, memory_mib: 8192} # WORKDIR of the sandbox image. sandbox_workdir: /workspace # The lent sandbox reaches only the model server, so the model cannot look answers up. It cannot download # OpenCode either, so the image carries the binary (sandbox_executable_path below). sandbox_network_access: model_only sandbox_model_server: type: responses_api_models name: policy_model visual_coding_opencode_agent: responses_api_agents: opencode_agent: entrypoint: app.py host: 127.0.0.1 description: Visual coding with OpenCode in a sandbox session that has a Python image toolkit resources_server: type: resources_servers name: visual_coding_opencode model_server: type: responses_api_models name: policy_model model: ${policy_model_name} opencode_version: 1.17.11 concurrency: 64 # Adds `opencode run --thinking`, which only shows reasoning in OpenCode's own output. The policy's serving # setup decides whether the model reasons. thinking: true # The only hard stop besides the context window: OpenCode itself has no turn cap. timeout: 1800 context_window: 131072 max_output_tokens: 32768 session_close_timeout_seconds: 90 # The model looks at the crops and plots it makes through OpenCode's read tool, and OpenCode passes those images # on only when image input is declared. The prompt image declares it anyway; this keeps it for any row. image_input: true # The resources server lends the sandbox and sets its network, so the agent keeps network_access inherit and # needs no sandbox_provider of its own. The lent sandbox cannot download OpenCode, so the image carries the # binary, pinned by its SHA-256. sandbox_executable_path: /opt/opencode/bin/opencode sandbox_executable_sha256: 0254a429cd0e6cf0ba53fc01672cf98e4a8dc728f7fa94be88f1e4b3645e6ded # pragma: allowlist secret # OpenCode's plugin dependencies, from the same image. Without them, OpenCode tries to install them from npm at # each start, and an offline start waits about 70 s before its first model call. sandbox_plugin_dependencies_path: /opt/opencode/opencode-1.17.11-plugin-dependencies.tar.gz sandbox_plugin_dependencies_sha256: 77b03eba45c8d7fe3d0a58bfc215f6fb99a1db4dd18a7e3d91fd29981fddafb1 # pragma: allowlist secret opencode_config: tools: webfetch: false websearch: false # Subagents run in sessions of their own, outside the scored chain of model calls. task: false permission: bash: allow edit: allow # OpenCode asks before it reads or writes outside the working directory, as when the model saves a crop # under /tmp. `opencode run` rejects every request that would ask, and a rejection ends the run with no # answer. The sandbox holds only this task, so allow every path. external_directory: allow # OpenCode also asks when the model repeats an identical tool call, and that rejection ends the run too. doom_loop: allow compaction: # Automatic compaction continues from a summary, in model calls that training cannot join to the scored # call. With it off, a run that fills the context ends with a context-limit error. auto: false datasets: - name: example type: example jsonl_fpath: resources_servers/visual_coding_opencode/data/example.jsonl num_repeats: 1 visual_coding_opencode_judge_model: responses_api_models: openai_model: entrypoint: app.py host: 127.0.0.1 # Any OpenAI-compatible endpoint: set visual_coding_opencode_judge_base_url and # visual_coding_opencode_judge_model_name to use another judge. openai_base_url: ${oc.select:visual_coding_opencode_judge_base_url,https://api.openai.com/v1} # Default empty: `gym eval prepare` loads this config without the run's key override. openai_api_key: ${oc.env:OPENAI_API_KEY,""} openai_model: ${oc.select:visual_coding_opencode_judge_model_name,gpt-4o-mini} openai_default_headers: {} max_concurrent_requests: 32 # The Environment Server runs each episode as an agent session: it seeds the session, sends the task, and closes it. visual_coding_opencode_environment_server: environment_servers: single_agent_turn_legacy: entrypoint: app.py host: 127.0.0.1 resources_server: type: resources_servers name: visual_coding_opencode agent_server: type: responses_api_agents name: visual_coding_opencode_agent # Admit episodes at the agent's rate (its concurrency, 64). Each seed lends a sandbox, so an episode admitted # beyond it would wait for OpenCode with its sandbox started and its deadline running. Admission comes before # the deadline starts. max_concurrent_episodes: 64 queue_timeout_seconds: 86400 # Above the sandbox start (ready_timeout_s), OpenCode's run (timeout) and judging; below the sandbox's ttl_s. default_episode_timeout_seconds: 5400