gdp_pdf: resources_servers: gdp_pdf: allowed_agents: - gdp_pdf_agent entrypoint: app.py domain: other description: 'GDP.pdf: professional multimodal reasoning over real-world PDFs (contracts, filings, records) across ten domains, graded per atomic rubric criterion by an LLM judge' value: Measure document reasoning on the dense professional documents the economy runs on judge_model_server: type: responses_api_models name: judge_model judge_chat_create_params: messages: [] # Artificial Analysis grades GDP.pdf with GPT-5.6 Luna at medium reasoning effort. The token # budget covers the judge's reasoning as well as its JSON verdict: at 1024 tokens the judge # ran out before writing the verdict, and each such criterion failed the whole row. reasoning_effort: medium max_completion_tokens: 16384 judge_prompt_template_fpath: prompt_templates/judge.txt # Training reward only; evaluation keeps mean_criteria, and all_pass and # mean_criterion_pass are reported in every mode. reward_mode: mean_criteria # mean_criteria | strict_pass | mixed mixed_reward_alpha: 0.5 # mixed: (1 - alpha) * mean_criterion_pass + alpha * all_pass judge_max_attempts: 5 # 500 rollouts x ~12.8 criteria/rollout is ~6,400 judge calls; at 64 concurrent # against the shared gpt-5.6-luna Azure deployment this sustained enough 429s to # exhaust all judge_max_attempts on 134/500 Ultra rollouts (all failures were # rate_limit, not judge malformed output -- see logs/judge_model.log). Lowered # to reduce sustained pressure on the shared deployment. judge_endpoint_max_concurrency: 16 verified: false gdp_pdf_simple_agent: responses_api_agents: gdp_pdf_agent: entrypoint: app.py max_steps: 1 resources_server: type: resources_servers name: gdp_pdf model_server: type: responses_api_models name: policy_model documents_base_dir: resources_servers/gdp_pdf/data dpi: 150 min_dpi: 72 max_images: null # compositing only kicks in reactively, on an actual server-reported image-count limit max_pages: null include_text: true include_images: true image_format: png # lossless; set jpeg to cut the base64 payload ~28% at the cost of artifacts jpeg_quality: 90 strip_images_from_output: true # Training only: mask attempts that end without a gradable answer (see the agent README). mask_failed_attempts: false datasets: - name: example type: example jsonl_fpath: resources_servers/gdp_pdf/data/example.jsonl num_repeats: 1 license: MIT gdp_pdf_simple_environment_server: environment_servers: legacy_agent: entrypoint: app.py agent_server: type: responses_api_agents name: gdp_pdf_simple_agent