# GenRM Pairwise Comparison Resources Server Config # # This server compares multiple candidate responses using a GenRM model. # The GenRM model uses a special chat template with roles 'response_1', 'response_2', # and optionally 'principle' (see use_principle and default_principle below). # # The GenRM model server is defined by the top-level "genrm_model" block below (same # structure as responses_api_models/genrm_model/configs/genrm_model.yaml). Override # genrm_model_server.name to use a different model server (e.g. from another config). # # Expected GenRM output format (JSON): # { # "score_1": <1-5>, # Individual helpfulness score for response 1 # "score_2": <1-5>, # Individual helpfulness score for response 2 # "ranking": <1-6> # 1=R1 much better, 6=R2 much better # } genrm_compare_resources_server: resources_servers: genrm_compare: entrypoint: app.py # GenRM model server: points to the "genrm_model" block above (or another config with that name) genrm_model_server: type: responses_api_models name: genrm_model # Cohort-based verify (Difference 1): number of rollouts per prompt before running comparison. # For isolated retries/replay, share a fresh _ng_group_id across exactly this many members. # Use local _ng_rollout_index slots 0..15 and enough concurrency to admit the whole group. # File collector: --num-repeats 16 --concurrency 16 (one task, one agent). num_rollouts_per_prompt: 16 # Separate finite deadlines: collection starts at first arrival; evaluation # starts only when all members are present. Increase for long tool rollouts. cohort_collection_timeout_s: 1800 cohort_evaluation_timeout_s: 1800 # Includes connection retries and queue wait. A 24576-token judge answer # takes about 614 seconds at 40 tokens/s; allow the full evaluation budget. judge_request_timeout_s: 1800 # Retain compact rewards/digests for idempotent late retries. The cache is also # bounded, so disabling the TTL does not make terminal state unbounded. cohort_result_ttl_s: 21600 max_terminal_cohorts: 4096 # Generation params for GenRM calls genrm_responses_create_params: input: [] max_output_tokens: 16384 temperature: 0.6 top_p: 0.95 # "rollout_cohort" or "fixed_baseline" (metadata.baseline_response) comparison_mode: rollout_cohort # Comparison strategy: "all_pairs" (C(n,2) comparisons) or "circular" (n comparisons) comparison_strategy: circular # Number of judge passes per pair (for majority voting) num_judges_per_comparison: 1 # Principle-based comparison use_principle: false # default_principle: "..." (see code for full default text) # Aggregator method (only "simple_tiebreaker" is currently supported) aggregator_method: simple_tiebreaker # rubric_mean requires expected_rubric_ids on each dataset record. score_source: overall # "overall" or "rubric_mean" # Bonus for shortest reasoning trace if in top percentile reasoning_bonus: 0.0 # Bonus for shortest final answer if in top percentile answer_bonus: 0.0 # Top percentile threshold (e.g., 0.2 = top 20%) top_percentile: 0.2 # Group-relative length penalty coefficients # Shorter responses get bonus, longer get penalty (mean-centered) group_reasoning_length_penalty_coeff: 0.0 group_answer_length_penalty_coeff: 0.0 # Group-relative style penalty coefficient # Lower formatting density (headers, lists, bold, emoji) gets bonus group_style_penalty_coeff: 0.0 # Default scores when parsing fails default_score: 3.0 default_ranking: 3.5 # Debug logging debug_logging: false # Shared retry budget for parse failures and judge HTTP 408, 429, or 5xx genrm_parse_retries: 3 genrm_parse_retry_sleep_s: 0.2 # Server metadata domain: rlhf verified: false description: GenRM pairwise comparison for RLHF training value: Compare multiple candidate responses using GenRM model # GenRM model server (local vLLM). Same structure as responses_api_models/genrm_model/configs/genrm_model.yaml. # Uses ${genrm_model_name} from env. Load this block when running genrm_compare, or load that file instead. genrm_model: responses_api_models: genrm_model: entrypoint: app.py model: ${genrm_model_name} return_token_id_information: false uses_reasoning_parser: false # GenRM outputs structured JSON, not reasoning supports_principle_role: true hf_home: null vllm_serve_env_vars: VLLM_RAY_DP_PACK_STRATEGY: strict vllm_serve_kwargs: data_parallel_size: 2 data_parallel_size_local: 2 tensor_parallel_size: 4 pipeline_parallel_size: 1 trust_remote_code: true gpu_memory_utilization: 0.9 debug: false genrm_simple_agent: responses_api_agents: simple_agent: entrypoint: app.py resources_server: type: resources_servers name: genrm_compare_resources_server model_server: type: responses_api_models name: policy_model datasets: - name: example type: example jsonl_fpath: resources_servers/genrm_compare/data/example.jsonl genrm_simple_agent_reasoning_off: responses_api_agents: simple_agent: entrypoint: app.py resources_server: type: resources_servers name: genrm_compare_resources_server model_server: type: responses_api_models name: policy_model_reasoning_off datasets: - name: example type: example jsonl_fpath: resources_servers/genrm_compare/data/example.jsonl genrm_environment_server: environment_servers: legacy_agent: entrypoint: app.py agent_server: type: responses_api_agents name: genrm_simple_agent genrm_simple_agent_reasoning_off_environment_server: environment_servers: legacy_agent: entrypoint: app.py agent_server: type: responses_api_agents name: genrm_simple_agent_reasoning_off