#!/bin/bash # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Single-node quick-start: Qwen3-4B dense math RL. # # 8 GPUs on one machine, split 4 actor + 4 vLLM rollout. No slurm. Same # standard batch / context / dataset as slurm/rl_qwen3_4b.sh — only the topology # differs (1 node here, 2 nodes there). # # MODEL_PATH=/path/to/Qwen3-4B bash examples/scripts/quick_start/rl_qwen3_4b.sh set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="${MOLT_PATH:-$(cd "$SCRIPT_DIR/../../.." && pwd)}" MODEL_PATH="${MODEL_PATH:?Set MODEL_PATH to a Qwen3-4B checkpoint.}" # Text-math RL. Auto-prep if missing, as the geo3k quick starts do. Point # PROMPT_DATASET / EVAL_DATASET at your own export (e.g. the proRL text split) # to train on that instead; any set with prompt + reward_model loads the same. DATA_DIR="${DATA_DIR:-$REPO_ROOT/.tmp/dapo_math_17k}" if [ ! -d "$DATA_DIR/train" ] && [ -z "${PROMPT_DATASET:-}" ]; then echo "[quickstart] preparing text math (BytedTsinghua-SIA/DAPO-Math-17k, AIME-2024 eval) — one-time" python3 "$REPO_ROOT/examples/python/utils/prepare_dapo.py" --out-dir "$DATA_DIR" fi PROMPT_DATASET="${PROMPT_DATASET:-$DATA_DIR/train}" EVAL_DATASET="${EVAL_DATASET:-$DATA_DIR/eval}" test -e "$PROMPT_DATASET" || { echo "PROMPT_DATASET not found: $PROMPT_DATASET"; exit 1; } SAVE_ROOT="${SAVE_ROOT:-$REPO_ROOT/outputs/quick_start-qwen3-4b/run}" GPUS_PER_NODE="${GPUS_PER_NODE:-8}" ACTOR_GPUS="${ACTOR_GPUS:-4}" VLLM_TP="${VLLM_TP:-4}" export HF_HOME="${HF_HOME:-$HOME/.cache/huggingface}" export TOKENIZERS_PARALLELISM=true export RAY_USAGE_STATS_ENABLED=0 export VLLM_WORKER_MULTIPROC_METHOD=spawn export VLLM_USE_FLASHINFER_MOE_FP16=0 export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True export MAX_AGENT_TURNS=1 if ! ray status >/dev/null 2>&1; then ray start --head --num-gpus="$GPUS_PER_NODE" --disable-usage-stats >/dev/null STARTED_RAY=1 else STARTED_RAY=0 fi trap '[ "$STARTED_RAY" = "1" ] && ray stop --force >/dev/null 2>&1 || true' EXIT cd "$REPO_ROOT" # Standard config: MAX_LEN=16384, ROLLOUT=16, N=8, TRAIN_BATCH=128, text math dataset. python3 -u -m molt.cli.train_rl_ray \ --actor.model_name_or_path "$MODEL_PATH" \ --data.prompt_dataset "$PROMPT_DATASET" \ --data.input_key prompt \ --data.label_key reward_model \ --data.apply_chat_template \ --data.max_samples 4800 \ --data.max_len 16384 \ --rollout.batch_size 16 \ --rollout.vllm_generate_batch_size 8 \ --rollout.micro_batch_size 1 \ --rollout.n_samples_per_prompt 8 \ --rollout.temperature 1.0 \ --train.batch_size 128 \ --train.micro_batch_size 1 \ --train.max_epochs 1 \ --train.num_episodes 1 \ --train.async_queue_size 2 \ --train.colocate_fsdp_models \ --actor.num_nodes 1 \ --actor.num_gpus_per_node "$ACTOR_GPUS" \ --ref.num_nodes 1 \ --ref.num_gpus_per_node "$ACTOR_GPUS" \ --vllm.num_engines 1 \ --vllm.tensor_parallel_size "$VLLM_TP" \ --vllm.sync_backend nccl \ --vllm.gpu_memory_utilization 0.8 \ --vllm.distributed_executor_backend mp \ --fsdp.param_dtype bf16 \ --fsdp.attn_implementation te \ --fsdp.tp_size 1 \ --fsdp.ep_size 1 \ --fsdp.cp_size 1 \ --fsdp.packing_samples \ --actor.gradient_checkpoint full \ --actor.adam.lr 1e-6 \ --actor.eps_clip_low_high 0.2 0.27 \ --actor.dual_clip 10.0 \ --algo.advantage.estimator reinforce_baseline \ --algo.advantage.is_correction_level geo \ --algo.advantage.is_correction_threshold 0.99 1.01 \ --algo.kl.use_loss \ --algo.kl.estimator k2 \ --algo.kl.init_coef 0.001 \ --algo.dynamic_filtering_enable \ --algo.dynamic_filtering_range 0.01 0.99 \ --reward.clip_range -10 10 \ --train.agent_path "$REPO_ROOT/examples/python/agents/math.py" \ --eval.dataset "$EVAL_DATASET" \ --eval.steps 5 \ --eval.n_samples_per_prompt 1 \ --ckpt.output_dir "$SAVE_ROOT/hf" \ --ckpt.path "$SAVE_ROOT/state" \ --ckpt.save_steps 5 \ --logger.logging_steps 1 \ --logger.wandb.project "${WANDB_PROJECT:-molt_quickstart_qwen3_4b}" \ --logger.wandb.run_name "${WANDB_RUN_NAME:-qwen3_4b_quickstart_$$}"