# This is a config file for the models available in HarmBench. The entries in this config specify model names, how to load the models, how many GPUs # to request for the models (e.g., if using SLURM), the model type, and other information. # ===================== Documentation for the fields ===================== # # Required fields for all models:: # model_type: The type of model. This can be "open_source", "closed_source", "open_source_multimodal", or "closed_source_multimodal". # All models are required to have this field in their config. This is used to determine which other fields to expect. # Required fields for open_source models:: # model: A dictionary containing arguments to load_model_and_tokenizer in `HarmBench/baselines/model_utils.py`. This is used to load the model # and tokenizer. This dictionary is required to have the `model_name_or_path` field. Other fields are optional and can be used to # override the default arguments to load_model_and_tokenizer. # num_gpus: The number of GPUs to request for the model. This is used to determine the number of GPUs to request when using SLURM. # Some methods also load open source LLMs using vLLM or create mini-clusters using ray. For these methods, we use `num_gpus` # to allocate the right number of GPUs for the model. # NOTE: The default values were chosen to work with 80GB A100 GPUs. If you are using different GPUs, you may need to adjust `num_gpus`. # # Required fields for closed_source models:: # model: A dictionary containing arguments to load_model_and_tokenizer in `model_utils.py`. This is used to load the model and tokenizer. # This dictionary is required to have the following fields, which are handled in `HarmBench/api_models.py`: # - model_name_or_path: The name of the API. This is used to select the API class in `HarmBench/api_models.py`. # - token: The token for accessing the model # # Required fields for open_source_multimodal models:: # model: A dictionary containing arguments to load_model_and_tokenizer in `HarmBench/baselines/model_utils.py`. This is used to load the model and tokenizer. # # Required fields for closed_source_multimodal models:: # model: A dictionary containing arguments to load_model_and_tokenizer in `model_utils.py`. This is used to load the model and tokenizer. # ===================== Open-Source Models ===================== # # Llama 2 llama2_7b: model: model_name_or_path: meta-llama/Llama-2-7b-chat-hf use_fast_tokenizer: False dtype: float16 chat_template: llama-2 num_gpus: 1 model_type: open_source llama2_13b: model: model_name_or_path: meta-llama/Llama-2-13b-chat-hf use_fast_tokenizer: False dtype: float16 chat_template: llama-2 num_gpus: 1 model_type: open_source llama2_70b: model: model_name_or_path: meta-llama/Llama-2-70b-chat-hf use_fast_tokenizer: False dtype: float16 chat_template: llama-2 num_gpus: 2 model_type: open_source # Vicuna vicuna_7b_v1_5: model: model_name_or_path: lmsys/vicuna-7b-v1.5 use_fast_tokenizer: False dtype: float16 chat_template: vicuna num_gpus: 1 model_type: open_source vicuna_13b_v1_5: model: model_name_or_path: lmsys/vicuna-13b-v1.5 use_fast_tokenizer: False dtype: float16 chat_template: vicuna num_gpus: 1 model_type: open_source # Koala koala_7b: model: model_name_or_path: TheBloke/koala-7B-HF use_fast_tokenizer: False dtype: bfloat16 chat_template: koala num_gpus: 1 model_type: open_source koala_13b: model: model_name_or_path: TheBloke/koala-13B-HF use_fast_tokenizer: False dtype: bfloat16 chat_template: koala num_gpus: 1 model_type: open_source # Orca orca_2_7b: model: model_name_or_path: microsoft/Orca-2-7b use_fast_tokenizer: False dtype: bfloat16 chat_template: orca-2 num_gpus: 1 model_type: open_source orca_2_13b: model: model_name_or_path: microsoft/Orca-2-13b use_fast_tokenizer: False dtype: bfloat16 chat_template: orca-2 num_gpus: 1 model_type: open_source # SOLAR solar_10_7b_instruct: model: model_name_or_path: upstage/SOLAR-10.7B-Instruct-v1.0 use_fast_tokenizer: False dtype: float16 num_gpus: 1 model_type: open_source # OpenChat openchat_3_5_1210: model: model_name_or_path: openchat/openchat-3.5-1210 use_fast_tokenizer: False dtype: bfloat16 num_gpus: 1 model_type: open_source # Starling starling_7b: model: model_name_or_path: berkeley-nest/Starling-LM-7B-alpha use_fast_tokenizer: False dtype: bfloat16 num_gpus: 1 model_type: open_source # Mistral mistral_7b_v2: model: model_name_or_path: mistralai/Mistral-7B-Instruct-v0.2 use_fast_tokenizer: False dtype: bfloat16 chat_template: mistral num_gpus: 1 model_type: open_source mixtral_8x7b: model: model_name_or_path: mistralai/Mixtral-8x7B-Instruct-v0.1 use_fast_tokenizer: False dtype: bfloat16 chat_template: mistral num_gpus: 2 model_type: open_source # Zephyr zephyr_7b: model: model_name_or_path: HuggingFaceH4/zephyr-7b-beta use_fast_tokenizer: False dtype: bfloat16 num_gpus: 1 model_type: open_source # Zephyr 7B + R2D2 (adversarial training against GCG) zephyr_7b_robust: model: model_name_or_path: cais/zephyr_7b_r2d2 use_fast_tokenizer: False dtype: bfloat16 chat_template: zephyr_7b_robust num_gpus: 1 model_type: open_source # Baichuan 2 baichuan2_7b: model: model_name_or_path: baichuan-inc/Baichuan2-7B-Chat use_fast_tokenizer: False dtype: bfloat16 trust_remote_code: True chat_template: baichuan2 num_gpus: 1 model_type: open_source baichuan2_13b: model: model_name_or_path: baichuan-inc/Baichuan2-13B-Chat use_fast_tokenizer: False dtype: bfloat16 trust_remote_code: True chat_template: baichuan2 num_gpus: 1 model_type: open_source # Qwen qwen_7b_chat: model: model_name_or_path: Qwen/Qwen-7B-Chat use_fast_tokenizer: True dtype: bfloat16 pad_token: <|extra_0|> eos_token: <|im_end|> trust_remote_code: True chat_template: qwen num_gpus: 1 model_type: open_source qwen_14b_chat: model: model_name_or_path: Qwen/Qwen-14B-Chat use_fast_tokenizer: True trust_remote_code: True dtype: bfloat16 pad_token: <|extra_0|> eos_token: <|im_end|> chat_template: qwen num_gpus: 1 model_type: open_source qwen_72b_chat: model: model_name_or_path: Qwen/Qwen-72B-Chat use_fast_tokenizer: True dtype: bfloat16 pad_token: <|extra_0|> eos_token: <|im_end|> trust_remote_code: True chat_template: qwen num_gpus: 4 model_type: open_source # ===================== Closed-Source Models ===================== # # TODO: remove tokens before release gpt-3.5-turbo-1106: model: model_name_or_path: gpt-3.5-turbo-1106 token: model_type: closed_source gpt-3.5-turbo-0613: model: model_name_or_path: gpt-3.5-turbo-0613 token: model_type: closed_source gpt-4-1106-preview: model: model_name_or_path: gpt-4-1106-preview token: model_type: closed_source gpt-4-0613: model: model_name_or_path: gpt-4-0613 token: model_type: closed_source gpt-4-vision-preview: model: model_name_or_path: gpt-4-vision-preview token: model_type: closed_source claude-instant-1: model: model_name_or_path: claude-instant-1 token: model_type: closed_source claude-2.1: model: model_name_or_path: claude-2.1 token: model_type: closed_source claude-2: model: model_name_or_path: claude-2 token: model_type: closed_source gemini: model: model_name_or_path: gemini-pro token: model_type: closed_source mistral-medium: mode: model_name_or_path: mistral-medium token: model_type: closed_source # ===================== Open-Source Multimodal Models ===================== # llava_v1_5: model: model_name_or_path: LLaVA_v1_5 model_type: open_source_multimodal num_gpus: 1 instructblip: model: model_name_or_path: InstructBLIP model_type: open_source_multimodal num_gpus: 1 qwen_vl_chat: model: model_name_or_path: Qwen_VL_Chat model_type: open_source_multimodal num_gpus: 1 # ===================== Closed-Source Multimodal Models ===================== # gpt4v: model: model_name_or_path: GPT4V model_type: closed_source_multimodal