Skip to content

Commit 65b8497

Browse files
JamePengJamePeng
authored
Merge pull request #61 from TAO71-AI/main
Implemented Qwen35ChatHandler for Qwen3.5
2 parents 5bf6b6a + 2258973 commit 65b8497

1 file changed

Lines changed: 199 additions & 11 deletions

File tree

‎llama_cpp/llama_chat_format.py‎

Lines changed: 199 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4669,7 +4669,6 @@ def __init__(
46694669
self,
46704670
force_reasoning: bool = False,
46714671
add_vision_id: bool = True,
4672-
image_min_tokens: int = -1,
46734672
**kwargs,
46744673
):
46754674
"""
@@ -4680,20 +4679,13 @@ def __init__(
46804679
- add_vision_id (bool):
46814680
- True (default): Count all the images. Recommended for multi-image.
46824681
- False: Doesn't count the images. Can save tokens with single-image.
4683-
- image_min_tokens (int):
4684-
It only takes effect when the value is greater than zero. the default value is -1 (i.e., using the default parameters in the model's preprocessor_config.json).
4685-
Note: Qwen-VL models require at minimum 1024 image tokens to function correctly on bbox grounding tasks
46864682
"""
4683+
super().__init__(**kwargs)
46874684
self.force_reasoning = force_reasoning
4688-
self.add_vision_id = add_vision_id
4689-
self.image_min_tokens = image_min_tokens
4690-
4691-
super().__init__(image_min_tokens=self.image_min_tokens, **kwargs)
4685+
self.extra_template_arguments["force_reasoning"] = force_reasoning
4686+
self.extra_template_arguments["add_vision_id"] = add_vision_id
46924687

46934688
def __call__(self, **kwargs):
4694-
self.extra_template_arguments["force_reasoning"] = self.force_reasoning
4695-
self.extra_template_arguments["add_vision_id"] = self.add_vision_id
4696-
46974689
llama = kwargs['llama']
46984690

46994691
if hasattr(llama, 'input_ids'):
@@ -4705,6 +4697,202 @@ def __call__(self, **kwargs):
47054697
# Use parent implementation
47064698
return super().__call__(**kwargs)
47074699

4700+
class Qwen35ChatHandler(MTMDChatHandler):
4701+
CHAT_FORMAT = (
4702+
"{%- set image_count = namespace(value=0) -%}"
4703+
"{%- set video_count = namespace(value=0) -%}"
4704+
"{%- macro render_content(content, do_vision_count, is_system_content=false) -%}"
4705+
" {%- if content is string -%}"
4706+
" {{- content -}}"
4707+
" {%- elif content is iterable and content is not mapping -%}"
4708+
" {%- for item in content -%}"
4709+
" {%- if 'image_url' in item or item.type == 'image_url' -%}"
4710+
" {%- if is_system_content -%}"
4711+
" {{- raise_exception('System message cannot contain images.') -}}"
4712+
" {%- endif -%}"
4713+
" {%- if do_vision_count -%}"
4714+
" {%- set image_count.value = image_count.value + 1 -%}"
4715+
" {%- endif -%}"
4716+
" {%- if add_vision_id -%}"
4717+
" {{- 'Picture ' -}}"
4718+
" {{- image_count.value | string -}}"
4719+
" {{- ': ' -}}"
4720+
" {%- endif -%}"
4721+
" {{- '<|vision_start|>' -}}"
4722+
" {%- if item.image_url is string -%}"
4723+
" {{- item.image_url -}}"
4724+
" {%- else -%}"
4725+
" {{- item.image_url.url -}}"
4726+
" {%- endif -%}"
4727+
" {{- '<|vision_end|>' -}}"
4728+
" {%- elif 'video' in item -%}"
4729+
" {{- raise_exception('llama.cpp does not currently support video.') -}}" # Video not supported, raise exception
4730+
" {%- if is_system_content -%}"
4731+
" {{- raise_exception('System message cannot contain videos.') -}}"
4732+
" {%- endif -%}"
4733+
" {%- if do_vision_count -%}"
4734+
" {%- set video_count.value = video_count.value + 1 -%}"
4735+
" {%- endif -%}"
4736+
" {%- if add_vision_id -%}"
4737+
" {{- 'Video ' ~ video_count.value ~ ': ' -}}"
4738+
" {%- endif -%}"
4739+
" {{- '<|vision_start|>' -}}"
4740+
" {{- item.video -}}"
4741+
" {{- '<|vision_end|>' -}}"
4742+
" {%- elif 'text' in item -%}"
4743+
" {{- item.text -}}"
4744+
" {%- else -%}"
4745+
" {{- raise_exception('Unexpected item type in content.') -}}"
4746+
" {%- endif -%}"
4747+
" {%- endfor -%}"
4748+
" {%- elif content is none or content is undefined -%}"
4749+
" {{- '' -}}"
4750+
" {%- else -%}"
4751+
" {{- raise_exception('Unexpected content type.') -}}"
4752+
" {%- endif -%}"
4753+
"{%- endmacro -%}"
4754+
"{%- if not messages -%}"
4755+
" {{- raise_exception('No messages provided.') -}}"
4756+
"{%- endif -%}"
4757+
"{%- if tools and tools is iterable and tools is not mapping -%}"
4758+
" {{- '<|im_start|>system\n' -}}"
4759+
" {{- '# Tools\n\nYou have access to the following functions:\n\n<tools>' -}}"
4760+
" {%- for tool in tools -%}"
4761+
" {{- '\n' -}}"
4762+
" {{- tool | tojson -}}"
4763+
" {%- endfor -%}"
4764+
" {{- '\n</tools>' -}}"
4765+
" {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' -}}"
4766+
" {%- if messages[0].role == 'system' -%}"
4767+
" {%- set content = render_content(messages[0].content, false, true) | trim -%}"
4768+
" {%- if content -%}"
4769+
" {{- '\n\n' + content -}}"
4770+
" {%- endif -%}"
4771+
" {%- endif -%}"
4772+
" {{- '<|im_end|>\n' -}}"
4773+
"{%- elif messages[0].role == 'system' -%}"
4774+
" {%- set content = render_content(messages[0].content, false, true) -%}"
4775+
" {{- '<|im_start|>system\n' + content + '<|im_end|>\n' -}}"
4776+
"{%- endif -%}"
4777+
"{%- set ns = namespace(multi_step_tool=true, last_query_index=messages | length - 1) -%}"
4778+
"{%- for message in messages[::-1] -%}"
4779+
" {%- set index = messages | length - 1 - loop.index0 -%}"
4780+
" {%- if ns.multi_step_tool and message.role == 'user' -%}"
4781+
" {%- set content = render_content(message.content, false) | trim -%}"
4782+
" {%- if not (content.startswith('<tool_response>') and content.endswith('</tool_response>')) -%}"
4783+
" {%- set ns.multi_step_tool = false -%}"
4784+
" {%- set ns.last_query_index = index -%}"
4785+
" {%- endif -%}"
4786+
" {%- endif -%}"
4787+
"{%- endfor -%}"
4788+
"{%- if ns.multi_step_tool -%}"
4789+
" {{- raise_exception('No user query found in messages.') -}}"
4790+
"{%- endif -%}"
4791+
"{%- for message in messages -%}"
4792+
" {%- set content = render_content(message.content, true) | trim -%}"
4793+
" {%- if message.role == 'system' -%}"
4794+
" {%- if not loop.first -%}"
4795+
" {{- raise_exception('System message must be at the beginning.') -}}"
4796+
" {%- endif -%}"
4797+
" {%- elif message.role == 'user' -%}"
4798+
" {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>\n' -}}"
4799+
" {%- elif message.role == 'assistant' -%}"
4800+
" {%- set reasoning_content = '' -%}"
4801+
" {%- if message.reasoning_content is string -%}"
4802+
" {%- set reasoning_content = message.reasoning_content -%}"
4803+
" {%- elif '</think>' in content -%}"
4804+
" {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') -%}"
4805+
" {%- set content = content.split('</think>')[-1].lstrip('\n') -%}"
4806+
" {%- endif -%}"
4807+
" {%- set reasoning_content = reasoning_content | trim -%}"
4808+
" {%- if loop.index0 > ns.last_query_index -%}"
4809+
" {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n\n' + content -}}"
4810+
" {%- else -%}"
4811+
" {{- '<|im_start|>' + message.role + '\n' + content -}}"
4812+
" {%- endif -%}"
4813+
" {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping -%}"
4814+
" {%- for tool_call in message.tool_call -%}"
4815+
" {%- if tool_call.function is defined -%}"
4816+
" {%- set tool_call = tool_call.function -%}"
4817+
" {%- endif -%}"
4818+
" {%- if loop.first -%}"
4819+
" {%- if content | trim -%}"
4820+
" {{- '\n\n<tool_call>\n<function=' + tool_call.name + '>\n' -}}"
4821+
" {%- else -%}"
4822+
" {{- '<tool_call>\n<function=' + tool_call.name + '>\n' -}}"
4823+
" {%- endif -%}"
4824+
" {%- else -%}"
4825+
" {{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' -}}"
4826+
" {%- endif -%}"
4827+
" {%- if tool_call.arguments is defined -%}"
4828+
" {%- for (args_name, args_value) in tool_calls.arguments | items -%}"
4829+
" {{- '<parameter=' + args.name + '>\n' -}}"
4830+
" {%- set args_value = args_value | tojson | safe if args_value is mapping or args_value is sequence and args_value is not string else args_value | string -%}"
4831+
" {{- args_value -}}"
4832+
" {{- '\n</parameter>' -}}"
4833+
" {%- endfor -%}"
4834+
" {%- endif -%}"
4835+
" {{- '</function>\n</tool_call>' -}}"
4836+
" {%- endfor -%}"
4837+
" {%- endif -%}"
4838+
" {{- '<|im_end|>\n' -}}"
4839+
" {%- elif message.role == 'tool' -%}"
4840+
" {%- if loop.previtem and loop.previtem.role != 'tool' -%}"
4841+
" {{- '<|im_start|>user' -}}"
4842+
" {%- endif -%}"
4843+
" {{- '\n<tool_response>\n' -}}"
4844+
" {{- content -}}"
4845+
" {{- '\n</tool_response>' -}}"
4846+
" {%- if not loop.last and loop.nextitem.role != 'tool' -%}"
4847+
" {{- '<|im_end|>\n' -}}"
4848+
" {%- elif loop.last -%}"
4849+
" {{- '<|im_end|>\n' -}}"
4850+
" {%- endif -%}"
4851+
" {%- else -%}"
4852+
" {{- raise_exception('Unexpected message role.') -}}"
4853+
" {%- endif -%}"
4854+
"{%- endfor -%}"
4855+
"{%- if add_generation_prompt -%}"
4856+
" {{- '<|im_start|>assistant\n' -}}"
4857+
" {%- if enable_thinking is false -%}"
4858+
" {{- '<think>\n\n</think>\n\n' -}}"
4859+
" {%- else -%}"
4860+
" {{- '<think>\n' -}}"
4861+
" {%- endif -%}"
4862+
"{%- endif -%}"
4863+
)
4864+
4865+
def __init__(
4866+
self,
4867+
enable_thinking: bool = True,
4868+
add_vision_id: bool = True,
4869+
**kwargs,
4870+
):
4871+
"""
4872+
Parameters:
4873+
- enable_thinking (bool):
4874+
- True (default): Enables reasoning for better results.
4875+
- False: Disables reasoning for faster results.
4876+
- add_vision_id (bool):
4877+
- True (default): Count all the images. Recommended for multi-image.
4878+
- False: Doesn't count the images. Can save tokens with single-image.
4879+
"""
4880+
super().__init__(**kwargs)
4881+
self.enable_thinking = enable_thinking
4882+
self.extra_template_arguments["enable_thinking"] = enable_thinking
4883+
self.extra_template_arguments["add_vision_id"] = add_vision_id
4884+
4885+
def __call__(self, **kwargs):
4886+
llama = kwargs['llama']
4887+
4888+
if hasattr(llama, 'input_ids'):
4889+
llama.input_ids.fill(0)
4890+
4891+
if self.verbose:
4892+
print(f"{self.log_prefix}(enable_thinking={self.enable_thinking}) - Start processing")
4893+
4894+
# Use parent implementation
4895+
return super().__call__(**kwargs)
47084896

47094897
@register_chat_completion_handler("chatml-function-calling")
47104898
def chatml_function_calling(

0 commit comments

Comments
 (0)