not run
from transformers import AutoProcessor, Glm4vForConditionalGeneration
import torch
MODEL_PATH = "zai-org/GLM-4.6V-Flash"
messages = [
{
"role": "user",
"content": [
{
"type": "image",
"url": "https://upload.wikimedia.org/wikipedia/commons/f/fa/Grayscale_8bits_palette_sample_image.png"
},
{
"type": "text",
"text": "describe this image"
}
],
}
]
processor = AutoProcessor.from_pretrained(MODEL_PATH)
model = Glm4vForConditionalGeneration.from_pretrained(
pretrained_model_name_or_path=MODEL_PATH,
torch_dtype="auto",
device_map="auto",
)
inputs = processor.apply_chat_template(
messages,
tokenize=True,
add_generation_prompt=True,
return_dict=True,
return_tensors="pt"
).to(model.device)
inputs.pop("token_type_ids", None)
generated_ids = model.generate(**inputs, max_new_tokens=55)
output_text = processor.decode(generated_ids[0][inputs["input_ids"].shape[1]:], skip_special_tokens=False)
print(output_text)
torch_dtype is deprecated! Use dtype instead!
Loading checkpoint shards: 100%
4/4 [00:51<00:00, 10.35s/it]
WARNING:accelerate.big_modeling:Some parameters are on the meta device because they were offloaded to the cpu.
TypeError Traceback (most recent call last)
/usr/local/lib/python3.12/dist-packages/transformers/utils/generic.py in wrapper(self, *args, **kwargs)
1078 try:
-> 1079 outputs = func(self, *args, **kwargs_without_recordable)
1080 except TypeError:
36 frames
TypeError: 'NoneType' object is not subscriptable
During handling of the above exception, another exception occurred:
TypeError Traceback (most recent call last)
/usr/local/lib/python3.12/dist-packages/transformers/models/glm4v/modeling_glm4v.py in forward(self, hidden_states, position_embeddings, attention_mask, position_ids, past_key_values, cache_position, **kwargs)
536 cos, sin = position_embeddings
537 query_states, key_states = apply_multimodal_rotary_pos_emb( # diff with Llama
--> 538 query_states, key_states, cos, sin, self.rope_scaling["mrope_section"]
539 )
540
TypeError: 'NoneType' object is not subscriptable