conda create --name vll-Janus-Pro-7B python=3.12.7

conda activate vll-Janus-Pro-7B

nvcc: NVIDIA (R) Cuda compiler driver
Copyright (c) 2005-2025 NVIDIA Corporation
Built on Wed_Jan_15_19:20:09_PST_2025
Cuda compilation tools, release 12.8, V12.8.61
Build cuda_12.8.r12.8/compiler.35404655_0

mkdir vllm
cd vllm

git clone https://github.com/deepseek-ai/Janus

pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu128

pip3 install transformers attrdict einops timm

pip install modelscope
modelscope download --model deepseek-ai/Janus-Pro-7B

Downloading Model from https://www.modelscope.cn to directory: /root/.cache/modelscope/hub/models/deepseek-ai/Janus-Pro-7B
Downloading [config.json]: 100%|███████████████████████████████████████████████████| 1.42k/1.42k [00:00<00:00, 5.29kB/s]
Downloading [configuration.json]: 100%|████████████████████████████████████████████████| 68.0/68.0 [00:00<00:00, 221B/s]
Downloading [README.md]: 100%|█████████████████████████████████████████████████████| 2.49k/2.49k [00:00<00:00, 7.20kB/s]
Downloading [processor_config.json]: 100%|███████████████████████████████████████████████| 210/210 [00:00<00:00, 590B/s]
Downloading [janus_pro_teaser1.png]: 100%|██████████████████████████████████████████| 95.7k/95.7k [00:00<00:00, 267kB/s]
Downloading [preprocessor_config.json]: 100%|████████████████████████████████████████████| 346/346 [00:00<00:00, 867B/s]
Downloading [janus_pro_teaser2.png]: 100%|███████████████████████████████████████████| 518k/518k [00:00<00:00, 1.18MB/s]
Downloading [special_tokens_map.json]: 100%|███████████████████████████████████████████| 344/344 [00:00<00:00, 1.50kB/s]
Downloading [tokenizer_config.json]: 100%|███████████████████████████████████████████████| 285/285 [00:00<00:00, 926B/s]
Downloading [pytorch_model.bin]:   0%|▏                                            | 16.0M/3.89G [00:00<03:55, 17.7MB/s]
Downloading [tokenizer.json]: 100%|████████████████████████████████████████████████| 4.50M/4.50M [00:00<00:00, 6.55MB/s]
Processing 11 items:  91%|█████████████████████████████████████████████████████▋     | 10.0/11.0 [00:19<00:00, 14.1it/s]
Downloading [pytorch_model.bin]: 100%|█████████████████████████████████████████████| 3.89G/3.89G [09:18<00:00, 7.48MB/s]
Processing 11 items: 100%|███████████████████████████████████████████████████████████| 11.0/11.0 [09:24<00:00, 51.3s/it]

mv /root/.cache/modelscope/hub/models/deepseek-ai /home/xiao/vllm

total 20
drwxr-xr-x 4 root root 4096 May  8 18:59 ./
drwxr-x--- 5 xiao xiao 4096 May  8 14:50 ../
drwxr-xr-x 8 root root 4096 May  8 18:59 Janus/
drwxr-xr-x 4 root root 4096 May  8 16:01 deepseek-ai/

# ll
total 16
drwxr-xr-x 4 root root 4096 May  8 16:01 ./
drwxr-xr-x 4 root root 4096 May  8 18:59 ../
drwxr-xr-x 2 root root 4096 May  7 18:32 Janus-Pro-7B/

from transformers import AutoModelForCausalLM
from janus.models import MultiModalityCausalLM, VLChatProcessor
from janus.utils.io import load_pil_images
model_path = "../deepseek-ai/Janus-Pro-7B"
image='aa.jpeg'
question='請(qǐng)說(shuō)明一下這張圖片'
vl_chat_processor: VLChatProcessor = VLChatProcessor.from_pretrained(model_path)
tokenizer = vl_chat_processor.tokenizer
vl_gpt: MultiModalityCausalLM = AutoModelForCausalLM.from_pretrained(
    model_path, trust_remote_code=True
)
vl_gpt = vl_gpt.to(torch.bfloat16).cuda().eval()
conversation = [
    {
        "role": "<|User|>",
        "content": f"<image_placeholder>\n{question}",
        "images": [image],
    },
    {"role": "<|Assistant|>", "content": ""},
]
# load images and prepare for inputs
pil_images = load_pil_images(conversation)
prepare_inputs = vl_chat_processor(
    conversations=conversation, images=pil_images, force_batchify=True
).to(vl_gpt.device)
# # run image encoder to get the image embeddings
inputs_embeds = vl_gpt.prepare_inputs_embeds(**prepare_inputs)
# # run the model to get the response
outputs = vl_gpt.language_model.generate(
    inputs_embeds=inputs_embeds,
    attention_mask=prepare_inputs.attention_mask,
    pad_token_id=tokenizer.eos_token_id,
    bos_token_id=tokenizer.bos_token_id,
    eos_token_id=tokenizer.eos_token_id,
    max_new_tokens=512,
    do_sample=False,
    use_cache=True,
)
answer = tokenizer.decode(outputs[0].cpu().tolist(), skip_special_tokens=True)
print(f"{prepare_inputs['sft_format'][0]}", answer)

total 2976
drwxr-xr-x 8 root root    4096 May  8 18:59 ./
drwxr-xr-x 4 root root    4096 May  8 18:59 ../
drwxr-xr-x 8 root root    4096 May  7 18:11 .git/
-rw-r--r-- 1 root root     115 May  7 18:11 .gitattributes
-rw-r--r-- 1 root root    7301 May  7 18:11 .gitignore
-rw-r--r-- 1 root root    1065 May  7 18:11 LICENSE-CODE
-rw-r--r-- 1 root root   13718 May  7 18:11 LICENSE-MODEL
-rw-r--r-- 1 root root    3069 May  7 18:11 Makefile
-rwxr-xr-x 1 root root   26781 May  7 18:11 README.md*
-rw-r--r-- 1 root root   62816 May  8 14:59 aa.jpeg
drwxr-xr-x 2 root root    4096 May  7 18:11 demo/
drwxr-xr-x 2 root root    4096 May  8 17:19 generated_samples/
-rw-r--r-- 1 root root    4515 May  7 18:11 generation_inference.py
-rw-r--r-- 1 xiao xiao    4066 May  8 18:50 image_generation.py
-rw-r--r-- 1 root root    1594 May  8 18:58 image_understanding.py
drwxr-xr-x 2 root root    4096 May  7 18:11 images/
-rw-r--r-- 1 root root    2642 May  7 18:11 inference.py
-rw-r--r-- 1 root root    5188 May  7 18:11 interactivechat.py
drwxr-xr-x 6 root root    4096 May  7 19:01 janus/
drwxr-xr-x 2 root root    4096 May  7 18:11 janus.egg-info/
-rw-r--r-- 1 root root 2846268 May  7 18:11 janus_pro_tech_report.pdf
-rw-r--r-- 1 root root    1111 May  7 18:11 pyproject.toml
-rw-r--r-- 1 root root     278 May  7 18:11 requirements.txt

# python image_understanding.py
Python version is above 3.10, patching the collections module.
/root/anaconda3/envs/vll-Janus-Pro-7B/lib/python3.12/site-packages/transformers/models/auto/image_processing_auto.py:604: FutureWarning: The image_processor_class argument is deprecated and will be removed in v4.42. Please use `slow_image_processor_class`, or `fast_image_processor_class` instead
  warnings.warn(
Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.
You are using the default legacy behaviour of the <class 'transformers.models.llama.tokenization_llama_fast.LlamaTokenizerFast'>. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565 - if you loaded a llama tokenizer from a GGUF file you can ignore this message.
Loading checkpoint shards: 100%|██████████████████████████████████████████████████████████| 2/2 [00:10<00:00,  5.18s/it]
You are a helpful language and vision assistant. You are able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language.
<|User|>: <image_placeholder>
請(qǐng)說(shuō)明一下這張圖片
<|Assistant|>: 這張圖片展示了一位身穿傳統(tǒng)服飾的女性，她正坐在戶(hù)外，雙手合十，閉著眼睛，似乎在進(jìn)行冥想或祈禱。背景是綠色的樹(shù)木和植物，陽(yáng)光透過(guò)樹(shù)葉灑在她的身上，營(yíng)造出一種寧?kù)o、祥和的氛圍。她的服裝以淡雅的白色和粉色為主，帶有精致的花紋，整體風(fēng)格非常優(yōu)雅。

import torch
import numpy as np
from PIL import Image
from transformers import AutoModelForCausalLM
from janus.models import MultiModalityCausalLM, VLChatProcessor
model_path = "../deepseek-ai/Janus-Pro-7B"
vl_chat_processor: VLChatProcessor = VLChatProcessor.from_pretrained(model_path)
tokenizer = vl_chat_processor.tokenizer
vl_gpt: MultiModalityCausalLM = AutoModelForCausalLM.from_pretrained(
    model_path, trust_remote_code=True
)
vl_gpt = vl_gpt.to(torch.bfloat16).cuda().eval()
conversation = [
    {"role": "<|User|>", "content": "超寫(xiě)實(shí)8K渲染，一位具有東方古典美的中國(guó)女性，瓜子臉，西昌的眉毛如彎彎的月牙，雙眼明亮而深邃，猶如夜空中閃爍的星星。高挺的鼻梁，櫻桃小嘴微微上揚(yáng)，透露出一絲誘人的微笑。她的頭發(fā)如黑色的瀑布般垂直落在減胖兩側(cè)，微風(fēng)輕輕浮動(dòng)發(fā)色。肌膚白皙如雪，在陽(yáng)光下泛著微微的光澤。她身著乙烯白色的透薄如紗的連衣裙，裙擺在海風(fēng)中輕輕飄動(dòng)。"},
    {"role": "<|Assistant|>", "content": ""},
]
sft_format = vl_chat_processor.apply_sft_template_for_multi_turn_prompts(
    conversations=conversation,
    sft_format=vl_chat_processor.sft_format,
    system_prompt=""
)
prompt = sft_format + vl_chat_processor.image_start_tag
@torch.inference_mode()
def generate(
        mmgpt: MultiModalityCausalLM,
        vl_chat_processor: VLChatProcessor,
        prompt: str,
        temperature: float = 1,
        parallel_size: int = 1, # 減小 parallel_size
        cfg_weight: float = 5,
        image_token_num_per_image: int = 576,
        img_size: int = 384,
        patch_size: int = 16,
):
    input_ids = vl_chat_processor.tokenizer.encode(prompt)
    input_ids = torch.LongTensor(input_ids)
    tokens = torch.zeros((parallel_size * 2, len(input_ids)), dtype=torch.int).cuda()
    for i in range(parallel_size * 2):
        tokens[i, :] = input_ids
        if i % 2 != 0:
            tokens[i, 1:-1] = vl_chat_processor.pad_id
    inputs_embeds = mmgpt.language_model.get_input_embeddings()(tokens)
    generated_tokens = torch.zeros((parallel_size, image_token_num_per_image), dtype=torch.int).cuda()
    for i in range(image_token_num_per_image):
        outputs = mmgpt.language_model.model(inputs_embeds=inputs_embeds, use_cache=True,
                                             past_key_values=outputs.past_key_values if i != 0 else None)
        hidden_states = outputs.last_hidden_state
        logits = mmgpt.gen_head(hidden_states[:, -1, :])
        logit_cond = logits[0::2, :]
        logit_uncond = logits[1::2, :]
        logits = logit_uncond + cfg_weight * (logit_cond - logit_uncond)
        probs = torch.softmax(logits / temperature, dim=-1)
        next_token = torch.multinomial(probs, num_samples=1)
        generated_tokens[:, i] = next_token.squeeze(dim=-1)
        next_token = torch.cat([next_token.unsqueeze(dim=1),
                                next_token.unsqueeze(dim=1)], dim=1).view(-1)
        img_embeds = mmgpt.prepare_gen_img_embeds(next_token)
        inputs_embeds = img_embeds.unsqueeze(dim=1)
        # 添加顯存清理
        del logits, logit_cond, logit_uncond, probs
        torch.cuda.empty_cache()
    dec = mmgpt.gen_vision_model.decode_code(generated_tokens.to(dtype=torch.int),
                                             shape=[parallel_size, 8, img_size // patch_size, img_size // patch_size])
    dec = dec.to(torch.float32).cpu().numpy().transpose(0, 2, 3, 1)
    dec = np.clip((dec + 1) / 2 * 255, 0, 255)
    visual_img = np.zeros((parallel_size, img_size, img_size, 3), dtype=np.uint8)
    visual_img[:, :, :] = dec
    os.makedirs('generated_samples', exist_ok=True)
    for i in range(parallel_size):
        save_path = os.path.join('generated_samples', f"img_{i}.jpg")
        img = Image.fromarray(visual_img[i])
        img.save(save_path)
generate(
    vl_gpt,
    vl_chat_processor,
    prompt,
)

Python version is above 3.10, patching the collections module.
/root/anaconda3/envs/vll-Janus-Pro-7B/lib/python3.12/site-packages/transformers/models/auto/image_processing_auto.py:604: FutureWarning: The image_processor_class argument is deprecated and will be removed in v4.42. Please use `slow_image_processor_class`, or `fast_image_processor_class` instead
  warnings.warn(
Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.
You are using the default legacy behaviour of the <class 'transformers.models.llama.tokenization_llama_fast.LlamaTokenizerFast'>. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565 - if you loaded a llama tokenizer from a GGUF file you can ignore this message.
Loading checkpoint shards: 100%|██████████████████████████████████████████████████████████| 1/1 [00:09<00:00,  4.58s/it]

parallel_size: int = 3