G9v3-3B

pip install "vllm>=0.21"
vllm serve ai9stars/G9v3-3B --port 8000

curl http://localhost:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "ai9stars/G9v3-3B",
    "messages": [{"role": "user", "content": "Who are you?"}],
    "max_tokens": 128,
    "temperature": 0.7
  }'

pip install "sglang[srt]>=0.5.12"
python -m sglang.launch_server --model-path ai9stars/G9v3-3B --port 30000

curl http://localhost:30000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "ai9stars/G9v3-3B",
    "messages": [{"role": "user", "content": "Who are you?"}],
    "max_tokens": 128,
    "temperature": 0.7
  }'

pip install -U "transformers>=5.6" accelerate torch

from transformers import AutoModelForCausalLM, AutoTokenizer

model_id = "ai9stars/G9v3-3B"
tokenizer = AutoTokenizer.from_pretrained(model_id)
model = AutoModelForCausalLM.from_pretrained(
    model_id,
    torch_dtype="auto",
    device_map="auto",
)

messages = [{"role": "user", "content": "Who are you?"}]
inputs = tokenizer.apply_chat_template(
    messages,
    tokenize=True,
    add_generation_prompt=True,
    enable_thinking=False,
    return_dict=True,
    return_tensors="pt",
).to(model.device)

outputs = model.generate(**inputs, max_new_tokens=128)
print(tokenizer.decode(outputs[0][inputs["input_ids"].shape[-1]:], skip_special_tokens=True))