Florence-2-large-ft

npm i @huggingface/transformers

import {
    Florence2ForConditionalGeneration,
    AutoProcessor,
    load_image,
} from '@huggingface/transformers';

// Load model, processor, and tokenizer
const model_id = 'onnx-community/Florence-2-large-ft';
const model = await Florence2ForConditionalGeneration.from_pretrained(model_id, {
    dtype: {
        embed_tokens: 'fp16', // or 'fp32'
        vision_encoder: 'fp16', // or 'fp32'
        encoder_model: 'q4',
        decoder_model_merged: 'q4',
    },
});
const processor = await AutoProcessor.from_pretrained(model_id);

// Load image and prepare vision inputs
const url = 'https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/car.jpg';
const image = await load_image(url);

// Specify task and prepare text inputs
const task = '<MORE_DETAILED_CAPTION>';
const prompts = processor.construct_prompts(task);

// Pre-process the image and text inputs
const inputs = await processor(image, prompts);

// Generate text
const generated_ids = await model.generate({
    ...inputs,
    max_new_tokens: 256,
});

// Decode generated text
const generated_text = processor.batch_decode(generated_ids, { skip_special_tokens: false })[0];

// Post-process the generated text
const result = processor.post_process_generation(generated_text, task, image.size);
console.log(result);
// { '<MORE_DETAILED_CAPTION>': 'A car is parked on the street. The car is a light green color. The doors on the building are brown. The building is a yellow color. There are two doors on both sides of the car. The wheels on the car are very shiny. The ground is made of bricks. The sky is blue. The sun is shining on the top of the building.' }