Florence-2-large

npm i @huggingface/transformers

import {
    Florence2ForConditionalGeneration,
    AutoProcessor,
    load_image,
} from '@huggingface/transformers';

// Load model, processor, and tokenizer
const model_id = 'onnx-community/Florence-2-large';
const model = await Florence2ForConditionalGeneration.from_pretrained(model_id, {
    dtype: {
        embed_tokens: 'fp16', // or 'fp32'
        vision_encoder: 'fp16', // or 'fp32'
        encoder_model: 'q4',
        decoder_model_merged: 'q4',
    },
});
const processor = await AutoProcessor.from_pretrained(model_id);

// Load image and prepare vision inputs
const url = 'https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/car.jpg';
const image = await load_image(url);

// Specify task and prepare text inputs
const task = '<MORE_DETAILED_CAPTION>';
const prompts = processor.construct_prompts(task);

// Pre-process the image and text inputs
const inputs = await processor(image, prompts);

// Generate text
const generated_ids = await model.generate({
    ...inputs,
    max_new_tokens: 256,
});

// Decode generated text
const generated_text = processor.batch_decode(generated_ids, { skip_special_tokens: false })[0];

// Post-process the generated text
const result = processor.post_process_generation(generated_text, task, image.size);
console.log(result);
// { '<MORE_DETAILED_CAPTION>': 'The image shows a vintage Volkswagen Beetle car parked on a cobblestone street in front of a yellow building with two wooden doors. The car is a bright turquoise color and has a classic design with a round body and a sloping roofline. It has two doors on either side of the car, one on the left side and one in the center, with a brown door on the right side. The doors are made of wood and have a rustic, weathered look. The building behind the car is painted in a light yellow color and appears to be old and dilapidated. The sky is blue and there are trees in the background. The image is taken from a low angle, looking up at the car and the building.' }