pixano_inference.providers.vllm

Provider for vLLM models.

VLLMProvider(*args, **kwargs)

Bases: ModelProvider

Provider for vLLM models.

Source code in pixano_inference/providers/vllm.py

def __init__(self, *args: Any, **kwargs: Any) -> None:
    """Initialize the vLLM provider."""
    assert_vllm_installed()
    super().__init__(*args, **kwargs)

load_model(name, task, device, path=None, processor_config={}, config={})

Load a model from vLLM.

Parameters:

Name	Type	Description	Default
name	str	Name of the model.	required
task	Task | str	Task of the model.	required
device	device	Device to use for the model.	required
path	str | None	Path to the model or its Hugging Face hub's identifier.	None
processor_config	dict	Configuration for the processor.	{}
config	dict	Configuration for the model.	{}

Returns:

Type	Description
VLLMModel	Loaded model.

Source code in pixano_inference/providers/vllm.py

def load_model(
    self,
    name: str,
    task: Task | str,
    device: "torch.device",
    path: str | None = None,  # type: ignore[override]
    processor_config: dict = {},
    config: dict = {},
) -> VLLMModel:
    """Load a model from vLLM.

    Args:
        name: Name of the model.
        task: Task of the model.
        device: Device to use for the model.
        path: Path to the model or its Hugging Face hub's identifier.
        processor_config: Configuration for the processor.
        config: Configuration for the model.

    Returns:
        Loaded model.
    """
    if path is None:
        raise ValueError("Path is required to load a model from vLLm.")
    if isinstance(task, str):
        task = str_to_task(task)

    our_model = VLLMModel(
        name=name, vllm_model=path, model_config=config, processor_config=processor_config, device=device
    )

    return our_model

text_image_conditional_generation(request, model, *args, **kwargs)

Generate text from an image and a prompt.

Parameters:

Name	Type	Description	Default
request	TextImageConditionalGenerationRequest	Request for text-image conditional generation.	required
model	VLLMModel	Model for text-image conditional generation	required
args	Any	Additional arguments.	()
kwargs	Any	Additional keyword arguments.	{}

Returns:

Type	Description
TextImageConditionalGenerationOutput	Output of text-image conditional generation.

Source code in pixano_inference/providers/vllm.py

def text_image_conditional_generation(
    self,
    request: TextImageConditionalGenerationRequest,
    model: VLLMModel,  # type: ignore[override]
    *args: Any,
    **kwargs: Any,
) -> TextImageConditionalGenerationOutput:
    """Generate text from an image and a prompt.

    Args:
        request: Request for text-image conditional generation.
        model: Model for text-image conditional generation
        args: Additional arguments.
        kwargs: Additional keyword arguments.

    Returns:
        Output of text-image conditional generation.
    """
    model_input = request.to_input()
    if model_input.images is not None:
        raise ValueError("images should be passed in the prompt for vLLM.")
    if isinstance(model_input.prompt, str):
        raise ValueError("Pixano-inference only support a chat template for vLLM.")

    output = model.text_image_conditional_generation(**model_input.model_dump(exclude="images"))
    return output