Phi-3.5-vision-instruct-q4f16_1-MLC

[raulgupta]

[raulgupta]

`// WebLLM Import
import * as webllm from "https://esm.run/@mlc-ai/web-llm";
window.webllm = webllm;

// Engine Initialization
const initializeEngine = async () => {
    try {
        setInitStatus("Initializing WebLLM engine...");
        const webllm = window.webllm;
        const newEngine = await webllm.CreateMLCEngine(
            "Phi-3.5-vision-instruct-q4f16_1-MLC",
            {
                initProgressCallback: (report) => setInitStatus(report.text),
                logLevel: "INFO"
            },
            { context_window_size: 6144 }
        );
        setEngine(newEngine);
        setInitStatus("Engine ready!");
    } catch (error) {
        setInitStatus("Failed to initialize engine: " + error.message);
        console.error("Engine initialization error:", error);
    }
};

// Image Analysis Function
const handleAnalyze = async () => {
    if (!engine) {
        setAnalysis("Engine not initialized. Please wait...");
        return;
    }

    try {
        setIsAnalyzing(true);
        setAnalysis("Converting image...");
        
        const base64Image = await fileToBase64(file);
        
        setAnalysis("Analyzing image...");
        const request = {
            stream: false,
            messages: [{
                role: "user",
                content: [
                    { type: "text", text: "What's in this image? Describe it in detail." },
                    {
                        type: "image_url",
                        image_url: { url: base64Image }
                    }
                ]
            }]
        };

        const reply = await engine.chat.completions.create(request);
        const message = await engine.getMessage();
        setAnalysis(message);
    } catch (error) {
        setAnalysis("Error analyzing image: " + error.message);
        console.error("Analysis error:", error);
    } finally {
        setIsAnalyzing(false);
    }
};

// WebGPU Support Check
async function checkSystem() {
    if ('gpu' in navigator) {
        const adapter = await navigator.gpu.requestAdapter();
        if (adapter) {
            gpuInfoEl.textContent = adapter.name || 'WebGPU Ready';
            gpuDetailEl.textContent = 'WebGPU Supported';
        } else {
            gpuInfoEl.textContent = 'No WebGPU Adapter';
            gpuDetailEl.textContent = 'Hardware acceleration unavailable';
        }
    } else {
        gpuInfoEl.textContent = 'WebGPU Not Supported';
        gpuDetailEl.textContent = 'Browser does not support WebGPU';
    }
}`

[djaffer]

yes it is not working.

[numinousmuses]

Getting this error too, not sure what's the cause

[jakobhoeg]

Any progress on this issue? The error is also present in the examples/vision-model example.

[djaffer]

Once HuggingFace has its transformer js documentation improved that might be a viable alternative and simpler to use. They support using quantized models too.

[gornostay25]

Any updates?

[InfTkm]

Any updates?