import { tileImage, scanTiles, saveScanImage } from "./tiling"; import { parseJsonItems, deduplicateItems } from "./parsing"; import { matchToKnownProducts } from "./matching"; import { geminiIdentifyProducts } from "./gemini"; const PANTRY_PROMPT = `Identify ALL food and grocery products visible in this photo of a pantry, fridge, or kitchen. For each product, extract: - Product name (as shown on the label) - Brand (if visible) - Category (produce/dairy/meat/seafood/bakery/snacks/beverages/frozen/pantry/condiments/other) - Approximate quantity (e.g. "2 cans", "1 bottle", "half gallon") Return ONLY a JSON array of objects with keys: "name", "brand", "category", "quantity_desc" Example: [{"name": "Greek Yogurt", "brand": "Fage", "category": "dairy", "quantity_desc": "2 containers"}] Return ONLY the JSON array, no other text.`; const MIN_LOCAL_RESULTS = 2; interface PantryItem extends Record { name: string; brand: string; category: string; quantity_desc: string; } function cleanPantryItems(items: Record[]): PantryItem[] { const cleaned: PantryItem[] = []; for (const item of items) { if (typeof item !== "object" || item === null) continue; const name = String(item.name || "").trim(); if (!name) continue; cleaned.push({ name, brand: String(item.brand || "").trim(), category: String(item.category || "other").trim().toLowerCase(), quantity_desc: String(item.quantity_desc || "1").trim(), }); } return cleaned; } export async function scanPantryPhoto( imageBuffer: Buffer ): Promise> { const { scanId } = saveScanImage(imageBuffer, "pantry"); let modelUsed = "local"; // Step 1: Tile and scan with local model const tiles = await tileImage(imageBuffer); const rawTexts = await scanTiles(tiles, PANTRY_PROMPT); let items: Record[] = cleanPantryItems(parseJsonItems(rawTexts)); items = deduplicateItems(items); // Step 2: Gemini fallback if too few results if (items.length < MIN_LOCAL_RESULTS) { console.log(`Local model found ${items.length} pantry items, falling back to Gemini`); try { const fullBase64 = imageBuffer.toString("base64"); const geminiItems = await geminiIdentifyProducts(fullBase64); const normalized: PantryItem[] = geminiItems.map((gi) => ({ name: String(gi.name || ""), brand: String(gi.brand || ""), category: String(gi.category || "other"), quantity_desc: "1", })); if (normalized.length > items.length) { items = normalized; modelUsed = "gemini"; } } catch (err) { console.error("Gemini fallback failed for pantry scan:", err instanceof Error ? err.message : err); } } // Step 3: Match against known products (ChromaDB only) const matched = await matchToKnownProducts(items); // Add index const indexed = matched.map((item, i) => ({ ...item, index: i })); console.log(`Pantry scan ${scanId}: ${indexed.length} products via ${modelUsed}`); return { scan_id: scanId, items: indexed, total_found: indexed.length, model_used: modelUsed, }; }