Initial commit: Image moderation service using local vision LLM
This commit is contained in:
218
src/moderate.ts
Normal file
218
src/moderate.ts
Normal file
@@ -0,0 +1,218 @@
|
||||
import sharp from "sharp";
|
||||
|
||||
// --- Config ---
|
||||
|
||||
const VISION_AI_URL =
|
||||
process.env.VISION_AI_URL || "http://localhost:8000/v1/chat/completions";
|
||||
const VISION_AI_MODEL = process.env.VISION_AI_MODEL || "gemma3:4b";
|
||||
const VISION_AI_TIMEOUT = parseInt(
|
||||
process.env.VISION_AI_TIMEOUT || "30000",
|
||||
10
|
||||
);
|
||||
const TILE_THRESHOLD = parseInt(process.env.TILE_THRESHOLD || "1024", 10);
|
||||
|
||||
// --- Types ---
|
||||
|
||||
export interface ModerationResult {
|
||||
safe: boolean;
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
// --- Prompts ---
|
||||
|
||||
const PROMPTS: Record<string, string> = {
|
||||
general: `You are a content moderation system. Analyze this image for safety.
|
||||
|
||||
Check for:
|
||||
- Nudity or sexually explicit content
|
||||
- Graphic violence or gore
|
||||
- Hate symbols or extremist imagery
|
||||
- Drug use or paraphernalia
|
||||
|
||||
Respond with exactly one word on the first line: SAFE or UNSAFE
|
||||
If UNSAFE, add a brief reason on the second line.`,
|
||||
|
||||
church: `You are a content moderation system for Catholic parish websites. Analyze this image for safety.
|
||||
|
||||
Check for:
|
||||
- Nudity or sexually explicit content
|
||||
- Graphic violence or gore
|
||||
- Hate symbols or extremist imagery
|
||||
- Drug use or paraphernalia
|
||||
|
||||
Note: Religious imagery (crucifixes, saints, stained glass, statues, liturgical art) is SAFE and expected. Depictions of the crucifixion or martyrdom in a religious context are SAFE.
|
||||
|
||||
Respond with exactly one word on the first line: SAFE or UNSAFE
|
||||
If UNSAFE, add a brief reason on the second line.`,
|
||||
};
|
||||
|
||||
// --- Vision AI call ---
|
||||
|
||||
async function callVisionAI(
|
||||
imageBase64: string,
|
||||
mimeType: string,
|
||||
context: string
|
||||
): Promise<string> {
|
||||
const prompt = PROMPTS[context] || PROMPTS.general;
|
||||
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), VISION_AI_TIMEOUT);
|
||||
|
||||
try {
|
||||
const response = await fetch(VISION_AI_URL, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
signal: controller.signal,
|
||||
body: JSON.stringify({
|
||||
model: VISION_AI_MODEL,
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: prompt },
|
||||
{
|
||||
type: "image_url",
|
||||
image_url: {
|
||||
url: `data:${mimeType};base64,${imageBase64}`,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
max_tokens: 100,
|
||||
temperature: 0.1,
|
||||
}),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Vision AI returned ${response.status}`);
|
||||
}
|
||||
|
||||
const data = (await response.json()) as {
|
||||
choices?: { message?: { content?: string } }[];
|
||||
};
|
||||
return data.choices?.[0]?.message?.content?.trim() || "";
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
function parseVerdict(response: string): ModerationResult {
|
||||
const firstLine = response.split("\n")[0].trim().toUpperCase();
|
||||
|
||||
if (firstLine.startsWith("UNSAFE")) {
|
||||
const lines = response.split("\n");
|
||||
const reason = lines
|
||||
.slice(1)
|
||||
.map((l: string) => l.trim())
|
||||
.filter(Boolean)
|
||||
.join(" ");
|
||||
return { safe: false, reason: reason || "Content flagged as unsafe" };
|
||||
}
|
||||
|
||||
// Only explicit UNSAFE triggers rejection — everything else (SAFE, garbled, empty) passes
|
||||
return { safe: true };
|
||||
}
|
||||
|
||||
// --- Tile slicing ---
|
||||
|
||||
interface TileBuffer {
|
||||
buffer: Buffer;
|
||||
mimeType: string;
|
||||
}
|
||||
|
||||
async function sliceIntoTiles(
|
||||
imageBuffer: Buffer,
|
||||
width: number,
|
||||
height: number
|
||||
): Promise<TileBuffer[]> {
|
||||
const gridSize = width > 2048 || height > 2048 ? 3 : 2;
|
||||
const tileW = Math.floor(width / gridSize);
|
||||
const tileH = Math.floor(height / gridSize);
|
||||
const targetSize = 512;
|
||||
|
||||
const tiles: Promise<TileBuffer>[] = [];
|
||||
|
||||
for (let row = 0; row < gridSize; row++) {
|
||||
for (let col = 0; col < gridSize; col++) {
|
||||
const left = col * tileW;
|
||||
const top = row * tileH;
|
||||
// Last tile in row/col extends to edge to avoid missing pixels
|
||||
const extractW = col === gridSize - 1 ? width - left : tileW;
|
||||
const extractH = row === gridSize - 1 ? height - top : tileH;
|
||||
|
||||
tiles.push(
|
||||
sharp(imageBuffer)
|
||||
.extract({ left, top, width: extractW, height: extractH })
|
||||
.resize(targetSize, targetSize, { fit: "inside" })
|
||||
.jpeg({ quality: 80 })
|
||||
.toBuffer()
|
||||
.then((buffer) => ({ buffer, mimeType: "image/jpeg" }))
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return Promise.all(tiles);
|
||||
}
|
||||
|
||||
// --- Main moderation function ---
|
||||
|
||||
export async function moderateImage(
|
||||
imageBuffer: Buffer,
|
||||
context: string = "general"
|
||||
): Promise<ModerationResult> {
|
||||
try {
|
||||
const metadata = await sharp(imageBuffer).metadata();
|
||||
const width = metadata.width || 0;
|
||||
const height = metadata.height || 0;
|
||||
|
||||
if (width === 0 || height === 0) {
|
||||
console.warn("Could not read image dimensions — allowing image");
|
||||
return { safe: true };
|
||||
}
|
||||
|
||||
const isLarge = width > TILE_THRESHOLD || height > TILE_THRESHOLD;
|
||||
|
||||
// Always analyze the full image (resized to fit within 1024px)
|
||||
const fullImageBuffer = await sharp(imageBuffer)
|
||||
.resize(1024, 1024, { fit: "inside" })
|
||||
.jpeg({ quality: 85 })
|
||||
.toBuffer();
|
||||
const fullBase64 = fullImageBuffer.toString("base64");
|
||||
|
||||
if (!isLarge) {
|
||||
// Small image: just analyze the full image
|
||||
const response = await callVisionAI(fullBase64, "image/jpeg", context);
|
||||
return parseVerdict(response);
|
||||
}
|
||||
|
||||
// Large image: full image + tiles in parallel
|
||||
const tiles = await sliceIntoTiles(imageBuffer, width, height);
|
||||
|
||||
const allPromises = [
|
||||
callVisionAI(fullBase64, "image/jpeg", context),
|
||||
...tiles.map((tile) =>
|
||||
callVisionAI(tile.buffer.toString("base64"), tile.mimeType, context)
|
||||
),
|
||||
];
|
||||
|
||||
const results = await Promise.all(allPromises);
|
||||
|
||||
// If ANY result is UNSAFE, reject the image
|
||||
for (const response of results) {
|
||||
const verdict = parseVerdict(response);
|
||||
if (!verdict.safe) {
|
||||
return verdict;
|
||||
}
|
||||
}
|
||||
|
||||
return { safe: true };
|
||||
} catch (error) {
|
||||
console.error(
|
||||
"Moderation error:",
|
||||
error instanceof Error ? error.message : error
|
||||
);
|
||||
// Fail open — allow the image if vision AI is unavailable
|
||||
return { safe: true };
|
||||
}
|
||||
}
|
||||
180
src/server.ts
Normal file
180
src/server.ts
Normal file
@@ -0,0 +1,180 @@
|
||||
import "dotenv/config";
|
||||
import express, { Request, Response, NextFunction } from "express";
|
||||
import multer from "multer";
|
||||
import { moderateImage } from "./moderate";
|
||||
|
||||
const PORT = parseInt(process.env.PORT || "8100", 10);
|
||||
const MAX_FILE_SIZE = 20 * 1024 * 1024; // 20MB
|
||||
|
||||
// Callback config — where to report unsafe images
|
||||
const UPLOAD_SERVICE_REPLACE_URL =
|
||||
process.env.UPLOAD_SERVICE_REPLACE_URL || "";
|
||||
const UPLOAD_SERVICE_SECRET = process.env.UPLOAD_SERVICE_SECRET || "";
|
||||
const PAROCHIA_CALLBACK_URL = process.env.PAROCHIA_CALLBACK_URL || "";
|
||||
const PAROCHIA_CALLBACK_SECRET = process.env.PAROCHIA_CALLBACK_SECRET || "";
|
||||
|
||||
const ALLOWED_MIMES = new Set([
|
||||
"image/jpeg",
|
||||
"image/png",
|
||||
"image/webp",
|
||||
"image/gif",
|
||||
]);
|
||||
|
||||
const app = express();
|
||||
|
||||
const upload = multer({
|
||||
storage: multer.memoryStorage(),
|
||||
limits: {
|
||||
fileSize: MAX_FILE_SIZE,
|
||||
files: 1,
|
||||
},
|
||||
fileFilter: (_req, file, cb) => {
|
||||
if (ALLOWED_MIMES.has(file.mimetype)) {
|
||||
cb(null, true);
|
||||
} else {
|
||||
cb(new Error("Invalid file type. Allowed: JPEG, PNG, WebP, GIF"));
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
// --- Callback helpers ---
|
||||
|
||||
async function notifyUnsafe(
|
||||
imagePath: string,
|
||||
userId: string,
|
||||
siteId: string,
|
||||
reason: string
|
||||
): Promise<void> {
|
||||
// Tell upload service to replace image with stub
|
||||
if (UPLOAD_SERVICE_REPLACE_URL && UPLOAD_SERVICE_SECRET) {
|
||||
try {
|
||||
const res = await fetch(UPLOAD_SERVICE_REPLACE_URL, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
"x-internal-secret": UPLOAD_SERVICE_SECRET,
|
||||
},
|
||||
body: JSON.stringify({ imagePath }),
|
||||
signal: AbortSignal.timeout(10000),
|
||||
});
|
||||
if (!res.ok) {
|
||||
console.error(`Replace callback failed: ${res.status}`);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(
|
||||
"Replace callback error:",
|
||||
err instanceof Error ? err.message : err
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Tell Parochia to flag the user
|
||||
if (PAROCHIA_CALLBACK_URL && PAROCHIA_CALLBACK_SECRET) {
|
||||
try {
|
||||
const res = await fetch(PAROCHIA_CALLBACK_URL, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
"x-callback-secret": PAROCHIA_CALLBACK_SECRET,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
userId,
|
||||
siteId,
|
||||
reason: "explicit_image",
|
||||
details: reason,
|
||||
}),
|
||||
signal: AbortSignal.timeout(10000),
|
||||
});
|
||||
if (!res.ok) {
|
||||
console.error(`Parochia callback failed: ${res.status}`);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(
|
||||
"Parochia callback error:",
|
||||
err instanceof Error ? err.message : err
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Routes ---
|
||||
|
||||
// Health check
|
||||
app.get("/health", (_req: Request, res: Response) => {
|
||||
res.json({ status: "ok", timestamp: new Date().toISOString() });
|
||||
});
|
||||
|
||||
// Moderation endpoint
|
||||
// Accepts: file (image), context, imagePath, userId, siteId
|
||||
// If called with metadata fields, will fire callbacks on UNSAFE result
|
||||
app.post(
|
||||
"/moderate",
|
||||
upload.single("file"),
|
||||
async (req: Request, res: Response, next: NextFunction) => {
|
||||
try {
|
||||
const file = req.file;
|
||||
if (!file) {
|
||||
res.status(400).json({ error: "No file provided" });
|
||||
return;
|
||||
}
|
||||
|
||||
const context =
|
||||
typeof req.body?.context === "string" ? req.body.context : "general";
|
||||
|
||||
const result = await moderateImage(file.buffer, context);
|
||||
|
||||
// If unsafe and metadata was provided, fire callbacks
|
||||
if (!result.safe && req.body?.imagePath) {
|
||||
const { imagePath, userId, siteId } = req.body;
|
||||
// Fire-and-forget — don't block the response
|
||||
notifyUnsafe(
|
||||
imagePath,
|
||||
userId || "",
|
||||
siteId || "",
|
||||
result.reason || "Content flagged as unsafe"
|
||||
).catch((err) =>
|
||||
console.error("Callback dispatch error:", err)
|
||||
);
|
||||
}
|
||||
|
||||
res.json(result);
|
||||
} catch (err) {
|
||||
next(err);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
// 404 catch-all
|
||||
app.use((_req: Request, res: Response) => {
|
||||
res.status(404).json({ error: "Not found" });
|
||||
});
|
||||
|
||||
// Error handler
|
||||
app.use((err: Error, _req: Request, res: Response, _next: NextFunction) => {
|
||||
if (
|
||||
err instanceof multer.MulterError &&
|
||||
err.code === "LIMIT_FILE_SIZE"
|
||||
) {
|
||||
res.status(413).json({ error: "File too large. Maximum size is 20MB." });
|
||||
return;
|
||||
}
|
||||
|
||||
if (err.message.startsWith("Invalid file type")) {
|
||||
res.status(400).json({ error: err.message });
|
||||
return;
|
||||
}
|
||||
|
||||
console.error("Internal error:", err.message);
|
||||
res.status(500).json({ error: "Internal server error" });
|
||||
});
|
||||
|
||||
app.listen(PORT, "0.0.0.0", () => {
|
||||
console.log(`Image moderation service listening on port ${PORT}`);
|
||||
console.log(
|
||||
`Vision AI: ${process.env.VISION_AI_URL || "http://localhost:8000/v1/chat/completions"}`
|
||||
);
|
||||
console.log(`Model: ${process.env.VISION_AI_MODEL || "gemma3:4b"}`);
|
||||
console.log(
|
||||
`Callbacks: replace=${UPLOAD_SERVICE_REPLACE_URL ? "configured" : "none"}, parochia=${PAROCHIA_CALLBACK_URL ? "configured" : "none"}`
|
||||
);
|
||||
});
|
||||
Reference in New Issue
Block a user