Moderation API؛ تشخیص محتوای ناامن با omni-moderation و Llama Guard

curl https://1ai.ir/v1/moderations \
  -H "Authorization: Bearer $ONEAI_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{ "model": "openai/omni-moderation-latest", "input": ["متن اول کاربر", "متن دوم کاربر"] }'

# pip install openai
import os
from openai import OpenAI

client = OpenAI(base_url="https://1ai.ir/v1", api_key=os.environ["ONEAI_API_KEY"])

result = client.moderations.create(
    model="openai/omni-moderation-latest",
    input=[
        {"type": "text", "text": "Caption of the uploaded photo"},
        {"type": "image_url", "image_url": {"url": "https://example.com/photo.jpg"}},
    ],
)
print(result.results[0].flagged, result.results[0].categories)

import OpenAI from "openai";

const client = new OpenAI({ baseURL: "https://1ai.ir/v1", apiKey: process.env.ONEAI_API_KEY });
const { results } = await client.moderations.create({ model: "meta-llama/llama-guard-3-8b", input: userMessage });
if (results[0].flagged) console.log("blocked:", results[0].guard.codes);

{
  "id": "modr-…",
  "model": "openai/omni-moderation-latest",
  "results": [
    {
      "flagged": true,
      "categories": { "violence": true, "harassment": false, "hate": false, "self-harm": false, "sexual": false, "illicit": false },
      "category_scores": { "violence": 0.92, "harassment": 0.012, "hate": 0.003 },
      "category_applied_input_types": { "violence": ["text"] }
    }
  ]
}

{ "flagged": true, "categories": { "illicit/violent": true, "violence": true }, "category_scores": { "violence": 1 }, "guard": { "verdict": "unsafe", "codes": ["S9"] } }