diff options
| -rw-r--r-- | packages/gateway/src/listeners/aiModeration.ts | 191 |
1 files changed, 113 insertions, 78 deletions
diff --git a/packages/gateway/src/listeners/aiModeration.ts b/packages/gateway/src/listeners/aiModeration.ts index 389f801..dc0c92b 100644 --- a/packages/gateway/src/listeners/aiModeration.ts +++ b/packages/gateway/src/listeners/aiModeration.ts @@ -1,4 +1,10 @@ -import { Client, Events, Message, TextChannel, ThreadChannel } from "discord.js"; +import { + Client, + Events, + Message, + TextChannel, + ThreadChannel, +} from "discord.js"; import { sendAuditLog } from "../commands/utilities"; const EXCLUDED_CATEGORIES = [ @@ -42,25 +48,30 @@ Regardless of which rule users have broken, if a user accumulates 15 warnings wi If you think you have received an unfair warning or punishment, please submit an appeal by using the \`/appeal\` slash command in a direct message to @UmaBot#9396. A member of the moderation team will review your appeal. `; -async function fetchMessageContext(channel: TextChannel | ThreadChannel, messageId: string): Promise<string> { +async function fetchMessageContext( + channel: TextChannel | ThreadChannel, + messageId: string, +): Promise<string> { try { - const messages = await channel.messages.fetch({ - limit: 20, - before: messageId + const messages = await channel.messages.fetch({ + limit: 20, + before: messageId, }); const contextMessages = Array.from(messages.values()) .reverse() - .map(msg => { + .map((msg) => { const timestamp = msg.createdAt.toISOString(); const author = msg.author.username; const content = msg.content || "[No text content]"; - const attachments = msg.attachments.size > 0 ? - ` [${msg.attachments.size} attachment(s)]` : ""; - + const attachments = + msg.attachments.size > 0 + ? ` [${msg.attachments.size} attachment(s)]` + : ""; + return `[${timestamp}] ${author}: ${content}${attachments}`; }) - .join('\n'); - + .join("\n"); + return contextMessages; } catch (error) { console.error("Error fetching message context:", error); @@ -69,10 +80,13 @@ async function fetchMessageContext(channel: TextChannel | ThreadChannel, message } } -async function analyzeMessageWithAI(message: Message, context: string): Promise<{ +async function analyzeMessageWithAI( + message: Message, + context: string, +): Promise<{ violation: boolean; rule: string; - severity: 'low' | 'medium' | 'high' | 'critical'; + severity: "low" | "medium" | "high" | "critical"; explanation: string; confidence: number; } | null> { @@ -85,9 +99,9 @@ ${SERVER_RULES} CURRENT MESSAGE TO ANALYZE: Author: ${message.author.username} (${message.author.id}) -Channel: ${'name' in message.channel ? message.channel.name : 'Unknown'} (${message.channelId}) +Channel: ${"name" in message.channel ? message.channel.name : "Unknown"} (${message.channelId}) Content: ${message.content || "[No text content]"} -Attachments: ${message.attachments.size > 0 ? message.attachments.map(a => a.name).join(', ') : "None"} +Attachments: ${message.attachments.size > 0 ? message.attachments.map((a) => a.name).join(", ") : "None"} MESSAGE CONTEXT (recent messages in this channel): ${context} @@ -109,54 +123,56 @@ Respond with a JSON object containing: If no violation is found, set "violation" to false and provide a brief explanation of why the message is acceptable. `; - const response = await fetch('https://api.openai.com/v1/chat/completions', { - method: 'POST', + const response = await fetch("https://api.openai.com/v1/chat/completions", { + method: "POST", headers: { - 'Authorization': `Bearer ${process.env.OPENAI_API_KEY}`, - 'Content-Type': 'application/json', + Authorization: `Bearer ${process.env.OPENAI_API_KEY}`, + "Content-Type": "application/json", }, - body: JSON.stringify({ - model: 'gpt-5-nano', - messages: [ - { - role: 'system', - content: 'You are a helpful AI moderator that analyzes Discord messages for rule violations. Always respond with valid JSON.' - }, - { - role: 'user', - content: prompt - } - ], - max_completion_tokens: 2000 - }) + body: JSON.stringify({ + model: "gpt-5-nano", + messages: [ + { + role: "system", + content: + "You are a helpful AI moderator that analyzes Discord messages for rule violations. Always respond with valid JSON.", + }, + { + role: "user", + content: prompt, + }, + ], + max_completion_tokens: 2000, + }), }); - if (!response.ok) { - const errorText = await response.text(); + if (!response.ok) { + const errorText = await response.text(); - console.error('OpenAI API error:', response.status, response.statusText); - console.error('Error response body:', errorText); + console.error("OpenAI API error:", response.status, response.statusText); + console.error("Error response body:", errorText); - return null; - } + return null; + } - const data = await response.json(); + const data = await response.json(); + + console.log("OpenAI API response:", JSON.stringify(data, null, 2)); + + if (data.usage) + console.log("Token usage:", { + prompt_tokens: data.usage.prompt_tokens, + completion_tokens: data.usage.completion_tokens, + reasoning_tokens: + data.usage.completion_tokens_details?.reasoning_tokens || 0, + total_tokens: data.usage.total_tokens, + }); + + const content = data.choices[0]?.message?.content; - console.log('OpenAI API response:', JSON.stringify(data, null, 2)); - - if (data.usage) - console.log('Token usage:', { - prompt_tokens: data.usage.prompt_tokens, - completion_tokens: data.usage.completion_tokens, - reasoning_tokens: data.usage.completion_tokens_details?.reasoning_tokens || 0, - total_tokens: data.usage.total_tokens - }); - - const content = data.choices[0]?.message?.content; - if (!content) { - console.error('No content in OpenAI response'); - console.error('Finish reason:', data.choices[0]?.finish_reason); + console.error("No content in OpenAI response"); + console.error("Finish reason:", data.choices[0]?.finish_reason); return null; } @@ -164,12 +180,13 @@ If no violation is found, set "violation" to false and provide a brief explanati try { return JSON.parse(content); } catch (parseError) { - console.error('Failed to parse OpenAI response as JSON:', content); + console.error("Failed to parse OpenAI response as JSON:", content); + console.error("Parse error:", parseError); return null; } } catch (error) { - console.error('Error in AI analysis:', error); + console.error("Error in AI analysis:", error); return null; } @@ -178,52 +195,69 @@ If no violation is found, set "violation" to false and provide a brief explanati export const handleAIModeration = (client: Client) => { client.on(Events.MessageCreate, async (message: Message) => { if (message.author.bot) return; - + if (!message.content && message.attachments.size === 0) return; - + if (message.channel.isThread()) { const parentChannel = message.channel.parent; - if (parentChannel && 'parentId' in parentChannel && parentChannel.parentId) + if ( + parentChannel && + "parentId" in parentChannel && + parentChannel.parentId + ) if (EXCLUDED_CATEGORIES.includes(parentChannel.parentId)) return; - } else if ('parentId' in message.channel && message.channel.parentId) { + } else if ("parentId" in message.channel && message.channel.parentId) { if (EXCLUDED_CATEGORIES.includes(message.channel.parentId)) return; } try { - console.log(`AI Moderation: Analyzing message from ${message.author.username} in #${'name' in message.channel ? message.channel.name : 'Unknown'}`); - - const context = await fetchMessageContext(message.channel as TextChannel | ThreadChannel, message.id); + console.log( + `AI Moderation: Analyzing message from ${message.author.username} in #${"name" in message.channel ? message.channel.name : "Unknown"}`, + ); + + const context = await fetchMessageContext( + message.channel as TextChannel | ThreadChannel, + message.id, + ); const analysis = await analyzeMessageWithAI(message, context); - + if (!analysis) { - console.log('AI analysis failed, skipping moderation'); + console.log("AI analysis failed, skipping moderation"); return; } - + if (!analysis.violation) { - console.log(`AI Moderation: No violation detected (confidence: ${analysis.confidence}%)`); + console.log( + `AI Moderation: No violation detected (confidence: ${analysis.confidence}%)`, + ); return; } - - console.log(`AI Moderation: Violation detected - ${analysis.rule} (severity: ${analysis.severity}, confidence: ${analysis.confidence}%)`); - + + console.log( + `AI Moderation: Violation detected - ${analysis.rule} (severity: ${analysis.severity}, confidence: ${analysis.confidence}%)`, + ); + try { await message.delete(); console.log(`AI Moderation: Deleted violating message`); } catch (error) { - console.error('Failed to delete message:', error); + console.error("Failed to delete message:", error); } - + const { EmbedBuilder } = await import("discord.js"); const embed = new EmbedBuilder() .setTitle("🤖 Message Deleted - Rule Violation") .setColor( - analysis.severity === 'critical' ? '#ff0000' : - analysis.severity === 'high' ? '#ff6600' : - analysis.severity === 'medium' ? '#ffaa00' : '#ffff00' + analysis.severity === "critical" + ? "#ff0000" + : analysis.severity === "high" + ? "#ff6600" + : analysis.severity === "medium" + ? "#ffaa00" + : "#ffff00", ) .addFields( { @@ -260,7 +294,7 @@ export const handleAIModeration = (client: Client) => { name: "Explanation", value: analysis.explanation, inline: false, - } + }, ) .setTimestamp() .setFooter({ @@ -277,10 +311,11 @@ export const handleAIModeration = (client: Client) => { await sendAuditLog( client, embed, - message.content && message.content.length > 1000 ? message.content : undefined, + message.content && message.content.length > 1000 + ? message.content + : undefined, MODERATION_LOG_CHANNEL_ID, ); - } catch (error) { console.error("Error in AI moderation:", error); } |