diff options
Diffstat (limited to 'packages/gateway')
| -rw-r--r-- | packages/gateway/src/listeners/aiModeration.ts | 288 | ||||
| -rw-r--r-- | packages/gateway/src/listeners/index.ts | 2 |
2 files changed, 290 insertions, 0 deletions
diff --git a/packages/gateway/src/listeners/aiModeration.ts b/packages/gateway/src/listeners/aiModeration.ts new file mode 100644 index 0000000..389f801 --- /dev/null +++ b/packages/gateway/src/listeners/aiModeration.ts @@ -0,0 +1,288 @@ +import { Client, Events, Message, TextChannel, ThreadChannel } from "discord.js"; +import { sendAuditLog } from "../commands/utilities"; + +const EXCLUDED_CATEGORIES = [ + "1406422619934167103", // Staff + "1420604833286852608", // Staff Automation +]; +const MODERATION_LOG_CHANNEL_ID = "1406422619934167106"; +const SERVER_RULES = ` +# Rules + +1. Do not circumvent the moderation system. +2. Follow Discord's Terms of Service & Discord Community Guidelines +3. Mark spoiler content using spoiler tags, except in #uma-musume-anime +4. Avoid self-promotion. Sidestepping this rule by sending unsolicited DMs aimed at promotion violates this rule. (artist promotion is fine in the designated art channels) +5. No hate speech, regardless of whether it is targeted directly at an individual in the community or not +6. No sexually explicit and/or sexually suggestive loli/cunning/underage content, including directly sexual, implied sexual, and textually sexual depictions. + + Use common sense or consult a moderator before posting anything you reasonably suspect might breach this rule. Alternatively, consider avoiding discussions about the topic altogether. + + We have a **zero-tolerance** stance on this rule and reserve the desecration to take administrative action against anyone at any time who violates this rule. +7. Treat channel descriptions as additional rules +8. NSFW content (including emoji, stickers, reactions, etc.) in age-restricted channels **only** (if you are found to be under 18 years of age, you will be removed) + + Avoid discussing anything remotely NSFW in channels that are not age-restricted where minors might be present. Just because this is the internet doesn't mean it isn't illegal. +9. Spam only in #spam +10. Do not send unsolicited communications or interact with users who have asked you to stop. If a user persists, instruct them to stop, block them, and report their behaviour using @UmaBot#9396. (\`/complain\` in DMs) +11. Refrain from discussing or promoting illegal activities. + +## Additional Guidelines + +**If AutoMod hits you or a moderator deletes your message, don't try again.** Attempting to sidestep any of these rules is arguably worse than breaking the rule, especially rule #6, which already has zero tolerance, and will be met with potentially harsher administrative action. + +By implementing these rules, we are not targeting anyone; instead, we are trying to maintain the community's good standing with the platforms on which it operates, as we are the guests. + +By the definition of this community, the term "loli" refers to a distinct body type **and/or** the implied context of being a child, not directly to a set of ages. + +The moderation team reserves the right to enforce these rules on any part of your Discord profile. This includes, but is not limited to, profile pictures, banners, usernames, descriptions, and pronouns. Failure to comply may result in a kick or ban from the server. + +Regardless of which rule users have broken, if a user accumulates 15 warnings within a rolling 6-month period, they will be subject to a ban. Similarly, warnings start to expire after 6 months, giving users the opportunity to improve their behaviour. + +If you think you have received an unfair warning or punishment, please submit an appeal by using the \`/appeal\` slash command in a direct message to @UmaBot#9396. A member of the moderation team will review your appeal. +`; + +async function fetchMessageContext(channel: TextChannel | ThreadChannel, messageId: string): Promise<string> { + try { + const messages = await channel.messages.fetch({ + limit: 20, + before: messageId + }); + const contextMessages = Array.from(messages.values()) + .reverse() + .map(msg => { + const timestamp = msg.createdAt.toISOString(); + const author = msg.author.username; + const content = msg.content || "[No text content]"; + const attachments = msg.attachments.size > 0 ? + ` [${msg.attachments.size} attachment(s)]` : ""; + + return `[${timestamp}] ${author}: ${content}${attachments}`; + }) + .join('\n'); + + return contextMessages; + } catch (error) { + console.error("Error fetching message context:", error); + + return "Unable to fetch message context"; + } +} + +async function analyzeMessageWithAI(message: Message, context: string): Promise<{ + violation: boolean; + rule: string; + severity: 'low' | 'medium' | 'high' | 'critical'; + explanation: string; + confidence: number; +} | null> { + try { + const prompt = ` +You are an AI moderator for a Discord server. Analyze the following message for rule violations. + +SERVER RULES: +${SERVER_RULES} + +CURRENT MESSAGE TO ANALYZE: +Author: ${message.author.username} (${message.author.id}) +Channel: ${'name' in message.channel ? message.channel.name : 'Unknown'} (${message.channelId}) +Content: ${message.content || "[No text content]"} +Attachments: ${message.attachments.size > 0 ? message.attachments.map(a => a.name).join(', ') : "None"} + +MESSAGE CONTEXT (recent messages in this channel): +${context} + +Please analyze this message for any rule violations. Consider: +1. The full context of the conversation +2. The specific channel this was posted in +3. The content and any attachments +4. Whether this violates any of the server rules + +Respond with a JSON object containing: +{ + "violation": boolean, + "rule": "Rule number and brief description if violation found, empty string if none", + "severity": "low|medium|high|critical", + "explanation": "Detailed explanation of the violation or why it's acceptable", + "confidence": number (0-100, how confident you are in this assessment) +} + +If no violation is found, set "violation" to false and provide a brief explanation of why the message is acceptable. +`; + const response = await fetch('https://api.openai.com/v1/chat/completions', { + method: 'POST', + headers: { + 'Authorization': `Bearer ${process.env.OPENAI_API_KEY}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + model: 'gpt-5-nano', + messages: [ + { + role: 'system', + content: 'You are a helpful AI moderator that analyzes Discord messages for rule violations. Always respond with valid JSON.' + }, + { + role: 'user', + content: prompt + } + ], + max_completion_tokens: 2000 + }) + }); + + if (!response.ok) { + const errorText = await response.text(); + + console.error('OpenAI API error:', response.status, response.statusText); + console.error('Error response body:', errorText); + + return null; + } + + const data = await response.json(); + + console.log('OpenAI API response:', JSON.stringify(data, null, 2)); + + if (data.usage) + console.log('Token usage:', { + prompt_tokens: data.usage.prompt_tokens, + completion_tokens: data.usage.completion_tokens, + reasoning_tokens: data.usage.completion_tokens_details?.reasoning_tokens || 0, + total_tokens: data.usage.total_tokens + }); + + const content = data.choices[0]?.message?.content; + + if (!content) { + console.error('No content in OpenAI response'); + console.error('Finish reason:', data.choices[0]?.finish_reason); + + return null; + } + + try { + return JSON.parse(content); + } catch (parseError) { + console.error('Failed to parse OpenAI response as JSON:', content); + + return null; + } + } catch (error) { + console.error('Error in AI analysis:', error); + + return null; + } +} + +export const handleAIModeration = (client: Client) => { + client.on(Events.MessageCreate, async (message: Message) => { + if (message.author.bot) return; + + if (!message.content && message.attachments.size === 0) return; + + if (message.channel.isThread()) { + const parentChannel = message.channel.parent; + + if (parentChannel && 'parentId' in parentChannel && parentChannel.parentId) + if (EXCLUDED_CATEGORIES.includes(parentChannel.parentId)) return; + } else if ('parentId' in message.channel && message.channel.parentId) { + if (EXCLUDED_CATEGORIES.includes(message.channel.parentId)) return; + } + + try { + console.log(`AI Moderation: Analyzing message from ${message.author.username} in #${'name' in message.channel ? message.channel.name : 'Unknown'}`); + + const context = await fetchMessageContext(message.channel as TextChannel | ThreadChannel, message.id); + const analysis = await analyzeMessageWithAI(message, context); + + if (!analysis) { + console.log('AI analysis failed, skipping moderation'); + + return; + } + + if (!analysis.violation) { + console.log(`AI Moderation: No violation detected (confidence: ${analysis.confidence}%)`); + + return; + } + + console.log(`AI Moderation: Violation detected - ${analysis.rule} (severity: ${analysis.severity}, confidence: ${analysis.confidence}%)`); + + try { + await message.delete(); + console.log(`AI Moderation: Deleted violating message`); + } catch (error) { + console.error('Failed to delete message:', error); + } + + const { EmbedBuilder } = await import("discord.js"); + const embed = new EmbedBuilder() + .setTitle("🤖 Message Deleted - Rule Violation") + .setColor( + analysis.severity === 'critical' ? '#ff0000' : + analysis.severity === 'high' ? '#ff6600' : + analysis.severity === 'medium' ? '#ffaa00' : '#ffff00' + ) + .addFields( + { + name: "Channel", + value: `<#${message.channelId}>`, + inline: true, + }, + { + name: "Author", + value: `<@${message.author.id}>`, + inline: true, + }, + { + name: "Message ID", + value: `[${message.id}](https://discord.com/channels/${message.guildId}/${message.channelId}/${message.id})`, + inline: true, + }, + { + name: "Rule Violation", + value: analysis.rule, + inline: false, + }, + { + name: "Severity", + value: analysis.severity.toUpperCase(), + inline: true, + }, + { + name: "Confidence", + value: `${analysis.confidence}%`, + inline: true, + }, + { + name: "Explanation", + value: analysis.explanation, + inline: false, + } + ) + .setTimestamp() + .setFooter({ + text: `Guild: ${message.guild?.name || "Unknown"}`, + }); + + if (message.content && message.content.length <= 1000) + embed.addFields({ + name: "Deleted Message Content", + value: message.content, + inline: false, + }); + + await sendAuditLog( + client, + embed, + message.content && message.content.length > 1000 ? message.content : undefined, + MODERATION_LOG_CHANNEL_ID, + ); + + } catch (error) { + console.error("Error in AI moderation:", error); + } + }); +}; diff --git a/packages/gateway/src/listeners/index.ts b/packages/gateway/src/listeners/index.ts index 6d796b1..6371753 100644 --- a/packages/gateway/src/listeners/index.ts +++ b/packages/gateway/src/listeners/index.ts @@ -2,6 +2,7 @@ import { Client } from "discord.js"; import { handleIqdbModeration } from "./iqdbModeration"; import { handleRoleplayUmagram } from "./roleplayUmagram"; import { handleArtMediaModeration } from "./artMediaModeration"; +import { handleAIModeration } from "./aiModeration"; import { handleAnnouncementReaction } from "./announcementReaction"; import { handleRoleProtection } from "./roleProtection"; import { handleChannelDeletion } from "./channelDeletion"; @@ -14,6 +15,7 @@ export const handleListeners = (client: Client) => { handleIqdbModeration(client); handleRoleplayUmagram(client); handleArtMediaModeration(client); + handleAIModeration(client); handleAnnouncementReaction(client); handleRoleProtection(client); handleChannelDeletion(client); |