diff options
Diffstat (limited to 'packages/gateway')
3 files changed, 65 insertions, 52 deletions
diff --git a/packages/gateway/src/listeners/moderationAgent/constants.ts b/packages/gateway/src/listeners/moderationAgent/constants.ts index d01e751..40cd82d 100644 --- a/packages/gateway/src/listeners/moderationAgent/constants.ts +++ b/packages/gateway/src/listeners/moderationAgent/constants.ts @@ -1,3 +1,4 @@ +export const SKIP_PRIMARY_NOTIFICATION = false; export const SKIP_ACTION = false; export const EXCLUDED_CATEGORIES = [ "1406422619934167103", // Staff diff --git a/packages/gateway/src/listeners/moderationAgent/index.ts b/packages/gateway/src/listeners/moderationAgent/index.ts index 6cf6682..c6915c0 100644 --- a/packages/gateway/src/listeners/moderationAgent/index.ts +++ b/packages/gateway/src/listeners/moderationAgent/index.ts @@ -14,6 +14,7 @@ import { MODERATION_LOG_CHANNEL_ID, SAFE_WORDS, SKIP_ACTION, + SKIP_PRIMARY_NOTIFICATION, } from "./constants"; import { analyzeMessageWithAI, fetchMessageContext } from "./utilities"; @@ -141,37 +142,37 @@ export const handleAIModeration = (client: Client) => { `AI Moderation: Violation detected - ${analysis.rule} (severity: ${analysis.severity}, confidence: ${analysis.confidence}%)`, ); - if ( + if (SKIP_ACTION) { + console.log( + `AI Moderation: SKIP_ACTION enabled - logging violation without taking action (severity: ${analysis.severity}, confidence: ${analysis.confidence}%)`, + ); + } else if ( (analysis.severity === "critical" || analysis.severity === "high") && analysis.confidence >= 75 ) { - if (!SKIP_ACTION) + try { + await message.delete(); + console.log(`AI Moderation: Auto-deleted high severity violation`); + try { - await message.delete(); - console.log(`AI Moderation: Auto-deleted high severity violation`); - - try { - const notificationText = `${message.author}, your message was deleted: **${analysis.brief}**. This notification will be deleted in 10 seconds.\n\nIf you believe this was a mistake, let me know in #umabot.`; - const notificationMessage = await (message.channel as any).send( - notificationText, - ); - - setTimeout(async () => { - try { - await notificationMessage.delete(); - } catch (error) { - console.error( - "Failed to delete notification message:", - error, - ); - } - }, 10000); - } catch (error) { - console.error("Failed to send notification message:", error); - } + const notificationText = `${message.author}, your message was deleted: **${analysis.brief}**. This notification will be deleted in 10 seconds.\n\nIf you believe this was a mistake, let me know in #umabot.`; + const notificationMessage = await (message.channel as any).send( + notificationText, + ); + + setTimeout(async () => { + try { + await notificationMessage.delete(); + } catch (error) { + console.error("Failed to delete notification message:", error); + } + }, 10000); } catch (error) { - console.error("Failed to delete message:", error); + console.error("Failed to send notification message:", error); } + } catch (error) { + console.error("Failed to delete message:", error); + } } else { console.log( `AI Moderation: Logging violation for human review (severity: ${analysis.severity}, confidence: ${analysis.confidence}%)`, @@ -180,6 +181,7 @@ export const handleAIModeration = (client: Client) => { const { EmbedBuilder } = await import("discord.js"); const wasDeleted = + !SKIP_ACTION && (analysis.severity === "critical" || analysis.severity === "high") && analysis.confidence >= 85; const embed = new EmbedBuilder() @@ -251,7 +253,7 @@ export const handleAIModeration = (client: Client) => { inline: false, }); - if (!SKIP_ACTION) + if (!SKIP_PRIMARY_NOTIFICATION) await sendAuditLog( client, embed, diff --git a/packages/gateway/src/listeners/moderationAgent/utilities.ts b/packages/gateway/src/listeners/moderationAgent/utilities.ts index e89730b..cbf99f8 100644 --- a/packages/gateway/src/listeners/moderationAgent/utilities.ts +++ b/packages/gateway/src/listeners/moderationAgent/utilities.ts @@ -50,50 +50,56 @@ export const analyzeMessageWithAI = async ( const channel = message.channel; const guild = message.guild; const author = message.author; - const channelName = 'name' in channel ? channel.name : 'Unknown'; + const channelName = "name" in channel ? channel.name : "Unknown"; const channelId = channel.id; const channelType = channel.type; const isThread = channel.isThread(); - const parentChannelName = isThread && channel.parent ? channel.parent.name : null; - const parentChannelId = isThread && channel.parent ? channel.parent.id : null; + const parentChannelName = + isThread && channel.parent ? channel.parent.name : null; + const parentChannelId = + isThread && channel.parent ? channel.parent.id : null; let isNSFW = false; if (isThread && channel.parent) { - isNSFW = 'nsfw' in channel.parent ? channel.parent.nsfw : false; + isNSFW = "nsfw" in channel.parent ? channel.parent.nsfw : false; } else { - isNSFW = 'nsfw' in channel ? channel.nsfw : false; + isNSFW = "nsfw" in channel ? channel.nsfw : false; } - - const categoryId = 'parentId' in channel ? channel.parentId : null; - const categoryName = categoryId && guild ? - guild.channels.cache.get(categoryId)?.name || 'Unknown Category' : - 'No Category'; - const guildName = guild?.name || 'Unknown Server'; - const guildId = guild?.id || 'Unknown'; + + const categoryId = "parentId" in channel ? channel.parentId : null; + const categoryName = + categoryId && guild + ? guild.channels.cache.get(categoryId)?.name || "Unknown Category" + : "No Category"; + const guildName = guild?.name || "Unknown Server"; + const guildId = guild?.id || "Unknown"; const messageLength = message.content?.length || 0; const hasAttachments = message.attachments.size > 0; const hasEmbeds = message.embeds.length > 0; const authorId = author.id; const member = guild?.members.cache.get(authorId); - const authorRoles = member?.roles.cache.map(role => role.name).join(', ') || 'No roles'; - const authorJoinedAt = member?.joinedAt?.toISOString() || 'Unknown'; + const authorRoles = + member?.roles.cache.map((role) => role.name).join(", ") || "No roles"; + const authorJoinedAt = member?.joinedAt?.toISOString() || "Unknown"; let repliedToMessage = null; if (message.reference && message.reference.messageId) try { - repliedToMessage = await message.channel.messages.fetch(message.reference.messageId); + repliedToMessage = await message.channel.messages.fetch( + message.reference.messageId, + ); } catch (error) { console.error("Error fetching replied-to message:", error); } - + const fullContext = ` === SERVER CONTEXT === Server: ${guildName} (ID: ${guildId}) Channel: #${channelName} (ID: ${channelId}) Channel Type: ${channelType} -NSFW Status: ${isNSFW ? 'NSFW Channel' : 'SFW Channel'} -Category: ${categoryName} (ID: ${categoryId || 'None'}) -${isThread ? `Thread Parent: #${parentChannelName} (ID: ${parentChannelId})` : ''} +NSFW Status: ${isNSFW ? "NSFW Channel" : "SFW Channel"} +Category: ${categoryName} (ID: ${categoryId || "None"}) +${isThread ? `Thread Parent: #${parentChannelName} (ID: ${parentChannelId})` : ""} === MESSAGE CONTEXT === Message ID: ${message.id} @@ -101,14 +107,18 @@ Timestamp: ${message.createdAt.toISOString()} Length: ${messageLength} characters Has Attachments: ${hasAttachments} (${message.attachments.size} files) Has Embeds: ${hasEmbeds} (${message.embeds.length} embeds) -${repliedToMessage ? `Is Reply: Yes (replying to message from ${repliedToMessage.author.username})` : 'Is Reply: No'} +${repliedToMessage ? `Is Reply: Yes (replying to message from ${repliedToMessage.author.username})` : "Is Reply: No"} -${repliedToMessage ? `=== REPLIED-TO MESSAGE === +${ + repliedToMessage + ? `=== REPLIED-TO MESSAGE === Replied-to Author: ${repliedToMessage.author.username} Replied-to Message ID: ${repliedToMessage.id} Replied-to Timestamp: ${repliedToMessage.createdAt.toISOString()} -Replied-to Content: "${repliedToMessage.content || '[No text content]'}" -` : ''} +Replied-to Content: "${repliedToMessage.content || "[No text content]"}" +` + : "" +} === AUTHOR CONTEXT === Username: ${author.username} @@ -120,16 +130,16 @@ Roles: ${authorRoles} Joined Server: ${authorJoinedAt} === RECENT MESSAGE HISTORY === -${context || 'No recent message history available'} +${context || "No recent message history available"} === MESSAGE TO ANALYZE === -"${message.content || '[No text content - attachment only message]'}" +"${message.content || "[No text content - attachment only message]"}" === SERVER RULES === ${SERVER_RULES} `; -const prompt = `You are an AI moderator for a Discord server. Your job is to analyze messages for rule violations with extreme precision and accuracy. + const prompt = `You are an AI moderator for a Discord server. Your job is to analyze messages for rule violations with extreme precision and accuracy. CRITICAL INSTRUCTIONS: 1. You MUST ONLY enforce the exact rules provided in the SERVER_RULES section above |