Crosspost cog looks for both duplicate and any messages posted in multiple channels

Log cog groups similar events to prevent getting throttled by API

config.sample.py

@@ -59,7 +59,9 @@ CONFIG = {
 		},
 		'CrossPostCog': {
 			'enabled': False,
-			'warncount': 3,
+			'dupewarncount': 3,
+			'warncount': 5,
+			'dupebancount': 9999,
 			'bancount': 9999,
 			'minlength': 1,
 			'timespan': 60,

main.py

@@ -11,7 +11,7 @@ from config import CONFIG
 from rocketbot.bot import start_bot
 from rocketbot.utils import bot_log
 
-CURRENT_CONFIG_VERSION = 3
+CURRENT_CONFIG_VERSION = 4
 if (CONFIG.get('__config_version') or 0) < CURRENT_CONFIG_VERSION:
 	# If you're getting this error, it means something changed in config.py's
 	# format. Consult config.sample.py and compare it to your own config.py.

rocketbot/cogs/basecog.py

@@ -9,6 +9,7 @@ from discord.abc import GuildChannel
 from discord.ext import commands
 
 from config import CONFIG
+from rocketbot.bot import Rocketbot
 from rocketbot.botmessage import BotMessage, BotMessageReaction
 from rocketbot.cogsetting import CogSetting
 from rocketbot.collections import AgeBoundDict
@@ -28,7 +29,7 @@ class BaseCog(commands.Cog):
 	common tasks.
 	"""
 	def __init__(self, bot):
-		self.bot = bot
+		self.bot: Rocketbot = bot
 		self.are_settings_setup = False
 		self.settings = []
 

rocketbot/cogs/crosspostcog.py

@@ -1,7 +1,8 @@
 """
 Cog for detecting spam messages posted in multiple channels.
 """
-from datetime import datetime, timedelta
+import re
+from datetime import datetime, timedelta, timezone
 from typing import Optional
 
 from discord import Member, Message, utils as discordutils, TextChannel
@@ -9,16 +10,17 @@ from discord.ext import commands
 
 from config import CONFIG
 from rocketbot.cogs.basecog import BaseCog, BotMessage, BotMessageReaction, CogSetting
-from rocketbot.collections import AgeBoundList, SizeBoundDict
+from rocketbot.collections import AgeBoundList, AgeBoundDict
 from rocketbot.storage import Storage
+from rocketbot.utils import str_from_timedelta
+
 
 class SpamContext:
 	"""
 	Data about a set of duplicate messages from a user.
 	"""
-	def __init__(self, member: Member, message_hash: int) -> None:
+	def __init__(self, member: Member) -> None:
 		self.member: Member = member
-		self.message_hash: int = message_hash
 		self.age: datetime = datetime.now()
 		self.bot_message: Optional[BotMessage] = None
 		self.is_kicked: bool = False
@@ -27,28 +29,49 @@ class SpamContext:
 		self.spam_messages: set[Message] = set()
 		self.deleted_messages: set[Message] = set()
 		self.unique_channels: set[TextChannel] = set()
+		self.duplicate_count: int = 0
 
 class CrossPostCog(BaseCog, name='Crosspost Detection'):
 	"""
-	Detects a user posting the same text in multiple channels in a short period
-	of time: a common pattern for spammers. Repeated posts in the same channel
-	aren't detected, as this can often be for a reason or due to trying a
-	failed post when connectivity is poor. Minimum message length can be
-	enforced for detection. Minimum is always at least 1 to ignore posts with
-	just embeds or images and no text.
+	Detects a user posting in multiple channels in a short period
+	of time: a common pattern for spammers.
+
+	These used to be identical text, but more recent attacks have had small
+	variations, such as different imgur URLs. It's reasonable to treat
+	posting in many channels in a short period as suspicious on its own,
+	regardless of whether they are identical.
+
+	Repeated posts in the same channel aren't currently detected, as this can
+	often be for a reason or due to trying a failed post when connectivity is
+	poor. Minimum message length can be enforced for detection.
 	"""
 	SETTING_ENABLED = CogSetting('enabled', bool,
 		brief='crosspost detection',
 		description='Whether crosspost detection is enabled.')
 	SETTING_WARN_COUNT = CogSetting('warncount', int,
 		brief='number of messages to trigger a warning',
-		description='The number of unique channels the same message is ' + \
+		description='The number of unique channels messages are ' + \
+			'posted in by the same user to trigger a mod warning. The ' + \
+			'messages need not be identical (see dupewarncount).',
+		usage='<count:int>',
+		min_value=2)
+	SETTING_DUPE_WARN_COUNT = CogSetting('dupewarncount', int,
+		brief='number of identical messages to trigger a warning',
+		description='The number of unique channels identical messages are ' + \
 			'posted in by the same user to trigger a mod warning.',
 		usage='<count:int>',
 		min_value=2)
 	SETTING_BAN_COUNT = CogSetting('bancount', int,
 		brief='number of messages to trigger a ban',
-		description='The number of unique channels the same message is ' + \
+		description='The number of unique channels messages are ' + \
+			'posted in by the same user to trigger an automatic ban. The ' + \
+			'messages need not be identical (see dupebancount). Set ' + \
+			'to a large value to effectively disable, e.g. 9999.',
+		usage='<count:int>',
+		min_value=2)
+	SETTING_DUPE_BAN_COUNT = CogSetting('dupebancount', int,
+		brief='number of identical messages to trigger a ban',
+		description='The number of unique channels identical messages are ' + \
 			'posted in by the same user to trigger an automatic ban. Set ' + \
 			'to a large value to effectively disable, e.g. 9999.',
 		usage='<count:int>',
@@ -75,7 +98,9 @@ class CrossPostCog(BaseCog, name='Crosspost Detection'):
 		super().__init__(bot)
 		self.add_setting(CrossPostCog.SETTING_ENABLED)
 		self.add_setting(CrossPostCog.SETTING_WARN_COUNT)
+		self.add_setting(CrossPostCog.SETTING_DUPE_WARN_COUNT)
 		self.add_setting(CrossPostCog.SETTING_BAN_COUNT)
+		self.add_setting(CrossPostCog.SETTING_DUPE_BAN_COUNT)
 		self.add_setting(CrossPostCog.SETTING_MIN_LENGTH)
 		self.add_setting(CrossPostCog.SETTING_TIMESPAN)
 		self.max_spam_contexts = 12
@@ -83,99 +108,121 @@ class CrossPostCog(BaseCog, name='Crosspost Detection'):
 	async def __record_message(self, message: Message) -> None:
 		if message.channel.permissions_for(message.author).ban_members:
 			# User exempt from spam detection
+			self.__trace("User exempt from crosspost checks")
 			return
 		def compute_message_hash(m: Message) -> int:
 			to_hash = m.content
+			# URLs sometimes differ per spam message, so simplify them
+			url_regex = r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+'
+			to_hash = re.sub(url_regex, '<url>', to_hash)
+			# Add attachment metadata
 			for attachment in m.attachments:
 				to_hash += f'\n[[ATT: ct={attachment.content_type} s={attachment.size} w={attachment.width} h={attachment.height}]]'
 			h = hash(to_hash)
+			self.__trace(f"Message hash for {m.id} is {h}")
 			return h
-		compute_message_hash(message)
-		if len(message.attachments) == 0 and len(message.content) < self.get_guild_setting(message.guild, self.SETTING_MIN_LENGTH):
+
+		min_length = self.get_guild_setting(message.guild, self.SETTING_MIN_LENGTH)
+		if len(message.attachments) == 0 and len(message.content) < min_length:
 			# Message too short to count towards spam total
+			self.__trace(f"Message len {len(message.content)} < {min_length}")
 			return
+
+		# Get config
 		max_age = timedelta(seconds=self.get_guild_setting(message.guild, self.SETTING_TIMESPAN))
 		warn_count: int = self.get_guild_setting(message.guild, self.SETTING_WARN_COUNT)
+		dupe_warn_count: int = self.get_guild_setting(message.guild, self.SETTING_DUPE_WARN_COUNT)
+
+		# Record message
 		recent_messages: AgeBoundList[Message, datetime, timedelta] = Storage.get_state_value(message.guild, self.STATE_KEY_RECENT_MESSAGES)
 		if recent_messages is None:
 			recent_messages = AgeBoundList(max_age, lambda index, message : message.created_at)
 			Storage.set_state_value(message.guild, self.STATE_KEY_RECENT_MESSAGES, recent_messages)
 		recent_messages.max_age = max_age
 		recent_messages.append(message)
+		self.__trace(f"Recent messages now length {len(recent_messages)}")
 
 		# Get all recent messages by user
 		member_messages = [m for m in recent_messages if m.author.id == message.author.id]
-		if len(member_messages) < warn_count:
+		message_count = len(member_messages)
+		self.__trace(f"Found {message_count} messages for {message.author.name}")
+		if message_count < warn_count and message_count < dupe_warn_count:
+			self.__trace(f"Bailing because message count {message_count} < warn count {warn_count} and < dupe warn count {dupe_warn_count}")
 			return
 
-		# Look for repeats
+		# Look for identical(ish) messages and unique channels
 		hash_to_channels: dict[int, set[TextChannel]] = {}
-		max_count = 0
+		unique_channels: set[TextChannel] = set()
+		max_duplicate_count = 0
 		for m in member_messages:
 			message_hash = compute_message_hash(m)
-			channels: set[TextChannel] = hash_to_channels.get(message_hash)
-			if channels is None:
-				channels = set()
-				hash_to_channels[message_hash] = channels
-			channels.add(m.channel)
-			max_count = max(max_count, len(channels))
-		if max_count < warn_count:
+			dupe_message_channels: set[TextChannel] = hash_to_channels.get(message_hash)
+			if dupe_message_channels is None:
+				dupe_message_channels = set()
+				hash_to_channels[message_hash] = dupe_message_channels
+			dupe_message_channels.add(m.channel)
+			unique_channels.add(m.channel)
+			max_duplicate_count = max(max_duplicate_count, len(dupe_message_channels))
+		channel_count = len(unique_channels)
+		self.__trace(f"Found {len(hash_to_channels)} unique messages, {channel_count} unique channels, {max_duplicate_count} duplicated messages")
+		if channel_count < warn_count and max_duplicate_count < dupe_warn_count:
+			self.__trace(f"Bailing because channels {channel_count} < warn count {warn_count} and max dupes {max_duplicate_count} < dupe warn count {dupe_warn_count}")
 			return
 
-		# Handle the spam
-		spam_lookup: SizeBoundDict[str, SpamContext, datetime] = Storage.get_state_value(message.guild, self.STATE_KEY_SPAM_CONTEXT)
+		# This person is a problem
+
+		spam_lookup: AgeBoundDict[str, SpamContext, datetime, timedelta] = Storage.get_state_value(message.guild, self.STATE_KEY_SPAM_CONTEXT)
 		if spam_lookup is None:
-			spam_lookup = SizeBoundDict(
-				self.max_spam_contexts,
+			spam_lookup = AgeBoundDict(
+				max_age,
 				lambda key, context : context.age)
 			Storage.set_state_value(message.guild, self.STATE_KEY_SPAM_CONTEXT, spam_lookup)
-		for message_hash, channels in hash_to_channels.items():
-			channel_count = len(channels)
-			if channel_count < warn_count:
-				continue
-			key = f'{message.author.id}|{message_hash}'
-			context = spam_lookup.get(key)
-			if context is None:
-				context = SpamContext(message.author, message_hash)
-				spam_lookup[key] = context
-				context.age = message.created_at
-				self.log(message.guild,
-					f'\u0007{message.author.name} ({message.author.id}) ' + \
-					f'posted the same message in {channel_count} or more channels.')
-			for m in member_messages:
-				if compute_message_hash(m) == message_hash:
-					context.spam_messages.add(m)
-					context.unique_channels.add(m.channel)
-			await self.__update_from_context(context)
+		key = f'{message.author.id}'
+		context = spam_lookup.get(key)
+		if context is not None and message.created_at - context.age > max_age:
+			context = None
+		if context is None:
+			context = SpamContext(message.author)
+			spam_lookup[key] = context
+			self.log(message.guild,
+				f'\u0007{message.author.name} ({message.author.id}) ' + \
+				f'posted messages in {channel_count} channels.')
+		context.age = message.created_at
+		context.duplicate_count = max_duplicate_count
+		context.spam_messages.update(member_messages)
+		context.unique_channels.update(unique_channels)
+		await self.__update_from_context(context)
 
 	async def __update_from_context(self, context: SpamContext):
 		ban_count = self.get_guild_setting(context.member.guild, self.SETTING_BAN_COUNT)
+		dupe_ban_count = self.get_guild_setting(context.member.guild, self.SETTING_DUPE_BAN_COUNT)
 		channel_count = len(context.unique_channels)
-		if channel_count >= ban_count:
+		if channel_count >= ban_count or context.duplicate_count >= dupe_ban_count:
 			if not context.is_banned:
+				max_age = timedelta(seconds=self.get_guild_setting(context.member.guild, self.SETTING_TIMESPAN))
+				max_age_str = str_from_timedelta(max_age)
 				await context.member.ban(
-					reason='Rocketbot: Posted same message in ' + \
-						f'{channel_count} channels. Banned by ' + \
-						f'{self.bot.user.name}.',
+					reason=f'Rocketbot: Posted in {channel_count} channels within {max_age_str} ' + \
+					   f'({context.duplicate_count} identical). Banned by {self.bot.user.name}.',
 					delete_message_days=1)
 				context.is_kicked = True
 				context.is_banned = True
 				context.is_autobanned = True
 				context.deleted_messages |= context.spam_messages
-				self.log(context.member.guild,
-					f'{context.member.name} ({context.member.id}) posted ' + \
-					f'same message in {channel_count} channels. Banned by ' + \
-					f'{self.bot.user.name}.')
+				self.__log_ban(context, self.bot.user.name)
 			else:
 				# Already banned. Nothing to update in the message.
 				return
 		await self.__update_message_from_context(context)
 
 	async def __update_message_from_context(self, context: SpamContext) -> None:
-		first_spam_message: Message = next(iter(context.spam_messages))
+		first_spam_message: Message = sorted(list(context.spam_messages), key=lambda m: m.created_at)[0]
 		spam_count = len(context.spam_messages)
 		channel_count = len(context.unique_channels)
-		deleted_count = len(context.spam_messages)
+		deleted_count = len(context.deleted_messages)
+		duplicate_count = context.duplicate_count
+		max_age = timedelta(seconds=self.get_guild_setting(context.member.guild, self.SETTING_TIMESPAN))
+		max_age_str = str_from_timedelta(max_age)
 		message = context.bot_message
 		if message is None:
 			message_type: int = BotMessage.TYPE_INFO if self.was_warned_recently(context.member) \
@@ -185,15 +232,25 @@ class CrossPostCog(BaseCog, name='Crosspost Detection'):
 			self.record_warning(context.member)
 		if context.is_autobanned:
 			text = f'User {context.member.mention} auto banned for ' + \
-				f'posting the same message in {channel_count} channels. ' + \
-				'Messages from past 24 hours deleted.'
+				f'posting messages in {channel_count} channels within {max_age_str} ' + \
+				f'({duplicate_count} identical). Messages from past 24 hours deleted.'
 			await message.set_reactions([])
 			await message.set_text(text)
 		else:
-			body: str = f'User {context.member.mention} posted ' + \
-				f'the same message in {channel_count} channels.'
-			for msg in context.spam_messages:
+			body: str = f'User {context.member.mention} posted '
+			if duplicate_count == channel_count:
+				body += f'identical messages in {channel_count} channels within {max_age_str} .'
+			elif duplicate_count == 1:
+				body += f'**different** messages in {channel_count} channels within ' + \
+						f'{max_age_str}. (Showing first one).'
+			else:
+				body += f'messages in {channel_count} channels within {max_age_str} ' + \
+						f'({duplicate_count} are identical, showing first one).'
+			max_links = 10
+			for msg in sorted(list(context.spam_messages), key=lambda m: m.created_at)[:max_links]:
 				body += f'\n- {msg.jump_url}'
+			if len(context.spam_messages) > max_links:
+				body += f'\n- ...{len(context.spam_messages) - max_links} more...'
 			await message.set_text(body)
 			await message.set_reactions(BotMessageReaction.standard_set(
 				did_delete = deleted_count >= spam_count,
@@ -218,30 +275,58 @@ class CrossPostCog(BaseCog, name='Crosspost Detection'):
 				await message.delete()
 				context.deleted_messages.add(message)
 			await self.__update_from_context(context)
-			self.log(context.member.guild,
-				f'{context.member.name} ({context.member.id}) posted same ' + \
-				f'message in {channel_count} channels. Deleted by {reacted_by.name}.')
+			self.__log_deletion(context, reacted_by.name)
 		elif reaction.emoji == CONFIG['kick_emoji']:
 			await context.member.kick(
-				reason=f'Rocketbot: Posted same message in {channel_count} ' + \
+				reason=f'Rocketbot: Posted messages in {channel_count} ' + \
 					f'channels. Kicked by {reacted_by.name}.')
 			context.is_kicked = True
 			await self.__update_from_context(context)
-			self.log(context.member.guild,
-				f'{context.member.name} ({context.member.id}) posted same ' + \
-				f'message in {channel_count} channels. Kicked by {reacted_by.name}.')
+			self.__log_kick(context, reacted_by.name)
 		elif reaction.emoji == CONFIG['ban_emoji']:
 			await context.member.ban(
-				reason=f'Rocketbot: Posted same message in {channel_count} ' + \
+				reason=f'Rocketbot: Posted messages in {channel_count} ' + \
 					f'channels. Banned by {reacted_by.name}.',
 				delete_message_days=1)
 			context.deleted_messages |= context.spam_messages
 			context.is_kicked = True
 			context.is_banned = True
 			await self.__update_from_context(context)
-			self.log(context.member.guild,
-				f'{context.member.name} ({context.member.id}) posted same ' + \
-				f'message in {channel_count} channels. Kicked by {reacted_by.name}.')
+			self.__log_ban(context, reacted_by.name)
+
+	def __log_deletion(self, context: SpamContext, by_who: str) -> None:
+		max_age = timedelta(seconds=self.get_guild_setting(context.member.guild, self.SETTING_TIMESPAN))
+		max_age_str = str_from_timedelta(max_age)
+		channel_count = len(context.unique_channels)
+		duplicate_count = context.duplicate_count
+		self.log(context.member.guild,
+				 f'{context.member.name} ({context.member.id}) posted ' + \
+				 f'messages in {channel_count} channels withint {max_age_str} ' + \
+				 f'({duplicate_count} identical). Deleted by {by_who}.')
+
+	def __log_kick(self, context: SpamContext, by_who: str) -> None:
+		max_age = timedelta(seconds=self.get_guild_setting(context.member.guild, self.SETTING_TIMESPAN))
+		max_age_str = str_from_timedelta(max_age)
+		channel_count = len(context.unique_channels)
+		duplicate_count = context.duplicate_count
+		self.log(context.member.guild,
+				 f'{context.member.name} ({context.member.id}) posted ' + \
+				 f'messages in {channel_count} channels within {max_age_str} ' + \
+				 f'({duplicate_count} identical). Kicked by {by_who}.')
+
+	def __log_ban(self, context: SpamContext, by_who: str) -> None:
+		max_age = timedelta(seconds=self.get_guild_setting(context.member.guild, self.SETTING_TIMESPAN))
+		max_age_str = str_from_timedelta(max_age)
+		channel_count = len(context.unique_channels)
+		duplicate_count = context.duplicate_count
+		self.log(context.member.guild,
+				 f'{context.member.name} ({context.member.id}) posted ' + \
+				 f'messages in {channel_count} channels within {max_age_str} ' + \
+				 f'({duplicate_count} identical). Banned by {by_who}.')
+
+	def __trace(self, message):
+		# print(message)
+		pass
 
 	@commands.Cog.listener()
 	async def on_message(self, message: Message):
@@ -249,11 +334,11 @@ class CrossPostCog(BaseCog, name='Crosspost Detection'):
 		if message.author is None or \
 				message.author.bot or \
 				message.channel is None or \
-				message.guild is None or \
-				message.content is None:
+				message.guild is None:
 			return
 		if not self.get_guild_setting(message.guild, self.SETTING_ENABLED):
 			return
+		self.__trace("--ON MESSAGE--")
 		await self.__record_message(message)
 
 	@commands.group(
@@ -262,6 +347,6 @@ class CrossPostCog(BaseCog, name='Crosspost Detection'):
 	@commands.has_permissions(ban_members=True)
 	@commands.guild_only()
 	async def crosspost(self, context: commands.Context):
-		"""Crosspost detection command group"""
+		"""Detects members posting messages in multiple channels in a short period of time."""
 		if context.invoked_subcommand is None:
 			await context.send_help()

rocketbot/cogs/logcog.py

@@ -2,16 +2,34 @@
 Cog for detecting large numbers of guild joins in a short period of time.
 """
 from collections.abc import Sequence
-from datetime import datetime
+from datetime import datetime, timezone, timedelta
+
 from discord import AuditLogAction, AuditLogEntry, Emoji, Guild, GuildSticker, Invite, Member, Message, RawBulkMessageDeleteEvent, RawMessageDeleteEvent, RawMessageUpdateEvent, Role, Thread, User
 from discord.abc import GuildChannel
-from discord.ext import commands
+from discord.ext import commands, tasks
 from discord.utils import escape_markdown
-from typing import Optional, Tuple, Union
+from typing import Optional, Tuple, Union, Callable
 import difflib
 import re
 
 from rocketbot.cogs.basecog import BaseCog, BotMessage, CogSetting
+from rocketbot.storage import Storage
+
+class BufferedMessageEditEvent:
+	def __init__(self, guild: Guild, channel: GuildChannel, before: Optional[Message], after: Message, data = None) -> None:
+		self.guild = guild
+		self.channel = channel
+		self.before = before
+		self.after = after
+		self.data = data
+
+class BufferedMessageDeleteEvent:
+	def __init__(self, guild: Guild, channel: GuildChannel, message_id: int, message: Optional[Message] = None) -> None:
+		self.guild = guild
+		self.channel = channel
+		self.message_id = message_id
+		self.message = message
+		self.author = message.author if message is not None else None
 
 class LoggingCog(BaseCog, name='Logging'):
 	"""
@@ -21,9 +39,16 @@ class LoggingCog(BaseCog, name='Logging'):
 			brief='logging',
 			description='Whether this cog is enabled for a guild.')
 
+	STATE_EVENT_BUFFER = 'LoggingCog.eventBuffer'
+
 	def __init__(self, bot):
 		super().__init__(bot)
 		self.add_setting(LoggingCog.SETTING_ENABLED)
+		self.flush_buffers.start()
+		self.buffered_guilds: set[Guild] = set()
+
+	def cog_unload(self) -> None:
+		self.flush_buffers.cancel()
 
 	@commands.group(
 		brief='Manages event logging',
@@ -407,6 +432,44 @@ class LoggingCog(BaseCog, name='Logging'):
 
 	# Events - Messages
 
+	def __buffer_event(self, guild: Guild, event_type: str, event) -> None:
+		buffers: dict[str, list] = Storage.get_state_value(guild, self.STATE_EVENT_BUFFER)
+		if buffers is None:
+			buffers = {}
+			Storage.set_state_value(guild, self.STATE_EVENT_BUFFER, buffers)
+		if buffers.get(event_type) is None:
+			buffers[event_type] = [ event ]
+		else:
+			buffers[event_type].append(event)
+		self.buffered_guilds.add(guild)
+
+	@tasks.loop(seconds=3.0)
+	async def flush_buffers(self) -> None:
+		try:
+			if len(self.buffered_guilds) == 0:
+				return
+			guilds = set(self.buffered_guilds)
+			self.buffered_guilds.clear()
+			for guild in guilds:
+				await self.__flush_buffers_for_guild(guild)
+		except Exception as e:
+			print(e)
+
+	async def __flush_buffers_for_guild(self, guild: Guild) -> None:
+		buffers: dict[str, list] = Storage.get_state_value(guild, self.STATE_EVENT_BUFFER)
+		if buffers is None:
+			return
+		Storage.set_state_value(guild, self.STATE_EVENT_BUFFER, None)
+		for event_type, buffer in buffers.items():
+			if event_type == 'edit':
+				await self.__flush_edit_buffers(guild, buffer)
+			elif event_type == 'delete':
+				await self.__flush_delete_buffers(guild, buffer)
+
+	@flush_buffers.before_loop
+	async def before_flush_buffers_start(self) -> None:
+		await self.bot.wait_until_ready()
+
 	@commands.Cog.listener()
 	async def on_message(self, message: Message) -> None:
 		"""
@@ -443,6 +506,79 @@ class LoggingCog(BaseCog, name='Logging'):
 			return
 		if after.author.id == self.bot.user.id:
 			return
+		channel = after.channel
+
+		self.__buffer_event(guild, 'edit', BufferedMessageEditEvent(guild, channel, before, after))
+
+	@commands.Cog.listener()
+	async def on_raw_message_edit(self, payload: RawMessageUpdateEvent) -> None:
+		"""
+		Called when a message is edited. Unlike on_message_edit(), this is called
+		regardless of the state of the internal message cache.
+
+		If the message is found in the message cache, it can be accessed via
+		RawMessageUpdateEvent.cached_message. The cached message represents the
+		message before it has been edited. For example, if the content of a
+		message is modified and triggers the on_raw_message_edit() coroutine,
+		the RawMessageUpdateEvent.cached_message will return a Message object
+		that represents the message before the content was modified.
+
+		Due to the inherently raw nature of this event, the data parameter
+		coincides with the raw data given by the gateway.
+
+		Since the data payload can be partial, care must be taken when accessing
+		stuff in the dictionary. One example of a common case of partial data is
+		when the 'content' key is inaccessible. This denotes an “embed” only
+		edit, which is an edit in which only the embeds are updated by the
+		Discord embed server.
+
+		https://discordpy.readthedocs.io/en/stable/api.html#discord.on_raw_message_edit
+		"""
+		if payload.cached_message:
+			return  # already handled by on_message_edit
+		guild = self.bot.get_guild(payload.guild_id) or await self.bot.fetch_guild(payload.guild_id)
+		if not guild:
+			return
+		if not self.get_guild_setting(guild, self.SETTING_ENABLED):
+			return
+		channel = guild.get_channel(payload.channel_id) or await guild.fetch_channel(payload.channel_id)
+		if not channel:
+			return
+
+		self.__buffer_event(guild, 'edit', BufferedMessageEditEvent(
+			guild, channel, None, payload.message, payload.data))
+
+	async def __flush_edit_buffers(self, guild: Guild, events: list[BufferedMessageEditEvent]) -> None:
+		simple_edits: list[BufferedMessageEditEvent] = []
+		complex_edits: list[BufferedMessageEditEvent] = []
+		old_cutoff = timedelta(days=1)
+		now = datetime.now(timezone.utc)
+		for event in events:
+			if event.before is not None and (now - event.after.created_at) < old_cutoff:
+				simple_edits.append(event)
+			else:
+				complex_edits.append(event)
+		if len(simple_edits) <= 3:
+			# A small number of edits with full details. Log them individually.
+			for event in events:
+				await self.__handle_complete_edit_event(event)
+		else:
+			complex_edits = events
+		if len(complex_edits) > 0:
+			# These messages are not cached, too old, or too numerous
+			text = 'Multiple messages edited' if len(complex_edits) > 1 else 'Message edited'
+			for event in complex_edits[:10]:
+				text += f'\n- {event.after.jump_url} by {event.after.author.name} ' + \
+						f'first posted <t:{int(event.after.created_at.timestamp())}:f>'
+			if len(complex_edits) > 10:
+				text += f'\n- ...{len(complex_edits) - 10} more...'
+			bot_message = BotMessage(guild, text, BotMessage.TYPE_LOG, suppress_embeds=True)
+			await bot_message.update()
+
+	async def __handle_complete_edit_event(self, event: BufferedMessageEditEvent) -> None:
+		before = event.before
+		after = event.after
+		guild = after.guild
 
 		content_changed = (after.content != before.content)
 		attachments_changed = (after.attachments != before.attachments)
@@ -495,49 +631,6 @@ class LoggingCog(BaseCog, name='Logging'):
 		await bot_message.update()
 
 	@commands.Cog.listener()
-	async def on_raw_message_edit(self, payload: RawMessageUpdateEvent) -> None:
-		"""
-		Called when a message is edited. Unlike on_message_edit(), this is called
-		regardless of the state of the internal message cache.
-
-		If the message is found in the message cache, it can be accessed via
-		RawMessageUpdateEvent.cached_message. The cached message represents the
-		message before it has been edited. For example, if the content of a
-		message is modified and triggers the on_raw_message_edit() coroutine,
-		the RawMessageUpdateEvent.cached_message will return a Message object
-		that represents the message before the content was modified.
-
-		Due to the inherently raw nature of this event, the data parameter
-		coincides with the raw data given by the gateway.
-
-		Since the data payload can be partial, care must be taken when accessing
-		stuff in the dictionary. One example of a common case of partial data is
-		when the 'content' key is inaccessible. This denotes an “embed” only
-		edit, which is an edit in which only the embeds are updated by the
-		Discord embed server.
-
-		https://discordpy.readthedocs.io/en/stable/api.html#discord.on_raw_message_edit
-		"""
-		if payload.cached_message:
-			return  # already handled by on_message_edit
-		guild = self.bot.get_guild(payload.guild_id) or await self.bot.fetch_guild(payload.guild_id)
-		if not guild:
-			return
-		if not self.get_guild_setting(guild, self.SETTING_ENABLED):
-			return
-		channel = guild.get_channel(payload.channel_id) or await guild.fetch_channel(payload.channel_id)
-		if not channel:
-			return
-		message = await channel.fetch_message(payload.message_id)
-		if not message:
-			return
-		text = f'Message {message.jump_url} edited by {self.__describe_user(message.author)}.\n' + \
-			'Original markdown unavailable in cache.\n' + \
-			f'Updated markdown:\n{self.__quote_markdown(message.content)}'
-		bot_message = BotMessage(guild, text, BotMessage.TYPE_LOG, suppress_embeds=True)
-		await bot_message.update()
-
-	@commands.Cog.listener()
 	async def on_raw_message_delete(self, payload: RawMessageDeleteEvent) -> None:
 		"""
 		Called when a message is deleted. Unlike on_message_delete(), this is
@@ -548,33 +641,22 @@ class LoggingCog(BaseCog, name='Logging'):
 
 		https://discordpy.readthedocs.io/en/stable/api.html#discord.on_raw_message_delete
 		"""
-		if payload.cached_message:
-			message = payload.cached_message
-			guild = message.guild
-			if not self.get_guild_setting(guild, self.SETTING_ENABLED):
-				return
-			if message.author.id == self.bot.user.id:
-				return
-			text = f'Message by {self.__describe_user(message.author)} deleted from {message.channel.mention}. ' + \
-				f'Markdown:\n{self.__quote_markdown(message.content)}'
-			for attachment in message.attachments or []:
-				text += f'\n> * 📎 {attachment.url}'
-			for embed in message.embeds or []:
-				text += f'\n> * 🔗 {embed.url}'
-			bot_message = BotMessage(message.guild, text, BotMessage.TYPE_LOG, suppress_embeds=True)
-			await bot_message.update()
-		else:
-			guild = self.bot.get_guild(payload.guild_id) or await self.bot.fetch_guild(payload.guild_id)
-			if not guild:
-				return
-			if not self.get_guild_setting(guild, self.SETTING_ENABLED):
-				return
-			channel = guild.get_channel(payload.channel_id) or await guild.fetch_channel(payload.channel_id)
-			if not channel:
-				return
-			text = f'Message {payload.message_id} deleted in ' + channel.mention + ' but content and author not available in cache.'
-			bot_message = BotMessage(guild, text, BotMessage.TYPE_LOG, suppress_embeds=True)
-			await bot_message.update()
+		message = payload.cached_message
+		if message and message.author.id == self.bot.user.id:
+			return
+		guild = (message.guild if message else None) or \
+				self.bot.get_guild(payload.guild_id) or \
+				await self.bot.fetch_guild(payload.guild_id)
+		if guild is None:
+			return
+		if not self.get_guild_setting(guild, self.SETTING_ENABLED):
+			return
+		channel = (message.channel if message else None) or \
+				  self.bot.get_channel(payload.channel_id) or \
+				  await guild.fetch_channel(payload.channel_id)
+		if channel is None:
+			return
+		self.__buffer_event(guild, 'delete', BufferedMessageDeleteEvent(guild, channel, payload.message_id, message))
 
 	@commands.Cog.listener()
 	async def on_raw_bulk_message_delete(self, payload: RawBulkMessageDeleteEvent) -> None:
@@ -594,27 +676,66 @@ class LoggingCog(BaseCog, name='Logging'):
 		if not self.get_guild_setting(guild, self.SETTING_ENABLED):
 			return
 		channel = guild.get_channel(payload.channel_id) or await guild.fetch_channel(payload.channel_id)
-		count = len(payload.message_ids)
-		cached_count = len(payload.cached_messages)
-		uncached_count = count - cached_count
-		text = f'Bulk deletion of {count} message(s) from {channel.mention}.'
-		if uncached_count == count:
-			text += f' No cached content available for any of them.'
-		elif uncached_count > 0:
-			text += f' No cached content available for {uncached_count} of them.'
-		bot_message = BotMessage(guild, text, BotMessage.TYPE_LOG, suppress_embeds=True)
-		await bot_message.update()
-
-		for message in payload.cached_messages:
-			text = f'Message by {self.__describe_user(message.author)} bulk deleted from {message.channel.mention}. ' + \
-				f'Markdown:\n{self.__quote_markdown(message.content)}'
-			for attachment in message.attachments or []:
-				text += f'\n> * 📎 {attachment.url}'
-			for embed in message.embeds or []:
-				text += f'\n> * 🔗 {embed.url}'
+		for message_id in payload.message_ids:
+			message = None
+			for cached_message in payload.cached_messages:
+				if cached_message.id == message_id:
+					message = cached_message
+			self.__buffer_event(guild, 'delete', BufferedMessageDeleteEvent(guild, channel, message_id, message))
+
+	async def __flush_delete_buffers(self, guild: Guild, events: list[BufferedMessageDeleteEvent]) -> None:
+		simple_deletes: list[BufferedMessageDeleteEvent] = []
+		complex_deletes: list[BufferedMessageDeleteEvent] = []
+		for event in events:
+			if event.message is not None:
+				simple_deletes.append(event)
+			else:
+				complex_deletes.append(event)
+		if len(simple_deletes) <= 3:
+			# Small number of deletes with complete info
+			for event in simple_deletes:
+				await self.__handle_complete_delete_event(event)
+		else:
+			complex_deletes = events
+		if len(complex_deletes) > 0:
+			messages_per_author: dict[Optional[User], list[BufferedMessageDeleteEvent]] = self.__groupby(complex_deletes, lambda e: e.author)
+			text = 'Multiple messages deleted' if len(complex_deletes) > 1 else 'Message deleted'
+			row_count = 0
+			for author, messages in messages_per_author.items():
+				row_count += 1
+				if row_count > 10:
+					break
+				count = len(messages)
+				text += f'\n- {count} {"message" if count == 1 else "messages"} by {author.mention if author else "unavailable user"}'
+				if count == 1:
+					text += f' in {messages[0].channel.mention}'
+				else:
+					messages_by_channel: dict[GuildChannel, list[BufferedMessageDeleteEvent]] = self.__groupby(messages, lambda e: e.channel)
+					if len(messages_by_channel) == 1:
+						text += f' in {messages[0].channel.mention}'
+					else:
+						for channel, ch_messages in messages_by_channel.items():
+							row_count += 1
+							if row_count > 10:
+								break
+							ch_count = len(ch_messages)
+							text += f'\n   - {ch_count} in {channel.mention}'
+			if row_count > 10:
+				text += '- ...more omitted...'
 			bot_message = BotMessage(guild, text, BotMessage.TYPE_LOG, suppress_embeds=True)
 			await bot_message.update()
 
+	async def __handle_complete_delete_event(self, event: BufferedMessageDeleteEvent) -> None:
+		message: Message = event.message
+		text = f'Message by {self.__describe_user(message.author)} deleted from {message.channel.mention}. ' + \
+			   f'Markdown:\n{self.__quote_markdown(message.content)}'
+		for attachment in message.attachments or []:
+			text += f'\n> * 📎 {attachment.url}'
+		for embed in message.embeds or []:
+			text += f'\n> * 🔗 {embed.url}'
+		bot_message = BotMessage(message.guild, text, BotMessage.TYPE_LOG, suppress_embeds=True)
+		await bot_message.update()
+
 	# Events - Roles
 
 	@commands.Cog.listener()
@@ -778,3 +899,14 @@ class LoggingCog(BaseCog, name='Logging'):
 		markdown_b = re.sub(r'[\ue000-\uefff]', unsub_token, markdown_b)
 
 		return markdown_a, markdown_b
+
+	def __groupby(self, a_list: list, grouper: Callable[[any], any]) -> dict:
+		"""itertools.groupby just less annoying"""
+		d = {}
+		for elem in a_list:
+			key = grouper(elem)
+			if key in d:
+				d[key].append(elem)
+			else:
+				d[key] = [elem]
+		return d

rocketbot/storage.py

@@ -92,7 +92,7 @@ class Storage:
 			'load from disk.')
 		config = cls.__read_guild_config(guild)
 		if config is None:
-			return {}
+			config = {}
 		cls.__guild_id_to_config[guild.id] = config
 		return config
 
@@ -154,6 +154,7 @@ class Storage:
 		path: str = cls.__guild_config_path(guild)
 		cls.__trace(f'Saving config for guild {guild.id} to {path}')
 		cls.__trace(f'config = {config}')
+		config['_guild_name'] = guild.name  # Just for making JSON files easier to identify
 		with open(path, 'w', encoding='utf8') as file:
 			# Pretty printing to make more legible for debugging
 			# Sorting keys to help with diffs