Adding more utils. Pattern now supports content.plain and content.markdown

patterns.md

@@ -41,16 +41,17 @@ Available actions:
 The simplest expression just consists of a message field, a comparison operator, and a value to compare it to. For example:
 
 ```
-content contains "forbidden"
+content.plain contains "forbidden"
 ```
 
-The message will match if its `content` `contains` the word `"forbidden"`.
+The message will match if its `content.plain` `contains` the word `"forbidden"`.
 
 The available operators and type of value depends on the field being accessed.
 
 #### Fields
 
-* `content` - The text of the message. Note this is in raw Discord markdown, so it may contain underscores, emote names, and mention codes. Available operators: `==`, `!=`, `contains`, `!contains`, `matches`, `!matches`. Comparison value must be a quoted string.
+* `content.plain` - The plain text of the message. All markdown formatting is removed, and mentions look like the `@Username` text name that gets displayed.
+* `content.markdown` - The raw markdown of the message. This contains all markdown characters, and mentions are of the `<@!0000000>` form. Available operators: `==`, `!=`, `contains`, `!contains`, `matches`, `!matches`. Comparison value must be a quoted string.
 * `author` - Who sent the message. Available operators: `==`, `!=`. Comparison value must be a user mention (an @ that Discord will tab-complete for you).
 * `author.id` - The numeric ID of the user who sent the message. Available operators: `==`, `!=`. Comparison value must be a numeric user ID.
 * `author.name` - The username of the author. Available operators: `==`, `!=`, `contains`, `!contains`, `matches`, `!matches`. Comparison value must be a quoted string.
@@ -84,7 +85,7 @@ Timespans consist of one or more pairs of a number and a unit letter ("d" for da
 Multiple expressions can be combined with "and" or "or". For example:
 
 ```
-content contains "alpha" and content contains "bravo"
+content.plain contains "alpha" and content.plain contains "bravo"
 ```
 
 This will only match messages that contain both "alpha" and "bravo" somewhere in the text.
@@ -92,7 +93,7 @@ This will only match messages that contain both "alpha" and "bravo" somewhere in
 More complex expressions can use parentheses to clarify the order of interpretation. For example:
 
 ```
-(content contains "foo" and author.joinage < 30m) or (content contains "bar" and author.joinage > 30m)
+(content.plain contains "foo" and author.joinage < 30m) or (content.plain contains "bar" and author.joinage > 30m)
 ```
 
 A message will match if it contains "foo" and the user joined in the last half hour, OR it will match if the message contains "bar" and the author joined more than a half hour ago.
@@ -100,7 +101,7 @@ A message will match if it contains "foo" and the user joined in the last half h
 Lastly, expressions can be inverted by prefixing a `!`. For example:
 
 ```
-!(content contains "me" and content contains "you")
+!(content.plain contains "me" and content.plain contains "you")
 ```
 
 This will only match messages that do not contain both "me" and "you". If it contains just "me" or just "you" or neither word then it will match. If both are present it will not match.
@@ -112,19 +113,19 @@ Here are examples of `add` commands:
 Automatically delete a banned word:
 
 ```
-$rb_pattern add "bad word" delete if content contains "darn"
+$rb_pattern add "bad word" delete if content.plain contains "darn"
 ```
 
 Ban anyone who posts a URL within the first 30 minutes of joining the server.
 
 ```
-$rb_pattern add "url spam" ban if author.joinage < 30m and (content contains "http://" or content contains "https://")
+$rb_pattern add "url spam" ban if author.joinage < 30m and (content.plain contains "http://" or content.plain contains "https://")
 ```
 
 Automatically reply to anyone asking when lunch is.
 
 ```
-$rb_pattern add "lunch" reply "Lunch is at noon." if content == "When is lunch?"
+$rb_pattern add "lunch" reply "Lunch is at noon." if content.plain == "When is lunch?"
 ```
 
 ## Grammar
@@ -151,7 +152,7 @@ $rb_pattern add "lunch" reply "Lunch is at noon." if content == "When is lunch?"
 
 `<or_expr>` ::= `<expression>` " or " `<expression>`
 
-`<field_name>` ::= "content" | "author" | "author.id" | "author.name" | "author.joinage"
+`<field_name>` ::= "content.plain" | "content.markdown" | "author" | "author.id" | "author.name" | "author.joinage"
 
 `<op>` ::= "==" | "!=" | "<" | ">" | "<=" | ">=" | "contains" | "!contains" | "matches" | "!matches"
 

rocketbot/cogs/crosspostcog.py

@@ -2,7 +2,7 @@
 Cog for detecting spam messages posted in multiple channels.
 """
 from datetime import datetime, timedelta
-from discord import Member, Message
+from discord import Member, Message, utils as discordutils
 from discord.ext import commands
 
 from config import CONFIG
@@ -163,7 +163,7 @@ class CrossPostCog(BaseCog, name='Crosspost Detection'):
 		await self.__update_message_from_context(context)
 
 	async def __update_message_from_context(self, context: SpamContext) -> None:
-		first_spam_message = next(iter(context.spam_messages))
+		first_spam_message: Message = next(iter(context.spam_messages))
 		spam_count = len(context.spam_messages)
 		channel_count = len(context.unique_channels)
 		deleted_count = len(context.spam_messages)
@@ -171,7 +171,7 @@ class CrossPostCog(BaseCog, name='Crosspost Detection'):
 		if message is None:
 			message = BotMessage(context.member.guild, '',
 				BotMessage.TYPE_MOD_WARNING, context)
-			message.quote = first_spam_message.content
+			message.quote = discordutils.remove_markdown(first_spam_message.clean_content)
 		if context.is_autobanned:
 			text = f'User {context.member.mention} auto banned for ' + \
 				f'posting the same message in {channel_count} channels. ' + \

rocketbot/cogs/generalcog.py

@@ -9,7 +9,7 @@ from discord.ext import commands
 
 from config import CONFIG
 from rocketbot.cogs.basecog import BaseCog, BotMessage
-from rocketbot.utils import parse_timedelta, describe_timedelta
+from rocketbot.utils import timedelta_from_str, describe_timedelta
 from rocketbot.storage import ConfigKey, Storage
 
 class GeneralCog(BaseCog, name='General'):
@@ -97,7 +97,7 @@ class GeneralCog(BaseCog, name='General'):
 				mention_author=False)
 			return
 		try:
-			age_delta: timedelta = parse_timedelta(age)
+			age_delta: timedelta = timedelta_from_str(age)
 		except ValueError:
 			await context.message.reply(
 				f'{CONFIG["failure_emoji"]} age must be a timespan, like "30s", "10m", "1h30m"',

rocketbot/cogs/patterncog.py

@@ -4,13 +4,15 @@ automated actions on them.
 """
 import re
 from abc import ABCMeta, abstractmethod
-from discord import Guild, Member, Message
+from discord import Guild, Member, Message, utils as discordutils
 from discord.ext import commands
 
 from config import CONFIG
 from rocketbot.cogs.basecog import BaseCog, BotMessage, BotMessageReaction
+from rocketbot.cogsetting import CogSetting
 from rocketbot.storage import Storage
-from rocketbot.utils import parse_timedelta
+from rocketbot.utils import is_user_id, str_from_quoted_str, timedelta_from_str, \
+	user_id_from_mention
 
 class PatternAction:
 	"""
@@ -50,8 +52,10 @@ class PatternSimpleExpression(PatternExpression):
 		self.value = value
 
 	def __field_value(self, message: Message):
-		if self.field == 'content':
+		if self.field in ('content.markdown', 'content'):
 			return message.content
+		if self.field == 'content.plain':
+			return discordutils.remove_markdown(message.clean_content)
 		if self.field == 'author':
 			return str(message.author.id)
 		if self.field == 'author.id':
@@ -119,7 +123,7 @@ class PatternCompoundExpression(PatternExpression):
 				if op.matches(message):
 					return True
 			return False
-		raise RuntimeError(f'Bad operator "{self.operator}"')
+		raise ValueError(f'Bad operator "{self.operator}"')
 
 	def __str__(self) -> str:
 		if self.operator == '!':
@@ -133,12 +137,32 @@ class PatternStatement:
 	A full message match statement. If a message matches the given expression,
 	the given actions should be performed.
 	"""
-	def __init__(self, name: str, actions: list, expression: PatternExpression, original: str):
+	def __init__(self,
+			name: str,
+			actions: list,
+			expression: PatternExpression,
+			original: str):
 		self.name = name
 		self.actions = list(actions)  # PatternAction[]
 		self.expression = expression
 		self.original = original
 
+	def to_json(self) -> dict:
+		"""
+		Returns a JSON representation of this statement.
+		"""
+		return {
+			'name': self.name,
+			'statement': self.original,
+		}
+
+	@classmethod
+	def from_json(cls, json: dict):
+		"""
+		Gets a PatternStatement from its JSON representation.
+		"""
+		return PatternCompiler.parse_statement(json['name'], json['statement'])
+
 class PatternContext:
 	"""
 	Data about a message that has matched a configured statement and what
@@ -157,33 +181,30 @@ class PatternCog(BaseCog, name='Pattern Matching'):
 	various critera. Patterns can be defined by mods for each guild.
 	"""
 
-	def __get_patterns(self, guild: Guild) -> dict:
-		patterns = Storage.get_state_value(guild, 'PatternCog.patterns')
+	SETTING_PATTERNS = CogSetting('patterns', None)
+
+	def __get_patterns(self, guild: Guild) -> dict[str, PatternStatement]:
+		"""
+		Returns a name -> PatternStatement lookup for the guild.
+		"""
+		patterns: dict[str, PatternStatement] = Storage.get_state_value(guild,
+			'PatternCog.patterns')
 		if patterns is None:
-			patterns = {}
-			patterns_encoded = Storage.get_config_value(guild, 'PatternCog.patterns')
-			if patterns_encoded:
-				for pe in patterns_encoded:
-					name = pe.get('name')
-					statement = pe.get('statement')
-					try:
-						ps = PatternCompiler.parse_statement(name, statement)
-						patterns[name] = ps
-					except PatternError as e:
-						self.log(guild, 'Error parsing saved statement ' + \
-							f'"{name}": "{e}" Statement: {statement}')
+			jsons: list[dict] = self.get_guild_setting(guild, self.SETTING_PATTERNS)
+			pattern_list: list[PatternStatement] = []
+			for json in jsons:
+				try:
+					pattern_list.append(PatternStatement.from_json(json))
+				except PatternError as e:
+					self.log(guild, f'Error decoding pattern "{json["name"]}": {e}')
+			patterns = { p.name:p for p in pattern_list}
 			Storage.set_state_value(guild, 'PatternCog.patterns', patterns)
 		return patterns
 
 	@classmethod
-	def __save_patterns(cls, guild: Guild, patterns: dict) -> None:
-		to_save = []
-		for name, statement in patterns.items():
-			to_save.append({
-				'name': name,
-				'statement': statement.original,
-			})
-		Storage.set_config_value(guild, 'PatternCog.patterns', to_save)
+	def __save_patterns(cls, guild: Guild, patterns: dict[str, PatternStatement]) -> None:
+		to_save: list[dict] = list(map(PatternStatement.to_json, patterns.values()))
+		cls.set_guild_setting(guild, cls.SETTING_PATTERNS, to_save)
 
 	@commands.Cog.listener()
 	async def on_message(self, message: Message) -> None:
@@ -200,12 +221,11 @@ class PatternCog(BaseCog, name='Pattern Matching'):
 			return
 
 		patterns = self.__get_patterns(message.guild)
-		for _, statement in patterns.items():
+		for statement in patterns.values():
 			if statement.expression.matches(message):
 				await self.__trigger_actions(message, statement)
 				break
 
-
 	async def __trigger_actions(self, message: Message, statement: PatternStatement) -> None:
 		context = PatternContext(message, statement)
 		should_alert_mods = False
@@ -239,7 +259,7 @@ class PatternCog(BaseCog, name='Pattern Matching'):
 					f'{action.arguments[0]}',
 					mention_author=False)
 				action_descriptions.append('Autoreplied')
-				self.log(message.guild, f'{message.author.name} autoreplied to')
+				self.log(message.guild, f'autoreplied to {message.author.name}')
 		bm = BotMessage(
 			message.guild,
 			f'User {message.author.name} tripped custom pattern ' + \
@@ -247,7 +267,7 @@ class PatternCog(BaseCog, name='Pattern Matching'):
 				('\n• '.join(action_descriptions)),
 			type=BotMessage.TYPE_MOD_WARNING if should_alert_mods else BotMessage.TYPE_INFO,
 			context=context)
-		bm.quote = message.content
+		bm.quote = discordutils.remove_markdown(message.clean_content)
 		await bm.set_reactions(BotMessageReaction.standard_set(
 			did_delete=context.is_deleted,
 			did_kick=context.is_kicked,
@@ -361,11 +381,14 @@ class PatternCompiler:
 	TYPE_TIMESPAN = 'timespan'
 
 	FIELD_TO_TYPE = {
-		'content': TYPE_TEXT,
+		'content.plain': TYPE_TEXT,
+		'content.markdown': TYPE_TEXT,
 		'author': TYPE_MEMBER,
 		'author.id': TYPE_ID,
 		'author.name': TYPE_TEXT,
 		'author.joinage': TYPE_TIMESPAN,
+
+		'content': TYPE_TEXT, # deprecated, use content.markdown or content.plain
 	}
 
 	ACTION_TO_ARGS = {
@@ -675,29 +698,17 @@ class PatternCompiler:
 		Converts a value token to its Python value. Raises ValueError on failure.
 		"""
 		if datatype == cls.TYPE_ID:
-			p = re.compile('^[0-9]+$')
-			if p.match(value) is None:
-				raise ValueError(f'Illegal id value "{value}"')
-			# Store it as a str so it can be larger than an int
+			if not is_user_id(value):
+				raise ValueError(f'Illegal user id value: {value}')
 			return value
 		if datatype == cls.TYPE_MEMBER:
-			p = re.compile('^<@!?([0-9]+)>$')
-			m = p.match(value)
-			if m is None:
-				raise ValueError('Illegal member value. Must be an @ mention.')
-			return m.group(1)
+			return user_id_from_mention(value)
 		if datatype == cls.TYPE_TEXT:
-			# Must be quoted.
-			if len(value) < 2 or \
-					value[0:1] not in cls.STRING_QUOTE_CHARS or \
-					value[-1:] not in cls.STRING_QUOTE_CHARS or \
-					value[0:1] != value[-1:]:
-				raise ValueError(f'Not a quoted string value: {value}')
-			return value[1:-1]
+			return str_from_quoted_str(value)
 		if datatype == cls.TYPE_INT:
 			return int(value)
 		if datatype == cls.TYPE_FLOAT:
 			return float(value)
 		if datatype == cls.TYPE_TIMESPAN:
-			return parse_timedelta(value)
+			return timedelta_from_str(value)
 		raise ValueError(f'Unhandled datatype {datatype}')

rocketbot/cogs/urlspamcog.py

@@ -3,7 +3,7 @@ Cog for detecting URLs posted by new users.
 """
 import re
 from datetime import timedelta
-from discord import Member, Message
+from discord import Member, Message, utils as discordutils
 from discord.ext import commands
 
 from config import CONFIG
@@ -120,7 +120,7 @@ class URLSpamCog(BaseCog, name='URL Spam'):
 					f'{join_age_str} after joining.',
 					type = BotMessage.TYPE_MOD_WARNING if needs_attention else BotMessage.TYPE_INFO,
 					context = context)
-			bm.quote = message.content
+			bm.quote = discordutils.remove_markdown(message.clean_content)
 			await bm.set_reactions(BotMessageReaction.standard_set(
 				did_delete=context.is_deleted,
 				did_kick=context.is_kicked,

rocketbot/utils.py

@@ -6,7 +6,7 @@ from datetime import datetime, timedelta
 from discord import Guild
 from discord.ext.commands import Cog, Group
 
-def parse_timedelta(s: str) -> timedelta:
+def timedelta_from_str(s: str) -> timedelta:
 	"""
 	Parses a timespan. Format examples:
 	"30m"
@@ -36,6 +36,25 @@ def parse_timedelta(s: str) -> timedelta:
 			seconds = scalar
 	return timedelta(days=days, hours=hours, minutes=minutes, seconds=seconds)
 
+def str_from_timedelta(td: timedelta) -> str:
+	"""
+	Encodes a timedelta as a str. E.g. "3d2h"
+	"""
+	d = td.days
+	h = td.seconds // 3600
+	m = (td.seconds // 60) % 60
+	s = td.seconds % 60
+	components = []
+	if d != 0:
+		components.append(f'{d}d')
+	if h != 0:
+		components.append(f'{h}h')
+	if m != 0:
+		components.append(f'{m}m')
+	if s != 0 or len(components) == 0:
+		components.append(f'{s}s')
+	return ''.join(components)
+
 def describe_timedelta(td: timedelta, max_components: int = 2) -> str:
 	"""
 	Formats a human-readable description of a time span. E.g. "3 days 2 hours".
@@ -58,9 +77,7 @@ def describe_timedelta(td: timedelta, max_components: int = 2) -> str:
 	return ' '.join(components)
 
 def first_command_group(cog: Cog) -> Group:
-	"""
-	Returns the first command Group found in a cog.
-	"""
+	'Returns the first command Group found in a cog.'
 	for member_name in dir(cog):
 		member = getattr(cog, member_name)
 		if isinstance(member, Group):
@@ -68,12 +85,53 @@ def first_command_group(cog: Cog) -> Group:
 	return None
 
 def bot_log(guild: Guild, cog_class, message: str) -> None:
-	"""
-	Logs a message to stdout with time, cog, and guild info.
-	"""
+	'Logs a message to stdout with time, cog, and guild info.'
 	now = datetime.now() # local
 	s = f'[{now.strftime("%Y-%m-%dT%H:%M:%S")}|'
 	s += f'{cog_class.__name__}|' if cog_class else '-|'
 	s += f'{guild.name}] ' if guild else '-] '
 	s += message
 	print(s)
+
+__QUOTE_CHARS = '\'"'
+__ID_REGEX = re.compile('^[0-9]{17,20}$')
+__MENTION_REGEX = re.compile('^<@[!&]([0-9]{17,20})>$')
+__USER_MENTION_REGEX = re.compile('^<@!([0-9]{17,20})>$')
+__ROLE_MENTION_REGEX = re.compile('^<@&([0-9]{17,20})>$')
+
+def is_user_id(val: str) -> bool:
+	'Tests if a string is in user/role ID format.'
+	return __ID_REGEX.match(val) is not None
+
+def is_mention(val: str) -> bool:
+	'Tests if a string is a user or role mention.'
+	return __MENTION_REGEX.match(val) is not None
+
+def is_role_mention(val: str) -> bool:
+	'Tests if a string is a role mention.'
+	return __ROLE_MENTION_REGEX.match(val) is not None
+
+def is_user_mention(val: str) -> bool:
+	'Tests if a string is a user mention.'
+	return __USER_MENTION_REGEX.match(val) is not None
+
+def user_id_from_mention(mention: str) -> str:
+	'Extracts the user ID from a mention. Raises a ValueError if malformed.'
+	m = __USER_MENTION_REGEX.match(mention)
+	if m:
+		return m.group(1)
+	raise ValueError(f'"{mention}" is not an @ user mention')
+
+def mention_from_user_id(user_id: str) -> str:
+	'Returns a markdown user mention from a user id.'
+	return f'<@!{user_id}>'
+
+def mention_from_role_id(role_id: str) -> str:
+	'Returns a markdown role mention from a role id.'
+	return f'<@&{role_id}>'
+
+def str_from_quoted_str(val: str) -> str:
+	'Removes the leading and trailing quotes from a string.'
+	if len(val) < 2 or val[0:1] not in __QUOTE_CHARS or val[-1:] not in __QUOTE_CHARS:
+		raise ValueError(f'Not a quoted string: {val}')
+	return val[1:-1]