From 9949f6ef51ca7bd83d6cfa2830c25d495f79ce83 Mon Sep 17 00:00:00 2001 From: A git user Date: Mon, 7 Aug 2023 00:36:34 +0200 Subject: [PATCH 01/18] Add html column to entry --- rss/db.py | 7 ++++--- rss/migrations.py | 6 +++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/rss/db.py b/rss/db.py index e6faa88..6af3dad 100644 --- a/rss/db.py +++ b/rss/db.py @@ -97,6 +97,7 @@ class Entry: title: str summary: str link: str + html: str @classmethod def from_row(cls, row: Record | None) -> Entry | None: @@ -143,7 +144,7 @@ async def get_feeds_by_room(self, room_id: RoomID) -> list[tuple[Feed, UserID]]: return [(Feed.from_row(row), row["user_id"]) for row in rows] async def get_entries(self, feed_id: int) -> list[Entry]: - q = "SELECT feed_id, id, date, title, summary, link FROM entry WHERE feed_id = $1" + q = "SELECT feed_id, id, date, title, summary, link, html FROM entry WHERE feed_id = $1" return [Entry.from_row(row) for row in await self.db.fetch(q, feed_id)] async def add_entries(self, entries: list[Entry], override_feed_id: int | None = None) -> None: @@ -153,13 +154,13 @@ async def add_entries(self, entries: list[Entry], override_feed_id: int | None = for entry in entries: entry.feed_id = override_feed_id records = [attr.astuple(entry) for entry in entries] - columns = ("feed_id", "id", "date", "title", "summary", "link") + columns = ("feed_id", "id", "date", "title", "summary", "link", "html") async with self.db.acquire() as conn: if self.db.scheme == Scheme.POSTGRES: await conn.copy_records_to_table("entry", records=records, columns=columns) else: q = ( - "INSERT INTO entry (feed_id, id, date, title, summary, link) " + "INSERT INTO entry (feed_id, id, date, title, summary, link, html) " "VALUES ($1, $2, $3, $4, $5, $6)" ) await conn.executemany(q, records) diff --git a/rss/migrations.py b/rss/migrations.py index 689f784..b2a9141 100644 --- a/rss/migrations.py +++ b/rss/migrations.py @@ -17,7 +17,6 @@ upgrade_table = UpgradeTable() - @upgrade_table.register(description="Latest revision", upgrades_to=3) async def upgrade_latest(conn: Connection, scheme: Scheme) -> None: gen = "GENERATED ALWAYS AS IDENTITY" if scheme != Scheme.SQLITE else "" @@ -72,3 +71,8 @@ async def upgrade_v2(conn: Connection) -> None: async def upgrade_v3(conn: Connection) -> None: await conn.execute("ALTER TABLE feed ADD COLUMN next_retry BIGINT DEFAULT 0") await conn.execute("ALTER TABLE feed ADD COLUMN error_count BIGINT DEFAULT 0") + + +@upgrade_table.register(description="Add html field to entry") +async def upgrade_v4(conn: Connection) -> None: + await conn.execute("ALTER TABLE entry ADD COLUMN html TEXT") From 01a4c89f91e9d739874b902953e83950bf687605 Mon Sep 17 00:00:00 2001 From: A git user Date: Mon, 7 Aug 2023 00:36:48 +0200 Subject: [PATCH 02/18] Revert "Add html column to entry" This reverts commit 9949f6ef51ca7bd83d6cfa2830c25d495f79ce83. --- rss/db.py | 7 +++---- rss/migrations.py | 6 +----- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/rss/db.py b/rss/db.py index 6af3dad..e6faa88 100644 --- a/rss/db.py +++ b/rss/db.py @@ -97,7 +97,6 @@ class Entry: title: str summary: str link: str - html: str @classmethod def from_row(cls, row: Record | None) -> Entry | None: @@ -144,7 +143,7 @@ async def get_feeds_by_room(self, room_id: RoomID) -> list[tuple[Feed, UserID]]: return [(Feed.from_row(row), row["user_id"]) for row in rows] async def get_entries(self, feed_id: int) -> list[Entry]: - q = "SELECT feed_id, id, date, title, summary, link, html FROM entry WHERE feed_id = $1" + q = "SELECT feed_id, id, date, title, summary, link FROM entry WHERE feed_id = $1" return [Entry.from_row(row) for row in await self.db.fetch(q, feed_id)] async def add_entries(self, entries: list[Entry], override_feed_id: int | None = None) -> None: @@ -154,13 +153,13 @@ async def add_entries(self, entries: list[Entry], override_feed_id: int | None = for entry in entries: entry.feed_id = override_feed_id records = [attr.astuple(entry) for entry in entries] - columns = ("feed_id", "id", "date", "title", "summary", "link", "html") + columns = ("feed_id", "id", "date", "title", "summary", "link") async with self.db.acquire() as conn: if self.db.scheme == Scheme.POSTGRES: await conn.copy_records_to_table("entry", records=records, columns=columns) else: q = ( - "INSERT INTO entry (feed_id, id, date, title, summary, link, html) " + "INSERT INTO entry (feed_id, id, date, title, summary, link) " "VALUES ($1, $2, $3, $4, $5, $6)" ) await conn.executemany(q, records) diff --git a/rss/migrations.py b/rss/migrations.py index b2a9141..689f784 100644 --- a/rss/migrations.py +++ b/rss/migrations.py @@ -17,6 +17,7 @@ upgrade_table = UpgradeTable() + @upgrade_table.register(description="Latest revision", upgrades_to=3) async def upgrade_latest(conn: Connection, scheme: Scheme) -> None: gen = "GENERATED ALWAYS AS IDENTITY" if scheme != Scheme.SQLITE else "" @@ -71,8 +72,3 @@ async def upgrade_v2(conn: Connection) -> None: async def upgrade_v3(conn: Connection) -> None: await conn.execute("ALTER TABLE feed ADD COLUMN next_retry BIGINT DEFAULT 0") await conn.execute("ALTER TABLE feed ADD COLUMN error_count BIGINT DEFAULT 0") - - -@upgrade_table.register(description="Add html field to entry") -async def upgrade_v4(conn: Connection) -> None: - await conn.execute("ALTER TABLE entry ADD COLUMN html TEXT") From 1d3d0cc18f8d1e30103d4e9b9c1e30f82916c5d7 Mon Sep 17 00:00:00 2001 From: A git user Date: Tue, 8 Aug 2023 06:43:40 +0200 Subject: [PATCH 03/18] Search for video tags in RSS body --- rss/bot.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rss/bot.py b/rss/bot.py index 945903b..7ac4304 100644 --- a/rss/bot.py +++ b/rss/bot.py @@ -22,6 +22,7 @@ import asyncio import hashlib import html +import re import aiohttp import attr @@ -117,6 +118,8 @@ async def _send(self, feed: Feed, entry: Entry, sub: Subscription) -> EventID: ) msgtype = MessageType.NOTICE if sub.send_notice else MessageType.TEXT try: + m = re.search(r" Date: Tue, 8 Aug 2023 06:43:48 +0200 Subject: [PATCH 04/18] Revert "Search for video tags in RSS body" This reverts commit 1d3d0cc18f8d1e30103d4e9b9c1e30f82916c5d7. --- rss/bot.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/rss/bot.py b/rss/bot.py index 7ac4304..945903b 100644 --- a/rss/bot.py +++ b/rss/bot.py @@ -22,7 +22,6 @@ import asyncio import hashlib import html -import re import aiohttp import attr @@ -118,8 +117,6 @@ async def _send(self, feed: Feed, entry: Entry, sub: Subscription) -> EventID: ) msgtype = MessageType.NOTICE if sub.send_notice else MessageType.TEXT try: - m = re.search(r" Date: Tue, 8 Aug 2023 07:18:08 +0200 Subject: [PATCH 05/18] Add postall command to post all previously seen entries to the room --- rss/bot.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/rss/bot.py b/rss/bot.py index 945903b..1058b16 100644 --- a/rss/bot.py +++ b/rss/bot.py @@ -445,6 +445,20 @@ async def command_notice(self, evt: MessageEvent, feed_id: int, setting: bool) - send_type = "m.notice" if setting else "m.text" await evt.reply(f"Updates for feed ID {feed.id} will now be sent as `{send_type}`") + @rss.subcommand( + "postall", aliases=("p",), help="Post all previously seen entries from the given feed to this room" + ) + @command.argument("feed_id", "feed ID", parser=int) + async def command_postall(self, evt: MessageEvent, feed_id: int) -> None: + if not await self.can_manage(evt): + return + sub, feed = await self.dbm.get_subscription(feed_id, evt.room_id) + if not sub: + await evt.reply("This room is not subscribed to that feed") + return + for entry in await self.dbm.get_entries(feed.id): + await self._broadcast(feed, entry, [sub]) + @staticmethod def _format_subscription(feed: Feed, subscriber: str) -> str: msg = ( From 07c7c7e4f15a08124a4de0a15d70f52a1527a7c5 Mon Sep 17 00:00:00 2001 From: A git user Date: Tue, 8 Aug 2023 07:30:01 +0200 Subject: [PATCH 06/18] Add postall command to README --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index aab772a..fd4e134 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,8 @@ Basic commands: * `!rss template [new template]` - Change the post template for a feed in the current room. If the new template is omitted, the bot replies with the current template. +* `!rss postall ` - Post all entries in the specified feed to the + current room ### Templates The default template is `New post in $feed_title: [$title]($link)`. From 721c4a400fd6ffa5e44f2511616eacbaa936897d Mon Sep 17 00:00:00 2001 From: A git user Date: Tue, 15 Aug 2023 13:32:09 +0200 Subject: [PATCH 07/18] Revert "Revert "Add html column to entry"" This reverts commit 01a4c89f91e9d739874b902953e83950bf687605. --- rss/db.py | 7 ++++--- rss/migrations.py | 6 +++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/rss/db.py b/rss/db.py index e6faa88..6af3dad 100644 --- a/rss/db.py +++ b/rss/db.py @@ -97,6 +97,7 @@ class Entry: title: str summary: str link: str + html: str @classmethod def from_row(cls, row: Record | None) -> Entry | None: @@ -143,7 +144,7 @@ async def get_feeds_by_room(self, room_id: RoomID) -> list[tuple[Feed, UserID]]: return [(Feed.from_row(row), row["user_id"]) for row in rows] async def get_entries(self, feed_id: int) -> list[Entry]: - q = "SELECT feed_id, id, date, title, summary, link FROM entry WHERE feed_id = $1" + q = "SELECT feed_id, id, date, title, summary, link, html FROM entry WHERE feed_id = $1" return [Entry.from_row(row) for row in await self.db.fetch(q, feed_id)] async def add_entries(self, entries: list[Entry], override_feed_id: int | None = None) -> None: @@ -153,13 +154,13 @@ async def add_entries(self, entries: list[Entry], override_feed_id: int | None = for entry in entries: entry.feed_id = override_feed_id records = [attr.astuple(entry) for entry in entries] - columns = ("feed_id", "id", "date", "title", "summary", "link") + columns = ("feed_id", "id", "date", "title", "summary", "link", "html") async with self.db.acquire() as conn: if self.db.scheme == Scheme.POSTGRES: await conn.copy_records_to_table("entry", records=records, columns=columns) else: q = ( - "INSERT INTO entry (feed_id, id, date, title, summary, link) " + "INSERT INTO entry (feed_id, id, date, title, summary, link, html) " "VALUES ($1, $2, $3, $4, $5, $6)" ) await conn.executemany(q, records) diff --git a/rss/migrations.py b/rss/migrations.py index 689f784..b2a9141 100644 --- a/rss/migrations.py +++ b/rss/migrations.py @@ -17,7 +17,6 @@ upgrade_table = UpgradeTable() - @upgrade_table.register(description="Latest revision", upgrades_to=3) async def upgrade_latest(conn: Connection, scheme: Scheme) -> None: gen = "GENERATED ALWAYS AS IDENTITY" if scheme != Scheme.SQLITE else "" @@ -72,3 +71,8 @@ async def upgrade_v2(conn: Connection) -> None: async def upgrade_v3(conn: Connection) -> None: await conn.execute("ALTER TABLE feed ADD COLUMN next_retry BIGINT DEFAULT 0") await conn.execute("ALTER TABLE feed ADD COLUMN error_count BIGINT DEFAULT 0") + + +@upgrade_table.register(description="Add html field to entry") +async def upgrade_v4(conn: Connection) -> None: + await conn.execute("ALTER TABLE entry ADD COLUMN html TEXT") From 777feeffb99974ec294c8242a5efde7a449bb39e Mon Sep 17 00:00:00 2001 From: A git user Date: Tue, 15 Aug 2023 14:52:21 +0200 Subject: [PATCH 08/18] Add support for non-required boolean arguments --- rss/bot.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rss/bot.py b/rss/bot.py index 1058b16..6258e93 100644 --- a/rss/bot.py +++ b/rss/bot.py @@ -66,6 +66,8 @@ def match(self, val: str, **kwargs) -> tuple[str, Any]: res = False elif part in ("t", "true", "y", "yes", "1"): res = True + elif self.required == False: + res = None else: raise ValueError("invalid boolean") return val[len(part) :], res From 4d2def1fe78a59b7e0da533dba430cb0b86af984 Mon Sep 17 00:00:00 2001 From: A git user Date: Tue, 15 Aug 2023 14:56:38 +0200 Subject: [PATCH 09/18] Add setting for encoded content --- rss/bot.py | 20 ++++++++++++++++++++ rss/db.py | 37 +++++++++++++++++++++++++++++-------- rss/migrations.py | 5 +++-- 3 files changed, 52 insertions(+), 10 deletions(-) diff --git a/rss/bot.py b/rss/bot.py index 6258e93..2dc3810 100644 --- a/rss/bot.py +++ b/rss/bot.py @@ -446,6 +446,26 @@ async def command_notice(self, evt: MessageEvent, feed_id: int, setting: bool) - await self.dbm.set_send_notice(feed.id, evt.room_id, setting) send_type = "m.notice" if setting else "m.text" await evt.reply(f"Updates for feed ID {feed.id} will now be sent as `{send_type}`") + + @rss.subcommand( + "formatted", aliases=("f","encoded"), help="Set whether or not the bot should send formatted updates when available" + ) + @command.argument("feed_id", "feed ID", parser=int) + @BoolArgument("setting", "true/false", required=False) + async def command_formatted(self, evt: MessageEvent, feed_id: int, setting: bool | None = None) -> None: + if not await self.can_manage(evt): + return + sub, feed = await self.dbm.get_subscription(feed_id, evt.room_id) + if not sub: + await evt.reply("This room is not subscribed to that feed") + return + if setting is None: + setting = await self.dbm.get_send_encoded(feed.id, evt.room_id) + else: + await self.dbm.set_send_encoded(feed.id, evt.room_id, setting) + + send_type = "formatted" if setting else "plain text" + await evt.reply(f"Updates for feed ID {feed.id} will be sent as {send_type}") @rss.subcommand( "postall", aliases=("p",), help="Post all previously seen entries from the given feed to this room" diff --git a/rss/db.py b/rss/db.py index 6af3dad..ec63e7b 100644 --- a/rss/db.py +++ b/rss/db.py @@ -39,6 +39,7 @@ class Subscription: user_id: UserID notification_template: Template send_notice: bool + send_encoded: bool @classmethod def from_row(cls, row: Record | None) -> Subscription | None: @@ -50,6 +51,7 @@ def from_row(cls, row: Record | None) -> Subscription | None: if not room_id or not user_id: return None send_notice = bool(row["send_notice"]) + send_encoded = bool(row["send_encoded"]) tpl = Template(row["notification_template"]) return cls( feed_id=feed_id, @@ -57,6 +59,7 @@ def from_row(cls, row: Record | None) -> Subscription | None: user_id=user_id, notification_template=tpl, send_notice=send_notice, + send_encoded=send_encoded ) @@ -81,6 +84,7 @@ def from_row(cls, row: Record | None) -> Feed | None: data.pop("room_id", None) data.pop("user_id", None) data.pop("send_notice", None) + data.pop("send_encoded", None) data.pop("notification_template", None) return cls(**data, subscriptions=[]) @@ -97,7 +101,7 @@ class Entry: title: str summary: str link: str - html: str + content_encoded: str @classmethod def from_row(cls, row: Record | None) -> Entry | None: @@ -122,7 +126,8 @@ def __init__(self, db: Database) -> None: async def get_feeds(self) -> list[Feed]: q = """ SELECT id, url, title, subtitle, link, next_retry, error_count, - room_id, user_id, notification_template, send_notice + room_id, user_id, notification_template, send_notice, + send_encoded FROM feed INNER JOIN subscription ON feed.id = subscription.feed_id """ rows = await self.db.fetch(q) @@ -144,7 +149,7 @@ async def get_feeds_by_room(self, room_id: RoomID) -> list[tuple[Feed, UserID]]: return [(Feed.from_row(row), row["user_id"]) for row in rows] async def get_entries(self, feed_id: int) -> list[Entry]: - q = "SELECT feed_id, id, date, title, summary, link, html FROM entry WHERE feed_id = $1" + q = "SELECT feed_id, id, date, title, summary, link, content_encoded FROM entry WHERE feed_id = $1" return [Entry.from_row(row) for row in await self.db.fetch(q, feed_id)] async def add_entries(self, entries: list[Entry], override_feed_id: int | None = None) -> None: @@ -154,13 +159,13 @@ async def add_entries(self, entries: list[Entry], override_feed_id: int | None = for entry in entries: entry.feed_id = override_feed_id records = [attr.astuple(entry) for entry in entries] - columns = ("feed_id", "id", "date", "title", "summary", "link", "html") + columns = ("feed_id", "id", "date", "title", "summary", "link", "content_encoded") async with self.db.acquire() as conn: if self.db.scheme == Scheme.POSTGRES: await conn.copy_records_to_table("entry", records=records, columns=columns) else: q = ( - "INSERT INTO entry (feed_id, id, date, title, summary, link, html) " + "INSERT INTO entry (feed_id, id, date, title, summary, link, content_encoded) " "VALUES ($1, $2, $3, $4, $5, $6)" ) await conn.executemany(q, records) @@ -174,7 +179,8 @@ async def get_subscription( ) -> tuple[Subscription | None, Feed | None]: q = """ SELECT id, url, title, subtitle, link, next_retry, error_count, - room_id, user_id, notification_template, send_notice + room_id, user_id, notification_template, send_notice, + send_encoded FROM feed LEFT JOIN subscription ON feed.id = subscription.feed_id AND room_id = $2 WHERE feed.id = $1 """ @@ -220,13 +226,19 @@ async def subscribe( user_id: UserID, template: str | None = None, send_notice: bool = True, + send_encoded: bool = False ) -> None: q = """ - INSERT INTO subscription (feed_id, room_id, user_id, notification_template, send_notice) + INSERT INTO subscription ( + feed_id, room_id, user_id, notification_template, + send_notice, send_encoded) VALUES ($1, $2, $3, $4, $5) """ template = template or "New post in $feed_title: [$title]($link)" - await self.db.execute(q, feed_id, room_id, user_id, template, send_notice) + await self.db.execute( + q, feed_id, room_id, user_id, template, send_notice, + send_encoded + ) async def unsubscribe(self, feed_id: int, room_id: RoomID) -> None: q = "DELETE FROM subscription WHERE feed_id = $1 AND room_id = $2" @@ -239,3 +251,12 @@ async def update_template(self, feed_id: int, room_id: RoomID, template: str) -> async def set_send_notice(self, feed_id: int, room_id: RoomID, send_notice: bool) -> None: q = "UPDATE subscription SET send_notice=$3 WHERE feed_id=$1 AND room_id=$2" await self.db.execute(q, feed_id, room_id, send_notice) + + async def set_send_encoded(self, feed_id: int, room_id: RoomID, send_encoded: bool) -> None: + q = "UPDATE subscription SET send_encoded=$3 WHERE feed_id=$1 AND room_id=$2" + await self.db.execute(q, feed_id, room_id, send_encoded) + + async def get_send_encoded(self, feed_id: int, room_id: RoomID) -> bool: + q = "SELECT send_encoded FROM subscription WHERE feed_id=$1 and room_id=$2" + row = await self.db.fetchrow(q, feed_id, room_id) + return bool(row["send_encoded"]) diff --git a/rss/migrations.py b/rss/migrations.py index b2a9141..22c10e9 100644 --- a/rss/migrations.py +++ b/rss/migrations.py @@ -73,6 +73,7 @@ async def upgrade_v3(conn: Connection) -> None: await conn.execute("ALTER TABLE feed ADD COLUMN error_count BIGINT DEFAULT 0") -@upgrade_table.register(description="Add html field to entry") +@upgrade_table.register(description="Add support for encoded content") async def upgrade_v4(conn: Connection) -> None: - await conn.execute("ALTER TABLE entry ADD COLUMN html TEXT") + await conn.execute("ALTER TABLE entry ADD COLUMN content_encoded TEXT") + await conn.execute("ALTER TABLE subscription ADD COLUMN send_encoded BOOLEAN DEFAULT false") From 8a95b2edbce47f4be258efa632fde96ffdbece96 Mon Sep 17 00:00:00 2001 From: A git user Date: Tue, 15 Aug 2023 15:29:45 +0200 Subject: [PATCH 10/18] Add support for encoded content --- rss/bot.py | 3 +++ rss/db.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/rss/bot.py b/rss/bot.py index 2dc3810..cf1a6f8 100644 --- a/rss/bot.py +++ b/rss/bot.py @@ -296,6 +296,7 @@ def _parse_rss_entry(cls, feed_id: int, entry: Any) -> Entry: title=getattr(entry, "title", ""), summary=getattr(entry, "description", "").strip(), link=getattr(entry, "link", ""), + content_encoded=entry["content"][0]["value"].strip(), ) @staticmethod @@ -419,6 +420,7 @@ async def command_template(self, evt: MessageEvent, feed_id: int, template: str) user_id=sub.user_id, notification_template=Template(template), send_notice=sub.send_notice, + send_encoded=sub.send_encoded, ) sample_entry = Entry( feed_id=feed.id, @@ -427,6 +429,7 @@ async def command_template(self, evt: MessageEvent, feed_id: int, template: str) title="Sample entry", summary="This is a sample entry to demonstrate your new template", link="http://example.com", + content_encoded="Sample encoded content" ) await evt.reply(f"Template for feed ID {feed.id} updated. Sample notification:") await self._send(feed, sample_entry, sub) diff --git a/rss/db.py b/rss/db.py index ec63e7b..e2a3a1d 100644 --- a/rss/db.py +++ b/rss/db.py @@ -166,7 +166,7 @@ async def add_entries(self, entries: list[Entry], override_feed_id: int | None = else: q = ( "INSERT INTO entry (feed_id, id, date, title, summary, link, content_encoded) " - "VALUES ($1, $2, $3, $4, $5, $6)" + "VALUES ($1, $2, $3, $4, $5, $6, $7)" ) await conn.executemany(q, records) @@ -232,7 +232,7 @@ async def subscribe( INSERT INTO subscription ( feed_id, room_id, user_id, notification_template, send_notice, send_encoded) - VALUES ($1, $2, $3, $4, $5) + VALUES ($1, $2, $3, $4, $5, $6) """ template = template or "New post in $feed_title: [$title]($link)" await self.db.execute( From 9f1d4a0e93454aefdb200c34faffec613a84c0a9 Mon Sep 17 00:00:00 2001 From: A git user Date: Tue, 15 Aug 2023 15:55:58 +0200 Subject: [PATCH 11/18] Generalize content_encoded into just content to support unencoded content as well --- rss/bot.py | 4 ++-- rss/db.py | 8 ++++---- rss/migrations.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/rss/bot.py b/rss/bot.py index cf1a6f8..746ddcf 100644 --- a/rss/bot.py +++ b/rss/bot.py @@ -296,7 +296,7 @@ def _parse_rss_entry(cls, feed_id: int, entry: Any) -> Entry: title=getattr(entry, "title", ""), summary=getattr(entry, "description", "").strip(), link=getattr(entry, "link", ""), - content_encoded=entry["content"][0]["value"].strip(), + content=entry["content"][0]["value"].strip(), ) @staticmethod @@ -429,7 +429,7 @@ async def command_template(self, evt: MessageEvent, feed_id: int, template: str) title="Sample entry", summary="This is a sample entry to demonstrate your new template", link="http://example.com", - content_encoded="Sample encoded content" + content="Sample formatted content" ) await evt.reply(f"Template for feed ID {feed.id} updated. Sample notification:") await self._send(feed, sample_entry, sub) diff --git a/rss/db.py b/rss/db.py index e2a3a1d..88e5592 100644 --- a/rss/db.py +++ b/rss/db.py @@ -101,7 +101,7 @@ class Entry: title: str summary: str link: str - content_encoded: str + content: str @classmethod def from_row(cls, row: Record | None) -> Entry | None: @@ -149,7 +149,7 @@ async def get_feeds_by_room(self, room_id: RoomID) -> list[tuple[Feed, UserID]]: return [(Feed.from_row(row), row["user_id"]) for row in rows] async def get_entries(self, feed_id: int) -> list[Entry]: - q = "SELECT feed_id, id, date, title, summary, link, content_encoded FROM entry WHERE feed_id = $1" + q = "SELECT feed_id, id, date, title, summary, link, content FROM entry WHERE feed_id = $1" return [Entry.from_row(row) for row in await self.db.fetch(q, feed_id)] async def add_entries(self, entries: list[Entry], override_feed_id: int | None = None) -> None: @@ -159,13 +159,13 @@ async def add_entries(self, entries: list[Entry], override_feed_id: int | None = for entry in entries: entry.feed_id = override_feed_id records = [attr.astuple(entry) for entry in entries] - columns = ("feed_id", "id", "date", "title", "summary", "link", "content_encoded") + columns = ("feed_id", "id", "date", "title", "summary", "link", "content") async with self.db.acquire() as conn: if self.db.scheme == Scheme.POSTGRES: await conn.copy_records_to_table("entry", records=records, columns=columns) else: q = ( - "INSERT INTO entry (feed_id, id, date, title, summary, link, content_encoded) " + "INSERT INTO entry (feed_id, id, date, title, summary, link, content) " "VALUES ($1, $2, $3, $4, $5, $6, $7)" ) await conn.executemany(q, records) diff --git a/rss/migrations.py b/rss/migrations.py index 22c10e9..2f1af16 100644 --- a/rss/migrations.py +++ b/rss/migrations.py @@ -75,5 +75,5 @@ async def upgrade_v3(conn: Connection) -> None: @upgrade_table.register(description="Add support for encoded content") async def upgrade_v4(conn: Connection) -> None: - await conn.execute("ALTER TABLE entry ADD COLUMN content_encoded TEXT") + await conn.execute("ALTER TABLE entry ADD COLUMN content TEXT") await conn.execute("ALTER TABLE subscription ADD COLUMN send_encoded BOOLEAN DEFAULT false") From 4dfa74a63b6aa1fa9eed090f7655297bb2b17b01 Mon Sep 17 00:00:00 2001 From: A git user Date: Tue, 15 Aug 2023 16:07:25 +0200 Subject: [PATCH 12/18] Remote send_encoded flag, as encoded content is now a template variable that can be included upon user's choice --- rss/bot.py | 21 --------------------- rss/db.py | 27 +++++---------------------- rss/migrations.py | 1 - 3 files changed, 5 insertions(+), 44 deletions(-) diff --git a/rss/bot.py b/rss/bot.py index 746ddcf..730a3c0 100644 --- a/rss/bot.py +++ b/rss/bot.py @@ -420,7 +420,6 @@ async def command_template(self, evt: MessageEvent, feed_id: int, template: str) user_id=sub.user_id, notification_template=Template(template), send_notice=sub.send_notice, - send_encoded=sub.send_encoded, ) sample_entry = Entry( feed_id=feed.id, @@ -449,26 +448,6 @@ async def command_notice(self, evt: MessageEvent, feed_id: int, setting: bool) - await self.dbm.set_send_notice(feed.id, evt.room_id, setting) send_type = "m.notice" if setting else "m.text" await evt.reply(f"Updates for feed ID {feed.id} will now be sent as `{send_type}`") - - @rss.subcommand( - "formatted", aliases=("f","encoded"), help="Set whether or not the bot should send formatted updates when available" - ) - @command.argument("feed_id", "feed ID", parser=int) - @BoolArgument("setting", "true/false", required=False) - async def command_formatted(self, evt: MessageEvent, feed_id: int, setting: bool | None = None) -> None: - if not await self.can_manage(evt): - return - sub, feed = await self.dbm.get_subscription(feed_id, evt.room_id) - if not sub: - await evt.reply("This room is not subscribed to that feed") - return - if setting is None: - setting = await self.dbm.get_send_encoded(feed.id, evt.room_id) - else: - await self.dbm.set_send_encoded(feed.id, evt.room_id, setting) - - send_type = "formatted" if setting else "plain text" - await evt.reply(f"Updates for feed ID {feed.id} will be sent as {send_type}") @rss.subcommand( "postall", aliases=("p",), help="Post all previously seen entries from the given feed to this room" diff --git a/rss/db.py b/rss/db.py index 88e5592..275010e 100644 --- a/rss/db.py +++ b/rss/db.py @@ -39,7 +39,6 @@ class Subscription: user_id: UserID notification_template: Template send_notice: bool - send_encoded: bool @classmethod def from_row(cls, row: Record | None) -> Subscription | None: @@ -51,7 +50,6 @@ def from_row(cls, row: Record | None) -> Subscription | None: if not room_id or not user_id: return None send_notice = bool(row["send_notice"]) - send_encoded = bool(row["send_encoded"]) tpl = Template(row["notification_template"]) return cls( feed_id=feed_id, @@ -59,7 +57,6 @@ def from_row(cls, row: Record | None) -> Subscription | None: user_id=user_id, notification_template=tpl, send_notice=send_notice, - send_encoded=send_encoded ) @@ -84,7 +81,6 @@ def from_row(cls, row: Record | None) -> Feed | None: data.pop("room_id", None) data.pop("user_id", None) data.pop("send_notice", None) - data.pop("send_encoded", None) data.pop("notification_template", None) return cls(**data, subscriptions=[]) @@ -126,8 +122,7 @@ def __init__(self, db: Database) -> None: async def get_feeds(self) -> list[Feed]: q = """ SELECT id, url, title, subtitle, link, next_retry, error_count, - room_id, user_id, notification_template, send_notice, - send_encoded + room_id, user_id, notification_template, send_notice FROM feed INNER JOIN subscription ON feed.id = subscription.feed_id """ rows = await self.db.fetch(q) @@ -179,8 +174,7 @@ async def get_subscription( ) -> tuple[Subscription | None, Feed | None]: q = """ SELECT id, url, title, subtitle, link, next_retry, error_count, - room_id, user_id, notification_template, send_notice, - send_encoded + room_id, user_id, notification_template, send_notice FROM feed LEFT JOIN subscription ON feed.id = subscription.feed_id AND room_id = $2 WHERE feed.id = $1 """ @@ -225,19 +219,17 @@ async def subscribe( room_id: RoomID, user_id: UserID, template: str | None = None, - send_notice: bool = True, - send_encoded: bool = False + send_notice: bool = True ) -> None: q = """ INSERT INTO subscription ( feed_id, room_id, user_id, notification_template, - send_notice, send_encoded) - VALUES ($1, $2, $3, $4, $5, $6) + send_notice) + VALUES ($1, $2, $3, $4, $5) """ template = template or "New post in $feed_title: [$title]($link)" await self.db.execute( q, feed_id, room_id, user_id, template, send_notice, - send_encoded ) async def unsubscribe(self, feed_id: int, room_id: RoomID) -> None: @@ -251,12 +243,3 @@ async def update_template(self, feed_id: int, room_id: RoomID, template: str) -> async def set_send_notice(self, feed_id: int, room_id: RoomID, send_notice: bool) -> None: q = "UPDATE subscription SET send_notice=$3 WHERE feed_id=$1 AND room_id=$2" await self.db.execute(q, feed_id, room_id, send_notice) - - async def set_send_encoded(self, feed_id: int, room_id: RoomID, send_encoded: bool) -> None: - q = "UPDATE subscription SET send_encoded=$3 WHERE feed_id=$1 AND room_id=$2" - await self.db.execute(q, feed_id, room_id, send_encoded) - - async def get_send_encoded(self, feed_id: int, room_id: RoomID) -> bool: - q = "SELECT send_encoded FROM subscription WHERE feed_id=$1 and room_id=$2" - row = await self.db.fetchrow(q, feed_id, room_id) - return bool(row["send_encoded"]) diff --git a/rss/migrations.py b/rss/migrations.py index 2f1af16..ea456de 100644 --- a/rss/migrations.py +++ b/rss/migrations.py @@ -76,4 +76,3 @@ async def upgrade_v3(conn: Connection) -> None: @upgrade_table.register(description="Add support for encoded content") async def upgrade_v4(conn: Connection) -> None: await conn.execute("ALTER TABLE entry ADD COLUMN content TEXT") - await conn.execute("ALTER TABLE subscription ADD COLUMN send_encoded BOOLEAN DEFAULT false") From e0a992396ebf5bff3cde7d3ba73fdb2dcc00f108 Mon Sep 17 00:00:00 2001 From: A git user Date: Thu, 17 Aug 2023 12:28:46 +0200 Subject: [PATCH 13/18] Fix support for RSS feeds without content attribute --- rss/bot.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rss/bot.py b/rss/bot.py index 730a3c0..a71a2ee 100644 --- a/rss/bot.py +++ b/rss/bot.py @@ -296,7 +296,9 @@ def _parse_rss_entry(cls, feed_id: int, entry: Any) -> Entry: title=getattr(entry, "title", ""), summary=getattr(entry, "description", "").strip(), link=getattr(entry, "link", ""), - content=entry["content"][0]["value"].strip(), + content=entry["content"][0]["value"].strip() if hasattr(entry, "content") + and len(entry["content"]) > 0 + and hasattr(entry["content"][0], "value") else "", ) @staticmethod From ed4b34a788cf33b08ae26a15971152b1b7b53d86 Mon Sep 17 00:00:00 2001 From: A git user Date: Sun, 3 Sep 2023 17:40:51 +0200 Subject: [PATCH 14/18] Split long messages to prevent matrix event too large errors --- rss/bot.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/rss/bot.py b/rss/bot.py index a71a2ee..95eb33d 100644 --- a/rss/bot.py +++ b/rss/bot.py @@ -117,11 +117,16 @@ async def _send(self, feed: Feed, entry: Entry, sub: Subscription) -> EventID: **attr.asdict(entry), } ) + msgtype = MessageType.NOTICE if sub.send_notice else MessageType.TEXT + msgchunks = [message[i:i + 30000] for i in range(0, len(message), 30000)] + self.log.debug(f"Message length: {len(message)} Content length: {len(entry.content)} Chunks: {len(msgchunks)}") try: - return await self.client.send_markdown( - sub.room_id, message, msgtype=msgtype, allow_html=True - ) + for chunk in msgchunks: + returnval = await self.client.send_markdown( + sub.room_id, chunk, msgtype=msgtype, allow_html=True + ) + return returnval except Exception as e: self.log.warning(f"Failed to send {entry.id} of {feed.id} to {sub.room_id}: {e}") From b598472e612b24d911850297c0babe110e682709 Mon Sep 17 00:00:00 2001 From: Lu Ro Date: Fri, 18 Oct 2024 02:20:55 +0200 Subject: [PATCH 15/18] add regex support --- rss/bot.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/rss/bot.py b/rss/bot.py index 945903b..5051f89 100644 --- a/rss/bot.py +++ b/rss/bot.py @@ -18,6 +18,7 @@ from typing import Any, Iterable from datetime import datetime from string import Template +import re from time import mktime, time import asyncio import hashlib @@ -115,6 +116,14 @@ async def _send(self, feed: Feed, entry: Entry, sub: Subscription) -> EventID: **attr.asdict(entry), } ) + entrytext = ' '.join(attr.asdict(entry).values()) + self.log.info(f"obtained entry text: {entrytext}") + + for match in re.finditer(r'\{\{ ([^}]*) \}\}', message): + value = re.search(match.groups(0), entrytext) + self.log.info(f"found match: {match} with regex {match.groups(0)} matching {value} in entrytext") + message = message[:match.span()[0]] + value + message[match.span()[1]:] + msgtype = MessageType.NOTICE if sub.send_notice else MessageType.TEXT try: return await self.client.send_markdown( From 748294c5af155012180c0787e94910b61341b930 Mon Sep 17 00:00:00 2001 From: A git user Date: Fri, 18 Oct 2024 03:16:51 +0200 Subject: [PATCH 16/18] fix regex matching --- rss/bot.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/rss/bot.py b/rss/bot.py index 47f62c1..58b1d80 100644 --- a/rss/bot.py +++ b/rss/bot.py @@ -118,14 +118,12 @@ async def _send(self, feed: Feed, entry: Entry, sub: Subscription) -> EventID: **attr.asdict(entry), } ) - entrytext = ' '.join(attr.asdict(entry).values()) - self.log.info(f"obtained entry text: {entrytext}") + entrytext = ' '.join([str(value) for value in attr.asdict(entry).values()]) for match in re.finditer(r'\{\{ ([^}]*) \}\}', message): - value = re.search(match.groups(0), entrytext) - self.log.info(f"found match: {match} with regex {match.groups(0)} matching {value} in entrytext") - message = message[:match.span()[0]] + value + message[match.span()[1]:] - + value = re.search(re.compile(match.groups(0)[0]), entrytext) + message = message[:match.span()[0]] + (value.group(0) if value is not None else "") + message[match.span()[1]:] + msgtype = MessageType.NOTICE if sub.send_notice else MessageType.TEXT msgchunks = [message[i:i + 30000] for i in range(0, len(message), 30000)] self.log.debug(f"Message length: {len(message)} Content length: {len(entry.content)} Chunks: {len(msgchunks)}") From 7ebfbb0502e27e459e63f30c251c3c96bafca795 Mon Sep 17 00:00:00 2001 From: A git user Date: Fri, 18 Oct 2024 03:17:41 +0200 Subject: [PATCH 17/18] bump version --- maubot.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maubot.yaml b/maubot.yaml index 4cefc3d..8c81470 100644 --- a/maubot.yaml +++ b/maubot.yaml @@ -1,6 +1,6 @@ maubot: 0.3.0 id: xyz.maubot.rss -version: 0.3.2 +version: 0.3.3 license: AGPL-3.0-or-later modules: - rss From d3389776c54c7351366065320625df3ac17114bb Mon Sep 17 00:00:00 2001 From: A git user Date: Fri, 18 Oct 2024 03:21:30 +0200 Subject: [PATCH 18/18] Update readme --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index fd4e134..2a0fb47 100644 --- a/README.md +++ b/README.md @@ -30,3 +30,7 @@ The following variables are available: * `$title` - The title of the entry. * `$summary` - The summary/description of the entry. * `$link` - The link of the entry. + +Furthermore, regular expressions can be used inside the template using double curly braces. +The regular expression will be matched against the entry content and replaced with the match. +E.g., the template string `{{ https://[^ ].jpg }}` will be replaced by the first link to an image file in the entry.