Add postall command and support for rss content

README.md

@@ -12,6 +12,8 @@ Basic commands:
 * `!rss template <feed ID> [new template]` - Change the post template for a
   feed in the current room. If the new template is omitted, the bot replies
   with the current template.
+* `!rss postall <feed ID>` - Post all entries in the specified feed to the
+  current room
 
 ### Templates
 The default template is `New post in $feed_title: [$title]($link)`.
@@ -28,3 +30,7 @@ The following variables are available:
 * `$title` - The title of the entry.
 * `$summary` - The summary/description of the entry.
 * `$link` - The link of the entry.
+
+Furthermore, regular expressions can be used inside the template using double curly braces.
+The regular expression will be matched against the entry content and replaced with the match.
+E.g., the template string `{{ https://[^ ].jpg }}` will be replaced by the first link to an image file in the entry.

maubot.yaml

@@ -1,6 +1,6 @@
 maubot: 0.3.0
 id: xyz.maubot.rss
-version: 0.3.2
+version: 0.3.3
 license: AGPL-3.0-or-later
 modules:
 - rss

rss/bot.py

@@ -18,6 +18,7 @@
 from typing import Any, Iterable
 from datetime import datetime
 from string import Template
+import re
 from time import mktime, time
 import asyncio
 import hashlib
@@ -66,6 +67,8 @@ def match(self, val: str, **kwargs) -> tuple[str, Any]:
             res = False
         elif part in ("t", "true", "y", "yes", "1"):
             res = True
+        elif self.required == False:
+            res = None
         else:
             raise ValueError("invalid boolean")
         return val[len(part) :], res
@@ -115,11 +118,21 @@ async def _send(self, feed: Feed, entry: Entry, sub: Subscription) -> EventID:
                 **attr.asdict(entry),
             }
         )
+        entrytext = ' '.join([str(value) for value in attr.asdict(entry).values()])
+
+        for match in re.finditer(r'\{\{ ([^}]*) \}\}', message):
+            value = re.search(re.compile(match.groups(0)[0]), entrytext)
+            message = message[:match.span()[0]] + (value.group(0) if value is not None else "") + message[match.span()[1]:]
+
         msgtype = MessageType.NOTICE if sub.send_notice else MessageType.TEXT
+        msgchunks = [message[i:i + 30000] for i in range(0, len(message), 30000)]
+        self.log.debug(f"Message length: {len(message)} Content length: {len(entry.content)} Chunks: {len(msgchunks)}")
         try:
-            return await self.client.send_markdown(
-                sub.room_id, message, msgtype=msgtype, allow_html=True
-            )
+            for chunk in msgchunks:
+                returnval = await self.client.send_markdown(
+                    sub.room_id, chunk, msgtype=msgtype, allow_html=True
+                )
+            return returnval
         except Exception as e:
             self.log.warning(f"Failed to send {entry.id} of {feed.id} to {sub.room_id}: {e}")
 
@@ -294,6 +307,9 @@ def _parse_rss_entry(cls, feed_id: int, entry: Any) -> Entry:
             title=getattr(entry, "title", ""),
             summary=getattr(entry, "description", "").strip(),
             link=getattr(entry, "link", ""),
+            content=entry["content"][0]["value"].strip() if hasattr(entry, "content")
+                                                            and len(entry["content"]) > 0
+                                                            and hasattr(entry["content"][0], "value") else "",
         )
 
     @staticmethod
@@ -425,6 +441,7 @@ async def command_template(self, evt: MessageEvent, feed_id: int, template: str)
             title="Sample entry",
             summary="This is a sample entry to demonstrate your new template",
             link="http://example.com",
+            content="<b>Sample formatted content</b>"
         )
         await evt.reply(f"Template for feed ID {feed.id} updated. Sample notification:")
         await self._send(feed, sample_entry, sub)
@@ -445,6 +462,20 @@ async def command_notice(self, evt: MessageEvent, feed_id: int, setting: bool) -
         send_type = "m.notice" if setting else "m.text"
         await evt.reply(f"Updates for feed ID {feed.id} will now be sent as `{send_type}`")
 
+    @rss.subcommand(
+        "postall", aliases=("p",), help="Post all previously seen entries from the given feed to this room"
+    )
+    @command.argument("feed_id", "feed ID", parser=int)
+    async def command_postall(self, evt: MessageEvent, feed_id: int) -> None:
+        if not await self.can_manage(evt):
+            return
+        sub, feed = await self.dbm.get_subscription(feed_id, evt.room_id)
+        if not sub:
+            await evt.reply("This room is not subscribed to that feed")
+            return
+        for entry in await self.dbm.get_entries(feed.id):
+            await self._broadcast(feed, entry, [sub])
+
     @staticmethod
     def _format_subscription(feed: Feed, subscriber: str) -> str:
         msg = (

rss/db.py

@@ -97,6 +97,7 @@ class Entry:
     title: str
     summary: str
     link: str
+    content: str
 
     @classmethod
     def from_row(cls, row: Record | None) -> Entry | None:
@@ -143,7 +144,7 @@ async def get_feeds_by_room(self, room_id: RoomID) -> list[tuple[Feed, UserID]]:
         return [(Feed.from_row(row), row["user_id"]) for row in rows]
 
     async def get_entries(self, feed_id: int) -> list[Entry]:
-        q = "SELECT feed_id, id, date, title, summary, link FROM entry WHERE feed_id = $1"
+        q = "SELECT feed_id, id, date, title, summary, link, content FROM entry WHERE feed_id = $1"
         return [Entry.from_row(row) for row in await self.db.fetch(q, feed_id)]
 
     async def add_entries(self, entries: list[Entry], override_feed_id: int | None = None) -> None:
@@ -153,14 +154,14 @@ async def add_entries(self, entries: list[Entry], override_feed_id: int | None =
             for entry in entries:
                 entry.feed_id = override_feed_id
         records = [attr.astuple(entry) for entry in entries]
-        columns = ("feed_id", "id", "date", "title", "summary", "link")
+        columns = ("feed_id", "id", "date", "title", "summary", "link", "content")
         async with self.db.acquire() as conn:
             if self.db.scheme == Scheme.POSTGRES:
                 await conn.copy_records_to_table("entry", records=records, columns=columns)
             else:
                 q = (
-                    "INSERT INTO entry (feed_id, id, date, title, summary, link) "
-                    "VALUES ($1, $2, $3, $4, $5, $6)"
+                    "INSERT INTO entry (feed_id, id, date, title, summary, link, content) "
+                    "VALUES ($1, $2, $3, $4, $5, $6, $7)"
                 )
                 await conn.executemany(q, records)
 
@@ -218,14 +219,18 @@ async def subscribe(
         room_id: RoomID,
         user_id: UserID,
         template: str | None = None,
-        send_notice: bool = True,
+        send_notice: bool = True
     ) -> None:
         q = """
-        INSERT INTO subscription (feed_id, room_id, user_id, notification_template, send_notice)
+        INSERT INTO subscription (
+            feed_id, room_id, user_id, notification_template,
+            send_notice)
         VALUES ($1, $2, $3, $4, $5)
         """
         template = template or "New post in $feed_title: [$title]($link)"
-        await self.db.execute(q, feed_id, room_id, user_id, template, send_notice)
+        await self.db.execute(
+            q, feed_id, room_id, user_id, template, send_notice,
+        )
 
     async def unsubscribe(self, feed_id: int, room_id: RoomID) -> None:
         q = "DELETE FROM subscription WHERE feed_id = $1 AND room_id = $2"

rss/migrations.py

@@ -17,7 +17,6 @@
 
 upgrade_table = UpgradeTable()
 
-
 @upgrade_table.register(description="Latest revision", upgrades_to=3)
 async def upgrade_latest(conn: Connection, scheme: Scheme) -> None:
     gen = "GENERATED ALWAYS AS IDENTITY" if scheme != Scheme.SQLITE else ""
@@ -72,3 +71,8 @@ async def upgrade_v2(conn: Connection) -> None:
 async def upgrade_v3(conn: Connection) -> None:
     await conn.execute("ALTER TABLE feed ADD COLUMN next_retry BIGINT DEFAULT 0")
     await conn.execute("ALTER TABLE feed ADD COLUMN error_count BIGINT DEFAULT 0")
+
+
+@upgrade_table.register(description="Add support for encoded content")
+async def upgrade_v4(conn: Connection) -> None:
+    await conn.execute("ALTER TABLE entry ADD COLUMN content TEXT")