* [PATCH v1 2/3] First release Discord bot
2022-08-25 16:09 [PATCH v1 0/3] New Discord bot and full refactor scripts Muhammad Rizki
2022-08-25 16:09 ` [PATCH v1 1/3] Move the Telegram bot source code Muhammad Rizki
@ 2022-08-25 16:09 ` Muhammad Rizki
2022-08-26 1:49 ` Alviro Iskandar Setiawan
2022-08-25 16:09 ` [PATCH v1 3/3] Full refactor bot scripts Muhammad Rizki
2 siblings, 1 reply; 8+ messages in thread
From: Muhammad Rizki @ 2022-08-25 16:09 UTC (permalink / raw)
To: Ammar Faizi
Cc: Muhammad Rizki, GNU/Weeb Mailing List, Alviro Iskandar Setiawan
The Discord bot for lore email message has been released.
Signed-off-by: Muhammad Rizki <[email protected]>
---
daemon/discord/.env.example | 8 +
daemon/discord/config.py.example | 9 +
daemon/discord/execute_me.sql | 64 +++++
daemon/discord/gnuweeb/__init__.py | 6 +
daemon/discord/gnuweeb/client.py | 110 ++++++++
daemon/discord/gnuweeb/filters.py | 67 +++++
daemon/discord/gnuweeb/models/__init__.py | 6 +
daemon/discord/gnuweeb/models/ui/__init__.py | 6 +
.../gnuweeb/models/ui/buttons/__init__.py | 6 +
.../models/ui/buttons/full_message_btn.py | 21 ++
daemon/discord/gnuweeb/plugins/__init__.py | 20 ++
.../plugins/basic_commands/__init__.py | 13 +
.../plugins/basic_commands/debugger.py | 20 ++
.../gnuweeb/plugins/basic_commands/sync_it.py | 20 ++
.../gnuweeb/plugins/events/__init__.py | 13 +
.../gnuweeb/plugins/events/on_error.py | 17 ++
.../gnuweeb/plugins/events/on_ready.py | 24 ++
.../plugins/slash_commands/__init__.py | 15 ++
.../plugins/slash_commands/get_lore_mail.py | 47 ++++
.../plugins/slash_commands/manage_atom.py | 81 ++++++
.../slash_commands/manage_broadcast.py | 83 ++++++
daemon/discord/gnuweeb/utils.py | 52 ++++
daemon/discord/mailer/__init__.py | 7 +
daemon/discord/mailer/database.py | 203 +++++++++++++++
daemon/discord/mailer/listener.py | 152 +++++++++++
daemon/discord/mailer/scraper.py | 63 +++++
daemon/discord/mailer/utils.py | 241 ++++++++++++++++++
daemon/discord/requirements.txt | 8 +
daemon/discord/run.py | 48 ++++
daemon/discord/storage/.gitignore | 2 +
daemon/telegram/packages/plugins/admin.py | 42 ---
daemon/telegram/packages/plugins/scrape.py | 86 -------
32 files changed, 1432 insertions(+), 128 deletions(-)
create mode 100644 daemon/discord/.env.example
create mode 100644 daemon/discord/config.py.example
create mode 100644 daemon/discord/execute_me.sql
create mode 100644 daemon/discord/gnuweeb/__init__.py
create mode 100644 daemon/discord/gnuweeb/client.py
create mode 100644 daemon/discord/gnuweeb/filters.py
create mode 100644 daemon/discord/gnuweeb/models/__init__.py
create mode 100644 daemon/discord/gnuweeb/models/ui/__init__.py
create mode 100644 daemon/discord/gnuweeb/models/ui/buttons/__init__.py
create mode 100644 daemon/discord/gnuweeb/models/ui/buttons/full_message_btn.py
create mode 100644 daemon/discord/gnuweeb/plugins/__init__.py
create mode 100644 daemon/discord/gnuweeb/plugins/basic_commands/__init__.py
create mode 100644 daemon/discord/gnuweeb/plugins/basic_commands/debugger.py
create mode 100644 daemon/discord/gnuweeb/plugins/basic_commands/sync_it.py
create mode 100644 daemon/discord/gnuweeb/plugins/events/__init__.py
create mode 100644 daemon/discord/gnuweeb/plugins/events/on_error.py
create mode 100644 daemon/discord/gnuweeb/plugins/events/on_ready.py
create mode 100644 daemon/discord/gnuweeb/plugins/slash_commands/__init__.py
create mode 100644 daemon/discord/gnuweeb/plugins/slash_commands/get_lore_mail.py
create mode 100644 daemon/discord/gnuweeb/plugins/slash_commands/manage_atom.py
create mode 100644 daemon/discord/gnuweeb/plugins/slash_commands/manage_broadcast.py
create mode 100644 daemon/discord/gnuweeb/utils.py
create mode 100644 daemon/discord/mailer/__init__.py
create mode 100644 daemon/discord/mailer/database.py
create mode 100644 daemon/discord/mailer/listener.py
create mode 100644 daemon/discord/mailer/scraper.py
create mode 100644 daemon/discord/mailer/utils.py
create mode 100644 daemon/discord/requirements.txt
create mode 100644 daemon/discord/run.py
create mode 100644 daemon/discord/storage/.gitignore
delete mode 100644 daemon/telegram/packages/plugins/admin.py
delete mode 100644 daemon/telegram/packages/plugins/scrape.py
diff --git a/daemon/discord/.env.example b/daemon/discord/.env.example
new file mode 100644
index 0000000..c2a9de8
--- /dev/null
+++ b/daemon/discord/.env.example
@@ -0,0 +1,8 @@
+# Input your Discord bot token below
+DISCORD_TOKEN=
+
+# Input your MySQL connection below
+DB_HOST=
+DB_USER=
+DB_PASS=
+DB_NAME=
diff --git a/daemon/discord/config.py.example b/daemon/discord/config.py.example
new file mode 100644
index 0000000..0e2f5f5
--- /dev/null
+++ b/daemon/discord/config.py.example
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Copyright (C) 2022 Muhammad Rizki <[email protected]>
+#
+
+# Paste the admin role ID below
+# to filter only admin who can
+# access add/delete lore commands.
+ADMIN_ROLE_ID = 0
diff --git a/daemon/discord/execute_me.sql b/daemon/discord/execute_me.sql
new file mode 100644
index 0000000..80f2a32
--- /dev/null
+++ b/daemon/discord/execute_me.sql
@@ -0,0 +1,64 @@
+-- Adminer 4.7.6 MySQL dump
+
+SET NAMES utf8;
+SET time_zone = '+00:00';
+SET foreign_key_checks = 0;
+SET sql_mode = 'NO_AUTO_VALUE_ON_ZERO';
+
+SET NAMES utf8mb4;
+
+DROP TABLE IF EXISTS `emails`;
+CREATE TABLE `emails` (
+ `id` bigint unsigned NOT NULL AUTO_INCREMENT,
+ `message_id` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci NOT NULL,
+ `created_at` datetime NOT NULL,
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `message_id` (`message_id`),
+ KEY `created_at` (`created_at`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_520_ci;
+
+
+DROP TABLE IF EXISTS `discord_emails`;
+CREATE TABLE `discord_emails` (
+ `id` bigint unsigned NOT NULL AUTO_INCREMENT,
+ `email_id` bigint unsigned NOT NULL,
+ `channel_id` bigint NOT NULL,
+ `dc_msg_id` bigint unsigned NOT NULL,
+ `created_at` datetime NOT NULL,
+ PRIMARY KEY (`id`),
+ KEY `email_id` (`email_id`),
+ KEY `channel_id` (`channel_id`),
+ KEY `dc_msg_id` (`dc_msg_id`),
+ KEY `created_at` (`created_at`),
+ CONSTRAINT `discord_emails_ibfk_2` FOREIGN KEY (`email_id`) REFERENCES `emails` (`id`) ON DELETE CASCADE ON UPDATE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_520_ci;
+
+
+DROP TABLE IF EXISTS `atom_urls`;
+CREATE TABLE `atom_urls` (
+ `id` bigint unsigned NOT NULL AUTO_INCREMENT,
+ `url` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci NOT NULL,
+ `created_at` datetime NOT NULL,
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `url` (`url`),
+ KEY `created_at` (`created_at`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_520_ci;
+
+
+DROP TABLE IF EXISTS `broadcast_chats`;
+CREATE TABLE `broadcast_chats` (
+ `id` bigint unsigned NOT NULL AUTO_INCREMENT,
+ `guild_id` BIGINT UNSIGNED NOT NULL,
+ `channel_id` BIGINT UNSIGNED NOT NULL,
+ `channel_name` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci NOT NULL,
+ `channel_link` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci NOT NULL,
+ `created_at` datetime NOT NULL,
+ PRIMARY KEY (`id`),
+ KEY `guild_id` (`guild_id`),
+ UNIQUE KEY `channel_id` (`channel_id`),
+ KEY `channel_name` (`channel_name`),
+ KEY `channel_link` (`channel_link`),
+ KEY `created_at` (`created_at`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_520_ci;
+
+-- 2022-07-07 14:25:28
diff --git a/daemon/discord/gnuweeb/__init__.py b/daemon/discord/gnuweeb/__init__.py
new file mode 100644
index 0000000..f3d814a
--- /dev/null
+++ b/daemon/discord/gnuweeb/__init__.py
@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Copyright (C) 2022 Muhammad Rizki <[email protected]>
+#
+
+from .client import GWClient
diff --git a/daemon/discord/gnuweeb/client.py b/daemon/discord/gnuweeb/client.py
new file mode 100644
index 0000000..cf88d36
--- /dev/null
+++ b/daemon/discord/gnuweeb/client.py
@@ -0,0 +1,110 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Copyright (C) 2022 Muhammad Rizki <[email protected]>
+#
+
+import discord
+from discord.ext import commands
+from discord import Intents
+from discord import Interaction
+from typing import Union
+
+from .models.ui import buttons
+from . import filters
+from mailer import utils
+from mailer import Database
+
+
+class GWClient(commands.Bot):
+ def __init__(self, db_conn) -> None:
+ self.db = Database(db_conn)
+ self.mailer = None
+ intents = Intents.default()
+ intents.message_content = True
+ super().__init__(
+ command_prefix=["$", "."],
+ description="Just a bot for receiving lore emails.",
+ intents=intents,
+ activity=discord.Game(
+ name="with Columbina Damselette",
+ type=0
+ )
+ )
+
+
+ async def setup_hook(self):
+ await self.load_extension("gnuweeb.plugins")
+
+ # WARNING! NOT RECOMMENDED SYNCING WHEN THE BOT IS START!!
+ # guild = discord.Object(id=845302963739033611)
+ # self.tree.copy_global_to(guild=guild)
+ # await self.tree.sync(guild=guild)
+
+
+ @filters.wait_on_limit
+ async def send_text_email(self, guild_id: int, chat_id: int, text: str,
+ reply_to: Union[int, None] = None, url: str = None):
+ print("[send_text_email]")
+ text = utils.bottom_border(text)
+ channel = self.get_channel(chat_id)
+
+ m = await channel.send(
+ content=text,
+ reference=discord.MessageReference(
+ guild_id=guild_id,
+ channel_id=chat_id,
+ message_id=reply_to
+ ) if reply_to else None,
+
+ view=buttons.FullMessageBtn(url)
+ )
+ return m
+
+
+ @filters.wait_on_limit
+ async def send_patch_email(self, mail, guild_id: int, chat_id: int, text: str,
+ reply_to: Union[int, None] = None, url: str = None):
+ print("[send_patch_email]")
+ tmp, doc, caption, url = utils.prepare_patch(mail, text, url)
+ channel = self.get_channel(chat_id)
+
+ m = await channel.send(
+ content=caption,
+ file=discord.File(doc),
+ reference=discord.MessageReference(
+ guild_id=guild_id,
+ channel_id=chat_id,
+ message_id=reply_to
+ ) if reply_to else None,
+
+ view=buttons.FullMessageBtn(url)
+ )
+
+ utils.remove_patch(tmp)
+ return m
+
+
+ async def send_text_mail_interaction(self, i: "Interaction",
+ text: str, url: str = None):
+ text = utils.border_and_trim(text)
+
+ m = await i.response.send_message(
+ content=text,
+ view=buttons.FullMessageBtn(url)
+ )
+
+ return m
+
+
+ async def send_patch_mail_interaction(self, mail, i: "Interaction",
+ text: str, url: str = None):
+ tmp, doc, caption, url = utils.prepare_patch(mail, text, url)
+
+ m = await i.response.send_message(
+ content=caption,
+ file=discord.File(doc),
+ view=buttons.FullMessageBtn(url)
+ )
+
+ utils.remove_patch(tmp)
+ return m
diff --git a/daemon/discord/gnuweeb/filters.py b/daemon/discord/gnuweeb/filters.py
new file mode 100644
index 0000000..735c77e
--- /dev/null
+++ b/daemon/discord/gnuweeb/filters.py
@@ -0,0 +1,67 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Copyright (C) 2022 Muhammad Rizki <[email protected]>
+#
+
+import config
+import discord
+import asyncio
+from discord import Interaction
+from discord.ext import commands
+from typing import Any, Callable, TypeVar
+from functools import wraps
+
+
+T = TypeVar("T")
+
+def channel(channel_id: int, admin_role_only: bool) -> Callable[[T], T]:
+ def _(func):
+ async def callback(*args: Any, **kwargs: Any) -> Any:
+ ctx: "commands.Context" = args[1]
+
+ if not channel_id == ctx.channel.id: return
+ if not admin_role_only: return
+
+ user_roles = [role.id for role in ctx.author.roles]
+ if config.ADMIN_ROLE_ID in user_roles:
+ return await func(*args, **kwargs,
+ content=ctx.message.content)
+
+ return callback
+ return _
+
+
+def lore_admin(func: Callable[[T], T]) -> Callable[[T], T]:
+ @wraps(func)
+ async def callback(*args: Any, **kwargs: Any) -> Any:
+ i: "Interaction" = args[1]
+ user_roles = [role.id for role in i.user.roles]
+
+ if config.ADMIN_ROLE_ID not in user_roles:
+ return await i.response.send_message(
+ "Sorry, you don't have this permission\n"\
+ "Tell the server admin to add you lore admin role.",
+ ephemeral=True
+ )
+ if config.ADMIN_ROLE_ID in user_roles:
+ return await func(*args, **kwargs)
+
+ return callback
+
+
+def wait_on_limit(func: Callable[[T], T]) -> Callable[[T], T]:
+ @wraps(func)
+ async def callback(*args: Any) -> Any:
+ while True:
+ try:
+ return await func(*args)
+ except discord.errors.RateLimited as e:
+ _flood_exceptions(e)
+ print("[wait_on_limit]: Woken up from flood wait...")
+ return callback
+
+
+async def _flood_exceptions(e: "discord.errors.RateLimited"):
+ wait = e.retry_after
+ print(f"[wait_on_limit]: Sleeping for {wait} seconds due to Telegram limit")
+ await asyncio.sleep(wait)
diff --git a/daemon/discord/gnuweeb/models/__init__.py b/daemon/discord/gnuweeb/models/__init__.py
new file mode 100644
index 0000000..7199f33
--- /dev/null
+++ b/daemon/discord/gnuweeb/models/__init__.py
@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Copyright (C) 2022 Muhammad Rizki <[email protected]>
+#
+
+from .ui import *
diff --git a/daemon/discord/gnuweeb/models/ui/__init__.py b/daemon/discord/gnuweeb/models/ui/__init__.py
new file mode 100644
index 0000000..9625317
--- /dev/null
+++ b/daemon/discord/gnuweeb/models/ui/__init__.py
@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Copyright (C) 2022 Muhammad Rizki <[email protected]>
+#
+
+from .buttons import FullMessageBtn
diff --git a/daemon/discord/gnuweeb/models/ui/buttons/__init__.py b/daemon/discord/gnuweeb/models/ui/buttons/__init__.py
new file mode 100644
index 0000000..c1d2a56
--- /dev/null
+++ b/daemon/discord/gnuweeb/models/ui/buttons/__init__.py
@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Copyright (C) 2022 Muhammad Rizki <[email protected]>
+#
+
+from .full_message_btn import FullMessageBtn
diff --git a/daemon/discord/gnuweeb/models/ui/buttons/full_message_btn.py b/daemon/discord/gnuweeb/models/ui/buttons/full_message_btn.py
new file mode 100644
index 0000000..310d761
--- /dev/null
+++ b/daemon/discord/gnuweeb/models/ui/buttons/full_message_btn.py
@@ -0,0 +1,21 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Copyright (C) 2022 Muhammad Rizki <[email protected]>
+#
+
+from discord import ui
+from discord import ButtonStyle
+
+
+class FullMessageBtn(ui.View):
+ def __init__(self, url: str):
+ super().__init__()
+ self.__add_button(
+ label='See the full message',
+ style=ButtonStyle.gray,
+ url=url
+ )
+
+
+ def __add_button(self, **kwargs):
+ self.add_item(ui.Button(**kwargs))
diff --git a/daemon/discord/gnuweeb/plugins/__init__.py b/daemon/discord/gnuweeb/plugins/__init__.py
new file mode 100644
index 0000000..2902e99
--- /dev/null
+++ b/daemon/discord/gnuweeb/plugins/__init__.py
@@ -0,0 +1,20 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Copyright (C) 2022 Muhammad Rizki <[email protected]>
+#
+
+from discord.ext import commands
+from .basic_commands import BasicCommands
+from .events import Events
+from .slash_commands import SlashCommands
+
+
+class Plugins(
+ BasicCommands,
+ Events,
+ SlashCommands
+): pass
+
+
+async def setup(bot: "commands.Bot"):
+ await bot.add_cog(Plugins(bot))
diff --git a/daemon/discord/gnuweeb/plugins/basic_commands/__init__.py b/daemon/discord/gnuweeb/plugins/basic_commands/__init__.py
new file mode 100644
index 0000000..06a735b
--- /dev/null
+++ b/daemon/discord/gnuweeb/plugins/basic_commands/__init__.py
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Copyright (C) 2022 Muhammad Rizki <[email protected]>
+#
+
+from .debugger import DebuggerCommand
+from .sync_it import SyncCommand
+
+
+class BasicCommands(
+ DebuggerCommand,
+ SyncCommand
+): pass
diff --git a/daemon/discord/gnuweeb/plugins/basic_commands/debugger.py b/daemon/discord/gnuweeb/plugins/basic_commands/debugger.py
new file mode 100644
index 0000000..be5b9f1
--- /dev/null
+++ b/daemon/discord/gnuweeb/plugins/basic_commands/debugger.py
@@ -0,0 +1,20 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Copyright (C) 2022 Muhammad Rizki <[email protected]>
+#
+
+from discord.ext import commands
+from gnuweeb import utils
+# from gnuweeb import filters
+
+
+class DebuggerCommand(commands.Cog):
+ def __init__(self, bot) -> None:
+ self.bot = bot
+
+
+ @commands.command("exec", aliases=["exc", "d"])
+ @commands.is_owner() # only the bot owner who have this access
+ # @filters.channel(channel_id=865963658726211604, admin_role_only=True)
+ async def exec(self, ctx: "commands.Context", *, content: str):
+ await utils.execute_python_code(ctx, content)
diff --git a/daemon/discord/gnuweeb/plugins/basic_commands/sync_it.py b/daemon/discord/gnuweeb/plugins/basic_commands/sync_it.py
new file mode 100644
index 0000000..2ed35cc
--- /dev/null
+++ b/daemon/discord/gnuweeb/plugins/basic_commands/sync_it.py
@@ -0,0 +1,20 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Copyright (C) 2022 Muhammad Rizki <[email protected]>
+#
+
+from discord.ext import commands
+
+
+class SyncCommand(commands.Cog):
+ def __init__(self, bot) -> None:
+ self.bot = bot
+
+
+ @commands.command("sync", aliases=["s"])
+ @commands.is_owner()
+ async def sync_it(self, ctx: "commands.Context"):
+ ctx.bot.tree.copy_global_to(guild=ctx.guild)
+ s = await ctx.bot.tree.sync(guild=ctx.guild)
+
+ await ctx.send(f"Synced {len(s)} commands.")
diff --git a/daemon/discord/gnuweeb/plugins/events/__init__.py b/daemon/discord/gnuweeb/plugins/events/__init__.py
new file mode 100644
index 0000000..7074e4d
--- /dev/null
+++ b/daemon/discord/gnuweeb/plugins/events/__init__.py
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Copyright (C) 2022 Muhammad Rizki <[email protected]>
+#
+
+from .on_ready import OnReady
+from .on_error import OnError
+
+
+class Events(
+ OnReady,
+ OnError
+): pass
diff --git a/daemon/discord/gnuweeb/plugins/events/on_error.py b/daemon/discord/gnuweeb/plugins/events/on_error.py
new file mode 100644
index 0000000..e1e4e28
--- /dev/null
+++ b/daemon/discord/gnuweeb/plugins/events/on_error.py
@@ -0,0 +1,17 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Copyright (C) 2022 Muhammad Rizki <[email protected]>
+#
+
+from discord.ext import commands
+
+
+class OnError(commands.Cog):
+ def __init__(self, bot: "commands.Bot") -> None:
+ self.bot = bot
+
+
+ @commands.Cog.listener()
+ async def on_command_error(self, _, err):
+ if isinstance(err, commands.CommandNotFound):
+ pass
diff --git a/daemon/discord/gnuweeb/plugins/events/on_ready.py b/daemon/discord/gnuweeb/plugins/events/on_ready.py
new file mode 100644
index 0000000..1554050
--- /dev/null
+++ b/daemon/discord/gnuweeb/plugins/events/on_ready.py
@@ -0,0 +1,24 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Copyright (C) 2022 Muhammad Rizki <[email protected]>
+#
+
+from discord.ext import commands
+
+
+class OnReady(commands.Cog):
+ def __init__(self, bot: "commands.Bot") -> None:
+ self.bot = bot
+
+
+ @commands.Cog.listener()
+ async def on_ready(self):
+ t = "[ GNU/Weeb Bot is connected ]\n\n"
+ t += f"ID : {self.bot.user.id}\n"
+ t += f"Name : {self.bot.user.display_name}\n"
+ t += f"Tags : {self.bot.user}\n\n"
+ t += "Ready to get the latest of lore kernel emails."
+ # starting to listen new emails when the bot
+ # is ready to sending the messages.
+ self.bot.mailer.run()
+ print(t)
diff --git a/daemon/discord/gnuweeb/plugins/slash_commands/__init__.py b/daemon/discord/gnuweeb/plugins/slash_commands/__init__.py
new file mode 100644
index 0000000..6e5929a
--- /dev/null
+++ b/daemon/discord/gnuweeb/plugins/slash_commands/__init__.py
@@ -0,0 +1,15 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Copyright (C) 2022 Muhammad Rizki <[email protected]>
+#
+
+from .manage_atom import ManageAtomSC
+from .manage_broadcast import ManageBroadcastSC
+from .get_lore_mail import GetLoreSC
+
+
+class SlashCommands(
+ ManageAtomSC,
+ ManageBroadcastSC,
+ GetLoreSC
+): pass
diff --git a/daemon/discord/gnuweeb/plugins/slash_commands/get_lore_mail.py b/daemon/discord/gnuweeb/plugins/slash_commands/get_lore_mail.py
new file mode 100644
index 0000000..a2671f4
--- /dev/null
+++ b/daemon/discord/gnuweeb/plugins/slash_commands/get_lore_mail.py
@@ -0,0 +1,47 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Copyright (C) 2022 Muhammad Rizki <[email protected]>
+#
+
+import shutil
+import asyncio
+import discord
+from discord.ext import commands
+from discord import Interaction
+from discord import app_commands
+
+from mailer import utils
+from mailer import Scraper
+
+
+class GetLoreSC(commands.Cog):
+ def __init__(self, bot) -> None:
+ self.bot = bot
+
+
+ @app_commands.command(
+ name="lore",
+ description="Get lore email from raw email URL."
+ )
+ @app_commands.describe(url="Raw lore email URL")
+ async def get_lore(self, i: "Interaction", url: str):
+ s = Scraper()
+ mail = await s.get_email_from_url(url)
+ text, files, is_patch = utils.create_template(mail)
+
+ if is_patch:
+ m = await self.bot.send_patch_mail_interaction(
+ mail=mail, i=i, text=text, url=url
+ )
+ else:
+ text = "#ml\n" + text
+ m = await self.bot.send_text_mail_interaction(
+ i=i, text=text, url=url
+ )
+
+ for d, f in files:
+ await m.reply(f"{d}/{f}", file=discord.File(f))
+ await asyncio.sleep(1)
+
+ if files:
+ shutil.rmtree(str(files[0][0]))
diff --git a/daemon/discord/gnuweeb/plugins/slash_commands/manage_atom.py b/daemon/discord/gnuweeb/plugins/slash_commands/manage_atom.py
new file mode 100644
index 0000000..2d9da80
--- /dev/null
+++ b/daemon/discord/gnuweeb/plugins/slash_commands/manage_atom.py
@@ -0,0 +1,81 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Copyright (C) 2022 Muhammad Rizki <[email protected]>
+#
+
+from discord.ext import commands
+from discord import Interaction
+from discord import app_commands
+
+from gnuweeb import filters
+from mailer import utils
+
+
+class ManageAtomSC(commands.Cog):
+ atom = app_commands.Group(
+ name="atom",
+ description="Manage lore atom URL."
+ )
+
+ def __init__(self, bot) -> None:
+ self.bot = bot
+
+
+ @atom.command(
+ name="list",
+ description="List of lore atom URL."
+ )
+ @filters.lore_admin
+ async def list_atom(self, i: "Interaction"):
+ atoms = self.bot.db.get_atom_urls()
+ if len(atoms) == 0:
+ t = "Currently empty."
+ await i.response.send_message(t, ephemeral=True)
+ return
+
+ text = "List of atom URL that currently listened:\n"
+ for u,n in zip(atoms, range(1, len(atoms)+1)):
+ text += f"{n}. {u}\n"
+
+ await i.response.send_message(text, ephemeral=True)
+
+
+ @atom.command(
+ name="add",
+ description="Add lore atom URL for receiving lore emails."
+ )
+ @app_commands.describe(url='Lore atom URL.')
+ @filters.lore_admin
+ async def add_atom(self, i: "Interaction", url: str):
+ is_atom = await utils.is_atom_url(url)
+ if not is_atom:
+ t = "Invalid Atom URL."
+ await i.response.send_message(t, ephemeral=True)
+ return
+
+ inserted = self.bot.db.insert_atom(url)
+ if inserted is None:
+ t = f"This URL already listened for new email."
+ await i.response.send_message(t, ephemeral=True)
+ return
+
+ t = f"Success add **{url}** for listening new email."
+ await i.response.send_message(t, ephemeral=True)
+
+
+ @atom.command(
+ name="delete",
+ description="Delete lore atom URL from receiving lore emails."
+ )
+ @app_commands.describe(url='Lore atom URL.')
+ @filters.lore_admin
+ async def del_atom(self, i: "Interaction", url: str):
+ success = self.bot.db.delete_atom(url)
+ if not success:
+ t = "Failed to delete atom URL\n"
+ t += "Maybe because already deleted or not exists."
+ await i.response.send_message(t, ephemeral=True)
+ return
+
+ t = f"Success delete **{url}** from receiving lore emails."
+ await i.response.send_message(t, ephemeral=True)
diff --git a/daemon/discord/gnuweeb/plugins/slash_commands/manage_broadcast.py b/daemon/discord/gnuweeb/plugins/slash_commands/manage_broadcast.py
new file mode 100644
index 0000000..9eb6b98
--- /dev/null
+++ b/daemon/discord/gnuweeb/plugins/slash_commands/manage_broadcast.py
@@ -0,0 +1,83 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Copyright (C) 2022 Muhammad Rizki <[email protected]>
+#
+
+from discord.ext import commands
+from discord import Interaction
+from discord import app_commands
+
+from gnuweeb import utils
+from gnuweeb import filters
+
+
+class ManageBroadcastSC(commands.Cog):
+ broadcast = app_commands.Group(
+ name="broadcast",
+ description="Manage broadcast channel."
+ )
+
+ def __init__(self, bot) -> None:
+ self.bot = bot
+
+
+ @broadcast.command(
+ name="list",
+ description="List of broadcast channel."
+ )
+ @filters.lore_admin
+ async def list_channel(self, i: "Interaction"):
+ chats = self.bot.db.get_broadcast_chats()
+ if len(chats) == 0:
+ t = "Currently empty."
+ await i.response.send_message(t, ephemeral=True)
+ return
+
+ text = "List of channels that will send email messages:\n"
+ for u,n in zip(chats, range(1, len(chats)+1)):
+ text += f"{n}. **{u[3]}**\n"
+ text += f"Link: {u[4]}\n\n"
+
+ await i.response.send_message(text, ephemeral=True)
+
+
+ @broadcast.command(
+ name="add",
+ description="Add broadcast channel for sending lore emails."
+ )
+ @filters.lore_admin
+ async def add_channel(self, i: "Interaction"):
+ inserted = self.bot.db.insert_broadcast(
+ guild_id=i.guild_id,
+ channel_id=i.channel_id,
+ channel_name=i.channel.name,
+ channel_link=utils.channel_link(
+ guild_id=i.guild_id,
+ channel_id=i.channel_id
+ )
+ )
+
+ if inserted is None:
+ t = f"This channel already added for send email messages."
+ await i.response.send_message(t, ephemeral=True)
+ return
+
+ t = f"Success add this channel for send email messages."
+ await i.response.send_message(t, ephemeral=True)
+
+
+ @broadcast.command(
+ name="delete",
+ description="Delete broadcast channel from sending email messages."
+ )
+ @filters.lore_admin
+ async def del_channel(self, i: "Interaction"):
+ success = self.bot.db.delete_broadcast(i.channel_id)
+ if not success:
+ t = "Failed to delete this channel\n"
+ t += "Maybe because already deleted or not exists."
+ await i.response.send_message(t, ephemeral=True)
+ return
+
+ t = f"Success delete this channel from sending email messages."
+ await i.response.send_message(t, ephemeral=True)
diff --git a/daemon/discord/gnuweeb/utils.py b/daemon/discord/gnuweeb/utils.py
new file mode 100644
index 0000000..d60e932
--- /dev/null
+++ b/daemon/discord/gnuweeb/utils.py
@@ -0,0 +1,52 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Copyright (C) 2022 Muhammad Rizki <[email protected]>
+#
+
+import re
+import io
+import import_expression
+import contextlib
+import traceback
+from discord.ext import commands
+from textwrap import indent
+
+
+def channel_link(guild_id: int, channel_id: int):
+ return f"https://discord.com/channels/{guild_id}/{channel_id}"
+
+
+async def execute_python_code(ctx: "commands.Context", content: str):
+ reg = re.compile(r"`{3}py\n([\w\W]*?)`{3}", flags=re.DOTALL)
+ code = reg.search(content).group(1)
+
+ env = {"ctx": ctx}
+ env.update(globals())
+
+ stdout = io.StringIO()
+ to_compile = f'async def func(ctx):\n{indent(code, " ")}'
+
+ try:
+ import_expression.exec(to_compile, env)
+ except Exception as e:
+ o = f"```py\n{e.__class__.__name__}: {e}"[0:1904]+"\n```"
+ return await ctx.send(o)
+
+ func = env["func"]
+
+ try:
+ with contextlib.redirect_stdout(stdout):
+ ret = await func(ctx)
+ except Exception:
+ value = stdout.getvalue()
+ o = f"```{value}{traceback.format_exc()}"[0:1904]+"```"
+ return await ctx.send(o)
+ else:
+ value = stdout.getvalue()
+ if ret is None:
+ if value:
+ o = f"```py\n{value}"[0:1904]+"\n```"
+ return await ctx.send(o)
+ else:
+ o = f"```py\n{value}{ret}"[0:1904]+"\n```"
+ return await ctx.send(o)
diff --git a/daemon/discord/mailer/__init__.py b/daemon/discord/mailer/__init__.py
new file mode 100644
index 0000000..0da4329
--- /dev/null
+++ b/daemon/discord/mailer/__init__.py
@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Copyright (C) 2022 Muhammad Rizki <[email protected]>
+#
+
+from .database import Database
+from .scraper import Scraper
diff --git a/daemon/discord/mailer/database.py b/daemon/discord/mailer/database.py
new file mode 100644
index 0000000..0b91558
--- /dev/null
+++ b/daemon/discord/mailer/database.py
@@ -0,0 +1,203 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Copyright (C) 2022 Muhammad Rizki <[email protected]>
+# Copyright (C) 2022 Ammar Faizi <[email protected]>
+#
+
+from datetime import datetime
+import mysql
+
+
+class Database:
+ def __init__(self, conn):
+ self.conn = conn
+ self.conn.autocommit = True
+ self.cur = self.conn.cursor(buffered=True)
+
+
+ def __del__(self):
+ self.cur.close()
+ self.conn.close()
+
+
+ def save_email_msg_id(self, email_msg_id):
+ try:
+ return self.__save_email_msg_id(email_msg_id)
+ except mysql.connector.errors.IntegrityError:
+ #
+ # Duplicate data, skip!
+ #
+ return None
+
+
+ def __save_email_msg_id(self, email_msg_id):
+ q = "INSERT INTO emails (message_id, created_at) VALUES (%s, %s)"
+ self.cur.execute(q, (email_msg_id, datetime.utcnow()))
+ return self.cur.lastrowid
+
+
+ def insert_discord(self, email_id, channel_id, dc_msg_id):
+ q = """
+ INSERT INTO discord_emails
+ (email_id, channel_id, dc_msg_id, created_at)
+ VALUES (%s, %s, %s, %s);
+ """
+ self.cur.execute(q, (email_id, channel_id, dc_msg_id,
+ datetime.utcnow()))
+ return self.cur.lastrowid
+
+
+ #
+ # Determine whether the email needs to be sent to @dc_chat_id.
+ #
+ # - Return an email id (PK) if it needs to be sent.
+ # - Return None if it doesn't need to be sent.
+ #
+ def get_email_id_sent(self, email_msg_id, channel_id):
+ q = """
+ SELECT emails.id, discord_emails.id FROM emails
+ LEFT JOIN discord_emails
+ ON emails.id = discord_emails.email_id
+ WHERE emails.message_id = %(email_msg_id)s
+ AND discord_emails.channel_id = %(channel_id)s
+ LIMIT 1
+ """
+
+ self.cur.execute(
+ q,
+ {
+ "email_msg_id": email_msg_id,
+ "channel_id": channel_id
+ }
+ )
+ res = self.cur.fetchone()
+ if bool(res):
+ #
+ # This email has already been sent to
+ # @dc_chat_id.
+ #
+ return None
+
+ q = """
+ SELECT id FROM emails WHERE message_id = %(email_msg_id)s
+ """
+ self.cur.execute(q, {"email_msg_id": email_msg_id})
+ res = self.cur.fetchone()
+ if not bool(res):
+ #
+ # Something goes wrong, skip!
+ #
+ return None
+
+ return int(res[0])
+
+
+ def get_discord_reply(self, email_msg_id, channel_id):
+ q = """
+ SELECT discord_emails.dc_msg_id
+ FROM emails INNER JOIN discord_emails
+ ON emails.id = discord_emails.email_id
+ WHERE emails.message_id = %(email_msg_id)s
+ AND discord_emails.channel_id = %(channel_id)s
+ """
+
+ self.cur.execute(
+ q,
+ {
+ "email_msg_id": email_msg_id,
+ "channel_id": channel_id
+ }
+ )
+ res = self.cur.fetchone()
+ if not bool(res):
+ return None
+
+ return res[0]
+
+
+ def insert_atom(self, atom: str):
+ try:
+ return self.__save_atom(atom)
+ except mysql.connector.errors.IntegrityError:
+ #
+ # Duplicate data, skip!
+ #
+ return None
+
+
+ def __save_atom(self, atom: str):
+ q = "INSERT INTO atom_urls (url, created_at) VALUES (%s, %s)"
+ self.cur.execute(q, (atom, datetime.utcnow()))
+ return self.cur.lastrowid
+
+
+ def delete_atom(self, atom: str):
+ q = """
+ DELETE FROM atom_urls
+ WHERE url = %(atom)s
+ """
+ self.cur.execute(q, {"atom": atom})
+ return self.cur.rowcount > 0
+
+
+ def get_atom_urls(self):
+ q = """
+ SELECT atom_urls.url
+ FROM atom_urls
+ """
+ self.cur.execute(q)
+ urls = self.cur.fetchall()
+
+ return [u[0] for u in urls]
+
+
+ def insert_broadcast(
+ self,
+ guild_id: int,
+ channel_id: int,
+ channel_name: str,
+ channel_link: str = None,
+ ):
+ try:
+ return self.__save_broadcast(
+ guild_id=guild_id,
+ channel_id=channel_id,
+ channel_name=channel_name,
+ channel_link=channel_link
+ )
+ except mysql.connector.errors.IntegrityError:
+ #
+ # Duplicate data, skip!
+ #
+ return None
+
+
+ def __save_broadcast(
+ self,
+ guild_id: int,
+ channel_id: int,
+ channel_name: str,
+ channel_link: str = None,
+ ):
+ q = """
+ INSERT INTO broadcast_chats
+ (guild_id, channel_id, channel_name, channel_link, created_at)
+ VALUES (%s, %s, %s, %s, %s)
+ """
+ values = (guild_id, channel_id, channel_name, channel_link, datetime.utcnow())
+ self.cur.execute(q, values)
+ return self.cur.lastrowid
+
+
+ def delete_broadcast(self, channel_id: int):
+ q = """
+ DELETE FROM broadcast_chats
+ WHERE channel_id = %(channel_id)s
+ """
+ self.cur.execute(q, {"channel_id": channel_id})
+ return self.cur.rowcount > 0
+
+
+ def get_broadcast_chats(self):
+ self.cur.execute("SELECT * FROM broadcast_chats")
+ return self.cur.fetchall()
diff --git a/daemon/discord/mailer/listener.py b/daemon/discord/mailer/listener.py
new file mode 100644
index 0000000..523a87d
--- /dev/null
+++ b/daemon/discord/mailer/listener.py
@@ -0,0 +1,152 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Copyright (C) 2022 Muhammad Rizki <[email protected]>
+# Copyright (C) 2022 Ammar Faizi <[email protected]>
+#
+
+import asyncio
+import traceback
+import shutil
+import re
+from apscheduler.schedulers.asyncio import AsyncIOScheduler
+from discord import File
+
+from gnuweeb import GWClient
+from .scraper import Scraper
+from . import utils
+
+
+class BotMutexes():
+ def __init__(self):
+ self.lock = asyncio.Lock()
+
+
+class Listener():
+ def __init__(
+ self,
+ client: "GWClient",
+ sched: "AsyncIOScheduler",
+ scraper: Scraper,
+ mutexes: "BotMutexes"
+ ):
+ self.client = client
+ self.sched = sched
+ self.scraper = scraper
+ self.mutexes = mutexes
+ self.db = client.db
+ self.isRunnerFixed = False
+ self.runner = None
+
+
+ def run(self):
+ #
+ # Execute __run() once to avoid high latency at
+ # initilization.
+ #
+ self.sched.start()
+ self.runner = self.sched.add_job(func=self.__run)
+
+
+ async def __run(self):
+ print("[__run]: Running...")
+ for url in self.db.get_atom_urls():
+ try:
+ await self.__handle_atom_url(url)
+ except:
+ print(traceback.format_exc())
+
+ if not self.isRunnerFixed:
+ self.isRunnerFixed = True
+ self.runner = self.sched.add_job(
+ func=self.__run,
+ trigger="interval",
+ seconds=30,
+ misfire_grace_time=None,
+ max_instances=1
+ )
+
+
+ async def __handle_atom_url(self, url):
+ urls = await self.scraper.get_new_threads_urls(url)
+ for url in urls:
+ mail = await self.scraper.get_email_from_url(url)
+ await self.__handle_mail(url, mail)
+
+
+ async def __handle_mail(self, url, mail):
+ chats = self.db.get_broadcast_chats()
+ for chat in chats:
+ async with self.mutexes.lock:
+ should_wait = \
+ await self.__send_to_discord(url, mail,
+ chat[1], chat[2])
+
+ if should_wait:
+ await asyncio.sleep(1)
+
+
+ # @__must_hold(self.mutexes.lock)
+ async def __send_to_discord(self, url, mail, dc_guild_id, dc_chat_id):
+ email_msg_id = utils.get_email_msg_id(mail)
+ if not email_msg_id:
+ #
+ # It doesn't have a Message-Id.
+ # A malformed email. Skip!
+ #
+ return False
+
+ email_id = self.__get_email_id_sent(
+ email_msg_id=email_msg_id,
+ dc_chat_id=dc_chat_id
+ )
+ if not email_id:
+ #
+ # Email has already been sent to Discord.
+ # Skip!
+ #
+ return False
+
+ text, files, is_patch = utils.create_template(mail)
+ reply_to = self.get_discord_reply(mail, dc_chat_id)
+ url = str(re.sub(r"/raw$", "", url))
+
+ if is_patch:
+ m = await self.client.send_patch_email(
+ mail, dc_guild_id, dc_chat_id, text, reply_to, url
+ )
+ else:
+ text = "#ml\n" + text
+ m = await self.client.send_text_email(
+ dc_guild_id, dc_chat_id, text, reply_to, url
+ )
+
+ self.db.insert_discord(email_id, m.channel.id, m.id)
+ for d, f in files:
+ await m.reply(f"{d}/{f}", file=File(f))
+ await asyncio.sleep(1)
+
+ if files:
+ shutil.rmtree(str(files[0][0]))
+
+ return True
+
+
+ def __get_email_id_sent(self, email_msg_id, dc_chat_id):
+ email_id = self.db.save_email_msg_id(email_msg_id)
+ if email_id:
+ return email_id
+
+ email_id = self.db.get_email_id_sent(email_msg_id, dc_chat_id)
+ return email_id
+
+
+ def get_discord_reply(self, mail, dc_chat_id):
+ reply_to = mail.get("in-reply-to")
+ if not reply_to:
+ return None
+
+ reply_to = utils.extract_email_msg_id(reply_to)
+ if not reply_to:
+ return None
+
+ return self.db.get_discord_reply(reply_to, dc_chat_id)
diff --git a/daemon/discord/mailer/scraper.py b/daemon/discord/mailer/scraper.py
new file mode 100644
index 0000000..3873161
--- /dev/null
+++ b/daemon/discord/mailer/scraper.py
@@ -0,0 +1,63 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Copyright (C) 2022 Muhammad Rizki <[email protected]>
+# Copyright (C) 2022 Ammar Faizi <[email protected]>
+#
+
+from typing import Dict, List
+import email.policy
+import xmltodict
+import httpx
+import email
+
+
+class Scraper:
+ async def get_new_threads_urls(self, atom_url):
+ ret = await self.__get_atom_content(atom_url)
+ return await self.__get_new_threads_from_atom(ret)
+
+
+ async def __get_atom_content(self, atom_url):
+ async with httpx.AsyncClient() as client:
+ res = await client.get(atom_url)
+ if res.status_code == 200:
+ return res.text
+ raise Exception(f"[get_atom_content]: Returned {res.status_code} HTTP code")
+
+
+ async def __get_new_threads_from_atom(self, atom):
+ j: Dict[str, List[
+ Dict[str, str]
+ ]] = xmltodict.parse(atom)["feed"]
+
+ entry = []
+ e = j["entry"]
+ for i in e:
+ entry.append({
+ "link": i["link"]["@href"],
+ "title": i["title"],
+ "updated": i["updated"],
+ })
+ #
+ # TODO(ammarfaizi2): Sort by title as well if the @updated is
+ # identic.
+ #
+ entry.sort(key=lambda x: x["updated"])
+
+ ret = []
+ for i in entry:
+ link = i["link"].replace("http://", "https://")
+ ret.append(link + "raw")
+
+ return ret
+
+
+ async def get_email_from_url(self, url):
+ async with httpx.AsyncClient() as client:
+ res = await client.get(url)
+ if res.status_code == 200:
+ return email.message_from_string(
+ res.text,
+ policy=email.policy.default
+ )
+ raise Exception(f"[get_atom_content]: Returned {res.status_code} HTTP code")
diff --git a/daemon/discord/mailer/utils.py b/daemon/discord/mailer/utils.py
new file mode 100644
index 0000000..0036f24
--- /dev/null
+++ b/daemon/discord/mailer/utils.py
@@ -0,0 +1,241 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Copyright (C) 2022 Muhammad Rizki <[email protected]>
+# Copyright (C) 2022 Ammar Faizi <[email protected]>
+#
+
+from email.message import Message
+from typing import Dict
+from slugify import slugify
+import hashlib
+import uuid
+import os
+import re
+import shutil
+import httpx
+
+
+def get_email_msg_id(mail):
+ ret = mail.get("message-id")
+ if not ret:
+ return None
+
+ ret = re.search(r"<([^\<\>]+)>", ret)
+ if not ret:
+ return None
+
+ return ret.group(1)
+
+
+#
+# This increments the @i while we are seeing a whitespace.
+#
+def __skip_whitespace(i, ss_len, ss):
+ while i < ss_len:
+ c = ss[i]
+ if c != ' ' and c != '\t' and c != '\n':
+ break
+ i += 1
+
+ return i
+
+
+#
+# Pick a single element in the list. The delimiter here is
+# a comma char ','. But note that when are inside a double
+# quotes, we must not take the comma as a delimiter.
+#
+def __pick_element(i, ss_len, ss, ret):
+ acc = ""
+ in_quotes = False
+
+ while i < ss_len:
+ c = ss[i]
+ i += 1
+
+ if c == '"':
+ in_quotes = (not in_quotes)
+
+ if not in_quotes and c == ',':
+ break
+
+ acc += c
+
+ if acc != "":
+ ret.append(acc)
+
+ return i
+
+
+def __extract_list(ss):
+ ss = ss.strip()
+ ss_len = len(ss)
+ ret = []
+ i = 0
+
+ while i < ss_len:
+ i = __skip_whitespace(i, ss_len, ss)
+ i = __pick_element(i, ss_len, ss, ret)
+
+ return ret
+
+
+def extract_list(key: str, content: Dict[str, str]):
+ people = content.get(key.lower())
+ if not people:
+ return []
+ return __extract_list(people)
+
+
+def consruct_to_n_cc(to: list, cc: list):
+ NR_MAX_LIST = 20
+
+ n = 0
+ ret = ""
+ for i in to:
+ if n >= NR_MAX_LIST:
+ ret += "To: ...\n"
+ break
+
+ n += 1
+ ret += f"To: {i}\n"
+
+ for i in cc:
+ if n >= NR_MAX_LIST:
+ ret += "Cc: ...\n"
+ break
+
+ n += 1
+ ret += f"Cc: {i}\n"
+
+ return ret
+
+
+def gen_temp(name: str):
+ md5 = hashlib.md5(name.encode()).hexdigest()
+ ret = os.getenv("STORAGE_DIR", "storage") + "/" + md5
+ try:
+ os.mkdir(ret)
+ except FileExistsError:
+ pass
+
+ return ret
+
+
+def extract_body(thread: Message):
+ if not thread.is_multipart():
+ p = thread.get_payload(decode=True)
+ return f"{p.decode(errors='replace')}\n".lstrip(), []
+
+ ret = ""
+ files = []
+ temp = gen_temp(str(uuid.uuid4()))
+ for p in thread.get_payload():
+ fname = p.get_filename()
+ payload = p.get_payload(decode=True)
+
+ if not payload:
+ continue
+
+ if 'inline' in [p.get('content-disposition')] or not bool(fname):
+ ret += f"{payload.decode(errors='replace')}\n".lstrip()
+ continue
+
+ with open(f"{temp}/{fname}", "wb") as f:
+ f.write(payload)
+ files.append((temp, fname))
+
+ ret = re.sub("^(>)", ">>> \\1", ret, 1, re.MULTILINE)
+ return ret, files
+
+
+
+PATCH_PATTERN = r"^\[.*(?:patch|rfc).*?(?:(\d+)\/(\d+))?\](.+)"
+def __is_patch(subject, content):
+ x = re.search(PATCH_PATTERN, subject, re.IGNORECASE)
+ if not x or x.group(1) == "0":
+ return False
+
+ x = re.search(r"diff --git", content)
+ if not x:
+ return False
+
+ return True
+
+
+def create_template(thread: Message, to=None, cc=None):
+ if not to:
+ to = extract_list("to", thread)
+ if not cc:
+ cc = extract_list("cc", thread)
+
+ subject = thread.get('subject')
+ ret = f"From: {thread.get('from')}\n"
+ ret += consruct_to_n_cc(to, cc)
+ ret += f"Date: {thread.get('date')}\n"
+ ret += f"Subject: {subject}\n\n"
+ content, files = extract_body(thread)
+ is_patch = __is_patch(subject, content)
+
+ if is_patch:
+ ret += content
+ else:
+ ret += content.strip().replace("\t", " ")
+ if len(ret) >= 1900:
+ ret = ret[:1900] + "..."
+
+ ret = fix_utf8_char(ret)
+
+ return ret, files, is_patch
+
+
+def prepare_patch(mail, text, url):
+ tmp = gen_temp(url)
+ fnm = str(mail.get("subject"))
+ sch = re.search(PATCH_PATTERN, fnm, re.IGNORECASE)
+
+ nr_patch = sch.group(1)
+ if not nr_patch:
+ nr_patch = 1
+ else:
+ nr_patch = int(nr_patch)
+
+ num = "%04d" % nr_patch
+ fnm = slugify(sch.group(3)).replace("_", "-")
+ file = f"{tmp}/{num}-{fnm}.patch"
+
+ with open(file, "wb") as f:
+ f.write(bytes(text, encoding="utf8"))
+
+ caption = "#patch #ml"
+ return tmp, file, caption, url
+
+
+def remove_patch(tmp):
+ shutil.rmtree(tmp)
+
+
+def fix_utf8_char(text: str):
+ return text.rstrip().replace("�"," ")
+
+
+def bottom_border(text: str):
+ return text + "\n" + "-"*72
+
+
+EMAIL_MSG_ID_PATTERN = r"<([^\<\>]+)>"
+def extract_email_msg_id(msg_id):
+ ret = re.search(EMAIL_MSG_ID_PATTERN, msg_id)
+ if not ret:
+ return None
+ return ret.group(1)
+
+
+async def is_atom_url(text: str):
+ try:
+ async with httpx.AsyncClient() as ses:
+ res = await ses.get(text)
+ mime = res.headers.get("Content-Type")
+
+ return mime == "application/atom+xml"
+ except: return False
diff --git a/daemon/discord/requirements.txt b/daemon/discord/requirements.txt
new file mode 100644
index 0000000..adc5a91
--- /dev/null
+++ b/daemon/discord/requirements.txt
@@ -0,0 +1,8 @@
+discord.py
+apscheduler
+python-slugify
+httpx
+import-expression
+mysql-connector-python
+python-dotenv
+xmltodict
diff --git a/daemon/discord/run.py b/daemon/discord/run.py
new file mode 100644
index 0000000..e3eb7a6
--- /dev/null
+++ b/daemon/discord/run.py
@@ -0,0 +1,48 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Copyright (C) 2022 Muhammad Rizki <[email protected]>
+#
+
+import os
+from mysql import connector
+from dotenv import load_dotenv
+from apscheduler.schedulers.asyncio import AsyncIOScheduler
+from mailer.listener import BotMutexes
+from mailer.listener import Listener
+
+from gnuweeb import GWClient
+from mailer import Scraper
+
+
+def main():
+ load_dotenv()
+
+ sched = AsyncIOScheduler(
+ job_defaults={
+ "max_instances": 3,
+ "misfire_grace_time": None
+ }
+ )
+
+ client = GWClient(
+ db_conn=connector.connect(
+ host=os.getenv("DB_HOST"),
+ user=os.getenv("DB_USER"),
+ password=os.getenv("DB_PASS"),
+ database=os.getenv("DB_NAME")
+ )
+ )
+
+ mailer = Listener(
+ client=client,
+ sched=sched,
+ scraper=Scraper(),
+ mutexes=BotMutexes()
+ )
+ client.mailer = mailer
+
+ client.run(os.getenv("DISCORD_TOKEN"))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/daemon/discord/storage/.gitignore b/daemon/discord/storage/.gitignore
new file mode 100644
index 0000000..d6b7ef3
--- /dev/null
+++ b/daemon/discord/storage/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore
diff --git a/daemon/telegram/packages/plugins/admin.py b/daemon/telegram/packages/plugins/admin.py
deleted file mode 100644
index e0f145e..0000000
--- a/daemon/telegram/packages/plugins/admin.py
+++ /dev/null
@@ -1,42 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-#
-# Copyright (C) 2022 Muhammad Rizki <[email protected]>
-#
-
-from pyrogram import Client, filters, enums
-from pyrogram.types import Message
-from textwrap import indent
-import io, import_expression, contextlib, traceback
-
[email protected]_message(
- filters.command(['d','debug']) &
- filters.user(["nekoha", "kiizuah"])
-)
-async def execute_v2(c: Client, m: Message):
- sep = m.text.split('\n')
- body = m.text.replace(sep[0] + '\n','')
-
- env = {"bot": c}
- env.update(globals())
-
- stdout = io.StringIO()
- to_compile = f'async def func(_, m):\n{indent(body, " ")}'
-
- try:
- import_expression.exec(to_compile, env)
- except Exception as e:
- text = f"```{e.__class__.__name__}: {e}"[0:4096]+"```"
-
- func = env["func"]
-
- try:
- with contextlib.redirect_stdout(stdout):
- await func(c, m)
- except Exception:
- value = stdout.getvalue()
- text = f"```{value}{traceback.format_exc()}"[0:4096]+"```"
- else:
- value = stdout.getvalue()
- text = f"```{value}"[0:4096]+"```"
-
- await c.send_message(m.chat.id, text, parse_mode=enums.ParseMode.MARKDOWN)
diff --git a/daemon/telegram/packages/plugins/scrape.py b/daemon/telegram/packages/plugins/scrape.py
deleted file mode 100644
index 1698c6d..0000000
--- a/daemon/telegram/packages/plugins/scrape.py
+++ /dev/null
@@ -1,86 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-#
-# Copyright (C) 2022 Muhammad Rizki <[email protected]>
-# Copyright (C) 2022 Ammar Faizi <[email protected]>
-#
-
-from pyrogram.types import InlineKeyboardMarkup
-from pyrogram.types import InlineKeyboardButton
-from pyrogram.types import Message
-from pyrogram import filters
-from pyrogram import Client
-from scraper import Scraper
-from pyrogram import enums
-from scraper import utils
-from scraper import Bot
-import shutil
-import re
-import asyncio
-
-
-#
-# This allows user to invoke the following commands:
-# /lore https://lore.kernel.org/path/message_id/raw
-# !lore https://lore.kernel.org/path/message_id/raw
-# .lore https://lore.kernel.org/path/message_id/raw
-#
-LORE_CMD_URL_PATTERN = r"^(?:\/|\.|\!)lore\s+(https?:\/\/lore\.kernel\.org\/\S+)"
[email protected]_message(
- filters.regex(LORE_CMD_URL_PATTERN) &
- filters.chat(["kiizuah", "nekoha", -1001673279485])
-)
-async def scrap_email(_, m: Message):
- p = re.search(LORE_CMD_URL_PATTERN, m.text)
- if not p:
- return
-
- url = p.group(1)
- if not url:
- return
-
- s = Scraper()
- mail = await s.get_email_from_url(url)
- text, files, is_patch = utils.create_template(mail)
-
- if is_patch:
- m = await __send_patch_msg(m, mail, text, url)
- else:
- text = "#ml\n" + text
- m = await __send_text_msg(m, text, url)
-
- for d, f in files:
- await m.reply_document(f"{d}/{f}", file_name=f)
- await asyncio.sleep(1)
-
- if files:
- shutil.rmtree(str(files[0][0]))
-
-
-async def __send_patch_msg(m, mail, text, url):
- tmp, fnm, caption, url = Bot.prepare_send_patch(mail, text, url)
- ret = await m.reply_document(
- fnm,
- caption=caption,
- parse_mode=enums.ParseMode.HTML,
- reply_markup=InlineKeyboardMarkup([
- [InlineKeyboardButton(
- "See the full message",
- url=url
- )]
- ])
- )
- Bot.clean_up_after_send_patch(tmp)
- return ret
-
-
-async def __send_text_msg(m, text, url):
- return await m.reply(
- text,
- parse_mode=enums.ParseMode.HTML,
- reply_markup=InlineKeyboardMarkup([
- [InlineKeyboardButton(
- "See the full message",
- url=url.replace("/raw","")
- )]
- ])
- )
--
Muhammad Rizki
^ permalink raw reply related [flat|nested] 8+ messages in thread* [PATCH v1 3/3] Full refactor bot scripts
2022-08-25 16:09 [PATCH v1 0/3] New Discord bot and full refactor scripts Muhammad Rizki
2022-08-25 16:09 ` [PATCH v1 1/3] Move the Telegram bot source code Muhammad Rizki
2022-08-25 16:09 ` [PATCH v1 2/3] First release Discord bot Muhammad Rizki
@ 2022-08-25 16:09 ` Muhammad Rizki
2 siblings, 0 replies; 8+ messages in thread
From: Muhammad Rizki @ 2022-08-25 16:09 UTC (permalink / raw)
To: Ammar Faizi
Cc: Muhammad Rizki, GNU/Weeb Mailing List, Alviro Iskandar Setiawan
Refactoring the bot scripts to make it more clean
Signed-off-by: Muhammad Rizki <[email protected]>
---
.gitignore | 6 +-
daemon/atom/__init__.py | 1 +
daemon/{discord/mailer => atom}/scraper.py | 8 +-
daemon/{telegram/scraper => atom}/utils.py | 87 +++++--
daemon/{discord/run.py => dc.py} | 12 +-
.../.env.example => discord.env.example} | 0
daemon/discord/mailer/utils.py | 241 ------------------
daemon/{discord => dscord}/config.py.example | 0
daemon/dscord/database/__init__.py | 1 +
daemon/dscord/database/core.py | 19 ++
daemon/dscord/database/methods/__init__.py | 10 +
.../database/methods/deletion/__init__.py | 8 +
.../database/methods/deletion/delet_atom.py | 12 +
.../methods/deletion/delete_broadcast.py | 12 +
.../database/methods/getter/__init__.py | 12 +
.../database/methods/getter/get_atom_urls.py | 18 ++
.../methods/getter/get_broadcast_chats.py | 18 ++
.../database/methods/getter/get_email_id.py | 56 ++++
.../database/methods/getter/get_reply.py | 29 +++
.../database/methods/insertion/__init__.py | 12 +
.../database/methods/insertion/insert_atom.py | 20 ++
.../methods/insertion/insert_broadcast.py | 42 +++
.../methods/insertion/insert_discord.py | 14 +
.../methods/insertion/insert_email.py | 20 ++
daemon/{discord => dscord}/execute_me.sql | 0
.../{discord => dscord}/gnuweeb/__init__.py | 0
daemon/{discord => dscord}/gnuweeb/client.py | 32 ++-
daemon/{discord => dscord}/gnuweeb/filters.py | 2 +-
.../gnuweeb/models/__init__.py | 0
.../gnuweeb/models/ui/__init__.py | 0
.../gnuweeb/models/ui/buttons/__init__.py | 0
.../models/ui/buttons/full_message_btn.py | 0
.../gnuweeb/plugins/__init__.py | 0
.../plugins/basic_commands/__init__.py | 0
.../plugins/basic_commands/debugger.py | 2 +-
.../gnuweeb/plugins/basic_commands/sync_it.py | 6 +-
.../gnuweeb/plugins/events/__init__.py | 0
.../gnuweeb/plugins/events/on_error.py | 0
.../gnuweeb/plugins/events/on_ready.py | 0
.../plugins/slash_commands/__init__.py | 0
.../plugins/slash_commands/get_lore_mail.py | 4 +-
.../plugins/slash_commands/manage_atom.py | 4 +-
.../slash_commands/manage_broadcast.py | 4 +-
daemon/{discord => dscord}/gnuweeb/utils.py | 0
daemon/{discord => dscord}/mailer/__init__.py | 1 -
daemon/{discord => dscord}/mailer/database.py | 0
daemon/{discord => dscord}/mailer/listener.py | 26 +-
daemon/{discord => dscord}/requirements.txt | 0
daemon/{discord => dscord}/storage/.gitignore | 0
.../.env.example => telegram.env.example} | 0
daemon/telegram/__init__.py | 0
daemon/telegram/database/__init__.py | 1 +
daemon/telegram/database/core.py | 19 ++
daemon/telegram/database/methods/__init__.py | 10 +
.../database/methods/deletion/__init__.py | 8 +
.../database/methods/deletion/delet_atom.py | 12 +
.../methods/deletion/delete_broadcast.py | 12 +
.../database/methods/getter/__init__.py | 12 +
.../database/methods/getter/get_atom_urls.py | 18 ++
.../methods/getter/get_broadcast_chats.py | 18 ++
.../database/methods/getter/get_email_id.py | 58 +++++
.../methods/getter/get_telegram_reply.py | 29 +++
.../database/methods/insertion/__init__.py | 12 +
.../database/methods/insertion/insert_atom.py | 20 ++
.../methods/insertion/insert_broadcast.py | 49 ++++
.../methods/insertion/insert_email.py | 20 ++
.../methods/insertion/insert_telegram.py | 14 +
.../telegram/{scraper => mailer}/__init__.py | 5 +-
.../{scraper/bot.py => mailer/listener.py} | 61 +++--
daemon/telegram/packages/client.py | 16 +-
.../packages/plugins/callbacks/del_atom.py | 6 +-
.../packages/plugins/callbacks/del_chat.py | 6 +-
.../packages/plugins/commands/debugger.py | 2 +-
.../packages/plugins/commands/manage_atom.py | 8 +-
.../plugins/commands/manage_broadcast.py | 8 +-
.../packages/plugins/commands/scrape.py | 10 +-
daemon/telegram/scraper/db.py | 217 ----------------
daemon/telegram/scraper/scraper.py | 63 -----
daemon/{telegram/run.py => tg.py} | 20 +-
79 files changed, 799 insertions(+), 674 deletions(-)
create mode 100644 daemon/atom/__init__.py
rename daemon/{discord/mailer => atom}/scraper.py (83%)
rename daemon/{telegram/scraper => atom}/utils.py (72%)
rename daemon/{discord/run.py => dc.py} (78%)
rename daemon/{discord/.env.example => discord.env.example} (100%)
delete mode 100644 daemon/discord/mailer/utils.py
rename daemon/{discord => dscord}/config.py.example (100%)
create mode 100644 daemon/dscord/database/__init__.py
create mode 100644 daemon/dscord/database/core.py
create mode 100644 daemon/dscord/database/methods/__init__.py
create mode 100644 daemon/dscord/database/methods/deletion/__init__.py
create mode 100644 daemon/dscord/database/methods/deletion/delet_atom.py
create mode 100644 daemon/dscord/database/methods/deletion/delete_broadcast.py
create mode 100644 daemon/dscord/database/methods/getter/__init__.py
create mode 100644 daemon/dscord/database/methods/getter/get_atom_urls.py
create mode 100644 daemon/dscord/database/methods/getter/get_broadcast_chats.py
create mode 100644 daemon/dscord/database/methods/getter/get_email_id.py
create mode 100644 daemon/dscord/database/methods/getter/get_reply.py
create mode 100644 daemon/dscord/database/methods/insertion/__init__.py
create mode 100644 daemon/dscord/database/methods/insertion/insert_atom.py
create mode 100644 daemon/dscord/database/methods/insertion/insert_broadcast.py
create mode 100644 daemon/dscord/database/methods/insertion/insert_discord.py
create mode 100644 daemon/dscord/database/methods/insertion/insert_email.py
rename daemon/{discord => dscord}/execute_me.sql (100%)
rename daemon/{discord => dscord}/gnuweeb/__init__.py (100%)
rename daemon/{discord => dscord}/gnuweeb/client.py (84%)
rename daemon/{discord => dscord}/gnuweeb/filters.py (98%)
rename daemon/{discord => dscord}/gnuweeb/models/__init__.py (100%)
rename daemon/{discord => dscord}/gnuweeb/models/ui/__init__.py (100%)
rename daemon/{discord => dscord}/gnuweeb/models/ui/buttons/__init__.py (100%)
rename daemon/{discord => dscord}/gnuweeb/models/ui/buttons/full_message_btn.py (100%)
rename daemon/{discord => dscord}/gnuweeb/plugins/__init__.py (100%)
rename daemon/{discord => dscord}/gnuweeb/plugins/basic_commands/__init__.py (100%)
rename daemon/{discord => dscord}/gnuweeb/plugins/basic_commands/debugger.py (94%)
rename daemon/{discord => dscord}/gnuweeb/plugins/basic_commands/sync_it.py (70%)
rename daemon/{discord => dscord}/gnuweeb/plugins/events/__init__.py (100%)
rename daemon/{discord => dscord}/gnuweeb/plugins/events/on_error.py (100%)
rename daemon/{discord => dscord}/gnuweeb/plugins/events/on_ready.py (100%)
rename daemon/{discord => dscord}/gnuweeb/plugins/slash_commands/__init__.py (100%)
rename daemon/{discord => dscord}/gnuweeb/plugins/slash_commands/get_lore_mail.py (95%)
rename daemon/{discord => dscord}/gnuweeb/plugins/slash_commands/manage_atom.py (97%)
rename daemon/{discord => dscord}/gnuweeb/plugins/slash_commands/manage_broadcast.py (96%)
rename daemon/{discord => dscord}/gnuweeb/utils.py (100%)
rename daemon/{discord => dscord}/mailer/__init__.py (82%)
rename daemon/{discord => dscord}/mailer/database.py (100%)
rename daemon/{discord => dscord}/mailer/listener.py (85%)
rename daemon/{discord => dscord}/requirements.txt (100%)
rename daemon/{discord => dscord}/storage/.gitignore (100%)
rename daemon/{telegram/.env.example => telegram.env.example} (100%)
create mode 100644 daemon/telegram/__init__.py
create mode 100644 daemon/telegram/database/__init__.py
create mode 100644 daemon/telegram/database/core.py
create mode 100644 daemon/telegram/database/methods/__init__.py
create mode 100644 daemon/telegram/database/methods/deletion/__init__.py
create mode 100644 daemon/telegram/database/methods/deletion/delet_atom.py
create mode 100644 daemon/telegram/database/methods/deletion/delete_broadcast.py
create mode 100644 daemon/telegram/database/methods/getter/__init__.py
create mode 100644 daemon/telegram/database/methods/getter/get_atom_urls.py
create mode 100644 daemon/telegram/database/methods/getter/get_broadcast_chats.py
create mode 100644 daemon/telegram/database/methods/getter/get_email_id.py
create mode 100644 daemon/telegram/database/methods/getter/get_telegram_reply.py
create mode 100644 daemon/telegram/database/methods/insertion/__init__.py
create mode 100644 daemon/telegram/database/methods/insertion/insert_atom.py
create mode 100644 daemon/telegram/database/methods/insertion/insert_broadcast.py
create mode 100644 daemon/telegram/database/methods/insertion/insert_email.py
create mode 100644 daemon/telegram/database/methods/insertion/insert_telegram.py
rename daemon/telegram/{scraper => mailer}/__init__.py (68%)
rename daemon/telegram/{scraper/bot.py => mailer/listener.py} (66%)
delete mode 100644 daemon/telegram/scraper/db.py
delete mode 100644 daemon/telegram/scraper/scraper.py
rename daemon/{telegram/run.py => tg.py} (75%)
diff --git a/.gitignore b/.gitignore
index 4201a17..ff5f6de 100644
--- a/.gitignore
+++ b/.gitignore
@@ -103,8 +103,8 @@ celerybeat.pid
*.sage.py
# Environments
-.env
-.venv
+*.env
+*.venv
env/
venv/
ENV/
@@ -141,4 +141,4 @@ data.json
# configuration file
daemon/telegram/config.py
-daemon/discord/config.py
+daemon/dscord/config.py
diff --git a/daemon/atom/__init__.py b/daemon/atom/__init__.py
new file mode 100644
index 0000000..b4fcd12
--- /dev/null
+++ b/daemon/atom/__init__.py
@@ -0,0 +1 @@
+from .scraper import Scraper
diff --git a/daemon/discord/mailer/scraper.py b/daemon/atom/scraper.py
similarity index 83%
rename from daemon/discord/mailer/scraper.py
rename to daemon/atom/scraper.py
index 3873161..8508ae9 100644
--- a/daemon/discord/mailer/scraper.py
+++ b/daemon/atom/scraper.py
@@ -19,10 +19,10 @@ class Scraper:
async def __get_atom_content(self, atom_url):
async with httpx.AsyncClient() as client:
- res = await client.get(atom_url)
+ res = await client.get(atom_url, timeout=20)
if res.status_code == 200:
return res.text
- raise Exception(f"[get_atom_content]: Returned {res.status_code} HTTP code")
+ raise Exception(f"[__get_atom_content]: Returned {res.status_code} HTTP code")
async def __get_new_threads_from_atom(self, atom):
@@ -54,10 +54,10 @@ class Scraper:
async def get_email_from_url(self, url):
async with httpx.AsyncClient() as client:
- res = await client.get(url)
+ res = await client.get(url, timeout=20)
if res.status_code == 200:
return email.message_from_string(
res.text,
policy=email.policy.default
)
- raise Exception(f"[get_atom_content]: Returned {res.status_code} HTTP code")
+ raise Exception(f"[get_email_from_url]: Returned {res.status_code} HTTP code")
diff --git a/daemon/telegram/scraper/utils.py b/daemon/atom/utils.py
similarity index 72%
rename from daemon/telegram/scraper/utils.py
rename to daemon/atom/utils.py
index c428a33..b62c850 100644
--- a/daemon/telegram/scraper/utils.py
+++ b/daemon/atom/utils.py
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: GPL-2.0-only
#
-# Copyright (C) 2022 Muhammad Rizki <[email protected]>
+# Copyright (C) 2022 Muhammad Rizki <[email protected]>
# Copyright (C) 2022 Ammar Faizi <[email protected]>
#
@@ -8,13 +8,19 @@ from pyrogram.types import Chat, InlineKeyboardMarkup, InlineKeyboardButton
from email.message import Message
from typing import Dict
from slugify import slugify
+import html
import hashlib
import uuid
import os
import re
import shutil
import httpx
-import html
+import asyncio
+
+
+class Mutexes():
+ def __init__(self):
+ self.lock = asyncio.Lock()
def get_email_msg_id(mail):
@@ -113,25 +119,37 @@ def consruct_to_n_cc(to: list, cc: list):
return ret
-def gen_temp(name: str):
+def gen_temp(name: str, platform: str):
+ platform = platform.lower()
+ plt_ls = ["telegram", "discord"]
+
+ if platform not in plt_ls:
+ t = f"Platform {platform} is not found, "
+ t += f"only {', '.join(plt_ls)} is available"
+ raise ValueError(f"Platform {platform} is not found")
+
md5 = hashlib.md5(name.encode()).hexdigest()
- ret = os.getenv("STORAGE_DIR", "storage") + "/" + md5
+ store_dir = os.getenv("STORAGE_DIR", "storage")
+ platform = platform.replace("discord", "dscord")
+ path = f"{platform}/{store_dir}/{md5}"
try:
- os.mkdir(ret)
+ os.mkdir(path)
except FileExistsError:
pass
- return ret
+ return path
-def extract_body(thread: Message):
+def extract_body(thread: Message, platform: str):
if not thread.is_multipart():
- p = thread.get_payload(decode=True)
- return f"{p.decode(errors='replace')}\n".lstrip(), []
+ p = thread.get_payload(decode=True).decode(errors='replace')
+ if platform == "discord":
+ p = quote_reply(p)
+ return f"{p}\n".lstrip(), []
ret = ""
files = []
- temp = gen_temp(str(uuid.uuid4()))
+ temp = gen_temp(str(uuid.uuid4()), platform)
for p in thread.get_payload():
fname = p.get_filename()
payload = p.get_payload(decode=True)
@@ -164,35 +182,42 @@ def __is_patch(subject, content):
return True
-def create_template(thread: Message, to=None, cc=None):
+def create_template(thread: Message, platform: str, to=None, cc=None):
if not to:
to = extract_list("to", thread)
if not cc:
cc = extract_list("cc", thread)
+ if platform == "telegram":
+ substr = 4000
+ border = f"\n<code>{'-'*72}</code>"
+ else:
+ substr = 1900
+ border = f"\n{'-'*80}"
subject = thread.get('subject')
ret = f"From: {thread.get('from')}\n"
ret += consruct_to_n_cc(to, cc)
ret += f"Date: {thread.get('date')}\n"
ret += f"Subject: {subject}\n\n"
- content, files = extract_body(thread)
+ content, files = extract_body(thread, platform)
is_patch = __is_patch(subject, content)
if is_patch:
ret += content
else:
ret += content.strip().replace("\t", " ")
- if len(ret) >= 4000:
- ret = ret[:4000] + "..."
- ret = fix_utf8_char(ret)
- ret += f"\n<code>{'-'*72}</code>"
+ if len(ret) >= substr:
+ ret = ret[:substr] + "..."
+
+ ret = fix_utf8_char(ret, platform == "telegram")
+ ret += border
return ret, files, is_patch
-def prepare_send_patch(mail, text, url):
- tmp = gen_temp(url)
+def prepare_patch(mail: "Message", text: str, url: str, platform: str):
+ tmp = gen_temp(url, platform)
fnm = str(mail.get("subject"))
sch = re.search(PATCH_PATTERN, fnm, re.IGNORECASE)
@@ -210,17 +235,31 @@ def prepare_send_patch(mail, text, url):
with open(file, "wb") as f:
f.write(bytes(text, encoding="utf8"))
- caption = "#patch #ml\n" + fix_utf8_char(cap)
+ caption = "#patch #ml"
+ if platform == "telegram":
+ caption += fix_utf8_char("\n" + cap, True)
return tmp, file, caption, url
-def clean_up_after_send_patch(tmp):
+def remove_patch(tmp):
shutil.rmtree(tmp)
-def fix_utf8_char(text: str):
- text = text.rstrip().replace("�"," ")
- return html.escape(html.escape(text))
+def fix_utf8_char(text: str, html_escape: bool = True):
+ t = text.rstrip().replace("�"," ")
+ if html_escape:
+ t = html.escape(html.escape(text))
+ return t
+
+
+def quote_reply(text: str):
+ a = ""
+ for b in text.split("\n"):
+ b = b.replace(">\n", "> ")
+ if b.startswith(">"):
+ a += "> "
+ a += f"{b}\n"
+ return a
EMAIL_MSG_ID_PATTERN = r"<([^\<\>]+)>"
@@ -240,6 +279,8 @@ async def is_atom_url(text: str):
return mime == "application/atom+xml"
except: return False
+
+
def remove_command(text: str):
txt = text.split(" ")
txt = text.replace(txt[0] + " ","")
diff --git a/daemon/discord/run.py b/daemon/dc.py
similarity index 78%
rename from daemon/discord/run.py
rename to daemon/dc.py
index e3eb7a6..a24421b 100644
--- a/daemon/discord/run.py
+++ b/daemon/dc.py
@@ -7,15 +7,15 @@ import os
from mysql import connector
from dotenv import load_dotenv
from apscheduler.schedulers.asyncio import AsyncIOScheduler
-from mailer.listener import BotMutexes
-from mailer.listener import Listener
+from dscord.mailer.listener import Listener
+from atom.scraper import Scraper
+from atom.utils import Mutexes
-from gnuweeb import GWClient
-from mailer import Scraper
+from dscord.gnuweeb import GWClient
def main():
- load_dotenv()
+ load_dotenv("discord.env")
sched = AsyncIOScheduler(
job_defaults={
@@ -37,7 +37,7 @@ def main():
client=client,
sched=sched,
scraper=Scraper(),
- mutexes=BotMutexes()
+ mutexes=Mutexes()
)
client.mailer = mailer
diff --git a/daemon/discord/.env.example b/daemon/discord.env.example
similarity index 100%
rename from daemon/discord/.env.example
rename to daemon/discord.env.example
diff --git a/daemon/discord/mailer/utils.py b/daemon/discord/mailer/utils.py
deleted file mode 100644
index 0036f24..0000000
--- a/daemon/discord/mailer/utils.py
+++ /dev/null
@@ -1,241 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-#
-# Copyright (C) 2022 Muhammad Rizki <[email protected]>
-# Copyright (C) 2022 Ammar Faizi <[email protected]>
-#
-
-from email.message import Message
-from typing import Dict
-from slugify import slugify
-import hashlib
-import uuid
-import os
-import re
-import shutil
-import httpx
-
-
-def get_email_msg_id(mail):
- ret = mail.get("message-id")
- if not ret:
- return None
-
- ret = re.search(r"<([^\<\>]+)>", ret)
- if not ret:
- return None
-
- return ret.group(1)
-
-
-#
-# This increments the @i while we are seeing a whitespace.
-#
-def __skip_whitespace(i, ss_len, ss):
- while i < ss_len:
- c = ss[i]
- if c != ' ' and c != '\t' and c != '\n':
- break
- i += 1
-
- return i
-
-
-#
-# Pick a single element in the list. The delimiter here is
-# a comma char ','. But note that when are inside a double
-# quotes, we must not take the comma as a delimiter.
-#
-def __pick_element(i, ss_len, ss, ret):
- acc = ""
- in_quotes = False
-
- while i < ss_len:
- c = ss[i]
- i += 1
-
- if c == '"':
- in_quotes = (not in_quotes)
-
- if not in_quotes and c == ',':
- break
-
- acc += c
-
- if acc != "":
- ret.append(acc)
-
- return i
-
-
-def __extract_list(ss):
- ss = ss.strip()
- ss_len = len(ss)
- ret = []
- i = 0
-
- while i < ss_len:
- i = __skip_whitespace(i, ss_len, ss)
- i = __pick_element(i, ss_len, ss, ret)
-
- return ret
-
-
-def extract_list(key: str, content: Dict[str, str]):
- people = content.get(key.lower())
- if not people:
- return []
- return __extract_list(people)
-
-
-def consruct_to_n_cc(to: list, cc: list):
- NR_MAX_LIST = 20
-
- n = 0
- ret = ""
- for i in to:
- if n >= NR_MAX_LIST:
- ret += "To: ...\n"
- break
-
- n += 1
- ret += f"To: {i}\n"
-
- for i in cc:
- if n >= NR_MAX_LIST:
- ret += "Cc: ...\n"
- break
-
- n += 1
- ret += f"Cc: {i}\n"
-
- return ret
-
-
-def gen_temp(name: str):
- md5 = hashlib.md5(name.encode()).hexdigest()
- ret = os.getenv("STORAGE_DIR", "storage") + "/" + md5
- try:
- os.mkdir(ret)
- except FileExistsError:
- pass
-
- return ret
-
-
-def extract_body(thread: Message):
- if not thread.is_multipart():
- p = thread.get_payload(decode=True)
- return f"{p.decode(errors='replace')}\n".lstrip(), []
-
- ret = ""
- files = []
- temp = gen_temp(str(uuid.uuid4()))
- for p in thread.get_payload():
- fname = p.get_filename()
- payload = p.get_payload(decode=True)
-
- if not payload:
- continue
-
- if 'inline' in [p.get('content-disposition')] or not bool(fname):
- ret += f"{payload.decode(errors='replace')}\n".lstrip()
- continue
-
- with open(f"{temp}/{fname}", "wb") as f:
- f.write(payload)
- files.append((temp, fname))
-
- ret = re.sub("^(>)", ">>> \\1", ret, 1, re.MULTILINE)
- return ret, files
-
-
-
-PATCH_PATTERN = r"^\[.*(?:patch|rfc).*?(?:(\d+)\/(\d+))?\](.+)"
-def __is_patch(subject, content):
- x = re.search(PATCH_PATTERN, subject, re.IGNORECASE)
- if not x or x.group(1) == "0":
- return False
-
- x = re.search(r"diff --git", content)
- if not x:
- return False
-
- return True
-
-
-def create_template(thread: Message, to=None, cc=None):
- if not to:
- to = extract_list("to", thread)
- if not cc:
- cc = extract_list("cc", thread)
-
- subject = thread.get('subject')
- ret = f"From: {thread.get('from')}\n"
- ret += consruct_to_n_cc(to, cc)
- ret += f"Date: {thread.get('date')}\n"
- ret += f"Subject: {subject}\n\n"
- content, files = extract_body(thread)
- is_patch = __is_patch(subject, content)
-
- if is_patch:
- ret += content
- else:
- ret += content.strip().replace("\t", " ")
- if len(ret) >= 1900:
- ret = ret[:1900] + "..."
-
- ret = fix_utf8_char(ret)
-
- return ret, files, is_patch
-
-
-def prepare_patch(mail, text, url):
- tmp = gen_temp(url)
- fnm = str(mail.get("subject"))
- sch = re.search(PATCH_PATTERN, fnm, re.IGNORECASE)
-
- nr_patch = sch.group(1)
- if not nr_patch:
- nr_patch = 1
- else:
- nr_patch = int(nr_patch)
-
- num = "%04d" % nr_patch
- fnm = slugify(sch.group(3)).replace("_", "-")
- file = f"{tmp}/{num}-{fnm}.patch"
-
- with open(file, "wb") as f:
- f.write(bytes(text, encoding="utf8"))
-
- caption = "#patch #ml"
- return tmp, file, caption, url
-
-
-def remove_patch(tmp):
- shutil.rmtree(tmp)
-
-
-def fix_utf8_char(text: str):
- return text.rstrip().replace("�"," ")
-
-
-def bottom_border(text: str):
- return text + "\n" + "-"*72
-
-
-EMAIL_MSG_ID_PATTERN = r"<([^\<\>]+)>"
-def extract_email_msg_id(msg_id):
- ret = re.search(EMAIL_MSG_ID_PATTERN, msg_id)
- if not ret:
- return None
- return ret.group(1)
-
-
-async def is_atom_url(text: str):
- try:
- async with httpx.AsyncClient() as ses:
- res = await ses.get(text)
- mime = res.headers.get("Content-Type")
-
- return mime == "application/atom+xml"
- except: return False
diff --git a/daemon/discord/config.py.example b/daemon/dscord/config.py.example
similarity index 100%
rename from daemon/discord/config.py.example
rename to daemon/dscord/config.py.example
diff --git a/daemon/dscord/database/__init__.py b/daemon/dscord/database/__init__.py
new file mode 100644
index 0000000..13cff25
--- /dev/null
+++ b/daemon/dscord/database/__init__.py
@@ -0,0 +1 @@
+from .core import DB
diff --git a/daemon/dscord/database/core.py b/daemon/dscord/database/core.py
new file mode 100644
index 0000000..cfd9cc2
--- /dev/null
+++ b/daemon/dscord/database/core.py
@@ -0,0 +1,19 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Copyright (C) 2022 Muhammad Rizki <[email protected]>
+# Copyright (C) 2022 Ammar Faizi <[email protected]>
+#
+
+from .methods import DBMethods
+
+
+class DB(DBMethods):
+ def __init__(self, conn):
+ self.conn = conn
+ self.conn.autocommit = True
+ self.cur = self.conn.cursor(buffered=True)
+
+
+ def __del__(self):
+ self.cur.close()
+ self.conn.close()
diff --git a/daemon/dscord/database/methods/__init__.py b/daemon/dscord/database/methods/__init__.py
new file mode 100644
index 0000000..6099941
--- /dev/null
+++ b/daemon/dscord/database/methods/__init__.py
@@ -0,0 +1,10 @@
+from .deletion import Deletion
+from .getter import Getter
+from .insertion import Insertion
+
+
+class DBMethods(
+ Deletion,
+ Getter,
+ Insertion
+): pass
diff --git a/daemon/dscord/database/methods/deletion/__init__.py b/daemon/dscord/database/methods/deletion/__init__.py
new file mode 100644
index 0000000..fa80464
--- /dev/null
+++ b/daemon/dscord/database/methods/deletion/__init__.py
@@ -0,0 +1,8 @@
+from .delet_atom import DeleteAtom
+from .delete_broadcast import DeleteBroadcast
+
+
+class Deletion(
+ DeleteAtom,
+ DeleteBroadcast
+): pass
diff --git a/daemon/dscord/database/methods/deletion/delet_atom.py b/daemon/dscord/database/methods/deletion/delet_atom.py
new file mode 100644
index 0000000..4923045
--- /dev/null
+++ b/daemon/dscord/database/methods/deletion/delet_atom.py
@@ -0,0 +1,12 @@
+
+
+
+class DeleteAtom:
+
+ def delete_atom(self, atom: str):
+ q = """
+ DELETE FROM atom_urls
+ WHERE url = %(atom)s
+ """
+ self.cur.execute(q, {"atom": atom})
+ return self.cur.rowcount > 0
diff --git a/daemon/dscord/database/methods/deletion/delete_broadcast.py b/daemon/dscord/database/methods/deletion/delete_broadcast.py
new file mode 100644
index 0000000..2f17eff
--- /dev/null
+++ b/daemon/dscord/database/methods/deletion/delete_broadcast.py
@@ -0,0 +1,12 @@
+
+
+
+class DeleteBroadcast:
+
+ def delete_broadcast(self, channel_id: int):
+ q = """
+ DELETE FROM broadcast_chats
+ WHERE channel_id = %(channel_id)s
+ """
+ self.cur.execute(q, {"channel_id": channel_id})
+ return self.cur.rowcount > 0
diff --git a/daemon/dscord/database/methods/getter/__init__.py b/daemon/dscord/database/methods/getter/__init__.py
new file mode 100644
index 0000000..fd58e84
--- /dev/null
+++ b/daemon/dscord/database/methods/getter/__init__.py
@@ -0,0 +1,12 @@
+from .get_atom_urls import GetAtomURL
+from .get_broadcast_chats import GetBroadcastChats
+from .get_email_id import GetEmailID
+from .get_reply import GetDiscordReply
+
+
+class Getter(
+ GetAtomURL,
+ GetBroadcastChats,
+ GetEmailID,
+ GetDiscordReply
+): pass
diff --git a/daemon/dscord/database/methods/getter/get_atom_urls.py b/daemon/dscord/database/methods/getter/get_atom_urls.py
new file mode 100644
index 0000000..b7353d2
--- /dev/null
+++ b/daemon/dscord/database/methods/getter/get_atom_urls.py
@@ -0,0 +1,18 @@
+
+
+
+class GetAtomURL:
+
+ def get_atom_urls(self):
+ '''
+ Get lore kernel raw email URLs.
+ - Return list of raw email URLs: `List[str]`
+ '''
+ q = """
+ SELECT atom_urls.url
+ FROM atom_urls
+ """
+ self.cur.execute(q)
+ urls = self.cur.fetchall()
+
+ return [u[0] for u in urls]
diff --git a/daemon/dscord/database/methods/getter/get_broadcast_chats.py b/daemon/dscord/database/methods/getter/get_broadcast_chats.py
new file mode 100644
index 0000000..43efe0c
--- /dev/null
+++ b/daemon/dscord/database/methods/getter/get_broadcast_chats.py
@@ -0,0 +1,18 @@
+
+
+
+class GetBroadcastChats:
+
+ def get_broadcast_chats(self):
+ '''
+ Get broadcast chats that are currently
+ listening for new email.
+ - Return list of chat object: `List[Object]`
+ '''
+ q = """
+ SELECT *
+ FROM broadcast_chats
+ """
+ self.cur.execute(q)
+
+ return self.cur.fetchall()
diff --git a/daemon/dscord/database/methods/getter/get_email_id.py b/daemon/dscord/database/methods/getter/get_email_id.py
new file mode 100644
index 0000000..637f47f
--- /dev/null
+++ b/daemon/dscord/database/methods/getter/get_email_id.py
@@ -0,0 +1,56 @@
+
+
+
+class GetEmailID:
+
+ def get_email_id(self, email_id, chat_id):
+ '''
+ Determine whether the email needs to be sent to @tg_chat_id.
+ - Return an email id (PK) if it needs to be sent
+ - Return None if it doesn't need to be sent
+ '''
+ if self.__is_sent(email_id, chat_id):
+ return
+
+ res = self.__email_id(email_id)
+ if not bool(res):
+ return
+
+ return int(res[0])
+
+
+ def __is_sent(self, email_id, channel_id):
+ '''
+ Checking if this email has already been sent
+ or not.
+ - Return True if it's already been sent
+ '''
+ q = """
+ SELECT emails.id, discord_emails.id FROM emails
+ LEFT JOIN discord_emails
+ ON emails.id = discord_emails.email_id
+ WHERE emails.message_id = %(email_msg_id)s
+ AND discord_emails.channel_id = %(channel_id)s
+ LIMIT 1
+ """
+
+ self.cur.execute(q, {
+ "email_msg_id": email_id,
+ "channel_id": channel_id
+ })
+ res = self.cur.fetchone()
+ return bool(res)
+
+
+ def __email_id(self, email_id):
+ '''
+ Get the email id if match with the email message_id.
+ - Return the result if it's match and exists
+ '''
+ q = """
+ SELECT id FROM emails WHERE message_id = %(email_id)s
+ """
+
+ self.cur.execute(q, {"email_id": email_id})
+ res = self.cur.fetchone()
+ return res
diff --git a/daemon/dscord/database/methods/getter/get_reply.py b/daemon/dscord/database/methods/getter/get_reply.py
new file mode 100644
index 0000000..c5b1830
--- /dev/null
+++ b/daemon/dscord/database/methods/getter/get_reply.py
@@ -0,0 +1,29 @@
+
+
+
+class GetDiscordReply:
+
+ def get_reply_id(self, email_id, channel_id):
+ '''
+ Get Telegram message ID sent match with
+ email message ID and Telegram chat ID.
+ - Return Telegram message ID if exists: `int`
+ - Return None if not exists`
+ '''
+ q = """
+ SELECT discord_emails.dc_msg_id
+ FROM emails INNER JOIN discord_emails
+ ON emails.id = discord_emails.email_id
+ WHERE emails.message_id = %(email_msg_id)s
+ AND discord_emails.channel_id = %(channel_id)s
+ """
+
+ self.cur.execute(q, {
+ "email_msg_id": email_id,
+ "channel_id": channel_id
+ })
+ res = self.cur.fetchone()
+ if not bool(res):
+ return None
+
+ return res[0]
diff --git a/daemon/dscord/database/methods/insertion/__init__.py b/daemon/dscord/database/methods/insertion/__init__.py
new file mode 100644
index 0000000..46b6e05
--- /dev/null
+++ b/daemon/dscord/database/methods/insertion/__init__.py
@@ -0,0 +1,12 @@
+from .insert_atom import InsertAtom
+from .insert_broadcast import InsertBroadcast
+from .insert_email import InsertEmail
+from .insert_discord import InsertDiscord
+
+
+class Insertion(
+ InsertAtom,
+ InsertBroadcast,
+ InsertEmail,
+ InsertDiscord
+): pass
diff --git a/daemon/dscord/database/methods/insertion/insert_atom.py b/daemon/dscord/database/methods/insertion/insert_atom.py
new file mode 100644
index 0000000..6313c07
--- /dev/null
+++ b/daemon/dscord/database/methods/insertion/insert_atom.py
@@ -0,0 +1,20 @@
+from mysql.connector import errors
+from datetime import datetime
+
+
+class InsertAtom:
+
+ def insert_atom(self, atom: str):
+ try:
+ return self.__save_atom(atom)
+ except errors.IntegrityError:
+ #
+ # Duplicate data, skip!
+ #
+ return None
+
+
+ def __save_atom(self, atom: str):
+ q = "INSERT INTO atom_urls (url, created_at) VALUES (%s, %s)"
+ self.cur.execute(q, (atom, datetime.utcnow()))
+ return self.cur.lastrowid
diff --git a/daemon/dscord/database/methods/insertion/insert_broadcast.py b/daemon/dscord/database/methods/insertion/insert_broadcast.py
new file mode 100644
index 0000000..e3177d3
--- /dev/null
+++ b/daemon/dscord/database/methods/insertion/insert_broadcast.py
@@ -0,0 +1,42 @@
+from mysql.connector import errors
+from datetime import datetime
+
+
+class InsertBroadcast:
+
+ def save_broadcast(
+ self,
+ guild_id: int,
+ channel_id: int,
+ channel_name: str,
+ channel_link: str = None,
+ ):
+ try:
+ return self.__save_broadcast(
+ guild_id=guild_id,
+ channel_id=channel_id,
+ channel_name=channel_name,
+ channel_link=channel_link
+ )
+ except errors.IntegrityError:
+ #
+ # Duplicate data, skip!
+ #
+ return None
+
+
+ def __save_broadcast(
+ self,
+ guild_id: int,
+ channel_id: int,
+ channel_name: str,
+ channel_link: str = None,
+ ):
+ q = """
+ INSERT INTO broadcast_chats
+ (guild_id, channel_id, channel_name, channel_link, created_at)
+ VALUES (%s, %s, %s, %s, %s)
+ """
+ values = (guild_id, channel_id, channel_name, channel_link, datetime.utcnow())
+ self.cur.execute(q, values)
+ return self.cur.lastrowid
diff --git a/daemon/dscord/database/methods/insertion/insert_discord.py b/daemon/dscord/database/methods/insertion/insert_discord.py
new file mode 100644
index 0000000..83e4b22
--- /dev/null
+++ b/daemon/dscord/database/methods/insertion/insert_discord.py
@@ -0,0 +1,14 @@
+from datetime import datetime
+
+
+class InsertDiscord:
+
+ def save_discord_mail(self, email_id, channel_id, dc_msg_id):
+ q = """
+ INSERT INTO discord_emails
+ (email_id, channel_id, dc_msg_id, created_at)
+ VALUES (%s, %s, %s, %s);
+ """
+ self.cur.execute(q, (email_id, channel_id, dc_msg_id,
+ datetime.utcnow()))
+ return self.cur.lastrowid
diff --git a/daemon/dscord/database/methods/insertion/insert_email.py b/daemon/dscord/database/methods/insertion/insert_email.py
new file mode 100644
index 0000000..bd88339
--- /dev/null
+++ b/daemon/dscord/database/methods/insertion/insert_email.py
@@ -0,0 +1,20 @@
+from mysql.connector import errors
+from datetime import datetime
+
+
+class InsertEmail:
+
+ def save_email(self, email_msg_id):
+ try:
+ return self.__insert_email(email_msg_id)
+ except errors.IntegrityError:
+ #
+ # Duplicate data, skip!
+ #
+ return None
+
+
+ def __insert_email(self, email_msg_id):
+ q = "INSERT INTO emails (message_id, created_at) VALUES (%s, %s)"
+ self.cur.execute(q, (email_msg_id, datetime.utcnow()))
+ return self.cur.lastrowid
diff --git a/daemon/discord/execute_me.sql b/daemon/dscord/execute_me.sql
similarity index 100%
rename from daemon/discord/execute_me.sql
rename to daemon/dscord/execute_me.sql
diff --git a/daemon/discord/gnuweeb/__init__.py b/daemon/dscord/gnuweeb/__init__.py
similarity index 100%
rename from daemon/discord/gnuweeb/__init__.py
rename to daemon/dscord/gnuweeb/__init__.py
diff --git a/daemon/discord/gnuweeb/client.py b/daemon/dscord/gnuweeb/client.py
similarity index 84%
rename from daemon/discord/gnuweeb/client.py
rename to daemon/dscord/gnuweeb/client.py
index cf88d36..c308f5a 100644
--- a/daemon/discord/gnuweeb/client.py
+++ b/daemon/dscord/gnuweeb/client.py
@@ -11,13 +11,13 @@ from typing import Union
from .models.ui import buttons
from . import filters
-from mailer import utils
-from mailer import Database
+from atom import utils
+from dscord.database import DB
class GWClient(commands.Bot):
def __init__(self, db_conn) -> None:
- self.db = Database(db_conn)
+ self.db = DB(db_conn)
self.mailer = None
intents = Intents.default()
intents.message_content = True
@@ -33,7 +33,10 @@ class GWClient(commands.Bot):
async def setup_hook(self):
- await self.load_extension("gnuweeb.plugins")
+ await self.load_extension(
+ name=".gnuweeb.plugins",
+ package="dscord"
+ )
# WARNING! NOT RECOMMENDED SYNCING WHEN THE BOT IS START!!
# guild = discord.Object(id=845302963739033611)
@@ -45,27 +48,26 @@ class GWClient(commands.Bot):
async def send_text_email(self, guild_id: int, chat_id: int, text: str,
reply_to: Union[int, None] = None, url: str = None):
print("[send_text_email]")
- text = utils.bottom_border(text)
channel = self.get_channel(chat_id)
- m = await channel.send(
+ return await channel.send(
content=text,
reference=discord.MessageReference(
guild_id=guild_id,
channel_id=chat_id,
message_id=reply_to
) if reply_to else None,
-
view=buttons.FullMessageBtn(url)
)
- return m
@filters.wait_on_limit
async def send_patch_email(self, mail, guild_id: int, chat_id: int, text: str,
reply_to: Union[int, None] = None, url: str = None):
print("[send_patch_email]")
- tmp, doc, caption, url = utils.prepare_patch(mail, text, url)
+ tmp, doc, caption, url = utils.prepare_patch(
+ mail, text, url, "discord"
+ )
channel = self.get_channel(chat_id)
m = await channel.send(
@@ -86,25 +88,21 @@ class GWClient(commands.Bot):
async def send_text_mail_interaction(self, i: "Interaction",
text: str, url: str = None):
- text = utils.border_and_trim(text)
-
- m = await i.response.send_message(
+ return await i.response.send_message(
content=text,
view=buttons.FullMessageBtn(url)
)
- return m
-
async def send_patch_mail_interaction(self, mail, i: "Interaction",
text: str, url: str = None):
- tmp, doc, caption, url = utils.prepare_patch(mail, text, url)
-
+ tmp, doc, caption, url = utils.prepare_patch(
+ mail, text, url, "discord"
+ )
m = await i.response.send_message(
content=caption,
file=discord.File(doc),
view=buttons.FullMessageBtn(url)
)
-
utils.remove_patch(tmp)
return m
diff --git a/daemon/discord/gnuweeb/filters.py b/daemon/dscord/gnuweeb/filters.py
similarity index 98%
rename from daemon/discord/gnuweeb/filters.py
rename to daemon/dscord/gnuweeb/filters.py
index 735c77e..76bfa5d 100644
--- a/daemon/discord/gnuweeb/filters.py
+++ b/daemon/dscord/gnuweeb/filters.py
@@ -3,7 +3,7 @@
# Copyright (C) 2022 Muhammad Rizki <[email protected]>
#
-import config
+from dscord import config
import discord
import asyncio
from discord import Interaction
diff --git a/daemon/discord/gnuweeb/models/__init__.py b/daemon/dscord/gnuweeb/models/__init__.py
similarity index 100%
rename from daemon/discord/gnuweeb/models/__init__.py
rename to daemon/dscord/gnuweeb/models/__init__.py
diff --git a/daemon/discord/gnuweeb/models/ui/__init__.py b/daemon/dscord/gnuweeb/models/ui/__init__.py
similarity index 100%
rename from daemon/discord/gnuweeb/models/ui/__init__.py
rename to daemon/dscord/gnuweeb/models/ui/__init__.py
diff --git a/daemon/discord/gnuweeb/models/ui/buttons/__init__.py b/daemon/dscord/gnuweeb/models/ui/buttons/__init__.py
similarity index 100%
rename from daemon/discord/gnuweeb/models/ui/buttons/__init__.py
rename to daemon/dscord/gnuweeb/models/ui/buttons/__init__.py
diff --git a/daemon/discord/gnuweeb/models/ui/buttons/full_message_btn.py b/daemon/dscord/gnuweeb/models/ui/buttons/full_message_btn.py
similarity index 100%
rename from daemon/discord/gnuweeb/models/ui/buttons/full_message_btn.py
rename to daemon/dscord/gnuweeb/models/ui/buttons/full_message_btn.py
diff --git a/daemon/discord/gnuweeb/plugins/__init__.py b/daemon/dscord/gnuweeb/plugins/__init__.py
similarity index 100%
rename from daemon/discord/gnuweeb/plugins/__init__.py
rename to daemon/dscord/gnuweeb/plugins/__init__.py
diff --git a/daemon/discord/gnuweeb/plugins/basic_commands/__init__.py b/daemon/dscord/gnuweeb/plugins/basic_commands/__init__.py
similarity index 100%
rename from daemon/discord/gnuweeb/plugins/basic_commands/__init__.py
rename to daemon/dscord/gnuweeb/plugins/basic_commands/__init__.py
diff --git a/daemon/discord/gnuweeb/plugins/basic_commands/debugger.py b/daemon/dscord/gnuweeb/plugins/basic_commands/debugger.py
similarity index 94%
rename from daemon/discord/gnuweeb/plugins/basic_commands/debugger.py
rename to daemon/dscord/gnuweeb/plugins/basic_commands/debugger.py
index be5b9f1..0bde454 100644
--- a/daemon/discord/gnuweeb/plugins/basic_commands/debugger.py
+++ b/daemon/dscord/gnuweeb/plugins/basic_commands/debugger.py
@@ -4,7 +4,7 @@
#
from discord.ext import commands
-from gnuweeb import utils
+from dscord.gnuweeb import utils
# from gnuweeb import filters
diff --git a/daemon/discord/gnuweeb/plugins/basic_commands/sync_it.py b/daemon/dscord/gnuweeb/plugins/basic_commands/sync_it.py
similarity index 70%
rename from daemon/discord/gnuweeb/plugins/basic_commands/sync_it.py
rename to daemon/dscord/gnuweeb/plugins/basic_commands/sync_it.py
index 2ed35cc..fbaec03 100644
--- a/daemon/discord/gnuweeb/plugins/basic_commands/sync_it.py
+++ b/daemon/dscord/gnuweeb/plugins/basic_commands/sync_it.py
@@ -7,14 +7,14 @@ from discord.ext import commands
class SyncCommand(commands.Cog):
- def __init__(self, bot) -> None:
+ def __init__(self, bot: "commands.Bot") -> None:
self.bot = bot
@commands.command("sync", aliases=["s"])
@commands.is_owner()
async def sync_it(self, ctx: "commands.Context"):
- ctx.bot.tree.copy_global_to(guild=ctx.guild)
- s = await ctx.bot.tree.sync(guild=ctx.guild)
+ self.bot.tree.copy_global_to(guild=ctx.guild)
+ s = await self.bot.tree.sync(guild=ctx.guild)
await ctx.send(f"Synced {len(s)} commands.")
diff --git a/daemon/discord/gnuweeb/plugins/events/__init__.py b/daemon/dscord/gnuweeb/plugins/events/__init__.py
similarity index 100%
rename from daemon/discord/gnuweeb/plugins/events/__init__.py
rename to daemon/dscord/gnuweeb/plugins/events/__init__.py
diff --git a/daemon/discord/gnuweeb/plugins/events/on_error.py b/daemon/dscord/gnuweeb/plugins/events/on_error.py
similarity index 100%
rename from daemon/discord/gnuweeb/plugins/events/on_error.py
rename to daemon/dscord/gnuweeb/plugins/events/on_error.py
diff --git a/daemon/discord/gnuweeb/plugins/events/on_ready.py b/daemon/dscord/gnuweeb/plugins/events/on_ready.py
similarity index 100%
rename from daemon/discord/gnuweeb/plugins/events/on_ready.py
rename to daemon/dscord/gnuweeb/plugins/events/on_ready.py
diff --git a/daemon/discord/gnuweeb/plugins/slash_commands/__init__.py b/daemon/dscord/gnuweeb/plugins/slash_commands/__init__.py
similarity index 100%
rename from daemon/discord/gnuweeb/plugins/slash_commands/__init__.py
rename to daemon/dscord/gnuweeb/plugins/slash_commands/__init__.py
diff --git a/daemon/discord/gnuweeb/plugins/slash_commands/get_lore_mail.py b/daemon/dscord/gnuweeb/plugins/slash_commands/get_lore_mail.py
similarity index 95%
rename from daemon/discord/gnuweeb/plugins/slash_commands/get_lore_mail.py
rename to daemon/dscord/gnuweeb/plugins/slash_commands/get_lore_mail.py
index a2671f4..38eb465 100644
--- a/daemon/discord/gnuweeb/plugins/slash_commands/get_lore_mail.py
+++ b/daemon/dscord/gnuweeb/plugins/slash_commands/get_lore_mail.py
@@ -10,8 +10,8 @@ from discord.ext import commands
from discord import Interaction
from discord import app_commands
-from mailer import utils
-from mailer import Scraper
+from atom import utils
+from atom import Scraper
class GetLoreSC(commands.Cog):
diff --git a/daemon/discord/gnuweeb/plugins/slash_commands/manage_atom.py b/daemon/dscord/gnuweeb/plugins/slash_commands/manage_atom.py
similarity index 97%
rename from daemon/discord/gnuweeb/plugins/slash_commands/manage_atom.py
rename to daemon/dscord/gnuweeb/plugins/slash_commands/manage_atom.py
index 2d9da80..9666264 100644
--- a/daemon/discord/gnuweeb/plugins/slash_commands/manage_atom.py
+++ b/daemon/dscord/gnuweeb/plugins/slash_commands/manage_atom.py
@@ -7,8 +7,8 @@ from discord.ext import commands
from discord import Interaction
from discord import app_commands
-from gnuweeb import filters
-from mailer import utils
+from dscord.gnuweeb import filters
+from atom import utils
class ManageAtomSC(commands.Cog):
diff --git a/daemon/discord/gnuweeb/plugins/slash_commands/manage_broadcast.py b/daemon/dscord/gnuweeb/plugins/slash_commands/manage_broadcast.py
similarity index 96%
rename from daemon/discord/gnuweeb/plugins/slash_commands/manage_broadcast.py
rename to daemon/dscord/gnuweeb/plugins/slash_commands/manage_broadcast.py
index 9eb6b98..86f5fec 100644
--- a/daemon/discord/gnuweeb/plugins/slash_commands/manage_broadcast.py
+++ b/daemon/dscord/gnuweeb/plugins/slash_commands/manage_broadcast.py
@@ -7,8 +7,8 @@ from discord.ext import commands
from discord import Interaction
from discord import app_commands
-from gnuweeb import utils
-from gnuweeb import filters
+from dscord.gnuweeb import utils
+from dscord.gnuweeb import filters
class ManageBroadcastSC(commands.Cog):
diff --git a/daemon/discord/gnuweeb/utils.py b/daemon/dscord/gnuweeb/utils.py
similarity index 100%
rename from daemon/discord/gnuweeb/utils.py
rename to daemon/dscord/gnuweeb/utils.py
diff --git a/daemon/discord/mailer/__init__.py b/daemon/dscord/mailer/__init__.py
similarity index 82%
rename from daemon/discord/mailer/__init__.py
rename to daemon/dscord/mailer/__init__.py
index 0da4329..e9c0190 100644
--- a/daemon/discord/mailer/__init__.py
+++ b/daemon/dscord/mailer/__init__.py
@@ -4,4 +4,3 @@
#
from .database import Database
-from .scraper import Scraper
diff --git a/daemon/discord/mailer/database.py b/daemon/dscord/mailer/database.py
similarity index 100%
rename from daemon/discord/mailer/database.py
rename to daemon/dscord/mailer/database.py
diff --git a/daemon/discord/mailer/listener.py b/daemon/dscord/mailer/listener.py
similarity index 85%
rename from daemon/discord/mailer/listener.py
rename to daemon/dscord/mailer/listener.py
index 523a87d..7538ba9 100644
--- a/daemon/discord/mailer/listener.py
+++ b/daemon/dscord/mailer/listener.py
@@ -11,14 +11,10 @@ import re
from apscheduler.schedulers.asyncio import AsyncIOScheduler
from discord import File
-from gnuweeb import GWClient
-from .scraper import Scraper
-from . import utils
-
-
-class BotMutexes():
- def __init__(self):
- self.lock = asyncio.Lock()
+from dscord.gnuweeb import GWClient
+from atom.utils import Mutexes
+from atom.scraper import Scraper
+from atom import utils
class Listener():
@@ -26,8 +22,8 @@ class Listener():
self,
client: "GWClient",
sched: "AsyncIOScheduler",
- scraper: Scraper,
- mutexes: "BotMutexes"
+ scraper: "Scraper",
+ mutexes: "Mutexes"
):
self.client = client
self.sched = sched
@@ -106,7 +102,7 @@ class Listener():
#
return False
- text, files, is_patch = utils.create_template(mail)
+ text, files, is_patch = utils.create_template(mail, "discord")
reply_to = self.get_discord_reply(mail, dc_chat_id)
url = str(re.sub(r"/raw$", "", url))
@@ -120,7 +116,7 @@ class Listener():
dc_guild_id, dc_chat_id, text, reply_to, url
)
- self.db.insert_discord(email_id, m.channel.id, m.id)
+ self.db.save_discord_mail(email_id, m.channel.id, m.id)
for d, f in files:
await m.reply(f"{d}/{f}", file=File(f))
await asyncio.sleep(1)
@@ -132,11 +128,11 @@ class Listener():
def __get_email_id_sent(self, email_msg_id, dc_chat_id):
- email_id = self.db.save_email_msg_id(email_msg_id)
+ email_id = self.db.save_email(email_msg_id)
if email_id:
return email_id
- email_id = self.db.get_email_id_sent(email_msg_id, dc_chat_id)
+ email_id = self.db.get_email_id(email_msg_id, dc_chat_id)
return email_id
@@ -149,4 +145,4 @@ class Listener():
if not reply_to:
return None
- return self.db.get_discord_reply(reply_to, dc_chat_id)
+ return self.db.get_reply_id(reply_to, dc_chat_id)
diff --git a/daemon/discord/requirements.txt b/daemon/dscord/requirements.txt
similarity index 100%
rename from daemon/discord/requirements.txt
rename to daemon/dscord/requirements.txt
diff --git a/daemon/discord/storage/.gitignore b/daemon/dscord/storage/.gitignore
similarity index 100%
rename from daemon/discord/storage/.gitignore
rename to daemon/dscord/storage/.gitignore
diff --git a/daemon/telegram/.env.example b/daemon/telegram.env.example
similarity index 100%
rename from daemon/telegram/.env.example
rename to daemon/telegram.env.example
diff --git a/daemon/telegram/__init__.py b/daemon/telegram/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/daemon/telegram/database/__init__.py b/daemon/telegram/database/__init__.py
new file mode 100644
index 0000000..13cff25
--- /dev/null
+++ b/daemon/telegram/database/__init__.py
@@ -0,0 +1 @@
+from .core import DB
diff --git a/daemon/telegram/database/core.py b/daemon/telegram/database/core.py
new file mode 100644
index 0000000..cfd9cc2
--- /dev/null
+++ b/daemon/telegram/database/core.py
@@ -0,0 +1,19 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Copyright (C) 2022 Muhammad Rizki <[email protected]>
+# Copyright (C) 2022 Ammar Faizi <[email protected]>
+#
+
+from .methods import DBMethods
+
+
+class DB(DBMethods):
+ def __init__(self, conn):
+ self.conn = conn
+ self.conn.autocommit = True
+ self.cur = self.conn.cursor(buffered=True)
+
+
+ def __del__(self):
+ self.cur.close()
+ self.conn.close()
diff --git a/daemon/telegram/database/methods/__init__.py b/daemon/telegram/database/methods/__init__.py
new file mode 100644
index 0000000..6099941
--- /dev/null
+++ b/daemon/telegram/database/methods/__init__.py
@@ -0,0 +1,10 @@
+from .deletion import Deletion
+from .getter import Getter
+from .insertion import Insertion
+
+
+class DBMethods(
+ Deletion,
+ Getter,
+ Insertion
+): pass
diff --git a/daemon/telegram/database/methods/deletion/__init__.py b/daemon/telegram/database/methods/deletion/__init__.py
new file mode 100644
index 0000000..fa80464
--- /dev/null
+++ b/daemon/telegram/database/methods/deletion/__init__.py
@@ -0,0 +1,8 @@
+from .delet_atom import DeleteAtom
+from .delete_broadcast import DeleteBroadcast
+
+
+class Deletion(
+ DeleteAtom,
+ DeleteBroadcast
+): pass
diff --git a/daemon/telegram/database/methods/deletion/delet_atom.py b/daemon/telegram/database/methods/deletion/delet_atom.py
new file mode 100644
index 0000000..4923045
--- /dev/null
+++ b/daemon/telegram/database/methods/deletion/delet_atom.py
@@ -0,0 +1,12 @@
+
+
+
+class DeleteAtom:
+
+ def delete_atom(self, atom: str):
+ q = """
+ DELETE FROM atom_urls
+ WHERE url = %(atom)s
+ """
+ self.cur.execute(q, {"atom": atom})
+ return self.cur.rowcount > 0
diff --git a/daemon/telegram/database/methods/deletion/delete_broadcast.py b/daemon/telegram/database/methods/deletion/delete_broadcast.py
new file mode 100644
index 0000000..a2c0751
--- /dev/null
+++ b/daemon/telegram/database/methods/deletion/delete_broadcast.py
@@ -0,0 +1,12 @@
+
+
+
+class DeleteBroadcast:
+
+ def delete_broadcast(self, chat_id: int):
+ q = """
+ DELETE FROM broadcast_chats
+ WHERE chat_id = %(chat_id)s
+ """
+ self.cur.execute(q, {"chat_id": chat_id})
+ return self.cur.rowcount > 0
diff --git a/daemon/telegram/database/methods/getter/__init__.py b/daemon/telegram/database/methods/getter/__init__.py
new file mode 100644
index 0000000..6a8a97d
--- /dev/null
+++ b/daemon/telegram/database/methods/getter/__init__.py
@@ -0,0 +1,12 @@
+from .get_atom_urls import GetAtomURL
+from .get_broadcast_chats import GetBroadcastChats
+from .get_email_id import GetEmailID
+from .get_telegram_reply import GetTelegramReply
+
+
+class Getter(
+ GetAtomURL,
+ GetBroadcastChats,
+ GetEmailID,
+ GetTelegramReply
+): pass
diff --git a/daemon/telegram/database/methods/getter/get_atom_urls.py b/daemon/telegram/database/methods/getter/get_atom_urls.py
new file mode 100644
index 0000000..b7353d2
--- /dev/null
+++ b/daemon/telegram/database/methods/getter/get_atom_urls.py
@@ -0,0 +1,18 @@
+
+
+
+class GetAtomURL:
+
+ def get_atom_urls(self):
+ '''
+ Get lore kernel raw email URLs.
+ - Return list of raw email URLs: `List[str]`
+ '''
+ q = """
+ SELECT atom_urls.url
+ FROM atom_urls
+ """
+ self.cur.execute(q)
+ urls = self.cur.fetchall()
+
+ return [u[0] for u in urls]
diff --git a/daemon/telegram/database/methods/getter/get_broadcast_chats.py b/daemon/telegram/database/methods/getter/get_broadcast_chats.py
new file mode 100644
index 0000000..43efe0c
--- /dev/null
+++ b/daemon/telegram/database/methods/getter/get_broadcast_chats.py
@@ -0,0 +1,18 @@
+
+
+
+class GetBroadcastChats:
+
+ def get_broadcast_chats(self):
+ '''
+ Get broadcast chats that are currently
+ listening for new email.
+ - Return list of chat object: `List[Object]`
+ '''
+ q = """
+ SELECT *
+ FROM broadcast_chats
+ """
+ self.cur.execute(q)
+
+ return self.cur.fetchall()
diff --git a/daemon/telegram/database/methods/getter/get_email_id.py b/daemon/telegram/database/methods/getter/get_email_id.py
new file mode 100644
index 0000000..115569e
--- /dev/null
+++ b/daemon/telegram/database/methods/getter/get_email_id.py
@@ -0,0 +1,58 @@
+
+
+
+class GetEmailID:
+
+ def get_email_id(self, email_id, chat_id):
+ '''
+ Determine whether the email needs to be sent to @tg_chat_id.
+ - Return an email id (PK) if it needs to be sent
+ - Return None if it doesn't need to be sent
+ '''
+ if self.__is_sent(email_id, chat_id):
+ return
+
+ res = self.__email_id(email_id)
+ if not bool(res):
+ return
+
+ return int(res[0])
+
+
+ def __is_sent(self, email_id, chat_id):
+ '''
+ Checking if this email has already been sent
+ or not.
+ - Return True if it's already been sent
+ '''
+
+ q = """
+ SELECT emails.id, tg_emails.id FROM emails LEFT JOIN tg_emails
+ ON emails.id = tg_emails.email_id
+ WHERE emails.message_id = %(email_id)s
+ AND tg_emails.chat_id = %(chat_id)s
+ LIMIT 1
+ """
+
+ self.cur.execute(q, {
+ "email_id": email_id,
+ "chat_id": chat_id
+ })
+
+ res = self.cur.fetchone()
+ return bool(res)
+
+
+ def __email_id(self, email_id):
+ '''
+ Get the email id if match with the email message_id.
+ - Return the result if it's match and exists
+ '''
+
+ q = """
+ SELECT id FROM emails WHERE message_id = %(email_id)s
+ """
+
+ self.cur.execute(q, {"email_id": email_id})
+ res = self.cur.fetchone()
+ return res
\ No newline at end of file
diff --git a/daemon/telegram/database/methods/getter/get_telegram_reply.py b/daemon/telegram/database/methods/getter/get_telegram_reply.py
new file mode 100644
index 0000000..2409225
--- /dev/null
+++ b/daemon/telegram/database/methods/getter/get_telegram_reply.py
@@ -0,0 +1,29 @@
+
+
+
+class GetTelegramReply:
+
+ def get_reply_id(self, email_msg_id, chat_id):
+ '''
+ Get Telegram message ID sent match with
+ email message ID and Telegram chat ID.
+ - Return Telegram message ID if exists: `int`
+ - Return None if not exists`
+ '''
+ q = """
+ SELECT tg_emails.tg_msg_id
+ FROM emails INNER JOIN tg_emails
+ ON emails.id = tg_emails.email_id
+ WHERE emails.message_id = %(email_msg_id)s
+ AND tg_emails.chat_id = %(chat_id)s
+ """
+
+ self.cur.execute(q, {
+ "email_msg_id": email_msg_id,
+ "chat_id": chat_id
+ })
+ res = self.cur.fetchone()
+ if not bool(res):
+ return None
+
+ return res[0]
diff --git a/daemon/telegram/database/methods/insertion/__init__.py b/daemon/telegram/database/methods/insertion/__init__.py
new file mode 100644
index 0000000..562bf41
--- /dev/null
+++ b/daemon/telegram/database/methods/insertion/__init__.py
@@ -0,0 +1,12 @@
+from .insert_atom import InsertAtom
+from .insert_broadcast import InsertBroadcast
+from .insert_email import InsertEmail
+from .insert_telegram import InsertTelegram
+
+
+class Insertion(
+ InsertAtom,
+ InsertBroadcast,
+ InsertEmail,
+ InsertTelegram
+): pass
diff --git a/daemon/telegram/database/methods/insertion/insert_atom.py b/daemon/telegram/database/methods/insertion/insert_atom.py
new file mode 100644
index 0000000..480aa02
--- /dev/null
+++ b/daemon/telegram/database/methods/insertion/insert_atom.py
@@ -0,0 +1,20 @@
+from mysql.connector import errors
+from datetime import datetime
+
+
+class InsertAtom:
+
+ def save_atom(self, atom: str):
+ try:
+ return self.__save_atom(atom)
+ except errors.IntegrityError:
+ #
+ # Duplicate data, skip!
+ #
+ return None
+
+
+ def __save_atom(self, atom: str):
+ q = "INSERT INTO atom_urls (url, created_at) VALUES (%s, %s)"
+ self.cur.execute(q, (atom, datetime.utcnow()))
+ return self.cur.lastrowid
diff --git a/daemon/telegram/database/methods/insertion/insert_broadcast.py b/daemon/telegram/database/methods/insertion/insert_broadcast.py
new file mode 100644
index 0000000..f4ee06d
--- /dev/null
+++ b/daemon/telegram/database/methods/insertion/insert_broadcast.py
@@ -0,0 +1,49 @@
+from mysql.connector import errors
+from datetime import datetime
+
+
+class InsertBroadcast:
+
+ def save_broadcast(
+ self,
+ chat_id: int,
+ name: str,
+ type: str,
+ created_at: "datetime",
+ username: str = None,
+ link: str = None,
+ ):
+ try:
+ return self.__insert_broadcast(
+ chat_id=chat_id,
+ name=name,
+ type=type,
+ created_at=created_at,
+ username=username,
+ link=link
+ )
+ except errors.IntegrityError:
+ #
+ # Duplicate data, skip!
+ #
+ return None
+
+
+ def __insert_broadcast(
+ self,
+ chat_id: int,
+ name: str,
+ type: str,
+ created_at: "datetime",
+ username: str = None,
+ link: str = None,
+ ):
+ q = """
+ INSERT INTO broadcast_chats
+ (chat_id, username, name, type, link, created_at)
+ VALUES
+ (%s, %s, %s, %s, %s, %s)
+ """
+ values = (chat_id, username, name, type, link, created_at)
+ self.cur.execute(q, values)
+ return self.cur.lastrowid
diff --git a/daemon/telegram/database/methods/insertion/insert_email.py b/daemon/telegram/database/methods/insertion/insert_email.py
new file mode 100644
index 0000000..bd88339
--- /dev/null
+++ b/daemon/telegram/database/methods/insertion/insert_email.py
@@ -0,0 +1,20 @@
+from mysql.connector import errors
+from datetime import datetime
+
+
+class InsertEmail:
+
+ def save_email(self, email_msg_id):
+ try:
+ return self.__insert_email(email_msg_id)
+ except errors.IntegrityError:
+ #
+ # Duplicate data, skip!
+ #
+ return None
+
+
+ def __insert_email(self, email_msg_id):
+ q = "INSERT INTO emails (message_id, created_at) VALUES (%s, %s)"
+ self.cur.execute(q, (email_msg_id, datetime.utcnow()))
+ return self.cur.lastrowid
diff --git a/daemon/telegram/database/methods/insertion/insert_telegram.py b/daemon/telegram/database/methods/insertion/insert_telegram.py
new file mode 100644
index 0000000..bf51526
--- /dev/null
+++ b/daemon/telegram/database/methods/insertion/insert_telegram.py
@@ -0,0 +1,14 @@
+from datetime import datetime
+
+
+class InsertTelegram:
+
+ def save_telegram_mail(self, email_id, tg_chat_id, tg_msg_id):
+ q = """
+ INSERT INTO tg_emails
+ (email_id, chat_id, tg_msg_id, created_at)
+ VALUES (%s, %s, %s, %s);
+ """
+ self.cur.execute(q, (email_id, tg_chat_id, tg_msg_id,
+ datetime.utcnow()))
+ return self.cur.lastrowid
diff --git a/daemon/telegram/scraper/__init__.py b/daemon/telegram/mailer/__init__.py
similarity index 68%
rename from daemon/telegram/scraper/__init__.py
rename to daemon/telegram/mailer/__init__.py
index 4294302..e6b4567 100644
--- a/daemon/telegram/scraper/__init__.py
+++ b/daemon/telegram/mailer/__init__.py
@@ -4,6 +4,5 @@
# Copyright (C) 2022 Ammar Faizi <[email protected]>
#
-from .scraper import Scraper
-from .bot import BotMutexes
-from .bot import Bot
+from .listener import Mutexes
+from .listener import Listener
diff --git a/daemon/telegram/scraper/bot.py b/daemon/telegram/mailer/listener.py
similarity index 66%
rename from daemon/telegram/scraper/bot.py
rename to daemon/telegram/mailer/listener.py
index 7adfb12..e36cac7 100644
--- a/daemon/telegram/scraper/bot.py
+++ b/daemon/telegram/mailer/listener.py
@@ -5,36 +5,33 @@
#
from apscheduler.schedulers.asyncio import AsyncIOScheduler
-from packages import DaemonClient
-from scraper import Scraper
-from . import utils
+from pyrogram.types import Message
+from ..packages import DaemonClient
+from atom import Scraper
+from atom import utils
+from atom.utils import Mutexes
import asyncio
import shutil
import re
import traceback
-class BotMutexes():
- def __init__(self):
- self.send_to_tg = asyncio.Lock()
-
-
-class Bot():
+class Listener:
def __init__(self, client: DaemonClient, sched: AsyncIOScheduler,
- scraper: Scraper, mutexes: BotMutexes):
+ mutexes: Mutexes):
self.client = client
self.sched = sched
- self.scraper = scraper
- self.mutexes = mutexes
+ self.mutex = mutexes
self.db = client.db
+ self.scraper = Scraper()
self.isRunnerFixed = False
def run(self):
- #
- # Execute __run() once to avoid high latency at
- # initilization.
- #
+ '''
+ Execute __run() once to avoid high latency at
+ initilization.
+ '''
self.runner = self.sched.add_job(
func=self.__run,
misfire_grace_time=None,
@@ -71,16 +68,16 @@ class Bot():
async def __handle_mail(self, url, mail):
chats = self.db.get_broadcast_chats()
for chat in chats:
- async with self.mutexes.send_to_tg:
- should_wait = await self.__send_to_tg(url, mail,
+ async with self.mutex.lock:
+ should_wait = await self.__send_mail(url, mail,
chat[1])
if should_wait:
await asyncio.sleep(1)
- # @__must_hold(self.mutexes.send_to_tg)
- async def __send_to_tg(self, url, mail, tg_chat_id):
+ # @__must_hold(self.mutex.lock)
+ async def __send_mail(self, url, mail, tg_chat_id):
email_msg_id = utils.get_email_msg_id(mail)
if not email_msg_id:
#
@@ -89,7 +86,7 @@ class Bot():
#
return False
- email_id = self.__need_to_send_to_telegram(email_msg_id,
+ email_id = self.__mail_id_from_db(email_msg_id,
tg_chat_id)
if not email_id:
#
@@ -98,21 +95,23 @@ class Bot():
#
return False
- text, files, is_patch = utils.create_template(mail)
- reply_to = self.get_tg_reply_to(mail, tg_chat_id)
+ text, files, is_patch = utils.create_template(
+ mail, "telegram"
+ )
+ reply_to = self.get_reply(mail, tg_chat_id)
url = str(re.sub(r"/raw$", "", url))
if is_patch:
- m = await self.client.send_patch_email(
+ m: "Message" = await self.client.send_patch_email(
mail, tg_chat_id, text, reply_to, url
)
else:
text = "#ml\n" + text
- m = await self.client.send_text_email(
+ m: "Message" = await self.client.send_text_email(
tg_chat_id, text,reply_to, url
)
- self.db.insert_telegram(email_id, m.chat.id, m.id)
+ self.db.save_telegram_mail(email_id, m.chat.id, m.id)
for d, f in files:
await m.reply_document(f"{d}/{f}", file_name=f)
await asyncio.sleep(1)
@@ -123,16 +122,16 @@ class Bot():
return True
- def __need_to_send_to_telegram(self, email_msg_id, tg_chat_id):
- email_id = self.db.save_email_msg_id(email_msg_id)
+ def __mail_id_from_db(self, email_msg_id, tg_chat_id):
+ email_id = self.db.save_email(email_msg_id)
if email_id:
return email_id
- email_id = self.db.need_to_send_to_tg(email_msg_id, tg_chat_id)
+ email_id = self.db.get_email_id(email_msg_id, tg_chat_id)
return email_id
- def get_tg_reply_to(self, mail, tg_chat_id):
+ def get_reply(self, mail, tg_chat_id):
reply_to = mail.get("in-reply-to")
if not reply_to:
return None
@@ -141,4 +140,4 @@ class Bot():
if not reply_to:
return None
- return self.db.get_tg_reply_to(reply_to, tg_chat_id)
+ return self.db.get_reply_id(reply_to, tg_chat_id)
diff --git a/daemon/telegram/packages/client.py b/daemon/telegram/packages/client.py
index 282daf6..75e1c63 100644
--- a/daemon/telegram/packages/client.py
+++ b/daemon/telegram/packages/client.py
@@ -6,19 +6,19 @@
from pyrogram import Client
from pyrogram.enums import ParseMode
from pyrogram.types import Message, InlineKeyboardMarkup, InlineKeyboardButton
+from atom import utils
from typing import Union
from email.message import Message
-from scraper import utils
-from scraper.db import Db
+from telegram.database import DB
from .decorator import handle_flood
class DaemonClient(Client):
- def __init__(self, name: str, api_id: int,
- api_hash: str, conn, **kwargs):
+ def __init__(self, name: str, api_id: int, api_hash: str,
+ conn, **kwargs):
super().__init__(name, api_id,
api_hash, **kwargs)
- self.db = Db(conn)
+ self.db = DB(conn)
@handle_flood
@@ -56,7 +56,9 @@ class DaemonClient(Client):
parse_mode: ParseMode = ParseMode.HTML
) -> Message:
print("[send_patch_email]")
- tmp, doc, caption, url = utils.prepare_send_patch(mail, text, url)
+ tmp, doc, caption, url = utils.prepare_patch(
+ mail, text, url, "telegram"
+ )
m = await self.send_document(
chat_id=chat_id,
document=doc,
@@ -71,5 +73,5 @@ class DaemonClient(Client):
])
)
- utils.clean_up_after_send_patch(tmp)
+ utils.remove_patch(tmp)
return m
diff --git a/daemon/telegram/packages/plugins/callbacks/del_atom.py b/daemon/telegram/packages/plugins/callbacks/del_atom.py
index 1510d60..b1076f9 100644
--- a/daemon/telegram/packages/plugins/callbacks/del_atom.py
+++ b/daemon/telegram/packages/plugins/callbacks/del_atom.py
@@ -3,10 +3,10 @@
# Copyright (C) 2022 Muhammad Rizki <[email protected]>
#
-from packages import DaemonClient
-from scraper import utils
+from telegram.packages import DaemonClient
from pyrogram.types import CallbackQuery
-import config
+from telegram import config
+from atom import utils
@DaemonClient.on_callback_query(config.admin_only, group=1)
diff --git a/daemon/telegram/packages/plugins/callbacks/del_chat.py b/daemon/telegram/packages/plugins/callbacks/del_chat.py
index 26c6dd8..de7d51e 100644
--- a/daemon/telegram/packages/plugins/callbacks/del_chat.py
+++ b/daemon/telegram/packages/plugins/callbacks/del_chat.py
@@ -3,10 +3,10 @@
# Copyright (C) 2022 Muhammad Rizki <[email protected]>
#
-from packages import DaemonClient
-from scraper import utils
+from telegram.packages import DaemonClient
from pyrogram.types import CallbackQuery
-import config
+from telegram import config
+from atom import utils
@DaemonClient.on_callback_query(config.admin_only, group=2)
diff --git a/daemon/telegram/packages/plugins/commands/debugger.py b/daemon/telegram/packages/plugins/commands/debugger.py
index ae2d31d..7f6f367 100644
--- a/daemon/telegram/packages/plugins/commands/debugger.py
+++ b/daemon/telegram/packages/plugins/commands/debugger.py
@@ -7,7 +7,7 @@ from pyrogram import Client, filters, enums
from pyrogram.types import Message
from textwrap import indent
import io, import_expression, contextlib, traceback
-import config
+from telegram import config
@Client.on_message(
diff --git a/daemon/telegram/packages/plugins/commands/manage_atom.py b/daemon/telegram/packages/plugins/commands/manage_atom.py
index bcb2f35..ef60260 100644
--- a/daemon/telegram/packages/plugins/commands/manage_atom.py
+++ b/daemon/telegram/packages/plugins/commands/manage_atom.py
@@ -5,9 +5,9 @@
from pyrogram.types import Message
from pyrogram import filters
-from packages import DaemonClient
-from scraper import utils
-import config
+from telegram.packages import DaemonClient
+from telegram import config
+from atom import utils
@DaemonClient.on_message(
@@ -25,7 +25,7 @@ async def add_atom_url(c: DaemonClient, m: Message):
if not is_atom:
return await m.reply("Invalid Atom URL")
- inserted = c.db.insert_atom(text)
+ inserted = c.db.save_atom(text)
if inserted is None:
return await m.reply(f"This URL already listened for new email.")
diff --git a/daemon/telegram/packages/plugins/commands/manage_broadcast.py b/daemon/telegram/packages/plugins/commands/manage_broadcast.py
index ffb5a6b..922f4fe 100644
--- a/daemon/telegram/packages/plugins/commands/manage_broadcast.py
+++ b/daemon/telegram/packages/plugins/commands/manage_broadcast.py
@@ -5,9 +5,9 @@
from pyrogram.types import Message
from pyrogram import filters, enums
-from packages import DaemonClient
-from scraper import utils
-import config
+from telegram.packages import DaemonClient
+from telegram import config
+from atom import utils
@DaemonClient.on_message(
@@ -20,7 +20,7 @@ async def add_broadcast(c: DaemonClient, m: Message):
else:
chat_name = m.chat.title
- inserted = c.db.insert_broadcast(
+ inserted = c.db.save_broadcast(
chat_id=m.chat.id,
name=chat_name,
type=str(m.chat.type),
diff --git a/daemon/telegram/packages/plugins/commands/scrape.py b/daemon/telegram/packages/plugins/commands/scrape.py
index 45b1581..f23ad6f 100644
--- a/daemon/telegram/packages/plugins/commands/scrape.py
+++ b/daemon/telegram/packages/plugins/commands/scrape.py
@@ -6,10 +6,10 @@
from pyrogram.types import Message
from pyrogram import filters
-from packages import DaemonClient
-from scraper import Scraper
-from scraper import utils
-import config
+from telegram.packages import DaemonClient
+from telegram import config
+from atom import Scraper
+from atom import utils
import shutil
import re
import asyncio
@@ -37,7 +37,7 @@ async def scrap_email(c: DaemonClient, m: Message):
s = Scraper()
mail = await s.get_email_from_url(url)
- text, files, is_patch = utils.create_template(mail)
+ text, files, is_patch = utils.create_template(mail, "telegram")
if is_patch:
m = await c.send_patch_email(
diff --git a/daemon/telegram/scraper/db.py b/daemon/telegram/scraper/db.py
deleted file mode 100644
index 58601d1..0000000
--- a/daemon/telegram/scraper/db.py
+++ /dev/null
@@ -1,217 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-#
-# Copyright (C) 2022 Muhammad Rizki <[email protected]>
-# Copyright (C) 2022 Ammar Faizi <[email protected]>
-#
-
-from datetime import datetime
-import mysql
-
-
-class Db():
- def __init__(self, conn):
- self.conn = conn
- self.conn.autocommit = True
- self.cur = self.conn.cursor(buffered=True)
-
-
- def __del__(self):
- self.cur.close()
- self.conn.close()
-
-
- def save_email_msg_id(self, email_msg_id):
- try:
- return self.__save_email_msg_id(email_msg_id)
- except mysql.connector.errors.IntegrityError:
- #
- # Duplicate data, skip!
- #
- return None
-
-
- def __save_email_msg_id(self, email_msg_id):
- q = "INSERT INTO emails (message_id, created_at) VALUES (%s, %s)"
- self.cur.execute(q, (email_msg_id, datetime.utcnow()))
- return self.cur.lastrowid
-
-
- def insert_telegram(self, email_id, tg_chat_id, tg_msg_id):
- q = """
- INSERT INTO tg_emails
- (email_id, chat_id, tg_msg_id, created_at)
- VALUES (%s, %s, %s, %s);
- """
- self.cur.execute(q, (email_id, tg_chat_id, tg_msg_id,
- datetime.utcnow()))
- return self.cur.lastrowid
-
-
- #
- # Determine whether the email needs to be sent to @tg_chat_id.
- #
- # - Return an email id (PK) if it needs to be sent.
- # - Return None if it doesn't need to be sent.
- #
- def need_to_send_to_tg(self, email_msg_id, tg_chat_id):
- q = """
- SELECT emails.id, tg_emails.id FROM emails LEFT JOIN tg_emails
- ON emails.id = tg_emails.email_id
- WHERE emails.message_id = %(email_msg_id)s
- AND tg_emails.chat_id = %(tg_chat_id)s
- LIMIT 1
- """
-
- self.cur.execute(
- q,
- {
- "email_msg_id": email_msg_id,
- "tg_chat_id": tg_chat_id
- }
- )
- res = self.cur.fetchone()
- if bool(res):
- #
- # This email has already been sent to
- # @tg_chat_id.
- #
- return None
-
- q = """
- SELECT id FROM emails WHERE message_id = %(email_msg_id)s
- """
- self.cur.execute(q, {"email_msg_id": email_msg_id})
- res = self.cur.fetchone()
- if not bool(res):
- #
- # Something goes wrong, skip!
- #
- return None
-
- return int(res[0])
-
-
- def get_tg_reply_to(self, email_msg_id, tg_chat_id):
- q = """
- SELECT tg_emails.tg_msg_id
- FROM emails INNER JOIN tg_emails
- ON emails.id = tg_emails.email_id
- WHERE emails.message_id = %(email_msg_id)s
- AND tg_emails.chat_id = %(chat_id)s
- """
-
- self.cur.execute(
- q,
- {
- "email_msg_id": email_msg_id,
- "chat_id": tg_chat_id
- }
- )
- res = self.cur.fetchone()
- if not bool(res):
- return None
-
- return res[0]
-
-
- def insert_atom(self, atom: str):
- try:
- return self.__save_atom(atom)
- except mysql.connector.errors.IntegrityError:
- #
- # Duplicate data, skip!
- #
- return None
-
-
- def __save_atom(self, atom: str):
- q = "INSERT INTO atom_urls (url, created_at) VALUES (%s, %s)"
- self.cur.execute(q, (atom, datetime.utcnow()))
- return self.cur.lastrowid
-
-
- def delete_atom(self, atom: str):
- q = """
- DELETE FROM atom_urls
- WHERE url = %(atom)s
- """
- try:
- self.cur.execute(q, {"atom": atom})
- return True
- except:
- return False
-
-
- def get_atom_urls(self):
- q = """
- SELECT atom_urls.url
- FROM atom_urls
- """
- self.cur.execute(q)
- urls = self.cur.fetchall()
-
- return [u[0] for u in urls]
-
-
- def insert_broadcast(
- self,
- chat_id: int,
- name: str,
- type: str,
- created_at: "datetime",
- username: str = None,
- link: str = None,
- ):
- try:
- return self.__save_broadcast(
- chat_id=chat_id,
- name=name,
- type=type,
- created_at=created_at,
- username=username,
- link=link
- )
- except mysql.connector.errors.IntegrityError:
- #
- # Duplicate data, skip!
- #
- return None
-
-
- def __save_broadcast(
- self,
- chat_id: int,
- name: str,
- type: str,
- created_at: "datetime",
- username: str = None,
- link: str = None,
- ):
- q = """
- INSERT INTO broadcast_chats
- (chat_id, username, name, type, link, created_at)
- VALUES
- (%s, %s, %s, %s, %s, %s)
- """
- values = (chat_id, username, name, type, link, created_at)
- self.cur.execute(q, values)
- return self.cur.lastrowid
-
-
- def delete_broadcast(self, chat_id: int):
- q = """
- DELETE FROM broadcast_chats
- WHERE chat_id = %(chat_id)s
- """
- self.cur.execute(q, {"chat_id": chat_id})
- return self.cur.rowcount > 0
-
-
- def get_broadcast_chats(self):
- q = """
- SELECT *
- FROM broadcast_chats
- """
- self.cur.execute(q)
-
- return self.cur.fetchall()
diff --git a/daemon/telegram/scraper/scraper.py b/daemon/telegram/scraper/scraper.py
deleted file mode 100644
index 2d5942b..0000000
--- a/daemon/telegram/scraper/scraper.py
+++ /dev/null
@@ -1,63 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-#
-# Copyright (C) 2022 Muhammad Rizki <[email protected]>
-# Copyright (C) 2022 Ammar Faizi <[email protected]>
-#
-
-from typing import Dict, List
-import email.policy
-import xmltodict
-import httpx
-import email
-
-
-class Scraper():
- async def get_new_threads_urls(self, atom_url):
- ret = await self.__get_atom_content(atom_url)
- return await self.__get_new_threads_from_atom(ret)
-
-
- async def __get_atom_content(self, atom_url):
- async with httpx.AsyncClient() as client:
- res = await client.get(atom_url)
- if res.status_code == 200:
- return res.text
- raise Exception(f"[get_atom_content]: Returned {res.status_code} HTTP code")
-
-
- async def __get_new_threads_from_atom(self, atom):
- j: Dict[str, List[
- Dict[str, str]
- ]] = xmltodict.parse(atom)["feed"]
-
- entry = []
- e = j["entry"]
- for i in e:
- entry.append({
- "link": i["link"]["@href"],
- "title": i["title"],
- "updated": i["updated"],
- })
- #
- # TODO(ammarfaizi2): Sort by title as well if the @updated is
- # identic.
- #
- entry.sort(key=lambda x: x["updated"])
-
- ret = []
- for i in entry:
- link = i["link"].replace("http://", "https://")
- ret.append(link + "raw")
-
- return ret
-
-
- async def get_email_from_url(self, url):
- async with httpx.AsyncClient() as client:
- res = await client.get(url)
- if res.status_code == 200:
- return email.message_from_string(
- res.text,
- policy=email.policy.default
- )
- raise Exception(f"[get_atom_content]: Returned {res.status_code} HTTP code")
diff --git a/daemon/telegram/run.py b/daemon/tg.py
similarity index 75%
rename from daemon/telegram/run.py
rename to daemon/tg.py
index 5360395..2e0418a 100644
--- a/daemon/telegram/run.py
+++ b/daemon/tg.py
@@ -5,20 +5,19 @@
#
from apscheduler.schedulers.asyncio import AsyncIOScheduler
-from scraper import BotMutexes
+from atom.utils import Mutexes
from dotenv import load_dotenv
from mysql import connector
-from packages import DaemonClient
-from scraper import Scraper
-from scraper import Bot
+from telegram.packages import DaemonClient
+from telegram.mailer import Listener
import os
def main():
- load_dotenv()
+ load_dotenv("telegram.env")
client = DaemonClient(
- "storage/EmailScraper",
+ "telegram/storage/EmailScraper",
api_id=int(os.getenv("API_ID")),
api_hash=os.getenv("API_HASH"),
bot_token=os.getenv("BOT_TOKEN"),
@@ -28,9 +27,7 @@ def main():
password=os.getenv("DB_PASS"),
database=os.getenv("DB_NAME")
),
- plugins=dict(
- root="packages.plugins"
- ),
+ plugins=dict(root="telegram.packages.plugins")
)
sched = AsyncIOScheduler(
@@ -40,11 +37,10 @@ def main():
}
)
- bot = Bot(
+ bot = Listener(
client=client,
sched=sched,
- scraper=Scraper(),
- mutexes=BotMutexes()
+ mutexes=Mutexes()
)
sched.start()
bot.run()
--
Muhammad Rizki
^ permalink raw reply related [flat|nested] 8+ messages in thread