From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.6 (2021-04-09) on gnuweeb.org X-Spam-Level: X-Spam-Status: No, score=-0.8 required=5.0 tests=ALL_TRUSTED,DKIM_SIGNED, DKIM_VALID,DKIM_VALID_AU,DKIM_VALID_EF,NO_DNS_FOR_FROM,URIBL_BLOCKED autolearn=no autolearn_force=no version=3.4.6 Received: from localhost.localdomain (unknown [101.128.125.217]) by gnuweeb.org (Postfix) with ESMTPSA id B90157E257; Mon, 3 Oct 2022 23:53:31 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=gnuweeb.org; s=default; t=1664841213; bh=jVskvbpEG0Sml6zVxB2C32XNGP+dP+Y9F3EBnwwyVso=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=WvFs1RqB9slH5pCUfaBqQ0BXInMbyn4TRAZMUWTKUlD+MFVc3Rgl3zsN+/+K6eJAq slhFdrRL1kA4scBiOdV6ozj5AmiloztieTwCCdSHTwvGcP79wh2AdU/MYRuid8T/83 iJcLAy3PZcVNUnOkSZkQwyxFQOycAD5ZanDKrOiW8UpTa7EcXDX/CJFDKUt28wOVhU aFiqiZHFfBPuCSJuJfrc8Juhuy92vW8atu9qu62D3jiDnks+T5s77Yu4hL0Ex1KFto ZrxHc9oh9A474Dy71FJG/oQ3c6FWOcRs1zxkcGUpt+VlHJEuQ/273fUw3SXe3lWGKP gCo9Hg4KoAc9A== From: Muhammad Rizki To: Ammar Faizi Cc: Muhammad Rizki , Alviro Iskandar Setiawan , GNU/Weeb Mailing List Subject: [PATCH v2 18/28] discord: Add mail listener Date: Tue, 4 Oct 2022 06:52:19 +0700 Message-Id: <20221003235230.1824-19-kiizuha@gnuweeb.org> X-Mailer: git-send-email 2.34.1.windows.1 In-Reply-To: <20221003235230.1824-1-kiizuha@gnuweeb.org> References: <20221003235230.1824-1-kiizuha@gnuweeb.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: Add the mail listener to receive new lore kernel email messages, then send them to the Discord channel. If there is no new email message, the listener will ignore it. Changelog: v1 -> v2 discord: Fix the patch directory deletion after sending patch files Fix a bug when the bot doesn't remove the patch directory after sending the patch files inside the directory. I don't know if this is fixed or not. I just thought I had patched it with many debugging methods. It's hard to catch these bugs. Main issues: Files are not deleted ammarfaizi2@integral2:~/app/lore-daemon.dev/daemon$ ls dscord/storage -l total 0 drwxrwxr-x 1 ammarfaizi2 ammarfaizi2 78 Sep 3 17:25 23035eeb480f2b3501ba0841ef10ce3f drwxrwxr-x 1 ammarfaizi2 ammarfaizi2 12 Oct 3 19:42 6766c45fa800d71173893158b2c7d640 drwxrwxr-x 1 ammarfaizi2 ammarfaizi2 0 Sep 3 17:22 78380c23529749c280e4cca46f7519c9 drwxrwxr-x 1 ammarfaizi2 ammarfaizi2 0 Sep 3 17:23 a23e986acc2c75d08a3e98377f06bbfe drwxrwxr-x 1 ammarfaizi2 ammarfaizi2 356 Sep 3 17:22 b241bd7c932c70e6a81d107da019cb67 drwxrwxr-x 1 ammarfaizi2 ammarfaizi2 12 Sep 3 17:25 bcaac8364eaf04951d504faa1f1df826 drwxrwxr-x 1 ammarfaizi2 ammarfaizi2 0 Sep 3 17:23 c36cdede05b536d395c5b55dbae92288 These directories is not removed after sending the patch files. From: https://lore.gnuweeb.org/gwml/7c6e2632-b8b2-a7dd-f972-07b2b864c20a@gnuweeb.org/ Logs: Traceback (most recent call last): File "/home/ammarfaizi2/app/lore-daemon.dev/daemon/dscord/mailer/listener.py", line 56, in __run await self.__handle_atom_url(url) File "/home/ammarfaizi2/app/lore-daemon.dev/daemon/dscord/mailer/listener.py", line 75, in __handle_atom_url await self.__handle_mail(url, mail) File "/home/ammarfaizi2/app/lore-daemon.dev/daemon/dscord/mailer/listener.py", line 83, in __handle_mail await self.__send_to_discord(url, mail, File "/home/ammarfaizi2/app/lore-daemon.dev/daemon/dscord/mailer/listener.py", line 127, in __send_to_discord await m.reply(f"{d}/{f}", file=File(f)) File "/home/ammarfaizi2/.local/lib/python3.10/site-packages/discord/file.py", line 97, in __init__ self.fp = open(fp, 'rb') FileNotFoundError: [Errno 2] No such file or directory: 'config' Signed-off-by: Muhammad Rizki --- daemon/dscord/mailer/__init__.py | 7 ++ daemon/dscord/mailer/listener.py | 153 +++++++++++++++++++++++++++++++ 2 files changed, 160 insertions(+) create mode 100644 daemon/dscord/mailer/__init__.py create mode 100644 daemon/dscord/mailer/listener.py diff --git a/daemon/dscord/mailer/__init__.py b/daemon/dscord/mailer/__init__.py new file mode 100644 index 0000000..5d7e8d8 --- /dev/null +++ b/daemon/dscord/mailer/__init__.py @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Copyright (C) 2022 Muhammad Rizki +# + +from .listener import Listener +from .listener import Mutexes diff --git a/daemon/dscord/mailer/listener.py b/daemon/dscord/mailer/listener.py new file mode 100644 index 0000000..a280a58 --- /dev/null +++ b/daemon/dscord/mailer/listener.py @@ -0,0 +1,153 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Copyright (C) 2022 Muhammad Rizki +# Copyright (C) 2022 Ammar Faizi +# + +import asyncio +import traceback +import re +from apscheduler.schedulers.asyncio import AsyncIOScheduler +from discord import File +from discord import Message + +from dscord.gnuweeb import GWClient +from atom.scraper import Scraper +from atom import utils + + +class Mutexes: + def __init__(self): + self.lock = asyncio.Lock() + + +class Listener: + def __init__( + self, + client: "GWClient", + sched: "AsyncIOScheduler", + scraper: "Scraper", + mutexes: "Mutexes" + ): + self.client = client + self.sched = sched + self.scraper = scraper + self.mutexes = mutexes + self.db = client.db + self.isRunnerFixed = False + self.runner = None + + + def run(self): + # + # Execute __run() once to avoid high latency at + # initilization. + # + print("Initialize listener...\n") + self.sched.start() + self.runner = self.sched.add_job(func=self.__run) + + + async def __run(self): + print("[__run]: Running...") + for url in self.db.get_atom_urls(): + try: + await self.__handle_atom_url(url) + except: + print(traceback.format_exc()) + + if not self.isRunnerFixed: + self.isRunnerFixed = True + self.runner = self.sched.add_job( + func=self.__run, + trigger="interval", + seconds=30, + misfire_grace_time=None, + max_instances=1 + ) + + + async def __handle_atom_url(self, url): + urls = await self.scraper.get_new_threads_urls(url) + for url in urls: + mail = await self.scraper.get_email_from_url(url) + await self.__handle_mail(url, mail) + + + async def __handle_mail(self, url, mail): + chats = self.db.get_broadcast_chats() + for chat in chats: + async with self.mutexes.lock: + should_wait = \ + await self.__send_to_discord(url, mail, + chat[1], chat[2]) + + if should_wait: + await asyncio.sleep(1) + + + # @__must_hold(self.mutexes.lock) + async def __send_to_discord(self, url, mail, dc_guild_id, dc_chat_id): + email_msg_id = utils.get_email_msg_id(mail) + if not email_msg_id: + # + # It doesn't have a Message-Id. + # A malformed email. Skip! + # + return False + + email_id = self.__get_email_id_sent( + email_msg_id=email_msg_id, + dc_chat_id=dc_chat_id + ) + if not email_id: + # + # Email has already been sent to Discord. + # Skip! + # + return False + + text, files, is_patch = utils.create_template(mail, "discord") + reply_to = self.get_discord_reply(mail, dc_chat_id) + url = str(re.sub(r"/raw$", "", url)) + + if is_patch: + m: Message = await self.client.send_patch_email( + mail, dc_guild_id, dc_chat_id, text, reply_to, url + ) + else: + text = "#ml\n" + text + m: Message = await self.client.send_text_email( + dc_guild_id, dc_chat_id, text, reply_to, url + ) + + self.db.save_discord_mail(email_id, m.channel.id, m.id) + + for d, f in files: + await m.reply(file=File(f"{d}/{f}")) + if files.index((d,f)) == len(files)-1: + utils.remove_patch(d) + await asyncio.sleep(1) + + return True + + + def __get_email_id_sent(self, email_msg_id, dc_chat_id): + email_id = self.db.save_email(email_msg_id) + if email_id: + return email_id + + email_id = self.db.get_email_id(email_msg_id, dc_chat_id) + return email_id + + + def get_discord_reply(self, mail, dc_chat_id): + reply_to = mail.get("in-reply-to") + if not reply_to: + return None + + reply_to = utils.extract_email_msg_id(reply_to) + if not reply_to: + return None + + return self.db.get_reply_id(reply_to, dc_chat_id) -- Muhammad Rizki