[PATCH v2 18/28] discord: Add mail listener

[Muhammad Rizki <kiizuha@gnuweeb.org>]

Add the mail listener to receive new lore kernel email messages, then
send them to the Discord channel. If there is no new email message, the
listener will ignore it.

Changelog:

v1 -> v2
  discord: Fix the patch directory deletion after sending patch files

  Fix a bug when the bot doesn't remove the patch directory after
  sending the patch files inside the directory. I don't know if this is
  fixed or not. I just thought I had patched it with many debugging
  methods. It's hard to catch these bugs.

  Main issues:
  Files are not deleted

  ammarfaizi2@integral2:~/app/lore-daemon.dev/daemon$ ls dscord/storage -l
  total 0
  drwxrwxr-x 1 ammarfaizi2 ammarfaizi2  78 Sep  3 17:25 23035eeb480f2b3501ba0841ef10ce3f
  drwxrwxr-x 1 ammarfaizi2 ammarfaizi2  12 Oct  3 19:42 6766c45fa800d71173893158b2c7d640
  drwxrwxr-x 1 ammarfaizi2 ammarfaizi2   0 Sep  3 17:22 78380c23529749c280e4cca46f7519c9
  drwxrwxr-x 1 ammarfaizi2 ammarfaizi2   0 Sep  3 17:23 a23e986acc2c75d08a3e98377f06bbfe
  drwxrwxr-x 1 ammarfaizi2 ammarfaizi2 356 Sep  3 17:22 b241bd7c932c70e6a81d107da019cb67
  drwxrwxr-x 1 ammarfaizi2 ammarfaizi2  12 Sep  3 17:25 bcaac8364eaf04951d504faa1f1df826
  drwxrwxr-x 1 ammarfaizi2 ammarfaizi2   0 Sep  3 17:23 c36cdede05b536d395c5b55dbae92288

  These directories is not removed after sending the patch files.

  From:
  https://lore.gnuweeb.org/gwml/7c6e2632-b8b2-a7dd-f972-07b2b864c20a@gnuweeb.org/

  Logs:
  Traceback (most recent call last):
   File "/home/ammarfaizi2/app/lore-daemon.dev/daemon/dscord/mailer/listener.py", line 56, in __run
     await self.__handle_atom_url(url)
   File "/home/ammarfaizi2/app/lore-daemon.dev/daemon/dscord/mailer/listener.py", line 75, in __handle_atom_url
     await self.__handle_mail(url, mail)
   File "/home/ammarfaizi2/app/lore-daemon.dev/daemon/dscord/mailer/listener.py", line 83, in __handle_mail
     await self.__send_to_discord(url, mail,
   File "/home/ammarfaizi2/app/lore-daemon.dev/daemon/dscord/mailer/listener.py", line 127, in __send_to_discord
     await m.reply(f"{d}/{f}", file=File(f))
   File "/home/ammarfaizi2/.local/lib/python3.10/site-packages/discord/file.py", line 97, in __init__
     self.fp = open(fp, 'rb')
  FileNotFoundError: [Errno 2] No such file or directory: 'config'

Signed-off-by: Muhammad Rizki <kiizuha@gnuweeb.org>
---
 daemon/dscord/mailer/__init__.py |   7 ++
 daemon/dscord/mailer/listener.py | 153 +++++++++++++++++++++++++++++++
 2 files changed, 160 insertions(+)
 create mode 100644 daemon/dscord/mailer/__init__.py
 create mode 100644 daemon/dscord/mailer/listener.py

diff --git a/daemon/dscord/mailer/__init__.py b/daemon/dscord/mailer/__init__.py
new file mode 100644
index 0000000..5d7e8d8
--- /dev/null
+++ b/daemon/dscord/mailer/__init__.py
@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Copyright (C) 2022  Muhammad Rizki <kiizuha@gnuweeb.org>
+#
+
+from .listener import Listener
+from .listener import Mutexes
diff --git a/daemon/dscord/mailer/listener.py b/daemon/dscord/mailer/listener.py
new file mode 100644
index 0000000..a280a58
--- /dev/null
+++ b/daemon/dscord/mailer/listener.py
@@ -0,0 +1,153 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Copyright (C) 2022  Muhammad Rizki <kiizuha@gnuweeb.org>
+# Copyright (C) 2022  Ammar Faizi <ammarfaizi2@gnuweeb.org>
+#
+
+import asyncio
+import traceback
+import re
+from apscheduler.schedulers.asyncio import AsyncIOScheduler
+from discord import File
+from discord import Message
+
+from dscord.gnuweeb import GWClient
+from atom.scraper import Scraper
+from atom import utils
+
+
+class Mutexes:
+	def __init__(self):
+		self.lock = asyncio.Lock()
+
+
+class Listener:
+	def __init__(
+                self,
+                client: "GWClient",
+                sched: "AsyncIOScheduler",
+		scraper: "Scraper",
+                mutexes: "Mutexes"
+        ):
+		self.client = client
+		self.sched = sched
+		self.scraper = scraper
+		self.mutexes = mutexes
+		self.db = client.db
+		self.isRunnerFixed = False
+		self.runner = None
+
+
+	def run(self):
+		#
+		# Execute __run() once to avoid high latency at
+		# initilization.
+		#
+		print("Initialize listener...\n")
+		self.sched.start()
+		self.runner = self.sched.add_job(func=self.__run)
+
+
+	async def __run(self):
+		print("[__run]: Running...")
+		for url in self.db.get_atom_urls():
+			try:
+				await self.__handle_atom_url(url)
+			except:
+				print(traceback.format_exc())
+
+		if not self.isRunnerFixed:
+			self.isRunnerFixed = True
+			self.runner = self.sched.add_job(
+				func=self.__run,
+				trigger="interval",
+				seconds=30,
+				misfire_grace_time=None,
+				max_instances=1
+			)
+
+
+	async def __handle_atom_url(self, url):
+		urls = await self.scraper.get_new_threads_urls(url)
+		for url in urls:
+			mail = await self.scraper.get_email_from_url(url)
+			await self.__handle_mail(url, mail)
+
+
+	async def __handle_mail(self, url, mail):
+		chats = self.db.get_broadcast_chats()
+		for chat in chats:
+			async with self.mutexes.lock:
+				should_wait = \
+				await self.__send_to_discord(url, mail,
+							chat[1], chat[2])
+
+			if should_wait:
+				await asyncio.sleep(1)
+
+
+	# @__must_hold(self.mutexes.lock)
+	async def __send_to_discord(self, url, mail, dc_guild_id, dc_chat_id):
+		email_msg_id = utils.get_email_msg_id(mail)
+		if not email_msg_id:
+			#
+			# It doesn't have a Message-Id.
+			# A malformed email. Skip!
+			#
+			return False
+
+		email_id = self.__get_email_id_sent(
+			email_msg_id=email_msg_id,
+			dc_chat_id=dc_chat_id
+		)
+		if not email_id:
+			#
+			# Email has already been sent to Discord.
+			# Skip!
+			#
+			return False
+
+		text, files, is_patch = utils.create_template(mail, "discord")
+		reply_to = self.get_discord_reply(mail, dc_chat_id)
+		url = str(re.sub(r"/raw$", "", url))
+
+		if is_patch:
+			m: Message = await self.client.send_patch_email(
+				mail, dc_guild_id, dc_chat_id, text, reply_to, url
+			)
+		else:
+			text = "#ml\n" + text
+			m: Message = await self.client.send_text_email(
+				dc_guild_id, dc_chat_id, text, reply_to, url
+			)
+
+		self.db.save_discord_mail(email_id, m.channel.id, m.id)
+
+		for d, f in files:
+			await m.reply(file=File(f"{d}/{f}"))
+			if files.index((d,f)) == len(files)-1:
+				utils.remove_patch(d)
+			await asyncio.sleep(1)
+
+		return True
+
+
+	def __get_email_id_sent(self, email_msg_id, dc_chat_id):
+		email_id = self.db.save_email(email_msg_id)
+		if email_id:
+			return email_id
+
+		email_id = self.db.get_email_id(email_msg_id, dc_chat_id)
+		return email_id
+
+
+	def get_discord_reply(self, mail, dc_chat_id):
+		reply_to = mail.get("in-reply-to")
+		if not reply_to:
+			return None
+
+		reply_to = utils.extract_email_msg_id(reply_to)
+		if not reply_to:
+			return None
+
+		return self.db.get_reply_id(reply_to, dc_chat_id)
-- 
Muhammad Rizki