From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.6 (2021-04-09) on gnuweeb.org X-Spam-Level: X-Spam-Status: No, score=-0.8 required=5.0 tests=ALL_TRUSTED,DKIM_SIGNED, DKIM_VALID,DKIM_VALID_AU,DKIM_VALID_EF,NO_DNS_FOR_FROM,URIBL_BLOCKED autolearn=no autolearn_force=no version=3.4.6 Received: from localhost.localdomain (unknown [101.128.125.254]) by gnuweeb.org (Postfix) with ESMTPSA id E91C180AB2; Thu, 25 Aug 2022 16:10:14 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=gnuweeb.org; s=default; t=1661443816; bh=reRCqrAETDv7Ipj9DLW/Shejqf9VdMew4GYakTqDWd0=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=OpfVb4An+fcJZZeDhGnT14HFAbvsOWFPXkhBERVlG5+wmcnPFq7mZqx93KjotVP5q R6DE6cm2e+Ceabn4XgLCYrRwa+x+a2Y4U2sgDW0XgtdI/fGRWr0zkCs2vvC9Y09JG8 d3i0p3BoPgzUfvIW5MF8++eZrujv+04kmG0PJjcCRnlNV58a0eVfAaHh8r4TOanHjU ay2JNqWJkwsK2NXB/XheoWTrsJVXfnLPzWmsbQ/xCYMUZHIg+7uIqYLDBCCxR5aDGw f4ECOTyxVdsQ5uz/fEAPttSpc+nOqlyJdVPkjvORBCH2b0iJzCb+244p6ZamgmY76/ Vn1ErN5vBGQtw== From: Muhammad Rizki To: Ammar Faizi Cc: Muhammad Rizki , GNU/Weeb Mailing List , Alviro Iskandar Setiawan Subject: [PATCH v1 3/3] Full refactor bot scripts Date: Thu, 25 Aug 2022 23:09:53 +0700 Message-Id: <20220825160953.1458-4-kiizuha@gnuweeb.org> X-Mailer: git-send-email 2.34.1.windows.1 In-Reply-To: <20220825160953.1458-1-kiizuha@gnuweeb.org> References: <20220825160953.1458-1-kiizuha@gnuweeb.org> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit List-Id: Refactoring the bot scripts to make it more clean Signed-off-by: Muhammad Rizki --- .gitignore | 6 +- daemon/atom/__init__.py | 1 + daemon/{discord/mailer => atom}/scraper.py | 8 +- daemon/{telegram/scraper => atom}/utils.py | 87 +++++-- daemon/{discord/run.py => dc.py} | 12 +- .../.env.example => discord.env.example} | 0 daemon/discord/mailer/utils.py | 241 ------------------ daemon/{discord => dscord}/config.py.example | 0 daemon/dscord/database/__init__.py | 1 + daemon/dscord/database/core.py | 19 ++ daemon/dscord/database/methods/__init__.py | 10 + .../database/methods/deletion/__init__.py | 8 + .../database/methods/deletion/delet_atom.py | 12 + .../methods/deletion/delete_broadcast.py | 12 + .../database/methods/getter/__init__.py | 12 + .../database/methods/getter/get_atom_urls.py | 18 ++ .../methods/getter/get_broadcast_chats.py | 18 ++ .../database/methods/getter/get_email_id.py | 56 ++++ .../database/methods/getter/get_reply.py | 29 +++ .../database/methods/insertion/__init__.py | 12 + .../database/methods/insertion/insert_atom.py | 20 ++ .../methods/insertion/insert_broadcast.py | 42 +++ .../methods/insertion/insert_discord.py | 14 + .../methods/insertion/insert_email.py | 20 ++ daemon/{discord => dscord}/execute_me.sql | 0 .../{discord => dscord}/gnuweeb/__init__.py | 0 daemon/{discord => dscord}/gnuweeb/client.py | 32 ++- daemon/{discord => dscord}/gnuweeb/filters.py | 2 +- .../gnuweeb/models/__init__.py | 0 .../gnuweeb/models/ui/__init__.py | 0 .../gnuweeb/models/ui/buttons/__init__.py | 0 .../models/ui/buttons/full_message_btn.py | 0 .../gnuweeb/plugins/__init__.py | 0 .../plugins/basic_commands/__init__.py | 0 .../plugins/basic_commands/debugger.py | 2 +- .../gnuweeb/plugins/basic_commands/sync_it.py | 6 +- .../gnuweeb/plugins/events/__init__.py | 0 .../gnuweeb/plugins/events/on_error.py | 0 .../gnuweeb/plugins/events/on_ready.py | 0 .../plugins/slash_commands/__init__.py | 0 .../plugins/slash_commands/get_lore_mail.py | 4 +- .../plugins/slash_commands/manage_atom.py | 4 +- .../slash_commands/manage_broadcast.py | 4 +- daemon/{discord => dscord}/gnuweeb/utils.py | 0 daemon/{discord => dscord}/mailer/__init__.py | 1 - daemon/{discord => dscord}/mailer/database.py | 0 daemon/{discord => dscord}/mailer/listener.py | 26 +- daemon/{discord => dscord}/requirements.txt | 0 daemon/{discord => dscord}/storage/.gitignore | 0 .../.env.example => telegram.env.example} | 0 daemon/telegram/__init__.py | 0 daemon/telegram/database/__init__.py | 1 + daemon/telegram/database/core.py | 19 ++ daemon/telegram/database/methods/__init__.py | 10 + .../database/methods/deletion/__init__.py | 8 + .../database/methods/deletion/delet_atom.py | 12 + .../methods/deletion/delete_broadcast.py | 12 + .../database/methods/getter/__init__.py | 12 + .../database/methods/getter/get_atom_urls.py | 18 ++ .../methods/getter/get_broadcast_chats.py | 18 ++ .../database/methods/getter/get_email_id.py | 58 +++++ .../methods/getter/get_telegram_reply.py | 29 +++ .../database/methods/insertion/__init__.py | 12 + .../database/methods/insertion/insert_atom.py | 20 ++ .../methods/insertion/insert_broadcast.py | 49 ++++ .../methods/insertion/insert_email.py | 20 ++ .../methods/insertion/insert_telegram.py | 14 + .../telegram/{scraper => mailer}/__init__.py | 5 +- .../{scraper/bot.py => mailer/listener.py} | 61 +++-- daemon/telegram/packages/client.py | 16 +- .../packages/plugins/callbacks/del_atom.py | 6 +- .../packages/plugins/callbacks/del_chat.py | 6 +- .../packages/plugins/commands/debugger.py | 2 +- .../packages/plugins/commands/manage_atom.py | 8 +- .../plugins/commands/manage_broadcast.py | 8 +- .../packages/plugins/commands/scrape.py | 10 +- daemon/telegram/scraper/db.py | 217 ---------------- daemon/telegram/scraper/scraper.py | 63 ----- daemon/{telegram/run.py => tg.py} | 20 +- 79 files changed, 799 insertions(+), 674 deletions(-) create mode 100644 daemon/atom/__init__.py rename daemon/{discord/mailer => atom}/scraper.py (83%) rename daemon/{telegram/scraper => atom}/utils.py (72%) rename daemon/{discord/run.py => dc.py} (78%) rename daemon/{discord/.env.example => discord.env.example} (100%) delete mode 100644 daemon/discord/mailer/utils.py rename daemon/{discord => dscord}/config.py.example (100%) create mode 100644 daemon/dscord/database/__init__.py create mode 100644 daemon/dscord/database/core.py create mode 100644 daemon/dscord/database/methods/__init__.py create mode 100644 daemon/dscord/database/methods/deletion/__init__.py create mode 100644 daemon/dscord/database/methods/deletion/delet_atom.py create mode 100644 daemon/dscord/database/methods/deletion/delete_broadcast.py create mode 100644 daemon/dscord/database/methods/getter/__init__.py create mode 100644 daemon/dscord/database/methods/getter/get_atom_urls.py create mode 100644 daemon/dscord/database/methods/getter/get_broadcast_chats.py create mode 100644 daemon/dscord/database/methods/getter/get_email_id.py create mode 100644 daemon/dscord/database/methods/getter/get_reply.py create mode 100644 daemon/dscord/database/methods/insertion/__init__.py create mode 100644 daemon/dscord/database/methods/insertion/insert_atom.py create mode 100644 daemon/dscord/database/methods/insertion/insert_broadcast.py create mode 100644 daemon/dscord/database/methods/insertion/insert_discord.py create mode 100644 daemon/dscord/database/methods/insertion/insert_email.py rename daemon/{discord => dscord}/execute_me.sql (100%) rename daemon/{discord => dscord}/gnuweeb/__init__.py (100%) rename daemon/{discord => dscord}/gnuweeb/client.py (84%) rename daemon/{discord => dscord}/gnuweeb/filters.py (98%) rename daemon/{discord => dscord}/gnuweeb/models/__init__.py (100%) rename daemon/{discord => dscord}/gnuweeb/models/ui/__init__.py (100%) rename daemon/{discord => dscord}/gnuweeb/models/ui/buttons/__init__.py (100%) rename daemon/{discord => dscord}/gnuweeb/models/ui/buttons/full_message_btn.py (100%) rename daemon/{discord => dscord}/gnuweeb/plugins/__init__.py (100%) rename daemon/{discord => dscord}/gnuweeb/plugins/basic_commands/__init__.py (100%) rename daemon/{discord => dscord}/gnuweeb/plugins/basic_commands/debugger.py (94%) rename daemon/{discord => dscord}/gnuweeb/plugins/basic_commands/sync_it.py (70%) rename daemon/{discord => dscord}/gnuweeb/plugins/events/__init__.py (100%) rename daemon/{discord => dscord}/gnuweeb/plugins/events/on_error.py (100%) rename daemon/{discord => dscord}/gnuweeb/plugins/events/on_ready.py (100%) rename daemon/{discord => dscord}/gnuweeb/plugins/slash_commands/__init__.py (100%) rename daemon/{discord => dscord}/gnuweeb/plugins/slash_commands/get_lore_mail.py (95%) rename daemon/{discord => dscord}/gnuweeb/plugins/slash_commands/manage_atom.py (97%) rename daemon/{discord => dscord}/gnuweeb/plugins/slash_commands/manage_broadcast.py (96%) rename daemon/{discord => dscord}/gnuweeb/utils.py (100%) rename daemon/{discord => dscord}/mailer/__init__.py (82%) rename daemon/{discord => dscord}/mailer/database.py (100%) rename daemon/{discord => dscord}/mailer/listener.py (85%) rename daemon/{discord => dscord}/requirements.txt (100%) rename daemon/{discord => dscord}/storage/.gitignore (100%) rename daemon/{telegram/.env.example => telegram.env.example} (100%) create mode 100644 daemon/telegram/__init__.py create mode 100644 daemon/telegram/database/__init__.py create mode 100644 daemon/telegram/database/core.py create mode 100644 daemon/telegram/database/methods/__init__.py create mode 100644 daemon/telegram/database/methods/deletion/__init__.py create mode 100644 daemon/telegram/database/methods/deletion/delet_atom.py create mode 100644 daemon/telegram/database/methods/deletion/delete_broadcast.py create mode 100644 daemon/telegram/database/methods/getter/__init__.py create mode 100644 daemon/telegram/database/methods/getter/get_atom_urls.py create mode 100644 daemon/telegram/database/methods/getter/get_broadcast_chats.py create mode 100644 daemon/telegram/database/methods/getter/get_email_id.py create mode 100644 daemon/telegram/database/methods/getter/get_telegram_reply.py create mode 100644 daemon/telegram/database/methods/insertion/__init__.py create mode 100644 daemon/telegram/database/methods/insertion/insert_atom.py create mode 100644 daemon/telegram/database/methods/insertion/insert_broadcast.py create mode 100644 daemon/telegram/database/methods/insertion/insert_email.py create mode 100644 daemon/telegram/database/methods/insertion/insert_telegram.py rename daemon/telegram/{scraper => mailer}/__init__.py (68%) rename daemon/telegram/{scraper/bot.py => mailer/listener.py} (66%) delete mode 100644 daemon/telegram/scraper/db.py delete mode 100644 daemon/telegram/scraper/scraper.py rename daemon/{telegram/run.py => tg.py} (75%) diff --git a/.gitignore b/.gitignore index 4201a17..ff5f6de 100644 --- a/.gitignore +++ b/.gitignore @@ -103,8 +103,8 @@ celerybeat.pid *.sage.py # Environments -.env -.venv +*.env +*.venv env/ venv/ ENV/ @@ -141,4 +141,4 @@ data.json # configuration file daemon/telegram/config.py -daemon/discord/config.py +daemon/dscord/config.py diff --git a/daemon/atom/__init__.py b/daemon/atom/__init__.py new file mode 100644 index 0000000..b4fcd12 --- /dev/null +++ b/daemon/atom/__init__.py @@ -0,0 +1 @@ +from .scraper import Scraper diff --git a/daemon/discord/mailer/scraper.py b/daemon/atom/scraper.py similarity index 83% rename from daemon/discord/mailer/scraper.py rename to daemon/atom/scraper.py index 3873161..8508ae9 100644 --- a/daemon/discord/mailer/scraper.py +++ b/daemon/atom/scraper.py @@ -19,10 +19,10 @@ class Scraper: async def __get_atom_content(self, atom_url): async with httpx.AsyncClient() as client: - res = await client.get(atom_url) + res = await client.get(atom_url, timeout=20) if res.status_code == 200: return res.text - raise Exception(f"[get_atom_content]: Returned {res.status_code} HTTP code") + raise Exception(f"[__get_atom_content]: Returned {res.status_code} HTTP code") async def __get_new_threads_from_atom(self, atom): @@ -54,10 +54,10 @@ class Scraper: async def get_email_from_url(self, url): async with httpx.AsyncClient() as client: - res = await client.get(url) + res = await client.get(url, timeout=20) if res.status_code == 200: return email.message_from_string( res.text, policy=email.policy.default ) - raise Exception(f"[get_atom_content]: Returned {res.status_code} HTTP code") + raise Exception(f"[get_email_from_url]: Returned {res.status_code} HTTP code") diff --git a/daemon/telegram/scraper/utils.py b/daemon/atom/utils.py similarity index 72% rename from daemon/telegram/scraper/utils.py rename to daemon/atom/utils.py index c428a33..b62c850 100644 --- a/daemon/telegram/scraper/utils.py +++ b/daemon/atom/utils.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only # -# Copyright (C) 2022 Muhammad Rizki +# Copyright (C) 2022 Muhammad Rizki # Copyright (C) 2022 Ammar Faizi # @@ -8,13 +8,19 @@ from pyrogram.types import Chat, InlineKeyboardMarkup, InlineKeyboardButton from email.message import Message from typing import Dict from slugify import slugify +import html import hashlib import uuid import os import re import shutil import httpx -import html +import asyncio + + +class Mutexes(): + def __init__(self): + self.lock = asyncio.Lock() def get_email_msg_id(mail): @@ -113,25 +119,37 @@ def consruct_to_n_cc(to: list, cc: list): return ret -def gen_temp(name: str): +def gen_temp(name: str, platform: str): + platform = platform.lower() + plt_ls = ["telegram", "discord"] + + if platform not in plt_ls: + t = f"Platform {platform} is not found, " + t += f"only {', '.join(plt_ls)} is available" + raise ValueError(f"Platform {platform} is not found") + md5 = hashlib.md5(name.encode()).hexdigest() - ret = os.getenv("STORAGE_DIR", "storage") + "/" + md5 + store_dir = os.getenv("STORAGE_DIR", "storage") + platform = platform.replace("discord", "dscord") + path = f"{platform}/{store_dir}/{md5}" try: - os.mkdir(ret) + os.mkdir(path) except FileExistsError: pass - return ret + return path -def extract_body(thread: Message): +def extract_body(thread: Message, platform: str): if not thread.is_multipart(): - p = thread.get_payload(decode=True) - return f"{p.decode(errors='replace')}\n".lstrip(), [] + p = thread.get_payload(decode=True).decode(errors='replace') + if platform == "discord": + p = quote_reply(p) + return f"{p}\n".lstrip(), [] ret = "" files = [] - temp = gen_temp(str(uuid.uuid4())) + temp = gen_temp(str(uuid.uuid4()), platform) for p in thread.get_payload(): fname = p.get_filename() payload = p.get_payload(decode=True) @@ -164,35 +182,42 @@ def __is_patch(subject, content): return True -def create_template(thread: Message, to=None, cc=None): +def create_template(thread: Message, platform: str, to=None, cc=None): if not to: to = extract_list("to", thread) if not cc: cc = extract_list("cc", thread) + if platform == "telegram": + substr = 4000 + border = f"\n{'-'*72}" + else: + substr = 1900 + border = f"\n{'-'*80}" subject = thread.get('subject') ret = f"From: {thread.get('from')}\n" ret += consruct_to_n_cc(to, cc) ret += f"Date: {thread.get('date')}\n" ret += f"Subject: {subject}\n\n" - content, files = extract_body(thread) + content, files = extract_body(thread, platform) is_patch = __is_patch(subject, content) if is_patch: ret += content else: ret += content.strip().replace("\t", " ") - if len(ret) >= 4000: - ret = ret[:4000] + "..." - ret = fix_utf8_char(ret) - ret += f"\n{'-'*72}" + if len(ret) >= substr: + ret = ret[:substr] + "..." + + ret = fix_utf8_char(ret, platform == "telegram") + ret += border return ret, files, is_patch -def prepare_send_patch(mail, text, url): - tmp = gen_temp(url) +def prepare_patch(mail: "Message", text: str, url: str, platform: str): + tmp = gen_temp(url, platform) fnm = str(mail.get("subject")) sch = re.search(PATCH_PATTERN, fnm, re.IGNORECASE) @@ -210,17 +235,31 @@ def prepare_send_patch(mail, text, url): with open(file, "wb") as f: f.write(bytes(text, encoding="utf8")) - caption = "#patch #ml\n" + fix_utf8_char(cap) + caption = "#patch #ml" + if platform == "telegram": + caption += fix_utf8_char("\n" + cap, True) return tmp, file, caption, url -def clean_up_after_send_patch(tmp): +def remove_patch(tmp): shutil.rmtree(tmp) -def fix_utf8_char(text: str): - text = text.rstrip().replace("�"," ") - return html.escape(html.escape(text)) +def fix_utf8_char(text: str, html_escape: bool = True): + t = text.rstrip().replace("�"," ") + if html_escape: + t = html.escape(html.escape(text)) + return t + + +def quote_reply(text: str): + a = "" + for b in text.split("\n"): + b = b.replace(">\n", "> ") + if b.startswith(">"): + a += "> " + a += f"{b}\n" + return a EMAIL_MSG_ID_PATTERN = r"<([^\<\>]+)>" @@ -240,6 +279,8 @@ async def is_atom_url(text: str): return mime == "application/atom+xml" except: return False + + def remove_command(text: str): txt = text.split(" ") txt = text.replace(txt[0] + " ","") diff --git a/daemon/discord/run.py b/daemon/dc.py similarity index 78% rename from daemon/discord/run.py rename to daemon/dc.py index e3eb7a6..a24421b 100644 --- a/daemon/discord/run.py +++ b/daemon/dc.py @@ -7,15 +7,15 @@ import os from mysql import connector from dotenv import load_dotenv from apscheduler.schedulers.asyncio import AsyncIOScheduler -from mailer.listener import BotMutexes -from mailer.listener import Listener +from dscord.mailer.listener import Listener +from atom.scraper import Scraper +from atom.utils import Mutexes -from gnuweeb import GWClient -from mailer import Scraper +from dscord.gnuweeb import GWClient def main(): - load_dotenv() + load_dotenv("discord.env") sched = AsyncIOScheduler( job_defaults={ @@ -37,7 +37,7 @@ def main(): client=client, sched=sched, scraper=Scraper(), - mutexes=BotMutexes() + mutexes=Mutexes() ) client.mailer = mailer diff --git a/daemon/discord/.env.example b/daemon/discord.env.example similarity index 100% rename from daemon/discord/.env.example rename to daemon/discord.env.example diff --git a/daemon/discord/mailer/utils.py b/daemon/discord/mailer/utils.py deleted file mode 100644 index 0036f24..0000000 --- a/daemon/discord/mailer/utils.py +++ /dev/null @@ -1,241 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# -# Copyright (C) 2022 Muhammad Rizki -# Copyright (C) 2022 Ammar Faizi -# - -from email.message import Message -from typing import Dict -from slugify import slugify -import hashlib -import uuid -import os -import re -import shutil -import httpx - - -def get_email_msg_id(mail): - ret = mail.get("message-id") - if not ret: - return None - - ret = re.search(r"<([^\<\>]+)>", ret) - if not ret: - return None - - return ret.group(1) - - -# -# This increments the @i while we are seeing a whitespace. -# -def __skip_whitespace(i, ss_len, ss): - while i < ss_len: - c = ss[i] - if c != ' ' and c != '\t' and c != '\n': - break - i += 1 - - return i - - -# -# Pick a single element in the list. The delimiter here is -# a comma char ','. But note that when are inside a double -# quotes, we must not take the comma as a delimiter. -# -def __pick_element(i, ss_len, ss, ret): - acc = "" - in_quotes = False - - while i < ss_len: - c = ss[i] - i += 1 - - if c == '"': - in_quotes = (not in_quotes) - - if not in_quotes and c == ',': - break - - acc += c - - if acc != "": - ret.append(acc) - - return i - - -def __extract_list(ss): - ss = ss.strip() - ss_len = len(ss) - ret = [] - i = 0 - - while i < ss_len: - i = __skip_whitespace(i, ss_len, ss) - i = __pick_element(i, ss_len, ss, ret) - - return ret - - -def extract_list(key: str, content: Dict[str, str]): - people = content.get(key.lower()) - if not people: - return [] - return __extract_list(people) - - -def consruct_to_n_cc(to: list, cc: list): - NR_MAX_LIST = 20 - - n = 0 - ret = "" - for i in to: - if n >= NR_MAX_LIST: - ret += "To: ...\n" - break - - n += 1 - ret += f"To: {i}\n" - - for i in cc: - if n >= NR_MAX_LIST: - ret += "Cc: ...\n" - break - - n += 1 - ret += f"Cc: {i}\n" - - return ret - - -def gen_temp(name: str): - md5 = hashlib.md5(name.encode()).hexdigest() - ret = os.getenv("STORAGE_DIR", "storage") + "/" + md5 - try: - os.mkdir(ret) - except FileExistsError: - pass - - return ret - - -def extract_body(thread: Message): - if not thread.is_multipart(): - p = thread.get_payload(decode=True) - return f"{p.decode(errors='replace')}\n".lstrip(), [] - - ret = "" - files = [] - temp = gen_temp(str(uuid.uuid4())) - for p in thread.get_payload(): - fname = p.get_filename() - payload = p.get_payload(decode=True) - - if not payload: - continue - - if 'inline' in [p.get('content-disposition')] or not bool(fname): - ret += f"{payload.decode(errors='replace')}\n".lstrip() - continue - - with open(f"{temp}/{fname}", "wb") as f: - f.write(payload) - files.append((temp, fname)) - - ret = re.sub("^(>)", ">>> \\1", ret, 1, re.MULTILINE) - return ret, files - - - -PATCH_PATTERN = r"^\[.*(?:patch|rfc).*?(?:(\d+)\/(\d+))?\](.+)" -def __is_patch(subject, content): - x = re.search(PATCH_PATTERN, subject, re.IGNORECASE) - if not x or x.group(1) == "0": - return False - - x = re.search(r"diff --git", content) - if not x: - return False - - return True - - -def create_template(thread: Message, to=None, cc=None): - if not to: - to = extract_list("to", thread) - if not cc: - cc = extract_list("cc", thread) - - subject = thread.get('subject') - ret = f"From: {thread.get('from')}\n" - ret += consruct_to_n_cc(to, cc) - ret += f"Date: {thread.get('date')}\n" - ret += f"Subject: {subject}\n\n" - content, files = extract_body(thread) - is_patch = __is_patch(subject, content) - - if is_patch: - ret += content - else: - ret += content.strip().replace("\t", " ") - if len(ret) >= 1900: - ret = ret[:1900] + "..." - - ret = fix_utf8_char(ret) - - return ret, files, is_patch - - -def prepare_patch(mail, text, url): - tmp = gen_temp(url) - fnm = str(mail.get("subject")) - sch = re.search(PATCH_PATTERN, fnm, re.IGNORECASE) - - nr_patch = sch.group(1) - if not nr_patch: - nr_patch = 1 - else: - nr_patch = int(nr_patch) - - num = "%04d" % nr_patch - fnm = slugify(sch.group(3)).replace("_", "-") - file = f"{tmp}/{num}-{fnm}.patch" - - with open(file, "wb") as f: - f.write(bytes(text, encoding="utf8")) - - caption = "#patch #ml" - return tmp, file, caption, url - - -def remove_patch(tmp): - shutil.rmtree(tmp) - - -def fix_utf8_char(text: str): - return text.rstrip().replace("�"," ") - - -def bottom_border(text: str): - return text + "\n" + "-"*72 - - -EMAIL_MSG_ID_PATTERN = r"<([^\<\>]+)>" -def extract_email_msg_id(msg_id): - ret = re.search(EMAIL_MSG_ID_PATTERN, msg_id) - if not ret: - return None - return ret.group(1) - - -async def is_atom_url(text: str): - try: - async with httpx.AsyncClient() as ses: - res = await ses.get(text) - mime = res.headers.get("Content-Type") - - return mime == "application/atom+xml" - except: return False diff --git a/daemon/discord/config.py.example b/daemon/dscord/config.py.example similarity index 100% rename from daemon/discord/config.py.example rename to daemon/dscord/config.py.example diff --git a/daemon/dscord/database/__init__.py b/daemon/dscord/database/__init__.py new file mode 100644 index 0000000..13cff25 --- /dev/null +++ b/daemon/dscord/database/__init__.py @@ -0,0 +1 @@ +from .core import DB diff --git a/daemon/dscord/database/core.py b/daemon/dscord/database/core.py new file mode 100644 index 0000000..cfd9cc2 --- /dev/null +++ b/daemon/dscord/database/core.py @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Copyright (C) 2022 Muhammad Rizki +# Copyright (C) 2022 Ammar Faizi +# + +from .methods import DBMethods + + +class DB(DBMethods): + def __init__(self, conn): + self.conn = conn + self.conn.autocommit = True + self.cur = self.conn.cursor(buffered=True) + + + def __del__(self): + self.cur.close() + self.conn.close() diff --git a/daemon/dscord/database/methods/__init__.py b/daemon/dscord/database/methods/__init__.py new file mode 100644 index 0000000..6099941 --- /dev/null +++ b/daemon/dscord/database/methods/__init__.py @@ -0,0 +1,10 @@ +from .deletion import Deletion +from .getter import Getter +from .insertion import Insertion + + +class DBMethods( + Deletion, + Getter, + Insertion +): pass diff --git a/daemon/dscord/database/methods/deletion/__init__.py b/daemon/dscord/database/methods/deletion/__init__.py new file mode 100644 index 0000000..fa80464 --- /dev/null +++ b/daemon/dscord/database/methods/deletion/__init__.py @@ -0,0 +1,8 @@ +from .delet_atom import DeleteAtom +from .delete_broadcast import DeleteBroadcast + + +class Deletion( + DeleteAtom, + DeleteBroadcast +): pass diff --git a/daemon/dscord/database/methods/deletion/delet_atom.py b/daemon/dscord/database/methods/deletion/delet_atom.py new file mode 100644 index 0000000..4923045 --- /dev/null +++ b/daemon/dscord/database/methods/deletion/delet_atom.py @@ -0,0 +1,12 @@ + + + +class DeleteAtom: + + def delete_atom(self, atom: str): + q = """ + DELETE FROM atom_urls + WHERE url = %(atom)s + """ + self.cur.execute(q, {"atom": atom}) + return self.cur.rowcount > 0 diff --git a/daemon/dscord/database/methods/deletion/delete_broadcast.py b/daemon/dscord/database/methods/deletion/delete_broadcast.py new file mode 100644 index 0000000..2f17eff --- /dev/null +++ b/daemon/dscord/database/methods/deletion/delete_broadcast.py @@ -0,0 +1,12 @@ + + + +class DeleteBroadcast: + + def delete_broadcast(self, channel_id: int): + q = """ + DELETE FROM broadcast_chats + WHERE channel_id = %(channel_id)s + """ + self.cur.execute(q, {"channel_id": channel_id}) + return self.cur.rowcount > 0 diff --git a/daemon/dscord/database/methods/getter/__init__.py b/daemon/dscord/database/methods/getter/__init__.py new file mode 100644 index 0000000..fd58e84 --- /dev/null +++ b/daemon/dscord/database/methods/getter/__init__.py @@ -0,0 +1,12 @@ +from .get_atom_urls import GetAtomURL +from .get_broadcast_chats import GetBroadcastChats +from .get_email_id import GetEmailID +from .get_reply import GetDiscordReply + + +class Getter( + GetAtomURL, + GetBroadcastChats, + GetEmailID, + GetDiscordReply +): pass diff --git a/daemon/dscord/database/methods/getter/get_atom_urls.py b/daemon/dscord/database/methods/getter/get_atom_urls.py new file mode 100644 index 0000000..b7353d2 --- /dev/null +++ b/daemon/dscord/database/methods/getter/get_atom_urls.py @@ -0,0 +1,18 @@ + + + +class GetAtomURL: + + def get_atom_urls(self): + ''' + Get lore kernel raw email URLs. + - Return list of raw email URLs: `List[str]` + ''' + q = """ + SELECT atom_urls.url + FROM atom_urls + """ + self.cur.execute(q) + urls = self.cur.fetchall() + + return [u[0] for u in urls] diff --git a/daemon/dscord/database/methods/getter/get_broadcast_chats.py b/daemon/dscord/database/methods/getter/get_broadcast_chats.py new file mode 100644 index 0000000..43efe0c --- /dev/null +++ b/daemon/dscord/database/methods/getter/get_broadcast_chats.py @@ -0,0 +1,18 @@ + + + +class GetBroadcastChats: + + def get_broadcast_chats(self): + ''' + Get broadcast chats that are currently + listening for new email. + - Return list of chat object: `List[Object]` + ''' + q = """ + SELECT * + FROM broadcast_chats + """ + self.cur.execute(q) + + return self.cur.fetchall() diff --git a/daemon/dscord/database/methods/getter/get_email_id.py b/daemon/dscord/database/methods/getter/get_email_id.py new file mode 100644 index 0000000..637f47f --- /dev/null +++ b/daemon/dscord/database/methods/getter/get_email_id.py @@ -0,0 +1,56 @@ + + + +class GetEmailID: + + def get_email_id(self, email_id, chat_id): + ''' + Determine whether the email needs to be sent to @tg_chat_id. + - Return an email id (PK) if it needs to be sent + - Return None if it doesn't need to be sent + ''' + if self.__is_sent(email_id, chat_id): + return + + res = self.__email_id(email_id) + if not bool(res): + return + + return int(res[0]) + + + def __is_sent(self, email_id, channel_id): + ''' + Checking if this email has already been sent + or not. + - Return True if it's already been sent + ''' + q = """ + SELECT emails.id, discord_emails.id FROM emails + LEFT JOIN discord_emails + ON emails.id = discord_emails.email_id + WHERE emails.message_id = %(email_msg_id)s + AND discord_emails.channel_id = %(channel_id)s + LIMIT 1 + """ + + self.cur.execute(q, { + "email_msg_id": email_id, + "channel_id": channel_id + }) + res = self.cur.fetchone() + return bool(res) + + + def __email_id(self, email_id): + ''' + Get the email id if match with the email message_id. + - Return the result if it's match and exists + ''' + q = """ + SELECT id FROM emails WHERE message_id = %(email_id)s + """ + + self.cur.execute(q, {"email_id": email_id}) + res = self.cur.fetchone() + return res diff --git a/daemon/dscord/database/methods/getter/get_reply.py b/daemon/dscord/database/methods/getter/get_reply.py new file mode 100644 index 0000000..c5b1830 --- /dev/null +++ b/daemon/dscord/database/methods/getter/get_reply.py @@ -0,0 +1,29 @@ + + + +class GetDiscordReply: + + def get_reply_id(self, email_id, channel_id): + ''' + Get Telegram message ID sent match with + email message ID and Telegram chat ID. + - Return Telegram message ID if exists: `int` + - Return None if not exists` + ''' + q = """ + SELECT discord_emails.dc_msg_id + FROM emails INNER JOIN discord_emails + ON emails.id = discord_emails.email_id + WHERE emails.message_id = %(email_msg_id)s + AND discord_emails.channel_id = %(channel_id)s + """ + + self.cur.execute(q, { + "email_msg_id": email_id, + "channel_id": channel_id + }) + res = self.cur.fetchone() + if not bool(res): + return None + + return res[0] diff --git a/daemon/dscord/database/methods/insertion/__init__.py b/daemon/dscord/database/methods/insertion/__init__.py new file mode 100644 index 0000000..46b6e05 --- /dev/null +++ b/daemon/dscord/database/methods/insertion/__init__.py @@ -0,0 +1,12 @@ +from .insert_atom import InsertAtom +from .insert_broadcast import InsertBroadcast +from .insert_email import InsertEmail +from .insert_discord import InsertDiscord + + +class Insertion( + InsertAtom, + InsertBroadcast, + InsertEmail, + InsertDiscord +): pass diff --git a/daemon/dscord/database/methods/insertion/insert_atom.py b/daemon/dscord/database/methods/insertion/insert_atom.py new file mode 100644 index 0000000..6313c07 --- /dev/null +++ b/daemon/dscord/database/methods/insertion/insert_atom.py @@ -0,0 +1,20 @@ +from mysql.connector import errors +from datetime import datetime + + +class InsertAtom: + + def insert_atom(self, atom: str): + try: + return self.__save_atom(atom) + except errors.IntegrityError: + # + # Duplicate data, skip! + # + return None + + + def __save_atom(self, atom: str): + q = "INSERT INTO atom_urls (url, created_at) VALUES (%s, %s)" + self.cur.execute(q, (atom, datetime.utcnow())) + return self.cur.lastrowid diff --git a/daemon/dscord/database/methods/insertion/insert_broadcast.py b/daemon/dscord/database/methods/insertion/insert_broadcast.py new file mode 100644 index 0000000..e3177d3 --- /dev/null +++ b/daemon/dscord/database/methods/insertion/insert_broadcast.py @@ -0,0 +1,42 @@ +from mysql.connector import errors +from datetime import datetime + + +class InsertBroadcast: + + def save_broadcast( + self, + guild_id: int, + channel_id: int, + channel_name: str, + channel_link: str = None, + ): + try: + return self.__save_broadcast( + guild_id=guild_id, + channel_id=channel_id, + channel_name=channel_name, + channel_link=channel_link + ) + except errors.IntegrityError: + # + # Duplicate data, skip! + # + return None + + + def __save_broadcast( + self, + guild_id: int, + channel_id: int, + channel_name: str, + channel_link: str = None, + ): + q = """ + INSERT INTO broadcast_chats + (guild_id, channel_id, channel_name, channel_link, created_at) + VALUES (%s, %s, %s, %s, %s) + """ + values = (guild_id, channel_id, channel_name, channel_link, datetime.utcnow()) + self.cur.execute(q, values) + return self.cur.lastrowid diff --git a/daemon/dscord/database/methods/insertion/insert_discord.py b/daemon/dscord/database/methods/insertion/insert_discord.py new file mode 100644 index 0000000..83e4b22 --- /dev/null +++ b/daemon/dscord/database/methods/insertion/insert_discord.py @@ -0,0 +1,14 @@ +from datetime import datetime + + +class InsertDiscord: + + def save_discord_mail(self, email_id, channel_id, dc_msg_id): + q = """ + INSERT INTO discord_emails + (email_id, channel_id, dc_msg_id, created_at) + VALUES (%s, %s, %s, %s); + """ + self.cur.execute(q, (email_id, channel_id, dc_msg_id, + datetime.utcnow())) + return self.cur.lastrowid diff --git a/daemon/dscord/database/methods/insertion/insert_email.py b/daemon/dscord/database/methods/insertion/insert_email.py new file mode 100644 index 0000000..bd88339 --- /dev/null +++ b/daemon/dscord/database/methods/insertion/insert_email.py @@ -0,0 +1,20 @@ +from mysql.connector import errors +from datetime import datetime + + +class InsertEmail: + + def save_email(self, email_msg_id): + try: + return self.__insert_email(email_msg_id) + except errors.IntegrityError: + # + # Duplicate data, skip! + # + return None + + + def __insert_email(self, email_msg_id): + q = "INSERT INTO emails (message_id, created_at) VALUES (%s, %s)" + self.cur.execute(q, (email_msg_id, datetime.utcnow())) + return self.cur.lastrowid diff --git a/daemon/discord/execute_me.sql b/daemon/dscord/execute_me.sql similarity index 100% rename from daemon/discord/execute_me.sql rename to daemon/dscord/execute_me.sql diff --git a/daemon/discord/gnuweeb/__init__.py b/daemon/dscord/gnuweeb/__init__.py similarity index 100% rename from daemon/discord/gnuweeb/__init__.py rename to daemon/dscord/gnuweeb/__init__.py diff --git a/daemon/discord/gnuweeb/client.py b/daemon/dscord/gnuweeb/client.py similarity index 84% rename from daemon/discord/gnuweeb/client.py rename to daemon/dscord/gnuweeb/client.py index cf88d36..c308f5a 100644 --- a/daemon/discord/gnuweeb/client.py +++ b/daemon/dscord/gnuweeb/client.py @@ -11,13 +11,13 @@ from typing import Union from .models.ui import buttons from . import filters -from mailer import utils -from mailer import Database +from atom import utils +from dscord.database import DB class GWClient(commands.Bot): def __init__(self, db_conn) -> None: - self.db = Database(db_conn) + self.db = DB(db_conn) self.mailer = None intents = Intents.default() intents.message_content = True @@ -33,7 +33,10 @@ class GWClient(commands.Bot): async def setup_hook(self): - await self.load_extension("gnuweeb.plugins") + await self.load_extension( + name=".gnuweeb.plugins", + package="dscord" + ) # WARNING! NOT RECOMMENDED SYNCING WHEN THE BOT IS START!! # guild = discord.Object(id=845302963739033611) @@ -45,27 +48,26 @@ class GWClient(commands.Bot): async def send_text_email(self, guild_id: int, chat_id: int, text: str, reply_to: Union[int, None] = None, url: str = None): print("[send_text_email]") - text = utils.bottom_border(text) channel = self.get_channel(chat_id) - m = await channel.send( + return await channel.send( content=text, reference=discord.MessageReference( guild_id=guild_id, channel_id=chat_id, message_id=reply_to ) if reply_to else None, - view=buttons.FullMessageBtn(url) ) - return m @filters.wait_on_limit async def send_patch_email(self, mail, guild_id: int, chat_id: int, text: str, reply_to: Union[int, None] = None, url: str = None): print("[send_patch_email]") - tmp, doc, caption, url = utils.prepare_patch(mail, text, url) + tmp, doc, caption, url = utils.prepare_patch( + mail, text, url, "discord" + ) channel = self.get_channel(chat_id) m = await channel.send( @@ -86,25 +88,21 @@ class GWClient(commands.Bot): async def send_text_mail_interaction(self, i: "Interaction", text: str, url: str = None): - text = utils.border_and_trim(text) - - m = await i.response.send_message( + return await i.response.send_message( content=text, view=buttons.FullMessageBtn(url) ) - return m - async def send_patch_mail_interaction(self, mail, i: "Interaction", text: str, url: str = None): - tmp, doc, caption, url = utils.prepare_patch(mail, text, url) - + tmp, doc, caption, url = utils.prepare_patch( + mail, text, url, "discord" + ) m = await i.response.send_message( content=caption, file=discord.File(doc), view=buttons.FullMessageBtn(url) ) - utils.remove_patch(tmp) return m diff --git a/daemon/discord/gnuweeb/filters.py b/daemon/dscord/gnuweeb/filters.py similarity index 98% rename from daemon/discord/gnuweeb/filters.py rename to daemon/dscord/gnuweeb/filters.py index 735c77e..76bfa5d 100644 --- a/daemon/discord/gnuweeb/filters.py +++ b/daemon/dscord/gnuweeb/filters.py @@ -3,7 +3,7 @@ # Copyright (C) 2022 Muhammad Rizki # -import config +from dscord import config import discord import asyncio from discord import Interaction diff --git a/daemon/discord/gnuweeb/models/__init__.py b/daemon/dscord/gnuweeb/models/__init__.py similarity index 100% rename from daemon/discord/gnuweeb/models/__init__.py rename to daemon/dscord/gnuweeb/models/__init__.py diff --git a/daemon/discord/gnuweeb/models/ui/__init__.py b/daemon/dscord/gnuweeb/models/ui/__init__.py similarity index 100% rename from daemon/discord/gnuweeb/models/ui/__init__.py rename to daemon/dscord/gnuweeb/models/ui/__init__.py diff --git a/daemon/discord/gnuweeb/models/ui/buttons/__init__.py b/daemon/dscord/gnuweeb/models/ui/buttons/__init__.py similarity index 100% rename from daemon/discord/gnuweeb/models/ui/buttons/__init__.py rename to daemon/dscord/gnuweeb/models/ui/buttons/__init__.py diff --git a/daemon/discord/gnuweeb/models/ui/buttons/full_message_btn.py b/daemon/dscord/gnuweeb/models/ui/buttons/full_message_btn.py similarity index 100% rename from daemon/discord/gnuweeb/models/ui/buttons/full_message_btn.py rename to daemon/dscord/gnuweeb/models/ui/buttons/full_message_btn.py diff --git a/daemon/discord/gnuweeb/plugins/__init__.py b/daemon/dscord/gnuweeb/plugins/__init__.py similarity index 100% rename from daemon/discord/gnuweeb/plugins/__init__.py rename to daemon/dscord/gnuweeb/plugins/__init__.py diff --git a/daemon/discord/gnuweeb/plugins/basic_commands/__init__.py b/daemon/dscord/gnuweeb/plugins/basic_commands/__init__.py similarity index 100% rename from daemon/discord/gnuweeb/plugins/basic_commands/__init__.py rename to daemon/dscord/gnuweeb/plugins/basic_commands/__init__.py diff --git a/daemon/discord/gnuweeb/plugins/basic_commands/debugger.py b/daemon/dscord/gnuweeb/plugins/basic_commands/debugger.py similarity index 94% rename from daemon/discord/gnuweeb/plugins/basic_commands/debugger.py rename to daemon/dscord/gnuweeb/plugins/basic_commands/debugger.py index be5b9f1..0bde454 100644 --- a/daemon/discord/gnuweeb/plugins/basic_commands/debugger.py +++ b/daemon/dscord/gnuweeb/plugins/basic_commands/debugger.py @@ -4,7 +4,7 @@ # from discord.ext import commands -from gnuweeb import utils +from dscord.gnuweeb import utils # from gnuweeb import filters diff --git a/daemon/discord/gnuweeb/plugins/basic_commands/sync_it.py b/daemon/dscord/gnuweeb/plugins/basic_commands/sync_it.py similarity index 70% rename from daemon/discord/gnuweeb/plugins/basic_commands/sync_it.py rename to daemon/dscord/gnuweeb/plugins/basic_commands/sync_it.py index 2ed35cc..fbaec03 100644 --- a/daemon/discord/gnuweeb/plugins/basic_commands/sync_it.py +++ b/daemon/dscord/gnuweeb/plugins/basic_commands/sync_it.py @@ -7,14 +7,14 @@ from discord.ext import commands class SyncCommand(commands.Cog): - def __init__(self, bot) -> None: + def __init__(self, bot: "commands.Bot") -> None: self.bot = bot @commands.command("sync", aliases=["s"]) @commands.is_owner() async def sync_it(self, ctx: "commands.Context"): - ctx.bot.tree.copy_global_to(guild=ctx.guild) - s = await ctx.bot.tree.sync(guild=ctx.guild) + self.bot.tree.copy_global_to(guild=ctx.guild) + s = await self.bot.tree.sync(guild=ctx.guild) await ctx.send(f"Synced {len(s)} commands.") diff --git a/daemon/discord/gnuweeb/plugins/events/__init__.py b/daemon/dscord/gnuweeb/plugins/events/__init__.py similarity index 100% rename from daemon/discord/gnuweeb/plugins/events/__init__.py rename to daemon/dscord/gnuweeb/plugins/events/__init__.py diff --git a/daemon/discord/gnuweeb/plugins/events/on_error.py b/daemon/dscord/gnuweeb/plugins/events/on_error.py similarity index 100% rename from daemon/discord/gnuweeb/plugins/events/on_error.py rename to daemon/dscord/gnuweeb/plugins/events/on_error.py diff --git a/daemon/discord/gnuweeb/plugins/events/on_ready.py b/daemon/dscord/gnuweeb/plugins/events/on_ready.py similarity index 100% rename from daemon/discord/gnuweeb/plugins/events/on_ready.py rename to daemon/dscord/gnuweeb/plugins/events/on_ready.py diff --git a/daemon/discord/gnuweeb/plugins/slash_commands/__init__.py b/daemon/dscord/gnuweeb/plugins/slash_commands/__init__.py similarity index 100% rename from daemon/discord/gnuweeb/plugins/slash_commands/__init__.py rename to daemon/dscord/gnuweeb/plugins/slash_commands/__init__.py diff --git a/daemon/discord/gnuweeb/plugins/slash_commands/get_lore_mail.py b/daemon/dscord/gnuweeb/plugins/slash_commands/get_lore_mail.py similarity index 95% rename from daemon/discord/gnuweeb/plugins/slash_commands/get_lore_mail.py rename to daemon/dscord/gnuweeb/plugins/slash_commands/get_lore_mail.py index a2671f4..38eb465 100644 --- a/daemon/discord/gnuweeb/plugins/slash_commands/get_lore_mail.py +++ b/daemon/dscord/gnuweeb/plugins/slash_commands/get_lore_mail.py @@ -10,8 +10,8 @@ from discord.ext import commands from discord import Interaction from discord import app_commands -from mailer import utils -from mailer import Scraper +from atom import utils +from atom import Scraper class GetLoreSC(commands.Cog): diff --git a/daemon/discord/gnuweeb/plugins/slash_commands/manage_atom.py b/daemon/dscord/gnuweeb/plugins/slash_commands/manage_atom.py similarity index 97% rename from daemon/discord/gnuweeb/plugins/slash_commands/manage_atom.py rename to daemon/dscord/gnuweeb/plugins/slash_commands/manage_atom.py index 2d9da80..9666264 100644 --- a/daemon/discord/gnuweeb/plugins/slash_commands/manage_atom.py +++ b/daemon/dscord/gnuweeb/plugins/slash_commands/manage_atom.py @@ -7,8 +7,8 @@ from discord.ext import commands from discord import Interaction from discord import app_commands -from gnuweeb import filters -from mailer import utils +from dscord.gnuweeb import filters +from atom import utils class ManageAtomSC(commands.Cog): diff --git a/daemon/discord/gnuweeb/plugins/slash_commands/manage_broadcast.py b/daemon/dscord/gnuweeb/plugins/slash_commands/manage_broadcast.py similarity index 96% rename from daemon/discord/gnuweeb/plugins/slash_commands/manage_broadcast.py rename to daemon/dscord/gnuweeb/plugins/slash_commands/manage_broadcast.py index 9eb6b98..86f5fec 100644 --- a/daemon/discord/gnuweeb/plugins/slash_commands/manage_broadcast.py +++ b/daemon/dscord/gnuweeb/plugins/slash_commands/manage_broadcast.py @@ -7,8 +7,8 @@ from discord.ext import commands from discord import Interaction from discord import app_commands -from gnuweeb import utils -from gnuweeb import filters +from dscord.gnuweeb import utils +from dscord.gnuweeb import filters class ManageBroadcastSC(commands.Cog): diff --git a/daemon/discord/gnuweeb/utils.py b/daemon/dscord/gnuweeb/utils.py similarity index 100% rename from daemon/discord/gnuweeb/utils.py rename to daemon/dscord/gnuweeb/utils.py diff --git a/daemon/discord/mailer/__init__.py b/daemon/dscord/mailer/__init__.py similarity index 82% rename from daemon/discord/mailer/__init__.py rename to daemon/dscord/mailer/__init__.py index 0da4329..e9c0190 100644 --- a/daemon/discord/mailer/__init__.py +++ b/daemon/dscord/mailer/__init__.py @@ -4,4 +4,3 @@ # from .database import Database -from .scraper import Scraper diff --git a/daemon/discord/mailer/database.py b/daemon/dscord/mailer/database.py similarity index 100% rename from daemon/discord/mailer/database.py rename to daemon/dscord/mailer/database.py diff --git a/daemon/discord/mailer/listener.py b/daemon/dscord/mailer/listener.py similarity index 85% rename from daemon/discord/mailer/listener.py rename to daemon/dscord/mailer/listener.py index 523a87d..7538ba9 100644 --- a/daemon/discord/mailer/listener.py +++ b/daemon/dscord/mailer/listener.py @@ -11,14 +11,10 @@ import re from apscheduler.schedulers.asyncio import AsyncIOScheduler from discord import File -from gnuweeb import GWClient -from .scraper import Scraper -from . import utils - - -class BotMutexes(): - def __init__(self): - self.lock = asyncio.Lock() +from dscord.gnuweeb import GWClient +from atom.utils import Mutexes +from atom.scraper import Scraper +from atom import utils class Listener(): @@ -26,8 +22,8 @@ class Listener(): self, client: "GWClient", sched: "AsyncIOScheduler", - scraper: Scraper, - mutexes: "BotMutexes" + scraper: "Scraper", + mutexes: "Mutexes" ): self.client = client self.sched = sched @@ -106,7 +102,7 @@ class Listener(): # return False - text, files, is_patch = utils.create_template(mail) + text, files, is_patch = utils.create_template(mail, "discord") reply_to = self.get_discord_reply(mail, dc_chat_id) url = str(re.sub(r"/raw$", "", url)) @@ -120,7 +116,7 @@ class Listener(): dc_guild_id, dc_chat_id, text, reply_to, url ) - self.db.insert_discord(email_id, m.channel.id, m.id) + self.db.save_discord_mail(email_id, m.channel.id, m.id) for d, f in files: await m.reply(f"{d}/{f}", file=File(f)) await asyncio.sleep(1) @@ -132,11 +128,11 @@ class Listener(): def __get_email_id_sent(self, email_msg_id, dc_chat_id): - email_id = self.db.save_email_msg_id(email_msg_id) + email_id = self.db.save_email(email_msg_id) if email_id: return email_id - email_id = self.db.get_email_id_sent(email_msg_id, dc_chat_id) + email_id = self.db.get_email_id(email_msg_id, dc_chat_id) return email_id @@ -149,4 +145,4 @@ class Listener(): if not reply_to: return None - return self.db.get_discord_reply(reply_to, dc_chat_id) + return self.db.get_reply_id(reply_to, dc_chat_id) diff --git a/daemon/discord/requirements.txt b/daemon/dscord/requirements.txt similarity index 100% rename from daemon/discord/requirements.txt rename to daemon/dscord/requirements.txt diff --git a/daemon/discord/storage/.gitignore b/daemon/dscord/storage/.gitignore similarity index 100% rename from daemon/discord/storage/.gitignore rename to daemon/dscord/storage/.gitignore diff --git a/daemon/telegram/.env.example b/daemon/telegram.env.example similarity index 100% rename from daemon/telegram/.env.example rename to daemon/telegram.env.example diff --git a/daemon/telegram/__init__.py b/daemon/telegram/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/daemon/telegram/database/__init__.py b/daemon/telegram/database/__init__.py new file mode 100644 index 0000000..13cff25 --- /dev/null +++ b/daemon/telegram/database/__init__.py @@ -0,0 +1 @@ +from .core import DB diff --git a/daemon/telegram/database/core.py b/daemon/telegram/database/core.py new file mode 100644 index 0000000..cfd9cc2 --- /dev/null +++ b/daemon/telegram/database/core.py @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Copyright (C) 2022 Muhammad Rizki +# Copyright (C) 2022 Ammar Faizi +# + +from .methods import DBMethods + + +class DB(DBMethods): + def __init__(self, conn): + self.conn = conn + self.conn.autocommit = True + self.cur = self.conn.cursor(buffered=True) + + + def __del__(self): + self.cur.close() + self.conn.close() diff --git a/daemon/telegram/database/methods/__init__.py b/daemon/telegram/database/methods/__init__.py new file mode 100644 index 0000000..6099941 --- /dev/null +++ b/daemon/telegram/database/methods/__init__.py @@ -0,0 +1,10 @@ +from .deletion import Deletion +from .getter import Getter +from .insertion import Insertion + + +class DBMethods( + Deletion, + Getter, + Insertion +): pass diff --git a/daemon/telegram/database/methods/deletion/__init__.py b/daemon/telegram/database/methods/deletion/__init__.py new file mode 100644 index 0000000..fa80464 --- /dev/null +++ b/daemon/telegram/database/methods/deletion/__init__.py @@ -0,0 +1,8 @@ +from .delet_atom import DeleteAtom +from .delete_broadcast import DeleteBroadcast + + +class Deletion( + DeleteAtom, + DeleteBroadcast +): pass diff --git a/daemon/telegram/database/methods/deletion/delet_atom.py b/daemon/telegram/database/methods/deletion/delet_atom.py new file mode 100644 index 0000000..4923045 --- /dev/null +++ b/daemon/telegram/database/methods/deletion/delet_atom.py @@ -0,0 +1,12 @@ + + + +class DeleteAtom: + + def delete_atom(self, atom: str): + q = """ + DELETE FROM atom_urls + WHERE url = %(atom)s + """ + self.cur.execute(q, {"atom": atom}) + return self.cur.rowcount > 0 diff --git a/daemon/telegram/database/methods/deletion/delete_broadcast.py b/daemon/telegram/database/methods/deletion/delete_broadcast.py new file mode 100644 index 0000000..a2c0751 --- /dev/null +++ b/daemon/telegram/database/methods/deletion/delete_broadcast.py @@ -0,0 +1,12 @@ + + + +class DeleteBroadcast: + + def delete_broadcast(self, chat_id: int): + q = """ + DELETE FROM broadcast_chats + WHERE chat_id = %(chat_id)s + """ + self.cur.execute(q, {"chat_id": chat_id}) + return self.cur.rowcount > 0 diff --git a/daemon/telegram/database/methods/getter/__init__.py b/daemon/telegram/database/methods/getter/__init__.py new file mode 100644 index 0000000..6a8a97d --- /dev/null +++ b/daemon/telegram/database/methods/getter/__init__.py @@ -0,0 +1,12 @@ +from .get_atom_urls import GetAtomURL +from .get_broadcast_chats import GetBroadcastChats +from .get_email_id import GetEmailID +from .get_telegram_reply import GetTelegramReply + + +class Getter( + GetAtomURL, + GetBroadcastChats, + GetEmailID, + GetTelegramReply +): pass diff --git a/daemon/telegram/database/methods/getter/get_atom_urls.py b/daemon/telegram/database/methods/getter/get_atom_urls.py new file mode 100644 index 0000000..b7353d2 --- /dev/null +++ b/daemon/telegram/database/methods/getter/get_atom_urls.py @@ -0,0 +1,18 @@ + + + +class GetAtomURL: + + def get_atom_urls(self): + ''' + Get lore kernel raw email URLs. + - Return list of raw email URLs: `List[str]` + ''' + q = """ + SELECT atom_urls.url + FROM atom_urls + """ + self.cur.execute(q) + urls = self.cur.fetchall() + + return [u[0] for u in urls] diff --git a/daemon/telegram/database/methods/getter/get_broadcast_chats.py b/daemon/telegram/database/methods/getter/get_broadcast_chats.py new file mode 100644 index 0000000..43efe0c --- /dev/null +++ b/daemon/telegram/database/methods/getter/get_broadcast_chats.py @@ -0,0 +1,18 @@ + + + +class GetBroadcastChats: + + def get_broadcast_chats(self): + ''' + Get broadcast chats that are currently + listening for new email. + - Return list of chat object: `List[Object]` + ''' + q = """ + SELECT * + FROM broadcast_chats + """ + self.cur.execute(q) + + return self.cur.fetchall() diff --git a/daemon/telegram/database/methods/getter/get_email_id.py b/daemon/telegram/database/methods/getter/get_email_id.py new file mode 100644 index 0000000..115569e --- /dev/null +++ b/daemon/telegram/database/methods/getter/get_email_id.py @@ -0,0 +1,58 @@ + + + +class GetEmailID: + + def get_email_id(self, email_id, chat_id): + ''' + Determine whether the email needs to be sent to @tg_chat_id. + - Return an email id (PK) if it needs to be sent + - Return None if it doesn't need to be sent + ''' + if self.__is_sent(email_id, chat_id): + return + + res = self.__email_id(email_id) + if not bool(res): + return + + return int(res[0]) + + + def __is_sent(self, email_id, chat_id): + ''' + Checking if this email has already been sent + or not. + - Return True if it's already been sent + ''' + + q = """ + SELECT emails.id, tg_emails.id FROM emails LEFT JOIN tg_emails + ON emails.id = tg_emails.email_id + WHERE emails.message_id = %(email_id)s + AND tg_emails.chat_id = %(chat_id)s + LIMIT 1 + """ + + self.cur.execute(q, { + "email_id": email_id, + "chat_id": chat_id + }) + + res = self.cur.fetchone() + return bool(res) + + + def __email_id(self, email_id): + ''' + Get the email id if match with the email message_id. + - Return the result if it's match and exists + ''' + + q = """ + SELECT id FROM emails WHERE message_id = %(email_id)s + """ + + self.cur.execute(q, {"email_id": email_id}) + res = self.cur.fetchone() + return res \ No newline at end of file diff --git a/daemon/telegram/database/methods/getter/get_telegram_reply.py b/daemon/telegram/database/methods/getter/get_telegram_reply.py new file mode 100644 index 0000000..2409225 --- /dev/null +++ b/daemon/telegram/database/methods/getter/get_telegram_reply.py @@ -0,0 +1,29 @@ + + + +class GetTelegramReply: + + def get_reply_id(self, email_msg_id, chat_id): + ''' + Get Telegram message ID sent match with + email message ID and Telegram chat ID. + - Return Telegram message ID if exists: `int` + - Return None if not exists` + ''' + q = """ + SELECT tg_emails.tg_msg_id + FROM emails INNER JOIN tg_emails + ON emails.id = tg_emails.email_id + WHERE emails.message_id = %(email_msg_id)s + AND tg_emails.chat_id = %(chat_id)s + """ + + self.cur.execute(q, { + "email_msg_id": email_msg_id, + "chat_id": chat_id + }) + res = self.cur.fetchone() + if not bool(res): + return None + + return res[0] diff --git a/daemon/telegram/database/methods/insertion/__init__.py b/daemon/telegram/database/methods/insertion/__init__.py new file mode 100644 index 0000000..562bf41 --- /dev/null +++ b/daemon/telegram/database/methods/insertion/__init__.py @@ -0,0 +1,12 @@ +from .insert_atom import InsertAtom +from .insert_broadcast import InsertBroadcast +from .insert_email import InsertEmail +from .insert_telegram import InsertTelegram + + +class Insertion( + InsertAtom, + InsertBroadcast, + InsertEmail, + InsertTelegram +): pass diff --git a/daemon/telegram/database/methods/insertion/insert_atom.py b/daemon/telegram/database/methods/insertion/insert_atom.py new file mode 100644 index 0000000..480aa02 --- /dev/null +++ b/daemon/telegram/database/methods/insertion/insert_atom.py @@ -0,0 +1,20 @@ +from mysql.connector import errors +from datetime import datetime + + +class InsertAtom: + + def save_atom(self, atom: str): + try: + return self.__save_atom(atom) + except errors.IntegrityError: + # + # Duplicate data, skip! + # + return None + + + def __save_atom(self, atom: str): + q = "INSERT INTO atom_urls (url, created_at) VALUES (%s, %s)" + self.cur.execute(q, (atom, datetime.utcnow())) + return self.cur.lastrowid diff --git a/daemon/telegram/database/methods/insertion/insert_broadcast.py b/daemon/telegram/database/methods/insertion/insert_broadcast.py new file mode 100644 index 0000000..f4ee06d --- /dev/null +++ b/daemon/telegram/database/methods/insertion/insert_broadcast.py @@ -0,0 +1,49 @@ +from mysql.connector import errors +from datetime import datetime + + +class InsertBroadcast: + + def save_broadcast( + self, + chat_id: int, + name: str, + type: str, + created_at: "datetime", + username: str = None, + link: str = None, + ): + try: + return self.__insert_broadcast( + chat_id=chat_id, + name=name, + type=type, + created_at=created_at, + username=username, + link=link + ) + except errors.IntegrityError: + # + # Duplicate data, skip! + # + return None + + + def __insert_broadcast( + self, + chat_id: int, + name: str, + type: str, + created_at: "datetime", + username: str = None, + link: str = None, + ): + q = """ + INSERT INTO broadcast_chats + (chat_id, username, name, type, link, created_at) + VALUES + (%s, %s, %s, %s, %s, %s) + """ + values = (chat_id, username, name, type, link, created_at) + self.cur.execute(q, values) + return self.cur.lastrowid diff --git a/daemon/telegram/database/methods/insertion/insert_email.py b/daemon/telegram/database/methods/insertion/insert_email.py new file mode 100644 index 0000000..bd88339 --- /dev/null +++ b/daemon/telegram/database/methods/insertion/insert_email.py @@ -0,0 +1,20 @@ +from mysql.connector import errors +from datetime import datetime + + +class InsertEmail: + + def save_email(self, email_msg_id): + try: + return self.__insert_email(email_msg_id) + except errors.IntegrityError: + # + # Duplicate data, skip! + # + return None + + + def __insert_email(self, email_msg_id): + q = "INSERT INTO emails (message_id, created_at) VALUES (%s, %s)" + self.cur.execute(q, (email_msg_id, datetime.utcnow())) + return self.cur.lastrowid diff --git a/daemon/telegram/database/methods/insertion/insert_telegram.py b/daemon/telegram/database/methods/insertion/insert_telegram.py new file mode 100644 index 0000000..bf51526 --- /dev/null +++ b/daemon/telegram/database/methods/insertion/insert_telegram.py @@ -0,0 +1,14 @@ +from datetime import datetime + + +class InsertTelegram: + + def save_telegram_mail(self, email_id, tg_chat_id, tg_msg_id): + q = """ + INSERT INTO tg_emails + (email_id, chat_id, tg_msg_id, created_at) + VALUES (%s, %s, %s, %s); + """ + self.cur.execute(q, (email_id, tg_chat_id, tg_msg_id, + datetime.utcnow())) + return self.cur.lastrowid diff --git a/daemon/telegram/scraper/__init__.py b/daemon/telegram/mailer/__init__.py similarity index 68% rename from daemon/telegram/scraper/__init__.py rename to daemon/telegram/mailer/__init__.py index 4294302..e6b4567 100644 --- a/daemon/telegram/scraper/__init__.py +++ b/daemon/telegram/mailer/__init__.py @@ -4,6 +4,5 @@ # Copyright (C) 2022 Ammar Faizi # -from .scraper import Scraper -from .bot import BotMutexes -from .bot import Bot +from .listener import Mutexes +from .listener import Listener diff --git a/daemon/telegram/scraper/bot.py b/daemon/telegram/mailer/listener.py similarity index 66% rename from daemon/telegram/scraper/bot.py rename to daemon/telegram/mailer/listener.py index 7adfb12..e36cac7 100644 --- a/daemon/telegram/scraper/bot.py +++ b/daemon/telegram/mailer/listener.py @@ -5,36 +5,33 @@ # from apscheduler.schedulers.asyncio import AsyncIOScheduler -from packages import DaemonClient -from scraper import Scraper -from . import utils +from pyrogram.types import Message +from ..packages import DaemonClient +from atom import Scraper +from atom import utils +from atom.utils import Mutexes import asyncio import shutil import re import traceback -class BotMutexes(): - def __init__(self): - self.send_to_tg = asyncio.Lock() - - -class Bot(): +class Listener: def __init__(self, client: DaemonClient, sched: AsyncIOScheduler, - scraper: Scraper, mutexes: BotMutexes): + mutexes: Mutexes): self.client = client self.sched = sched - self.scraper = scraper - self.mutexes = mutexes + self.mutex = mutexes self.db = client.db + self.scraper = Scraper() self.isRunnerFixed = False def run(self): - # - # Execute __run() once to avoid high latency at - # initilization. - # + ''' + Execute __run() once to avoid high latency at + initilization. + ''' self.runner = self.sched.add_job( func=self.__run, misfire_grace_time=None, @@ -71,16 +68,16 @@ class Bot(): async def __handle_mail(self, url, mail): chats = self.db.get_broadcast_chats() for chat in chats: - async with self.mutexes.send_to_tg: - should_wait = await self.__send_to_tg(url, mail, + async with self.mutex.lock: + should_wait = await self.__send_mail(url, mail, chat[1]) if should_wait: await asyncio.sleep(1) - # @__must_hold(self.mutexes.send_to_tg) - async def __send_to_tg(self, url, mail, tg_chat_id): + # @__must_hold(self.mutex.lock) + async def __send_mail(self, url, mail, tg_chat_id): email_msg_id = utils.get_email_msg_id(mail) if not email_msg_id: # @@ -89,7 +86,7 @@ class Bot(): # return False - email_id = self.__need_to_send_to_telegram(email_msg_id, + email_id = self.__mail_id_from_db(email_msg_id, tg_chat_id) if not email_id: # @@ -98,21 +95,23 @@ class Bot(): # return False - text, files, is_patch = utils.create_template(mail) - reply_to = self.get_tg_reply_to(mail, tg_chat_id) + text, files, is_patch = utils.create_template( + mail, "telegram" + ) + reply_to = self.get_reply(mail, tg_chat_id) url = str(re.sub(r"/raw$", "", url)) if is_patch: - m = await self.client.send_patch_email( + m: "Message" = await self.client.send_patch_email( mail, tg_chat_id, text, reply_to, url ) else: text = "#ml\n" + text - m = await self.client.send_text_email( + m: "Message" = await self.client.send_text_email( tg_chat_id, text,reply_to, url ) - self.db.insert_telegram(email_id, m.chat.id, m.id) + self.db.save_telegram_mail(email_id, m.chat.id, m.id) for d, f in files: await m.reply_document(f"{d}/{f}", file_name=f) await asyncio.sleep(1) @@ -123,16 +122,16 @@ class Bot(): return True - def __need_to_send_to_telegram(self, email_msg_id, tg_chat_id): - email_id = self.db.save_email_msg_id(email_msg_id) + def __mail_id_from_db(self, email_msg_id, tg_chat_id): + email_id = self.db.save_email(email_msg_id) if email_id: return email_id - email_id = self.db.need_to_send_to_tg(email_msg_id, tg_chat_id) + email_id = self.db.get_email_id(email_msg_id, tg_chat_id) return email_id - def get_tg_reply_to(self, mail, tg_chat_id): + def get_reply(self, mail, tg_chat_id): reply_to = mail.get("in-reply-to") if not reply_to: return None @@ -141,4 +140,4 @@ class Bot(): if not reply_to: return None - return self.db.get_tg_reply_to(reply_to, tg_chat_id) + return self.db.get_reply_id(reply_to, tg_chat_id) diff --git a/daemon/telegram/packages/client.py b/daemon/telegram/packages/client.py index 282daf6..75e1c63 100644 --- a/daemon/telegram/packages/client.py +++ b/daemon/telegram/packages/client.py @@ -6,19 +6,19 @@ from pyrogram import Client from pyrogram.enums import ParseMode from pyrogram.types import Message, InlineKeyboardMarkup, InlineKeyboardButton +from atom import utils from typing import Union from email.message import Message -from scraper import utils -from scraper.db import Db +from telegram.database import DB from .decorator import handle_flood class DaemonClient(Client): - def __init__(self, name: str, api_id: int, - api_hash: str, conn, **kwargs): + def __init__(self, name: str, api_id: int, api_hash: str, + conn, **kwargs): super().__init__(name, api_id, api_hash, **kwargs) - self.db = Db(conn) + self.db = DB(conn) @handle_flood @@ -56,7 +56,9 @@ class DaemonClient(Client): parse_mode: ParseMode = ParseMode.HTML ) -> Message: print("[send_patch_email]") - tmp, doc, caption, url = utils.prepare_send_patch(mail, text, url) + tmp, doc, caption, url = utils.prepare_patch( + mail, text, url, "telegram" + ) m = await self.send_document( chat_id=chat_id, document=doc, @@ -71,5 +73,5 @@ class DaemonClient(Client): ]) ) - utils.clean_up_after_send_patch(tmp) + utils.remove_patch(tmp) return m diff --git a/daemon/telegram/packages/plugins/callbacks/del_atom.py b/daemon/telegram/packages/plugins/callbacks/del_atom.py index 1510d60..b1076f9 100644 --- a/daemon/telegram/packages/plugins/callbacks/del_atom.py +++ b/daemon/telegram/packages/plugins/callbacks/del_atom.py @@ -3,10 +3,10 @@ # Copyright (C) 2022 Muhammad Rizki # -from packages import DaemonClient -from scraper import utils +from telegram.packages import DaemonClient from pyrogram.types import CallbackQuery -import config +from telegram import config +from atom import utils @DaemonClient.on_callback_query(config.admin_only, group=1) diff --git a/daemon/telegram/packages/plugins/callbacks/del_chat.py b/daemon/telegram/packages/plugins/callbacks/del_chat.py index 26c6dd8..de7d51e 100644 --- a/daemon/telegram/packages/plugins/callbacks/del_chat.py +++ b/daemon/telegram/packages/plugins/callbacks/del_chat.py @@ -3,10 +3,10 @@ # Copyright (C) 2022 Muhammad Rizki # -from packages import DaemonClient -from scraper import utils +from telegram.packages import DaemonClient from pyrogram.types import CallbackQuery -import config +from telegram import config +from atom import utils @DaemonClient.on_callback_query(config.admin_only, group=2) diff --git a/daemon/telegram/packages/plugins/commands/debugger.py b/daemon/telegram/packages/plugins/commands/debugger.py index ae2d31d..7f6f367 100644 --- a/daemon/telegram/packages/plugins/commands/debugger.py +++ b/daemon/telegram/packages/plugins/commands/debugger.py @@ -7,7 +7,7 @@ from pyrogram import Client, filters, enums from pyrogram.types import Message from textwrap import indent import io, import_expression, contextlib, traceback -import config +from telegram import config @Client.on_message( diff --git a/daemon/telegram/packages/plugins/commands/manage_atom.py b/daemon/telegram/packages/plugins/commands/manage_atom.py index bcb2f35..ef60260 100644 --- a/daemon/telegram/packages/plugins/commands/manage_atom.py +++ b/daemon/telegram/packages/plugins/commands/manage_atom.py @@ -5,9 +5,9 @@ from pyrogram.types import Message from pyrogram import filters -from packages import DaemonClient -from scraper import utils -import config +from telegram.packages import DaemonClient +from telegram import config +from atom import utils @DaemonClient.on_message( @@ -25,7 +25,7 @@ async def add_atom_url(c: DaemonClient, m: Message): if not is_atom: return await m.reply("Invalid Atom URL") - inserted = c.db.insert_atom(text) + inserted = c.db.save_atom(text) if inserted is None: return await m.reply(f"This URL already listened for new email.") diff --git a/daemon/telegram/packages/plugins/commands/manage_broadcast.py b/daemon/telegram/packages/plugins/commands/manage_broadcast.py index ffb5a6b..922f4fe 100644 --- a/daemon/telegram/packages/plugins/commands/manage_broadcast.py +++ b/daemon/telegram/packages/plugins/commands/manage_broadcast.py @@ -5,9 +5,9 @@ from pyrogram.types import Message from pyrogram import filters, enums -from packages import DaemonClient -from scraper import utils -import config +from telegram.packages import DaemonClient +from telegram import config +from atom import utils @DaemonClient.on_message( @@ -20,7 +20,7 @@ async def add_broadcast(c: DaemonClient, m: Message): else: chat_name = m.chat.title - inserted = c.db.insert_broadcast( + inserted = c.db.save_broadcast( chat_id=m.chat.id, name=chat_name, type=str(m.chat.type), diff --git a/daemon/telegram/packages/plugins/commands/scrape.py b/daemon/telegram/packages/plugins/commands/scrape.py index 45b1581..f23ad6f 100644 --- a/daemon/telegram/packages/plugins/commands/scrape.py +++ b/daemon/telegram/packages/plugins/commands/scrape.py @@ -6,10 +6,10 @@ from pyrogram.types import Message from pyrogram import filters -from packages import DaemonClient -from scraper import Scraper -from scraper import utils -import config +from telegram.packages import DaemonClient +from telegram import config +from atom import Scraper +from atom import utils import shutil import re import asyncio @@ -37,7 +37,7 @@ async def scrap_email(c: DaemonClient, m: Message): s = Scraper() mail = await s.get_email_from_url(url) - text, files, is_patch = utils.create_template(mail) + text, files, is_patch = utils.create_template(mail, "telegram") if is_patch: m = await c.send_patch_email( diff --git a/daemon/telegram/scraper/db.py b/daemon/telegram/scraper/db.py deleted file mode 100644 index 58601d1..0000000 --- a/daemon/telegram/scraper/db.py +++ /dev/null @@ -1,217 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# -# Copyright (C) 2022 Muhammad Rizki -# Copyright (C) 2022 Ammar Faizi -# - -from datetime import datetime -import mysql - - -class Db(): - def __init__(self, conn): - self.conn = conn - self.conn.autocommit = True - self.cur = self.conn.cursor(buffered=True) - - - def __del__(self): - self.cur.close() - self.conn.close() - - - def save_email_msg_id(self, email_msg_id): - try: - return self.__save_email_msg_id(email_msg_id) - except mysql.connector.errors.IntegrityError: - # - # Duplicate data, skip! - # - return None - - - def __save_email_msg_id(self, email_msg_id): - q = "INSERT INTO emails (message_id, created_at) VALUES (%s, %s)" - self.cur.execute(q, (email_msg_id, datetime.utcnow())) - return self.cur.lastrowid - - - def insert_telegram(self, email_id, tg_chat_id, tg_msg_id): - q = """ - INSERT INTO tg_emails - (email_id, chat_id, tg_msg_id, created_at) - VALUES (%s, %s, %s, %s); - """ - self.cur.execute(q, (email_id, tg_chat_id, tg_msg_id, - datetime.utcnow())) - return self.cur.lastrowid - - - # - # Determine whether the email needs to be sent to @tg_chat_id. - # - # - Return an email id (PK) if it needs to be sent. - # - Return None if it doesn't need to be sent. - # - def need_to_send_to_tg(self, email_msg_id, tg_chat_id): - q = """ - SELECT emails.id, tg_emails.id FROM emails LEFT JOIN tg_emails - ON emails.id = tg_emails.email_id - WHERE emails.message_id = %(email_msg_id)s - AND tg_emails.chat_id = %(tg_chat_id)s - LIMIT 1 - """ - - self.cur.execute( - q, - { - "email_msg_id": email_msg_id, - "tg_chat_id": tg_chat_id - } - ) - res = self.cur.fetchone() - if bool(res): - # - # This email has already been sent to - # @tg_chat_id. - # - return None - - q = """ - SELECT id FROM emails WHERE message_id = %(email_msg_id)s - """ - self.cur.execute(q, {"email_msg_id": email_msg_id}) - res = self.cur.fetchone() - if not bool(res): - # - # Something goes wrong, skip! - # - return None - - return int(res[0]) - - - def get_tg_reply_to(self, email_msg_id, tg_chat_id): - q = """ - SELECT tg_emails.tg_msg_id - FROM emails INNER JOIN tg_emails - ON emails.id = tg_emails.email_id - WHERE emails.message_id = %(email_msg_id)s - AND tg_emails.chat_id = %(chat_id)s - """ - - self.cur.execute( - q, - { - "email_msg_id": email_msg_id, - "chat_id": tg_chat_id - } - ) - res = self.cur.fetchone() - if not bool(res): - return None - - return res[0] - - - def insert_atom(self, atom: str): - try: - return self.__save_atom(atom) - except mysql.connector.errors.IntegrityError: - # - # Duplicate data, skip! - # - return None - - - def __save_atom(self, atom: str): - q = "INSERT INTO atom_urls (url, created_at) VALUES (%s, %s)" - self.cur.execute(q, (atom, datetime.utcnow())) - return self.cur.lastrowid - - - def delete_atom(self, atom: str): - q = """ - DELETE FROM atom_urls - WHERE url = %(atom)s - """ - try: - self.cur.execute(q, {"atom": atom}) - return True - except: - return False - - - def get_atom_urls(self): - q = """ - SELECT atom_urls.url - FROM atom_urls - """ - self.cur.execute(q) - urls = self.cur.fetchall() - - return [u[0] for u in urls] - - - def insert_broadcast( - self, - chat_id: int, - name: str, - type: str, - created_at: "datetime", - username: str = None, - link: str = None, - ): - try: - return self.__save_broadcast( - chat_id=chat_id, - name=name, - type=type, - created_at=created_at, - username=username, - link=link - ) - except mysql.connector.errors.IntegrityError: - # - # Duplicate data, skip! - # - return None - - - def __save_broadcast( - self, - chat_id: int, - name: str, - type: str, - created_at: "datetime", - username: str = None, - link: str = None, - ): - q = """ - INSERT INTO broadcast_chats - (chat_id, username, name, type, link, created_at) - VALUES - (%s, %s, %s, %s, %s, %s) - """ - values = (chat_id, username, name, type, link, created_at) - self.cur.execute(q, values) - return self.cur.lastrowid - - - def delete_broadcast(self, chat_id: int): - q = """ - DELETE FROM broadcast_chats - WHERE chat_id = %(chat_id)s - """ - self.cur.execute(q, {"chat_id": chat_id}) - return self.cur.rowcount > 0 - - - def get_broadcast_chats(self): - q = """ - SELECT * - FROM broadcast_chats - """ - self.cur.execute(q) - - return self.cur.fetchall() diff --git a/daemon/telegram/scraper/scraper.py b/daemon/telegram/scraper/scraper.py deleted file mode 100644 index 2d5942b..0000000 --- a/daemon/telegram/scraper/scraper.py +++ /dev/null @@ -1,63 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# -# Copyright (C) 2022 Muhammad Rizki -# Copyright (C) 2022 Ammar Faizi -# - -from typing import Dict, List -import email.policy -import xmltodict -import httpx -import email - - -class Scraper(): - async def get_new_threads_urls(self, atom_url): - ret = await self.__get_atom_content(atom_url) - return await self.__get_new_threads_from_atom(ret) - - - async def __get_atom_content(self, atom_url): - async with httpx.AsyncClient() as client: - res = await client.get(atom_url) - if res.status_code == 200: - return res.text - raise Exception(f"[get_atom_content]: Returned {res.status_code} HTTP code") - - - async def __get_new_threads_from_atom(self, atom): - j: Dict[str, List[ - Dict[str, str] - ]] = xmltodict.parse(atom)["feed"] - - entry = [] - e = j["entry"] - for i in e: - entry.append({ - "link": i["link"]["@href"], - "title": i["title"], - "updated": i["updated"], - }) - # - # TODO(ammarfaizi2): Sort by title as well if the @updated is - # identic. - # - entry.sort(key=lambda x: x["updated"]) - - ret = [] - for i in entry: - link = i["link"].replace("http://", "https://") - ret.append(link + "raw") - - return ret - - - async def get_email_from_url(self, url): - async with httpx.AsyncClient() as client: - res = await client.get(url) - if res.status_code == 200: - return email.message_from_string( - res.text, - policy=email.policy.default - ) - raise Exception(f"[get_atom_content]: Returned {res.status_code} HTTP code") diff --git a/daemon/telegram/run.py b/daemon/tg.py similarity index 75% rename from daemon/telegram/run.py rename to daemon/tg.py index 5360395..2e0418a 100644 --- a/daemon/telegram/run.py +++ b/daemon/tg.py @@ -5,20 +5,19 @@ # from apscheduler.schedulers.asyncio import AsyncIOScheduler -from scraper import BotMutexes +from atom.utils import Mutexes from dotenv import load_dotenv from mysql import connector -from packages import DaemonClient -from scraper import Scraper -from scraper import Bot +from telegram.packages import DaemonClient +from telegram.mailer import Listener import os def main(): - load_dotenv() + load_dotenv("telegram.env") client = DaemonClient( - "storage/EmailScraper", + "telegram/storage/EmailScraper", api_id=int(os.getenv("API_ID")), api_hash=os.getenv("API_HASH"), bot_token=os.getenv("BOT_TOKEN"), @@ -28,9 +27,7 @@ def main(): password=os.getenv("DB_PASS"), database=os.getenv("DB_NAME") ), - plugins=dict( - root="packages.plugins" - ), + plugins=dict(root="telegram.packages.plugins") ) sched = AsyncIOScheduler( @@ -40,11 +37,10 @@ def main(): } ) - bot = Bot( + bot = Listener( client=client, sched=sched, - scraper=Scraper(), - mutexes=BotMutexes() + mutexes=Mutexes() ) sched.start() bot.run() -- Muhammad Rizki