From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.6 (2021-04-09) on gnuweeb.org X-Spam-Level: X-Spam-Status: No, score=-0.8 required=5.0 tests=ALL_TRUSTED,DKIM_SIGNED, DKIM_VALID,DKIM_VALID_AU,DKIM_VALID_EF,NO_DNS_FOR_FROM,URIBL_BLOCKED autolearn=no autolearn_force=no version=3.4.6 Received: from localhost.localdomain (unknown [101.128.125.254]) by gnuweeb.org (Postfix) with ESMTPSA id 9DCA480AE5; Sat, 27 Aug 2022 03:03:00 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=gnuweeb.org; s=default; t=1661569382; bh=TC1VqTjKEZs9qG3Xr/aoqdyckn2iVcDtQtBHchLhRO0=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=mxZ2XGkjZSxSYiVRFxJmCqOVfmQ8IkPpIF0vmDojSaS5QsiFWCB+0CqJMuO2OSvDN VYrV0yqnqIoSbVLWXmgRLV9xlbF3Fi6OMh3UGeMpkZuZvx6mCtw2sSLRj7dlXDC2KJ DJGMyPhWyRhfZ07onYcV2M9li76nwzZ566WEUdJdAkZZjw/GJ+fmy1WN86XiMnIj5N m1JbAlneZmJlIdCFCUi++5+BFNdgZFYOcupIq/bW9oK4nVxBICf4CVctOMDJGnO64f O0OrXr7ivHQs2Lm1gwCRH7aNRbUPP6NL54PtiJ7to6X3coFWQDUpyyDRTZxGuw/c1d ldB0wtxV/1syw== From: Muhammad Rizki To: Ammar Faizi Cc: Muhammad Rizki , GNU/Weeb Mailing List , Alviro Iskandar Setiawan Subject: [PATCH v2 3/3] Full refactor bot scripts Date: Sat, 27 Aug 2022 10:02:36 +0700 Message-Id: <20220827030236.1094-4-kiizuha@gnuweeb.org> X-Mailer: git-send-email 2.34.1.windows.1 In-Reply-To: <20220827030236.1094-1-kiizuha@gnuweeb.org> References: <20220827030236.1094-1-kiizuha@gnuweeb.org> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit List-Id: Refactoring the bot scripts to make it more clean Signed-off-by: Muhammad Rizki --- .gitignore | 6 +- daemon/{discord/mailer => atom}/__init__.py | 2 +- daemon/{discord/mailer => atom}/scraper.py | 8 +- daemon/{telegram/scraper => atom}/utils.py | 87 +++++-- daemon/db.sql | 119 +++++++++ daemon/{discord/run.py => dc.py} | 12 +- .../.env.example => discord.env.example} | 0 daemon/discord/execute_me.sql | 64 ----- daemon/discord/mailer/database.py | 203 --------------- daemon/discord/mailer/utils.py | 241 ------------------ daemon/{discord => dscord}/config.py.example | 0 daemon/dscord/database/__init__.py | 6 + daemon/dscord/database/core.py | 19 ++ daemon/dscord/database/methods/__init__.py | 16 ++ .../database/methods/deletion/__init__.py | 8 + .../database/methods/deletion/delet_atom.py | 12 + .../methods/deletion/delete_broadcast.py | 12 + .../database/methods/getter/__init__.py | 12 + .../database/methods/getter/get_atom_urls.py | 18 ++ .../methods/getter/get_broadcast_chats.py | 18 ++ .../database/methods/getter/get_email_id.py | 56 ++++ .../database/methods/getter/get_reply.py | 29 +++ .../database/methods/insertion/__init__.py | 12 + .../database/methods/insertion/insert_atom.py | 20 ++ .../methods/insertion/insert_broadcast.py | 42 +++ .../methods/insertion/insert_discord.py | 14 + .../methods/insertion/insert_email.py | 20 ++ .../{discord => dscord}/gnuweeb/__init__.py | 0 daemon/{discord => dscord}/gnuweeb/client.py | 32 ++- daemon/{discord => dscord}/gnuweeb/filters.py | 2 +- .../gnuweeb/models/__init__.py | 0 .../gnuweeb/models/ui/__init__.py | 0 .../gnuweeb/models/ui/buttons/__init__.py | 0 .../models/ui/buttons/full_message_btn.py | 0 .../gnuweeb/plugins/__init__.py | 0 .../plugins/basic_commands/__init__.py | 0 .../plugins/basic_commands/debugger.py | 2 +- .../gnuweeb/plugins/basic_commands/sync_it.py | 6 +- .../gnuweeb/plugins/events/__init__.py | 0 .../gnuweeb/plugins/events/on_error.py | 0 .../gnuweeb/plugins/events/on_ready.py | 0 .../plugins/slash_commands/__init__.py | 0 .../plugins/slash_commands/get_lore_mail.py | 4 +- .../plugins/slash_commands/manage_atom.py | 6 +- .../slash_commands/manage_broadcast.py | 6 +- daemon/{discord => dscord}/gnuweeb/utils.py | 0 daemon/dscord/mailer/__init__.py | 6 + daemon/{discord => dscord}/mailer/listener.py | 26 +- daemon/{discord => dscord}/requirements.txt | 0 daemon/{discord => dscord}/storage/.gitignore | 0 .../.env.example => telegram.env.example} | 0 daemon/telegram/database/__init__.py | 7 + daemon/telegram/database/core.py | 20 ++ daemon/telegram/database/methods/__init__.py | 17 ++ .../database/methods/deletion/__init__.py | 14 + .../database/methods/deletion/delet_atom.py | 15 ++ .../methods/deletion/delete_broadcast.py | 15 ++ .../database/methods/getter/__init__.py | 18 ++ .../database/methods/getter/get_atom_urls.py | 21 ++ .../methods/getter/get_broadcast_chats.py | 21 ++ .../database/methods/getter/get_email_id.py | 62 +++++ .../methods/getter/get_telegram_reply.py | 33 +++ .../database/methods/insertion/__init__.py | 18 ++ .../database/methods/insertion/insert_atom.py | 27 ++ .../methods/insertion/insert_broadcast.py | 56 ++++ .../methods/insertion/insert_email.py | 27 ++ .../methods/insertion/insert_telegram.py | 21 ++ daemon/telegram/db.sql | 62 ----- daemon/telegram/mailer/__init__.py | 8 + .../{scraper/bot.py => mailer/listener.py} | 63 +++-- daemon/telegram/packages/__init__.py | 5 + daemon/telegram/packages/client.py | 18 +- daemon/telegram/packages/decorator.py | 6 +- .../packages/plugins/callbacks/del_atom.py | 8 +- .../packages/plugins/callbacks/del_chat.py | 8 +- .../packages/plugins/commands/debugger.py | 4 +- .../packages/plugins/commands/manage_atom.py | 10 +- .../plugins/commands/manage_broadcast.py | 10 +- .../packages/plugins/commands/scrape.py | 12 +- daemon/telegram/scraper/__init__.py | 9 - daemon/telegram/scraper/db.py | 217 ---------------- daemon/telegram/scraper/scraper.py | 63 ----- daemon/{telegram/run.py => tg.py} | 22 +- 83 files changed, 1039 insertions(+), 1024 deletions(-) rename daemon/{discord/mailer => atom}/__init__.py (68%) rename daemon/{discord/mailer => atom}/scraper.py (83%) rename daemon/{telegram/scraper => atom}/utils.py (72%) create mode 100644 daemon/db.sql rename daemon/{discord/run.py => dc.py} (78%) rename daemon/{discord/.env.example => discord.env.example} (100%) delete mode 100644 daemon/discord/execute_me.sql delete mode 100644 daemon/discord/mailer/database.py delete mode 100644 daemon/discord/mailer/utils.py rename daemon/{discord => dscord}/config.py.example (100%) create mode 100644 daemon/dscord/database/__init__.py create mode 100644 daemon/dscord/database/core.py create mode 100644 daemon/dscord/database/methods/__init__.py create mode 100644 daemon/dscord/database/methods/deletion/__init__.py create mode 100644 daemon/dscord/database/methods/deletion/delet_atom.py create mode 100644 daemon/dscord/database/methods/deletion/delete_broadcast.py create mode 100644 daemon/dscord/database/methods/getter/__init__.py create mode 100644 daemon/dscord/database/methods/getter/get_atom_urls.py create mode 100644 daemon/dscord/database/methods/getter/get_broadcast_chats.py create mode 100644 daemon/dscord/database/methods/getter/get_email_id.py create mode 100644 daemon/dscord/database/methods/getter/get_reply.py create mode 100644 daemon/dscord/database/methods/insertion/__init__.py create mode 100644 daemon/dscord/database/methods/insertion/insert_atom.py create mode 100644 daemon/dscord/database/methods/insertion/insert_broadcast.py create mode 100644 daemon/dscord/database/methods/insertion/insert_discord.py create mode 100644 daemon/dscord/database/methods/insertion/insert_email.py rename daemon/{discord => dscord}/gnuweeb/__init__.py (100%) rename daemon/{discord => dscord}/gnuweeb/client.py (84%) rename daemon/{discord => dscord}/gnuweeb/filters.py (98%) rename daemon/{discord => dscord}/gnuweeb/models/__init__.py (100%) rename daemon/{discord => dscord}/gnuweeb/models/ui/__init__.py (100%) rename daemon/{discord => dscord}/gnuweeb/models/ui/buttons/__init__.py (100%) rename daemon/{discord => dscord}/gnuweeb/models/ui/buttons/full_message_btn.py (100%) rename daemon/{discord => dscord}/gnuweeb/plugins/__init__.py (100%) rename daemon/{discord => dscord}/gnuweeb/plugins/basic_commands/__init__.py (100%) rename daemon/{discord => dscord}/gnuweeb/plugins/basic_commands/debugger.py (94%) rename daemon/{discord => dscord}/gnuweeb/plugins/basic_commands/sync_it.py (70%) rename daemon/{discord => dscord}/gnuweeb/plugins/events/__init__.py (100%) rename daemon/{discord => dscord}/gnuweeb/plugins/events/on_error.py (100%) rename daemon/{discord => dscord}/gnuweeb/plugins/events/on_ready.py (100%) rename daemon/{discord => dscord}/gnuweeb/plugins/slash_commands/__init__.py (100%) rename daemon/{discord => dscord}/gnuweeb/plugins/slash_commands/get_lore_mail.py (95%) rename daemon/{discord => dscord}/gnuweeb/plugins/slash_commands/manage_atom.py (95%) rename daemon/{discord => dscord}/gnuweeb/plugins/slash_commands/manage_broadcast.py (95%) rename daemon/{discord => dscord}/gnuweeb/utils.py (100%) create mode 100644 daemon/dscord/mailer/__init__.py rename daemon/{discord => dscord}/mailer/listener.py (85%) rename daemon/{discord => dscord}/requirements.txt (100%) rename daemon/{discord => dscord}/storage/.gitignore (100%) rename daemon/{telegram/.env.example => telegram.env.example} (100%) create mode 100644 daemon/telegram/database/__init__.py create mode 100644 daemon/telegram/database/core.py create mode 100644 daemon/telegram/database/methods/__init__.py create mode 100644 daemon/telegram/database/methods/deletion/__init__.py create mode 100644 daemon/telegram/database/methods/deletion/delet_atom.py create mode 100644 daemon/telegram/database/methods/deletion/delete_broadcast.py create mode 100644 daemon/telegram/database/methods/getter/__init__.py create mode 100644 daemon/telegram/database/methods/getter/get_atom_urls.py create mode 100644 daemon/telegram/database/methods/getter/get_broadcast_chats.py create mode 100644 daemon/telegram/database/methods/getter/get_email_id.py create mode 100644 daemon/telegram/database/methods/getter/get_telegram_reply.py create mode 100644 daemon/telegram/database/methods/insertion/__init__.py create mode 100644 daemon/telegram/database/methods/insertion/insert_atom.py create mode 100644 daemon/telegram/database/methods/insertion/insert_broadcast.py create mode 100644 daemon/telegram/database/methods/insertion/insert_email.py create mode 100644 daemon/telegram/database/methods/insertion/insert_telegram.py delete mode 100644 daemon/telegram/db.sql create mode 100644 daemon/telegram/mailer/__init__.py rename daemon/telegram/{scraper/bot.py => mailer/listener.py} (64%) delete mode 100644 daemon/telegram/scraper/__init__.py delete mode 100644 daemon/telegram/scraper/db.py delete mode 100644 daemon/telegram/scraper/scraper.py rename daemon/{telegram/run.py => tg.py} (69%) diff --git a/.gitignore b/.gitignore index 4201a17..ff5f6de 100644 --- a/.gitignore +++ b/.gitignore @@ -103,8 +103,8 @@ celerybeat.pid *.sage.py # Environments -.env -.venv +*.env +*.venv env/ venv/ ENV/ @@ -141,4 +141,4 @@ data.json # configuration file daemon/telegram/config.py -daemon/discord/config.py +daemon/dscord/config.py diff --git a/daemon/discord/mailer/__init__.py b/daemon/atom/__init__.py similarity index 68% rename from daemon/discord/mailer/__init__.py rename to daemon/atom/__init__.py index 0da4329..2fe4e31 100644 --- a/daemon/discord/mailer/__init__.py +++ b/daemon/atom/__init__.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only # # Copyright (C) 2022 Muhammad Rizki +# Copyright (C) 2022 Ammar Faizi # -from .database import Database from .scraper import Scraper diff --git a/daemon/discord/mailer/scraper.py b/daemon/atom/scraper.py similarity index 83% rename from daemon/discord/mailer/scraper.py rename to daemon/atom/scraper.py index 3873161..8508ae9 100644 --- a/daemon/discord/mailer/scraper.py +++ b/daemon/atom/scraper.py @@ -19,10 +19,10 @@ class Scraper: async def __get_atom_content(self, atom_url): async with httpx.AsyncClient() as client: - res = await client.get(atom_url) + res = await client.get(atom_url, timeout=20) if res.status_code == 200: return res.text - raise Exception(f"[get_atom_content]: Returned {res.status_code} HTTP code") + raise Exception(f"[__get_atom_content]: Returned {res.status_code} HTTP code") async def __get_new_threads_from_atom(self, atom): @@ -54,10 +54,10 @@ class Scraper: async def get_email_from_url(self, url): async with httpx.AsyncClient() as client: - res = await client.get(url) + res = await client.get(url, timeout=20) if res.status_code == 200: return email.message_from_string( res.text, policy=email.policy.default ) - raise Exception(f"[get_atom_content]: Returned {res.status_code} HTTP code") + raise Exception(f"[get_email_from_url]: Returned {res.status_code} HTTP code") diff --git a/daemon/telegram/scraper/utils.py b/daemon/atom/utils.py similarity index 72% rename from daemon/telegram/scraper/utils.py rename to daemon/atom/utils.py index c428a33..b62c850 100644 --- a/daemon/telegram/scraper/utils.py +++ b/daemon/atom/utils.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only # -# Copyright (C) 2022 Muhammad Rizki +# Copyright (C) 2022 Muhammad Rizki # Copyright (C) 2022 Ammar Faizi # @@ -8,13 +8,19 @@ from pyrogram.types import Chat, InlineKeyboardMarkup, InlineKeyboardButton from email.message import Message from typing import Dict from slugify import slugify +import html import hashlib import uuid import os import re import shutil import httpx -import html +import asyncio + + +class Mutexes(): + def __init__(self): + self.lock = asyncio.Lock() def get_email_msg_id(mail): @@ -113,25 +119,37 @@ def consruct_to_n_cc(to: list, cc: list): return ret -def gen_temp(name: str): +def gen_temp(name: str, platform: str): + platform = platform.lower() + plt_ls = ["telegram", "discord"] + + if platform not in plt_ls: + t = f"Platform {platform} is not found, " + t += f"only {', '.join(plt_ls)} is available" + raise ValueError(f"Platform {platform} is not found") + md5 = hashlib.md5(name.encode()).hexdigest() - ret = os.getenv("STORAGE_DIR", "storage") + "/" + md5 + store_dir = os.getenv("STORAGE_DIR", "storage") + platform = platform.replace("discord", "dscord") + path = f"{platform}/{store_dir}/{md5}" try: - os.mkdir(ret) + os.mkdir(path) except FileExistsError: pass - return ret + return path -def extract_body(thread: Message): +def extract_body(thread: Message, platform: str): if not thread.is_multipart(): - p = thread.get_payload(decode=True) - return f"{p.decode(errors='replace')}\n".lstrip(), [] + p = thread.get_payload(decode=True).decode(errors='replace') + if platform == "discord": + p = quote_reply(p) + return f"{p}\n".lstrip(), [] ret = "" files = [] - temp = gen_temp(str(uuid.uuid4())) + temp = gen_temp(str(uuid.uuid4()), platform) for p in thread.get_payload(): fname = p.get_filename() payload = p.get_payload(decode=True) @@ -164,35 +182,42 @@ def __is_patch(subject, content): return True -def create_template(thread: Message, to=None, cc=None): +def create_template(thread: Message, platform: str, to=None, cc=None): if not to: to = extract_list("to", thread) if not cc: cc = extract_list("cc", thread) + if platform == "telegram": + substr = 4000 + border = f"\n{'-'*72}" + else: + substr = 1900 + border = f"\n{'-'*80}" subject = thread.get('subject') ret = f"From: {thread.get('from')}\n" ret += consruct_to_n_cc(to, cc) ret += f"Date: {thread.get('date')}\n" ret += f"Subject: {subject}\n\n" - content, files = extract_body(thread) + content, files = extract_body(thread, platform) is_patch = __is_patch(subject, content) if is_patch: ret += content else: ret += content.strip().replace("\t", " ") - if len(ret) >= 4000: - ret = ret[:4000] + "..." - ret = fix_utf8_char(ret) - ret += f"\n{'-'*72}" + if len(ret) >= substr: + ret = ret[:substr] + "..." + + ret = fix_utf8_char(ret, platform == "telegram") + ret += border return ret, files, is_patch -def prepare_send_patch(mail, text, url): - tmp = gen_temp(url) +def prepare_patch(mail: "Message", text: str, url: str, platform: str): + tmp = gen_temp(url, platform) fnm = str(mail.get("subject")) sch = re.search(PATCH_PATTERN, fnm, re.IGNORECASE) @@ -210,17 +235,31 @@ def prepare_send_patch(mail, text, url): with open(file, "wb") as f: f.write(bytes(text, encoding="utf8")) - caption = "#patch #ml\n" + fix_utf8_char(cap) + caption = "#patch #ml" + if platform == "telegram": + caption += fix_utf8_char("\n" + cap, True) return tmp, file, caption, url -def clean_up_after_send_patch(tmp): +def remove_patch(tmp): shutil.rmtree(tmp) -def fix_utf8_char(text: str): - text = text.rstrip().replace("�"," ") - return html.escape(html.escape(text)) +def fix_utf8_char(text: str, html_escape: bool = True): + t = text.rstrip().replace("�"," ") + if html_escape: + t = html.escape(html.escape(text)) + return t + + +def quote_reply(text: str): + a = "" + for b in text.split("\n"): + b = b.replace(">\n", "> ") + if b.startswith(">"): + a += "> " + a += f"{b}\n" + return a EMAIL_MSG_ID_PATTERN = r"<([^\<\>]+)>" @@ -240,6 +279,8 @@ async def is_atom_url(text: str): return mime == "application/atom+xml" except: return False + + def remove_command(text: str): txt = text.split(" ") txt = text.replace(txt[0] + " ","") diff --git a/daemon/db.sql b/daemon/db.sql new file mode 100644 index 0000000..96eeb81 --- /dev/null +++ b/daemon/db.sql @@ -0,0 +1,119 @@ +-- Adminer 4.7.6 MySQL dump + +SET NAMES utf8; +SET time_zone = '+00:00'; +SET foreign_key_checks = 0; +SET sql_mode = 'NO_AUTO_VALUE_ON_ZERO'; + +SET NAMES utf8mb4; + +DROP TABLE IF EXISTS `tg_emails`; +CREATE TABLE `tg_emails` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT, + `message_id` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci NOT NULL, + `created_at` datetime NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `message_id` (`message_id`), + KEY `created_at` (`created_at`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_520_ci; + + +DROP TABLE IF EXISTS `tg_mail_msg`; +CREATE TABLE `tg_mail_msg` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT, + `email_id` bigint unsigned NOT NULL, + `chat_id` bigint NOT NULL, + `tg_msg_id` bigint unsigned NOT NULL, + `created_at` datetime NOT NULL, + PRIMARY KEY (`id`), + KEY `email_id` (`email_id`), + KEY `chat_id` (`chat_id`), + KEY `tg_msg_id` (`tg_msg_id`), + KEY `created_at` (`created_at`), + CONSTRAINT `tg_mail_msg_ibfk_2` FOREIGN KEY (`email_id`) REFERENCES `tg_emails` (`id`) ON DELETE CASCADE ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_520_ci; + + +DROP TABLE IF EXISTS `tg_atoms`; +CREATE TABLE `tg_atoms` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT, + `url` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci NOT NULL, + `created_at` datetime NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `url` (`url`), + KEY `created_at` (`created_at`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_520_ci; + + +DROP TABLE IF EXISTS `tg_broadcasts`; +CREATE TABLE `tg_broadcasts` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT, + `chat_id` bigint NOT NULL, + `username` varchar(32), + `name` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci NOT NULL, + `type` varchar(32) NOT NULL, + `link` varchar(64), + `created_at` datetime NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `chat_id` (`chat_id`), + KEY `created_at` (`created_at`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_520_ci; + +-- [ SPLITTER ] ------------------------------------------------------------------------------ + +DROP TABLE IF EXISTS `dc_emails`; +CREATE TABLE `dc_emails` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT, + `message_id` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci NOT NULL, + `created_at` datetime NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `message_id` (`message_id`), + KEY `created_at` (`created_at`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_520_ci; + + +DROP TABLE IF EXISTS `dc_mail_msg`; +CREATE TABLE `dc_mail_msg` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT, + `email_id` bigint unsigned NOT NULL, + `channel_id` bigint NOT NULL, + `dc_msg_id` bigint unsigned NOT NULL, + `created_at` datetime NOT NULL, + PRIMARY KEY (`id`), + KEY `email_id` (`email_id`), + KEY `channel_id` (`channel_id`), + KEY `dc_msg_id` (`dc_msg_id`), + KEY `created_at` (`created_at`), + CONSTRAINT `dc_mail_msg_ibfk_2` FOREIGN KEY (`email_id`) REFERENCES `dc_emails` (`id`) ON DELETE CASCADE ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_520_ci; + + +DROP TABLE IF EXISTS `dc_atoms`; +CREATE TABLE `dc_atoms` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT, + `url` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci NOT NULL, + `created_at` datetime NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `url` (`url`), + KEY `created_at` (`created_at`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_520_ci; + + +DROP TABLE IF EXISTS `dc_broadcasts`; +CREATE TABLE `dc_broadcasts` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT, + `guild_id` BIGINT UNSIGNED NOT NULL, + `channel_id` BIGINT UNSIGNED NOT NULL, + `channel_name` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci NOT NULL, + `channel_link` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci NOT NULL, + `created_at` datetime NOT NULL, + PRIMARY KEY (`id`), + KEY `guild_id` (`guild_id`), + UNIQUE KEY `channel_id` (`channel_id`), + KEY `channel_name` (`channel_name`), + KEY `channel_link` (`channel_link`), + KEY `created_at` (`created_at`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_520_ci; + + +-- 2022-07-07 14:25:28 diff --git a/daemon/discord/run.py b/daemon/dc.py similarity index 78% rename from daemon/discord/run.py rename to daemon/dc.py index e3eb7a6..a24421b 100644 --- a/daemon/discord/run.py +++ b/daemon/dc.py @@ -7,15 +7,15 @@ import os from mysql import connector from dotenv import load_dotenv from apscheduler.schedulers.asyncio import AsyncIOScheduler -from mailer.listener import BotMutexes -from mailer.listener import Listener +from dscord.mailer.listener import Listener +from atom.scraper import Scraper +from atom.utils import Mutexes -from gnuweeb import GWClient -from mailer import Scraper +from dscord.gnuweeb import GWClient def main(): - load_dotenv() + load_dotenv("discord.env") sched = AsyncIOScheduler( job_defaults={ @@ -37,7 +37,7 @@ def main(): client=client, sched=sched, scraper=Scraper(), - mutexes=BotMutexes() + mutexes=Mutexes() ) client.mailer = mailer diff --git a/daemon/discord/.env.example b/daemon/discord.env.example similarity index 100% rename from daemon/discord/.env.example rename to daemon/discord.env.example diff --git a/daemon/discord/execute_me.sql b/daemon/discord/execute_me.sql deleted file mode 100644 index 80f2a32..0000000 --- a/daemon/discord/execute_me.sql +++ /dev/null @@ -1,64 +0,0 @@ --- Adminer 4.7.6 MySQL dump - -SET NAMES utf8; -SET time_zone = '+00:00'; -SET foreign_key_checks = 0; -SET sql_mode = 'NO_AUTO_VALUE_ON_ZERO'; - -SET NAMES utf8mb4; - -DROP TABLE IF EXISTS `emails`; -CREATE TABLE `emails` ( - `id` bigint unsigned NOT NULL AUTO_INCREMENT, - `message_id` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci NOT NULL, - `created_at` datetime NOT NULL, - PRIMARY KEY (`id`), - UNIQUE KEY `message_id` (`message_id`), - KEY `created_at` (`created_at`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_520_ci; - - -DROP TABLE IF EXISTS `discord_emails`; -CREATE TABLE `discord_emails` ( - `id` bigint unsigned NOT NULL AUTO_INCREMENT, - `email_id` bigint unsigned NOT NULL, - `channel_id` bigint NOT NULL, - `dc_msg_id` bigint unsigned NOT NULL, - `created_at` datetime NOT NULL, - PRIMARY KEY (`id`), - KEY `email_id` (`email_id`), - KEY `channel_id` (`channel_id`), - KEY `dc_msg_id` (`dc_msg_id`), - KEY `created_at` (`created_at`), - CONSTRAINT `discord_emails_ibfk_2` FOREIGN KEY (`email_id`) REFERENCES `emails` (`id`) ON DELETE CASCADE ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_520_ci; - - -DROP TABLE IF EXISTS `atom_urls`; -CREATE TABLE `atom_urls` ( - `id` bigint unsigned NOT NULL AUTO_INCREMENT, - `url` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci NOT NULL, - `created_at` datetime NOT NULL, - PRIMARY KEY (`id`), - UNIQUE KEY `url` (`url`), - KEY `created_at` (`created_at`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_520_ci; - - -DROP TABLE IF EXISTS `broadcast_chats`; -CREATE TABLE `broadcast_chats` ( - `id` bigint unsigned NOT NULL AUTO_INCREMENT, - `guild_id` BIGINT UNSIGNED NOT NULL, - `channel_id` BIGINT UNSIGNED NOT NULL, - `channel_name` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci NOT NULL, - `channel_link` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci NOT NULL, - `created_at` datetime NOT NULL, - PRIMARY KEY (`id`), - KEY `guild_id` (`guild_id`), - UNIQUE KEY `channel_id` (`channel_id`), - KEY `channel_name` (`channel_name`), - KEY `channel_link` (`channel_link`), - KEY `created_at` (`created_at`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_520_ci; - --- 2022-07-07 14:25:28 diff --git a/daemon/discord/mailer/database.py b/daemon/discord/mailer/database.py deleted file mode 100644 index 0b91558..0000000 --- a/daemon/discord/mailer/database.py +++ /dev/null @@ -1,203 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# -# Copyright (C) 2022 Muhammad Rizki -# Copyright (C) 2022 Ammar Faizi -# - -from datetime import datetime -import mysql - - -class Database: - def __init__(self, conn): - self.conn = conn - self.conn.autocommit = True - self.cur = self.conn.cursor(buffered=True) - - - def __del__(self): - self.cur.close() - self.conn.close() - - - def save_email_msg_id(self, email_msg_id): - try: - return self.__save_email_msg_id(email_msg_id) - except mysql.connector.errors.IntegrityError: - # - # Duplicate data, skip! - # - return None - - - def __save_email_msg_id(self, email_msg_id): - q = "INSERT INTO emails (message_id, created_at) VALUES (%s, %s)" - self.cur.execute(q, (email_msg_id, datetime.utcnow())) - return self.cur.lastrowid - - - def insert_discord(self, email_id, channel_id, dc_msg_id): - q = """ - INSERT INTO discord_emails - (email_id, channel_id, dc_msg_id, created_at) - VALUES (%s, %s, %s, %s); - """ - self.cur.execute(q, (email_id, channel_id, dc_msg_id, - datetime.utcnow())) - return self.cur.lastrowid - - - # - # Determine whether the email needs to be sent to @dc_chat_id. - # - # - Return an email id (PK) if it needs to be sent. - # - Return None if it doesn't need to be sent. - # - def get_email_id_sent(self, email_msg_id, channel_id): - q = """ - SELECT emails.id, discord_emails.id FROM emails - LEFT JOIN discord_emails - ON emails.id = discord_emails.email_id - WHERE emails.message_id = %(email_msg_id)s - AND discord_emails.channel_id = %(channel_id)s - LIMIT 1 - """ - - self.cur.execute( - q, - { - "email_msg_id": email_msg_id, - "channel_id": channel_id - } - ) - res = self.cur.fetchone() - if bool(res): - # - # This email has already been sent to - # @dc_chat_id. - # - return None - - q = """ - SELECT id FROM emails WHERE message_id = %(email_msg_id)s - """ - self.cur.execute(q, {"email_msg_id": email_msg_id}) - res = self.cur.fetchone() - if not bool(res): - # - # Something goes wrong, skip! - # - return None - - return int(res[0]) - - - def get_discord_reply(self, email_msg_id, channel_id): - q = """ - SELECT discord_emails.dc_msg_id - FROM emails INNER JOIN discord_emails - ON emails.id = discord_emails.email_id - WHERE emails.message_id = %(email_msg_id)s - AND discord_emails.channel_id = %(channel_id)s - """ - - self.cur.execute( - q, - { - "email_msg_id": email_msg_id, - "channel_id": channel_id - } - ) - res = self.cur.fetchone() - if not bool(res): - return None - - return res[0] - - - def insert_atom(self, atom: str): - try: - return self.__save_atom(atom) - except mysql.connector.errors.IntegrityError: - # - # Duplicate data, skip! - # - return None - - - def __save_atom(self, atom: str): - q = "INSERT INTO atom_urls (url, created_at) VALUES (%s, %s)" - self.cur.execute(q, (atom, datetime.utcnow())) - return self.cur.lastrowid - - - def delete_atom(self, atom: str): - q = """ - DELETE FROM atom_urls - WHERE url = %(atom)s - """ - self.cur.execute(q, {"atom": atom}) - return self.cur.rowcount > 0 - - - def get_atom_urls(self): - q = """ - SELECT atom_urls.url - FROM atom_urls - """ - self.cur.execute(q) - urls = self.cur.fetchall() - - return [u[0] for u in urls] - - - def insert_broadcast( - self, - guild_id: int, - channel_id: int, - channel_name: str, - channel_link: str = None, - ): - try: - return self.__save_broadcast( - guild_id=guild_id, - channel_id=channel_id, - channel_name=channel_name, - channel_link=channel_link - ) - except mysql.connector.errors.IntegrityError: - # - # Duplicate data, skip! - # - return None - - - def __save_broadcast( - self, - guild_id: int, - channel_id: int, - channel_name: str, - channel_link: str = None, - ): - q = """ - INSERT INTO broadcast_chats - (guild_id, channel_id, channel_name, channel_link, created_at) - VALUES (%s, %s, %s, %s, %s) - """ - values = (guild_id, channel_id, channel_name, channel_link, datetime.utcnow()) - self.cur.execute(q, values) - return self.cur.lastrowid - - - def delete_broadcast(self, channel_id: int): - q = """ - DELETE FROM broadcast_chats - WHERE channel_id = %(channel_id)s - """ - self.cur.execute(q, {"channel_id": channel_id}) - return self.cur.rowcount > 0 - - - def get_broadcast_chats(self): - self.cur.execute("SELECT * FROM broadcast_chats") - return self.cur.fetchall() diff --git a/daemon/discord/mailer/utils.py b/daemon/discord/mailer/utils.py deleted file mode 100644 index 0036f24..0000000 --- a/daemon/discord/mailer/utils.py +++ /dev/null @@ -1,241 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# -# Copyright (C) 2022 Muhammad Rizki -# Copyright (C) 2022 Ammar Faizi -# - -from email.message import Message -from typing import Dict -from slugify import slugify -import hashlib -import uuid -import os -import re -import shutil -import httpx - - -def get_email_msg_id(mail): - ret = mail.get("message-id") - if not ret: - return None - - ret = re.search(r"<([^\<\>]+)>", ret) - if not ret: - return None - - return ret.group(1) - - -# -# This increments the @i while we are seeing a whitespace. -# -def __skip_whitespace(i, ss_len, ss): - while i < ss_len: - c = ss[i] - if c != ' ' and c != '\t' and c != '\n': - break - i += 1 - - return i - - -# -# Pick a single element in the list. The delimiter here is -# a comma char ','. But note that when are inside a double -# quotes, we must not take the comma as a delimiter. -# -def __pick_element(i, ss_len, ss, ret): - acc = "" - in_quotes = False - - while i < ss_len: - c = ss[i] - i += 1 - - if c == '"': - in_quotes = (not in_quotes) - - if not in_quotes and c == ',': - break - - acc += c - - if acc != "": - ret.append(acc) - - return i - - -def __extract_list(ss): - ss = ss.strip() - ss_len = len(ss) - ret = [] - i = 0 - - while i < ss_len: - i = __skip_whitespace(i, ss_len, ss) - i = __pick_element(i, ss_len, ss, ret) - - return ret - - -def extract_list(key: str, content: Dict[str, str]): - people = content.get(key.lower()) - if not people: - return [] - return __extract_list(people) - - -def consruct_to_n_cc(to: list, cc: list): - NR_MAX_LIST = 20 - - n = 0 - ret = "" - for i in to: - if n >= NR_MAX_LIST: - ret += "To: ...\n" - break - - n += 1 - ret += f"To: {i}\n" - - for i in cc: - if n >= NR_MAX_LIST: - ret += "Cc: ...\n" - break - - n += 1 - ret += f"Cc: {i}\n" - - return ret - - -def gen_temp(name: str): - md5 = hashlib.md5(name.encode()).hexdigest() - ret = os.getenv("STORAGE_DIR", "storage") + "/" + md5 - try: - os.mkdir(ret) - except FileExistsError: - pass - - return ret - - -def extract_body(thread: Message): - if not thread.is_multipart(): - p = thread.get_payload(decode=True) - return f"{p.decode(errors='replace')}\n".lstrip(), [] - - ret = "" - files = [] - temp = gen_temp(str(uuid.uuid4())) - for p in thread.get_payload(): - fname = p.get_filename() - payload = p.get_payload(decode=True) - - if not payload: - continue - - if 'inline' in [p.get('content-disposition')] or not bool(fname): - ret += f"{payload.decode(errors='replace')}\n".lstrip() - continue - - with open(f"{temp}/{fname}", "wb") as f: - f.write(payload) - files.append((temp, fname)) - - ret = re.sub("^(>)", ">>> \\1", ret, 1, re.MULTILINE) - return ret, files - - - -PATCH_PATTERN = r"^\[.*(?:patch|rfc).*?(?:(\d+)\/(\d+))?\](.+)" -def __is_patch(subject, content): - x = re.search(PATCH_PATTERN, subject, re.IGNORECASE) - if not x or x.group(1) == "0": - return False - - x = re.search(r"diff --git", content) - if not x: - return False - - return True - - -def create_template(thread: Message, to=None, cc=None): - if not to: - to = extract_list("to", thread) - if not cc: - cc = extract_list("cc", thread) - - subject = thread.get('subject') - ret = f"From: {thread.get('from')}\n" - ret += consruct_to_n_cc(to, cc) - ret += f"Date: {thread.get('date')}\n" - ret += f"Subject: {subject}\n\n" - content, files = extract_body(thread) - is_patch = __is_patch(subject, content) - - if is_patch: - ret += content - else: - ret += content.strip().replace("\t", " ") - if len(ret) >= 1900: - ret = ret[:1900] + "..." - - ret = fix_utf8_char(ret) - - return ret, files, is_patch - - -def prepare_patch(mail, text, url): - tmp = gen_temp(url) - fnm = str(mail.get("subject")) - sch = re.search(PATCH_PATTERN, fnm, re.IGNORECASE) - - nr_patch = sch.group(1) - if not nr_patch: - nr_patch = 1 - else: - nr_patch = int(nr_patch) - - num = "%04d" % nr_patch - fnm = slugify(sch.group(3)).replace("_", "-") - file = f"{tmp}/{num}-{fnm}.patch" - - with open(file, "wb") as f: - f.write(bytes(text, encoding="utf8")) - - caption = "#patch #ml" - return tmp, file, caption, url - - -def remove_patch(tmp): - shutil.rmtree(tmp) - - -def fix_utf8_char(text: str): - return text.rstrip().replace("�"," ") - - -def bottom_border(text: str): - return text + "\n" + "-"*72 - - -EMAIL_MSG_ID_PATTERN = r"<([^\<\>]+)>" -def extract_email_msg_id(msg_id): - ret = re.search(EMAIL_MSG_ID_PATTERN, msg_id) - if not ret: - return None - return ret.group(1) - - -async def is_atom_url(text: str): - try: - async with httpx.AsyncClient() as ses: - res = await ses.get(text) - mime = res.headers.get("Content-Type") - - return mime == "application/atom+xml" - except: return False diff --git a/daemon/discord/config.py.example b/daemon/dscord/config.py.example similarity index 100% rename from daemon/discord/config.py.example rename to daemon/dscord/config.py.example diff --git a/daemon/dscord/database/__init__.py b/daemon/dscord/database/__init__.py new file mode 100644 index 0000000..c5221d6 --- /dev/null +++ b/daemon/dscord/database/__init__.py @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Copyright (C) 2022 Muhammad Rizki +# + +from .core import DB diff --git a/daemon/dscord/database/core.py b/daemon/dscord/database/core.py new file mode 100644 index 0000000..cfd9cc2 --- /dev/null +++ b/daemon/dscord/database/core.py @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Copyright (C) 2022 Muhammad Rizki +# Copyright (C) 2022 Ammar Faizi +# + +from .methods import DBMethods + + +class DB(DBMethods): + def __init__(self, conn): + self.conn = conn + self.conn.autocommit = True + self.cur = self.conn.cursor(buffered=True) + + + def __del__(self): + self.cur.close() + self.conn.close() diff --git a/daemon/dscord/database/methods/__init__.py b/daemon/dscord/database/methods/__init__.py new file mode 100644 index 0000000..2c97652 --- /dev/null +++ b/daemon/dscord/database/methods/__init__.py @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Copyright (C) 2022 Muhammad Rizki +# + + +from .deletion import Deletion +from .getter import Getter +from .insertion import Insertion + + +class DBMethods( + Deletion, + Getter, + Insertion +): pass diff --git a/daemon/dscord/database/methods/deletion/__init__.py b/daemon/dscord/database/methods/deletion/__init__.py new file mode 100644 index 0000000..fa80464 --- /dev/null +++ b/daemon/dscord/database/methods/deletion/__init__.py @@ -0,0 +1,8 @@ +from .delet_atom import DeleteAtom +from .delete_broadcast import DeleteBroadcast + + +class Deletion( + DeleteAtom, + DeleteBroadcast +): pass diff --git a/daemon/dscord/database/methods/deletion/delet_atom.py b/daemon/dscord/database/methods/deletion/delet_atom.py new file mode 100644 index 0000000..95496a2 --- /dev/null +++ b/daemon/dscord/database/methods/deletion/delet_atom.py @@ -0,0 +1,12 @@ + + + +class DeleteAtom: + + def delete_atom(self, atom: str): + q = """ + DELETE FROM dc_atoms + WHERE url = %(atom)s + """ + self.cur.execute(q, {"atom": atom}) + return self.cur.rowcount > 0 diff --git a/daemon/dscord/database/methods/deletion/delete_broadcast.py b/daemon/dscord/database/methods/deletion/delete_broadcast.py new file mode 100644 index 0000000..65cb2a4 --- /dev/null +++ b/daemon/dscord/database/methods/deletion/delete_broadcast.py @@ -0,0 +1,12 @@ + + + +class DeleteBroadcast: + + def delete_broadcast(self, channel_id: int): + q = """ + DELETE FROM dc_broadcasts + WHERE channel_id = %(channel_id)s + """ + self.cur.execute(q, {"channel_id": channel_id}) + return self.cur.rowcount > 0 diff --git a/daemon/dscord/database/methods/getter/__init__.py b/daemon/dscord/database/methods/getter/__init__.py new file mode 100644 index 0000000..fd58e84 --- /dev/null +++ b/daemon/dscord/database/methods/getter/__init__.py @@ -0,0 +1,12 @@ +from .get_atom_urls import GetAtomURL +from .get_broadcast_chats import GetBroadcastChats +from .get_email_id import GetEmailID +from .get_reply import GetDiscordReply + + +class Getter( + GetAtomURL, + GetBroadcastChats, + GetEmailID, + GetDiscordReply +): pass diff --git a/daemon/dscord/database/methods/getter/get_atom_urls.py b/daemon/dscord/database/methods/getter/get_atom_urls.py new file mode 100644 index 0000000..b02c48c --- /dev/null +++ b/daemon/dscord/database/methods/getter/get_atom_urls.py @@ -0,0 +1,18 @@ + + + +class GetAtomURL: + + def get_atom_urls(self): + ''' + Get lore kernel raw email URLs. + - Return list of raw email URLs: `List[str]` + ''' + q = """ + SELECT dc_atoms.url + FROM dc_atoms + """ + self.cur.execute(q) + urls = self.cur.fetchall() + + return [u[0] for u in urls] diff --git a/daemon/dscord/database/methods/getter/get_broadcast_chats.py b/daemon/dscord/database/methods/getter/get_broadcast_chats.py new file mode 100644 index 0000000..84fcac0 --- /dev/null +++ b/daemon/dscord/database/methods/getter/get_broadcast_chats.py @@ -0,0 +1,18 @@ + + + +class GetBroadcastChats: + + def get_broadcast_chats(self): + ''' + Get broadcast chats that are currently + listening for new email. + - Return list of chat object: `List[Object]` + ''' + q = """ + SELECT * + FROM dc_broadcasts + """ + self.cur.execute(q) + + return self.cur.fetchall() diff --git a/daemon/dscord/database/methods/getter/get_email_id.py b/daemon/dscord/database/methods/getter/get_email_id.py new file mode 100644 index 0000000..46beac9 --- /dev/null +++ b/daemon/dscord/database/methods/getter/get_email_id.py @@ -0,0 +1,56 @@ + + + +class GetEmailID: + + def get_email_id(self, email_id, chat_id): + ''' + Determine whether the email needs to be sent to @tg_chat_id. + - Return an email id (PK) if it needs to be sent + - Return None if it doesn't need to be sent + ''' + if self.__is_sent(email_id, chat_id): + return + + res = self.__email_id(email_id) + if not bool(res): + return + + return int(res[0]) + + + def __is_sent(self, email_id, channel_id): + ''' + Checking if this email has already been sent + or not. + - Return True if it's already been sent + ''' + q = """ + SELECT dc_emails.id, dc_mail_msg.id FROM dc_emails + LEFT JOIN dc_mail_msg + ON dc_emails.id = dc_mail_msg.email_id + WHERE dc_emails.message_id = %(email_msg_id)s + AND dc_mail_msg.channel_id = %(channel_id)s + LIMIT 1 + """ + + self.cur.execute(q, { + "email_msg_id": email_id, + "channel_id": channel_id + }) + res = self.cur.fetchone() + return bool(res) + + + def __email_id(self, email_id): + ''' + Get the email id if match with the email message_id. + - Return the result if it's match and exists + ''' + q = """ + SELECT id FROM dc_emails WHERE message_id = %(email_id)s + """ + + self.cur.execute(q, {"email_id": email_id}) + res = self.cur.fetchone() + return res diff --git a/daemon/dscord/database/methods/getter/get_reply.py b/daemon/dscord/database/methods/getter/get_reply.py new file mode 100644 index 0000000..e363963 --- /dev/null +++ b/daemon/dscord/database/methods/getter/get_reply.py @@ -0,0 +1,29 @@ + + + +class GetDiscordReply: + + def get_reply_id(self, email_id, channel_id): + ''' + Get Telegram message ID sent match with + email message ID and Telegram chat ID. + - Return Telegram message ID if exists: `int` + - Return None if not exists` + ''' + q = """ + SELECT dc_mail_msg.dc_msg_id + FROM dc_emails INNER JOIN dc_mail_msg + ON dc_emails.id = dc_mail_msg.email_id + WHERE dc_emails.message_id = %(email_msg_id)s + AND dc_mail_msg.channel_id = %(channel_id)s + """ + + self.cur.execute(q, { + "email_msg_id": email_id, + "channel_id": channel_id + }) + res = self.cur.fetchone() + if not bool(res): + return None + + return res[0] diff --git a/daemon/dscord/database/methods/insertion/__init__.py b/daemon/dscord/database/methods/insertion/__init__.py new file mode 100644 index 0000000..46b6e05 --- /dev/null +++ b/daemon/dscord/database/methods/insertion/__init__.py @@ -0,0 +1,12 @@ +from .insert_atom import InsertAtom +from .insert_broadcast import InsertBroadcast +from .insert_email import InsertEmail +from .insert_discord import InsertDiscord + + +class Insertion( + InsertAtom, + InsertBroadcast, + InsertEmail, + InsertDiscord +): pass diff --git a/daemon/dscord/database/methods/insertion/insert_atom.py b/daemon/dscord/database/methods/insertion/insert_atom.py new file mode 100644 index 0000000..a69cde7 --- /dev/null +++ b/daemon/dscord/database/methods/insertion/insert_atom.py @@ -0,0 +1,20 @@ +from mysql.connector import errors +from datetime import datetime + + +class InsertAtom: + + def save_atom(self, atom: str): + try: + return self.__insert_atom(atom) + except errors.IntegrityError: + # + # Duplicate data, skip! + # + return None + + + def __insert_atom(self, atom: str): + q = "INSERT INTO dc_atoms (url, created_at) VALUES (%s, %s)" + self.cur.execute(q, (atom, datetime.utcnow())) + return self.cur.lastrowid diff --git a/daemon/dscord/database/methods/insertion/insert_broadcast.py b/daemon/dscord/database/methods/insertion/insert_broadcast.py new file mode 100644 index 0000000..e68f9b4 --- /dev/null +++ b/daemon/dscord/database/methods/insertion/insert_broadcast.py @@ -0,0 +1,42 @@ +from mysql.connector import errors +from datetime import datetime + + +class InsertBroadcast: + + def save_broadcast( + self, + guild_id: int, + channel_id: int, + channel_name: str, + channel_link: str = None, + ): + try: + return self.__insert_broadcast( + guild_id=guild_id, + channel_id=channel_id, + channel_name=channel_name, + channel_link=channel_link + ) + except errors.IntegrityError: + # + # Duplicate data, skip! + # + return None + + + def __insert_broadcast( + self, + guild_id: int, + channel_id: int, + channel_name: str, + channel_link: str = None, + ): + q = """ + INSERT INTO dc_broadcasts + (guild_id, channel_id, channel_name, channel_link, created_at) + VALUES (%s, %s, %s, %s, %s) + """ + values = (guild_id, channel_id, channel_name, channel_link, datetime.utcnow()) + self.cur.execute(q, values) + return self.cur.lastrowid diff --git a/daemon/dscord/database/methods/insertion/insert_discord.py b/daemon/dscord/database/methods/insertion/insert_discord.py new file mode 100644 index 0000000..bb423fd --- /dev/null +++ b/daemon/dscord/database/methods/insertion/insert_discord.py @@ -0,0 +1,14 @@ +from datetime import datetime + + +class InsertDiscord: + + def save_discord_mail(self, email_id, channel_id, dc_msg_id): + q = """ + INSERT INTO dc_mail_msg + (email_id, channel_id, dc_msg_id, created_at) + VALUES (%s, %s, %s, %s); + """ + self.cur.execute(q, (email_id, channel_id, dc_msg_id, + datetime.utcnow())) + return self.cur.lastrowid diff --git a/daemon/dscord/database/methods/insertion/insert_email.py b/daemon/dscord/database/methods/insertion/insert_email.py new file mode 100644 index 0000000..0fab378 --- /dev/null +++ b/daemon/dscord/database/methods/insertion/insert_email.py @@ -0,0 +1,20 @@ +from mysql.connector import errors +from datetime import datetime + + +class InsertEmail: + + def save_email(self, email_msg_id): + try: + return self.__insert_email(email_msg_id) + except errors.IntegrityError: + # + # Duplicate data, skip! + # + return None + + + def __insert_email(self, email_msg_id): + q = "INSERT INTO dc_emails (message_id, created_at) VALUES (%s, %s)" + self.cur.execute(q, (email_msg_id, datetime.utcnow())) + return self.cur.lastrowid diff --git a/daemon/discord/gnuweeb/__init__.py b/daemon/dscord/gnuweeb/__init__.py similarity index 100% rename from daemon/discord/gnuweeb/__init__.py rename to daemon/dscord/gnuweeb/__init__.py diff --git a/daemon/discord/gnuweeb/client.py b/daemon/dscord/gnuweeb/client.py similarity index 84% rename from daemon/discord/gnuweeb/client.py rename to daemon/dscord/gnuweeb/client.py index cf88d36..c308f5a 100644 --- a/daemon/discord/gnuweeb/client.py +++ b/daemon/dscord/gnuweeb/client.py @@ -11,13 +11,13 @@ from typing import Union from .models.ui import buttons from . import filters -from mailer import utils -from mailer import Database +from atom import utils +from dscord.database import DB class GWClient(commands.Bot): def __init__(self, db_conn) -> None: - self.db = Database(db_conn) + self.db = DB(db_conn) self.mailer = None intents = Intents.default() intents.message_content = True @@ -33,7 +33,10 @@ class GWClient(commands.Bot): async def setup_hook(self): - await self.load_extension("gnuweeb.plugins") + await self.load_extension( + name=".gnuweeb.plugins", + package="dscord" + ) # WARNING! NOT RECOMMENDED SYNCING WHEN THE BOT IS START!! # guild = discord.Object(id=845302963739033611) @@ -45,27 +48,26 @@ class GWClient(commands.Bot): async def send_text_email(self, guild_id: int, chat_id: int, text: str, reply_to: Union[int, None] = None, url: str = None): print("[send_text_email]") - text = utils.bottom_border(text) channel = self.get_channel(chat_id) - m = await channel.send( + return await channel.send( content=text, reference=discord.MessageReference( guild_id=guild_id, channel_id=chat_id, message_id=reply_to ) if reply_to else None, - view=buttons.FullMessageBtn(url) ) - return m @filters.wait_on_limit async def send_patch_email(self, mail, guild_id: int, chat_id: int, text: str, reply_to: Union[int, None] = None, url: str = None): print("[send_patch_email]") - tmp, doc, caption, url = utils.prepare_patch(mail, text, url) + tmp, doc, caption, url = utils.prepare_patch( + mail, text, url, "discord" + ) channel = self.get_channel(chat_id) m = await channel.send( @@ -86,25 +88,21 @@ class GWClient(commands.Bot): async def send_text_mail_interaction(self, i: "Interaction", text: str, url: str = None): - text = utils.border_and_trim(text) - - m = await i.response.send_message( + return await i.response.send_message( content=text, view=buttons.FullMessageBtn(url) ) - return m - async def send_patch_mail_interaction(self, mail, i: "Interaction", text: str, url: str = None): - tmp, doc, caption, url = utils.prepare_patch(mail, text, url) - + tmp, doc, caption, url = utils.prepare_patch( + mail, text, url, "discord" + ) m = await i.response.send_message( content=caption, file=discord.File(doc), view=buttons.FullMessageBtn(url) ) - utils.remove_patch(tmp) return m diff --git a/daemon/discord/gnuweeb/filters.py b/daemon/dscord/gnuweeb/filters.py similarity index 98% rename from daemon/discord/gnuweeb/filters.py rename to daemon/dscord/gnuweeb/filters.py index 735c77e..76bfa5d 100644 --- a/daemon/discord/gnuweeb/filters.py +++ b/daemon/dscord/gnuweeb/filters.py @@ -3,7 +3,7 @@ # Copyright (C) 2022 Muhammad Rizki # -import config +from dscord import config import discord import asyncio from discord import Interaction diff --git a/daemon/discord/gnuweeb/models/__init__.py b/daemon/dscord/gnuweeb/models/__init__.py similarity index 100% rename from daemon/discord/gnuweeb/models/__init__.py rename to daemon/dscord/gnuweeb/models/__init__.py diff --git a/daemon/discord/gnuweeb/models/ui/__init__.py b/daemon/dscord/gnuweeb/models/ui/__init__.py similarity index 100% rename from daemon/discord/gnuweeb/models/ui/__init__.py rename to daemon/dscord/gnuweeb/models/ui/__init__.py diff --git a/daemon/discord/gnuweeb/models/ui/buttons/__init__.py b/daemon/dscord/gnuweeb/models/ui/buttons/__init__.py similarity index 100% rename from daemon/discord/gnuweeb/models/ui/buttons/__init__.py rename to daemon/dscord/gnuweeb/models/ui/buttons/__init__.py diff --git a/daemon/discord/gnuweeb/models/ui/buttons/full_message_btn.py b/daemon/dscord/gnuweeb/models/ui/buttons/full_message_btn.py similarity index 100% rename from daemon/discord/gnuweeb/models/ui/buttons/full_message_btn.py rename to daemon/dscord/gnuweeb/models/ui/buttons/full_message_btn.py diff --git a/daemon/discord/gnuweeb/plugins/__init__.py b/daemon/dscord/gnuweeb/plugins/__init__.py similarity index 100% rename from daemon/discord/gnuweeb/plugins/__init__.py rename to daemon/dscord/gnuweeb/plugins/__init__.py diff --git a/daemon/discord/gnuweeb/plugins/basic_commands/__init__.py b/daemon/dscord/gnuweeb/plugins/basic_commands/__init__.py similarity index 100% rename from daemon/discord/gnuweeb/plugins/basic_commands/__init__.py rename to daemon/dscord/gnuweeb/plugins/basic_commands/__init__.py diff --git a/daemon/discord/gnuweeb/plugins/basic_commands/debugger.py b/daemon/dscord/gnuweeb/plugins/basic_commands/debugger.py similarity index 94% rename from daemon/discord/gnuweeb/plugins/basic_commands/debugger.py rename to daemon/dscord/gnuweeb/plugins/basic_commands/debugger.py index be5b9f1..0bde454 100644 --- a/daemon/discord/gnuweeb/plugins/basic_commands/debugger.py +++ b/daemon/dscord/gnuweeb/plugins/basic_commands/debugger.py @@ -4,7 +4,7 @@ # from discord.ext import commands -from gnuweeb import utils +from dscord.gnuweeb import utils # from gnuweeb import filters diff --git a/daemon/discord/gnuweeb/plugins/basic_commands/sync_it.py b/daemon/dscord/gnuweeb/plugins/basic_commands/sync_it.py similarity index 70% rename from daemon/discord/gnuweeb/plugins/basic_commands/sync_it.py rename to daemon/dscord/gnuweeb/plugins/basic_commands/sync_it.py index 2ed35cc..fbaec03 100644 --- a/daemon/discord/gnuweeb/plugins/basic_commands/sync_it.py +++ b/daemon/dscord/gnuweeb/plugins/basic_commands/sync_it.py @@ -7,14 +7,14 @@ from discord.ext import commands class SyncCommand(commands.Cog): - def __init__(self, bot) -> None: + def __init__(self, bot: "commands.Bot") -> None: self.bot = bot @commands.command("sync", aliases=["s"]) @commands.is_owner() async def sync_it(self, ctx: "commands.Context"): - ctx.bot.tree.copy_global_to(guild=ctx.guild) - s = await ctx.bot.tree.sync(guild=ctx.guild) + self.bot.tree.copy_global_to(guild=ctx.guild) + s = await self.bot.tree.sync(guild=ctx.guild) await ctx.send(f"Synced {len(s)} commands.") diff --git a/daemon/discord/gnuweeb/plugins/events/__init__.py b/daemon/dscord/gnuweeb/plugins/events/__init__.py similarity index 100% rename from daemon/discord/gnuweeb/plugins/events/__init__.py rename to daemon/dscord/gnuweeb/plugins/events/__init__.py diff --git a/daemon/discord/gnuweeb/plugins/events/on_error.py b/daemon/dscord/gnuweeb/plugins/events/on_error.py similarity index 100% rename from daemon/discord/gnuweeb/plugins/events/on_error.py rename to daemon/dscord/gnuweeb/plugins/events/on_error.py diff --git a/daemon/discord/gnuweeb/plugins/events/on_ready.py b/daemon/dscord/gnuweeb/plugins/events/on_ready.py similarity index 100% rename from daemon/discord/gnuweeb/plugins/events/on_ready.py rename to daemon/dscord/gnuweeb/plugins/events/on_ready.py diff --git a/daemon/discord/gnuweeb/plugins/slash_commands/__init__.py b/daemon/dscord/gnuweeb/plugins/slash_commands/__init__.py similarity index 100% rename from daemon/discord/gnuweeb/plugins/slash_commands/__init__.py rename to daemon/dscord/gnuweeb/plugins/slash_commands/__init__.py diff --git a/daemon/discord/gnuweeb/plugins/slash_commands/get_lore_mail.py b/daemon/dscord/gnuweeb/plugins/slash_commands/get_lore_mail.py similarity index 95% rename from daemon/discord/gnuweeb/plugins/slash_commands/get_lore_mail.py rename to daemon/dscord/gnuweeb/plugins/slash_commands/get_lore_mail.py index a2671f4..38eb465 100644 --- a/daemon/discord/gnuweeb/plugins/slash_commands/get_lore_mail.py +++ b/daemon/dscord/gnuweeb/plugins/slash_commands/get_lore_mail.py @@ -10,8 +10,8 @@ from discord.ext import commands from discord import Interaction from discord import app_commands -from mailer import utils -from mailer import Scraper +from atom import utils +from atom import Scraper class GetLoreSC(commands.Cog): diff --git a/daemon/discord/gnuweeb/plugins/slash_commands/manage_atom.py b/daemon/dscord/gnuweeb/plugins/slash_commands/manage_atom.py similarity index 95% rename from daemon/discord/gnuweeb/plugins/slash_commands/manage_atom.py rename to daemon/dscord/gnuweeb/plugins/slash_commands/manage_atom.py index 2d9da80..90b19de 100644 --- a/daemon/discord/gnuweeb/plugins/slash_commands/manage_atom.py +++ b/daemon/dscord/gnuweeb/plugins/slash_commands/manage_atom.py @@ -7,8 +7,8 @@ from discord.ext import commands from discord import Interaction from discord import app_commands -from gnuweeb import filters -from mailer import utils +from dscord.gnuweeb import filters +from atom import utils class ManageAtomSC(commands.Cog): @@ -53,7 +53,7 @@ class ManageAtomSC(commands.Cog): await i.response.send_message(t, ephemeral=True) return - inserted = self.bot.db.insert_atom(url) + inserted = self.bot.db.save_atom(url) if inserted is None: t = f"This URL already listened for new email." await i.response.send_message(t, ephemeral=True) diff --git a/daemon/discord/gnuweeb/plugins/slash_commands/manage_broadcast.py b/daemon/dscord/gnuweeb/plugins/slash_commands/manage_broadcast.py similarity index 95% rename from daemon/discord/gnuweeb/plugins/slash_commands/manage_broadcast.py rename to daemon/dscord/gnuweeb/plugins/slash_commands/manage_broadcast.py index 9eb6b98..7448e23 100644 --- a/daemon/discord/gnuweeb/plugins/slash_commands/manage_broadcast.py +++ b/daemon/dscord/gnuweeb/plugins/slash_commands/manage_broadcast.py @@ -7,8 +7,8 @@ from discord.ext import commands from discord import Interaction from discord import app_commands -from gnuweeb import utils -from gnuweeb import filters +from dscord.gnuweeb import utils +from dscord.gnuweeb import filters class ManageBroadcastSC(commands.Cog): @@ -47,7 +47,7 @@ class ManageBroadcastSC(commands.Cog): ) @filters.lore_admin async def add_channel(self, i: "Interaction"): - inserted = self.bot.db.insert_broadcast( + inserted = self.bot.db.save_broadcast( guild_id=i.guild_id, channel_id=i.channel_id, channel_name=i.channel.name, diff --git a/daemon/discord/gnuweeb/utils.py b/daemon/dscord/gnuweeb/utils.py similarity index 100% rename from daemon/discord/gnuweeb/utils.py rename to daemon/dscord/gnuweeb/utils.py diff --git a/daemon/dscord/mailer/__init__.py b/daemon/dscord/mailer/__init__.py new file mode 100644 index 0000000..77478d0 --- /dev/null +++ b/daemon/dscord/mailer/__init__.py @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Copyright (C) 2022 Muhammad Rizki +# + +from .listener import Listener diff --git a/daemon/discord/mailer/listener.py b/daemon/dscord/mailer/listener.py similarity index 85% rename from daemon/discord/mailer/listener.py rename to daemon/dscord/mailer/listener.py index 523a87d..7538ba9 100644 --- a/daemon/discord/mailer/listener.py +++ b/daemon/dscord/mailer/listener.py @@ -11,14 +11,10 @@ import re from apscheduler.schedulers.asyncio import AsyncIOScheduler from discord import File -from gnuweeb import GWClient -from .scraper import Scraper -from . import utils - - -class BotMutexes(): - def __init__(self): - self.lock = asyncio.Lock() +from dscord.gnuweeb import GWClient +from atom.utils import Mutexes +from atom.scraper import Scraper +from atom import utils class Listener(): @@ -26,8 +22,8 @@ class Listener(): self, client: "GWClient", sched: "AsyncIOScheduler", - scraper: Scraper, - mutexes: "BotMutexes" + scraper: "Scraper", + mutexes: "Mutexes" ): self.client = client self.sched = sched @@ -106,7 +102,7 @@ class Listener(): # return False - text, files, is_patch = utils.create_template(mail) + text, files, is_patch = utils.create_template(mail, "discord") reply_to = self.get_discord_reply(mail, dc_chat_id) url = str(re.sub(r"/raw$", "", url)) @@ -120,7 +116,7 @@ class Listener(): dc_guild_id, dc_chat_id, text, reply_to, url ) - self.db.insert_discord(email_id, m.channel.id, m.id) + self.db.save_discord_mail(email_id, m.channel.id, m.id) for d, f in files: await m.reply(f"{d}/{f}", file=File(f)) await asyncio.sleep(1) @@ -132,11 +128,11 @@ class Listener(): def __get_email_id_sent(self, email_msg_id, dc_chat_id): - email_id = self.db.save_email_msg_id(email_msg_id) + email_id = self.db.save_email(email_msg_id) if email_id: return email_id - email_id = self.db.get_email_id_sent(email_msg_id, dc_chat_id) + email_id = self.db.get_email_id(email_msg_id, dc_chat_id) return email_id @@ -149,4 +145,4 @@ class Listener(): if not reply_to: return None - return self.db.get_discord_reply(reply_to, dc_chat_id) + return self.db.get_reply_id(reply_to, dc_chat_id) diff --git a/daemon/discord/requirements.txt b/daemon/dscord/requirements.txt similarity index 100% rename from daemon/discord/requirements.txt rename to daemon/dscord/requirements.txt diff --git a/daemon/discord/storage/.gitignore b/daemon/dscord/storage/.gitignore similarity index 100% rename from daemon/discord/storage/.gitignore rename to daemon/dscord/storage/.gitignore diff --git a/daemon/telegram/.env.example b/daemon/telegram.env.example similarity index 100% rename from daemon/telegram/.env.example rename to daemon/telegram.env.example diff --git a/daemon/telegram/database/__init__.py b/daemon/telegram/database/__init__.py new file mode 100644 index 0000000..930e3d9 --- /dev/null +++ b/daemon/telegram/database/__init__.py @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Copyright (C) 2022 Muhammad Rizki +# + + +from .core import DB diff --git a/daemon/telegram/database/core.py b/daemon/telegram/database/core.py new file mode 100644 index 0000000..c34d7a8 --- /dev/null +++ b/daemon/telegram/database/core.py @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Copyright (C) 2022 Muhammad Rizki +# Copyright (C) 2022 Ammar Faizi +# + + +from .methods import DBMethods + + +class DB(DBMethods): + def __init__(self, conn): + self.conn = conn + self.conn.autocommit = True + self.cur = self.conn.cursor(buffered=True) + + + def __del__(self): + self.cur.close() + self.conn.close() diff --git a/daemon/telegram/database/methods/__init__.py b/daemon/telegram/database/methods/__init__.py new file mode 100644 index 0000000..961b4e0 --- /dev/null +++ b/daemon/telegram/database/methods/__init__.py @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Copyright (C) 2022 Muhammad Rizki +# Copyright (C) 2022 Ammar Faizi +# + + +from .deletion import Deletion +from .getter import Getter +from .insertion import Insertion + + +class DBMethods( + Deletion, + Getter, + Insertion +): pass diff --git a/daemon/telegram/database/methods/deletion/__init__.py b/daemon/telegram/database/methods/deletion/__init__.py new file mode 100644 index 0000000..b206929 --- /dev/null +++ b/daemon/telegram/database/methods/deletion/__init__.py @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Copyright (C) 2022 Muhammad Rizki +# + + +from .delet_atom import DeleteAtom +from .delete_broadcast import DeleteBroadcast + + +class Deletion( + DeleteAtom, + DeleteBroadcast +): pass diff --git a/daemon/telegram/database/methods/deletion/delet_atom.py b/daemon/telegram/database/methods/deletion/delet_atom.py new file mode 100644 index 0000000..d8ad4bf --- /dev/null +++ b/daemon/telegram/database/methods/deletion/delet_atom.py @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Copyright (C) 2022 Muhammad Rizki +# + + +class DeleteAtom: + + def delete_atom(self, atom: str): + q = """ + DELETE FROM tg_atoms + WHERE url = %(atom)s + """ + self.cur.execute(q, {"atom": atom}) + return self.cur.rowcount > 0 diff --git a/daemon/telegram/database/methods/deletion/delete_broadcast.py b/daemon/telegram/database/methods/deletion/delete_broadcast.py new file mode 100644 index 0000000..d076dec --- /dev/null +++ b/daemon/telegram/database/methods/deletion/delete_broadcast.py @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Copyright (C) 2022 Muhammad Rizki +# + + +class DeleteBroadcast: + + def delete_broadcast(self, chat_id: int): + q = """ + DELETE FROM tg_broadcasts + WHERE chat_id = %(chat_id)s + """ + self.cur.execute(q, {"chat_id": chat_id}) + return self.cur.rowcount > 0 diff --git a/daemon/telegram/database/methods/getter/__init__.py b/daemon/telegram/database/methods/getter/__init__.py new file mode 100644 index 0000000..e978c72 --- /dev/null +++ b/daemon/telegram/database/methods/getter/__init__.py @@ -0,0 +1,18 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Copyright (C) 2022 Muhammad Rizki +# + + +from .get_atom_urls import GetAtomURL +from .get_broadcast_chats import GetBroadcastChats +from .get_email_id import GetEmailID +from .get_telegram_reply import GetTelegramReply + + +class Getter( + GetAtomURL, + GetBroadcastChats, + GetEmailID, + GetTelegramReply +): pass diff --git a/daemon/telegram/database/methods/getter/get_atom_urls.py b/daemon/telegram/database/methods/getter/get_atom_urls.py new file mode 100644 index 0000000..04a9315 --- /dev/null +++ b/daemon/telegram/database/methods/getter/get_atom_urls.py @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Copyright (C) 2022 Muhammad Rizki +# + + +class GetAtomURL: + + def get_atom_urls(self): + ''' + Get lore kernel raw email URLs. + - Return list of raw email URLs: `List[str]` + ''' + q = """ + SELECT tg_atoms.url + FROM tg_atoms + """ + self.cur.execute(q) + urls = self.cur.fetchall() + + return [u[0] for u in urls] diff --git a/daemon/telegram/database/methods/getter/get_broadcast_chats.py b/daemon/telegram/database/methods/getter/get_broadcast_chats.py new file mode 100644 index 0000000..d92e879 --- /dev/null +++ b/daemon/telegram/database/methods/getter/get_broadcast_chats.py @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Copyright (C) 2022 Muhammad Rizki +# + + +class GetBroadcastChats: + + def get_broadcast_chats(self): + ''' + Get broadcast chats that are currently + listening for new email. + - Return list of chat object: `List[Object]` + ''' + q = """ + SELECT * + FROM tg_broadcasts + """ + self.cur.execute(q) + + return self.cur.fetchall() diff --git a/daemon/telegram/database/methods/getter/get_email_id.py b/daemon/telegram/database/methods/getter/get_email_id.py new file mode 100644 index 0000000..509aa99 --- /dev/null +++ b/daemon/telegram/database/methods/getter/get_email_id.py @@ -0,0 +1,62 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Copyright (C) 2022 Muhammad Rizki +# Copyright (C) 2022 Ammar Faizi +# + + +class GetEmailID: + + def get_email_id(self, email_id, chat_id): + ''' + Determine whether the email needs to be sent to @tg_chat_id. + - Return an email id (PK) if it needs to be sent + - Return None if it doesn't need to be sent + ''' + if self.__is_sent(email_id, chat_id): + return + + res = self.__email_id(email_id) + if not bool(res): + return + + return int(res[0]) + + + def __is_sent(self, email_id, chat_id): + ''' + Checking if this email has already been sent + or not. + - Return True if it's already been sent + ''' + + q = """ + SELECT tg_emails.id, tg_mail_msg.id FROM tg_emails LEFT JOIN tg_mail_msg + ON tg_emails.id = tg_mail_msg.email_id + WHERE tg_emails.message_id = %(email_id)s + AND tg_mail_msg.chat_id = %(chat_id)s + LIMIT 1 + """ + + self.cur.execute(q, { + "email_id": email_id, + "chat_id": chat_id + }) + + res = self.cur.fetchone() + return bool(res) + + + def __email_id(self, email_id): + ''' + Get the email id if match with the email message_id. + - Return the result if it's match and exists + ''' + + q = """ + SELECT id FROM tg_emails WHERE message_id = %(email_id)s + """ + + self.cur.execute(q, {"email_id": email_id}) + res = self.cur.fetchone() + return res \ No newline at end of file diff --git a/daemon/telegram/database/methods/getter/get_telegram_reply.py b/daemon/telegram/database/methods/getter/get_telegram_reply.py new file mode 100644 index 0000000..94e3138 --- /dev/null +++ b/daemon/telegram/database/methods/getter/get_telegram_reply.py @@ -0,0 +1,33 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Copyright (C) 2022 Muhammad Rizki +# Copyright (C) 2022 Ammar Faizi +# + + +class GetTelegramReply: + + def get_reply_id(self, email_msg_id, chat_id): + ''' + Get Telegram message ID sent match with + email message ID and Telegram chat ID. + - Return Telegram message ID if exists: `int` + - Return None if not exists` + ''' + q = """ + SELECT tg_mail_msg.tg_msg_id + FROM tg_emails INNER JOIN tg_mail_msg + ON tg_emails.id = tg_mail_msg.email_id + WHERE tg_emails.message_id = %(email_msg_id)s + AND tg_mail_msg.chat_id = %(chat_id)s + """ + + self.cur.execute(q, { + "email_msg_id": email_msg_id, + "chat_id": chat_id + }) + res = self.cur.fetchone() + if not bool(res): + return None + + return res[0] diff --git a/daemon/telegram/database/methods/insertion/__init__.py b/daemon/telegram/database/methods/insertion/__init__.py new file mode 100644 index 0000000..3604f82 --- /dev/null +++ b/daemon/telegram/database/methods/insertion/__init__.py @@ -0,0 +1,18 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Copyright (C) 2022 Muhammad Rizki +# Copyright (C) 2022 Ammar Faizi +# + +from .insert_atom import InsertAtom +from .insert_broadcast import InsertBroadcast +from .insert_email import InsertEmail +from .insert_telegram import InsertTelegram + + +class Insertion( + InsertAtom, + InsertBroadcast, + InsertEmail, + InsertTelegram +): pass diff --git a/daemon/telegram/database/methods/insertion/insert_atom.py b/daemon/telegram/database/methods/insertion/insert_atom.py new file mode 100644 index 0000000..ac068ae --- /dev/null +++ b/daemon/telegram/database/methods/insertion/insert_atom.py @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Copyright (C) 2022 Muhammad Rizki +# Copyright (C) 2022 Ammar Faizi +# + + +from mysql.connector import errors +from datetime import datetime + + +class InsertAtom: + + def save_atom(self, atom: str): + try: + return self.__insert_atom(atom) + except errors.IntegrityError: + # + # Duplicate data, skip! + # + return None + + + def __insert_atom(self, atom: str): + q = "INSERT INTO tg_atoms (url, created_at) VALUES (%s, %s)" + self.cur.execute(q, (atom, datetime.utcnow())) + return self.cur.lastrowid diff --git a/daemon/telegram/database/methods/insertion/insert_broadcast.py b/daemon/telegram/database/methods/insertion/insert_broadcast.py new file mode 100644 index 0000000..11ebc6e --- /dev/null +++ b/daemon/telegram/database/methods/insertion/insert_broadcast.py @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Copyright (C) 2022 Muhammad Rizki +# Copyright (C) 2022 Ammar Faizi +# + + +from mysql.connector import errors +from datetime import datetime + + +class InsertBroadcast: + + def save_broadcast( + self, + chat_id: int, + name: str, + type: str, + created_at: "datetime", + username: str = None, + link: str = None, + ): + try: + return self.__insert_broadcast( + chat_id=chat_id, + name=name, + type=type, + created_at=created_at, + username=username, + link=link + ) + except errors.IntegrityError: + # + # Duplicate data, skip! + # + return None + + + def __insert_broadcast( + self, + chat_id: int, + name: str, + type: str, + created_at: "datetime", + username: str = None, + link: str = None, + ): + q = """ + INSERT INTO tg_broadcasts + (chat_id, username, name, type, link, created_at) + VALUES + (%s, %s, %s, %s, %s, %s) + """ + values = (chat_id, username, name, type, link, created_at) + self.cur.execute(q, values) + return self.cur.lastrowid diff --git a/daemon/telegram/database/methods/insertion/insert_email.py b/daemon/telegram/database/methods/insertion/insert_email.py new file mode 100644 index 0000000..adbe86b --- /dev/null +++ b/daemon/telegram/database/methods/insertion/insert_email.py @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Copyright (C) 2022 Muhammad Rizki +# Copyright (C) 2022 Ammar Faizi +# + + +from mysql.connector import errors +from datetime import datetime + + +class InsertEmail: + + def save_email(self, email_msg_id): + try: + return self.__insert_email(email_msg_id) + except errors.IntegrityError: + # + # Duplicate data, skip! + # + return None + + + def __insert_email(self, email_msg_id): + q = "INSERT INTO tg_emails (message_id, created_at) VALUES (%s, %s)" + self.cur.execute(q, (email_msg_id, datetime.utcnow())) + return self.cur.lastrowid diff --git a/daemon/telegram/database/methods/insertion/insert_telegram.py b/daemon/telegram/database/methods/insertion/insert_telegram.py new file mode 100644 index 0000000..8e0615c --- /dev/null +++ b/daemon/telegram/database/methods/insertion/insert_telegram.py @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Copyright (C) 2022 Muhammad Rizki +# Copyright (C) 2022 Ammar Faizi +# + + +from datetime import datetime + + +class InsertTelegram: + + def save_telegram_mail(self, email_id, tg_chat_id, tg_msg_id): + q = """ + INSERT INTO tg_mail_msg + (email_id, chat_id, tg_msg_id, created_at) + VALUES (%s, %s, %s, %s); + """ + self.cur.execute(q, (email_id, tg_chat_id, tg_msg_id, + datetime.utcnow())) + return self.cur.lastrowid diff --git a/daemon/telegram/db.sql b/daemon/telegram/db.sql deleted file mode 100644 index ac6ed0d..0000000 --- a/daemon/telegram/db.sql +++ /dev/null @@ -1,62 +0,0 @@ --- Adminer 4.7.6 MySQL dump - -SET NAMES utf8; -SET time_zone = '+00:00'; -SET foreign_key_checks = 0; -SET sql_mode = 'NO_AUTO_VALUE_ON_ZERO'; - -SET NAMES utf8mb4; - -DROP TABLE IF EXISTS `emails`; -CREATE TABLE `emails` ( - `id` bigint unsigned NOT NULL AUTO_INCREMENT, - `message_id` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci NOT NULL, - `created_at` datetime NOT NULL, - PRIMARY KEY (`id`), - UNIQUE KEY `message_id` (`message_id`), - KEY `created_at` (`created_at`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_520_ci; - - -DROP TABLE IF EXISTS `tg_emails`; -CREATE TABLE `tg_emails` ( - `id` bigint unsigned NOT NULL AUTO_INCREMENT, - `email_id` bigint unsigned NOT NULL, - `chat_id` bigint NOT NULL, - `tg_msg_id` bigint unsigned NOT NULL, - `created_at` datetime NOT NULL, - PRIMARY KEY (`id`), - KEY `email_id` (`email_id`), - KEY `chat_id` (`chat_id`), - KEY `tg_msg_id` (`tg_msg_id`), - KEY `created_at` (`created_at`), - CONSTRAINT `tg_emails_ibfk_2` FOREIGN KEY (`email_id`) REFERENCES `emails` (`id`) ON DELETE CASCADE ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_520_ci; - - -DROP TABLE IF EXISTS `atom_urls`; -CREATE TABLE `atom_urls` ( - `id` bigint unsigned NOT NULL AUTO_INCREMENT, - `url` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci NOT NULL, - `created_at` datetime NOT NULL, - PRIMARY KEY (`id`), - UNIQUE KEY `url` (`url`), - KEY `created_at` (`created_at`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_520_ci; - - -DROP TABLE IF EXISTS `broadcast_chats`; -CREATE TABLE `broadcast_chats` ( - `id` bigint unsigned NOT NULL AUTO_INCREMENT, - `chat_id` bigint NOT NULL, - `username` varchar(32), - `name` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci NOT NULL, - `type` varchar(32) NOT NULL, - `link` varchar(64), - `created_at` datetime NOT NULL, - PRIMARY KEY (`id`), - UNIQUE KEY `chat_id` (`chat_id`), - KEY `created_at` (`created_at`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_520_ci; - --- 2022-07-07 14:25:28 diff --git a/daemon/telegram/mailer/__init__.py b/daemon/telegram/mailer/__init__.py new file mode 100644 index 0000000..27c2630 --- /dev/null +++ b/daemon/telegram/mailer/__init__.py @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Copyright (C) 2022 Muhammad Rizki +# Copyright (C) 2022 Ammar Faizi +# + +from .listener import Mutexes +from .listener import Listener diff --git a/daemon/telegram/scraper/bot.py b/daemon/telegram/mailer/listener.py similarity index 64% rename from daemon/telegram/scraper/bot.py rename to daemon/telegram/mailer/listener.py index 7adfb12..b3c42d9 100644 --- a/daemon/telegram/scraper/bot.py +++ b/daemon/telegram/mailer/listener.py @@ -1,40 +1,37 @@ # SPDX-License-Identifier: GPL-2.0-only # -# Copyright (C) 2022 Muhammad Rizki +# Copyright (C) 2022 Muhammad Rizki # Copyright (C) 2022 Ammar Faizi # from apscheduler.schedulers.asyncio import AsyncIOScheduler -from packages import DaemonClient -from scraper import Scraper -from . import utils +from pyrogram.types import Message +from ..packages import DaemonClient +from atom import Scraper +from atom import utils +from atom.utils import Mutexes import asyncio import shutil import re import traceback -class BotMutexes(): - def __init__(self): - self.send_to_tg = asyncio.Lock() - - -class Bot(): +class Listener: def __init__(self, client: DaemonClient, sched: AsyncIOScheduler, - scraper: Scraper, mutexes: BotMutexes): + mutexes: Mutexes): self.client = client self.sched = sched - self.scraper = scraper - self.mutexes = mutexes + self.mutex = mutexes self.db = client.db + self.scraper = Scraper() self.isRunnerFixed = False def run(self): - # - # Execute __run() once to avoid high latency at - # initilization. - # + ''' + Execute __run() once to avoid high latency at + initilization. + ''' self.runner = self.sched.add_job( func=self.__run, misfire_grace_time=None, @@ -71,16 +68,16 @@ class Bot(): async def __handle_mail(self, url, mail): chats = self.db.get_broadcast_chats() for chat in chats: - async with self.mutexes.send_to_tg: - should_wait = await self.__send_to_tg(url, mail, + async with self.mutex.lock: + should_wait = await self.__send_mail(url, mail, chat[1]) if should_wait: await asyncio.sleep(1) - # @__must_hold(self.mutexes.send_to_tg) - async def __send_to_tg(self, url, mail, tg_chat_id): + # @__must_hold(self.mutex.lock) + async def __send_mail(self, url, mail, tg_chat_id): email_msg_id = utils.get_email_msg_id(mail) if not email_msg_id: # @@ -89,7 +86,7 @@ class Bot(): # return False - email_id = self.__need_to_send_to_telegram(email_msg_id, + email_id = self.__mail_id_from_db(email_msg_id, tg_chat_id) if not email_id: # @@ -98,21 +95,23 @@ class Bot(): # return False - text, files, is_patch = utils.create_template(mail) - reply_to = self.get_tg_reply_to(mail, tg_chat_id) + text, files, is_patch = utils.create_template( + mail, "telegram" + ) + reply_to = self.get_reply(mail, tg_chat_id) url = str(re.sub(r"/raw$", "", url)) if is_patch: - m = await self.client.send_patch_email( + m: "Message" = await self.client.send_patch_email( mail, tg_chat_id, text, reply_to, url ) else: text = "#ml\n" + text - m = await self.client.send_text_email( + m: "Message" = await self.client.send_text_email( tg_chat_id, text,reply_to, url ) - self.db.insert_telegram(email_id, m.chat.id, m.id) + self.db.save_telegram_mail(email_id, m.chat.id, m.id) for d, f in files: await m.reply_document(f"{d}/{f}", file_name=f) await asyncio.sleep(1) @@ -123,16 +122,16 @@ class Bot(): return True - def __need_to_send_to_telegram(self, email_msg_id, tg_chat_id): - email_id = self.db.save_email_msg_id(email_msg_id) + def __mail_id_from_db(self, email_msg_id, tg_chat_id): + email_id = self.db.save_email(email_msg_id) if email_id: return email_id - email_id = self.db.need_to_send_to_tg(email_msg_id, tg_chat_id) + email_id = self.db.get_email_id(email_msg_id, tg_chat_id) return email_id - def get_tg_reply_to(self, mail, tg_chat_id): + def get_reply(self, mail, tg_chat_id): reply_to = mail.get("in-reply-to") if not reply_to: return None @@ -141,4 +140,4 @@ class Bot(): if not reply_to: return None - return self.db.get_tg_reply_to(reply_to, tg_chat_id) + return self.db.get_reply_id(reply_to, tg_chat_id) diff --git a/daemon/telegram/packages/__init__.py b/daemon/telegram/packages/__init__.py index efef9ae..9610883 100644 --- a/daemon/telegram/packages/__init__.py +++ b/daemon/telegram/packages/__init__.py @@ -1 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Copyright (C) 2022 Muhammad Rizki +# + from .client import DaemonClient diff --git a/daemon/telegram/packages/client.py b/daemon/telegram/packages/client.py index 282daf6..61ef356 100644 --- a/daemon/telegram/packages/client.py +++ b/daemon/telegram/packages/client.py @@ -1,24 +1,24 @@ # SPDX-License-Identifier: GPL-2.0-only # -# Copyright (C) 2022 Muhammad Rizki +# Copyright (C) 2022 Muhammad Rizki # from pyrogram import Client from pyrogram.enums import ParseMode from pyrogram.types import Message, InlineKeyboardMarkup, InlineKeyboardButton +from atom import utils from typing import Union from email.message import Message -from scraper import utils -from scraper.db import Db +from telegram.database import DB from .decorator import handle_flood class DaemonClient(Client): - def __init__(self, name: str, api_id: int, - api_hash: str, conn, **kwargs): + def __init__(self, name: str, api_id: int, api_hash: str, + conn, **kwargs): super().__init__(name, api_id, api_hash, **kwargs) - self.db = Db(conn) + self.db = DB(conn) @handle_flood @@ -56,7 +56,9 @@ class DaemonClient(Client): parse_mode: ParseMode = ParseMode.HTML ) -> Message: print("[send_patch_email]") - tmp, doc, caption, url = utils.prepare_send_patch(mail, text, url) + tmp, doc, caption, url = utils.prepare_patch( + mail, text, url, "telegram" + ) m = await self.send_document( chat_id=chat_id, document=doc, @@ -71,5 +73,5 @@ class DaemonClient(Client): ]) ) - utils.clean_up_after_send_patch(tmp) + utils.remove_patch(tmp) return m diff --git a/daemon/telegram/packages/decorator.py b/daemon/telegram/packages/decorator.py index c7a5f02..4406c2b 100644 --- a/daemon/telegram/packages/decorator.py +++ b/daemon/telegram/packages/decorator.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only # -# Copyright (C) 2022 Muhammad Rizki +# Copyright (C) 2022 Muhammad Rizki # from pyrogram.errors.exceptions.flood_420 import FloodWait @@ -14,9 +14,7 @@ __all__ = ["handle_flood"] T = TypeVar("T", bound=Message) -# -# TODO(Muhammad Rizki): Add more typing for @handle_flood -# + def handle_flood(func: Callable[[T], T]) -> Callable[[T], T]: @wraps(func) async def callback(*args: Any) -> Any: diff --git a/daemon/telegram/packages/plugins/callbacks/del_atom.py b/daemon/telegram/packages/plugins/callbacks/del_atom.py index 1510d60..3a656e1 100644 --- a/daemon/telegram/packages/plugins/callbacks/del_atom.py +++ b/daemon/telegram/packages/plugins/callbacks/del_atom.py @@ -1,12 +1,12 @@ # SPDX-License-Identifier: GPL-2.0-only # -# Copyright (C) 2022 Muhammad Rizki +# Copyright (C) 2022 Muhammad Rizki # -from packages import DaemonClient -from scraper import utils +from telegram.packages import DaemonClient from pyrogram.types import CallbackQuery -import config +from telegram import config +from atom import utils @DaemonClient.on_callback_query(config.admin_only, group=1) diff --git a/daemon/telegram/packages/plugins/callbacks/del_chat.py b/daemon/telegram/packages/plugins/callbacks/del_chat.py index 26c6dd8..44977b2 100644 --- a/daemon/telegram/packages/plugins/callbacks/del_chat.py +++ b/daemon/telegram/packages/plugins/callbacks/del_chat.py @@ -1,12 +1,12 @@ # SPDX-License-Identifier: GPL-2.0-only # -# Copyright (C) 2022 Muhammad Rizki +# Copyright (C) 2022 Muhammad Rizki # -from packages import DaemonClient -from scraper import utils +from telegram.packages import DaemonClient from pyrogram.types import CallbackQuery -import config +from telegram import config +from atom import utils @DaemonClient.on_callback_query(config.admin_only, group=2) diff --git a/daemon/telegram/packages/plugins/commands/debugger.py b/daemon/telegram/packages/plugins/commands/debugger.py index ae2d31d..4fe1cea 100644 --- a/daemon/telegram/packages/plugins/commands/debugger.py +++ b/daemon/telegram/packages/plugins/commands/debugger.py @@ -1,13 +1,13 @@ # SPDX-License-Identifier: GPL-2.0-only # -# Copyright (C) 2022 Muhammad Rizki +# Copyright (C) 2022 Muhammad Rizki # from pyrogram import Client, filters, enums from pyrogram.types import Message from textwrap import indent import io, import_expression, contextlib, traceback -import config +from telegram import config @Client.on_message( diff --git a/daemon/telegram/packages/plugins/commands/manage_atom.py b/daemon/telegram/packages/plugins/commands/manage_atom.py index bcb2f35..6fe9a1e 100644 --- a/daemon/telegram/packages/plugins/commands/manage_atom.py +++ b/daemon/telegram/packages/plugins/commands/manage_atom.py @@ -1,13 +1,13 @@ # SPDX-License-Identifier: GPL-2.0-only # -# Copyright (C) 2022 Muhammad Rizki +# Copyright (C) 2022 Muhammad Rizki # from pyrogram.types import Message from pyrogram import filters -from packages import DaemonClient -from scraper import utils -import config +from telegram.packages import DaemonClient +from telegram import config +from atom import utils @DaemonClient.on_message( @@ -25,7 +25,7 @@ async def add_atom_url(c: DaemonClient, m: Message): if not is_atom: return await m.reply("Invalid Atom URL") - inserted = c.db.insert_atom(text) + inserted = c.db.save_atom(text) if inserted is None: return await m.reply(f"This URL already listened for new email.") diff --git a/daemon/telegram/packages/plugins/commands/manage_broadcast.py b/daemon/telegram/packages/plugins/commands/manage_broadcast.py index ffb5a6b..99b960f 100644 --- a/daemon/telegram/packages/plugins/commands/manage_broadcast.py +++ b/daemon/telegram/packages/plugins/commands/manage_broadcast.py @@ -1,13 +1,13 @@ # SPDX-License-Identifier: GPL-2.0-only # -# Copyright (C) 2022 Muhammad Rizki +# Copyright (C) 2022 Muhammad Rizki # from pyrogram.types import Message from pyrogram import filters, enums -from packages import DaemonClient -from scraper import utils -import config +from telegram.packages import DaemonClient +from telegram import config +from atom import utils @DaemonClient.on_message( @@ -20,7 +20,7 @@ async def add_broadcast(c: DaemonClient, m: Message): else: chat_name = m.chat.title - inserted = c.db.insert_broadcast( + inserted = c.db.save_broadcast( chat_id=m.chat.id, name=chat_name, type=str(m.chat.type), diff --git a/daemon/telegram/packages/plugins/commands/scrape.py b/daemon/telegram/packages/plugins/commands/scrape.py index 45b1581..df49eb6 100644 --- a/daemon/telegram/packages/plugins/commands/scrape.py +++ b/daemon/telegram/packages/plugins/commands/scrape.py @@ -1,15 +1,15 @@ # SPDX-License-Identifier: GPL-2.0-only # -# Copyright (C) 2022 Muhammad Rizki +# Copyright (C) 2022 Muhammad Rizki # Copyright (C) 2022 Ammar Faizi # from pyrogram.types import Message from pyrogram import filters -from packages import DaemonClient -from scraper import Scraper -from scraper import utils -import config +from telegram.packages import DaemonClient +from telegram import config +from atom import Scraper +from atom import utils import shutil import re import asyncio @@ -37,7 +37,7 @@ async def scrap_email(c: DaemonClient, m: Message): s = Scraper() mail = await s.get_email_from_url(url) - text, files, is_patch = utils.create_template(mail) + text, files, is_patch = utils.create_template(mail, "telegram") if is_patch: m = await c.send_patch_email( diff --git a/daemon/telegram/scraper/__init__.py b/daemon/telegram/scraper/__init__.py deleted file mode 100644 index 4294302..0000000 --- a/daemon/telegram/scraper/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# -# Copyright (C) 2022 Muhammad Rizki -# Copyright (C) 2022 Ammar Faizi -# - -from .scraper import Scraper -from .bot import BotMutexes -from .bot import Bot diff --git a/daemon/telegram/scraper/db.py b/daemon/telegram/scraper/db.py deleted file mode 100644 index 58601d1..0000000 --- a/daemon/telegram/scraper/db.py +++ /dev/null @@ -1,217 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# -# Copyright (C) 2022 Muhammad Rizki -# Copyright (C) 2022 Ammar Faizi -# - -from datetime import datetime -import mysql - - -class Db(): - def __init__(self, conn): - self.conn = conn - self.conn.autocommit = True - self.cur = self.conn.cursor(buffered=True) - - - def __del__(self): - self.cur.close() - self.conn.close() - - - def save_email_msg_id(self, email_msg_id): - try: - return self.__save_email_msg_id(email_msg_id) - except mysql.connector.errors.IntegrityError: - # - # Duplicate data, skip! - # - return None - - - def __save_email_msg_id(self, email_msg_id): - q = "INSERT INTO emails (message_id, created_at) VALUES (%s, %s)" - self.cur.execute(q, (email_msg_id, datetime.utcnow())) - return self.cur.lastrowid - - - def insert_telegram(self, email_id, tg_chat_id, tg_msg_id): - q = """ - INSERT INTO tg_emails - (email_id, chat_id, tg_msg_id, created_at) - VALUES (%s, %s, %s, %s); - """ - self.cur.execute(q, (email_id, tg_chat_id, tg_msg_id, - datetime.utcnow())) - return self.cur.lastrowid - - - # - # Determine whether the email needs to be sent to @tg_chat_id. - # - # - Return an email id (PK) if it needs to be sent. - # - Return None if it doesn't need to be sent. - # - def need_to_send_to_tg(self, email_msg_id, tg_chat_id): - q = """ - SELECT emails.id, tg_emails.id FROM emails LEFT JOIN tg_emails - ON emails.id = tg_emails.email_id - WHERE emails.message_id = %(email_msg_id)s - AND tg_emails.chat_id = %(tg_chat_id)s - LIMIT 1 - """ - - self.cur.execute( - q, - { - "email_msg_id": email_msg_id, - "tg_chat_id": tg_chat_id - } - ) - res = self.cur.fetchone() - if bool(res): - # - # This email has already been sent to - # @tg_chat_id. - # - return None - - q = """ - SELECT id FROM emails WHERE message_id = %(email_msg_id)s - """ - self.cur.execute(q, {"email_msg_id": email_msg_id}) - res = self.cur.fetchone() - if not bool(res): - # - # Something goes wrong, skip! - # - return None - - return int(res[0]) - - - def get_tg_reply_to(self, email_msg_id, tg_chat_id): - q = """ - SELECT tg_emails.tg_msg_id - FROM emails INNER JOIN tg_emails - ON emails.id = tg_emails.email_id - WHERE emails.message_id = %(email_msg_id)s - AND tg_emails.chat_id = %(chat_id)s - """ - - self.cur.execute( - q, - { - "email_msg_id": email_msg_id, - "chat_id": tg_chat_id - } - ) - res = self.cur.fetchone() - if not bool(res): - return None - - return res[0] - - - def insert_atom(self, atom: str): - try: - return self.__save_atom(atom) - except mysql.connector.errors.IntegrityError: - # - # Duplicate data, skip! - # - return None - - - def __save_atom(self, atom: str): - q = "INSERT INTO atom_urls (url, created_at) VALUES (%s, %s)" - self.cur.execute(q, (atom, datetime.utcnow())) - return self.cur.lastrowid - - - def delete_atom(self, atom: str): - q = """ - DELETE FROM atom_urls - WHERE url = %(atom)s - """ - try: - self.cur.execute(q, {"atom": atom}) - return True - except: - return False - - - def get_atom_urls(self): - q = """ - SELECT atom_urls.url - FROM atom_urls - """ - self.cur.execute(q) - urls = self.cur.fetchall() - - return [u[0] for u in urls] - - - def insert_broadcast( - self, - chat_id: int, - name: str, - type: str, - created_at: "datetime", - username: str = None, - link: str = None, - ): - try: - return self.__save_broadcast( - chat_id=chat_id, - name=name, - type=type, - created_at=created_at, - username=username, - link=link - ) - except mysql.connector.errors.IntegrityError: - # - # Duplicate data, skip! - # - return None - - - def __save_broadcast( - self, - chat_id: int, - name: str, - type: str, - created_at: "datetime", - username: str = None, - link: str = None, - ): - q = """ - INSERT INTO broadcast_chats - (chat_id, username, name, type, link, created_at) - VALUES - (%s, %s, %s, %s, %s, %s) - """ - values = (chat_id, username, name, type, link, created_at) - self.cur.execute(q, values) - return self.cur.lastrowid - - - def delete_broadcast(self, chat_id: int): - q = """ - DELETE FROM broadcast_chats - WHERE chat_id = %(chat_id)s - """ - self.cur.execute(q, {"chat_id": chat_id}) - return self.cur.rowcount > 0 - - - def get_broadcast_chats(self): - q = """ - SELECT * - FROM broadcast_chats - """ - self.cur.execute(q) - - return self.cur.fetchall() diff --git a/daemon/telegram/scraper/scraper.py b/daemon/telegram/scraper/scraper.py deleted file mode 100644 index 2d5942b..0000000 --- a/daemon/telegram/scraper/scraper.py +++ /dev/null @@ -1,63 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# -# Copyright (C) 2022 Muhammad Rizki -# Copyright (C) 2022 Ammar Faizi -# - -from typing import Dict, List -import email.policy -import xmltodict -import httpx -import email - - -class Scraper(): - async def get_new_threads_urls(self, atom_url): - ret = await self.__get_atom_content(atom_url) - return await self.__get_new_threads_from_atom(ret) - - - async def __get_atom_content(self, atom_url): - async with httpx.AsyncClient() as client: - res = await client.get(atom_url) - if res.status_code == 200: - return res.text - raise Exception(f"[get_atom_content]: Returned {res.status_code} HTTP code") - - - async def __get_new_threads_from_atom(self, atom): - j: Dict[str, List[ - Dict[str, str] - ]] = xmltodict.parse(atom)["feed"] - - entry = [] - e = j["entry"] - for i in e: - entry.append({ - "link": i["link"]["@href"], - "title": i["title"], - "updated": i["updated"], - }) - # - # TODO(ammarfaizi2): Sort by title as well if the @updated is - # identic. - # - entry.sort(key=lambda x: x["updated"]) - - ret = [] - for i in entry: - link = i["link"].replace("http://", "https://") - ret.append(link + "raw") - - return ret - - - async def get_email_from_url(self, url): - async with httpx.AsyncClient() as client: - res = await client.get(url) - if res.status_code == 200: - return email.message_from_string( - res.text, - policy=email.policy.default - ) - raise Exception(f"[get_atom_content]: Returned {res.status_code} HTTP code") diff --git a/daemon/telegram/run.py b/daemon/tg.py similarity index 69% rename from daemon/telegram/run.py rename to daemon/tg.py index 5360395..724782f 100644 --- a/daemon/telegram/run.py +++ b/daemon/tg.py @@ -1,24 +1,23 @@ # SPDX-License-Identifier: GPL-2.0-only # -# Copyright (C) 2022 Muhammad Rizki +# Copyright (C) 2022 Muhammad Rizki # Copyright (C) 2022 Ammar Faizi # from apscheduler.schedulers.asyncio import AsyncIOScheduler -from scraper import BotMutexes +from atom.utils import Mutexes from dotenv import load_dotenv from mysql import connector -from packages import DaemonClient -from scraper import Scraper -from scraper import Bot +from telegram.packages import DaemonClient +from telegram.mailer import Listener import os def main(): - load_dotenv() + load_dotenv("telegram.env") client = DaemonClient( - "storage/EmailScraper", + "telegram/storage/EmailScraper", api_id=int(os.getenv("API_ID")), api_hash=os.getenv("API_HASH"), bot_token=os.getenv("BOT_TOKEN"), @@ -28,9 +27,7 @@ def main(): password=os.getenv("DB_PASS"), database=os.getenv("DB_NAME") ), - plugins=dict( - root="packages.plugins" - ), + plugins=dict(root="telegram.packages.plugins") ) sched = AsyncIOScheduler( @@ -40,11 +37,10 @@ def main(): } ) - bot = Bot( + bot = Listener( client=client, sched=sched, - scraper=Scraper(), - mutexes=BotMutexes() + mutexes=Mutexes() ) sched.start() bot.run() -- Muhammad Rizki