* [PATCH v2 01/11] discord: Remove unnecessary try/except block
2023-01-08 6:15 [PATCH v2 00/11] Improvements, refactors, and features Muhammad Rizki
@ 2023-01-08 6:15 ` Muhammad Rizki
2023-01-08 6:48 ` Ammar Faizi
2023-01-08 6:15 ` [PATCH v2 02/11] telegram: Perform graceful exit when interrupted by a signal Muhammad Rizki
` (9 subsequent siblings)
10 siblings, 1 reply; 25+ messages in thread
From: Muhammad Rizki @ 2023-01-08 6:15 UTC (permalink / raw)
To: Ammar Faizi
Cc: Muhammad Rizki, Alviro Iskandar Setiawan, GNU/Weeb Mailing List
We don't need this anymore since it's already has a try/except earlier.
Signed-off-by: Muhammad Rizki <[email protected]>
---
daemon/dscord/mailer/listener.py | 9 +--------
1 file changed, 1 insertion(+), 8 deletions(-)
diff --git a/daemon/dscord/mailer/listener.py b/daemon/dscord/mailer/listener.py
index 24f0a6c..a225f46 100644
--- a/daemon/dscord/mailer/listener.py
+++ b/daemon/dscord/mailer/listener.py
@@ -106,14 +106,7 @@ class Listener:
self.logger.debug(md)
return False
- try:
- text, files, is_patch = utils.create_template(mail, Platform.DISCORD)
- except:
- exc_str = utils.catch_err()
- self.client.logger.warning(exc_str)
- await self.client.send_log_file(url)
- return
-
+ text, files, is_patch = utils.create_template(mail, Platform.DISCORD)
reply_to = self.get_discord_reply(mail, dc_chat_id)
url = str(re.sub(r"/raw$", "", url))
--
Muhammad Rizki
^ permalink raw reply related [flat|nested] 25+ messages in thread* [PATCH v2 02/11] telegram: Perform graceful exit when interrupted by a signal
2023-01-08 6:15 [PATCH v2 00/11] Improvements, refactors, and features Muhammad Rizki
2023-01-08 6:15 ` [PATCH v2 01/11] discord: Remove unnecessary try/except block Muhammad Rizki
@ 2023-01-08 6:15 ` Muhammad Rizki
2023-01-08 6:58 ` Alviro Iskandar Setiawan
2023-01-08 6:15 ` [PATCH v2 03/11] Add DaemonException() class Muhammad Rizki
` (8 subsequent siblings)
10 siblings, 1 reply; 25+ messages in thread
From: Muhammad Rizki @ 2023-01-08 6:15 UTC (permalink / raw)
To: Ammar Faizi
Cc: Muhammad Rizki, Alviro Iskandar Setiawan, GNU/Weeb Mailing List
We need to start the Telegram bot before starting the listener. This
way, if an error occurs, especially while parsing the email message, the
listener will be able to send a report message to the channel, since the
Telegram bot will already be running.
Also, move them to the run() function in the listener.
Corrected-by: Alviro Iskandar Setiawan <[email protected]>
Signed-off-by: Muhammad Rizki <[email protected]>
---
daemon/telegram/mailer/listener.py | 7 +++++++
daemon/tg.py | 4 +---
2 files changed, 8 insertions(+), 3 deletions(-)
diff --git a/daemon/telegram/mailer/listener.py b/daemon/telegram/mailer/listener.py
index 044de6a..6482a66 100644
--- a/daemon/telegram/mailer/listener.py
+++ b/daemon/telegram/mailer/listener.py
@@ -4,6 +4,7 @@
# Copyright (C) 2022 Ammar Faizi <[email protected]>
#
+from pyrogram import idle
from pyrogram.types import Message
from mysql.connector.errors import OperationalError, DatabaseError
from apscheduler.schedulers.asyncio import AsyncIOScheduler
@@ -37,12 +38,18 @@ class Bot():
# Execute __run() once to avoid high latency at
# initilization.
#
+ self.sched.start()
+ self.client.start()
+
self.runner = self.sched.add_job(
func=self.__run,
misfire_grace_time=None,
max_instances=1
)
+ idle()
+ self.client.stop()
+
async def handle_db_error(self, e):
#
diff --git a/daemon/tg.py b/daemon/tg.py
index 73ff2b9..a676cf5 100644
--- a/daemon/tg.py
+++ b/daemon/tg.py
@@ -7,6 +7,7 @@
from apscheduler.schedulers.asyncio import AsyncIOScheduler
from dotenv import load_dotenv
from mysql import connector
+from pyrogram import idle
from atom import Scraper
from telegram.packages import DaemonClient
from telegram.mailer import BotMutexes
@@ -58,10 +59,7 @@ def main():
scraper=Scraper(),
mutexes=BotMutexes()
)
- sched.start()
bot.run()
- client.run()
-
if __name__ == '__main__':
main()
--
Muhammad Rizki
^ permalink raw reply related [flat|nested] 25+ messages in thread* Re: [PATCH v2 02/11] telegram: Perform graceful exit when interrupted by a signal
2023-01-08 6:15 ` [PATCH v2 02/11] telegram: Perform graceful exit when interrupted by a signal Muhammad Rizki
@ 2023-01-08 6:58 ` Alviro Iskandar Setiawan
2023-01-08 7:46 ` Muhammad Rizki
0 siblings, 1 reply; 25+ messages in thread
From: Alviro Iskandar Setiawan @ 2023-01-08 6:58 UTC (permalink / raw)
To: Muhammad Rizki; +Cc: Ammar Faizi, GNU/Weeb Mailing List
On Sun, Jan 8, 2023 at 1:16 PM Muhammad Rizki wrote:
> We need to start the Telegram bot before starting the listener. This
> way, if an error occurs, especially while parsing the email message, the
> listener will be able to send a report message to the channel, since the
> Telegram bot will already be running.
>
> Also, move them to the run() function in the listener.
>
> Corrected-by: Alviro Iskandar Setiawan <[email protected]>
> Signed-off-by: Muhammad Rizki <[email protected]>
Acked-by: Alviro Iskandar Setiawan <[email protected]>
^ permalink raw reply [flat|nested] 25+ messages in thread
* Re: [PATCH v2 02/11] telegram: Perform graceful exit when interrupted by a signal
2023-01-08 6:58 ` Alviro Iskandar Setiawan
@ 2023-01-08 7:46 ` Muhammad Rizki
0 siblings, 0 replies; 25+ messages in thread
From: Muhammad Rizki @ 2023-01-08 7:46 UTC (permalink / raw)
To: Alviro Iskandar Setiawan; +Cc: Ammar Faizi, GNU/Weeb Mailing List
On 08/01/2023 13.58, Alviro Iskandar Setiawan wrote:
> On Sun, Jan 8, 2023 at 1:16 PM Muhammad Rizki wrote:
>> We need to start the Telegram bot before starting the listener. This
>> way, if an error occurs, especially while parsing the email message, the
>> listener will be able to send a report message to the channel, since the
>> Telegram bot will already be running.
>>
>> Also, move them to the run() function in the listener.
>>
>> Corrected-by: Alviro Iskandar Setiawan <[email protected]>
>> Signed-off-by: Muhammad Rizki <[email protected]>
>
> Acked-by: Alviro Iskandar Setiawan <[email protected]>
OK
^ permalink raw reply [flat|nested] 25+ messages in thread
* [PATCH v2 03/11] Add DaemonException() class
2023-01-08 6:15 [PATCH v2 00/11] Improvements, refactors, and features Muhammad Rizki
2023-01-08 6:15 ` [PATCH v2 01/11] discord: Remove unnecessary try/except block Muhammad Rizki
2023-01-08 6:15 ` [PATCH v2 02/11] telegram: Perform graceful exit when interrupted by a signal Muhammad Rizki
@ 2023-01-08 6:15 ` Muhammad Rizki
2023-01-08 6:47 ` Ammar Faizi
2023-01-08 6:15 ` [PATCH v2 04/11] telegram: Refactor report_err() and move to the DaemonClient() class Muhammad Rizki
` (7 subsequent siblings)
10 siblings, 1 reply; 25+ messages in thread
From: Muhammad Rizki @ 2023-01-08 6:15 UTC (permalink / raw)
To: Ammar Faizi
Cc: Muhammad Rizki, Alviro Iskandar Setiawan, GNU/Weeb Mailing List
Added DaemonException() class in the daemon/exceptions.py to be able to
throw custom error, the use case is to easily catch the email thread
URL.
This error class is reusable for both Discord and Telegram bot.
Signed-off-by: Muhammad Rizki <[email protected]>
---
daemon/exceptions.py | 18 ++++++++++++++++++
1 file changed, 18 insertions(+)
create mode 100644 daemon/exceptions.py
diff --git a/daemon/exceptions.py b/daemon/exceptions.py
new file mode 100644
index 0000000..34f6823
--- /dev/null
+++ b/daemon/exceptions.py
@@ -0,0 +1,18 @@
+from typing import Optional
+
+class DaemonException(Exception):
+ thread_url: Optional[str] = None
+ atom_url: Optional[str] = None
+ original_exception: Optional[str] = None
+
+ def __str__(self) -> str:
+ return self.original_exception
+
+ def set_atom_url(self, url: str):
+ self.atom_url = url
+
+ def set_thread_url(self, url: str):
+ self.thread_url = url
+
+ def set_message(self, msg: str):
+ self.original_exception = msg
--
Muhammad Rizki
^ permalink raw reply related [flat|nested] 25+ messages in thread* [PATCH v2 04/11] telegram: Refactor report_err() and move to the DaemonClient() class
2023-01-08 6:15 [PATCH v2 00/11] Improvements, refactors, and features Muhammad Rizki
` (2 preceding siblings ...)
2023-01-08 6:15 ` [PATCH v2 03/11] Add DaemonException() class Muhammad Rizki
@ 2023-01-08 6:15 ` Muhammad Rizki
2023-01-08 6:15 ` [PATCH v2 05/11] telegram: Implement the DaemonException() and refactor report_err() Muhammad Rizki
` (6 subsequent siblings)
10 siblings, 0 replies; 25+ messages in thread
From: Muhammad Rizki @ 2023-01-08 6:15 UTC (permalink / raw)
To: Ammar Faizi
Cc: Muhammad Rizki, Alviro Iskandar Setiawan, GNU/Weeb Mailing List
Instead of adding the report_err() function in the Bot() class, move it
to the DaemonClient() class. Also, change the report_err() function as
follows:
def report_err(self, e: DaemonException):
# code for reporting error goes here
Signed-off-by: Muhammad Rizki <[email protected]>
---
daemon/telegram/mailer/listener.py | 8 +-------
daemon/telegram/packages/client.py | 8 ++++++++
2 files changed, 9 insertions(+), 7 deletions(-)
diff --git a/daemon/telegram/mailer/listener.py b/daemon/telegram/mailer/listener.py
index 6482a66..5ac618f 100644
--- a/daemon/telegram/mailer/listener.py
+++ b/daemon/telegram/mailer/listener.py
@@ -68,12 +68,6 @@ class Bot():
self.db.ping(reconnect=True, attempts=reconnect_attempts,
delay=delay_in_secs)
- async def report_err(caption):
- if not caption:
- caption = "No lore URL"
- exc_str = utils.catch_err()
- self.logger.warning(exc_str)
- await self.client.send_log_file(caption)
async def __run(self):
self.logger.info("Running...")
@@ -84,7 +78,7 @@ class Bot():
except (OperationalError, DatabaseError) as e:
await self.handle_db_error(e)
except:
- await self.report_err(url)
+ await self.client.report_err(url)
if not self.isRunnerFixed:
self.isRunnerFixed = True
diff --git a/daemon/telegram/packages/client.py b/daemon/telegram/packages/client.py
index 58195b9..c5b0b33 100644
--- a/daemon/telegram/packages/client.py
+++ b/daemon/telegram/packages/client.py
@@ -14,6 +14,7 @@ from logger import BotLogger
from telegram import config
from telegram.database import DB
from .decorator import handle_flood
+from exceptions import DaemonException
class DaemonClient(Client):
@@ -27,6 +28,13 @@ class DaemonClient(Client):
self.logger = logger
+ async def report_err(self, e: DaemonException):
+ capt = f"Atom URL: {e.atom_url}\n"
+ capt += f"Thread URL: {e.thread_url}"
+ self.logger.warning(e.original_exception)
+ await self.send_log_file(capt)
+
+
@handle_flood
async def send_log_file(self, caption: str):
filename = self.logger.handlers[0].baseFilename
--
Muhammad Rizki
^ permalink raw reply related [flat|nested] 25+ messages in thread* [PATCH v2 05/11] telegram: Implement the DaemonException() and refactor report_err()
2023-01-08 6:15 [PATCH v2 00/11] Improvements, refactors, and features Muhammad Rizki
` (3 preceding siblings ...)
2023-01-08 6:15 ` [PATCH v2 04/11] telegram: Refactor report_err() and move to the DaemonClient() class Muhammad Rizki
@ 2023-01-08 6:15 ` Muhammad Rizki
2023-01-08 6:36 ` Ammar Faizi
2023-01-08 6:15 ` [PATCH v2 06/11] telegram: listener: Remove unnecessary try/except block Muhammad Rizki
` (5 subsequent siblings)
10 siblings, 1 reply; 25+ messages in thread
From: Muhammad Rizki @ 2023-01-08 6:15 UTC (permalink / raw)
To: Ammar Faizi
Cc: Muhammad Rizki, Alviro Iskandar Setiawan, GNU/Weeb Mailing List
This exception class is used to handle errors that are easy to catch,
such as the email thread URL. This class has been customized and
inherited from the built-in Exception() class. Also, change the
report_err() value from the DaemonException.
Signed-off-by: Muhammad Rizki <[email protected]>
---
daemon/telegram/mailer/listener.py | 16 ++++++++++++----
1 file changed, 12 insertions(+), 4 deletions(-)
diff --git a/daemon/telegram/mailer/listener.py b/daemon/telegram/mailer/listener.py
index 5ac618f..746385c 100644
--- a/daemon/telegram/mailer/listener.py
+++ b/daemon/telegram/mailer/listener.py
@@ -9,6 +9,7 @@ from pyrogram.types import Message
from mysql.connector.errors import OperationalError, DatabaseError
from apscheduler.schedulers.asyncio import AsyncIOScheduler
from telegram.packages import DaemonClient
+from exceptions import DaemonException
from atom import Scraper
from atom import utils
from enums import Platform
@@ -77,8 +78,9 @@ class Bot():
await self.__handle_atom_url(url)
except (OperationalError, DatabaseError) as e:
await self.handle_db_error(e)
- except:
- await self.client.report_err(url)
+ except DaemonException as e:
+ e.set_atom_url(url)
+ await self.client.report_err(e)
if not self.isRunnerFixed:
self.isRunnerFixed = True
@@ -94,8 +96,14 @@ class Bot():
async def __handle_atom_url(self, url):
urls = await self.scraper.get_new_threads_urls(url)
for url in urls:
- mail = await self.scraper.get_email_from_url(url)
- await self.__handle_mail(url, mail)
+ try:
+ mail = await self.scraper.get_email_from_url(url)
+ await self.__handle_mail(url, mail)
+ except:
+ e = DaemonException()
+ e.set_thread_url(url)
+ e.set_message(utils.catch_err())
+ raise e
async def __handle_mail(self, url, mail):
--
Muhammad Rizki
^ permalink raw reply related [flat|nested] 25+ messages in thread* Re: [PATCH v2 05/11] telegram: Implement the DaemonException() and refactor report_err()
2023-01-08 6:15 ` [PATCH v2 05/11] telegram: Implement the DaemonException() and refactor report_err() Muhammad Rizki
@ 2023-01-08 6:36 ` Ammar Faizi
2023-01-08 7:15 ` Alviro Iskandar Setiawan
0 siblings, 1 reply; 25+ messages in thread
From: Ammar Faizi @ 2023-01-08 6:36 UTC (permalink / raw)
To: Muhammad Rizki; +Cc: Alviro Iskandar Setiawan, GNU/Weeb Mailing List
On Sun, Jan 08, 2023 at 01:15:37PM +0700, Muhammad Rizki wrote:
> @@ -77,8 +78,9 @@ class Bot():
> await self.__handle_atom_url(url)
> except (OperationalError, DatabaseError) as e:
> await self.handle_db_error(e)
> - except:
> - await self.client.report_err(url)
> + except DaemonException as e:
> + e.set_atom_url(url)
> + await self.client.report_err(e)
What is wrong with you?
I have told you like 3 times about this. You need to handle a generic
exception as well. Because if it throws an exception inside
`self.db.get_atom_urls()`, we will not get the report.
An exception that is raised inside `self.db.get_atom_urls()` will not
be converted into a DaemonException instance. That is why it's possible
to get an exception that's not an instance of (OperationalError,
DatabaseError, DaemonException).
--
Ammar Faizi
^ permalink raw reply [flat|nested] 25+ messages in thread
* Re: [PATCH v2 05/11] telegram: Implement the DaemonException() and refactor report_err()
2023-01-08 6:36 ` Ammar Faizi
@ 2023-01-08 7:15 ` Alviro Iskandar Setiawan
2023-01-08 7:23 ` Ammar Faizi
2023-01-08 7:25 ` Ammar Faizi
0 siblings, 2 replies; 25+ messages in thread
From: Alviro Iskandar Setiawan @ 2023-01-08 7:15 UTC (permalink / raw)
To: Ammar Faizi; +Cc: Muhammad Rizki, GNU/Weeb Mailing List
On Sun, Jan 8, 2023 at 1:36 PM Ammar Faizi wrote:
> On Sun, Jan 08, 2023 at 01:15:37PM +0700, Muhammad Rizki wrote:
> > @@ -77,8 +78,9 @@ class Bot():
> > await self.__handle_atom_url(url)
> > except (OperationalError, DatabaseError) as e:
> > await self.handle_db_error(e)
> > - except:
> > - await self.client.report_err(url)
> > + except DaemonException as e:
> > + e.set_atom_url(url)
> > + await self.client.report_err(e)
>
> What is wrong with you?
>
> I have told you like 3 times about this. You need to handle a generic
> exception as well. Because if it throws an exception inside
> `self.db.get_atom_urls()`, we will not get the report.
>
> An exception that is raised inside `self.db.get_atom_urls()` will not
> be converted into a DaemonException instance. That is why it's possible
> to get an exception that's not an instance of (OperationalError,
> DatabaseError, DaemonException).
Maybe you need to give a small pseudo-code so it's clear to him. My
understanding will be adding an "except" statement without specifying
any exception class name, no?
^ permalink raw reply [flat|nested] 25+ messages in thread
* Re: [PATCH v2 05/11] telegram: Implement the DaemonException() and refactor report_err()
2023-01-08 7:15 ` Alviro Iskandar Setiawan
@ 2023-01-08 7:23 ` Ammar Faizi
2023-01-08 7:48 ` Muhammad Rizki
2023-01-08 7:25 ` Ammar Faizi
1 sibling, 1 reply; 25+ messages in thread
From: Ammar Faizi @ 2023-01-08 7:23 UTC (permalink / raw)
To: Alviro Iskandar Setiawan; +Cc: Muhammad Rizki, GNU/Weeb Mailing List
On Sun, Jan 08, 2023 at 02:15:38PM +0700, Alviro Iskandar Setiawan wrote:
> Maybe you need to give a small pseudo-code so it's clear to him.
Like I didn't? I definitely did that. Please stop this retardation. I'm
very tired of getting ignored when giving reviews. Let me emphasize it
with a block of comment this time.
url = None
try:
for url in self.db.get_atom_urls():
await self.__handle_atom_url(url)
except (OperationalError, DatabaseError) as e:
await self.handle_db_error(e)
except DaemonException as e:
e.set_atom_url(url)
await self.client.report_err(e)
except:
#
# This is needed because if an exception is
# raised inside self.db.get_atom_urls(), we
# will catch it here.
#
# At this point, we have an atom_url, but
# no thread_url.
#
e = DaemonException()
e.set_atom_url(url)
e.set_thread_url(None)
await self.client.report_err(e)
> My understanding will be adding an "except" statement without specifying
> any exception class name, no?
Yes, you are correct.
--
Ammar Faizi
^ permalink raw reply [flat|nested] 25+ messages in thread* Re: [PATCH v2 05/11] telegram: Implement the DaemonException() and refactor report_err()
2023-01-08 7:23 ` Ammar Faizi
@ 2023-01-08 7:48 ` Muhammad Rizki
0 siblings, 0 replies; 25+ messages in thread
From: Muhammad Rizki @ 2023-01-08 7:48 UTC (permalink / raw)
To: Ammar Faizi, Alviro Iskandar Setiawan; +Cc: GNU/Weeb Mailing List
On 08/01/2023 14.23, Ammar Faizi wrote:
> On Sun, Jan 08, 2023 at 02:15:38PM +0700, Alviro Iskandar Setiawan wrote:
>> Maybe you need to give a small pseudo-code so it's clear to him.
>
> Like I didn't? I definitely did that. Please stop this retardation. I'm
> very tired of getting ignored when giving reviews. Let me emphasize it
> with a block of comment this time.
>
> url = None
> try:
> for url in self.db.get_atom_urls():
> await self.__handle_atom_url(url)
> except (OperationalError, DatabaseError) as e:
> await self.handle_db_error(e)
> except DaemonException as e:
> e.set_atom_url(url)
> await self.client.report_err(e)
> except:
> #
> # This is needed because if an exception is
> # raised inside self.db.get_atom_urls(), we
> # will catch it here.
> #
> # At this point, we have an atom_url, but
> # no thread_url.
> #
>
> e = DaemonException()
> e.set_atom_url(url)
> e.set_thread_url(None)
> await self.client.report_err(e)
OK, I'll implement this.
>
>
>> My understanding will be adding an "except" statement without specifying
>> any exception class name, no?
>
> Yes, you are correct.
>
^ permalink raw reply [flat|nested] 25+ messages in thread
* Re: [PATCH v2 05/11] telegram: Implement the DaemonException() and refactor report_err()
2023-01-08 7:15 ` Alviro Iskandar Setiawan
2023-01-08 7:23 ` Ammar Faizi
@ 2023-01-08 7:25 ` Ammar Faizi
1 sibling, 0 replies; 25+ messages in thread
From: Ammar Faizi @ 2023-01-08 7:25 UTC (permalink / raw)
To: Alviro Iskandar Setiawan; +Cc: Muhammad Rizki, GNU/Weeb Mailing List
[ Resending this, just not to make you misunderstand, please don't ignore.
The previous message doesn't have a call to e.set_message() ]
On Sun, Jan 08, 2023 at 02:15:38PM +0700, Alviro Iskandar Setiawan wrote:
> Maybe you need to give a small pseudo-code so it's clear to him.
Like I didn't? I definitely did that. Please stop this retardation. I'm
very tired of getting ignored when giving reviews. Let me emphasize it
with a block of comment this time.
url = None
try:
for url in self.db.get_atom_urls():
await self.__handle_atom_url(url)
except (OperationalError, DatabaseError) as e:
await self.handle_db_error(e)
except DaemonException as e:
e.set_atom_url(url)
await self.client.report_err(e)
except:
#
# This is needed because if an exception is
# raised inside self.db.get_atom_urls(), we
# will catch it here.
#
# At this point, we have an atom_url, but
# no thread_url.
#
e = DaemonException()
e.set_atom_url(url)
e.set_thread_url(None)
e.set_message(utils.catch_err())
await self.client.report_err(e)
> My understanding will be adding an "except" statement without specifying
> any exception class name, no?
Yes, you are correct.
--
Ammar Faizi
^ permalink raw reply [flat|nested] 25+ messages in thread
* [PATCH v2 06/11] telegram: listener: Remove unnecessary try/except block
2023-01-08 6:15 [PATCH v2 00/11] Improvements, refactors, and features Muhammad Rizki
` (4 preceding siblings ...)
2023-01-08 6:15 ` [PATCH v2 05/11] telegram: Implement the DaemonException() and refactor report_err() Muhammad Rizki
@ 2023-01-08 6:15 ` Muhammad Rizki
2023-01-08 6:15 ` [PATCH v2 07/11] discord: add report_err() for the Discord bot Muhammad Rizki
` (4 subsequent siblings)
10 siblings, 0 replies; 25+ messages in thread
From: Muhammad Rizki @ 2023-01-08 6:15 UTC (permalink / raw)
To: Ammar Faizi
Cc: Muhammad Rizki, Alviro Iskandar Setiawan, GNU/Weeb Mailing List
The error handling is already been moved to the __handle_atom_url()
so it's not necessary to use try/except block again.
Signed-off-by: Muhammad Rizki <[email protected]>
---
daemon/telegram/mailer/listener.py | 9 +--------
1 file changed, 1 insertion(+), 8 deletions(-)
diff --git a/daemon/telegram/mailer/listener.py b/daemon/telegram/mailer/listener.py
index 746385c..9e5cbf5 100644
--- a/daemon/telegram/mailer/listener.py
+++ b/daemon/telegram/mailer/listener.py
@@ -133,14 +133,7 @@ class Bot():
self.logger.debug(md)
return False
- try:
- text, files, is_patch = utils.create_template(mail, Platform.TELEGRAM)
- except:
- exc_str = utils.catch_err()
- self.logger.warning(exc_str)
- await self.client.send_log_file(url)
- return
-
+ text, files, is_patch = utils.create_template(mail, Platform.TELEGRAM)
reply_to = self.get_reply(mail, tg_chat_id)
url = str(re.sub(r"/raw$", "", url))
--
Muhammad Rizki
^ permalink raw reply related [flat|nested] 25+ messages in thread* [PATCH v2 07/11] discord: add report_err() for the Discord bot
2023-01-08 6:15 [PATCH v2 00/11] Improvements, refactors, and features Muhammad Rizki
` (5 preceding siblings ...)
2023-01-08 6:15 ` [PATCH v2 06/11] telegram: listener: Remove unnecessary try/except block Muhammad Rizki
@ 2023-01-08 6:15 ` Muhammad Rizki
2023-01-08 6:49 ` Ammar Faizi
2023-01-08 6:15 ` [PATCH v2 08/11] discord: Implement the report_err() and DaemonException() error class Muhammad Rizki
` (3 subsequent siblings)
10 siblings, 1 reply; 25+ messages in thread
From: Muhammad Rizki @ 2023-01-08 6:15 UTC (permalink / raw)
To: Ammar Faizi
Cc: Muhammad Rizki, Alviro Iskandar Setiawan, GNU/Weeb Mailing List
Added report_err() in the Discord bot GWClient() class for reporting
error to the Discord channel.
Signed-off-by: Muhammad Rizki <[email protected]>
---
daemon/dscord/gnuweeb/client.py | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/daemon/dscord/gnuweeb/client.py b/daemon/dscord/gnuweeb/client.py
index 406c606..78ec3f9 100644
--- a/daemon/dscord/gnuweeb/client.py
+++ b/daemon/dscord/gnuweeb/client.py
@@ -13,6 +13,7 @@ from . import filters
from . import models
from atom import utils
from enums import Platform
+from exceptions import DaemonException
from logger.log import BotLogger
from dscord.config import ACTIVITY_NAME, LOG_CHANNEL_ID
from dscord.database import DB
@@ -40,6 +41,13 @@ class GWClient(commands.Bot):
)
+ async def report_err(self, e: DaemonException):
+ self.logger.warning(e.original_exception)
+ capt = f"Atom URL: {e.atom_url}\n"
+ capt += f"Thread URL: {e.thread_url}"
+ await self.send_log_file(capt)
+
+
@filters.wait_on_limit
async def send_log_file(self, caption: str):
filename = self.logger.handlers[0].baseFilename
--
Muhammad Rizki
^ permalink raw reply related [flat|nested] 25+ messages in thread* [PATCH v2 08/11] discord: Implement the report_err() and DaemonException() error class
2023-01-08 6:15 [PATCH v2 00/11] Improvements, refactors, and features Muhammad Rizki
` (6 preceding siblings ...)
2023-01-08 6:15 ` [PATCH v2 07/11] discord: add report_err() for the Discord bot Muhammad Rizki
@ 2023-01-08 6:15 ` Muhammad Rizki
2023-01-08 6:15 ` [PATCH v2 09/11] discord: database: Add ping() method for reconnect MySQL Muhammad Rizki
` (2 subsequent siblings)
10 siblings, 0 replies; 25+ messages in thread
From: Muhammad Rizki @ 2023-01-08 6:15 UTC (permalink / raw)
To: Ammar Faizi
Cc: Muhammad Rizki, Alviro Iskandar Setiawan, GNU/Weeb Mailing List
After creating the `report_err()` function within the `GWClient()`
class, implement it in `dscord/mailer/listener.py` to report error logs
to the Discord channel. Additionally, also implement the
DaemonException() function to handle errors.
Signed-off-by: Muhammad Rizki <[email protected]>
---
daemon/dscord/mailer/listener.py | 18 +++++++++++++-----
1 file changed, 13 insertions(+), 5 deletions(-)
diff --git a/daemon/dscord/mailer/listener.py b/daemon/dscord/mailer/listener.py
index a225f46..dc77ad2 100644
--- a/daemon/dscord/mailer/listener.py
+++ b/daemon/dscord/mailer/listener.py
@@ -14,6 +14,7 @@ from dscord.gnuweeb import GWClient
from atom.scraper import Scraper
from atom import utils
from enums import Platform
+from exceptions import DaemonException
class Mutexes:
@@ -51,8 +52,14 @@ class Listener:
async def __run(self):
self.logger.info("Running...")
- for url in self.db.get_atom_urls():
- await self.__handle_atom_url(url)
+ url = None
+
+ try:
+ for url in self.db.get_atom_urls():
+ await self.__handle_atom_url(url)
+ except DaemonException as e:
+ e.set_atom_url(url)
+ await self.client.report_err(e)
if not self.isRunnerFixed:
self.isRunnerFixed = True
@@ -72,9 +79,10 @@ class Listener:
mail = await self.scraper.get_email_from_url(url)
await self.__handle_mail(url, mail)
except:
- exc_str = utils.catch_err()
- self.client.logger.warning(exc_str)
- await self.client.send_log_file(url)
+ e = DaemonException()
+ e.set_thread_url(url)
+ e.set_message(utils.catch_err())
+ raise e
async def __handle_mail(self, url, mail):
--
Muhammad Rizki
^ permalink raw reply related [flat|nested] 25+ messages in thread* [PATCH v2 09/11] discord: database: Add ping() method for reconnect MySQL
2023-01-08 6:15 [PATCH v2 00/11] Improvements, refactors, and features Muhammad Rizki
` (7 preceding siblings ...)
2023-01-08 6:15 ` [PATCH v2 08/11] discord: Implement the report_err() and DaemonException() error class Muhammad Rizki
@ 2023-01-08 6:15 ` Muhammad Rizki
2023-01-08 6:51 ` Ammar Faizi
2023-01-08 6:15 ` [PATCH v2 10/11] discord: listener: Add handle_db_error to handle the MySQL errors Muhammad Rizki
2023-01-08 6:15 ` [PATCH v2 11/11] telegram: Rename the Telegram bot custom client Muhammad Rizki
10 siblings, 1 reply; 25+ messages in thread
From: Muhammad Rizki @ 2023-01-08 6:15 UTC (permalink / raw)
To: Ammar Faizi
Cc: Muhammad Rizki, Alviro Iskandar Setiawan, GNU/Weeb Mailing List
Added ping() in the Discord database core.py for reconnecting the MySQL
database. This function is to reconnect the MySQL connection.
Signed-off-by: Muhammad Rizki <[email protected]>
---
daemon/dscord/database/core.py | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/daemon/dscord/database/core.py b/daemon/dscord/database/core.py
index c34d7a8..c8826bb 100644
--- a/daemon/dscord/database/core.py
+++ b/daemon/dscord/database/core.py
@@ -15,6 +15,12 @@ class DB(DBMethods):
self.cur = self.conn.cursor(buffered=True)
+ def ping(self, reconnect=True, attempts=3, delay=3):
+ self.conn.ping(reconnect=reconnect, attempts=attempts,
+ delay=delay)
+ self.cur = self.conn.cursor(buffered=True)
+
+
def __del__(self):
self.cur.close()
self.conn.close()
--
Muhammad Rizki
^ permalink raw reply related [flat|nested] 25+ messages in thread* [PATCH v2 10/11] discord: listener: Add handle_db_error to handle the MySQL errors
2023-01-08 6:15 [PATCH v2 00/11] Improvements, refactors, and features Muhammad Rizki
` (8 preceding siblings ...)
2023-01-08 6:15 ` [PATCH v2 09/11] discord: database: Add ping() method for reconnect MySQL Muhammad Rizki
@ 2023-01-08 6:15 ` Muhammad Rizki
2023-01-08 6:15 ` [PATCH v2 11/11] telegram: Rename the Telegram bot custom client Muhammad Rizki
10 siblings, 0 replies; 25+ messages in thread
From: Muhammad Rizki @ 2023-01-08 6:15 UTC (permalink / raw)
To: Ammar Faizi
Cc: Muhammad Rizki, Alviro Iskandar Setiawan, GNU/Weeb Mailing List
After adding a ping() method to the database, you should handle database
errors such as connection errors. This function will be used to handle
connection errors that occur when the MySQL service is restarted.
Signed-off-by: Muhammad Rizki <[email protected]>
---
daemon/dscord/mailer/listener.py | 21 +++++++++++++++++++++
1 file changed, 21 insertions(+)
diff --git a/daemon/dscord/mailer/listener.py b/daemon/dscord/mailer/listener.py
index dc77ad2..cb2849a 100644
--- a/daemon/dscord/mailer/listener.py
+++ b/daemon/dscord/mailer/listener.py
@@ -6,6 +6,7 @@
import asyncio
import re
+from mysql.connector.errors import OperationalError, DatabaseError
from apscheduler.schedulers.asyncio import AsyncIOScheduler
from discord import File
from discord import Message
@@ -50,6 +51,24 @@ class Listener:
self.runner = self.sched.add_job(func=self.__run)
+ async def handle_db_error(self, e):
+ #
+ # TODO(ammarfaizi2):
+ # Ideally, we also want to log and report this situation.
+ #
+ self.logger.error(f"Database error: {str(e)}")
+ self.logger.info("Reconnecting to the database...")
+
+ #
+ # Don't do this too often to avoid reconnect burst.
+ #
+ delay_in_secs = 3
+ reconnect_attempts = 3
+
+ self.db.ping(reconnect=True, attempts=reconnect_attempts,
+ delay=delay_in_secs)
+
+
async def __run(self):
self.logger.info("Running...")
url = None
@@ -57,6 +76,8 @@ class Listener:
try:
for url in self.db.get_atom_urls():
await self.__handle_atom_url(url)
+ except (OperationalError, DatabaseError) as e:
+ await self.handle_db_error(e)
except DaemonException as e:
e.set_atom_url(url)
await self.client.report_err(e)
--
Muhammad Rizki
^ permalink raw reply related [flat|nested] 25+ messages in thread* [PATCH v2 11/11] telegram: Rename the Telegram bot custom client
2023-01-08 6:15 [PATCH v2 00/11] Improvements, refactors, and features Muhammad Rizki
` (9 preceding siblings ...)
2023-01-08 6:15 ` [PATCH v2 10/11] discord: listener: Add handle_db_error to handle the MySQL errors Muhammad Rizki
@ 2023-01-08 6:15 ` Muhammad Rizki
10 siblings, 0 replies; 25+ messages in thread
From: Muhammad Rizki @ 2023-01-08 6:15 UTC (permalink / raw)
To: Ammar Faizi
Cc: Muhammad Rizki, Alviro Iskandar Setiawan, GNU/Weeb Mailing List
Rename the custom client for the Telegram bot and update all affected
code in the other files to reflect the change.
Signed-off-by: Muhammad Rizki <[email protected]>
---
daemon/telegram/mailer/listener.py | 4 ++--
daemon/telegram/packages/__init__.py | 2 +-
daemon/telegram/packages/client.py | 2 +-
daemon/telegram/packages/decorator.py | 2 +-
daemon/telegram/packages/plugins/callbacks/del_atom.py | 6 +++---
daemon/telegram/packages/plugins/callbacks/del_chat.py | 6 +++---
.../telegram/packages/plugins/commands/manage_atom.py | 10 +++++-----
.../packages/plugins/commands/manage_broadcast.py | 10 +++++-----
daemon/telegram/packages/plugins/commands/scrape.py | 6 +++---
daemon/tg.py | 4 ++--
10 files changed, 26 insertions(+), 26 deletions(-)
diff --git a/daemon/telegram/mailer/listener.py b/daemon/telegram/mailer/listener.py
index 9e5cbf5..c32ad94 100644
--- a/daemon/telegram/mailer/listener.py
+++ b/daemon/telegram/mailer/listener.py
@@ -8,7 +8,7 @@ from pyrogram import idle
from pyrogram.types import Message
from mysql.connector.errors import OperationalError, DatabaseError
from apscheduler.schedulers.asyncio import AsyncIOScheduler
-from telegram.packages import DaemonClient
+from telegram.packages import DaemonTelegram
from exceptions import DaemonException
from atom import Scraper
from atom import utils
@@ -23,7 +23,7 @@ class BotMutexes():
class Bot():
- def __init__(self, client: DaemonClient, sched: AsyncIOScheduler,
+ def __init__(self, client: DaemonTelegram, sched: AsyncIOScheduler,
scraper: Scraper, mutexes: BotMutexes):
self.client = client
self.sched = sched
diff --git a/daemon/telegram/packages/__init__.py b/daemon/telegram/packages/__init__.py
index efef9ae..15a270f 100644
--- a/daemon/telegram/packages/__init__.py
+++ b/daemon/telegram/packages/__init__.py
@@ -1 +1 @@
-from .client import DaemonClient
+from .client import DaemonTelegram
diff --git a/daemon/telegram/packages/client.py b/daemon/telegram/packages/client.py
index c5b0b33..a58aaf8 100644
--- a/daemon/telegram/packages/client.py
+++ b/daemon/telegram/packages/client.py
@@ -17,7 +17,7 @@ from .decorator import handle_flood
from exceptions import DaemonException
-class DaemonClient(Client):
+class DaemonTelegram(Client):
def __init__(self, name: str, api_id: int,
api_hash: str, conn, logger: BotLogger,
**kwargs
diff --git a/daemon/telegram/packages/decorator.py b/daemon/telegram/packages/decorator.py
index 153fa95..efcfc38 100644
--- a/daemon/telegram/packages/decorator.py
+++ b/daemon/telegram/packages/decorator.py
@@ -24,7 +24,7 @@ def handle_flood(func: Callable[[T], T]) -> Callable[[T], T]:
try:
return await func(*args)
except FloodWait as e:
- # Calling logger attr from the DaemonClient() class
+ # Calling logger attr from the DaemonTelegram() class
logger = args[0].logger
_flood_exceptions(e, logger)
diff --git a/daemon/telegram/packages/plugins/callbacks/del_atom.py b/daemon/telegram/packages/plugins/callbacks/del_atom.py
index b750e1c..84524c2 100644
--- a/daemon/telegram/packages/plugins/callbacks/del_atom.py
+++ b/daemon/telegram/packages/plugins/callbacks/del_atom.py
@@ -3,14 +3,14 @@
# Copyright (C) 2022 Muhammad Rizki <[email protected]>
#
-from telegram.packages import DaemonClient
+from telegram.packages import DaemonTelegram
from atom import utils
from pyrogram.types import CallbackQuery
from telegram import config
[email protected]_callback_query(config.admin_only, group=1)
-async def on_del_atom(c: DaemonClient, cb: CallbackQuery):
[email protected]_callback_query(config.admin_only, group=1)
+async def on_del_atom(c: DaemonTelegram, cb: CallbackQuery):
if not "del_atom" in cb.data:
return
diff --git a/daemon/telegram/packages/plugins/callbacks/del_chat.py b/daemon/telegram/packages/plugins/callbacks/del_chat.py
index 90b557e..c7135e0 100644
--- a/daemon/telegram/packages/plugins/callbacks/del_chat.py
+++ b/daemon/telegram/packages/plugins/callbacks/del_chat.py
@@ -3,14 +3,14 @@
# Copyright (C) 2022 Muhammad Rizki <[email protected]>
#
-from telegram.packages import DaemonClient
+from telegram.packages import DaemonTelegram
from atom import utils
from pyrogram.types import CallbackQuery
from telegram import config
[email protected]_callback_query(config.admin_only, group=2)
-async def on_del_chat(c: DaemonClient, cb: CallbackQuery):
[email protected]_callback_query(config.admin_only, group=2)
+async def on_del_chat(c: DaemonTelegram, cb: CallbackQuery):
if not "del_chat" in cb.data:
return
diff --git a/daemon/telegram/packages/plugins/commands/manage_atom.py b/daemon/telegram/packages/plugins/commands/manage_atom.py
index 99df7f7..f686adf 100644
--- a/daemon/telegram/packages/plugins/commands/manage_atom.py
+++ b/daemon/telegram/packages/plugins/commands/manage_atom.py
@@ -5,16 +5,16 @@
from pyrogram.types import Message
from pyrogram import filters
-from telegram.packages import DaemonClient
+from telegram.packages import DaemonTelegram
from atom import utils
from telegram import config
[email protected]_message(
[email protected]_message(
filters.command("add_atom") &
config.admin_only
)
-async def add_atom_url(c: DaemonClient, m: Message):
+async def add_atom_url(c: DaemonTelegram, m: Message):
if len(m.command) <= 1:
tutor = "Please specify the URL\n"
tutor += "Example: `/add_atom https://lore.kernel.org/linux-sgx/new.atom`"
@@ -32,11 +32,11 @@ async def add_atom_url(c: DaemonClient, m: Message):
await m.reply(f"Success add **{text}** for listening new email")
[email protected]_message(
[email protected]_message(
filters.command("del_atom") &
config.admin_only
)
-async def del_atom_url(c: DaemonClient, m: Message):
+async def del_atom_url(c: DaemonTelegram, m: Message):
atoms = c.db.get_atom_urls()
if len(atoms) == 0:
return await m.reply("Currently empty.")
diff --git a/daemon/telegram/packages/plugins/commands/manage_broadcast.py b/daemon/telegram/packages/plugins/commands/manage_broadcast.py
index 0aa70de..63f9547 100644
--- a/daemon/telegram/packages/plugins/commands/manage_broadcast.py
+++ b/daemon/telegram/packages/plugins/commands/manage_broadcast.py
@@ -5,16 +5,16 @@
from pyrogram.types import Message
from pyrogram import filters, enums
-from telegram.packages import DaemonClient
+from telegram.packages import DaemonTelegram
from atom import utils
from telegram import config
[email protected]_message(
[email protected]_message(
filters.command("add_bc") &
config.admin_only
)
-async def add_broadcast(c: DaemonClient, m: Message):
+async def add_broadcast(c: DaemonTelegram, m: Message):
if m.chat.type == enums.ChatType.PRIVATE:
chat_name = m.chat.first_name
else:
@@ -37,11 +37,11 @@ async def add_broadcast(c: DaemonClient, m: Message):
await m.reply(msg)
[email protected]_message(
[email protected]_message(
filters.command("del_bc") &
config.admin_only
)
-async def del_broadcast(c: DaemonClient, m: Message):
+async def del_broadcast(c: DaemonTelegram, m: Message):
if "--list" in m.text:
chats = c.db.get_broadcast_chats()
if len(chats) == 0:
diff --git a/daemon/telegram/packages/plugins/commands/scrape.py b/daemon/telegram/packages/plugins/commands/scrape.py
index f89727d..89581b6 100644
--- a/daemon/telegram/packages/plugins/commands/scrape.py
+++ b/daemon/telegram/packages/plugins/commands/scrape.py
@@ -6,7 +6,7 @@
from pyrogram.types import Message
from pyrogram import filters
-from telegram.packages import DaemonClient
+from telegram.packages import DaemonTelegram
from atom import Scraper
from atom import utils
from enums import Platform
@@ -22,11 +22,11 @@ import asyncio
# .lore https://lore.kernel.org/path/message_id/raw
#
LORE_CMD_URL_PATTERN = r"^(?:\/|\.|\!)lore\s+(https?:\/\/lore\.kernel\.org\/\S+)"
[email protected]_message(
[email protected]_message(
filters.regex(LORE_CMD_URL_PATTERN) &
config.admin_only
)
-async def scrap_email(c: DaemonClient, m: Message):
+async def scrap_email(c: DaemonTelegram, m: Message):
p = re.search(LORE_CMD_URL_PATTERN, m.text)
if not p:
return
diff --git a/daemon/tg.py b/daemon/tg.py
index a676cf5..9f36e54 100644
--- a/daemon/tg.py
+++ b/daemon/tg.py
@@ -9,7 +9,7 @@ from dotenv import load_dotenv
from mysql import connector
from pyrogram import idle
from atom import Scraper
-from telegram.packages import DaemonClient
+from telegram.packages import DaemonTelegram
from telegram.mailer import BotMutexes
from telegram.mailer import Bot
from logger import BotLogger
@@ -28,7 +28,7 @@ def main():
else:
port = int(port)
- client = DaemonClient(
+ client = DaemonTelegram(
"telegram/storage/EmailScraper",
api_id=int(os.getenv("API_ID")),
api_hash=os.getenv("API_HASH"),
--
Muhammad Rizki
^ permalink raw reply related [flat|nested] 25+ messages in thread