From 71884389e16339a70035aa6cabd776c6c8f292b7 Mon Sep 17 00:00:00 2001 From: Hinrich Mahler Date: Sun, 25 Oct 2020 19:35:01 +0100 Subject: [PATCH 01/15] Get started on refactoring MQ --- telegram/bot.py | 400 +++++++++++++++++++++++++++++++++-- telegram/ext/defaults.py | 46 +++- telegram/ext/messagequeue.py | 100 +++++---- telegram/utils/promise.py | 17 +- tests/test_defaults.py | 4 + tests/test_messagequeue.py | 5 +- tests/test_official.py | 1 + 7 files changed, 494 insertions(+), 79 deletions(-) diff --git a/telegram/bot.py b/telegram/bot.py index 03e39889b51..5e91a3c9162 100644 --- a/telegram/bot.py +++ b/telegram/bot.py @@ -92,7 +92,7 @@ ) if TYPE_CHECKING: - from telegram.ext import Defaults + from telegram.ext import Defaults, MessageQueue RT = TypeVar('RT') @@ -101,10 +101,10 @@ def info(func: Callable[..., RT]) -> Callable[..., RT]: @functools.wraps(func) def decorator(self: 'Bot', *args: Any, **kwargs: Any) -> RT: if not self.bot: - self.get_me() + self.get_me(delay_queue=None) if self._commands is None: - self.get_my_commands() + self.get_my_commands(delay_queue=None) result = func(self, *args, **kwargs) return result @@ -125,6 +125,50 @@ def decorator(self: 'Bot', *args: Any, **kwargs: Any) -> RT: return decorate(func, decorator) +def mq(func: Callable[..., RT], *args: Any, **kwargs: Any) -> Callable[..., RT]: + def decorator(self: Union[Callable, Bot], *args: Any, **kwargs: Any) -> RT: + if callable(self): + self = cast('Bot', args[0]) + + if not self.message_queue or not self.message_queue.running: + return func(*args, **kwargs) + + delay_queue = kwargs.pop('delay_queue', None) + if not delay_queue: + return func(*args, **kwargs) + + if delay_queue == self.message_queue.DEFAULT_QUEUE: + # For default queue, check if we're in a group setting or not + arg_spec = inspect.getfullargspec(func) + chat_id: Union[str, int] = '' + if 'chat_id' in kwargs: + chat_id = kwargs['chat_id'] + elif 'chat_id' in arg_spec.args: + idx = arg_spec.args.index('chat_id') + chat_id = args[idx] + + if not chat_id: + is_group = False + elif isinstance(chat_id, str) and chat_id.startswith('@'): + is_group = True + else: + try: + is_group = int(chat_id) < 0 + except ValueError: + is_group = False + + queue = self.message_queue.GROUP_QUEUE if is_group else delay_queue + return self.message_queue.process( # type: ignore[return-value] + func, queue, self, *args, **kwargs + ) + + return self.message_queue.process( # type: ignore[return-value] + func, delay_queue, self, *args, **kwargs + ) + + return decorate(func, decorator) + + class Bot(TelegramObject): """This object represents a Telegram Bot. @@ -138,12 +182,19 @@ class Bot(TelegramObject): private_key_password (:obj:`bytes`, optional): Password for above private key. defaults (:class:`telegram.ext.Defaults`, optional): An object containing default values to be used if not set explicitly in the bot methods. + message_queue (:class:`telegram.ext.MessageQueue`, optional): A message queue to pass + requests through in order to avoid flood limits. Note: - Most bot methods have the argument ``api_kwargs`` which allows to pass arbitrary keywords - to the Telegram API. This can be used to access new features of the API before they were - incorporated into PTB. However, this is not guaranteed to work, i.e. it will fail for - passing files. + * Most bot methods have the argument ``api_kwargs`` which allows to pass arbitrary keywords + to the Telegram API. This can be used to access new features of the API before they were + incorporated into PTB. However, this is not guaranteed to work, i.e. it will fail for + passing files. + * Most bot methods have the argument ``delay_queue`` which allows you to pass the request + to the specified delay queue of the :attr:`message_queue`. This will have an effect only, + if the :class:`telegram.ext.MessageQueue` is set and running. When passing a request + through the :attr:`message_queue`, the bot method will return a + :class:`telegram.utils.Promise` instead of the documented return value. """ @@ -172,6 +223,9 @@ def __new__(cls, *args: Any, **kwargs: Any) -> 'Bot': for kwarg_name in needs_default if (getattr(defaults, kwarg_name) is not DEFAULT_NONE) } + # ... do some special casing for delay_queue because that may depend on the method + if 'delay_queue' in default_kwargs: + default_kwargs['delay_queue'] = defaults.delay_queue_per_method[method_name] # ... apply the defaults using a partial if default_kwargs: setattr(instance, method_name, functools.partial(method, **default_kwargs)) @@ -187,11 +241,12 @@ def __init__( private_key: bytes = None, private_key_password: bytes = None, defaults: 'Defaults' = None, + message_queue: 'MessageQueue' = None, ): self.token = self._validate_token(token) - # Gather default self.defaults = defaults + self.message_queue = message_queue if base_url is None: base_url = 'https://api.telegram.org/bot' @@ -354,7 +409,10 @@ def name(self) -> str: return '@{}'.format(self.username) @log - def get_me(self, timeout: int = None, api_kwargs: JSONDict = None) -> Optional[User]: + @mq + def get_me( + self, timeout: int = None, api_kwargs: JSONDict = None, delay_queue: str = None + ) -> Optional[User]: """A simple method for testing your bot's auth token. Requires no parameters. Args: @@ -363,6 +421,8 @@ def get_me(self, timeout: int = None, api_kwargs: JSONDict = None) -> Optional[U the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.User`: A :class:`telegram.User` instance representing that bot if the @@ -379,6 +439,7 @@ def get_me(self, timeout: int = None, api_kwargs: JSONDict = None) -> Optional[U return self.bot @log + @mq def send_message( self, chat_id: Union[int, str], @@ -390,6 +451,7 @@ def send_message( reply_markup: ReplyMarkup = None, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Optional[Message]: """Use this method to send text messages. @@ -415,6 +477,8 @@ def send_message( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, the sent message is returned. @@ -441,12 +505,14 @@ def send_message( ) @log + @mq def delete_message( self, chat_id: Union[str, int], message_id: Union[str, int], timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """ Use this method to delete a message, including service messages, with the following @@ -471,6 +537,8 @@ def delete_message( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -486,6 +554,7 @@ def delete_message( return result # type: ignore[return-value] @log + @mq def forward_message( self, chat_id: Union[int, str], @@ -494,6 +563,7 @@ def forward_message( disable_notification: bool = False, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Optional[Message]: """Use this method to forward messages of any kind. @@ -510,6 +580,8 @@ def forward_message( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, the sent Message is returned. @@ -536,6 +608,7 @@ def forward_message( ) @log + @mq def send_photo( self, chat_id: int, @@ -547,6 +620,7 @@ def send_photo( timeout: float = 20, parse_mode: str = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Optional[Message]: """Use this method to send photos. @@ -577,6 +651,8 @@ def send_photo( timeout (:obj:`int` | :obj:`float`, optional): Send file timeout (default: 20 seconds). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, the sent Message is returned. @@ -609,6 +685,7 @@ def send_photo( ) @log + @mq def send_audio( self, chat_id: Union[int, str], @@ -624,6 +701,7 @@ def send_audio( parse_mode: str = None, thumb: FileLike = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Optional[Message]: """ Use this method to send audio files, if you want Telegram clients to display them in the @@ -669,6 +747,8 @@ def send_audio( timeout (:obj:`int` | :obj:`float`, optional): Send file timeout (default: 20 seconds). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, the sent Message is returned. @@ -712,6 +792,7 @@ def send_audio( ) @log + @mq def send_document( self, chat_id: Union[int, str], @@ -725,6 +806,7 @@ def send_document( parse_mode: str = None, thumb: FileLike = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Optional[Message]: """ Use this method to send general files. @@ -766,6 +848,8 @@ def send_document( timeout (:obj:`int` | :obj:`float`, optional): Send file timeout (default: 20 seconds). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, the sent Message is returned. @@ -803,6 +887,7 @@ def send_document( ) @log + @mq def send_sticker( self, chat_id: Union[int, str], @@ -812,6 +897,7 @@ def send_sticker( reply_markup: ReplyMarkup = None, timeout: float = 20, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Optional[Message]: """ Use this method to send static .WEBP or animated .TGS stickers. @@ -838,6 +924,8 @@ def send_sticker( timeout (:obj:`int` | :obj:`float`, optional): Send file timeout (default: 20 seconds). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, the sent Message is returned. @@ -865,6 +953,7 @@ def send_sticker( ) @log + @mq def send_video( self, chat_id: Union[int, str], @@ -881,6 +970,7 @@ def send_video( supports_streaming: bool = None, thumb: FileLike = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Optional[Message]: """ Use this method to send video files, Telegram clients support mp4 videos @@ -929,6 +1019,8 @@ def send_video( timeout (:obj:`int` | :obj:`float`, optional): Send file timeout (default: 20 seconds). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, the sent Message is returned. @@ -974,6 +1066,7 @@ def send_video( ) @log + @mq def send_video_note( self, chat_id: Union[int, str], @@ -986,6 +1079,7 @@ def send_video_note( timeout: float = 20, thumb: FileLike = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Optional[Message]: """ As of v.4.0, Telegram clients support rounded square mp4 videos of up to 1 minute long. @@ -1024,6 +1118,8 @@ def send_video_note( timeout (:obj:`int` | :obj:`float`, optional): Send file timeout (default: 20 seconds). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, the sent Message is returned. @@ -1061,6 +1157,7 @@ def send_video_note( ) @log + @mq def send_animation( self, chat_id: Union[int, str], @@ -1076,6 +1173,7 @@ def send_animation( reply_markup: ReplyMarkup = None, timeout: float = 20, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Optional[Message]: """ Use this method to send animation files (GIF or H.264/MPEG-4 AVC video without sound). @@ -1118,6 +1216,8 @@ def send_animation( timeout (:obj:`int` | :obj:`float`, optional): Send file timeout (default: 20 seconds). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, the sent Message is returned. @@ -1161,6 +1261,7 @@ def send_animation( ) @log + @mq def send_voice( self, chat_id: Union[int, str], @@ -1173,6 +1274,7 @@ def send_voice( timeout: float = 20, parse_mode: str = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Optional[Message]: """ Use this method to send audio files, if you want Telegram clients to display the file @@ -1208,6 +1310,8 @@ def send_voice( timeout (:obj:`int` | :obj:`float`, optional): Send file timeout (default: 20 seconds). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, the sent Message is returned. @@ -1242,6 +1346,7 @@ def send_voice( ) @log + @mq def send_media_group( self, chat_id: Union[int, str], @@ -1250,6 +1355,7 @@ def send_media_group( reply_to_message_id: Union[int, str] = None, timeout: float = 20, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> List[Optional[Message]]: """Use this method to send a group of photos or videos as an album. @@ -1265,6 +1371,8 @@ def send_media_group( timeout (:obj:`int` | :obj:`float`, optional): Send file timeout (default: 20 seconds). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: List[:class:`telegram.Message`]: An array of the sent Messages. @@ -1295,6 +1403,7 @@ def send_media_group( return [Message.de_json(res, self) for res in result] # type: ignore @log + @mq def send_location( self, chat_id: Union[int, str], @@ -1307,6 +1416,7 @@ def send_location( location: Location = None, live_period: int = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Optional[Message]: """Use this method to send point on the map. @@ -1333,6 +1443,8 @@ def send_location( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, the sent Message is returned. @@ -1371,6 +1483,7 @@ def send_location( ) @log + @mq def edit_message_live_location( self, chat_id: Union[str, int] = None, @@ -1382,6 +1495,7 @@ def edit_message_live_location( reply_markup: ReplyMarkup = None, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Union[Optional[Message], bool]: """Use this method to edit live location messages sent by the bot or via the bot (for inline bots). A location can be edited until its :attr:`live_period` expires or @@ -1408,6 +1522,8 @@ def edit_message_live_location( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, if edited message is sent by the bot, the @@ -1444,6 +1560,7 @@ def edit_message_live_location( ) @log + @mq def stop_message_live_location( self, chat_id: Union[str, int] = None, @@ -1452,6 +1569,7 @@ def stop_message_live_location( reply_markup: ReplyMarkup = None, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Union[Optional[Message], bool]: """Use this method to stop updating a live location message sent by the bot or via the bot (for inline bots) before live_period expires. @@ -1471,6 +1589,8 @@ def stop_message_live_location( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, if edited message is sent by the bot, the @@ -1494,6 +1614,7 @@ def stop_message_live_location( ) @log + @mq def send_venue( self, chat_id: Union[int, str], @@ -1509,6 +1630,7 @@ def send_venue( venue: Venue = None, foursquare_type: str = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Optional[Message]: """Use this method to send information about a venue. @@ -1541,6 +1663,8 @@ def send_venue( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, the sent Message is returned. @@ -1587,6 +1711,7 @@ def send_venue( ) @log + @mq def send_contact( self, chat_id: Union[int, str], @@ -1600,6 +1725,7 @@ def send_contact( contact: Contact = None, vcard: str = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Optional[Message]: """Use this method to send phone contacts. @@ -1628,6 +1754,8 @@ def send_contact( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, the sent Message is returned. @@ -1669,6 +1797,7 @@ def send_contact( ) @log + @mq def send_game( self, chat_id: Union[int, str], @@ -1678,6 +1807,7 @@ def send_game( reply_markup: ReplyMarkup = None, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Optional[Message]: """Use this method to send a game. @@ -1698,6 +1828,8 @@ def send_game( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, the sent Message is returned. @@ -1719,12 +1851,14 @@ def send_game( ) @log + @mq def send_chat_action( self, chat_id: Union[str, int], action: str, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """ Use this method when you need to tell the user that something is happening on the bot's @@ -1743,6 +1877,8 @@ def send_chat_action( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -1758,6 +1894,7 @@ def send_chat_action( return result # type: ignore[return-value] @log + @mq def answer_inline_query( self, inline_query_id: str, @@ -1770,6 +1907,7 @@ def answer_inline_query( timeout: float = None, current_offset: str = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """ Use this method to send answers to an inline query. No more than 50 results per query are @@ -1811,6 +1949,8 @@ def answer_inline_query( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Example: An inline bot that sends YouTube videos can ask the user to connect the bot to their @@ -1913,6 +2053,7 @@ def _set_defaults(res): ) @log + @mq def get_user_profile_photos( self, user_id: Union[str, int], @@ -1920,6 +2061,7 @@ def get_user_profile_photos( limit: int = 100, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Optional[UserProfilePhotos]: """Use this method to get a list of profile pictures for a user. @@ -1934,6 +2076,8 @@ def get_user_profile_photos( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.UserProfilePhotos` @@ -1954,6 +2098,7 @@ def get_user_profile_photos( return UserProfilePhotos.de_json(result, self) # type: ignore @log + @mq def get_file( self, file_id: Union[ @@ -1961,6 +2106,7 @@ def get_file( ], timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> File: """ Use this method to get basic info about a file and prepare it for downloading. For the @@ -1987,6 +2133,8 @@ def get_file( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.File` @@ -2012,6 +2160,7 @@ def get_file( return File.de_json(result, self) # type: ignore @log + @mq def kick_chat_member( self, chat_id: Union[str, int], @@ -2019,6 +2168,7 @@ def kick_chat_member( timeout: float = None, until_date: Union[int, datetime] = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """ Use this method to kick a user from a group or a supergroup or a channel. In the case of @@ -2040,6 +2190,8 @@ def kick_chat_member( bot will be used. api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool` On success, :obj:`True` is returned. @@ -2062,12 +2214,14 @@ def kick_chat_member( return result # type: ignore[return-value] @log + @mq def unban_chat_member( self, chat_id: Union[str, int], user_id: Union[str, int], timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """Use this method to unban a previously kicked user in a supergroup or channel. @@ -2083,6 +2237,8 @@ def unban_chat_member( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool` On success, :obj:`True` is returned. @@ -2098,6 +2254,7 @@ def unban_chat_member( return result # type: ignore[return-value] @log + @mq def answer_callback_query( self, callback_query_id: str, @@ -2107,6 +2264,7 @@ def answer_callback_query( cache_time: int = None, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """ Use this method to send answers to callback queries sent from inline keyboards. The answer @@ -2137,6 +2295,8 @@ def answer_callback_query( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool` On success, :obj:`True` is returned. @@ -2161,6 +2321,7 @@ def answer_callback_query( return result # type: ignore[return-value] @log + @mq def edit_message_text( self, text: str, @@ -2172,6 +2333,7 @@ def edit_message_text( reply_markup: ReplyMarkup = None, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Union[Optional[Message], bool]: """ Use this method to edit text and game messages sent by the bot or via the bot (for inline @@ -2198,6 +2360,8 @@ def edit_message_text( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, if edited message is sent by the bot, the @@ -2229,6 +2393,7 @@ def edit_message_text( ) @log + @mq def edit_message_caption( self, chat_id: Union[str, int] = None, @@ -2239,6 +2404,7 @@ def edit_message_caption( timeout: float = None, parse_mode: str = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Union[Message, bool]: """ Use this method to edit captions of messages sent by the bot or via the bot @@ -2264,6 +2430,8 @@ def edit_message_caption( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, if edited message is sent by the bot, the @@ -2301,6 +2469,7 @@ def edit_message_caption( ) @log + @mq def edit_message_media( self, chat_id: Union[str, int] = None, @@ -2310,6 +2479,7 @@ def edit_message_media( reply_markup: ReplyMarkup = None, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Union[Message, bool]: """ Use this method to edit animation, audio, document, photo, or video messages. If a @@ -2334,6 +2504,8 @@ def edit_message_media( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, if edited message is sent by the bot, the @@ -2367,6 +2539,7 @@ def edit_message_media( ) @log + @mq def edit_message_reply_markup( self, chat_id: Union[str, int] = None, @@ -2375,6 +2548,7 @@ def edit_message_reply_markup( reply_markup: ReplyMarkup = None, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Union[Message, bool]: """ Use this method to edit only the reply markup of messages sent by the bot or via the bot @@ -2395,6 +2569,8 @@ def edit_message_reply_markup( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, if edited message is sent by the bot, the @@ -2428,6 +2604,7 @@ def edit_message_reply_markup( ) @log + @mq def get_updates( self, offset: int = None, @@ -2436,6 +2613,7 @@ def get_updates( read_latency: float = 2.0, allowed_updates: List[str] = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> List[Update]: """Use this method to receive incoming updates using long polling. @@ -2462,6 +2640,8 @@ def get_updates( updates may be received for a short period of time. api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Note: 1. This method will not work if an outgoing webhook is set up. @@ -2617,8 +2797,13 @@ def delete_webhook(self, timeout: float = None, api_kwargs: JSONDict = None) -> return result # type: ignore[return-value] @log + @mq def leave_chat( - self, chat_id: Union[str, int], timeout: float = None, api_kwargs: JSONDict = None + self, + chat_id: Union[str, int], + timeout: float = None, + api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """Use this method for your bot to leave a group, supergroup or channel. @@ -2630,6 +2815,8 @@ def leave_chat( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool` On success, :obj:`True` is returned. @@ -2645,8 +2832,13 @@ def leave_chat( return result # type: ignore[return-value] @log + @mq def get_chat( - self, chat_id: Union[str, int], timeout: float = None, api_kwargs: JSONDict = None + self, + chat_id: Union[str, int], + timeout: float = None, + api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Chat: """ Use this method to get up to date information about the chat (current name of the user for @@ -2660,6 +2852,8 @@ def get_chat( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Chat` @@ -2678,8 +2872,13 @@ def get_chat( return Chat.de_json(result, self) # type: ignore @log + @mq def get_chat_administrators( - self, chat_id: Union[str, int], timeout: float = None, api_kwargs: JSONDict = None + self, + chat_id: Union[str, int], + timeout: float = None, + api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> List[ChatMember]: """ Use this method to get a list of administrators in a chat. @@ -2692,6 +2891,8 @@ def get_chat_administrators( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: List[:class:`telegram.ChatMember`]: On success, returns a list of ``ChatMember`` @@ -2710,8 +2911,13 @@ def get_chat_administrators( return [ChatMember.de_json(x, self) for x in result] # type: ignore @log + @mq def get_chat_members_count( - self, chat_id: Union[str, int], timeout: float = None, api_kwargs: JSONDict = None + self, + chat_id: Union[str, int], + timeout: float = None, + api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> int: """Use this method to get the number of members in a chat. @@ -2723,6 +2929,8 @@ def get_chat_members_count( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`int`: Number of members in the chat. @@ -2738,12 +2946,14 @@ def get_chat_members_count( return result # type: ignore[return-value] @log + @mq def get_chat_member( self, chat_id: Union[str, int], user_id: Union[str, int], timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> ChatMember: """Use this method to get information about a member of a chat. @@ -2756,6 +2966,8 @@ def get_chat_member( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.ChatMember` @@ -2771,12 +2983,14 @@ def get_chat_member( return ChatMember.de_json(result, self) # type: ignore @log + @mq def set_chat_sticker_set( self, chat_id: Union[str, int], sticker_set_name: str, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """Use this method to set a new group sticker set for a supergroup. The bot must be an administrator in the chat for this to work and must have the appropriate @@ -2793,6 +3007,8 @@ def set_chat_sticker_set( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -2804,8 +3020,13 @@ def set_chat_sticker_set( return result # type: ignore[return-value] @log + @mq def delete_chat_sticker_set( - self, chat_id: Union[str, int], timeout: float = None, api_kwargs: JSONDict = None + self, + chat_id: Union[str, int], + timeout: float = None, + api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """Use this method to delete a group sticker set from a supergroup. The bot must be an administrator in the chat for this to work and must have the appropriate admin rights. @@ -2820,6 +3041,8 @@ def delete_chat_sticker_set( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -2830,6 +3053,7 @@ def delete_chat_sticker_set( return result # type: ignore[return-value] + @log def get_webhook_info(self, timeout: float = None, api_kwargs: JSONDict = None) -> WebhookInfo: """Use this method to get current webhook status. Requires no parameters. @@ -2851,6 +3075,7 @@ def get_webhook_info(self, timeout: float = None, api_kwargs: JSONDict = None) - return WebhookInfo.de_json(result, self) # type: ignore @log + @mq def set_game_score( self, user_id: Union[int, str], @@ -2862,6 +3087,7 @@ def set_game_score( disable_edit_message: bool = None, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Union[Message, bool]: """ Use this method to set the score of the specified user in a game. @@ -2884,6 +3110,8 @@ def set_game_score( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: The edited message, or if the message wasn't sent by the bot @@ -2915,6 +3143,7 @@ def set_game_score( ) @log + @mq def get_game_high_scores( self, user_id: Union[int, str], @@ -2923,6 +3152,7 @@ def get_game_high_scores( inline_message_id: Union[str, int] = None, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> List[GameHighScore]: """ Use this method to get data for high score tables. Will return the score of the specified @@ -2941,6 +3171,8 @@ def get_game_high_scores( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: List[:class:`telegram.GameHighScore`] @@ -2963,6 +3195,7 @@ def get_game_high_scores( return [GameHighScore.de_json(hs, self) for hs in result] # type: ignore @log + @mq def send_invoice( self, chat_id: Union[int, str], @@ -2990,6 +3223,7 @@ def send_invoice( send_email_to_provider: bool = None, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Message: """Use this method to send invoices. @@ -3043,6 +3277,8 @@ def send_invoice( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, the sent Message is returned. @@ -3100,6 +3336,7 @@ def send_invoice( ) @log + @mq def answer_shipping_query( self, shipping_query_id: str, @@ -3108,6 +3345,7 @@ def answer_shipping_query( error_message: str = None, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """ If you sent an invoice requesting a shipping address and the parameter is_flexible was @@ -3130,6 +3368,8 @@ def answer_shipping_query( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -3165,6 +3405,7 @@ def answer_shipping_query( return result # type: ignore[return-value] @log + @mq def answer_pre_checkout_query( self, pre_checkout_query_id: str, @@ -3172,6 +3413,7 @@ def answer_pre_checkout_query( error_message: str = None, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """ Once the user has confirmed their payment and shipping details, the Bot API sends the final @@ -3197,6 +3439,8 @@ def answer_pre_checkout_query( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -3224,6 +3468,7 @@ def answer_pre_checkout_query( return result # type: ignore[return-value] @log + @mq def restrict_chat_member( self, chat_id: Union[str, int], @@ -3232,6 +3477,7 @@ def restrict_chat_member( until_date: Union[int, datetime] = None, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """ Use this method to restrict a user in a supergroup. The bot must be an administrator in @@ -3260,6 +3506,8 @@ def restrict_chat_member( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -3285,6 +3533,7 @@ def restrict_chat_member( return result # type: ignore[return-value] @log + @mq def promote_chat_member( self, chat_id: Union[str, int], @@ -3299,6 +3548,7 @@ def promote_chat_member( can_promote_members: bool = None, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """ Use this method to promote or demote a user in a supergroup or a channel. The bot must be @@ -3332,6 +3582,8 @@ def promote_chat_member( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -3364,12 +3616,14 @@ def promote_chat_member( return result # type: ignore[return-value] @log + @mq def set_chat_permissions( self, chat_id: Union[str, int], permissions: ChatPermissions, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """ Use this method to set default chat permissions for all members. The bot must be an @@ -3385,6 +3639,8 @@ def set_chat_permissions( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -3400,6 +3656,7 @@ def set_chat_permissions( return result # type: ignore[return-value] @log + @mq def set_chat_administrator_custom_title( self, chat_id: Union[int, str], @@ -3407,6 +3664,7 @@ def set_chat_administrator_custom_title( custom_title: str, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """ Use this method to set a custom title for administrators promoted by the bot in a @@ -3423,6 +3681,8 @@ def set_chat_administrator_custom_title( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -3440,8 +3700,13 @@ def set_chat_administrator_custom_title( return result # type: ignore[return-value] @log + @mq def export_chat_invite_link( - self, chat_id: Union[str, int], timeout: float = None, api_kwargs: JSONDict = None + self, + chat_id: Union[str, int], + timeout: float = None, + api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> str: """ Use this method to generate a new invite link for a chat; any previously generated link @@ -3456,6 +3721,8 @@ def export_chat_invite_link( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`str`: New invite link on success. @@ -3471,12 +3738,14 @@ def export_chat_invite_link( return result # type: ignore[return-value] @log + @mq def set_chat_photo( self, chat_id: Union[str, int], photo: FileLike, timeout: float = 20, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """Use this method to set a new profile photo for the chat. @@ -3492,6 +3761,8 @@ def set_chat_photo( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -3511,8 +3782,13 @@ def set_chat_photo( return result # type: ignore[return-value] @log + @mq def delete_chat_photo( - self, chat_id: Union[str, int], timeout: float = None, api_kwargs: JSONDict = None + self, + chat_id: Union[str, int], + timeout: float = None, + api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """ Use this method to delete a chat photo. Photos can't be changed for private chats. The bot @@ -3527,6 +3803,8 @@ def delete_chat_photo( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -3542,12 +3820,14 @@ def delete_chat_photo( return result # type: ignore[return-value] @log + @mq def set_chat_title( self, chat_id: Union[str, int], title: str, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """ Use this method to change the title of a chat. Titles can't be changed for private chats. @@ -3563,6 +3843,8 @@ def set_chat_title( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -3578,12 +3860,14 @@ def set_chat_title( return result # type: ignore[return-value] @log + @mq def set_chat_description( self, chat_id: Union[str, int], description: str, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """ Use this method to change the description of a group, a supergroup or a channel. The bot @@ -3599,6 +3883,8 @@ def set_chat_description( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -3614,6 +3900,7 @@ def set_chat_description( return result # type: ignore[return-value] @log + @mq def pin_chat_message( self, chat_id: Union[str, int], @@ -3621,6 +3908,7 @@ def pin_chat_message( disable_notification: bool = None, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """ Use this method to pin a message in a group, a supergroup, or a channel. @@ -3640,6 +3928,8 @@ def pin_chat_message( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -3658,8 +3948,13 @@ def pin_chat_message( return result # type: ignore[return-value] @log + @mq def unpin_chat_message( - self, chat_id: Union[str, int], timeout: float = None, api_kwargs: JSONDict = None + self, + chat_id: Union[str, int], + timeout: float = None, + api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """ Use this method to unpin a message in a group, a supergroup, or a channel. @@ -3675,6 +3970,8 @@ def unpin_chat_message( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -3690,8 +3987,13 @@ def unpin_chat_message( return result # type: ignore[return-value] @log + @mq def get_sticker_set( - self, name: str, timeout: float = None, api_kwargs: JSONDict = None + self, + name: str, + timeout: float = None, + api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> StickerSet: """Use this method to get a sticker set. @@ -3702,6 +4004,8 @@ def get_sticker_set( creation of the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.StickerSet` @@ -3717,12 +4021,14 @@ def get_sticker_set( return StickerSet.de_json(result, self) # type: ignore @log + @mq def upload_sticker_file( self, user_id: Union[str, int], png_sticker: Union[str, FileLike], timeout: float = 20, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> File: """ Use this method to upload a .png file with a sticker for later use in @@ -3743,6 +4049,8 @@ def upload_sticker_file( creation of the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.File`: On success, the uploaded File is returned. @@ -3761,6 +4069,7 @@ def upload_sticker_file( return File.de_json(result, self) # type: ignore @log + @mq def create_new_sticker_set( self, user_id: Union[str, int], @@ -3773,6 +4082,7 @@ def create_new_sticker_set( timeout: float = 20, tgs_sticker: Union[str, FileLike] = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """ Use this method to create new sticker set owned by a user. @@ -3816,6 +4126,8 @@ def create_new_sticker_set( creation of the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -3848,6 +4160,7 @@ def create_new_sticker_set( return result # type: ignore[return-value] @log + @mq def add_sticker_to_set( self, user_id: Union[str, int], @@ -3858,6 +4171,7 @@ def add_sticker_to_set( timeout: float = 20, tgs_sticker: Union[str, FileLike] = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """ Use this method to add a new sticker to a set created by the bot. @@ -3895,6 +4209,8 @@ def add_sticker_to_set( creation of the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -3925,8 +4241,14 @@ def add_sticker_to_set( return result # type: ignore[return-value] @log + @mq def set_sticker_position_in_set( - self, sticker: str, position: int, timeout: float = None, api_kwargs: JSONDict = None + self, + sticker: str, + position: int, + timeout: float = None, + api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """Use this method to move a sticker in a set created by the bot to a specific position. @@ -3938,6 +4260,8 @@ def set_sticker_position_in_set( creation of the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -3955,8 +4279,13 @@ def set_sticker_position_in_set( return result # type: ignore[return-value] @log + @mq def delete_sticker_from_set( - self, sticker: str, timeout: float = None, api_kwargs: JSONDict = None + self, + sticker: str, + timeout: float = None, + api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """Use this method to delete a sticker from a set created by the bot. @@ -3967,6 +4296,8 @@ def delete_sticker_from_set( creation of the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -3982,6 +4313,7 @@ def delete_sticker_from_set( return result # type: ignore[return-value] @log + @mq def set_sticker_set_thumb( self, name: str, @@ -3989,6 +4321,7 @@ def set_sticker_set_thumb( thumb: FileLike = None, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """Use this method to set the thumbnail of a sticker set. Animated thumbnails can be set for animated sticker sets only. @@ -4012,6 +4345,8 @@ def set_sticker_set_thumb( creation of the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -4032,12 +4367,14 @@ def set_sticker_set_thumb( return result # type: ignore[return-value] @log + @mq def set_passport_data_errors( self, user_id: Union[str, int], errors: List[PassportElementError], timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """ Informs a user that some of the Telegram Passport elements they provided contains errors. @@ -4058,6 +4395,8 @@ def set_passport_data_errors( creation of the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`bool`: On success, :obj:`True` is returned. @@ -4073,6 +4412,7 @@ def set_passport_data_errors( return result # type: ignore[return-value] @log + @mq def send_poll( self, chat_id: Union[int, str], @@ -4092,6 +4432,7 @@ def send_poll( open_period: int = None, close_date: Union[int, datetime] = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Message: """ Use this method to send a native poll. @@ -4137,6 +4478,8 @@ def send_poll( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, the sent Message is returned. @@ -4187,6 +4530,7 @@ def send_poll( ) @log + @mq def stop_poll( self, chat_id: Union[int, str], @@ -4194,6 +4538,7 @@ def stop_poll( reply_markup: ReplyMarkup = None, timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Poll: """ Use this method to stop a poll which was sent by the bot. @@ -4209,6 +4554,8 @@ def stop_poll( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Poll`: On success, the stopped Poll with the final results is @@ -4233,6 +4580,7 @@ def stop_poll( return Poll.de_json(result, self) # type: ignore @log + @mq def send_dice( self, chat_id: Union[int, str], @@ -4242,6 +4590,7 @@ def send_dice( timeout: float = None, emoji: str = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> Message: """ Use this method to send an animated emoji, which will have a random value. On success, the @@ -4264,6 +4613,8 @@ def send_dice( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :class:`telegram.Message`: On success, the sent Message is returned. @@ -4290,8 +4641,9 @@ def send_dice( ) @log + @mq def get_my_commands( - self, timeout: float = None, api_kwargs: JSONDict = None + self, timeout: float = None, api_kwargs: JSONDict = None, delay_queue: str = None ) -> List[BotCommand]: """ Use this method to get the current list of the bot's commands. @@ -4302,6 +4654,8 @@ def get_my_commands( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: List[:class:`telegram.BotCommand]`: On success, the commands set for the bot @@ -4317,11 +4671,13 @@ def get_my_commands( return self._commands @log + @mq def set_my_commands( self, commands: List[Union[BotCommand, Tuple[str, str]]], timeout: float = None, api_kwargs: JSONDict = None, + delay_queue: str = None, ) -> bool: """ Use this method to change the list of the bot's commands. @@ -4335,6 +4691,8 @@ def set_my_commands( the connection pool). api_kwargs (:obj:`dict`, optional): Arbitrary keyword arguments to be passed to the Telegram API. + delay_queue (:obj:`str`, optional): The name of the delay queue to pass this request + through. Defaults to :obj:`None`. Returns: :obj:`True`: On success diff --git a/telegram/ext/defaults.py b/telegram/ext/defaults.py index 6b041db71d6..57a8803d4c3 100644 --- a/telegram/ext/defaults.py +++ b/telegram/ext/defaults.py @@ -17,8 +17,10 @@ # You should have received a copy of the GNU Lesser Public License # along with this program. If not, see [http://www.gnu.org/licenses/]. """This module contains the class Defaults, which allows to pass default values to Updater.""" +from collections import defaultdict + import pytz -from typing import Union, Optional, Any, NoReturn +from typing import Union, Optional, Any, NoReturn, Dict, DefaultDict from telegram.utils.helpers import DEFAULT_NONE, DefaultValue @@ -41,6 +43,12 @@ class Defaults: be ignored. Default: :obj:`True` in group chats and :obj:`False` in private chats. tzinfo (:obj:`tzinfo`): A timezone to be used for all date(time) objects appearing throughout PTB. + delay_queue (:obj:`str`, optional): A :class:`telegram.ext.DelayQueue` the bots + :class:`telegram.ext.MessageQueue` should use. + delay_queue_per_method (Dict[:obj:`str`, :obj:`str`], optional): A dictionary specifying + for each bot method a :class:`telegram.ext.DelayQueue` the bots + :class:`telegram.ext.MessageQueue` should use. Methods not specified here will use + :attr:`delay_queue`. Parameters: parse_mode (:obj:`str`, optional): Send Markdown or HTML, if you want Telegram apps to show @@ -59,6 +67,12 @@ class Defaults: appearing throughout PTB, i.e. if a timezone naive date(time) object is passed somewhere, it will be assumed to be in ``tzinfo``. Must be a timezone provided by the ``pytz`` module. Defaults to UTC. + delay_queue (:obj:`str`, optional): A :class:`telegram.ext.DelayQueue` the bots + :class:`telegram.ext.MessageQueue` should use. Defaults to :obj:`None`. + delay_queue_per_method (Dict[:obj:`str`, :obj:`str`], optional): A dictionary specifying + for each bot method a :class:`telegram.ext.DelayQueue` the bots + :class:`telegram.ext.MessageQueue` should use. Methods not specified here will use + :attr:`delay_queue`. Defaults to :obj:`None`. """ def __init__( @@ -71,6 +85,8 @@ def __init__( timeout: Union[float, DefaultValue] = DEFAULT_NONE, quote: bool = None, tzinfo: pytz.BaseTzInfo = pytz.utc, + delay_queue: str = None, + delay_queue_per_method: Dict[str, Optional[str]] = None, ): self._parse_mode = parse_mode self._disable_notification = disable_notification @@ -78,6 +94,10 @@ def __init__( self._timeout = timeout self._quote = quote self._tzinfo = tzinfo + self._delay_queue = delay_queue + self._delay_queue_per_method = defaultdict( + lambda: self.delay_queue, delay_queue_per_method or {} + ) @property def parse_mode(self) -> Optional[str]: @@ -145,6 +165,28 @@ def tzinfo(self, value: Any) -> NoReturn: "not have any effect." ) + @property + def delay_queue(self) -> Optional[str]: + return self._delay_queue + + @delay_queue.setter + def delay_queue(self, value: Any) -> NoReturn: + raise AttributeError( + "You can not assign a new value to defaults after because it would " + "not have any effect." + ) + + @property + def delay_queue_per_method(self) -> DefaultDict[str, Optional[str]]: + return self._delay_queue_per_method + + @delay_queue_per_method.setter + def delay_queue_per_method(self, value: Any) -> NoReturn: + raise AttributeError( + "You can not assign a new value to defaults after because it would " + "not have any effect." + ) + def __hash__(self) -> int: return hash( ( @@ -154,6 +196,8 @@ def __hash__(self) -> int: self._timeout, self._quote, self._tzinfo, + self._delay_queue, + ((key, value) for key, value in self._delay_queue_per_method), ) ) diff --git a/telegram/ext/messagequeue.py b/telegram/ext/messagequeue.py index d268b18fdb6..faad634e041 100644 --- a/telegram/ext/messagequeue.py +++ b/telegram/ext/messagequeue.py @@ -20,14 +20,15 @@ # You should have received a copy of the GNU Lesser Public License # along with this program. If not, see [http://www.gnu.org/licenses/] """A throughput-limiting message processor for Telegram bots.""" -from telegram.utils import promise import functools import time import threading import queue as q -from typing import Callable, Any, TYPE_CHECKING, List, NoReturn +from typing import Callable, Any, TYPE_CHECKING, List, NoReturn, ClassVar, Dict + +from telegram.utils.promise import Promise if TYPE_CHECKING: from telegram import Bot @@ -107,7 +108,7 @@ def run(self) -> None: times: List[float] = [] # used to store each callable processing time while True: - item = self._queue.get() + promise = self._queue.get() if self.__exit_req: return # shutdown thread # delay routine @@ -124,11 +125,9 @@ def run(self) -> None: if len(times) >= self.burst_limit: # if throughput limit was hit time.sleep(times[1] - t_delta) # finally process one - try: - func, args, kwargs = item - func(*args, **kwargs) - except Exception as exc: # re-route any exceptions - self.exc_route(exc) # to prevent thread exit + promise.run() + if promise.exception: + self.exc_route(promise.exception) # re-route any exceptions def stop(self, timeout: float = None) -> None: """Used to gently stop processor and shutdown its thread. @@ -156,7 +155,7 @@ def _default_exception_handler(exc: Exception) -> NoReturn: raise exc - def __call__(self, func: Callable, *args: Any, **kwargs: Any) -> None: + def process(self, func: Callable, args: Any, kwargs: Any) -> Promise: """Used to process callbacks in throughput-limiting thread through queue. Args: @@ -169,7 +168,9 @@ def __call__(self, func: Callable, *args: Any, **kwargs: Any) -> None: if not self.is_alive() or self.__exit_req: raise DelayQueueError('Could not process callback in stopped thread') - self._queue.put((func, args, kwargs)) + promise = Promise(func, args, kwargs) + self._queue.put(promise) + return promise # The most straightforward way to implement this is to use 2 sequential delay @@ -185,6 +186,9 @@ class MessageQueue: Callables are processed through *group* ``DelayQueue``, then through *all* ``DelayQueue`` for group-type messages. For non-group messages, only the *all* ``DelayQueue`` is used. + Attributes: + running (:obj:`bool`): Whether this message queue has started it's delay queues or not. + Args: all_burst_limit (:obj:`int`, optional): Number of maximum *all-type* callbacks to process per time-window defined by :attr:`all_time_limit_ms`. Defaults to 30. @@ -213,61 +217,48 @@ def __init__( exc_route: Callable[[Exception], None] = None, autostart: bool = True, ): - # create according delay queues, use composition - self._all_delayq = DelayQueue( - burst_limit=all_burst_limit, - time_limit_ms=all_time_limit_ms, - exc_route=exc_route, - autostart=autostart, - ) - self._group_delayq = DelayQueue( - burst_limit=group_burst_limit, - time_limit_ms=group_time_limit_ms, - exc_route=exc_route, - autostart=autostart, - ) + self.running = False + self._delay_queues: Dict[str, DelayQueue] = { + self.DEFAULT_QUEUE: DelayQueue( + burst_limit=all_burst_limit, + time_limit_ms=all_time_limit_ms, + exc_route=exc_route, + autostart=autostart, + ), + self.GROUP_QUEUE: DelayQueue( + burst_limit=group_burst_limit, + time_limit_ms=group_time_limit_ms, + exc_route=exc_route, + autostart=autostart, + ), + } def start(self) -> None: """Method is used to manually start the ``MessageQueue`` processing.""" - self._all_delayq.start() - self._group_delayq.start() + for dq in self._delay_queues.values(): + dq.start() def stop(self, timeout: float = None) -> None: - self._group_delayq.stop(timeout=timeout) - self._all_delayq.stop(timeout=timeout) + for dq in self._delay_queues.values(): + dq.stop() stop.__doc__ = DelayQueue.stop.__doc__ or '' # reuse docstring if any - def __call__(self, promise: Callable, is_group_msg: bool = False) -> Callable: + def process(self, func: Callable, delay_queue: str, *args: Any, **kwargs: Any) -> Promise: """ Processes callables in throughput-limiting queues to avoid hitting limits (specified with :attr:`burst_limit` and :attr:`time_limit`. - Args: - promise (:obj:`callable`): Mainly the ``telegram.utils.promise.Promise`` (see Notes for - other callables), that is processed in delay queues. - is_group_msg (:obj:`bool`, optional): Defines whether ``promise`` would be processed in - group*+*all* ``DelayQueue``s (if set to :obj:`True`), or only through *all* - ``DelayQueue`` (if set to :obj:`False`), resulting in needed delays to avoid - hitting specified limits. Defaults to :obj:`False`. - - Note: - Method is designed to accept ``telegram.utils.promise.Promise`` as ``promise`` - argument, but other callables could be used too. For example, lambdas or simple - functions could be used to wrap original func to be called with needed args. In that - case, be sure that either wrapper func does not raise outside exceptions or the proper - :attr:`exc_route` handler is provided. - Returns: - :obj:`callable`: Used as ``promise`` argument. + :class:`telegram.ext.Promise`. """ + return self._delay_queues[delay_queue].process(func, args, kwargs) - if not is_group_msg: # ignore middle group delay - self._all_delayq(promise) - else: # use middle group delay - self._group_delayq(self._all_delayq, promise) - return promise + DEFAULT_QUEUE: ClassVar[str] = 'default_delay_queue' + """:obj:`str`: The default delay queue.""" + GROUP_QUEUE: ClassVar[str] = 'group_delay_queue' + """:obj:`str`: The default delay queue for group requests.""" def queuedmessage(method: Callable) -> Callable: @@ -307,10 +298,15 @@ def wrapped(self: 'Bot', *args: Any, **kwargs: Any) -> Any: queued = kwargs.pop( 'queued', self._is_messages_queued_default # type: ignore[attr-defined] ) - isgroup = kwargs.pop('isgroup', False) + is_group = kwargs.pop('isgroup', False) if queued: - prom = promise.Promise(method, (self,) + args, kwargs) - return self._msg_queue(prom, isgroup) # type: ignore[attr-defined] + if not is_group: + return self._msg_queue.process( # type: ignore[attr-defined] + method, MessageQueue.DEFAULT_QUEUE, self, *args, **kwargs + ) + return self._msg_queue.process( # type: ignore[attr-defined] + method, MessageQueue.GROUP_QUEUE, self, *args, **kwargs + ) return method(self, *args, **kwargs) return wrapped diff --git a/telegram/utils/promise.py b/telegram/utils/promise.py index 02905ef601b..e875836dbfb 100644 --- a/telegram/utils/promise.py +++ b/telegram/utils/promise.py @@ -20,6 +20,8 @@ import logging from threading import Event + +from telegram import InputFile from telegram.utils.types import JSONDict, HandlerArg from typing import Callable, List, Tuple, Optional, Union, TypeVar @@ -60,9 +62,20 @@ def __init__( update: HandlerArg = None, error_handling: bool = True, ): - self.pooled_function = pooled_function - self.args = args + + parsed_args = [] + for arg in args: + if InputFile.is_file(arg): + parsed_args.append(InputFile(arg)) + else: + parsed_args.append(arg) + self.args = tuple(parsed_args) self.kwargs = kwargs + for key, value in self.kwargs.items(): + if InputFile.is_file(value): + self.kwargs[key] = InputFile(value) + + self.pooled_function = pooled_function self.update = update self.error_handling = error_handling self.done = Event() diff --git a/tests/test_defaults.py b/tests/test_defaults.py index 5344f538d38..2d96684c387 100644 --- a/tests/test_defaults.py +++ b/tests/test_defaults.py @@ -39,6 +39,10 @@ def test_data_assignment(self, cdp): defaults.quote = True with pytest.raises(AttributeError): defaults.tzinfo = True + with pytest.raises(AttributeError): + defaults.delay_queue = True + with pytest.raises(AttributeError): + defaults.delay_queue_per_method = True def test_equality(self): a = Defaults(parse_mode='HTML', quote=True) diff --git a/tests/test_messagequeue.py b/tests/test_messagequeue.py index f9ebfc90159..661d69c4e39 100644 --- a/tests/test_messagequeue.py +++ b/tests/test_messagequeue.py @@ -17,7 +17,6 @@ # You should have received a copy of the GNU Lesser Public License # along with this program. If not, see [http://www.gnu.org/licenses/]. -import os from time import sleep, perf_counter import pytest @@ -26,8 +25,8 @@ @pytest.mark.skipif( - os.getenv('GITHUB_ACTIONS', False) and os.name == 'nt', - reason="On windows precise timings are not accurate.", + True, + reason="Didn't adjust the tests yet.", ) class TestDelayQueue: N = 128 diff --git a/tests/test_official.py b/tests/test_official.py index 39f5864e396..25c853d3622 100644 --- a/tests/test_official.py +++ b/tests/test_official.py @@ -36,6 +36,7 @@ 'timeout', 'bot', 'api_kwargs', + 'delay_queue', } From 315090164473dc587a9592fedc50250b9ff5f326 Mon Sep 17 00:00:00 2001 From: Hinrich Mahler Date: Tue, 27 Oct 2020 17:31:28 +0100 Subject: [PATCH 02/15] fix stuff --- telegram/bot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telegram/bot.py b/telegram/bot.py index 5e91a3c9162..feef65a8889 100644 --- a/telegram/bot.py +++ b/telegram/bot.py @@ -126,7 +126,7 @@ def decorator(self: 'Bot', *args: Any, **kwargs: Any) -> RT: def mq(func: Callable[..., RT], *args: Any, **kwargs: Any) -> Callable[..., RT]: - def decorator(self: Union[Callable, Bot], *args: Any, **kwargs: Any) -> RT: + def decorator(self: Union[Callable, 'Bot'], *args: Any, **kwargs: Any) -> RT: if callable(self): self = cast('Bot', args[0]) From e81f315d1a50e09b35f438d97c42c92dd5385503 Mon Sep 17 00:00:00 2001 From: Hinrich Mahler Date: Thu, 29 Oct 2020 19:26:28 +0100 Subject: [PATCH 03/15] add/remove_dq, error handling & some more untested changes --- telegram/bot.py | 4 +- telegram/ext/dispatcher.py | 53 +++++++------ telegram/ext/messagequeue.py | 147 +++++++++++++++++++++++++++-------- 3 files changed, 145 insertions(+), 59 deletions(-) diff --git a/telegram/bot.py b/telegram/bot.py index feef65a8889..cf9e537c437 100644 --- a/telegram/bot.py +++ b/telegram/bot.py @@ -158,11 +158,11 @@ def decorator(self: Union[Callable, 'Bot'], *args: Any, **kwargs: Any) -> RT: is_group = False queue = self.message_queue.GROUP_QUEUE if is_group else delay_queue - return self.message_queue.process( # type: ignore[return-value] + return self.message_queue.put( # type: ignore[return-value] func, queue, self, *args, **kwargs ) - return self.message_queue.process( # type: ignore[return-value] + return self.message_queue.put( # type: ignore[return-value] func, delay_queue, self, *args, **kwargs ) diff --git a/telegram/ext/dispatcher.py b/telegram/ext/dispatcher.py index c7658e41133..4dde2e54d67 100644 --- a/telegram/ext/dispatcher.py +++ b/telegram/ext/dispatcher.py @@ -264,35 +264,38 @@ def _pooled(self) -> None: promise.run() - if not promise.exception: - self.update_persistence(update=promise.update) - continue + self.post_process_promise(promise) - if isinstance(promise.exception, DispatcherHandlerStop): - self.logger.warning( - 'DispatcherHandlerStop is not supported with async functions; func: %s', - promise.pooled_function.__name__, - ) - continue + def post_process_promise(self, promise: Promise) -> None: + if not promise.exception: + self.update_persistence(update=promise.update) + return - # Avoid infinite recursion of error handlers. - if promise.pooled_function in self.error_handlers: - self.logger.error('An uncaught error was raised while handling the error.') - continue + if isinstance(promise.exception, DispatcherHandlerStop): + self.logger.warning( + 'DispatcherHandlerStop is not supported with async functions; func: %s', + promise.pooled_function.__name__, + ) + return - # Don't perform error handling for a `Promise` with deactivated error handling. This - # should happen only via the deprecated `@run_async` decorator or `Promises` created - # within error handlers - if not promise.error_handling: - self.logger.error('A promise with deactivated error handling raised an error.') - continue + # Avoid infinite recursion of error handlers. + if promise.pooled_function in self.error_handlers: + self.logger.error('An uncaught error was raised while handling the error.') + return - # If we arrive here, an exception happened in the promise and was neither - # DispatcherHandlerStop nor raised by an error handler. So we can and must handle it - try: - self.dispatch_error(promise.update, promise.exception, promise=promise) - except Exception: - self.logger.exception('An uncaught error was raised while handling the error.') + # Don't perform error handling for a `Promise` with deactivated error handling. This + # should happen only via the deprecated `@run_async` decorator or `Promises` created + # within error handlers + if not promise.error_handling: + self.logger.error('A promise with deactivated error handling raised an error.') + return + + # If we arrive here, an exception happened in the promise and was neither + # DispatcherHandlerStop nor raised by an error handler. So we can and must handle it + try: + self.dispatch_error(promise.update, promise.exception, promise=promise) + except Exception: + self.logger.exception('An uncaught error was raised while handling the error.') def run_async( self, func: Callable[..., Any], *args: Any, update: HandlerArg = None, **kwargs: Any diff --git a/telegram/ext/messagequeue.py b/telegram/ext/messagequeue.py index faad634e041..59a90bdc7e2 100644 --- a/telegram/ext/messagequeue.py +++ b/telegram/ext/messagequeue.py @@ -26,12 +26,13 @@ import threading import queue as q -from typing import Callable, Any, TYPE_CHECKING, List, NoReturn, ClassVar, Dict +from typing import Callable, Any, TYPE_CHECKING, List, NoReturn, ClassVar, Dict, Optional from telegram.utils.promise import Promise if TYPE_CHECKING: from telegram import Bot + from telegram.ext import Dispatcher # We need to count < 1s intervals, so the most accurate timer is needed curtime = time.perf_counter @@ -55,10 +56,15 @@ class DelayQueue(threading.Thread): exc_route (:obj:`callable`): A callable, accepting 1 positional argument; used to route exceptions from processor thread to main thread; name (:obj:`str`): Thread's name. + dispatcher (:class:`telegram.ext.Disptacher`): Optional. The dispatcher to use for error + handling. Args: queue (:obj:`Queue`, optional): Used to pass callbacks to thread. Creates ``Queue`` implicitly if not provided. + parent (:class:`telegram.ext.DelayQueue`, optional): Pass another delay queue to put all + requests through that delay queue after they were processed by this queue. Defaults to + :obj:`None`. burst_limit (:obj:`int`, optional): Number of maximum callbacks to process per time-window defined by :attr:`time_limit_ms`. Defaults to 30. time_limit_ms (:obj:`int`, optional): Defines width of time-window used when each @@ -66,7 +72,8 @@ class DelayQueue(threading.Thread): exc_route (:obj:`callable`, optional): A callable, accepting 1 positional argument; used to route exceptions from processor thread to main thread; is called on `Exception` subclass exceptions. If not provided, exceptions are routed through dummy handler, - which re-raises them. + which re-raises them. If :attr:`dispatcher` is set, error handling will *always* be + done by the dispatcher. autostart (:obj:`bool`, optional): If :obj:`True`, processor is started immediately after object's creation; if :obj:`False`, should be started manually by `start` method. Defaults to :obj:`True`. @@ -75,7 +82,7 @@ class DelayQueue(threading.Thread): """ - _instcnt = 0 # instance counter + INSTANCE_COUNT: ClassVar[int] = 0 # instance counter def __init__( self, @@ -85,20 +92,34 @@ def __init__( exc_route: Callable[[Exception], None] = None, autostart: bool = True, name: str = None, + parent: 'DelayQueue' = None, ): self._queue = queue if queue is not None else q.Queue() self.burst_limit = burst_limit self.time_limit = time_limit_ms / 1000 - self.exc_route = exc_route if exc_route is not None else self._default_exception_handler + self.exc_route = exc_route if exc_route else self._default_exception_handler + self.parent = parent + self.dispatcher: Optional['Dispatcher'] = None + self.__exit_req = False # flag to gently exit thread - self.__class__._instcnt += 1 + self.__class__.INSTANCE_COUNT += 1 + if name is None: - name = '{}-{}'.format(self.__class__.__name__, self.__class__._instcnt) + name = '{}-{}'.format('DelayQueue', self.INSTANCE_COUNT) super().__init__(name=name) - self.daemon = False + if autostart: # immediately start processing super().start() + def set_dispatcher(self, dispatcher: 'Dispatcher') -> None: + """ + Sets the dispatcher to use for error handling. + + Args: + dispatcher (:class:`telegram.ext.Dispatcher`): The dispatcher. + """ + self.dispatcher = dispatcher + def run(self) -> None: """ Do not use the method except for unthreaded testing purposes, the method normally is @@ -125,9 +146,14 @@ def run(self) -> None: if len(times) >= self.burst_limit: # if throughput limit was hit time.sleep(times[1] - t_delta) # finally process one - promise.run() - if promise.exception: - self.exc_route(promise.exception) # re-route any exceptions + if self.parent: + self.parent.put(promise) + else: + promise.run() + if self.dispatcher: + self.dispatcher.post_process_promise(promise) + elif promise.exception: + self.exc_route(promise.exception) # re-route any exceptions def stop(self, timeout: float = None) -> None: """Used to gently stop processor and shutdown its thread. @@ -155,20 +181,28 @@ def _default_exception_handler(exc: Exception) -> NoReturn: raise exc - def process(self, func: Callable, args: Any, kwargs: Any) -> Promise: - """Used to process callbacks in throughput-limiting thread through queue. + def put( + self, func: Callable = None, args: Any = None, kwargs: Any = None, promise: Promise = None + ) -> Promise: + """Used to process callbacks in throughput-limiting thread through queue. You must either + pass a :class:`telegram.utils.Promise` or all of ``func``, ``args`` and ``kwargs``. Args: - func (:obj:`callable`): The actual function (or any callable) that is processed through - queue. - *args (:obj:`list`): Variable-length `func` arguments. - **kwargs (:obj:`dict`): Arbitrary keyword-arguments to `func`. + func (:obj:`callable`, optional): The actual function (or any callable) that is + processed through queue. + args (:obj:`list`, optional): Variable-length `func` arguments. + kwargs (:obj:`dict`, optional): Arbitrary keyword-arguments to `func`. + promise (:class:`telegram.utils.Promise`, optional): A promise. """ + if not bool(promise) ^ all([func, args, kwargs]): + raise ValueError('You must pass either a promise or all all func, args, kwargs.') if not self.is_alive() or self.__exit_req: raise DelayQueueError('Could not process callback in stopped thread') - promise = Promise(func, args, kwargs) + + if not promise: + promise = Promise(func, args, kwargs) # type: ignore[arg-type] self._queue.put(promise) return promise @@ -188,6 +222,8 @@ class MessageQueue: Attributes: running (:obj:`bool`): Whether this message queue has started it's delay queues or not. + dispatcher (:class:`telegram.ext.Disptacher`): Optional. The Dispatcher to use for error + handling. Args: all_burst_limit (:obj:`int`, optional): Number of maximum *all-type* callbacks to process @@ -218,33 +254,71 @@ def __init__( autostart: bool = True, ): self.running = False + self.dispatcher: Optional['Dispatcher'] = None self._delay_queues: Dict[str, DelayQueue] = { self.DEFAULT_QUEUE: DelayQueue( burst_limit=all_burst_limit, time_limit_ms=all_time_limit_ms, exc_route=exc_route, autostart=autostart, - ), - self.GROUP_QUEUE: DelayQueue( - burst_limit=group_burst_limit, - time_limit_ms=group_time_limit_ms, - exc_route=exc_route, - autostart=autostart, - ), + name=self.DEFAULT_QUEUE, + ) } + self._delay_queues[self.GROUP_QUEUE] = DelayQueue( + burst_limit=group_burst_limit, + time_limit_ms=group_time_limit_ms, + exc_route=exc_route, + autostart=autostart, + name=self.GROUP_QUEUE, + parent=self._delay_queues[self.DEFAULT_QUEUE], + ) + + def add_delay_queue(self, delay_queue: DelayQueue) -> None: + """ + Adds a new :class:`telegram.ext.DelayQueue` to this message queue. If the message queue is + already running, also starts the delay queue. Also takes care of setting the + :class:`telegram.ext.Dispatcher`, if :attr:`dispatcher` is set. + + Args: + delay_queue (:class:`telegram.ext.DelayQueue`): The delay queue to add. + """ + self._delay_queues[delay_queue.name] = delay_queue + if self.dispatcher: + delay_queue.set_dispatcher(self.dispatcher) + if self.running: + delay_queue.start() + + def remove_delay_queue(self, name: str, timeout: float = None) -> None: + """ + Removes the :class:`telegram.ext.DelayQueue` with the given name. If the message queue is + still running, also stops the delay queue. + + Args: + name (:obj:`str`): The name of the delay queue to remove. + timeout (:obj:`float`, optional): The timeout to pass to + :meth:`telegram.ext.DelayQueue.stop`. + """ + delay_queue = self._delay_queues.pop(name) + if self.running: + delay_queue.stop(timeout) def start(self) -> None: - """Method is used to manually start the ``MessageQueue`` processing.""" + """Starts the all :class:`telegram.ext.DelayQueue` registered for this message queue.""" for dq in self._delay_queues.values(): dq.start() def stop(self, timeout: float = None) -> None: - for dq in self._delay_queues.values(): - dq.stop() + """ + Stops the all :class:`telegram.ext.DelayQueue` registered for this message queue. - stop.__doc__ = DelayQueue.stop.__doc__ or '' # reuse docstring if any + Args: + timeout (:obj:`float`, optional): The timeout to pass to + :meth:`telegram.ext.DelayQueue.stop`. + """ + for dq in self._delay_queues.values(): + dq.stop(timeout) - def process(self, func: Callable, delay_queue: str, *args: Any, **kwargs: Any) -> Promise: + def put(self, func: Callable, delay_queue: str, *args: Any, **kwargs: Any) -> Promise: """ Processes callables in throughput-limiting queues to avoid hitting limits (specified with :attr:`burst_limit` and :attr:`time_limit`. @@ -253,7 +327,16 @@ def process(self, func: Callable, delay_queue: str, *args: Any, **kwargs: Any) - :class:`telegram.ext.Promise`. """ - return self._delay_queues[delay_queue].process(func, args, kwargs) + return self._delay_queues[delay_queue].put(func, args, kwargs) + + def set_dispatcher(self, dispatcher: 'Dispatcher') -> None: + """ + Sets the dispatcher to use for error handling. + + Args: + dispatcher (:class:`telegram.ext.Dispatcher`): The dispatcher. + """ + self.dispatcher = dispatcher DEFAULT_QUEUE: ClassVar[str] = 'default_delay_queue' """:obj:`str`: The default delay queue.""" @@ -301,10 +384,10 @@ def wrapped(self: 'Bot', *args: Any, **kwargs: Any) -> Any: is_group = kwargs.pop('isgroup', False) if queued: if not is_group: - return self._msg_queue.process( # type: ignore[attr-defined] + return self._msg_queue.put( # type: ignore[attr-defined] method, MessageQueue.DEFAULT_QUEUE, self, *args, **kwargs ) - return self._msg_queue.process( # type: ignore[attr-defined] + return self._msg_queue.put( # type: ignore[attr-defined] method, MessageQueue.GROUP_QUEUE, self, *args, **kwargs ) return method(self, *args, **kwargs) From 05ef91d13b208dc060583b9ebaf4e3c156ab6751 Mon Sep 17 00:00:00 2001 From: Hinrich Mahler Date: Sun, 1 Nov 2020 16:08:26 +0100 Subject: [PATCH 04/15] Integrate MQ with Updater --- telegram/ext/messagequeue.py | 97 +++++++++++++++++++++--------------- telegram/ext/updater.py | 16 +++++- 2 files changed, 73 insertions(+), 40 deletions(-) diff --git a/telegram/ext/messagequeue.py b/telegram/ext/messagequeue.py index 3d99e682ebc..5fd5b4ae1aa 100644 --- a/telegram/ext/messagequeue.py +++ b/telegram/ext/messagequeue.py @@ -22,21 +22,21 @@ """A throughput-limiting message processor for Telegram bots.""" import functools +import logging import queue as q import threading import time +import warnings from typing import Callable, Any, TYPE_CHECKING, List, NoReturn, ClassVar, Dict, Optional +from telegram.utils.deprecate import TelegramDeprecationWarning from telegram.utils.promise import Promise if TYPE_CHECKING: from telegram import Bot from telegram.ext import Dispatcher -# We need to count < 1s intervals, so the most accurate timer is needed -curtime = time.perf_counter - class DelayQueueError(RuntimeError): """Indicates processing errors.""" @@ -51,9 +51,9 @@ class DelayQueue(threading.Thread): burst_limit (:obj:`int`): Number of maximum callbacks to process per time-window. time_limit (:obj:`int`): Defines width of time-window used when each processing limit is calculated. - exc_route (:obj:`callable`): A callable, accepting 1 positional argument; used to route - exceptions from processor thread to main thread; name (:obj:`str`): Thread's name. + error_handler (:obj:`callable`): Optional. A callable, accepting 1 positional argument. + Used to route exceptions from processor thread to main thread. dispatcher (:class:`telegram.ext.Disptacher`): Optional. The dispatcher to use for error handling. @@ -67,11 +67,12 @@ class DelayQueue(threading.Thread): defined by :attr:`time_limit_ms`. Defaults to 30. time_limit_ms (:obj:`int`, optional): Defines width of time-window used when each processing limit is calculated. Defaults to 1000. - exc_route (:obj:`callable`, optional): A callable, accepting 1 positional argument; used to - route exceptions from processor thread to main thread; is called on `Exception` + error_handler (:obj:`callable`, optional): A callable, accepting 1 positional argument. + Used to route exceptions from processor thread to main thread. Is called on `Exception` subclass exceptions. If not provided, exceptions are routed through dummy handler, which re-raises them. If :attr:`dispatcher` is set, error handling will *always* be done by the dispatcher. + exc_route (:obj:`callable`, optional): Deprecated alias of :attr:`error_handler`. autostart (:obj:`bool`, optional): If :obj:`True`, processor is started immediately after object's creation; if :obj:`False`, should be started manually by `start` method. Defaults to :obj:`True`. @@ -91,14 +92,25 @@ def __init__( autostart: bool = True, name: str = None, parent: 'DelayQueue' = None, + error_handler: Callable[[Exception], None] = None, ): + self.logger = logging.getLogger(__name__) self._queue = queue if queue is not None else q.Queue() self.burst_limit = burst_limit self.time_limit = time_limit_ms / 1000 - self.exc_route = exc_route if exc_route else self._default_exception_handler self.parent = parent self.dispatcher: Optional['Dispatcher'] = None + if not (bool(exc_route) ^ bool(error_handler)): # pylint: disable=C0325 + raise RuntimeError('Only one of exc_route or error_handler can be passed.') + if exc_route: + warnings.warn( + 'The exc_route argument is deprecated. Use error_handler instead.', + TelegramDeprecationWarning, + stacklevel=2, + ) + self.exc_route = exc_route or error_handler or self._default_exception_handler + self.__exit_req = False # flag to gently exit thread self.__class__.INSTANCE_COUNT += 1 @@ -119,20 +131,17 @@ def set_dispatcher(self, dispatcher: 'Dispatcher') -> None: self.dispatcher = dispatcher def run(self) -> None: - """ - Do not use the method except for unthreaded testing purposes, the method normally is - automatically called by autostart argument. - - """ - times: List[float] = [] # used to store each callable processing time + while True: promise = self._queue.get() if self.__exit_req: return # shutdown thread + # delay routine now = time.perf_counter() t_delta = now - self.time_limit # calculate early to improve perf. + if times and t_delta > times[-1]: # if last call was before the limit time-window # used to impr. perf. in long-interval calls case @@ -143,11 +152,14 @@ def run(self) -> None: times.append(now) if len(times) >= self.burst_limit: # if throughput limit was hit time.sleep(times[1] - t_delta) + # finally process one if self.parent: + # put through parent, if specified self.parent.put(promise) else: promise.run() + # error handling if self.dispatcher: self.dispatcher.post_process_promise(promise) elif promise.exception: @@ -167,16 +179,12 @@ def stop(self, timeout: float = None) -> None: self.__exit_req = True # gently request self._queue.put(None) # put something to unfreeze if frozen + self.logger.debug('Waiting for DelayQueue %s to shut down.', self.name) super().join(timeout=timeout) + self.logger.debug('DelayQueue %s shut down.', self.name) @staticmethod def _default_exception_handler(exc: Exception) -> NoReturn: - """ - Dummy exception handler which re-raises exception in thread. Could be possibly overwritten - by subclasses. - - """ - raise exc def put( @@ -205,18 +213,11 @@ def put( return promise -# The most straightforward way to implement this is to use 2 sequential delay -# queues, like on classic delay chain schematics in electronics. -# So, message path is: -# msg --> group delay if group msg, else no delay --> normal msg delay --> out -# This way OS threading scheduler cares of timings accuracy. -# (see time.time, time.clock, time.perf_counter, time.sleep @ docs.python.org) class MessageQueue: """ Implements callback processing with proper delays to avoid hitting Telegram's message limits. - Contains two ``DelayQueue``, for group and for all messages, interconnected in delay chain. - Callables are processed through *group* ``DelayQueue``, then through *all* ``DelayQueue`` for - group-type messages. For non-group messages, only the *all* ``DelayQueue`` is used. + By default contains two :class:`telegram.ext.DelayQueue` instances, for general requests and + group requests where the default delay queue is the parent of the group requests one. Attributes: running (:obj:`bool`): Whether this message queue has started it's delay queues or not. @@ -232,13 +233,14 @@ class MessageQueue: process per time-window defined by :attr:`group_time_limit_ms`. Defaults to 20. group_time_limit_ms (:obj:`int`, optional): Defines width of *group-type* time-window used when each processing limit is calculated. Defaults to 60000 ms. - exc_route (:obj:`callable`, optional): A callable, accepting one positional argument; used - to route exceptions from processor threads to main thread; is called on ``Exception`` + error_handler (:obj:`callable`, optional): A callable, accepting 1 positional argument. + Used to route exceptions from processor thread to main thread. Is called on `Exception` subclass exceptions. If not provided, exceptions are routed through dummy handler, - which re-raises them. - autostart (:obj:`bool`, optional): If :obj:`True`, processors are started immediately after - object's creation; if :obj:`False`, should be started manually by :attr:`start` method. - Defaults to :obj:`True`. + which re-raises them. If :attr:`dispatcher` is set, error handling will *always* be + done by the dispatcher. + exc_route (:obj:`callable`, optional): Deprecated alias of :attr:`error_handler`. + autostart (:obj:`bool`, optional): If :obj:`True`, both default delay queues are started + immediately after object's creation. Defaults to :obj:`True`. """ @@ -250,14 +252,26 @@ def __init__( group_time_limit_ms: int = 60000, exc_route: Callable[[Exception], None] = None, autostart: bool = True, + error_handler: Callable[[Exception], None] = None, ): + self.logger = logging.getLogger(__name__) self.running = False self.dispatcher: Optional['Dispatcher'] = None + + if not (bool(exc_route) ^ bool(error_handler)): # pylint: disable=C0325 + raise RuntimeError('Only one of exc_route or error_handler can be passed.') + if exc_route: + warnings.warn( + 'The exc_route argument is deprecated. Use error_handler instead.', + TelegramDeprecationWarning, + stacklevel=2, + ) + self._delay_queues: Dict[str, DelayQueue] = { self.DEFAULT_QUEUE: DelayQueue( burst_limit=all_burst_limit, time_limit_ms=all_time_limit_ms, - exc_route=exc_route, + error_handler=exc_route or error_handler, autostart=autostart, name=self.DEFAULT_QUEUE, ) @@ -265,7 +279,7 @@ def __init__( self._delay_queues[self.GROUP_QUEUE] = DelayQueue( burst_limit=group_burst_limit, time_limit_ms=group_time_limit_ms, - exc_route=exc_route, + error_handler=exc_route or error_handler, autostart=autostart, name=self.GROUP_QUEUE, parent=self._delay_queues[self.DEFAULT_QUEUE], @@ -318,8 +332,13 @@ def stop(self, timeout: float = None) -> None: def put(self, func: Callable, delay_queue: str, *args: Any, **kwargs: Any) -> Promise: """ - Processes callables in throughput-limiting queues to avoid hitting limits (specified with - :attr:`burst_limit` and :attr:`time_limit`. + Processes callables in throughput-limiting queues to avoid hitting limits. + + Args: + func (:obj:`callable`): The callable to process + delay_queue (:obj:`str`): The name of the :class:`telegram.ext.DelayQueue` to use. + *args (:obj:`tuple`, optional): Arguments to ``func``. + **kwargs (:obj:`dict`, optional): Keyword arguments to ``func``. Returns: :class:`telegram.ext.Promise`. diff --git a/telegram/ext/updater.py b/telegram/ext/updater.py index 365caaa904c..a9d0e412f79 100644 --- a/telegram/ext/updater.py +++ b/telegram/ext/updater.py @@ -29,7 +29,7 @@ from telegram import Bot, TelegramError from telegram.error import InvalidToken, RetryAfter, TimedOut, Unauthorized -from telegram.ext import Dispatcher, JobQueue +from telegram.ext import Dispatcher, JobQueue, MessageQueue from telegram.utils.deprecate import TelegramDeprecationWarning from telegram.utils.helpers import get_signal_name from telegram.utils.request import Request @@ -94,6 +94,8 @@ class Updater: used). defaults (:class:`telegram.ext.Defaults`, optional): An object containing default values to be used if not set explicitly in the bot methods. + message_queue (:class:`telegram.ext.MessageQueue`, optional): A message queue to use with + the bot. Will be started automatically and the dispatcher will be set. Note: * You must supply either a :attr:`bot` or a :attr:`token` argument. @@ -122,6 +124,7 @@ def __init__( use_context: bool = True, dispatcher: Dispatcher = None, base_file_url: str = None, + message_queue: MessageQueue = None, ): if defaults and bot: @@ -181,6 +184,7 @@ def __init__( private_key=private_key, private_key_password=private_key_password, defaults=defaults, + message_queue=message_queue, ) self.update_queue: Queue = Queue() self.job_queue = JobQueue() @@ -196,6 +200,10 @@ def __init__( use_context=use_context, ) self.job_queue.set_dispatcher(self.dispatcher) + if self.bot.message_queue: + self.bot.message_queue.set_dispatcher(self.dispatcher) + if not self.bot.message_queue.running: + self.bot.message_queue.start() else: con_pool_size = dispatcher.workers + 4 @@ -634,6 +642,7 @@ def stop(self) -> None: self.running = False self._stop_httpd() + self._stop_message_queue() self._stop_dispatcher() self._join_threads() @@ -657,6 +666,11 @@ def _stop_dispatcher(self) -> None: self.logger.debug('Requesting Dispatcher to stop...') self.dispatcher.stop() + def _stop_message_queue(self) -> None: + if self.bot.message_queue: + self.logger.debug('Requesting MessageQueue to stop...') + self.bot.message_queue.stop() + @no_type_check def _join_threads(self) -> None: for thr in self.__threads: From ed86882869bdc0ae9068b5c8610d6c4e137afd27 Mon Sep 17 00:00:00 2001 From: Hinrich Mahler Date: Mon, 23 Nov 2020 00:37:37 +0100 Subject: [PATCH 05/15] Tests for MQ and DQ --- telegram/ext/__init__.py | 4 +- telegram/ext/messagequeue.py | 18 +-- tests/test_messagequeue.py | 279 +++++++++++++++++++++++++++++++---- 3 files changed, 258 insertions(+), 43 deletions(-) diff --git a/telegram/ext/__init__.py b/telegram/ext/__init__.py index b614e292c74..58a4d238a80 100644 --- a/telegram/ext/__init__.py +++ b/telegram/ext/__init__.py @@ -18,6 +18,7 @@ # along with this program. If not, see [http://www.gnu.org/licenses/]. """Extensions over the Telegram Bot API to facilitate bot making""" +from .messagequeue import MessageQueue, DelayQueue, DelayQueueError from .basepersistence import BasePersistence from .picklepersistence import PicklePersistence from .dictpersistence import DictPersistence @@ -39,8 +40,6 @@ from .conversationhandler import ConversationHandler from .precheckoutqueryhandler import PreCheckoutQueryHandler from .shippingqueryhandler import ShippingQueryHandler -from .messagequeue import MessageQueue -from .messagequeue import DelayQueue from .pollanswerhandler import PollAnswerHandler from .pollhandler import PollHandler from .defaults import Defaults @@ -69,6 +68,7 @@ 'ShippingQueryHandler', 'MessageQueue', 'DelayQueue', + 'DelayQueueError', 'DispatcherHandlerStop', 'run_async', 'CallbackContext', diff --git a/telegram/ext/messagequeue.py b/telegram/ext/messagequeue.py index 5fd5b4ae1aa..34b1f054fb4 100644 --- a/telegram/ext/messagequeue.py +++ b/telegram/ext/messagequeue.py @@ -101,8 +101,8 @@ def __init__( self.parent = parent self.dispatcher: Optional['Dispatcher'] = None - if not (bool(exc_route) ^ bool(error_handler)): # pylint: disable=C0325 - raise RuntimeError('Only one of exc_route or error_handler can be passed.') + if exc_route and error_handler: + raise ValueError('Only one of exc_route or error_handler can be passed.') if exc_route: warnings.warn( 'The exc_route argument is deprecated. Use error_handler instead.', @@ -201,7 +201,7 @@ def put( promise (:class:`telegram.utils.Promise`, optional): A promise. """ - if not bool(promise) ^ all([func, args, kwargs]): + if not bool(promise) ^ all(v is not None for v in [func, args, kwargs]): raise ValueError('You must pass either a promise or all all func, args, kwargs.') if not self.is_alive() or self.__exit_req: @@ -254,12 +254,11 @@ def __init__( autostart: bool = True, error_handler: Callable[[Exception], None] = None, ): - self.logger = logging.getLogger(__name__) - self.running = False + self.running = autostart self.dispatcher: Optional['Dispatcher'] = None - if not (bool(exc_route) ^ bool(error_handler)): # pylint: disable=C0325 - raise RuntimeError('Only one of exc_route or error_handler can be passed.') + if exc_route and error_handler: + raise ValueError('Only one of exc_route or error_handler can be passed.') if exc_route: warnings.warn( 'The exc_route argument is deprecated. Use error_handler instead.', @@ -297,7 +296,7 @@ def add_delay_queue(self, delay_queue: DelayQueue) -> None: self._delay_queues[delay_queue.name] = delay_queue if self.dispatcher: delay_queue.set_dispatcher(self.dispatcher) - if self.running: + if self.running and not delay_queue.is_alive(): delay_queue.start() def remove_delay_queue(self, name: str, timeout: float = None) -> None: @@ -311,13 +310,14 @@ def remove_delay_queue(self, name: str, timeout: float = None) -> None: :meth:`telegram.ext.DelayQueue.stop`. """ delay_queue = self._delay_queues.pop(name) - if self.running: + if self.running and delay_queue.is_alive(): delay_queue.stop(timeout) def start(self) -> None: """Starts the all :class:`telegram.ext.DelayQueue` registered for this message queue.""" for delay_queue in self._delay_queues.values(): delay_queue.start() + self.running = True def stop(self, timeout: float = None) -> None: """ diff --git a/tests/test_messagequeue.py b/tests/test_messagequeue.py index 661d69c4e39..5407b5abf3a 100644 --- a/tests/test_messagequeue.py +++ b/tests/test_messagequeue.py @@ -16,54 +16,269 @@ # # You should have received a copy of the GNU Lesser Public License # along with this program. If not, see [http://www.gnu.org/licenses/]. - from time import sleep, perf_counter import pytest -import telegram.ext.messagequeue as mq +from telegram.ext import MessageQueue, DelayQueue, DelayQueueError -@pytest.mark.skipif( - True, - reason="Didn't adjust the tests yet.", -) class TestDelayQueue: N = 128 burst_limit = 30 time_limit_ms = 1000 margin_ms = 0 - testtimes = [] + test_times = [] + test_flag = None + + @pytest.fixture(autouse=True) + def reset(self): + DelayQueue.INSTANCE_COUNT = 0 + self.test_flag = None def call(self): - self.testtimes.append(perf_counter()) + self.test_times.append(perf_counter()) + + def callback_raises_exception(self): + raise DelayQueueError('TestError') + + def test_auto_start_false(self): + delay_queue = DelayQueue(autostart=False) + assert not delay_queue.is_alive() + + def test_name(self): + delay_queue = DelayQueue(autostart=False) + assert delay_queue.name == 'DelayQueue-1' + delay_queue = DelayQueue(autostart=False) + assert delay_queue.name == 'DelayQueue-2' + delay_queue = DelayQueue(name='test_queue', autostart=False) + assert delay_queue.name == 'test_queue' + + def test_exc_route_deprecation(self, recwarn): + with pytest.raises(ValueError, match='Only one of exc_route or '): + DelayQueue(exc_route=True, error_handler=True, autostart=False) - def test_delayqueue_limits(self): - dsp = mq.DelayQueue( + DelayQueue(exc_route=True, autostart=False) + assert len(recwarn) == 1 + assert str(recwarn[0].message).startswith('The exc_route argument is') + + def test_delay_queue_limits(self): + delay_queue = DelayQueue( burst_limit=self.burst_limit, time_limit_ms=self.time_limit_ms, autostart=True ) - assert dsp.is_alive() is True + assert delay_queue.is_alive() is True + + try: + for _ in range(self.N): + delay_queue.put(self.call, [], {}) + + start_time = perf_counter() + # wait up to 20 sec more than needed + app_end_time = ( + (self.N * self.burst_limit / (1000 * self.time_limit_ms)) + start_time + 20 + ) + while not delay_queue._queue.empty() and perf_counter() < app_end_time: + sleep(1) + assert delay_queue._queue.empty() is True # check loop exit condition + + delay_queue.stop() + assert delay_queue.is_alive() is False + + assert self.test_times or self.N == 0 + passes, fails = [], [] + delta = (self.time_limit_ms - self.margin_ms) / 1000 + for start, stop in enumerate(range(self.burst_limit + 1, len(self.test_times))): + part = self.test_times[start:stop] + if (part[-1] - part[0]) >= delta: + passes.append(part) + else: + fails.append(part) + assert fails == [] + finally: + delay_queue.stop() + + def test_put_errors(self): + delay_queue = DelayQueue(autostart=False) + with pytest.raises(DelayQueueError, match='stopped thread'): + delay_queue.put(promise=True) + + delay_queue.start() + try: + with pytest.raises(ValueError, match='You must pass either'): + delay_queue.put() + with pytest.raises(ValueError, match='You must pass either'): + delay_queue.put(promise=True, args=True, kwargs=True, func=True) + with pytest.raises(ValueError, match='You must pass either'): + delay_queue.put(args=True) + finally: + delay_queue.stop() + + def test_default_error_handler_without_dispatcher(self, monkeypatch): + @staticmethod + def exc_route(exception): + self.test_flag = ( + isinstance(exception, DelayQueueError) and str(exception) == 'TestError' + ) + + monkeypatch.setattr(DelayQueue, '_default_exception_handler', exc_route) + + delay_queue = DelayQueue() + try: + delay_queue.put(self.callback_raises_exception, [], {}) + sleep(1) + assert self.test_flag + finally: + delay_queue.stop() + + def test_custom_error_handler_without_dispatcher(self): + def exc_route(exception): + self.test_flag = ( + isinstance(exception, DelayQueueError) and str(exception) == 'TestError' + ) + + delay_queue = DelayQueue(exc_route=exc_route) + try: + delay_queue.put(self.callback_raises_exception, [], {}) + sleep(1) + assert self.test_flag + finally: + delay_queue.stop() + + def test_custom_error_handler_with_dispatcher(self, cdp): + def error_handler(_, context): + self.test_flag = ( + isinstance(context.error, DelayQueueError) and str(context.error) == 'TestError' + ) + + cdp.add_error_handler(error_handler) + delay_queue = DelayQueue() + delay_queue.set_dispatcher(cdp) + try: + delay_queue.put(self.callback_raises_exception, [], {}) + sleep(1) + assert self.test_flag + finally: + delay_queue.stop() + + def test_parent(self, monkeypatch): + def put(*args, **kwargs): + self.test_flag = True + + parent = DelayQueue(name='parent') + monkeypatch.setattr(parent, 'put', put) + + delay_queue = DelayQueue(parent=parent) + try: + delay_queue.put(self.call, [], {}) + sleep(1) + assert self.test_flag + finally: + parent.stop() + delay_queue.stop() + + +class TestMessageQueue: + test_flag = None + + @pytest.fixture(autouse=True) + def reset(self): + DelayQueue.INSTANCE_COUNT = 0 + self.test_flag = None + + def call(self, arg, kwarg=None): + self.test_flag = arg == 1 and kwarg == 'foo' + + def callback_raises_exception(self): + raise DelayQueueError('TestError') + + def test_auto_start_false(self): + message_queue = MessageQueue(autostart=False) + assert not any(thread.is_alive() for thread in message_queue._delay_queues.values()) + + def test_exc_route_deprecation(self, recwarn): + with pytest.raises(ValueError, match='Only one of exc_route or '): + MessageQueue(exc_route=True, error_handler=True, autostart=False) + + MessageQueue(exc_route=True, autostart=False) + assert len(recwarn) == 1 + assert str(recwarn[0].message).startswith('The exc_route argument is') + + def test_add_delay_queue_autostart_false(self): + message_queue = MessageQueue(autostart=False) + delay_queue = DelayQueue(autostart=False, name='dq') + try: + message_queue.add_delay_queue(delay_queue) + assert 'dq' in message_queue._delay_queues + assert not any(thread.is_alive() for thread in message_queue._delay_queues.values()) + message_queue.start() + assert all(thread.is_alive() for thread in message_queue._delay_queues.values()) + + message_queue.stop() + assert not any(thread.is_alive() for thread in message_queue._delay_queues.values()) + finally: + delay_queue.stop() + message_queue.stop() + + @pytest.mark.parametrize('autostart', [True, False]) + def test_add_delay_queue_autostart_true(self, autostart): + message_queue = MessageQueue() + delay_queue = DelayQueue(name='dq', autostart=autostart) + try: + message_queue.add_delay_queue(delay_queue) + assert 'dq' in message_queue._delay_queues + assert delay_queue.is_alive() + assert all(thread.is_alive() for thread in message_queue._delay_queues.values()) + + message_queue.stop() + assert not any(thread.is_alive() for thread in message_queue._delay_queues.values()) + finally: + delay_queue.stop() + message_queue.stop() + + def test_add_delay_queue_dispatcher(self, dp): + message_queue = MessageQueue(autostart=False) + message_queue.set_dispatcher(dispatcher=dp) + delay_queue = DelayQueue(autostart=False, name='dq') + message_queue.add_delay_queue(delay_queue) + assert delay_queue.dispatcher is dp + + @pytest.mark.parametrize('autostart', [True, False]) + def test_remove_delay_queue(self, autostart): + message_queue = MessageQueue(autostart=autostart) + delay_queue = DelayQueue(name='dq') + try: + message_queue.add_delay_queue(delay_queue) + assert 'dq' in message_queue._delay_queues + assert delay_queue.is_alive() + + message_queue.remove_delay_queue('dq') + assert 'dq' not in message_queue._delay_queues + if autostart: + assert not delay_queue.is_alive() + finally: + delay_queue.stop() + if autostart: + message_queue.stop() + + def test_put(self): + group_flag = None + + message_queue = MessageQueue() + original_put = message_queue._delay_queues[MessageQueue.DEFAULT_QUEUE].put + + def put(*args, **kwargs): + nonlocal group_flag + group_flag = True + return original_put(*args, **kwargs) - for _ in range(self.N): - dsp(self.call) + message_queue._delay_queues[MessageQueue.GROUP_QUEUE].put = put - starttime = perf_counter() - # wait up to 20 sec more than needed - app_endtime = (self.N * self.burst_limit / (1000 * self.time_limit_ms)) + starttime + 20 - while not dsp._queue.empty() and perf_counter() < app_endtime: + try: + message_queue.put(self.call, MessageQueue.GROUP_QUEUE, 1, kwarg='foo') sleep(1) - assert dsp._queue.empty() is True # check loop exit condition - - dsp.stop() - assert dsp.is_alive() is False - - assert self.testtimes or self.N == 0 - passes, fails = [], [] - delta = (self.time_limit_ms - self.margin_ms) / 1000 - for start, stop in enumerate(range(self.burst_limit + 1, len(self.testtimes))): - part = self.testtimes[start:stop] - if (part[-1] - part[0]) >= delta: - passes.append(part) - else: - fails.append(part) - assert fails == [] + assert self.test_flag is True + # make sure that group queue was called, too + assert group_flag is True + finally: + message_queue._delay_queues[MessageQueue.GROUP_QUEUE].put = original_put + message_queue.stop() From ada030a90a6a3dabe37e3031009fcfcbf2a2f3ff Mon Sep 17 00:00:00 2001 From: Hinrich Mahler Date: Mon, 23 Nov 2020 16:31:08 +0100 Subject: [PATCH 06/15] Add deprecation warning to decorator --- telegram/ext/messagequeue.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/telegram/ext/messagequeue.py b/telegram/ext/messagequeue.py index 34b1f054fb4..3af87f41991 100644 --- a/telegram/ext/messagequeue.py +++ b/telegram/ext/messagequeue.py @@ -395,6 +395,13 @@ def queuedmessage(method: Callable) -> Callable: @functools.wraps(method) def wrapped(self: 'Bot', *args: Any, **kwargs: Any) -> Any: + warnings.warn( + 'The @queuedmessage decorator is deprecated. Use the `delay_queue` parameter of' + 'the various bot methods instead.', + TelegramDeprecationWarning, + stacklevel=2, + ) + # pylint: disable=W0212 queued = kwargs.pop( 'queued', self._is_messages_queued_default # type: ignore[attr-defined] From d571d70c1e179ea1695e54310fd7ba9eeb557d82 Mon Sep 17 00:00:00 2001 From: Hinrich Mahler Date: Tue, 24 Nov 2020 20:16:26 +0100 Subject: [PATCH 07/15] tests for MQ-decorator --- telegram/ext/messagequeue.py | 4 +- tests/test_messagequeue.py | 83 +++++++++++++++++++++++++++++++++--- 2 files changed, 78 insertions(+), 9 deletions(-) diff --git a/telegram/ext/messagequeue.py b/telegram/ext/messagequeue.py index 6f721d2aa09..2f89c1cab18 100644 --- a/telegram/ext/messagequeue.py +++ b/telegram/ext/messagequeue.py @@ -115,7 +115,7 @@ def __init__( self.__class__.INSTANCE_COUNT += 1 if name is None: - name = f'{self.__class__.__name__}-{self.__class__._instcnt}' + name = f'{self.__class__.__name__}-{self.__class__.INSTANCE_COUNT}' super().__init__(name=name) if autostart: # immediately start processing @@ -156,7 +156,7 @@ def run(self) -> None: # finally process one if self.parent: # put through parent, if specified - self.parent.put(promise) + self.parent.put(promise=promise) else: promise.run() # error handling diff --git a/tests/test_messagequeue.py b/tests/test_messagequeue.py index 5407b5abf3a..a1fdde8834b 100644 --- a/tests/test_messagequeue.py +++ b/tests/test_messagequeue.py @@ -20,7 +20,9 @@ import pytest +from telegram import Bot from telegram.ext import MessageQueue, DelayQueue, DelayQueueError +from telegram.ext.messagequeue import queuedmessage class TestDelayQueue: @@ -78,7 +80,7 @@ def test_delay_queue_limits(self): (self.N * self.burst_limit / (1000 * self.time_limit_ms)) + start_time + 20 ) while not delay_queue._queue.empty() and perf_counter() < app_end_time: - sleep(1) + sleep(0.5) assert delay_queue._queue.empty() is True # check loop exit condition delay_queue.stop() @@ -125,7 +127,7 @@ def exc_route(exception): delay_queue = DelayQueue() try: delay_queue.put(self.callback_raises_exception, [], {}) - sleep(1) + sleep(0.5) assert self.test_flag finally: delay_queue.stop() @@ -139,7 +141,7 @@ def exc_route(exception): delay_queue = DelayQueue(exc_route=exc_route) try: delay_queue.put(self.callback_raises_exception, [], {}) - sleep(1) + sleep(0.5) assert self.test_flag finally: delay_queue.stop() @@ -155,14 +157,14 @@ def error_handler(_, context): delay_queue.set_dispatcher(cdp) try: delay_queue.put(self.callback_raises_exception, [], {}) - sleep(1) + sleep(0.5) assert self.test_flag finally: delay_queue.stop() def test_parent(self, monkeypatch): def put(*args, **kwargs): - self.test_flag = True + self.test_flag = bool(kwargs.pop('promise', False)) parent = DelayQueue(name='parent') monkeypatch.setattr(parent, 'put', put) @@ -170,7 +172,7 @@ def put(*args, **kwargs): delay_queue = DelayQueue(parent=parent) try: delay_queue.put(self.call, [], {}) - sleep(1) + sleep(0.5) assert self.test_flag finally: parent.stop() @@ -275,10 +277,77 @@ def put(*args, **kwargs): try: message_queue.put(self.call, MessageQueue.GROUP_QUEUE, 1, kwarg='foo') - sleep(1) + sleep(0.5) assert self.test_flag is True # make sure that group queue was called, too assert group_flag is True finally: message_queue._delay_queues[MessageQueue.GROUP_QUEUE].put = original_put message_queue.stop() + + +@pytest.fixture(scope='function') +def mq_bot(bot, monkeypatch): + class MQBot(Bot): + def __init__(self, *args, **kwargs): + self.test = None + self.default_count = 0 + self.group_count = 0 + super().__init__(*args, **kwargs) + # below 2 attributes should be provided for decorator usage + self._is_messages_queued_default = True + self._msg_queue = MessageQueue() + + @queuedmessage + def test_method(self, input, *args, **kwargs): + self.test = input + + bot = MQBot(token=bot.token) + + orig_default_put = bot._msg_queue._delay_queues[MessageQueue.DEFAULT_QUEUE].put + orig_group_put = bot._msg_queue._delay_queues[MessageQueue.GROUP_QUEUE].put + + def step_default_counter(*args, **kwargs): + orig_default_put(*args, **kwargs) + bot.default_count += 1 + + def step_group_counter(*args, **kwargs): + orig_group_put(*args, **kwargs) + bot.group_count += 1 + + monkeypatch.setattr( + bot._msg_queue._delay_queues[MessageQueue.DEFAULT_QUEUE], 'put', step_default_counter + ) + monkeypatch.setattr( + bot._msg_queue._delay_queues[MessageQueue.GROUP_QUEUE], 'put', step_group_counter + ) + yield bot + bot._msg_queue.stop() + + +class TestDecorator: + def test_queued_kwarg(self, mq_bot): + mq_bot.test_method('received', queued=False) + sleep(0.5) + assert mq_bot.default_count == 0 + assert mq_bot.group_count == 0 + assert mq_bot.test == 'received' + + mq_bot.test_method('received1') + sleep(0.5) + assert mq_bot.default_count == 1 + assert mq_bot.group_count == 0 + assert mq_bot.test == 'received1' + + def test_isgroup_kwarg(self, mq_bot): + mq_bot.test_method('received', isgroup=False) + sleep(0.5) + assert mq_bot.default_count == 1 + assert mq_bot.group_count == 0 + assert mq_bot.test == 'received' + + mq_bot.test_method('received1', isgroup=True) + sleep(0.5) + assert mq_bot.default_count == 2 + assert mq_bot.group_count == 1 + assert mq_bot.test == 'received1' From 0140ad7c6e5063e878bfc829e7f491e95538b78d Mon Sep 17 00:00:00 2001 From: Hinrich Mahler Date: Thu, 26 Nov 2020 18:53:21 +0100 Subject: [PATCH 08/15] Get @mq to work & tests --- telegram/bot.py | 40 ++++++++++-------- telegram/ext/messagequeue.py | 1 + tests/test_bot.py | 81 ++++++++++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+), 17 deletions(-) diff --git a/telegram/bot.py b/telegram/bot.py index 41a06cb5602..dd1c79938ba 100644 --- a/telegram/bot.py +++ b/telegram/bot.py @@ -114,12 +114,10 @@ def decorator(self: 'Bot', *args: Any, **kwargs: Any) -> RT: return decorator -def log( - func: Callable[..., RT], *args: Any, **kwargs: Any # pylint: disable=W0613 -) -> Callable[..., RT]: +def log(func: Callable[..., RT]) -> Callable[..., RT]: logger = logging.getLogger(func.__module__) - def decorator(self: 'Bot', *args: Any, **kwargs: Any) -> RT: # pylint: disable=W0613 + def decorator(_: Callable, *args: Any, **kwargs: Any) -> RT: # pylint: disable=W0613 logger.debug('Entering: %s', func.__name__) result = func(*args, **kwargs) logger.debug(result) @@ -129,27 +127,29 @@ def decorator(self: 'Bot', *args: Any, **kwargs: Any) -> RT: # pylint: disable= return decorate(func, decorator) -def mq( - func: Callable[..., RT], *args: Any, **kwargs: Any # pylint: disable=W0613 -) -> Callable[..., RT]: - def decorator(self: Union[Callable, 'Bot'], *args: Any, **kwargs: Any) -> RT: - if callable(self): - self = cast('Bot', args[0]) +def mq(func: Callable[..., RT]) -> Callable[..., RT]: + logger = logging.getLogger(func.__module__) + + def decorator(_: Callable, *args: Any, **kwargs: Any) -> RT: + self = cast('Bot', args[0]) + arg_spec = inspect.getfullargspec(func) + idx = arg_spec.args.index('delay_queue') + delay_queue = args[idx] if not self.message_queue or not self.message_queue.running: + if delay_queue: + logger.warning( + 'Ignoring call to MessageQueue, because it is either not set or not running.' + ) return func(*args, **kwargs) - delay_queue = kwargs.pop('delay_queue', None) if not delay_queue: return func(*args, **kwargs) if delay_queue == self.message_queue.DEFAULT_QUEUE: # For default queue, check if we're in a group setting or not - arg_spec = inspect.getfullargspec(func) chat_id: Union[str, int] = '' - if 'chat_id' in kwargs: - chat_id = kwargs['chat_id'] - elif 'chat_id' in arg_spec.args: + if 'chat_id' in arg_spec.args: idx = arg_spec.args.index('chat_id') chat_id = args[idx] @@ -163,13 +163,19 @@ def decorator(self: Union[Callable, 'Bot'], *args: Any, **kwargs: Any) -> RT: except ValueError: is_group = False + logger.debug( + 'Processing MessageQueue call with chat id %s through the %s queue.', + chat_id, + 'group' if is_group else 'default', + ) + queue = self.message_queue.GROUP_QUEUE if is_group else delay_queue return self.message_queue.put( # type: ignore[return-value] - func, queue, self, *args, **kwargs + func, queue, *args, **kwargs ) return self.message_queue.put( # type: ignore[return-value] - func, delay_queue, self, *args, **kwargs + func, delay_queue, *args, **kwargs ) return decorate(func, decorator) diff --git a/telegram/ext/messagequeue.py b/telegram/ext/messagequeue.py index 2f89c1cab18..98a0bce6a9b 100644 --- a/telegram/ext/messagequeue.py +++ b/telegram/ext/messagequeue.py @@ -329,6 +329,7 @@ def stop(self, timeout: float = None) -> None: """ for delay_queue in self._delay_queues.values(): delay_queue.stop(timeout) + self.running = False def put(self, func: Callable, delay_queue: str, *args: Any, **kwargs: Any) -> Promise: """ diff --git a/tests/test_bot.py b/tests/test_bot.py index 43dfc2b71ff..25a7759f216 100644 --- a/tests/test_bot.py +++ b/tests/test_bot.py @@ -16,6 +16,7 @@ # # You should have received a copy of the GNU Lesser Public License # along with this program. If not, see [http://www.gnu.org/licenses/]. +import logging import time import datetime as dtm from platform import python_implementation @@ -45,7 +46,9 @@ ) from telegram.constants import MAX_INLINE_QUERY_RESULTS from telegram.error import BadRequest, InvalidToken, NetworkError, RetryAfter +from telegram.ext import MessageQueue, DelayQueue from telegram.utils.helpers import from_timestamp, escape_markdown, to_timestamp +from telegram.utils.promise import Promise from tests.conftest import expect_bad_request BASE_TIME = time.time() @@ -93,6 +96,15 @@ def inline_results(): return inline_results_callback() +@pytest.fixture(scope='function') +def mq_bot(bot, monkeypatch): + bot.message_queue = MessageQueue() + bot.message_queue.add_delay_queue(DelayQueue(name='custom_dq')) + yield bot + bot.message_queue.stop() + bot.message_queue = None + + class TestBot: @pytest.mark.parametrize( 'token', @@ -1379,3 +1391,72 @@ def test_set_and_get_my_commands_strings(self, bot): assert bc[0].description == 'descr1' assert bc[1].command == 'cmd2' assert bc[1].description == 'descr2' + + @pytest.mark.parametrize( + 'chat_id,expected', + [ + ('123', 'default'), + (123, 'default'), + ('-123', 'group'), + (-123, 'group'), + ('@supergroup', 'group'), + ('foobar', 'default'), + ], + ) + def test_message_queue_group_chat_detection( + self, mq_bot, monkeypatch, chat_id, expected, caplog + ): + def _post(*args, **kwargs): + pass + + monkeypatch.setattr(mq_bot, '_post', _post) + with caplog.at_level(logging.DEBUG): + result = mq_bot.send_message(chat_id, 'text', delay_queue=MessageQueue.DEFAULT_QUEUE) + assert isinstance(result, Promise) + assert len(caplog.records) == 4 + assert expected in caplog.records[1].getMessage() + + with caplog.at_level(logging.DEBUG): + result = mq_bot.send_message( + text='text', chat_id=chat_id, delay_queue=MessageQueue.DEFAULT_QUEUE + ) + assert isinstance(result, Promise) + assert len(caplog.records) == 8 + assert expected in caplog.records[5].getMessage() + + with caplog.at_level(logging.DEBUG): + assert isinstance(mq_bot.get_me(delay_queue=MessageQueue.DEFAULT_QUEUE), Promise) + assert len(caplog.records) == 12 + assert 'default' in caplog.records[9].getMessage() + + def test_stopped_message_queue(self, mq_bot, caplog): + mq_bot.message_queue.stop() + with caplog.at_level(logging.DEBUG): + result = mq_bot.get_me(delay_queue=MessageQueue.DEFAULT_QUEUE) + assert not isinstance(result, Promise) + assert len(caplog.records) >= 2 + assert 'Ignoring call to MessageQueue' in caplog.records[1].getMessage() + + def test_no_message_queue(self, bot, caplog): + with caplog.at_level(logging.DEBUG): + result = bot.get_me(delay_queue=MessageQueue.DEFAULT_QUEUE) + assert not isinstance(result, Promise) + assert len(caplog.records) >= 2 + assert 'Ignoring call to MessageQueue' in caplog.records[1].getMessage() + + def test_message_queue_custom_delay_queue(self, chat_id, mq_bot, monkeypatch): + test_flag = False + orig_put = mq_bot.message_queue._delay_queues['custom_dq'].put + + def put(*args, **kwargs): + nonlocal test_flag + test_flag = True + return orig_put(*args, **kwargs) + + result = mq_bot.send_message(chat_id, 'general kenobi') + assert not isinstance(result, Promise) + + monkeypatch.setattr(mq_bot.message_queue._delay_queues['custom_dq'], 'put', put) + result = mq_bot.send_message(chat_id, 'hello there', delay_queue='custom_dq') + assert isinstance(result, Promise) + assert test_flag From 6233920eab0914551330e6cef0db720fabb1cbd3 Mon Sep 17 00:00:00 2001 From: Hinrich Mahler Date: Thu, 26 Nov 2020 19:08:33 +0100 Subject: [PATCH 09/15] Test for Updater --- telegram/ext/messagequeue.py | 4 ++-- tests/test_updater.py | 16 ++++++++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/telegram/ext/messagequeue.py b/telegram/ext/messagequeue.py index 98a0bce6a9b..29cdba4942c 100644 --- a/telegram/ext/messagequeue.py +++ b/telegram/ext/messagequeue.py @@ -315,9 +315,9 @@ def remove_delay_queue(self, name: str, timeout: float = None) -> None: def start(self) -> None: """Starts the all :class:`telegram.ext.DelayQueue` registered for this message queue.""" + self.running = True for delay_queue in self._delay_queues.values(): delay_queue.start() - self.running = True def stop(self, timeout: float = None) -> None: """ @@ -327,9 +327,9 @@ def stop(self, timeout: float = None) -> None: timeout (:obj:`float`, optional): The timeout to pass to :meth:`telegram.ext.DelayQueue.stop`. """ + self.running = False for delay_queue in self._delay_queues.values(): delay_queue.stop(timeout) - self.running = False def put(self, func: Callable, delay_queue: str, *args: Any, **kwargs: Any) -> Promise: """ diff --git a/tests/test_updater.py b/tests/test_updater.py index 745836acd3d..b3cae39a64a 100644 --- a/tests/test_updater.py +++ b/tests/test_updater.py @@ -38,13 +38,13 @@ from telegram import TelegramError, Message, User, Chat, Update, Bot from telegram.error import Unauthorized, InvalidToken, TimedOut, RetryAfter -from telegram.ext import Updater, Dispatcher, DictPersistence, Defaults +from telegram.ext import Updater, Dispatcher, DictPersistence, Defaults, MessageQueue from telegram.utils.deprecate import TelegramDeprecationWarning from telegram.utils.webhookhandler import WebhookServer signalskip = pytest.mark.skipif( sys.platform == 'win32', - reason='Can\'t send signals without stopping ' 'whole process on windows', + reason='Can\'t send signals without stopping whole process on windows', ) @@ -583,3 +583,15 @@ def test_mutual_exclude_use_context_dispatcher(self): def test_defaults_warning(self, bot): with pytest.warns(TelegramDeprecationWarning, match='no effect when a Bot is passed'): Updater(bot=bot, defaults=Defaults()) + + def test_message_queue(self, bot, caplog): + updater = Updater(bot.token, message_queue=MessageQueue()) + updater.running = True + try: + assert updater.bot.message_queue.dispatcher is updater.dispatcher + with caplog.at_level(logging.DEBUG): + updater.stop() + assert caplog.records[1].getMessage() == 'Requesting MessageQueue to stop...' + assert not updater.bot.message_queue.running + finally: + updater.stop() From 478d1ffb4657fb588b67cb966fdb4211199bf6e6 Mon Sep 17 00:00:00 2001 From: Hinrich Mahler Date: Thu, 26 Nov 2020 19:21:01 +0100 Subject: [PATCH 10/15] Use tg.constants where possible --- telegram/ext/messagequeue.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/telegram/ext/messagequeue.py b/telegram/ext/messagequeue.py index 29cdba4942c..a9e1065e46a 100644 --- a/telegram/ext/messagequeue.py +++ b/telegram/ext/messagequeue.py @@ -32,6 +32,7 @@ from telegram.utils.deprecate import TelegramDeprecationWarning from telegram.utils.promise import Promise +from telegram import constants if TYPE_CHECKING: from telegram import Bot @@ -64,7 +65,8 @@ class DelayQueue(threading.Thread): requests through that delay queue after they were processed by this queue. Defaults to :obj:`None`. burst_limit (:obj:`int`, optional): Number of maximum callbacks to process per time-window - defined by :attr:`time_limit_ms`. Defaults to 30. + defined by :attr:`time_limit_ms`. Defaults to + :attr:`telegram.constants.MAX_MESSAGES_PER_SECOND`. time_limit_ms (:obj:`int`, optional): Defines width of time-window used when each processing limit is calculated. Defaults to 1000. error_handler (:obj:`callable`, optional): A callable, accepting 1 positional argument. @@ -86,7 +88,7 @@ class DelayQueue(threading.Thread): def __init__( self, queue: q.Queue = None, - burst_limit: int = 30, + burst_limit: int = constants.MAX_MESSAGES_PER_SECOND, time_limit_ms: int = 1000, exc_route: Callable[[Exception], None] = None, autostart: bool = True, @@ -226,11 +228,13 @@ class MessageQueue: Args: all_burst_limit (:obj:`int`, optional): Number of maximum *all-type* callbacks to process - per time-window defined by :attr:`all_time_limit_ms`. Defaults to 30. + per time-window defined by :attr:`all_time_limit_ms`. Defaults to + :attr:`telegram.constants.MAX_MESSAGES_PER_SECOND`. all_time_limit_ms (:obj:`int`, optional): Defines width of *all-type* time-window used when each processing limit is calculated. Defaults to 1000 ms. group_burst_limit (:obj:`int`, optional): Number of maximum *group-type* callbacks to - process per time-window defined by :attr:`group_time_limit_ms`. Defaults to 20. + process per time-window defined by :attr:`group_time_limit_ms`. Defaults to + :attr:`telegram.constants.MAX_MESSAGES_PER_MINUTE_PER_GROUP`. group_time_limit_ms (:obj:`int`, optional): Defines width of *group-type* time-window used when each processing limit is calculated. Defaults to 60000 ms. error_handler (:obj:`callable`, optional): A callable, accepting 1 positional argument. @@ -246,9 +250,9 @@ class MessageQueue: def __init__( self, - all_burst_limit: int = 30, + all_burst_limit: int = constants.MAX_MESSAGES_PER_SECOND, all_time_limit_ms: int = 1000, - group_burst_limit: int = 20, + group_burst_limit: int = constants.MAX_MESSAGES_PER_MINUTE_PER_GROUP, group_time_limit_ms: int = 60000, exc_route: Callable[[Exception], None] = None, autostart: bool = True, From 870a077eaa5758e0c438ba82cdc8deb2ae0d1d41 Mon Sep 17 00:00:00 2001 From: Hinrich Mahler Date: Thu, 26 Nov 2020 19:32:58 +0100 Subject: [PATCH 11/15] Increase coverage --- tests/test_bot.py | 12 ++++++++++++ tests/test_updater.py | 12 ++++++++++++ 2 files changed, 24 insertions(+) diff --git a/tests/test_bot.py b/tests/test_bot.py index 25a7759f216..f96e3c5e989 100644 --- a/tests/test_bot.py +++ b/tests/test_bot.py @@ -1460,3 +1460,15 @@ def put(*args, **kwargs): result = mq_bot.send_message(chat_id, 'hello there', delay_queue='custom_dq') assert isinstance(result, Promise) assert test_flag + + @pytest.mark.timeout(10) + def test_message_queue_context_manager(self, mq_bot, chat_id): + with open('tests/data/telegram.gif', 'rb') as document: + with open('tests/data/telegram.jpg', 'rb') as thumb: + promise = mq_bot.send_document( + chat_id, document, thumb=thumb, delay_queue=MessageQueue.DEFAULT_QUEUE + ) + + message = promise.result() + assert message.document + assert message.document.thumb diff --git a/tests/test_updater.py b/tests/test_updater.py index b3cae39a64a..18b949761cc 100644 --- a/tests/test_updater.py +++ b/tests/test_updater.py @@ -595,3 +595,15 @@ def test_message_queue(self, bot, caplog): assert not updater.bot.message_queue.running finally: updater.stop() + + updater = Updater(bot.token, message_queue=MessageQueue(autostart=False)) + updater.running = True + try: + assert updater.bot.message_queue.running + assert updater.bot.message_queue.dispatcher is updater.dispatcher + with caplog.at_level(logging.DEBUG): + updater.stop() + assert caplog.records[1].getMessage() == 'Requesting MessageQueue to stop...' + assert not updater.bot.message_queue.running + finally: + updater.stop() From 3433f78b30c3abf95267fbcc2b73dde5a557fae5 Mon Sep 17 00:00:00 2001 From: Hinrich Mahler Date: Wed, 23 Dec 2020 12:52:39 +0100 Subject: [PATCH 12/15] Drop context manager support & document MQ.delay_queues --- telegram/bot.py | 4 +- telegram/ext/messagequeue.py | 85 +++++++++++++++++++++++------------- telegram/utils/promise.py | 15 +------ tests/test_bot.py | 28 ++++-------- tests/test_messagequeue.py | 40 ++++++++--------- 5 files changed, 85 insertions(+), 87 deletions(-) diff --git a/telegram/bot.py b/telegram/bot.py index a74439703c9..a10d5cb76a3 100644 --- a/telegram/bot.py +++ b/telegram/bot.py @@ -157,7 +157,7 @@ def decorator(_: Callable, *args: Any, **kwargs: Any) -> RT: if not delay_queue: return func(*args, **kwargs) - if delay_queue == self.message_queue.DEFAULT_QUEUE: + if delay_queue == self.message_queue.DEFAULT_QUEUE_NAME: # For default queue, check if we're in a group setting or not chat_id: Union[str, int] = '' if 'chat_id' in arg_spec.args: @@ -180,7 +180,7 @@ def decorator(_: Callable, *args: Any, **kwargs: Any) -> RT: 'group' if is_group else 'default', ) - queue = self.message_queue.GROUP_QUEUE if is_group else delay_queue + queue = self.message_queue.GROUP_QUEUE_NAME if is_group else delay_queue return self.message_queue.put( # type: ignore[return-value] func, queue, *args, **kwargs ) diff --git a/telegram/ext/messagequeue.py b/telegram/ext/messagequeue.py index a9e1065e46a..b2613f01b9f 100644 --- a/telegram/ext/messagequeue.py +++ b/telegram/ext/messagequeue.py @@ -48,15 +48,10 @@ class DelayQueue(threading.Thread): Processes callbacks from queue with specified throughput limits. Creates a separate thread to process callbacks with delays. - Attributes: - burst_limit (:obj:`int`): Number of maximum callbacks to process per time-window. - time_limit (:obj:`int`): Defines width of time-window used when each processing limit is - calculated. - name (:obj:`str`): Thread's name. - error_handler (:obj:`callable`): Optional. A callable, accepting 1 positional argument. - Used to route exceptions from processor thread to main thread. - dispatcher (:class:`telegram.ext.Disptacher`): Optional. The dispatcher to use for error - handling. + Note: + For most use cases, the :attr:`parent` argument should be set to + :attr:`MessageQueue.DEFAULT_QUEUE_NAME` to ensure that the global flood limits are not + exceeded. Args: queue (:obj:`Queue`, optional): Used to pass callbacks to thread. Creates ``Queue`` @@ -81,6 +76,16 @@ class DelayQueue(threading.Thread): name (:obj:`str`, optional): Thread's name. Defaults to ``'DelayQueue-N'``, where N is sequential number of object created. + Attributes: + burst_limit (:obj:`int`): Number of maximum callbacks to process per time-window. + time_limit (:obj:`int`): Defines width of time-window used when each processing limit is + calculated. + name (:obj:`str`): Thread's name. + error_handler (:obj:`callable`): Optional. A callable, accepting 1 positional argument. + Used to route exceptions from processor thread to main thread. + dispatcher (:class:`telegram.ext.Disptacher`): Optional. The dispatcher to use for error + handling. + """ INSTANCE_COUNT: ClassVar[int] = 0 # instance counter @@ -221,11 +226,6 @@ class MessageQueue: By default contains two :class:`telegram.ext.DelayQueue` instances, for general requests and group requests where the default delay queue is the parent of the group requests one. - Attributes: - running (:obj:`bool`): Whether this message queue has started it's delay queues or not. - dispatcher (:class:`telegram.ext.Disptacher`): Optional. The Dispatcher to use for error - handling. - Args: all_burst_limit (:obj:`int`, optional): Number of maximum *all-type* callbacks to process per time-window defined by :attr:`all_time_limit_ms`. Defaults to @@ -246,6 +246,15 @@ class MessageQueue: autostart (:obj:`bool`, optional): If :obj:`True`, both default delay queues are started immediately after object's creation. Defaults to :obj:`True`. + Attributes: + running (:obj:`bool`): Whether this message queue has started it's delay queues or not. + dispatcher (:class:`telegram.ext.Disptacher`): Optional. The Dispatcher to use for error + handling. + delay_queues (Dict[:obj:`str`, :class:`telegram.ext.DelayQueue`]): A dictionary containing + all registered delay queues, where the keys are the names of the delay queues. By + default includes bot :attr:`default_queue` and :attr:`group_queue` under the keys + :attr:`DEFAULT_QUEUE_NAME` and :attr:`GROUP_QUEUE_NAME`, respectively. + """ def __init__( @@ -270,22 +279,22 @@ def __init__( stacklevel=2, ) - self._delay_queues: Dict[str, DelayQueue] = { - self.DEFAULT_QUEUE: DelayQueue( + self.delay_queues: Dict[str, DelayQueue] = { + self.DEFAULT_QUEUE_NAME: DelayQueue( burst_limit=all_burst_limit, time_limit_ms=all_time_limit_ms, error_handler=exc_route or error_handler, autostart=autostart, - name=self.DEFAULT_QUEUE, + name=self.DEFAULT_QUEUE_NAME, ) } - self._delay_queues[self.GROUP_QUEUE] = DelayQueue( + self.delay_queues[self.GROUP_QUEUE_NAME] = DelayQueue( burst_limit=group_burst_limit, time_limit_ms=group_time_limit_ms, error_handler=exc_route or error_handler, autostart=autostart, - name=self.GROUP_QUEUE, - parent=self._delay_queues[self.DEFAULT_QUEUE], + name=self.GROUP_QUEUE_NAME, + parent=self.delay_queues[self.DEFAULT_QUEUE_NAME], ) def add_delay_queue(self, delay_queue: DelayQueue) -> None: @@ -297,7 +306,7 @@ def add_delay_queue(self, delay_queue: DelayQueue) -> None: Args: delay_queue (:class:`telegram.ext.DelayQueue`): The delay queue to add. """ - self._delay_queues[delay_queue.name] = delay_queue + self.delay_queues[delay_queue.name] = delay_queue if self.dispatcher: delay_queue.set_dispatcher(self.dispatcher) if self.running and not delay_queue.is_alive(): @@ -313,14 +322,14 @@ def remove_delay_queue(self, name: str, timeout: float = None) -> None: timeout (:obj:`float`, optional): The timeout to pass to :meth:`telegram.ext.DelayQueue.stop`. """ - delay_queue = self._delay_queues.pop(name) + delay_queue = self.delay_queues.pop(name) if self.running and delay_queue.is_alive(): delay_queue.stop(timeout) def start(self) -> None: """Starts the all :class:`telegram.ext.DelayQueue` registered for this message queue.""" self.running = True - for delay_queue in self._delay_queues.values(): + for delay_queue in self.delay_queues.values(): delay_queue.start() def stop(self, timeout: float = None) -> None: @@ -332,7 +341,7 @@ def stop(self, timeout: float = None) -> None: :meth:`telegram.ext.DelayQueue.stop`. """ self.running = False - for delay_queue in self._delay_queues.values(): + for delay_queue in self.delay_queues.values(): delay_queue.stop(timeout) def put(self, func: Callable, delay_queue: str, *args: Any, **kwargs: Any) -> Promise: @@ -349,7 +358,7 @@ def put(self, func: Callable, delay_queue: str, *args: Any, **kwargs: Any) -> Pr :class:`telegram.ext.Promise`. """ - return self._delay_queues[delay_queue].put(func, args, kwargs) + return self.delay_queues[delay_queue].put(func, args, kwargs) def set_dispatcher(self, dispatcher: 'Dispatcher') -> None: """ @@ -360,10 +369,24 @@ def set_dispatcher(self, dispatcher: 'Dispatcher') -> None: """ self.dispatcher = dispatcher - DEFAULT_QUEUE: ClassVar[str] = 'default_delay_queue' - """:obj:`str`: The default delay queue.""" - GROUP_QUEUE: ClassVar[str] = 'group_delay_queue' - """:obj:`str`: The default delay queue for group requests.""" + DEFAULT_QUEUE_NAME: ClassVar[str] = 'default_delay_queue' + """:obj:`str`: The default delay queues name.""" + GROUP_QUEUE_NAME: ClassVar[str] = 'group_delay_queue' + """:obj:`str`: The name of the default delay queue for group requests.""" + + @property + def default_queue(self) -> DelayQueue: + """ + Shortcut for ``MessageQueue.delay_queues[MessageQueue.DEFAULT_QUEUE_NAME]``. + """ + return self.delay_queues[self.DEFAULT_QUEUE_NAME] + + @property + def group_queue(self) -> DelayQueue: + """ + Shortcut for ``MessageQueue.delay_queues[MessageQueue.GROUP_QUEUE_NAME]``. + """ + return self.delay_queues[self.GROUP_QUEUE_NAME] def queuedmessage(method: Callable) -> Callable: @@ -415,10 +438,10 @@ def wrapped(self: 'Bot', *args: Any, **kwargs: Any) -> Any: if queued: if not is_group: return self._msg_queue.put( # type: ignore[attr-defined] - method, MessageQueue.DEFAULT_QUEUE, self, *args, **kwargs + method, MessageQueue.DEFAULT_QUEUE_NAME, self, *args, **kwargs ) return self._msg_queue.put( # type: ignore[attr-defined] - method, MessageQueue.GROUP_QUEUE, self, *args, **kwargs + method, MessageQueue.GROUP_QUEUE_NAME, self, *args, **kwargs ) return method(self, *args, **kwargs) diff --git a/telegram/utils/promise.py b/telegram/utils/promise.py index d7333a05234..e989ce1aa5e 100644 --- a/telegram/utils/promise.py +++ b/telegram/utils/promise.py @@ -63,20 +63,9 @@ def __init__( update: Any = None, error_handling: bool = True, ): - - parsed_args = [] - for arg in args: - if InputFile.is_file(arg): - parsed_args.append(InputFile(arg)) - else: - parsed_args.append(arg) - self.args = tuple(parsed_args) - self.kwargs = kwargs - for key, value in self.kwargs.items(): - if InputFile.is_file(value): - self.kwargs[key] = InputFile(value) - self.pooled_function = pooled_function + self.args = args + self.kwargs = kwargs self.update = update self.error_handling = error_handling self.done = Event() diff --git a/tests/test_bot.py b/tests/test_bot.py index d2a47767a05..f6bd5c20f4b 100644 --- a/tests/test_bot.py +++ b/tests/test_bot.py @@ -1836,42 +1836,44 @@ def _post(*args, **kwargs): monkeypatch.setattr(mq_bot, '_post', _post) with caplog.at_level(logging.DEBUG): - result = mq_bot.send_message(chat_id, 'text', delay_queue=MessageQueue.DEFAULT_QUEUE) + result = mq_bot.send_message( + chat_id, 'text', delay_queue=MessageQueue.DEFAULT_QUEUE_NAME + ) assert isinstance(result, Promise) assert len(caplog.records) == 4 assert expected in caplog.records[1].getMessage() with caplog.at_level(logging.DEBUG): result = mq_bot.send_message( - text='text', chat_id=chat_id, delay_queue=MessageQueue.DEFAULT_QUEUE + text='text', chat_id=chat_id, delay_queue=MessageQueue.DEFAULT_QUEUE_NAME ) assert isinstance(result, Promise) assert len(caplog.records) == 8 assert expected in caplog.records[5].getMessage() with caplog.at_level(logging.DEBUG): - assert isinstance(mq_bot.get_me(delay_queue=MessageQueue.DEFAULT_QUEUE), Promise) + assert isinstance(mq_bot.get_me(delay_queue=MessageQueue.DEFAULT_QUEUE_NAME), Promise) assert len(caplog.records) == 12 assert 'default' in caplog.records[9].getMessage() def test_stopped_message_queue(self, mq_bot, caplog): mq_bot.message_queue.stop() with caplog.at_level(logging.DEBUG): - result = mq_bot.get_me(delay_queue=MessageQueue.DEFAULT_QUEUE) + result = mq_bot.get_me(delay_queue=MessageQueue.DEFAULT_QUEUE_NAME) assert not isinstance(result, Promise) assert len(caplog.records) >= 2 assert 'Ignoring call to MessageQueue' in caplog.records[1].getMessage() def test_no_message_queue(self, bot, caplog): with caplog.at_level(logging.DEBUG): - result = bot.get_me(delay_queue=MessageQueue.DEFAULT_QUEUE) + result = bot.get_me(delay_queue=MessageQueue.DEFAULT_QUEUE_NAME) assert not isinstance(result, Promise) assert len(caplog.records) >= 2 assert 'Ignoring call to MessageQueue' in caplog.records[1].getMessage() def test_message_queue_custom_delay_queue(self, chat_id, mq_bot, monkeypatch): test_flag = False - orig_put = mq_bot.message_queue._delay_queues['custom_dq'].put + orig_put = mq_bot.message_queue.delay_queues['custom_dq'].put def put(*args, **kwargs): nonlocal test_flag @@ -1881,19 +1883,7 @@ def put(*args, **kwargs): result = mq_bot.send_message(chat_id, 'general kenobi') assert not isinstance(result, Promise) - monkeypatch.setattr(mq_bot.message_queue._delay_queues['custom_dq'], 'put', put) + monkeypatch.setattr(mq_bot.message_queue.delay_queues['custom_dq'], 'put', put) result = mq_bot.send_message(chat_id, 'hello there', delay_queue='custom_dq') assert isinstance(result, Promise) assert test_flag - - @pytest.mark.timeout(10) - def test_message_queue_context_manager(self, mq_bot, chat_id): - with open('tests/data/telegram.gif', 'rb') as document: - with open('tests/data/telegram.jpg', 'rb') as thumb: - promise = mq_bot.send_document( - chat_id, document, thumb=thumb, delay_queue=MessageQueue.DEFAULT_QUEUE - ) - - message = promise.result() - assert message.document - assert message.document.thumb diff --git a/tests/test_messagequeue.py b/tests/test_messagequeue.py index a1fdde8834b..4152306983a 100644 --- a/tests/test_messagequeue.py +++ b/tests/test_messagequeue.py @@ -195,7 +195,7 @@ def callback_raises_exception(self): def test_auto_start_false(self): message_queue = MessageQueue(autostart=False) - assert not any(thread.is_alive() for thread in message_queue._delay_queues.values()) + assert not any(thread.is_alive() for thread in message_queue.delay_queues.values()) def test_exc_route_deprecation(self, recwarn): with pytest.raises(ValueError, match='Only one of exc_route or '): @@ -210,13 +210,13 @@ def test_add_delay_queue_autostart_false(self): delay_queue = DelayQueue(autostart=False, name='dq') try: message_queue.add_delay_queue(delay_queue) - assert 'dq' in message_queue._delay_queues - assert not any(thread.is_alive() for thread in message_queue._delay_queues.values()) + assert 'dq' in message_queue.delay_queues + assert not any(thread.is_alive() for thread in message_queue.delay_queues.values()) message_queue.start() - assert all(thread.is_alive() for thread in message_queue._delay_queues.values()) + assert all(thread.is_alive() for thread in message_queue.delay_queues.values()) message_queue.stop() - assert not any(thread.is_alive() for thread in message_queue._delay_queues.values()) + assert not any(thread.is_alive() for thread in message_queue.delay_queues.values()) finally: delay_queue.stop() message_queue.stop() @@ -227,12 +227,12 @@ def test_add_delay_queue_autostart_true(self, autostart): delay_queue = DelayQueue(name='dq', autostart=autostart) try: message_queue.add_delay_queue(delay_queue) - assert 'dq' in message_queue._delay_queues + assert 'dq' in message_queue.delay_queues assert delay_queue.is_alive() - assert all(thread.is_alive() for thread in message_queue._delay_queues.values()) + assert all(thread.is_alive() for thread in message_queue.delay_queues.values()) message_queue.stop() - assert not any(thread.is_alive() for thread in message_queue._delay_queues.values()) + assert not any(thread.is_alive() for thread in message_queue.delay_queues.values()) finally: delay_queue.stop() message_queue.stop() @@ -250,11 +250,11 @@ def test_remove_delay_queue(self, autostart): delay_queue = DelayQueue(name='dq') try: message_queue.add_delay_queue(delay_queue) - assert 'dq' in message_queue._delay_queues + assert 'dq' in message_queue.delay_queues assert delay_queue.is_alive() message_queue.remove_delay_queue('dq') - assert 'dq' not in message_queue._delay_queues + assert 'dq' not in message_queue.delay_queues if autostart: assert not delay_queue.is_alive() finally: @@ -266,23 +266,23 @@ def test_put(self): group_flag = None message_queue = MessageQueue() - original_put = message_queue._delay_queues[MessageQueue.DEFAULT_QUEUE].put + original_put = message_queue.default_queue.put def put(*args, **kwargs): nonlocal group_flag group_flag = True return original_put(*args, **kwargs) - message_queue._delay_queues[MessageQueue.GROUP_QUEUE].put = put + message_queue.group_queue.put = put try: - message_queue.put(self.call, MessageQueue.GROUP_QUEUE, 1, kwarg='foo') + message_queue.put(self.call, MessageQueue.GROUP_QUEUE_NAME, 1, kwarg='foo') sleep(0.5) assert self.test_flag is True # make sure that group queue was called, too assert group_flag is True finally: - message_queue._delay_queues[MessageQueue.GROUP_QUEUE].put = original_put + message_queue.group_queue.put = original_put message_queue.stop() @@ -304,8 +304,8 @@ def test_method(self, input, *args, **kwargs): bot = MQBot(token=bot.token) - orig_default_put = bot._msg_queue._delay_queues[MessageQueue.DEFAULT_QUEUE].put - orig_group_put = bot._msg_queue._delay_queues[MessageQueue.GROUP_QUEUE].put + orig_default_put = bot._msg_queue.default_queue.put + orig_group_put = bot._msg_queue.group_queue.put def step_default_counter(*args, **kwargs): orig_default_put(*args, **kwargs) @@ -315,12 +315,8 @@ def step_group_counter(*args, **kwargs): orig_group_put(*args, **kwargs) bot.group_count += 1 - monkeypatch.setattr( - bot._msg_queue._delay_queues[MessageQueue.DEFAULT_QUEUE], 'put', step_default_counter - ) - monkeypatch.setattr( - bot._msg_queue._delay_queues[MessageQueue.GROUP_QUEUE], 'put', step_group_counter - ) + monkeypatch.setattr(bot._msg_queue.default_queue, 'put', step_default_counter) + monkeypatch.setattr(bot._msg_queue.group_queue, 'put', step_group_counter) yield bot bot._msg_queue.stop() From 238cf151f15a479a81967a46b0c096c1bcd5a14e Mon Sep 17 00:00:00 2001 From: Hinrich Mahler Date: Wed, 23 Dec 2020 14:53:59 +0100 Subject: [PATCH 13/15] Update docs --- docs/source/telegram.ext.delayqueue.rst | 1 - docs/source/telegram.ext.messagequeue.rst | 1 - telegram/ext/messagequeue.py | 12 +++++++++++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/docs/source/telegram.ext.delayqueue.rst b/docs/source/telegram.ext.delayqueue.rst index ee79b849478..7426a923c74 100644 --- a/docs/source/telegram.ext.delayqueue.rst +++ b/docs/source/telegram.ext.delayqueue.rst @@ -4,4 +4,3 @@ telegram.ext.DelayQueue .. autoclass:: telegram.ext.DelayQueue :members: :show-inheritance: - :special-members: diff --git a/docs/source/telegram.ext.messagequeue.rst b/docs/source/telegram.ext.messagequeue.rst index 98bcb6e6357..f9ff9721044 100644 --- a/docs/source/telegram.ext.messagequeue.rst +++ b/docs/source/telegram.ext.messagequeue.rst @@ -4,4 +4,3 @@ telegram.ext.MessageQueue .. autoclass:: telegram.ext.MessageQueue :members: :show-inheritance: - :special-members: diff --git a/telegram/ext/messagequeue.py b/telegram/ext/messagequeue.py index b2613f01b9f..06b9f6ca6bf 100644 --- a/telegram/ext/messagequeue.py +++ b/telegram/ext/messagequeue.py @@ -48,9 +48,13 @@ class DelayQueue(threading.Thread): Processes callbacks from queue with specified throughput limits. Creates a separate thread to process callbacks with delays. + .. versionchanged:: 13.2 + DelayQueue was almost completely overhauled in v13.2. Please read the docs carefully, if + you're upgrading from lower versions. + Note: For most use cases, the :attr:`parent` argument should be set to - :attr:`MessageQueue.DEFAULT_QUEUE_NAME` to ensure that the global flood limits are not + :attr:`MessageQueue.default_queue` to ensure that the global flood limits are not exceeded. Args: @@ -226,6 +230,10 @@ class MessageQueue: By default contains two :class:`telegram.ext.DelayQueue` instances, for general requests and group requests where the default delay queue is the parent of the group requests one. + .. versionchanged:: 13.2 + MessageQueue was almost completely overhauled in v13.2. Please read the docs carefully, if + you're upgrading from lower versions. + Args: all_burst_limit (:obj:`int`, optional): Number of maximum *all-type* callbacks to process per time-window defined by :attr:`all_time_limit_ms`. Defaults to @@ -392,6 +400,8 @@ def group_queue(self) -> DelayQueue: def queuedmessage(method: Callable) -> Callable: """A decorator to be used with :attr:`telegram.Bot` send* methods. + .. deprecated:: 13.2 + Note: As it probably wouldn't be a good idea to make this decorator a property, it has been coded as decorator function, so it implies that first positional argument to wrapped MUST be From e7d4c2396bfd9c92f531fae05ee05425f95baff7 Mon Sep 17 00:00:00 2001 From: Hinrich Mahler Date: Wed, 23 Dec 2020 15:07:41 +0100 Subject: [PATCH 14/15] Add test for defaults --- tests/test_bot.py | 54 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/tests/test_bot.py b/tests/test_bot.py index f6bd5c20f4b..be2c647d22e 100644 --- a/tests/test_bot.py +++ b/tests/test_bot.py @@ -44,6 +44,7 @@ Dice, MessageEntity, ParseMode, + Message, ) from telegram.constants import MAX_INLINE_QUERY_RESULTS from telegram.error import BadRequest, InvalidToken, NetworkError, RetryAfter @@ -1887,3 +1888,56 @@ def put(*args, **kwargs): result = mq_bot.send_message(chat_id, 'hello there', delay_queue='custom_dq') assert isinstance(result, Promise) assert test_flag + + @pytest.mark.parametrize( + 'default_bot', + [ + { + 'delay_queue': MessageQueue.DEFAULT_QUEUE_NAME, + 'delay_queue_per_method': { + 'send_dice': MessageQueue.GROUP_QUEUE_NAME, + 'send_poll': None, + }, + } + ], + indirect=True, + ) + def test_message_queue_with_defaults(self, chat_id, default_bot, monkeypatch): + default_bot.message_queue = MessageQueue() + + default_counter = 0 + group_counter = 0 + orig_default_put = default_bot.message_queue.default_queue.put + orig_group_put = default_bot.message_queue.default_queue.put + + def default_put(*args, **kwargs): + nonlocal default_counter + default_counter += 1 + return orig_default_put(*args, **kwargs) + + def group_put(*args, **kwargs): + nonlocal group_counter + group_counter += 1 + return orig_group_put(*args, **kwargs) + + try: + monkeypatch.setattr(default_bot.message_queue.default_queue, 'put', default_put) + monkeypatch.setattr(default_bot.message_queue.group_queue, 'put', group_put) + + result = default_bot.send_message(chat_id, 'general kenobi') + assert isinstance(result, Promise) + assert default_counter == 1 + assert group_counter == 0 + + result = default_bot.send_poll(chat_id, 'question', options=['1', '2']) + assert isinstance(result, Message) + assert default_counter == 1 + assert group_counter == 0 + + result = default_bot.send_dice(chat_id) + assert isinstance(result, Promise) + assert default_counter == 1 + assert group_counter == 1 + finally: + default_bot.message_queue.stop() + default_bot.message_queue = None From 216ed4e6130e1ad148d79dd1c3a8ea9e12a10cc7 Mon Sep 17 00:00:00 2001 From: Hinrich Mahler Date: Wed, 23 Dec 2020 17:10:28 +0100 Subject: [PATCH 15/15] Add priority functionality --- telegram/ext/messagequeue.py | 50 ++++++++++++++++++++++++++++-------- tests/test_messagequeue.py | 44 +++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 10 deletions(-) diff --git a/telegram/ext/messagequeue.py b/telegram/ext/messagequeue.py index 06b9f6ca6bf..3a91ac34d87 100644 --- a/telegram/ext/messagequeue.py +++ b/telegram/ext/messagequeue.py @@ -23,10 +23,10 @@ import functools import logging -import queue as q import threading import time import warnings +from queue import PriorityQueue from typing import Callable, Any, TYPE_CHECKING, List, NoReturn, ClassVar, Dict, Optional @@ -43,6 +43,23 @@ class DelayQueueError(RuntimeError): """Indicates processing errors.""" +@functools.total_ordering +class PriorityWrapper: + def __init__(self, priority: int, promise: Promise): + self.priority = priority + self.promise = promise + + def __lt__(self, other: object) -> bool: + if not isinstance(other, PriorityWrapper): + raise NotImplementedError + return self.priority < other.priority + + def __eq__(self, other: object) -> bool: + if not isinstance(other, PriorityWrapper): + raise NotImplementedError + return self.priority == other.priority + + class DelayQueue(threading.Thread): """ Processes callbacks from queue with specified throughput limits. Creates a separate thread to @@ -58,8 +75,6 @@ class DelayQueue(threading.Thread): exceeded. Args: - queue (:obj:`Queue`, optional): Used to pass callbacks to thread. Creates ``Queue`` - implicitly if not provided. parent (:class:`telegram.ext.DelayQueue`, optional): Pass another delay queue to put all requests through that delay queue after they were processed by this queue. Defaults to :obj:`None`. @@ -79,6 +94,10 @@ class DelayQueue(threading.Thread): Defaults to :obj:`True`. name (:obj:`str`, optional): Thread's name. Defaults to ``'DelayQueue-N'``, where N is sequential number of object created. + priority (:obj:`int`, optional): Priority of the delay queue. Higher priority callbacks are + processed before lower priority callbacks, even if scheduled later. Higher number means + lower priority (i.e. ``priority = 0`` is processed *before* ``priority = 1``). Only + relevant, if the delay queue has a :attr:`parent``. Defaults to ``0``. Attributes: burst_limit (:obj:`int`): Number of maximum callbacks to process per time-window. @@ -96,7 +115,6 @@ class DelayQueue(threading.Thread): def __init__( self, - queue: q.Queue = None, burst_limit: int = constants.MAX_MESSAGES_PER_SECOND, time_limit_ms: int = 1000, exc_route: Callable[[Exception], None] = None, @@ -104,13 +122,15 @@ def __init__( name: str = None, parent: 'DelayQueue' = None, error_handler: Callable[[Exception], None] = None, + priority: int = 0, ): self.logger = logging.getLogger(__name__) - self._queue = queue if queue is not None else q.Queue() self.burst_limit = burst_limit self.time_limit = time_limit_ms / 1000 self.parent = parent self.dispatcher: Optional['Dispatcher'] = None + self.priority = priority + self._queue: 'PriorityQueue[PriorityWrapper]' = PriorityQueue() if exc_route and error_handler: raise ValueError('Only one of exc_route or error_handler can be passed.') @@ -145,7 +165,7 @@ def run(self) -> None: times: List[float] = [] # used to store each callable processing time while True: - promise = self._queue.get() + promise = self._queue.get().promise if self.__exit_req: return # shutdown thread @@ -167,7 +187,7 @@ def run(self) -> None: # finally process one if self.parent: # put through parent, if specified - self.parent.put(promise=promise) + self.parent.put(promise=promise, priority=self.priority) else: promise.run() # error handling @@ -189,7 +209,11 @@ def stop(self, timeout: float = None) -> None: """ self.__exit_req = True # gently request - self._queue.put(None) # put something to unfreeze if frozen + self._queue.put( + PriorityWrapper( + 0, Promise(PriorityQueue, [], {}) # put something to unfreeze if frozen + ) + ) self.logger.debug('Waiting for DelayQueue %s to shut down.', self.name) super().join(timeout=timeout) self.logger.debug('DelayQueue %s shut down.', self.name) @@ -199,7 +223,12 @@ def _default_exception_handler(exc: Exception) -> NoReturn: raise exc def put( - self, func: Callable = None, args: Any = None, kwargs: Any = None, promise: Promise = None + self, + func: Callable = None, + args: Any = None, + kwargs: Any = None, + promise: Promise = None, + priority: int = 0, ) -> Promise: """Used to process callbacks in throughput-limiting thread through queue. You must either pass a :class:`telegram.utils.Promise` or all of ``func``, ``args`` and ``kwargs``. @@ -210,6 +239,7 @@ def put( args (:obj:`list`, optional): Variable-length `func` arguments. kwargs (:obj:`dict`, optional): Arbitrary keyword-arguments to `func`. promise (:class:`telegram.utils.Promise`, optional): A promise. + priority (:obj:`int`, optional): Priority of the callback. Defaults to ``0``. """ if not bool(promise) ^ all(v is not None for v in [func, args, kwargs]): @@ -220,7 +250,7 @@ def put( if not promise: promise = Promise(func, args, kwargs) # type: ignore[arg-type] - self._queue.put(promise) + self._queue.put(PriorityWrapper(priority, promise)) return promise diff --git a/tests/test_messagequeue.py b/tests/test_messagequeue.py index 4152306983a..942f6555924 100644 --- a/tests/test_messagequeue.py +++ b/tests/test_messagequeue.py @@ -99,6 +99,50 @@ def test_delay_queue_limits(self): finally: delay_queue.stop() + def test_with_priority(self): + parent_queue = DelayQueue() + high_priority_queue = DelayQueue(parent=parent_queue, priority=0) + low_priority_queue = DelayQueue(parent=parent_queue, priority=1) + high_priority_count = 0 + low_priority_count = 0 + event_list = [] + + def low_priority_callback(): + nonlocal low_priority_count + nonlocal event_list + event_list.append((low_priority_count, 'low')) + low_priority_count += 1 + + def high_priority_callback(): + nonlocal high_priority_count + nonlocal event_list + event_list.append((high_priority_count, 'high')) + high_priority_count += 1 + + # enqueue low priority first + for _ in range(3): + low_priority_queue.put(low_priority_callback, args=[], kwargs={}) + + # enqueue high priority second + for _ in range(3): + high_priority_queue.put(high_priority_callback, args=[], kwargs={}) + + try: + sleep(1) + # high priority events should be handled first + assert event_list == [ + (0, 'high'), + (1, 'high'), + (2, 'high'), + (0, 'low'), + (1, 'low'), + (2, 'low'), + ] + finally: + parent_queue.stop() + low_priority_queue.stop() + high_priority_queue.stop() + def test_put_errors(self): delay_queue = DelayQueue(autostart=False) with pytest.raises(DelayQueueError, match='stopped thread'):