From 9e47f02fbd99dc94a4d2ea4db9294f4068aa5de7 Mon Sep 17 00:00:00 2001 From: Harshil <37377066+harshil21@users.noreply.github.com> Date: Sat, 22 Jan 2022 02:16:47 +0530 Subject: [PATCH 1/6] simplify persistence --- telegram/ext/_basepersistence.py | 31 +++---- telegram/ext/_dictpersistence.py | 43 ++++----- telegram/ext/_picklepersistence.py | 54 +++++------ tests/test_persistence.py | 141 ++++++++++++----------------- 4 files changed, 116 insertions(+), 153 deletions(-) diff --git a/telegram/ext/_basepersistence.py b/telegram/ext/_basepersistence.py index 81a16a66371..47f2f7b40c3 100644 --- a/telegram/ext/_basepersistence.py +++ b/telegram/ext/_basepersistence.py @@ -19,7 +19,7 @@ """This module contains the BasePersistence class.""" from abc import ABC, abstractmethod from copy import copy -from typing import Dict, Optional, Tuple, cast, ClassVar, Generic, DefaultDict, NamedTuple +from typing import Dict, Optional, Tuple, cast, ClassVar, Generic, DefaultDict, NamedTuple, Union from telegram import Bot from telegram.ext import ExtBot @@ -132,10 +132,10 @@ def __new__( update_bot_data = instance.update_bot_data update_callback_data = instance.update_callback_data - def get_user_data_insert_bot() -> DefaultDict[int, UD]: + def get_user_data_insert_bot() -> Union[DefaultDict[int, UD], Dict[int, UD]]: return instance.insert_bot(get_user_data()) - def get_chat_data_insert_bot() -> DefaultDict[int, CD]: + def get_chat_data_insert_bot() -> Union[DefaultDict[int, CD], Dict[int, CD]]: return instance.insert_bot(get_chat_data()) def get_bot_data_insert_bot() -> BD: @@ -171,10 +171,7 @@ def update_callback_data_replace_bot(data: CDCData) -> None: setattr(instance, 'update_callback_data', update_callback_data_replace_bot) return instance - def __init__( - self, - store_data: PersistenceInput = None, - ): + def __init__(self, store_data: PersistenceInput = None): self.store_data = store_data or PersistenceInput() self.bot: Bot = None # type: ignore[assignment] @@ -398,34 +395,34 @@ def _insert_bot(self, obj: object, memo: Dict[int, object]) -> object: return obj @abstractmethod - def get_user_data(self) -> DefaultDict[int, UD]: + def get_user_data(self) -> Union[DefaultDict[int, UD], Dict[int, UD]]: """Will be called by :class:`telegram.ext.Dispatcher` upon creation with a persistence object. It should return the ``user_data`` if stored, or an empty - :obj:`defaultdict`. In the latter case, the :obj:`defaultdict` should produce values + :obj:`defaultdict` or :obj:`dict`. In the latter case, the dictionary should produce values corresponding to one of the following: * :obj:`dict` * The type from :attr:`telegram.ext.ContextTypes.user_data` - if :class:`telegram.ext.ContextTypes` are used. + if :class:`telegram.ext.ContextTypes` is used. Returns: - DefaultDict[:obj:`int`, :obj:`dict` | :attr:`telegram.ext.ContextTypes.user_data`]: + Dict[:obj:`int`, :obj:`dict` | :attr:`telegram.ext.ContextTypes.user_data`]: The restored user data. """ @abstractmethod - def get_chat_data(self) -> DefaultDict[int, CD]: + def get_chat_data(self) -> Union[DefaultDict[int, CD], Dict[int, CD]]: """Will be called by :class:`telegram.ext.Dispatcher` upon creation with a persistence object. It should return the ``chat_data`` if stored, or an empty - :obj:`defaultdict`. In the latter case, the :obj:`defaultdict` should produce values + :obj:`defaultdict` or :obj:`dict`. In the latter case, the dictionary should produce values corresponding to one of the following: * :obj:`dict` * The type from :attr:`telegram.ext.ContextTypes.chat_data` - if :class:`telegram.ext.ContextTypes` are used. + if :class:`telegram.ext.ContextTypes` is used. Returns: - DefaultDict[:obj:`int`, :obj:`dict` | :attr:`telegram.ext.ContextTypes.chat_data`]: + Dict[:obj:`int`, :obj:`dict` | :attr:`telegram.ext.ContextTypes.chat_data`]: The restored chat data. """ @@ -433,7 +430,7 @@ def get_chat_data(self) -> DefaultDict[int, CD]: def get_bot_data(self) -> BD: """Will be called by :class:`telegram.ext.Dispatcher` upon creation with a persistence object. It should return the ``bot_data`` if stored, or an empty - :obj:`defaultdict`. In the latter case, the :obj:`defaultdict` should produce values + :obj:`dict`. In the latter case, the :obj:`dict` should produce values corresponding to one of the following: * :obj:`dict` @@ -441,7 +438,7 @@ def get_bot_data(self) -> BD: if :class:`telegram.ext.ContextTypes` are used. Returns: - DefaultDict[:obj:`int`, :obj:`dict` | :attr:`telegram.ext.ContextTypes.bot_data`]: + Dict[:obj:`int`, :obj:`dict` | :attr:`telegram.ext.ContextTypes.bot_data`]: The restored bot data. """ diff --git a/telegram/ext/_dictpersistence.py b/telegram/ext/_dictpersistence.py index df3f9aa71f5..48110618600 100644 --- a/telegram/ext/_dictpersistence.py +++ b/telegram/ext/_dictpersistence.py @@ -18,8 +18,7 @@ # along with this program. If not, see [http://www.gnu.org/licenses/]. """This module contains the DictPersistence class.""" -from typing import DefaultDict, Dict, Optional, Tuple, cast -from collections import defaultdict +from typing import Dict, Optional, Tuple, cast from telegram.ext import BasePersistence, PersistenceInput from telegram._utils.types import JSONDict @@ -166,7 +165,7 @@ def __init__( ) from exc @property - def user_data(self) -> Optional[DefaultDict[int, Dict]]: + def user_data(self) -> Optional[Dict[int, Dict]]: """:obj:`dict`: The user_data as a dict.""" return self._user_data @@ -178,7 +177,7 @@ def user_data_json(self) -> str: return json.dumps(self.user_data) @property - def chat_data(self) -> Optional[DefaultDict[int, Dict]]: + def chat_data(self) -> Optional[Dict[int, Dict]]: """:obj:`dict`: The chat_data as a dict.""" return self._chat_data @@ -204,7 +203,7 @@ def bot_data_json(self) -> str: @property def callback_data(self) -> Optional[CDCData]: """Tuple[List[Tuple[:obj:`str`, :obj:`float`, Dict[:obj:`str`, :obj:`Any`]]], \ - Dict[:obj:`str`, :obj:`str`]]: The meta data on the stored callback data. + Dict[:obj:`str`, :obj:`str`]]: The metadata on the stored callback data. .. versionadded:: 13.6 """ @@ -212,7 +211,7 @@ def callback_data(self) -> Optional[CDCData]: @property def callback_data_json(self) -> str: - """:obj:`str`: The meta data on the stored callback data as a JSON-string. + """:obj:`str`: The metadata on the stored callback data as a JSON-string. .. versionadded:: 13.6 """ @@ -232,26 +231,24 @@ def conversations_json(self) -> str: return self._conversations_json return self._encode_conversations_to_json(self.conversations) # type: ignore[arg-type] - def get_user_data(self) -> DefaultDict[int, Dict[object, object]]: - """Returns the user_data created from the ``user_data_json`` or an empty - :obj:`defaultdict`. + def get_user_data(self) -> Dict[int, Dict[object, object]]: + """Returns the user_data created from the ``user_data_json`` or an empty :obj:`dict`. Returns: - :obj:`defaultdict`: The restored user data. + :obj:`dict`: The restored user data. """ if self.user_data is None: - self._user_data = defaultdict(dict) + self._user_data = {} return self.user_data # type: ignore[return-value] - def get_chat_data(self) -> DefaultDict[int, Dict[object, object]]: - """Returns the chat_data created from the ``chat_data_json`` or an empty - :obj:`defaultdict`. + def get_chat_data(self) -> Dict[int, Dict[object, object]]: + """Returns the chat_data created from the ``chat_data_json`` or an empty :obj:`dict`. Returns: - :obj:`defaultdict`: The restored chat data. + :obj:`dict`: The restored chat data. """ if self.chat_data is None: - self._chat_data = defaultdict(dict) + self._chat_data = {} return self.chat_data # type: ignore[return-value] def get_bot_data(self) -> Dict[object, object]: @@ -271,7 +268,7 @@ def get_callback_data(self) -> Optional[CDCData]: Returns: Tuple[List[Tuple[:obj:`str`, :obj:`float`, Dict[:obj:`str`, :obj:`Any`]]], \ - Dict[:obj:`str`, :obj:`str`]]: The restored meta data or :obj:`None`, \ + Dict[:obj:`str`, :obj:`str`]]: The restored metadata or :obj:`None`, \ if no data was stored. """ if self.callback_data is None: @@ -315,8 +312,8 @@ def update_user_data(self, user_id: int, data: Dict) -> None: data (:obj:`dict`): The :attr:`telegram.ext.Dispatcher.user_data` ``[user_id]``. """ if self._user_data is None: - self._user_data = defaultdict(dict) - if self._user_data.get(user_id) == data: + self._user_data = {} + if self._user_data.get(user_id, None) == data: return self._user_data[user_id] = data self._user_data_json = None @@ -329,8 +326,8 @@ def update_chat_data(self, chat_id: int, data: Dict) -> None: data (:obj:`dict`): The :attr:`telegram.ext.Dispatcher.chat_data` ``[chat_id]``. """ if self._chat_data is None: - self._chat_data = defaultdict(dict) - if self._chat_data.get(chat_id) == data: + self._chat_data = {} + if self._chat_data.get(chat_id, None) == data: return self._chat_data[chat_id] = data self._chat_data_json = None @@ -453,7 +450,7 @@ def _decode_conversations_from_json(json_string: str) -> Dict[str, Dict[Tuple, o return conversations @staticmethod - def _decode_user_chat_data_from_json(data: str) -> DefaultDict[int, Dict[object, object]]: + def _decode_user_chat_data_from_json(data: str) -> Dict[int, Dict[object, object]]: """Helper method to decode chat or user data (that uses ints as keys) from a JSON-string. @@ -463,7 +460,7 @@ def _decode_user_chat_data_from_json(data: str) -> DefaultDict[int, Dict[object, Returns: :obj:`dict`: The user/chat_data defaultdict after decoding """ - tmp: DefaultDict[int, Dict[object, object]] = defaultdict(dict) + tmp: Dict[int, Dict[object, object]] = {} decoded_data = json.loads(data) for user, user_data in decoded_data.items(): user = int(user) diff --git a/telegram/ext/_picklepersistence.py b/telegram/ext/_picklepersistence.py index 3fcbb995424..0bb2a4544e4 100644 --- a/telegram/ext/_picklepersistence.py +++ b/telegram/ext/_picklepersistence.py @@ -18,7 +18,6 @@ # along with this program. If not, see [http://www.gnu.org/licenses/]. """This module contains the PicklePersistence class.""" import pickle -from collections import defaultdict from pathlib import Path from typing import ( Any, @@ -27,7 +26,6 @@ Tuple, overload, cast, - DefaultDict, ) from telegram._utils.types import FilePathInput @@ -138,8 +136,8 @@ def __init__( self.filepath = Path(filepath) self.single_file = single_file self.on_flush = on_flush - self.user_data: Optional[DefaultDict[int, UD]] = None - self.chat_data: Optional[DefaultDict[int, CD]] = None + self.user_data: Optional[Dict[int, UD]] = None + self.chat_data: Optional[Dict[int, CD]] = None self.bot_data: Optional[BD] = None self.callback_data: Optional[CDCData] = None self.conversations: Optional[Dict[str, Dict[Tuple, object]]] = None @@ -149,16 +147,16 @@ def _load_singlefile(self) -> None: try: with self.filepath.open("rb") as file: data = pickle.load(file) - self.user_data = defaultdict(self.context_types.user_data, data['user_data']) - self.chat_data = defaultdict(self.context_types.chat_data, data['chat_data']) + self.user_data = data['user_data'] + self.chat_data = data['chat_data'] # For backwards compatibility with files not containing bot data self.bot_data = data.get('bot_data', self.context_types.bot_data()) self.callback_data = data.get('callback_data', {}) self.conversations = data['conversations'] except OSError: self.conversations = {} - self.user_data = defaultdict(self.context_types.user_data) - self.chat_data = defaultdict(self.context_types.chat_data) + self.user_data = {} + self.chat_data = {} self.bot_data = self.context_types.bot_data() self.callback_data = None except pickle.UnpicklingError as exc: @@ -195,41 +193,35 @@ def _dump_file(filepath: Path, data: object) -> None: with filepath.open("wb") as file: pickle.dump(data, file) - def get_user_data(self) -> DefaultDict[int, UD]: - """Returns the user_data from the pickle file if it exists or an empty :obj:`defaultdict`. + def get_user_data(self) -> Dict[int, UD]: + """Returns the user_data from the pickle file if it exists or an empty :obj:`dict`. Returns: - DefaultDict[:obj:`int`, :obj:`dict` | :attr:`telegram.ext.ContextTypes.user_data`]: - The restored user data. + Dict[:obj:`int`, :obj:`dict`]: The restored user data. """ if self.user_data: pass elif not self.single_file: data = self._load_file(Path(f"{self.filepath}_user_data")) if not data: - data = defaultdict(self.context_types.user_data) - else: - data = defaultdict(self.context_types.user_data, data) + data = {} self.user_data = data else: self._load_singlefile() return self.user_data # type: ignore[return-value] - def get_chat_data(self) -> DefaultDict[int, CD]: - """Returns the chat_data from the pickle file if it exists or an empty :obj:`defaultdict`. + def get_chat_data(self) -> Dict[int, CD]: + """Returns the chat_data from the pickle file if it exists or an empty :obj:`dict`. Returns: - DefaultDict[:obj:`int`, :obj:`dict` | :attr:`telegram.ext.ContextTypes.chat_data`]: - The restored chat data. + Dict[:obj:`int`, :obj:`dict`]: The restored chat data. """ if self.chat_data: pass elif not self.single_file: data = self._load_file(Path(f"{self.filepath}_chat_data")) if not data: - data = defaultdict(self.context_types.chat_data) - else: - data = defaultdict(self.context_types.chat_data, data) + data = {} self.chat_data = data else: self._load_singlefile() @@ -260,8 +252,8 @@ def get_callback_data(self) -> Optional[CDCData]: Returns: Optional[Tuple[List[Tuple[:obj:`str`, :obj:`float`, \ - Dict[:obj:`str`, :obj:`Any`]]], Dict[:obj:`str`, :obj:`str`]]: - The restored meta data or :obj:`None`, if no data was stored. + Dict[:obj:`str`, :obj:`Any`]]], Dict[:obj:`str`, :obj:`str`]]]: + The restored metadata or :obj:`None`, if no data was stored. """ if self.callback_data: pass @@ -323,12 +315,11 @@ def update_user_data(self, user_id: int, data: UD) -> None: Args: user_id (:obj:`int`): The user the data might have been changed for. - data (:obj:`dict` | :attr:`telegram.ext.ContextTypes.user_data`): The - :attr:`telegram.ext.Dispatcher.user_data` ``[user_id]``. + data (:obj:`dict`): The :attr:`telegram.ext.Dispatcher.user_data` ``[user_id]``. """ if self.user_data is None: - self.user_data = defaultdict(self.context_types.user_data) - if self.user_data.get(user_id) == data: + self.user_data = {} + if self.user_data.get(user_id, None) == data: # type: ignore[arg-type] return self.user_data[user_id] = data if not self.on_flush: @@ -342,12 +333,11 @@ def update_chat_data(self, chat_id: int, data: CD) -> None: Args: chat_id (:obj:`int`): The chat the data might have been changed for. - data (:obj:`dict` | :attr:`telegram.ext.ContextTypes.chat_data`): The - :attr:`telegram.ext.Dispatcher.chat_data` ``[chat_id]``. + data (:obj:`dict`): The :attr:`telegram.ext.Dispatcher.chat_data` ``[chat_id]``. """ if self.chat_data is None: - self.chat_data = defaultdict(self.context_types.chat_data) - if self.chat_data.get(chat_id) == data: + self.chat_data = {} + if self.chat_data.get(chat_id, None) == data: # type: ignore[arg-type] return self.chat_data[chat_id] = data if not self.on_flush: diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 55ab9f1f15e..fc7f32fcf97 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -16,26 +16,24 @@ # # You should have received a copy of the GNU Lesser Public License # along with this program. If not, see [http://www.gnu.org/licenses/]. +import logging +import os +import pickle import gzip import signal import uuid +from collections.abc import Container +from collections import defaultdict from pathlib import Path +from time import sleep from threading import Lock -from telegram.ext import PersistenceInput, UpdaterBuilder, CallbackDataCache +import pytest try: import ujson as json except ImportError: import json -import logging -import os -import pickle -from collections import defaultdict -from collections.abc import Container -from time import sleep - -import pytest from telegram import Update, Message, User, Chat, MessageEntity, Bot from telegram.ext import ( @@ -49,6 +47,9 @@ TypeHandler, JobQueue, ContextTypes, + PersistenceInput, + UpdaterBuilder, + CallbackDataCache, ) from telegram.ext._callbackdatacache import _KeyboardData @@ -134,8 +135,8 @@ class BotPersistence(BasePersistence): def __init__(self): super().__init__() self.bot_data = None - self.chat_data = defaultdict(dict) - self.user_data = defaultdict(dict) + self.chat_data = {} + self.user_data = {} self.callback_data = None def get_bot_data(self): @@ -196,16 +197,12 @@ def bot_data(): @pytest.fixture(scope="function") def chat_data(): - return defaultdict( - dict, {-12345: {'test1': 'test2', 'test3': {'test4': 'test5'}}, -67890: {3: 'test4'}} - ) + return {-12345: {'test1': 'test2', 'test3': {'test4': 'test5'}}, -67890: {3: 'test4'}} @pytest.fixture(scope="function") def user_data(): - return defaultdict( - dict, {12345: {'test1': 'test2', 'test3': {'test4': 'test5'}}, 67890: {3: 'test4'}} - ) + return {12345: {'test1': 'test2', 'test3': {'test4': 'test5'}}, 67890: {3: 'test4'}} @pytest.fixture(scope="function") @@ -1040,10 +1037,6 @@ def update(bot): return Update(0, message=message) -class CustomMapping(defaultdict): - pass - - class TestPicklePersistence: def test_slot_behaviour(self, mro_slots, pickle_persistence): inst = pickle_persistence @@ -1059,16 +1052,16 @@ def test_pickle_behaviour_with_slots(self, pickle_persistence): assert retrieved == bot_data def test_no_files_present_multi_file(self, pickle_persistence): - assert pickle_persistence.get_user_data() == defaultdict(dict) - assert pickle_persistence.get_chat_data() == defaultdict(dict) + assert pickle_persistence.get_user_data() == {} + assert pickle_persistence.get_chat_data() == {} assert pickle_persistence.get_bot_data() == {} assert pickle_persistence.get_callback_data() is None assert pickle_persistence.get_conversations('noname') == {} def test_no_files_present_single_file(self, pickle_persistence): pickle_persistence.single_file = True - assert pickle_persistence.get_user_data() == defaultdict(dict) - assert pickle_persistence.get_chat_data() == defaultdict(dict) + assert pickle_persistence.get_user_data() == {} + assert pickle_persistence.get_chat_data() == {} assert pickle_persistence.get_bot_data() == {} assert pickle_persistence.get_callback_data() is None assert pickle_persistence.get_conversations('noname') == {} @@ -1125,16 +1118,14 @@ def test_with_invalid_single_file(self, pickle_persistence, invalid_pickle_files def test_with_good_multi_file(self, pickle_persistence, good_pickle_files): user_data = pickle_persistence.get_user_data() - assert isinstance(user_data, defaultdict) + assert isinstance(user_data, dict) assert user_data[12345]['test1'] == 'test2' assert user_data[67890][3] == 'test4' - assert user_data[54321] == {} chat_data = pickle_persistence.get_chat_data() - assert isinstance(chat_data, defaultdict) + assert isinstance(chat_data, dict) assert chat_data[-12345]['test1'] == 'test2' assert chat_data[-67890][3] == 'test4' - assert chat_data[-54321] == {} bot_data = pickle_persistence.get_bot_data() assert isinstance(bot_data, dict) @@ -1163,16 +1154,14 @@ def test_with_good_multi_file(self, pickle_persistence, good_pickle_files): def test_with_good_single_file(self, pickle_persistence, good_pickle_files): pickle_persistence.single_file = True user_data = pickle_persistence.get_user_data() - assert isinstance(user_data, defaultdict) + assert isinstance(user_data, dict) assert user_data[12345]['test1'] == 'test2' assert user_data[67890][3] == 'test4' - assert user_data[54321] == {} chat_data = pickle_persistence.get_chat_data() - assert isinstance(chat_data, defaultdict) + assert isinstance(chat_data, dict) assert chat_data[-12345]['test1'] == 'test2' assert chat_data[-67890][3] == 'test4' - assert chat_data[-54321] == {} bot_data = pickle_persistence.get_bot_data() assert isinstance(bot_data, dict) @@ -1200,16 +1189,14 @@ def test_with_good_single_file(self, pickle_persistence, good_pickle_files): def test_with_multi_file_wo_bot_data(self, pickle_persistence, pickle_files_wo_bot_data): user_data = pickle_persistence.get_user_data() - assert isinstance(user_data, defaultdict) + assert isinstance(user_data, dict) assert user_data[12345]['test1'] == 'test2' assert user_data[67890][3] == 'test4' - assert user_data[54321] == {} chat_data = pickle_persistence.get_chat_data() - assert isinstance(chat_data, defaultdict) + assert isinstance(chat_data, dict) assert chat_data[-12345]['test1'] == 'test2' assert chat_data[-67890][3] == 'test4' - assert chat_data[-54321] == {} bot_data = pickle_persistence.get_bot_data() assert isinstance(bot_data, dict) @@ -1237,16 +1224,14 @@ def test_with_multi_file_wo_callback_data( self, pickle_persistence, pickle_files_wo_callback_data ): user_data = pickle_persistence.get_user_data() - assert isinstance(user_data, defaultdict) + assert isinstance(user_data, dict) assert user_data[12345]['test1'] == 'test2' assert user_data[67890][3] == 'test4' - assert user_data[54321] == {} chat_data = pickle_persistence.get_chat_data() - assert isinstance(chat_data, defaultdict) + assert isinstance(chat_data, dict) assert chat_data[-12345]['test1'] == 'test2' assert chat_data[-67890][3] == 'test4' - assert chat_data[-54321] == {} bot_data = pickle_persistence.get_bot_data() assert isinstance(bot_data, dict) @@ -1273,16 +1258,14 @@ def test_with_multi_file_wo_callback_data( def test_with_single_file_wo_bot_data(self, pickle_persistence, pickle_files_wo_bot_data): pickle_persistence.single_file = True user_data = pickle_persistence.get_user_data() - assert isinstance(user_data, defaultdict) + assert isinstance(user_data, dict) assert user_data[12345]['test1'] == 'test2' assert user_data[67890][3] == 'test4' - assert user_data[54321] == {} chat_data = pickle_persistence.get_chat_data() - assert isinstance(chat_data, defaultdict) + assert isinstance(chat_data, dict) assert chat_data[-12345]['test1'] == 'test2' assert chat_data[-67890][3] == 'test4' - assert chat_data[-54321] == {} bot_data = pickle_persistence.get_bot_data() assert isinstance(bot_data, dict) @@ -1310,16 +1293,14 @@ def test_with_single_file_wo_callback_data( self, pickle_persistence, pickle_files_wo_callback_data ): user_data = pickle_persistence.get_user_data() - assert isinstance(user_data, defaultdict) + assert isinstance(user_data, dict) assert user_data[12345]['test1'] == 'test2' assert user_data[67890][3] == 'test4' - assert user_data[54321] == {} chat_data = pickle_persistence.get_chat_data() - assert isinstance(chat_data, defaultdict) + assert isinstance(chat_data, dict) assert chat_data[-12345]['test1'] == 'test2' assert chat_data[-67890][3] == 'test4' - assert chat_data[-54321] == {} bot_data = pickle_persistence.get_bot_data() assert isinstance(bot_data, dict) @@ -1353,7 +1334,7 @@ def test_updating_multi_file(self, pickle_persistence, good_pickle_files): pickle_persistence.update_user_data(12345, user_data[12345]) assert pickle_persistence.user_data == user_data with Path('pickletest_user_data').open('rb') as f: - user_data_test = defaultdict(dict, pickle.load(f)) + user_data_test = dict(pickle.load(f)) assert user_data_test == user_data pickle_persistence.drop_user_data(67890) assert 67890 not in pickle_persistence.get_user_data() @@ -1367,7 +1348,7 @@ def test_updating_multi_file(self, pickle_persistence, good_pickle_files): pickle_persistence.update_chat_data(-12345, chat_data[-12345]) assert pickle_persistence.chat_data == chat_data with Path('pickletest_chat_data').open('rb') as f: - chat_data_test = defaultdict(dict, pickle.load(f)) + chat_data_test = dict(pickle.load(f)) assert chat_data_test == chat_data pickle_persistence.drop_chat_data(-67890) assert -67890 not in pickle_persistence.get_chat_data() @@ -1403,7 +1384,7 @@ def test_updating_multi_file(self, pickle_persistence, good_pickle_files): assert pickle_persistence.conversations['name1'] == conversation1 assert pickle_persistence.get_conversations('name1') == conversation1 with Path('pickletest_conversations').open('rb') as f: - conversations_test = defaultdict(dict, pickle.load(f)) + conversations_test = dict(pickle.load(f)) assert conversations_test['name1'] == conversation1 pickle_persistence.conversations = None @@ -1423,7 +1404,7 @@ def test_updating_single_file(self, pickle_persistence, good_pickle_files): pickle_persistence.update_user_data(12345, user_data[12345]) assert pickle_persistence.user_data == user_data with Path('pickletest').open('rb') as f: - user_data_test = defaultdict(dict, pickle.load(f)['user_data']) + user_data_test = dict(pickle.load(f))['user_data'] assert user_data_test == user_data pickle_persistence.drop_user_data(67890) assert 67890 not in pickle_persistence.get_user_data() @@ -1437,7 +1418,7 @@ def test_updating_single_file(self, pickle_persistence, good_pickle_files): pickle_persistence.update_chat_data(-12345, chat_data[-12345]) assert pickle_persistence.chat_data == chat_data with Path('pickletest').open('rb') as f: - chat_data_test = defaultdict(dict, pickle.load(f)['chat_data']) + chat_data_test = dict(pickle.load(f))['chat_data'] assert chat_data_test == chat_data pickle_persistence.drop_chat_data(-67890) assert -67890 not in pickle_persistence.get_chat_data() @@ -1473,7 +1454,7 @@ def test_updating_single_file(self, pickle_persistence, good_pickle_files): assert pickle_persistence.conversations['name1'] == conversation1 assert pickle_persistence.get_conversations('name1') == conversation1 with Path('pickletest').open('rb') as f: - conversations_test = defaultdict(dict, pickle.load(f)['conversations']) + conversations_test = dict(pickle.load(f))['conversations'] assert conversations_test['name1'] == conversation1 pickle_persistence.conversations = None @@ -1502,6 +1483,7 @@ def test_save_on_flush_multi_files(self, pickle_persistence, good_pickle_files): pickle_persistence.on_flush = True user_data = pickle_persistence.get_user_data() + user_data[54321] = {} user_data[54321]['test9'] = 'test 10' assert not pickle_persistence.user_data == user_data @@ -1512,10 +1494,11 @@ def test_save_on_flush_multi_files(self, pickle_persistence, good_pickle_files): assert pickle_persistence.user_data == user_data with Path('pickletest_user_data').open('rb') as f: - user_data_test = defaultdict(dict, pickle.load(f)) + user_data_test = dict(pickle.load(f)) assert not user_data_test == user_data chat_data = pickle_persistence.get_chat_data() + chat_data[54321] = {} chat_data[54321]['test9'] = 'test 10' assert not pickle_persistence.chat_data == chat_data @@ -1526,7 +1509,7 @@ def test_save_on_flush_multi_files(self, pickle_persistence, good_pickle_files): assert pickle_persistence.user_data == user_data with Path('pickletest_chat_data').open('rb') as f: - chat_data_test = defaultdict(dict, pickle.load(f)) + chat_data_test = dict(pickle.load(f)) assert not chat_data_test == chat_data bot_data = pickle_persistence.get_bot_data() @@ -1559,16 +1542,16 @@ def test_save_on_flush_multi_files(self, pickle_persistence, good_pickle_files): assert pickle_persistence.conversations['name1'] == conversation1 with Path('pickletest_conversations').open('rb') as f: - conversations_test = defaultdict(dict, pickle.load(f)) + conversations_test = dict(pickle.load(f)) assert not conversations_test['name1'] == conversation1 pickle_persistence.flush() with Path('pickletest_user_data').open('rb') as f: - user_data_test = defaultdict(dict, pickle.load(f)) + user_data_test = dict(pickle.load(f)) assert user_data_test == user_data with Path('pickletest_chat_data').open('rb') as f: - chat_data_test = defaultdict(dict, pickle.load(f)) + chat_data_test = dict(pickle.load(f)) assert chat_data_test == chat_data with Path('pickletest_bot_data').open('rb') as f: @@ -1576,7 +1559,7 @@ def test_save_on_flush_multi_files(self, pickle_persistence, good_pickle_files): assert bot_data_test == bot_data with Path('pickletest_conversations').open('rb') as f: - conversations_test = defaultdict(dict, pickle.load(f)) + conversations_test = dict(pickle.load(f)) assert conversations_test['name1'] == conversation1 def test_save_on_flush_single_files(self, pickle_persistence, good_pickle_files): @@ -1587,21 +1570,23 @@ def test_save_on_flush_single_files(self, pickle_persistence, good_pickle_files) pickle_persistence.single_file = True user_data = pickle_persistence.get_user_data() + user_data[54321] = {} user_data[54321]['test9'] = 'test 10' assert not pickle_persistence.user_data == user_data pickle_persistence.update_user_data(54321, user_data[54321]) assert pickle_persistence.user_data == user_data with Path('pickletest').open('rb') as f: - user_data_test = defaultdict(dict, pickle.load(f)['user_data']) + user_data_test = dict(pickle.load(f))['user_data'] assert not user_data_test == user_data chat_data = pickle_persistence.get_chat_data() + chat_data[54321] = {} chat_data[54321]['test9'] = 'test 10' assert not pickle_persistence.chat_data == chat_data pickle_persistence.update_chat_data(54321, chat_data[54321]) assert pickle_persistence.chat_data == chat_data with Path('pickletest').open('rb') as f: - chat_data_test = defaultdict(dict, pickle.load(f)['chat_data']) + chat_data_test = dict(pickle.load(f))['chat_data'] assert not chat_data_test == chat_data bot_data = pickle_persistence.get_bot_data() @@ -1628,16 +1613,16 @@ def test_save_on_flush_single_files(self, pickle_persistence, good_pickle_files) pickle_persistence.update_conversation('name1', (123, 123), 5) assert pickle_persistence.conversations['name1'] == conversation1 with Path('pickletest').open('rb') as f: - conversations_test = defaultdict(dict, pickle.load(f)['conversations']) + conversations_test = dict(pickle.load(f))['conversations'] assert not conversations_test['name1'] == conversation1 pickle_persistence.flush() with Path('pickletest').open('rb') as f: - user_data_test = defaultdict(dict, pickle.load(f)['user_data']) + user_data_test = dict(pickle.load(f))['user_data'] assert user_data_test == user_data with Path('pickletest').open('rb') as f: - chat_data_test = defaultdict(dict, pickle.load(f)['chat_data']) + chat_data_test = dict(pickle.load(f))['chat_data'] assert chat_data_test == chat_data with Path('pickletest').open('rb') as f: @@ -1645,7 +1630,7 @@ def test_save_on_flush_single_files(self, pickle_persistence, good_pickle_files) assert bot_data_test == bot_data with Path('pickletest').open('rb') as f: - conversations_test = defaultdict(dict, pickle.load(f)['conversations']) + conversations_test = dict(pickle.load(f))['conversations'] assert conversations_test['name1'] == conversation1 def test_with_handler(self, bot, update, bot_data, pickle_persistence, good_pickle_files): @@ -1925,10 +1910,6 @@ def test_with_context_types(self, ud, cd, bd, singlefile): cc = ContextTypes(user_data=ud, chat_data=cd, bot_data=bd) persistence = PicklePersistence('pickletest', single_file=singlefile, context_types=cc) - assert isinstance(persistence.get_user_data()[1], ud) - assert persistence.get_user_data()[1] == 0 - assert isinstance(persistence.get_chat_data()[1], cd) - assert persistence.get_chat_data()[1] == 0 assert isinstance(persistence.get_bot_data(), bd) assert persistence.get_bot_data() == 0 @@ -1936,8 +1917,8 @@ def test_with_context_types(self, ud, cd, bd, singlefile): persistence.chat_data = None persistence.drop_user_data(123) persistence.drop_chat_data(123) - assert isinstance(persistence.get_user_data(), defaultdict) - assert isinstance(persistence.get_chat_data(), defaultdict) + assert isinstance(persistence.get_user_data(), dict) + assert isinstance(persistence.get_chat_data(), dict) persistence.user_data = None persistence.chat_data = None persistence.update_user_data(1, ud(1)) @@ -1993,8 +1974,8 @@ def test_slot_behaviour(self, mro_slots, recwarn): def test_no_json_given(self): dict_persistence = DictPersistence() - assert dict_persistence.get_user_data() == defaultdict(dict) - assert dict_persistence.get_chat_data() == defaultdict(dict) + assert dict_persistence.get_user_data() == {} + assert dict_persistence.get_chat_data() == {} assert dict_persistence.get_bot_data() == {} assert dict_persistence.get_callback_data() is None assert dict_persistence.get_conversations('noname') == {} @@ -2055,16 +2036,14 @@ def test_good_json_input( callback_data_json=callback_data_json, ) user_data = dict_persistence.get_user_data() - assert isinstance(user_data, defaultdict) + assert isinstance(user_data, dict) assert user_data[12345]['test1'] == 'test2' assert user_data[67890][3] == 'test4' - assert user_data[54321] == {} chat_data = dict_persistence.get_chat_data() - assert isinstance(chat_data, defaultdict) + assert isinstance(chat_data, dict) assert chat_data[-12345]['test1'] == 'test2' assert chat_data[-67890][3] == 'test4' - assert chat_data[-54321] == {} bot_data = dict_persistence.get_bot_data() assert isinstance(bot_data, dict) @@ -2169,7 +2148,7 @@ def test_updating( assert 67890 not in dict_persistence.user_data dict_persistence._user_data = None dict_persistence.drop_user_data(123) - assert isinstance(dict_persistence.get_user_data(), defaultdict) + assert isinstance(dict_persistence.get_user_data(), dict) chat_data = dict_persistence.get_chat_data() chat_data[-12345]['test3']['test4'] = 'test6' @@ -2186,7 +2165,7 @@ def test_updating( assert -67890 not in dict_persistence.chat_data dict_persistence._chat_data = None dict_persistence.drop_chat_data(123) - assert isinstance(dict_persistence.get_chat_data(), defaultdict) + assert isinstance(dict_persistence.get_chat_data(), dict) bot_data = dict_persistence.get_bot_data() bot_data['test3']['test4'] = 'test6' From e2558eb6fccf586a5f0368176263f1e43436199c Mon Sep 17 00:00:00 2001 From: Harshil <37377066+harshil21@users.noreply.github.com> Date: Sat, 22 Jan 2022 02:47:24 +0530 Subject: [PATCH 2/6] some rearrangment in dispatcher --- telegram/ext/_dispatcher.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/telegram/ext/_dispatcher.py b/telegram/ext/_dispatcher.py index f635cf32cf9..859384d1a6f 100644 --- a/telegram/ext/_dispatcher.py +++ b/telegram/ext/_dispatcher.py @@ -321,7 +321,7 @@ def get_instance(cls) -> 'Dispatcher': def _pooled(self) -> None: thr_name = current_thread().name - while 1: + while True: promise = self.__async_queue.get() # If unpacking fails, the thread pool is being closed from Updater._join_async_threads @@ -417,7 +417,7 @@ def start(self, ready: Event = None) -> None: if ready is not None: ready.set() - while 1: + while True: try: # Pop update from update queue. update = self.update_queue.get(True, 1) @@ -527,9 +527,9 @@ def process_update(self, update: object) -> None: self.logger.debug('Error handler stopped further handlers.') break - # Update persistence, if handled - handled_only_async = all(sync_modes) + # Update persistence, if an update was handled if handled: + handled_only_async = all(sync_modes) # Respect default settings if ( all(mode is DEFAULT_FALSE for mode in sync_modes) @@ -739,7 +739,7 @@ def update_persistence(self, update: object = None) -> None: def __update_persistence(self, update: object = None) -> None: if self.persistence: - # We use list() here in order to decouple chat_ids from self._chat_data, as dict view + # We use list() here in order to decouple chat_ids from self.chat_data, as dict view # objects will change, when the dict does and we want to loop over chat_ids chat_ids = list(self.chat_data.keys()) user_ids = list(self.user_data.keys()) @@ -856,10 +856,10 @@ def dispatch_error( :obj:`bool`: :obj:`True` if one of the error handlers raised :class:`telegram.ext.DispatcherHandlerStop`. :obj:`False`, otherwise. """ - async_args = None if not promise else promise.args - async_kwargs = None if not promise else promise.kwargs - if self.error_handlers: + async_args = None if not promise else promise.args + async_kwargs = None if not promise else promise.kwargs + for ( callback, run_async, From b62576073c76c5677c46f9d81d0558f26c8cf119 Mon Sep 17 00:00:00 2001 From: Harshil <37377066+harshil21@users.noreply.github.com> Date: Sat, 22 Jan 2022 22:09:37 +0530 Subject: [PATCH 3/6] keep self.chat_data --- telegram/ext/_dispatcher.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/telegram/ext/_dispatcher.py b/telegram/ext/_dispatcher.py index 859384d1a6f..9d9d2e9a5aa 100644 --- a/telegram/ext/_dispatcher.py +++ b/telegram/ext/_dispatcher.py @@ -321,7 +321,7 @@ def get_instance(cls) -> 'Dispatcher': def _pooled(self) -> None: thr_name = current_thread().name - while True: + while 1: promise = self.__async_queue.get() # If unpacking fails, the thread pool is being closed from Updater._join_async_threads @@ -417,7 +417,7 @@ def start(self, ready: Event = None) -> None: if ready is not None: ready.set() - while True: + while 1: try: # Pop update from update queue. update = self.update_queue.get(True, 1) @@ -527,9 +527,9 @@ def process_update(self, update: object) -> None: self.logger.debug('Error handler stopped further handlers.') break - # Update persistence, if an update was handled + # Update persistence, if handled + handled_only_async = all(sync_modes) if handled: - handled_only_async = all(sync_modes) # Respect default settings if ( all(mode is DEFAULT_FALSE for mode in sync_modes) @@ -856,10 +856,10 @@ def dispatch_error( :obj:`bool`: :obj:`True` if one of the error handlers raised :class:`telegram.ext.DispatcherHandlerStop`. :obj:`False`, otherwise. """ - if self.error_handlers: - async_args = None if not promise else promise.args - async_kwargs = None if not promise else promise.kwargs + async_args = None if not promise else promise.args + async_kwargs = None if not promise else promise.kwargs + if self.error_handlers: for ( callback, run_async, From 128dd0ca300fc1ccfb7c9a574c1da9cdef1b58c0 Mon Sep 17 00:00:00 2001 From: Harshil <37377066+harshil21@users.noreply.github.com> Date: Sat, 22 Jan 2022 22:28:07 +0530 Subject: [PATCH 4/6] review --- telegram/ext/_basepersistence.py | 4 ++-- telegram/ext/_picklepersistence.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/telegram/ext/_basepersistence.py b/telegram/ext/_basepersistence.py index 47f2f7b40c3..a6d588cf0f2 100644 --- a/telegram/ext/_basepersistence.py +++ b/telegram/ext/_basepersistence.py @@ -132,10 +132,10 @@ def __new__( update_bot_data = instance.update_bot_data update_callback_data = instance.update_callback_data - def get_user_data_insert_bot() -> Union[DefaultDict[int, UD], Dict[int, UD]]: + def get_user_data_insert_bot() -> Dict[int, UD]: return instance.insert_bot(get_user_data()) - def get_chat_data_insert_bot() -> Union[DefaultDict[int, CD], Dict[int, CD]]: + def get_chat_data_insert_bot() -> Dict[int, CD]: return instance.insert_bot(get_chat_data()) def get_bot_data_insert_bot() -> BD: diff --git a/telegram/ext/_picklepersistence.py b/telegram/ext/_picklepersistence.py index 0bb2a4544e4..bf2d6b70dc2 100644 --- a/telegram/ext/_picklepersistence.py +++ b/telegram/ext/_picklepersistence.py @@ -319,7 +319,7 @@ def update_user_data(self, user_id: int, data: UD) -> None: """ if self.user_data is None: self.user_data = {} - if self.user_data.get(user_id, None) == data: # type: ignore[arg-type] + if self.user_data.get(user_id) == data: return self.user_data[user_id] = data if not self.on_flush: @@ -337,7 +337,7 @@ def update_chat_data(self, chat_id: int, data: CD) -> None: """ if self.chat_data is None: self.chat_data = {} - if self.chat_data.get(chat_id, None) == data: # type: ignore[arg-type] + if self.chat_data.get(chat_id) == data: return self.chat_data[chat_id] = data if not self.on_flush: From 3b87696dca86207dd51ab28e76cdbdb04d467329 Mon Sep 17 00:00:00 2001 From: Harshil <37377066+harshil21@users.noreply.github.com> Date: Sat, 22 Jan 2022 22:44:36 +0530 Subject: [PATCH 5/6] versionchanged tags --- telegram/ext/_basepersistence.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/telegram/ext/_basepersistence.py b/telegram/ext/_basepersistence.py index a6d588cf0f2..2236b608b28 100644 --- a/telegram/ext/_basepersistence.py +++ b/telegram/ext/_basepersistence.py @@ -19,7 +19,7 @@ """This module contains the BasePersistence class.""" from abc import ABC, abstractmethod from copy import copy -from typing import Dict, Optional, Tuple, cast, ClassVar, Generic, DefaultDict, NamedTuple, Union +from typing import Dict, Optional, Tuple, cast, ClassVar, Generic, NamedTuple from telegram import Bot from telegram.ext import ExtBot @@ -395,7 +395,7 @@ def _insert_bot(self, obj: object, memo: Dict[int, object]) -> object: return obj @abstractmethod - def get_user_data(self) -> Union[DefaultDict[int, UD], Dict[int, UD]]: + def get_user_data(self) -> Dict[int, UD]: """Will be called by :class:`telegram.ext.Dispatcher` upon creation with a persistence object. It should return the ``user_data`` if stored, or an empty :obj:`defaultdict` or :obj:`dict`. In the latter case, the dictionary should produce values @@ -405,13 +405,16 @@ def get_user_data(self) -> Union[DefaultDict[int, UD], Dict[int, UD]]: * The type from :attr:`telegram.ext.ContextTypes.user_data` if :class:`telegram.ext.ContextTypes` is used. + .. versionchanged:: 14.0 + This method may now return a :obj:`dict` instead of a :obj:`collections.defaultdict` + Returns: Dict[:obj:`int`, :obj:`dict` | :attr:`telegram.ext.ContextTypes.user_data`]: The restored user data. """ @abstractmethod - def get_chat_data(self) -> Union[DefaultDict[int, CD], Dict[int, CD]]: + def get_chat_data(self) -> Dict[int, CD]: """Will be called by :class:`telegram.ext.Dispatcher` upon creation with a persistence object. It should return the ``chat_data`` if stored, or an empty :obj:`defaultdict` or :obj:`dict`. In the latter case, the dictionary should produce values @@ -421,6 +424,9 @@ def get_chat_data(self) -> Union[DefaultDict[int, CD], Dict[int, CD]]: * The type from :attr:`telegram.ext.ContextTypes.chat_data` if :class:`telegram.ext.ContextTypes` is used. + .. versionchanged:: 14.0 + This method may now return a :obj:`dict` instead of a :obj:`collections.defaultdict` + Returns: Dict[:obj:`int`, :obj:`dict` | :attr:`telegram.ext.ContextTypes.chat_data`]: The restored chat data. From 2978f7e401483ac4e9f9194b083a58445512e3cd Mon Sep 17 00:00:00 2001 From: Harshil <37377066+harshil21@users.noreply.github.com> Date: Sun, 23 Jan 2022 03:15:35 +0530 Subject: [PATCH 6/6] review 2 --- telegram/ext/_basepersistence.py | 4 ++-- telegram/ext/_dictpersistence.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/telegram/ext/_basepersistence.py b/telegram/ext/_basepersistence.py index 2236b608b28..598d84dd264 100644 --- a/telegram/ext/_basepersistence.py +++ b/telegram/ext/_basepersistence.py @@ -398,7 +398,7 @@ def _insert_bot(self, obj: object, memo: Dict[int, object]) -> object: def get_user_data(self) -> Dict[int, UD]: """Will be called by :class:`telegram.ext.Dispatcher` upon creation with a persistence object. It should return the ``user_data`` if stored, or an empty - :obj:`defaultdict` or :obj:`dict`. In the latter case, the dictionary should produce values + :obj:`dict`. In the latter case, the dictionary should produce values corresponding to one of the following: * :obj:`dict` @@ -417,7 +417,7 @@ def get_user_data(self) -> Dict[int, UD]: def get_chat_data(self) -> Dict[int, CD]: """Will be called by :class:`telegram.ext.Dispatcher` upon creation with a persistence object. It should return the ``chat_data`` if stored, or an empty - :obj:`defaultdict` or :obj:`dict`. In the latter case, the dictionary should produce values + :obj:`dict`. In the latter case, the dictionary should produce values corresponding to one of the following: * :obj:`dict` diff --git a/telegram/ext/_dictpersistence.py b/telegram/ext/_dictpersistence.py index 48110618600..6ea7bb17c53 100644 --- a/telegram/ext/_dictpersistence.py +++ b/telegram/ext/_dictpersistence.py @@ -313,7 +313,7 @@ def update_user_data(self, user_id: int, data: Dict) -> None: """ if self._user_data is None: self._user_data = {} - if self._user_data.get(user_id, None) == data: + if self._user_data.get(user_id) == data: return self._user_data[user_id] = data self._user_data_json = None @@ -327,7 +327,7 @@ def update_chat_data(self, chat_id: int, data: Dict) -> None: """ if self._chat_data is None: self._chat_data = {} - if self._chat_data.get(chat_id, None) == data: + if self._chat_data.get(chat_id) == data: return self._chat_data[chat_id] = data self._chat_data_json = None