diff --git a/tests/test_persistence.py b/tests/test_persistence.py
index 1927e154932..9e9b2665c68 100644
--- a/tests/test_persistence.py
+++ b/tests/test_persistence.py
@@ -37,6 +37,17 @@
     PicklePersistence, CommandHandler, DictPersistence, TypeHandler
 
 
+@pytest.fixture(autouse=True)
+def change_directory(tmp_path):
+    orig_dir = os.getcwd()
+    # Switch to a temporary directory so we don't have to worry about cleaning up files
+    # (str() for py<3.6)
+    os.chdir(str(tmp_path))
+    yield
+    # Go back to original directory
+    os.chdir(orig_dir)
+
+
 @pytest.fixture(scope="function")
 def base_persistence():
     return BasePersistence(store_chat_data=True, store_user_data=True, store_bot_data=True)
@@ -310,9 +321,6 @@ def bad_pickle_files():
         with open(name, 'w') as f:
             f.write('(())')
     yield True
-    for name in ['pickletest_user_data', 'pickletest_chat_data', 'pickletest_bot_data',
-                 'pickletest_conversations', 'pickletest']:
-        os.remove(name)
 
 
 @pytest.fixture(scope='function')
@@ -330,9 +338,6 @@ def good_pickle_files(user_data, chat_data, bot_data, conversations):
     with open('pickletest', 'wb') as f:
         pickle.dump(data, f)
     yield True
-    for name in ['pickletest_user_data', 'pickletest_chat_data', 'pickletest_bot_data',
-                 'pickletest_conversations', 'pickletest']:
-        os.remove(name)
 
 
 @pytest.fixture(scope='function')
@@ -347,9 +352,6 @@ def pickle_files_wo_bot_data(user_data, chat_data, conversations):
     with open('pickletest', 'wb') as f:
         pickle.dump(data, f)
     yield True
-    for name in ['pickletest_user_data', 'pickletest_chat_data',
-                 'pickletest_conversations', 'pickletest']:
-        os.remove(name)
 
 
 @pytest.fixture(scope='function')
@@ -776,9 +778,6 @@ def test_flush_on_stop(self, bot, update, pickle_persistence):
         assert pickle_persistence_2.get_bot_data()['test'] == 'Working3!'
 
     def test_flush_on_stop_only_bot(self, bot, update, pickle_persistence_only_bot):
-        os.remove('pickletest_user_data')
-        os.remove('pickletest_chat_data')
-        os.remove('pickletest_bot_data')
         u = Updater(bot=bot, persistence=pickle_persistence_only_bot)
         dp = u.dispatcher
         u.running = True
@@ -800,7 +799,6 @@ def test_flush_on_stop_only_bot(self, bot, update, pickle_persistence_only_bot):
         assert pickle_persistence_2.get_bot_data()['my_test3'] == 'Working3!'
 
     def test_flush_on_stop_only_chat(self, bot, update, pickle_persistence_only_chat):
-        os.remove('pickletest_bot_data')
         u = Updater(bot=bot, persistence=pickle_persistence_only_chat)
         dp = u.dispatcher
         u.running = True
@@ -821,7 +819,6 @@ def test_flush_on_stop_only_chat(self, bot, update, pickle_persistence_only_chat
         assert pickle_persistence_2.get_bot_data() == {}
 
     def test_flush_on_stop_only_user(self, bot, update, pickle_persistence_only_user):
-        os.remove('pickletest_chat_data')
         u = Updater(bot=bot, persistence=pickle_persistence_only_user)
         dp = u.dispatcher
         u.running = True
@@ -923,17 +920,6 @@ def next2(update, context):
         assert nested_ch.conversations[nested_ch._get_key(update)] == 1
         assert nested_ch.conversations == pickle_persistence.conversations['name3']
 
-    @classmethod
-    def teardown_class(cls):
-        try:
-            for name in ['pickletest_user_data', 'pickletest_chat_data',
-                         'pickletest_bot_data',
-                         'pickletest_conversations',
-                         'pickletest']:
-                os.remove(name)
-        except Exception:
-            pass
-
 
 @pytest.fixture(scope='function')
 def user_data_json(user_data):