diff --git a/contrib/pg_tde/src/access/pg_tde_tdemap.c b/contrib/pg_tde/src/access/pg_tde_tdemap.c
index 1a9e118771c49..082a9ad8f2fd5 100644
--- a/contrib/pg_tde/src/access/pg_tde_tdemap.c
+++ b/contrib/pg_tde/src/access/pg_tde_tdemap.c
@@ -297,10 +297,9 @@ pg_tde_perform_rotate_key(const TDEPrincipalKey *principal_key, const TDEPrincip
 	CloseTransientFile(new_fd);
 
 	/*
-	 * Do the final steps - replace the current _map with the file with new
+	 * Do the final step - replace the current _keys with the file with new
 	 * data
 	 */
-	durable_unlink(old_path, ERROR);
 	durable_rename(new_path, old_path, ERROR);
 
 	/*
diff --git a/contrib/pg_tde/src/access/pg_tde_xlog_keys.c b/contrib/pg_tde/src/access/pg_tde_xlog_keys.c
index 4c14625e0080f..f1c64aa5a32b2 100644
--- a/contrib/pg_tde/src/access/pg_tde_xlog_keys.c
+++ b/contrib/pg_tde/src/access/pg_tde_xlog_keys.c
@@ -718,10 +718,9 @@ pg_tde_perform_rotate_server_key(const TDEPrincipalKey *principal_key,
 	CloseTransientFile(new_fd);
 
 	/*
-	 * Do the final steps - replace the current WAL key file with the file
-	 * with new data.
+	 * Do the final step - replace the current WAL key file with the file with
+	 * new data.
 	 */
-	durable_unlink(get_wal_key_file_path(), ERROR);
 	durable_rename(tmp_path, get_wal_key_file_path(), ERROR);
 
 	/*