diff --git a/testapps/on_device_unit_tests/test_app/app_flask.py b/testapps/on_device_unit_tests/test_app/app_flask.py
index ee23532035..4ce8f5a1a0 100644
--- a/testapps/on_device_unit_tests/test_app/app_flask.py
+++ b/testapps/on_device_unit_tests/test_app/app_flask.py
@@ -110,6 +110,7 @@ def loadUrl():
     print('asked to open url', args['url'])
     activity = get_android_python_activity()
     activity.loadUrl(args['url'])
+    return ('', 204)
 
 
 @app.route('/vibrate')
@@ -122,7 +123,8 @@ def vibrate():
     if 'time' not in args:
         print('ERROR: asked to vibrate but without time argument')
     print('asked to vibrate', args['time'])
-    return vibrate_with_pyjnius(int(float(args['time']) * 1000))
+    vibrate_with_pyjnius(int(float(args['time']) * 1000))
+    return ('', 204)
 
 
 @app.route('/orientation')
@@ -135,4 +137,5 @@ def orientation():
         print('ERROR: asked to orient but no dir specified')
         return 'No direction specified '
     direction = args['dir']
-    return set_device_orientation(direction)
+    set_device_orientation(direction)
+    return ('', 204)
diff --git a/testapps/on_device_unit_tests/test_app/main.py b/testapps/on_device_unit_tests/test_app/main.py
index d0fa34e97d..48bf0dc33d 100644
--- a/testapps/on_device_unit_tests/test_app/main.py
+++ b/testapps/on_device_unit_tests/test_app/main.py
@@ -85,10 +85,15 @@
     app_flask.TESTS_TO_PERFORM = tests_to_perform
 
     print('Current directory is ', realpath(curdir))
-    if realpath(curdir).startswith('/data'):
-        app_flask.app.run(debug=False)
-    else:
-        app_flask.app.run(debug=True)
+    flask_debug = not realpath(curdir).startswith('/data')
+
+    # Flask is run non-threaded since it tries to resolve app classes
+    # through pyjnius from request handlers. That doesn't work since the
+    # JNI ends up using the Java system class loader in new native
+    # threads.
+    #
+    # https://github.com/kivy/python-for-android/issues/2533
+    app_flask.app.run(threaded=False, debug=flask_debug)
 else:
     # we don't have kivy or flask in our
     # requirements, so we run unittests in terminal