diff --git a/Include/modsupport.h b/Include/modsupport.h
index f90ede4831e32b..2052725eec2558 100644
--- a/Include/modsupport.h
+++ b/Include/modsupport.h
@@ -134,6 +134,7 @@ void _PyArg_Fini(void);
 #endif   /* Py_LIMITED_API */
 
 PyAPI_FUNC(int) PyModule_AddObject(PyObject *, const char *, PyObject *);
+PyAPI_FUNC(int) _PyModule_StealObject(PyObject *, const char *, PyObject *);
 PyAPI_FUNC(int) PyModule_AddIntConstant(PyObject *, const char *, long);
 PyAPI_FUNC(int) PyModule_AddStringConstant(PyObject *, const char *, const char *);
 #define PyModule_AddIntMacro(m, c) PyModule_AddIntConstant(m, #c, c)
diff --git a/Misc/NEWS.d/next/C API/2019-11-20-18-47-48.bpo-38823.x1nghH.rst b/Misc/NEWS.d/next/C API/2019-11-20-18-47-48.bpo-38823.x1nghH.rst
new file mode 100644
index 00000000000000..82d0b07cc13997
--- /dev/null
+++ b/Misc/NEWS.d/next/C API/2019-11-20-18-47-48.bpo-38823.x1nghH.rst	
@@ -0,0 +1,3 @@
+Add a new private ``_PyModule_StealObject`` API. It is identical to :c:func:`PyModule_AddObject`, but steals a reference to the added object on both success *and* failure.
+
+Patch by Brandt Bucher.
\ No newline at end of file
diff --git a/Modules/_bz2module.c b/Modules/_bz2module.c
index 31bbf66104119c..afc0614b4119d3 100644
--- a/Modules/_bz2module.c
+++ b/Modules/_bz2module.c
@@ -755,11 +755,20 @@ PyInit__bz2(void)
         return NULL;
 
     Py_INCREF(&BZ2Compressor_Type);
-    PyModule_AddObject(m, "BZ2Compressor", (PyObject *)&BZ2Compressor_Type);
+    if (_PyModule_StealObject(m, "BZ2Compressor",
+                            (PyObject *)&BZ2Compressor_Type) < 0) {
+        goto fail;
+    }
 
     Py_INCREF(&BZ2Decompressor_Type);
-    PyModule_AddObject(m, "BZ2Decompressor",
-                       (PyObject *)&BZ2Decompressor_Type);
+    if (_PyModule_StealObject(m, "BZ2Decompressor",
+                            (PyObject *)&BZ2Decompressor_Type) < 0) {
+        goto fail;
+    }
 
     return m;
+
+fail:
+    Py_DECREF(m);
+    return NULL;
 }
diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c
index 1d23973fd05661..1b3c053bc4d605 100644
--- a/Modules/_collectionsmodule.c
+++ b/Modules/_collectionsmodule.c
@@ -2539,34 +2539,55 @@ PyInit__collections(void)
     if (m == NULL)
         return NULL;
 
-    if (PyType_Ready(&deque_type) < 0)
-        return NULL;
+    if (PyType_Ready(&deque_type) < 0) {
+        goto fail;
+    }
     Py_INCREF(&deque_type);
-    PyModule_AddObject(m, "deque", (PyObject *)&deque_type);
+    if (_PyModule_StealObject(m, "deque", (PyObject *)&deque_type) < 0) {
+        goto fail;
+    }
 
     defdict_type.tp_base = &PyDict_Type;
-    if (PyType_Ready(&defdict_type) < 0)
-        return NULL;
+    if (PyType_Ready(&defdict_type) < 0) {
+        goto fail;
+    }
     Py_INCREF(&defdict_type);
-    PyModule_AddObject(m, "defaultdict", (PyObject *)&defdict_type);
+    if (_PyModule_StealObject(m, "defaultdict", (PyObject *)&defdict_type) < 0) {
+        goto fail;
+    }
 
     Py_INCREF(&PyODict_Type);
-    PyModule_AddObject(m, "OrderedDict", (PyObject *)&PyODict_Type);
+    if (_PyModule_StealObject(m, "OrderedDict", (PyObject *)&PyODict_Type) < 0) {
+        goto fail;
+    }
 
-    if (PyType_Ready(&dequeiter_type) < 0)
-        return NULL;
+    if (PyType_Ready(&dequeiter_type) < 0) {
+        goto fail;
+    }
     Py_INCREF(&dequeiter_type);
-    PyModule_AddObject(m, "_deque_iterator", (PyObject *)&dequeiter_type);
+    if (_PyModule_StealObject(m, "_deque_iterator", (PyObject *)&dequeiter_type) < 0) {
+        goto fail;
+    }
 
-    if (PyType_Ready(&dequereviter_type) < 0)
-        return NULL;
+    if (PyType_Ready(&dequereviter_type) < 0) {
+        goto fail;
+    }
     Py_INCREF(&dequereviter_type);
-    PyModule_AddObject(m, "_deque_reverse_iterator", (PyObject *)&dequereviter_type);
+    if (_PyModule_StealObject(m, "_deque_reverse_iterator", (PyObject *)&dequereviter_type) < 0) {
+        goto fail;
+    }
 
-    if (PyType_Ready(&tuplegetter_type) < 0)
-        return NULL;
+    if (PyType_Ready(&tuplegetter_type) < 0) {
+        goto fail;
+    }
     Py_INCREF(&tuplegetter_type);
-    PyModule_AddObject(m, "_tuplegetter", (PyObject *)&tuplegetter_type);
+    if (_PyModule_StealObject(m, "_tuplegetter", (PyObject *)&tuplegetter_type) < 0) {
+        goto fail;
+    }
 
     return m;
+
+fail:
+    Py_DECREF(m);
+    return NULL;
 }
diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c
index 987087b1ac97ba..6e8b7c0f69a234 100644
--- a/Modules/_functoolsmodule.c
+++ b/Modules/_functoolsmodule.c
@@ -1457,7 +1457,10 @@ PyInit__functools(void)
         }
         name = _PyType_Name(typelist[i]);
         Py_INCREF(typelist[i]);
-        PyModule_AddObject(m, name, (PyObject *)typelist[i]);
+        if (_PyModule_StealObject(m, name, (PyObject *)typelist[i]) < 0) {
+            Py_DECREF(m);
+            return NULL;
+        }
     }
     return m;
 }
diff --git a/PC/_msi.c b/PC/_msi.c
index accbe7a7206944..ddd2b26831d82b 100644
--- a/PC/_msi.c
+++ b/PC/_msi.c
@@ -1120,6 +1120,9 @@ PyInit__msi(void)
     MSIError = PyErr_NewException ("_msi.MSIError", NULL, NULL);
     if (!MSIError)
         return NULL;
-    PyModule_AddObject(m, "MSIError", MSIError);
+    if (_PyModule_StealObject(m, "MSIError", MSIError) < 0) {
+        Py_DECREF(m);
+        return NULL;
+    }
     return m;
 }
diff --git a/Python/_warnings.c b/Python/_warnings.c
index b8585d204787db..0ec7452d0e9d10 100644
--- a/Python/_warnings.c
+++ b/Python/_warnings.c
@@ -1353,17 +1353,17 @@ _PyWarnings_Init(void)
     }
 
     Py_INCREF(st->filters);
-    if (PyModule_AddObject(m, "filters", st->filters) < 0) {
+    if (_PyModule_StealObject(m, "filters", st->filters) < 0) {
         goto error;
     }
 
     Py_INCREF(st->once_registry);
-    if (PyModule_AddObject(m, "_onceregistry", st->once_registry) < 0) {
+    if (_PyModule_StealObject(m, "_onceregistry", st->once_registry) < 0) {
         goto error;
     }
 
     Py_INCREF(st->default_action);
-    if (PyModule_AddObject(m, "_defaultaction", st->default_action) < 0) {
+    if (_PyModule_StealObject(m, "_defaultaction", st->default_action) < 0) {
         goto error;
     }
 
diff --git a/Python/modsupport.c b/Python/modsupport.c
index 62558221077468..a3a9eca772021b 100644
--- a/Python/modsupport.c
+++ b/Python/modsupport.c
@@ -655,6 +655,19 @@ PyModule_AddObject(PyObject *m, const char *name, PyObject *o)
     return 0;
 }
 
+/* Like PyModule_AddObject, but steals o on success AND failure.
+   This is probably what you want! */
+
+int
+_PyModule_StealObject(PyObject *m, const char *name, PyObject *o)
+{
+    if (PyModule_AddObject(m, name, o) < 0) {
+        Py_XDECREF(o);
+        return -1;
+    }
+    return 0;
+}
+
 int
 PyModule_AddIntConstant(PyObject *m, const char *name, long value)
 {