diff --git a/Misc/NEWS.d/next/Library/2019-03-04-10-42-46.bpo-36179.jEyuI-.rst b/Misc/NEWS.d/next/Library/2019-03-04-10-42-46.bpo-36179.jEyuI-.rst
new file mode 100644
index 00000000000000..61a98778b78e9d
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-03-04-10-42-46.bpo-36179.jEyuI-.rst
@@ -0,0 +1,2 @@
+Fix two unlikely reference leaks in _hashopenssl. The leaks only occur in
+out-of-memory cases.
diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c
index de69f6fcd03766..78445ebabdd33c 100644
--- a/Modules/_hashopenssl.c
+++ b/Modules/_hashopenssl.c
@@ -133,12 +133,6 @@ newEVPobject(PyObject *name)
     if (retval == NULL)
         return NULL;
 
-    retval->ctx = EVP_MD_CTX_new();
-    if (retval->ctx == NULL) {
-        PyErr_NoMemory();
-        return NULL;
-    }
-
     /* save the name for .name to return */
     Py_INCREF(name);
     retval->name = name;
@@ -146,6 +140,13 @@ newEVPobject(PyObject *name)
     retval->lock = NULL;
 #endif
 
+    retval->ctx = EVP_MD_CTX_new();
+    if (retval->ctx == NULL) {
+        Py_DECREF(retval);
+        PyErr_NoMemory();
+        return NULL;
+    }
+
     return retval;
 }
 
@@ -205,6 +206,7 @@ EVP_copy(EVPobject *self, PyObject *unused)
         return NULL;
 
     if (!locked_EVP_MD_CTX_copy(newobj->ctx, self)) {
+        Py_DECREF(newobj);
         return _setException(PyExc_ValueError);
     }
     return (PyObject *)newobj;