diff --git a/src/main/java/org/apache/commons/dbcp2/PoolableConnectionFactory.java b/src/main/java/org/apache/commons/dbcp2/PoolableConnectionFactory.java
index a2ab9772a..bdcdddc6a 100644
--- a/src/main/java/org/apache/commons/dbcp2/PoolableConnectionFactory.java
+++ b/src/main/java/org/apache/commons/dbcp2/PoolableConnectionFactory.java
@@ -425,16 +425,6 @@ protected void initializeConnection(final Connection conn) throws SQLException {
                 for (final String sql : sqls) {
                     statement.execute(Objects.requireNonNull(sql, "null connectionInitSqls element"));
                 }
-            } catch (SQLException sqle) {
-                /*
-                 * Need to close the connection here as the reference to it will be lost once the SQLEXception is
-                 * thrown.
-                 *
-                 * Cast to AutoCloseable to avoid calling the deprecated method. The cast can be removed once the
-                 * deprecated method has been removed.s
-                 */
-                Utils.closeQuietly((AutoCloseable) conn);
-                throw sqle;
             }
         }
     }
diff --git a/src/main/java/org/apache/commons/dbcp2/managed/PoolableManagedConnectionFactory.java b/src/main/java/org/apache/commons/dbcp2/managed/PoolableManagedConnectionFactory.java
index 96cd4769a..c91425241 100644
--- a/src/main/java/org/apache/commons/dbcp2/managed/PoolableManagedConnectionFactory.java
+++ b/src/main/java/org/apache/commons/dbcp2/managed/PoolableManagedConnectionFactory.java
@@ -28,6 +28,7 @@
 import org.apache.commons.dbcp2.PoolableConnection;
 import org.apache.commons.dbcp2.PoolableConnectionFactory;
 import org.apache.commons.dbcp2.PoolingConnection;
+import org.apache.commons.dbcp2.Utils;
 import org.apache.commons.pool2.KeyedObjectPool;
 import org.apache.commons.pool2.PooledObject;
 import org.apache.commons.pool2.impl.DefaultPooledObject;
@@ -80,7 +81,14 @@ public synchronized PooledObject<PoolableConnection> makeObject() throws SQLExce
         if (conn == null) {
             throw new IllegalStateException("Connection factory returned null from createConnection");
         }
-        initializeConnection(conn);
+        try {
+            initializeConnection(conn);
+        } catch (final SQLException e) {
+            // Make sure the connection is closed
+            Utils.closeQuietly((AutoCloseable) conn);
+            // Rethrow original exception so it is visible to caller
+            throw e;
+        }
         if (getPoolStatements()) {
             conn = new PoolingConnection(conn);
             final GenericKeyedObjectPoolConfig<DelegatingPreparedStatement> config = new GenericKeyedObjectPoolConfig<>();