diff --git a/lib/vtls/openssl.c b/lib/vtls/openssl.c
index 41d948b3a..897ca6880 100644
--- a/lib/vtls/openssl.c
+++ b/lib/vtls/openssl.c
@@ -3024,7 +3024,7 @@ static CURLcode ossl_connect_step1(struct connectdata *conn, int sockindex)
 #endif
 
 #ifdef CURL_CA_FALLBACK
-  else if(verifypeer) {
+  if(verifypeer && !ssl_cafile && !ssl_capath) {
     /* verifying the peer without any CA certificates won't
        work so use openssl's built in default as fallback */
     SSL_CTX_set_default_verify_paths(backend->ctx);