diff --git a/.travis.yml b/.travis.yml
index 408d4b7ff..c86b0273c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -170,7 +170,7 @@ jobs:
         - libtasn1-6-dev
         - nettle-dev
   - env:
-    - T=debug-wolfssl C="--with-wolfssl --without-ssl"
+    - T=debug-wolfssl C="--with-wolfssl --without-ssl" WOLFSSL="yes"
     - OVERRIDE_CC="CC=gcc-8" OVERRIDE_CXX="CXX=g++-8"
     addons:
       apt:
diff --git a/scripts/travis/before_script.sh b/scripts/travis/before_script.sh
index 231b6ba27..945132e46 100755
--- a/scripts/travis/before_script.sh
+++ b/scripts/travis/before_script.sh
@@ -137,11 +137,7 @@ if [ "$TRAVIS_OS_NAME" = linux -a "$RUSTLS" ]; then
   make DESTDIR=$HOME/crust install
 fi
 
-# Install common libraries.
-# The library build directories are set to be cached by .travis.yml. If you are
-# changing a build directory name below (eg a version change) then you must
-# change it in .travis.yml `cache: directories:` as well.
-if [ $TRAVIS_OS_NAME = linux ]; then
+if [ $TRAVIS_OS_NAME = linux -a "$WOLFSSL" ]; then
   if [ ! -e $HOME/wolfssl-4.4.0-stable/Makefile ]; then
     cd $HOME
     curl -LO https://github.com/wolfSSL/wolfssl/archive/v4.4.0-stable.tar.gz
@@ -155,6 +151,13 @@ if [ $TRAVIS_OS_NAME = linux ]; then
 
   cd $HOME/wolfssl-4.4.0-stable
   sudo make install
+fi
+
+# Install common libraries.
+# The library build directories are set to be cached by .travis.yml. If you are
+# changing a build directory name below (eg a version change) then you must
+# change it in .travis.yml `cache: directories:` as well.
+if [ $TRAVIS_OS_NAME = linux ]; then
 
   if [ "$MESALINK" = "yes" ]; then
     if [ ! -e $HOME/mesalink-1.0.0/Makefile ]; then