diff --git a/src/shrpx.cc b/src/shrpx.cc
index f533988..13e3a10 100644
--- a/src/shrpx.cc
+++ b/src/shrpx.cc
@@ -259,10 +259,6 @@ int event_loop()
     }
   }
 
-  // ListenHandler loads private key. After that, we drop the root
-  // privileges if needed.
-  drop_privileges();
-
   evconnlistener *evlistener6, *evlistener4;
   evlistener6 = create_evlistener(listener_handler, AF_INET6);
   evlistener4 = create_evlistener(listener_handler, AF_INET);
@@ -272,6 +268,10 @@ int event_loop()
     exit(EXIT_FAILURE);
   }
 
+  // ListenHandler loads private key, and we listen on a priveleged port.
+  // After that, we drop the root privileges if needed.
+  drop_privileges();
+
   if(get_config()->num_worker > 1) {
     listener_handler->create_worker_thread(get_config()->num_worker);
   } else if(get_config()->downstream_proto == PROTO_SPDY) {