diff --git a/src/ngircd/conf.c b/src/ngircd/conf.c
index bea4d619..758ee743 100644
--- a/src/ngircd/conf.c
+++ b/src/ngircd/conf.c
@@ -108,6 +108,28 @@ ConfSSL_Init(void)
 	array_free(&Conf_SSLOptions.ListenPorts);
 }
 
+/**
+ * Check if the current configuration uses/requires SSL.
+ *
+ * @returns true if SSL is used and should be initialized.
+ */
+GLOBAL bool
+Conf_SSLInUse(void)
+{
+	int i;
+
+	/* SSL listen ports configured? */
+	if (array_bytes(&Conf_SSLOptions.ListenPorts))
+		return true;
+
+	for (i = 0; i < MAX_SERVERS; i++) {
+		if (Conf_Server[i].port > 0
+		    && Conf_Server[i].SSLConnect)
+			return true;
+	}
+	return false;
+}
+
 /**
  * Make sure that a configured file is readable.
  *
diff --git a/src/ngircd/conf.h b/src/ngircd/conf.h
index f85a25fa..0c80c243 100644
--- a/src/ngircd/conf.h
+++ b/src/ngircd/conf.h
@@ -253,6 +253,10 @@ GLOBAL bool Conf_AddServer PARAMS(( const char *Name, UINT16 Port, const char *H
 GLOBAL bool Conf_NickIsService PARAMS((int ConfServer, const char *Nick));
 GLOBAL bool Conf_NickIsBlocked PARAMS((const char *Nick));
 
+#ifdef SSL_SUPPORT
+GLOBAL bool Conf_SSLInUse PARAMS((void));
+#endif
+
 /* Password required by WEBIRC command */
 GLOBAL char Conf_WebircPwd[CLIENT_PASS_LEN];
 
diff --git a/src/ngircd/conn-ssl.c b/src/ngircd/conn-ssl.c
index 59729e04..45e6458a 100644
--- a/src/ngircd/conn-ssl.c
+++ b/src/ngircd/conn-ssl.c
@@ -241,8 +241,10 @@ void ConnSSL_Free(CONNECTION *c)
 bool
 ConnSSL_InitLibrary( void )
 {
-	if (!array_bytes(&Conf_SSLOptions.ListenPorts))
+	if (!Conf_SSLInUse()) {
+		LogDebug("SSL not in use, skipping initialization.");
 		return true;
+	}
 
 #ifdef HAVE_LIBSSL
 	SSL_CTX *newctx;