tls: evolve handshake serialization into simultaneous_ssl_handshake_restriction

This patch adapts the recent change about serializing the number of
simultaneous tls handshakes allowed to 1, so you can set the number in the
context creation info, and the accounting for it is handled by counters
same as the overally tls restriction.

The name of the context info var to control it changes to simultaneous_ssl_handshake_restriction
which is now a count, the default 0 means no limit.

The count rejects tls connection attempts when the tls borrow is attempted,
and separately hands back the hs borrow from the tls borrow when the
connection attempt fails or succeeds.

13 files changed, 127 insertions, 96 deletions

diff --git a/include/libwebsockets/lws-context-vhost.h b/include/libwebsockets/lws-context-vhost.h
index 522a8fad..b3de140b 100644
--- a/include/libwebsockets/lws-context-vhost.h
+++ b/include/libwebsockets/lws-context-vhost.h
@@ -453,9 +453,8 @@ struct lws_context_creation_info {
 	int simultaneous_ssl_restriction;
 	/**< CONTEXT: 0 (no limit) or limit of simultaneous SSL sessions
 	 * possible.*/
-	int ssl_handshake_serialize;
-	/**< CONTEXT: 0 disables ssl handshake serialization (default).
-	 *            1 enables ssl handshake serialization. */
+	int simultaneous_ssl_handshake_restriction;
+	/**< CONTEXT: 0 (no limit) or limit of simultaneous SSL handshakes ongoing */
 	int ssl_info_event_mask;
 	/**< VHOST: mask of ssl events to be reported on LWS_CALLBACK_SSL_INFO
 	 * callback for connections on this vhost.  The mask values are of
diff --git a/lib/core-net/client/connect.c b/lib/core-net/client/connect.c
index 4b45bbe0..1844abef 100644
--- a/lib/core-net/client/connect.c
+++ b/lib/core-net/client/connect.c
@@ -530,8 +530,8 @@ bail3:
 
 bail:
 #if defined(LWS_WITH_TLS)
-	if (wsi->tls.ssl && wsi->tls_borrowed)
-		lws_tls_restrict_return(i->context);
+	if (wsi->tls.ssl)
+		lws_tls_restrict_return(wsi);
 #endif
 
 	lws_free_set_NULL(wsi->stash);
diff --git a/lib/core-net/private-lib-core-net.h b/lib/core-net/private-lib-core-net.h
index 1f566178..3f507311 100644
--- a/lib/core-net/private-lib-core-net.h
+++ b/lib/core-net/private-lib-core-net.h
@@ -809,6 +809,7 @@ struct lws {
 	unsigned int			client_proxy_onward:1;
 #endif
 	unsigned int                    tls_borrowed:1;
+	unsigned int                    tls_borrowed_hs:1;
 	unsigned int                    tls_read_wanted_write:1;
 
 #ifdef LWS_WITH_ACCESS_LOG
diff --git a/lib/core/context.c b/lib/core/context.c
index 95219e47..d55ee515 100644
--- a/lib/core/context.c
+++ b/lib/core/context.c
@@ -915,7 +915,8 @@ lws_create_context(const struct lws_context_creation_info *info)
 #if defined(LWS_WITH_TLS) && defined(LWS_WITH_NETWORK)
 	context->simultaneous_ssl_restriction =
 			info->simultaneous_ssl_restriction;
-	context->ssl_handshake_serialize = info->ssl_handshake_serialize;
+	context->simultaneous_ssl_handshake_restriction =
+			info->simultaneous_ssl_handshake_restriction;
 #endif
 
 	context->options = info->options;
diff --git a/lib/core/private-lib-core.h b/lib/core/private-lib-core.h
index 51078a5a..8ba08520 100644
--- a/lib/core/private-lib-core.h
+++ b/lib/core/private-lib-core.h
@@ -702,7 +702,8 @@ struct lws_context {
 	unsigned int max_http_header_pool;
 	int simultaneous_ssl_restriction;
 	int simultaneous_ssl;
-	int ssl_handshake_serialize;
+	int simultaneous_ssl_handshake_restriction;
+	int simultaneous_ssl_handshake;
 #if defined(LWS_WITH_TLS_JIT_TRUST)
 	int		vh_idle_grace_ms;
 #endif
@@ -721,6 +722,8 @@ struct lws_context {
 	lws_route_uidx_t			route_uidx;
 #endif
 
+	char		tls_gate_accepts;
+
 	unsigned int deprecated:1;
 	unsigned int inside_context_destroy:1;
 	unsigned int being_destroyed:1;
diff --git a/lib/tls/mbedtls/mbedtls-ssl.c b/lib/tls/mbedtls/mbedtls-ssl.c
index 436f34aa..5fe92205 100644
--- a/lib/tls/mbedtls/mbedtls-ssl.c
+++ b/lib/tls/mbedtls/mbedtls-ssl.c
@@ -278,8 +278,7 @@ lws_ssl_close(struct lws *wsi)
 	SSL_free(wsi->tls.ssl);
 	wsi->tls.ssl = NULL;
 
-	if (wsi->tls_borrowed)
-		lws_tls_restrict_return(wsi->a.context);
+	lws_tls_restrict_return(wsi);
 
 	return 1; /* handled */
 }
diff --git a/lib/tls/openssl/openssl-ssl.c b/lib/tls/openssl/openssl-ssl.c
index 09844a42..cf4d2b8c 100644
--- a/lib/tls/openssl/openssl-ssl.c
+++ b/lib/tls/openssl/openssl-ssl.c
@@ -469,8 +469,7 @@ lws_ssl_close(struct lws *wsi)
 	SSL_free(wsi->tls.ssl);
 	wsi->tls.ssl = NULL;
 
-	if (wsi->tls_borrowed)
-		lws_tls_restrict_return(wsi->a.context);
+	lws_tls_restrict_return(wsi);
 
 	// lwsl_notice("%s: ssl restr %d, simul %d\n", __func__,
 	//		wsi->a.context->simultaneous_ssl_restriction,
diff --git a/lib/tls/private-lib-tls.h b/lib/tls/private-lib-tls.h
index 23fc2a72..13ac6926 100644
--- a/lib/tls/private-lib-tls.h
+++ b/lib/tls/private-lib-tls.h
@@ -126,10 +126,13 @@ enum lws_tls_extant {
 #endif
 
 int
-lws_tls_restrict_borrow(struct lws_context *context);
+lws_tls_restrict_borrow(struct lws *wsi);
 
 void
-lws_tls_restrict_return(struct lws_context *context);
+lws_tls_restrict_return(struct lws *wsi);
+
+void
+lws_tls_restrict_return_handshake(struct lws *wsi);
 
 typedef SSL lws_tls_conn;
 typedef SSL_CTX lws_tls_ctx;
diff --git a/lib/tls/tls-client.c b/lib/tls/tls-client.c
index 6009e435..9d93a3da 100644
--- a/lib/tls/tls-client.c
+++ b/lib/tls/tls-client.c
@@ -25,43 +25,6 @@
 #include "private-lib-core.h"
 
 static int
-lws_ssl_handshake_serialize(struct lws_context *ctx, struct lws *wsi)
-{
-	struct lws_vhost *vh = ctx->vhost_list;
-#if LWS_MAX_SMP > 1
-	int tsi = lws_pthread_self_to_tsi(ctx);
-#else
-	int tsi = 0;
-#endif
-	struct lws_context_per_thread *pt = &ctx->pt[tsi];
-	unsigned int n;
-
-	while (vh) {
-		for (n = 0; n < pt->fds_count; n++) {
-			struct lws *w = wsi_from_fd(ctx, pt->fds[n].fd);
-
-			if (!w || w->tsi != tsi || w->a.vhost != vh || wsi == w)
-				continue;
-
-			/* Now we found other vhost's wsi in process */
-			if (lwsi_role_mqtt(w)) {
-				/* MQTT TLS connection not established yet.
-				 * Let it finish.
-				 */
-				if (lwsi_state(w) != LRS_ESTABLISHED)
-					return 1;
-			} else {
-				/* H1/H2 not finished yet. Let it finish. */
-				if (lwsi_state(w) != LRS_DEAD_SOCKET)
-					return 1;
-			}
-		}
-		vh = vh->vhost_next;
-	}
-	return 0;
-}
-
-static int
 lws_ssl_client_connect1(struct lws *wsi, char *errbuf, size_t len)
 {
 	int n;
@@ -69,8 +32,10 @@ lws_ssl_client_connect1(struct lws *wsi, char *errbuf, size_t len)
 	n = lws_tls_client_connect(wsi, errbuf, len);
 	switch (n) {
 	case LWS_SSL_CAPABLE_ERROR:
+		lws_tls_restrict_return_handshake(wsi);
 		return -1;
 	case LWS_SSL_CAPABLE_DONE:
+		lws_tls_restrict_return_handshake(wsi);
 		lws_metrics_caliper_report(wsi->cal_conn, METRES_GO);
 #if defined(LWS_WITH_CONMON)
 	wsi->conmon.ciu_tls = (lws_conmon_interval_us_t)
@@ -100,6 +65,7 @@ lws_ssl_client_connect2(struct lws *wsi, char *errbuf, size_t len)
 
 		switch (n) {
 		case LWS_SSL_CAPABLE_ERROR:
+			lws_tls_restrict_return_handshake(wsi);
 			// lws_snprintf(errbuf, len, "client connect failed");
 			return -1;
 		case LWS_SSL_CAPABLE_DONE:
@@ -115,6 +81,8 @@ lws_ssl_client_connect2(struct lws *wsi, char *errbuf, size_t len)
 		}
 	}
 
+	lws_tls_restrict_return_handshake(wsi);
+
 	if (lws_tls_client_confirm_peer_cert(wsi, errbuf, len)) {
 		lws_metrics_caliper_report(wsi->cal_conn, METRES_NOGO);
 		return -1;
@@ -221,23 +189,12 @@ lws_client_create_tls(struct lws *wsi, const char **pcce, int do_c1)
 		if (!wsi->tls.ssl) {
 
 #if defined(LWS_WITH_TLS)
-			if (!wsi->transaction_from_pipeline_queue) {
-				if (lws_tls_restrict_borrow(wsi->a.context)) {
-					*pcce = "tls restriction limit";
-					return CCTLS_RETURN_ERROR;
-				}
-				wsi->tls_borrowed = 1;
-				if (wsi->a.context->ssl_handshake_serialize) {
-					if (lws_ssl_handshake_serialize(wsi->a.context, wsi)) {
-						lws_tls_restrict_return(wsi->a.context);
-						wsi->tls_borrowed = 0;
-						*pcce = "ssl handshake serialization";
-						return CCTLS_RETURN_ERROR;
-					}
-				}
+			if (!wsi->transaction_from_pipeline_queue &&
+			    lws_tls_restrict_borrow(wsi)) {
+				*pcce = "tls restriction limit";
+				return CCTLS_RETURN_ERROR;
 			}
 #endif
-
 			if (lws_ssl_client_bio_create(wsi) < 0) {
 				*pcce = "bio_create failed";
 				return CCTLS_RETURN_ERROR;
diff --git a/lib/tls/tls-network.c b/lib/tls/tls-network.c
index 0698e1ab..b2038d53 100644
--- a/lib/tls/tls-network.c
+++ b/lib/tls/tls-network.c
@@ -203,6 +203,11 @@ lws_gate_accepts(struct lws_context *context, int on)
 
 	lwsl_notice("%s: on = %d\n", __func__, on);
 
+	if (context->tls_gate_accepts == (char)on)
+		return 0;
+
+	context->tls_gate_accepts = (char)on;
+
 	while (v) {
 		lws_start_foreach_dll(struct lws_dll2 *, d,
 				      lws_dll2_get_head(&v->listen_wsi)) {
@@ -210,8 +215,8 @@ lws_gate_accepts(struct lws_context *context, int on)
 							   listen_list);
 
 			if (v->tls.use_ssl &&
-			    lws_change_pollfd(wsi, on ? 0 : LWS_POLLIN,
-						   on ? LWS_POLLIN : 0))
+			    lws_change_pollfd(wsi, on ? LWS_POLLIN : 0,
+						   on ? 0 : LWS_POLLIN))
 				lwsl_notice("%s: Unable to set POLLIN %d\n",
 					    __func__, on);
 		} lws_end_foreach_dll(d);
diff --git a/lib/tls/tls-server.c b/lib/tls/tls-server.c
index 39578c1b..dbb51783 100644
--- a/lib/tls/tls-server.c
+++ b/lib/tls/tls-server.c
@@ -144,18 +144,16 @@ lws_server_socket_service_ssl(struct lws *wsi, lws_sockfd_type accept_fd, char f
 		if (accept_fd == LWS_SOCK_INVALID)
 			assert(0);
 
-		if (lws_tls_restrict_borrow(context)) {
+		if (lws_tls_restrict_borrow(wsi)) {
 			lwsl_err("%s: failed on ssl restriction\n", __func__);
 			return 1;
 		}
-		wsi->tls_borrowed = 1;
 
 		if (lws_tls_server_new_nonblocking(wsi, accept_fd)) {
 			lwsl_err("%s: failed on lws_tls_server_new_nonblocking\n", __func__);
 			if (accept_fd != LWS_SOCK_INVALID)
 				compatible_close(accept_fd);
-			if (wsi->tls_borrowed)
-				lws_tls_restrict_return(context);
+			lws_tls_restrict_return(wsi);
 			goto fail;
 		}
 
@@ -322,8 +320,10 @@ punt:
 		lwsl_info("SSL_accept says %d\n", n);
 		switch (n) {
 		case LWS_SSL_CAPABLE_DONE:
+			lws_tls_restrict_return_handshake(wsi);
 			break;
 		case LWS_SSL_CAPABLE_ERROR:
+			lws_tls_restrict_return_handshake(wsi);
 	                lwsl_info("%s: SSL_accept failed socket %u: %d\n",
 	                		__func__, wsi->desc.sockfd, n);
 			wsi->socket_is_permanently_unusable = 1;
diff --git a/lib/tls/tls.c b/lib/tls/tls.c
index 9e9a32ca..6afc7640 100644
--- a/lib/tls/tls.c
+++ b/lib/tls/tls.c
@@ -46,52 +46,110 @@ alpn_cb(SSL *s, const unsigned char **out, unsigned char *outlen,
 #endif
 
 int
-lws_tls_restrict_borrow(struct lws_context *context)
+lws_tls_restrict_borrow(struct lws *wsi)
 {
-	if (!context->simultaneous_ssl_restriction)
-		return 0;
+	struct lws_context *cx = wsi->a.context;
 
-	if (context->simultaneous_ssl >= context->simultaneous_ssl_restriction) {
+	if (cx->simultaneous_ssl_restriction &&
+	    cx->simultaneous_ssl >= cx->simultaneous_ssl_restriction) {
 		lwsl_notice("%s: tls connection limit %d\n", __func__,
-			    context->simultaneous_ssl);
+			    cx->simultaneous_ssl);
+		return 1;
+	}
+
+	if (cx->simultaneous_ssl_handshake_restriction &&
+	    cx->simultaneous_ssl_handshake >=
+			    cx->simultaneous_ssl_handshake_restriction) {
+		lwsl_notice("%s: tls handshake limit %d\n", __func__,
+			    cx->simultaneous_ssl);
 		return 1;
 	}
 
-	context->simultaneous_ssl++;
+	cx->simultaneous_ssl++;
+	cx->simultaneous_ssl_handshake++;
+	wsi->tls_borrowed_hs = 1;
+	wsi->tls_borrowed = 1;
 
 	lwsl_info("%s: %d -> %d\n", __func__,
-		  context->simultaneous_ssl - 1,
-		  context->simultaneous_ssl);
+		  cx->simultaneous_ssl - 1,
+		  cx->simultaneous_ssl);
 
-	assert(context->simultaneous_ssl <=
-	       context->simultaneous_ssl_restriction);
+	assert(!cx->simultaneous_ssl_restriction ||
+			cx->simultaneous_ssl <=
+				cx->simultaneous_ssl_restriction);
+	assert(!cx->simultaneous_ssl_handshake_restriction ||
+			cx->simultaneous_ssl_handshake <=
+				cx->simultaneous_ssl_handshake_restriction);
 
 #if defined(LWS_WITH_SERVER)
-	if (context->simultaneous_ssl == context->simultaneous_ssl_restriction)
-		/* that was the last allowed SSL connection */
-		lws_gate_accepts(context, 0);
+	lws_gate_accepts(cx,
+			(cx->simultaneous_ssl_restriction &&
+			 cx->simultaneous_ssl == cx->simultaneous_ssl_restriction) ||
+			(cx->simultaneous_ssl_handshake_restriction &&
+			 cx->simultaneous_ssl_handshake == cx->simultaneous_ssl_handshake_restriction));
 #endif
 
 	return 0;
 }
 
-void
-lws_tls_restrict_return(struct lws_context *context)
+static void
+_lws_tls_restrict_return(struct lws *wsi)
 {
-	if (context->simultaneous_ssl_restriction) {
-		int n = context->simultaneous_ssl--;
-
-		lwsl_info("%s: %d -> %d\n", __func__, n,
-			  context->simultaneous_ssl);
+	struct lws_context *cx = wsi->a.context;
 
-		assert(context->simultaneous_ssl >= 0);
+	assert(cx->simultaneous_ssl_handshake >= 0);
+	assert(cx->simultaneous_ssl >= 0);
 
 #if defined(LWS_WITH_SERVER)
-		if (n == context->simultaneous_ssl_restriction)
-			/* we made space and can do an accept */
-			lws_gate_accepts(context, 1);
+	lws_gate_accepts(cx,
+			(cx->simultaneous_ssl_restriction &&
+			 cx->simultaneous_ssl == cx->simultaneous_ssl_restriction) ||
+			(cx->simultaneous_ssl_handshake_restriction &&
+			 cx->simultaneous_ssl_handshake == cx->simultaneous_ssl_handshake_restriction));
 #endif
-	}
+}
+
+void
+lws_tls_restrict_return_handshake(struct lws *wsi)
+{
+	struct lws_context *cx = wsi->a.context;
+
+	/* we're just returning the hs part */
+
+	if (!wsi->tls_borrowed_hs)
+		return;
+
+	wsi->tls_borrowed_hs = 0; /* return it one time per wsi */
+	cx->simultaneous_ssl_handshake--;
+
+	lwsl_info("%s:  %d -> %d\n", __func__,
+		  cx->simultaneous_ssl_handshake + 1,
+		  cx->simultaneous_ssl_handshake);
+
+	_lws_tls_restrict_return(wsi);
+}
+
+void
+lws_tls_restrict_return(struct lws *wsi)
+{
+	struct lws_context *cx = wsi->a.context;
+
+	if (!wsi->tls_borrowed)
+		return;
+
+	wsi->tls_borrowed = 0;
+	cx->simultaneous_ssl--;
+
+	lwsl_info("%s: %d -> %d\n", __func__,
+		  cx->simultaneous_ssl + 1,
+		  cx->simultaneous_ssl);
+
+	/* We're returning everything, even if hs didn't complete */
+
+	if (wsi->tls_borrowed_hs)
+		lws_tls_restrict_return_handshake(wsi);
+	else
+		_lws_tls_restrict_return(wsi);
 }
 
 void
diff --git a/minimal-examples/http-client/minimal-http-client-multi/minimal-http-client-multi.c b/minimal-examples/http-client/minimal-http-client-multi/minimal-http-client-multi.c
index fb7c65b9..d0d11f3f 100644
--- a/minimal-examples/http-client/minimal-http-client-multi/minimal-http-client-multi.c
+++ b/minimal-examples/http-client/minimal-http-client-multi/minimal-http-client-multi.c
@@ -586,6 +586,12 @@ int main(int argc, const char **argv)
 	if ((p = lws_cmdline_option(argc, argv, "--limit")))
 		info.simultaneous_ssl_restriction = atoi(p);
 
+	if ((p = lws_cmdline_option(argc, argv, "--ssl-handshake-serialize")))
+		/* We only consider simultaneous_ssl_restriction > 1 use cases.
+		 * If ssl isn't limited or only 1 is allowed, we don't care.
+		 */
+		info.simultaneous_ssl_handshake_restriction = atoi(p);
+
 	context = lws_create_context(&info);
 	if (!context) {
 		lwsl_err("lws init failed\n");