Date: Fri, 12 May 2006 03:41:01 -0700 (PDT)
From: Aaron Hopkins <lists@die.net>
To: nsd-users@NLnetLabs.nl
Subject: Reducing select() usage under load

NSD currently only processes one UDP packet per socket per select().  Since
select() is kind of expensive, under load this means it burns a lot of CPU
unnecessarily.

There's a simple trick to avoid this.  Make the UDP socket non-blocking, and
loop on recvfrom() until it returns -1, ignoring any EAGAIN errors.  Under
light load, this results in an extra recvfrom() every packet.  But under
heavy load, this avoids select() until the input buffer is drained.

Attached is an example patch against NSD 2.3.4 that implements this. 
According to the queryperf tool that comes with BIND, on a simple query
against localhost on an old Linux box, this takes NSD's peak throughput from
39kpps to 48kpps, a 23% improvement.  These are obviously ideal conditions,
but please feel free to test for yourself.

diff -ur nsd-2.3.4/server.c nsd-2.3.4.faster/server.c
--- nsd-2.3.4/server.c	2006-04-06 07:26:35.000000000 -0700
+++ nsd-2.3.4.faster/server.c	2006-05-12 03:03:42.000000000 -0700
@@ -276,6 +276,11 @@
 		}
 #endif
 
+		if (fcntl(nsd->udp[i].s, F_SETFL, O_NONBLOCK) == -1) {
+			log_msg(LOG_ERR, "fcntl failed: %s", strerror(errno));
+			return -1;
+		}
+
 		/* Bind it... */
 		if (bind(nsd->udp[i].s, (struct sockaddr *) nsd->udp[i].addr->ai_addr, nsd->udp[i].addr->ai_addrlen) != 0) {
 			log_msg(LOG_ERR, "can't bind the socket: %s", strerror(errno));
@@ -707,28 +712,31 @@
 		return;
 	}
 	
-	/* Account... */
-	if (data->socket->addr->ai_family == AF_INET) {
-		STATUP(data->nsd, qudp);
-	} else if (data->socket->addr->ai_family == AF_INET6) {
-		STATUP(data->nsd, qudp6);
-	}
+	while (1) {
+		/* Initialize the query... */
+		query_reset(q, UDP_MAX_MESSAGE_LEN, 0);
+
+		received = recvfrom(handler->fd,
+				    buffer_begin(q->packet),
+				    buffer_remaining(q->packet),
+				    0,
+				    (struct sockaddr *)&q->addr,
+				    &q->addrlen);
+		if (received == -1) {
+			if (errno != EAGAIN && errno != EINTR) {
+				log_msg(LOG_ERR, "recvfrom failed: %s", strerror(errno));
+				STATUP(data->nsd, rxerr);
+			}
+			return;
+		}
 
-	/* Initialize the query... */
-	query_reset(q, UDP_MAX_MESSAGE_LEN, 0);
-
-	received = recvfrom(handler->fd,
-			    buffer_begin(q->packet),
-			    buffer_remaining(q->packet),
-			    0,
-			    (struct sockaddr *)&q->addr,
-			    &q->addrlen);
-	if (received == -1) {
-		if (errno != EAGAIN && errno != EINTR) {
-			log_msg(LOG_ERR, "recvfrom failed: %s", strerror(errno));
-			STATUP(data->nsd, rxerr);
+		/* Account... */
+		if (data->socket->addr->ai_family == AF_INET) {
+			STATUP(data->nsd, qudp);
+		} else if (data->socket->addr->ai_family == AF_INET6) {
+			STATUP(data->nsd, qudp6);
 		}
-	} else {
+
 		buffer_skip(q->packet, received);
 		buffer_flip(q->packet);