diff --git a/ntp_core.c b/ntp_core.c
index f179dbf..e936a09 100644
--- a/ntp_core.c
+++ b/ntp_core.c
@@ -229,6 +229,10 @@ static ARR_Instance broadcasts;
 
 /* ================================================== */
 
+/* Server IPv4/IPv6 sockets */
+static int server_sock_fd4;
+static int server_sock_fd6;
+
 static ADF_AuthTable access_auth_table;
 
 /* ================================================== */
@@ -305,6 +309,8 @@ do_time_checks(void)
 void
 NCR_Initialise(void)
 {
+  NTP_Remote_Address addr;
+
   do_size_checks();
   do_time_checks();
 
@@ -314,6 +320,11 @@ NCR_Initialise(void)
 
   access_auth_table = ADF_CreateTable();
   broadcasts = ARR_CreateInstance(sizeof (BroadcastDestination));
+
+  addr.ip_addr.family = IPADDR_INET4;
+  server_sock_fd4 = NIO_OpenServerSocket(&addr);
+  addr.ip_addr.family = IPADDR_INET6;
+  server_sock_fd6 = NIO_OpenServerSocket(&addr);
 }
 
 /* ================================================== */
@@ -321,6 +332,16 @@ NCR_Initialise(void)
 void
 NCR_Finalise(void)
 {
+  unsigned int i;
+
+  if (server_sock_fd4 != INVALID_SOCK_FD)
+    NIO_CloseServerSocket(server_sock_fd4);
+  if (server_sock_fd6 != INVALID_SOCK_FD)
+    NIO_CloseServerSocket(server_sock_fd6);
+
+  for (i = 0; i < ARR_GetSize(broadcasts); i++)
+    NIO_CloseServerSocket(((BroadcastDestination *)ARR_GetElement(broadcasts, i))->local_addr.sock_fd);
+
   ARR_DestroyInstance(broadcasts);
   ADF_DestroyTable(access_auth_table);
 }
@@ -490,6 +511,9 @@ NCR_DestroyInstance(NCR_Instance instance)
   if (instance->opmode != MD_OFFLINE)
     take_offline(instance);
 
+  if (instance->mode == MODE_ACTIVE)
+    NIO_CloseServerSocket(instance->local_addr.sock_fd);
+
   /* This will destroy the source instance inside the
      structure, which will cause reselection if this was the
      synchronising source etc. */
@@ -550,8 +574,10 @@ NCR_ChangeRemoteAddress(NCR_Instance inst, NTP_Remote_Address *remote_addr)
 
   if (inst->mode == MODE_CLIENT)
     close_client_socket(inst);
-  else
+  else {
+    NIO_CloseServerSocket(inst->local_addr.sock_fd);
     inst->local_addr.sock_fd = NIO_OpenServerSocket(remote_addr);
+  }
 
   /* Update the reference ID and reset the source/sourcestats instances */
   SRC_SetRefid(inst->source, UTI_IPToRefid(&remote_addr->ip_addr),
diff --git a/ntp_io.c b/ntp_io.c
index d45dba6..fdf4bb6 100644
--- a/ntp_io.c
+++ b/ntp_io.c
@@ -57,10 +57,22 @@ static int server_sock_fd6;
 static int client_sock_fd6;
 #endif
 
+/* Reference counters for server sockets to keep them open only when needed */
+static int server_sock_ref4;
+#ifdef FEAT_IPV6
+static int server_sock_ref6;
+#endif
+
 /* Flag indicating we create a new connected client socket for each
    server instead of sharing client_sock_fd4 and client_sock_fd6 */
 static int separate_client_sockets;
 
+/* Flag indicating the server sockets are not created dynamically when needed,
+   either to have a socket for client requests when separate client sockets
+   are disabled and client port is equal to server port, or the server port is
+   disabled */
+static int permanent_server_sockets;
+
 /* Flag indicating that we have been initialised */
 static int initialised=0;
 
@@ -283,15 +295,20 @@ NIO_Initialise(int family)
   if (client_port < 0)
     client_port = 0;
 
+  permanent_server_sockets = !server_port || (!separate_client_sockets &&
+                                              client_port == server_port);
+
   server_sock_fd4 = INVALID_SOCK_FD;
   client_sock_fd4 = INVALID_SOCK_FD;
+  server_sock_ref4 = 0;
 #ifdef FEAT_IPV6
   server_sock_fd6 = INVALID_SOCK_FD;
   client_sock_fd6 = INVALID_SOCK_FD;
+  server_sock_ref6 = 0;
 #endif
 
   if (family == IPADDR_UNSPEC || family == IPADDR_INET4) {
-    if (server_port)
+    if (permanent_server_sockets && server_port)
       server_sock_fd4 = prepare_socket(AF_INET, server_port, 0);
     if (!separate_client_sockets) {
       if (client_port != server_port || !server_port)
@@ -302,7 +319,7 @@ NIO_Initialise(int family)
   }
 #ifdef FEAT_IPV6
   if (family == IPADDR_UNSPEC || family == IPADDR_INET6) {
-    if (server_port)
+    if (permanent_server_sockets && server_port)
       server_sock_fd6 = prepare_socket(AF_INET6, server_port, 0);
     if (!separate_client_sockets) {
       if (client_port != server_port || !server_port)
@@ -313,7 +330,8 @@ NIO_Initialise(int family)
   }
 #endif
 
-  if ((server_port && server_sock_fd4 == INVALID_SOCK_FD
+  if ((server_port && server_sock_fd4 == INVALID_SOCK_FD &&
+       permanent_server_sockets 
 #ifdef FEAT_IPV6
        && server_sock_fd6 == INVALID_SOCK_FD
 #endif
@@ -382,9 +400,21 @@ NIO_OpenServerSocket(NTP_Remote_Address *remote_addr)
 {
   switch (remote_addr->ip_addr.family) {
     case IPADDR_INET4:
+      if (permanent_server_sockets)
+        return server_sock_fd4;
+      if (server_sock_fd4 == INVALID_SOCK_FD)
+        server_sock_fd4 = prepare_socket(AF_INET, CNF_GetNTPPort(), 0);
+      if (server_sock_fd4 != INVALID_SOCK_FD)
+        server_sock_ref4++;
       return server_sock_fd4;
 #ifdef FEAT_IPV6
     case IPADDR_INET6:
+      if (permanent_server_sockets)
+        return server_sock_fd6;
+      if (server_sock_fd6 == INVALID_SOCK_FD)
+        server_sock_fd6 = prepare_socket(AF_INET6, CNF_GetNTPPort(), 0);
+      if (server_sock_fd6 != INVALID_SOCK_FD)
+        server_sock_ref6++;
       return server_sock_fd6;
 #endif
     default:
@@ -403,6 +433,33 @@ NIO_CloseClientSocket(int sock_fd)
 
 /* ================================================== */
 
+void
+NIO_CloseServerSocket(int sock_fd)
+{
+  if (permanent_server_sockets || sock_fd == INVALID_SOCK_FD)
+    return;
+
+  if (sock_fd == server_sock_fd4) {
+    if (--server_sock_ref4 <= 0) {
+      close_socket(server_sock_fd4);
+      server_sock_fd4 = INVALID_SOCK_FD;
+    }
+  }
+#ifdef FEAT_IPV6
+  else if (sock_fd == server_sock_fd6) {
+    if (--server_sock_ref6 <= 0) {
+      close_socket(server_sock_fd6);
+      server_sock_fd6 = INVALID_SOCK_FD;
+    }
+  }
+#endif
+  else {
+    assert(0);
+  }
+}
+
+/* ================================================== */
+
 int
 NIO_IsServerSocket(int sock_fd)
 {
diff --git a/ntp_io.h b/ntp_io.h
index 6714bfe..cbdee56 100644
--- a/ntp_io.h
+++ b/ntp_io.h
@@ -47,6 +47,9 @@ extern int NIO_OpenServerSocket(NTP_Remote_Address *remote_addr);
 /* Function to close a socket returned by NIO_OpenClientSocket() */
 extern void NIO_CloseClientSocket(int sock_fd);
 
+/* Function to close a socket returned by NIO_OpenServerSocket() */
+extern void NIO_CloseServerSocket(int sock_fd);
+
 /* Function to check if socket is a server socket */
 extern int NIO_IsServerSocket(int sock_fd);