ntp: support pools

The pool directive can be used to configure chronyd for a pool of NTP
servers (e.g. pool.ntp.org). The name is expected to resolve to multiple
addresses which change over time.

On start, a source will be added for each resolved address. When a
source from the pool is unreachable or marked as falseticker, chronyd
will try to replace the source with a newly resolved address of the
pool.

The minimum interval between replacements is currently set to 244
seconds to avoid frequent DNS requests.

chrony.texi.in

@@ -1164,6 +1164,7 @@ the configuration file is ignored.
 * noclientlog directive::       Prevent chronyd from gathering data about clients
 * peer directive::              Specify an NTP peer
 * pidfile directive::           Specify the file where chronyd's pid is written
+* pool directive::              Specify an NTP pool
 * port directive::              Set NTP server port
 * refclock directive::          Specify a reference clock
 * reselectdist directive::      Set improvement in distance needed to reselect a source
@@ -2487,6 +2488,25 @@ chronyd always writes its process ID (pid) to a file, and checks this file on st
 @example
 pidfile /var/tmp/chronyd.pid
 @end example
+@c }}}
+@c {{{ pool
+@node pool directive
+@subsection pool
+The syntax of this directive is identical to that for the @code{server}
+directive (@pxref{server directive}), except that it is used to specify a pool
+of NTP servers rather than a single NTP server.  The pool name is expected to
+resolve to multiple addresses which change over time.
+
+On start, a source will be added for each resolved address.  When a source from
+the pool is unreachable or marked as falseticker, @code{chronyd} will try to
+replace the source with a newly resolved address of the pool.
+
+An example of the pool directive is
+
+@example
+pool pool.ntp.org iburst
+@end example
+
 @c }}}
 @c {{{ port
 @node port directive

conf.c

@@ -69,6 +69,7 @@ static void parse_mailonchange(char *);
 static void parse_makestep(char *);
 static void parse_maxchange(char *);
 static void parse_peer(char *);
+static void parse_pool(char *);
 static void parse_refclock(char *);
 static void parse_server(char *);
 static void parse_tempcomp(char *);
@@ -199,6 +200,7 @@ static char *user;
 
 typedef struct {
   NTP_Source_Type type;
+  int pool;
   CPS_NTP_Source params;
 } NTP_Source;
 
@@ -467,6 +469,8 @@ CNF_ParseLine(const char *filename, int number, char *line)
     parse_peer(p);
   } else if (!strcasecmp(command, "pidfile")) {
     parse_string(p, &pidfile);
+  } else if (!strcasecmp(command, "pool")) {
+    parse_pool(p);
   } else if (!strcasecmp(command, "port")) {
     parse_int(p, &ntp_port);
   } else if (!strcasecmp(command, "refclock")) {
@@ -560,12 +564,13 @@ parse_null(char *line)
 /* ================================================== */
 
 static void
-parse_source(char *line, NTP_Source_Type type)
+parse_source(char *line, NTP_Source_Type type, int pool)
 {
   CPS_Status status;
   NTP_Source source;
 
   source.type = type;
+  source.pool = pool;
   status = CPS_ParseNTPSourceAdd(line, &source.params);
 
   switch (status) {
@@ -620,7 +625,7 @@ parse_source(char *line, NTP_Source_Type type)
 static void
 parse_server(char *line)
 {
-  parse_source(line, NTP_SERVER);
+  parse_source(line, NTP_SERVER, 0);
 }
 
 /* ================================================== */
@@ -628,7 +633,15 @@ parse_server(char *line)
 static void
 parse_peer(char *line)
 {
-  parse_source(line, NTP_PEER);
+  parse_source(line, NTP_PEER, 0);
+}
+
+/* ================================================== */
+
+static void
+parse_pool(char *line)
+{
+  parse_source(line, NTP_SERVER, 1);
 }
 
 /* ================================================== */
@@ -1207,8 +1220,9 @@ CNF_AddSources(void)
 
   for (i = 0; i < ARR_GetSize(ntp_sources); i++) {
     source = (NTP_Source *)ARR_GetElement(ntp_sources, i);
-    NSR_AddUnresolvedSource(source->params.name, source->params.port,
+    NSR_AddSourceByName(source->params.name, source->params.port,
-        source->type, &source->params.params);
+                        source->pool, source->type, &source->params.params);
+    Free(source->params.name);
   }
 
   ARR_SetSize(ntp_sources, 0);

ntp_sources.c

@@ -49,6 +49,8 @@ typedef struct {
   NTP_Remote_Address *remote_addr; /* The address of this source, non-NULL
                                       means this slot in table is in use */
   NCR_Instance data;            /* Data for the protocol engine for this source */
+  int pool;                     /* Number of the pool from which was this source
+                                   added or INVALID_POOL */
 } SourceRecord;
 
 /* Hash table of SourceRecord, the size should be a power of two */
@@ -64,8 +66,16 @@ static int auto_start_sources = 0;
 struct UnresolvedSource {
   char *name;
   int port;
-  NTP_Source_Type type;
+  int replacement;
-  SourceParameters params;
+  union {
+    struct {
+      NTP_Source_Type type;
+      SourceParameters params;
+      int pool;
+      int max_new_sources;
+    } new_source;
+    NTP_Remote_Address replace_source;
+  };
   struct UnresolvedSource *next;
 };
 
@@ -79,6 +89,20 @@ static SCH_TimeoutID resolving_id;
 static struct UnresolvedSource *resolving_source = NULL;
 static NSR_SourceResolvingEndHandler resolving_end_handler = NULL;
 
+#define MIN_POOL_RESOLVE_INTERVAL 5
+#define MAX_POOL_SOURCES 16
+#define INVALID_POOL (-1)
+
+/* Pool of sources, the name is expected to resolve to multiple addresses
+   which change over time */
+struct SourcePool {
+  char *name;
+  int port;
+};
+
+/* Array of SourcePool */
+static ARR_Instance pools;
+
 /* ================================================== */
 /* Forward prototypes */
 
@@ -117,6 +141,8 @@ NSR_Initialise(void)
   records = ARR_CreateInstance(sizeof (SourceRecord));
   rehash_records();
 
+  pools = ARR_CreateInstance(sizeof (struct SourcePool));
+
   LCL_AddParameterChangeHandler(slew_sources, NULL);
 }
 
@@ -129,6 +155,10 @@ NSR_Finalise(void)
   struct UnresolvedSource *us;
   unsigned int i;
 
+  for (i = 0; i < ARR_GetSize(pools); i++)
+    Free(((struct SourcePool *)ARR_GetElement(pools, i))->name);
+  ARR_DestroyInstance(pools);
+
   for (i = 0; i < ARR_GetSize(records); i++) {
     record = get_record(i);
     if (!record->remote_addr)
@@ -224,7 +254,7 @@ check_hashtable_size(unsigned int sources, unsigned int size)
 static void
 rehash_records(void)
 {
-  SourceRecord *record, *temp_records;
+  SourceRecord *temp_records;
   unsigned int i, old_size, new_size;
   int slot, found;
 
@@ -249,9 +279,7 @@ rehash_records(void)
     find_slot(temp_records[i].remote_addr, &slot, &found);
     assert(!found);
 
-    record = get_record(slot);
+    *get_record(slot) = temp_records[i];
-    record->remote_addr = temp_records[i].remote_addr;
-    record->data = temp_records[i].data;
   }
 
   Free(temp_records);
@@ -260,8 +288,8 @@ rehash_records(void)
 /* ================================================== */
 
 /* Procedure to add a new source */
-NSR_Status
+static NSR_Status
-NSR_AddSource(NTP_Remote_Address *remote_addr, NTP_Source_Type type, SourceParameters *params)
+add_source(NTP_Remote_Address *remote_addr, NTP_Source_Type type, SourceParameters *params, int pool)
 {
   SourceRecord *record;
   int slot, found;
@@ -288,6 +316,7 @@ NSR_AddSource(NTP_Remote_Address *remote_addr, NTP_Source_Type type, SourceParam
       record = get_record(slot);
       record->data = NCR_GetInstance(remote_addr, type, params);
       record->remote_addr = NCR_GetRemoteAddress(record->data);
+      record->pool = pool;
 
       if (auto_start_sources)
         NCR_StartInstance(record->data);
@@ -299,21 +328,59 @@ NSR_AddSource(NTP_Remote_Address *remote_addr, NTP_Source_Type type, SourceParam
 
 /* ================================================== */
 
+static NSR_Status
+replace_source(NTP_Remote_Address *old_addr, NTP_Remote_Address *new_addr)
+{
+  int slot1, slot2, found;
+  SourceRecord *record;
+
+  find_slot(old_addr, &slot1, &found);
+  if (!found)
+    return NSR_NoSuchSource;
+
+  find_slot(new_addr, &slot2, &found);
+  if (found)
+    return NSR_AlreadyInUse;
+
+  record = get_record(slot1);
+  NCR_ChangeRemoteAddress(record->data, new_addr);
+  record->remote_addr = NCR_GetRemoteAddress(record->data);
+
+  /* The hash table must be rebuilt for the new address */
+  rehash_records();
+
+  LOG(LOGS_INFO, LOGF_NtpSources, "Source %s replaced with %s",
+      UTI_IPToString(&old_addr->ip_addr),
+      UTI_IPToString(&new_addr->ip_addr));
+
+  return NSR_Success;
+}
+
+/* ================================================== */
+
 static void
 process_resolved_name(struct UnresolvedSource *us, IPAddr *ip_addrs, int n_addrs)
 {
   NTP_Remote_Address address;
-  int i;
+  int i, added;
 
-  for (i = 0; i < n_addrs; i++) {
+  for (i = added = 0; i < n_addrs; i++) {
     DEBUG_LOG(LOGF_NtpSources, "%s resolved to %s", us->name, UTI_IPToString(&ip_addrs[i]));
 
     address.ip_addr = ip_addrs[i];
     address.port = us->port;
 
-    /* Add only one new source for this name */
+    if (us->replacement) {
-    if (NSR_AddSource(&address, us->type, &us->params) == NSR_Success)
+      if (replace_source(&us->replace_source, &address) != NSR_AlreadyInUse)
-      break;
+        break;
+    } else {
+      if (add_source(&address, us->new_source.type, &us->new_source.params,
+                     us->new_source.pool) == NSR_Success)
+        added++;
+
+      if (added >= us->new_source.max_new_sources)
+        break;
+    }
   }
 }
 
@@ -343,8 +410,9 @@ name_resolve_handler(DNS_Status status, int n_addrs, IPAddr *ip_addrs, void *any
 
   next = us->next;
 
-  if (status != DNS_TryAgain) {
+  /* Remove the source from the list on success or failure, replacements
-    /* Remove the source from the list */
+     are removed on any status */
+  if (us->replacement || status != DNS_TryAgain) {
     for (i = &unresolved_sources; *i; i = &(*i)->next) {
       if (*i == us) {
         *i = us->next;
@@ -403,24 +471,52 @@ resolve_sources(void *arg)
 
 /* ================================================== */
 
-/* Procedure to add a new server or peer source, but instead of an IP address
+static void
-   only a name is provided */
+append_unresolved_source(struct UnresolvedSource *us)
-void
-NSR_AddUnresolvedSource(char *name, int port, NTP_Source_Type type, SourceParameters *params)
 {
-  struct UnresolvedSource *us, **i;
+  struct UnresolvedSource **i;
-
-  us = MallocNew(struct UnresolvedSource);
-
-  us->name = name;
-  us->port = port;
-  us->type = type;
-  us->params = *params;
-  us->next = NULL;
 
   for (i = &unresolved_sources; *i; i = &(*i)->next)
     ;
   *i = us;
+  us->next = NULL;
+}
+
+/* ================================================== */
+
+NSR_Status
+NSR_AddSource(NTP_Remote_Address *remote_addr, NTP_Source_Type type, SourceParameters *params)
+{
+  return add_source(remote_addr, type, params, INVALID_POOL);
+}
+
+/* ================================================== */
+
+void
+NSR_AddSourceByName(char *name, int port, int pool, NTP_Source_Type type, SourceParameters *params)
+{
+  struct UnresolvedSource *us;
+  struct SourcePool *sp;
+
+  us = MallocNew(struct UnresolvedSource);
+  us->name = Strdup(name);
+  us->port = port;
+  us->replacement = 0;
+  us->new_source.type = type;
+  us->new_source.params = *params;
+
+  if (!pool) {
+    us->new_source.pool = INVALID_POOL;
+    us->new_source.max_new_sources = 1;
+  } else {
+    sp = (struct SourcePool *)ARR_GetNewElement(pools);
+    sp->name = Strdup(name);
+    sp->port = port;
+    us->new_source.pool = ARR_GetSize(pools) - 1;
+    us->new_source.max_new_sources = MAX_POOL_SOURCES;
+  }
+
+  append_unresolved_source(us);
 }
 
 /* ================================================== */
@@ -522,6 +618,59 @@ NSR_RemoveAllSources(void)
 
 /* ================================================== */
 
+static void
+resolve_pool_replacement(struct SourcePool *sp, NTP_Remote_Address *addr)
+{
+  struct UnresolvedSource *us;
+
+  us = MallocNew(struct UnresolvedSource);
+  us->name = Strdup(sp->name);
+  us->port = sp->port;
+  us->replacement = 1;
+  us->replace_source = *addr;
+
+  append_unresolved_source(us);
+  NSR_ResolveSources();
+}
+
+/* ================================================== */
+
+void
+NSR_HandleBadSource(IPAddr *address)
+{
+  static struct timeval last_replacement;
+  struct timeval now;
+  NTP_Remote_Address remote_addr;
+  struct SourcePool *pool;
+  int pool_index, slot, found;
+  double diff;
+
+  remote_addr.ip_addr = *address;
+  remote_addr.port = 0;
+
+  /* Only sources from a pool can be replaced */
+  find_slot(&remote_addr, &slot, &found);
+  if (!found || (pool_index = get_record(slot)->pool) == INVALID_POOL)
+    return;
+
+  pool = (struct SourcePool *)ARR_GetElement(pools, pool_index);
+
+  /* Don't resolve the pool name too frequently */
+  SCH_GetLastEventTime(NULL, NULL, &now);
+  UTI_DiffTimevalsToDouble(&diff, &now, &last_replacement);
+  if (fabs(diff) < RESOLVE_INTERVAL_UNIT * (1 << MIN_POOL_RESOLVE_INTERVAL)) {
+    DEBUG_LOG(LOGF_NtpSources, "replacement postponed");
+    return;
+  }
+  last_replacement = now;
+
+  DEBUG_LOG(LOGF_NtpSources, "pool replacement for %s", UTI_IPToString(address));
+
+  resolve_pool_replacement(pool, &remote_addr);
+}
+
+/* ================================================== */
+
 /* This routine is called by ntp_io when a new packet arrives off the network,
    possibly with an authentication tail */
 void
@@ -592,8 +741,10 @@ NSR_TakeSourcesOnline(IPAddr *mask, IPAddr *address)
     struct UnresolvedSource *us;
 
     for (us = unresolved_sources; us; us = us->next) {
+      if (us->replacement)
+        continue;
       any = 1;
-      us->params.online = 1;
+      us->new_source.params.online = 1;
     }
   }
 
@@ -634,8 +785,10 @@ NSR_TakeSourcesOffline(IPAddr *mask, IPAddr *address)
     struct UnresolvedSource *us;
 
     for (us = unresolved_sources; us; us = us->next) {
+      if (us->replacement)
+        continue;
       any = 1;
-      us->params.online = 0;
+      us->new_source.params.online = 0;
     }
   }
 

ntp_sources.h

@@ -50,10 +50,10 @@ typedef enum {
 /* Procedure to add a new server or peer source. */
 extern NSR_Status NSR_AddSource(NTP_Remote_Address *remote_addr, NTP_Source_Type type, SourceParameters *params);
 
-/* Procedure to add a new server or peer source with currently unknown address.
+/* Procedure to add a new server, peer source, or pool of servers specified by
-   The name will be periodically resolved in exponentially increasing intervals
+   name instead of address.  The name is resolved in exponentially increasing
-   until it succeeds or fails with a non-temporary error. */
+   intervals until it succeeds or fails with a non-temporary error. */
-extern void NSR_AddUnresolvedSource(char *name, int port, NTP_Source_Type type, SourceParameters *params);
+extern void NSR_AddSourceByName(char *name, int port, int pool, NTP_Source_Type type, SourceParameters *params);
 
 /* Function type for handlers to be called back when an attempt
  * (possibly unsuccessful) to resolve unresolved sources ends */
@@ -77,6 +77,9 @@ extern NSR_Status NSR_RemoveSource(NTP_Remote_Address *remote_addr);
 /* Procedure to remove all sources */
 extern void NSR_RemoveAllSources(void);
 
+/* Procedure to try to find a replacement for a bad source */
+extern void NSR_HandleBadSource(IPAddr *address);
+
 /* This routine is called by ntp_io when a new packet arrives off the network */
 extern void NSR_ProcessReceive(NTP_Packet *message, struct timeval *now, double now_err, NTP_Remote_Address *remote_addr, NTP_Local_Address *local_addr, int length);
 

sources.c

@@ -36,6 +36,7 @@
 #include "sourcestats.h"
 #include "memory.h"
 #include "ntp.h" /* For NTP_Leap */
+#include "ntp_sources.h"
 #include "local.h"
 #include "reference.h"
 #include "util.h"
@@ -399,6 +400,12 @@ SRC_UpdateReachability(SRC_Instance inst, int reachable)
   if (REF_GetMode() != REF_ModeNormal && special_mode_end()) {
     REF_SetUnsynchronised();
   }
+
+  /* Try to replace NTP sources that are unreachable or falsetickers */
+  if (inst->type == SRC_NTP && (inst->status == SRC_FALSETICKER ||
+      (!inst->reachability && inst->reachability_size == SOURCE_REACH_BITS))) {
+    NSR_HandleBadSource(inst->ip_addr);
+  }
 }
 
 /* ================================================== */

stubs.c

@@ -149,7 +149,7 @@ NSR_AddSource(NTP_Remote_Address *remote_addr, NTP_Source_Type type, SourceParam
 }
 
 void
-NSR_AddUnresolvedSource(char *name, int port, NTP_Source_Type type, SourceParameters *params)
+NSR_AddSourceByName(char *name, int port, int pool, NTP_Source_Type type, SourceParameters *params)
 {
 }
 
@@ -164,6 +164,11 @@ NSR_RemoveAllSources(void)
 {
 }
 
+void
+NSR_HandleBadSource(IPAddr *address)
+{
+}
+
 void
 NSR_SetSourceResolvingEndHandler(NSR_SourceResolvingEndHandler handler)
 {