ntp: wait for late HW TX timestamps

When sending client requests to a close and fast server, it is possible
that a response will be received before the HW transmit timestamp of
the request itself. To avoid processing of the response without the HW
timestamp, monitor events returned by select() and suspend reading of
packets from the receive queue for up to 200 microseconds. As the
requests are normally separated by at least 200 milliseconds, it is
sufficient to monitor and suspend one socket at a time.

ntp_io.c

@@ -318,6 +318,9 @@ close_socket(int sock_fd)
   if (sock_fd == INVALID_SOCK_FD)
     return;
 
+#ifdef HAVE_LINUX_TIMESTAMPING
+  NIO_Linux_NotifySocketClosing(sock_fd);
+#endif
   SCH_RemoveFileHandler(sock_fd);
   close(sock_fd);
 }
@@ -685,6 +688,11 @@ read_from_socket(int sock_fd, int event, void *anything)
   unsigned int i, n;
   int status, flags = 0;
 
+#ifdef HAVE_LINUX_TIMESTAMPING
+  if (NIO_Linux_ProcessEvent(sock_fd, event))
+    return;
+#endif
+
   hdr = ARR_GetElements(recv_headers);
   n = ARR_GetSize(recv_headers);
   assert(n >= 1);

ntp_io_linux.c

@@ -94,6 +94,21 @@ static int ts_tx_flags;
 /* Flag indicating the socket options can't be changed in control messages */
 static int permanent_ts_options;
 
+/* When sending client requests to a close and fast server, it is possible that
+   a response will be received before the HW transmit timestamp of the request
+   itself.  To avoid processing of the response without the HW timestamp, we
+   monitor events returned by select() and suspend reading of packets from the
+   receive queue for up to 200 microseconds.  As the requests are normally
+   separated by at least 200 milliseconds, it is sufficient to monitor and
+   suspend one socket at a time. */
+static int monitored_socket;
+static int suspended_socket;
+static SCH_TimeoutID resume_timeout_id;
+
+#define RESUME_TIMEOUT 200.0e-6
+
+#define INVALID_SOCK_FD -3
+
 /* ================================================== */
 
 static int
@@ -350,6 +365,9 @@ NIO_Linux_Initialise(void)
 
   /* Kernels before 4.7 ignore timestamping flags set in control messages */
   permanent_ts_options = !SYS_Linux_CheckKernelVersion(4, 7);
+
+  monitored_socket = INVALID_SOCK_FD;
+  suspended_socket = INVALID_SOCK_FD;
 }
 
 /* ================================================== */
@@ -406,6 +424,73 @@ NIO_Linux_SetTimestampSocketOptions(int sock_fd, int client_only, int *events)
 
 /* ================================================== */
 
+static void
+resume_socket(int sock_fd)
+{
+  if (monitored_socket == sock_fd)
+    monitored_socket = INVALID_SOCK_FD;
+
+  if (sock_fd == INVALID_SOCK_FD || sock_fd != suspended_socket)
+    return;
+
+  suspended_socket = INVALID_SOCK_FD;
+
+  SCH_SetFileHandlerEvent(sock_fd, SCH_FILE_INPUT, 1);
+
+  DEBUG_LOG("Resumed RX processing %s timeout fd=%d",
+            resume_timeout_id ? "before" : "on", sock_fd);
+
+  if (resume_timeout_id) {
+    SCH_RemoveTimeout(resume_timeout_id);
+    resume_timeout_id = 0;
+  }
+}
+
+/* ================================================== */
+
+static void
+resume_timeout(void *arg)
+{
+  resume_timeout_id = 0;
+  resume_socket(suspended_socket);
+}
+
+/* ================================================== */
+
+static void
+suspend_socket(int sock_fd)
+{
+  resume_socket(suspended_socket);
+
+  suspended_socket = sock_fd;
+
+  SCH_SetFileHandlerEvent(suspended_socket, SCH_FILE_INPUT, 0);
+  resume_timeout_id = SCH_AddTimeoutByDelay(RESUME_TIMEOUT, resume_timeout, NULL);
+
+  DEBUG_LOG("Suspended RX processing fd=%d", sock_fd);
+}
+
+/* ================================================== */
+
+int
+NIO_Linux_ProcessEvent(int sock_fd, int event)
+{
+  if (sock_fd != monitored_socket)
+    return 0;
+
+  if (event == SCH_FILE_INPUT) {
+    suspend_socket(monitored_socket);
+    monitored_socket = INVALID_SOCK_FD;
+
+    /* Don't process the message yet */
+    return 1;
+  }
+
+  return 0;
+}
+
+/* ================================================== */
+
 static struct Interface *
 get_interface(int if_index)
 {
@@ -614,6 +699,11 @@ NIO_Linux_ProcessMessage(NTP_Remote_Address *remote_addr, NTP_Local_Address *loc
         } else {
           DEBUG_LOG("HW clock not found for interface %d", ts_if_index);
         }
+
+        /* If a HW transmit timestamp was received, resume processing
+           of non-error messages on this socket */
+        if (is_tx)
+          resume_socket(local_addr->sock_fd);
       }
 
       if (local_ts->source == NTP_TS_DAEMON && !UTI_IsZeroTimespec(&ts3.ts[0]) &&
@@ -685,6 +775,12 @@ NIO_Linux_RequestTxTimestamp(struct msghdr *msg, int cmsglen, int sock_fd)
   if (!ts_flags)
     return cmsglen;
 
+  /* If a HW transmit timestamp is requested on a client socket, monitor
+     events on the socket in order to avoid processing of a fast response
+     without the HW timestamp of the request */
+  if (ts_tx_flags & SOF_TIMESTAMPING_TX_HARDWARE && !NIO_IsServerSocket(sock_fd))
+    monitored_socket = sock_fd;
+
   /* Check if TX timestamping is disabled on this socket */
   if (permanent_ts_options || !NIO_IsServerSocket(sock_fd))
     return cmsglen;
@@ -704,3 +800,11 @@ NIO_Linux_RequestTxTimestamp(struct msghdr *msg, int cmsglen, int sock_fd)
 
   return cmsglen;
 }
+
+/* ================================================== */
+
+void
+NIO_Linux_NotifySocketClosing(int sock_fd)
+{
+  resume_socket(sock_fd);
+}

ntp_io_linux.h

@@ -33,9 +33,13 @@ extern void NIO_Linux_Finalise(void);
 
 extern int NIO_Linux_SetTimestampSocketOptions(int sock_fd, int client_only, int *events);
 
+extern int NIO_Linux_ProcessEvent(int sock_fd, int event);
+
 extern int NIO_Linux_ProcessMessage(NTP_Remote_Address *remote_addr, NTP_Local_Address *local_addr,
                                     NTP_Local_Timestamp *local_ts, struct msghdr *hdr, int length);
 
 extern int NIO_Linux_RequestTxTimestamp(struct msghdr *msg, int cmsglen, int sock_fd);
 
+extern void NIO_Linux_NotifySocketClosing(int sock_fd);
+
 #endif