From b1647dbcb757dbebb918d397e165808f0901b59d Mon Sep 17 00:00:00 2001 From: Miroslav Lichvar Date: Fri, 2 Feb 2018 11:29:23 +0100 Subject: [PATCH] ntp: wait for late HW TX timestamps When sending client requests to a close and fast server, it is possible that a response will be received before the HW transmit timestamp of the request itself. To avoid processing of the response without the HW timestamp, monitor events returned by select() and suspend reading of packets from the receive queue for up to 200 microseconds. As the requests are normally separated by at least 200 milliseconds, it is sufficient to monitor and suspend one socket at a time. --- ntp_io.c | 8 ++++ ntp_io_linux.c | 104 +++++++++++++++++++++++++++++++++++++++++++++++++ ntp_io_linux.h | 4 ++ 3 files changed, 116 insertions(+) diff --git a/ntp_io.c b/ntp_io.c index c93e52f..d63eb1d 100644 --- a/ntp_io.c +++ b/ntp_io.c @@ -318,6 +318,9 @@ close_socket(int sock_fd) if (sock_fd == INVALID_SOCK_FD) return; +#ifdef HAVE_LINUX_TIMESTAMPING + NIO_Linux_NotifySocketClosing(sock_fd); +#endif SCH_RemoveFileHandler(sock_fd); close(sock_fd); } @@ -685,6 +688,11 @@ read_from_socket(int sock_fd, int event, void *anything) unsigned int i, n; int status, flags = 0; +#ifdef HAVE_LINUX_TIMESTAMPING + if (NIO_Linux_ProcessEvent(sock_fd, event)) + return; +#endif + hdr = ARR_GetElements(recv_headers); n = ARR_GetSize(recv_headers); assert(n >= 1); diff --git a/ntp_io_linux.c b/ntp_io_linux.c index 8edcfcc..66c2c1e 100644 --- a/ntp_io_linux.c +++ b/ntp_io_linux.c @@ -94,6 +94,21 @@ static int ts_tx_flags; /* Flag indicating the socket options can't be changed in control messages */ static int permanent_ts_options; +/* When sending client requests to a close and fast server, it is possible that + a response will be received before the HW transmit timestamp of the request + itself. To avoid processing of the response without the HW timestamp, we + monitor events returned by select() and suspend reading of packets from the + receive queue for up to 200 microseconds. As the requests are normally + separated by at least 200 milliseconds, it is sufficient to monitor and + suspend one socket at a time. */ +static int monitored_socket; +static int suspended_socket; +static SCH_TimeoutID resume_timeout_id; + +#define RESUME_TIMEOUT 200.0e-6 + +#define INVALID_SOCK_FD -3 + /* ================================================== */ static int @@ -350,6 +365,9 @@ NIO_Linux_Initialise(void) /* Kernels before 4.7 ignore timestamping flags set in control messages */ permanent_ts_options = !SYS_Linux_CheckKernelVersion(4, 7); + + monitored_socket = INVALID_SOCK_FD; + suspended_socket = INVALID_SOCK_FD; } /* ================================================== */ @@ -406,6 +424,73 @@ NIO_Linux_SetTimestampSocketOptions(int sock_fd, int client_only, int *events) /* ================================================== */ +static void +resume_socket(int sock_fd) +{ + if (monitored_socket == sock_fd) + monitored_socket = INVALID_SOCK_FD; + + if (sock_fd == INVALID_SOCK_FD || sock_fd != suspended_socket) + return; + + suspended_socket = INVALID_SOCK_FD; + + SCH_SetFileHandlerEvent(sock_fd, SCH_FILE_INPUT, 1); + + DEBUG_LOG("Resumed RX processing %s timeout fd=%d", + resume_timeout_id ? "before" : "on", sock_fd); + + if (resume_timeout_id) { + SCH_RemoveTimeout(resume_timeout_id); + resume_timeout_id = 0; + } +} + +/* ================================================== */ + +static void +resume_timeout(void *arg) +{ + resume_timeout_id = 0; + resume_socket(suspended_socket); +} + +/* ================================================== */ + +static void +suspend_socket(int sock_fd) +{ + resume_socket(suspended_socket); + + suspended_socket = sock_fd; + + SCH_SetFileHandlerEvent(suspended_socket, SCH_FILE_INPUT, 0); + resume_timeout_id = SCH_AddTimeoutByDelay(RESUME_TIMEOUT, resume_timeout, NULL); + + DEBUG_LOG("Suspended RX processing fd=%d", sock_fd); +} + +/* ================================================== */ + +int +NIO_Linux_ProcessEvent(int sock_fd, int event) +{ + if (sock_fd != monitored_socket) + return 0; + + if (event == SCH_FILE_INPUT) { + suspend_socket(monitored_socket); + monitored_socket = INVALID_SOCK_FD; + + /* Don't process the message yet */ + return 1; + } + + return 0; +} + +/* ================================================== */ + static struct Interface * get_interface(int if_index) { @@ -614,6 +699,11 @@ NIO_Linux_ProcessMessage(NTP_Remote_Address *remote_addr, NTP_Local_Address *loc } else { DEBUG_LOG("HW clock not found for interface %d", ts_if_index); } + + /* If a HW transmit timestamp was received, resume processing + of non-error messages on this socket */ + if (is_tx) + resume_socket(local_addr->sock_fd); } if (local_ts->source == NTP_TS_DAEMON && !UTI_IsZeroTimespec(&ts3.ts[0]) && @@ -685,6 +775,12 @@ NIO_Linux_RequestTxTimestamp(struct msghdr *msg, int cmsglen, int sock_fd) if (!ts_flags) return cmsglen; + /* If a HW transmit timestamp is requested on a client socket, monitor + events on the socket in order to avoid processing of a fast response + without the HW timestamp of the request */ + if (ts_tx_flags & SOF_TIMESTAMPING_TX_HARDWARE && !NIO_IsServerSocket(sock_fd)) + monitored_socket = sock_fd; + /* Check if TX timestamping is disabled on this socket */ if (permanent_ts_options || !NIO_IsServerSocket(sock_fd)) return cmsglen; @@ -704,3 +800,11 @@ NIO_Linux_RequestTxTimestamp(struct msghdr *msg, int cmsglen, int sock_fd) return cmsglen; } + +/* ================================================== */ + +void +NIO_Linux_NotifySocketClosing(int sock_fd) +{ + resume_socket(sock_fd); +} diff --git a/ntp_io_linux.h b/ntp_io_linux.h index 1702a58..ed37e6a 100644 --- a/ntp_io_linux.h +++ b/ntp_io_linux.h @@ -33,9 +33,13 @@ extern void NIO_Linux_Finalise(void); extern int NIO_Linux_SetTimestampSocketOptions(int sock_fd, int client_only, int *events); +extern int NIO_Linux_ProcessEvent(int sock_fd, int event); + extern int NIO_Linux_ProcessMessage(NTP_Remote_Address *remote_addr, NTP_Local_Address *local_addr, NTP_Local_Timestamp *local_ts, struct msghdr *hdr, int length); extern int NIO_Linux_RequestTxTimestamp(struct msghdr *msg, int cmsglen, int sock_fd); +extern void NIO_Linux_NotifySocketClosing(int sock_fd); + #endif