diff --git a/ntp_io.c b/ntp_io.c
index 902425f..c93e52f 100644
--- a/ntp_io.c
+++ b/ntp_io.c
@@ -204,21 +204,18 @@ prepare_socket(int family, int port_number, int client_only)
     /* Don't quit - we might survive anyway */
   }
 
-#ifdef SO_TIMESTAMP
-  /* Enable receiving of timestamp control messages */
-#ifdef SO_TIMESTAMPNS
-  /* Try nanosecond resolution first */
-  if (setsockopt(sock_fd, SOL_SOCKET, SO_TIMESTAMPNS, (char *)&on_off, sizeof(on_off)) < 0)
-#endif
-  if (setsockopt(sock_fd, SOL_SOCKET, SO_TIMESTAMP, (char *)&on_off, sizeof(on_off)) < 0) {
-    LOG(LOGS_ERR, "Could not set %s socket option", "SO_TIMESTAMP");
-    /* Don't quit - we might survive anyway */
-  }
-#endif
-
+  /* Enable kernel/HW timestamping of packets */
 #ifdef HAVE_LINUX_TIMESTAMPING
-  NIO_Linux_SetTimestampSocketOptions(sock_fd, client_only, &events);
+  if (!NIO_Linux_SetTimestampSocketOptions(sock_fd, client_only, &events))
 #endif
+#ifdef SO_TIMESTAMPNS
+    if (setsockopt(sock_fd, SOL_SOCKET, SO_TIMESTAMPNS, (char *)&on_off, sizeof(on_off)) < 0)
+#endif
+#ifdef SO_TIMESTAMP
+      if (setsockopt(sock_fd, SOL_SOCKET, SO_TIMESTAMP, (char *)&on_off, sizeof(on_off)) < 0)
+        LOG(LOGS_ERR, "Could not set %s socket option", "SO_TIMESTAMP");
+#endif
+      ;
 
 #ifdef IP_FREEBIND
   /* Allow binding to address that doesn't exist yet */
diff --git a/ntp_io_linux.c b/ntp_io_linux.c
index cb77dd7..8c12e5c 100644
--- a/ntp_io_linux.c
+++ b/ntp_io_linux.c
@@ -280,12 +280,12 @@ NIO_Linux_Initialise(void)
     break;
   }
 
+  ts_flags = SOF_TIMESTAMPING_SOFTWARE | SOF_TIMESTAMPING_RX_SOFTWARE;
+  ts_tx_flags = SOF_TIMESTAMPING_TX_SOFTWARE;
+
   if (hwts) {
-    ts_flags = SOF_TIMESTAMPING_RAW_HARDWARE | SOF_TIMESTAMPING_RX_HARDWARE;
-    ts_tx_flags = SOF_TIMESTAMPING_TX_HARDWARE;
-  } else {
-    ts_flags = SOF_TIMESTAMPING_SOFTWARE | SOF_TIMESTAMPING_RX_SOFTWARE;
-    ts_tx_flags = SOF_TIMESTAMPING_TX_SOFTWARE;
+    ts_flags |= SOF_TIMESTAMPING_RAW_HARDWARE | SOF_TIMESTAMPING_RX_HARDWARE;
+    ts_tx_flags |= SOF_TIMESTAMPING_TX_HARDWARE;
   }
 
   /* Enable IP_PKTINFO in messages looped back to the error queue */
@@ -504,10 +504,7 @@ NIO_Linux_ProcessMessage(NTP_Remote_Address *remote_addr, NTP_Local_Address *loc
 
       memcpy(&ts3, CMSG_DATA(cmsg), sizeof (ts3));
 
-      if (!UTI_IsZeroTimespec(&ts3.ts[0])) {
-        LCL_CookTime(&ts3.ts[0], &local_ts->ts, &local_ts->err);
-        local_ts->source = NTP_TS_KERNEL;
-      } else if (!UTI_IsZeroTimespec(&ts3.ts[2])) {
+      if (!UTI_IsZeroTimespec(&ts3.ts[2])) {
         iface = get_interface(local_addr->if_index);
         if (iface) {
           process_hw_timestamp(iface, &ts3.ts[2], local_ts, !is_tx ? length : 0,
@@ -516,6 +513,12 @@ NIO_Linux_ProcessMessage(NTP_Remote_Address *remote_addr, NTP_Local_Address *loc
           DEBUG_LOG("HW clock not found for interface %d", local_addr->if_index);
         }
       }
+
+      if (local_ts->source == NTP_TS_DAEMON && !UTI_IsZeroTimespec(&ts3.ts[0]) &&
+          (!is_tx || UTI_IsZeroTimespec(&ts3.ts[2]))) {
+        LCL_CookTime(&ts3.ts[0], &local_ts->ts, &local_ts->err);
+        local_ts->source = NTP_TS_KERNEL;
+      }
     }
 
     if ((cmsg->cmsg_level == SOL_IP && cmsg->cmsg_type == IP_RECVERR) ||
@@ -555,9 +558,9 @@ NIO_Linux_ProcessMessage(NTP_Remote_Address *remote_addr, NTP_Local_Address *loc
       iface->l2_udp6_ntp_start = l2_length - length;
   }
 
-  /* Drop the message if HW timestamp is missing or its processing failed */
-  if ((ts_flags & SOF_TIMESTAMPING_RAW_HARDWARE) && local_ts->source != NTP_TS_HARDWARE) {
-    DEBUG_LOG("Missing HW timestamp");
+  /* Drop the message if it has no timestamp or its processing failed */
+  if (local_ts->source == NTP_TS_DAEMON) {
+    DEBUG_LOG("Missing TX timestamp");
     return 1;
   }