diff --git a/ntp_sources.c b/ntp_sources.c
index 852602a..3cbb2ae 100644
--- a/ntp_sources.c
+++ b/ntp_sources.c
@@ -1100,8 +1100,10 @@ NSR_ProcessRx(NTP_Remote_Address *remote_addr, NTP_Local_Address *local_addr,
 
   assert(initialised);
 
-  /* Must match IP address AND port number */
-  if (find_slot2(remote_addr, &slot) == 2) {
+  /* Avoid unnecessary lookup if the packet cannot be a response from our
+     source.  Otherwise, it must match both IP address and port number. */
+  if (NTP_LVM_TO_MODE(message->lvm) != MODE_CLIENT &&
+      find_slot2(remote_addr, &slot) == 2) {
     record = get_record(slot);
 
     if (!NCR_ProcessRxKnown(record->data, local_addr, rx_ts, message, length))
@@ -1137,8 +1139,10 @@ NSR_ProcessTx(NTP_Remote_Address *remote_addr, NTP_Local_Address *local_addr,
   SourceRecord *record;
   int slot;
 
-  /* Must match IP address AND port number */
-  if (find_slot2(remote_addr, &slot) == 2) {
+  /* Avoid unnecessary lookup if the packet cannot be a request to our
+     source.  Otherwise, it must match both IP address and port number. */
+  if (NTP_LVM_TO_MODE(message->lvm) != MODE_SERVER &&
+      find_slot2(remote_addr, &slot) == 2) {
     record = get_record(slot);
     NCR_ProcessTxKnown(record->data, local_addr, tx_ts, message, length);
   } else {
diff --git a/test/unit/ntp_sources.c b/test/unit/ntp_sources.c
index e8b7420..ea3910f 100644
--- a/test/unit/ntp_sources.c
+++ b/test/unit/ntp_sources.c
@@ -110,7 +110,7 @@ change_remote_address(NCR_Instance inst, NTP_Remote_Address *remote_addr, int nt
 void
 test_unit(void)
 {
-  char source_line[] = "127.0.0.1 offline", conf[] = "port 0", name[64], msg[1];
+  char source_line[] = "127.0.0.1 offline", conf[] = "port 0", name[64];
   int i, j, k, slot, found, pool, prev_n;
   uint32_t hash = 0, conf_id;
   NTP_Remote_Address addrs[256], addr;
@@ -120,6 +120,7 @@ test_unit(void)
   RPT_ActivityReport report;
   CPS_NTP_Source source;
   NSR_Status status;
+  NTP_Packet msg;
 
   CNF_Initialise(0, 0);
   CNF_ParseLine(NULL, 1, conf);
@@ -272,12 +273,14 @@ test_unit(void)
 
         switch (random() % 5) {
           case 0:
+            msg.lvm = NTP_LVM(0, NTP_VERSION, random() % 2 ? MODE_CLIENT : MODE_SERVER);
             NSR_ProcessTx(get_record(slot)->remote_addr, &local_addr,
-                          &local_ts, (NTP_Packet *)msg, 0);
+                          &local_ts, &msg, 0);
             break;
           case 1:
+            msg.lvm = NTP_LVM(0, NTP_VERSION, random() % 2 ? MODE_CLIENT : MODE_SERVER);
             NSR_ProcessRx(get_record(slot)->remote_addr, &local_addr,
-                          &local_ts, (NTP_Packet *)msg, 0);
+                          &local_ts, &msg, 0);
             break;
           case 2:
             NSR_HandleBadSource(&get_record(slot)->remote_addr->ip_addr);