diff --git a/chrony.texi.in b/chrony.texi.in
index b56fb81..25f807b 100644
--- a/chrony.texi.in
+++ b/chrony.texi.in
@@ -2508,6 +2508,9 @@ The syntax of this directive is identical to that for the @code{server}
 directive (@pxref{server directive}), except that it is used to specify
 an NTP peer rather than an NTP server.
 
+When a key is specified by the @code{key} option to enable authentication, both
+peers must be configured to use the same key and the same key number.
+
 Please note that NTP peers that are not configured with a key to enable
 authentication are vulnerable to a denial-of-service attack.  An attacker
 knowing that NTP hosts A and B are peering with each other can send a packet
diff --git a/ntp_core.c b/ntp_core.c
index b477666..2843858 100644
--- a/ntp_core.c
+++ b/ntp_core.c
@@ -1099,7 +1099,7 @@ static int
 receive_packet(NTP_Packet *message, struct timeval *now, double now_err, NCR_Instance inst, NTP_Local_Address *local_addr, int length)
 {
   int pkt_leap;
-  uint32_t pkt_refid;
+  uint32_t pkt_refid, pkt_key_id;
   double pkt_root_delay;
   double pkt_root_dispersion;
 
@@ -1190,11 +1190,13 @@ receive_packet(NTP_Packet *message, struct timeval *now, double now_err, NCR_Ins
      function is called only for known sources. */
 
   /* Test 5 checks for authentication failure.  If we expect authenticated info
-     from this peer/server and the packet doesn't have it or the authentication
-     is bad, it's got to fail.  If the peer or server sends us an authenticated
-     frame, but we're not bothered about whether he authenticates or not, just
-     ignore the test. */
-  test5 = inst->do_auth ? check_packet_auth(message, length, NULL, NULL) : 1;
+     from this peer/server and the packet doesn't have it, the authentication
+     is bad, or it's authenticated with a different key than expected, it's got
+     to fail.  If we don't expect the packet to be authenticated, just ignore
+     the test. */
+  test5 = !inst->do_auth ||
+          (check_packet_auth(message, length, NULL, &pkt_key_id) &&
+           pkt_key_id == inst->auth_key_id);
 
   /* Test 6 checks for unsynchronised server */
   test6 = pkt_leap != LEAP_Unsynchronised &&