diff --git a/sources.c b/sources.c
index 8cb542d..b56fb6a 100644
--- a/sources.c
+++ b/sources.c
@@ -458,12 +458,12 @@ combine_sources(int n_sel_sources, struct timeval *ref_time, double *offset,
          fabs(*frequency - src_frequency) >
            combine_limit * (*skew + src_skew + LCL_GetMaxClockError()))) {
       sources[index]->outlier = !sources[index]->beginning ? OUTLIER_PENALTY : 1;
+    } else if (sources[index]->outlier) {
+      sources[index]->outlier--;
     }
 
-    if (sources[index]->outlier) {
-      sources[index]->outlier--;
+    if (sources[index]->outlier)
       continue;
-    }
 
     UTI_DiffTimevalsToDouble(&elapsed, ref_time, &src_ref_time);
     src_offset += elapsed * src_frequency;