Add simulation tests

Use clknetsim to run multiple chronyd instances with simulated clocks
and network. It allows fast and reproducible testing, without real
network.

Included are several tests of performance in different clock/network
conditions, chronyd options, NTP authentication, chronyc, and past bug
fixes.

Makefile.in

@@ -129,6 +129,9 @@ install: chronyd chronyc chrony.txt
 %.s : %.c
 	$(CC) $(CFLAGS) $(CPPFLAGS) -S $<
 
+check :
+	cd test/simulation; ./run
+
 install-docs : docs
 	[ -d $(DESTDIR)$(DOCDIR) ] || mkdir -p $(DESTDIR)$(DOCDIR)
 	cp chrony.txt $(DESTDIR)$(DOCDIR)/chrony.txt

test/simulation/001-defaults

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+. test.common
+
+test_start "default test settings"
+
+run_test || test_fail
+check_chronyd_exit || test_fail
+check_source_selection || test_fail
+check_packet_interval || test_fail
+check_sync || test_fail
+
+test_pass

test/simulation/002-largenetwork

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+. test.common
+
+test_start "large network"
+
+time_rms_limit=5e-4
+
+server_strata=3
+servers=4
+clients=5
+
+client_start=2000
+min_sync_time=2100
+max_sync_time=2300
+
+run_test || test_fail
+check_chronyd_exit || test_fail
+check_packet_interval || test_fail
+check_sync || test_fail
+
+test_pass

test/simulation/003-largefreqoffset

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+. test.common
+
+test_start "large frequency offset"
+
+max_sync_time=1000
+
+for freq_offset in -5e-2 -5e-3 5e-3 5e-2; do
+	# Adjust offset so it's close to 0 on first clock update
+	time_offset=$(awk "BEGIN {print -($freq_offset * 130)}")
+	run_test || test_fail
+	check_chronyd_exit || test_fail
+	check_source_selection || test_fail
+	check_sync || test_fail
+done
+
+test_pass

test/simulation/004-largetimeoffset

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+. test.common
+
+test_start "large time offset"
+
+min_sync_time=1300
+max_sync_time=1400
+
+for time_offset in -1e2 1e2; do
+	run_test || test_fail
+	check_chronyd_exit || test_fail
+	check_source_selection || test_fail
+	check_sync || test_fail
+done
+
+test_pass

test/simulation/005-externalstep

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+. test.common
+
+test_start "external time step"
+
+min_sync_time=1300
+max_sync_time=1500
+
+for step in -1e2 1e2; do
+	# Make one step in 150th second
+	client_step="(* $step (equal 0.1 (sum 1.0) 150))"
+	run_test || test_fail
+	check_chronyd_exit || test_fail
+	check_sync || test_fail
+done
+
+test_pass

test/simulation/006-largejitter

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+. test.common
+
+test_start "large jitter"
+
+time_offset=1e0
+jitter=1e-1
+
+time_max_limit=5e-1
+freq_max_limit=2e-1
+time_rms_limit=1e-1
+freq_rms_limit=5e-3
+
+run_test || test_fail
+check_chronyd_exit || test_fail
+check_source_selection || test_fail
+check_packet_interval || test_fail
+check_sync || test_fail
+
+test_pass

test/simulation/007-largewander

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+. test.common
+
+test_start "large wander"
+
+wander=1e-7
+
+time_max_limit=5e-3
+freq_max_limit=5e-3
+time_rms_limit=1e-3
+freq_rms_limit=1e-4
+
+run_test || test_fail
+check_chronyd_exit || test_fail
+check_source_selection || test_fail
+check_packet_interval || test_fail
+check_sync || test_fail
+
+test_pass

test/simulation/101-poll

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+. test.common
+test_start "minpoll/maxpoll options"
+
+wander=0.0
+jitter=1e-6
+
+time_max_limit=1e-5
+freq_max_limit=1e-5
+time_rms_limit=5e-6
+freq_rms_limit=5e-6
+client_conf="makestep 1e-2 1"
+
+for poll in $(seq 2 14); do
+	client_server_options="minpoll $poll maxpoll $poll"
+	limit=$[2**$poll * 10]
+	min_sync_time=$[2**$poll * 2]
+	max_sync_time=$[2**$poll * 21 / 10 + 1]
+
+	run_test || test_fail
+	check_chronyd_exit || test_fail
+	check_source_selection || test_fail
+	check_packet_interval || test_fail
+	check_sync || test_fail
+done
+
+test_pass

test/simulation/102-iburst

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+. test.common
+test_start "iburst option"
+
+freq_offset=1e-4
+
+client_conf="makestep 1e-2 1
+driftfile tmp/drift"
+client_server_options="iburst"
+
+min_sync_time=4
+max_sync_time=6
+
+echo "100 1.0" > tmp/drift
+
+run_test || test_fail
+check_chronyd_exit || test_fail
+check_source_selection || test_fail
+check_packet_interval || test_fail
+check_sync || test_fail
+
+test_pass

test/simulation/103-initstepslew

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+. test.common
+test_start "initstepslew directive"
+
+freq_offset=0.0
+wander=0.0
+limit=100
+
+# clknetsim requires source port (if bound) to match dest port
+client_server_options="port 124"
+client_conf="initstepslew 5 192.168.123.1
+port 124"
+
+min_sync_time=15
+max_sync_time=30
+
+for time_offset in -2.0 -0.2 0.2 2.0; do
+	run_test || test_fail
+	check_chronyd_exit || test_fail
+	check_sync || test_fail
+done
+
+min_sync_time=1
+max_sync_time=1
+
+for time_offset in -1e8 -1e2 1e2 1e8; do
+	run_test || test_fail
+	check_sync || test_fail
+done
+
+test_pass

test/simulation/104-driftfile

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+. test.common
+test_start "driftfile directive"
+
+servers=0
+time_offset=0.0
+wander=0.0
+limit=10
+freq_max_limit=1e-9
+min_sync_time=1
+max_sync_time=1
+client_conf="driftfile tmp/drift"
+
+for freq_offset in -5e-2 -5e-4 -5e-6 5e-6 5e-4 5e-2; do
+	awk "BEGIN {printf \"%.9e 1\", 1e6 - 1 / (1 + $freq_offset) * 1e6}" > tmp/drift
+	run_test || test_fail
+	check_chronyd_exit || test_fail
+	check_sync || test_fail
+done
+
+test_pass

test/simulation/105-ntpauth

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+. test.common
+
+test_start "NTP authentication"
+
+server_conf="keyfile tmp/keys"
+client_conf="keyfile tmp/keys"
+
+cat > tmp/keys <<-EOF
+1 $(tr -c -d 'a-zA-Z0-9' < /dev/urandom 2> /dev/null | head -c 24)
+2 ASCII:$(tr -c -d 'a-zA-Z0-9' < /dev/urandom 2> /dev/null | head -c 24)
+3 MD5 ASCII:$(tr -c -d 'a-zA-Z' < /dev/urandom 2> /dev/null | head -c 24)
+4 MD5 HEX:$(tr -c -d '0-9A-F' < /dev/urandom 2> /dev/null | head -c 32)
+EOF
+
+for key in 1 2 3 4; do
+	client_server_options="key $key"
+	run_test || test_fail
+	check_chronyd_exit || test_fail
+	check_source_selection || test_fail
+	check_packet_interval || test_fail
+	check_sync || test_fail
+done
+
+server_conf=""
+
+run_test || test_fail
+check_chronyd_exit || test_fail
+# This check must fail as server doesn't know the key
+check_sync && test_fail
+
+test_pass

test/simulation/106-refclock

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+. test.common
+test_start "SHM refclock"
+
+servers=0
+limit=1000
+refclock_jitter=$jitter
+min_sync_time=60
+max_sync_time=80
+client_conf="refclock SHM 0"
+
+run_test || test_fail
+check_chronyd_exit || test_fail
+check_source_selection || test_fail
+check_sync || test_fail
+
+test_pass

test/simulation/107-allowdeny

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+. test.common
+
+test_start "allow/deny directives"
+
+limit=500
+
+# Note that start_client in clknetsim.bash always adds allow to the config
+
+for server_conf in \
+	"deny" \
+	"deny all" \
+	"deny 192.168.0.0/16" \
+	"deny 192.168.123" \
+	"deny 192.168.123.2" \
+	"deny all
+allow 192.168.124.0/24"
+do
+	run_test || test_fail
+	check_chronyd_exit || test_fail
+	# These checks are expected to fail
+	check_source_selection && test_fail
+	check_packet_interval && test_fail
+	check_sync && test_fail
+done
+
+for server_conf in \
+	"deny all
+allow" \
+	"deny all
+allow all" \
+	"deny all
+allow 192.168.123" \
+	"deny all
+allow 192.168.123/24" \
+	"deny 192.168.124.0/24"
+do
+	run_test || test_fail
+	check_chronyd_exit || test_fail
+	check_source_selection || test_fail
+	check_packet_interval || test_fail
+	check_sync || test_fail
+done
+
+test_pass

test/simulation/108-peer

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+. test.common
+
+test_start "NTP peers"
+
+# Allow and drop packets to the server in 1000 second intervals, so only one
+# client has access to it and the other is forced to switch to the peer.
+base_delay=$(cat <<-EOF | tr -d '\n'
+  (+ 1e-4
+     (* -1
+        (equal 0.1 from 2)
+        (equal 0.1 to 1)
+        (equal 0.1 (min (% time 2000) 1000) 1000))
+     (* -1
+        (equal 0.1 from 3)
+        (equal 0.1 to 1)
+	(equal 0.1 (max (% time 2000) 1000) 1000)))
+EOF
+)
+
+clients=2
+peers=2
+max_sync_time=1000
+client_server_options="minpoll 6 maxpoll 6"
+client_peer_options="minpoll 6 maxpoll 6"
+
+run_test || test_fail
+check_chronyd_exit || test_fail
+check_source_selection || test_fail
+check_sync || test_fail
+
+test_pass

test/simulation/109-makestep

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+. test.common
+test_start "makestep directive"
+
+limit=200
+jitter=1e-5
+client_conf="makestep 2 1"
+
+min_sync_time=140
+max_sync_time=160
+
+for time_offset in -1.0 -0.1 0.1 1.0; do
+	run_test || test_fail
+	check_chronyd_exit || test_fail
+	check_source_selection || test_fail
+	check_sync || test_fail
+done
+
+min_sync_time=120
+max_sync_time=140
+
+for time_offset in -1e8 -1e2 1e2 1e8; do
+	run_test || test_fail
+	check_chronyd_exit || test_fail
+	check_source_selection || test_fail
+	check_sync || test_fail
+done
+
+test_pass

test/simulation/110-chronyc

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+. test.common
+
+test_start "chronyc"
+
+chronyc_conf="tracking
+sources
+sourcestats"
+
+run_test || test_fail
+check_chronyd_exit || test_fail
+
+check_chronyc_output "^Reference ID    : 192\.168\.123\.1 \(192\.168\.123\.1\)
+Stratum         : 2
+Ref time \(UTC\)  : Fri Jan  1 00:1.:.. 2010
+System time     : 0\.0000..... seconds (slow|fast) of NTP time
+Last offset     : -?0\.0000..... seconds
+RMS offset      : 0\.000...... seconds
+Frequency       : (99|100)\.... ppm fast
+Residual freq   : -?0.00. ppm
+Skew            : 0\.... ppm
+Root delay      : 0\.000... seconds
+Root dispersion : 0.000... seconds
+Update interval : 6.\.. seconds
+Leap status     : Normal
+210 Number of sources = 1
+MS Name/IP address         Stratum Poll Reach LastRx Last sample
+===============================================================================
+\^\* 192\.168\.123\.1                 1   [67]   377    [0-9]+ [0-9 +-]+[un]s\[[0-9 +-]+[un]s\] \+/-[ 0-9]+[un]s
+210 Number of sources = 1
+Name/IP Address            NP  NR  Span  Frequency  Freq Skew  Offset  Std Dev
+==============================================================================
+192\.168\.123\.1              [0-9 ]+ [0-9 ]+ [0-9 ]+ [ +-]0\.0..      0\....  [0-9 +-]+[un]s [0-9 ]+[un]s$" \
+|| test_fail
+
+test_pass

test/simulation/201-freqaccumulation

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+. test.common
+
+# Test fix in commit 60d0fa299307076143da94d36deb7b908fa9bdb7
+
+test_start "frequency accumulation"
+
+time_offset=100.0
+jitter=1e-6
+base_delay=1e-6
+wander=0.0
+
+limit=200
+time_max_limit=1e-5
+freq_max_limit=1e-7
+time_rms_limit=1e-5
+freq_rms_limit=1e-7
+min_sync_time=120
+max_sync_time=140
+
+client_server_options="minpoll 6 maxpoll 6"
+client_conf="driftfile tmp/drift
+makestep 1 1"
+
+for freq_offset in -5e-2 -5e-4 5e-4 5e-2; do
+	for drift in -1e+4 -1e+2 1e+2 1e+4; do
+		echo "$drift 100000" > tmp/drift
+		run_test || test_fail
+		check_chronyd_exit || test_fail
+		check_sync || test_fail
+	done
+done
+
+test_pass

test/simulation/README

@@ -0,0 +1,12 @@
+This is a collection of simulation tests. They use clknetsim to simulate
+multiple systems connected in a network. It's available at
+
+https://github.com/mlichvar/clknetsim
+
+If this directory doesn't have a clknetsim subdirectory, a known working
+revision will be downloaded and compiled automatically.
+
+Currently it runs only on Linux.
+
+The tests are written in bash and they can be run directly. The ./run script
+runs all tests.

test/simulation/test.common

@@ -0,0 +1,413 @@
+# Copyright (C) 2013-2014  Miroslav Lichvar <mlichvar@redhat.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+export LC_ALL=C
+export PATH=../../:$PATH
+export CLKNETSIM_PATH=clknetsim
+
+# Known working clknetsim revision
+clknetsim_revision=eb0b85335c40027ac941b2276142542fff482107
+clknetsim_url=https://github.com/mlichvar/clknetsim/archive/$clknetsim_revision.tar.gz
+
+# Only Linux is supported
+if [ "$(uname -s)" != Linux ]; then
+	echo "Simulation tests supported only on Linux"
+	exit 3
+fi
+
+# Try to download clknetsim if not found
+if [ ! -e $CLKNETSIM_PATH ]; then
+	curl -L "$clknetsim_url" | tar xz || exit 3
+	ln -s clknetsim-$clknetsim_revision clknetsim || exit 3
+fi
+
+# Try to build clknetsim if not built
+if [ ! -x $CLKNETSIM_PATH/clknetsim -o ! -e $CLKNETSIM_PATH/clknetsim.so ]; then
+	make -C clknetsim || exit 3
+fi
+
+. $CLKNETSIM_PATH/clknetsim.bash
+
+# Default test testings
+
+default_limit=10000
+default_time_offset=1e-1
+default_freq_offset=1e-4
+default_base_delay=1e-4
+default_jitter=1e-4
+default_wander=1e-9
+default_refclock_jitter=""
+
+default_update_interval=0
+default_log_packets=0
+default_shift_pll=2
+
+default_server_strata=1
+default_servers=1
+default_clients=1
+default_peers=0
+default_server_start=0.0
+default_client_start=0.0
+default_chronyc_start=1000.0
+default_server_step=""
+default_client_step=""
+
+default_server_server_options=""
+default_client_server_options=""
+default_server_peer_options=""
+default_client_peer_options=""
+default_server_conf=""
+default_client_conf=""
+default_chronyc_conf=""
+default_chronyd_options=""
+
+default_time_max_limit=1e-3
+default_freq_max_limit=5e-4
+default_time_rms_limit=2e-4
+default_freq_rms_limit=1e-5
+default_min_sync_time=120
+default_max_sync_time=210
+
+# Initialize test settings from their defaults
+for defopt in $(declare | grep '^default_'); do
+	defoptname=${defopt%%=*}
+	optname=${defoptname#default_}
+	eval "[ -z \"\${$optname:+a}\" ] && $optname=\"\$$defoptname\""
+done
+
+test_start() {
+	rm -f tmp/*
+	echo "Testing $@:"
+}
+
+test_pass() {
+	echo "PASS"
+	exit 0
+}
+
+test_fail() {
+	echo "FAIL"
+	exit 1
+}
+
+test_ok() {
+	pad_line
+	echo -e "\tOK"
+	return 0
+}
+
+test_bad() {
+	pad_line
+	echo -e "\tBAD"
+	return 1
+}
+
+test_error() {
+	pad_line
+	echo -e "\tERROR"
+	return 1
+}
+
+msg_length=0
+pad_line() {
+	local line_length=56
+	[ $msg_length -lt $line_length ] && \
+		printf "%$[$line_length - $msg_length]s" ""
+	msg_length=0
+}
+
+# Print aligned message
+test_message() {
+	local level=$1 eol=$2
+	shift 2
+	local msg="$*"
+
+	while [ $level -gt 0 ]; do
+		echo -n "  "
+		level=$[$level - 1]
+		msg_length=$[$msg_length + 2]
+	done
+	echo -n "$msg"
+
+	msg_length=$[$msg_length + ${#msg}]
+	if [ $eol -ne 0 ]; then
+		echo
+		msg_length=0
+	fi
+}
+
+get_wander_expr() {
+	local scaled_wander
+
+	scaled_wander=$(awk "BEGIN {print $wander / \
+		sqrt($update_interval < 0 ? 2^-($update_interval) : 1)}")
+
+	echo "(+ $freq_offset (sum (* $scaled_wander (normal))))"
+}
+
+
+get_delay_expr() {
+	echo "(+ $base_delay (* $jitter (exponential)))"
+}
+
+get_refclock_expr() {
+	echo "(* $refclock_jitter (normal))"
+}
+
+get_chronyd_nodes() {
+	echo $[$servers * $server_strata + $clients]
+}
+
+get_chronyd_conf() {
+	local i stratum=$1 peer=$2
+
+	if [ $stratum -eq 1 ]; then
+		echo "local stratum 1"
+		echo "$server_conf"
+	elif [ $stratum -le $server_strata ]; then
+		for i in $(seq 1 $servers); do
+			echo "server 192.168.123.$[$servers * ($stratum - 2) + $i] $server_server_options"
+		done
+		for i in $(seq 1 $peers); do
+			[ $i -eq $peer -o $i -gt $servers ] && continue
+			echo "peer 192.168.123.$[$servers * ($stratum - 1) + $i] $server_peer_options"
+		done
+		echo "$server_conf"
+	else
+		for i in $(seq 1 $servers); do
+			echo "server 192.168.123.$[$servers * ($stratum - 2) + $i] $client_server_options"
+		done
+		for i in $(seq 1 $peers); do
+			[ $i -eq $peer -o $i -gt $clients ] && continue
+			echo "peer 192.168.123.$[$servers * ($stratum - 1) + $i] $client_peer_options"
+		done
+		echo "$client_conf"
+	fi
+}
+
+# Check if the clock was well synchronized
+check_sync() {
+	local i sync_time max_time_error max_freq_error ret=0
+	local rms_time_error rms_freq_error
+
+	test_message 2 1 "checking clock sync time, max/rms time/freq error:"
+
+	for i in $(seq 1 $(get_chronyd_nodes)); do
+		[ $i -gt $[$servers * $server_strata] ] || continue
+
+		sync_time=$(find_sync tmp/log.offset tmp/log.freq $i \
+			$time_max_limit $freq_max_limit 1.0)
+		max_time_error=$(get_stat 'Maximum absolute offset' $i)
+		max_freq_error=$(get_stat 'Maximum absolute frequency' $i)
+		rms_time_error=$(get_stat 'RMS offset' $i)
+		rms_freq_error=$(get_stat 'RMS frequency' $i)
+
+		test_message 3 0 "node $i: $sync_time $(printf '%.2e %.2e %.2e %.2e' \
+			$max_time_error $max_freq_error $rms_time_error $rms_freq_error)"
+
+		check_stat $sync_time $min_sync_time $max_sync_time && \
+			check_stat $max_time_error 0.0 $time_max_limit && \
+			check_stat $max_freq_error 0.0 $freq_max_limit && \
+			check_stat $rms_time_error 0.0 $time_rms_limit && \
+			check_stat $rms_freq_error 0.0 $freq_rms_limit && \
+			test_ok || test_bad
+
+		[ $? -eq 0 ] || ret=1
+	done
+
+	return $ret
+}
+
+# Check if chronyd exited properly
+check_chronyd_exit() {
+	local i ret=0
+
+	test_message 2 1 "checking chronyd exit:"
+
+	for i in $(seq 1 $(get_chronyd_nodes)); do
+		test_message 3 0 "node $i:"
+
+		tail -n 1 tmp/log.$i | grep -q 'chronyd exiting' && \
+			test_ok || test_bad
+		[ $? -eq 0 ] || ret=1
+	done
+
+	return $ret
+}
+
+# Check for problems in source selection
+check_source_selection() {
+	local i ret=0
+
+	test_message 2 1 "checking source selection:"
+
+	for i in $(seq $[$servers * $server_strata + 1] $(get_chronyd_nodes)); do
+		test_message 3 0 "node $i:"
+
+		! grep -q 'no majority\|no reachable sources' tmp/log.$i && \
+			grep -q 'Selected source' tmp/log.$i && \
+			test_ok || test_bad
+		[ $? -eq 0 ] || ret=1
+	done
+
+	return $ret
+}
+
+# Check if incoming and outgoing packet intervals are sane
+check_packet_interval() {
+	local i ret=0 in_interval out_interval
+
+	test_message 2 1 "checking incoming and outgoing packet interval:"
+
+	for i in $(seq 1 $(get_chronyd_nodes)); do
+		in_interval=$(get_stat 'Mean incoming packet interval' $i)
+		out_interval=$(get_stat 'Mean outgoing packet interval' $i)
+
+		test_message 3 0 "node $i: $(printf '%.2e %.2e' \
+			$in_interval $out_interval)"
+
+		# Check that the intervals are non-zero and shorter than limit,
+		# incoming is not longer than outgoing for stratum 1 servers, and
+		# outgoing is not longer than incoming for clients.
+		nodes=$[$servers * $server_strata + $clients]
+		check_stat $in_interval 0.1 $limit && \
+			check_stat $out_interval 0.1 $limit && \
+			([ $i -gt $servers ] || \
+				check_stat $in_interval 0.0 $out_interval) && \
+			([ $i -le $[$servers * $server_strata] ] || \
+				check_stat $in_interval 0.0 $out_interval) && \
+			test_ok || test_bad
+
+		[ $? -eq 0 ] || ret=1
+	done
+
+	return $ret
+}
+
+# Compare chronyc output with specified pattern
+check_chronyc_output() {
+	local i ret=0 pattern=$1
+
+	test_message 2 1 "checking chronyc output:"
+
+	for i in $(seq $[$(get_chronyd_nodes) + 1] $[$(get_chronyd_nodes) + $clients]); do
+		test_message 3 0 "node $i:"
+
+		[[ "$(cat tmp/log.$i)" =~ $pattern ]] && \
+			test_ok || test_bad
+		[ $? -eq 0 ] || ret=1
+	done
+
+	return $ret
+}
+
+# Print test settings which differ from default value
+print_nondefaults() {
+	local defopt defoptname optname
+
+	test_message 2 1 "non-default settings:"
+	declare | grep '^default_*' | while read defopt; do
+		defoptname=${defopt%%=*}
+		optname=${defoptname#default_}
+		eval "[ \"\$$optname\" = \"\$$defoptname\" ]" || \
+			test_message 3 1 $(eval "echo $optname=\$$optname")
+	done
+}
+
+run_simulation() {
+	local nodes=$1
+
+	test_message 2 0 "running simulation:"
+
+	start_server $nodes \
+		-o tmp/log.offset -f tmp/log.freq \
+		$([ $log_packets -ne 0 ] && echo -p tmp/log.packets) \
+		-R $(awk "BEGIN {print $update_interval < 0 ? 2^-($update_interval) : 1}") \
+		-r $(awk "BEGIN {print $max_sync_time * 2^$update_interval}") \
+		-l $(awk "BEGIN {print $limit * 2^$update_interval}") && test_ok || test_error
+}
+
+run_test() {
+	local i j n stratum node nodes step start freq offset conf
+
+	test_message 1 1 "network with $servers*$server_strata servers and $clients clients:"
+	print_nondefaults
+
+	nodes=$(get_chronyd_nodes)
+	[ -n "$chronyc_conf" ] && nodes=$[$nodes + $clients]
+
+	for i in $(seq 1 $nodes); do
+		echo "node${i}_shift_pll = $shift_pll"
+		for j in $(seq 1 $nodes); do
+			[ $i -eq $j ] && continue
+			echo "node${i}_delay${j} = $(get_delay_expr)"
+			echo "node${j}_delay${i} = $(get_delay_expr)"
+		done
+	done > tmp/conf
+
+	node=1
+
+	for stratum in $(seq 1 $[$server_strata + 1]); do
+		[ $stratum -le $server_strata ] && n=$servers || n=$clients
+
+		for i in $(seq 1 $n); do
+			test_message 2 0 "starting node $node:"
+			if [ $stratum -eq 1 ]; then
+				step=$server_step
+				start=$server_start
+				freq=""
+				offset=0.0
+			elif [ $stratum -le $server_strata ]; then
+				step=$server_step
+				start=$server_start
+				freq=$(get_wander_expr)
+				offset=0.0
+			else
+				step=$client_step
+				start=$client_start
+				freq=$(get_wander_expr)
+				offset=$time_offset
+			fi
+
+			conf=$(get_chronyd_conf $stratum $i $n)
+
+			[ -z "$freq" ] || echo "node${node}_freq = $freq" >> tmp/conf
+			[ -z "$step" ] || echo "node${node}_step = $step" >> tmp/conf
+			[ -z "$refclock_jitter" ] || \
+				echo "node${node}_refclock = $(get_refclock_expr)" >> tmp/conf
+			echo "node${node}_offset = $offset" >> tmp/conf
+			echo "node${node}_start = $start" >> tmp/conf
+			start_client $node chronyd "$conf" "" "$chronyd_options" && \
+				test_ok || test_error
+
+			[ $? -ne 0 ] && return 1
+			node=$[$node + 1]
+		done
+	done
+
+	for i in $(seq 1 $[$nodes - $node + 1]); do
+		test_message 2 0 "starting node $node:"
+
+		echo "node${node}_start = $chronyc_start" >> tmp/conf
+		start_client $node chronyc "$chronyc_conf" "" \
+			"-n -h 192.168.123.$[$node - $clients]" && \
+			test_ok || test_error
+
+		[ $? -ne 0 ] && return 1
+		node=$[$node + 1]
+	done
+
+	run_simulation $nodes
+}