tdt-analyze-serverlog

#! /usr/bin/env perl

# Analyze the log entries produced by trustydns-server and produce a report that is easier to read
# than the condensed log entries. Typically usage might be to pipe in a 24hour period of log entries
# into this program to produce a daily report.
#
# This program is obviously version dependent as trustydns-server logging details can readily change
# between versions.

use warnings;
use strict 'vars';
use Getopt::Std;

# We purposely hard-code the version in this analysis program as an automated import risks a version
# change without anyone inspecting this code. It's rarely likely to be the case that a version
# change occurs without *some* changes to the stats reporting. At least in the early stages of this
# package.

my $expectedVersion = "v0.2.1";
my $currentVersion = "";

##########

sub usage($$)
{
    print "Error: $_[0]\n" if length $_[0];
    shift;
    print "
Usage: tdt-analyze-serverlog [-h] [-deSW]

This program analyzes the logging output generated by trustydns-server and produces a more
user-friendly report containing columns with headings and summaries across the whole logging period.

Log lines are supplied via stdin or filenames on the command line.

Where:
      -h Print usage message and exit 0

      -S Suppress summary report (normally printed)

      -W Suppress warnings messages when patterns mismatch (normally printed)

Version: $expectedVersion

";

    exit(shift);
}

##########
# MAIN

my %opts;
usage("", 1) unless getopts('hSW', \%opts);
usage("", 0) if defined $opts{h};
my $reportSummary = ! defined $opts{S};
my $printWarnings = ! defined $opts{W};

my $listeners = {};		# Running totals are kept in these set of variables
my $localResolver = {};
$localResolver->{Servers} = {};
my $biasUptime = 0;
my $runningUptime = 0;
my $totalUptime = 0;
my $totalLines = 0;
my $analyzedLines = 0;
my $skippedLines = 0;
my $startCount = 0;
my $exitCount = 0;

my $lines = {};			# Track how many successful lines were processed
$lines->{listeners} = 0;
$lines->{localResolver} = 0;

my $dispatch = {};

$dispatch->{listener}->{matchRegex} = 'Listening: (.*)';
$dispatch->{listener}->{detailRegex} = '\((\w+) on (\S+)\)';
$dispatch->{listener}->{handler} = \&Listener;

$dispatch->{listenerStats}->{matchRegex} = 'Status Listener: (.*)';
$dispatch->{listenerStats}->{detailRegex} = 'req=(\d+) ok=(\d+) \((\d+)\/(\d+)\/(\d+)\/(\d+)\/(\d+)\/(\d+)\) al=(\S+) errs=(\d+) \((\d+)\/(\d+)\/(\d+)\/(\d+)\/(\d+)\/(\d+)\/(\d+)\/(\d+)\/(\d+)\/(\d+)\/(\d+)\/(\d+)\) Concurrency=(\d+) \((\w+) on (\S+)\)';
$dispatch->{listenerStats}->{handler} = \&ListenerStats;

$dispatch->{connectionTracker}->{matchRegex} = 'Status Conn Track: (.*)';
$dispatch->{connectionTracker}->{detailRegex} = 'curr=(\d+) pk=(\d+) sess=(\d+) errs=(\d+) \((\d+)\/(\d+)\/(\d+)\/(\d+)\/(\d+)\/(\d+)\) connFor=(\S+) activeFor=(\S+) \((\w+) on (\S+)\)';
$dispatch->{connectionTracker}->{handler} = \&ConnectionTracker;

$dispatch->{localResolverTotals}->{matchRegex} = 'Status Local Resolver: Totals: (.*)';
$dispatch->{localResolverTotals}->{detailRegex} = 'req=(\d+) ok=(\d+) errs=(\d+) \((\d+)\/(\d+)\)';
$dispatch->{localResolverTotals}->{handler} = \&LocalResolverTotals;

$dispatch->{localResolverServer}->{matchRegex} = 'Status Local Resolver: Server: (.*)';
$dispatch->{localResolverServer}->{detailRegex} = 'req=(\d+) ok=(\d+) al=(\S+) errs=(\d+) \((\d+)\/(\d+)\/(\d+)\/(\d+)\/(\d+)\/(\d+)\) \(ev (\d+)\/(\d+)\) (\S+)';
$dispatch->{localResolverServer}->{handler} = \&LocalResolverServer;


while (<>) {
    chomp;
    &matchAndDispatch($_);
}

# Post analysis cleanups

$totalUptime += $runningUptime - $biasUptime;	# Include final instance uptime
$totalUptime = 0.001 if $totalUptime <= 0;	# Eliminate tedious divide by zero checking

if ($reportSummary) {
    &printServer();
    &printListener();
    &printLocalResolver();
}

exit(0);


##########
# trustydns-server v0.0.9 Starting

sub Starting($) {
    $currentVersion = shift;
    if ($currentVersion ne $expectedVersion) {
	print STDERR "Warning @ $.:Version mismatch. Want $expectedVersion, got $currentVersion\n" if $printWarnings;
	return;
    }

    $startCount++;
    $totalUptime += $runningUptime - $biasUptime; # Accumulate previous instance uptime
    $runningUptime = 0;
    $biasUptime = 0;
}


##########
# trustydns-server v0.0.1 Exiting after ...

sub Exiting($$) {
    my $version = shift;
    my $uptime = shift;
    $exitCount++;
}


##########
# Status Up: trustydns-server v0.0.7 19h52m4s

sub StatusUp($$) {
    $currentVersion = shift;
    return unless $currentVersion eq $expectedVersion;

    my $uptime = shift;
    my ($hh, $mm, $ss) = (0, 0, 0);
    $hh = $1 if $uptime =~ /(\d+)h/;
    $mm = $1 if $uptime =~ /(\d+)m/;
    $ss = $1 if $uptime =~ /(\d+)s/;
    my $nextRunningUptime = ((($hh * 60) + $mm) * 60) + $ss;
    if ($nextRunningUptime < $runningUptime) {	# Restarted?
	$totalUptime += $runningUptime - $biasUptime;
	$biasUptime = 0;
    }
    $runningUptime = $nextRunningUptime;
    if ($startCount == 0 && $biasUptime == 0) { # If we haven't seen a start then we need to bias the uptime
	$biasUptime = $runningUptime;
    }
}


##########
# Listening: (HTTP on 103.16.128.179:1443)
#             $1      2

sub Listener {
    my $listener = lc($1) . "://" . $2;
    $listeners->{$listener} = {} unless exists $listeners->{$listener}; # Create an entry for this listen address
}


##########
# Status Listener: req=120 ok=120 (0/0/120/120/0/120) al=0.070 errs=0 (0/0/0/0/0/0/0/0/0/0/0/0) Concurrency=7 (HTTPS on 103.16.128.179:1443)
#                      $1     2    3 4 ....      8       9          10 11    ...             22             23 24       25

sub ListenerStats {
    my $listener = lc($24) . "://" . $25;
    $lines->{listeners}++;
    my $al = $9;
    $al = 0.0 if $al eq "NaN";
    $listeners->{$listener}->{req} += $1;
    $listeners->{$listener}->{ok} += $2;
    $listeners->{$listener}->{evGet} = $3;
    $listeners->{$listener}->{evTsig} = $4;
    $listeners->{$listener}->{evEDNS0Removed} = $5;
    $listeners->{$listener}->{evECSv4Synth} = $6;
    $listeners->{$listener}->{evECSv6Synth} = $7;
    $listeners->{$listener}->{evPadding} = $8;

    $listeners->{$listener}->{totalLatency} += $2 * $al;
    $listeners->{$listener}->{totalErrors} += $10;
    $listeners->{$listener}->{BadContentType} += $11;
    $listeners->{$listener}->{BadMethod} += $12;
    $listeners->{$listener}->{BadPrefixLengths} += $13;
    $listeners->{$listener}->{BadQueryParamDecode} += $14;
    $listeners->{$listener}->{BodyReadError} += $15;
    $listeners->{$listener}->{ClientTLSBad} += $16;
    $listeners->{$listener}->{DNSPackResponseFailed} += $17;
    $listeners->{$listener}->{DNSUnpackRequestFailed} += $18;
    $listeners->{$listener}->{ECSSynthesisFailed} += $19;
    $listeners->{$listener}->{HTTPWriterFailed} += $20;
    $listeners->{$listener}->{LocalResolutionFailed} += $21;
    $listeners->{$listener}->{QueryParamMissing} += $22;
    $listeners->{$listener}->{Concurrency} = $23
	if !defined $listeners->{$listener}->{Concurrency} or $23 > $listeners->{$listener}->{Concurrency};
}


##########
# Status Conn Track: curr=1 pk=2 sess=0 errs=0 (0/0/0/0/0/0) connFor=0.0s activeFor=0.0s (HTTPS on 103.16.128.179:1443)
#                        $1    2      3      4  5/6/7/8/9/20         11             12    13       14

sub ConnectionTracker {
    my $lKey = lc($13) . "://" . $14;
    $lines->{listeners}++;
    $listeners->{$lKey}->{peakConn} = $2
	if !defined $listeners->{$lKey}->{peakConn} or $2 > $listeners->{$lKey}->{peakConn};

    $listeners->{$lKey}->{peakSess} = $3
	if !defined $listeners->{$lKey}->{peakSess} or $3 > $listeners->{$lKey}->{peakSess};

    $listeners->{$lKey}->{errNoConnInMap} += $5;
    $listeners->{$lKey}->{errNoConnForSession} += $6;
    $listeners->{$lKey}->{errDanglingConn} += $7;;
    $listeners->{$lKey}->{errNegativeConcurrency} += $8;
    $listeners->{$lKey}->{errConnsLost} += $9;
    $listeners->{$lKey}->{errUnknownState} += $10;

    # We're running more REs which clobber the $1, $2 vars so do this code last and copy the $11, $12 vars
    my $cf = $11;
    my $af = $12;
    my $connFor = 0;
    my $activeFor = 0;
    $connFor = $1 if $cf =~ /([0-9.]+)s/;
    $activeFor = $1 if $af =~ /([0-9.]+)s/;
    $listeners->{$lKey}->{connFor} += $connFor;
    $listeners->{$lKey}->{activeFor} += $activeFor;
}


##########
# v0.0.9: Status Local Resolver: Totals: req=123 ok=99 errs=24 (8/16)
#                                            $1     2       3   4 5
#
sub LocalResolverTotals {
    $lines->{localResolver}++;
    $localResolver->{req} += $1;
    $localResolver->{ok} += $2;

    $localResolver->{totalErrors} += $3;
    $localResolver->{errRTT} += $4;
    $localResolver->{errRetries} += $5;
}


##########
# v0.0.9 Status Local Resolver: Server: req=6 ok=6 al=0.194 errs=0 (0/0/0/0/0/0) (ev 0/0) 127.0.0.1:53
#                                           $1   2    3          4  5/6/7/8/9/10    11/12 13
sub LocalResolverServer {
    $lines->{localResolver}++;
    my $name = $13;
    my $al = $3;
    $al = 0.0 if $al eq "NaN";
    $localResolver->{Servers}->{$name}->{req} += $1;
    $localResolver->{Servers}->{$name}->{ok} += $2;
    $localResolver->{Servers}->{$name}->{latency} += $2 * $3;
    $localResolver->{Servers}->{$name}->{errs} += $4;
    $localResolver->{Servers}->{$name}->{sfxExchangeError} += $5;
    $localResolver->{Servers}->{$name}->{sfxFormatError} += $6;
    $localResolver->{Servers}->{$name}->{sfxServerFail} += $7;
    $localResolver->{Servers}->{$name}->{sfxRefused} += $8;
    $localResolver->{Servers}->{$name}->{sfxNotImplemented} += $9;
    $localResolver->{Servers}->{$name}->{sfxOther} += $10;
    $localResolver->{Servers}->{$name}->{evxTCPFallback} += $9;
    $localResolver->{Servers}->{$name}->{evxTCPSuperior} += $10;
}


##########
# Tease out the type and type-specific data then dispatch to type-specific handler. Pattern matching
# purposely ignores prefix goop such as timestamps and application identifiers which are added by
# the likes of syslog or multilog.

sub matchAndDispatch($) {
    $_ = shift;
    $totalLines++;
    if (/trustydns-server (\S+) Starting/) {
	&Starting($1);
	return;
    }

    if (/trustydns-server (\S+) Exiting after (\S+)/) {
	&Exiting($1, $2);
	return;
    }
	
    if (/Status Up: trustydns-server (\S+) (\w+)/) {
	&StatusUp($1, $2);
	return;
    }

    # Don't try and analyze any other lines if we're in the wrong version

    if ($currentVersion ne $expectedVersion) {
	print "Skip $. $currentVersion vs $expectedVersion\n";
	$skippedLines++;
	return;
    }

    foreach my $handlerName (keys %{$dispatch}) { # Find the matching pattern and handler
	my $matchRegex = $dispatch->{$handlerName}->{matchRegex};
	if (/$matchRegex/) {
	    $_ = $1;
	    my $detailRegex = $dispatch->{$handlerName}->{detailRegex};
	    my $handler = $dispatch->{$handlerName}->{handler};
	    if (/$detailRegex/) {
		$analyzedLines++;
		&${handler}();
	    } else {
		print STDERR "Warning @ $.: Regex failed for $handlerName\n" if $printWarnings;
	    }
	    return;
	}
    }

    $skippedLines++;
}


##########

sub printServer() {
    my $ss = $totalUptime;
    my $day = $ss / 86400;
    $ss %= 86400;
    my $hh = $ss / 3600;
    $ss %= 3600;
    my $mm = $ss / 60;
    $ss %= 60;
    
    printf("Lines: %d Analyzed: %d, Skipped: %d. Total UpTime: %dd%dh%dm%ds, Server Starts: %d, exits: %d.\n",
	   $totalLines, $analyzedLines, $skippedLines, $day, $hh, $mm, $ss, $startCount, $exitCount);
}


##########

sub printListener() {
    if ($lines->{listeners} == 0) {
	print STDERR "Warning: No Listen stats found in log file\n" if $printWarnings;
	return;
    }
    &printListenerSummary();
    my @listenerKeys = keys %{$listeners};
    if (!@listenerKeys) {
	print STDERR "Warning: No listen interfaces found in log file\n" if $printWarnings;
	return;
    }
    &printListenerEvents();
    &printListenerErrors();
}

sub printListenerSummary() {
    my @listenerKeys = sort keys %{$listeners};
    if (!@listenerKeys) {
	print STDERR "Warning: No listen interfaces found in log file\n" if $printWarnings;
	return;
    }

    printf("\n                        [Listener Interfaces - input lines=%d]\n\n", $lines->{listeners});

    my $header = "%7s %7s %7s %7s %7s %7s %7s %7s %8s\n";
    my $detail = "%7d %7.3f %7d %7.3f %7d %7d %7d %7d %8s\n";
    my $sep = "------- ------- ------- ------- ------- ------- ------- ------- --------\n";

    printf($header, "Total", "Req/", "Good", "Good", "Peak", "Failed", "Peak", "Peak", "Listener");
    printf($header, "Reqs", "Second", "Reqs", "Latency", "Concurr", "Reqs", "Conns", "Sess", "Index");
    print $sep;

    my $totals = {};
    $totals->{req} = 0;
    $totals->{ok}  = 0;
    $totals->{totalLatency} = 0;
    $totals->{totalErrors} = 0;
    my $peakConnections  = 0;
    my $peakSessions = 0;
    my $peakConcurrency = 0;

    my $lix = 0;
    foreach my $listener (@listenerKeys) {
	my $l = $listeners->{$listener};
	my $rps = $l->{req} / $totalUptime;
	my $latency = 0;
	$latency = $l->{totalLatency} / $l->{ok} if $l->{ok} > 0;
	printf($detail, $l->{req}, $rps, $l->{ok}, $latency, $l->{Concurrency}, $l->{totalErrors},
	       $l->{peakConn}, $l->{peakSess},  $lix);
	$lix++;

	$totals->{req} += $l->{req};
	$totals->{ok} += $l->{ok};
	$totals->{totalLatency} += $l->{totalLatency};
	$totals->{totalErrors} += $l->{totalErrors};
	$peakConcurrency = $l->{Concurrency} if $l->{Concurrency} > $peakConcurrency;
	$peakConnections = $l->{peakConn} if $l->{peakConn} > $peakConnections;
	$peakSessions = $l->{peakSess} if $l->{peakSess} > $peakSessions;
    }

    print $sep;
    my $rps = $totals->{req} / $totalUptime;
    my $latency = 0;
    $latency = $totals->{totalLatency} / $totals->{ok} if $totals->{ok} > 0;
    printf($detail, $totals->{req}, $rps, $totals->{ok}, $latency, $peakConcurrency, $totals->{totalErrors},
	   $peakConnections, $peakSessions, "Totals");
    print $sep;
    print "\n";
}


##########

sub printListenerEvents() {
    my @listenerKeys = sort keys %{$listeners};

    print "\n                        [Listener Events]\n\n";

    my @counterNames = ("evGet", "evTsig", "evEDNS0Removed", "evECSv4Synth", "evECSv6Synth", "evPadding");
    my @counterTitles = ("HTTP GET", "TSIG", "EDNS0 Removed", "ipv4 ECS", "ipv6 ECS", "Padding");
    &printListenerCounters(\@listenerKeys, \@counterNames, \@counterTitles, 0);
}


##########

sub printListenerErrors() {
    my @listenerKeys = sort keys %{$listeners};

    print "\n                        [Listener Errors]\n\n";

    my @counterNames = ("BadContentType", "BadMethod", "BadPrefixLengths", "BadQueryParamDecode",
			"BodyReadError", "ClientTLSBad", "DNSPackResponseFailed", "DNSUnpackRequestFailed",
			"ECSSynthesisFailed", "HTTPWriterFailed", "LocalResolutionFailed", "QueryParamMissing",
			"errNoConnInMap", "errNoConnForSession", "errDanglingConn", "errNegativeConcurrency",
			"errConnsLost", "errUnknownState");

    my @counterTitles = ("Bad Content Type", "Bad HTTP Method", "Bad Prefix Length", "Bad QP Data",
			 "Body Read Error", "Bad Client TLS", "Pack Response Failed", "Unpack Request Failed",
			 "ECS Synthesis failed", "HTTP Writer Failed", "Local Resolve Failed", "Query Param Missing",
			 "No Conn In Map", "No Conn For Session", "Dangling Conn", "Negative Concurrency",
			 "Conns Lost", "Unknown Conn State");

    &printListenerCounters(\@listenerKeys, \@counterNames, \@counterTitles, 1);
}


##########
# General printer routine for listener counters. There are so many counters that we have to go down
# the page to make them fit. This means the listener IDs go across the page, but since they too can
# be quite long we actually use a legend to conserve title space.

sub printListenerCounters($$$) {
    $_ = shift;
    my @listenerKeys = @{$_};
    $_ = shift;
    my @names = @{$_};
    $_ = shift;
    my @titles = @{$_};
    my $printLegend = shift;

    my $l1 = "                      ";
    my $l2 = $l1;
    my @totals;
    for (my $ix = 0; $ix <= $#listenerKeys; $ix++) {
	$l1 .= sprintf("    %4d", $ix);
	$l2 .= "  ------";
	$totals[$ix] = 0;
    }
    
    $l1 .= "     Totals\n";
    $l2 .= "     ------\n";

    if (!$printLegend) {
	print "                         Index (See Legend)\n";
	print $l1, $l2;
    }

    # Print array of errors

    for (my $cx = 0; $cx <= $#names; $cx++) {
	printf("%-22s", $titles[$cx]);
	my $tot = 0;
	for (my $ix = 0; $ix <= $#listenerKeys; $ix++) {
	    my $listener = $listenerKeys[$ix];
	    my $v = $listeners->{$listener}->{$names[$cx]};
	    printf("  %6d", $v);
	    $tot += $v;
	    $totals[$ix] += $v;
	}
	printf("     %6d\n", $tot);
    }
    print $l2;
    printf("%-22s", "Totals");
    my $tot = 0;
    for (my $ix = 0; $ix <= $#listenerKeys; $ix++) {
	my $v = $totals[$ix];
	printf("  %6d", $v);
	$tot += $v;
    }
    printf("     %6d\n", $tot);
    print $l2;
    if ($printLegend) {
	print "\n                              [Listener Legend]\n\n";
	print "Index  Listener\n";
	print "-----  ---------------------------------------------------------\n";
	for (my $ix = 0; $ix <= $#listenerKeys; $ix++) {
	    printf(" %4d  %s\n", $ix, $listenerKeys[$ix]);
	}
	print "-----  ---------------------------------------------------------\n\n";
    }
}


##########

sub printLocalResolver() {
    printf("\n                              [Local Resolver - input lines=%d]\n\n", $lines->{localResolver});
    if ($lines->{localResolver} == 0) {
	print STDERR "Warning: No Local Resolvers found in log file\n" if $printWarnings;
	return;
    }

    &printLocalResolverGeneral();
    if (%{$localResolver->{Servers}}) {
	print "\n                              [Local Resolver Servers]\n\n";
	&printLocalResolverServers();
	print "\n                              [Local Resolver Legend]\n\n";
	&printLocalResolverLegend();
    }
}

##########

sub printLocalResolverGeneral() {
    my $header = "%7s %7s %7s %7s %7s\n";
    my $detail = "%7d %7d %7d %7d %7d\n";
    my $sep = "------- ------- ------- ------- -------\n";
    printf($header, "Total", "Good", "Error", "Exceed", "Exceed");
    printf($header, "Reqs", "Reqs", "Reqs", "RTT", "Retries");
    print $sep;

    printf($detail, $localResolver->{req}, $localResolver->{ok},
	   $localResolver->{totalErrors}, $localResolver->{errRTT}, $localResolver->{errRetries});
    print $sep;
}

##########

sub printLocalResolverServers() {
    my $header = "%9s %9s %9s %9s %9s  %8s\n";
    my $detail = "%9d %9.3f %9d %9d %9.3f  %8s\n";
    my $sep = "--------- --------- --------- --------- ---------   -------\n";
    printf($header, "Total", "Req/", "Good", "Error", "Avg", "Resolver");
    printf($header, "Reqs", "Second", "Reqs", "Reqs", "Latency", "Index");
    print $sep;
    my $totals = {};
    my $six = 0;
    foreach my $server (sort (keys %{$localResolver->{Servers}})) {
	my $d = $localResolver->{Servers}->{$server}; # Reduce RSI
	my $latency = $d->{latency};
	$latency /= $d->{ok} if $d->{ok} > 0;
	printf($detail,
	       $d->{req},
	       $d->{req} / $totalUptime,
	       $d->{ok},
	       $d->{errs},
	       $latency,
	       $six);
	$six++;

	$totals->{req} += $d->{req};
	$totals->{ok} += $d->{ok};
	$totals->{latency} += $d->{latency};
	$totals->{errs} += $d->{errs};
    }
    print $sep;
    my $latency = $totals->{latency};
    $latency /= $totals->{ok} if $totals->{ok} > 0;
    printf($detail,
	   $totals->{req},
	   $totals->{req} / $totalUptime,
	   $totals->{ok},
	   $totals->{errs},
	   $latency,
	   "Totals");
    print $sep;

    print "\n";

    $header = "%8s %8s %8s %8s %8s %8s %8s %8s  %8s\n";
    $detail = "%8d %8d %8d %8d %8d %8d %8d %8d  %8s\n";
    $sep = "-------- -------- -------- -------- -------- -------- -------- --------  --------\n";
    printf($header, "sfx", "sfx", "sfx", "", "sfx Not", "", "evx", "evx", "");
    printf($header, "Exchange", "Format", "Server", "sfx", "Imple-", "sfx", "TCP", "TCP", "Resolver");
    printf($header, "Error", "Error", "Fail", "Refused", "mented", "Other", "Fallback","Superior", "Index");

    print $sep;
    $totals = {};
    $six = 0;
    foreach my $server (sort (keys %{$localResolver->{Servers}})) {
	my $d = $localResolver->{Servers}->{$server}; # Reduce RSI
	printf($detail,
	       $d->{sfxExchangeError},
	       $d->{sfxFormatError},
	       $d->{sfxServerFail},
	       $d->{sfxRefused},
	       $d->{sfxNotImplemented},
	       $d->{sfxOther},
	       $d->{evxTCPFallback},
	       $d->{evxTCPSuperior},
	       $six);
	$six++;

	$totals->{sfxExchangeError} += $d->{sfxExchangeError};
	$totals->{sfxFormatError} += $d->{sfxFormatError};
	$totals->{sfxServerFail} += $d->{sfxServerFail};
	$totals->{sfxRefused} += $d->{sfxRefused};
	$totals->{sfxNotImplemented} += $d->{sfxNotImplemented};
	$totals->{sfxOther} += $d->{sfxOther};
	$totals->{evxTCPFallback} += $d->{evxTCPFallback};
	$totals->{evxTCPSuperior} += $d->{evxTCPSuperior};
    }
    print $sep;
    printf($detail,
	   $totals->{sfxExchangeError},
	   $totals->{sfxFormatError},
	   $totals->{sfxServerFail},
	   $totals->{sfxRefused},
	   $totals->{sfxNotImplemented},
	   $totals->{sfxOther},
	   $totals->{evxTCPFallback},
	   $totals->{evxTCPSuperior},
	   "Totals");
    print $sep;
}

sub printLocalResolverLegend {
    print "Index   Resolver IP\n";
    print "-----   ----------------------------------------\n";
    my $six = 0;
    foreach my $server (sort (keys %{$localResolver->{Servers}})) {
	printf("%5d   %s\n", $six, $server);
	$six++;
    }
    print "-----   ----------------------------------------\n";
}