#!/usr/bin/perl
# This script reads the output from the dstate monitor and reports how
# many unique stack traces there were and what the stall times were.
# The objective is to identify the worst sources of stalls.
#
# Copyright Mel Gorman 2011
use strict;

my $line;
my %unique_event_counts;
my %unique_event_stall;
my %unique_event_stall_details;
my $total_stalled;

my ($process, $stalltime, $function, $event);
my ($stall_details, $trace, $first_trace, $reading_trace, $skip);
while ($line = <>) {
	# Watch for the beginning of a new event
	if ($line =~ /^time ([0-9]*): ([0-9]*) \((.*)\) Stalled: ([-0-9]*) ms: (.*)/) {

		# Skip uninteresting traces
		if (!$skip) {
			# Record the last event
			$unique_event_counts{$trace}++;
			$unique_event_stall_details{$trace} .= $event;
			if ($stalltime >= 0) {
				$unique_event_stall{$trace} += $stalltime;
				$total_stalled += $stalltime;
			}
		}

		# Start the next event
		$event = sprintf "%-20s %-20s %6d ms\n", $3, $5, $4;
		$reading_trace = 0;
		$stalltime = $4;

		if ($stalltime == 0) {
			print "DEBUG $line";
		}

		$first_trace = "";
		$trace = "";
	}

	# If we have reached a trace, blindly read it
	if ($reading_trace) {
		# Ignore traces that look like they are from user space
		if ($line =~ /^\[<0/) {
			$reading_trace = 0;
			next;
		}
		$trace .= $line;
		if ($first_trace eq "") {
			$first_trace = $line;
			$skip = 1;

			# Skip uninteresting traces
			if ($first_trace !~ / do_poll\+/ &&
				$first_trace !~ / kthread\+/ &&
				$first_trace !~ / khugepaged\+/ &&
				$first_trace !~ / sys_epoll_wait\+/ &&
				$first_trace !~ / kswapd\+/) {
				$skip = 0;
			}
		}
		next;
	}

	if ($line =~ /^\[<f/) {
		$reading_trace = 1;
		next;
	}
}

print "Overall stalled time: $total_stalled ms\n\n";
foreach my $trace (sort {$unique_event_stall{$b} <=> $unique_event_stall{$a}} keys %unique_event_stall) {
	#printf "Event $short_event us count %4d\n", $unique_event_counts{$event};
	#print $unique_event_stall_details{$event};
	printf "Time stalled in this event: %8d ms\n", $unique_event_stall{$trace};
	printf "Event count:                %8d\n", $unique_event_counts{$trace};
	print $unique_event_stall_details{$trace};
	print "$trace\n";
}

#print "\nDetails\n=======\n";
#foreach my $event (sort {$unique_event_stall{$b} <=> $unique_event_stall{$a}} keys %unique_event_stall) {
#	print "Event $event us count $unique_event_counts{$event}\n";
#	print "\n";
#}