Skip to content

Commit

Permalink
Merge branch 'cc/codespeed'
Browse files Browse the repository at this point in the history 
"perf" test output can be sent to codespeed server.

* cc/codespeed:
  perf/run: read GIT_PERF_REPO_NAME from perf.repoName
  perf/run: learn to send output to codespeed server
  perf/run: learn about perf.codespeedOutput
  perf/run: add conf_opts argument to get_var_from_env_or_config()
  perf/aggregate: implement codespeed JSON output
  perf/aggregate: refactor printing results
  perf/aggregate: fix checking ENV{GIT_PERF_SUBSECTION}
  • Loading branch information
@gitster
gitster committed Jan 23, 2018
2 parents 59b43c0 + 19cf57a commit 86d7fcc
Show file tree
Hide file tree
Showing 2 changed files with 137 additions and 54 deletions.
160 changes: 112 additions & 48 deletions t/perf/aggregate.perl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
use lib '../../perl/blib/lib';
use strict;
use warnings;
use JSON;
use Git;

sub get_times {
Expand Down Expand Up @@ -35,10 +36,15 @@ sub format_times {
return $out;
}

my (@dirs, %dirnames, %dirabbrevs, %prefixes, @tests);
my (@dirs, %dirnames, %dirabbrevs, %prefixes, @tests, $codespeed);
while (scalar @ARGV) {
my $arg = $ARGV[0];
my $dir;
if ($arg eq "--codespeed") {
$codespeed = 1;
shift @ARGV;
next;
}
last if -f $arg or $arg eq "--";
if (! -d $arg) {
my $rev = Git::command_oneline(qw(rev-parse --verify), $arg);
Expand Down Expand Up @@ -70,8 +76,10 @@ sub format_times {
}

my $resultsdir = "test-results";
if ($ENV{GIT_PERF_SUBSECTION} ne "") {
my $results_section = "";
if (exists $ENV{GIT_PERF_SUBSECTION} and $ENV{GIT_PERF_SUBSECTION} ne "") {
$resultsdir .= "/" . $ENV{GIT_PERF_SUBSECTION};
$results_section = $ENV{GIT_PERF_SUBSECTION};
}

my @subtests;
Expand Down Expand Up @@ -100,13 +108,6 @@ sub read_descr {
return $line;
}

my %descrs;
my $descrlen = 4; # "Test"
for my $t (@subtests) {
$descrs{$t} = $shorttests{$t}.": ".read_descr("$resultsdir/$t.descr");
$descrlen = length $descrs{$t} if length $descrs{$t}>$descrlen;
}

sub have_duplicate {
my %seen;
for (@_) {
Expand All @@ -122,54 +123,117 @@ sub have_slash {
return 0;
}

my %newdirabbrevs = %dirabbrevs;
while (!have_duplicate(values %newdirabbrevs)) {
%dirabbrevs = %newdirabbrevs;
last if !have_slash(values %dirabbrevs);
%newdirabbrevs = %dirabbrevs;
for (values %newdirabbrevs) {
s{^[^/]*/}{};
sub print_default_results {
my %descrs;
my $descrlen = 4; # "Test"
for my $t (@subtests) {
$descrs{$t} = $shorttests{$t}.": ".read_descr("$resultsdir/$t.descr");
$descrlen = length $descrs{$t} if length $descrs{$t}>$descrlen;
}
}

my %times;
my @colwidth = ((0)x@dirs);
for my $i (0..$#dirs) {
my $d = $dirs[$i];
my $w = length (exists $dirabbrevs{$d} ? $dirabbrevs{$d} : $dirnames{$d});
$colwidth[$i] = $w if $w > $colwidth[$i];
}
for my $t (@subtests) {
my $firstr;
my %newdirabbrevs = %dirabbrevs;
while (!have_duplicate(values %newdirabbrevs)) {
%dirabbrevs = %newdirabbrevs;
last if !have_slash(values %dirabbrevs);
%newdirabbrevs = %dirabbrevs;
for (values %newdirabbrevs) {
s{^[^/]*/}{};
}
}

my %times;
my @colwidth = ((0)x@dirs);
for my $i (0..$#dirs) {
my $d = $dirs[$i];
$times{$prefixes{$d}.$t} = [get_times("$resultsdir/$prefixes{$d}$t.times")];
my ($r,$u,$s) = @{$times{$prefixes{$d}.$t}};
my $w = length format_times($r,$u,$s,$firstr);
my $w = length (exists $dirabbrevs{$d} ? $dirabbrevs{$d} : $dirnames{$d});
$colwidth[$i] = $w if $w > $colwidth[$i];
$firstr = $r unless defined $firstr;
}
}
my $totalwidth = 3*@dirs+$descrlen;
$totalwidth += $_ for (@colwidth);

binmode STDOUT, ":utf8" or die "PANIC on binmode: $!";
for my $t (@subtests) {
my $firstr;
for my $i (0..$#dirs) {
my $d = $dirs[$i];
$times{$prefixes{$d}.$t} = [get_times("$resultsdir/$prefixes{$d}$t.times")];
my ($r,$u,$s) = @{$times{$prefixes{$d}.$t}};
my $w = length format_times($r,$u,$s,$firstr);
$colwidth[$i] = $w if $w > $colwidth[$i];
$firstr = $r unless defined $firstr;
}
}
my $totalwidth = 3*@dirs+$descrlen;
$totalwidth += $_ for (@colwidth);

printf "%-${descrlen}s", "Test";
for my $i (0..$#dirs) {
my $d = $dirs[$i];
printf " %-$colwidth[$i]s", (exists $dirabbrevs{$d} ? $dirabbrevs{$d} : $dirnames{$d});
}
print "\n";
print "-"x$totalwidth, "\n";
for my $t (@subtests) {
printf "%-${descrlen}s", $descrs{$t};
my $firstr;
printf "%-${descrlen}s", "Test";
for my $i (0..$#dirs) {
my $d = $dirs[$i];
my ($r,$u,$s) = @{$times{$prefixes{$d}.$t}};
printf " %-$colwidth[$i]s", format_times($r,$u,$s,$firstr);
$firstr = $r unless defined $firstr;
printf " %-$colwidth[$i]s", (exists $dirabbrevs{$d} ? $dirabbrevs{$d} : $dirnames{$d});
}
print "\n";
print "-"x$totalwidth, "\n";
for my $t (@subtests) {
printf "%-${descrlen}s", $descrs{$t};
my $firstr;
for my $i (0..$#dirs) {
my $d = $dirs[$i];
my ($r,$u,$s) = @{$times{$prefixes{$d}.$t}};
printf " %-$colwidth[$i]s", format_times($r,$u,$s,$firstr);
$firstr = $r unless defined $firstr;
}
print "\n";
}
}

sub print_codespeed_results {
my ($results_section) = @_;

my $project = "Git";

my $executable = `uname -s -m`;
chomp $executable;

if ($results_section ne "") {
$executable .= ", " . $results_section;
}

my $environment;
if (exists $ENV{GIT_PERF_REPO_NAME} and $ENV{GIT_PERF_REPO_NAME} ne "") {
$environment = $ENV{GIT_PERF_REPO_NAME};
} elsif (exists $ENV{GIT_TEST_INSTALLED} and $ENV{GIT_TEST_INSTALLED} ne "") {
$environment = $ENV{GIT_TEST_INSTALLED};
$environment =~ s|/bin-wrappers$||;
} else {
$environment = `uname -r`;
chomp $environment;
}

my @data;

for my $t (@subtests) {
for my $d (@dirs) {
my $commitid = $prefixes{$d};
$commitid =~ s/^build_//;
$commitid =~ s/\.$//;
my ($result_value, $u, $s) = get_times("$resultsdir/$prefixes{$d}$t.times");

my %vals = (
"commitid" => $commitid,
"project" => $project,
"branch" => $dirnames{$d},
"executable" => $executable,
"benchmark" => $shorttests{$t} . " " . read_descr("$resultsdir/$t.descr"),
"environment" => $environment,
"result_value" => $result_value,
);
push @data, \%vals;
}
}

print to_json(\@data, {utf8 => 1, pretty => 1}), "\n";
}

binmode STDOUT, ":utf8" or die "PANIC on binmode: $!";

if ($codespeed) {
print_codespeed_results($results_section);
} else {
print_default_results();
}
31 changes: 25 additions & 6 deletions t/perf/run
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,8 @@ get_var_from_env_or_config () {
env_var="$1"
conf_sec="$2"
conf_var="$3"
# $4 can be set to a default value
conf_opts="$4" # optional
# $5 can be set to a default value

# Do nothing if the env variable is already set
eval "test -z \"\${$env_var+x}\"" || return
Expand All @@ -116,18 +117,18 @@ get_var_from_env_or_config () {
if test -n "$GIT_PERF_SUBSECTION"
then
var="$conf_sec.$GIT_PERF_SUBSECTION.$conf_var"
conf_value=$(git config -f "$GIT_PERF_CONFIG_FILE" "$var") &&
conf_value=$(git config $conf_opts -f "$GIT_PERF_CONFIG_FILE" "$var") &&
eval "$env_var=\"$conf_value\"" && return
fi
var="$conf_sec.$conf_var"
conf_value=$(git config -f "$GIT_PERF_CONFIG_FILE" "$var") &&
conf_value=$(git config $conf_opts -f "$GIT_PERF_CONFIG_FILE" "$var") &&
eval "$env_var=\"$conf_value\"" && return

test -n "${4+x}" && eval "$env_var=\"$4\""
test -n "${5+x}" && eval "$env_var=\"$5\""
}

run_subsection () {
get_var_from_env_or_config "GIT_PERF_REPEAT_COUNT" "perf" "repeatCount" 3
get_var_from_env_or_config "GIT_PERF_REPEAT_COUNT" "perf" "repeatCount" "--int" 3
export GIT_PERF_REPEAT_COUNT

get_var_from_env_or_config "GIT_PERF_DIRS_OR_REVS" "perf" "dirsOrRevs"
Expand All @@ -136,17 +137,35 @@ run_subsection () {
get_var_from_env_or_config "GIT_PERF_MAKE_COMMAND" "perf" "makeCommand"
get_var_from_env_or_config "GIT_PERF_MAKE_OPTS" "perf" "makeOpts"

get_var_from_env_or_config "GIT_PERF_REPO_NAME" "perf" "repoName"
export GIT_PERF_REPO_NAME

GIT_PERF_AGGREGATING_LATER=t
export GIT_PERF_AGGREGATING_LATER

if test $# = 0 -o "$1" = -- -o -f "$1"; then
set -- . "$@"
fi

codespeed_opt=
test "$GIT_PERF_CODESPEED_OUTPUT" = "true" && codespeed_opt="--codespeed"

run_dirs "$@"
./aggregate.perl "$@"

if test -z "$GIT_PERF_SEND_TO_CODESPEED"
then
./aggregate.perl $codespeed_opt "$@"
else
json_res_file="test-results/$GIT_PERF_SUBSECTION/aggregate.json"
./aggregate.perl --codespeed "$@" | tee "$json_res_file"
send_data_url="$GIT_PERF_SEND_TO_CODESPEED/result/add/json/"
curl -v --request POST --data-urlencode "json=$(cat "$json_res_file")" "$send_data_url"
fi
}

get_var_from_env_or_config "GIT_PERF_CODESPEED_OUTPUT" "perf" "codespeedOutput" "--bool"
get_var_from_env_or_config "GIT_PERF_SEND_TO_CODESPEED" "perf" "sendToCodespeed"

cd "$(dirname $0)"
. ../../GIT-BUILD-OPTIONS

Expand Down

0 comments on commit 86d7fcc

Please sign in to comment.