diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index b914a2fa1b4d4..88ba750335268 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -231,12 +231,12 @@ jobs: sudo apt-get update sudo apt-get install \ bison \ - libgmp-dev \ libonig-dev \ libsqlite3-dev \ openssl \ re2c \ - valgrind + valgrind \ + debuginfod - name: ccache uses: hendrikmuhs/ccache-action@v1.2 with: @@ -255,7 +255,6 @@ jobs: --enable-werror \ --prefix=/usr \ --with-config-file-scan-dir=/etc/php.d \ - --with-gmp \ --with-mysqli=mysqlnd \ --with-openssl \ --with-pdo-sqlite \ @@ -287,7 +286,9 @@ jobs: ssh-key: ${{ secrets.BENCHMARKING_DATA_DEPLOY_KEY }} path: benchmark/repos/data - name: Benchmark - run: php benchmark/benchmark.php true + run: | + export DEBUGINFOD_URLS="https://debuginfod.ubuntu.com" + php benchmark/benchmark.php true - name: Store result if: github.event_name == 'push' run: | diff --git a/benchmark/benchmark.php b/benchmark/benchmark.php index ec39e9f3310e5..c240602fbd254 100644 --- a/benchmark/benchmark.php +++ b/benchmark/benchmark.php @@ -63,7 +63,8 @@ function runSymfonyDemo(bool $jit): array { cloneRepo($dir, 'https://github.com/php/benchmarking-symfony-demo-2.2.3.git'); runPhpCommand([$dir . '/bin/console', 'cache:clear']); runPhpCommand([$dir . '/bin/console', 'cache:warmup']); - return runValgrindPhpCgiCommand('symfony-demo', [$dir . '/public/index.php'], cwd: $dir, jit: $jit, warmup: 50, repeat: 50); + + return runValgrindPhpCgiCommand('symfony-demo', [$dir . '/public/index.php'], cwd: $dir, jit: $jit, repeat: 100); } function runWordpress(bool $jit): array { @@ -86,7 +87,8 @@ function runWordpress(bool $jit): array { // Warmup runPhpCommand([$dir . '/index.php'], $dir); - return runValgrindPhpCgiCommand('wordpress', [$dir . '/index.php'], cwd: $dir, jit: $jit, warmup: 50, repeat: 50); + + return runValgrindPhpCgiCommand('wordpress', [$dir . '/index.php'], cwd: $dir, jit: $jit, repeat: 100); } function runPhpCommand(array $args, ?string $cwd = null): ProcessResult { @@ -98,41 +100,94 @@ function runValgrindPhpCgiCommand( array $args, ?string $cwd = null, bool $jit = false, - int $warmup = 0, int $repeat = 1, ): array { global $phpCgi; - $profileOut = __DIR__ . "/profiles/callgrind.out.$name"; + $profileOut = __DIR__ . "/profiles/callgrind.$name"; if ($jit) { - $profileOut .= '.jit'; + $profileOut .= '-jit'; } $process = runCommand([ 'valgrind', '--tool=callgrind', '--dump-instr=yes', + '--collect-jumps=yes', + '--trace-children=yes', "--callgrind-out-file=$profileOut", + '--verbose', '--', $phpCgi, - '-T' . ($warmup ? $warmup . ',' : '') . $repeat, + '-T' . $repeat, '-d max_execution_time=0', '-d opcache.enable=1', '-d opcache.jit=' . ($jit ? 'tracing' : 'disable'), '-d opcache.jit_buffer_size=128M', '-d opcache.validate_timestamps=0', ...$args, - ]); - $instructions = extractInstructionsFromValgrindOutput($process->stderr); - if ($repeat > 1) { - $instructions = gmp_strval(gmp_div_q($instructions, $repeat)); + ], null, array_merge(getenv(), ['BENCHMARK_DUMP_SEPARATE_PROFILES' => '1'])); + + // collect metrics for startup, each benchmark run and shutdown + $metricsArr = []; + foreach (['startup' => 1, ...range(2, $repeat + 1), 'shutdown' => ''] as $k => $kCallgrindOut) { + $profileOutSpecific = $profileOut . '.' . $k; + rename($profileOut . ($kCallgrindOut === '' ? '' : '.' . $kCallgrindOut), $profileOutSpecific); + + $metricsArr[$k] = extractMetricsFromCallgrindFile($profileOutSpecific); + } + + // print all collected metrics + print_r(array_map(fn ($metrics) => array_map(fn ($v) => number_format($v, 0, '.', '_'), $metrics), $metricsArr)); + + // find median benchmark run + $metricsForMedianArr = $metricsArr; + unset($metricsForMedianArr['startup']); + unset($metricsForMedianArr['shutdown']); + uasort($metricsForMedianArr, fn ($a, $b) => $a['Ir'] <=> $b['Ir']); + $medianRunIndex = array_keys($metricsForMedianArr)[max(0, floor((count($metricsForMedianArr) - 3 /* -1 for count to index, -1 for first slow run due compliation, -1 for second run which is a little slower too */) / 2.0))]; + + // remove non-first-non-median profiles from artifacts + foreach (range(0, $repeat - 1) as $k) { + $profileOutSpecific = $profileOut . '.' . $k; + + if ($k !== 0 && $k !== $medianRunIndex) { + unlink($profileOutSpecific); + } + } + + // annotate profiles for artifacts + foreach (['startup', 0, $medianRunIndex, 'shutdown'] as $k) { + $profileOutSpecific = $profileOut . '.' . $k; + + runCommand([ + 'callgrind_annotate', + '--threshold=100', + '--auto=yes', + '--show-percs=no', + $profileOutSpecific, + new UnescapedArg('>'), + "$profileOutSpecific.txt", + ]); + } + + $instructions = $metricsArr[$medianRunIndex]['Ir']; + if ($repeat === 1) { // return the same data as before https://github.com/php/php-src/pull/13162 + $instructions += $metricsArr['startup']['Ir']; } + return ['instructions' => $instructions]; } -function extractInstructionsFromValgrindOutput(string $output): string { - preg_match("(==[0-9]+== Events : Ir\n==[0-9]+== Collected : (?[0-9]+))", $output, $matches); - return $matches['instructions'] ?? throw new \Exception('Unexpected valgrind output'); +/** + * @return array + */ +function extractMetricsFromCallgrindFile(string $path): array { + if (!preg_match('/\nevents:((?: +\w+)+)\nsummary:((?: +\d+)+)\n/', file_get_contents($path, length: 10_000), $matches)) { + throw new \Exception('Unexpected callgrind data'); + } + + return array_combine(explode(' ', ltrim($matches[1], ' ')), explode(' ', ltrim($matches[2], ' '))); } main(); diff --git a/benchmark/shared.php b/benchmark/shared.php index 450101770b28b..c4779c2d9caf7 100644 --- a/benchmark/shared.php +++ b/benchmark/shared.php @@ -1,17 +1,28 @@ arg + : escapeshellarg($v); + }, $args)); $pipes = null; - $result = new ProcessResult(); $descriptorSpec = [0 => ['pipe', 'r'], 1 => ['pipe', 'w'], 2 => ['pipe', 'w']]; fwrite(STDOUT, "> $cmd\n"); - $processHandle = proc_open($cmd, $descriptorSpec, $pipes, $cwd ?? getcwd(), null); + $processHandle = proc_open($cmd, $descriptorSpec, $pipes, $cwd ?? getcwd(), $envVars); $stdin = $pipes[0]; $stdout = $pipes[1]; @@ -22,6 +33,8 @@ function runCommand(array $args, ?string $cwd = null): ProcessResult { stream_set_blocking($stdout, false); stream_set_blocking($stderr, false); + $stdoutStr = ''; + $stderrStr = ''; $stdoutEof = false; $stderrEof = false; @@ -35,9 +48,9 @@ function runCommand(array $args, ?string $cwd = null): ProcessResult { foreach ($read as $stream) { $chunk = fgets($stream); if ($stream === $stdout) { - $result->stdout .= $chunk; + $stdoutStr .= $chunk; } elseif ($stream === $stderr) { - $result->stderr .= $chunk; + $stderrStr .= $chunk; } } @@ -48,6 +61,8 @@ function runCommand(array $args, ?string $cwd = null): ProcessResult { fclose($stdout); fclose($stderr); + $result = new ProcessResult($stdoutStr, $stderrStr); + $statusCode = proc_close($processHandle); if ($statusCode !== 0) { fwrite(STDOUT, $result->stdout); diff --git a/sapi/cgi/cgi_main.c b/sapi/cgi/cgi_main.c index 9d7e66ce2b5bc..ac70f592a4121 100644 --- a/sapi/cgi/cgi_main.c +++ b/sapi/cgi/cgi_main.c @@ -1696,6 +1696,16 @@ PHP_FUNCTION(apache_response_headers) /* {{{ */ } /* }}} */ +#ifdef HAVE_VALGRIND +static inline void callgrind_dump_stats(void) +{ + char *tmp = getenv("BENCHMARK_DUMP_SEPARATE_PROFILES"); + if (tmp && ZEND_ATOL(tmp)) { + CALLGRIND_DUMP_STATS; + } +} +#endif + static zend_module_entry cgi_module_entry = { STANDARD_MODULE_HEADER, "cgi-fcgi", @@ -2216,13 +2226,6 @@ consult the installation file that came with this distribution, or visit \n\ if (comma) { warmup_repeats = atoi(php_optarg); repeats = atoi(comma + 1); -#ifdef HAVE_VALGRIND - if (warmup_repeats > 0) { - CALLGRIND_STOP_INSTRUMENTATION; - /* We're not interested in measuring startup */ - CALLGRIND_ZERO_STATS; - } -#endif } else { repeats = atoi(php_optarg); } @@ -2264,6 +2267,13 @@ consult the installation file that came with this distribution, or visit \n\ } #endif while (!fastcgi || fcgi_accept_request(request) >= 0) { +#ifdef HAVE_VALGRIND + if (benchmark) { + /* measure startup and each benchmark run separately */ + callgrind_dump_stats(); + } +#endif + SG(server_context) = fastcgi ? (void *)request : (void *) 1; init_request_info(request); @@ -2430,12 +2440,6 @@ consult the installation file that came with this distribution, or visit \n\ } } /* end !cgi && !fastcgi */ -#ifdef HAVE_VALGRIND - if (warmup_repeats == 0) { - CALLGRIND_START_INSTRUMENTATION; - } -#endif - /* request startup only after we've done all we can to * get path_translated */ if (php_request_startup() == FAILURE) { @@ -2554,11 +2558,6 @@ consult the installation file that came with this distribution, or visit \n\ SG(request_info).query_string = NULL; } -#ifdef HAVE_VALGRIND - /* We're not interested in measuring shutdown */ - CALLGRIND_STOP_INSTRUMENTATION; -#endif - if (!fastcgi) { if (benchmark) { if (warmup_repeats) { @@ -2586,6 +2585,12 @@ consult the installation file that came with this distribution, or visit \n\ script_file = NULL; goto do_repeat; } + +#ifdef HAVE_VALGRIND + /* measure shutdown separately */ + callgrind_dump_stats(); +#endif + break; }