Feat: memory profiler mode

xtrime-ru · xtrime-ru · commit da3dbfeed3d5 · 2025-07-10T23:23:21.000+02:00
diff --git a/README.md b/README.md
@@ -37,6 +37,10 @@ Options:
                              Pyroscope Auth Token. 
                              Example: psx-BWlqy_dW1Wxg6oBjuCWD28HxGCkB1Jfzt-jjtqHzrkzI
                              
+  -m, --memory               Enable memory traces mode
+                             Get memory usage from 'mem' tag and send average memory usage for trace group
+                             Do not provide this option for CPU mode
+                             
   -a, --app=STRING           Name of app. 
                              All samples will be saved under given app name.
                              Example: app
@@ -71,7 +75,21 @@ phpspy --max-depth=-1 --time-limit-ms=59000 --threads=1024 --rate-hz=4 --buffer-
 phpspy --max-depth=-1 --time-limit-ms=59000 --threads=100 --rate-hz=25 --buffer-size=65536 -J m -P '-x "php-fpm|php-fpm[0-9]\.[0-9]" | shuf' 2> error_log.log | php pyrospy.php run --pyroscope=https://pyroscope.yourdomain.com --rateHz=25 --app=testApp --tags=host=server39 --tags=role=web
 ```
 
-## Plugins
+### Memory profiling
+ - add `--memory-usage` option to phpspy command to add current and maximum memory usage to tags
+ - add `--memory` option to pyrospy command to send average memory usage for traces instead of sum count
+```shell
+phpspy --max-depth=-1 --time-limit-ms=59000 --threads=1024 --memory-usage --rate-hz=4 --buffer-size=65536 -J m -P '-x "php|php[0-9]\.[0-9]" | shuf' 2> error_log.log | php pyrospy.php run --pyroscope=https://pyroscope.yourdomain.com --rateHz=4 --app=testAppMemory --tags=host=server39 --tags=role=cli --memory
+
+phpspy --max-depth=-1 --time-limit-ms=59000 --threads=100 --memory-usage --rate-hz=25 --buffer-size=65536 -J m -P '-x "php-fpm|php-fpm[0-9]\.[0-9]" | shuf' 2> error_log.log | php pyrospy.php run --pyroscope=https://pyroscope.yourdomain.com --rateHz=25 --app=testAppMemory --tags=host=server39 --tags=role=web --memory
+```
+
+Tee can be used to get memory usage and cpu samples from same phpspy stdout
+```shell
+phpspy --max-depth=-1 --time-limit-ms=59000 --threads=1024 --memory-usage --rate-hz=4 --buffer-size=65536 -J m -P '-x "php|php[0-9]\.[0-9]" | shuf' 2> error_log.log | tee >(php pyrospy.php run --pyroscope=https://pyroscope.yourdomain.com --rateHz=4 --app=testAppMemory --tags=host=server39 --tags=role=cli --memory) >(php pyrospy.php run --pyroscope=https://pyroscope.yourdomain.com --rateHz=4 --app=testApp --tags=host=server39 --tags=role=cli) >/dev/null
+```
+
+### Plugins
 
 1. Create `.php` plugin class. Put it in any place. Make sure it has `namespace Zoon\PyroSpy\Plugins;` and classname match filename.
 ```php
diff --git a/app/Commands/RunCommand.php b/app/Commands/RunCommand.php
@@ -8,9 +8,13 @@
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
+use Zoon\PyroSpy\CpuTraceAggregator;
+use Zoon\PyroSpy\MemoryTraceAggregator;
 use Zoon\PyroSpy\Plugins\PluginInterface;
 use Zoon\PyroSpy\Processor;
 use Zoon\PyroSpy\SampleSender;
+use Zoon\PyroSpy\SenderAggregationEnum;
+use Zoon\PyroSpy\SenderUnitsEnum;
 
 class RunCommand extends Command
 {
@@ -33,6 +37,12 @@ protected function configure(): void
                     InputOption::VALUE_OPTIONAL,
                     'Pyroscope Auth Token. Example: psx-BWlqy_dW1Wxg6oBjuCWD28HxGCkB1Jfzt-jjtqHzrkzI',
                 ),
+                new InputOption(
+                    'memory',
+                    'm',
+                    InputOption::VALUE_NONE,
+                    'Process memory traces instead of CPU traces'
+                ),
                 new InputOption(
                     'app',
                     'a',
@@ -151,10 +161,27 @@ protected function execute(InputInterface $input, OutputInterface $output): int
             }
         }
 
+        if ($input->getOption('memory')) {
+            $aggregator = new MemoryTraceAggregator();
+            $sender = new SampleSender(
+                $pyroscope,
+                $app,
+                $rateHz,
+                $tags,
+                $pyroscopeAuthToken,
+                SenderUnitsEnum::Bytes,
+                SenderAggregationEnum::Average
+            );
+        } else {
+            $aggregator = new CpuTraceAggregator();
+            $sender = new SampleSender($pyroscope, $app, $rateHz, $tags, $pyroscopeAuthToken);
+        }
+
         $processor = new Processor(
             $interval,
             $batch,
-            new SampleSender($pyroscope, $app, $rateHz, $tags, $pyroscopeAuthToken),
+            $aggregator,
+            $sender,
             array_values(array_filter($plugins)),
             $sendSampleFutureLimit,
             $concurrentRequestLimit,
diff --git a/app/CpuTraceAggregator.php b/app/CpuTraceAggregator.php
@@ -0,0 +1,23 @@
+<?php
+
+namespace Zoon\PyroSpy;
+
+class CpuTraceAggregator extends TraceAggregatorAbstract
+{
+    /**
+     * @inheritDoc
+     */
+    public function addTrace(array $tags, string $key): void
+    {
+        ksort($tags);
+        unset($tags['mem']);
+        $tagsKey = serialize($tags);
+        if (!array_key_exists($tagsKey, $this->results)) {
+            $this->results[$tagsKey] = [];
+        }
+        if (!array_key_exists($key, $this->results[$tagsKey])) {
+            $this->results[$tagsKey][$key] = 0;
+        }
+        $this->results[$tagsKey][$key]++;
+    }
+}
diff --git a/app/MemoryTraceAggregator.php b/app/MemoryTraceAggregator.php
@@ -0,0 +1,55 @@
+<?php
+
+namespace Zoon\PyroSpy;
+
+/**
+ * @psalm-import-type ResultsArray from TraceAggregatorAbstract
+ */
+class MemoryTraceAggregator extends TraceAggregatorAbstract
+{
+    /**
+     * @var array<string, array<string, list<int>>>
+     */
+    protected array $results;
+    /**
+     * @inheritDoc
+     */
+    public function addTrace(array $tags, string $key): void
+    {
+        ksort($tags);
+        if (!array_key_exists('mem', $tags)) {
+            return;
+        }
+        $memSize = $tags['mem'];
+        unset($tags['mem']);
+
+        $tagsKey = serialize($tags);
+        if (!array_key_exists($tagsKey, $this->results)) {
+            $this->results[$tagsKey] = [];
+        }
+        if (!array_key_exists($key, $this->results[$tagsKey])) {
+            $this->results[$tagsKey][$key] = [];
+        }
+        $this->results[$tagsKey][$key][] = $memSize;
+    }
+
+    /**
+     * @return ResultsArray
+     */
+    public function getGrouppedTraces(): array
+    {
+        $results = [];
+
+        foreach ($this->results as $tagsKey => $tagResuts) {
+            foreach ($tagResuts as $key => $traces) {
+                if ($traces) {
+                    $results[$tagsKey][$key] = (int)(array_sum($traces)/count($traces));
+                } else {
+                    $results[$tagsKey][$key] = 0;
+                }
+            }
+        }
+
+        return $results;
+    }
+}
diff --git a/app/Plugins/ClearEmptyTags.php b/app/Plugins/ClearEmptyTags.php
@@ -6,8 +6,6 @@ class ClearEmptyTags implements PluginInterface
 {
     public function process(array $tags, array $trace): array
     {
-        unset($tags['ts']);
-
         foreach ($tags as $name => $value) {
             $value = trim($value);
             if (!$value || $value === '-') {
diff --git a/app/Processor.php b/app/Processor.php
@@ -37,6 +37,7 @@ final class Processor
     public function __construct(
         private readonly int $interval,
         private readonly int $batchLimit,
+        private readonly TraceAggregatorAbstract $aggregator,
         private readonly SampleSenderInterface $sender,
         private readonly array $plugins,
         int $sendSampleFutureLimit,
@@ -52,7 +53,7 @@ public function __construct(
 
     private function init(): void
     {
-        $this->results = [];
+        $this->aggregator->clear();
         $this->tsStart = time();
         $this->tsEnd = $this->tsStart + $this->interval;
     }
@@ -103,15 +104,15 @@ private function runProducer(): Future
                         }
 
                         $key = self::stringifyTrace($tracePrepared);
-                        $this->groupTrace($tags, $key);
+                        $this->aggregator->addTrace($tags, $key);
 
                         $currentTime = time();
 
-                        if ($currentTime < $this->tsEnd && $this->countResults() < $this->batchLimit) {
+                        if ($currentTime < $this->tsEnd && $this->aggregator->countGrouppedTraces() < $this->batchLimit) {
                             continue;
                         }
 
-                        foreach ($this->results as $tagSerialized => $results) {
+                        foreach ($this->aggregator->getGrouppedTraces() as $tagSerialized => $results) {
                             $this->queue->push(new Sample($this->tsStart, $currentTime, $results, unserialize($tagSerialized)));
                         }
 
@@ -123,7 +124,7 @@ private function runProducer(): Future
             }
 
             $currentTime = time();
-            foreach ($this->results as $tagSerialized => $results) {
+            foreach ($this->aggregator->getGrouppedTraces() as $tagSerialized => $results) {
                 $this->queue->push(new Sample($this->tsStart, $currentTime, $results, unserialize($tagSerialized)));
             }
 
@@ -184,7 +185,19 @@ private static function extractTags(array $sample): array
             if (count($item) !== 4) {
                 continue;
             }
+
+            //# uri = /stat.php
             [$hashtag, $tag, $equalsign, $value] = $item;
+
+            if ($tag === 'ts') {
+                continue;
+            }
+            if ($tag === 'mem') {
+                // # mem 1653800 1659424
+                [$hashtag, $tag, $value, $maxUsage] = $item;
+                $value = (int) $value;
+            }
+
             $tags[$tag] = $value;
         }
         return $tags;
@@ -236,32 +249,6 @@ private function getLine(): Generator
         }
     }
 
-    /**
-     * @param TagsArray $tags
-     * @param string $key
-     */
-    private function groupTrace(array $tags, string $key): void
-    {
-        ksort($tags);
-        $tagsKey = serialize($tags);
-        if (!array_key_exists($tagsKey, $this->results)) {
-            $this->results[$tagsKey] = [];
-        }
-        if (!array_key_exists($key, $this->results[$tagsKey])) {
-            $this->results[$tagsKey][$key] = 0;
-        }
-        $this->results[$tagsKey][$key]++;
-    }
-
-    private function countResults(): int
-    {
-        $count = 0;
-        foreach ($this->results as $tagResuts) {
-            $count += count($tagResuts);
-        }
-        return $count;
-    }
-
     private static function fixEvalLine(string $line): string
     {
 
diff --git a/app/SampleSender.php b/app/SampleSender.php
@@ -23,6 +23,8 @@ public function __construct(
         private readonly int $rateHz,
         private readonly array $tags,
         private readonly string $authToken = '',
+        private readonly SenderUnitsEnum $units = SenderUnitsEnum::Samples,
+        private readonly SenderAggregationEnum $aggregation = SenderAggregationEnum::Sum,
     ) {
         $this->client = (new HttpClientBuilder())
             ->retry(0)
@@ -95,6 +97,8 @@ private function getUrl(array $tags, int $fromTs, int $toTs): string
             'until' => $toTs,
             'sampleRate' => $this->rateHz,
             'format' => 'folded',
+            'units' => $this->units->value,
+            'aggregationType' => $this->aggregation->value,
         ];
         return $this->pyroscopeHost . "/ingest?" . http_build_query($params);
     }
diff --git a/app/SenderAggregationEnum.php b/app/SenderAggregationEnum.php
@@ -0,0 +1,9 @@
+<?php
+
+namespace Zoon\PyroSpy;
+
+enum SenderAggregationEnum: string
+{
+    case Sum = 'sum';
+    case Average = 'average';
+}
diff --git a/app/SenderUnitsEnum.php b/app/SenderUnitsEnum.php
@@ -0,0 +1,10 @@
+<?php
+
+namespace Zoon\PyroSpy;
+
+enum SenderUnitsEnum: string
+{
+    case Samples = 'samples';
+    case Objects = 'objects';
+    case Bytes = 'bytes';
+}
diff --git a/app/TraceAggregatorAbstract.php b/app/TraceAggregatorAbstract.php
@@ -0,0 +1,41 @@
+<?php
+
+namespace Zoon\PyroSpy;
+
+/**
+ * @psalm-import-type TagsArray from Sample
+ * @psalm-type ResultsArray=array<string, array<string, int>>
+ */
+abstract class TraceAggregatorAbstract
+{
+    /**
+     * @var ResultsArray
+     */
+    protected array $results;
+
+    public function clear(): void
+    {
+        $this->results = [];
+    }
+
+    /**
+     * @param TagsArray $tags
+     * @param string $key
+     */
+    abstract public function addTrace(array $tags, string $key): void;
+
+    public function countGrouppedTraces(): int {
+        $count = 0;
+        foreach ($this->results as $tagResuts) {
+            $count += count($tagResuts);
+        }
+        return $count;
+    }
+
+    /**
+     * @return ResultsArray
+     */
+    public function getGrouppedTraces(): array {
+        return $this->results;
+    }
+}
diff --git a/tests/ProcessorTest.php b/tests/ProcessorTest.php

Original file line number	Diff line number	Diff line change
`@@ -6,8 +6,6 @@ class ClearEmptyTags implements PluginInterface`
`6`	`6`	`{`
`7`	`7`	`public function process(array $tags, array $trace): array`
`8`	`8`	`{`
`9`		`- unset($tags['ts']);`
`10`		`-`
`11`	`9`	`foreach ($tags as $name => $value) {`
`12`	`10`	`$value = trim($value);`
`13`	`11`	`if (!$value \|\| $value === '-') {`