Only include requested columns in CSV/JSON exports when using ?columns=…

doc/05-Upgrading.md

@@ -16,6 +16,11 @@ If you are upgrading across multiple versions, make sure to follow the steps for
   if another role denies access to different variables. The same applies to `icingadb/protect/variables`, in which
   case variables protected in one role will now be protected even if another role protects different variables.
   This has been done to simplify the configuration and to get it more in line with how refusals work in Icinga Web.
+* When using the `?columns` parameter to filter for specific columns and exporting to CSV/JSON, the exported file will
+  only contain the listed columns of the Hosts/Services, if the parameter is not set, all columns will be included.
+* If a relation is entirely empty and would result in an empty JSON-object, the JSON-export will not create an object
+  for it at all. (instead of `{"someKey":"someValue","emptyObject":{}}` we will now export `{"someKey":"someValue"}`)
+
 
 **Removed Features**
 

library/Icingadb/Data/CsvResultSetUtils.php

@@ -85,12 +85,14 @@ public static function stream(Query $query): void
             $offset = 0;
         }
 
+        $headerPrinted = false;
         do {
             $query->offset($offset);
             $result = $query->execute()->disableCache();
-            foreach ($result as $i => $keysAndValues) {
-                if ($i === 0) {
+            foreach ($result as $keysAndValues) {
+                if (! $headerPrinted) {
                     echo implode(',', array_keys($keysAndValues));
+                    $headerPrinted = true;
                 }
 
                 echo "\r\n";

library/Icingadb/Data/JsonResultSetUtils.php

@@ -51,7 +51,12 @@ protected function createObject(Model $model): array
         $keysAndValues = [];
         foreach ($model as $key => $value) {
             if ($value instanceof Model) {
-                $keysAndValues[$key] = $this->createObject($value);
+                $object = $this->createObject($value);
+                // If there is no value in the model or it's descendents,
+                // it was not a part of the query, so no JSON object will be created for this model.
+                if (! empty($object)) {
+                    $keysAndValues[$key] = $object;
+                }
             } else {
                 $keysAndValues[$key] = $this->formatValue($key, $value);
             }

library/Icingadb/Redis/VolatileStateResults.php

@@ -36,11 +36,32 @@ class VolatileStateResults extends ResultSet
     /** @var string Object type service */
     protected const TYPE_SERVICE = 'service';
 
+    /** @var array|null Columns to be selected if they were explicitly set, if empty all columns are selected */
+    protected ?array $columns;
+
+    /** @var bool Whether the model's ID should be contained in the results */
+    protected bool $includeModelID = true;
+
     public static function fromQuery(Query $query)
     {
         $self = parent::fromQuery($query);
         $self->resolver = $query->getResolver();
         $self->redisUnavailable = Backend::getRedis()->isUnavailable();
+        $self->columns = $query->getColumns();
+
+        if (! empty($self->columns)) {
+            // The id is necessary to apply the redis-updates
+            if ($query->getModel() instanceof Host && empty(array_intersect(['host.id', 'id'], $self->columns))) {
+                $query->withColumns('host.id');
+                $self->includeModelID = false;
+            } elseif (
+                $query->getModel() instanceof Service &&
+                empty(array_intersect(['service.id', 'id'], $self->columns))
+            ) {
+                $query->withColumns('service.id');
+                $self->includeModelID = false;
+            }
+        }
 
         return $self;
     }
@@ -62,7 +83,12 @@ public function current()
             $this->rewind();
         }
 
-        return parent::current();
+        $result = parent::current();
+        if (! $this->includeModelID) {
+            unset($result['id']);
+        }
+
+        return $result;
     }
 
     public function next(): void
@@ -109,8 +135,22 @@ protected function applyRedisUpdates($rows)
         $type = null;
         $showSourceGranted = $this->getAuth()->hasPermission('icingadb/object/show-source');
 
-        $getKeysAndBehaviors = function (State $state): array {
-            return [$state->getColumns(), $this->resolver->getBehaviors($state)];
+        $getKeysAndBehaviors = function (State $state, $type): array {
+            $columns = array_filter($state->getColumns(), function ($column) {
+                return ! str_ends_with($column, '_id');
+            });
+
+            if (! empty($this->columns)) {
+                $normalizedColumns = array_map(
+                    fn($column) => preg_replace("/^($type\.state\.|state\.)/", '', $column),
+                    $this->columns
+                );
+
+                $stateColumns = array_intersect($normalizedColumns, $columns);
+                return [$stateColumns, $this->resolver->getBehaviors($state)];
+            }
+
+            return [$columns, $this->resolver->getBehaviors($state)];
         };
 
         $states = [];
@@ -141,7 +181,7 @@ protected function applyRedisUpdates($rows)
             $states[$type][bin2hex($row->id)] = $row->state;
 
             if (! isset($states[$type]['keys'])) {
-                [$keys, $behaviors] = $getKeysAndBehaviors($row->state);
+                [$keys, $behaviors] = $getKeysAndBehaviors($row->state, $type);
 
                 if (! $showSourceGranted) {
                     $keys = array_diff($keys, ['check_commandline']);
@@ -155,7 +195,7 @@ protected function applyRedisUpdates($rows)
                 $states[self::TYPE_HOST][bin2hex($row->host->id)] = $row->host->state;
 
                 if (! isset($states[self::TYPE_HOST]['keys'])) {
-                    [$keys, $behaviors] = $getKeysAndBehaviors($row->host->state);
+                    [$keys, $behaviors] = $getKeysAndBehaviors($row->host->state, $type);
 
                     $states[self::TYPE_HOST]['keys'] = $keys;
                     $states[self::TYPE_HOST]['behaviors'] = $behaviors;

library/Icingadb/Web/Controller.php

@@ -22,6 +22,8 @@
 use Icinga\Module\Icingadb\Common\SearchControls;
 use Icinga\Module\Icingadb\Data\CsvResultSet;
 use Icinga\Module\Icingadb\Data\JsonResultSet;
+use Icinga\Module\Icingadb\Model\Host;
+use Icinga\Module\Icingadb\Model\Service;
 use Icinga\Module\Icingadb\Web\Control\GridViewModeSwitcher;
 use Icinga\Module\Icingadb\Web\Control\ViewModeSwitcher;
 use Icinga\Module\Icingadb\Widget\ItemTable\StateItemTable;
@@ -102,7 +104,12 @@ public function createColumnControl(Query $query, ViewModeSwitcher $viewModeSwit
             }
         }
 
-        $query->withColumns($columns);
+        // When exporting as CSV or JSON, and the user requested specific columns, only those should be included
+        if ($this->format === 'csv' || $this->format === 'json') {
+            $query->columns($columns);
+        } else {
+            $query->withColumns($columns);
+        }
 
         if (! $viewMode) {
             $viewModeSwitcher->setViewMode('tabular');