Optimize Apigee entity caching for 3x branch (#1165)

kedarkhaire · web-flow · commit 677a2f7546ea · 2025-09-17T21:06:44.000+05:30
diff --git a/apigee_edge.install b/apigee_edge.install
@@ -389,3 +389,17 @@ function apigee_edge_update_9002() {
   $edge_settings['content']['callbackUrl'] = $new_edge_settings['content']['callbackUrl'];
   $config_storage->write('core.entity_view_display.developer_app.developer_app.default', $edge_settings);
 }
+
+/**
+ * Set cache_insert_chunk_size for API Products, Developers, Developer apps.
+ */
+function apigee_edge_update_10401(): void {
+  $configs = [
+    'apigee_edge.api_product_settings',
+    'apigee_edge.developer_settings',
+    'apigee_edge.developer_app_settings',
+  ];
+  foreach ($configs as $config) {
+    \Drupal::configFactory()->getEditable($config)->set('cache_insert_chunk_size', 100)->save(TRUE);
+  }
+}
diff --git a/config/install/apigee_edge.api_product_settings.yml b/config/install/apigee_edge.api_product_settings.yml
@@ -2,6 +2,7 @@ langcode: en
 entity_label_singular: ''
 entity_label_plural: ''
 cache_expiration: 900
+cache_insert_chunk_size: 100
 access:
   public:
     - anonymous
diff --git a/config/install/apigee_edge.developer_app_settings.yml b/config/install/apigee_edge.developer_app_settings.yml
@@ -2,6 +2,7 @@ langcode: en
 entity_label_singular: 'App'
 entity_label_plural: 'Apps'
 cache_expiration: 900
+cache_insert_chunk_size: 100
 credential_lifetime: 0
 locked_base_fields:
   - displayName
diff --git a/config/install/apigee_edge.developer_settings.yml b/config/install/apigee_edge.developer_settings.yml
@@ -24,3 +24,4 @@ user_edit_error_message:
   value: 'This email address already exists in our system. You can register a new account if you would like to use it on the Developer Portal.'
   format: 'plain_text'
 cache_expiration: 900
+cache_insert_chunk_size: 100
diff --git a/config/schema/apigee_edge.schema.yml b/config/schema/apigee_edge.schema.yml
@@ -1,3 +1,10 @@
+apigee_edge.cache_insert_chunk_size:
+  type: integer
+  label: 'Cache insert chunk size'
+  constraints:
+    Range:
+      min: 1
+
 apigee_edge.error_page:
   type: config_object
   label: 'Apigee Edge: Error page'
@@ -79,6 +86,8 @@ apigee_edge.developer_app_settings:
       label: 'How to refer to a Developer App on the UI (plural)'
     cache_expiration:
       type: integer
+    cache_insert_chunk_size:
+      type: apigee_edge.cache_insert_chunk_size
     required_base_fields:
       type: sequence
       sequence:
@@ -161,6 +170,8 @@ apigee_edge.developer_settings:
           label: 'Message'
     cache_expiration:
       type: integer
+    cache_insert_chunk_size:
+      type: apigee_edge.cache_insert_chunk_size
 
 apigee_edge.api_product_settings:
   type: config_object
@@ -174,6 +185,8 @@ apigee_edge.api_product_settings:
       label: 'How to refer to an API Product on the UI (plural)'
     cache_expiration:
       type: integer
+    cache_insert_chunk_size:
+      type: apigee_edge.cache_insert_chunk_size
     access:
       type: mapping
       label: 'Filter displayed API products by access attribute'
diff --git a/modules/apigee_edge_teams/apigee_edge_teams.install b/modules/apigee_edge_teams/apigee_edge_teams.install
@@ -233,3 +233,16 @@ function apigee_edge_teams_update_9003() {
   $team_settings['enablefilter'] = $new_team_settings['enablefilter'];
   $config_storage->write('apigee_edge_teams.team_settings', $team_settings);
 }
+
+/**
+ * Set cache_insert_chunk_size for Teams and Team apps.
+ */
+function apigee_edge_teams_update_11401(): void {
+  $configs = [
+    'apigee_edge_teams.team_settings',
+    'apigee_edge_teams.team_app_settings',
+  ];
+  foreach ($configs as $config) {
+    \Drupal::configFactory()->getEditable($config)->set('cache_insert_chunk_size', 100)->save(TRUE);
+  }
+}
diff --git a/modules/apigee_edge_teams/config/install/apigee_edge_teams.team_app_settings.yml b/modules/apigee_edge_teams/config/install/apigee_edge_teams.team_app_settings.yml
@@ -2,6 +2,7 @@ langcode: en
 entity_label_singular: ''
 entity_label_plural: ''
 cache_expiration: 900
+cache_insert_chunk_size: 100
 credential_lifetime: 0
 locked_base_fields:
   - displayName
diff --git a/modules/apigee_edge_teams/config/install/apigee_edge_teams.team_settings.yml b/modules/apigee_edge_teams/config/install/apigee_edge_teams.team_settings.yml
@@ -5,6 +5,7 @@ enablefilter: ''
 entity_label_singular: ''
 entity_label_plural: ''
 cache_expiration: 900
+cache_insert_chunk_size: 100
 non_member_team_apps_visible_api_products:
   - public
   - private
diff --git a/modules/apigee_edge_teams/config/schema/apigee_edge_teams.schema.yml b/modules/apigee_edge_teams/config/schema/apigee_edge_teams.schema.yml
@@ -19,6 +19,8 @@ apigee_edge_teams.team_settings:
       label: 'How to refer to a Team on the UI (plural)'
     cache_expiration:
       type: integer
+    cache_insert_chunk_size:
+      type: apigee_edge.cache_insert_chunk_size
     non_member_team_apps_visible_api_products:
       type: sequence
       sequence:
@@ -45,6 +47,8 @@ apigee_edge_teams.team_app_settings:
       label: 'How to refer to a Team App on the UI (plural)'
     cache_expiration:
       type: integer
+    cache_insert_chunk_size:
+      type: apigee_edge.cache_insert_chunk_size
     required_base_fields:
       type: sequence
       sequence:
diff --git a/modules/apigee_edge_teams/src/Entity/Storage/TeamAppStorage.php b/modules/apigee_edge_teams/src/Entity/Storage/TeamAppStorage.php
@@ -23,6 +23,7 @@
 use Drupal\Component\Datetime\TimeInterface;
 use Drupal\Core\Cache\CacheBackendInterface;
 use Drupal\Core\Cache\MemoryCache\MemoryCacheInterface;
+use Drupal\Core\Config\ConfigFactoryInterface;
 use Drupal\Core\Entity\EntityTypeInterface;
 use Drupal\apigee_edge\Entity\AppInterface;
 use Drupal\apigee_edge\Entity\Controller\AppControllerInterface;
@@ -72,11 +73,21 @@ class TeamAppStorage extends AppStorage implements TeamAppStorageInterface {
    *   The app controller service.
    * @param \Drupal\apigee_edge\Entity\Controller\OrganizationControllerInterface $org_controller
    *   The organization controller service.
+   * @param \Drupal\Core\Config\ConfigFactoryInterface $config
+   *   Configuration factory.
    */
-  public function __construct(EntityTypeInterface $entity_type, CacheBackendInterface $cache_backend, MemoryCacheInterface $memory_cache, TimeInterface $system_time, TeamAppControllerFactoryInterface $team_app_controller_factory, AppControllerInterface $app_controller, OrganizationControllerInterface $org_controller) {
+  public function __construct(EntityTypeInterface $entity_type, CacheBackendInterface $cache_backend, MemoryCacheInterface $memory_cache, TimeInterface $system_time, TeamAppControllerFactoryInterface $team_app_controller_factory, AppControllerInterface $app_controller, OrganizationControllerInterface $org_controller, protected ?ConfigFactoryInterface $config = NULL) {
     parent::__construct($entity_type, $cache_backend, $memory_cache, $system_time, $app_controller);
     $this->teamAppControllerFactory = $team_app_controller_factory;
     $this->orgController = $org_controller;
+    if ($config === NULL) {
+      @trigger_error('Calling ' . __METHOD__ . ' without the $config is deprecated in apigee_edge:3.0.12 and it will be required in apigee_edge:3.1.0. See https://github.com/apigee/apigee-edge-drupal/pull/1155.', E_USER_DEPRECATED);
+      $config = \Drupal::configFactory();
+    }
+
+    $config = $config->get('apigee_edge_teams.team_app_settings');
+    $this->cacheExpiration = $config->get('cache_expiration');
+    $this->cacheInsertChunkSize = $config->get('cache_insert_chunk_size') ?? static::DEFAULT_PERSISTENT_CACHE_INSERT_CHUNK_SIZE;
   }
 
   /**
@@ -90,7 +101,8 @@ public static function createInstance(ContainerInterface $container, EntityTypeI
       $container->get('datetime.time'),
       $container->get('apigee_edge_teams.controller.team_app_controller_factory'),
       $container->get('apigee_edge.controller.app'),
-      $container->get('apigee_edge.controller.organization')
+      $container->get('apigee_edge.controller.organization'),
+      $container->get('config.factory'),
     );
   }
 
diff --git a/modules/apigee_edge_teams/src/Entity/Storage/TeamStorage.php b/modules/apigee_edge_teams/src/Entity/Storage/TeamStorage.php
@@ -66,7 +66,7 @@ class TeamStorage extends AttributesAwareFieldableEdgeEntityStorageBase implemen
   private $logger;
 
   /**
-   * Constructs an DeveloperStorage instance.
+   * Constructs an TeamStorage instance.
    *
    * @param \Drupal\Core\Entity\EntityTypeInterface $entity_type
    *   The entity type definition.
@@ -88,7 +88,9 @@ class TeamStorage extends AttributesAwareFieldableEdgeEntityStorageBase implemen
   public function __construct(EntityTypeInterface $entity_type, CacheBackendInterface $cache_backend, MemoryCacheInterface $memory_cache, TimeInterface $system_time, TeamControllerInterface $team_controller, EntityTypeManagerInterface $entity_type_manager, ConfigFactoryInterface $config, LoggerInterface $logger) {
     parent::__construct($entity_type, $cache_backend, $memory_cache, $system_time);
     $this->teamController = $team_controller;
-    $this->cacheExpiration = $config->get('apigee_edge_teams.team_settings')->get('cache_expiration');
+    $config = $config->get('apigee_edge_teams.team_settings');
+    $this->cacheExpiration = $config->get('cache_expiration');
+    $this->cacheInsertChunkSize = $config->get('cache_insert_chunk_size') ?? static::DEFAULT_PERSISTENT_CACHE_INSERT_CHUNK_SIZE;
     $this->entityTypeManager = $entity_type_manager;
     $this->logger = $logger;
   }
diff --git a/src/Entity/Storage/ApiProductStorage.php b/src/Entity/Storage/ApiProductStorage.php
@@ -61,7 +61,9 @@ class ApiProductStorage extends EdgeEntityStorageBase implements ApiProductStora
    */
   public function __construct(EntityTypeInterface $entity_type, CacheBackendInterface $cache_backend, MemoryCacheInterface $memory_cache, TimeInterface $system_time, ApiProductControllerInterface $api_product_controller, ConfigFactoryInterface $config) {
     parent::__construct($entity_type, $cache_backend, $memory_cache, $system_time);
-    $this->cacheExpiration = $config->get('apigee_edge.api_product_settings')->get('cache_expiration');
+    $config = $config->get('apigee_edge.api_product_settings');
+    $this->cacheExpiration = $config->get('cache_expiration');
+    $this->cacheInsertChunkSize = $config->get('cache_insert_chunk_size') ?? static::DEFAULT_PERSISTENT_CACHE_INSERT_CHUNK_SIZE;
     $this->apiProductController = $api_product_controller;
   }
 
diff --git a/src/Entity/Storage/AppStorage.php b/src/Entity/Storage/AppStorage.php
@@ -240,17 +240,25 @@ public function resetCache(?array $ids = NULL) {
   protected function setPersistentCache(array $entities) {
     parent::setPersistentCache($entities);
 
-    if (!$this->entityType->isPersistentlyCacheable()) {
+    if ($this->cacheExpiration === 0 || !$this->entityType->isPersistentlyCacheable()) {
       return;
     }
 
-    /** @var \Drupal\apigee_edge\Entity\AppInterface $entity */
-    foreach ($entities as $entity) {
-      // Create an additional cache entry for each app that stores the app id
-      // for each developerId or company (team) name + app name combinations.
-      // Thanks for this we can run queries faster that tries to an load app
-      // by using these two properties instead of the app id.
-      $this->cacheBackend->set($this->buildCacheIdForAppName($entity->getAppOwner(), $entity->getName()), $entity->getAppId(), $this->getPersistentCacheExpiration(), $this->getPersistentCacheTagsForAppName($entity));
+    // Create secondary cache entries to enable fast lookups by
+    // developer/company + app name. This allows querying apps without knowing
+    // the app ID, improving performance for searches based on owner and
+    // application name combinations.
+    while (!empty($entities)) {
+      $cache_items = [];
+      foreach (array_splice($entities, 0, $this->cacheInsertChunkSize) as $entity) {
+        $cache_items[$this->buildCacheIdForAppName($entity->getAppOwner(), $entity->getName())] = [
+          'data' => $entity->getAppId(),
+          'expire' => $this->getPersistentCacheExpiration(),
+          'tags' => $this->getPersistentCacheTagsForAppName($entity),
+        ];
+      }
+
+      $this->cacheBackend->setMultiple($cache_items);
     }
   }
 
diff --git a/src/Entity/Storage/DeveloperAppStorage.php b/src/Entity/Storage/DeveloperAppStorage.php
@@ -78,7 +78,9 @@ class DeveloperAppStorage extends AppStorage implements DeveloperAppStorageInter
   public function __construct(EntityTypeInterface $entity_type, CacheBackendInterface $cache_backend, MemoryCacheInterface $memory_cache, TimeInterface $system_time, DeveloperAppControllerFactoryInterface $developer_app_controller_factory, AppControllerInterface $app_controller, ConfigFactoryInterface $config, EmailValidatorInterface $email_validator, OrganizationControllerInterface $org_controller) {
     parent::__construct($entity_type, $cache_backend, $memory_cache, $system_time, $app_controller);
     $this->appEntityController = new DeveloperAppEdgeEntityControllerProxy($developer_app_controller_factory, $app_controller, $org_controller);
-    $this->cacheExpiration = $config->get('apigee_edge.developer_app_settings')->get('cache_expiration');
+    $config = $config->get('apigee_edge.developer_app_settings');
+    $this->cacheExpiration = $config->get('cache_expiration');
+    $this->cacheInsertChunkSize = $config->get('cache_insert_chunk_size') ?? static::DEFAULT_PERSISTENT_CACHE_INSERT_CHUNK_SIZE;
     $this->emailValidator = $email_validator;
   }
 
diff --git a/src/Entity/Storage/DeveloperStorage.php b/src/Entity/Storage/DeveloperStorage.php
@@ -75,7 +75,9 @@ class DeveloperStorage extends EdgeEntityStorageBase implements DeveloperStorage
    */
   public function __construct(EntityTypeInterface $entity_type, CacheBackendInterface $cache_backend, MemoryCacheInterface $memory_cache, TimeInterface $system_time, DeveloperControllerInterface $developer_controller, ConfigFactoryInterface $config, DeveloperCompaniesCacheInterface $developer_companies_cache) {
     parent::__construct($entity_type, $cache_backend, $memory_cache, $system_time);
-    $this->cacheExpiration = $config->get('apigee_edge.developer_settings')->get('cache_expiration');
+    $config = $config->get('apigee_edge.developer_settings');
+    $this->cacheExpiration = $config->get('cache_expiration');
+    $this->cacheInsertChunkSize = $config->get('cache_insert_chunk_size') ?? static::DEFAULT_PERSISTENT_CACHE_INSERT_CHUNK_SIZE;
     $this->developerController = $developer_controller;
     $this->developerCompanies = $developer_companies_cache;
   }
@@ -232,16 +234,24 @@ private function sanitizeCacheTags(string $id, array $cache_tags): array {
   protected function setPersistentCache(array $entities) {
     parent::setPersistentCache($entities);
 
-    if (!$this->entityType->isPersistentlyCacheable()) {
+    if ($this->cacheExpiration === 0 || !$this->entityType->isPersistentlyCacheable()) {
       return;
     }
 
     // Create a separate cache entry that uses developer id in the cache id
     // instead of the email address. This way we can load a developer from
     // cache by using both ids.
-    foreach ($entities as $entity) {
-      /** @var \Drupal\apigee_edge\Entity\Developer $entity */
-      $this->cacheBackend->set($this->buildCacheId($entity->getDeveloperId()), $entity, $this->getPersistentCacheExpiration(), $this->getPersistentCacheTags($entity));
+    while (!empty($entities)) {
+      $cache_items = [];
+      foreach (array_splice($entities, 0, $this->cacheInsertChunkSize) as $id => $entity) {
+        $cache_items[$this->buildCacheId($entity->getDeveloperId())] = [
+          'data' => $entity,
+          'expire' => $this->getPersistentCacheExpiration(),
+          'tags' => $this->getPersistentCacheTags($entity),
+        ];
+      }
+
+      $this->cacheBackend->setMultiple($cache_items);
     }
   }
 
diff --git a/src/Entity/Storage/EdgeEntityStorageBase.php b/src/Entity/Storage/EdgeEntityStorageBase.php
@@ -50,6 +50,13 @@ abstract class EdgeEntityStorageBase extends DrupalEntityStorageBase implements
    */
   public const SAVED_UNKNOWN = 0;
 
+  /**
+   * The default cache insert chunk size to the persistent cache.
+   *
+   * @var int
+   */
+  protected const DEFAULT_PERSISTENT_CACHE_INSERT_CHUNK_SIZE = 100;
+
   /**
    * Cache backend.
    *
@@ -67,6 +74,13 @@ abstract class EdgeEntityStorageBase extends DrupalEntityStorageBase implements
    */
   protected $cacheExpiration = CacheBackendInterface::CACHE_PERMANENT;
 
+  /**
+   * The cache insert chunk size to the persistent cache.
+   *
+   * @var int
+   */
+  protected int $cacheInsertChunkSize = self::DEFAULT_PERSISTENT_CACHE_INSERT_CHUNK_SIZE;
+
   /**
    * The system time.
    *
@@ -389,12 +403,21 @@ protected function getFromPersistentCache(?array &$ids = NULL) {
    *   Entities to store in the cache.
    */
   protected function setPersistentCache(array $entities) {
-    if (!$this->entityType->isPersistentlyCacheable()) {
+    if ($this->cacheExpiration === 0 || !$this->entityType->isPersistentlyCacheable()) {
       return;
     }
 
-    foreach ($entities as $id => $entity) {
-      $this->cacheBackend->set($this->buildCacheId($id), $entity, $this->getPersistentCacheExpiration(), $this->getPersistentCacheTags($entity));
+    while (!empty($entities)) {
+      $cache_items = [];
+      foreach (array_splice($entities, 0, $this->cacheInsertChunkSize) as $id => $entity) {
+        $cache_items[$this->buildCacheId($id)] = [
+          'data' => $entity,
+          'expire' => $this->getPersistentCacheExpiration(),
+          'tags' => $this->getPersistentCacheTags($entity),
+        ];
+      }
+
+      $this->cacheBackend->setMultiple($cache_items);
     }
   }
 
diff --git a/src/Form/EdgeEntityCacheConfigFormBase.php b/src/Form/EdgeEntityCacheConfigFormBase.php
@@ -47,6 +47,14 @@ public function buildForm(array $form, FormStateInterface $form_state) {
       '#min' => -1,
       '#required' => TRUE,
     ];
+    $form['cache']['cache_insert_chunk_size'] = [
+      '#type' => 'number',
+      '#title' => $this->t('Cache insert chunk size'),
+      '#description' => $this->t('Number of entities to process in each batch when storing to persistent cache.'),
+      '#default_value' => $config->get('cache_insert_chunk_size'),
+      '#min' => 1,
+      '#required' => TRUE,
+    ];
     $form['cache']['actions'] = [
       '#type' => 'actions',
     ];
@@ -65,6 +73,7 @@ public function buildForm(array $form, FormStateInterface $form_state) {
   public function submitForm(array &$form, FormStateInterface $form_state) {
     $this->config($this->getConfigNameWithCacheSettings())
       ->set('cache_expiration', $form_state->getValue('cache_expiration'))
+      ->set('cache_insert_chunk_size', $form_state->getValue('cache_insert_chunk_size'))
       ->save();
     parent::submitForm($form, $form_state);
   }
