Merge pull request #5 from dpfaffenbauer/generator

use generators to index elements, makes initial loading and memory usage much better
dachcom-digital · Jul 13, 2021 · 0d15db3 · 0d15db3
2 parents 0cf79d4 + eb747d6
commit 0d15db3
Show file tree

Hide file tree

Showing 6 changed files with 66 additions and 16 deletions.
diff --git a/UPGRADE.md b/UPGRADE.md
@@ -1,5 +1,7 @@
 # Upgrade Notes
-***
+
+#### Update from Version 1.0.1 to Version 1.0.2
+- **[IMPROVEMENT]**: Fix missing Resource Proxy Option Resolver [@dpfaffenbauer](https://github.com/dachcom-digital/pimcore-dynamic-search-data-provider-trinity/pull/5)
 
 #### Update from Version 1.0.0 to Version 1.0.1
 - **[BUGFIX]**: Fix missing Resource Proxy Option Resolver [#10](https://github.com/dachcom-digital/pimcore-dynamic-search-data-provider-trinity/issues/10)

diff --git a/src/DsTrinityDataBundle/Service/Builder/AssetListBuilder.php b/src/DsTrinityDataBundle/Service/Builder/AssetListBuilder.php
@@ -33,11 +33,23 @@ public function __construct(Connection $db, EventDispatcherInterface $eventDispa
     /**
      * {@inheritdoc}
      */
-    public function buildByList(array $options): array
+    public function buildByList(array $options): \Generator
     {
         $list = $this->getList($options);
 
-        return $list->getAssets();
+        $c = 0;
+        foreach ($list->loadIdList() as $id) {
+            if ($asset = Asset::getById($id)) {
+                yield $asset;
+            }
+
+            // call the garbage collector if memory consumption is > 100MB
+            if (memory_get_usage() > 100000000 && ($c % 300 === 0)) {
+                \Pimcore::collectGarbage();
+            }
+
+            $c++;
+        }
     }
 
     /**

diff --git a/src/DsTrinityDataBundle/Service/Builder/DataBuilderInterface.php b/src/DsTrinityDataBundle/Service/Builder/DataBuilderInterface.php
@@ -9,9 +9,9 @@ interface DataBuilderInterface
     /**
      * @param array $options
      *
-     * @return array
+     * @return \Generator
      */
-    public function buildByList(array $options): array;
+    public function buildByList(array $options): \Generator;
 
     /**
      * @param int   $id

diff --git a/src/DsTrinityDataBundle/Service/Builder/DocumentListBuilder.php b/src/DsTrinityDataBundle/Service/Builder/DocumentListBuilder.php
@@ -33,11 +33,23 @@ public function __construct(Connection $db, EventDispatcherInterface $eventDispa
     /**
      * {@inheritdoc}
      */
-    public function buildByList(array $options): array
+    public function buildByList(array $options): \Generator
     {
         $list = $this->getList($options);
 
-        return $list->getDocuments();
+        $c = 0;
+        foreach ($list->loadIdList() as $id) {
+            if ($doc = Document::getById($id)) {
+                yield $doc;
+            }
+
+            // call the garbage collector if memory consumption is > 100MB
+            if (memory_get_usage() > 100000000 && ($c % 300 === 0)) {
+                \Pimcore::collectGarbage();
+            }
+
+            $c++;
+        }
     }
 
     /**

diff --git a/src/DsTrinityDataBundle/Service/Builder/ObjectListBuilder.php b/src/DsTrinityDataBundle/Service/Builder/ObjectListBuilder.php
@@ -33,11 +33,23 @@ public function __construct(Connection $db, EventDispatcherInterface $eventDispa
     /**
      * {@inheritdoc}
      */
-    public function buildByList(array $options): array
+    public function buildByList(array $options): \Generator
     {
         $list = $this->getList($options);
 
-        return $list->getObjects();
+        $c = 0;
+        foreach ($list->loadIdList() as $id) {
+            if ($object = DataObject::getById($id)) {
+                yield $object;
+            }
+
+            // call the garbage collector if memory consumption is > 100MB
+            if (memory_get_usage() > 100000000 && ($c % 300 === 0)) {
+                \Pimcore::collectGarbage();
+            }
+
+            $c++;
+        }
     }
 
     /**

diff --git a/src/DsTrinityDataBundle/Service/DataProviderService.php b/src/DsTrinityDataBundle/Service/DataProviderService.php
@@ -253,7 +253,9 @@ protected function fetchByTypeAndId(string $type, string $providerBehaviour, $id
 
         $element = $builder->buildById((int) $id);
 
-        $this->dispatchData([$element], $providerBehaviour, $resourceMeta);
+        if ($element instanceof ElementInterface) {
+            $this->dispatchElement($element, $providerBehaviour, $resourceMeta);
+        }
     }
 
     /**
@@ -266,20 +268,30 @@ protected function log($level, $message)
     }
 
     /**
-     * @param array                      $elements
+     * @param \Generator                 $elements
      * @param string                     $providerBehaviour
      * @param ResourceMetaInterface|null $resourceMeta
      */
-    protected function dispatchData(array $elements, string $providerBehaviour, ?ResourceMetaInterface $resourceMeta = null)
+    protected function dispatchData(\Generator $elements, string $providerBehaviour, ?ResourceMetaInterface $resourceMeta = null)
     {
         foreach ($elements as $element) {
-            $newDataEvent = new NewDataEvent($this->contextDispatchType, $this->contextName, $element, $providerBehaviour, $resourceMeta);
-            $this->eventDispatcher->dispatch($newDataEvent, DynamicSearchEvents::NEW_DATA_AVAILABLE);
-
-            $this->dispatchProcessControlSignal();
+            $this->dispatchElement($element, $providerBehaviour, $resourceMeta);
         }
     }
 
+    /**
+     * @param ElementInterface           $element
+     * @param string                     $providerBehaviour
+     * @param ResourceMetaInterface|null $resourceMeta
+     */
+    protected function dispatchElement(ElementInterface $element, string $providerBehaviour, ?ResourceMetaInterface $resourceMeta = null)
+    {
+        $newDataEvent = new NewDataEvent($this->contextDispatchType, $this->contextName, $element, $providerBehaviour, $resourceMeta);
+        $this->eventDispatcher->dispatch($newDataEvent, DynamicSearchEvents::NEW_DATA_AVAILABLE);
+
+        $this->dispatchProcessControlSignal();
+    }
+
     protected function addSignalListener()
     {
         if (php_sapi_name() !== 'cli') {