Add data provider for indexes

src/Config/Index.php

@@ -5,6 +5,7 @@
 namespace Setono\SyliusMeilisearchPlugin\Config;
 
 use Psr\Container\ContainerInterface;
+use Setono\SyliusMeilisearchPlugin\DataProvider\DataProviderInterface;
 use Setono\SyliusMeilisearchPlugin\Document\Document;
 use Setono\SyliusMeilisearchPlugin\Indexer\IndexerInterface;
 use Setono\SyliusMeilisearchPlugin\Model\IndexableInterface;
@@ -53,15 +54,6 @@ public function __construct(
         }
     }
 
-    /**
-     * @psalm-suppress MixedInferredReturnType
-     */
-    public function indexer(): IndexerInterface
-    {
-        /** @psalm-suppress MixedReturnStatement */
-        return $this->locator->get(IndexerInterface::class);
-    }
-
     /**
      * @param class-string|object $entity
      */
@@ -80,6 +72,24 @@ public function hasEntity(string|object $entity): bool
         return false;
     }
 
+    /**
+     * @psalm-suppress MixedInferredReturnType
+     */
+    public function indexer(): IndexerInterface
+    {
+        /** @psalm-suppress MixedReturnStatement */
+        return $this->locator->get(IndexerInterface::class);
+    }
+
+    /**
+     * @psalm-suppress MixedInferredReturnType
+     */
+    public function dataProvider(): DataProviderInterface
+    {
+        /** @psalm-suppress MixedReturnStatement */
+        return $this->locator->get(DataProviderInterface::class);
+    }
+
     public function __toString(): string
     {
         return $this->name;

src/DataProvider/DataProviderInterface.php

@@ -0,0 +1,27 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Setono\SyliusMeilisearchPlugin\DataProvider;
+
+use Setono\SyliusMeilisearchPlugin\Config\Index;
+use Setono\SyliusMeilisearchPlugin\Model\IndexableInterface;
+
+interface DataProviderInterface
+{
+    /**
+     * Returns ids of objects of the given entity
+     *
+     * @param class-string<IndexableInterface> $entity
+     *
+     * @return iterable<array-key, string|int>
+     */
+    public function getIds(string $entity, Index $index): iterable;
+
+    /**
+     * @param class-string<IndexableInterface> $entity
+     *
+     * @return iterable<array-key, IndexableInterface>
+     */
+    public function getObjects(string $entity, Index $index): iterable;
+}

src/DataProvider/DefaultDataProvider.php

@@ -0,0 +1,57 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Setono\SyliusMeilisearchPlugin\DataProvider;
+
+use Doctrine\Persistence\ManagerRegistry;
+use DoctrineBatchUtils\BatchProcessing\SelectBatchIteratorAggregate;
+use Psr\EventDispatcher\EventDispatcherInterface;
+use Setono\Doctrine\ORMTrait;
+use Setono\SyliusMeilisearchPlugin\Config\Index;
+use Setono\SyliusMeilisearchPlugin\Event\DataProviderEvent;
+use Setono\SyliusMeilisearchPlugin\Model\IndexableInterface;
+
+final class DefaultDataProvider implements DataProviderInterface
+{
+    use ORMTrait;
+
+    public function __construct(
+        ManagerRegistry $managerRegistry,
+        private readonly EventDispatcherInterface $eventDispatcher,
+    ) {
+        $this->managerRegistry = $managerRegistry;
+    }
+
+    public function getIds(string $entity, Index $index): \Generator
+    {
+        yield from $this->createGenerator($entity, $index, true);
+    }
+
+    public function getObjects(string $entity, Index $index): \Generator
+    {
+        yield from $this->createGenerator($entity, $index);
+    }
+
+    /**
+     * @psalm-suppress MixedReturnTypeCoercion
+     *
+     * @param class-string<IndexableInterface> $entity
+     *
+     * @return ($onlyId is true ? SelectBatchIteratorAggregate<array-key, string|int> : SelectBatchIteratorAggregate<array-key, IndexableInterface>)
+     */
+    private function createGenerator(string $entity, Index $index, bool $onlyId = false): SelectBatchIteratorAggregate
+    {
+        $qb = $this
+            ->getManager($entity)
+            ->createQueryBuilder()
+            ->select($onlyId ? 'o.id' : 'o')
+            ->from($entity, 'o')
+        ;
+
+        $this->eventDispatcher->dispatch(new DataProviderEvent($entity, $index, $qb));
+
+        /** @psalm-suppress MixedReturnTypeCoercion */
+        return SelectBatchIteratorAggregate::fromQuery($qb->getQuery(), 100);
+    }
+}

src/DependencyInjection/Configuration.php

@@ -4,6 +4,7 @@
 
 namespace Setono\SyliusMeilisearchPlugin\DependencyInjection;
 
+use Setono\SyliusMeilisearchPlugin\DataProvider\DefaultDataProvider;
 use Setono\SyliusMeilisearchPlugin\Document\Product;
 use Setono\SyliusMeilisearchPlugin\Form\Type\SynonymType;
 use Setono\SyliusMeilisearchPlugin\Indexer\DefaultIndexer;
@@ -36,20 +37,26 @@ public function getConfigTreeBuilder(): TreeBuilder
                     ->beforeNormalization()->castToArray()->end()
                     ->defaultValue([])
                     ->arrayPrototype()
+                        ->addDefaultsIfNotSet()
                         ->children()
                             ->scalarNode('document')
                                 ->info(sprintf('The fully qualified class name for the document that maps to the index. If you are creating a product index, a good starting point is the %s', Product::class))
                                 ->cannotBeEmpty()
                                 ->isRequired()
                             ->end()
-                            ->scalarNode('indexer')
-                                ->info(sprintf('You can set a custom indexer here. If you do not set one, the default indexer will be used. The default indexer is %s', DefaultIndexer::class))
-                                ->defaultNull()
-                            ->end()
                             ->arrayNode('entities')
                                 ->info('The Doctrine entities that make up this index. Examples could be "App\Entity\Product\Product", "App\Entity\Taxonomy\Taxon", etc.')
                                 ->scalarPrototype()->end()
                             ->end()
+                            ->scalarNode('data_provider')
+                                ->info('You can set a custom data provider here. If you do not set one, the default data provider will be used.')
+                                ->defaultValue(DefaultDataProvider::class)
+                            ->end()
+                            ->scalarNode('indexer')
+                                ->info(sprintf('You can set a custom indexer here. If you do not set one, the default indexer will be used. The default indexer is %s', DefaultIndexer::class))
+                                ->cannotBeEmpty()
+                                ->defaultNull()
+                            ->end()
                             ->scalarNode('prefix')
                                 ->defaultNull()
                                 ->info('If you want to prepend a string to the index name, you can set it here. This can be useful in a development setup where each developer has their own prefix. Notice that the environment is already prefixed by default, so you do not have to prefix that.')

src/DependencyInjection/SetonoSyliusMeilisearchExtension.php

@@ -7,6 +7,7 @@
 use Meilisearch\Client;
 use Setono\SyliusMeilisearchPlugin\Config\Index;
 use Setono\SyliusMeilisearchPlugin\DataMapper\DataMapperInterface;
+use Setono\SyliusMeilisearchPlugin\DataProvider\DataProviderInterface;
 use Setono\SyliusMeilisearchPlugin\Document\Document;
 use Setono\SyliusMeilisearchPlugin\Filter\Doctrine\FilterInterface as DoctrineFilterInterface;
 use Setono\SyliusMeilisearchPlugin\Filter\Object\FilterInterface as ObjectFilterInterface;
@@ -34,7 +35,7 @@ public function load(array $configs, ContainerBuilder $container): void
          * @psalm-suppress PossiblyNullArgument
          *
          * @var array{
-         *      indexes: array<string, array{document: class-string<Document>, indexer: string|null, entities: list<class-string>, prefix: string|null}>,
+         *      indexes: array<string, array{document: class-string<Document>, entities: list<class-string>, data_provider: class-string, indexer: class-string|null, prefix: string|null}>,
          *      server: array{ host: string, master_key: string },
          *      search: array{ enabled: bool, path: string, index: string, hits_per_page: integer },
          *      resources: array,
@@ -159,7 +160,7 @@ public function prepend(ContainerBuilder $container): void
     }
 
     /**
-     *  @param array<string, array{document: class-string<Document>, indexer: string|null, entities: list<class-string>, prefix: string|null}> $config
+     *  @param array<string, array{document: class-string<Document>, entities: list<class-string>, data_provider: class-string, indexer: class-string|null, prefix: string|null}> $config
      */
     private static function registerIndexesConfiguration(array $config, ContainerBuilder $container): void
     {
@@ -178,7 +179,10 @@ private static function registerIndexesConfiguration(array $config, ContainerBui
                 $indexName,
                 $index['document'],
                 $index['entities'],
-                ServiceLocatorTagPass::register($container, [IndexerInterface::class => new Reference($indexerServiceId)]),
+                ServiceLocatorTagPass::register($container, [
+                    DataProviderInterface::class => new Reference($index['data_provider']),
+                    IndexerInterface::class => new Reference($indexerServiceId),
+                ]),
                 $index['prefix'],
             ]));
 

src/Event/DataProviderEvent.php

@@ -0,0 +1,20 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Setono\SyliusMeilisearchPlugin\Event;
+
+use Doctrine\ORM\QueryBuilder;
+use Setono\SyliusMeilisearchPlugin\Config\Index;
+use Setono\SyliusMeilisearchPlugin\Model\IndexableInterface;
+
+final class DataProviderEvent
+{
+    public function __construct(
+        /** @var class-string<IndexableInterface> $entity */
+        public readonly string $entity,
+        public readonly Index $index,
+        public readonly QueryBuilder $qb,
+    ) {
+    }
+}

src/Indexer/DefaultIndexer.php

@@ -5,18 +5,17 @@
 namespace Setono\SyliusMeilisearchPlugin\Indexer;
 
 use Doctrine\Persistence\ManagerRegistry;
-use DoctrineBatchUtils\BatchProcessing\SelectBatchIteratorAggregate;
 use Meilisearch\Client;
 use Psr\EventDispatcher\EventDispatcherInterface;
 use Setono\Doctrine\ORMTrait;
 use Setono\SyliusMeilisearchPlugin\Config\Index;
 use Setono\SyliusMeilisearchPlugin\DataMapper\DataMapperInterface;
 use Setono\SyliusMeilisearchPlugin\Document\Document;
-use Setono\SyliusMeilisearchPlugin\Event\EntityBasedQueryBuilderForIndexingCreatedEvent;
 use Setono\SyliusMeilisearchPlugin\Filter\Object\FilterInterface as ObjectFilterInterface;
-use Setono\SyliusMeilisearchPlugin\Model\IndexableInterface;
+use Setono\SyliusMeilisearchPlugin\Message\Command\IndexEntities;
 use Setono\SyliusMeilisearchPlugin\Provider\IndexScope\IndexScopeProviderInterface;
 use Setono\SyliusMeilisearchPlugin\Resolver\IndexName\IndexNameResolverInterface;
+use Symfony\Component\Messenger\MessageBusInterface;
 use Symfony\Component\Serializer\Normalizer\NormalizerInterface;
 use Webmozart\Assert\Assert;
 
@@ -37,14 +36,22 @@ public function __construct(
         protected readonly Client $client,
         protected readonly ObjectFilterInterface $objectFilter,
         protected readonly EventDispatcherInterface $eventDispatcher,
+        protected readonly MessageBusInterface $commandBus,
     ) {
         $this->managerRegistry = $managerRegistry;
     }
 
     public function index(): void
     {
         foreach ($this->index->entities as $entity) {
-            $this->indexEntityClass($entity);
+            /** @var IndexBuffer<string|int> $buffer */
+            $buffer = new IndexBuffer(100, fn (array $ids) => $this->commandBus->dispatch(IndexEntities::fromIds($entity, $ids)));
+
+            foreach ($this->index->dataProvider()->getIds($entity, $this->index) as $id) {
+                $buffer->push($id);
+            }
+
+            $buffer->flush();
         }
     }
 
@@ -61,7 +68,9 @@ public function indexEntities(array $entities): void
                 $document = new $this->index->document();
                 $this->dataMapper->map($entity, $document, $indexScope);
 
-                $this->objectFilter->filter($entity, $document, $indexScope);
+                if (!$this->objectFilter->filter($entity, $document, $indexScope)) {
+                    continue;
+                }
 
                 $documents[] = $this->normalize($document);
             }
@@ -83,28 +92,6 @@ public function removeEntities(array $entities): void
         }
     }
 
-    /**
-     * @param class-string<IndexableInterface> $entity
-     */
-    protected function indexEntityClass(string $entity): void
-    {
-        $qb = $this
-            ->getManager($entity)
-            ->createQueryBuilder()
-            ->select('o')
-            ->from($entity, 'o')
-        ;
-
-        $this->eventDispatcher->dispatch(new EntityBasedQueryBuilderForIndexingCreatedEvent($entity, $qb));
-
-        /** @var SelectBatchIteratorAggregate<array-key, IndexableInterface> $objects */
-        $objects = SelectBatchIteratorAggregate::fromQuery($qb->getQuery(), 100);
-
-        foreach ($objects as $object) {
-            $this->indexEntity($object);
-        }
-    }
-
     // todo move this to a service
     protected function normalize(Document $document): array
     {

src/Indexer/IndexBuffer.php

@@ -0,0 +1,47 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Setono\SyliusMeilisearchPlugin\Indexer;
+
+/**
+ * @internal
+ *
+ * @template T
+ */
+final class IndexBuffer
+{
+    private int $count = 0;
+
+    /** @var list<T> */
+    private array $buffer = [];
+
+    /**
+     * @param \Closure(list<T>):void $callback
+     */
+    public function __construct(private readonly int $bufferSize, private readonly \Closure $callback)
+    {
+    }
+
+    /**
+     * @param T $item
+     */
+    public function push(mixed $item): void
+    {
+        $this->buffer[] = $item;
+        ++$this->count;
+
+        if ($this->count >= $this->bufferSize) {
+            $this->flush();
+        }
+    }
+
+    public function flush(): void
+    {
+        if ($this->count > 0) {
+            ($this->callback)($this->buffer);
+            $this->buffer = [];
+            $this->count = 0;
+        }
+    }
+}