[6.x] Improve search indexing performance

config/search.php

@@ -79,4 +79,31 @@
  'fields' => ['title'],
  ],
 
+ /*
+ |--------------------------------------------------------------------------
+ | Indexing Queue
+ |--------------------------------------------------------------------------
+ |
+ | Here you may configure the queue name and connection used when indexing
+ | documents.
+ |
+ */
+
+ 'queue' => env('STATAMIC_SEARCH_QUEUE'),
+
+ 'queue_connection' => env('STATAMIC_SEARCH_QUEUE_CONNECTION'),
+
+ /*
+ |--------------------------------------------------------------------------
+ | Chunk Size
+ |--------------------------------------------------------------------------
+ |
+ | Here you can configure the chunk size used when indexing documents.
+ | The higher you make it, the more memory it will use, but the quicker
+ | the indexing process will be.
+ |
+ */
+
+ 'chunk_size' => 100,
+
 ];

src/Search/Algolia/Index.php

@@ -39,7 +39,7 @@ public function search($query)
  return (new Query($this))->query($query);
  }
 
-protected function insertDocuments(Documents $documents)
+public function insertDocuments(Documents $documents)
  {
  $documents = $documents->map(function ($item, $id) {
  $item['objectID'] = $id;

src/Search/Comb/Index.php

@@ -99,7 +99,7 @@ public function delete($document)
  $this->save($data);
  }
 
-protected function insertDocuments(Documents $documents)
+public function insertDocuments(Documents $documents)
  {
  try {
  $data = $this->data();

src/Search/Index.php

@@ -4,7 +4,6 @@
 
 use Closure;
 use Statamic\Contracts\Search\Searchable;
-use Statamic\Support\Arr;
 use Statamic\Support\Str;
 
 abstract class Index
@@ -20,7 +19,7 @@ abstract public function delete($document);
 
  abstract public function exists();
 
- abstract protected function insertDocuments(Documents $documents);
+ abstract public function insertDocuments(Documents $documents);
 
  abstract protected function deleteIndex();
 
@@ -84,20 +83,27 @@ public function ensureExists()
 
  public function insert($document)
  {
- return $this->insertMultiple(Arr::wrap($document));
+ return $this->insertMultiple(collect($document));
  }
 
  public function insertMultiple($documents)
  {
- $documents = (new Documents($documents))->mapWithKeys(function (Searchable $item) {
- return [$item->getSearchReference() => $this->searchables()->fields($item)];
- });
-
- $this->insertDocuments($documents);
+ $documents
+ ->chunk(config('statamic.search.chunk_size'))
+ ->each(fn ($documents) => InsertMultipleJob::dispatch(
+ name: Str::beforeLast($this->name, '_'),
+ locale: $this->locale,
+ documents: $documents
+ ));
 
  return $this;
  }
 
+ public function fields(Searchable $searchable)
+ {
+ return $this->searchables()->fields($searchable);
+ }
+
  public function shouldIndex($searchable)
  {
  return $this->searchables()->contains($searchable);

src/Search/InsertMultipleJob.php

@@ -0,0 +1,52 @@
+<?php
+
+namespace Statamic\Search;
+
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Queue\Queueable;
+use Illuminate\Support\Collection;
+use Illuminate\Support\LazyCollection;
+use Statamic\Contracts\Search\Searchable;
+use Statamic\Facades\Search;
+use Statamic\Search\Searchables\Providers;
+use Statamic\Support\Str;
+
+class InsertMultipleJob implements ShouldQueue
+{
+ use Queueable;
+
+ /**
+ * Create a new job instance.
+ */
+ public function __construct(
+ public string $name,
+ public ?string $locale,
+ public Collection|LazyCollection $documents,
+ ) {
+ $this->onConnection($connection = config('statamic.search.queue_connection', config('queue.default')));
+ $this->onQueue(config('statamic.search.queue', config("queue.connections.{$connection}.queue")));
+ }
+
+ /**
+ * Execute the job.
+ */
+ public function handle(): void
+ {
+ $providers = app(Providers::class);
+ $index = Search::index($this->name, $this->locale);
+
+ $documents = $this->documents
+ ->groupBy(fn ($document) => explode('::', $document)[0])
+ ->flatMap(function ($documents, $prefix) use ($providers) {
+ return $providers
+ ->getByPrefix($prefix)
+ ->find($documents->map(fn ($reference) => Str::after($reference, '::'))->all())
+ ->all();
+ })
+ ->mapWithKeys(function (Searchable $item) use ($index) {
+ return [$item->getSearchReference() => $index->fields($item)];
+ });
+
+ $index->insertDocuments(new Documents($documents));
+ }
+}

src/Search/Null/NullIndex.php

@@ -22,7 +22,7 @@ public function exists()
  return true;
  }
 
-protected function insertDocuments(Documents $documents)
+public function insertDocuments(Documents $documents)
  {
  //
  }

src/Search/Search.php

@@ -51,7 +51,7 @@ public function updateWithinIndexes(Searchable $searchable)
  $exists = $index->exists();
 
  if ($shouldIndex && $exists) {
- $index->insert($searchable);
+ $index->insert($searchable->getSearchReference());
  } elseif ($shouldIndex && ! $exists) {
  $index->update();
  } elseif ($exists) {

src/Search/Searchables/Assets.php

@@ -26,7 +26,9 @@ public function provide(): Collection
  : AssetCollection::make($this->keys)
  ->flatMap(fn ($key) => Asset::whereContainer($key));
 
- return $assets->filter($this->filter())->values();
+ // TODO: query scope support?
+
+ return $assets->filter($this->filter())->values()->map->reference();
  }
 
  public function contains($searchable): bool

src/Search/Searchables/Entries.php

@@ -31,7 +31,19 @@ public function provide(): Collection|LazyCollection
  $query->where('site', $site);
  }
 
- return $query->lazy(100)->filter($this->filter())->values();
+ $this->applyQueryScope($query);
+
+ if ($this->hasFilter()) {
+ return $query
+ ->lazy(config('statamic.search.chunk_size'))
+ ->filter($this->filter())
+ ->values()
+ ->map->reference();
+ }
+
+ $query->where('status', 'published');
+
+ return $query->pluck('reference');
  }
 
  public function contains($searchable): bool
@@ -48,7 +60,17 @@ public function contains($searchable): bool
  return false;
  }
 
- return $this->filter()($searchable);
+ if ($this->hasFilter()) {
+ return $this->filter()($searchable);
+ }
+
+ $query = Entry::query()
+ ->where('status', 'published')
+ ->where('id', $searchable->id());
+
+ $this->applyQueryScope($query);
+
+ return $query->exists();
  }
 
  public function find(array $ids): Collection

src/Search/Searchables/Provider.php

@@ -2,9 +2,11 @@
 
 namespace Statamic\Search\Searchables;
 
+use Statamic\Facades\Scope;
 use Statamic\Facades\Search;
 use Statamic\Search\Index;
 use Statamic\Search\ProvidesSearchables;
+use Statamic\Support\Arr;
 
 abstract class Provider implements ProvidesSearchables
 {
@@ -55,6 +57,20 @@ protected function usesWildcard()
  return in_array('*', $this->keys);
  }
 
+ protected function applyQueryScope($query)
+ {
+ if (! $scope = $this->index->config()['query_scope'] ?? null) {
+ return;
+ }
+
+ Scope::find($scope)->apply($query, []);
+ }
+
+ protected function hasFilter()
+ {
+ return Arr::has($this->index->config(), 'filter');
+ }
+
  protected function filter()
  {
  $filter = $this->index->config()['filter'] ?? null;

src/Search/Searchables/Terms.php

@@ -32,7 +32,17 @@ public function provide(): Collection|LazyCollection
  $query->where('site', $site);
  }
 
- return $query->lazy(100)->filter($this->filter())->values();
+ $this->applyQueryScope($query);
+
+ if ($this->hasFilter()) {
+ return $query
+ ->lazy(config('statamic.search.chunk_size'))
+ ->filter($this->filter())
+ ->values()
+ ->map->reference();
+ }
+
+ return $query->pluck('reference');
  }
 
  public function contains($searchable): bool
@@ -49,7 +59,15 @@ public function contains($searchable): bool
  return false;
  }
 
- return $this->filter()($searchable);
+ if ($this->hasFilter()) {
+ return $this->filter()($searchable);
+ }
+
+ $query = Term::query()->where('reference', $searchable->reference());
+
+ $this->applyQueryScope($query);
+
+ return $query->exists();
  }
 
  public function find(array $refs): Collection

src/Search/Searchables/Users.php

@@ -3,6 +3,7 @@
 namespace Statamic\Search\Searchables;
 
 use Illuminate\Support\Collection;
+use Illuminate\Support\LazyCollection;
 use Statamic\Contracts\Auth\User as UserContract;
 use Statamic\Facades\User;
 
@@ -18,9 +19,21 @@ public static function referencePrefix(): string
  return 'user';
  }
 
- public function provide(): Collection
+ public function provide(): Collection|LazyCollection
  {
- return User::all()->filter($this->filter())->values();
+ $query = User::query();
+
+ $this->applyQueryScope($query);
+
+ if ($this->hasFilter()) {
+ return $query
+ ->lazy(config('statamic.search.chunk_size'))
+ ->filter($this->filter())
+ ->values()
+ ->map->reference();
+ }
+
+ return $query->pluck('reference');
  }
 
  public function contains($searchable): bool
@@ -29,7 +42,15 @@ public function contains($searchable): bool
  return false;
  }
 
- return $this->filter()($searchable);
+ if ($this->hasFilter()) {
+ return $this->filter()($searchable);
+ }
+
+ $query = User::query()->where('id', $searchable->id());
+
+ $this->applyQueryScope($query);
+
+ return $query->exists();
  }
 
  public function find(array $ids): Collection

tests/Search/SearchTest.php

@@ -19,6 +19,7 @@ public function it_updates_indexes($updateMock)
  {
  $index = Mockery::mock(Index::class);
  $item = Mockery::mock(Searchable::class);
+ $item->shouldReceive('getSearchReference')->andReturn('a');
 
  $updateMock($index, $item);
 
@@ -37,7 +38,7 @@ public static function saveProvider()
  function ($mock, $entry) {
  $mock->shouldReceive('shouldIndex')->with($entry)->andReturnTrue();
  $mock->shouldReceive('exists')->andReturnTrue();
- $mock->shouldReceive('insert')->once()->with($entry);
+ $mock->shouldReceive('insert')->once()->with($entry->getSearchReference());
  },
  ],
 

config/search.php

@@ -79,4 +79,31 @@
  'fields' => ['title'],
  ],
 
+ /*
+ |--------------------------------------------------------------------------
+ | Indexing Queue
+ |--------------------------------------------------------------------------
+ |
+ | Here you may configure the queue name and connection used when indexing
+ | documents.
+ |
+ */
+
+ 'queue' => env('STATAMIC_SEARCH_QUEUE'),
+
+ 'queue_connection' => env('STATAMIC_SEARCH_QUEUE_CONNECTION'),
+
+ /*
+ |--------------------------------------------------------------------------
+ | Chunk Size
+ |--------------------------------------------------------------------------
+ |
+ | Here you can configure the chunk size used when indexing documents.
+ | The higher you make it, the more memory it will use, but the quicker
+ | the indexing process will be.
+ |
+ */
+
+ 'chunk_size' => 100,
+
 ];

src/Search/Algolia/Index.php

@@ -39,7 +39,7 @@ public function search($query)
  return (new Query($this))->query($query);
  }
 
-protected function insertDocuments(Documents $documents)
+public function insertDocuments(Documents $documents)
  {
  $documents = $documents->map(function ($item, $id) {
  $item['objectID'] = $id;

src/Search/Comb/Index.php

@@ -99,7 +99,7 @@ public function delete($document)
  $this->save($data);
  }
 
-protected function insertDocuments(Documents $documents)
+public function insertDocuments(Documents $documents)
  {
  try {
  $data = $this->data();

src/Search/Index.php

@@ -4,7 +4,6 @@
 
 use Closure;
 use Statamic\Contracts\Search\Searchable;
-use Statamic\Support\Arr;
 use Statamic\Support\Str;
 
 abstract class Index
@@ -20,7 +19,7 @@ abstract public function delete($document);
 
  abstract public function exists();
 
- abstract protected function insertDocuments(Documents $documents);
+ abstract public function insertDocuments(Documents $documents);
 
  abstract protected function deleteIndex();
 
@@ -84,20 +83,27 @@ public function ensureExists()
 
  public function insert($document)
  {
- return $this->insertMultiple(Arr::wrap($document));
+ return $this->insertMultiple(collect($document));
  }
 
  public function insertMultiple($documents)
  {
- $documents = (new Documents($documents))->mapWithKeys(function (Searchable $item) {
- return [$item->getSearchReference() => $this->searchables()->fields($item)];
- });
-
- $this->insertDocuments($documents);
+ $documents
+ ->chunk(config('statamic.search.chunk_size'))
+ ->each(fn ($documents) => InsertMultipleJob::dispatch(
+ name: Str::beforeLast($this->name, '_'),
+ locale: $this->locale,
+ documents: $documents
+ ));
 
  return $this;
  }
 
+ public function fields(Searchable $searchable)
+ {
+ return $this->searchables()->fields($searchable);
+ }
+
  public function shouldIndex($searchable)
  {
  return $this->searchables()->contains($searchable);

src/Search/InsertMultipleJob.php

@@ -0,0 +1,52 @@
+<?php
+
+namespace Statamic\Search;
+
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Queue\Queueable;
+use Illuminate\Support\Collection;
+use Illuminate\Support\LazyCollection;
+use Statamic\Contracts\Search\Searchable;
+use Statamic\Facades\Search;
+use Statamic\Search\Searchables\Providers;
+use Statamic\Support\Str;
+
+class InsertMultipleJob implements ShouldQueue
+{
+ use Queueable;
+
+ /**
+ * Create a new job instance.
+ */
+ public function __construct(
+ public string $name,
+ public ?string $locale,
+ public Collection|LazyCollection $documents,
+ ) {
+ $this->onConnection($connection = config('statamic.search.queue_connection', config('queue.default')));
+ $this->onQueue(config('statamic.search.queue', config("queue.connections.{$connection}.queue")));
+ }
+
+ /**
+ * Execute the job.
+ */
+ public function handle(): void
+ {
+ $providers = app(Providers::class);
+ $index = Search::index($this->name, $this->locale);
+
+ $documents = $this->documents
+ ->groupBy(fn ($document) => explode('::', $document)[0])
+ ->flatMap(function ($documents, $prefix) use ($providers) {
+ return $providers
+ ->getByPrefix($prefix)
+ ->find($documents->map(fn ($reference) => Str::after($reference, '::'))->all())
+ ->all();
+ })
+ ->mapWithKeys(function (Searchable $item) use ($index) {
+ return [$item->getSearchReference() => $index->fields($item)];
+ });
+
+ $index->insertDocuments(new Documents($documents));
+ }
+}

src/Search/Null/NullIndex.php

@@ -22,7 +22,7 @@ public function exists()
  return true;
  }
 
-protected function insertDocuments(Documents $documents)
+public function insertDocuments(Documents $documents)
  {
  //
  }

src/Search/Search.php

@@ -51,7 +51,7 @@ public function updateWithinIndexes(Searchable $searchable)
  $exists = $index->exists();
 
  if ($shouldIndex && $exists) {
- $index->insert($searchable);
+ $index->insert($searchable->getSearchReference());
  } elseif ($shouldIndex && ! $exists) {
  $index->update();
  } elseif ($exists) {

src/Search/Searchables/Assets.php

@@ -26,7 +26,9 @@ public function provide(): Collection
  : AssetCollection::make($this->keys)
  ->flatMap(fn ($key) => Asset::whereContainer($key));
 
- return $assets->filter($this->filter())->values();
+ // TODO: query scope support?
+
+ return $assets->filter($this->filter())->values()->map->reference();
  }
 
  public function contains($searchable): bool

src/Search/Searchables/Entries.php

@@ -31,7 +31,19 @@ public function provide(): Collection|LazyCollection
  $query->where('site', $site);
  }
 
- return $query->lazy(100)->filter($this->filter())->values();
+ $this->applyQueryScope($query);
+
+ if ($this->hasFilter()) {
+ return $query
+ ->lazy(config('statamic.search.chunk_size'))
+ ->filter($this->filter())
+ ->values()
+ ->map->reference();
+ }
+
+ $query->where('status', 'published');
+
+ return $query->pluck('reference');
  }
 
  public function contains($searchable): bool
@@ -48,7 +60,17 @@ public function contains($searchable): bool
  return false;
  }
 
- return $this->filter()($searchable);
+ if ($this->hasFilter()) {
+ return $this->filter()($searchable);
+ }
+
+ $query = Entry::query()
+ ->where('status', 'published')
+ ->where('id', $searchable->id());
+
+ $this->applyQueryScope($query);
+
+ return $query->exists();
  }
 
  public function find(array $ids): Collection

src/Search/Searchables/Provider.php

@@ -2,9 +2,11 @@
 
 namespace Statamic\Search\Searchables;
 
+use Statamic\Facades\Scope;
 use Statamic\Facades\Search;
 use Statamic\Search\Index;
 use Statamic\Search\ProvidesSearchables;
+use Statamic\Support\Arr;
 
 abstract class Provider implements ProvidesSearchables
 {
@@ -55,6 +57,20 @@ protected function usesWildcard()
  return in_array('*', $this->keys);
  }
 
+ protected function applyQueryScope($query)
+ {
+ if (! $scope = $this->index->config()['query_scope'] ?? null) {
+ return;
+ }
+
+ Scope::find($scope)->apply($query, []);
+ }
+
+ protected function hasFilter()
+ {
+ return Arr::has($this->index->config(), 'filter');
+ }
+
  protected function filter()
  {
  $filter = $this->index->config()['filter'] ?? null;

src/Search/Searchables/Terms.php

@@ -32,7 +32,17 @@ public function provide(): Collection|LazyCollection
  $query->where('site', $site);
  }
 
- return $query->lazy(100)->filter($this->filter())->values();
+ $this->applyQueryScope($query);
+
+ if ($this->hasFilter()) {
+ return $query
+ ->lazy(config('statamic.search.chunk_size'))
+ ->filter($this->filter())
+ ->values()
+ ->map->reference();
+ }
+
+ return $query->pluck('reference');
  }
 
  public function contains($searchable): bool
@@ -49,7 +59,15 @@ public function contains($searchable): bool
  return false;
  }
 
- return $this->filter()($searchable);
+ if ($this->hasFilter()) {
+ return $this->filter()($searchable);
+ }
+
+ $query = Term::query()->where('reference', $searchable->reference());
+
+ $this->applyQueryScope($query);
+
+ return $query->exists();
  }
 
  public function find(array $refs): Collection

src/Search/Searchables/Users.php

@@ -3,6 +3,7 @@
 namespace Statamic\Search\Searchables;
 
 use Illuminate\Support\Collection;
+use Illuminate\Support\LazyCollection;
 use Statamic\Contracts\Auth\User as UserContract;
 use Statamic\Facades\User;
 
@@ -18,9 +19,21 @@ public static function referencePrefix(): string
  return 'user';
  }
 
- public function provide(): Collection
+ public function provide(): Collection|LazyCollection
  {
- return User::all()->filter($this->filter())->values();
+ $query = User::query();
+
+ $this->applyQueryScope($query);
+
+ if ($this->hasFilter()) {
+ return $query
+ ->lazy(config('statamic.search.chunk_size'))
+ ->filter($this->filter())
+ ->values()
+ ->map->reference();
+ }
+
+ return $query->pluck('reference');
  }
 
  public function contains($searchable): bool
@@ -29,7 +42,15 @@ public function contains($searchable): bool
  return false;
  }
 
- return $this->filter()($searchable);
+ if ($this->hasFilter()) {
+ return $this->filter()($searchable);
+ }
+
+ $query = User::query()->where('id', $searchable->id());
+
+ $this->applyQueryScope($query);
+
+ return $query->exists();
  }
 
  public function find(array $ids): Collection

tests/Search/SearchTest.php

@@ -19,6 +19,7 @@ public function it_updates_indexes($updateMock)
  {
  $index = Mockery::mock(Index::class);
  $item = Mockery::mock(Searchable::class);
+ $item->shouldReceive('getSearchReference')->andReturn('a');
 
  $updateMock($index, $item);
 
@@ -37,7 +38,7 @@ public static function saveProvider()
  function ($mock, $entry) {
  $mock->shouldReceive('shouldIndex')->with($entry)->andReturnTrue();
  $mock->shouldReceive('exists')->andReturnTrue();
- $mock->shouldReceive('insert')->once()->with($entry);
+ $mock->shouldReceive('insert')->once()->with($entry->getSearchReference());
  },
  ],
 

config/search.php

@@ -79,4 +79,31 @@
  'fields' => ['title'],
  ],
 
+ /*
+ |--------------------------------------------------------------------------
+ | Indexing Queue
+ |--------------------------------------------------------------------------
+ |
+ | Here you may configure the queue name and connection used when indexing
+ | documents.
+ |
+ */
+
+ 'queue' => env('STATAMIC_SEARCH_QUEUE'),
+
+ 'queue_connection' => env('STATAMIC_SEARCH_QUEUE_CONNECTION'),
+
+ /*
+ |--------------------------------------------------------------------------
+ | Chunk Size
+ |--------------------------------------------------------------------------
+ |
+ | Here you can configure the chunk size used when indexing documents.
+ | The higher you make it, the more memory it will use, but the quicker
+ | the indexing process will be.
+ |
+ */
+
+ 'chunk_size' => 100,
+
 ];

src/Search/Algolia/Index.php

@@ -39,7 +39,7 @@ public function search($query)
  return (new Query($this))->query($query);
  }
 
-protected function insertDocuments(Documents $documents)
+public function insertDocuments(Documents $documents)
  {
  $documents = $documents->map(function ($item, $id) {
  $item['objectID'] = $id;

src/Search/Comb/Index.php

@@ -99,7 +99,7 @@ public function delete($document)
  $this->save($data);
  }
 
-protected function insertDocuments(Documents $documents)
+public function insertDocuments(Documents $documents)
  {
  try {
  $data = $this->data();

src/Search/Index.php

@@ -4,7 +4,6 @@
 
 use Closure;
 use Statamic\Contracts\Search\Searchable;
-use Statamic\Support\Arr;
 use Statamic\Support\Str;
 
 abstract class Index
@@ -20,7 +19,7 @@ abstract public function delete($document);
 
  abstract public function exists();
 
- abstract protected function insertDocuments(Documents $documents);
+ abstract public function insertDocuments(Documents $documents);
 
  abstract protected function deleteIndex();
 
@@ -84,20 +83,27 @@ public function ensureExists()
 
  public function insert($document)
  {
- return $this->insertMultiple(Arr::wrap($document));
+ return $this->insertMultiple(collect($document));
  }
 
  public function insertMultiple($documents)
  {
- $documents = (new Documents($documents))->mapWithKeys(function (Searchable $item) {
- return [$item->getSearchReference() => $this->searchables()->fields($item)];
- });
-
- $this->insertDocuments($documents);
+ $documents
+ ->chunk(config('statamic.search.chunk_size'))
+ ->each(fn ($documents) => InsertMultipleJob::dispatch(
+ name: Str::beforeLast($this->name, '_'),
+ locale: $this->locale,
+ documents: $documents
+ ));
 
  return $this;
  }
 
+ public function fields(Searchable $searchable)
+ {
+ return $this->searchables()->fields($searchable);
+ }
+
  public function shouldIndex($searchable)
  {
  return $this->searchables()->contains($searchable);

src/Search/InsertMultipleJob.php

@@ -0,0 +1,52 @@
+<?php
+
+namespace Statamic\Search;
+
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Queue\Queueable;
+use Illuminate\Support\Collection;
+use Illuminate\Support\LazyCollection;
+use Statamic\Contracts\Search\Searchable;
+use Statamic\Facades\Search;
+use Statamic\Search\Searchables\Providers;
+use Statamic\Support\Str;
+
+class InsertMultipleJob implements ShouldQueue
+{
+ use Queueable;
+
+ /**
+ * Create a new job instance.
+ */
+ public function __construct(
+ public string $name,
+ public ?string $locale,
+ public Collection|LazyCollection $documents,
+ ) {
+ $this->onConnection($connection = config('statamic.search.queue_connection', config('queue.default')));
+ $this->onQueue(config('statamic.search.queue', config("queue.connections.{$connection}.queue")));
+ }
+
+ /**
+ * Execute the job.
+ */
+ public function handle(): void
+ {
+ $providers = app(Providers::class);
+ $index = Search::index($this->name, $this->locale);
+
+ $documents = $this->documents
+ ->groupBy(fn ($document) => explode('::', $document)[0])
+ ->flatMap(function ($documents, $prefix) use ($providers) {
+ return $providers
+ ->getByPrefix($prefix)
+ ->find($documents->map(fn ($reference) => Str::after($reference, '::'))->all())
+ ->all();
+ })
+ ->mapWithKeys(function (Searchable $item) use ($index) {
+ return [$item->getSearchReference() => $index->fields($item)];
+ });
+
+ $index->insertDocuments(new Documents($documents));
+ }
+}

src/Search/Null/NullIndex.php

@@ -22,7 +22,7 @@ public function exists()
  return true;
  }
 
-protected function insertDocuments(Documents $documents)
+public function insertDocuments(Documents $documents)
  {
  //
  }

src/Search/Search.php

@@ -51,7 +51,7 @@ public function updateWithinIndexes(Searchable $searchable)
  $exists = $index->exists();
 
  if ($shouldIndex && $exists) {
- $index->insert($searchable);
+ $index->insert($searchable->getSearchReference());
  } elseif ($shouldIndex && ! $exists) {
  $index->update();
  } elseif ($exists) {

src/Search/Searchables/Assets.php

@@ -26,7 +26,9 @@ public function provide(): Collection
  : AssetCollection::make($this->keys)
  ->flatMap(fn ($key) => Asset::whereContainer($key));
 
- return $assets->filter($this->filter())->values();
+ // TODO: query scope support?
+
+ return $assets->filter($this->filter())->values()->map->reference();
  }
 
  public function contains($searchable): bool

src/Search/Searchables/Entries.php

@@ -31,7 +31,19 @@ public function provide(): Collection|LazyCollection
  $query->where('site', $site);
  }
 
- return $query->lazy(100)->filter($this->filter())->values();
+ $this->applyQueryScope($query);
+
+ if ($this->hasFilter()) {
+ return $query
+ ->lazy(config('statamic.search.chunk_size'))
+ ->filter($this->filter())
+ ->values()
+ ->map->reference();
+ }
+
+ $query->where('status', 'published');
+
+ return $query->pluck('reference');
  }
 
  public function contains($searchable): bool
@@ -48,7 +60,17 @@ public function contains($searchable): bool
  return false;
  }
 
- return $this->filter()($searchable);
+ if ($this->hasFilter()) {
+ return $this->filter()($searchable);
+ }
+
+ $query = Entry::query()
+ ->where('status', 'published')
+ ->where('id', $searchable->id());
+
+ $this->applyQueryScope($query);
+
+ return $query->exists();
  }
 
  public function find(array $ids): Collection

src/Search/Searchables/Provider.php

@@ -2,9 +2,11 @@
 
 namespace Statamic\Search\Searchables;
 
+use Statamic\Facades\Scope;
 use Statamic\Facades\Search;
 use Statamic\Search\Index;
 use Statamic\Search\ProvidesSearchables;
+use Statamic\Support\Arr;
 
 abstract class Provider implements ProvidesSearchables
 {
@@ -55,6 +57,20 @@ protected function usesWildcard()
  return in_array('*', $this->keys);
  }
 
+ protected function applyQueryScope($query)
+ {
+ if (! $scope = $this->index->config()['query_scope'] ?? null) {
+ return;
+ }
+
+ Scope::find($scope)->apply($query, []);
+ }
+
+ protected function hasFilter()
+ {
+ return Arr::has($this->index->config(), 'filter');
+ }
+
  protected function filter()
  {
  $filter = $this->index->config()['filter'] ?? null;

src/Search/Searchables/Terms.php

@@ -32,7 +32,17 @@ public function provide(): Collection|LazyCollection
  $query->where('site', $site);
  }
 
- return $query->lazy(100)->filter($this->filter())->values();
+ $this->applyQueryScope($query);
+
+ if ($this->hasFilter()) {
+ return $query
+ ->lazy(config('statamic.search.chunk_size'))
+ ->filter($this->filter())
+ ->values()
+ ->map->reference();
+ }
+
+ return $query->pluck('reference');
  }
 
  public function contains($searchable): bool
@@ -49,7 +59,15 @@ public function contains($searchable): bool
  return false;
  }
 
- return $this->filter()($searchable);
+ if ($this->hasFilter()) {
+ return $this->filter()($searchable);
+ }
+
+ $query = Term::query()->where('reference', $searchable->reference());
+
+ $this->applyQueryScope($query);
+
+ return $query->exists();
  }
 
  public function find(array $refs): Collection

src/Search/Searchables/Users.php

@@ -3,6 +3,7 @@
 namespace Statamic\Search\Searchables;
 
 use Illuminate\Support\Collection;
+use Illuminate\Support\LazyCollection;
 use Statamic\Contracts\Auth\User as UserContract;
 use Statamic\Facades\User;
 
@@ -18,9 +19,21 @@ public static function referencePrefix(): string
  return 'user';
  }
 
- public function provide(): Collection
+ public function provide(): Collection|LazyCollection
  {
- return User::all()->filter($this->filter())->values();
+ $query = User::query();
+
+ $this->applyQueryScope($query);
+
+ if ($this->hasFilter()) {
+ return $query
+ ->lazy(config('statamic.search.chunk_size'))
+ ->filter($this->filter())
+ ->values()
+ ->map->reference();
+ }
+
+ return $query->pluck('reference');
  }
 
  public function contains($searchable): bool
@@ -29,7 +42,15 @@ public function contains($searchable): bool
  return false;
  }
 
- return $this->filter()($searchable);
+ if ($this->hasFilter()) {
+ return $this->filter()($searchable);
+ }
+
+ $query = User::query()->where('id', $searchable->id());
+
+ $this->applyQueryScope($query);
+
+ return $query->exists();
  }
 
  public function find(array $ids): Collection

tests/Search/SearchTest.php

@@ -19,6 +19,7 @@ public function it_updates_indexes($updateMock)
  {
  $index = Mockery::mock(Index::class);
  $item = Mockery::mock(Searchable::class);
+ $item->shouldReceive('getSearchReference')->andReturn('a');
 
  $updateMock($index, $item);
 
@@ -37,7 +38,7 @@ public static function saveProvider()
  function ($mock, $entry) {
  $mock->shouldReceive('shouldIndex')->with($entry)->andReturnTrue();
  $mock->shouldReceive('exists')->andReturnTrue();
- $mock->shouldReceive('insert')->once()->with($entry);
+ $mock->shouldReceive('insert')->once()->with($entry->getSearchReference());
  },
  ],
 

config/search.php

@@ -79,4 +79,31 @@
  'fields' => ['title'],
  ],
 
+ /*
+ |--------------------------------------------------------------------------
+ | Indexing Queue
+ |--------------------------------------------------------------------------
+ |
+ | Here you may configure the queue name and connection used when indexing
+ | documents.
+ |
+ */
+
+ 'queue' => env('STATAMIC_SEARCH_QUEUE'),
+
+ 'queue_connection' => env('STATAMIC_SEARCH_QUEUE_CONNECTION'),
+
+ /*
+ |--------------------------------------------------------------------------
+ | Chunk Size
+ |--------------------------------------------------------------------------
+ |
+ | Here you can configure the chunk size used when indexing documents.
+ | The higher you make it, the more memory it will use, but the quicker
+ | the indexing process will be.
+ |
+ */
+
+ 'chunk_size' => 100,
+
 ];

src/Search/Algolia/Index.php

@@ -39,7 +39,7 @@ public function search($query)
  return (new Query($this))->query($query);
  }
 
-protected function insertDocuments(Documents $documents)
+public function insertDocuments(Documents $documents)
  {
  $documents = $documents->map(function ($item, $id) {
  $item['objectID'] = $id;

src/Search/Comb/Index.php

@@ -99,7 +99,7 @@ public function delete($document)
  $this->save($data);
  }
 
-protected function insertDocuments(Documents $documents)
+public function insertDocuments(Documents $documents)
  {
  try {
  $data = $this->data();

src/Search/Index.php

@@ -4,7 +4,6 @@
 
 use Closure;
 use Statamic\Contracts\Search\Searchable;
-use Statamic\Support\Arr;
 use Statamic\Support\Str;
 
 abstract class Index
@@ -20,7 +19,7 @@ abstract public function delete($document);
 
  abstract public function exists();
 
- abstract protected function insertDocuments(Documents $documents);
+ abstract public function insertDocuments(Documents $documents);
 
  abstract protected function deleteIndex();
 
@@ -84,20 +83,27 @@ public function ensureExists()
 
  public function insert($document)
  {
- return $this->insertMultiple(Arr::wrap($document));
+ return $this->insertMultiple(collect($document));
  }
 
  public function insertMultiple($documents)
  {
- $documents = (new Documents($documents))->mapWithKeys(function (Searchable $item) {
- return [$item->getSearchReference() => $this->searchables()->fields($item)];
- });
-
- $this->insertDocuments($documents);
+ $documents
+ ->chunk(config('statamic.search.chunk_size'))
+ ->each(fn ($documents) => InsertMultipleJob::dispatch(
+ name: Str::beforeLast($this->name, '_'),
+ locale: $this->locale,
+ documents: $documents
+ ));
 
  return $this;
  }
 
+ public function fields(Searchable $searchable)
+ {
+ return $this->searchables()->fields($searchable);
+ }
+
  public function shouldIndex($searchable)
  {
  return $this->searchables()->contains($searchable);

src/Search/InsertMultipleJob.php

@@ -0,0 +1,52 @@
+<?php
+
+namespace Statamic\Search;
+
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Queue\Queueable;
+use Illuminate\Support\Collection;
+use Illuminate\Support\LazyCollection;
+use Statamic\Contracts\Search\Searchable;
+use Statamic\Facades\Search;
+use Statamic\Search\Searchables\Providers;
+use Statamic\Support\Str;
+
+class InsertMultipleJob implements ShouldQueue
+{
+ use Queueable;
+
+ /**
+ * Create a new job instance.
+ */
+ public function __construct(
+ public string $name,
+ public ?string $locale,
+ public Collection|LazyCollection $documents,
+ ) {
+ $this->onConnection($connection = config('statamic.search.queue_connection', config('queue.default')));
+ $this->onQueue(config('statamic.search.queue', config("queue.connections.{$connection}.queue")));
+ }
+
+ /**
+ * Execute the job.
+ */
+ public function handle(): void
+ {
+ $providers = app(Providers::class);
+ $index = Search::index($this->name, $this->locale);
+
+ $documents = $this->documents
+ ->groupBy(fn ($document) => explode('::', $document)[0])
+ ->flatMap(function ($documents, $prefix) use ($providers) {
+ return $providers
+ ->getByPrefix($prefix)
+ ->find($documents->map(fn ($reference) => Str::after($reference, '::'))->all())
+ ->all();
+ })
+ ->mapWithKeys(function (Searchable $item) use ($index) {
+ return [$item->getSearchReference() => $index->fields($item)];
+ });
+
+ $index->insertDocuments(new Documents($documents));
+ }
+}

src/Search/Null/NullIndex.php

@@ -22,7 +22,7 @@ public function exists()
  return true;
  }
 
-protected function insertDocuments(Documents $documents)
+public function insertDocuments(Documents $documents)
  {
  //
  }

src/Search/Search.php

@@ -51,7 +51,7 @@ public function updateWithinIndexes(Searchable $searchable)
  $exists = $index->exists();
 
  if ($shouldIndex && $exists) {
- $index->insert($searchable);
+ $index->insert($searchable->getSearchReference());
  } elseif ($shouldIndex && ! $exists) {
  $index->update();
  } elseif ($exists) {

src/Search/Searchables/Assets.php

@@ -26,7 +26,9 @@ public function provide(): Collection
  : AssetCollection::make($this->keys)
  ->flatMap(fn ($key) => Asset::whereContainer($key));
 
- return $assets->filter($this->filter())->values();
+ // TODO: query scope support?
+
+ return $assets->filter($this->filter())->values()->map->reference();
  }
 
  public function contains($searchable): bool

src/Search/Searchables/Entries.php

@@ -31,7 +31,19 @@ public function provide(): Collection|LazyCollection
  $query->where('site', $site);
  }
 
- return $query->lazy(100)->filter($this->filter())->values();
+ $this->applyQueryScope($query);
+
+ if ($this->hasFilter()) {
+ return $query
+ ->lazy(config('statamic.search.chunk_size'))
+ ->filter($this->filter())
+ ->values()
+ ->map->reference();
+ }
+
+ $query->where('status', 'published');
+
+ return $query->pluck('reference');
  }
 
  public function contains($searchable): bool
@@ -48,7 +60,17 @@ public function contains($searchable): bool
  return false;
  }
 
- return $this->filter()($searchable);
+ if ($this->hasFilter()) {
+ return $this->filter()($searchable);
+ }
+
+ $query = Entry::query()
+ ->where('status', 'published')
+ ->where('id', $searchable->id());
+
+ $this->applyQueryScope($query);
+
+ return $query->exists();
  }
 
  public function find(array $ids): Collection

src/Search/Searchables/Provider.php

@@ -2,9 +2,11 @@
 
 namespace Statamic\Search\Searchables;
 
+use Statamic\Facades\Scope;
 use Statamic\Facades\Search;
 use Statamic\Search\Index;
 use Statamic\Search\ProvidesSearchables;
+use Statamic\Support\Arr;
 
 abstract class Provider implements ProvidesSearchables
 {
@@ -55,6 +57,20 @@ protected function usesWildcard()
  return in_array('*', $this->keys);
  }
 
+ protected function applyQueryScope($query)
+ {
+ if (! $scope = $this->index->config()['query_scope'] ?? null) {
+ return;
+ }
+
+ Scope::find($scope)->apply($query, []);
+ }
+
+ protected function hasFilter()
+ {
+ return Arr::has($this->index->config(), 'filter');
+ }
+
  protected function filter()
  {
  $filter = $this->index->config()['filter'] ?? null;

src/Search/Searchables/Terms.php

@@ -32,7 +32,17 @@ public function provide(): Collection|LazyCollection
  $query->where('site', $site);
  }
 
- return $query->lazy(100)->filter($this->filter())->values();
+ $this->applyQueryScope($query);
+
+ if ($this->hasFilter()) {
+ return $query
+ ->lazy(config('statamic.search.chunk_size'))
+ ->filter($this->filter())
+ ->values()
+ ->map->reference();
+ }
+
+ return $query->pluck('reference');
  }
 
  public function contains($searchable): bool
@@ -49,7 +59,15 @@ public function contains($searchable): bool
  return false;
  }
 
- return $this->filter()($searchable);
+ if ($this->hasFilter()) {
+ return $this->filter()($searchable);
+ }
+
+ $query = Term::query()->where('reference', $searchable->reference());
+
+ $this->applyQueryScope($query);
+
+ return $query->exists();
  }
 
  public function find(array $refs): Collection

src/Search/Searchables/Users.php

@@ -3,6 +3,7 @@
 namespace Statamic\Search\Searchables;
 
 use Illuminate\Support\Collection;
+use Illuminate\Support\LazyCollection;
 use Statamic\Contracts\Auth\User as UserContract;
 use Statamic\Facades\User;
 
@@ -18,9 +19,21 @@ public static function referencePrefix(): string
  return 'user';
  }
 
- public function provide(): Collection
+ public function provide(): Collection|LazyCollection
  {
- return User::all()->filter($this->filter())->values();
+ $query = User::query();
+
+ $this->applyQueryScope($query);
+
+ if ($this->hasFilter()) {
+ return $query
+ ->lazy(config('statamic.search.chunk_size'))
+ ->filter($this->filter())
+ ->values()
+ ->map->reference();
+ }
+
+ return $query->pluck('reference');
  }
 
  public function contains($searchable): bool
@@ -29,7 +42,15 @@ public function contains($searchable): bool
  return false;
  }
 
- return $this->filter()($searchable);
+ if ($this->hasFilter()) {
+ return $this->filter()($searchable);
+ }
+
+ $query = User::query()->where('id', $searchable->id());
+
+ $this->applyQueryScope($query);
+
+ return $query->exists();
  }
 
  public function find(array $ids): Collection

tests/Search/SearchTest.php

@@ -19,6 +19,7 @@ public function it_updates_indexes($updateMock)
  {
  $index = Mockery::mock(Index::class);
  $item = Mockery::mock(Searchable::class);
+ $item->shouldReceive('getSearchReference')->andReturn('a');
 
  $updateMock($index, $item);
 
@@ -37,7 +38,7 @@ public static function saveProvider()
  function ($mock, $entry) {
  $mock->shouldReceive('shouldIndex')->with($entry)->andReturnTrue();
  $mock->shouldReceive('exists')->andReturnTrue();
- $mock->shouldReceive('insert')->once()->with($entry);
+ $mock->shouldReceive('insert')->once()->with($entry->getSearchReference());
  },
  ],
 

config/search.php

@@ -79,4 +79,31 @@
  'fields' => ['title'],
  ],
 
+ /*
+ |--------------------------------------------------------------------------
+ | Indexing Queue
+ |--------------------------------------------------------------------------
+ |
+ | Here you may configure the queue name and connection used when indexing
+ | documents.
+ |
+ */
+
+ 'queue' => env('STATAMIC_SEARCH_QUEUE'),
+
+ 'queue_connection' => env('STATAMIC_SEARCH_QUEUE_CONNECTION'),
+
+ /*
+ |--------------------------------------------------------------------------
+ | Chunk Size
+ |--------------------------------------------------------------------------
+ |
+ | Here you can configure the chunk size used when indexing documents.
+ | The higher you make it, the more memory it will use, but the quicker
+ | the indexing process will be.
+ |
+ */
+
+ 'chunk_size' => 100,
+
 ];

src/Search/Algolia/Index.php

@@ -39,7 +39,7 @@ public function search($query)
  return (new Query($this))->query($query);
  }
 
-protected function insertDocuments(Documents $documents)
+public function insertDocuments(Documents $documents)
  {
  $documents = $documents->map(function ($item, $id) {
  $item['objectID'] = $id;

src/Search/Comb/Index.php

@@ -99,7 +99,7 @@ public function delete($document)
  $this->save($data);
  }
 
-protected function insertDocuments(Documents $documents)
+public function insertDocuments(Documents $documents)
  {
  try {
  $data = $this->data();

src/Search/Index.php

@@ -4,7 +4,6 @@
 
 use Closure;
 use Statamic\Contracts\Search\Searchable;
-use Statamic\Support\Arr;
 use Statamic\Support\Str;
 
 abstract class Index
@@ -20,7 +19,7 @@ abstract public function delete($document);
 
  abstract public function exists();
 
- abstract protected function insertDocuments(Documents $documents);
+ abstract public function insertDocuments(Documents $documents);
 
  abstract protected function deleteIndex();
 
@@ -84,20 +83,27 @@ public function ensureExists()
 
  public function insert($document)
  {
- return $this->insertMultiple(Arr::wrap($document));
+ return $this->insertMultiple(collect($document));
  }
 
  public function insertMultiple($documents)
  {
- $documents = (new Documents($documents))->mapWithKeys(function (Searchable $item) {
- return [$item->getSearchReference() => $this->searchables()->fields($item)];
- });
-
- $this->insertDocuments($documents);
+ $documents
+ ->chunk(config('statamic.search.chunk_size'))
+ ->each(fn ($documents) => InsertMultipleJob::dispatch(
+ name: Str::beforeLast($this->name, '_'),
+ locale: $this->locale,
+ documents: $documents
+ ));
 
  return $this;
  }
 
+ public function fields(Searchable $searchable)
+ {
+ return $this->searchables()->fields($searchable);
+ }
+
  public function shouldIndex($searchable)
  {
  return $this->searchables()->contains($searchable);

src/Search/InsertMultipleJob.php

@@ -0,0 +1,52 @@
+<?php
+
+namespace Statamic\Search;
+
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Queue\Queueable;
+use Illuminate\Support\Collection;
+use Illuminate\Support\LazyCollection;
+use Statamic\Contracts\Search\Searchable;
+use Statamic\Facades\Search;
+use Statamic\Search\Searchables\Providers;
+use Statamic\Support\Str;
+
+class InsertMultipleJob implements ShouldQueue
+{
+ use Queueable;
+
+ /**
+ * Create a new job instance.
+ */
+ public function __construct(
+ public string $name,
+ public ?string $locale,
+ public Collection|LazyCollection $documents,
+ ) {
+ $this->onConnection($connection = config('statamic.search.queue_connection', config('queue.default')));
+ $this->onQueue(config('statamic.search.queue', config("queue.connections.{$connection}.queue")));
+ }
+
+ /**
+ * Execute the job.
+ */
+ public function handle(): void
+ {
+ $providers = app(Providers::class);
+ $index = Search::index($this->name, $this->locale);
+
+ $documents = $this->documents
+ ->groupBy(fn ($document) => explode('::', $document)[0])
+ ->flatMap(function ($documents, $prefix) use ($providers) {
+ return $providers
+ ->getByPrefix($prefix)
+ ->find($documents->map(fn ($reference) => Str::after($reference, '::'))->all())
+ ->all();
+ })
+ ->mapWithKeys(function (Searchable $item) use ($index) {
+ return [$item->getSearchReference() => $index->fields($item)];
+ });
+
+ $index->insertDocuments(new Documents($documents));
+ }
+}

src/Search/Null/NullIndex.php

@@ -22,7 +22,7 @@ public function exists()
  return true;
  }
 
-protected function insertDocuments(Documents $documents)
+public function insertDocuments(Documents $documents)
  {
  //
  }

src/Search/Search.php

@@ -51,7 +51,7 @@ public function updateWithinIndexes(Searchable $searchable)
  $exists = $index->exists();
 
  if ($shouldIndex && $exists) {
- $index->insert($searchable);
+ $index->insert($searchable->getSearchReference());
  } elseif ($shouldIndex && ! $exists) {
  $index->update();
  } elseif ($exists) {

src/Search/Searchables/Assets.php

@@ -26,7 +26,9 @@ public function provide(): Collection
  : AssetCollection::make($this->keys)
  ->flatMap(fn ($key) => Asset::whereContainer($key));
 
- return $assets->filter($this->filter())->values();
+ // TODO: query scope support?
+
+ return $assets->filter($this->filter())->values()->map->reference();
  }
 
  public function contains($searchable): bool

src/Search/Searchables/Entries.php

@@ -31,7 +31,19 @@ public function provide(): Collection|LazyCollection
  $query->where('site', $site);
  }
 
- return $query->lazy(100)->filter($this->filter())->values();
+ $this->applyQueryScope($query);
+
+ if ($this->hasFilter()) {
+ return $query
+ ->lazy(config('statamic.search.chunk_size'))
+ ->filter($this->filter())
+ ->values()
+ ->map->reference();
+ }
+
+ $query->where('status', 'published');
+
+ return $query->pluck('reference');
  }
 
  public function contains($searchable): bool
@@ -48,7 +60,17 @@ public function contains($searchable): bool
  return false;
  }
 
- return $this->filter()($searchable);
+ if ($this->hasFilter()) {
+ return $this->filter()($searchable);
+ }
+
+ $query = Entry::query()
+ ->where('status', 'published')
+ ->where('id', $searchable->id());
+
+ $this->applyQueryScope($query);
+
+ return $query->exists();
  }
 
  public function find(array $ids): Collection

src/Search/Searchables/Provider.php

@@ -2,9 +2,11 @@
 
 namespace Statamic\Search\Searchables;
 
+use Statamic\Facades\Scope;
 use Statamic\Facades\Search;
 use Statamic\Search\Index;
 use Statamic\Search\ProvidesSearchables;
+use Statamic\Support\Arr;
 
 abstract class Provider implements ProvidesSearchables
 {
@@ -55,6 +57,20 @@ protected function usesWildcard()
  return in_array('*', $this->keys);
  }
 
+ protected function applyQueryScope($query)
+ {
+ if (! $scope = $this->index->config()['query_scope'] ?? null) {
+ return;
+ }
+
+ Scope::find($scope)->apply($query, []);
+ }
+
+ protected function hasFilter()
+ {
+ return Arr::has($this->index->config(), 'filter');
+ }
+
  protected function filter()
  {
  $filter = $this->index->config()['filter'] ?? null;

src/Search/Searchables/Terms.php

@@ -32,7 +32,17 @@ public function provide(): Collection|LazyCollection
  $query->where('site', $site);
  }
 
- return $query->lazy(100)->filter($this->filter())->values();
+ $this->applyQueryScope($query);
+
+ if ($this->hasFilter()) {
+ return $query
+ ->lazy(config('statamic.search.chunk_size'))
+ ->filter($this->filter())
+ ->values()
+ ->map->reference();
+ }
+
+ return $query->pluck('reference');
  }
 
  public function contains($searchable): bool
@@ -49,7 +59,15 @@ public function contains($searchable): bool
  return false;
  }
 
- return $this->filter()($searchable);
+ if ($this->hasFilter()) {
+ return $this->filter()($searchable);
+ }
+
+ $query = Term::query()->where('reference', $searchable->reference());
+
+ $this->applyQueryScope($query);
+
+ return $query->exists();
  }
 
  public function find(array $refs): Collection

src/Search/Searchables/Users.php

@@ -3,6 +3,7 @@
 namespace Statamic\Search\Searchables;
 
 use Illuminate\Support\Collection;
+use Illuminate\Support\LazyCollection;
 use Statamic\Contracts\Auth\User as UserContract;
 use Statamic\Facades\User;
 
@@ -18,9 +19,21 @@ public static function referencePrefix(): string
  return 'user';
  }
 
- public function provide(): Collection
+ public function provide(): Collection|LazyCollection
  {
- return User::all()->filter($this->filter())->values();
+ $query = User::query();
+
+ $this->applyQueryScope($query);
+
+ if ($this->hasFilter()) {
+ return $query
+ ->lazy(config('statamic.search.chunk_size'))
+ ->filter($this->filter())
+ ->values()
+ ->map->reference();
+ }
+
+ return $query->pluck('reference');
  }
 
  public function contains($searchable): bool
@@ -29,7 +42,15 @@ public function contains($searchable): bool
  return false;
  }
 
- return $this->filter()($searchable);
+ if ($this->hasFilter()) {
+ return $this->filter()($searchable);
+ }
+
+ $query = User::query()->where('id', $searchable->id());
+
+ $this->applyQueryScope($query);
+
+ return $query->exists();
  }
 
  public function find(array $ids): Collection

tests/Search/SearchTest.php

@@ -19,6 +19,7 @@ public function it_updates_indexes($updateMock)
  {
  $index = Mockery::mock(Index::class);
  $item = Mockery::mock(Searchable::class);
+ $item->shouldReceive('getSearchReference')->andReturn('a');
 
  $updateMock($index, $item);
 
@@ -37,7 +38,7 @@ public static function saveProvider()
  function ($mock, $entry) {
  $mock->shouldReceive('shouldIndex')->with($entry)->andReturnTrue();
  $mock->shouldReceive('exists')->andReturnTrue();
- $mock->shouldReceive('insert')->once()->with($entry);
+ $mock->shouldReceive('insert')->once()->with($entry->getSearchReference());
  },
  ],
 

config/search.php

@@ -79,4 +79,31 @@
  'fields' => ['title'],
  ],
 
+ /*
+ |--------------------------------------------------------------------------
+ | Indexing Queue
+ |--------------------------------------------------------------------------
+ |
+ | Here you may configure the queue name and connection used when indexing
+ | documents.
+ |
+ */
+
+ 'queue' => env('STATAMIC_SEARCH_QUEUE'),
+
+ 'queue_connection' => env('STATAMIC_SEARCH_QUEUE_CONNECTION'),
+
+ /*
+ |--------------------------------------------------------------------------
+ | Chunk Size
+ |--------------------------------------------------------------------------
+ |
+ | Here you can configure the chunk size used when indexing documents.
+ | The higher you make it, the more memory it will use, but the quicker
+ | the indexing process will be.
+ |
+ */
+
+ 'chunk_size' => 100,
+
 ];

src/Search/Algolia/Index.php

@@ -39,7 +39,7 @@ public function search($query)
  return (new Query($this))->query($query);
  }
 
-protected function insertDocuments(Documents $documents)
+public function insertDocuments(Documents $documents)
  {
  $documents = $documents->map(function ($item, $id) {
  $item['objectID'] = $id;

src/Search/Comb/Index.php

@@ -99,7 +99,7 @@ public function delete($document)
  $this->save($data);
  }
 
-protected function insertDocuments(Documents $documents)
+public function insertDocuments(Documents $documents)
  {
  try {
  $data = $this->data();

src/Search/Index.php

@@ -4,7 +4,6 @@
 
 use Closure;
 use Statamic\Contracts\Search\Searchable;
-use Statamic\Support\Arr;
 use Statamic\Support\Str;
 
 abstract class Index
@@ -20,7 +19,7 @@ abstract public function delete($document);
 
  abstract public function exists();
 
- abstract protected function insertDocuments(Documents $documents);
+ abstract public function insertDocuments(Documents $documents);
 
  abstract protected function deleteIndex();
 
@@ -84,20 +83,27 @@ public function ensureExists()
 
  public function insert($document)
  {
- return $this->insertMultiple(Arr::wrap($document));
+ return $this->insertMultiple(collect($document));
  }
 
  public function insertMultiple($documents)
  {
- $documents = (new Documents($documents))->mapWithKeys(function (Searchable $item) {
- return [$item->getSearchReference() => $this->searchables()->fields($item)];
- });
-
- $this->insertDocuments($documents);
+ $documents
+ ->chunk(config('statamic.search.chunk_size'))
+ ->each(fn ($documents) => InsertMultipleJob::dispatch(
+ name: Str::beforeLast($this->name, '_'),
+ locale: $this->locale,
+ documents: $documents
+ ));
 
  return $this;
  }
 
+ public function fields(Searchable $searchable)
+ {
+ return $this->searchables()->fields($searchable);
+ }
+
  public function shouldIndex($searchable)
  {
  return $this->searchables()->contains($searchable);

src/Search/InsertMultipleJob.php

@@ -0,0 +1,52 @@
+<?php
+
+namespace Statamic\Search;
+
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Queue\Queueable;
+use Illuminate\Support\Collection;
+use Illuminate\Support\LazyCollection;
+use Statamic\Contracts\Search\Searchable;
+use Statamic\Facades\Search;
+use Statamic\Search\Searchables\Providers;
+use Statamic\Support\Str;
+
+class InsertMultipleJob implements ShouldQueue
+{
+ use Queueable;
+
+ /**
+ * Create a new job instance.
+ */
+ public function __construct(
+ public string $name,
+ public ?string $locale,
+ public Collection|LazyCollection $documents,
+ ) {
+ $this->onConnection($connection = config('statamic.search.queue_connection', config('queue.default')));
+ $this->onQueue(config('statamic.search.queue', config("queue.connections.{$connection}.queue")));
+ }
+
+ /**
+ * Execute the job.
+ */
+ public function handle(): void
+ {
+ $providers = app(Providers::class);
+ $index = Search::index($this->name, $this->locale);
+
+ $documents = $this->documents
+ ->groupBy(fn ($document) => explode('::', $document)[0])
+ ->flatMap(function ($documents, $prefix) use ($providers) {
+ return $providers
+ ->getByPrefix($prefix)
+ ->find($documents->map(fn ($reference) => Str::after($reference, '::'))->all())
+ ->all();
+ })
+ ->mapWithKeys(function (Searchable $item) use ($index) {
+ return [$item->getSearchReference() => $index->fields($item)];
+ });
+
+ $index->insertDocuments(new Documents($documents));
+ }
+}

src/Search/Null/NullIndex.php

@@ -22,7 +22,7 @@ public function exists()
  return true;
  }
 
-protected function insertDocuments(Documents $documents)
+public function insertDocuments(Documents $documents)
  {
  //
  }

src/Search/Search.php

@@ -51,7 +51,7 @@ public function updateWithinIndexes(Searchable $searchable)
  $exists = $index->exists();
 
  if ($shouldIndex && $exists) {
- $index->insert($searchable);
+ $index->insert($searchable->getSearchReference());
  } elseif ($shouldIndex && ! $exists) {
  $index->update();
  } elseif ($exists) {

src/Search/Searchables/Assets.php

@@ -26,7 +26,9 @@ public function provide(): Collection
  : AssetCollection::make($this->keys)
  ->flatMap(fn ($key) => Asset::whereContainer($key));
 
- return $assets->filter($this->filter())->values();
+ // TODO: query scope support?
+
+ return $assets->filter($this->filter())->values()->map->reference();
  }
 
  public function contains($searchable): bool

src/Search/Searchables/Entries.php

@@ -31,7 +31,19 @@ public function provide(): Collection|LazyCollection
  $query->where('site', $site);
  }
 
- return $query->lazy(100)->filter($this->filter())->values();
+ $this->applyQueryScope($query);
+
+ if ($this->hasFilter()) {
+ return $query
+ ->lazy(config('statamic.search.chunk_size'))
+ ->filter($this->filter())
+ ->values()
+ ->map->reference();
+ }
+
+ $query->where('status', 'published');
+
+ return $query->pluck('reference');
  }
 
  public function contains($searchable): bool
@@ -48,7 +60,17 @@ public function contains($searchable): bool
  return false;
  }
 
- return $this->filter()($searchable);
+ if ($this->hasFilter()) {
+ return $this->filter()($searchable);
+ }
+
+ $query = Entry::query()
+ ->where('status', 'published')
+ ->where('id', $searchable->id());
+
+ $this->applyQueryScope($query);
+
+ return $query->exists();
  }
 
  public function find(array $ids): Collection

src/Search/Searchables/Provider.php

@@ -2,9 +2,11 @@
 
 namespace Statamic\Search\Searchables;
 
+use Statamic\Facades\Scope;
 use Statamic\Facades\Search;
 use Statamic\Search\Index;
 use Statamic\Search\ProvidesSearchables;
+use Statamic\Support\Arr;
 
 abstract class Provider implements ProvidesSearchables
 {
@@ -55,6 +57,20 @@ protected function usesWildcard()
  return in_array('*', $this->keys);
  }
 
+ protected function applyQueryScope($query)
+ {
+ if (! $scope = $this->index->config()['query_scope'] ?? null) {
+ return;
+ }
+
+ Scope::find($scope)->apply($query, []);
+ }
+
+ protected function hasFilter()
+ {
+ return Arr::has($this->index->config(), 'filter');
+ }
+
  protected function filter()
  {
  $filter = $this->index->config()['filter'] ?? null;

src/Search/Searchables/Terms.php

@@ -32,7 +32,17 @@ public function provide(): Collection|LazyCollection
  $query->where('site', $site);
  }
 
- return $query->lazy(100)->filter($this->filter())->values();
+ $this->applyQueryScope($query);
+
+ if ($this->hasFilter()) {
+ return $query
+ ->lazy(config('statamic.search.chunk_size'))
+ ->filter($this->filter())
+ ->values()
+ ->map->reference();
+ }
+
+ return $query->pluck('reference');
  }
 
  public function contains($searchable): bool
@@ -49,7 +59,15 @@ public function contains($searchable): bool
  return false;
  }
 
- return $this->filter()($searchable);
+ if ($this->hasFilter()) {
+ return $this->filter()($searchable);
+ }
+
+ $query = Term::query()->where('reference', $searchable->reference());
+
+ $this->applyQueryScope($query);
+
+ return $query->exists();
  }
 
  public function find(array $refs): Collection

src/Search/Searchables/Users.php

@@ -3,6 +3,7 @@
 namespace Statamic\Search\Searchables;
 
 use Illuminate\Support\Collection;
+use Illuminate\Support\LazyCollection;
 use Statamic\Contracts\Auth\User as UserContract;
 use Statamic\Facades\User;
 
@@ -18,9 +19,21 @@ public static function referencePrefix(): string
  return 'user';
  }
 
- public function provide(): Collection
+ public function provide(): Collection|LazyCollection
  {
- return User::all()->filter($this->filter())->values();
+ $query = User::query();
+
+ $this->applyQueryScope($query);
+
+ if ($this->hasFilter()) {
+ return $query
+ ->lazy(config('statamic.search.chunk_size'))
+ ->filter($this->filter())
+ ->values()
+ ->map->reference();
+ }
+
+ return $query->pluck('reference');
  }
 
  public function contains($searchable): bool
@@ -29,7 +42,15 @@ public function contains($searchable): bool
  return false;
  }
 
- return $this->filter()($searchable);
+ if ($this->hasFilter()) {
+ return $this->filter()($searchable);
+ }
+
+ $query = User::query()->where('id', $searchable->id());
+
+ $this->applyQueryScope($query);
+
+ return $query->exists();
  }
 
  public function find(array $ids): Collection

tests/Search/SearchTest.php

@@ -19,6 +19,7 @@ public function it_updates_indexes($updateMock)
  {
  $index = Mockery::mock(Index::class);
  $item = Mockery::mock(Searchable::class);
+ $item->shouldReceive('getSearchReference')->andReturn('a');
 
  $updateMock($index, $item);
 
@@ -37,7 +38,7 @@ public static function saveProvider()
  function ($mock, $entry) {
  $mock->shouldReceive('shouldIndex')->with($entry)->andReturnTrue();
  $mock->shouldReceive('exists')->andReturnTrue();
- $mock->shouldReceive('insert')->once()->with($entry);
+ $mock->shouldReceive('insert')->once()->with($entry->getSearchReference());
  },
  ],
 

config/search.php

@@ -79,4 +79,31 @@
  'fields' => ['title'],
  ],
 
+ /*
+ |--------------------------------------------------------------------------
+ | Indexing Queue
+ |--------------------------------------------------------------------------
+ |
+ | Here you may configure the queue name and connection used when indexing
+ | documents.
+ |
+ */
+
+ 'queue' => env('STATAMIC_SEARCH_QUEUE'),
+
+ 'queue_connection' => env('STATAMIC_SEARCH_QUEUE_CONNECTION'),
+
+ /*
+ |--------------------------------------------------------------------------
+ | Chunk Size
+ |--------------------------------------------------------------------------
+ |
+ | Here you can configure the chunk size used when indexing documents.
+ | The higher you make it, the more memory it will use, but the quicker
+ | the indexing process will be.
+ |
+ */
+
+ 'chunk_size' => 100,
+
 ];

src/Search/Algolia/Index.php

@@ -39,7 +39,7 @@ public function search($query)
  return (new Query($this))->query($query);
  }
 
-protected function insertDocuments(Documents $documents)
+public function insertDocuments(Documents $documents)
  {
  $documents = $documents->map(function ($item, $id) {
  $item['objectID'] = $id;

src/Search/Comb/Index.php

@@ -99,7 +99,7 @@ public function delete($document)
  $this->save($data);
  }
 
-protected function insertDocuments(Documents $documents)
+public function insertDocuments(Documents $documents)
  {
  try {
  $data = $this->data();

src/Search/Index.php

@@ -4,7 +4,6 @@
 
 use Closure;
 use Statamic\Contracts\Search\Searchable;
-use Statamic\Support\Arr;
 use Statamic\Support\Str;
 
 abstract class Index
@@ -20,7 +19,7 @@ abstract public function delete($document);
 
  abstract public function exists();
 
- abstract protected function insertDocuments(Documents $documents);
+ abstract public function insertDocuments(Documents $documents);
 
  abstract protected function deleteIndex();
 
@@ -84,20 +83,27 @@ public function ensureExists()
 
  public function insert($document)
  {
- return $this->insertMultiple(Arr::wrap($document));
+ return $this->insertMultiple(collect($document));
  }
 
  public function insertMultiple($documents)
  {
- $documents = (new Documents($documents))->mapWithKeys(function (Searchable $item) {
- return [$item->getSearchReference() => $this->searchables()->fields($item)];
- });
-
- $this->insertDocuments($documents);
+ $documents
+ ->chunk(config('statamic.search.chunk_size'))
+ ->each(fn ($documents) => InsertMultipleJob::dispatch(
+ name: Str::beforeLast($this->name, '_'),
+ locale: $this->locale,
+ documents: $documents
+ ));
 
  return $this;
  }
 
+ public function fields(Searchable $searchable)
+ {
+ return $this->searchables()->fields($searchable);
+ }
+
  public function shouldIndex($searchable)
  {
  return $this->searchables()->contains($searchable);

src/Search/InsertMultipleJob.php

@@ -0,0 +1,52 @@
+<?php
+
+namespace Statamic\Search;
+
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Queue\Queueable;
+use Illuminate\Support\Collection;
+use Illuminate\Support\LazyCollection;
+use Statamic\Contracts\Search\Searchable;
+use Statamic\Facades\Search;
+use Statamic\Search\Searchables\Providers;
+use Statamic\Support\Str;
+
+class InsertMultipleJob implements ShouldQueue
+{
+ use Queueable;
+
+ /**
+ * Create a new job instance.
+ */
+ public function __construct(
+ public string $name,
+ public ?string $locale,
+ public Collection|LazyCollection $documents,
+ ) {
+ $this->onConnection($connection = config('statamic.search.queue_connection', config('queue.default')));
+ $this->onQueue(config('statamic.search.queue', config("queue.connections.{$connection}.queue")));
+ }
+
+ /**
+ * Execute the job.
+ */
+ public function handle(): void
+ {
+ $providers = app(Providers::class);
+ $index = Search::index($this->name, $this->locale);
+
+ $documents = $this->documents
+ ->groupBy(fn ($document) => explode('::', $document)[0])
+ ->flatMap(function ($documents, $prefix) use ($providers) {
+ return $providers
+ ->getByPrefix($prefix)
+ ->find($documents->map(fn ($reference) => Str::after($reference, '::'))->all())
+ ->all();
+ })
+ ->mapWithKeys(function (Searchable $item) use ($index) {
+ return [$item->getSearchReference() => $index->fields($item)];
+ });
+
+ $index->insertDocuments(new Documents($documents));
+ }
+}

src/Search/Null/NullIndex.php

@@ -22,7 +22,7 @@ public function exists()
  return true;
  }
 
-protected function insertDocuments(Documents $documents)
+public function insertDocuments(Documents $documents)
  {
  //
  }

src/Search/Search.php

@@ -51,7 +51,7 @@ public function updateWithinIndexes(Searchable $searchable)
  $exists = $index->exists();
 
  if ($shouldIndex && $exists) {
- $index->insert($searchable);
+ $index->insert($searchable->getSearchReference());
  } elseif ($shouldIndex && ! $exists) {
  $index->update();
  } elseif ($exists) {

src/Search/Searchables/Assets.php

@@ -26,7 +26,9 @@ public function provide(): Collection
  : AssetCollection::make($this->keys)
  ->flatMap(fn ($key) => Asset::whereContainer($key));
 
- return $assets->filter($this->filter())->values();
+ // TODO: query scope support?
+
+ return $assets->filter($this->filter())->values()->map->reference();
  }
 
  public function contains($searchable): bool

src/Search/Searchables/Entries.php

@@ -31,7 +31,19 @@ public function provide(): Collection|LazyCollection
  $query->where('site', $site);
  }
 
- return $query->lazy(100)->filter($this->filter())->values();
+ $this->applyQueryScope($query);
+
+ if ($this->hasFilter()) {
+ return $query
+ ->lazy(config('statamic.search.chunk_size'))
+ ->filter($this->filter())
+ ->values()
+ ->map->reference();
+ }
+
+ $query->where('status', 'published');
+
+ return $query->pluck('reference');
  }
 
  public function contains($searchable): bool
@@ -48,7 +60,17 @@ public function contains($searchable): bool
  return false;
  }
 
- return $this->filter()($searchable);
+ if ($this->hasFilter()) {
+ return $this->filter()($searchable);
+ }
+
+ $query = Entry::query()
+ ->where('status', 'published')
+ ->where('id', $searchable->id());
+
+ $this->applyQueryScope($query);
+
+ return $query->exists();
  }
 
  public function find(array $ids): Collection

src/Search/Searchables/Provider.php

@@ -2,9 +2,11 @@
 
 namespace Statamic\Search\Searchables;
 
+use Statamic\Facades\Scope;
 use Statamic\Facades\Search;
 use Statamic\Search\Index;
 use Statamic\Search\ProvidesSearchables;
+use Statamic\Support\Arr;
 
 abstract class Provider implements ProvidesSearchables
 {
@@ -55,6 +57,20 @@ protected function usesWildcard()
  return in_array('*', $this->keys);
  }
 
+ protected function applyQueryScope($query)
+ {
+ if (! $scope = $this->index->config()['query_scope'] ?? null) {
+ return;
+ }
+
+ Scope::find($scope)->apply($query, []);
+ }
+
+ protected function hasFilter()
+ {
+ return Arr::has($this->index->config(), 'filter');
+ }
+
  protected function filter()
  {
  $filter = $this->index->config()['filter'] ?? null;

src/Search/Searchables/Terms.php

@@ -32,7 +32,17 @@ public function provide(): Collection|LazyCollection
  $query->where('site', $site);
  }
 
- return $query->lazy(100)->filter($this->filter())->values();
+ $this->applyQueryScope($query);
+
+ if ($this->hasFilter()) {
+ return $query
+ ->lazy(config('statamic.search.chunk_size'))
+ ->filter($this->filter())
+ ->values()
+ ->map->reference();
+ }
+
+ return $query->pluck('reference');
  }
 
  public function contains($searchable): bool
@@ -49,7 +59,15 @@ public function contains($searchable): bool
  return false;
  }
 
- return $this->filter()($searchable);
+ if ($this->hasFilter()) {
+ return $this->filter()($searchable);
+ }
+
+ $query = Term::query()->where('reference', $searchable->reference());
+
+ $this->applyQueryScope($query);
+
+ return $query->exists();
  }
 
  public function find(array $refs): Collection

src/Search/Searchables/Users.php

@@ -3,6 +3,7 @@
 namespace Statamic\Search\Searchables;
 
 use Illuminate\Support\Collection;
+use Illuminate\Support\LazyCollection;
 use Statamic\Contracts\Auth\User as UserContract;
 use Statamic\Facades\User;
 
@@ -18,9 +19,21 @@ public static function referencePrefix(): string
  return 'user';
  }
 
- public function provide(): Collection
+ public function provide(): Collection|LazyCollection
  {
- return User::all()->filter($this->filter())->values();
+ $query = User::query();
+
+ $this->applyQueryScope($query);
+
+ if ($this->hasFilter()) {
+ return $query
+ ->lazy(config('statamic.search.chunk_size'))
+ ->filter($this->filter())
+ ->values()
+ ->map->reference();
+ }
+
+ return $query->pluck('reference');
  }
 
  public function contains($searchable): bool
@@ -29,7 +42,15 @@ public function contains($searchable): bool
  return false;
  }
 
- return $this->filter()($searchable);
+ if ($this->hasFilter()) {
+ return $this->filter()($searchable);
+ }
+
+ $query = User::query()->where('id', $searchable->id());
+
+ $this->applyQueryScope($query);
+
+ return $query->exists();
  }
 
  public function find(array $ids): Collection

tests/Search/SearchTest.php

@@ -19,6 +19,7 @@ public function it_updates_indexes($updateMock)
  {
  $index = Mockery::mock(Index::class);
  $item = Mockery::mock(Searchable::class);
+ $item->shouldReceive('getSearchReference')->andReturn('a');
 
  $updateMock($index, $item);
 
@@ -37,7 +38,7 @@ public static function saveProvider()
  function ($mock, $entry) {
  $mock->shouldReceive('shouldIndex')->with($entry)->andReturnTrue();
  $mock->shouldReceive('exists')->andReturnTrue();
- $mock->shouldReceive('insert')->once()->with($entry);
+ $mock->shouldReceive('insert')->once()->with($entry->getSearchReference());
  },
  ],
 

config/search.php

@@ -79,4 +79,31 @@
  'fields' => ['title'],
  ],
 
+ /*
+ |--------------------------------------------------------------------------
+ | Indexing Queue
+ |--------------------------------------------------------------------------
+ |
+ | Here you may configure the queue name and connection used when indexing
+ | documents.
+ |
+ */
+
+ 'queue' => env('STATAMIC_SEARCH_QUEUE'),
+
+ 'queue_connection' => env('STATAMIC_SEARCH_QUEUE_CONNECTION'),
+
+ /*
+ |--------------------------------------------------------------------------
+ | Chunk Size
+ |--------------------------------------------------------------------------
+ |
+ | Here you can configure the chunk size used when indexing documents.
+ | The higher you make it, the more memory it will use, but the quicker
+ | the indexing process will be.
+ |
+ */
+
+ 'chunk_size' => 100,
+
 ];

src/Search/Algolia/Index.php

@@ -39,7 +39,7 @@ public function search($query)
  return (new Query($this))->query($query);
  }
 
-protected function insertDocuments(Documents $documents)
+public function insertDocuments(Documents $documents)
  {
  $documents = $documents->map(function ($item, $id) {
  $item['objectID'] = $id;

src/Search/Comb/Index.php

@@ -99,7 +99,7 @@ public function delete($document)
  $this->save($data);
  }
 
-protected function insertDocuments(Documents $documents)
+public function insertDocuments(Documents $documents)
  {
  try {
  $data = $this->data();

src/Search/Index.php

@@ -4,7 +4,6 @@
 
 use Closure;
 use Statamic\Contracts\Search\Searchable;
-use Statamic\Support\Arr;
 use Statamic\Support\Str;
 
 abstract class Index
@@ -20,7 +19,7 @@ abstract public function delete($document);
 
  abstract public function exists();
 
- abstract protected function insertDocuments(Documents $documents);
+ abstract public function insertDocuments(Documents $documents);
 
  abstract protected function deleteIndex();
 
@@ -84,20 +83,27 @@ public function ensureExists()
 
  public function insert($document)
  {
- return $this->insertMultiple(Arr::wrap($document));
+ return $this->insertMultiple(collect($document));
  }
 
  public function insertMultiple($documents)
  {
- $documents = (new Documents($documents))->mapWithKeys(function (Searchable $item) {
- return [$item->getSearchReference() => $this->searchables()->fields($item)];
- });
-
- $this->insertDocuments($documents);
+ $documents
+ ->chunk(config('statamic.search.chunk_size'))
+ ->each(fn ($documents) => InsertMultipleJob::dispatch(
+ name: Str::beforeLast($this->name, '_'),
+ locale: $this->locale,
+ documents: $documents
+ ));
 
  return $this;
  }
 
+ public function fields(Searchable $searchable)
+ {
+ return $this->searchables()->fields($searchable);
+ }
+
  public function shouldIndex($searchable)
  {
  return $this->searchables()->contains($searchable);

src/Search/InsertMultipleJob.php

@@ -0,0 +1,52 @@
+<?php
+
+namespace Statamic\Search;
+
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Queue\Queueable;
+use Illuminate\Support\Collection;
+use Illuminate\Support\LazyCollection;
+use Statamic\Contracts\Search\Searchable;
+use Statamic\Facades\Search;
+use Statamic\Search\Searchables\Providers;
+use Statamic\Support\Str;
+
+class InsertMultipleJob implements ShouldQueue
+{
+ use Queueable;
+
+ /**
+ * Create a new job instance.
+ */
+ public function __construct(
+ public string $name,
+ public ?string $locale,
+ public Collection|LazyCollection $documents,
+ ) {
+ $this->onConnection($connection = config('statamic.search.queue_connection', config('queue.default')));
+ $this->onQueue(config('statamic.search.queue', config("queue.connections.{$connection}.queue")));
+ }
+
+ /**
+ * Execute the job.
+ */
+ public function handle(): void
+ {
+ $providers = app(Providers::class);
+ $index = Search::index($this->name, $this->locale);
+
+ $documents = $this->documents
+ ->groupBy(fn ($document) => explode('::', $document)[0])
+ ->flatMap(function ($documents, $prefix) use ($providers) {
+ return $providers
+ ->getByPrefix($prefix)
+ ->find($documents->map(fn ($reference) => Str::after($reference, '::'))->all())
+ ->all();
+ })
+ ->mapWithKeys(function (Searchable $item) use ($index) {
+ return [$item->getSearchReference() => $index->fields($item)];
+ });
+
+ $index->insertDocuments(new Documents($documents));
+ }
+}

src/Search/Null/NullIndex.php

@@ -22,7 +22,7 @@ public function exists()
  return true;
  }
 
-protected function insertDocuments(Documents $documents)
+public function insertDocuments(Documents $documents)
  {
  //
  }

src/Search/Search.php

@@ -51,7 +51,7 @@ public function updateWithinIndexes(Searchable $searchable)
  $exists = $index->exists();
 
  if ($shouldIndex && $exists) {
- $index->insert($searchable);
+ $index->insert($searchable->getSearchReference());
  } elseif ($shouldIndex && ! $exists) {
  $index->update();
  } elseif ($exists) {

src/Search/Searchables/Assets.php

@@ -26,7 +26,9 @@ public function provide(): Collection
  : AssetCollection::make($this->keys)
  ->flatMap(fn ($key) => Asset::whereContainer($key));
 
- return $assets->filter($this->filter())->values();
+ // TODO: query scope support?
+
+ return $assets->filter($this->filter())->values()->map->reference();
  }
 
  public function contains($searchable): bool

src/Search/Searchables/Entries.php

@@ -31,7 +31,19 @@ public function provide(): Collection|LazyCollection
  $query->where('site', $site);
  }
 
- return $query->lazy(100)->filter($this->filter())->values();
+ $this->applyQueryScope($query);
+
+ if ($this->hasFilter()) {
+ return $query
+ ->lazy(config('statamic.search.chunk_size'))
+ ->filter($this->filter())
+ ->values()
+ ->map->reference();
+ }
+
+ $query->where('status', 'published');
+
+ return $query->pluck('reference');
  }
 
  public function contains($searchable): bool
@@ -48,7 +60,17 @@ public function contains($searchable): bool
  return false;
  }
 
- return $this->filter()($searchable);
+ if ($this->hasFilter()) {
+ return $this->filter()($searchable);
+ }
+
+ $query = Entry::query()
+ ->where('status', 'published')
+ ->where('id', $searchable->id());
+
+ $this->applyQueryScope($query);
+
+ return $query->exists();
  }
 
  public function find(array $ids): Collection

src/Search/Searchables/Provider.php

@@ -2,9 +2,11 @@
 
 namespace Statamic\Search\Searchables;
 
+use Statamic\Facades\Scope;
 use Statamic\Facades\Search;
 use Statamic\Search\Index;
 use Statamic\Search\ProvidesSearchables;
+use Statamic\Support\Arr;
 
 abstract class Provider implements ProvidesSearchables
 {
@@ -55,6 +57,20 @@ protected function usesWildcard()
  return in_array('*', $this->keys);
  }
 
+ protected function applyQueryScope($query)
+ {
+ if (! $scope = $this->index->config()['query_scope'] ?? null) {
+ return;
+ }
+
+ Scope::find($scope)->apply($query, []);
+ }
+
+ protected function hasFilter()
+ {
+ return Arr::has($this->index->config(), 'filter');
+ }
+
  protected function filter()
  {
  $filter = $this->index->config()['filter'] ?? null;

src/Search/Searchables/Terms.php

@@ -32,7 +32,17 @@ public function provide(): Collection|LazyCollection
  $query->where('site', $site);
  }
 
- return $query->lazy(100)->filter($this->filter())->values();
+ $this->applyQueryScope($query);
+
+ if ($this->hasFilter()) {
+ return $query
+ ->lazy(config('statamic.search.chunk_size'))
+ ->filter($this->filter())
+ ->values()
+ ->map->reference();
+ }
+
+ return $query->pluck('reference');
  }
 
  public function contains($searchable): bool
@@ -49,7 +59,15 @@ public function contains($searchable): bool
  return false;
  }
 
- return $this->filter()($searchable);
+ if ($this->hasFilter()) {
+ return $this->filter()($searchable);
+ }
+
+ $query = Term::query()->where('reference', $searchable->reference());
+
+ $this->applyQueryScope($query);
+
+ return $query->exists();
  }
 
  public function find(array $refs): Collection

src/Search/Searchables/Users.php

@@ -3,6 +3,7 @@
 namespace Statamic\Search\Searchables;
 
 use Illuminate\Support\Collection;
+use Illuminate\Support\LazyCollection;
 use Statamic\Contracts\Auth\User as UserContract;
 use Statamic\Facades\User;
 
@@ -18,9 +19,21 @@ public static function referencePrefix(): string
  return 'user';
  }
 
- public function provide(): Collection
+ public function provide(): Collection|LazyCollection
  {
- return User::all()->filter($this->filter())->values();
+ $query = User::query();
+
+ $this->applyQueryScope($query);
+
+ if ($this->hasFilter()) {
+ return $query
+ ->lazy(config('statamic.search.chunk_size'))
+ ->filter($this->filter())
+ ->values()
+ ->map->reference();
+ }
+
+ return $query->pluck('reference');
  }
 
  public function contains($searchable): bool
@@ -29,7 +42,15 @@ public function contains($searchable): bool
  return false;
  }
 
- return $this->filter()($searchable);
+ if ($this->hasFilter()) {
+ return $this->filter()($searchable);
+ }
+
+ $query = User::query()->where('id', $searchable->id());
+
+ $this->applyQueryScope($query);
+
+ return $query->exists();
  }
 
  public function find(array $ids): Collection

tests/Search/SearchTest.php

@@ -19,6 +19,7 @@ public function it_updates_indexes($updateMock)
  {
  $index = Mockery::mock(Index::class);
  $item = Mockery::mock(Searchable::class);
+ $item->shouldReceive('getSearchReference')->andReturn('a');
 
  $updateMock($index, $item);
 
@@ -37,7 +38,7 @@ public static function saveProvider()
  function ($mock, $entry) {
  $mock->shouldReceive('shouldIndex')->with($entry)->andReturnTrue();
  $mock->shouldReceive('exists')->andReturnTrue();
- $mock->shouldReceive('insert')->once()->with($entry);
+ $mock->shouldReceive('insert')->once()->with($entry->getSearchReference());
  },
  ],