[6.x] Search Indexing Performance Improvements

content/collections/pages/5-to-6.md

@@ -588,6 +588,68 @@ The REST API will now just give you sub-keys:
 }
 ```
 
+### Search: Changes to custom searchables
+
+Searchables should now return collections of references (eg. `entry::the-entry-id`) instead of objects.
+
+```php
+return Thing::query()->lazy(); // [tl! remove]
+return Thing::query()->lazy()->pluck('reference'); // [tl! add]
+```
+
+If your searchable allows for it, you may want to consider adding support for query scopes, which we now recommend over filtering.
+
+```php
+$query = Thing::query();
+
+$this->applyQueryScope($query);
+
+return $query->pluck('reference');
+```
+
+If you support filtering, you may want to split the "with filter" & "without filter" cases into separate return statements.
+
+The `->filter()` method evaluates every item in the collection, which is an unnecessary performance hit when no filter is configured:
+
+```php
+$query = Thing::query();
+
+if ($this->hasFilter()) {
+	return $query
+ ->lazy(config('statamic.search.chunk_size')) 
+ ->filter($this->filter()) 
+ ->values() 
+ ->map->reference();
+}
+
+return $query->pluck('reference');
+```
+
+### Search: Changes to custom search drivers
+
+The `insertDocument` method is now public:
+
+```php
+protected function insertDocuments(Documents $documents) // [tl! remove]
+public function insertDocuments(Documents $documents) // [tl! add]
+```
+
+If you were previously overriding the `insertMultiple` method to chunk documents, you don't need to do that anymore (chunking is now handled by the base method).
+
+If you need to manipulate the fields array before it gets sent to your index, you may define a `fields` method:
+
+```php
+public function fields(Searchable $searchable)
+{
+ return array_merge(
+ $this->searchables()->fields($searchable),
+ [
+ '_some_special_field_' => $searchable->id(),
+ ]
+ );
+}
+```
+
 ### Bard: Strikes now output `<strike>` tags, rather than `<s>`
 **Affects apps relying on the `<s>` tag for strikes.**
 

content/collections/pages/search.md

@@ -595,17 +595,24 @@ class ProductProvider extends Provider
  }
 
  /**
- * Get a collection of all searchables.
+ * Get all searchables and return a collection of 
+ * their references.
+ * 
+ * e.g. 'entry::123'
  */
  public function provide(): Collection
  {
- return Product::all();
+ return Product::query()
+ ->pluck('id')
+ ->map(fn ($id) => "product::{$id}");
 
  // If you wanted to allow subsets of products, you could specify them in your
  // config then retrieve them appropriately here using keys.
  // e.g. 'searchables' => ['products:hats', 'products:shoes'],
  // $this->keys would be ['keys', 'hats'].
- return Product::whereIn('type', $this->keys)->get();
+ return Product::whereIn('type', $this->keys)
+ ->pluck('id')
+ ->map(fn ($id) => "product::{$id}");
  }
 
  /**
@@ -726,7 +733,7 @@ class FastSearchIndex extends Index
  /**
  * Insert items into the index.
  */
-protected function insertDocuments(Documents $documents)
+public function insertDocuments(Documents $documents)
  {
  $this->client->insertObjects($documents->all());
  }

content/collections/pages/5-to-6.md

@@ -588,6 +588,68 @@ The REST API will now just give you sub-keys:
 }
 ```
 
+### Search: Changes to custom searchables
+
+Searchables should now return collections of references (eg. `entry::the-entry-id`) instead of objects.
+
+```php
+return Thing::query()->lazy(); // [tl! remove]
+return Thing::query()->lazy()->pluck('reference'); // [tl! add]
+```
+
+If your searchable allows for it, you may want to consider adding support for query scopes, which we now recommend over filtering.
+
+```php
+$query = Thing::query();
+
+$this->applyQueryScope($query);
+
+return $query->pluck('reference');
+```
+
+If you support filtering, you may want to split the "with filter" & "without filter" cases into separate return statements.
+
+The `->filter()` method evaluates every item in the collection, which is an unnecessary performance hit when no filter is configured:
+
+```php
+$query = Thing::query();
+
+if ($this->hasFilter()) {
+	return $query
+ ->lazy(config('statamic.search.chunk_size')) 
+ ->filter($this->filter()) 
+ ->values() 
+ ->map->reference();
+}
+
+return $query->pluck('reference');
+```
+
+### Search: Changes to custom search drivers
+
+The `insertDocument` method is now public:
+
+```php
+protected function insertDocuments(Documents $documents) // [tl! remove]
+public function insertDocuments(Documents $documents) // [tl! add]
+```
+
+If you were previously overriding the `insertMultiple` method to chunk documents, you don't need to do that anymore (chunking is now handled by the base method).
+
+If you need to manipulate the fields array before it gets sent to your index, you may define a `fields` method:
+
+```php
+public function fields(Searchable $searchable)
+{
+ return array_merge(
+ $this->searchables()->fields($searchable),
+ [
+ '_some_special_field_' => $searchable->id(),
+ ]
+ );
+}
+```
+
 ### Bard: Strikes now output `<strike>` tags, rather than `<s>`
 **Affects apps relying on the `<s>` tag for strikes.**
 

content/collections/pages/search.md

@@ -595,17 +595,24 @@ class ProductProvider extends Provider
  }
 
  /**
- * Get a collection of all searchables.
+ * Get all searchables and return a collection of 
+ * their references.
+ * 
+ * e.g. 'entry::123'
  */
  public function provide(): Collection
  {
- return Product::all();
+ return Product::query()
+ ->pluck('id')
+ ->map(fn ($id) => "product::{$id}");
 
  // If you wanted to allow subsets of products, you could specify them in your
  // config then retrieve them appropriately here using keys.
  // e.g. 'searchables' => ['products:hats', 'products:shoes'],
  // $this->keys would be ['keys', 'hats'].
- return Product::whereIn('type', $this->keys)->get();
+ return Product::whereIn('type', $this->keys)
+ ->pluck('id')
+ ->map(fn ($id) => "product::{$id}");
  }
 
  /**
@@ -726,7 +733,7 @@ class FastSearchIndex extends Index
  /**
  * Insert items into the index.
  */
-protected function insertDocuments(Documents $documents)
+public function insertDocuments(Documents $documents)
  {
  $this->client->insertObjects($documents->all());
  }

content/collections/pages/5-to-6.md

@@ -588,6 +588,68 @@ The REST API will now just give you sub-keys:
 }
 ```
 
+### Search: Changes to custom searchables
+
+Searchables should now return collections of references (eg. `entry::the-entry-id`) instead of objects.
+
+```php
+return Thing::query()->lazy(); // [tl! remove]
+return Thing::query()->lazy()->pluck('reference'); // [tl! add]
+```
+
+If your searchable allows for it, you may want to consider adding support for query scopes, which we now recommend over filtering.
+
+```php
+$query = Thing::query();
+
+$this->applyQueryScope($query);
+
+return $query->pluck('reference');
+```
+
+If you support filtering, you may want to split the "with filter" & "without filter" cases into separate return statements.
+
+The `->filter()` method evaluates every item in the collection, which is an unnecessary performance hit when no filter is configured:
+
+```php
+$query = Thing::query();
+
+if ($this->hasFilter()) {
+	return $query
+ ->lazy(config('statamic.search.chunk_size')) 
+ ->filter($this->filter()) 
+ ->values() 
+ ->map->reference();
+}
+
+return $query->pluck('reference');
+```
+
+### Search: Changes to custom search drivers
+
+The `insertDocument` method is now public:
+
+```php
+protected function insertDocuments(Documents $documents) // [tl! remove]
+public function insertDocuments(Documents $documents) // [tl! add]
+```
+
+If you were previously overriding the `insertMultiple` method to chunk documents, you don't need to do that anymore (chunking is now handled by the base method).
+
+If you need to manipulate the fields array before it gets sent to your index, you may define a `fields` method:
+
+```php
+public function fields(Searchable $searchable)
+{
+ return array_merge(
+ $this->searchables()->fields($searchable),
+ [
+ '_some_special_field_' => $searchable->id(),
+ ]
+ );
+}
+```
+
 ### Bard: Strikes now output `<strike>` tags, rather than `<s>`
 **Affects apps relying on the `<s>` tag for strikes.**
 

content/collections/pages/search.md

@@ -595,17 +595,24 @@ class ProductProvider extends Provider
  }
 
  /**
- * Get a collection of all searchables.
+ * Get all searchables and return a collection of 
+ * their references.
+ * 
+ * e.g. 'entry::123'
  */
  public function provide(): Collection
  {
- return Product::all();
+ return Product::query()
+ ->pluck('id')
+ ->map(fn ($id) => "product::{$id}");
 
  // If you wanted to allow subsets of products, you could specify them in your
  // config then retrieve them appropriately here using keys.
  // e.g. 'searchables' => ['products:hats', 'products:shoes'],
  // $this->keys would be ['keys', 'hats'].
- return Product::whereIn('type', $this->keys)->get();
+ return Product::whereIn('type', $this->keys)
+ ->pluck('id')
+ ->map(fn ($id) => "product::{$id}");
  }
 
  /**
@@ -726,7 +733,7 @@ class FastSearchIndex extends Index
  /**
  * Insert items into the index.
  */
-protected function insertDocuments(Documents $documents)
+public function insertDocuments(Documents $documents)
  {
  $this->client->insertObjects($documents->all());
  }

content/collections/pages/5-to-6.md

@@ -588,6 +588,68 @@ The REST API will now just give you sub-keys:
 }
 ```
 
+### Search: Changes to custom searchables
+
+Searchables should now return collections of references (eg. `entry::the-entry-id`) instead of objects.
+
+```php
+return Thing::query()->lazy(); // [tl! remove]
+return Thing::query()->lazy()->pluck('reference'); // [tl! add]
+```
+
+If your searchable allows for it, you may want to consider adding support for query scopes, which we now recommend over filtering.
+
+```php
+$query = Thing::query();
+
+$this->applyQueryScope($query);
+
+return $query->pluck('reference');
+```
+
+If you support filtering, you may want to split the "with filter" & "without filter" cases into separate return statements.
+
+The `->filter()` method evaluates every item in the collection, which is an unnecessary performance hit when no filter is configured:
+
+```php
+$query = Thing::query();
+
+if ($this->hasFilter()) {
+	return $query
+ ->lazy(config('statamic.search.chunk_size')) 
+ ->filter($this->filter()) 
+ ->values() 
+ ->map->reference();
+}
+
+return $query->pluck('reference');
+```
+
+### Search: Changes to custom search drivers
+
+The `insertDocument` method is now public:
+
+```php
+protected function insertDocuments(Documents $documents) // [tl! remove]
+public function insertDocuments(Documents $documents) // [tl! add]
+```
+
+If you were previously overriding the `insertMultiple` method to chunk documents, you don't need to do that anymore (chunking is now handled by the base method).
+
+If you need to manipulate the fields array before it gets sent to your index, you may define a `fields` method:
+
+```php
+public function fields(Searchable $searchable)
+{
+ return array_merge(
+ $this->searchables()->fields($searchable),
+ [
+ '_some_special_field_' => $searchable->id(),
+ ]
+ );
+}
+```
+
 ### Bard: Strikes now output `<strike>` tags, rather than `<s>`
 **Affects apps relying on the `<s>` tag for strikes.**
 

content/collections/pages/search.md

@@ -595,17 +595,24 @@ class ProductProvider extends Provider
  }
 
  /**
- * Get a collection of all searchables.
+ * Get all searchables and return a collection of 
+ * their references.
+ * 
+ * e.g. 'entry::123'
  */
  public function provide(): Collection
  {
- return Product::all();
+ return Product::query()
+ ->pluck('id')
+ ->map(fn ($id) => "product::{$id}");
 
  // If you wanted to allow subsets of products, you could specify them in your
  // config then retrieve them appropriately here using keys.
  // e.g. 'searchables' => ['products:hats', 'products:shoes'],
  // $this->keys would be ['keys', 'hats'].
- return Product::whereIn('type', $this->keys)->get();
+ return Product::whereIn('type', $this->keys)
+ ->pluck('id')
+ ->map(fn ($id) => "product::{$id}");
  }
 
  /**
@@ -726,7 +733,7 @@ class FastSearchIndex extends Index
  /**
  * Insert items into the index.
  */
-protected function insertDocuments(Documents $documents)
+public function insertDocuments(Documents $documents)
  {
  $this->client->insertObjects($documents->all());
  }

content/collections/pages/5-to-6.md

@@ -588,6 +588,68 @@ The REST API will now just give you sub-keys:
 }
 ```
 
+### Search: Changes to custom searchables
+
+Searchables should now return collections of references (eg. `entry::the-entry-id`) instead of objects.
+
+```php
+return Thing::query()->lazy(); // [tl! remove]
+return Thing::query()->lazy()->pluck('reference'); // [tl! add]
+```
+
+If your searchable allows for it, you may want to consider adding support for query scopes, which we now recommend over filtering.
+
+```php
+$query = Thing::query();
+
+$this->applyQueryScope($query);
+
+return $query->pluck('reference');
+```
+
+If you support filtering, you may want to split the "with filter" & "without filter" cases into separate return statements.
+
+The `->filter()` method evaluates every item in the collection, which is an unnecessary performance hit when no filter is configured:
+
+```php
+$query = Thing::query();
+
+if ($this->hasFilter()) {
+	return $query
+ ->lazy(config('statamic.search.chunk_size')) 
+ ->filter($this->filter()) 
+ ->values() 
+ ->map->reference();
+}
+
+return $query->pluck('reference');
+```
+
+### Search: Changes to custom search drivers
+
+The `insertDocument` method is now public:
+
+```php
+protected function insertDocuments(Documents $documents) // [tl! remove]
+public function insertDocuments(Documents $documents) // [tl! add]
+```
+
+If you were previously overriding the `insertMultiple` method to chunk documents, you don't need to do that anymore (chunking is now handled by the base method).
+
+If you need to manipulate the fields array before it gets sent to your index, you may define a `fields` method:
+
+```php
+public function fields(Searchable $searchable)
+{
+ return array_merge(
+ $this->searchables()->fields($searchable),
+ [
+ '_some_special_field_' => $searchable->id(),
+ ]
+ );
+}
+```
+
 ### Bard: Strikes now output `<strike>` tags, rather than `<s>`
 **Affects apps relying on the `<s>` tag for strikes.**
 

content/collections/pages/search.md

@@ -595,17 +595,24 @@ class ProductProvider extends Provider
  }
 
  /**
- * Get a collection of all searchables.
+ * Get all searchables and return a collection of 
+ * their references.
+ * 
+ * e.g. 'entry::123'
  */
  public function provide(): Collection
  {
- return Product::all();
+ return Product::query()
+ ->pluck('id')
+ ->map(fn ($id) => "product::{$id}");
 
  // If you wanted to allow subsets of products, you could specify them in your
  // config then retrieve them appropriately here using keys.
  // e.g. 'searchables' => ['products:hats', 'products:shoes'],
  // $this->keys would be ['keys', 'hats'].
- return Product::whereIn('type', $this->keys)->get();
+ return Product::whereIn('type', $this->keys)
+ ->pluck('id')
+ ->map(fn ($id) => "product::{$id}");
  }
 
  /**
@@ -726,7 +733,7 @@ class FastSearchIndex extends Index
  /**
  * Insert items into the index.
  */
-protected function insertDocuments(Documents $documents)
+public function insertDocuments(Documents $documents)
  {
  $this->client->insertObjects($documents->all());
  }

content/collections/pages/5-to-6.md

@@ -588,6 +588,68 @@ The REST API will now just give you sub-keys:
 }
 ```
 
+### Search: Changes to custom searchables
+
+Searchables should now return collections of references (eg. `entry::the-entry-id`) instead of objects.
+
+```php
+return Thing::query()->lazy(); // [tl! remove]
+return Thing::query()->lazy()->pluck('reference'); // [tl! add]
+```
+
+If your searchable allows for it, you may want to consider adding support for query scopes, which we now recommend over filtering.
+
+```php
+$query = Thing::query();
+
+$this->applyQueryScope($query);
+
+return $query->pluck('reference');
+```
+
+If you support filtering, you may want to split the "with filter" & "without filter" cases into separate return statements.
+
+The `->filter()` method evaluates every item in the collection, which is an unnecessary performance hit when no filter is configured:
+
+```php
+$query = Thing::query();
+
+if ($this->hasFilter()) {
+	return $query
+ ->lazy(config('statamic.search.chunk_size')) 
+ ->filter($this->filter()) 
+ ->values() 
+ ->map->reference();
+}
+
+return $query->pluck('reference');
+```
+
+### Search: Changes to custom search drivers
+
+The `insertDocument` method is now public:
+
+```php
+protected function insertDocuments(Documents $documents) // [tl! remove]
+public function insertDocuments(Documents $documents) // [tl! add]
+```
+
+If you were previously overriding the `insertMultiple` method to chunk documents, you don't need to do that anymore (chunking is now handled by the base method).
+
+If you need to manipulate the fields array before it gets sent to your index, you may define a `fields` method:
+
+```php
+public function fields(Searchable $searchable)
+{
+ return array_merge(
+ $this->searchables()->fields($searchable),
+ [
+ '_some_special_field_' => $searchable->id(),
+ ]
+ );
+}
+```
+
 ### Bard: Strikes now output `<strike>` tags, rather than `<s>`
 **Affects apps relying on the `<s>` tag for strikes.**
 

content/collections/pages/search.md

@@ -595,17 +595,24 @@ class ProductProvider extends Provider
  }
 
  /**
- * Get a collection of all searchables.
+ * Get all searchables and return a collection of 
+ * their references.
+ * 
+ * e.g. 'entry::123'
  */
  public function provide(): Collection
  {
- return Product::all();
+ return Product::query()
+ ->pluck('id')
+ ->map(fn ($id) => "product::{$id}");
 
  // If you wanted to allow subsets of products, you could specify them in your
  // config then retrieve them appropriately here using keys.
  // e.g. 'searchables' => ['products:hats', 'products:shoes'],
  // $this->keys would be ['keys', 'hats'].
- return Product::whereIn('type', $this->keys)->get();
+ return Product::whereIn('type', $this->keys)
+ ->pluck('id')
+ ->map(fn ($id) => "product::{$id}");
  }
 
  /**
@@ -726,7 +733,7 @@ class FastSearchIndex extends Index
  /**
  * Insert items into the index.
  */
-protected function insertDocuments(Documents $documents)
+public function insertDocuments(Documents $documents)
  {
  $this->client->insertObjects($documents->all());
  }

content/collections/pages/5-to-6.md

@@ -588,6 +588,68 @@ The REST API will now just give you sub-keys:
 }
 ```
 
+### Search: Changes to custom searchables
+
+Searchables should now return collections of references (eg. `entry::the-entry-id`) instead of objects.
+
+```php
+return Thing::query()->lazy(); // [tl! remove]
+return Thing::query()->lazy()->pluck('reference'); // [tl! add]
+```
+
+If your searchable allows for it, you may want to consider adding support for query scopes, which we now recommend over filtering.
+
+```php
+$query = Thing::query();
+
+$this->applyQueryScope($query);
+
+return $query->pluck('reference');
+```
+
+If you support filtering, you may want to split the "with filter" & "without filter" cases into separate return statements.
+
+The `->filter()` method evaluates every item in the collection, which is an unnecessary performance hit when no filter is configured:
+
+```php
+$query = Thing::query();
+
+if ($this->hasFilter()) {
+	return $query
+ ->lazy(config('statamic.search.chunk_size')) 
+ ->filter($this->filter()) 
+ ->values() 
+ ->map->reference();
+}
+
+return $query->pluck('reference');
+```
+
+### Search: Changes to custom search drivers
+
+The `insertDocument` method is now public:
+
+```php
+protected function insertDocuments(Documents $documents) // [tl! remove]
+public function insertDocuments(Documents $documents) // [tl! add]
+```
+
+If you were previously overriding the `insertMultiple` method to chunk documents, you don't need to do that anymore (chunking is now handled by the base method).
+
+If you need to manipulate the fields array before it gets sent to your index, you may define a `fields` method:
+
+```php
+public function fields(Searchable $searchable)
+{
+ return array_merge(
+ $this->searchables()->fields($searchable),
+ [
+ '_some_special_field_' => $searchable->id(),
+ ]
+ );
+}
+```
+
 ### Bard: Strikes now output `<strike>` tags, rather than `<s>`
 **Affects apps relying on the `<s>` tag for strikes.**
 

content/collections/pages/search.md

@@ -595,17 +595,24 @@ class ProductProvider extends Provider
  }
 
  /**
- * Get a collection of all searchables.
+ * Get all searchables and return a collection of 
+ * their references.
+ * 
+ * e.g. 'entry::123'
  */
  public function provide(): Collection
  {
- return Product::all();
+ return Product::query()
+ ->pluck('id')
+ ->map(fn ($id) => "product::{$id}");
 
  // If you wanted to allow subsets of products, you could specify them in your
  // config then retrieve them appropriately here using keys.
  // e.g. 'searchables' => ['products:hats', 'products:shoes'],
  // $this->keys would be ['keys', 'hats'].
- return Product::whereIn('type', $this->keys)->get();
+ return Product::whereIn('type', $this->keys)
+ ->pluck('id')
+ ->map(fn ($id) => "product::{$id}");
  }
 
  /**
@@ -726,7 +733,7 @@ class FastSearchIndex extends Index
  /**
  * Insert items into the index.
  */
-protected function insertDocuments(Documents $documents)
+public function insertDocuments(Documents $documents)
  {
  $this->client->insertObjects($documents->all());
  }

content/collections/pages/5-to-6.md

@@ -588,6 +588,68 @@ The REST API will now just give you sub-keys:
 }
 ```
 
+### Search: Changes to custom searchables
+
+Searchables should now return collections of references (eg. `entry::the-entry-id`) instead of objects.
+
+```php
+return Thing::query()->lazy(); // [tl! remove]
+return Thing::query()->lazy()->pluck('reference'); // [tl! add]
+```
+
+If your searchable allows for it, you may want to consider adding support for query scopes, which we now recommend over filtering.
+
+```php
+$query = Thing::query();
+
+$this->applyQueryScope($query);
+
+return $query->pluck('reference');
+```
+
+If you support filtering, you may want to split the "with filter" & "without filter" cases into separate return statements.
+
+The `->filter()` method evaluates every item in the collection, which is an unnecessary performance hit when no filter is configured:
+
+```php
+$query = Thing::query();
+
+if ($this->hasFilter()) {
+	return $query
+ ->lazy(config('statamic.search.chunk_size')) 
+ ->filter($this->filter()) 
+ ->values() 
+ ->map->reference();
+}
+
+return $query->pluck('reference');
+```
+
+### Search: Changes to custom search drivers
+
+The `insertDocument` method is now public:
+
+```php
+protected function insertDocuments(Documents $documents) // [tl! remove]
+public function insertDocuments(Documents $documents) // [tl! add]
+```
+
+If you were previously overriding the `insertMultiple` method to chunk documents, you don't need to do that anymore (chunking is now handled by the base method).
+
+If you need to manipulate the fields array before it gets sent to your index, you may define a `fields` method:
+
+```php
+public function fields(Searchable $searchable)
+{
+ return array_merge(
+ $this->searchables()->fields($searchable),
+ [
+ '_some_special_field_' => $searchable->id(),
+ ]
+ );
+}
+```
+
 ### Bard: Strikes now output `<strike>` tags, rather than `<s>`
 **Affects apps relying on the `<s>` tag for strikes.**
 

content/collections/pages/search.md

@@ -595,17 +595,24 @@ class ProductProvider extends Provider
  }
 
  /**
- * Get a collection of all searchables.
+ * Get all searchables and return a collection of 
+ * their references.
+ * 
+ * e.g. 'entry::123'
  */
  public function provide(): Collection
  {
- return Product::all();
+ return Product::query()
+ ->pluck('id')
+ ->map(fn ($id) => "product::{$id}");
 
  // If you wanted to allow subsets of products, you could specify them in your
  // config then retrieve them appropriately here using keys.
  // e.g. 'searchables' => ['products:hats', 'products:shoes'],
  // $this->keys would be ['keys', 'hats'].
- return Product::whereIn('type', $this->keys)->get();
+ return Product::whereIn('type', $this->keys)
+ ->pluck('id')
+ ->map(fn ($id) => "product::{$id}");
  }
 
  /**
@@ -726,7 +733,7 @@ class FastSearchIndex extends Index
  /**
  * Insert items into the index.
  */
-protected function insertDocuments(Documents $documents)
+public function insertDocuments(Documents $documents)
  {
  $this->client->insertObjects($documents->all());
  }