Skip to content
This repository was archived by the owner on Mar 2, 2026. It is now read-only.

Commit 9462d10

Browse files
feat: pipelines preview (#1156)
This PR adds support for Pipeline Queries
1 parent f0ed940 commit 9462d10

48 files changed

Lines changed: 13577 additions & 258 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎google/cloud/firestore_v1/_helpers.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,9 @@ def __ne__(self, other):
120120
else:
121121
returnnotequality_val
122122

123+
def__repr__(self):
124+
returnf"{type(self).__name__}(latitude={self.latitude}, longitude={self.longitude})"
125+
123126

124127
defverify_path(path, is_collection) ->None:
125128
"""Verifies that a ``path`` has the correct form.

‎google/cloud/firestore_v1/async_client.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,8 @@
5656
fromgoogle.cloud.firestore_v1.services.firestore.transportsimport (
5757
grpc_asyncioasfirestore_grpc_transport,
5858
)
59+
fromgoogle.cloud.firestore_v1.async_pipelineimportAsyncPipeline
60+
fromgoogle.cloud.firestore_v1.pipeline_sourceimportPipelineSource
5961

6062
ifTYPE_CHECKING: # pragma: NO COVER
6163
importdatetime
@@ -438,3 +440,10 @@ def transaction(
438440
A transaction attached to this client.
439441
"""
440442
returnAsyncTransaction(self, max_attempts=max_attempts, read_only=read_only)
443+
444+
@property
445+
def_pipeline_cls(self):
446+
returnAsyncPipeline
447+
448+
defpipeline(self) ->PipelineSource:
449+
returnPipelineSource(self)
Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
# Copyright 2025 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
"""
15+
.. warning::
16+
**Preview API**: Firestore Pipelines is currently in preview and is
17+
subject to potential breaking changes in future releases
18+
"""
19+
20+
from __future__ importannotations
21+
fromtypingimportTYPE_CHECKING
22+
fromgoogle.cloud.firestore_v1importpipeline_stagesasstages
23+
fromgoogle.cloud.firestore_v1.base_pipelineimport_BasePipeline
24+
fromgoogle.cloud.firestore_v1.pipeline_resultimportAsyncPipelineStream
25+
fromgoogle.cloud.firestore_v1.pipeline_resultimportPipelineSnapshot
26+
fromgoogle.cloud.firestore_v1.pipeline_resultimportPipelineResult
27+
28+
ifTYPE_CHECKING: # pragma: NO COVER
29+
importdatetime
30+
fromgoogle.cloud.firestore_v1.async_clientimportAsyncClient
31+
fromgoogle.cloud.firestore_v1.async_transactionimportAsyncTransaction
32+
fromgoogle.cloud.firestore_v1.pipeline_expressionsimportConstant
33+
fromgoogle.cloud.firestore_v1.types.documentimportValue
34+
fromgoogle.cloud.firestore_v1.query_profileimportPipelineExplainOptions
35+
36+
37+
classAsyncPipeline(_BasePipeline):
38+
"""
39+
Pipelines allow for complex data transformations and queries involving
40+
multiple stages like filtering, projection, aggregation, and vector search.
41+
42+
This class extends `_BasePipeline` and provides methods to execute the
43+
defined pipeline stages using an asynchronous `AsyncClient`.
44+
45+
Usage Example:
46+
>>> from google.cloud.firestore_v1.pipeline_expressions import Field
47+
>>>
48+
>>> async def run_pipeline():
49+
... client = AsyncClient(...)
50+
... pipeline = client.pipeline()
51+
... .collection("books")
52+
... .where(Field.of("published").gt(1980))
53+
... .select("title", "author")
54+
... async for result in pipeline.stream():
55+
... print(result)
56+
57+
Use `client.pipeline()` to create instances of this class.
58+
59+
.. warning::
60+
**Preview API**: Firestore Pipelines is currently in preview and is
61+
subject to potential breaking changes in future releases
62+
"""
63+
64+
def__init__(self, client: AsyncClient, *stages: stages.Stage):
65+
"""
66+
Initializes an asynchronous Pipeline.
67+
68+
Args:
69+
client: The asynchronous `AsyncClient` instance to use for execution.
70+
*stages: Initial stages for the pipeline.
71+
"""
72+
super().__init__(client, *stages)
73+
74+
asyncdefexecute(
75+
self,
76+
*,
77+
transaction: "AsyncTransaction"|None=None,
78+
read_time: datetime.datetime|None=None,
79+
explain_options: PipelineExplainOptions|None=None,
80+
additional_options: dict[str, Value|Constant] = {},
81+
) ->PipelineSnapshot[PipelineResult]:
82+
"""
83+
Executes this pipeline and returns results as a list
84+
85+
Args:
86+
transaction (Optional[:class:`~google.cloud.firestore_v1.transaction.Transaction`]):
87+
An existing transaction that this query will run in.
88+
If a ``transaction`` is used and it already has write operations
89+
added, this method cannot be used (i.e. read-after-write is not
90+
allowed).
91+
read_time (Optional[datetime.datetime]): If set, reads documents as they were at the given
92+
time. This must be a microsecond precision timestamp within the past one hour, or
93+
if Point-in-Time Recovery is enabled, can additionally be a whole minute timestamp
94+
within the past 7 days. For the most accurate results, use UTC timezone.
95+
explain_options (Optional[:class:`~google.cloud.firestore_v1.query_profile.PipelineExplainOptions`]):
96+
Options to enable query profiling for this query. When set,
97+
explain_metrics will be available on the returned list.
98+
additional_options (Optional[dict[str, Value | Constant]]): Additional options to pass to the query.
99+
These options will take precedence over method argument if there is a conflict (e.g. explain_options)
100+
"""
101+
kwargs= {k: vfork, vinlocals().items() ifk!="self"}
102+
stream=AsyncPipelineStream(PipelineResult, self, **kwargs)
103+
results= [resultasyncforresultinstream]
104+
returnPipelineSnapshot(results, stream)
105+
106+
defstream(
107+
self,
108+
*,
109+
read_time: datetime.datetime|None=None,
110+
transaction: "AsyncTransaction"|None=None,
111+
explain_options: PipelineExplainOptions|None=None,
112+
additional_options: dict[str, Value|Constant] = {},
113+
) ->AsyncPipelineStream[PipelineResult]:
114+
"""
115+
Process this pipeline as a stream, providing results through an AsyncIterable
116+
117+
Args:
118+
transaction (Optional[:class:`~google.cloud.firestore_v1.transaction.Transaction`]):
119+
An existing transaction that this query will run in.
120+
If a ``transaction`` is used and it already has write operations
121+
added, this method cannot be used (i.e. read-after-write is not
122+
allowed).
123+
read_time (Optional[datetime.datetime]): If set, reads documents as they were at the given
124+
time. This must be a microsecond precision timestamp within the past one hour, or
125+
if Point-in-Time Recovery is enabled, can additionally be a whole minute timestamp
126+
within the past 7 days. For the most accurate results, use UTC timezone.
127+
explain_options (Optional[:class:`~google.cloud.firestore_v1.query_profile.PipelineExplainOptions`]):
128+
Options to enable query profiling for this query. When set,
129+
explain_metrics will be available on the returned generator.
130+
additional_options (Optional[dict[str, Value | Constant]]): Additional options to pass to the query.
131+
These options will take precedence over method argument if there is a conflict (e.g. explain_options)
132+
"""
133+
kwargs= {k: vfork, vinlocals().items() ifk!="self"}
134+
returnAsyncPipelineStream(PipelineResult, self, **kwargs)

‎google/cloud/firestore_v1/base_aggregation.py‎

Lines changed: 58 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,10 @@
2121
from __future__ importannotations
2222

2323
importabc
24+
importitertools
2425

2526
fromabcimportABC
26-
fromtypingimportTYPE_CHECKING, Any, Coroutine, List, Optional, Tuple, Union
27+
fromtypingimportTYPE_CHECKING, Any, Coroutine, List, Optional, Tuple, Union, Iterable
2728

2829
fromgoogle.api_coreimportgapic_v1
2930
fromgoogle.api_coreimportretryasretries
@@ -33,6 +34,10 @@
3334
fromgoogle.cloud.firestore_v1.typesimport (
3435
StructuredAggregationQuery,
3536
)
37+
fromgoogle.cloud.firestore_v1.pipeline_expressionsimportAggregateFunction
38+
fromgoogle.cloud.firestore_v1.pipeline_expressionsimportCount
39+
fromgoogle.cloud.firestore_v1.pipeline_expressionsimportAliasedExpression
40+
fromgoogle.cloud.firestore_v1.pipeline_expressionsimportField
3641

3742
# Types needed only for Type Hints
3843
ifTYPE_CHECKING: # pragma: NO COVER
@@ -43,6 +48,7 @@
4348
fromgoogle.cloud.firestore_v1.stream_generatorimport (
4449
StreamGenerator,
4550
)
51+
fromgoogle.cloud.firestore_v1.pipeline_sourceimportPipelineSource
4652

4753
importdatetime
4854

@@ -66,6 +72,9 @@ def __init__(self, alias: str, value: float, read_time=None):
6672
def__repr__(self):
6773
returnf"<Aggregation alias={self.alias}, value={self.value}, readtime={self.read_time}>"
6874

75+
def_to_dict(self):
76+
return {self.alias: self.value}
77+
6978

7079
classBaseAggregation(ABC):
7180
def__init__(self, alias: str|None=None):
@@ -75,6 +84,27 @@ def __init__(self, alias: str | None = None):
7584
def_to_protobuf(self):
7685
"""Convert this instance to the protobuf representation"""
7786

87+
@abc.abstractmethod
88+
def_to_pipeline_expr(
89+
self, autoindexer: Iterable[int]
90+
) ->AliasedExpression[AggregateFunction]:
91+
"""
92+
Convert this instance to a pipeline expression for use with pipeline.aggregate()
93+
94+
Args:
95+
autoindexer: If an alias isn't supplied, one should be created with the format "field_n"
96+
The autoindexer is an iterable that provides the `n` value to use for each expression
97+
"""
98+
99+
def_pipeline_alias(self, autoindexer):
100+
"""
101+
Helper to build the alias for the pipeline expression
102+
"""
103+
ifself.aliasisnotNone:
104+
returnself.alias
105+
else:
106+
returnf"field_{next(autoindexer)}"
107+
78108

79109
classCountAggregation(BaseAggregation):
80110
def__init__(self, alias: str|None=None):
@@ -88,6 +118,9 @@ def _to_protobuf(self):
88118
aggregation_pb.count=StructuredAggregationQuery.Aggregation.Count()
89119
returnaggregation_pb
90120

121+
def_to_pipeline_expr(self, autoindexer: Iterable[int]):
122+
returnCount().as_(self._pipeline_alias(autoindexer))
123+
91124

92125
classSumAggregation(BaseAggregation):
93126
def__init__(self, field_ref: str|FieldPath, alias: str|None=None):
@@ -107,6 +140,9 @@ def _to_protobuf(self):
107140
aggregation_pb.sum.field.field_path=self.field_ref
108141
returnaggregation_pb
109142

143+
def_to_pipeline_expr(self, autoindexer: Iterable[int]):
144+
returnField.of(self.field_ref).sum().as_(self._pipeline_alias(autoindexer))
145+
110146

111147
classAvgAggregation(BaseAggregation):
112148
def__init__(self, field_ref: str|FieldPath, alias: str|None=None):
@@ -126,6 +162,9 @@ def _to_protobuf(self):
126162
aggregation_pb.avg.field.field_path=self.field_ref
127163
returnaggregation_pb
128164

165+
def_to_pipeline_expr(self, autoindexer: Iterable[int]):
166+
returnField.of(self.field_ref).average().as_(self._pipeline_alias(autoindexer))
167+
129168

130169
def_query_response_to_result(
131170
response_pb,
@@ -317,3 +356,21 @@ def stream(
317356
StreamGenerator[List[AggregationResult]] | AsyncStreamGenerator[List[AggregationResult]]:
318357
A generator of the query results.
319358
"""
359+
360+
def_build_pipeline(self, source: "PipelineSource"):
361+
"""
362+
Convert this query into a Pipeline
363+
364+
Queries containing a `cursor` or `limit_to_last` are not currently supported
365+
366+
Args:
367+
source: the PipelineSource to build the pipeline off of
368+
Raises:
369+
- NotImplementedError: raised if the query contains a `cursor` or `limit_to_last`
370+
Returns:
371+
a Pipeline representing the query
372+
"""
373+
# use autoindexer to keep track of which field number to use for un-aliased fields
374+
autoindexer=itertools.count(start=1)
375+
exprs= [a._to_pipeline_expr(autoindexer) forainself._aggregations]
376+
returnself._nested_query._build_pipeline(source).aggregate(*exprs)

‎google/cloud/firestore_v1/base_client.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
Optional,
3838
Tuple,
3939
Union,
40+
Type,
4041
)
4142

4243
importgoogle.api_core.client_options
@@ -61,6 +62,8 @@
6162
fromgoogle.cloud.firestore_v1.bulk_writerimportBulkWriter, BulkWriterOptions
6263
fromgoogle.cloud.firestore_v1.field_pathimportrender_field_path
6364
fromgoogle.cloud.firestore_v1.services.firestoreimportclientasfirestore_client
65+
fromgoogle.cloud.firestore_v1.pipeline_sourceimportPipelineSource
66+
fromgoogle.cloud.firestore_v1.base_pipelineimport_BasePipeline
6467

6568
DEFAULT_DATABASE="(default)"
6669
"""str: The default database used in a :class:`~google.cloud.firestore_v1.client.Client`."""
@@ -502,6 +505,20 @@ def transaction(
502505
) ->BaseTransaction:
503506
raiseNotImplementedError
504507

508+
defpipeline(self) ->PipelineSource:
509+
"""
510+
Start a pipeline with this client.
511+
512+
Returns:
513+
:class:`~google.cloud.firestore_v1.pipeline_source.PipelineSource`:
514+
A pipeline that uses this client`
515+
"""
516+
raiseNotImplementedError
517+
518+
@property
519+
def_pipeline_cls(self) ->Type["_BasePipeline"]:
520+
raiseNotImplementedError
521+
505522

506523
def_reference_info(references: list) ->Tuple[list, dict]:
507524
"""Get information about document references.

‎google/cloud/firestore_v1/base_collection.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
fromgoogle.cloud.firestore_v1.async_documentimportAsyncDocumentReference
5050
fromgoogle.cloud.firestore_v1.documentimportDocumentReference
5151
fromgoogle.cloud.firestore_v1.field_pathimportFieldPath
52+
fromgoogle.cloud.firestore_v1.pipeline_sourceimportPipelineSource
5253
fromgoogle.cloud.firestore_v1.query_profileimportExplainOptions
5354
fromgoogle.cloud.firestore_v1.query_resultsimportQueryResultsList
5455
fromgoogle.cloud.firestore_v1.stream_generatorimportStreamGenerator
@@ -603,6 +604,21 @@ def find_nearest(
603604
distance_threshold=distance_threshold,
604605
)
605606

607+
def_build_pipeline(self, source: "PipelineSource"):
608+
"""
609+
Convert this query into a Pipeline
610+
611+
Queries containing a `cursor` or `limit_to_last` are not currently supported
612+
613+
Args:
614+
source: the PipelineSource to build the pipeline off o
615+
Raises:
616+
- NotImplementedError: raised if the query contains a `cursor` or `limit_to_last`
617+
Returns:
618+
a Pipeline representing the query
619+
"""
620+
returnself._query()._build_pipeline(source)
621+
606622

607623
def_auto_id() ->str:
608624
"""Generate a "random" automatically generated ID.

0 commit comments

Comments
 (0)