Skip to content

PPL options #14902

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 54 commits into
base: cheryllin/ppl
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
54 commits
Select commit Hold shift + click to select a range
bd0ce8f
create main branch of ppl
cherylEnkidu Feb 27, 2025
84691ab
Add stage API
cherylEnkidu Feb 27, 2025
8519a5e
Add stage API
cherylEnkidu Mar 3, 2025
454893b
Add stage API
cherylEnkidu Mar 7, 2025
ed1a19b
API changes
cherylEnkidu Mar 12, 2025
819853c
Add cpp stages and expressions
wu-hui Mar 11, 2025
1be141b
Add more APIs
cherylEnkidu Mar 18, 2025
8fd5f8a
add Field
cherylEnkidu Mar 18, 2025
039b919
merge in main branch
cherylEnkidu Mar 19, 2025
5127bd0
Add UserDataReader support for constant and expr in pipelines
wu-hui Mar 17, 2025
14d2e5c
Add bridge
cherylEnkidu Mar 20, 2025
9895936
merge in push-pqrpmwtupmwv
cherylEnkidu Mar 20, 2025
5d701f8
Add UserDataReader support for constant and expr in pipelines
wu-hui Mar 17, 2025
b2af1c6
solve merge conflicts
cherylEnkidu Mar 20, 2025
cec7f6b
Fix merge conflicts
cherylEnkidu Mar 20, 2025
30b9251
solve conflicts
cherylEnkidu Mar 20, 2025
1ad093a
format and fix conflicts
cherylEnkidu Mar 20, 2025
b59a4fd
Remove placeholder for the test case
cherylEnkidu Mar 21, 2025
7d73155
Support tests
cherylEnkidu Mar 25, 2025
828bb12
Fix bug in decoding
cherylEnkidu Mar 27, 2025
d1d0aa4
Add some convert functions
cherylEnkidu Apr 1, 2025
113da45
add tests
cherylEnkidu Apr 2, 2025
39803e5
replace Any with sendable
cherylEnkidu Apr 2, 2025
6c78523
Update PipelineResult
cherylEnkidu Apr 3, 2025
9598764
add expression and stages
cherylEnkidu Apr 14, 2025
5a7f61f
merge changes
cherylEnkidu Apr 17, 2025
a5f558f
add tests
cherylEnkidu Apr 17, 2025
be5b0b2
change implementation details
cherylEnkidu Apr 24, 2025
e699c17
Add get for ppl result
cherylEnkidu Apr 24, 2025
daf1b68
revert test settings
cherylEnkidu Apr 24, 2025
5b2bd88
Change cmake version
cherylEnkidu Apr 25, 2025
04a16e2
merge in cmake settings
cherylEnkidu Apr 25, 2025
d324c5d
Merge branch 'cheryllin/ppl' into cheryllin/pplapi
cherylEnkidu Apr 26, 2025
23305c0
remove unused variable
cherylEnkidu Apr 28, 2025
6dd3515
revert settinggs
cherylEnkidu Apr 28, 2025
8ff71c2
add expressions
cherylEnkidu May 6, 2025
2372405
Add documentation to Pipeline.swift
cherylEnkidu May 6, 2025
c7ff4b7
merge in main
cherylEnkidu May 6, 2025
d5be6a0
Merge branch 'cheryllin/ppl' into cheryllin/pplapi
cherylEnkidu May 6, 2025
591816b
Fix replace with
cherylEnkidu May 7, 2025
0098829
change tests
cherylEnkidu May 8, 2025
4fb738d
Address feebacks 2
cherylEnkidu May 12, 2025
6ec8b6c
Address feedbacks 3
cherylEnkidu May 13, 2025
fd51b7b
change mutating fields
cherylEnkidu May 13, 2025
469b768
[Firestore] On SPM, import FirebaseFirestoreInternalWrapper (#14848)
ncooke3 May 13, 2025
ba94c32
add version restrictions
cherylEnkidu May 13, 2025
d1d1a89
Fix CI tests
cherylEnkidu May 15, 2025
65475b6
enable nightly build
cherylEnkidu May 15, 2025
f4b00ab
revert settings
cherylEnkidu May 15, 2025
f066ded
add tests and fix bugs
cherylEnkidu May 23, 2025
2437141
merge in base branch
cherylEnkidu May 23, 2025
71820c7
fix warning
cherylEnkidu May 23, 2025
7e5e71d
add option api
cherylEnkidu May 28, 2025
0583ecb
add documentations
cherylEnkidu Jun 2, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 38 additions & 10 deletions Firestore/Source/API/FIRPipelineBridge.mm
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

#include <memory>

#import "Firestore/Source/API/FIRCollectionReference+Internal.h"
#import "Firestore/Source/API/FIRDocumentReference+Internal.h"
#import "Firestore/Source/API/FIRFieldPath+Internal.h"
#import "Firestore/Source/API/FIRFirestore+Internal.h"
Expand All @@ -39,11 +40,13 @@
#include "Firestore/core/src/api/pipeline_result.h"
#include "Firestore/core/src/api/pipeline_snapshot.h"
#include "Firestore/core/src/api/stages.h"
#include "Firestore/core/src/util/comparison.h"
#include "Firestore/core/src/util/error_apple.h"
#include "Firestore/core/src/util/status.h"
#include "Firestore/core/src/util/string_apple.h"

using firebase::firestore::api::AddFields;
using firebase::firestore::api::AddStage;
using firebase::firestore::api::AggregateFunction;
using firebase::firestore::api::AggregateStage;
using firebase::firestore::api::CollectionGroupSource;
Expand All @@ -57,7 +60,6 @@
using firebase::firestore::api::Field;
using firebase::firestore::api::FindNearestStage;
using firebase::firestore::api::FunctionExpr;
using firebase::firestore::api::GenericStage;
using firebase::firestore::api::LimitStage;
using firebase::firestore::api::MakeFIRTimestamp;
using firebase::firestore::api::OffsetStage;
Expand All @@ -73,13 +75,21 @@
using firebase::firestore::api::Where;
using firebase::firestore::model::FieldPath;
using firebase::firestore::nanopb::SharedMessage;
using firebase::firestore::util::ComparisonResult;
using firebase::firestore::util::MakeCallback;
using firebase::firestore::util::MakeNSString;
using firebase::firestore::util::MakeString;
using firebase::firestore::util::ThrowInvalidArgument;

NS_ASSUME_NONNULL_BEGIN

inline std::string EnsureLeadingSlash(const std::string &path) {
if (!path.empty() && path[0] == '/') {
return path;
}
return "/" + path;
}

@implementation FIRExprBridge
@end

Expand Down Expand Up @@ -216,10 +226,19 @@ @implementation FIRCollectionSourceStageBridge {
std::shared_ptr<CollectionSource> collection_source;
}

- (id)initWithPath:(NSString *)path {
- (id)initWithRef:(FIRCollectionReference *)ref firestore:(FIRFirestore *)db {
self = [super init];
if (self) {
collection_source = std::make_shared<CollectionSource>(MakeString(path));
if (ref.firestore.databaseID.CompareTo(db.databaseID) != ComparisonResult::Same) {
ThrowInvalidArgument(
"Invalid CollectionReference. The project ID (\"%s\") or the database (\"%s\") does not "
"match "
"the project ID (\"%s\") and database (\"%s\") of the target database of this Pipeline.",
ref.firestore.databaseID.project_id(), ref.firestore.databaseID.database_id(),
db.databaseID.project_id(), db.databaseID.project_id());
}
collection_source =
std::make_shared<CollectionSource>(EnsureLeadingSlash(MakeString(ref.path)));
}
return self;
}
Expand Down Expand Up @@ -270,12 +289,21 @@ @implementation FIRDocumentsSourceStageBridge {
std::shared_ptr<DocumentsSource> cpp_document_source;
}

- (id)initWithDocuments:(NSArray<NSString *> *)documents {
- (id)initWithDocuments:(NSArray<FIRDocumentReference *> *)documents firestore:(FIRFirestore *)db {
self = [super init];
if (self) {
std::vector<std::string> cpp_documents;
for (NSString *doc in documents) {
cpp_documents.push_back(MakeString(doc));
for (FIRDocumentReference *doc in documents) {
if (doc.firestore.databaseID.CompareTo(db.databaseID) != ComparisonResult::Same) {
ThrowInvalidArgument("Invalid DocumentReference. The project ID (\"%s\") or the database "
"(\"%s\") does not match "
"the project ID (\"%s\") and database (\"%s\") of the target database "
"of this Pipeline.",
doc.firestore.databaseID.project_id(),
doc.firestore.databaseID.database_id(), db.databaseID.project_id(),
db.databaseID.project_id());
}
cpp_documents.push_back(EnsureLeadingSlash(MakeString(doc.path)));
}
cpp_document_source = std::make_shared<DocumentsSource>(std::move(cpp_documents));
}
Expand Down Expand Up @@ -754,12 +782,12 @@ - (id)initWithField:(FIRExprBridge *)field indexField:(NSString *_Nullable)index

@end

@implementation FIRGenericStageBridge {
@implementation FIRAddStageBridge {
NSString *_name;
NSArray<FIRExprBridge *> *_params;
NSDictionary<NSString *, FIRExprBridge *> *_Nullable _options;
Boolean isUserDataRead;
std::shared_ptr<GenericStage> cpp_generic_stage;
std::shared_ptr<AddStage> cpp_generic_stage;
}

- (id)initWithName:(NSString *)name
Expand Down Expand Up @@ -787,8 +815,8 @@ - (id)initWithName:(NSString *)name
cpp_options[MakeString(key)] = [_options[key] cppExprWithReader:reader];
}
}
cpp_generic_stage = std::make_shared<GenericStage>(MakeString(_name), std::move(cpp_params),
std::move(cpp_options));
cpp_generic_stage = std::make_shared<AddStage>(MakeString(_name), std::move(cpp_params),
std::move(cpp_options));
}

isUserDataRead = YES;
Expand Down
8 changes: 4 additions & 4 deletions Firestore/Source/Public/FirebaseFirestore/FIRPipelineBridge.h
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ NS_SWIFT_SENDABLE
NS_SWIFT_NAME(CollectionSourceStageBridge)
@interface FIRCollectionSourceStageBridge : FIRStageBridge

- (id)initWithPath:(NSString *)path;
- (id)initWithRef:(FIRCollectionReference *)ref firestore:(FIRFirestore *)db;

@end

Expand All @@ -94,7 +94,7 @@ NS_SWIFT_SENDABLE
NS_SWIFT_NAME(DocumentsSourceStageBridge)
@interface FIRDocumentsSourceStageBridge : FIRStageBridge

- (id)initWithDocuments:(NSArray<NSString *> *)documents;
- (id)initWithDocuments:(NSArray<FIRDocumentReference *> *)documents firestore:(FIRFirestore *)db;

@end

Expand Down Expand Up @@ -195,8 +195,8 @@ NS_SWIFT_NAME(UnnestStageBridge)
@end

NS_SWIFT_SENDABLE
NS_SWIFT_NAME(GenericStageBridge)
@interface FIRGenericStageBridge : FIRStageBridge
NS_SWIFT_NAME(AddStageBridge)
@interface FIRAddStageBridge : FIRStageBridge
- (id)initWithName:(NSString *)name
params:(NSArray<FIRExprBridge *> *)params
options:(NSDictionary<NSString *, FIRExprBridge *> *_Nullable)options;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

public struct ExplainOptions: OptionProtocol, Sendable, Equatable, Hashable {
public struct Mode: Sendable, Equatable, Hashable {
let rawValue: String

public static let execute = Mode(rawValue: "execute")
public static let explain = Mode(rawValue: "explain")
public static let analyze = Mode(rawValue: "analyze")

init(rawValue: String) {
self.rawValue = rawValue
}
}

public struct OutputFormat: Sendable, Equatable, Hashable {
let rawValue: String

public static let text = OutputFormat(rawValue: "text")
public static let json = OutputFormat(rawValue: "json")
public static let `struct` = OutputFormat(rawValue: "struct")

init(rawValue: String) {
self.rawValue = rawValue
}
}

public struct Verbosity: Sendable, Equatable, Hashable {
let rawValue: String

public static let summaryOnly = Verbosity(rawValue: "summary_only")
public static let executionTree = Verbosity(rawValue: "execution_tree")

init(rawValue: String) {
self.rawValue = rawValue
}
}

public struct Profiles: Sendable, Equatable, Hashable {
let rawValue: String

public static let latency = Profiles(rawValue: "latency")
public static let recordsCount = Profiles(rawValue: "records_count")
public static let bytesThroughput = Profiles(rawValue: "bytes_throughput")

init(rawValue: String) {
self.rawValue = rawValue
}
}

public let mode: Mode?
public let outputFormat: OutputFormat?
public let verbosity: Verbosity?
public let indexRecommendation: Bool?
public let profiles: Profiles?
public let redact: Bool?

public init(mode: Mode? = nil,
outputFormat: OutputFormat? = nil,
verbosity: Verbosity? = nil,
indexRecommendation: Bool? = nil,
profiles: Profiles? = nil,
redact: Bool? = nil) {
self.mode = mode
self.outputFormat = outputFormat
self.verbosity = verbosity
self.indexRecommendation = indexRecommendation
self.profiles = profiles
self.redact = redact
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

/// A container for providing arbitrary, backend-specific options to a pipeline.
///
/// Use this to pass options that are not explicitly defined in the other option structs.
public struct CustomOptions: OptionProtocol {
var values: [String: Sendable]
/// Creates a set of custom options from a dictionary.
/// - Parameter values: A dictionary containing the custom options.
public init(_ values: [String: Sendable] = [:]) {
self.values = values
}
}
25 changes: 25 additions & 0 deletions Firestore/Swift/Source/SwiftAPI/Pipeline/Options/IndexMode.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

/// Specifies which indexes the pipeline should use for execution.
public struct IndexMode: Sendable, Equatable, Hashable {
let rawValue: String

/// Use the set of indexes recommended by the backend.
public static let recommended = IndexMode(rawValue: "recommended")

init(rawValue: String) {
self.rawValue = rawValue
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

/// A protocol that all pipeline option types must conform to.
public protocol OptionProtocol: Sendable {}
23 changes: 17 additions & 6 deletions Firestore/Swift/Source/SwiftAPI/Pipeline/Pipeline.swift
Original file line number Diff line number Diff line change
Expand Up @@ -107,9 +107,19 @@ public struct Pipeline: @unchecked Sendable {
/// }
/// ```
///
/// - Parameters:
/// - explainOptions: Configures the execution plan report for debugging and optimization. Use
/// it to analyze performance, get index recommendations, and control the verbosity and format of
/// the output.
/// - indexMode: Specifies which indexes to use for the pipeline, such as the `.recommended`
/// set.
/// - customOptions: A dictionary for passing any other backend-specific or advanced options to
/// the pipeline execution.
/// - Throws: An error if the pipeline execution fails on the backend.
/// - Returns: A `PipelineSnapshot` containing the result of the pipeline execution.
public func execute() async throws -> PipelineSnapshot {
public func execute(explainOptions: ExplainOptions? = nil,
indexMode: IndexMode? = nil,
customOptions: CustomOptions? = nil) async throws -> PipelineSnapshot {
return try await withCheckedThrowingContinuation { continuation in
self.bridge.execute { result, error in
if let error {
Expand Down Expand Up @@ -147,7 +157,8 @@ public struct Pipeline: @unchecked Sendable {
/// - Parameter field: The first field to add to the documents, specified as a `Selectable`.
/// - Parameter additionalFields: Optional additional fields to add, specified as `Selectable`s.
/// - Returns: A new `Pipeline` object with this stage appended.
public func addFields(_ field: Selectable, _ additionalFields: Selectable...) -> Pipeline {
public func addFields(_ field: Selectable, _ additionalFields: Selectable...,
customOptions: [String: Sendable]? = nil) -> Pipeline {
let fields = [field] + additionalFields
return Pipeline(stages: stages + [AddFields(fields: fields)], db: db)
}
Expand Down Expand Up @@ -699,7 +710,7 @@ public struct Pipeline: @unchecked Sendable {
/// ```swift
/// // let pipeline: Pipeline = ...
/// // Example: Assuming a hypothetical backend stage "customFilterV2".
/// let genericPipeline = pipeline.genericStage(
/// let genericPipeline = pipeline.addStage(
/// name: "customFilterV2",
/// params: [Field("userScore"), 80], // Ordered parameters.
/// options: ["mode": "strict", "logLevel": 2] // Optional named parameters.
Expand All @@ -712,10 +723,10 @@ public struct Pipeline: @unchecked Sendable {
/// - params: An array of ordered, `Sendable` parameters for the stage.
/// - options: Optional dictionary of named, `Sendable` parameters.
/// - Returns: A new `Pipeline` object with this stage appended.
public func genericStage(name: String, params: [Sendable],
options: [String: Sendable]? = nil) -> Pipeline {
public func addStage(name: String, params: [Sendable],
options: [String: Sendable]? = nil) -> Pipeline {
return Pipeline(
stages: stages + [GenericStage(name: name, params: params, options: options)],
stages: stages + [AddStage(name: name, params: params, options: options)],
db: db
)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ public struct PipelineSnapshot: Sendable {
public let pipeline: Pipeline

/// An array of all the results in the `PipelineSnapshot`.
let results_cache: [PipelineResult]
public let results: [PipelineResult]

/// The time at which the pipeline producing this result was executed.
public let executionTime: Timestamp
Expand All @@ -36,10 +36,6 @@ public struct PipelineSnapshot: Sendable {
self.bridge = bridge
self.pipeline = pipeline
executionTime = self.bridge.execution_time
results_cache = self.bridge.results.map { PipelineResult($0) }
}

public func results() -> [PipelineResult] {
return results_cache
results = self.bridge.results.map { PipelineResult($0) }
}
}
Loading
Loading