Merge pull request #2410 from firebase/next

Release firestore-send-email and firestore-bigquery-export extensions

Author: cabljac

firestore-bigquery-export/CHANGELOG.md

@@ -1,3 +1,7 @@
+## Version 0.2.4
+
+feat: Add bigquery dataset locations and remove duplicates
+
 ## Version 0.2.3
 
 fix: pass full document resource name to bigquery

firestore-bigquery-export/extension.yaml

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 name: firestore-bigquery-export
-version: 0.2.3
+version: 0.2.4
 specVersion: v1beta
 
 displayName: Stream Firestore to BigQuery
@@ -163,18 +163,50 @@ params:
         value: asia-northeast2
       - label: Seoul (asia-northeast3)
         value: asia-northeast3
-      - label: Singapore (asia-southeast1)
-        value: asia-southeast1
       - label: Sydney (australia-southeast1)
         value: australia-southeast1
-      - label: Taiwan (asia-east1)
-        value: asia-east1
       - label: Tokyo (asia-northeast1)
         value: asia-northeast1
       - label: United States (multi-regional)
         value: us
       - label: Europe (multi-regional)
         value: eu
+      - label: Johannesburg (africa-south1)
+        value: africa-south1
+      - label: Tel Aviv (me-west1)
+        value: me-west1
+      - label: Doha (me-central1)
+        value: me-central1
+      - label: Dammam (me-central2)
+        value: me-central2
+      - label: Zürich (europe-west6)
+        value: europe-west6
+      - label: Turin (europe-west12)
+        value: europe-west12
+      - label: Stockholm (europe-north2)
+        value: europe-north2
+      - label: Paris (europe-west9)
+        value: europe-west9
+      - label: Milan (europe-west8)
+        value: europe-west8
+      - label: Madrid (europe-southwest1)
+        value: europe-southwest1
+      - label: Berlin (europe-west10)
+        value: europe-west10
+      - label: Melbourne (australia-southeast2)
+        value: australia-southeast2
+      - label: Delhi (asia-south2)
+        value: asia-south2
+      - label: Toronto (northamerica-northeast2)
+        value: northamerica-northeast2
+      - label: Santiago (southamerica-west1)
+        value: southamerica-west1
+      - label: Mexico (northamerica-south1)
+        value: northamerica-south1
+      - label: Dallas (us-south1)
+        value: us-south1
+      - label: Columbus, Ohio (us-east5)
+        value: us-east5
     default: us
     required: true
     immutable: true

firestore-bigquery-export/firestore-bigquery-change-tracker/package-lock.json

@@ -105,7 +105,7 @@ const run = async (): Promise<number> => {
     cursor = await firebase.firestore().doc(cursorDocumentId).get();
     logs.resumingImport(config, cursorDocumentId);
   }
-  const totalRowsImported = runSingleThread(dataSink, config, cursor);
+  const totalRowsImported = await runSingleThread(dataSink, config, cursor);
   try {
     await unlink(cursorPositionFile);
   } catch (e) {

firestore-bigquery-export/scripts/import/src/index.ts

@@ -47,6 +47,45 @@ export function getQuery(
   return query;
 }
 
+async function verifyCollectionExists(config: CliConfig): Promise<void> {
+  const { sourceCollectionPath, queryCollectionGroup } = config;
+
+  try {
+    if (queryCollectionGroup) {
+      const sourceCollectionPathParts = sourceCollectionPath.split("/");
+      const collectionName =
+        sourceCollectionPathParts[sourceCollectionPathParts.length - 1];
+      const snapshot = await firebase
+        .firestore()
+        .collectionGroup(collectionName)
+        .limit(1)
+        .get();
+      if (snapshot.empty) {
+        throw new Error(
+          `No documents found in collection group: ${collectionName}`
+        );
+      }
+    } else {
+      const snapshot = await firebase
+        .firestore()
+        .collection(sourceCollectionPath)
+        .limit(1)
+        .get();
+      if (snapshot.empty) {
+        throw new Error(
+          `Collection does not exist or is empty: ${sourceCollectionPath}`
+        );
+      }
+    }
+  } catch (error) {
+    if (error instanceof Error) {
+      error.message = `Failed to access collection: ${error.message}`;
+      throw error;
+    }
+    throw error;
+  }
+}
+
 export async function runSingleThread(
   dataSink: FirestoreBigQueryEventHistoryTracker,
   config: CliConfig,
@@ -56,6 +95,8 @@ export async function runSingleThread(
 ) {
   let totalRowsImported = 0;
 
+  await verifyCollectionExists(config);
+
   await initializeFailedBatchOutput(config.failedBatchOutput);
 
   while (true) {

firestore-bigquery-export/scripts/import/src/run-single-thread.ts

@@ -1,3 +1,11 @@
+## Version 0.2.1
+
+fix: return info on sendgrid messages
+
+feat: include sendgridQueueId if using that provider
+
+fix: improve validation of triggering firestore objects
+
 ## Version 0.2.0
 
 feat: use v2 firestore trigger

firestore-send-email/CHANGELOG.md

@@ -44,7 +44,7 @@ See the [official documentation](https://firebase.google.com/docs/extensions/off
 
 When using SendGrid (`SMTP_CONNECTION_URI` includes `sendgrid.net`), you can assign categories to your emails.
 
-## Example JSON with Categories:
+##### Example JSON with Categories:
 ```json
 {
   "to": ["[email protected]"],
@@ -61,6 +61,30 @@ Add this document to the Firestore mail collection to send categorized emails.
 
 For more details, see the [SendGrid Categories documentation](https://docs.sendgrid.com/ui/sending-email/categories).
 
+#### Understanding SendGrid Email IDs
+
+When an email is sent successfully, the extension tracks two different IDs in the delivery information:
+
+- **Queue ID**: This is SendGrid's internal queue identifier (from the `x-message-id` header). It's useful for tracking the email within SendGrid's system.
+- **Message ID**: This is the RFC-2822 Message-ID header, which is a standard email identifier used across email systems.
+
+You can find both IDs in the `delivery.info` field of your email document after successful delivery:
+
+```json
+{
+  "delivery": {
+    "info": {
+      "messageId": "<[email protected]>",
+      "sendgridQueueId": "sendgrid-queue-id",
+      "accepted": ["[email protected]"],
+      "rejected": [],
+      "pending": [],
+      "response": "status=202"
+    }
+  }
+}
+```
+
 ### Automatic Deletion of Email Documents
 
 To use Firestore's TTL feature for automatic deletion of expired email documents, the extension provides several configuration parameters.

firestore-send-email/POSTINSTALL.md

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 name: firestore-send-email
-version: 0.2.0
+version: 0.2.1
 specVersion: v1beta
 
 displayName: Trigger Email from Firestore