feat(datastore): Add Datastore Admin API samples (#4121)

* feat(datastore): Add Datastore Admin API samples

* add test for entities import/export

* add requirements into the README file

* fix imports order, add Storage roles tip, add retrying into requirements

* use backoff instead of retrying, add App Engine tip

* del retrying from requrements-test.txt

* fix imports order

* add timeouts

Co-authored-by: Leah E. Cole <[email protected]>
Co-authored-by: Takashi Matsuo <[email protected]>

Author: 3 people

datastore/cloud-client/README.rst

@@ -14,6 +14,18 @@ This directory contains samples for Google Cloud Datastore. `Google Cloud Datast
 
 .. _Google Cloud Datastore: https://cloud.google.com/datastore/docs
 
+
+
+
+Set environment variables:
+  `GOOGLE_CLOUD_PROJECT` - Google Cloud project id
+  `CLOUD_STORAGE_BUCKET` - Google Cloud Storage bucket name
+
+Roles to be set in your Service Account and App Engine default service account:
+  `Datastore Import Export Admin`, or `Cloud Datastore Owner`, or `Owner`
+  `Storage Admin`, or `Owner`
+
+
 Setup
 -------------------------------------------------------------------------------
 

datastore/cloud-client/README.rst.in

@@ -4,10 +4,19 @@ product:
   name: Google Cloud Datastore
   short_name: Cloud Datastore
   url: https://cloud.google.com/datastore/docs
-  description: > 
+  description: >
     `Google Cloud Datastore`_ is a NoSQL document database built for automatic
     scaling, high performance, and ease of application development.
 
+other_required_steps: >
+    Set environment variables:
+      `GOOGLE_CLOUD_PROJECT` - Google Cloud project id
+      `CLOUD_STORAGE_BUCKET` - Google Cloud Storage bucket name
+
+    Roles to be set in your Service Account and App Engine default service account:
+      `Datastore Import Export Admin`, or `Cloud Datastore Owner`, or `Owner`
+      `Storage Admin`, or `Owner`
+
 setup:
 - auth
 - install_deps

datastore/cloud-client/admin.py

@@ -0,0 +1,86 @@
+# Copyright 2016, Google, Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# [START datastore_admin_client_create]
+from google.cloud.datastore_admin_v1.gapic import datastore_admin_client
+
+
+def client_create():
+    """Creates a new Datastore admin client."""
+    client = datastore_admin_client.DatastoreAdminClient()
+
+    print("Admin client created\n")
+    return client
+# [END datastore_admin_client_create]
+
+
+# [START datastore_admin_entities_export]
+def export_entities(project_id, output_url_prefix):
+    """
+    Exports a copy of all or a subset of entities from
+    Datastore to another storage system, such as Cloud Storage.
+    """
+    # project_id = "project-id"
+    # output_url_prefix = "gs://bucket-name"
+    client = datastore_admin_client.DatastoreAdminClient()
+
+    op = client.export_entities(project_id, output_url_prefix)
+    response = op.result(timeout=200)
+
+    print("Entities were exported\n")
+    return response
+# [END datastore_admin_entities_export]
+
+
+# [START datastore_admin_entities_import]
+def import_entities(project_id, input_url):
+    """Imports entities into Datastore."""
+    # project_id := "project-id"
+    # input_url := "gs://bucket-name/overall-export-metadata-file"
+    client = datastore_admin_client.DatastoreAdminClient()
+
+    op = client.import_entities(project_id, input_url)
+    response = op.result(timeout=200)
+
+    print("Entities were imported\n")
+    return response
+# [END datastore_admin_entities_import]
+
+
+# [START datastore_admin_index_get]
+def get_index(project_id, index_id):
+    """Gets an index."""
+    # project_id := "my-project-id"
+    # index_id := "my-index"
+    client = datastore_admin_client.DatastoreAdminClient()
+    index = client.get_index(project_id, index_id)
+
+    print("Got index: %v\n", index.index_id)
+    return index
+# [END datastore_admin_index_get]
+
+
+# [START datastore_admin_index_list]
+def list_indexes(project_id):
+    """Lists the indexes."""
+    # project_id := "my-project-id"
+    client = datastore_admin_client.DatastoreAdminClient()
+
+    indexes = []
+    for index in client.list_indexes(project_id):
+        indexes.append(index)
+        print("Got index: %v\n", index.index_id)
+
+    print("Got list of indexes\n")
+    return indexes
+# [END datastore_admin_index_list]

datastore/cloud-client/admin_test.py

@@ -0,0 +1,48 @@
+# Copyright 2016, Google, Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+import backoff
+import pytest
+
+import admin
+
+
+PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"]
+BUCKET = os.environ["CLOUD_STORAGE_BUCKET"]
+
+
+class TestDatastoreAdminSnippets:
+    def test_client_create(self):
+        assert admin.client_create()
+
+    def test_get_index(self):
+        indexes = admin.list_indexes(PROJECT)
+        if not indexes:
+            pytest.skip(
+                "Skipping datastore test. At least "
+                "one index should present in database."
+            )
+
+        assert admin.get_index(PROJECT, indexes[0].index_id)
+
+    def test_list_index(self):
+        assert admin.list_indexes(PROJECT)
+
+    @backoff.on_exception(backoff.expo, AssertionError, max_tries=3, max_time=540000)
+    def test_export_import_entities(self):
+        response = admin.export_entities(PROJECT, "gs://" + BUCKET)
+        assert response
+
+        assert admin.import_entities(PROJECT, response.output_url)