Skip to content

Commit 884eba7

Browse files
author
Shun Fan
committed
Merge pull request #21 from GoogleCloudPlatform/sfan-sts03
Add storagetransfer samples
2 parents 105bcfd + d286b20 commit 884eba7

File tree

16 files changed

+1249
-20
lines changed

16 files changed

+1249
-20
lines changed

‎cloud-storage/storage-transfer/README.md

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
# Transfer Service sample using Java
2+
3+
This app creates two types of transfers using the Transfer Service tool.
4+
5+
## Prerequisites
6+
7+
1. Set up a project on Google Developers Console.
8+
1. Go to the [Developers Console](https://cloud.google.com/console) and create or select your project.
9+
You will need the project ID later.
10+
1. Within Developers Console, select APIs & auth > Credentials.
11+
1. Select Add credentials > Service account > JSON key.
12+
1. Set the environment variable GOOGLE_APPLICATION_CREDENTIALS to point to your JSON key.
13+
1. Add the Storage Transfer service account, [email protected] as an
14+
editor of your project.
15+
1. Set up gcloud for application default credentials.
16+
1. `gcloud components update`
17+
1. `gcloud auth login`
18+
1. `gcloud config set project PROJECT_ID`
19+
20+
## Transfer from Amazon S3 to Google Cloud Storage
21+
22+
Creating a one-time transfer from Amazon S3 to Google Cloud Storage.
23+
1. Set up data sink.
24+
1. Go to the Developers Console and create a bucket under Cloud Storage > Storage Browser.
25+
1. Set up data source.
26+
1. Go to AWS Management Console and create a bucket.
27+
1. Under Security Credentials, create an IAM User with access to the bucket.
28+
1. Create an Access Key for the user. Note the Access Key ID and Secret Access Key.
29+
1. In AwsRequester.java, fill in the user-provided constants.
30+
1. Run with `mvn compile` and
31+
`mvn exec:java -Dexec.mainClass="com.google.cloud.storage.storagetransfer.samples.AwsRequester"`
32+
1. Note the job ID in the returned Transfer Job.
33+
34+
## Transfer data from a standard Cloud Storage bucket to a Cloud Storage Nearline bucket
35+
36+
Creating a daily transfer from a standard Cloud Storage bucket to a Cloud Storage Nearline
37+
bucket for files untouched for 30 days.
38+
1. Set up data sink.
39+
1. Go to the Developers Console and create a bucket under Cloud Storage > Storage Browser.
40+
1. Select Nearline for Storage Class.
41+
1. Set up data source.
42+
1. Go to the Developers Console and create a bucket under Cloud Storage > Storage Browser.
43+
1. In NearlineRequester.java, fill in the user-provided constants.
44+
1. Run with `mvn compile` and
45+
`mvn exec:java -Dexec.mainClass="com.google.cloud.storage.storagetransfer.samples.NearlineRequester"`
46+
1. Note the job ID in the returned Transfer Job.
47+
48+
## Checking the status of a transfer
49+
50+
1. In RequestChecker.java, fill in the user-provided constants. Use the Job Name you recorded earlier.
51+
1. Run with `mvn compile` and
52+
`mvn exec:java -Dexec.mainClass="com.google.cloud.storage.storagetransfer.samples.RequestChecker"`

‎cloud-storage/storage-transfer/pom.xml

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
~ Copyright (c) 2015 Google Inc. All Rights Reserved.
4+
~
5+
~ Licensed under the Apache License, Version 2.0 (the "License"); you
6+
~ may not use this file except in compliance with the License. You may
7+
~ obtain a copy of the License at
8+
~
9+
~ http://www.apache.org/licenses/LICENSE-2.0
10+
~
11+
~ Unless required by applicable law or agreed to in writing, software
12+
~ distributed under the License is distributed on an "AS IS" BASIS,
13+
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
14+
~ implied. See the License for the specific language governing
15+
~ permissions and limitations under the License.
16+
-->
17+
18+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
19+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
20+
<modelVersion>4.0.0</modelVersion>
21+
<parent>
22+
<artifactId>doc-samples</artifactId>
23+
<groupId>com.google.cloud</groupId>
24+
<version>1.0.0</version>
25+
<relativePath>../..</relativePath>
26+
</parent>
27+
28+
<groupId>com.google.storagetransfer.samples</groupId>
29+
<artifactId>transfersample</artifactId>
30+
<version>0.1</version>
31+
<packaging>jar</packaging>
32+
33+
<name>transfersample</name>
34+
<url>http://maven.apache.org</url>
35+
36+
<properties>
37+
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
38+
<powermock.version>1.6.2</powermock.version>
39+
</properties>
40+
41+
<dependencies>
42+
<dependency>
43+
<groupId>com.google.apis</groupId>
44+
<artifactId>google-api-services-storagetransfer</artifactId>
45+
<version>v1-rev1-1.20.0</version>
46+
</dependency>
47+
48+
<!-- Test Dependencies -->
49+
<dependency>
50+
<groupId>org.powermock</groupId>
51+
<artifactId>powermock-module-junit4</artifactId>
52+
<version>${powermock.version}</version>
53+
<scope>test</scope>
54+
</dependency>
55+
<dependency>
56+
<groupId>org.powermock</groupId>
57+
<artifactId>powermock-api-mockito</artifactId>
58+
<version>${powermock.version}</version>
59+
<scope>test</scope>
60+
</dependency>
61+
</dependencies>
62+
<build>
63+
<plugins>
64+
<plugin>
65+
<groupId>org.apache.maven.plugins</groupId>
66+
<artifactId>maven-compiler-plugin</artifactId>
67+
<version>3.1</version>
68+
<configuration>
69+
<source>1.7</source>
70+
<target>1.7</target>
71+
</configuration>
72+
</plugin>
73+
</plugins>
74+
</build>
75+
</project>

‎cloud-storage/storage-transfer/src/main/java/com/google/cloud/storage/storagetransfer/samples/AwsRequester.java

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
/**
2+
* Copyright 2015 Google Inc. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.storage.storagetransfer.samples;
18+
19+
import com.google.api.services.storagetransfer.Storagetransfer;
20+
import com.google.api.services.storagetransfer.model.AwsAccessKey;
21+
import com.google.api.services.storagetransfer.model.AwsS3Data;
22+
import com.google.api.services.storagetransfer.model.Date;
23+
import com.google.api.services.storagetransfer.model.GcsData;
24+
import com.google.api.services.storagetransfer.model.Schedule;
25+
import com.google.api.services.storagetransfer.model.TimeOfDay;
26+
import com.google.api.services.storagetransfer.model.TransferJob;
27+
import com.google.api.services.storagetransfer.model.TransferSpec;
28+
29+
import java.io.IOException;
30+
import java.util.logging.Logger;
31+
32+
/**
33+
* Creates a one-off transfer job from Amazon S3 to Google Cloud Storage.
34+
*/
35+
public final class AwsRequester {
36+
37+
private static final String JOB_DESC = "YOUR DESCRIPTION";
38+
private static final String PROJECT_ID = "YOUR_PROJECT_ID";
39+
private static final String AWS_SOURCE_NAME = "YOUR SOURCE BUCKET";
40+
private static final String AWS_ACCESS_KEY_ID = "YOUR_ACCESS_KEY_ID";
41+
private static final String AWS_SECRET_ACCESS_KEY = "YOUR_SECRET_ACCESS_KEY";
42+
private static final String GCS_SINK_NAME = "YOUR_SINK_BUCKET";
43+
44+
/**
45+
* Specify times below using US Pacific Time Zone.
46+
*/
47+
private static final String START_DATE = "YYYY-MM-DD";
48+
private static final String START_TIME = "HH:MM:SS";
49+
50+
private static final Logger LOG = Logger.getLogger(AwsRequester.class.getName());
51+
52+
/**
53+
* Creates and executes a request for a TransferJob from Amazon S3 to Cloud Storage.
54+
*
55+
* @return the response TransferJob if the request is successful
56+
* @throws InstantiationException
57+
* if instantiation fails when building the TransferJob
58+
* @throws IllegalAccessException
59+
* if an illegal access occurs when building the TransferJob
60+
* @throws IOException
61+
* if the client failed to complete the request
62+
*/
63+
public static TransferJob createAwsTransferJob() throws InstantiationException,
64+
IllegalAccessException, IOException {
65+
Date date = TransferJobUtils.createDate(START_DATE);
66+
TimeOfDay time = TransferJobUtils.createTimeOfDay(START_TIME);
67+
TransferJob transferJob = TransferJob.class
68+
.newInstance()
69+
.setDescription(JOB_DESC)
70+
.setProjectId(PROJECT_ID)
71+
.setTransferSpec(
72+
TransferSpec.class
73+
.newInstance()
74+
.setAwsS3DataSource(
75+
AwsS3Data.class
76+
.newInstance()
77+
.setBucketName(AWS_SOURCE_NAME)
78+
.setAwsAccessKey(
79+
AwsAccessKey.class.newInstance().setAccessKeyId(AWS_ACCESS_KEY_ID)
80+
.setSecretAccessKey(AWS_SECRET_ACCESS_KEY)))
81+
.setGcsDataSink(GcsData.class.newInstance().setBucketName(GCS_SINK_NAME)))
82+
.setSchedule(
83+
Schedule.class.newInstance().setScheduleStartDate(date).setScheduleEndDate(date)
84+
.setStartTimeOfDay(time)).setStatus("ENABLED");
85+
86+
Storagetransfer client = TransferClientCreator.createStorageTransferClient();
87+
return client.transferJobs().create(transferJob).execute();
88+
}
89+
90+
/**
91+
* Output the contents of a successfully created TransferJob.
92+
*
93+
* @param args
94+
* arguments from the command line
95+
*/
96+
public static void main(String[] args) {
97+
try {
98+
TransferJob responseT = createAwsTransferJob();
99+
LOG.info("Return transferJob: " + responseT.toPrettyString());
100+
} catch (Exception e) {
101+
e.printStackTrace();
102+
}
103+
}
104+
}

‎cloud-storage/storage-transfer/src/main/java/com/google/cloud/storage/storagetransfer/samples/NearlineRequester.java

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
/**
2+
* Copyright 2015 Google Inc. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.storage.storagetransfer.samples;
18+
19+
import com.google.api.services.storagetransfer.Storagetransfer;
20+
import com.google.api.services.storagetransfer.model.Date;
21+
import com.google.api.services.storagetransfer.model.GcsData;
22+
import com.google.api.services.storagetransfer.model.ObjectConditions;
23+
import com.google.api.services.storagetransfer.model.Schedule;
24+
import com.google.api.services.storagetransfer.model.TimeOfDay;
25+
import com.google.api.services.storagetransfer.model.TransferJob;
26+
import com.google.api.services.storagetransfer.model.TransferOptions;
27+
import com.google.api.services.storagetransfer.model.TransferSpec;
28+
29+
import java.io.IOException;
30+
import java.util.logging.Logger;
31+
32+
/**
33+
* Creates a daily transfer from a standard Cloud Storage bucket to a Cloud Storage Nearline
34+
* bucket for files untouched for 30 days.
35+
*/
36+
public final class NearlineRequester {
37+
38+
private static final String JOB_DESC = "YOUR DESCRIPTION";
39+
private static final String PROJECT_ID = "YOUR_PROJECT_ID";
40+
private static final String GCS_SOURCE_NAME = "YOUR_SOURCE_BUCKET";
41+
private static final String NEARLINE_SINK_NAME = "YOUR_SINK_BUCKET";
42+
43+
/**
44+
* Specify times below using US Pacific Time Zone.
45+
*/
46+
private static final String START_DATE = "YYYY-MM-DD";
47+
private static final String START_TIME = "HH:MM:SS";
48+
49+
private static final Logger LOG = Logger.getLogger(AwsRequester.class.getName());
50+
51+
/**
52+
* Creates and executes a request for a TransferJob to Cloud Storage Nearline.
53+
*
54+
* @return the response TransferJob if the request is successful
55+
* @throws InstantiationException
56+
* if instantiation fails when building the TransferJob
57+
* @throws IllegalAccessException
58+
* if an illegal access occurs when building the TransferJob
59+
* @throws IOException
60+
* if the client failed to complete the request
61+
*/
62+
public static TransferJob createNearlineTransferJob() throws InstantiationException,
63+
IllegalAccessException, IOException {
64+
Date date = TransferJobUtils.createDate(START_DATE);
65+
TimeOfDay time = TransferJobUtils.createTimeOfDay(START_TIME);
66+
TransferJob transferJob = TransferJob.class
67+
.newInstance()
68+
.setDescription(JOB_DESC)
69+
.setProjectId(PROJECT_ID)
70+
.setTransferSpec(
71+
TransferSpec.class
72+
.newInstance()
73+
.setGcsDataSource(GcsData.class.newInstance().setBucketName(GCS_SOURCE_NAME))
74+
.setGcsDataSink(GcsData.class.newInstance().setBucketName(NEARLINE_SINK_NAME))
75+
.setObjectConditions(
76+
ObjectConditions.class.newInstance().setMinTimeElapsedSinceLastModification("2592000s"))
77+
.setTransferOptions(
78+
TransferOptions.class.newInstance().setDeleteObjectsFromSourceAfterTransfer(true)))
79+
.setSchedule(Schedule.class.newInstance().setScheduleStartDate(date)
80+
.setStartTimeOfDay(time))
81+
.setStatus("ENABLED");
82+
83+
Storagetransfer client = TransferClientCreator.createStorageTransferClient();
84+
return client.transferJobs().create(transferJob).execute();
85+
}
86+
87+
/**
88+
* Output the contents of a successfully created TransferJob.
89+
*
90+
* @param args
91+
* arguments from the command line
92+
*/
93+
public static void main(String[] args) {
94+
try {
95+
TransferJob responseT = createNearlineTransferJob();
96+
LOG.info("Return transferJob: " + responseT.toPrettyString());
97+
} catch (Exception e) {
98+
e.printStackTrace();
99+
}
100+
}
101+
}

0 commit comments

Comments
 (0)