Skip to content

Commit aa965b1

Browse files
pixccpixcc
authored
Merge ae4d162 into 7a00bd0 
2 parents 7a00bd0 + ae4d162 commit aa965b1

File tree

7 files changed

+148
-1
lines changed

7 files changed

+148
-1
lines changed

‎ydb/core/wrappers/s3_storage_config.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,13 +171,15 @@ IExternalStorageOperator::TPtr TS3ExternalStorageConfig::DoConstructStorageOpera
171171
TS3ExternalStorageConfig::TS3ExternalStorageConfig(const Ydb::Import::ImportFromS3Settings& settings)
172172
: Config(ConfigFromSettings(settings))
173173
, Credentials(CredentialsFromSettings(settings))
174+
, UseVirtualAddressing(!settings.disable_virtual_addressing())
174175
{
175176
Bucket = settings.bucket();
176177
}
177178

178179
TS3ExternalStorageConfig::TS3ExternalStorageConfig(const Ydb::Export::ExportToS3Settings& settings)
179180
: Config(ConfigFromSettings(settings))
180181
, Credentials(CredentialsFromSettings(settings))
182+
, UseVirtualAddressing(!settings.disable_virtual_addressing())
181183
{
182184
Bucket = settings.bucket();
183185
}

‎ydb/public/lib/ydb_cli/commands/ydb_service_import.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,7 @@ int TCommandImportFromS3::Run(TConfig& config) {
128128
settings.Bucket(AwsBucket);
129129
settings.AccessKey(AwsAccessKey);
130130
settings.SecretKey(AwsSecretKey);
131+
settings.UseVirtualAddressing(UseVirtualAddressing);
131132

132133
if (Description) {
133134
settings.Description(Description);

‎ydb/public/lib/ydb_cli/common/aws.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,12 +70,13 @@ class TS3ClientWrapper : public IS3ClientWrapper {
7070
} else {
7171
throw TMisuseException() << "\"" << settings.Scheme_ << "\" scheme type is not supported";
7272
}
73+
config.useVirtualAddressing = settings.UseVirtualAddressing_;
7374

7475
Client = std::make_unique<Aws::S3::S3Client>(
7576
Aws::Auth::AWSCredentials(settings.AccessKey_, settings.SecretKey_),
7677
config,
7778
Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
78-
true);
79+
settings.UseVirtualAddressing_);
7980
}
8081

8182
TListS3Result ListObjectKeys(const TString& prefix, const std::optional<TString>& token) override {

‎ydb/tests/functional/backup/s3_path_style/s3_path_style_backup_ut.cpp‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
#include <ydb/public/sdk/cpp/client/ydb_driver/driver.h>
2+
#include <ydb/public/sdk/cpp/client/ydb_export/export.h>
3+
#include <ydb/public/sdk/cpp/client/ydb_import/import.h>
4+
#include <ydb/public/sdk/cpp/client/ydb_operation/operation.h>
5+
#include <ydb/public/sdk/cpp/client/ydb_table/table.h>
6+
#include <ydb/library/testlib/s3_recipe_helper/s3_recipe_helper.h>
7+
8+
#include <library/cpp/testing/unittest/registar.h>
9+
#include <util/system/env.h>
10+
11+
using namespace NYdb;
12+
using namespace NYdb::NTable;
13+
14+
namespace {
15+
template<typename TOp>
16+
void WaitOp(TMaybe<TOperation>& op, NOperation::TOperationClient& opClient) {
17+
int attempt = 20;
18+
while (--attempt) {
19+
op = opClient.Get<TOp>(op->Id()).GetValueSync();
20+
if (op->Ready()) {
21+
break;
22+
}
23+
Sleep(TDuration::Seconds(1));
24+
}
25+
UNIT_ASSERT_C(attempt, "Unable to wait completion of backup");
26+
}
27+
}
28+
29+
Y_UNIT_TEST_SUITE(S3PathStyleBackup)
30+
{
31+
Y_UNIT_TEST(DisableVirtualAddressing)
32+
{
33+
TString connectionString = GetEnv("YDB_ENDPOINT") + "/?database=" + GetEnv("YDB_DATABASE");
34+
auto config = TDriverConfig(connectionString);
35+
auto driver = TDriver(config);
36+
auto tableClient = TTableClient(driver);
37+
auto session = tableClient.GetSession().GetValueSync().GetSession();
38+
39+
{
40+
auto res = session.ExecuteSchemeQuery(R"(
41+
CREATE TABLE `/local/Table` (
42+
Key Uint32,
43+
PRIMARY KEY (Key)
44+
);
45+
)").GetValueSync();
46+
UNIT_ASSERT_C(res.IsSuccess(), res.GetIssues().ToString());
47+
}
48+
49+
Aws::Client::ClientConfiguration s3ClientConfig;
50+
s3ClientConfig.endpointOverride = GetEnv("S3_ENDPOINT");
51+
s3ClientConfig.scheme = Aws::Http::Scheme::HTTP;
52+
auto s3Client = Aws::S3::S3Client(
53+
std::make_shared<Aws::Auth::AnonymousAWSCredentialsProvider>(),
54+
s3ClientConfig,
55+
Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
56+
false
57+
);
58+
const TString bucketName = "my-bucket";
59+
NTestUtils::CreateBucket(bucketName, s3Client);
60+
61+
auto fillS3Settings = [bucketName](auto& settings) {
62+
settings.Endpoint(GetEnv("S3_ENDPOINT"));
63+
settings.Bucket(bucketName);
64+
settings.AccessKey("minio");
65+
settings.SecretKey("minio123");
66+
settings.UseVirtualAddressing(false);
67+
};
68+
69+
{
70+
NExport::TExportToS3Settings settings;
71+
fillS3Settings(settings);
72+
73+
settings.AppendItem({"/local/Table", "Table"});
74+
75+
auto exportClient = NExport::TExportClient(driver);
76+
auto operationClient = NOperation::TOperationClient(driver);
77+
78+
const auto backupOp = exportClient.ExportToS3(settings).GetValueSync();
79+
80+
if (backupOp.Ready()) {
81+
UNIT_ASSERT_C(backupOp.Status().IsSuccess(), backupOp.Status().GetIssues().ToString());
82+
} else {
83+
TMaybe<TOperation> op = backupOp;
84+
WaitOp<NExport::TExportToS3Response>(op, operationClient);
85+
UNIT_ASSERT_C(op->Status().IsSuccess(), op->Status().GetIssues().ToString());
86+
}
87+
}
88+
89+
{
90+
NImport::TImportFromS3Settings settings;
91+
fillS3Settings(settings);
92+
93+
settings.AppendItem({"Table", "/local/Restored"});
94+
95+
auto importClient = NImport::TImportClient(driver);
96+
auto operationClient = NOperation::TOperationClient(driver);
97+
98+
const auto restoreOp = importClient.ImportFromS3(settings).GetValueSync();
99+
100+
if (restoreOp.Ready()) {
101+
UNIT_ASSERT_C(restoreOp.Status().IsSuccess(), restoreOp.Status().GetIssues().ToString());
102+
} else {
103+
TMaybe<TOperation> op = restoreOp;
104+
WaitOp<NImport::TImportFromS3Response>(op, operationClient);
105+
UNIT_ASSERT_C(op->Status().IsSuccess(), op->Status().GetIssues().ToString());
106+
}
107+
}
108+
}
109+
}
110+

‎ydb/tests/functional/backup/s3_path_style/ya.make‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
UNITTEST()
2+
3+
ENV(S3_IGNORE_SUBDOMAIN_BUCKETNAME=true)
4+
ENV(YDB_USE_IN_MEMORY_PDISKS=true)
5+
6+
ENV(YDB_ERASURE=block_4-2)
7+
8+
PEERDIR(
9+
ydb/library/testlib/s3_recipe_helper
10+
ydb/public/sdk/cpp/client/ydb_export
11+
ydb/public/sdk/cpp/client/ydb_table
12+
ydb/public/sdk/cpp/client/ydb_operation
13+
ydb/public/sdk/cpp/client/draft
14+
)
15+
16+
SRCS(
17+
s3_path_style_backup_ut.cpp
18+
)
19+
20+
INCLUDE(${ARCADIA_ROOT}/ydb/public/tools/ydb_recipe/recipe.inc)
21+
INCLUDE(${ARCADIA_ROOT}/ydb/tests/tools/s3_recipe/recipe.inc)
22+
23+
SIZE(MEDIUM)
24+
25+
IF (SANITIZER_TYPE)
26+
REQUIREMENTS(ram:16)
27+
ENDIF()
28+
29+
END()

‎ydb/tests/functional/backup/ya.make‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
RECURSE(
2+
s3_path_style
3+
)

‎ydb/tests/functional/ya.make‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ RECURSE(
22
api
33
audit
44
autoconfig
5+
backup
56
blobstorage
67
canonical
78
clickbench

0 commit comments

Comments
 (0)