Skip to content

Commit 350bae0

Browse files
wps132230wps132230
authored
Use HTTP GET instead of HEAD for the WaitFor*ToPropagate logics in S3 related integration tests.
And shuffle the integration tests in run_integration_tests.py to avoid multiple jobs running the same tests at the same time in the pipeline.
1 parent 644dda7 commit 350bae0

File tree

4 files changed

+75
-47
lines changed

4 files changed

+75
-47
lines changed

‎aws-cpp-sdk-s3-integration-tests/BucketAndObjectOperationTest.cpp

+46-22
Original file line numberDiff line numberDiff line change
@@ -258,10 +258,10 @@ namespace
258258
unsigned timeoutCount = 0;
259259
while (timeoutCount++ < TIMEOUT_MAX)
260260
{
261-
HeadBucketRequest headBucketRequest;
262-
headBucketRequest.SetBucket(bucketName);
263-
HeadBucketOutcome headBucketOutcome = client->HeadBucket(headBucketRequest);
264-
if (headBucketOutcome.IsSuccess())
261+
ListObjectsRequest listObjectsRequest;
262+
listObjectsRequest.SetBucket(bucketName);
263+
ListObjectsOutcome listObjectsOutcome = client->ListObjects(listObjectsRequest);
264+
if (listObjectsOutcome.IsSuccess())
265265
{
266266
return true;
267267
}
@@ -277,16 +277,40 @@ namespace
277277
unsigned timeoutCount = 0;
278278
while (timeoutCount++ < TIMEOUT_MAX)
279279
{
280-
HeadObjectRequest headObjectRequest;
281-
headObjectRequest.SetBucket(bucketName);
282-
headObjectRequest.SetKey(objectKey);
283-
HeadObjectOutcome headObjectOutcome = Client->HeadObject(headObjectRequest);
284-
if (headObjectOutcome.IsSuccess())
280+
GetObjectRequest getObjectRequest;
281+
getObjectRequest.SetBucket(bucketName);
282+
getObjectRequest.SetKey(objectKey);
283+
GetObjectOutcome getObjectOutcome = Client->GetObject(getObjectRequest);
284+
if (getObjectOutcome.IsSuccess())
285285
{
286286
return true;
287287
}
288288

289-
std::this_thread::sleep_for(std::chrono::seconds(1));
289+
std::this_thread::sleep_for(std::chrono::seconds(5));
290+
}
291+
292+
return false;
293+
}
294+
295+
static bool WaitForObjectWithSSECToPropagate(const Aws::String& bucketName, const char* objectKey, const ByteBuffer& sseKey)
296+
{
297+
unsigned timeoutCount = 0;
298+
Aws::String strBuffer(reinterpret_cast<char*>(sseKey.GetUnderlyingData()), sseKey.GetLength());
299+
while (timeoutCount++ < TIMEOUT_MAX)
300+
{
301+
GetObjectRequest getObjectRequest;
302+
getObjectRequest.SetBucket(bucketName);
303+
getObjectRequest.SetKey(objectKey);
304+
getObjectRequest.WithSSECustomerAlgorithm(Aws::S3::Model::ServerSideEncryptionMapper::GetNameForServerSideEncryption(Aws::S3::Model::ServerSideEncryption::AES256))
305+
.WithSSECustomerKey(HashingUtils::Base64Encode(sseKey))
306+
.WithSSECustomerKeyMD5(HashingUtils::Base64Encode(HashingUtils::CalculateMD5(strBuffer)));
307+
GetObjectOutcome getObjectOutcome = Client->GetObject(getObjectRequest);
308+
if (getObjectOutcome.IsSuccess())
309+
{
310+
return true;
311+
}
312+
313+
std::this_thread::sleep_for(std::chrono::seconds(5));
290314
}
291315

292316
return false;
@@ -324,7 +348,7 @@ namespace
324348

325349
if (listObjectsOutcome.GetResult().GetContents().size() > 0)
326350
{
327-
std::this_thread::sleep_for(std::chrono::seconds(1));
351+
std::this_thread::sleep_for(std::chrono::seconds(5));
328352
}
329353
else
330354
{
@@ -367,7 +391,7 @@ namespace
367391
EXPECT_TRUE(createBucketOutcome.IsSuccess());
368392
const CreateBucketResult& createBucketResult = createBucketOutcome.GetResult();
369393
EXPECT_TRUE(!createBucketResult.GetLocation().empty());
370-
WaitForBucketToPropagate(fullBucketName);
394+
EXPECT_TRUE(WaitForBucketToPropagate(fullBucketName));
371395
return fullBucketName;
372396
}
373397

@@ -386,7 +410,7 @@ namespace
386410

387411
ASSERT_EQ(HttpResponseCode::OK, putResponse->GetResponseCode());
388412

389-
WaitForObjectToPropagate(bucketName, TEST_OBJ_KEY);
413+
ASSERT_TRUE(WaitForObjectToPropagate(bucketName, TEST_OBJ_KEY));
390414

391415
// GetObject with presigned url
392416
Aws::String presignedUrlGet = Client->GeneratePresignedUrl(bucketName, TEST_OBJ_KEY, HttpMethod::HTTP_GET);
@@ -466,7 +490,7 @@ namespace
466490

467491
ASSERT_EQ(HttpResponseCode::OK, putResponse->GetResponseCode());
468492

469-
WaitForObjectToPropagate(fullBucketName, TEST_OBJ_KEY);
493+
ASSERT_TRUE(WaitForObjectWithSSECToPropagate(fullBucketName, TEST_OBJ_KEY, sseKey));
470494

471495
// Test GetObject with SSEC Presigned Url
472496
Aws::String presignedUrlGet = Client->GeneratePresignedUrlWithSSEC(fullBucketName, TEST_OBJ_KEY, HttpMethod::HTTP_GET, HashingUtils::Base64Encode(sseKey));
@@ -656,7 +680,7 @@ namespace
656680
const CreateBucketResult& createBucketResult = createBucketOutcome.GetResult();
657681
ASSERT_TRUE(!createBucketResult.GetLocation().empty());
658682

659-
WaitForBucketToPropagate(fullBucketName);
683+
ASSERT_TRUE(WaitForBucketToPropagate(fullBucketName));
660684

661685
PutObjectRequest putObjectRequest;
662686
putObjectRequest.SetBucket(fullBucketName);
@@ -682,7 +706,7 @@ namespace
682706
const CreateBucketResult& createBucketResult = createBucketOutcome.GetResult();
683707
ASSERT_TRUE(!createBucketResult.GetLocation().empty());
684708

685-
WaitForBucketToPropagate(fullBucketName);
709+
ASSERT_TRUE(WaitForBucketToPropagate(fullBucketName));
686710

687711
PutObjectRequest putObjectRequest;
688712
putObjectRequest.SetBucket(fullBucketName);
@@ -769,7 +793,7 @@ namespace
769793
const CreateBucketResult& createBucketResult = createBucketOutcome.GetResult();
770794
ASSERT_TRUE(!createBucketResult.GetLocation().empty());
771795

772-
WaitForBucketToPropagate(fullBucketName);
796+
ASSERT_TRUE(WaitForBucketToPropagate(fullBucketName));
773797

774798
//test unicode
775799
{
@@ -933,7 +957,7 @@ namespace
933957
const CreateBucketResult& createBucketResult = createBucketOutcome.GetResult();
934958
ASSERT_TRUE(!createBucketResult.GetLocation().empty());
935959

936-
WaitForBucketToPropagate(fullBucketName);
960+
ASSERT_TRUE(WaitForBucketToPropagate(fullBucketName));
937961

938962
CreateMultipartUploadRequest createMultipartUploadRequest;
939963
createMultipartUploadRequest.SetBucket(fullBucketName);
@@ -1001,7 +1025,7 @@ namespace
10011025
completeMultipartUploadRequest);
10021026
ASSERT_TRUE(completeMultipartUploadOutcome.IsSuccess());
10031027

1004-
WaitForObjectToPropagate(fullBucketName, multipartKeyName);
1028+
ASSERT_TRUE(WaitForObjectToPropagate(fullBucketName, multipartKeyName));
10051029

10061030
GetObjectRequest getObjectRequest;
10071031
getObjectRequest.SetBucket(fullBucketName);
@@ -1180,7 +1204,7 @@ namespace
11801204
createBucketRequest.SetACL(BucketCannedACL::private_);
11811205
CreateBucketOutcome createBucketOutcome = Client->CreateBucket(createBucketRequest);
11821206
ASSERT_TRUE(createBucketOutcome.IsSuccess());
1183-
WaitForBucketToPropagate(fullBucketName);
1207+
ASSERT_TRUE(WaitForBucketToPropagate(fullBucketName));
11841208

11851209
PutObjectRequest putObjectRequest;
11861210
putObjectRequest.SetBucket(fullBucketName);
@@ -1214,7 +1238,7 @@ namespace
12141238
createBucketRequest.SetACL(BucketCannedACL::private_);
12151239
CreateBucketOutcome createBucketOutcome = Client->CreateBucket(createBucketRequest);
12161240
ASSERT_TRUE(createBucketOutcome.IsSuccess());
1217-
WaitForBucketToPropagate(fullBucketName);
1241+
ASSERT_TRUE(WaitForBucketToPropagate(fullBucketName));
12181242

12191243
PutObjectRequest putObjectRequest;
12201244
putObjectRequest.SetBucket(fullBucketName);
@@ -1243,7 +1267,7 @@ namespace
12431267
CreateBucketOutcome createBucketOutcome = Client->CreateBucket(createBucketRequest);
12441268
ASSERT_TRUE(createBucketOutcome.IsSuccess());
12451269

1246-
WaitForBucketToPropagate(fullBucketName);
1270+
ASSERT_TRUE(WaitForBucketToPropagate(fullBucketName));
12471271

12481272
auto objectStream = Aws::MakeShared<Aws::StringStream>("BucketAndObjectOperationTest");
12491273
*objectStream << "Test Japanese & Chinese Unicode keys";

‎aws-cpp-sdk-s3control-integration-tests/S3ControlTest.cpp

+13-12
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,9 @@
2828
#include <aws/s3/S3Client.h>
2929
#include <aws/s3/model/CreateBucketRequest.h>
3030
#include <aws/s3/model/HeadBucketRequest.h>
31+
#include <aws/s3/model/ListObjectsRequest.h>
3132
#include <aws/s3/model/DeleteBucketRequest.h>
32-
#include <aws/s3/model/HeadObjectRequest.h>
33+
#include <aws/s3/model/GetObjectRequest.h>
3334
#include <aws/s3control/S3ControlClient.h>
3435
#include <aws/s3control/model/PutPublicAccessBlockRequest.h>
3536
#include <aws/s3control/model/GetPublicAccessBlockRequest.h>
@@ -106,10 +107,10 @@ namespace
106107
unsigned timeoutCount = 0;
107108
while (timeoutCount++ < TIMEOUT_MAX)
108109
{
109-
S3::Model::HeadBucketRequest headBucketRequest;
110-
headBucketRequest.SetBucket(bucketName);
111-
auto headBucketOutcome = client.HeadBucket(headBucketRequest);
112-
if (headBucketOutcome.IsSuccess())
110+
S3::Model::ListObjectsRequest ListObjectsRequest;
111+
ListObjectsRequest.SetBucket(bucketName);
112+
auto listObjectsOutcome = client.ListObjects(ListObjectsRequest);
113+
if (listObjectsOutcome.IsSuccess())
113114
{
114115
return true;
115116
}
@@ -125,16 +126,16 @@ namespace
125126
unsigned timeoutCount = 0;
126127
while (timeoutCount++ < TIMEOUT_MAX)
127128
{
128-
S3::Model::HeadObjectRequest headObjectRequest;
129-
headObjectRequest.SetBucket(accessPointArn);
130-
headObjectRequest.SetKey(objectKey);
131-
auto headObjectOutcome = client.HeadObject(headObjectRequest);
132-
if (headObjectOutcome.IsSuccess())
129+
S3::Model::GetObjectRequest getObjectRequest;
130+
getObjectRequest.SetBucket(accessPointArn);
131+
getObjectRequest.SetKey(objectKey);
132+
auto getObjectOutcome = client.GetObject(getObjectRequest);
133+
if (getObjectOutcome.IsSuccess())
133134
{
134135
return true;
135136
}
136137

137-
std::this_thread::sleep_for(std::chrono::seconds(1));
138+
std::this_thread::sleep_for(std::chrono::seconds(5));
138139
}
139140

140141
return false;
@@ -203,7 +204,7 @@ namespace
203204

204205
ASSERT_EQ(HttpResponseCode::OK, putResponse->GetResponseCode());
205206

206-
WaitForObjectToPropagate(accessPointArn, TEST_OBJECT_KEY, m_s3Client);
207+
ASSERT_TRUE(WaitForObjectToPropagate(accessPointArn, TEST_OBJECT_KEY, m_s3Client));
207208

208209
// GetObject with presigned url
209210
Aws::String presignedUrlGet = m_s3Client.GeneratePresignedUrl(accessPointArn, TEST_OBJECT_KEY, HttpMethod::HTTP_GET);

‎aws-cpp-sdk-transfer-tests/TransferTests.cpp

+13-13
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ static const unsigned BIG_TEST_SIZE = MB5 * PARTS_IN_BIG_TEST;
8181
static const char* testString = "S3 MultiPart upload Test File ";
8282
static const uint32_t testStrLen = static_cast<uint32_t>(strlen(testString));
8383
static const std::chrono::seconds TEST_WAIT_TIMEOUT = std::chrono::seconds(10);
84-
static const unsigned WAIT_MAX_RETRIES = 10;
84+
static const unsigned WAIT_MAX_RETRIES = 20;
8585

8686
namespace {
8787
static const char *ALLOCATION_TAG = "TransferTests";
@@ -479,7 +479,7 @@ class TransferTests : public ::testing::Test
479479
auto createBucketOutcome = m_s3Client->CreateBucket(createBucket);
480480
ASSERT_TRUE(createBucketOutcome.IsSuccess());
481481

482-
WaitForBucketToPropagate(GetTestBucketName());
482+
ASSERT_TRUE(WaitForBucketToPropagate(GetTestBucketName()));
483483

484484
auto testDirectory = GetTestFilesDirectory();
485485

@@ -491,10 +491,10 @@ class TransferTests : public ::testing::Test
491491
unsigned timeoutCount = 0;
492492
while (timeoutCount++ < WAIT_MAX_RETRIES)
493493
{
494-
HeadBucketRequest headBucketRequest;
495-
headBucketRequest.SetBucket(bucketName);
496-
HeadBucketOutcome headBucketOutcome = m_s3Client->HeadBucket(headBucketRequest);
497-
if (headBucketOutcome.IsSuccess())
494+
ListObjectsRequest listObjectsRequest;
495+
listObjectsRequest.SetBucket(bucketName);
496+
ListObjectsOutcome listObjectsOutcome = m_s3Client->ListObjects(listObjectsRequest);
497+
if (listObjectsOutcome.IsSuccess())
498498
{
499499
return true;
500500
}
@@ -510,16 +510,16 @@ class TransferTests : public ::testing::Test
510510
unsigned timeoutCount = 0;
511511
while (timeoutCount++ < WAIT_MAX_RETRIES)
512512
{
513-
HeadObjectRequest headObjectRequest;
514-
headObjectRequest.SetBucket(bucketName);
515-
headObjectRequest.SetKey(objectKey);
516-
HeadObjectOutcome headObjectOutcome = m_s3Client->HeadObject(headObjectRequest);
517-
if (headObjectOutcome.IsSuccess())
513+
GetObjectRequest getObjectRequest;
514+
getObjectRequest.SetBucket(bucketName);
515+
getObjectRequest.SetKey(objectKey);
516+
GetObjectOutcome getObjectOutcome = m_s3Client->GetObject(getObjectRequest);
517+
if (getObjectOutcome.IsSuccess())
518518
{
519519
return true;
520520
}
521521

522-
std::this_thread::sleep_for(std::chrono::seconds(1));
522+
std::this_thread::sleep_for(std::chrono::seconds(5));
523523
}
524524

525525
return false;
@@ -580,7 +580,7 @@ class TransferTests : public ::testing::Test
580580

581581
if (!listObjectsOutcome.IsSuccess() || listObjectsOutcome.GetResult().GetContents().size() > 0)
582582
{
583-
std::this_thread::sleep_for(std::chrono::seconds(1));
583+
std::this_thread::sleep_for(std::chrono::seconds(5));
584584
}
585585
else
586586
{

‎scripts/run_integration_tests.py

+3
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
import stat
1818
import subprocess
1919
import platform
20+
import random
2021

2122
def ParseArguments():
2223
argMap = {}
@@ -54,6 +55,8 @@ def Main():
5455
#"aws-cpp-sdk-redshift-integration-tests", # Don't run this test unless you really want to, it will cost you a lot of money. The test takes around a half hour to finish.
5556
#"aws-cpp-sdk-cloudfront-integration-tests", # This test will cost you a lot of money as well.
5657
"aws-cpp-sdk-ec2-integration-tests" ]
58+
# Shuffle the list to avoid multiple jobs running the same tests in the testing pipeline.
59+
random.shuffle(testList)
5760

5861
for testName in testList:
5962
testExe = os.path.join(arguments[ "testDir" ], testName if testHasParentDir else "", testName) + exeExtension

0 commit comments

Comments
 (0)