MB-24293: Retry atmost 10 times if directory removal fails

The database directory is removed in order to simulate a commit
failure. The directory removal could fail for various reasons.
Retry atmost 10 times to remove the directory

Change-Id: I16e81f3572e0b7d58af3d5ee1f7849aec8cabf97
Reviewed-on: http://review.couchbase.org/78254
Tested-by: Build Bot <build@couchbase.com>
Reviewed-by: Dave Rigby <daver@couchbase.com>

diff --git a/tests/ep_testsuite.cc b/tests/ep_testsuite.cc
index 8b58081..004fd8f 100644 (file)
--- a/tests/ep_testsuite.cc
+++ b/tests/ep_testsuite.cc
@@ -7218,8 +7218,15 @@ static enum test_result test_mb20697(ENGINE_HANDLE *h,
 
     std::string dbname = vals["ep_dbname"];
 
-    /* Nuke the database directory to simulate the commit failure */
-    rmdb(dbname.c_str());
+    /* Nuke the database directory to simulate the commit failure.
+     * In case of failure to remove the directory, then retry atmost
+     * 10 times.
+     */
+    int retries = 0;
+    while (rmdb(dbname.c_str()) == FAIL && retries < 10) {
+        retries++;
+        std::this_thread::sleep_for(std::chrono::milliseconds(10));
+    }
 
     checkeq(ENGINE_SUCCESS, store(h, h1, NULL, OPERATION_SET,"key", "somevalue",
                                   NULL, 0, 0, 0), "store should have succeeded");

diff --git a/tests/ep_testsuite_common.cc b/tests/ep_testsuite_common.cc
index 62bab4c..ada0421 100644 (file)
--- a/tests/ep_testsuite_common.cc
+++ b/tests/ep_testsuite_common.cc
@@ -136,7 +136,11 @@ static enum test_result skipped_test_function(ENGINE_HANDLE *h,
 }
 
 enum test_result rmdb(const char* path) {
-    cb::io::rmrf(path);
+    try {
+        cb::io::rmrf(path);
+    } catch (std::system_error& e) {
+        throw e;
+    }
     if (access(path, F_OK) != -1) {
         std::cerr << "Failed to remove: " << path << " " << std::endl;
         return FAIL;
@@ -220,7 +224,13 @@ enum test_result prepare(engine_test_t *test) {
 
     std::string dbname = get_dbname(test->cfg);
     /* Remove if the same DB directory already exists */
-    rmdb(dbname.c_str());
+    try {
+        rmdb(dbname.c_str());
+    } catch (std::system_error& e) {
+        if (e.code() != std::error_code(ENOENT, std::system_category())) {
+            throw e;
+        }
+    }
     mkdir(dbname.c_str(), 0777);
     return SUCCESS;
 }
@@ -280,7 +290,13 @@ void cleanup(engine_test_t *test, enum test_result result) {
     // Nuke the database files we created
     std::string dbname = get_dbname(test->cfg);
     /* Remove only the db file this test created */
-    rmdb(dbname.c_str());
+    try {
+        rmdb(dbname.c_str());
+    } catch (std::system_error& e) {
+        if (e.code() != std::error_code(ENOENT, std::system_category())) {
+            throw e;
+        }
+    }
 }
 
 // Array of testcases to return back to engine_testapp.
@@ -380,7 +396,13 @@ int create_buckets(const char* cfg, int n_buckets, std::vector<BucketHolder> &bu
             config << str_cfg << "dbname=" << dbpath.str();
         }
 
-        rmdb(dbpath.str().c_str());
+        try {
+            rmdb(dbpath.str().c_str());
+        } catch (std::system_error& e) {
+            if (e.code() != std::error_code(ENOENT, std::system_category())) {
+                throw e;
+            }
+        }
         ENGINE_HANDLE_V1* handle = testHarness.create_bucket(true, config.str().c_str());
         if (handle) {
             buckets.push_back(BucketHolder((ENGINE_HANDLE*)handle, handle, dbpath.str()));

diff --git a/tests/module_tests/evp_engine_test.cc b/tests/module_tests/evp_engine_test.cc
index 1403e0c..e5a7a17 100644 (file)
--- a/tests/module_tests/evp_engine_test.cc
+++ b/tests/module_tests/evp_engine_test.cc
@@ -31,7 +31,13 @@
 void EventuallyPersistentEngineTest::SetUp() {
     // Paranoia - kill any existing files in case they are left over
     // from a previous run.
-    cb::io::rmrf(test_dbname);
+    try {
+        cb::io::rmrf(test_dbname);
+    } catch (std::system_error& e) {
+        if (e.code() != std::error_code(ENOENT, std::system_category())) {
+            throw e;
+        }
+    }
 
     // Setup an engine with a single active vBucket.
     EXPECT_EQ(ENGINE_SUCCESS,

diff --git a/tests/module_tests/kv_bucket_test.cc b/tests/module_tests/kv_bucket_test.cc
index 121b10f..003a608 100644 (file)
--- a/tests/module_tests/kv_bucket_test.cc
+++ b/tests/module_tests/kv_bucket_test.cc
@@ -47,7 +47,13 @@
 void KVBucketTest::SetUp() {
     // Paranoia - kill any existing files in case they are left over
     // from a previous run.
-    cb::io::rmrf(test_dbname);
+    try {
+        cb::io::rmrf(test_dbname);
+    } catch (std::system_error& e) {
+        if (e.code() != std::error_code(ENOENT, std::system_category())) {
+            throw e;
+        }
+    }
 
     // Add dbname to config string.
     std::string config = config_string;

diff --git a/tests/module_tests/kvstore_test.cc b/tests/module_tests/kvstore_test.cc
index 475f694..1fae656 100644 (file)
--- a/tests/module_tests/kvstore_test.cc
+++ b/tests/module_tests/kvstore_test.cc
@@ -486,7 +486,13 @@ public:
                                false /*persistnamespace*/)
                          .setLogger(logger)
                          .setBuffered(false)) {
-        cb::io::rmrf(data_dir.c_str());
+        try {
+            cb::io::rmrf(data_dir.c_str());
+        } catch (std::system_error& e) {
+            if (e.code() != std::error_code(ENOENT, std::system_category())) {
+                throw e;
+            }
+        }
         kvstore.reset(new CouchKVStore(config, ops));
         initialize_kv_store(kvstore.get());
     }
@@ -1155,7 +1161,13 @@ public:
                                0,
                                false /*persistnamespace*/)
                          .setBuffered(false)) {
-        cb::io::rmrf(data_dir.c_str());
+        try {
+            cb::io::rmrf(data_dir.c_str());
+        } catch (std::system_error& e) {
+            if (e.code() != std::error_code(ENOENT, std::system_category())) {
+                throw e;
+            }
+        }
         kvstore.reset(new MockCouchKVStore(config));
         StatsCallback sc;
         std::string failoverLog("");