Fix test case test_dcp_early_termination 42/57642/5
authorabhinavdangeti <abhinav@couchbase.com>
Wed, 9 Dec 2015 19:42:43 +0000 (11:42 -0800)
committerChiyoung Seo <chiyoung@couchbase.com>
Wed, 9 Dec 2015 23:47:34 +0000 (23:47 +0000)
Account for tasks that are already in the future queue of the
auxIO dispatcher to ensure all DCP backfill tasks (auxIO) have
completed.

Change-Id: I9544a79436193f3ef42b08a2b6615eb4be4792ce
Reviewed-on: http://review.couchbase.org/57642
Well-Formed: buildbot <build@couchbase.com>
Tested-by: buildbot <build@couchbase.com>
Reviewed-by: Chiyoung Seo <chiyoung@couchbase.com>
tests/ep_test_apis.cc
tests/ep_test_apis.h
tests/ep_testsuite.cc

index 727ac03..94cc4f0 100644 (file)
@@ -830,6 +830,49 @@ void verify_curr_items(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, int exp,
     }
 }
 
+/** Helper class used when waiting on statistics to reach a certain value -
+ * aggregates how long we have been waiting and aborts if the maximum wait time
+ * is exceeded.
+ */
+template <typename T>
+class WaitTimeAccumulator
+{
+public:
+    WaitTimeAccumulator(const char* compare_name,
+                        const char* stat_, const char* stat_key,
+                        const T final_, const time_t wait_time_in_secs)
+    : compareName(compare_name),
+      stat(stat_),
+      statKey(stat_key),
+      final(final_),
+      maxWaitTime(wait_time_in_secs * 1000 * 1000),
+      totalSleepTime(0) {}
+
+    void incrementAndAbortIfLimitReached(const useconds_t sleep_time)
+    {
+        totalSleepTime += sleep_time;
+        if (totalSleepTime >= maxWaitTime) {
+            std::cerr << "Exceeded maximum wait time of " << maxWaitTime
+                      << "s waiting for stat '" << stat;
+            if (statKey != NULL) {
+                std::cerr << "(" << statKey << ")";
+            }
+            std::cerr << "' " << compareName << " " << final << " - aborting."
+                      << std::endl;
+            abort();
+        }
+    }
+
+private:
+    const char* compareName;
+    const char* stat;
+    const char* statKey;
+    const T final;
+    const useconds_t maxWaitTime;
+    useconds_t totalSleepTime;
+};
+
+
 void wait_for_stat_change(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
                           const char *stat, int initial,
                           const char *statkey) {
@@ -847,6 +890,20 @@ void wait_for_stat_to_be(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
     }
 }
 
+void wait_for_stat_to_be_lte(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
+                             const char *stat, int final,
+                             const char* stat_key,
+                             const time_t max_wait_time_in_secs) {
+    useconds_t sleepTime = 128;
+    WaitTimeAccumulator<int> accumulator("to be less than or equal to", stat,
+                                         stat_key, final,
+                                         max_wait_time_in_secs);
+    while (get_int_stat(h, h1, stat, stat_key) > final) {
+        accumulator.incrementAndAbortIfLimitReached(sleepTime);
+        decayingSleep(&sleepTime);
+    }
+}
+
 void wait_for_str_stat_to_be(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
                              const char *stat, const char* final,
                              const char* stat_key) {
index 5a78904..7c806d6 100644 (file)
@@ -176,6 +176,10 @@ void wait_for_stat_change(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
                           const char *statkey = NULL);
 void wait_for_stat_to_be(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1, const char *stat,
                          int final, const char* stat_key = NULL);
+void wait_for_stat_to_be_lte(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
+                             const char *stat, int final,
+                             const char* stat_key = NULL,
+                             const time_t max_wait_time_in_secs = 60);
 void wait_for_str_stat_to_be(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1,
                              const char *stat, const char* final,
                              const char* stat_key);
index c6e933c..0f18114 100644 (file)
@@ -8469,6 +8469,10 @@ static enum test_result test_dcp_early_termination(ENGINE_HANDLE* h,
     }
     wait_for_flusher_to_settle(h, h1);
 
+    // Account for tasks already in the auxIO futureQ
+    int auxIOFutureQ = get_int_stat(h, h1, "ep_workload:LowPrioQ_AuxIO:InQsize",
+                                    "workload");
+
     const void *cookie = testHarness.create_cookie();
     uint32_t opaque = 1;
     check(h1->dcp.open(h, cookie, ++opaque, 0, DCP_OPEN_PRODUCER,
@@ -8496,8 +8500,13 @@ static enum test_result test_dcp_early_termination(ENGINE_HANDLE* h,
     testHarness.destroy_cookie(cookie);
 
     // Let all AUXIO (backfills) finish
-    wait_for_stat_to_be(h, h1, "ep_workload:LowPrioQ_AuxIO:InQsize", 0, "workload");
-    wait_for_stat_to_be(h, h1, "ep_workload:LowPrioQ_AuxIO:OutQsize", 0, "workload");
+    // - FutureQ of auxIO should contain less than or equal to what
+    // was in it before DCP
+    // - ReadyQ of auxIO to contain 0 tasks
+    wait_for_stat_to_be_lte(h, h1, "ep_workload:LowPrioQ_AuxIO:InQsize",
+                            auxIOFutureQ, "workload");
+    wait_for_stat_to_be(h, h1, "ep_workload:LowPrioQ_AuxIO:OutQsize",
+                        0, "workload");
     return SUCCESS;
 }