Reorganise EPStore tests 25/76325/5
authorJames Harrison <00jamesh@gmail.com>
Wed, 5 Apr 2017 13:48:08 +0000 (14:48 +0100)
committerDave Rigby <daver@couchbase.com>
Mon, 10 Apr 2017 10:56:49 +0000 (10:56 +0000)
Separating out tests which are applicable to all of:

 EP full eviction
 EP value eviction
 Ephemeral

from those only applicable to the two former.

Change-Id: If1580b5d20b60778dd0a7d88260c80a172251b07
Reviewed-on: http://review.couchbase.org/76325
Reviewed-by: Dave Rigby <daver@couchbase.com>
Tested-by: Build Bot <build@couchbase.com>
CMakeLists.txt
tests/module_tests/evp_store_test.cc
tests/module_tests/evp_store_test.h
tests/module_tests/kv_bucket_test.cc [new file with mode: 0644]
tests/module_tests/kv_bucket_test.h [new file with mode: 0644]

index 81ebaed..756de1c 100644 (file)
@@ -263,6 +263,7 @@ ADD_EXECUTABLE(ep-engine_ep_unit_tests
                tests/module_tests/hash_table_test.cc
                tests/module_tests/item_pager_test.cc
                tests/module_tests/kvstore_test.cc
+               tests/module_tests/kv_bucket_test.cc
                tests/module_tests/memory_tracker_test.cc
                tests/module_tests/mock_hooks_api.cc
                tests/module_tests/mutation_log_test.cc
index 2ad72c5..6162890 100644 (file)
 
 #include "../mock/mock_dcp_producer.h"
 #include "bgfetcher.h"
-#include "checkpoint.h"
 #include "checkpoint_remover.h"
 #include "dcp/dcpconnmap.h"
-#include "dcp/flow-control-manager.h"
-#include "ep_engine.h"
 #include "flusher.h"
-#include "replicationthrottle.h"
 #include "tapconnmap.h"
-#include "tasks.h"
 #include "tests/mock/mock_global_task.h"
 #include "tests/module_tests/test_helpers.h"
 #include "vbucketmemorydeletiontask.h"
 
-#include <chrono>
-#include <platform/dirutils.h>
 #include <thread>
 
-void KVBucketTest::SetUp() {
-    // Paranoia - kill any existing files in case they are left over
-    // from a previous run.
-    cb::io::rmrf(test_dbname);
-
-    // Add dbname to config string.
-    std::string config = config_string;
-    if (config.size() > 0) {
-        config += ";";
-    }
-    config += "dbname=" + std::string(test_dbname);
-
-    engine.reset(new SynchronousEPEngine(config));
-    ObjectRegistry::onSwitchThread(engine.get());
-
-    engine->setKVBucket(engine->public_makeBucket(engine->getConfiguration()));
-    store = engine->getKVBucket();
-
-    store->chkTask = new ClosedUnrefCheckpointRemoverTask(
-            engine.get(),
-            engine->getEpStats(),
-            engine->getConfiguration().getChkRemoverStime());
-
-    // Ensure that EPEngine is hold about necessary server callbacks
-    // (client disconnect, bucket delete).
-    engine->public_initializeEngineCallbacks();
-
-    // Need to initialize ep_real_time and friends.
-    initialize_time_functions(get_mock_server_api()->core);
-
-    cookie = create_mock_cookie();
-}
-
-void KVBucketTest::TearDown() {
-    destroy_mock_cookie(cookie);
-    destroy_mock_event_callbacks();
-    engine->getDcpConnMap().manageConnections();
-    ObjectRegistry::onSwitchThread(nullptr);
-    engine.reset();
-
-    // Shutdown the ExecutorPool singleton (initialized when we create
-    // an EPBucket object). Must happen after engine
-    // has been destroyed (to allow the tasks the engine has
-    // registered a chance to be unregistered).
-    ExecutorPool::shutdown();
-}
-
-Item KVBucketTest::store_item(uint16_t vbid,
-                              const StoredDocKey& key,
-                              const std::string& value,
-                              uint32_t exptime,
-                              const std::vector<cb::engine_errc>& expected,
-                              protocol_binary_datatype_t datatype) {
-    auto item = make_item(vbid, key, value, exptime, datatype);
-    auto returnCode = store->set(item, nullptr);
-    EXPECT_NE(expected.end(),
-              std::find(expected.begin(),
-                        expected.end(),
-                        cb::engine_errc(returnCode)));
-    return item;
-}
-
-::testing::AssertionResult KVBucketTest::store_items(
-        int nitems,
-        uint16_t vbid,
-        const DocKey& key,
-        const std::string& value,
-        uint32_t exptime,
-        protocol_binary_datatype_t datatype) {
-    for (int ii = 0; ii < nitems; ii++) {
-        auto keyii = makeStoredDocKey(
-                std::string(reinterpret_cast<const char*>(key.data()),
-                            key.size()) +
-                        std::to_string(ii),
-                key.getDocNamespace());
-        auto item = make_item(vbid, keyii, value, exptime, datatype);
-        auto err = store->set(item, nullptr);
-        if (ENGINE_SUCCESS != err) {
-            return ::testing::AssertionFailure()
-                   << "Failed to store " << keyii.data() << " error:" << err;
-        }
-    }
-    return ::testing::AssertionSuccess();
-}
-
-void KVBucketTest::flush_vbucket_to_disk(uint16_t vbid, int expected) {
-    int result;
-    const auto time_limit = std::chrono::seconds(10);
-    const auto deadline = std::chrono::steady_clock::now() + time_limit;
-
-    // Need to retry as warmup may not have completed.
-    bool flush_successful = false;
-    do {
-        result = store->flushVBucket(vbid);
-        if (result != RETRY_FLUSH_VBUCKET) {
-            flush_successful = true;
-            break;
-        }
-        std::this_thread::sleep_for(std::chrono::microseconds(100));
-    } while (std::chrono::steady_clock::now() < deadline);
-
-    ASSERT_TRUE(flush_successful)
-        << "Hit timeout (" << time_limit.count() << " seconds) waiting for "
-           "warmup to complete while flushing VBucket.";
-
-    ASSERT_EQ(expected, result) << "Unexpected items in flush_vbucket_to_disk";
-}
-
-void KVBucketTest::delete_item(uint16_t vbid, const StoredDocKey& key) {
-    uint64_t cas = 0;
-    EXPECT_EQ(ENGINE_SUCCESS,
-              store->deleteItem(key,
-                                cas,
-                                vbid,
-                                cookie,
-                                /*Item*/ nullptr,
-                                /*itemMeta*/ nullptr,
-                                /*mutation_descr_t*/ nullptr));
-}
-
-void KVBucketTest::evict_key(uint16_t vbid, const StoredDocKey& key) {
-    const char* msg;
-    EXPECT_EQ(ENGINE_SUCCESS, store->evictKey(key, vbid, &msg));
-    EXPECT_STREQ("Ejected.", msg);
-}
-
-
-GetValue KVBucketTest::getInternal(const StoredDocKey& key,
-                                   uint16_t vbucket,
-                                   const void* cookie,
-                                   vbucket_state_t allowedState,
-                                   get_options_t options) {
-    return store->getInternal(key, vbucket, cookie, allowedState, options);
-}
-
-void KVBucketTest::createAndScheduleItemPager() {
-    store->itemPagerTask = new ItemPager(engine.get(), engine->getEpStats());
-    ExecutorPool::get()->schedule(store->itemPagerTask);
-}
-
-void KVBucketTest::initializeExpiryPager() {
-    store->initializeExpiryPager(engine->getConfiguration());
-}
-
 // Verify that when handling a bucket delete with open DCP
 // connections, we don't deadlock when notifying the front-end
 // connection.
@@ -243,30 +92,6 @@ class EPStoreEvictionTest : public EPBucketTest,
 
 // getKeyStats tests //////////////////////////////////////////////////////////
 
-// Check that keystats on resident items works correctly.
-TEST_P(KVBucketParamTest, GetKeyStatsResident) {
-    key_stats kstats;
-
-    // Should start with key not existing.
-    EXPECT_EQ(ENGINE_KEY_ENOENT,
-              store->getKeyStats(makeStoredDocKey("key"),
-                                 0,
-                                 cookie,
-                                 kstats,
-                                 WantsDeleted::No));
-
-    store_item(0, makeStoredDocKey("key"), "value");
-    EXPECT_EQ(ENGINE_SUCCESS,
-              store->getKeyStats(makeStoredDocKey("key"),
-                                 0,
-                                 cookie,
-                                 kstats,
-                                 WantsDeleted::No))
-            << "Expected to get key stats on existing item";
-    EXPECT_EQ(vbucket_state_active, kstats.vb_state);
-    EXPECT_FALSE(kstats.logically_deleted);
-}
-
 // Check that keystats on ejected items. When ejected should return ewouldblock
 // until bgfetch completes.
 TEST_P(EPStoreEvictionTest, GetKeyStatsEjected) {
@@ -324,57 +149,8 @@ TEST_P(EPStoreEvictionTest, GetKeyStatsEjected) {
     }
 }
 
-// Create then delete an item, checking we get keyStats reporting the item as
-// deleted.
-TEST_P(KVBucketParamTest, GetKeyStatsDeleted) {
-    auto& kvbucket = *engine->getKVBucket();
-    key_stats kstats;
-
-    store_item(0, makeStoredDocKey("key"), "value");
-    delete_item(vbid, makeStoredDocKey("key"));
-
-    // Should get ENOENT if we don't ask for deleted items.
-    EXPECT_EQ(ENGINE_KEY_ENOENT,
-              kvbucket.getKeyStats(makeStoredDocKey("key"),
-                                   0,
-                                   cookie,
-                                   kstats,
-                                   WantsDeleted::No));
-
-    // Should get success (and item flagged as deleted) if we ask for deleted
-    // items.
-    EXPECT_EQ(ENGINE_SUCCESS,
-              kvbucket.getKeyStats(makeStoredDocKey("key"),
-                                   0,
-                                   cookie,
-                                   kstats,
-                                   WantsDeleted::Yes));
-    EXPECT_EQ(vbucket_state_active, kstats.vb_state);
-    EXPECT_TRUE(kstats.logically_deleted);
-}
-
-// Check incorrect vbucket returns not-my-vbucket.
-TEST_P(KVBucketParamTest, GetKeyStatsNMVB) {
-    auto& kvbucket = *engine->getKVBucket();
-    key_stats kstats;
-
-    EXPECT_EQ(ENGINE_NOT_MY_VBUCKET,
-              kvbucket.getKeyStats(makeStoredDocKey("key"),
-                                   1,
-                                   cookie,
-                                   kstats,
-                                   WantsDeleted::No));
-}
-
 // Replace tests //////////////////////////////////////////////////////////////
 
-// Test replace against a non-existent key.
-TEST_P(KVBucketParamTest, ReplaceENOENT) {
-    // Should start with key not existing (and hence cannot replace).
-    auto item = make_item(vbid, makeStoredDocKey("key"), "value");
-    EXPECT_EQ(ENGINE_KEY_ENOENT, store->replace(item, cookie));
-}
-
 // Test replace against an ejected key.
 TEST_P(EPStoreEvictionTest, ReplaceEExists) {
 
@@ -416,29 +192,6 @@ TEST_P(EPStoreEvictionTest, ReplaceEExists) {
     }
 }
 
-// Create then delete an item, checking replace reports ENOENT.
-TEST_P(KVBucketParamTest, ReplaceDeleted) {
-    store_item(vbid, makeStoredDocKey("key"), "value");
-    delete_item(vbid, makeStoredDocKey("key"));
-
-    // Replace should fail.
-    auto item = make_item(vbid, makeStoredDocKey("key"), "value2");
-    EXPECT_EQ(ENGINE_KEY_ENOENT, store->replace(item, cookie));
-}
-
-// Check incorrect vbucket returns not-my-vbucket.
-TEST_P(KVBucketParamTest, ReplaceNMVB) {
-    auto item = make_item(vbid + 1, makeStoredDocKey("key"), "value2");
-    EXPECT_EQ(ENGINE_NOT_MY_VBUCKET, store->replace(item, cookie));
-}
-
-// Check pending vbucket returns EWOULDBLOCK.
-TEST_P(KVBucketParamTest, ReplacePendingVB) {
-    store->setVBucketState(vbid, vbucket_state_pending, false);
-    auto item = make_item(vbid, makeStoredDocKey("key"), "value2");
-    EXPECT_EQ(ENGINE_EWOULDBLOCK, store->replace(item, cookie));
-}
-
 // Set tests //////////////////////////////////////////////////////////////////
 
 // Test set against an ejected key.
@@ -478,26 +231,8 @@ TEST_P(EPStoreEvictionTest, SetEExists) {
     }
 }
 
-// Test CAS set against a non-existent key
-TEST_P(KVBucketParamTest, SetCASNonExistent) {
-    // Create an item with a non-zero CAS.
-    auto item = make_item(vbid, makeStoredDocKey("key"), "value");
-    item.setCas();
-    ASSERT_NE(0, item.getCas());
-
-    // Should get ENOENT as we should immediately know (either from metadata
-    // being resident, or by bloomfilter) that key doesn't exist.
-    EXPECT_EQ(ENGINE_KEY_ENOENT, store->set(item, cookie));
-}
-
 // Add tests //////////////////////////////////////////////////////////////////
 
-// Test successful add
-TEST_P(KVBucketParamTest, Add) {
-    auto item = make_item(vbid, makeStoredDocKey("key"), "value");
-    EXPECT_EQ(ENGINE_SUCCESS, store->add(item, nullptr));
-}
-
 // Test add against an ejected key.
 TEST_P(EPStoreEvictionTest, AddEExists) {
 
@@ -540,55 +275,8 @@ TEST_P(EPStoreEvictionTest, AddEExists) {
     }
 }
 
-// Check incorrect vbucket returns not-my-vbucket.
-TEST_P(KVBucketParamTest, AddNMVB) {
-    auto item = make_item(vbid + 1, makeStoredDocKey("key"), "value2");
-    EXPECT_EQ(ENGINE_NOT_MY_VBUCKET, store->add(item, cookie));
-}
-
 // SetWithMeta tests //////////////////////////////////////////////////////////
 
-// Test basic setWithMeta
-TEST_P(KVBucketParamTest, SetWithMeta) {
-    auto item = make_item(vbid, makeStoredDocKey("key"), "value");
-    item.setCas();
-    uint64_t seqno;
-    EXPECT_EQ(ENGINE_SUCCESS,
-              store->setWithMeta(item, 0, &seqno, cookie, /*force*/false,
-                                 /*allowExisting*/false));
-}
-
-// Test setWithMeta with a conflict with an existing item.
-TEST_P(KVBucketParamTest, SetWithMeta_Conflicted) {
-    auto item = make_item(vbid, makeStoredDocKey("key"), "value");
-    EXPECT_EQ(ENGINE_SUCCESS, store->set(item, nullptr));
-
-    uint64_t seqno;
-    // Attempt to set with the same rev Seqno - should get EEXISTS.
-    EXPECT_EQ(ENGINE_KEY_EEXISTS,
-              store->setWithMeta(item, item.getCas(), &seqno, cookie,
-                                 /*force*/false, /*allowExisting*/true));
-}
-
-// Test setWithMeta replacing existing item
-TEST_P(KVBucketParamTest, SetWithMeta_Replace) {
-    auto item = make_item(vbid, makeStoredDocKey("key"), "value");
-    EXPECT_EQ(ENGINE_SUCCESS, store->set(item, nullptr));
-
-    // Increase revSeqno so conflict resolution doesn't fail.
-    item.setRevSeqno(item.getRevSeqno() + 1);
-    uint64_t seqno;
-    // Should get EEXISTS if we don't force (and use wrong CAS).
-    EXPECT_EQ(ENGINE_KEY_EEXISTS,
-              store->setWithMeta(item, item.getCas() + 1, &seqno, cookie,
-                                 /*force*/false, /*allowExisting*/true));
-
-    // Should succeed with correct CAS, and different RevSeqno.
-    EXPECT_EQ(ENGINE_SUCCESS,
-              store->setWithMeta(item, item.getCas(), &seqno, cookie,
-                                 /*force*/false, /*allowExisting*/true));
-}
-
 // Test setWithMeta replacing an existing, non-resident item
 TEST_P(EPStoreEvictionTest, SetWithMeta_ReplaceNonResident) {
     // Store an item, then evict it.
@@ -634,16 +322,6 @@ TEST_P(EPStoreEvictionTest, SetWithMeta_ReplaceNonResident) {
     }
 }
 
-// Test forced setWithMeta
-TEST_P(KVBucketParamTest, SetWithMeta_Forced) {
-    auto item = make_item(vbid, makeStoredDocKey("key"), "value");
-    item.setCas();
-    uint64_t seqno;
-    EXPECT_EQ(ENGINE_SUCCESS,
-              store->setWithMeta(item, 0, &seqno, cookie, /*force*/true,
-                                 /*allowExisting*/false));
-}
-
 // Test to ensure all pendingBGfetches are deleted when the
 // VBucketMemoryDeletionTask is run
 TEST_P(EPStoreEvictionTest, MB_21976) {
@@ -715,38 +393,6 @@ TEST_P(EPStoreEvictionTest, TouchCmdDuringBgFetch) {
               store->getVBucket(vbid)->getHighSeqno());
 }
 
-// MB and test was raised because a few commits back this was broken but no
-// existing test covered the case. I.e. run this test  against 0810540 and it
-// fails, but now fixed
-TEST_P(KVBucketParamTest, mb22824) {
-    auto key = makeStoredDocKey("key");
-
-    // Store key and force expiry
-    store_item(0, key, "value", 1);
-    TimeTraveller docBrown(20);
-
-    uint32_t deleted = false;
-    ItemMetaData itemMeta1;
-    uint8_t datatype = PROTOCOL_BINARY_RAW_BYTES;
-    EXPECT_EQ(ENGINE_SUCCESS,
-              store->getMetaData(key, vbid, cookie, false,itemMeta1, deleted,
-                                 datatype));
-
-    uint64_t cas = 0;
-    ItemMetaData itemMeta2;
-    EXPECT_EQ(ENGINE_KEY_ENOENT,
-              store->deleteItem(key,
-                                cas,
-                                vbid,
-                                cookie,
-                                /*Item*/ nullptr,
-                                &itemMeta2,
-                                /*mutation_descr_t*/ nullptr));
-
-    // Should be getting the same CAS from the failed delete as getMetaData
-    EXPECT_EQ(itemMeta1.cas, itemMeta2.cas);
-}
-
 // Test cases which run in both Full and Value eviction
 INSTANTIATE_TEST_CASE_P(FullAndValueEviction,
                         EPStoreEvictionTest,
@@ -754,16 +400,3 @@ INSTANTIATE_TEST_CASE_P(FullAndValueEviction,
                         [] (const ::testing::TestParamInfo<std::string>& info) {
                             return info.param;
                         });
-
-// Test cases which run for EP (Full and Value eviction) and Ephemeral
-INSTANTIATE_TEST_CASE_P(EPAndEphemeral,
-                        KVBucketParamTest,
-                        ::testing::Values("item_eviction_policy=value_only",
-                                          "item_eviction_policy=full_eviction",
-                                          "bucket_type=ephemeral"),
-                        [] (const ::testing::TestParamInfo<std::string>& info) {
-                            return info.param.substr(info.param.find('=')+1);
-                        });
-
-
-const char KVBucketTest::test_dbname[] = "ep_engine_ep_unit_tests_db";
index 7771ecb..f3aa65e 100644 (file)
 
 /*
  * Unit tests for the EPBucket class.
+ *
+ * These are instantiated for value and full eviction persistent buckets.
  */
 
 #pragma once
 
 #include "config.h"
 
-#include "ep_bucket.h"
-#include "ep_engine.h"
-#include "item.h"
-
-#include <memcached/engine.h>
-#include <tests/mock/mock_synchronous_ep_engine.h>
-
-#include <gtest/gtest.h>
-#include <memory>
-
-/**
- * Test fixture for KVBucket unit tests.
- *
- * Will create the appropriate subclass of KVBucket (EPBucket /
- * EphemeralBucket) based on the Configuration passed (specifically the
- * bucket_type parameter), defaulting to EPBucket if no bucket_type is
- * specified.
- */
-class KVBucketTest : public ::testing::Test {
-public:
-    void SetUp() override;
-
-    void TearDown() override;
-
-    // Stores an item into the given vbucket. Returns the item stored.
-    Item store_item(uint16_t vbid,
-                    const StoredDocKey& key,
-                    const std::string& value,
-                    uint32_t exptime = 0,
-                    const std::vector<cb::engine_errc>& expected =
-                            {cb::engine_errc::success},
-                    protocol_binary_datatype_t datatype =
-                            PROTOCOL_BINARY_DATATYPE_JSON);
-
-    /**
-     * Store multiple items into the vbucket, the given key will have an
-     * iteration appended to it.
-     */
-    ::testing::AssertionResult store_items(
-            int nitems,
-            uint16_t vbid,
-            const DocKey& key,
-            const std::string& value,
-            uint32_t exptime = 0,
-            protocol_binary_datatype_t datatype =
-                    PROTOCOL_BINARY_DATATYPE_JSON);
-
-    /* Flush the given vbucket to disk, so any outstanding dirty items are
-     * written (and are clean).
-     */
-    void flush_vbucket_to_disk(uint16_t vbid, int expected = 1);
-
-    /* Delete the given item from the given vbucket, verifying it was
-     * successfully deleted.
-     */
-    void delete_item(uint16_t vbid, const StoredDocKey& key);
-
-    /* Evict the given key from memory according to the current eviction
-     * strategy. Verifies it was successfully evicted.
-     */
-    void evict_key(uint16_t vbid, const StoredDocKey& key);
-
-    /// Exposes the normally-protected getInternal method from the store.
-    GetValue getInternal(const StoredDocKey& key,
-                         uint16_t vbucket,
-                         const void* cookie,
-                         vbucket_state_t allowedState,
-                         get_options_t options);
-
-    /**
-     * Creates the ItemPager task and adds it to the scheduler. Allows testing
-     * of the item pager from subclasses, without KVBucket having to grant
-     * friendship to many different test classes.
-     */
-    void createAndScheduleItemPager();
-
-    void initializeExpiryPager();
-
-    static const char test_dbname[];
-
-    std::string config_string;
-
-    const uint16_t vbid = 0;
-
-    // The mock engine (needed to construct the store).
-    std::unique_ptr<SynchronousEPEngine> engine;
-
-    // The store under test. Wrapped in a mock to expose some normally
-    // protected members. Uses a raw pointer as this is owned by the engine.
-    KVBucket* store;
-
-    // The (mock) server cookie.
-    const void* cookie;
-};
+#include "kv_bucket_test.h"
 
 /**
  * Test fixture for EPBucket unit tests.
@@ -126,22 +35,4 @@ class EPBucketTest : public KVBucketTest {
     // default bucket_type in configuration.json is EPBucket, therefore
     // KVBucketTest already defaults to creating EPBucket. Introducing this
     // subclass to just make the name more descriptive.
-};
-
-/**
- * Test fixture for KVBucket unit tests.
- *
- * These tests are parameterized over an extra config string to allow them to
- * be run against ephemeral and value and full eviction persistent buckets.
- */
-class KVBucketParamTest : public KVBucketTest,
-                          public ::testing::WithParamInterface<std::string> {
-    void SetUp() override {
-        config_string += GetParam();
-        KVBucketTest::SetUp();
-
-        // Have all the objects, activate vBucket zero so we can store data.
-        store->setVBucketState(vbid, vbucket_state_active, false);
-
-    }
-};
+};
\ No newline at end of file
diff --git a/tests/module_tests/kv_bucket_test.cc b/tests/module_tests/kv_bucket_test.cc
new file mode 100644 (file)
index 0000000..78bd923
--- /dev/null
@@ -0,0 +1,430 @@
+/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *     Copyright 2017 Couchbase, Inc
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+/*
+ * Unit tests for the KVBucket class.
+ */
+
+#include "kv_bucket_test.h"
+
+#include "../mock/mock_dcp_producer.h"
+#include "bgfetcher.h"
+#include "checkpoint.h"
+#include "checkpoint_remover.h"
+#include "dcp/dcpconnmap.h"
+#include "dcp/flow-control-manager.h"
+#include "ep_engine.h"
+#include "flusher.h"
+#include "replicationthrottle.h"
+#include "tapconnmap.h"
+#include "tasks.h"
+#include "tests/mock/mock_global_task.h"
+#include "tests/module_tests/test_helpers.h"
+#include "vbucketmemorydeletiontask.h"
+
+#include <platform/dirutils.h>
+#include <chrono>
+#include <thread>
+
+void KVBucketTest::SetUp() {
+    // Paranoia - kill any existing files in case they are left over
+    // from a previous run.
+    cb::io::rmrf(test_dbname);
+
+    // Add dbname to config string.
+    std::string config = config_string;
+    if (config.size() > 0) {
+        config += ";";
+    }
+    config += "dbname=" + std::string(test_dbname);
+
+    engine.reset(new SynchronousEPEngine(config));
+    ObjectRegistry::onSwitchThread(engine.get());
+
+    engine->setKVBucket(engine->public_makeBucket(engine->getConfiguration()));
+    store = engine->getKVBucket();
+
+    store->chkTask = new ClosedUnrefCheckpointRemoverTask(
+            engine.get(),
+            engine->getEpStats(),
+            engine->getConfiguration().getChkRemoverStime());
+
+    // Ensure that EPEngine is hold about necessary server callbacks
+    // (client disconnect, bucket delete).
+    engine->public_initializeEngineCallbacks();
+
+    // Need to initialize ep_real_time and friends.
+    initialize_time_functions(get_mock_server_api()->core);
+
+    cookie = create_mock_cookie();
+}
+
+void KVBucketTest::TearDown() {
+    destroy_mock_cookie(cookie);
+    destroy_mock_event_callbacks();
+    engine->getDcpConnMap().manageConnections();
+    ObjectRegistry::onSwitchThread(nullptr);
+    engine.reset();
+
+    // Shutdown the ExecutorPool singleton (initialized when we create
+    // an EPBucket object). Must happen after engine
+    // has been destroyed (to allow the tasks the engine has
+    // registered a chance to be unregistered).
+    ExecutorPool::shutdown();
+}
+
+Item KVBucketTest::store_item(uint16_t vbid,
+                              const StoredDocKey& key,
+                              const std::string& value,
+                              uint32_t exptime,
+                              const std::vector<cb::engine_errc>& expected,
+                              protocol_binary_datatype_t datatype) {
+    auto item = make_item(vbid, key, value, exptime, datatype);
+    auto returnCode = store->set(item, nullptr);
+    EXPECT_NE(expected.end(),
+              std::find(expected.begin(),
+                        expected.end(),
+                        cb::engine_errc(returnCode)));
+    return item;
+}
+
+::testing::AssertionResult KVBucketTest::store_items(
+        int nitems,
+        uint16_t vbid,
+        const DocKey& key,
+        const std::string& value,
+        uint32_t exptime,
+        protocol_binary_datatype_t datatype) {
+    for (int ii = 0; ii < nitems; ii++) {
+        auto keyii = makeStoredDocKey(
+                std::string(reinterpret_cast<const char*>(key.data()),
+                            key.size()) +
+                        std::to_string(ii),
+                key.getDocNamespace());
+        auto item = make_item(vbid, keyii, value, exptime, datatype);
+        auto err = store->set(item, nullptr);
+        if (ENGINE_SUCCESS != err) {
+            return ::testing::AssertionFailure()
+                   << "Failed to store " << keyii.data() << " error:" << err;
+        }
+    }
+    return ::testing::AssertionSuccess();
+}
+
+void KVBucketTest::flush_vbucket_to_disk(uint16_t vbid, int expected) {
+    int result;
+    const auto time_limit = std::chrono::seconds(10);
+    const auto deadline = std::chrono::steady_clock::now() + time_limit;
+
+    // Need to retry as warmup may not have completed.
+    bool flush_successful = false;
+    do {
+        result = store->flushVBucket(vbid);
+        if (result != RETRY_FLUSH_VBUCKET) {
+            flush_successful = true;
+            break;
+        }
+        std::this_thread::sleep_for(std::chrono::microseconds(100));
+    } while (std::chrono::steady_clock::now() < deadline);
+
+    ASSERT_TRUE(flush_successful)
+            << "Hit timeout (" << time_limit.count()
+            << " seconds) waiting for "
+               "warmup to complete while flushing VBucket.";
+
+    ASSERT_EQ(expected, result) << "Unexpected items in flush_vbucket_to_disk";
+}
+
+void KVBucketTest::delete_item(uint16_t vbid, const StoredDocKey& key) {
+    uint64_t cas = 0;
+    EXPECT_EQ(ENGINE_SUCCESS,
+              store->deleteItem(key,
+                                cas,
+                                vbid,
+                                cookie,
+                                /*Item*/ nullptr,
+                                /*itemMeta*/ nullptr,
+                                /*mutation_descr_t*/ nullptr));
+}
+
+void KVBucketTest::evict_key(uint16_t vbid, const StoredDocKey& key) {
+    const char* msg;
+    EXPECT_EQ(ENGINE_SUCCESS, store->evictKey(key, vbid, &msg));
+    EXPECT_STREQ("Ejected.", msg);
+}
+
+GetValue KVBucketTest::getInternal(const StoredDocKey& key,
+                                   uint16_t vbucket,
+                                   const void* cookie,
+                                   vbucket_state_t allowedState,
+                                   get_options_t options) {
+    return store->getInternal(key, vbucket, cookie, allowedState, options);
+}
+
+void KVBucketTest::createAndScheduleItemPager() {
+    store->itemPagerTask = new ItemPager(engine.get(), engine->getEpStats());
+    ExecutorPool::get()->schedule(store->itemPagerTask);
+}
+
+void KVBucketTest::initializeExpiryPager() {
+    store->initializeExpiryPager(engine->getConfiguration());
+}
+
+// getKeyStats tests //////////////////////////////////////////////////////////
+
+// Check that keystats on resident items works correctly.
+TEST_P(KVBucketParamTest, GetKeyStatsResident) {
+    key_stats kstats;
+
+    // Should start with key not existing.
+    EXPECT_EQ(ENGINE_KEY_ENOENT,
+              store->getKeyStats(makeStoredDocKey("key"),
+                                 0,
+                                 cookie,
+                                 kstats,
+                                 WantsDeleted::No));
+
+    store_item(0, makeStoredDocKey("key"), "value");
+    EXPECT_EQ(ENGINE_SUCCESS,
+              store->getKeyStats(makeStoredDocKey("key"),
+                                 0,
+                                 cookie,
+                                 kstats,
+                                 WantsDeleted::No))
+            << "Expected to get key stats on existing item";
+    EXPECT_EQ(vbucket_state_active, kstats.vb_state);
+    EXPECT_FALSE(kstats.logically_deleted);
+}
+
+// Create then delete an item, checking we get keyStats reporting the item as
+// deleted.
+TEST_P(KVBucketParamTest, GetKeyStatsDeleted) {
+    auto& kvbucket = *engine->getKVBucket();
+    key_stats kstats;
+
+    store_item(0, makeStoredDocKey("key"), "value");
+    delete_item(vbid, makeStoredDocKey("key"));
+
+    // Should get ENOENT if we don't ask for deleted items.
+    EXPECT_EQ(ENGINE_KEY_ENOENT,
+              kvbucket.getKeyStats(makeStoredDocKey("key"),
+                                   0,
+                                   cookie,
+                                   kstats,
+                                   WantsDeleted::No));
+
+    // Should get success (and item flagged as deleted) if we ask for deleted
+    // items.
+    EXPECT_EQ(ENGINE_SUCCESS,
+              kvbucket.getKeyStats(makeStoredDocKey("key"),
+                                   0,
+                                   cookie,
+                                   kstats,
+                                   WantsDeleted::Yes));
+    EXPECT_EQ(vbucket_state_active, kstats.vb_state);
+    EXPECT_TRUE(kstats.logically_deleted);
+}
+
+// Check incorrect vbucket returns not-my-vbucket.
+TEST_P(KVBucketParamTest, GetKeyStatsNMVB) {
+    auto& kvbucket = *engine->getKVBucket();
+    key_stats kstats;
+
+    EXPECT_EQ(ENGINE_NOT_MY_VBUCKET,
+              kvbucket.getKeyStats(makeStoredDocKey("key"),
+                                   1,
+                                   cookie,
+                                   kstats,
+                                   WantsDeleted::No));
+}
+
+// Replace tests //////////////////////////////////////////////////////////////
+
+// Test replace against a non-existent key.
+TEST_P(KVBucketParamTest, ReplaceENOENT) {
+    // Should start with key not existing (and hence cannot replace).
+    auto item = make_item(vbid, makeStoredDocKey("key"), "value");
+    EXPECT_EQ(ENGINE_KEY_ENOENT, store->replace(item, cookie));
+}
+
+// Create then delete an item, checking replace reports ENOENT.
+TEST_P(KVBucketParamTest, ReplaceDeleted) {
+    store_item(vbid, makeStoredDocKey("key"), "value");
+    delete_item(vbid, makeStoredDocKey("key"));
+
+    // Replace should fail.
+    auto item = make_item(vbid, makeStoredDocKey("key"), "value2");
+    EXPECT_EQ(ENGINE_KEY_ENOENT, store->replace(item, cookie));
+}
+
+// Check incorrect vbucket returns not-my-vbucket.
+TEST_P(KVBucketParamTest, ReplaceNMVB) {
+    auto item = make_item(vbid + 1, makeStoredDocKey("key"), "value2");
+    EXPECT_EQ(ENGINE_NOT_MY_VBUCKET, store->replace(item, cookie));
+}
+
+// Check pending vbucket returns EWOULDBLOCK.
+TEST_P(KVBucketParamTest, ReplacePendingVB) {
+    store->setVBucketState(vbid, vbucket_state_pending, false);
+    auto item = make_item(vbid, makeStoredDocKey("key"), "value2");
+    EXPECT_EQ(ENGINE_EWOULDBLOCK, store->replace(item, cookie));
+}
+
+// Set tests //////////////////////////////////////////////////////////////////
+
+// Test CAS set against a non-existent key
+TEST_P(KVBucketParamTest, SetCASNonExistent) {
+    // Create an item with a non-zero CAS.
+    auto item = make_item(vbid, makeStoredDocKey("key"), "value");
+    item.setCas();
+    ASSERT_NE(0, item.getCas());
+
+    // Should get ENOENT as we should immediately know (either from metadata
+    // being resident, or by bloomfilter) that key doesn't exist.
+    EXPECT_EQ(ENGINE_KEY_ENOENT, store->set(item, cookie));
+}
+
+// Add tests //////////////////////////////////////////////////////////////////
+
+// Test successful add
+TEST_P(KVBucketParamTest, Add) {
+    auto item = make_item(vbid, makeStoredDocKey("key"), "value");
+    EXPECT_EQ(ENGINE_SUCCESS, store->add(item, nullptr));
+}
+
+// Check incorrect vbucket returns not-my-vbucket.
+TEST_P(KVBucketParamTest, AddNMVB) {
+    auto item = make_item(vbid + 1, makeStoredDocKey("key"), "value2");
+    EXPECT_EQ(ENGINE_NOT_MY_VBUCKET, store->add(item, cookie));
+}
+
+// SetWithMeta tests //////////////////////////////////////////////////////////
+
+// Test basic setWithMeta
+TEST_P(KVBucketParamTest, SetWithMeta) {
+    auto item = make_item(vbid, makeStoredDocKey("key"), "value");
+    item.setCas();
+    uint64_t seqno;
+    EXPECT_EQ(ENGINE_SUCCESS,
+              store->setWithMeta(item,
+                                 0,
+                                 &seqno,
+                                 cookie,
+                                 /*force*/ false,
+                                 /*allowExisting*/ false));
+}
+
+// Test setWithMeta with a conflict with an existing item.
+TEST_P(KVBucketParamTest, SetWithMeta_Conflicted) {
+    auto item = make_item(vbid, makeStoredDocKey("key"), "value");
+    EXPECT_EQ(ENGINE_SUCCESS, store->set(item, nullptr));
+
+    uint64_t seqno;
+    // Attempt to set with the same rev Seqno - should get EEXISTS.
+    EXPECT_EQ(ENGINE_KEY_EEXISTS,
+              store->setWithMeta(item,
+                                 item.getCas(),
+                                 &seqno,
+                                 cookie,
+                                 /*force*/ false,
+                                 /*allowExisting*/ true));
+}
+
+// Test setWithMeta replacing existing item
+TEST_P(KVBucketParamTest, SetWithMeta_Replace) {
+    auto item = make_item(vbid, makeStoredDocKey("key"), "value");
+    EXPECT_EQ(ENGINE_SUCCESS, store->set(item, nullptr));
+
+    // Increase revSeqno so conflict resolution doesn't fail.
+    item.setRevSeqno(item.getRevSeqno() + 1);
+    uint64_t seqno;
+    // Should get EEXISTS if we don't force (and use wrong CAS).
+    EXPECT_EQ(ENGINE_KEY_EEXISTS,
+              store->setWithMeta(item,
+                                 item.getCas() + 1,
+                                 &seqno,
+                                 cookie,
+                                 /*force*/ false,
+                                 /*allowExisting*/ true));
+
+    // Should succeed with correct CAS, and different RevSeqno.
+    EXPECT_EQ(ENGINE_SUCCESS,
+              store->setWithMeta(item,
+                                 item.getCas(),
+                                 &seqno,
+                                 cookie,
+                                 /*force*/ false,
+                                 /*allowExisting*/ true));
+}
+
+// Test forced setWithMeta
+TEST_P(KVBucketParamTest, SetWithMeta_Forced) {
+    auto item = make_item(vbid, makeStoredDocKey("key"), "value");
+    item.setCas();
+    uint64_t seqno;
+    EXPECT_EQ(ENGINE_SUCCESS,
+              store->setWithMeta(item,
+                                 0,
+                                 &seqno,
+                                 cookie,
+                                 /*force*/ true,
+                                 /*allowExisting*/ false));
+}
+
+// MB and test was raised because a few commits back this was broken but no
+// existing test covered the case. I.e. run this test  against 0810540 and it
+// fails, but now fixed
+TEST_P(KVBucketParamTest, mb22824) {
+    auto key = makeStoredDocKey("key");
+
+    // Store key and force expiry
+    store_item(0, key, "value", 1);
+    TimeTraveller docBrown(20);
+
+    uint32_t deleted = false;
+    ItemMetaData itemMeta1;
+    uint8_t datatype = PROTOCOL_BINARY_RAW_BYTES;
+    EXPECT_EQ(ENGINE_SUCCESS,
+              store->getMetaData(
+                      key, vbid, cookie, false, itemMeta1, deleted, datatype));
+
+    uint64_t cas = 0;
+    ItemMetaData itemMeta2;
+    EXPECT_EQ(ENGINE_KEY_ENOENT,
+              store->deleteItem(key,
+                                cas,
+                                vbid,
+                                cookie,
+                                /*Item*/ nullptr,
+                                &itemMeta2,
+                                /*mutation_descr_t*/ nullptr));
+
+    // Should be getting the same CAS from the failed delete as getMetaData
+    EXPECT_EQ(itemMeta1.cas, itemMeta2.cas);
+}
+
+// Test cases which run for EP (Full and Value eviction) and Ephemeral
+INSTANTIATE_TEST_CASE_P(EphemeralOrPersistent,
+                        KVBucketParamTest,
+                        ::testing::Values("item_eviction_policy=value_only",
+                                          "item_eviction_policy=full_eviction",
+                                          "bucket_type=ephemeral"),
+                        [](const ::testing::TestParamInfo<std::string>& info) {
+                            return info.param.substr(info.param.find('=') + 1);
+                        });
+
+const char KVBucketTest::test_dbname[] = "ep_engine_ep_unit_tests_db";
diff --git a/tests/module_tests/kv_bucket_test.h b/tests/module_tests/kv_bucket_test.h
new file mode 100644 (file)
index 0000000..bf6bce1
--- /dev/null
@@ -0,0 +1,140 @@
+/* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+ *     Copyright 2016 Couchbase, Inc
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ */
+
+/*
+ * Unit tests for the KVBucket class.
+ *
+ * These tests are instantiated with additional config strings to test over
+ * ephemeral and value and full eviction persistent buckets.
+ *
+ */
+
+#pragma once
+
+#include "config.h"
+
+#include "ep_bucket.h"
+#include "ep_engine.h"
+#include "item.h"
+
+#include <memcached/engine.h>
+#include <tests/mock/mock_synchronous_ep_engine.h>
+
+#include <gtest/gtest.h>
+#include <memory>
+
+/**
+ * Test fixture for KVBucket unit tests.
+ *
+ * Will create the appropriate subclass of KVBucket (EPBucket /
+ * EphemeralBucket) based on the Configuration passed (specifically the
+ * bucket_type parameter), defaulting to EPBucket if no bucket_type is
+ * specified.
+ */
+class KVBucketTest : public ::testing::Test {
+public:
+    void SetUp() override;
+
+    void TearDown() override;
+
+    // Stores an item into the given vbucket. Returns the item stored.
+    Item store_item(uint16_t vbid,
+                    const StoredDocKey& key,
+                    const std::string& value,
+                    uint32_t exptime = 0,
+                    const std::vector<cb::engine_errc>& expected =
+                            {cb::engine_errc::success},
+                    protocol_binary_datatype_t datatype =
+                            PROTOCOL_BINARY_DATATYPE_JSON);
+
+    /**
+     * Store multiple items into the vbucket, the given key will have an
+     * iteration appended to it.
+     */
+    ::testing::AssertionResult store_items(
+            int nitems,
+            uint16_t vbid,
+            const DocKey& key,
+            const std::string& value,
+            uint32_t exptime = 0,
+            protocol_binary_datatype_t datatype =
+                    PROTOCOL_BINARY_DATATYPE_JSON);
+
+    /* Flush the given vbucket to disk, so any outstanding dirty items are
+     * written (and are clean).
+     */
+    void flush_vbucket_to_disk(uint16_t vbid, int expected = 1);
+
+    /* Delete the given item from the given vbucket, verifying it was
+     * successfully deleted.
+     */
+    void delete_item(uint16_t vbid, const StoredDocKey& key);
+
+    /* Evict the given key from memory according to the current eviction
+     * strategy. Verifies it was successfully evicted.
+     */
+    void evict_key(uint16_t vbid, const StoredDocKey& key);
+
+    /// Exposes the normally-protected getInternal method from the store.
+    GetValue getInternal(const StoredDocKey& key,
+                         uint16_t vbucket,
+                         const void* cookie,
+                         vbucket_state_t allowedState,
+                         get_options_t options);
+
+    /**
+     * Creates the ItemPager task and adds it to the scheduler. Allows testing
+     * of the item pager from subclasses, without KVBucket having to grant
+     * friendship to many different test classes.
+     */
+    void createAndScheduleItemPager();
+
+    void initializeExpiryPager();
+
+    static const char test_dbname[];
+
+    std::string config_string;
+
+    const uint16_t vbid = 0;
+
+    // The mock engine (needed to construct the store).
+    std::unique_ptr<SynchronousEPEngine> engine;
+
+    // The store under test. Wrapped in a mock to expose some normally
+    // protected members. Uses a raw pointer as this is owned by the engine.
+    KVBucket* store;
+
+    // The (mock) server cookie.
+    const void* cookie;
+};
+
+/**
+ * Test fixture for KVBucket unit tests.
+ *
+ * These tests are parameterized over an extra config string to allow them to
+ * be run against ephemeral and value and full eviction persistent buckets.
+ */
+class KVBucketParamTest : public KVBucketTest,
+                          public ::testing::WithParamInterface<std::string> {
+    void SetUp() override {
+        config_string += GetParam();
+        KVBucketTest::SetUp();
+
+        // Have all the objects, activate vBucket zero so we can store data.
+        store->setVBucketState(vbid, vbucket_state_active, false);
+    }
+};