1 /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
3 * Copyright 2017 Couchbase, Inc
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
9 * http://www.apache.org/licenses/LICENSE-2.0
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
19 * Unit tests for the KVBucket class.
22 #include "kv_bucket_test.h"
24 #include "../mock/mock_dcp_producer.h"
25 #include "bgfetcher.h"
26 #include "checkpoint.h"
27 #include "checkpoint_remover.h"
28 #include "dcp/dcpconnmap.h"
29 #include "dcp/flow-control-manager.h"
30 #include "ep_engine.h"
32 #include "replicationthrottle.h"
33 #include "tapconnmap.h"
35 #include "tests/mock/mock_global_task.h"
36 #include "tests/module_tests/test_helpers.h"
37 #include "vbucketdeletiontask.h"
39 #include <platform/dirutils.h>
43 #include <string_utilities.h>
44 #include <xattr/blob.h>
45 #include <xattr/utils.h>
47 void KVBucketTest::SetUp() {
48 // Paranoia - kill any existing files in case they are left over
49 // from a previous run.
50 cb::io::rmrf(test_dbname);
52 // Add dbname to config string.
53 std::string config = config_string;
54 if (config.size() > 0) {
57 config += "dbname=" + std::string(test_dbname);
59 engine.reset(new SynchronousEPEngine(config));
60 ObjectRegistry::onSwitchThread(engine.get());
62 engine->setKVBucket(engine->public_makeBucket(engine->getConfiguration()));
63 store = engine->getKVBucket();
65 store->chkTask = new ClosedUnrefCheckpointRemoverTask(
68 engine->getConfiguration().getChkRemoverStime());
70 // Ensure that EPEngine is hold about necessary server callbacks
71 // (client disconnect, bucket delete).
72 engine->public_initializeEngineCallbacks();
74 // Need to initialize ep_real_time and friends.
75 initialize_time_functions(get_mock_server_api()->core);
77 cookie = create_mock_cookie();
80 void KVBucketTest::TearDown() {
81 destroy_mock_cookie(cookie);
82 destroy_mock_event_callbacks();
83 engine->getDcpConnMap().manageConnections();
84 ObjectRegistry::onSwitchThread(nullptr);
87 // Shutdown the ExecutorPool singleton (initialized when we create
88 // an EPBucket object). Must happen after engine
89 // has been destroyed (to allow the tasks the engine has
90 // registered a chance to be unregistered).
91 ExecutorPool::shutdown();
94 Item KVBucketTest::store_item(uint16_t vbid,
95 const StoredDocKey& key,
96 const std::string& value,
98 const std::vector<cb::engine_errc>& expected,
99 protocol_binary_datatype_t datatype) {
100 auto item = make_item(vbid, key, value, exptime, datatype);
101 auto returnCode = store->set(item, nullptr);
102 EXPECT_NE(expected.end(),
103 std::find(expected.begin(),
105 cb::engine_errc(returnCode)));
109 ::testing::AssertionResult KVBucketTest::store_items(
113 const std::string& value,
115 protocol_binary_datatype_t datatype) {
116 for (int ii = 0; ii < nitems; ii++) {
117 auto keyii = makeStoredDocKey(
118 std::string(reinterpret_cast<const char*>(key.data()),
121 key.getDocNamespace());
122 auto item = make_item(vbid, keyii, value, exptime, datatype);
123 auto err = store->set(item, nullptr);
124 if (ENGINE_SUCCESS != err) {
125 return ::testing::AssertionFailure()
126 << "Failed to store " << keyii.data() << " error:" << err;
129 return ::testing::AssertionSuccess();
132 void KVBucketTest::flush_vbucket_to_disk(uint16_t vbid, int expected) {
134 const auto time_limit = std::chrono::seconds(10);
135 const auto deadline = std::chrono::steady_clock::now() + time_limit;
137 // Need to retry as warmup may not have completed.
138 bool flush_successful = false;
140 result = store->flushVBucket(vbid);
141 if (result != RETRY_FLUSH_VBUCKET) {
142 flush_successful = true;
145 std::this_thread::sleep_for(std::chrono::microseconds(100));
146 } while (std::chrono::steady_clock::now() < deadline);
148 ASSERT_TRUE(flush_successful)
149 << "Hit timeout (" << time_limit.count()
150 << " seconds) waiting for "
151 "warmup to complete while flushing VBucket.";
153 ASSERT_EQ(expected, result) << "Unexpected items in flush_vbucket_to_disk";
156 void KVBucketTest::delete_item(uint16_t vbid, const StoredDocKey& key) {
158 EXPECT_EQ(ENGINE_SUCCESS,
159 store->deleteItem(key,
163 /*itemMeta*/ nullptr,
164 /*mutation_descr_t*/ nullptr));
167 void KVBucketTest::evict_key(uint16_t vbid, const StoredDocKey& key) {
169 EXPECT_EQ(ENGINE_SUCCESS, store->evictKey(key, vbid, &msg));
170 EXPECT_STREQ("Ejected.", msg);
173 GetValue KVBucketTest::getInternal(const StoredDocKey& key,
176 vbucket_state_t allowedState,
177 get_options_t options) {
178 return store->getInternal(key, vbucket, cookie, allowedState, options);
181 void KVBucketTest::createAndScheduleItemPager() {
182 store->itemPagerTask = new ItemPager(engine.get(), engine->getEpStats());
183 ExecutorPool::get()->schedule(store->itemPagerTask);
186 void KVBucketTest::initializeExpiryPager() {
187 store->initializeExpiryPager(engine->getConfiguration());
191 * Create a del_with_meta packet with the key/body (body can be empty)
193 std::vector<char> KVBucketTest::buildWithMetaPacket(
194 protocol_binary_command opcode,
195 protocol_binary_datatype_t datatype,
199 ItemMetaData metaData,
200 const std::string& key,
201 const std::string& body,
202 const std::vector<char>& emd,
204 EXPECT_EQ(sizeof(protocol_binary_request_set_with_meta),
205 sizeof(protocol_binary_request_delete_with_meta));
207 size_t size = sizeof(protocol_binary_request_set_with_meta);
208 // body at least the meta
209 size_t extlen = (sizeof(uint32_t) * 2) + (sizeof(uint64_t) * 2);
210 size_t bodylen = extlen;
212 size += sizeof(uint32_t);
213 bodylen += sizeof(uint32_t);
214 extlen += sizeof(uint32_t);
217 EXPECT_TRUE(emd.size() < std::numeric_limits<uint16_t>::max());
218 size += sizeof(uint16_t) + emd.size();
219 bodylen += sizeof(uint16_t) + emd.size();
220 extlen += sizeof(uint16_t);
223 bodylen += body.size();
225 bodylen += key.size();
227 protocol_binary_request_set_with_meta header;
228 header.message.header.request.magic = PROTOCOL_BINARY_REQ;
229 header.message.header.request.opcode = opcode;
230 header.message.header.request.keylen = htons(key.size());
231 header.message.header.request.extlen = uint8_t(extlen);
232 header.message.header.request.datatype = datatype;
233 header.message.header.request.vbucket = htons(vbucket);
234 header.message.header.request.bodylen = htonl(bodylen);
235 header.message.header.request.opaque = opaque;
236 header.message.header.request.cas = htonll(cas);
237 header.message.body.flags = metaData.flags;
238 header.message.body.expiration = htonl(metaData.exptime);
239 header.message.body.seqno = htonll(metaData.revSeqno);
240 header.message.body.cas = htonll(metaData.cas);
242 std::vector<char> packet;
243 packet.reserve(size);
244 packet.insert(packet.end(),
245 reinterpret_cast<char*>(&header),
246 reinterpret_cast<char*>(&header) +
247 sizeof(protocol_binary_request_set_with_meta));
250 options = htonl(options);
251 std::copy_n(reinterpret_cast<char*>(&options),
253 std::back_inserter(packet));
257 uint16_t emdSize = htons(emd.size());
258 std::copy_n(reinterpret_cast<char*>(&emdSize),
260 std::back_inserter(packet));
263 std::copy_n(key.c_str(), key.size(), std::back_inserter(packet));
264 std::copy_n(body.c_str(), body.size(), std::back_inserter(packet));
265 packet.insert(packet.end(), emd.begin(), emd.end());
269 std::string KVBucketTest::createXattrValue(const std::string& body) {
270 cb::xattr::Blob blob;
273 blob.set(to_const_byte_buffer("user"),
274 to_const_byte_buffer("{\"author\":\"bubba\"}"));
275 blob.set(to_const_byte_buffer("_sync"),
276 to_const_byte_buffer("{\"cas\":\"0xdeadbeefcafefeed\"}"));
277 blob.set(to_const_byte_buffer("meta"),
278 to_const_byte_buffer("{\"content-type\":\"text\"}"));
280 auto xattrValue = blob.finalize();
282 // append body to the xattrs and store in data
284 std::copy_n(xattrValue.buf, xattrValue.len, std::back_inserter(data));
285 std::copy_n(body.c_str(), body.size(), std::back_inserter(data));
290 bool KVBucketTest::addResponse(const void* k,
299 const void* cookie) {
300 addResponseStatus = protocol_binary_response_status(status);
304 protocol_binary_response_status KVBucketTest::getAddResponseStatus(
305 protocol_binary_response_status newval) {
306 protocol_binary_response_status rv = addResponseStatus;
307 addResponseStatus = newval;
311 protocol_binary_response_status KVBucketTest::addResponseStatus =
312 PROTOCOL_BINARY_RESPONSE_SUCCESS;
315 // getKeyStats tests //////////////////////////////////////////////////////////
317 // Check that keystats on resident items works correctly.
318 TEST_P(KVBucketParamTest, GetKeyStatsResident) {
321 // Should start with key not existing.
322 EXPECT_EQ(ENGINE_KEY_ENOENT,
323 store->getKeyStats(makeStoredDocKey("key"),
329 store_item(0, makeStoredDocKey("key"), "value");
330 EXPECT_EQ(ENGINE_SUCCESS,
331 store->getKeyStats(makeStoredDocKey("key"),
336 << "Expected to get key stats on existing item";
337 EXPECT_EQ(vbucket_state_active, kstats.vb_state);
338 EXPECT_FALSE(kstats.logically_deleted);
341 // Create then delete an item, checking we get keyStats reporting the item as
343 TEST_P(KVBucketParamTest, GetKeyStatsDeleted) {
344 auto& kvbucket = *engine->getKVBucket();
347 store_item(0, makeStoredDocKey("key"), "value");
348 delete_item(vbid, makeStoredDocKey("key"));
350 // Should get ENOENT if we don't ask for deleted items.
351 EXPECT_EQ(ENGINE_KEY_ENOENT,
352 kvbucket.getKeyStats(makeStoredDocKey("key"),
358 // Should get success (and item flagged as deleted) if we ask for deleted
360 EXPECT_EQ(ENGINE_SUCCESS,
361 kvbucket.getKeyStats(makeStoredDocKey("key"),
366 EXPECT_EQ(vbucket_state_active, kstats.vb_state);
367 EXPECT_TRUE(kstats.logically_deleted);
370 // Check incorrect vbucket returns not-my-vbucket.
371 TEST_P(KVBucketParamTest, GetKeyStatsNMVB) {
372 auto& kvbucket = *engine->getKVBucket();
375 EXPECT_EQ(ENGINE_NOT_MY_VBUCKET,
376 kvbucket.getKeyStats(makeStoredDocKey("key"),
383 // Replace tests //////////////////////////////////////////////////////////////
385 // Test replace against a non-existent key.
386 TEST_P(KVBucketParamTest, ReplaceENOENT) {
387 // Should start with key not existing (and hence cannot replace).
388 auto item = make_item(vbid, makeStoredDocKey("key"), "value");
389 EXPECT_EQ(ENGINE_KEY_ENOENT, store->replace(item, cookie));
392 // Create then delete an item, checking replace reports ENOENT.
393 TEST_P(KVBucketParamTest, ReplaceDeleted) {
394 store_item(vbid, makeStoredDocKey("key"), "value");
395 delete_item(vbid, makeStoredDocKey("key"));
397 // Replace should fail.
398 auto item = make_item(vbid, makeStoredDocKey("key"), "value2");
399 EXPECT_EQ(ENGINE_KEY_ENOENT, store->replace(item, cookie));
402 // Check incorrect vbucket returns not-my-vbucket.
403 TEST_P(KVBucketParamTest, ReplaceNMVB) {
404 auto item = make_item(vbid + 1, makeStoredDocKey("key"), "value2");
405 EXPECT_EQ(ENGINE_NOT_MY_VBUCKET, store->replace(item, cookie));
408 // Check pending vbucket returns EWOULDBLOCK.
409 TEST_P(KVBucketParamTest, ReplacePendingVB) {
410 store->setVBucketState(vbid, vbucket_state_pending, false);
411 auto item = make_item(vbid, makeStoredDocKey("key"), "value2");
412 EXPECT_EQ(ENGINE_EWOULDBLOCK, store->replace(item, cookie));
415 // Set tests //////////////////////////////////////////////////////////////////
417 // Test CAS set against a non-existent key
418 TEST_P(KVBucketParamTest, SetCASNonExistent) {
419 // Create an item with a non-zero CAS.
420 auto item = make_item(vbid, makeStoredDocKey("key"), "value");
422 ASSERT_NE(0, item.getCas());
424 // Should get ENOENT as we should immediately know (either from metadata
425 // being resident, or by bloomfilter) that key doesn't exist.
426 EXPECT_EQ(ENGINE_KEY_ENOENT, store->set(item, cookie));
429 // Test CAS set against a deleted item
430 TEST_P(KVBucketParamTest, SetCASDeleted) {
431 auto key = makeStoredDocKey("key");
432 auto item = make_item(vbid, key, "value");
435 EXPECT_EQ(ENGINE_SUCCESS, store->set(item, cookie));
439 EXPECT_EQ(ENGINE_SUCCESS,
440 store->deleteItem(key,
444 /*itemMeta*/ nullptr,
445 /*mutation_descr_t*/ nullptr));
447 if (engine->getConfiguration().getBucketType() == "persistent") {
448 // Trigger a flush to disk.
449 flush_vbucket_to_disk(vbid);
452 // check we have the cas
455 auto item2 = make_item(vbid, key, "value2");
459 if (engine->getConfiguration().getItemEvictionPolicy() ==
461 EXPECT_EQ(ENGINE_EWOULDBLOCK, store->set(item2, cookie));
463 // Manually run the bgfetch task.
464 MockGlobalTask mockTask(engine->getTaskable(),
465 TaskId::MultiBGFetcherTask);
466 store->getVBucket(vbid)->getShard()->getBgFetcher()->run(&mockTask);
469 EXPECT_EQ(ENGINE_KEY_ENOENT, store->set(item2, cookie));
472 // Add tests //////////////////////////////////////////////////////////////////
474 // Test successful add
475 TEST_P(KVBucketParamTest, Add) {
476 auto item = make_item(vbid, makeStoredDocKey("key"), "value");
477 EXPECT_EQ(ENGINE_SUCCESS, store->add(item, nullptr));
480 // Check incorrect vbucket returns not-my-vbucket.
481 TEST_P(KVBucketParamTest, AddNMVB) {
482 auto item = make_item(vbid + 1, makeStoredDocKey("key"), "value2");
483 EXPECT_EQ(ENGINE_NOT_MY_VBUCKET, store->add(item, cookie));
486 // SetWithMeta tests //////////////////////////////////////////////////////////
488 // Test basic setWithMeta
489 TEST_P(KVBucketParamTest, SetWithMeta) {
490 auto item = make_item(vbid, makeStoredDocKey("key"), "value");
493 EXPECT_EQ(ENGINE_SUCCESS,
494 store->setWithMeta(item,
499 /*allowExisting*/ false));
502 // Test setWithMeta with a conflict with an existing item.
503 TEST_P(KVBucketParamTest, SetWithMeta_Conflicted) {
504 auto item = make_item(vbid, makeStoredDocKey("key"), "value");
505 EXPECT_EQ(ENGINE_SUCCESS, store->set(item, nullptr));
508 // Attempt to set with the same rev Seqno - should get EEXISTS.
509 EXPECT_EQ(ENGINE_KEY_EEXISTS,
510 store->setWithMeta(item,
515 /*allowExisting*/ true));
518 // Test setWithMeta replacing existing item
519 TEST_P(KVBucketParamTest, SetWithMeta_Replace) {
520 auto item = make_item(vbid, makeStoredDocKey("key"), "value");
521 EXPECT_EQ(ENGINE_SUCCESS, store->set(item, nullptr));
523 // Increase revSeqno so conflict resolution doesn't fail.
524 item.setRevSeqno(item.getRevSeqno() + 1);
526 // Should get EEXISTS if we don't force (and use wrong CAS).
527 EXPECT_EQ(ENGINE_KEY_EEXISTS,
528 store->setWithMeta(item,
533 /*allowExisting*/ true));
535 // Should succeed with correct CAS, and different RevSeqno.
536 EXPECT_EQ(ENGINE_SUCCESS,
537 store->setWithMeta(item,
542 /*allowExisting*/ true));
545 // Test forced setWithMeta
546 TEST_P(KVBucketParamTest, SetWithMeta_Forced) {
547 auto item = make_item(vbid, makeStoredDocKey("key"), "value");
550 EXPECT_EQ(ENGINE_SUCCESS,
551 store->setWithMeta(item,
556 /*allowExisting*/ false));
559 // MB and test was raised because a few commits back this was broken but no
560 // existing test covered the case. I.e. run this test against 0810540 and it
561 // fails, but now fixed
562 TEST_P(KVBucketParamTest, mb22824) {
563 auto key = makeStoredDocKey("key");
565 // Store key and force expiry
566 store_item(0, key, "value", 1);
567 TimeTraveller docBrown(20);
569 uint32_t deleted = false;
570 ItemMetaData itemMeta1;
571 uint8_t datatype = PROTOCOL_BINARY_RAW_BYTES;
572 EXPECT_EQ(ENGINE_SUCCESS,
574 key, vbid, cookie, itemMeta1, deleted, datatype));
577 ItemMetaData itemMeta2;
578 EXPECT_EQ(ENGINE_KEY_ENOENT,
579 store->deleteItem(key,
584 /*mutation_descr_t*/ nullptr));
586 // Should be getting the same CAS from the failed delete as getMetaData
587 EXPECT_EQ(itemMeta1.cas, itemMeta2.cas);
590 // Test cases which run for EP (Full and Value eviction) and Ephemeral
591 INSTANTIATE_TEST_CASE_P(EphemeralOrPersistent,
593 ::testing::Values("item_eviction_policy=value_only",
594 "item_eviction_policy=full_eviction",
595 "bucket_type=ephemeral"),
596 [](const ::testing::TestParamInfo<std::string>& info) {
597 return info.param.substr(info.param.find('=') + 1);
600 const char KVBucketTest::test_dbname[] = "ep_engine_ep_unit_tests_db";