1 /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
3 * Copyright 2016 Couchbase, Inc
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
9 * http://www.apache.org/licenses/LICENSE-2.0
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
20 #include "vbucket_test.h"
21 #include "bgfetcher.h"
23 #include "failover-table.h"
25 #include "programs/engine_testapp/mock_server.h"
26 #include "tests/module_tests/test_helpers.h"
28 #include <platform/cb_malloc.h>
30 void VBucketTest::SetUp() {
31 const auto eviction_policy = GetParam();
32 vbucket.reset(new EPVBucket(0,
41 std::make_shared<DummyCB>(),
42 /*newSeqnoCb*/ nullptr,
47 void VBucketTest::TearDown() {
51 std::vector<StoredDocKey> VBucketTest::generateKeys(int num, int start) {
52 std::vector<StoredDocKey> rv;
54 for (int i = start; i < num + start; i++) {
55 rv.push_back(makeStoredDocKey(std::to_string(i)));
61 AddStatus VBucketTest::addOne(const StoredDocKey& k, int expiry) {
62 Item i(k, 0, expiry, k.data(), k.size());
63 return public_processAdd(i);
66 void VBucketTest::addMany(std::vector<StoredDocKey>& keys, AddStatus expect) {
67 for (const auto& k : keys) {
68 EXPECT_EQ(expect, addOne(k));
72 MutationStatus VBucketTest::setOne(const StoredDocKey& k, int expiry) {
73 Item i(k, 0, expiry, k.data(), k.size());
74 return public_processSet(i, i.getCas());
77 void VBucketTest::setMany(std::vector<StoredDocKey>& keys,
78 MutationStatus expect) {
79 for (const auto& k : keys) {
80 EXPECT_EQ(expect, setOne(k));
84 void VBucketTest::softDeleteOne(const StoredDocKey& k, MutationStatus expect) {
85 StoredValue* v(vbucket->ht.find(k, TrackReference::No, WantsDeleted::No));
86 EXPECT_NE(nullptr, v);
88 EXPECT_EQ(expect, public_processSoftDelete(v->getKey(), v, 0))
89 << "Failed to soft delete key " << k.c_str();
92 void VBucketTest::softDeleteMany(std::vector<StoredDocKey>& keys,
93 MutationStatus expect) {
94 for (const auto& k : keys) {
95 softDeleteOne(k, expect);
99 StoredValue* VBucketTest::findValue(StoredDocKey& key) {
100 return vbucket->ht.find(key, TrackReference::Yes, WantsDeleted::Yes);
103 void VBucketTest::verifyValue(StoredDocKey& key,
105 TrackReference trackReference,
106 WantsDeleted wantDeleted) {
107 StoredValue* v = vbucket->ht.find(key, trackReference, wantDeleted);
108 EXPECT_NE(nullptr, v);
109 value_t val = v->getValue();
111 EXPECT_EQ(nullptr, val.get());
113 EXPECT_STREQ(value, val->to_s().c_str());
117 std::pair<HashTable::HashBucketLock, StoredValue*> VBucketTest::lockAndFind(
118 const StoredDocKey& key) {
119 auto hbl = vbucket->ht.getLockedBucket(key);
120 auto* storedVal = vbucket->ht.unlocked_find(
121 key, hbl.getBucketNum(), WantsDeleted::Yes, TrackReference::No);
122 return std::make_pair(std::move(hbl), storedVal);
125 MutationStatus VBucketTest::public_processSet(Item& itm,
128 auto hbl_sv = lockAndFind(itm.getKey());
129 VBQueueItemCtx queueItmCtx(GenerateBySeqno::Yes,
132 /*isBackfillItem*/ false,
133 /*preLinkDocumentContext_*/ nullptr);
135 ->processSet(hbl_sv.first,
141 withCtx ? &queueItmCtx : nullptr)
145 AddStatus VBucketTest::public_processAdd(Item& itm) {
146 auto hbl_sv = lockAndFind(itm.getKey());
148 ->processAdd(hbl_sv.first,
151 /*maybeKeyExists*/ true,
152 /*isReplication*/ false)
156 MutationStatus VBucketTest::public_processSoftDelete(const DocKey& key,
159 auto hbl = vbucket->ht.getLockedBucket(key);
161 v = vbucket->ht.unlocked_find(
162 key, hbl.getBucketNum(), WantsDeleted::No, TrackReference::No);
164 return MutationStatus::NotFound;
167 ItemMetaData metadata;
168 metadata.revSeqno = v->getRevSeqno() + 1;
169 MutationStatus status;
170 std::tie(status, std::ignore, std::ignore) = vbucket->processSoftDelete(
175 VBQueueItemCtx(GenerateBySeqno::Yes,
178 /*isBackfillItem*/ false,
179 /*preLinkDocCtx*/ nullptr),
181 /*bySeqno*/ v->getBySeqno());
185 bool VBucketTest::public_deleteStoredValue(const DocKey& key) {
186 auto hbl_sv = lockAndFind(key);
187 if (!hbl_sv.second) {
190 return vbucket->deleteStoredValue(hbl_sv.first, *hbl_sv.second);
193 GetValue VBucketTest::public_getAndUpdateTtl(const DocKey& key,
195 auto hbl = lockAndFind(key);
197 MutationStatus status;
198 std::tie(status, gv) = vbucket->processGetAndUpdateTtl(
199 hbl.first, key, hbl.second, exptime);
203 size_t EPVBucketTest::public_queueBGFetchItem(
205 std::unique_ptr<VBucketBGFetchItem> fetchItem,
206 BgFetcher* bgFetcher) {
207 return dynamic_cast<EPVBucket&>(*vbucket).queueBGFetchItem(
208 key, std::move(fetchItem), bgFetcher);
211 class BlobTest : public Blob {
213 BlobTest() : Blob(0,0) {}
214 static size_t getAllocationSize(size_t len){
215 return Blob::getAllocationSize(len);
219 TEST(BlobTest, basicAllocationSize){
220 EXPECT_EQ(BlobTest::getAllocationSize(10), 20);
222 // Expected to be 10 because the 2 bytes of the data member array will not
223 // be allocated because they will not be used.
224 EXPECT_EQ(BlobTest::getAllocationSize(0), 10);
227 // Measure performance of VBucket::getBGFetchItems - queue and then get
228 // 10,000 items from the vbucket.
229 TEST_P(EPVBucketTest, GetBGFetchItemsPerformance) {
230 BgFetcher fetcher(/*store*/ nullptr, /*shard*/ nullptr, this->global_stats);
232 for (unsigned int ii = 0; ii < 100000; ii++) {
233 auto fetchItem = std::make_unique<VBucketBGFetchItem>(
236 this->public_queueBGFetchItem(
237 makeStoredDocKey(std::to_string(ii)),
238 std::move(fetchItem),
241 auto items = this->vbucket->getBGFetchItems();
244 // Check the existence of bloom filter after performing a
245 // swap of existing filter with a temporary filter.
246 TEST_P(VBucketTest, SwapFilter) {
247 this->vbucket->createFilter(1, 1.0);
248 ASSERT_FALSE(this->vbucket->isTempFilterAvailable());
249 ASSERT_NE("DOESN'T EXIST", this->vbucket->getFilterStatusString());
250 this->vbucket->swapFilter();
251 EXPECT_NE("DOESN'T EXIST", this->vbucket->getFilterStatusString());
254 TEST_P(VBucketTest, Add) {
255 const auto eviction_policy = GetParam();
256 if (eviction_policy != VALUE_ONLY) {
259 const int nkeys = 1000;
261 auto keys = generateKeys(nkeys);
262 addMany(keys, AddStatus::Success);
264 StoredDocKey missingKey = makeStoredDocKey("aMissingKey");
265 EXPECT_FALSE(this->vbucket->ht.find(
266 missingKey, TrackReference::Yes, WantsDeleted::No));
268 for (const auto& key : keys) {
269 EXPECT_TRUE(this->vbucket->ht.find(
270 key, TrackReference::Yes, WantsDeleted::No));
273 addMany(keys, AddStatus::Exists);
274 for (const auto& key : keys) {
275 EXPECT_TRUE(this->vbucket->ht.find(
276 key, TrackReference::Yes, WantsDeleted::No));
279 // Verify we can read after a soft deletion.
280 EXPECT_EQ(MutationStatus::WasDirty,
281 this->public_processSoftDelete(keys[0], nullptr, 0));
282 EXPECT_EQ(MutationStatus::NotFound,
283 this->public_processSoftDelete(keys[0], nullptr, 0));
284 EXPECT_FALSE(this->vbucket->ht.find(
285 keys[0], TrackReference::Yes, WantsDeleted::No));
287 Item i(keys[0], 0, 0, "newtest", 7);
288 EXPECT_EQ(AddStatus::UnDel, this->public_processAdd(i));
289 EXPECT_EQ(nkeys, this->vbucket->ht.getNumItems());
292 TEST_P(VBucketTest, AddExpiry) {
293 const auto eviction_policy = GetParam();
294 if (eviction_policy != VALUE_ONLY) {
297 StoredDocKey k = makeStoredDocKey("aKey");
299 ASSERT_EQ(AddStatus::Success, addOne(k, ep_real_time() + 5));
300 EXPECT_EQ(AddStatus::Exists, addOne(k, ep_real_time() + 5));
303 this->vbucket->ht.find(k, TrackReference::Yes, WantsDeleted::No);
305 EXPECT_FALSE(v->isExpired(ep_real_time()));
306 EXPECT_TRUE(v->isExpired(ep_real_time() + 6));
308 TimeTraveller biffTannen(6);
309 EXPECT_TRUE(v->isExpired(ep_real_time()));
311 EXPECT_EQ(AddStatus::UnDel, addOne(k, ep_real_time() + 5));
313 EXPECT_FALSE(v->isExpired(ep_real_time()));
314 EXPECT_TRUE(v->isExpired(ep_real_time() + 6));
318 * Test to check if an unlocked_softDelete performed on an
319 * existing item with a new value results in a success
321 TEST_P(VBucketTest, unlockedSoftDeleteWithValue) {
322 const auto eviction_policy = GetParam();
323 if (eviction_policy != VALUE_ONLY) {
327 // Setup - create a key and then delete it with a value.
328 StoredDocKey key = makeStoredDocKey("key");
329 Item stored_item(key, 0, 0, "value", strlen("value"));
330 ASSERT_EQ(MutationStatus::WasClean,
331 this->public_processSet(stored_item, stored_item.getCas()));
334 this->vbucket->ht.find(key, TrackReference::No, WantsDeleted::No));
335 EXPECT_NE(nullptr, v);
337 // Create an item and set its state to deleted
338 Item deleted_item(key, 0, 0, "deletedvalue", strlen("deletedvalue"));
339 deleted_item.setDeleted();
341 auto prev_revseqno = v->getRevSeqno();
342 deleted_item.setRevSeqno(prev_revseqno);
344 EXPECT_EQ(MutationStatus::WasDirty,
345 this->public_processSet(deleted_item, 0));
346 verifyValue(key, "deletedvalue", TrackReference::Yes, WantsDeleted::Yes);
347 EXPECT_EQ(prev_revseqno + 1, v->getRevSeqno());
351 * Test to check that if an item has expired, an incoming mutation
352 * on that item, if in deleted state results in an invalid cas and
353 * if not in deleted state, results in not found
355 TEST_P(VBucketTest, updateExpiredItem) {
356 // Setup - create a key
357 StoredDocKey key = makeStoredDocKey("key");
358 Item stored_item(key, 0, ep_real_time() - 1, "value", strlen("value"));
359 ASSERT_EQ(MutationStatus::WasClean,
360 this->public_processSet(stored_item, stored_item.getCas()));
362 StoredValue* v = this->vbucket->ht.find(key, TrackReference::No, WantsDeleted::No);
364 EXPECT_TRUE(v->isExpired(ep_real_time()));
366 auto cas = v->getCas();
367 // Create an item and set its state to deleted.
368 Item deleted_item(key, 0, 0, "deletedvalue", strlen("deletedvalue"));
370 EXPECT_EQ(MutationStatus::NotFound,
371 this->public_processSet(deleted_item, cas + 1));
373 deleted_item.setDeleted();
374 EXPECT_EQ(MutationStatus::InvalidCas,
375 this->public_processSet(deleted_item, cas + 1));
379 * Test to check if an unlocked_softDelete performed on a
380 * deleted item without a value and with a value
382 TEST_P(VBucketTest, updateDeletedItem) {
383 const auto eviction_policy = GetParam();
384 if (eviction_policy != VALUE_ONLY) {
388 // Setup - create a key and then delete it.
389 StoredDocKey key = makeStoredDocKey("key");
390 Item stored_item(key, 0, 0, "value", strlen("value"));
391 ASSERT_EQ(MutationStatus::WasClean,
392 this->public_processSet(stored_item, stored_item.getCas()));
395 this->vbucket->ht.find(key, TrackReference::No, WantsDeleted::No));
396 EXPECT_NE(nullptr, v);
398 ItemMetaData itm_meta;
399 EXPECT_EQ(MutationStatus::WasDirty,
400 this->public_processSoftDelete(v->getKey(), v, 0));
401 verifyValue(key, nullptr, TrackReference::Yes, WantsDeleted::Yes);
402 EXPECT_EQ(1, this->vbucket->getNumItems());
403 EXPECT_EQ(1, this->vbucket->getNumInMemoryDeletes());
405 Item deleted_item(key, 0, 0, "deletedvalue", strlen("deletedvalue"));
406 deleted_item.setDeleted();
408 auto prev_revseqno = v->getRevSeqno();
409 deleted_item.setRevSeqno(prev_revseqno);
411 EXPECT_EQ(MutationStatus::WasDirty,
412 this->public_processSet(deleted_item, 0));
418 EXPECT_EQ(1, this->vbucket->getNumItems());
419 EXPECT_EQ(1, this->vbucket->getNumInMemoryDeletes());
420 EXPECT_EQ(prev_revseqno + 1, v->getRevSeqno());
422 Item update_deleted_item(
423 key, 0, 0, "updatedeletedvalue", strlen("updatedeletedvalue"));
424 update_deleted_item.setDeleted();
426 prev_revseqno = v->getRevSeqno();
427 update_deleted_item.setRevSeqno(prev_revseqno);
429 EXPECT_EQ(MutationStatus::WasDirty,
430 this->public_processSet(update_deleted_item, 0));
432 key, "updatedeletedvalue", TrackReference::Yes, WantsDeleted::Yes);
433 EXPECT_EQ(1, this->vbucket->getNumItems());
434 EXPECT_EQ(1, this->vbucket->getNumInMemoryDeletes());
435 EXPECT_EQ(prev_revseqno + 1, v->getRevSeqno());
438 TEST_P(VBucketTest, SizeStatsSoftDel) {
439 this->global_stats.reset();
440 ASSERT_EQ(0, this->vbucket->ht.memSize.load());
441 ASSERT_EQ(0, this->vbucket->ht.cacheSize.load());
442 size_t initialSize = this->global_stats.currentSize.load();
444 const StoredDocKey k = makeStoredDocKey("somekey");
445 const size_t itemSize(16 * 1024);
446 char* someval(static_cast<char*>(cb_calloc(1, itemSize)));
447 EXPECT_TRUE(someval);
449 Item i(k, 0, 0, someval, itemSize);
451 EXPECT_EQ(MutationStatus::WasClean,
452 this->public_processSet(i, i.getCas()));
454 EXPECT_EQ(MutationStatus::WasDirty,
455 this->public_processSoftDelete(k, nullptr, 0));
456 this->public_deleteStoredValue(k);
458 EXPECT_EQ(0, this->vbucket->ht.memSize.load());
459 EXPECT_EQ(0, this->vbucket->ht.cacheSize.load());
460 EXPECT_EQ(initialSize, this->global_stats.currentSize.load());
465 TEST_P(VBucketTest, SizeStatsSoftDelFlush) {
466 this->global_stats.reset();
467 ASSERT_EQ(0, this->vbucket->ht.memSize.load());
468 ASSERT_EQ(0, this->vbucket->ht.cacheSize.load());
469 size_t initialSize = this->global_stats.currentSize.load();
471 StoredDocKey k = makeStoredDocKey("somekey");
472 const size_t itemSize(16 * 1024);
473 char* someval(static_cast<char*>(cb_calloc(1, itemSize)));
474 EXPECT_TRUE(someval);
476 Item i(k, 0, 0, someval, itemSize);
478 EXPECT_EQ(MutationStatus::WasClean,
479 this->public_processSet(i, i.getCas()));
481 EXPECT_EQ(MutationStatus::WasDirty,
482 this->public_processSoftDelete(k, nullptr, 0));
483 this->vbucket->ht.clear();
485 EXPECT_EQ(0, this->vbucket->ht.memSize.load());
486 EXPECT_EQ(0, this->vbucket->ht.cacheSize.load());
487 EXPECT_EQ(initialSize, this->global_stats.currentSize.load());
492 class VBucketEvictionTest : public VBucketTest {};
494 // Check that counts of items and resident items are as expected when items are
495 // ejected from the HashTable.
496 TEST_P(VBucketEvictionTest, EjectionResidentCount) {
497 const auto eviction_policy = GetParam();
498 ASSERT_EQ(0, this->vbucket->getNumItems());
499 ASSERT_EQ(0, this->vbucket->getNumNonResidentItems());
501 Item item(makeStoredDocKey("key"), /*flags*/0, /*exp*/0,
502 /*data*/nullptr, /*ndata*/0);
504 EXPECT_EQ(MutationStatus::WasClean,
505 this->public_processSet(item, item.getCas()));
507 EXPECT_EQ(1, this->vbucket->getNumItems());
508 EXPECT_EQ(0, this->vbucket->getNumNonResidentItems());
510 // TODO-MT: Should acquire lock really (ok given this is currently
512 auto* stored_item = this->vbucket->ht.find(
513 makeStoredDocKey("key"), TrackReference::Yes, WantsDeleted::No);
514 EXPECT_NE(nullptr, stored_item);
515 // Need to clear the dirty flag to allow it to be ejected.
516 stored_item->markClean();
517 EXPECT_TRUE(this->vbucket->ht.unlocked_ejectItem(stored_item,
520 // After ejection, should still have 1 item in VBucket, but also have
521 // 1 non-resident item.
522 EXPECT_EQ(1, this->vbucket->getNumItems());
523 EXPECT_EQ(1, this->vbucket->getNumNonResidentItems());
526 // Regression test for MB-21448 - if an attempt is made to perform a CAS
527 // operation on a logically deleted item we should return NOT_FOUND
528 // (aka KEY_ENOENT) and *not* INVALID_CAS (aka KEY_EEXISTS).
529 TEST_P(VBucketEvictionTest, MB21448_UnlockedSetWithCASDeleted) {
530 // Setup - create a key and then delete it.
531 StoredDocKey key = makeStoredDocKey("key");
532 Item item(key, 0, 0, "deleted", strlen("deleted"));
533 ASSERT_EQ(MutationStatus::WasClean,
534 this->public_processSet(item, item.getCas()));
535 ASSERT_EQ(MutationStatus::WasDirty,
536 this->public_processSoftDelete(key, nullptr, 0));
538 // Attempt to perform a set on a deleted key with a CAS.
539 Item replacement(key, 0, 0, "value", strlen("value"));
540 EXPECT_EQ(MutationStatus::NotFound,
541 this->public_processSet(replacement,
543 << "When trying to replace-with-CAS a deleted item";
546 // Test cases which run in both Full and Value eviction
547 INSTANTIATE_TEST_CASE_P(
548 FullAndValueEviction,
550 ::testing::Values(VALUE_ONLY, FULL_EVICTION),
551 [](const ::testing::TestParamInfo<item_eviction_policy_t>& info) {
552 if (info.param == VALUE_ONLY) {
555 return "FULL_EVICTION";
559 INSTANTIATE_TEST_CASE_P(
560 FullAndValueEviction,
562 ::testing::Values(VALUE_ONLY, FULL_EVICTION),
563 [](const ::testing::TestParamInfo<item_eviction_policy_t>& info) {
564 if (info.param == VALUE_ONLY) {
567 return "FULL_EVICTION";
571 INSTANTIATE_TEST_CASE_P(
572 FullAndValueEviction,
574 ::testing::Values(VALUE_ONLY, FULL_EVICTION),
575 [](const ::testing::TestParamInfo<item_eviction_policy_t>& info) {
576 if (info.param == VALUE_ONLY) {
579 return "FULL_EVICTION";