5068eee220feb93e5ff3bf50859d65297d8abf62
[ep-engine.git] / tests / module_tests / vbucket_test.cc
1 /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 /*
3  *     Copyright 2016 Couchbase, Inc
4  *
5  *   Licensed under the Apache License, Version 2.0 (the "License");
6  *   you may not use this file except in compliance with the License.
7  *   You may obtain a copy of the License at
8  *
9  *       http://www.apache.org/licenses/LICENSE-2.0
10  *
11  *   Unless required by applicable law or agreed to in writing, software
12  *   distributed under the License is distributed on an "AS IS" BASIS,
13  *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  *   See the License for the specific language governing permissions and
15  *   limitations under the License.
16  */
17
18 #include "config.h"
19
20 #include "vbucket_test.h"
21 #include "bgfetcher.h"
22 #include "ep_vb.h"
23 #include "failover-table.h"
24 #include "item.h"
25 #include "programs/engine_testapp/mock_server.h"
26 #include "tests/module_tests/test_helpers.h"
27
28 #include <platform/cb_malloc.h>
29
30 void VBucketTest::SetUp() {
31     const auto eviction_policy = GetParam();
32     vbucket.reset(new EPVBucket(0,
33                                 vbucket_state_active,
34                                 global_stats,
35                                 checkpoint_config,
36                                 /*kvshard*/ nullptr,
37                                 /*lastSeqno*/ 1000,
38                                 /*lastSnapStart*/ 0,
39                                 /*lastSnapEnd*/ 0,
40                                 /*table*/ nullptr,
41                                 std::make_shared<DummyCB>(),
42                                 /*newSeqnoCb*/ nullptr,
43                                 config,
44                                 eviction_policy));
45 }
46
47 void VBucketTest::TearDown() {
48     vbucket.reset();
49 }
50
51 std::vector<StoredDocKey> VBucketTest::generateKeys(int num, int start) {
52     std::vector<StoredDocKey> rv;
53
54     for (int i = start; i < num; i++) {
55         rv.push_back(makeStoredDocKey(std::to_string(i)));
56     }
57
58     return rv;
59 }
60
61 void VBucketTest::addOne(const StoredDocKey& k, AddStatus expect, int expiry) {
62     Item i(k, 0, expiry, k.data(), k.size());
63     EXPECT_EQ(expect, public_processAdd(i)) << "Failed to add key "
64                                                      << k.c_str();
65 }
66
67 void VBucketTest::addMany(std::vector<StoredDocKey>& keys, AddStatus expect) {
68     for (const auto& k : keys) {
69         addOne(k, expect);
70     }
71 }
72
73 void VBucketTest::setOne(const StoredDocKey& k,
74                          MutationStatus expect,
75                          int expiry) {
76     Item i(k, 0, expiry, k.data(), k.size());
77     EXPECT_EQ(expect, public_processSet(i, i.getCas())) << "Failed to set key "
78                                                         << k.c_str();
79 }
80
81 void VBucketTest::setMany(std::vector<StoredDocKey>& keys,
82                           MutationStatus expect) {
83     for (const auto& k : keys) {
84         setOne(k, expect);
85     }
86 }
87
88 void VBucketTest::softDeleteOne(const StoredDocKey& k, MutationStatus expect) {
89     StoredValue* v(vbucket->ht.find(k, TrackReference::No, WantsDeleted::No));
90     EXPECT_NE(nullptr, v);
91
92     EXPECT_EQ(expect, public_processSoftDelete(v->getKey(), v, 0))
93             << "Failed to soft delete key " << k.c_str();
94 }
95
96 void VBucketTest::softDeleteMany(std::vector<StoredDocKey>& keys,
97                                  MutationStatus expect) {
98     for (const auto& k : keys) {
99         softDeleteOne(k, expect);
100     }
101 }
102
103 StoredValue* VBucketTest::findValue(StoredDocKey& key) {
104     return vbucket->ht.find(key, TrackReference::Yes, WantsDeleted::Yes);
105 }
106
107 void VBucketTest::verifyValue(StoredDocKey& key,
108                               const char* value,
109                               TrackReference trackReference,
110                               WantsDeleted wantDeleted) {
111     StoredValue* v = vbucket->ht.find(key, trackReference, wantDeleted);
112     EXPECT_NE(nullptr, v);
113     value_t val = v->getValue();
114     if (!value) {
115         EXPECT_EQ(nullptr, val.get());
116     } else {
117         EXPECT_STREQ(value, val->to_s().c_str());
118     }
119 }
120
121 std::pair<HashTable::HashBucketLock, StoredValue*> VBucketTest::lockAndFind(
122         const StoredDocKey& key) {
123     auto hbl = vbucket->ht.getLockedBucket(key);
124     auto* storedVal = vbucket->ht.unlocked_find(
125             key, hbl.getBucketNum(), WantsDeleted::Yes, TrackReference::No);
126     return std::make_pair(std::move(hbl), storedVal);
127 }
128
129 MutationStatus VBucketTest::public_processSet(Item& itm, const uint64_t cas) {
130     auto hbl_sv = lockAndFind(itm.getKey());
131     VBQueueItemCtx queueItmCtx(GenerateBySeqno::Yes,
132                                GenerateCas::No,
133                                TrackCasDrift::No,
134                                /*isBackfillItem*/ false,
135                                /*preLinkDocumentContext_*/ nullptr);
136     return vbucket
137             ->processSet(hbl_sv.first,
138                          hbl_sv.second,
139                          itm,
140                          cas,
141                          true,
142                          false,
143                          &queueItmCtx)
144             .first;
145 }
146
147 AddStatus VBucketTest::public_processAdd(Item& itm) {
148     auto hbl_sv = lockAndFind(itm.getKey());
149     return vbucket
150             ->processAdd(hbl_sv.first,
151                          hbl_sv.second,
152                          itm,
153                          /*maybeKeyExists*/ true,
154                          /*isReplication*/ false)
155             .first;
156 }
157
158 MutationStatus VBucketTest::public_processSoftDelete(const DocKey& key,
159                                                      StoredValue* v,
160                                                      uint64_t cas) {
161     auto hbl = vbucket->ht.getLockedBucket(key);
162     if (!v) {
163         v = vbucket->ht.unlocked_find(
164                 key, hbl.getBucketNum(), WantsDeleted::No, TrackReference::No);
165         if (!v) {
166             return MutationStatus::NotFound;
167         }
168     }
169     ItemMetaData metadata;
170     metadata.revSeqno = v->getRevSeqno() + 1;
171     MutationStatus status;
172     std::tie(status, std::ignore, std::ignore) = vbucket->processSoftDelete(
173             hbl,
174             *v,
175             cas,
176             metadata,
177             VBQueueItemCtx(GenerateBySeqno::Yes,
178                            GenerateCas::Yes,
179                            TrackCasDrift::No,
180                            /*isBackfillItem*/ false,
181                            /*preLinkDocCtx*/ nullptr),
182             /*use_meta*/ false,
183             /*bySeqno*/ v->getBySeqno());
184     return status;
185 }
186
187 bool VBucketTest::public_deleteStoredValue(const DocKey& key) {
188     auto hbl_sv = lockAndFind(key);
189     if (!hbl_sv.second) {
190         return false;
191     }
192     return vbucket->deleteStoredValue(hbl_sv.first, *hbl_sv.second);
193 }
194
195 size_t EPVBucketTest::public_queueBGFetchItem(
196         const DocKey& key,
197         std::unique_ptr<VBucketBGFetchItem> fetchItem,
198         BgFetcher* bgFetcher) {
199     return dynamic_cast<EPVBucket&>(*vbucket).queueBGFetchItem(
200             key, std::move(fetchItem), bgFetcher);
201 }
202
203 class BlobTest : public Blob {
204 public:
205     BlobTest() : Blob(0,0) {}
206     static size_t getAllocationSize(size_t len){
207         return Blob::getAllocationSize(len);
208     }
209 };
210
211 TEST(BlobTest, basicAllocationSize){
212     EXPECT_EQ(BlobTest::getAllocationSize(10), 20);
213
214     // Expected to be 10 because the 2 bytes of the data member array will not
215     // be allocated because they will not be used.
216     EXPECT_EQ(BlobTest::getAllocationSize(0), 10);
217 }
218
219 // Measure performance of VBucket::getBGFetchItems - queue and then get
220 // 10,000 items from the vbucket.
221 TEST_P(EPVBucketTest, GetBGFetchItemsPerformance) {
222     BgFetcher fetcher(/*store*/ nullptr, /*shard*/ nullptr, this->global_stats);
223
224     for (unsigned int ii = 0; ii < 100000; ii++) {
225         auto fetchItem = std::make_unique<VBucketBGFetchItem>(
226                 /*cookie*/ nullptr,
227                 /*isMeta*/ false);
228         this->public_queueBGFetchItem(
229                 makeStoredDocKey(std::to_string(ii)),
230                 std::move(fetchItem),
231                 &fetcher);
232     }
233     auto items = this->vbucket->getBGFetchItems();
234 }
235
236 // Check the existence of bloom filter after performing a
237 // swap of existing filter with a temporary filter.
238 TEST_P(VBucketTest, SwapFilter) {
239     this->vbucket->createFilter(1, 1.0);
240     ASSERT_FALSE(this->vbucket->isTempFilterAvailable());
241     ASSERT_NE("DOESN'T EXIST", this->vbucket->getFilterStatusString());
242     this->vbucket->swapFilter();
243     EXPECT_NE("DOESN'T EXIST", this->vbucket->getFilterStatusString());
244 }
245
246 TEST_P(VBucketTest, Add) {
247     const auto eviction_policy = GetParam();
248     if (eviction_policy != VALUE_ONLY) {
249         return;
250     }
251     const int nkeys = 1000;
252
253     auto keys = generateKeys(nkeys);
254     addMany(keys, AddStatus::Success);
255
256     StoredDocKey missingKey = makeStoredDocKey("aMissingKey");
257     EXPECT_FALSE(this->vbucket->ht.find(
258             missingKey, TrackReference::Yes, WantsDeleted::No));
259
260     for (const auto& key : keys) {
261         EXPECT_TRUE(this->vbucket->ht.find(
262                 key, TrackReference::Yes, WantsDeleted::No));
263     }
264
265     addMany(keys, AddStatus::Exists);
266     for (const auto& key : keys) {
267         EXPECT_TRUE(this->vbucket->ht.find(
268                 key, TrackReference::Yes, WantsDeleted::No));
269     }
270
271     // Verify we can read after a soft deletion.
272     EXPECT_EQ(MutationStatus::WasDirty,
273               this->public_processSoftDelete(keys[0], nullptr, 0));
274     EXPECT_EQ(MutationStatus::NotFound,
275               this->public_processSoftDelete(keys[0], nullptr, 0));
276     EXPECT_FALSE(this->vbucket->ht.find(
277             keys[0], TrackReference::Yes, WantsDeleted::No));
278
279     Item i(keys[0], 0, 0, "newtest", 7);
280     EXPECT_EQ(AddStatus::UnDel, this->public_processAdd(i));
281     EXPECT_EQ(nkeys, this->vbucket->ht.getNumItems());
282 }
283
284 TEST_P(VBucketTest, AddExpiry) {
285     const auto eviction_policy = GetParam();
286     if (eviction_policy != VALUE_ONLY) {
287         return;
288     }
289     StoredDocKey k = makeStoredDocKey("aKey");
290
291     addOne(k, AddStatus::Success, ep_real_time() + 5);
292     addOne(k, AddStatus::Exists, ep_real_time() + 5);
293
294     StoredValue* v =
295             this->vbucket->ht.find(k, TrackReference::Yes, WantsDeleted::No);
296     EXPECT_TRUE(v);
297     EXPECT_FALSE(v->isExpired(ep_real_time()));
298     EXPECT_TRUE(v->isExpired(ep_real_time() + 6));
299
300     TimeTraveller biffTannen(6);
301     EXPECT_TRUE(v->isExpired(ep_real_time()));
302
303     addOne(k, AddStatus::UnDel, ep_real_time() + 5);
304     EXPECT_TRUE(v);
305     EXPECT_FALSE(v->isExpired(ep_real_time()));
306     EXPECT_TRUE(v->isExpired(ep_real_time() + 6));
307 }
308
309 /**
310  * Test to check if an unlocked_softDelete performed on an
311  * existing item with a new value results in a success
312  */
313 TEST_P(VBucketTest, unlockedSoftDeleteWithValue) {
314     const auto eviction_policy = GetParam();
315     if (eviction_policy != VALUE_ONLY) {
316         return;
317     }
318
319     // Setup - create a key and then delete it with a value.
320     StoredDocKey key = makeStoredDocKey("key");
321     Item stored_item(key, 0, 0, "value", strlen("value"));
322     ASSERT_EQ(MutationStatus::WasClean,
323               this->public_processSet(stored_item, stored_item.getCas()));
324
325     StoredValue* v(
326             this->vbucket->ht.find(key, TrackReference::No, WantsDeleted::No));
327     EXPECT_NE(nullptr, v);
328
329     // Create an item and set its state to deleted
330     Item deleted_item(key, 0, 0, "deletedvalue", strlen("deletedvalue"));
331     deleted_item.setDeleted();
332
333     auto prev_revseqno = v->getRevSeqno();
334     deleted_item.setRevSeqno(prev_revseqno);
335
336     // Set a new deleted value
337     v->setValue(deleted_item, this->vbucket->ht);
338
339     EXPECT_EQ(MutationStatus::WasDirty,
340               this->public_processSoftDelete(v->getKey(), v, 0));
341     verifyValue(key, "deletedvalue", TrackReference::Yes, WantsDeleted::Yes);
342     EXPECT_EQ(prev_revseqno + 1, v->getRevSeqno());
343 }
344
345 /**
346  * Test to check if an unlocked_softDelete performed on a
347  * deleted item without a value and with a value
348  */
349 TEST_P(VBucketTest, updateDeletedItem) {
350     const auto eviction_policy = GetParam();
351     if (eviction_policy != VALUE_ONLY) {
352         return;
353     }
354
355     // Setup - create a key and then delete it.
356     StoredDocKey key = makeStoredDocKey("key");
357     Item stored_item(key, 0, 0, "value", strlen("value"));
358     ASSERT_EQ(MutationStatus::WasClean,
359               this->public_processSet(stored_item, stored_item.getCas()));
360
361     StoredValue* v(
362             this->vbucket->ht.find(key, TrackReference::No, WantsDeleted::No));
363     EXPECT_NE(nullptr, v);
364
365     ItemMetaData itm_meta;
366     EXPECT_EQ(MutationStatus::WasDirty,
367               this->public_processSoftDelete(v->getKey(), v, 0));
368     verifyValue(key, nullptr, TrackReference::Yes, WantsDeleted::Yes);
369
370     Item deleted_item(key, 0, 0, "deletedvalue", strlen("deletedvalue"));
371     deleted_item.setDeleted();
372
373     auto prev_revseqno = v->getRevSeqno();
374     deleted_item.setRevSeqno(prev_revseqno);
375
376     // Set a new deleted value
377     v->setValue(deleted_item, this->vbucket->ht);
378
379     EXPECT_EQ(MutationStatus::WasDirty,
380               this->public_processSoftDelete(v->getKey(), v, 0));
381     verifyValue(
382                 key,
383                 "deletedvalue",
384                 TrackReference::Yes,
385                 WantsDeleted::Yes);
386
387     EXPECT_EQ(prev_revseqno + 1, v->getRevSeqno());
388
389     Item update_deleted_item(
390             key, 0, 0, "updatedeletedvalue", strlen("updatedeletedvalue"));
391     update_deleted_item.setDeleted();
392
393     prev_revseqno = v->getRevSeqno();
394     update_deleted_item.setRevSeqno(prev_revseqno);
395
396     // Set a new deleted value
397     v->setValue(update_deleted_item, this->vbucket->ht);
398
399     EXPECT_EQ(MutationStatus::WasDirty,
400               this->public_processSoftDelete(v->getKey(), v, 0));
401     verifyValue(
402             key, "updatedeletedvalue", TrackReference::Yes, WantsDeleted::Yes);
403     EXPECT_EQ(prev_revseqno + 1, v->getRevSeqno());
404 }
405
406 TEST_P(VBucketTest, SizeStatsSoftDel) {
407     this->global_stats.reset();
408     ASSERT_EQ(0, this->vbucket->ht.memSize.load());
409     ASSERT_EQ(0, this->vbucket->ht.cacheSize.load());
410     size_t initialSize = this->global_stats.currentSize.load();
411
412     const StoredDocKey k = makeStoredDocKey("somekey");
413     const size_t itemSize(16 * 1024);
414     char* someval(static_cast<char*>(cb_calloc(1, itemSize)));
415     EXPECT_TRUE(someval);
416
417     Item i(k, 0, 0, someval, itemSize);
418
419     EXPECT_EQ(MutationStatus::WasClean,
420               this->public_processSet(i, i.getCas()));
421
422     EXPECT_EQ(MutationStatus::WasDirty,
423               this->public_processSoftDelete(k, nullptr, 0));
424     this->public_deleteStoredValue(k);
425
426     EXPECT_EQ(0, this->vbucket->ht.memSize.load());
427     EXPECT_EQ(0, this->vbucket->ht.cacheSize.load());
428     EXPECT_EQ(initialSize, this->global_stats.currentSize.load());
429
430     cb_free(someval);
431 }
432
433 TEST_P(VBucketTest, SizeStatsSoftDelFlush) {
434     this->global_stats.reset();
435     ASSERT_EQ(0, this->vbucket->ht.memSize.load());
436     ASSERT_EQ(0, this->vbucket->ht.cacheSize.load());
437     size_t initialSize = this->global_stats.currentSize.load();
438
439     StoredDocKey k = makeStoredDocKey("somekey");
440     const size_t itemSize(16 * 1024);
441     char* someval(static_cast<char*>(cb_calloc(1, itemSize)));
442     EXPECT_TRUE(someval);
443
444     Item i(k, 0, 0, someval, itemSize);
445
446     EXPECT_EQ(MutationStatus::WasClean,
447               this->public_processSet(i, i.getCas()));
448
449     EXPECT_EQ(MutationStatus::WasDirty,
450               this->public_processSoftDelete(k, nullptr, 0));
451     this->vbucket->ht.clear();
452
453     EXPECT_EQ(0, this->vbucket->ht.memSize.load());
454     EXPECT_EQ(0, this->vbucket->ht.cacheSize.load());
455     EXPECT_EQ(initialSize, this->global_stats.currentSize.load());
456
457     cb_free(someval);
458 }
459
460 class VBucketEvictionTest : public VBucketTest {};
461
462 // Check that counts of items and resident items are as expected when items are
463 // ejected from the HashTable.
464 TEST_P(VBucketEvictionTest, EjectionResidentCount) {
465     const auto eviction_policy = GetParam();
466     ASSERT_EQ(0, this->vbucket->getNumItems());
467     ASSERT_EQ(0, this->vbucket->getNumNonResidentItems());
468
469     Item item(makeStoredDocKey("key"), /*flags*/0, /*exp*/0,
470               /*data*/nullptr, /*ndata*/0);
471
472     EXPECT_EQ(MutationStatus::WasClean,
473               this->public_processSet(item, item.getCas()));
474
475     EXPECT_EQ(1, this->vbucket->getNumItems());
476     EXPECT_EQ(0, this->vbucket->getNumNonResidentItems());
477
478     // TODO-MT: Should acquire lock really (ok given this is currently
479     // single-threaded).
480     auto* stored_item = this->vbucket->ht.find(
481             makeStoredDocKey("key"), TrackReference::Yes, WantsDeleted::No);
482     EXPECT_NE(nullptr, stored_item);
483     // Need to clear the dirty flag to allow it to be ejected.
484     stored_item->markClean();
485     EXPECT_TRUE(this->vbucket->ht.unlocked_ejectItem(stored_item,
486                                                      eviction_policy));
487
488     // After ejection, should still have 1 item in VBucket, but also have
489     // 1 non-resident item.
490     EXPECT_EQ(1, this->vbucket->getNumItems());
491     EXPECT_EQ(1, this->vbucket->getNumNonResidentItems());
492 }
493
494 // Regression test for MB-21448 - if an attempt is made to perform a CAS
495 // operation on a logically deleted item we should return NOT_FOUND
496 // (aka KEY_ENOENT) and *not* INVALID_CAS (aka KEY_EEXISTS).
497 TEST_P(VBucketEvictionTest, MB21448_UnlockedSetWithCASDeleted) {
498     // Setup - create a key and then delete it.
499     StoredDocKey key = makeStoredDocKey("key");
500     Item item(key, 0, 0, "deleted", strlen("deleted"));
501     ASSERT_EQ(MutationStatus::WasClean,
502               this->public_processSet(item, item.getCas()));
503     ASSERT_EQ(MutationStatus::WasDirty,
504               this->public_processSoftDelete(key, nullptr, 0));
505
506     // Attempt to perform a set on a deleted key with a CAS.
507     Item replacement(key, 0, 0, "value", strlen("value"));
508     EXPECT_EQ(MutationStatus::NotFound,
509               this->public_processSet(replacement,
510                                                /*cas*/ 10))
511             << "When trying to replace-with-CAS a deleted item";
512 }
513
514 // Test cases which run in both Full and Value eviction
515 INSTANTIATE_TEST_CASE_P(
516         FullAndValueEviction,
517         VBucketTest,
518         ::testing::Values(VALUE_ONLY, FULL_EVICTION),
519         [](const ::testing::TestParamInfo<item_eviction_policy_t>& info) {
520             if (info.param == VALUE_ONLY) {
521                 return "VALUE_ONLY";
522             } else {
523                 return "FULL_EVICTION";
524             }
525         });
526
527 INSTANTIATE_TEST_CASE_P(
528         FullAndValueEviction,
529         VBucketEvictionTest,
530         ::testing::Values(VALUE_ONLY, FULL_EVICTION),
531         [](const ::testing::TestParamInfo<item_eviction_policy_t>& info) {
532             if (info.param == VALUE_ONLY) {
533                 return "VALUE_ONLY";
534             } else {
535                 return "FULL_EVICTION";
536             }
537         });
538
539 INSTANTIATE_TEST_CASE_P(
540         FullAndValueEviction,
541         EPVBucketTest,
542         ::testing::Values(VALUE_ONLY, FULL_EVICTION),
543         [](const ::testing::TestParamInfo<item_eviction_policy_t>& info) {
544             if (info.param == VALUE_ONLY) {
545                 return "VALUE_ONLY";
546             } else {
547                 return "FULL_EVICTION";
548             }
549         });