MB-23906: Implement delete-with-value with store() instead of delete()
[ep-engine.git] / tests / module_tests / vbucket_test.cc
1 /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 /*
3  *     Copyright 2016 Couchbase, Inc
4  *
5  *   Licensed under the Apache License, Version 2.0 (the "License");
6  *   you may not use this file except in compliance with the License.
7  *   You may obtain a copy of the License at
8  *
9  *       http://www.apache.org/licenses/LICENSE-2.0
10  *
11  *   Unless required by applicable law or agreed to in writing, software
12  *   distributed under the License is distributed on an "AS IS" BASIS,
13  *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  *   See the License for the specific language governing permissions and
15  *   limitations under the License.
16  */
17
18 #include "config.h"
19
20 #include "vbucket_test.h"
21 #include "bgfetcher.h"
22 #include "ep_vb.h"
23 #include "failover-table.h"
24 #include "item.h"
25 #include "programs/engine_testapp/mock_server.h"
26 #include "tests/module_tests/test_helpers.h"
27
28 #include <platform/cb_malloc.h>
29
30 void VBucketTest::SetUp() {
31     const auto eviction_policy = GetParam();
32     vbucket.reset(new EPVBucket(0,
33                                 vbucket_state_active,
34                                 global_stats,
35                                 checkpoint_config,
36                                 /*kvshard*/ nullptr,
37                                 /*lastSeqno*/ 1000,
38                                 /*lastSnapStart*/ 0,
39                                 /*lastSnapEnd*/ 0,
40                                 /*table*/ nullptr,
41                                 std::make_shared<DummyCB>(),
42                                 /*newSeqnoCb*/ nullptr,
43                                 config,
44                                 eviction_policy));
45 }
46
47 void VBucketTest::TearDown() {
48     vbucket.reset();
49 }
50
51 std::vector<StoredDocKey> VBucketTest::generateKeys(int num, int start) {
52     std::vector<StoredDocKey> rv;
53
54     for (int i = start; i < num; i++) {
55         rv.push_back(makeStoredDocKey(std::to_string(i)));
56     }
57
58     return rv;
59 }
60
61 void VBucketTest::addOne(const StoredDocKey& k, AddStatus expect, int expiry) {
62     Item i(k, 0, expiry, k.data(), k.size());
63     EXPECT_EQ(expect, public_processAdd(i)) << "Failed to add key "
64                                                      << k.c_str();
65 }
66
67 void VBucketTest::addMany(std::vector<StoredDocKey>& keys, AddStatus expect) {
68     for (const auto& k : keys) {
69         addOne(k, expect);
70     }
71 }
72
73 void VBucketTest::setOne(const StoredDocKey& k,
74                          MutationStatus expect,
75                          int expiry) {
76     Item i(k, 0, expiry, k.data(), k.size());
77     EXPECT_EQ(expect, public_processSet(i, i.getCas())) << "Failed to set key "
78                                                         << k.c_str();
79 }
80
81 void VBucketTest::setMany(std::vector<StoredDocKey>& keys,
82                           MutationStatus expect) {
83     for (const auto& k : keys) {
84         setOne(k, expect);
85     }
86 }
87
88 void VBucketTest::softDeleteOne(const StoredDocKey& k, MutationStatus expect) {
89     StoredValue* v(vbucket->ht.find(k, TrackReference::No, WantsDeleted::No));
90     EXPECT_NE(nullptr, v);
91
92     EXPECT_EQ(expect, public_processSoftDelete(v->getKey(), v, 0))
93             << "Failed to soft delete key " << k.c_str();
94 }
95
96 void VBucketTest::softDeleteMany(std::vector<StoredDocKey>& keys,
97                                  MutationStatus expect) {
98     for (const auto& k : keys) {
99         softDeleteOne(k, expect);
100     }
101 }
102
103 StoredValue* VBucketTest::findValue(StoredDocKey& key) {
104     return vbucket->ht.find(key, TrackReference::Yes, WantsDeleted::Yes);
105 }
106
107 void VBucketTest::verifyValue(StoredDocKey& key,
108                               const char* value,
109                               TrackReference trackReference,
110                               WantsDeleted wantDeleted) {
111     StoredValue* v = vbucket->ht.find(key, trackReference, wantDeleted);
112     EXPECT_NE(nullptr, v);
113     value_t val = v->getValue();
114     if (!value) {
115         EXPECT_EQ(nullptr, val.get());
116     } else {
117         EXPECT_STREQ(value, val->to_s().c_str());
118     }
119 }
120
121 std::pair<HashTable::HashBucketLock, StoredValue*> VBucketTest::lockAndFind(
122         const StoredDocKey& key) {
123     auto hbl = vbucket->ht.getLockedBucket(key);
124     auto* storedVal = vbucket->ht.unlocked_find(
125             key, hbl.getBucketNum(), WantsDeleted::Yes, TrackReference::No);
126     return std::make_pair(std::move(hbl), storedVal);
127 }
128
129 MutationStatus VBucketTest::public_processSet(Item& itm, const uint64_t cas) {
130     auto hbl_sv = lockAndFind(itm.getKey());
131     VBQueueItemCtx queueItmCtx(GenerateBySeqno::Yes,
132                                GenerateCas::No,
133                                TrackCasDrift::No,
134                                /*isBackfillItem*/ false,
135                                /*preLinkDocumentContext_*/ nullptr);
136     return vbucket
137             ->processSet(hbl_sv.first,
138                          hbl_sv.second,
139                          itm,
140                          cas,
141                          true,
142                          false,
143                          &queueItmCtx)
144             .first;
145 }
146
147 AddStatus VBucketTest::public_processAdd(Item& itm) {
148     auto hbl_sv = lockAndFind(itm.getKey());
149     return vbucket
150             ->processAdd(hbl_sv.first,
151                          hbl_sv.second,
152                          itm,
153                          /*maybeKeyExists*/ true,
154                          /*isReplication*/ false)
155             .first;
156 }
157
158 MutationStatus VBucketTest::public_processSoftDelete(const DocKey& key,
159                                                      StoredValue* v,
160                                                      uint64_t cas) {
161     auto hbl = vbucket->ht.getLockedBucket(key);
162     if (!v) {
163         v = vbucket->ht.unlocked_find(
164                 key, hbl.getBucketNum(), WantsDeleted::No, TrackReference::No);
165         if (!v) {
166             return MutationStatus::NotFound;
167         }
168     }
169     ItemMetaData metadata;
170     metadata.revSeqno = v->getRevSeqno() + 1;
171     MutationStatus status;
172     std::tie(status, std::ignore, std::ignore) = vbucket->processSoftDelete(
173             hbl,
174             *v,
175             cas,
176             metadata,
177             VBQueueItemCtx(GenerateBySeqno::Yes,
178                            GenerateCas::Yes,
179                            TrackCasDrift::No,
180                            /*isBackfillItem*/ false,
181                            /*preLinkDocCtx*/ nullptr),
182             /*use_meta*/ false,
183             /*bySeqno*/ v->getBySeqno());
184     return status;
185 }
186
187 bool VBucketTest::public_deleteStoredValue(const DocKey& key) {
188     auto hbl_sv = lockAndFind(key);
189     if (!hbl_sv.second) {
190         return false;
191     }
192     return vbucket->deleteStoredValue(hbl_sv.first, *hbl_sv.second);
193 }
194
195 size_t EPVBucketTest::public_queueBGFetchItem(
196         const DocKey& key,
197         std::unique_ptr<VBucketBGFetchItem> fetchItem,
198         BgFetcher* bgFetcher) {
199     return dynamic_cast<EPVBucket&>(*vbucket).queueBGFetchItem(
200             key, std::move(fetchItem), bgFetcher);
201 }
202
203 class BlobTest : public Blob {
204 public:
205     BlobTest() : Blob(0,0) {}
206     static size_t getAllocationSize(size_t len){
207         return Blob::getAllocationSize(len);
208     }
209 };
210
211 TEST(BlobTest, basicAllocationSize){
212     EXPECT_EQ(BlobTest::getAllocationSize(10), 20);
213
214     // Expected to be 10 because the 2 bytes of the data member array will not
215     // be allocated because they will not be used.
216     EXPECT_EQ(BlobTest::getAllocationSize(0), 10);
217 }
218
219 // Measure performance of VBucket::getBGFetchItems - queue and then get
220 // 10,000 items from the vbucket.
221 TEST_P(EPVBucketTest, GetBGFetchItemsPerformance) {
222     BgFetcher fetcher(/*store*/ nullptr, /*shard*/ nullptr, this->global_stats);
223
224     for (unsigned int ii = 0; ii < 100000; ii++) {
225         auto fetchItem = std::make_unique<VBucketBGFetchItem>(
226                 /*cookie*/ nullptr,
227                 /*isMeta*/ false);
228         this->public_queueBGFetchItem(
229                 makeStoredDocKey(std::to_string(ii)),
230                 std::move(fetchItem),
231                 &fetcher);
232     }
233     auto items = this->vbucket->getBGFetchItems();
234 }
235
236 // Check the existence of bloom filter after performing a
237 // swap of existing filter with a temporary filter.
238 TEST_P(VBucketTest, SwapFilter) {
239     this->vbucket->createFilter(1, 1.0);
240     ASSERT_FALSE(this->vbucket->isTempFilterAvailable());
241     ASSERT_NE("DOESN'T EXIST", this->vbucket->getFilterStatusString());
242     this->vbucket->swapFilter();
243     EXPECT_NE("DOESN'T EXIST", this->vbucket->getFilterStatusString());
244 }
245
246 TEST_P(VBucketTest, Add) {
247     const auto eviction_policy = GetParam();
248     if (eviction_policy != VALUE_ONLY) {
249         return;
250     }
251     const int nkeys = 1000;
252
253     auto keys = generateKeys(nkeys);
254     addMany(keys, AddStatus::Success);
255
256     StoredDocKey missingKey = makeStoredDocKey("aMissingKey");
257     EXPECT_FALSE(this->vbucket->ht.find(
258             missingKey, TrackReference::Yes, WantsDeleted::No));
259
260     for (const auto& key : keys) {
261         EXPECT_TRUE(this->vbucket->ht.find(
262                 key, TrackReference::Yes, WantsDeleted::No));
263     }
264
265     addMany(keys, AddStatus::Exists);
266     for (const auto& key : keys) {
267         EXPECT_TRUE(this->vbucket->ht.find(
268                 key, TrackReference::Yes, WantsDeleted::No));
269     }
270
271     // Verify we can read after a soft deletion.
272     EXPECT_EQ(MutationStatus::WasDirty,
273               this->public_processSoftDelete(keys[0], nullptr, 0));
274     EXPECT_EQ(MutationStatus::NotFound,
275               this->public_processSoftDelete(keys[0], nullptr, 0));
276     EXPECT_FALSE(this->vbucket->ht.find(
277             keys[0], TrackReference::Yes, WantsDeleted::No));
278
279     Item i(keys[0], 0, 0, "newtest", 7);
280     EXPECT_EQ(AddStatus::UnDel, this->public_processAdd(i));
281     EXPECT_EQ(nkeys, this->vbucket->ht.getNumItems());
282 }
283
284 TEST_P(VBucketTest, AddExpiry) {
285     const auto eviction_policy = GetParam();
286     if (eviction_policy != VALUE_ONLY) {
287         return;
288     }
289     StoredDocKey k = makeStoredDocKey("aKey");
290
291     addOne(k, AddStatus::Success, ep_real_time() + 5);
292     addOne(k, AddStatus::Exists, ep_real_time() + 5);
293
294     StoredValue* v =
295             this->vbucket->ht.find(k, TrackReference::Yes, WantsDeleted::No);
296     EXPECT_TRUE(v);
297     EXPECT_FALSE(v->isExpired(ep_real_time()));
298     EXPECT_TRUE(v->isExpired(ep_real_time() + 6));
299
300     TimeTraveller biffTannen(6);
301     EXPECT_TRUE(v->isExpired(ep_real_time()));
302
303     addOne(k, AddStatus::UnDel, ep_real_time() + 5);
304     EXPECT_TRUE(v);
305     EXPECT_FALSE(v->isExpired(ep_real_time()));
306     EXPECT_TRUE(v->isExpired(ep_real_time() + 6));
307 }
308
309 /**
310  * Test to check if an unlocked_softDelete performed on an
311  * existing item with a new value results in a success
312  */
313 TEST_P(VBucketTest, unlockedSoftDeleteWithValue) {
314     const auto eviction_policy = GetParam();
315     if (eviction_policy != VALUE_ONLY) {
316         return;
317     }
318
319     // Setup - create a key and then delete it with a value.
320     StoredDocKey key = makeStoredDocKey("key");
321     Item stored_item(key, 0, 0, "value", strlen("value"));
322     ASSERT_EQ(MutationStatus::WasClean,
323               this->public_processSet(stored_item, stored_item.getCas()));
324
325     StoredValue* v(
326             this->vbucket->ht.find(key, TrackReference::No, WantsDeleted::No));
327     EXPECT_NE(nullptr, v);
328
329     // Create an item and set its state to deleted
330     Item deleted_item(key, 0, 0, "deletedvalue", strlen("deletedvalue"));
331     deleted_item.setDeleted();
332
333     auto prev_revseqno = v->getRevSeqno();
334     deleted_item.setRevSeqno(prev_revseqno);
335
336     EXPECT_EQ(MutationStatus::WasDirty,
337               this->public_processSet(deleted_item, 0));
338     verifyValue(key, "deletedvalue", TrackReference::Yes, WantsDeleted::Yes);
339     EXPECT_EQ(prev_revseqno + 1, v->getRevSeqno());
340 }
341
342 /**
343  * Test to check if an unlocked_softDelete performed on a
344  * deleted item without a value and with a value
345  */
346 TEST_P(VBucketTest, updateDeletedItem) {
347     const auto eviction_policy = GetParam();
348     if (eviction_policy != VALUE_ONLY) {
349         return;
350     }
351
352     // Setup - create a key and then delete it.
353     StoredDocKey key = makeStoredDocKey("key");
354     Item stored_item(key, 0, 0, "value", strlen("value"));
355     ASSERT_EQ(MutationStatus::WasClean,
356               this->public_processSet(stored_item, stored_item.getCas()));
357
358     StoredValue* v(
359             this->vbucket->ht.find(key, TrackReference::No, WantsDeleted::No));
360     EXPECT_NE(nullptr, v);
361
362     ItemMetaData itm_meta;
363     EXPECT_EQ(MutationStatus::WasDirty,
364               this->public_processSoftDelete(v->getKey(), v, 0));
365     verifyValue(key, nullptr, TrackReference::Yes, WantsDeleted::Yes);
366
367     Item deleted_item(key, 0, 0, "deletedvalue", strlen("deletedvalue"));
368     deleted_item.setDeleted();
369
370     auto prev_revseqno = v->getRevSeqno();
371     deleted_item.setRevSeqno(prev_revseqno);
372
373     EXPECT_EQ(MutationStatus::WasDirty,
374               this->public_processSet(deleted_item, 0));
375     verifyValue(
376                 key,
377                 "deletedvalue",
378                 TrackReference::Yes,
379                 WantsDeleted::Yes);
380
381     EXPECT_EQ(prev_revseqno + 1, v->getRevSeqno());
382
383     Item update_deleted_item(
384             key, 0, 0, "updatedeletedvalue", strlen("updatedeletedvalue"));
385     update_deleted_item.setDeleted();
386
387     prev_revseqno = v->getRevSeqno();
388     update_deleted_item.setRevSeqno(prev_revseqno);
389
390     EXPECT_EQ(MutationStatus::WasDirty,
391               this->public_processSet(update_deleted_item, 0));
392     verifyValue(
393             key, "updatedeletedvalue", TrackReference::Yes, WantsDeleted::Yes);
394     EXPECT_EQ(prev_revseqno + 1, v->getRevSeqno());
395 }
396
397 TEST_P(VBucketTest, SizeStatsSoftDel) {
398     this->global_stats.reset();
399     ASSERT_EQ(0, this->vbucket->ht.memSize.load());
400     ASSERT_EQ(0, this->vbucket->ht.cacheSize.load());
401     size_t initialSize = this->global_stats.currentSize.load();
402
403     const StoredDocKey k = makeStoredDocKey("somekey");
404     const size_t itemSize(16 * 1024);
405     char* someval(static_cast<char*>(cb_calloc(1, itemSize)));
406     EXPECT_TRUE(someval);
407
408     Item i(k, 0, 0, someval, itemSize);
409
410     EXPECT_EQ(MutationStatus::WasClean,
411               this->public_processSet(i, i.getCas()));
412
413     EXPECT_EQ(MutationStatus::WasDirty,
414               this->public_processSoftDelete(k, nullptr, 0));
415     this->public_deleteStoredValue(k);
416
417     EXPECT_EQ(0, this->vbucket->ht.memSize.load());
418     EXPECT_EQ(0, this->vbucket->ht.cacheSize.load());
419     EXPECT_EQ(initialSize, this->global_stats.currentSize.load());
420
421     cb_free(someval);
422 }
423
424 TEST_P(VBucketTest, SizeStatsSoftDelFlush) {
425     this->global_stats.reset();
426     ASSERT_EQ(0, this->vbucket->ht.memSize.load());
427     ASSERT_EQ(0, this->vbucket->ht.cacheSize.load());
428     size_t initialSize = this->global_stats.currentSize.load();
429
430     StoredDocKey k = makeStoredDocKey("somekey");
431     const size_t itemSize(16 * 1024);
432     char* someval(static_cast<char*>(cb_calloc(1, itemSize)));
433     EXPECT_TRUE(someval);
434
435     Item i(k, 0, 0, someval, itemSize);
436
437     EXPECT_EQ(MutationStatus::WasClean,
438               this->public_processSet(i, i.getCas()));
439
440     EXPECT_EQ(MutationStatus::WasDirty,
441               this->public_processSoftDelete(k, nullptr, 0));
442     this->vbucket->ht.clear();
443
444     EXPECT_EQ(0, this->vbucket->ht.memSize.load());
445     EXPECT_EQ(0, this->vbucket->ht.cacheSize.load());
446     EXPECT_EQ(initialSize, this->global_stats.currentSize.load());
447
448     cb_free(someval);
449 }
450
451 class VBucketEvictionTest : public VBucketTest {};
452
453 // Check that counts of items and resident items are as expected when items are
454 // ejected from the HashTable.
455 TEST_P(VBucketEvictionTest, EjectionResidentCount) {
456     const auto eviction_policy = GetParam();
457     ASSERT_EQ(0, this->vbucket->getNumItems());
458     ASSERT_EQ(0, this->vbucket->getNumNonResidentItems());
459
460     Item item(makeStoredDocKey("key"), /*flags*/0, /*exp*/0,
461               /*data*/nullptr, /*ndata*/0);
462
463     EXPECT_EQ(MutationStatus::WasClean,
464               this->public_processSet(item, item.getCas()));
465
466     EXPECT_EQ(1, this->vbucket->getNumItems());
467     EXPECT_EQ(0, this->vbucket->getNumNonResidentItems());
468
469     // TODO-MT: Should acquire lock really (ok given this is currently
470     // single-threaded).
471     auto* stored_item = this->vbucket->ht.find(
472             makeStoredDocKey("key"), TrackReference::Yes, WantsDeleted::No);
473     EXPECT_NE(nullptr, stored_item);
474     // Need to clear the dirty flag to allow it to be ejected.
475     stored_item->markClean();
476     EXPECT_TRUE(this->vbucket->ht.unlocked_ejectItem(stored_item,
477                                                      eviction_policy));
478
479     // After ejection, should still have 1 item in VBucket, but also have
480     // 1 non-resident item.
481     EXPECT_EQ(1, this->vbucket->getNumItems());
482     EXPECT_EQ(1, this->vbucket->getNumNonResidentItems());
483 }
484
485 // Regression test for MB-21448 - if an attempt is made to perform a CAS
486 // operation on a logically deleted item we should return NOT_FOUND
487 // (aka KEY_ENOENT) and *not* INVALID_CAS (aka KEY_EEXISTS).
488 TEST_P(VBucketEvictionTest, MB21448_UnlockedSetWithCASDeleted) {
489     // Setup - create a key and then delete it.
490     StoredDocKey key = makeStoredDocKey("key");
491     Item item(key, 0, 0, "deleted", strlen("deleted"));
492     ASSERT_EQ(MutationStatus::WasClean,
493               this->public_processSet(item, item.getCas()));
494     ASSERT_EQ(MutationStatus::WasDirty,
495               this->public_processSoftDelete(key, nullptr, 0));
496
497     // Attempt to perform a set on a deleted key with a CAS.
498     Item replacement(key, 0, 0, "value", strlen("value"));
499     EXPECT_EQ(MutationStatus::NotFound,
500               this->public_processSet(replacement,
501                                                /*cas*/ 10))
502             << "When trying to replace-with-CAS a deleted item";
503 }
504
505 // Test cases which run in both Full and Value eviction
506 INSTANTIATE_TEST_CASE_P(
507         FullAndValueEviction,
508         VBucketTest,
509         ::testing::Values(VALUE_ONLY, FULL_EVICTION),
510         [](const ::testing::TestParamInfo<item_eviction_policy_t>& info) {
511             if (info.param == VALUE_ONLY) {
512                 return "VALUE_ONLY";
513             } else {
514                 return "FULL_EVICTION";
515             }
516         });
517
518 INSTANTIATE_TEST_CASE_P(
519         FullAndValueEviction,
520         VBucketEvictionTest,
521         ::testing::Values(VALUE_ONLY, FULL_EVICTION),
522         [](const ::testing::TestParamInfo<item_eviction_policy_t>& info) {
523             if (info.param == VALUE_ONLY) {
524                 return "VALUE_ONLY";
525             } else {
526                 return "FULL_EVICTION";
527             }
528         });
529
530 INSTANTIATE_TEST_CASE_P(
531         FullAndValueEviction,
532         EPVBucketTest,
533         ::testing::Values(VALUE_ONLY, FULL_EVICTION),
534         [](const ::testing::TestParamInfo<item_eviction_policy_t>& info) {
535             if (info.param == VALUE_ONLY) {
536                 return "VALUE_ONLY";
537             } else {
538                 return "FULL_EVICTION";
539             }
540         });