b54fb03928355bcb6563a536bdc73c084bb01e24
[ep-engine.git] / tests / module_tests / vbucket_test.cc
1 /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 /*
3  *     Copyright 2016 Couchbase, Inc
4  *
5  *   Licensed under the Apache License, Version 2.0 (the "License");
6  *   you may not use this file except in compliance with the License.
7  *   You may obtain a copy of the License at
8  *
9  *       http://www.apache.org/licenses/LICENSE-2.0
10  *
11  *   Unless required by applicable law or agreed to in writing, software
12  *   distributed under the License is distributed on an "AS IS" BASIS,
13  *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  *   See the License for the specific language governing permissions and
15  *   limitations under the License.
16  */
17
18 #include "config.h"
19
20 #include "vbucket_test.h"
21 #include "bgfetcher.h"
22 #include "ep_vb.h"
23 #include "failover-table.h"
24 #include "item.h"
25 #include "programs/engine_testapp/mock_server.h"
26 #include "tests/module_tests/test_helpers.h"
27
28 #include <platform/cb_malloc.h>
29
30 void VBucketTest::SetUp() {
31     const auto eviction_policy = GetParam();
32     vbucket.reset(new EPVBucket(0,
33                                 vbucket_state_active,
34                                 global_stats,
35                                 checkpoint_config,
36                                 /*kvshard*/ nullptr,
37                                 /*lastSeqno*/ 1000,
38                                 /*lastSnapStart*/ 0,
39                                 /*lastSnapEnd*/ 0,
40                                 /*table*/ nullptr,
41                                 std::make_shared<DummyCB>(),
42                                 /*newSeqnoCb*/ nullptr,
43                                 config,
44                                 eviction_policy));
45 }
46
47 void VBucketTest::TearDown() {
48     vbucket.reset();
49 }
50
51 std::vector<StoredDocKey> VBucketTest::generateKeys(int num, int start) {
52     std::vector<StoredDocKey> rv;
53
54     for (int i = start; i < num + start; i++) {
55         rv.push_back(makeStoredDocKey(std::to_string(i)));
56     }
57
58     return rv;
59 }
60
61 AddStatus VBucketTest::addOne(const StoredDocKey& k, int expiry) {
62     Item i(k, 0, expiry, k.data(), k.size());
63     return public_processAdd(i);
64 }
65
66 void VBucketTest::addMany(std::vector<StoredDocKey>& keys, AddStatus expect) {
67     for (const auto& k : keys) {
68         EXPECT_EQ(expect, addOne(k));
69     }
70 }
71
72 MutationStatus VBucketTest::setOne(const StoredDocKey& k, int expiry) {
73     Item i(k, 0, expiry, k.data(), k.size());
74     return public_processSet(i, i.getCas());
75 }
76
77 void VBucketTest::setMany(std::vector<StoredDocKey>& keys,
78                           MutationStatus expect) {
79     for (const auto& k : keys) {
80         EXPECT_EQ(expect, setOne(k));
81     }
82 }
83
84 void VBucketTest::softDeleteOne(const StoredDocKey& k, MutationStatus expect) {
85     StoredValue* v(vbucket->ht.find(k, TrackReference::No, WantsDeleted::No));
86     EXPECT_NE(nullptr, v);
87
88     EXPECT_EQ(expect, public_processSoftDelete(v->getKey(), v, 0))
89             << "Failed to soft delete key " << k.c_str();
90 }
91
92 void VBucketTest::softDeleteMany(std::vector<StoredDocKey>& keys,
93                                  MutationStatus expect) {
94     for (const auto& k : keys) {
95         softDeleteOne(k, expect);
96     }
97 }
98
99 StoredValue* VBucketTest::findValue(StoredDocKey& key) {
100     return vbucket->ht.find(key, TrackReference::Yes, WantsDeleted::Yes);
101 }
102
103 void VBucketTest::verifyValue(StoredDocKey& key,
104                               const char* value,
105                               TrackReference trackReference,
106                               WantsDeleted wantDeleted) {
107     StoredValue* v = vbucket->ht.find(key, trackReference, wantDeleted);
108     EXPECT_NE(nullptr, v);
109     value_t val = v->getValue();
110     if (!value) {
111         EXPECT_EQ(nullptr, val.get());
112     } else {
113         EXPECT_STREQ(value, val->to_s().c_str());
114     }
115 }
116
117 std::pair<HashTable::HashBucketLock, StoredValue*> VBucketTest::lockAndFind(
118         const StoredDocKey& key) {
119     auto hbl = vbucket->ht.getLockedBucket(key);
120     auto* storedVal = vbucket->ht.unlocked_find(
121             key, hbl.getBucketNum(), WantsDeleted::Yes, TrackReference::No);
122     return std::make_pair(std::move(hbl), storedVal);
123 }
124
125 MutationStatus VBucketTest::public_processSet(Item& itm, const uint64_t cas) {
126     auto hbl_sv = lockAndFind(itm.getKey());
127     VBQueueItemCtx queueItmCtx(GenerateBySeqno::Yes,
128                                GenerateCas::No,
129                                TrackCasDrift::No,
130                                /*isBackfillItem*/ false,
131                                /*preLinkDocumentContext_*/ nullptr);
132     return vbucket
133             ->processSet(hbl_sv.first,
134                          hbl_sv.second,
135                          itm,
136                          cas,
137                          true,
138                          false,
139                          &queueItmCtx)
140             .first;
141 }
142
143 AddStatus VBucketTest::public_processAdd(Item& itm) {
144     auto hbl_sv = lockAndFind(itm.getKey());
145     return vbucket
146             ->processAdd(hbl_sv.first,
147                          hbl_sv.second,
148                          itm,
149                          /*maybeKeyExists*/ true,
150                          /*isReplication*/ false)
151             .first;
152 }
153
154 MutationStatus VBucketTest::public_processSoftDelete(const DocKey& key,
155                                                      StoredValue* v,
156                                                      uint64_t cas) {
157     auto hbl = vbucket->ht.getLockedBucket(key);
158     if (!v) {
159         v = vbucket->ht.unlocked_find(
160                 key, hbl.getBucketNum(), WantsDeleted::No, TrackReference::No);
161         if (!v) {
162             return MutationStatus::NotFound;
163         }
164     }
165     ItemMetaData metadata;
166     metadata.revSeqno = v->getRevSeqno() + 1;
167     MutationStatus status;
168     std::tie(status, std::ignore, std::ignore) = vbucket->processSoftDelete(
169             hbl,
170             *v,
171             cas,
172             metadata,
173             VBQueueItemCtx(GenerateBySeqno::Yes,
174                            GenerateCas::Yes,
175                            TrackCasDrift::No,
176                            /*isBackfillItem*/ false,
177                            /*preLinkDocCtx*/ nullptr),
178             /*use_meta*/ false,
179             /*bySeqno*/ v->getBySeqno());
180     return status;
181 }
182
183 bool VBucketTest::public_deleteStoredValue(const DocKey& key) {
184     auto hbl_sv = lockAndFind(key);
185     if (!hbl_sv.second) {
186         return false;
187     }
188     return vbucket->deleteStoredValue(hbl_sv.first, *hbl_sv.second);
189 }
190
191 GetValue VBucketTest::public_getAndUpdateTtl(const DocKey& key,
192                                              time_t exptime) {
193     auto hbl = lockAndFind(key);
194     GetValue gv;
195     MutationStatus status;
196     std::tie(status, gv) = vbucket->processGetAndUpdateTtl(
197             hbl.first, key, hbl.second, exptime);
198     return gv;
199 }
200
201 size_t EPVBucketTest::public_queueBGFetchItem(
202         const DocKey& key,
203         std::unique_ptr<VBucketBGFetchItem> fetchItem,
204         BgFetcher* bgFetcher) {
205     return dynamic_cast<EPVBucket&>(*vbucket).queueBGFetchItem(
206             key, std::move(fetchItem), bgFetcher);
207 }
208
209 class BlobTest : public Blob {
210 public:
211     BlobTest() : Blob(0,0) {}
212     static size_t getAllocationSize(size_t len){
213         return Blob::getAllocationSize(len);
214     }
215 };
216
217 TEST(BlobTest, basicAllocationSize){
218     EXPECT_EQ(BlobTest::getAllocationSize(10), 20);
219
220     // Expected to be 10 because the 2 bytes of the data member array will not
221     // be allocated because they will not be used.
222     EXPECT_EQ(BlobTest::getAllocationSize(0), 10);
223 }
224
225 // Measure performance of VBucket::getBGFetchItems - queue and then get
226 // 10,000 items from the vbucket.
227 TEST_P(EPVBucketTest, GetBGFetchItemsPerformance) {
228     BgFetcher fetcher(/*store*/ nullptr, /*shard*/ nullptr, this->global_stats);
229
230     for (unsigned int ii = 0; ii < 100000; ii++) {
231         auto fetchItem = std::make_unique<VBucketBGFetchItem>(
232                 /*cookie*/ nullptr,
233                 /*isMeta*/ false);
234         this->public_queueBGFetchItem(
235                 makeStoredDocKey(std::to_string(ii)),
236                 std::move(fetchItem),
237                 &fetcher);
238     }
239     auto items = this->vbucket->getBGFetchItems();
240 }
241
242 // Check the existence of bloom filter after performing a
243 // swap of existing filter with a temporary filter.
244 TEST_P(VBucketTest, SwapFilter) {
245     this->vbucket->createFilter(1, 1.0);
246     ASSERT_FALSE(this->vbucket->isTempFilterAvailable());
247     ASSERT_NE("DOESN'T EXIST", this->vbucket->getFilterStatusString());
248     this->vbucket->swapFilter();
249     EXPECT_NE("DOESN'T EXIST", this->vbucket->getFilterStatusString());
250 }
251
252 TEST_P(VBucketTest, Add) {
253     const auto eviction_policy = GetParam();
254     if (eviction_policy != VALUE_ONLY) {
255         return;
256     }
257     const int nkeys = 1000;
258
259     auto keys = generateKeys(nkeys);
260     addMany(keys, AddStatus::Success);
261
262     StoredDocKey missingKey = makeStoredDocKey("aMissingKey");
263     EXPECT_FALSE(this->vbucket->ht.find(
264             missingKey, TrackReference::Yes, WantsDeleted::No));
265
266     for (const auto& key : keys) {
267         EXPECT_TRUE(this->vbucket->ht.find(
268                 key, TrackReference::Yes, WantsDeleted::No));
269     }
270
271     addMany(keys, AddStatus::Exists);
272     for (const auto& key : keys) {
273         EXPECT_TRUE(this->vbucket->ht.find(
274                 key, TrackReference::Yes, WantsDeleted::No));
275     }
276
277     // Verify we can read after a soft deletion.
278     EXPECT_EQ(MutationStatus::WasDirty,
279               this->public_processSoftDelete(keys[0], nullptr, 0));
280     EXPECT_EQ(MutationStatus::NotFound,
281               this->public_processSoftDelete(keys[0], nullptr, 0));
282     EXPECT_FALSE(this->vbucket->ht.find(
283             keys[0], TrackReference::Yes, WantsDeleted::No));
284
285     Item i(keys[0], 0, 0, "newtest", 7);
286     EXPECT_EQ(AddStatus::UnDel, this->public_processAdd(i));
287     EXPECT_EQ(nkeys, this->vbucket->ht.getNumItems());
288 }
289
290 TEST_P(VBucketTest, AddExpiry) {
291     const auto eviction_policy = GetParam();
292     if (eviction_policy != VALUE_ONLY) {
293         return;
294     }
295     StoredDocKey k = makeStoredDocKey("aKey");
296
297     ASSERT_EQ(AddStatus::Success, addOne(k, ep_real_time() + 5));
298     EXPECT_EQ(AddStatus::Exists, addOne(k, ep_real_time() + 5));
299
300     StoredValue* v =
301             this->vbucket->ht.find(k, TrackReference::Yes, WantsDeleted::No);
302     EXPECT_TRUE(v);
303     EXPECT_FALSE(v->isExpired(ep_real_time()));
304     EXPECT_TRUE(v->isExpired(ep_real_time() + 6));
305
306     TimeTraveller biffTannen(6);
307     EXPECT_TRUE(v->isExpired(ep_real_time()));
308
309     EXPECT_EQ(AddStatus::UnDel, addOne(k, ep_real_time() + 5));
310     EXPECT_TRUE(v);
311     EXPECT_FALSE(v->isExpired(ep_real_time()));
312     EXPECT_TRUE(v->isExpired(ep_real_time() + 6));
313 }
314
315 /**
316  * Test to check if an unlocked_softDelete performed on an
317  * existing item with a new value results in a success
318  */
319 TEST_P(VBucketTest, unlockedSoftDeleteWithValue) {
320     const auto eviction_policy = GetParam();
321     if (eviction_policy != VALUE_ONLY) {
322         return;
323     }
324
325     // Setup - create a key and then delete it with a value.
326     StoredDocKey key = makeStoredDocKey("key");
327     Item stored_item(key, 0, 0, "value", strlen("value"));
328     ASSERT_EQ(MutationStatus::WasClean,
329               this->public_processSet(stored_item, stored_item.getCas()));
330
331     StoredValue* v(
332             this->vbucket->ht.find(key, TrackReference::No, WantsDeleted::No));
333     EXPECT_NE(nullptr, v);
334
335     // Create an item and set its state to deleted
336     Item deleted_item(key, 0, 0, "deletedvalue", strlen("deletedvalue"));
337     deleted_item.setDeleted();
338
339     auto prev_revseqno = v->getRevSeqno();
340     deleted_item.setRevSeqno(prev_revseqno);
341
342     EXPECT_EQ(MutationStatus::WasDirty,
343               this->public_processSet(deleted_item, 0));
344     verifyValue(key, "deletedvalue", TrackReference::Yes, WantsDeleted::Yes);
345     EXPECT_EQ(prev_revseqno + 1, v->getRevSeqno());
346 }
347
348 /**
349  * Test to check that if an item has expired, an incoming mutation
350  * on that item, if in deleted state results in an invalid cas and
351  * if not in deleted state, results in not found
352  */
353 TEST_P(VBucketTest, updateExpiredItem) {
354     // Setup - create a key
355     StoredDocKey key = makeStoredDocKey("key");
356     Item stored_item(key, 0, ep_real_time() - 1, "value", strlen("value"));
357     ASSERT_EQ(MutationStatus::WasClean,
358               this->public_processSet(stored_item, stored_item.getCas()));
359
360     StoredValue* v = this->vbucket->ht.find(key, TrackReference::No, WantsDeleted::No);
361     EXPECT_TRUE(v);
362     EXPECT_TRUE(v->isExpired(ep_real_time()));
363
364     auto cas = v->getCas();
365     // Create an item and set its state to deleted.
366     Item deleted_item(key, 0, 0, "deletedvalue", strlen("deletedvalue"));
367
368     EXPECT_EQ(MutationStatus::NotFound,
369               this->public_processSet(deleted_item, cas + 1));
370
371     deleted_item.setDeleted();
372     EXPECT_EQ(MutationStatus::InvalidCas,
373               this->public_processSet(deleted_item, cas + 1));
374 }
375
376 /**
377  * Test to check if an unlocked_softDelete performed on a
378  * deleted item without a value and with a value
379  */
380 TEST_P(VBucketTest, updateDeletedItem) {
381     const auto eviction_policy = GetParam();
382     if (eviction_policy != VALUE_ONLY) {
383         return;
384     }
385
386     // Setup - create a key and then delete it.
387     StoredDocKey key = makeStoredDocKey("key");
388     Item stored_item(key, 0, 0, "value", strlen("value"));
389     ASSERT_EQ(MutationStatus::WasClean,
390               this->public_processSet(stored_item, stored_item.getCas()));
391
392     StoredValue* v(
393             this->vbucket->ht.find(key, TrackReference::No, WantsDeleted::No));
394     EXPECT_NE(nullptr, v);
395
396     ItemMetaData itm_meta;
397     EXPECT_EQ(MutationStatus::WasDirty,
398               this->public_processSoftDelete(v->getKey(), v, 0));
399     verifyValue(key, nullptr, TrackReference::Yes, WantsDeleted::Yes);
400     EXPECT_EQ(1, this->vbucket->getNumItems());
401     EXPECT_EQ(1, this->vbucket->getNumInMemoryDeletes());
402
403     Item deleted_item(key, 0, 0, "deletedvalue", strlen("deletedvalue"));
404     deleted_item.setDeleted();
405
406     auto prev_revseqno = v->getRevSeqno();
407     deleted_item.setRevSeqno(prev_revseqno);
408
409     EXPECT_EQ(MutationStatus::WasDirty,
410               this->public_processSet(deleted_item, 0));
411     verifyValue(
412                 key,
413                 "deletedvalue",
414                 TrackReference::Yes,
415                 WantsDeleted::Yes);
416     EXPECT_EQ(1, this->vbucket->getNumItems());
417     EXPECT_EQ(1, this->vbucket->getNumInMemoryDeletes());
418     EXPECT_EQ(prev_revseqno + 1, v->getRevSeqno());
419
420     Item update_deleted_item(
421             key, 0, 0, "updatedeletedvalue", strlen("updatedeletedvalue"));
422     update_deleted_item.setDeleted();
423
424     prev_revseqno = v->getRevSeqno();
425     update_deleted_item.setRevSeqno(prev_revseqno);
426
427     EXPECT_EQ(MutationStatus::WasDirty,
428               this->public_processSet(update_deleted_item, 0));
429     verifyValue(
430             key, "updatedeletedvalue", TrackReference::Yes, WantsDeleted::Yes);
431     EXPECT_EQ(1, this->vbucket->getNumItems());
432     EXPECT_EQ(1, this->vbucket->getNumInMemoryDeletes());
433     EXPECT_EQ(prev_revseqno + 1, v->getRevSeqno());
434 }
435
436 TEST_P(VBucketTest, SizeStatsSoftDel) {
437     this->global_stats.reset();
438     ASSERT_EQ(0, this->vbucket->ht.memSize.load());
439     ASSERT_EQ(0, this->vbucket->ht.cacheSize.load());
440     size_t initialSize = this->global_stats.currentSize.load();
441
442     const StoredDocKey k = makeStoredDocKey("somekey");
443     const size_t itemSize(16 * 1024);
444     char* someval(static_cast<char*>(cb_calloc(1, itemSize)));
445     EXPECT_TRUE(someval);
446
447     Item i(k, 0, 0, someval, itemSize);
448
449     EXPECT_EQ(MutationStatus::WasClean,
450               this->public_processSet(i, i.getCas()));
451
452     EXPECT_EQ(MutationStatus::WasDirty,
453               this->public_processSoftDelete(k, nullptr, 0));
454     this->public_deleteStoredValue(k);
455
456     EXPECT_EQ(0, this->vbucket->ht.memSize.load());
457     EXPECT_EQ(0, this->vbucket->ht.cacheSize.load());
458     EXPECT_EQ(initialSize, this->global_stats.currentSize.load());
459
460     cb_free(someval);
461 }
462
463 TEST_P(VBucketTest, SizeStatsSoftDelFlush) {
464     this->global_stats.reset();
465     ASSERT_EQ(0, this->vbucket->ht.memSize.load());
466     ASSERT_EQ(0, this->vbucket->ht.cacheSize.load());
467     size_t initialSize = this->global_stats.currentSize.load();
468
469     StoredDocKey k = makeStoredDocKey("somekey");
470     const size_t itemSize(16 * 1024);
471     char* someval(static_cast<char*>(cb_calloc(1, itemSize)));
472     EXPECT_TRUE(someval);
473
474     Item i(k, 0, 0, someval, itemSize);
475
476     EXPECT_EQ(MutationStatus::WasClean,
477               this->public_processSet(i, i.getCas()));
478
479     EXPECT_EQ(MutationStatus::WasDirty,
480               this->public_processSoftDelete(k, nullptr, 0));
481     this->vbucket->ht.clear();
482
483     EXPECT_EQ(0, this->vbucket->ht.memSize.load());
484     EXPECT_EQ(0, this->vbucket->ht.cacheSize.load());
485     EXPECT_EQ(initialSize, this->global_stats.currentSize.load());
486
487     cb_free(someval);
488 }
489
490 class VBucketEvictionTest : public VBucketTest {};
491
492 // Check that counts of items and resident items are as expected when items are
493 // ejected from the HashTable.
494 TEST_P(VBucketEvictionTest, EjectionResidentCount) {
495     const auto eviction_policy = GetParam();
496     ASSERT_EQ(0, this->vbucket->getNumItems());
497     ASSERT_EQ(0, this->vbucket->getNumNonResidentItems());
498
499     Item item(makeStoredDocKey("key"), /*flags*/0, /*exp*/0,
500               /*data*/nullptr, /*ndata*/0);
501
502     EXPECT_EQ(MutationStatus::WasClean,
503               this->public_processSet(item, item.getCas()));
504
505     EXPECT_EQ(1, this->vbucket->getNumItems());
506     EXPECT_EQ(0, this->vbucket->getNumNonResidentItems());
507
508     // TODO-MT: Should acquire lock really (ok given this is currently
509     // single-threaded).
510     auto* stored_item = this->vbucket->ht.find(
511             makeStoredDocKey("key"), TrackReference::Yes, WantsDeleted::No);
512     EXPECT_NE(nullptr, stored_item);
513     // Need to clear the dirty flag to allow it to be ejected.
514     stored_item->markClean();
515     EXPECT_TRUE(this->vbucket->ht.unlocked_ejectItem(stored_item,
516                                                      eviction_policy));
517
518     // After ejection, should still have 1 item in VBucket, but also have
519     // 1 non-resident item.
520     EXPECT_EQ(1, this->vbucket->getNumItems());
521     EXPECT_EQ(1, this->vbucket->getNumNonResidentItems());
522 }
523
524 // Regression test for MB-21448 - if an attempt is made to perform a CAS
525 // operation on a logically deleted item we should return NOT_FOUND
526 // (aka KEY_ENOENT) and *not* INVALID_CAS (aka KEY_EEXISTS).
527 TEST_P(VBucketEvictionTest, MB21448_UnlockedSetWithCASDeleted) {
528     // Setup - create a key and then delete it.
529     StoredDocKey key = makeStoredDocKey("key");
530     Item item(key, 0, 0, "deleted", strlen("deleted"));
531     ASSERT_EQ(MutationStatus::WasClean,
532               this->public_processSet(item, item.getCas()));
533     ASSERT_EQ(MutationStatus::WasDirty,
534               this->public_processSoftDelete(key, nullptr, 0));
535
536     // Attempt to perform a set on a deleted key with a CAS.
537     Item replacement(key, 0, 0, "value", strlen("value"));
538     EXPECT_EQ(MutationStatus::NotFound,
539               this->public_processSet(replacement,
540                                                /*cas*/ 10))
541             << "When trying to replace-with-CAS a deleted item";
542 }
543
544 // Test cases which run in both Full and Value eviction
545 INSTANTIATE_TEST_CASE_P(
546         FullAndValueEviction,
547         VBucketTest,
548         ::testing::Values(VALUE_ONLY, FULL_EVICTION),
549         [](const ::testing::TestParamInfo<item_eviction_policy_t>& info) {
550             if (info.param == VALUE_ONLY) {
551                 return "VALUE_ONLY";
552             } else {
553                 return "FULL_EVICTION";
554             }
555         });
556
557 INSTANTIATE_TEST_CASE_P(
558         FullAndValueEviction,
559         VBucketEvictionTest,
560         ::testing::Values(VALUE_ONLY, FULL_EVICTION),
561         [](const ::testing::TestParamInfo<item_eviction_policy_t>& info) {
562             if (info.param == VALUE_ONLY) {
563                 return "VALUE_ONLY";
564             } else {
565                 return "FULL_EVICTION";
566             }
567         });
568
569 INSTANTIATE_TEST_CASE_P(
570         FullAndValueEviction,
571         EPVBucketTest,
572         ::testing::Values(VALUE_ONLY, FULL_EVICTION),
573         [](const ::testing::TestParamInfo<item_eviction_policy_t>& info) {
574             if (info.param == VALUE_ONLY) {
575                 return "VALUE_ONLY";
576             } else {
577                 return "FULL_EVICTION";
578             }
579         });