MB-24376: Stop rangeRead on invalid Seqno
[ep-engine.git] / tests / module_tests / vbucket_test.cc
1 /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 /*
3  *     Copyright 2016 Couchbase, Inc
4  *
5  *   Licensed under the Apache License, Version 2.0 (the "License");
6  *   you may not use this file except in compliance with the License.
7  *   You may obtain a copy of the License at
8  *
9  *       http://www.apache.org/licenses/LICENSE-2.0
10  *
11  *   Unless required by applicable law or agreed to in writing, software
12  *   distributed under the License is distributed on an "AS IS" BASIS,
13  *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  *   See the License for the specific language governing permissions and
15  *   limitations under the License.
16  */
17
18 #include "config.h"
19
20 #include "vbucket_test.h"
21 #include "bgfetcher.h"
22 #include "ep_vb.h"
23 #include "failover-table.h"
24 #include "item.h"
25 #include "programs/engine_testapp/mock_server.h"
26 #include "tests/module_tests/test_helpers.h"
27
28 #include <platform/cb_malloc.h>
29
30 void VBucketTest::SetUp() {
31     const auto eviction_policy = GetParam();
32     vbucket.reset(new EPVBucket(0,
33                                 vbucket_state_active,
34                                 global_stats,
35                                 checkpoint_config,
36                                 /*kvshard*/ nullptr,
37                                 /*lastSeqno*/ 1000,
38                                 /*lastSnapStart*/ 0,
39                                 /*lastSnapEnd*/ 0,
40                                 /*table*/ nullptr,
41                                 std::make_shared<DummyCB>(),
42                                 /*newSeqnoCb*/ nullptr,
43                                 config,
44                                 eviction_policy));
45 }
46
47 void VBucketTest::TearDown() {
48     vbucket.reset();
49 }
50
51 std::vector<StoredDocKey> VBucketTest::generateKeys(int num, int start) {
52     std::vector<StoredDocKey> rv;
53
54     for (int i = start; i < num + start; i++) {
55         rv.push_back(makeStoredDocKey(std::to_string(i)));
56     }
57
58     return rv;
59 }
60
61 AddStatus VBucketTest::addOne(const StoredDocKey& k, int expiry) {
62     Item i(k, 0, expiry, k.data(), k.size());
63     return public_processAdd(i);
64 }
65
66 void VBucketTest::addMany(std::vector<StoredDocKey>& keys, AddStatus expect) {
67     for (const auto& k : keys) {
68         EXPECT_EQ(expect, addOne(k));
69     }
70 }
71
72 MutationStatus VBucketTest::setOne(const StoredDocKey& k, int expiry) {
73     Item i(k, 0, expiry, k.data(), k.size());
74     return public_processSet(i, i.getCas());
75 }
76
77 void VBucketTest::setMany(std::vector<StoredDocKey>& keys,
78                           MutationStatus expect) {
79     for (const auto& k : keys) {
80         EXPECT_EQ(expect, setOne(k));
81     }
82 }
83
84 void VBucketTest::softDeleteOne(const StoredDocKey& k, MutationStatus expect) {
85     StoredValue* v(vbucket->ht.find(k, TrackReference::No, WantsDeleted::No));
86     EXPECT_NE(nullptr, v);
87
88     EXPECT_EQ(expect, public_processSoftDelete(v->getKey(), v, 0))
89             << "Failed to soft delete key " << k.c_str();
90 }
91
92 void VBucketTest::softDeleteMany(std::vector<StoredDocKey>& keys,
93                                  MutationStatus expect) {
94     for (const auto& k : keys) {
95         softDeleteOne(k, expect);
96     }
97 }
98
99 StoredValue* VBucketTest::findValue(StoredDocKey& key) {
100     return vbucket->ht.find(key, TrackReference::Yes, WantsDeleted::Yes);
101 }
102
103 void VBucketTest::verifyValue(StoredDocKey& key,
104                               const char* value,
105                               TrackReference trackReference,
106                               WantsDeleted wantDeleted) {
107     StoredValue* v = vbucket->ht.find(key, trackReference, wantDeleted);
108     EXPECT_NE(nullptr, v);
109     value_t val = v->getValue();
110     if (!value) {
111         EXPECT_EQ(nullptr, val.get());
112     } else {
113         EXPECT_STREQ(value, val->to_s().c_str());
114     }
115 }
116
117 std::pair<HashTable::HashBucketLock, StoredValue*> VBucketTest::lockAndFind(
118         const StoredDocKey& key) {
119     auto hbl = vbucket->ht.getLockedBucket(key);
120     auto* storedVal = vbucket->ht.unlocked_find(
121             key, hbl.getBucketNum(), WantsDeleted::Yes, TrackReference::No);
122     return std::make_pair(std::move(hbl), storedVal);
123 }
124
125 MutationStatus VBucketTest::public_processSet(Item& itm,
126                                               const uint64_t cas,
127                                               bool withCtx) {
128     auto hbl_sv = lockAndFind(itm.getKey());
129     VBQueueItemCtx queueItmCtx(GenerateBySeqno::Yes,
130                                GenerateCas::No,
131                                TrackCasDrift::No,
132                                /*isBackfillItem*/ false,
133                                /*preLinkDocumentContext_*/ nullptr);
134     return vbucket
135             ->processSet(hbl_sv.first,
136                          hbl_sv.second,
137                          itm,
138                          cas,
139                          true,
140                          false,
141                          withCtx ? &queueItmCtx : nullptr)
142             .first;
143 }
144
145 AddStatus VBucketTest::public_processAdd(Item& itm) {
146     auto hbl_sv = lockAndFind(itm.getKey());
147     return vbucket
148             ->processAdd(hbl_sv.first,
149                          hbl_sv.second,
150                          itm,
151                          /*maybeKeyExists*/ true,
152                          /*isReplication*/ false)
153             .first;
154 }
155
156 MutationStatus VBucketTest::public_processSoftDelete(const DocKey& key,
157                                                      StoredValue* v,
158                                                      uint64_t cas) {
159     auto hbl = vbucket->ht.getLockedBucket(key);
160     if (!v) {
161         v = vbucket->ht.unlocked_find(
162                 key, hbl.getBucketNum(), WantsDeleted::No, TrackReference::No);
163         if (!v) {
164             return MutationStatus::NotFound;
165         }
166     }
167     ItemMetaData metadata;
168     metadata.revSeqno = v->getRevSeqno() + 1;
169     MutationStatus status;
170     std::tie(status, std::ignore, std::ignore) = vbucket->processSoftDelete(
171             hbl,
172             *v,
173             cas,
174             metadata,
175             VBQueueItemCtx(GenerateBySeqno::Yes,
176                            GenerateCas::Yes,
177                            TrackCasDrift::No,
178                            /*isBackfillItem*/ false,
179                            /*preLinkDocCtx*/ nullptr),
180             /*use_meta*/ false,
181             /*bySeqno*/ v->getBySeqno());
182     return status;
183 }
184
185 bool VBucketTest::public_deleteStoredValue(const DocKey& key) {
186     auto hbl_sv = lockAndFind(key);
187     if (!hbl_sv.second) {
188         return false;
189     }
190     return vbucket->deleteStoredValue(hbl_sv.first, *hbl_sv.second);
191 }
192
193 GetValue VBucketTest::public_getAndUpdateTtl(const DocKey& key,
194                                              time_t exptime) {
195     auto hbl = lockAndFind(key);
196     GetValue gv;
197     MutationStatus status;
198     std::tie(status, gv) = vbucket->processGetAndUpdateTtl(
199             hbl.first, key, hbl.second, exptime);
200     return gv;
201 }
202
203 size_t EPVBucketTest::public_queueBGFetchItem(
204         const DocKey& key,
205         std::unique_ptr<VBucketBGFetchItem> fetchItem,
206         BgFetcher* bgFetcher) {
207     return dynamic_cast<EPVBucket&>(*vbucket).queueBGFetchItem(
208             key, std::move(fetchItem), bgFetcher);
209 }
210
211 class BlobTest : public Blob {
212 public:
213     BlobTest() : Blob(0,0) {}
214     static size_t getAllocationSize(size_t len){
215         return Blob::getAllocationSize(len);
216     }
217 };
218
219 TEST(BlobTest, basicAllocationSize){
220     EXPECT_EQ(BlobTest::getAllocationSize(10), 20);
221
222     // Expected to be 10 because the 2 bytes of the data member array will not
223     // be allocated because they will not be used.
224     EXPECT_EQ(BlobTest::getAllocationSize(0), 10);
225 }
226
227 // Measure performance of VBucket::getBGFetchItems - queue and then get
228 // 10,000 items from the vbucket.
229 TEST_P(EPVBucketTest, GetBGFetchItemsPerformance) {
230     BgFetcher fetcher(/*store*/ nullptr, /*shard*/ nullptr, this->global_stats);
231
232     for (unsigned int ii = 0; ii < 100000; ii++) {
233         auto fetchItem = std::make_unique<VBucketBGFetchItem>(
234                 /*cookie*/ nullptr,
235                 /*isMeta*/ false);
236         this->public_queueBGFetchItem(
237                 makeStoredDocKey(std::to_string(ii)),
238                 std::move(fetchItem),
239                 &fetcher);
240     }
241     auto items = this->vbucket->getBGFetchItems();
242 }
243
244 // Check the existence of bloom filter after performing a
245 // swap of existing filter with a temporary filter.
246 TEST_P(VBucketTest, SwapFilter) {
247     this->vbucket->createFilter(1, 1.0);
248     ASSERT_FALSE(this->vbucket->isTempFilterAvailable());
249     ASSERT_NE("DOESN'T EXIST", this->vbucket->getFilterStatusString());
250     this->vbucket->swapFilter();
251     EXPECT_NE("DOESN'T EXIST", this->vbucket->getFilterStatusString());
252 }
253
254 TEST_P(VBucketTest, Add) {
255     const auto eviction_policy = GetParam();
256     if (eviction_policy != VALUE_ONLY) {
257         return;
258     }
259     const int nkeys = 1000;
260
261     auto keys = generateKeys(nkeys);
262     addMany(keys, AddStatus::Success);
263
264     StoredDocKey missingKey = makeStoredDocKey("aMissingKey");
265     EXPECT_FALSE(this->vbucket->ht.find(
266             missingKey, TrackReference::Yes, WantsDeleted::No));
267
268     for (const auto& key : keys) {
269         EXPECT_TRUE(this->vbucket->ht.find(
270                 key, TrackReference::Yes, WantsDeleted::No));
271     }
272
273     addMany(keys, AddStatus::Exists);
274     for (const auto& key : keys) {
275         EXPECT_TRUE(this->vbucket->ht.find(
276                 key, TrackReference::Yes, WantsDeleted::No));
277     }
278
279     // Verify we can read after a soft deletion.
280     EXPECT_EQ(MutationStatus::WasDirty,
281               this->public_processSoftDelete(keys[0], nullptr, 0));
282     EXPECT_EQ(MutationStatus::NotFound,
283               this->public_processSoftDelete(keys[0], nullptr, 0));
284     EXPECT_FALSE(this->vbucket->ht.find(
285             keys[0], TrackReference::Yes, WantsDeleted::No));
286
287     Item i(keys[0], 0, 0, "newtest", 7);
288     EXPECT_EQ(AddStatus::UnDel, this->public_processAdd(i));
289     EXPECT_EQ(nkeys, this->vbucket->ht.getNumItems());
290 }
291
292 TEST_P(VBucketTest, AddExpiry) {
293     const auto eviction_policy = GetParam();
294     if (eviction_policy != VALUE_ONLY) {
295         return;
296     }
297     StoredDocKey k = makeStoredDocKey("aKey");
298
299     ASSERT_EQ(AddStatus::Success, addOne(k, ep_real_time() + 5));
300     EXPECT_EQ(AddStatus::Exists, addOne(k, ep_real_time() + 5));
301
302     StoredValue* v =
303             this->vbucket->ht.find(k, TrackReference::Yes, WantsDeleted::No);
304     EXPECT_TRUE(v);
305     EXPECT_FALSE(v->isExpired(ep_real_time()));
306     EXPECT_TRUE(v->isExpired(ep_real_time() + 6));
307
308     TimeTraveller biffTannen(6);
309     EXPECT_TRUE(v->isExpired(ep_real_time()));
310
311     EXPECT_EQ(AddStatus::UnDel, addOne(k, ep_real_time() + 5));
312     EXPECT_TRUE(v);
313     EXPECT_FALSE(v->isExpired(ep_real_time()));
314     EXPECT_TRUE(v->isExpired(ep_real_time() + 6));
315 }
316
317 /**
318  * Test to check if an unlocked_softDelete performed on an
319  * existing item with a new value results in a success
320  */
321 TEST_P(VBucketTest, unlockedSoftDeleteWithValue) {
322     const auto eviction_policy = GetParam();
323     if (eviction_policy != VALUE_ONLY) {
324         return;
325     }
326
327     // Setup - create a key and then delete it with a value.
328     StoredDocKey key = makeStoredDocKey("key");
329     Item stored_item(key, 0, 0, "value", strlen("value"));
330     ASSERT_EQ(MutationStatus::WasClean,
331               this->public_processSet(stored_item, stored_item.getCas()));
332
333     StoredValue* v(
334             this->vbucket->ht.find(key, TrackReference::No, WantsDeleted::No));
335     EXPECT_NE(nullptr, v);
336
337     // Create an item and set its state to deleted
338     Item deleted_item(key, 0, 0, "deletedvalue", strlen("deletedvalue"));
339     deleted_item.setDeleted();
340
341     auto prev_revseqno = v->getRevSeqno();
342     deleted_item.setRevSeqno(prev_revseqno);
343
344     EXPECT_EQ(MutationStatus::WasDirty,
345               this->public_processSet(deleted_item, 0));
346     verifyValue(key, "deletedvalue", TrackReference::Yes, WantsDeleted::Yes);
347     EXPECT_EQ(prev_revseqno + 1, v->getRevSeqno());
348 }
349
350 /**
351  * Test to check that if an item has expired, an incoming mutation
352  * on that item, if in deleted state results in an invalid cas and
353  * if not in deleted state, results in not found
354  */
355 TEST_P(VBucketTest, updateExpiredItem) {
356     // Setup - create a key
357     StoredDocKey key = makeStoredDocKey("key");
358     Item stored_item(key, 0, ep_real_time() - 1, "value", strlen("value"));
359     ASSERT_EQ(MutationStatus::WasClean,
360               this->public_processSet(stored_item, stored_item.getCas()));
361
362     StoredValue* v = this->vbucket->ht.find(key, TrackReference::No, WantsDeleted::No);
363     EXPECT_TRUE(v);
364     EXPECT_TRUE(v->isExpired(ep_real_time()));
365
366     auto cas = v->getCas();
367     // Create an item and set its state to deleted.
368     Item deleted_item(key, 0, 0, "deletedvalue", strlen("deletedvalue"));
369
370     EXPECT_EQ(MutationStatus::NotFound,
371               this->public_processSet(deleted_item, cas + 1));
372
373     deleted_item.setDeleted();
374     EXPECT_EQ(MutationStatus::InvalidCas,
375               this->public_processSet(deleted_item, cas + 1));
376 }
377
378 /**
379  * Test to check if an unlocked_softDelete performed on a
380  * deleted item without a value and with a value
381  */
382 TEST_P(VBucketTest, updateDeletedItem) {
383     const auto eviction_policy = GetParam();
384     if (eviction_policy != VALUE_ONLY) {
385         return;
386     }
387
388     // Setup - create a key and then delete it.
389     StoredDocKey key = makeStoredDocKey("key");
390     Item stored_item(key, 0, 0, "value", strlen("value"));
391     ASSERT_EQ(MutationStatus::WasClean,
392               this->public_processSet(stored_item, stored_item.getCas()));
393
394     StoredValue* v(
395             this->vbucket->ht.find(key, TrackReference::No, WantsDeleted::No));
396     EXPECT_NE(nullptr, v);
397
398     ItemMetaData itm_meta;
399     EXPECT_EQ(MutationStatus::WasDirty,
400               this->public_processSoftDelete(v->getKey(), v, 0));
401     verifyValue(key, nullptr, TrackReference::Yes, WantsDeleted::Yes);
402     EXPECT_EQ(1, this->vbucket->getNumItems());
403     EXPECT_EQ(1, this->vbucket->getNumInMemoryDeletes());
404
405     Item deleted_item(key, 0, 0, "deletedvalue", strlen("deletedvalue"));
406     deleted_item.setDeleted();
407
408     auto prev_revseqno = v->getRevSeqno();
409     deleted_item.setRevSeqno(prev_revseqno);
410
411     EXPECT_EQ(MutationStatus::WasDirty,
412               this->public_processSet(deleted_item, 0));
413     verifyValue(
414                 key,
415                 "deletedvalue",
416                 TrackReference::Yes,
417                 WantsDeleted::Yes);
418     EXPECT_EQ(1, this->vbucket->getNumItems());
419     EXPECT_EQ(1, this->vbucket->getNumInMemoryDeletes());
420     EXPECT_EQ(prev_revseqno + 1, v->getRevSeqno());
421
422     Item update_deleted_item(
423             key, 0, 0, "updatedeletedvalue", strlen("updatedeletedvalue"));
424     update_deleted_item.setDeleted();
425
426     prev_revseqno = v->getRevSeqno();
427     update_deleted_item.setRevSeqno(prev_revseqno);
428
429     EXPECT_EQ(MutationStatus::WasDirty,
430               this->public_processSet(update_deleted_item, 0));
431     verifyValue(
432             key, "updatedeletedvalue", TrackReference::Yes, WantsDeleted::Yes);
433     EXPECT_EQ(1, this->vbucket->getNumItems());
434     EXPECT_EQ(1, this->vbucket->getNumInMemoryDeletes());
435     EXPECT_EQ(prev_revseqno + 1, v->getRevSeqno());
436 }
437
438 TEST_P(VBucketTest, SizeStatsSoftDel) {
439     this->global_stats.reset();
440     ASSERT_EQ(0, this->vbucket->ht.memSize.load());
441     ASSERT_EQ(0, this->vbucket->ht.cacheSize.load());
442     size_t initialSize = this->global_stats.currentSize.load();
443
444     const StoredDocKey k = makeStoredDocKey("somekey");
445     const size_t itemSize(16 * 1024);
446     char* someval(static_cast<char*>(cb_calloc(1, itemSize)));
447     EXPECT_TRUE(someval);
448
449     Item i(k, 0, 0, someval, itemSize);
450
451     EXPECT_EQ(MutationStatus::WasClean,
452               this->public_processSet(i, i.getCas()));
453
454     EXPECT_EQ(MutationStatus::WasDirty,
455               this->public_processSoftDelete(k, nullptr, 0));
456     this->public_deleteStoredValue(k);
457
458     EXPECT_EQ(0, this->vbucket->ht.memSize.load());
459     EXPECT_EQ(0, this->vbucket->ht.cacheSize.load());
460     EXPECT_EQ(initialSize, this->global_stats.currentSize.load());
461
462     cb_free(someval);
463 }
464
465 TEST_P(VBucketTest, SizeStatsSoftDelFlush) {
466     this->global_stats.reset();
467     ASSERT_EQ(0, this->vbucket->ht.memSize.load());
468     ASSERT_EQ(0, this->vbucket->ht.cacheSize.load());
469     size_t initialSize = this->global_stats.currentSize.load();
470
471     StoredDocKey k = makeStoredDocKey("somekey");
472     const size_t itemSize(16 * 1024);
473     char* someval(static_cast<char*>(cb_calloc(1, itemSize)));
474     EXPECT_TRUE(someval);
475
476     Item i(k, 0, 0, someval, itemSize);
477
478     EXPECT_EQ(MutationStatus::WasClean,
479               this->public_processSet(i, i.getCas()));
480
481     EXPECT_EQ(MutationStatus::WasDirty,
482               this->public_processSoftDelete(k, nullptr, 0));
483     this->vbucket->ht.clear();
484
485     EXPECT_EQ(0, this->vbucket->ht.memSize.load());
486     EXPECT_EQ(0, this->vbucket->ht.cacheSize.load());
487     EXPECT_EQ(initialSize, this->global_stats.currentSize.load());
488
489     cb_free(someval);
490 }
491
492 class VBucketEvictionTest : public VBucketTest {};
493
494 // Check that counts of items and resident items are as expected when items are
495 // ejected from the HashTable.
496 TEST_P(VBucketEvictionTest, EjectionResidentCount) {
497     const auto eviction_policy = GetParam();
498     ASSERT_EQ(0, this->vbucket->getNumItems());
499     ASSERT_EQ(0, this->vbucket->getNumNonResidentItems());
500
501     Item item(makeStoredDocKey("key"), /*flags*/0, /*exp*/0,
502               /*data*/nullptr, /*ndata*/0);
503
504     EXPECT_EQ(MutationStatus::WasClean,
505               this->public_processSet(item, item.getCas()));
506
507     EXPECT_EQ(1, this->vbucket->getNumItems());
508     EXPECT_EQ(0, this->vbucket->getNumNonResidentItems());
509
510     // TODO-MT: Should acquire lock really (ok given this is currently
511     // single-threaded).
512     auto* stored_item = this->vbucket->ht.find(
513             makeStoredDocKey("key"), TrackReference::Yes, WantsDeleted::No);
514     EXPECT_NE(nullptr, stored_item);
515     // Need to clear the dirty flag to allow it to be ejected.
516     stored_item->markClean();
517     EXPECT_TRUE(this->vbucket->ht.unlocked_ejectItem(stored_item,
518                                                      eviction_policy));
519
520     // After ejection, should still have 1 item in VBucket, but also have
521     // 1 non-resident item.
522     EXPECT_EQ(1, this->vbucket->getNumItems());
523     EXPECT_EQ(1, this->vbucket->getNumNonResidentItems());
524 }
525
526 // Regression test for MB-21448 - if an attempt is made to perform a CAS
527 // operation on a logically deleted item we should return NOT_FOUND
528 // (aka KEY_ENOENT) and *not* INVALID_CAS (aka KEY_EEXISTS).
529 TEST_P(VBucketEvictionTest, MB21448_UnlockedSetWithCASDeleted) {
530     // Setup - create a key and then delete it.
531     StoredDocKey key = makeStoredDocKey("key");
532     Item item(key, 0, 0, "deleted", strlen("deleted"));
533     ASSERT_EQ(MutationStatus::WasClean,
534               this->public_processSet(item, item.getCas()));
535     ASSERT_EQ(MutationStatus::WasDirty,
536               this->public_processSoftDelete(key, nullptr, 0));
537
538     // Attempt to perform a set on a deleted key with a CAS.
539     Item replacement(key, 0, 0, "value", strlen("value"));
540     EXPECT_EQ(MutationStatus::NotFound,
541               this->public_processSet(replacement,
542                                                /*cas*/ 10))
543             << "When trying to replace-with-CAS a deleted item";
544 }
545
546 // Test cases which run in both Full and Value eviction
547 INSTANTIATE_TEST_CASE_P(
548         FullAndValueEviction,
549         VBucketTest,
550         ::testing::Values(VALUE_ONLY, FULL_EVICTION),
551         [](const ::testing::TestParamInfo<item_eviction_policy_t>& info) {
552             if (info.param == VALUE_ONLY) {
553                 return "VALUE_ONLY";
554             } else {
555                 return "FULL_EVICTION";
556             }
557         });
558
559 INSTANTIATE_TEST_CASE_P(
560         FullAndValueEviction,
561         VBucketEvictionTest,
562         ::testing::Values(VALUE_ONLY, FULL_EVICTION),
563         [](const ::testing::TestParamInfo<item_eviction_policy_t>& info) {
564             if (info.param == VALUE_ONLY) {
565                 return "VALUE_ONLY";
566             } else {
567                 return "FULL_EVICTION";
568             }
569         });
570
571 INSTANTIATE_TEST_CASE_P(
572         FullAndValueEviction,
573         EPVBucketTest,
574         ::testing::Values(VALUE_ONLY, FULL_EVICTION),
575         [](const ::testing::TestParamInfo<item_eviction_policy_t>& info) {
576             if (info.param == VALUE_ONLY) {
577                 return "VALUE_ONLY";
578             } else {
579                 return "FULL_EVICTION";
580             }
581         });