4739fd8bef28f30a4ee4dcf237d7a6bc4c16b1bc
[ep-engine.git] / tests / module_tests / kv_bucket_test.cc
1 /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 /*
3  *     Copyright 2017 Couchbase, Inc
4  *
5  *   Licensed under the Apache License, Version 2.0 (the "License");
6  *   you may not use this file except in compliance with the License.
7  *   You may obtain a copy of the License at
8  *
9  *       http://www.apache.org/licenses/LICENSE-2.0
10  *
11  *   Unless required by applicable law or agreed to in writing, software
12  *   distributed under the License is distributed on an "AS IS" BASIS,
13  *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  *   See the License for the specific language governing permissions and
15  *   limitations under the License.
16  */
17
18 /*
19  * Unit tests for the KVBucket class.
20  */
21
22 #include "kv_bucket_test.h"
23
24 #include "../mock/mock_dcp_producer.h"
25 #include "bgfetcher.h"
26 #include "checkpoint.h"
27 #include "checkpoint_remover.h"
28 #include "dcp/dcpconnmap.h"
29 #include "dcp/flow-control-manager.h"
30 #include "ep_engine.h"
31 #include "flusher.h"
32 #include "replicationthrottle.h"
33 #include "tapconnmap.h"
34 #include "tasks.h"
35 #include "tests/mock/mock_global_task.h"
36 #include "tests/module_tests/test_helpers.h"
37 #include "vbucketmemorydeletiontask.h"
38
39 #include <platform/dirutils.h>
40 #include <chrono>
41 #include <thread>
42
43 void KVBucketTest::SetUp() {
44     // Paranoia - kill any existing files in case they are left over
45     // from a previous run.
46     cb::io::rmrf(test_dbname);
47
48     // Add dbname to config string.
49     std::string config = config_string;
50     if (config.size() > 0) {
51         config += ";";
52     }
53     config += "dbname=" + std::string(test_dbname);
54
55     engine.reset(new SynchronousEPEngine(config));
56     ObjectRegistry::onSwitchThread(engine.get());
57
58     engine->setKVBucket(engine->public_makeBucket(engine->getConfiguration()));
59     store = engine->getKVBucket();
60
61     store->chkTask = new ClosedUnrefCheckpointRemoverTask(
62             engine.get(),
63             engine->getEpStats(),
64             engine->getConfiguration().getChkRemoverStime());
65
66     // Ensure that EPEngine is hold about necessary server callbacks
67     // (client disconnect, bucket delete).
68     engine->public_initializeEngineCallbacks();
69
70     // Need to initialize ep_real_time and friends.
71     initialize_time_functions(get_mock_server_api()->core);
72
73     cookie = create_mock_cookie();
74 }
75
76 void KVBucketTest::TearDown() {
77     destroy_mock_cookie(cookie);
78     destroy_mock_event_callbacks();
79     engine->getDcpConnMap().manageConnections();
80     ObjectRegistry::onSwitchThread(nullptr);
81     engine.reset();
82
83     // Shutdown the ExecutorPool singleton (initialized when we create
84     // an EPBucket object). Must happen after engine
85     // has been destroyed (to allow the tasks the engine has
86     // registered a chance to be unregistered).
87     ExecutorPool::shutdown();
88 }
89
90 Item KVBucketTest::store_item(uint16_t vbid,
91                               const StoredDocKey& key,
92                               const std::string& value,
93                               uint32_t exptime,
94                               const std::vector<cb::engine_errc>& expected,
95                               protocol_binary_datatype_t datatype) {
96     auto item = make_item(vbid, key, value, exptime, datatype);
97     auto returnCode = store->set(item, nullptr);
98     EXPECT_NE(expected.end(),
99               std::find(expected.begin(),
100                         expected.end(),
101                         cb::engine_errc(returnCode)));
102     return item;
103 }
104
105 ::testing::AssertionResult KVBucketTest::store_items(
106         int nitems,
107         uint16_t vbid,
108         const DocKey& key,
109         const std::string& value,
110         uint32_t exptime,
111         protocol_binary_datatype_t datatype) {
112     for (int ii = 0; ii < nitems; ii++) {
113         auto keyii = makeStoredDocKey(
114                 std::string(reinterpret_cast<const char*>(key.data()),
115                             key.size()) +
116                         std::to_string(ii),
117                 key.getDocNamespace());
118         auto item = make_item(vbid, keyii, value, exptime, datatype);
119         auto err = store->set(item, nullptr);
120         if (ENGINE_SUCCESS != err) {
121             return ::testing::AssertionFailure()
122                    << "Failed to store " << keyii.data() << " error:" << err;
123         }
124     }
125     return ::testing::AssertionSuccess();
126 }
127
128 void KVBucketTest::flush_vbucket_to_disk(uint16_t vbid, int expected) {
129     int result;
130     const auto time_limit = std::chrono::seconds(10);
131     const auto deadline = std::chrono::steady_clock::now() + time_limit;
132
133     // Need to retry as warmup may not have completed.
134     bool flush_successful = false;
135     do {
136         result = store->flushVBucket(vbid);
137         if (result != RETRY_FLUSH_VBUCKET) {
138             flush_successful = true;
139             break;
140         }
141         std::this_thread::sleep_for(std::chrono::microseconds(100));
142     } while (std::chrono::steady_clock::now() < deadline);
143
144     ASSERT_TRUE(flush_successful)
145             << "Hit timeout (" << time_limit.count()
146             << " seconds) waiting for "
147                "warmup to complete while flushing VBucket.";
148
149     ASSERT_EQ(expected, result) << "Unexpected items in flush_vbucket_to_disk";
150 }
151
152 void KVBucketTest::delete_item(uint16_t vbid, const StoredDocKey& key) {
153     uint64_t cas = 0;
154     EXPECT_EQ(ENGINE_SUCCESS,
155               store->deleteItem(key,
156                                 cas,
157                                 vbid,
158                                 cookie,
159                                 /*Item*/ nullptr,
160                                 /*itemMeta*/ nullptr,
161                                 /*mutation_descr_t*/ nullptr));
162 }
163
164 void KVBucketTest::evict_key(uint16_t vbid, const StoredDocKey& key) {
165     const char* msg;
166     EXPECT_EQ(ENGINE_SUCCESS, store->evictKey(key, vbid, &msg));
167     EXPECT_STREQ("Ejected.", msg);
168 }
169
170 GetValue KVBucketTest::getInternal(const StoredDocKey& key,
171                                    uint16_t vbucket,
172                                    const void* cookie,
173                                    vbucket_state_t allowedState,
174                                    get_options_t options) {
175     return store->getInternal(key, vbucket, cookie, allowedState, options);
176 }
177
178 void KVBucketTest::createAndScheduleItemPager() {
179     store->itemPagerTask = new ItemPager(engine.get(), engine->getEpStats());
180     ExecutorPool::get()->schedule(store->itemPagerTask);
181 }
182
183 void KVBucketTest::initializeExpiryPager() {
184     store->initializeExpiryPager(engine->getConfiguration());
185 }
186
187 // getKeyStats tests //////////////////////////////////////////////////////////
188
189 // Check that keystats on resident items works correctly.
190 TEST_P(KVBucketParamTest, GetKeyStatsResident) {
191     key_stats kstats;
192
193     // Should start with key not existing.
194     EXPECT_EQ(ENGINE_KEY_ENOENT,
195               store->getKeyStats(makeStoredDocKey("key"),
196                                  0,
197                                  cookie,
198                                  kstats,
199                                  WantsDeleted::No));
200
201     store_item(0, makeStoredDocKey("key"), "value");
202     EXPECT_EQ(ENGINE_SUCCESS,
203               store->getKeyStats(makeStoredDocKey("key"),
204                                  0,
205                                  cookie,
206                                  kstats,
207                                  WantsDeleted::No))
208             << "Expected to get key stats on existing item";
209     EXPECT_EQ(vbucket_state_active, kstats.vb_state);
210     EXPECT_FALSE(kstats.logically_deleted);
211 }
212
213 // Create then delete an item, checking we get keyStats reporting the item as
214 // deleted.
215 TEST_P(KVBucketParamTest, GetKeyStatsDeleted) {
216     auto& kvbucket = *engine->getKVBucket();
217     key_stats kstats;
218
219     store_item(0, makeStoredDocKey("key"), "value");
220     delete_item(vbid, makeStoredDocKey("key"));
221
222     // Should get ENOENT if we don't ask for deleted items.
223     EXPECT_EQ(ENGINE_KEY_ENOENT,
224               kvbucket.getKeyStats(makeStoredDocKey("key"),
225                                    0,
226                                    cookie,
227                                    kstats,
228                                    WantsDeleted::No));
229
230     // Should get success (and item flagged as deleted) if we ask for deleted
231     // items.
232     EXPECT_EQ(ENGINE_SUCCESS,
233               kvbucket.getKeyStats(makeStoredDocKey("key"),
234                                    0,
235                                    cookie,
236                                    kstats,
237                                    WantsDeleted::Yes));
238     EXPECT_EQ(vbucket_state_active, kstats.vb_state);
239     EXPECT_TRUE(kstats.logically_deleted);
240 }
241
242 // Check incorrect vbucket returns not-my-vbucket.
243 TEST_P(KVBucketParamTest, GetKeyStatsNMVB) {
244     auto& kvbucket = *engine->getKVBucket();
245     key_stats kstats;
246
247     EXPECT_EQ(ENGINE_NOT_MY_VBUCKET,
248               kvbucket.getKeyStats(makeStoredDocKey("key"),
249                                    1,
250                                    cookie,
251                                    kstats,
252                                    WantsDeleted::No));
253 }
254
255 // Replace tests //////////////////////////////////////////////////////////////
256
257 // Test replace against a non-existent key.
258 TEST_P(KVBucketParamTest, ReplaceENOENT) {
259     // Should start with key not existing (and hence cannot replace).
260     auto item = make_item(vbid, makeStoredDocKey("key"), "value");
261     EXPECT_EQ(ENGINE_KEY_ENOENT, store->replace(item, cookie));
262 }
263
264 // Create then delete an item, checking replace reports ENOENT.
265 TEST_P(KVBucketParamTest, ReplaceDeleted) {
266     store_item(vbid, makeStoredDocKey("key"), "value");
267     delete_item(vbid, makeStoredDocKey("key"));
268
269     // Replace should fail.
270     auto item = make_item(vbid, makeStoredDocKey("key"), "value2");
271     EXPECT_EQ(ENGINE_KEY_ENOENT, store->replace(item, cookie));
272 }
273
274 // Check incorrect vbucket returns not-my-vbucket.
275 TEST_P(KVBucketParamTest, ReplaceNMVB) {
276     auto item = make_item(vbid + 1, makeStoredDocKey("key"), "value2");
277     EXPECT_EQ(ENGINE_NOT_MY_VBUCKET, store->replace(item, cookie));
278 }
279
280 // Check pending vbucket returns EWOULDBLOCK.
281 TEST_P(KVBucketParamTest, ReplacePendingVB) {
282     store->setVBucketState(vbid, vbucket_state_pending, false);
283     auto item = make_item(vbid, makeStoredDocKey("key"), "value2");
284     EXPECT_EQ(ENGINE_EWOULDBLOCK, store->replace(item, cookie));
285 }
286
287 // Set tests //////////////////////////////////////////////////////////////////
288
289 // Test CAS set against a non-existent key
290 TEST_P(KVBucketParamTest, SetCASNonExistent) {
291     // Create an item with a non-zero CAS.
292     auto item = make_item(vbid, makeStoredDocKey("key"), "value");
293     item.setCas();
294     ASSERT_NE(0, item.getCas());
295
296     // Should get ENOENT as we should immediately know (either from metadata
297     // being resident, or by bloomfilter) that key doesn't exist.
298     EXPECT_EQ(ENGINE_KEY_ENOENT, store->set(item, cookie));
299 }
300
301 // Test CAS set against a deleted item
302 TEST_P(KVBucketParamTest, SetCASDeleted) {
303     auto key = makeStoredDocKey("key");
304     auto item = make_item(vbid, key, "value");
305
306     // Store item
307     EXPECT_EQ(ENGINE_SUCCESS, store->set(item, cookie));
308
309     // Delete item
310     uint64_t cas = 0;
311     EXPECT_EQ(ENGINE_SUCCESS,
312               store->deleteItem(key,
313                                 cas,
314                                 vbid,
315                                 cookie,
316                       /*Item*/ nullptr,
317                       /*itemMeta*/ nullptr,
318                       /*mutation_descr_t*/ nullptr));
319
320     if (engine->getConfiguration().getBucketType() == "persistent") {
321         // Trigger a flush to disk.
322         flush_vbucket_to_disk(vbid);
323     }
324
325     // check we have the cas
326     ASSERT_NE(0, cas);
327
328     auto item2 = make_item(vbid, key, "value2");
329     item2.setCas(cas);
330
331     // Store item
332     if (engine->getConfiguration().getItemEvictionPolicy() ==
333                "full_eviction") {
334         EXPECT_EQ(ENGINE_EWOULDBLOCK, store->set(item2, cookie));
335
336         // Manually run the bgfetch task.
337         MockGlobalTask mockTask(engine->getTaskable(),
338                                 TaskId::MultiBGFetcherTask);
339         store->getVBucket(vbid)->getShard()->getBgFetcher()->run(&mockTask);
340     }
341
342     EXPECT_EQ(ENGINE_KEY_ENOENT, store->set(item2, cookie));
343 }
344
345 // Add tests //////////////////////////////////////////////////////////////////
346
347 // Test successful add
348 TEST_P(KVBucketParamTest, Add) {
349     auto item = make_item(vbid, makeStoredDocKey("key"), "value");
350     EXPECT_EQ(ENGINE_SUCCESS, store->add(item, nullptr));
351 }
352
353 // Check incorrect vbucket returns not-my-vbucket.
354 TEST_P(KVBucketParamTest, AddNMVB) {
355     auto item = make_item(vbid + 1, makeStoredDocKey("key"), "value2");
356     EXPECT_EQ(ENGINE_NOT_MY_VBUCKET, store->add(item, cookie));
357 }
358
359 // SetWithMeta tests //////////////////////////////////////////////////////////
360
361 // Test basic setWithMeta
362 TEST_P(KVBucketParamTest, SetWithMeta) {
363     auto item = make_item(vbid, makeStoredDocKey("key"), "value");
364     item.setCas();
365     uint64_t seqno;
366     EXPECT_EQ(ENGINE_SUCCESS,
367               store->setWithMeta(item,
368                                  0,
369                                  &seqno,
370                                  cookie,
371                                  /*force*/ false,
372                                  /*allowExisting*/ false));
373 }
374
375 // Test setWithMeta with a conflict with an existing item.
376 TEST_P(KVBucketParamTest, SetWithMeta_Conflicted) {
377     auto item = make_item(vbid, makeStoredDocKey("key"), "value");
378     EXPECT_EQ(ENGINE_SUCCESS, store->set(item, nullptr));
379
380     uint64_t seqno;
381     // Attempt to set with the same rev Seqno - should get EEXISTS.
382     EXPECT_EQ(ENGINE_KEY_EEXISTS,
383               store->setWithMeta(item,
384                                  item.getCas(),
385                                  &seqno,
386                                  cookie,
387                                  /*force*/ false,
388                                  /*allowExisting*/ true));
389 }
390
391 // Test setWithMeta replacing existing item
392 TEST_P(KVBucketParamTest, SetWithMeta_Replace) {
393     auto item = make_item(vbid, makeStoredDocKey("key"), "value");
394     EXPECT_EQ(ENGINE_SUCCESS, store->set(item, nullptr));
395
396     // Increase revSeqno so conflict resolution doesn't fail.
397     item.setRevSeqno(item.getRevSeqno() + 1);
398     uint64_t seqno;
399     // Should get EEXISTS if we don't force (and use wrong CAS).
400     EXPECT_EQ(ENGINE_KEY_EEXISTS,
401               store->setWithMeta(item,
402                                  item.getCas() + 1,
403                                  &seqno,
404                                  cookie,
405                                  /*force*/ false,
406                                  /*allowExisting*/ true));
407
408     // Should succeed with correct CAS, and different RevSeqno.
409     EXPECT_EQ(ENGINE_SUCCESS,
410               store->setWithMeta(item,
411                                  item.getCas(),
412                                  &seqno,
413                                  cookie,
414                                  /*force*/ false,
415                                  /*allowExisting*/ true));
416 }
417
418 // Test forced setWithMeta
419 TEST_P(KVBucketParamTest, SetWithMeta_Forced) {
420     auto item = make_item(vbid, makeStoredDocKey("key"), "value");
421     item.setCas();
422     uint64_t seqno;
423     EXPECT_EQ(ENGINE_SUCCESS,
424               store->setWithMeta(item,
425                                  0,
426                                  &seqno,
427                                  cookie,
428                                  /*force*/ true,
429                                  /*allowExisting*/ false));
430 }
431
432 // MB and test was raised because a few commits back this was broken but no
433 // existing test covered the case. I.e. run this test  against 0810540 and it
434 // fails, but now fixed
435 TEST_P(KVBucketParamTest, mb22824) {
436     auto key = makeStoredDocKey("key");
437
438     // Store key and force expiry
439     store_item(0, key, "value", 1);
440     TimeTraveller docBrown(20);
441
442     uint32_t deleted = false;
443     ItemMetaData itemMeta1;
444     uint8_t datatype = PROTOCOL_BINARY_RAW_BYTES;
445     EXPECT_EQ(ENGINE_SUCCESS,
446               store->getMetaData(
447                       key, vbid, cookie, false, itemMeta1, deleted, datatype));
448
449     uint64_t cas = 0;
450     ItemMetaData itemMeta2;
451     EXPECT_EQ(ENGINE_KEY_ENOENT,
452               store->deleteItem(key,
453                                 cas,
454                                 vbid,
455                                 cookie,
456                                 /*Item*/ nullptr,
457                                 &itemMeta2,
458                                 /*mutation_descr_t*/ nullptr));
459
460     // Should be getting the same CAS from the failed delete as getMetaData
461     EXPECT_EQ(itemMeta1.cas, itemMeta2.cas);
462 }
463
464 // Test cases which run for EP (Full and Value eviction) and Ephemeral
465 INSTANTIATE_TEST_CASE_P(EphemeralOrPersistent,
466                         KVBucketParamTest,
467                         ::testing::Values("item_eviction_policy=value_only",
468                                           "item_eviction_policy=full_eviction",
469                                           "bucket_type=ephemeral"),
470                         [](const ::testing::TestParamInfo<std::string>& info) {
471                             return info.param.substr(info.param.find('=') + 1);
472                         });
473
474 const char KVBucketTest::test_dbname[] = "ep_engine_ep_unit_tests_db";