MB-24293: Retry atmost 10 times if directory removal fails
[ep-engine.git] / tests / module_tests / kv_bucket_test.cc
1 /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 /*
3  *     Copyright 2017 Couchbase, Inc
4  *
5  *   Licensed under the Apache License, Version 2.0 (the "License");
6  *   you may not use this file except in compliance with the License.
7  *   You may obtain a copy of the License at
8  *
9  *       http://www.apache.org/licenses/LICENSE-2.0
10  *
11  *   Unless required by applicable law or agreed to in writing, software
12  *   distributed under the License is distributed on an "AS IS" BASIS,
13  *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  *   See the License for the specific language governing permissions and
15  *   limitations under the License.
16  */
17
18 /*
19  * Unit tests for the KVBucket class.
20  */
21
22 #include "kv_bucket_test.h"
23
24 #include "../mock/mock_dcp_producer.h"
25 #include "bgfetcher.h"
26 #include "checkpoint.h"
27 #include "checkpoint_remover.h"
28 #include "dcp/dcpconnmap.h"
29 #include "dcp/flow-control-manager.h"
30 #include "ep_engine.h"
31 #include "flusher.h"
32 #include "replicationthrottle.h"
33 #include "tapconnmap.h"
34 #include "tasks.h"
35 #include "tests/mock/mock_global_task.h"
36 #include "tests/module_tests/test_helpers.h"
37 #include "vbucketdeletiontask.h"
38
39 #include <platform/dirutils.h>
40 #include <chrono>
41 #include <thread>
42
43 #include <string_utilities.h>
44 #include <xattr/blob.h>
45 #include <xattr/utils.h>
46
47 void KVBucketTest::SetUp() {
48     // Paranoia - kill any existing files in case they are left over
49     // from a previous run.
50     try {
51         cb::io::rmrf(test_dbname);
52     } catch (std::system_error& e) {
53         if (e.code() != std::error_code(ENOENT, std::system_category())) {
54             throw e;
55         }
56     }
57
58     // Add dbname to config string.
59     std::string config = config_string;
60     if (config.size() > 0) {
61         config += ";";
62     }
63     config += "dbname=" + std::string(test_dbname);
64
65     engine.reset(new SynchronousEPEngine(config));
66     ObjectRegistry::onSwitchThread(engine.get());
67
68     engine->setKVBucket(engine->public_makeBucket(engine->getConfiguration()));
69     store = engine->getKVBucket();
70
71     store->chkTask = new ClosedUnrefCheckpointRemoverTask(
72             engine.get(),
73             engine->getEpStats(),
74             engine->getConfiguration().getChkRemoverStime());
75
76     // Ensure that EPEngine is hold about necessary server callbacks
77     // (client disconnect, bucket delete).
78     engine->public_initializeEngineCallbacks();
79
80     // Need to initialize ep_real_time and friends.
81     initialize_time_functions(get_mock_server_api()->core);
82
83     cookie = create_mock_cookie();
84 }
85
86 void KVBucketTest::TearDown() {
87     destroy_mock_cookie(cookie);
88     destroy_mock_event_callbacks();
89     engine->getDcpConnMap().manageConnections();
90     ObjectRegistry::onSwitchThread(nullptr);
91     engine.reset();
92
93     // Shutdown the ExecutorPool singleton (initialized when we create
94     // an EPBucket object). Must happen after engine
95     // has been destroyed (to allow the tasks the engine has
96     // registered a chance to be unregistered).
97     ExecutorPool::shutdown();
98 }
99
100 Item KVBucketTest::store_item(uint16_t vbid,
101                               const StoredDocKey& key,
102                               const std::string& value,
103                               uint32_t exptime,
104                               const std::vector<cb::engine_errc>& expected,
105                               protocol_binary_datatype_t datatype) {
106     auto item = make_item(vbid, key, value, exptime, datatype);
107     auto returnCode = store->set(item, nullptr);
108     EXPECT_NE(expected.end(),
109               std::find(expected.begin(),
110                         expected.end(),
111                         cb::engine_errc(returnCode)));
112     return item;
113 }
114
115 ::testing::AssertionResult KVBucketTest::store_items(
116         int nitems,
117         uint16_t vbid,
118         const DocKey& key,
119         const std::string& value,
120         uint32_t exptime,
121         protocol_binary_datatype_t datatype) {
122     for (int ii = 0; ii < nitems; ii++) {
123         auto keyii = makeStoredDocKey(
124                 std::string(reinterpret_cast<const char*>(key.data()),
125                             key.size()) +
126                         std::to_string(ii),
127                 key.getDocNamespace());
128         auto item = make_item(vbid, keyii, value, exptime, datatype);
129         auto err = store->set(item, nullptr);
130         if (ENGINE_SUCCESS != err) {
131             return ::testing::AssertionFailure()
132                    << "Failed to store " << keyii.data() << " error:" << err;
133         }
134     }
135     return ::testing::AssertionSuccess();
136 }
137
138 void KVBucketTest::flush_vbucket_to_disk(uint16_t vbid, int expected) {
139     int result;
140     const auto time_limit = std::chrono::seconds(10);
141     const auto deadline = std::chrono::steady_clock::now() + time_limit;
142
143     // Need to retry as warmup may not have completed.
144     bool flush_successful = false;
145     do {
146         result = store->flushVBucket(vbid);
147         if (result != RETRY_FLUSH_VBUCKET) {
148             flush_successful = true;
149             break;
150         }
151         std::this_thread::sleep_for(std::chrono::microseconds(100));
152     } while (std::chrono::steady_clock::now() < deadline);
153
154     ASSERT_TRUE(flush_successful)
155             << "Hit timeout (" << time_limit.count()
156             << " seconds) waiting for "
157                "warmup to complete while flushing VBucket.";
158
159     ASSERT_EQ(expected, result) << "Unexpected items in flush_vbucket_to_disk";
160 }
161
162 void KVBucketTest::delete_item(uint16_t vbid, const StoredDocKey& key) {
163     uint64_t cas = 0;
164     EXPECT_EQ(ENGINE_SUCCESS,
165               store->deleteItem(key,
166                                 cas,
167                                 vbid,
168                                 cookie,
169                                 /*itemMeta*/ nullptr,
170                                 /*mutation_descr_t*/ nullptr));
171 }
172
173 void KVBucketTest::evict_key(uint16_t vbid, const StoredDocKey& key) {
174     const char* msg;
175     EXPECT_EQ(ENGINE_SUCCESS, store->evictKey(key, vbid, &msg));
176     EXPECT_STREQ("Ejected.", msg);
177 }
178
179 GetValue KVBucketTest::getInternal(const StoredDocKey& key,
180                                    uint16_t vbucket,
181                                    const void* cookie,
182                                    vbucket_state_t allowedState,
183                                    get_options_t options) {
184     return store->getInternal(key, vbucket, cookie, allowedState, options);
185 }
186
187 void KVBucketTest::createAndScheduleItemPager() {
188     store->itemPagerTask = new ItemPager(engine.get(), engine->getEpStats());
189     ExecutorPool::get()->schedule(store->itemPagerTask);
190 }
191
192 void KVBucketTest::initializeExpiryPager() {
193     store->initializeExpiryPager(engine->getConfiguration());
194 }
195
196 /**
197  * Create a del_with_meta packet with the key/body (body can be empty)
198  */
199 std::vector<char> KVBucketTest::buildWithMetaPacket(
200         protocol_binary_command opcode,
201         protocol_binary_datatype_t datatype,
202         uint16_t vbucket,
203         uint32_t opaque,
204         uint64_t cas,
205         ItemMetaData metaData,
206         const std::string& key,
207         const std::string& body,
208         const std::vector<char>& emd,
209         int options) {
210     EXPECT_EQ(sizeof(protocol_binary_request_set_with_meta),
211               sizeof(protocol_binary_request_delete_with_meta));
212
213     size_t size = sizeof(protocol_binary_request_set_with_meta);
214     // body at least the meta
215     size_t extlen = (sizeof(uint32_t) * 2) + (sizeof(uint64_t) * 2);
216     size_t bodylen = extlen;
217     if (options) {
218         size += sizeof(uint32_t);
219         bodylen += sizeof(uint32_t);
220         extlen += sizeof(uint32_t);
221     }
222     if (!emd.empty()) {
223         EXPECT_TRUE(emd.size() < std::numeric_limits<uint16_t>::max());
224         size += sizeof(uint16_t) + emd.size();
225         bodylen += sizeof(uint16_t) + emd.size();
226         extlen += sizeof(uint16_t);
227     }
228     size += body.size();
229     bodylen += body.size();
230     size += key.size();
231     bodylen += key.size();
232
233     protocol_binary_request_set_with_meta header;
234     header.message.header.request.magic = PROTOCOL_BINARY_REQ;
235     header.message.header.request.opcode = opcode;
236     header.message.header.request.keylen = htons(key.size());
237     header.message.header.request.extlen = uint8_t(extlen);
238     header.message.header.request.datatype = datatype;
239     header.message.header.request.vbucket = htons(vbucket);
240     header.message.header.request.bodylen = htonl(bodylen);
241     header.message.header.request.opaque = opaque;
242     header.message.header.request.cas = htonll(cas);
243     header.message.body.flags = metaData.flags;
244     header.message.body.expiration = htonl(metaData.exptime);
245     header.message.body.seqno = htonll(metaData.revSeqno);
246     header.message.body.cas = htonll(metaData.cas);
247
248     std::vector<char> packet;
249     packet.reserve(size);
250     packet.insert(packet.end(),
251                   reinterpret_cast<char*>(&header),
252                   reinterpret_cast<char*>(&header) +
253                           sizeof(protocol_binary_request_set_with_meta));
254
255     if (options) {
256         options = htonl(options);
257         std::copy_n(reinterpret_cast<char*>(&options),
258                     sizeof(uint32_t),
259                     std::back_inserter(packet));
260     }
261
262     if (!emd.empty()) {
263         uint16_t emdSize = htons(emd.size());
264         std::copy_n(reinterpret_cast<char*>(&emdSize),
265                     sizeof(uint16_t),
266                     std::back_inserter(packet));
267     }
268
269     std::copy_n(key.c_str(), key.size(), std::back_inserter(packet));
270     std::copy_n(body.c_str(), body.size(), std::back_inserter(packet));
271     packet.insert(packet.end(), emd.begin(), emd.end());
272     return packet;
273 }
274
275 std::string KVBucketTest::createXattrValue(const std::string& body) {
276     cb::xattr::Blob blob;
277
278     // Add a few XAttrs
279     blob.set(to_const_byte_buffer("user"),
280              to_const_byte_buffer("{\"author\":\"bubba\"}"));
281     blob.set(to_const_byte_buffer("_sync"),
282              to_const_byte_buffer("{\"cas\":\"0xdeadbeefcafefeed\"}"));
283     blob.set(to_const_byte_buffer("meta"),
284              to_const_byte_buffer("{\"content-type\":\"text\"}"));
285
286     auto xattrValue = blob.finalize();
287
288     // append body to the xattrs and store in data
289     std::string data;
290     std::copy_n(xattrValue.buf, xattrValue.len, std::back_inserter(data));
291     std::copy_n(body.c_str(), body.size(), std::back_inserter(data));
292
293     return data;
294 }
295
296 bool KVBucketTest::addResponse(const void* k,
297                                uint16_t keylen,
298                                const void* ext,
299                                uint8_t extlen,
300                                const void* body,
301                                uint32_t bodylen,
302                                uint8_t datatype,
303                                uint16_t status,
304                                uint64_t pcas,
305                                const void* cookie) {
306     addResponseStatus = protocol_binary_response_status(status);
307     return true;
308 }
309
310 protocol_binary_response_status KVBucketTest::getAddResponseStatus(
311         protocol_binary_response_status newval) {
312     protocol_binary_response_status rv = addResponseStatus;
313     addResponseStatus = newval;
314     return rv;
315 }
316
317 protocol_binary_response_status KVBucketTest::addResponseStatus =
318         PROTOCOL_BINARY_RESPONSE_SUCCESS;
319
320
321 // getKeyStats tests //////////////////////////////////////////////////////////
322
323 // Check that keystats on resident items works correctly.
324 TEST_P(KVBucketParamTest, GetKeyStatsResident) {
325     key_stats kstats;
326
327     // Should start with key not existing.
328     EXPECT_EQ(ENGINE_KEY_ENOENT,
329               store->getKeyStats(makeStoredDocKey("key"),
330                                  0,
331                                  cookie,
332                                  kstats,
333                                  WantsDeleted::No));
334
335     store_item(0, makeStoredDocKey("key"), "value");
336     EXPECT_EQ(ENGINE_SUCCESS,
337               store->getKeyStats(makeStoredDocKey("key"),
338                                  0,
339                                  cookie,
340                                  kstats,
341                                  WantsDeleted::No))
342             << "Expected to get key stats on existing item";
343     EXPECT_EQ(vbucket_state_active, kstats.vb_state);
344     EXPECT_FALSE(kstats.logically_deleted);
345 }
346
347 // Create then delete an item, checking we get keyStats reporting the item as
348 // deleted.
349 TEST_P(KVBucketParamTest, GetKeyStatsDeleted) {
350     auto& kvbucket = *engine->getKVBucket();
351     key_stats kstats;
352
353     store_item(0, makeStoredDocKey("key"), "value");
354     delete_item(vbid, makeStoredDocKey("key"));
355
356     // Should get ENOENT if we don't ask for deleted items.
357     EXPECT_EQ(ENGINE_KEY_ENOENT,
358               kvbucket.getKeyStats(makeStoredDocKey("key"),
359                                    0,
360                                    cookie,
361                                    kstats,
362                                    WantsDeleted::No));
363
364     // Should get success (and item flagged as deleted) if we ask for deleted
365     // items.
366     EXPECT_EQ(ENGINE_SUCCESS,
367               kvbucket.getKeyStats(makeStoredDocKey("key"),
368                                    0,
369                                    cookie,
370                                    kstats,
371                                    WantsDeleted::Yes));
372     EXPECT_EQ(vbucket_state_active, kstats.vb_state);
373     EXPECT_TRUE(kstats.logically_deleted);
374 }
375
376 // Check incorrect vbucket returns not-my-vbucket.
377 TEST_P(KVBucketParamTest, GetKeyStatsNMVB) {
378     auto& kvbucket = *engine->getKVBucket();
379     key_stats kstats;
380
381     EXPECT_EQ(ENGINE_NOT_MY_VBUCKET,
382               kvbucket.getKeyStats(makeStoredDocKey("key"),
383                                    1,
384                                    cookie,
385                                    kstats,
386                                    WantsDeleted::No));
387 }
388
389 // Replace tests //////////////////////////////////////////////////////////////
390
391 // Test replace against a non-existent key.
392 TEST_P(KVBucketParamTest, ReplaceENOENT) {
393     // Should start with key not existing (and hence cannot replace).
394     auto item = make_item(vbid, makeStoredDocKey("key"), "value");
395     EXPECT_EQ(ENGINE_KEY_ENOENT, store->replace(item, cookie));
396 }
397
398 // Create then delete an item, checking replace reports ENOENT.
399 TEST_P(KVBucketParamTest, ReplaceDeleted) {
400     store_item(vbid, makeStoredDocKey("key"), "value");
401     delete_item(vbid, makeStoredDocKey("key"));
402
403     // Replace should fail.
404     auto item = make_item(vbid, makeStoredDocKey("key"), "value2");
405     EXPECT_EQ(ENGINE_KEY_ENOENT, store->replace(item, cookie));
406 }
407
408 // Check incorrect vbucket returns not-my-vbucket.
409 TEST_P(KVBucketParamTest, ReplaceNMVB) {
410     auto item = make_item(vbid + 1, makeStoredDocKey("key"), "value2");
411     EXPECT_EQ(ENGINE_NOT_MY_VBUCKET, store->replace(item, cookie));
412 }
413
414 // Check pending vbucket returns EWOULDBLOCK.
415 TEST_P(KVBucketParamTest, ReplacePendingVB) {
416     store->setVBucketState(vbid, vbucket_state_pending, false);
417     auto item = make_item(vbid, makeStoredDocKey("key"), "value2");
418     EXPECT_EQ(ENGINE_EWOULDBLOCK, store->replace(item, cookie));
419 }
420
421 // Set tests //////////////////////////////////////////////////////////////////
422
423 // Test CAS set against a non-existent key
424 TEST_P(KVBucketParamTest, SetCASNonExistent) {
425     // Create an item with a non-zero CAS.
426     auto item = make_item(vbid, makeStoredDocKey("key"), "value");
427     item.setCas();
428     ASSERT_NE(0, item.getCas());
429
430     // Should get ENOENT as we should immediately know (either from metadata
431     // being resident, or by bloomfilter) that key doesn't exist.
432     EXPECT_EQ(ENGINE_KEY_ENOENT, store->set(item, cookie));
433 }
434
435 // Test CAS set against a deleted item
436 TEST_P(KVBucketParamTest, SetCASDeleted) {
437     auto key = makeStoredDocKey("key");
438     auto item = make_item(vbid, key, "value");
439
440     // Store item
441     EXPECT_EQ(ENGINE_SUCCESS, store->set(item, cookie));
442
443     // Delete item
444     uint64_t cas = 0;
445     EXPECT_EQ(ENGINE_SUCCESS,
446               store->deleteItem(key,
447                                 cas,
448                                 vbid,
449                                 cookie,
450                       /*itemMeta*/ nullptr,
451                       /*mutation_descr_t*/ nullptr));
452
453     if (engine->getConfiguration().getBucketType() == "persistent") {
454         // Trigger a flush to disk.
455         flush_vbucket_to_disk(vbid);
456     }
457
458     // check we have the cas
459     ASSERT_NE(0, cas);
460
461     auto item2 = make_item(vbid, key, "value2");
462     item2.setCas(cas);
463
464     // Store item
465     if (engine->getConfiguration().getItemEvictionPolicy() ==
466                "full_eviction") {
467         EXPECT_EQ(ENGINE_EWOULDBLOCK, store->set(item2, cookie));
468
469         // Manually run the bgfetch task.
470         MockGlobalTask mockTask(engine->getTaskable(),
471                                 TaskId::MultiBGFetcherTask);
472         store->getVBucket(vbid)->getShard()->getBgFetcher()->run(&mockTask);
473     }
474
475     EXPECT_EQ(ENGINE_KEY_ENOENT, store->set(item2, cookie));
476 }
477
478 // Add tests //////////////////////////////////////////////////////////////////
479
480 // Test successful add
481 TEST_P(KVBucketParamTest, Add) {
482     auto item = make_item(vbid, makeStoredDocKey("key"), "value");
483     EXPECT_EQ(ENGINE_SUCCESS, store->add(item, nullptr));
484 }
485
486 // Check incorrect vbucket returns not-my-vbucket.
487 TEST_P(KVBucketParamTest, AddNMVB) {
488     auto item = make_item(vbid + 1, makeStoredDocKey("key"), "value2");
489     EXPECT_EQ(ENGINE_NOT_MY_VBUCKET, store->add(item, cookie));
490 }
491
492 // SetWithMeta tests //////////////////////////////////////////////////////////
493
494 // Test basic setWithMeta
495 TEST_P(KVBucketParamTest, SetWithMeta) {
496     auto item = make_item(vbid, makeStoredDocKey("key"), "value");
497     item.setCas();
498     uint64_t seqno;
499     EXPECT_EQ(ENGINE_SUCCESS,
500               store->setWithMeta(item,
501                                  0,
502                                  &seqno,
503                                  cookie,
504                                  /*force*/ false,
505                                  /*allowExisting*/ false));
506 }
507
508 // Test setWithMeta with a conflict with an existing item.
509 TEST_P(KVBucketParamTest, SetWithMeta_Conflicted) {
510     auto item = make_item(vbid, makeStoredDocKey("key"), "value");
511     EXPECT_EQ(ENGINE_SUCCESS, store->set(item, nullptr));
512
513     uint64_t seqno;
514     // Attempt to set with the same rev Seqno - should get EEXISTS.
515     EXPECT_EQ(ENGINE_KEY_EEXISTS,
516               store->setWithMeta(item,
517                                  item.getCas(),
518                                  &seqno,
519                                  cookie,
520                                  /*force*/ false,
521                                  /*allowExisting*/ true));
522 }
523
524 // Test setWithMeta replacing existing item
525 TEST_P(KVBucketParamTest, SetWithMeta_Replace) {
526     auto item = make_item(vbid, makeStoredDocKey("key"), "value");
527     EXPECT_EQ(ENGINE_SUCCESS, store->set(item, nullptr));
528
529     // Increase revSeqno so conflict resolution doesn't fail.
530     item.setRevSeqno(item.getRevSeqno() + 1);
531     uint64_t seqno;
532     // Should get EEXISTS if we don't force (and use wrong CAS).
533     EXPECT_EQ(ENGINE_KEY_EEXISTS,
534               store->setWithMeta(item,
535                                  item.getCas() + 1,
536                                  &seqno,
537                                  cookie,
538                                  /*force*/ false,
539                                  /*allowExisting*/ true));
540
541     // Should succeed with correct CAS, and different RevSeqno.
542     EXPECT_EQ(ENGINE_SUCCESS,
543               store->setWithMeta(item,
544                                  item.getCas(),
545                                  &seqno,
546                                  cookie,
547                                  /*force*/ false,
548                                  /*allowExisting*/ true));
549 }
550
551 // Test forced setWithMeta
552 TEST_P(KVBucketParamTest, SetWithMeta_Forced) {
553     auto item = make_item(vbid, makeStoredDocKey("key"), "value");
554     item.setCas();
555     uint64_t seqno;
556     EXPECT_EQ(ENGINE_SUCCESS,
557               store->setWithMeta(item,
558                                  0,
559                                  &seqno,
560                                  cookie,
561                                  /*force*/ true,
562                                  /*allowExisting*/ false));
563 }
564
565 // MB and test was raised because a few commits back this was broken but no
566 // existing test covered the case. I.e. run this test  against 0810540 and it
567 // fails, but now fixed
568 TEST_P(KVBucketParamTest, mb22824) {
569     auto key = makeStoredDocKey("key");
570
571     // Store key and force expiry
572     store_item(0, key, "value", 1);
573     TimeTraveller docBrown(20);
574
575     uint32_t deleted = false;
576     ItemMetaData itemMeta1;
577     uint8_t datatype = PROTOCOL_BINARY_RAW_BYTES;
578     EXPECT_EQ(ENGINE_SUCCESS,
579               store->getMetaData(
580                       key, vbid, cookie, itemMeta1, deleted, datatype));
581
582     uint64_t cas = 0;
583     ItemMetaData itemMeta2;
584     EXPECT_EQ(ENGINE_KEY_ENOENT,
585               store->deleteItem(key,
586                                 cas,
587                                 vbid,
588                                 cookie,
589                                 &itemMeta2,
590                                 /*mutation_descr_t*/ nullptr));
591
592     // Should be getting the same CAS from the failed delete as getMetaData
593     EXPECT_EQ(itemMeta1.cas, itemMeta2.cas);
594 }
595
596 // Test cases which run for EP (Full and Value eviction) and Ephemeral
597 INSTANTIATE_TEST_CASE_P(EphemeralOrPersistent,
598                         KVBucketParamTest,
599                         ::testing::Values("item_eviction_policy=value_only",
600                                           "item_eviction_policy=full_eviction",
601                                           "bucket_type=ephemeral"),
602                         [](const ::testing::TestParamInfo<std::string>& info) {
603                             return info.param.substr(info.param.find('=') + 1);
604                         });
605
606 const char KVBucketTest::test_dbname[] = "ep_engine_ep_unit_tests_db";