Add WithMeta unit tests
[ep-engine.git] / tests / module_tests / kv_bucket_test.cc
1 /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 /*
3  *     Copyright 2017 Couchbase, Inc
4  *
5  *   Licensed under the Apache License, Version 2.0 (the "License");
6  *   you may not use this file except in compliance with the License.
7  *   You may obtain a copy of the License at
8  *
9  *       http://www.apache.org/licenses/LICENSE-2.0
10  *
11  *   Unless required by applicable law or agreed to in writing, software
12  *   distributed under the License is distributed on an "AS IS" BASIS,
13  *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  *   See the License for the specific language governing permissions and
15  *   limitations under the License.
16  */
17
18 /*
19  * Unit tests for the KVBucket class.
20  */
21
22 #include "kv_bucket_test.h"
23
24 #include "../mock/mock_dcp_producer.h"
25 #include "bgfetcher.h"
26 #include "checkpoint.h"
27 #include "checkpoint_remover.h"
28 #include "dcp/dcpconnmap.h"
29 #include "dcp/flow-control-manager.h"
30 #include "ep_engine.h"
31 #include "flusher.h"
32 #include "replicationthrottle.h"
33 #include "tapconnmap.h"
34 #include "tasks.h"
35 #include "tests/mock/mock_global_task.h"
36 #include "tests/module_tests/test_helpers.h"
37 #include "vbucketmemorydeletiontask.h"
38
39 #include <platform/dirutils.h>
40 #include <chrono>
41 #include <thread>
42
43 #include <string_utilities.h>
44 #include <xattr/blob.h>
45 #include <xattr/utils.h>
46
47 void KVBucketTest::SetUp() {
48     // Paranoia - kill any existing files in case they are left over
49     // from a previous run.
50     cb::io::rmrf(test_dbname);
51
52     // Add dbname to config string.
53     std::string config = config_string;
54     if (config.size() > 0) {
55         config += ";";
56     }
57     config += "dbname=" + std::string(test_dbname);
58
59     engine.reset(new SynchronousEPEngine(config));
60     ObjectRegistry::onSwitchThread(engine.get());
61
62     engine->setKVBucket(engine->public_makeBucket(engine->getConfiguration()));
63     store = engine->getKVBucket();
64
65     store->chkTask = new ClosedUnrefCheckpointRemoverTask(
66             engine.get(),
67             engine->getEpStats(),
68             engine->getConfiguration().getChkRemoverStime());
69
70     // Ensure that EPEngine is hold about necessary server callbacks
71     // (client disconnect, bucket delete).
72     engine->public_initializeEngineCallbacks();
73
74     // Need to initialize ep_real_time and friends.
75     initialize_time_functions(get_mock_server_api()->core);
76
77     cookie = create_mock_cookie();
78 }
79
80 void KVBucketTest::TearDown() {
81     destroy_mock_cookie(cookie);
82     destroy_mock_event_callbacks();
83     engine->getDcpConnMap().manageConnections();
84     ObjectRegistry::onSwitchThread(nullptr);
85     engine.reset();
86
87     // Shutdown the ExecutorPool singleton (initialized when we create
88     // an EPBucket object). Must happen after engine
89     // has been destroyed (to allow the tasks the engine has
90     // registered a chance to be unregistered).
91     ExecutorPool::shutdown();
92 }
93
94 Item KVBucketTest::store_item(uint16_t vbid,
95                               const StoredDocKey& key,
96                               const std::string& value,
97                               uint32_t exptime,
98                               const std::vector<cb::engine_errc>& expected,
99                               protocol_binary_datatype_t datatype) {
100     auto item = make_item(vbid, key, value, exptime, datatype);
101     auto returnCode = store->set(item, nullptr);
102     EXPECT_NE(expected.end(),
103               std::find(expected.begin(),
104                         expected.end(),
105                         cb::engine_errc(returnCode)));
106     return item;
107 }
108
109 ::testing::AssertionResult KVBucketTest::store_items(
110         int nitems,
111         uint16_t vbid,
112         const DocKey& key,
113         const std::string& value,
114         uint32_t exptime,
115         protocol_binary_datatype_t datatype) {
116     for (int ii = 0; ii < nitems; ii++) {
117         auto keyii = makeStoredDocKey(
118                 std::string(reinterpret_cast<const char*>(key.data()),
119                             key.size()) +
120                         std::to_string(ii),
121                 key.getDocNamespace());
122         auto item = make_item(vbid, keyii, value, exptime, datatype);
123         auto err = store->set(item, nullptr);
124         if (ENGINE_SUCCESS != err) {
125             return ::testing::AssertionFailure()
126                    << "Failed to store " << keyii.data() << " error:" << err;
127         }
128     }
129     return ::testing::AssertionSuccess();
130 }
131
132 void KVBucketTest::flush_vbucket_to_disk(uint16_t vbid, int expected) {
133     int result;
134     const auto time_limit = std::chrono::seconds(10);
135     const auto deadline = std::chrono::steady_clock::now() + time_limit;
136
137     // Need to retry as warmup may not have completed.
138     bool flush_successful = false;
139     do {
140         result = store->flushVBucket(vbid);
141         if (result != RETRY_FLUSH_VBUCKET) {
142             flush_successful = true;
143             break;
144         }
145         std::this_thread::sleep_for(std::chrono::microseconds(100));
146     } while (std::chrono::steady_clock::now() < deadline);
147
148     ASSERT_TRUE(flush_successful)
149             << "Hit timeout (" << time_limit.count()
150             << " seconds) waiting for "
151                "warmup to complete while flushing VBucket.";
152
153     ASSERT_EQ(expected, result) << "Unexpected items in flush_vbucket_to_disk";
154 }
155
156 void KVBucketTest::delete_item(uint16_t vbid, const StoredDocKey& key) {
157     uint64_t cas = 0;
158     EXPECT_EQ(ENGINE_SUCCESS,
159               store->deleteItem(key,
160                                 cas,
161                                 vbid,
162                                 cookie,
163                                 /*Item*/ nullptr,
164                                 /*itemMeta*/ nullptr,
165                                 /*mutation_descr_t*/ nullptr));
166 }
167
168 void KVBucketTest::evict_key(uint16_t vbid, const StoredDocKey& key) {
169     const char* msg;
170     EXPECT_EQ(ENGINE_SUCCESS, store->evictKey(key, vbid, &msg));
171     EXPECT_STREQ("Ejected.", msg);
172 }
173
174 GetValue KVBucketTest::getInternal(const StoredDocKey& key,
175                                    uint16_t vbucket,
176                                    const void* cookie,
177                                    vbucket_state_t allowedState,
178                                    get_options_t options) {
179     return store->getInternal(key, vbucket, cookie, allowedState, options);
180 }
181
182 void KVBucketTest::createAndScheduleItemPager() {
183     store->itemPagerTask = new ItemPager(engine.get(), engine->getEpStats());
184     ExecutorPool::get()->schedule(store->itemPagerTask);
185 }
186
187 void KVBucketTest::initializeExpiryPager() {
188     store->initializeExpiryPager(engine->getConfiguration());
189 }
190
191 /**
192  * Create a del_with_meta packet with the key/body (body can be empty)
193  */
194 std::vector<char> KVBucketTest::buildWithMetaPacket(
195         protocol_binary_command opcode,
196         protocol_binary_datatype_t datatype,
197         uint16_t vbucket,
198         uint32_t opaque,
199         uint64_t cas,
200         ItemMetaData metaData,
201         const std::string& key,
202         const std::string& body,
203         const std::vector<char>& emd,
204         int options) {
205     EXPECT_EQ(sizeof(protocol_binary_request_set_with_meta),
206               sizeof(protocol_binary_request_delete_with_meta));
207
208     size_t size = sizeof(protocol_binary_request_set_with_meta);
209     // body at least the meta
210     size_t extlen = (sizeof(uint32_t) * 2) + (sizeof(uint64_t) * 2);
211     size_t bodylen = extlen;
212     if (options) {
213         size += sizeof(uint32_t);
214         bodylen += sizeof(uint32_t);
215         extlen += sizeof(uint32_t);
216     }
217     if (!emd.empty()) {
218         EXPECT_TRUE(emd.size() < std::numeric_limits<uint16_t>::max());
219         size += sizeof(uint16_t) + emd.size();
220         bodylen += sizeof(uint16_t) + emd.size();
221         extlen += sizeof(uint16_t);
222     }
223     size += body.size();
224     bodylen += body.size();
225     size += key.size();
226     bodylen += key.size();
227
228     protocol_binary_request_set_with_meta header;
229     header.message.header.request.magic = PROTOCOL_BINARY_REQ;
230     header.message.header.request.opcode = opcode;
231     header.message.header.request.keylen = htons(key.size());
232     header.message.header.request.extlen = uint8_t(extlen);
233     header.message.header.request.datatype = datatype;
234     header.message.header.request.vbucket = htons(vbucket);
235     header.message.header.request.bodylen = htonl(bodylen);
236     header.message.header.request.opaque = opaque;
237     header.message.header.request.cas = htonll(cas);
238     header.message.body.flags = metaData.flags;
239     header.message.body.expiration = htonl(metaData.exptime);
240     header.message.body.seqno = htonll(metaData.revSeqno);
241     header.message.body.cas = htonll(metaData.cas);
242
243     std::vector<char> packet;
244     packet.reserve(size);
245     packet.insert(packet.end(),
246                   reinterpret_cast<char*>(&header),
247                   reinterpret_cast<char*>(&header) +
248                           sizeof(protocol_binary_request_set_with_meta));
249
250     if (options) {
251         options = htonl(options);
252         std::copy_n(reinterpret_cast<char*>(&options),
253                     sizeof(uint32_t),
254                     std::back_inserter(packet));
255     }
256
257     if (!emd.empty()) {
258         uint16_t emdSize = htons(emd.size());
259         std::copy_n(reinterpret_cast<char*>(&emdSize),
260                     sizeof(uint16_t),
261                     std::back_inserter(packet));
262     }
263
264     std::copy_n(key.c_str(), key.size(), std::back_inserter(packet));
265     std::copy_n(body.c_str(), body.size(), std::back_inserter(packet));
266     packet.insert(packet.end(), emd.begin(), emd.end());
267     return packet;
268 }
269
270 std::string KVBucketTest::createXattrValue(const std::string& body) {
271     cb::xattr::Blob blob;
272
273     // Add a few XAttrs
274     blob.set(to_const_byte_buffer("user"),
275              to_const_byte_buffer("{\"author\":\"bubba\"}"));
276     blob.set(to_const_byte_buffer("_sync"),
277              to_const_byte_buffer("{\"cas\":\"0xdeadbeefcafefeed\"}"));
278     blob.set(to_const_byte_buffer("meta"),
279              to_const_byte_buffer("{\"content-type\":\"text\"}"));
280
281     auto xattrValue = blob.finalize();
282
283     // append body to the xattrs and store in data
284     std::string data;
285     std::copy_n(xattrValue.buf, xattrValue.len, std::back_inserter(data));
286     std::copy_n(body.c_str(), body.size(), std::back_inserter(data));
287
288     return data;
289 }
290
291 bool KVBucketTest::addResponse(const void* k,
292                                uint16_t keylen,
293                                const void* ext,
294                                uint8_t extlen,
295                                const void* body,
296                                uint32_t bodylen,
297                                uint8_t datatype,
298                                uint16_t status,
299                                uint64_t pcas,
300                                const void* cookie) {
301     addResponseStatus = protocol_binary_response_status(status);
302     return true;
303 }
304
305 protocol_binary_response_status KVBucketTest::getAddResponseStatus(
306         protocol_binary_response_status newval) {
307     protocol_binary_response_status rv = addResponseStatus;
308     addResponseStatus = newval;
309     return rv;
310 }
311
312 protocol_binary_response_status KVBucketTest::addResponseStatus =
313         PROTOCOL_BINARY_RESPONSE_SUCCESS;
314
315
316 // getKeyStats tests //////////////////////////////////////////////////////////
317
318 // Check that keystats on resident items works correctly.
319 TEST_P(KVBucketParamTest, GetKeyStatsResident) {
320     key_stats kstats;
321
322     // Should start with key not existing.
323     EXPECT_EQ(ENGINE_KEY_ENOENT,
324               store->getKeyStats(makeStoredDocKey("key"),
325                                  0,
326                                  cookie,
327                                  kstats,
328                                  WantsDeleted::No));
329
330     store_item(0, makeStoredDocKey("key"), "value");
331     EXPECT_EQ(ENGINE_SUCCESS,
332               store->getKeyStats(makeStoredDocKey("key"),
333                                  0,
334                                  cookie,
335                                  kstats,
336                                  WantsDeleted::No))
337             << "Expected to get key stats on existing item";
338     EXPECT_EQ(vbucket_state_active, kstats.vb_state);
339     EXPECT_FALSE(kstats.logically_deleted);
340 }
341
342 // Create then delete an item, checking we get keyStats reporting the item as
343 // deleted.
344 TEST_P(KVBucketParamTest, GetKeyStatsDeleted) {
345     auto& kvbucket = *engine->getKVBucket();
346     key_stats kstats;
347
348     store_item(0, makeStoredDocKey("key"), "value");
349     delete_item(vbid, makeStoredDocKey("key"));
350
351     // Should get ENOENT if we don't ask for deleted items.
352     EXPECT_EQ(ENGINE_KEY_ENOENT,
353               kvbucket.getKeyStats(makeStoredDocKey("key"),
354                                    0,
355                                    cookie,
356                                    kstats,
357                                    WantsDeleted::No));
358
359     // Should get success (and item flagged as deleted) if we ask for deleted
360     // items.
361     EXPECT_EQ(ENGINE_SUCCESS,
362               kvbucket.getKeyStats(makeStoredDocKey("key"),
363                                    0,
364                                    cookie,
365                                    kstats,
366                                    WantsDeleted::Yes));
367     EXPECT_EQ(vbucket_state_active, kstats.vb_state);
368     EXPECT_TRUE(kstats.logically_deleted);
369 }
370
371 // Check incorrect vbucket returns not-my-vbucket.
372 TEST_P(KVBucketParamTest, GetKeyStatsNMVB) {
373     auto& kvbucket = *engine->getKVBucket();
374     key_stats kstats;
375
376     EXPECT_EQ(ENGINE_NOT_MY_VBUCKET,
377               kvbucket.getKeyStats(makeStoredDocKey("key"),
378                                    1,
379                                    cookie,
380                                    kstats,
381                                    WantsDeleted::No));
382 }
383
384 // Replace tests //////////////////////////////////////////////////////////////
385
386 // Test replace against a non-existent key.
387 TEST_P(KVBucketParamTest, ReplaceENOENT) {
388     // Should start with key not existing (and hence cannot replace).
389     auto item = make_item(vbid, makeStoredDocKey("key"), "value");
390     EXPECT_EQ(ENGINE_KEY_ENOENT, store->replace(item, cookie));
391 }
392
393 // Create then delete an item, checking replace reports ENOENT.
394 TEST_P(KVBucketParamTest, ReplaceDeleted) {
395     store_item(vbid, makeStoredDocKey("key"), "value");
396     delete_item(vbid, makeStoredDocKey("key"));
397
398     // Replace should fail.
399     auto item = make_item(vbid, makeStoredDocKey("key"), "value2");
400     EXPECT_EQ(ENGINE_KEY_ENOENT, store->replace(item, cookie));
401 }
402
403 // Check incorrect vbucket returns not-my-vbucket.
404 TEST_P(KVBucketParamTest, ReplaceNMVB) {
405     auto item = make_item(vbid + 1, makeStoredDocKey("key"), "value2");
406     EXPECT_EQ(ENGINE_NOT_MY_VBUCKET, store->replace(item, cookie));
407 }
408
409 // Check pending vbucket returns EWOULDBLOCK.
410 TEST_P(KVBucketParamTest, ReplacePendingVB) {
411     store->setVBucketState(vbid, vbucket_state_pending, false);
412     auto item = make_item(vbid, makeStoredDocKey("key"), "value2");
413     EXPECT_EQ(ENGINE_EWOULDBLOCK, store->replace(item, cookie));
414 }
415
416 // Set tests //////////////////////////////////////////////////////////////////
417
418 // Test CAS set against a non-existent key
419 TEST_P(KVBucketParamTest, SetCASNonExistent) {
420     // Create an item with a non-zero CAS.
421     auto item = make_item(vbid, makeStoredDocKey("key"), "value");
422     item.setCas();
423     ASSERT_NE(0, item.getCas());
424
425     // Should get ENOENT as we should immediately know (either from metadata
426     // being resident, or by bloomfilter) that key doesn't exist.
427     EXPECT_EQ(ENGINE_KEY_ENOENT, store->set(item, cookie));
428 }
429
430 // Test CAS set against a deleted item
431 TEST_P(KVBucketParamTest, SetCASDeleted) {
432     auto key = makeStoredDocKey("key");
433     auto item = make_item(vbid, key, "value");
434
435     // Store item
436     EXPECT_EQ(ENGINE_SUCCESS, store->set(item, cookie));
437
438     // Delete item
439     uint64_t cas = 0;
440     EXPECT_EQ(ENGINE_SUCCESS,
441               store->deleteItem(key,
442                                 cas,
443                                 vbid,
444                                 cookie,
445                       /*Item*/ nullptr,
446                       /*itemMeta*/ nullptr,
447                       /*mutation_descr_t*/ nullptr));
448
449     if (engine->getConfiguration().getBucketType() == "persistent") {
450         // Trigger a flush to disk.
451         flush_vbucket_to_disk(vbid);
452     }
453
454     // check we have the cas
455     ASSERT_NE(0, cas);
456
457     auto item2 = make_item(vbid, key, "value2");
458     item2.setCas(cas);
459
460     // Store item
461     if (engine->getConfiguration().getItemEvictionPolicy() ==
462                "full_eviction") {
463         EXPECT_EQ(ENGINE_EWOULDBLOCK, store->set(item2, cookie));
464
465         // Manually run the bgfetch task.
466         MockGlobalTask mockTask(engine->getTaskable(),
467                                 TaskId::MultiBGFetcherTask);
468         store->getVBucket(vbid)->getShard()->getBgFetcher()->run(&mockTask);
469     }
470
471     EXPECT_EQ(ENGINE_KEY_ENOENT, store->set(item2, cookie));
472 }
473
474 // Add tests //////////////////////////////////////////////////////////////////
475
476 // Test successful add
477 TEST_P(KVBucketParamTest, Add) {
478     auto item = make_item(vbid, makeStoredDocKey("key"), "value");
479     EXPECT_EQ(ENGINE_SUCCESS, store->add(item, nullptr));
480 }
481
482 // Check incorrect vbucket returns not-my-vbucket.
483 TEST_P(KVBucketParamTest, AddNMVB) {
484     auto item = make_item(vbid + 1, makeStoredDocKey("key"), "value2");
485     EXPECT_EQ(ENGINE_NOT_MY_VBUCKET, store->add(item, cookie));
486 }
487
488 // SetWithMeta tests //////////////////////////////////////////////////////////
489
490 // Test basic setWithMeta
491 TEST_P(KVBucketParamTest, SetWithMeta) {
492     auto item = make_item(vbid, makeStoredDocKey("key"), "value");
493     item.setCas();
494     uint64_t seqno;
495     EXPECT_EQ(ENGINE_SUCCESS,
496               store->setWithMeta(item,
497                                  0,
498                                  &seqno,
499                                  cookie,
500                                  /*force*/ false,
501                                  /*allowExisting*/ false));
502 }
503
504 // Test setWithMeta with a conflict with an existing item.
505 TEST_P(KVBucketParamTest, SetWithMeta_Conflicted) {
506     auto item = make_item(vbid, makeStoredDocKey("key"), "value");
507     EXPECT_EQ(ENGINE_SUCCESS, store->set(item, nullptr));
508
509     uint64_t seqno;
510     // Attempt to set with the same rev Seqno - should get EEXISTS.
511     EXPECT_EQ(ENGINE_KEY_EEXISTS,
512               store->setWithMeta(item,
513                                  item.getCas(),
514                                  &seqno,
515                                  cookie,
516                                  /*force*/ false,
517                                  /*allowExisting*/ true));
518 }
519
520 // Test setWithMeta replacing existing item
521 TEST_P(KVBucketParamTest, SetWithMeta_Replace) {
522     auto item = make_item(vbid, makeStoredDocKey("key"), "value");
523     EXPECT_EQ(ENGINE_SUCCESS, store->set(item, nullptr));
524
525     // Increase revSeqno so conflict resolution doesn't fail.
526     item.setRevSeqno(item.getRevSeqno() + 1);
527     uint64_t seqno;
528     // Should get EEXISTS if we don't force (and use wrong CAS).
529     EXPECT_EQ(ENGINE_KEY_EEXISTS,
530               store->setWithMeta(item,
531                                  item.getCas() + 1,
532                                  &seqno,
533                                  cookie,
534                                  /*force*/ false,
535                                  /*allowExisting*/ true));
536
537     // Should succeed with correct CAS, and different RevSeqno.
538     EXPECT_EQ(ENGINE_SUCCESS,
539               store->setWithMeta(item,
540                                  item.getCas(),
541                                  &seqno,
542                                  cookie,
543                                  /*force*/ false,
544                                  /*allowExisting*/ true));
545 }
546
547 // Test forced setWithMeta
548 TEST_P(KVBucketParamTest, SetWithMeta_Forced) {
549     auto item = make_item(vbid, makeStoredDocKey("key"), "value");
550     item.setCas();
551     uint64_t seqno;
552     EXPECT_EQ(ENGINE_SUCCESS,
553               store->setWithMeta(item,
554                                  0,
555                                  &seqno,
556                                  cookie,
557                                  /*force*/ true,
558                                  /*allowExisting*/ false));
559 }
560
561 // MB and test was raised because a few commits back this was broken but no
562 // existing test covered the case. I.e. run this test  against 0810540 and it
563 // fails, but now fixed
564 TEST_P(KVBucketParamTest, mb22824) {
565     auto key = makeStoredDocKey("key");
566
567     // Store key and force expiry
568     store_item(0, key, "value", 1);
569     TimeTraveller docBrown(20);
570
571     uint32_t deleted = false;
572     ItemMetaData itemMeta1;
573     uint8_t datatype = PROTOCOL_BINARY_RAW_BYTES;
574     EXPECT_EQ(ENGINE_SUCCESS,
575               store->getMetaData(
576                       key, vbid, cookie, false, itemMeta1, deleted, datatype));
577
578     uint64_t cas = 0;
579     ItemMetaData itemMeta2;
580     EXPECT_EQ(ENGINE_KEY_ENOENT,
581               store->deleteItem(key,
582                                 cas,
583                                 vbid,
584                                 cookie,
585                                 /*Item*/ nullptr,
586                                 &itemMeta2,
587                                 /*mutation_descr_t*/ nullptr));
588
589     // Should be getting the same CAS from the failed delete as getMetaData
590     EXPECT_EQ(itemMeta1.cas, itemMeta2.cas);
591 }
592
593 // Test cases which run for EP (Full and Value eviction) and Ephemeral
594 INSTANTIATE_TEST_CASE_P(EphemeralOrPersistent,
595                         KVBucketParamTest,
596                         ::testing::Values("item_eviction_policy=value_only",
597                                           "item_eviction_policy=full_eviction",
598                                           "bucket_type=ephemeral"),
599                         [](const ::testing::TestParamInfo<std::string>& info) {
600                             return info.param.substr(info.param.find('=') + 1);
601                         });
602
603 const char KVBucketTest::test_dbname[] = "ep_engine_ep_unit_tests_db";