MB-24221: Don't fetch deleted values as part of get_if
[ep-engine.git] / tests / ep_testsuite_basic.cc
1 /* -*- MODE: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 /*
3  *     Copyright 2016 Couchbase, Inc
4  *
5  *   Licensed under the Apache License, Version 2.0 (the "License");
6  *   you may not use this file except in compliance with the License.
7  *   You may obtain a copy of the License at
8  *
9  *       http://www.apache.org/licenses/LICENSE-2.0
10  *
11  *   Unless required by applicable law or agreed to in writing, software
12  *   distributed under the License is distributed on an "AS IS" BASIS,
13  *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  *   See the License for the specific language governing permissions and
15  *   limitations under the License.
16  */
17
18 /*
19  * Testsuite for 'basic' key-value functionality in ep-engine.
20  */
21
22 #include "config.h"
23
24 #include "ep_test_apis.h"
25 #include "ep_testsuite_common.h"
26
27 #include <platform/cb_malloc.h>
28 #include <platform/cbassert.h>
29 #include <JSON_checker.h>
30
31 #include <array>
32 #include <memcached/types.h>
33
34 #define WHITESPACE_DB "whitespace sucks.db"
35
36 // Types //////////////////////////////////////////////////////////////////////
37
38
39 // Helper functions ///////////////////////////////////////////////////////////
40
41 static bool epsilon(int val, int target, int ep=5) {
42     return abs(val - target) < ep;
43 }
44
45
46 // Testcases //////////////////////////////////////////////////////////////////
47
48 static enum test_result test_alloc_limit(ENGINE_HANDLE *h,
49                                          ENGINE_HANDLE_V1 *h1) {
50     item *it = NULL;
51     ENGINE_ERROR_CODE rv;
52
53     rv = allocate(h, h1, NULL, &it, "key", 20 * 1024 * 1024, 0, 0,
54                       PROTOCOL_BINARY_RAW_BYTES, 0);
55     checkeq(ENGINE_SUCCESS, rv, "Allocated 20MB item");
56     h1->release(h, NULL, it);
57
58     rv = allocate(h, h1, NULL, &it, "key", (20 * 1024 * 1024) + 1, 0, 0,
59                       PROTOCOL_BINARY_RAW_BYTES, 0);
60     checkeq(ENGINE_E2BIG, rv, "Object too big");
61
62     return SUCCESS;
63 }
64
65 static enum test_result test_memory_tracking(ENGINE_HANDLE *h,
66                                              ENGINE_HANDLE_V1 *h1) {
67     // Need memory tracker to be able to check our memory usage.
68     std::string tracker = get_str_stat(h, h1, "ep_mem_tracker_enabled");
69     if (tracker == "true") {
70         return SUCCESS;
71     } else {
72         std::cerr << "Memory tracker not enabled ...";
73         return SKIPPED;
74     }
75 }
76
77 // TODO: Ephemeral: Should refactor this into storage and memory tests,
78 // enable the memory checks for ephemeral.
79 static enum test_result test_memory_limit(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
80     checkeq(10240000, get_int_stat(h, h1, "ep_max_size"), "Max size not at 10MB");
81     set_param(h, h1, protocol_binary_engine_param_flush, "mutation_mem_threshold", "95");
82     if (isPersistentBucket(h, h1)) {
83         wait_for_stat_change(h, h1,"ep_db_data_size", 0);
84     }
85     check(get_int_stat(h, h1, "ep_oom_errors") == 0 &&
86           get_int_stat(h, h1, "ep_tmp_oom_errors") == 0, "Expected no OOM errors.");
87
88     size_t vlen = 4 * 1024 * 1024;
89     char *data = new char[vlen + 1]; // +1 for terminating '\0' byte
90     cb_assert(data);
91     memset(data, 'x', vlen);
92     data[vlen] = '\0';
93
94     item *i = NULL;
95     // So if we add an item,
96     checkeq(ENGINE_SUCCESS,
97             store(h, h1, NULL, OPERATION_SET, "key", data, &i),
98             "store failure");
99     check_key_value(h, h1, "key", data, vlen);
100     h1->release(h, NULL, i);
101
102     wait_for_flusher_to_settle(h, h1);
103
104     // Set max_size equal to used memory, so that the next store operation
105     // would throw an ENOMEM/ETMPFAIL.
106     int new_max_size = get_int_stat(h, h1, "mem_used");
107     set_param(h, h1, protocol_binary_engine_param_flush, "max_size",
108               std::to_string(new_max_size).c_str());
109
110     int num_pager_runs = get_int_stat(h, h1, "ep_num_pager_runs");
111     int num_ejects = get_int_stat(h, h1, "ep_num_value_ejects");
112
113     i = NULL;
114     // There should be no room for another.
115     ENGINE_ERROR_CODE second = store(h, h1, NULL, OPERATION_SET, "key2", data, &i);
116     check(second == ENGINE_ENOMEM || second == ENGINE_TMPFAIL,
117           "should have failed second set");
118     if (i) {
119         h1->release(h, NULL, i);
120         i = NULL;
121     }
122     check(get_int_stat(h, h1, "ep_oom_errors") == 1 ||
123           get_int_stat(h, h1, "ep_tmp_oom_errors") == 1, "Expected an OOM error.");
124
125     // Consider the number of ejects to estimate the outcome of the next
126     // store operation, as the previous one that failed because of ENOMEM
127     // would've woken up the item-pager.
128     bool opToSucceed = false;
129     if (get_int_stat(h, h1, "ep_num_pager_runs") > num_pager_runs &&
130         get_int_stat(h, h1, "ep_num_value_ejects") > num_ejects) {
131         opToSucceed = true;
132     }
133     ENGINE_ERROR_CODE overwrite = store(h, h1, NULL, OPERATION_SET, "key", data, &i);
134     if (opToSucceed) {
135         checkeq(ENGINE_SUCCESS,
136                 overwrite,
137                 "Item pager cleared up memory but this op still failed");
138     } else {
139         check(overwrite == ENGINE_ENOMEM || overwrite == ENGINE_TMPFAIL,
140               "should have failed second override");
141     }
142
143     if (i) {
144         h1->release(h, NULL, i);
145         i = NULL;
146     }
147
148     if (overwrite != ENGINE_SUCCESS) {
149         check(get_int_stat(h, h1, "ep_oom_errors") == 2 ||
150               get_int_stat(h, h1, "ep_tmp_oom_errors") == 2,
151               "Expected another OOM error.");
152     }
153
154     check_key_value(h, h1, "key", data, vlen);
155     check(ENGINE_SUCCESS != verify_key(h, h1, "key2"), "Expected a failure in GET");
156     int itemsRemoved = get_int_stat(h, h1, "ep_items_rm_from_checkpoints");
157     // Until we remove that item
158     checkeq(ENGINE_SUCCESS, del(h, h1, "key", 0, 0), "Failed remove with value.");
159     checkeq(ENGINE_KEY_ENOENT, verify_key(h, h1, "key"), "Expected missing key");
160     testHarness.time_travel(65);
161     wait_for_stat_change(h, h1, "ep_items_rm_from_checkpoints", itemsRemoved);
162
163     wait_for_flusher_to_settle(h, h1);
164
165     checkeq(store(h, h1, NULL, OPERATION_SET, "key2", "somevalue2", &i),
166             ENGINE_SUCCESS,
167             "should have succeded on the last set");
168     check_key_value(h, h1, "key2", "somevalue2", 10);
169     h1->release(h, NULL, i);
170     delete []data;
171     return SUCCESS;
172 }
173
174 static enum test_result test_max_size_and_water_marks_settings(
175                                         ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
176     checkeq(1000, get_int_stat(h, h1, "ep_max_size"), "Incorrect initial size.");
177     check(epsilon(get_int_stat(h, h1, "ep_mem_low_wat"), 750),
178           "Incorrect initial low wat.");
179     check(epsilon(get_int_stat(h, h1, "ep_mem_high_wat"), 850),
180           "Incorrect initial high wat.");
181     check((get_float_stat(h, h1, "ep_mem_low_wat_percent") == (float)0.75),
182           "Incorrect initial low wat. percent");
183     check((get_float_stat(h, h1, "ep_mem_high_wat_percent") == (float)0.85),
184           "Incorrect initial high wat. percent");
185
186     set_param(h, h1, protocol_binary_engine_param_flush, "max_size", "1000000");
187
188     checkeq(1000000, get_int_stat(h, h1, "ep_max_size"),
189             "Incorrect new size.");
190     check(epsilon(get_int_stat(h, h1, "ep_mem_low_wat"), 750000),
191           "Incorrect larger low wat.");
192     check(epsilon(get_int_stat(h, h1, "ep_mem_high_wat"), 850000),
193           "Incorrect larger high wat.");
194     check((get_float_stat(h, h1, "ep_mem_low_wat_percent") == (float)0.75),
195           "Incorrect larger low wat. percent");
196     check((get_float_stat(h, h1, "ep_mem_high_wat_percent") == (float)0.85),
197           "Incorrect larger high wat. percent");
198
199     set_param(h, h1, protocol_binary_engine_param_flush, "mem_low_wat", "700000");
200     set_param(h, h1, protocol_binary_engine_param_flush, "mem_high_wat", "800000");
201
202     checkeq(700000, get_int_stat(h, h1, "ep_mem_low_wat"),
203             "Incorrect even larger low wat.");
204     checkeq(800000, get_int_stat(h, h1, "ep_mem_high_wat"),
205             "Incorrect even larger high wat.");
206     check((get_float_stat(h, h1, "ep_mem_low_wat_percent") == (float)0.7),
207           "Incorrect even larger low wat. percent");
208     check((get_float_stat(h, h1, "ep_mem_high_wat_percent") == (float)0.8),
209           "Incorrect even larger high wat. percent");
210
211     set_param(h, h1, protocol_binary_engine_param_flush, "max_size", "100");
212
213     checkeq(100, get_int_stat(h, h1, "ep_max_size"),
214             "Incorrect smaller size.");
215     check(epsilon(get_int_stat(h, h1, "ep_mem_low_wat"), 70),
216           "Incorrect smaller low wat.");
217     check(epsilon(get_int_stat(h, h1, "ep_mem_high_wat"), 80),
218           "Incorrect smaller high wat.");
219     check((get_float_stat(h, h1, "ep_mem_low_wat_percent") == (float)0.7),
220           "Incorrect smaller low wat. percent");
221     check((get_float_stat(h, h1, "ep_mem_high_wat_percent") == (float)0.8),
222           "Incorrect smaller high wat. percent");
223
224     set_param(h, h1, protocol_binary_engine_param_flush, "mem_low_wat", "50");
225     set_param(h, h1, protocol_binary_engine_param_flush, "mem_high_wat", "70");
226
227     checkeq(50, get_int_stat(h, h1, "ep_mem_low_wat"),
228             "Incorrect even smaller low wat.");
229     checkeq(70, get_int_stat(h, h1, "ep_mem_high_wat"),
230             "Incorrect even smaller high wat.");
231     check((get_float_stat(h, h1, "ep_mem_low_wat_percent") == (float)0.5),
232           "Incorrect even smaller low wat. percent");
233     check((get_float_stat(h, h1, "ep_mem_high_wat_percent") == (float)0.7),
234           "Incorrect even smaller high wat. percent");
235
236     testHarness.reload_engine(&h, &h1,
237                               testHarness.engine_path,
238                               testHarness.get_current_testcase()->cfg,
239                               true, true);
240     wait_for_warmup_complete(h, h1);
241
242     checkeq(1000, get_int_stat(h, h1, "ep_max_size"),
243             "Incorrect initial size.");
244     check(epsilon(get_int_stat(h, h1, "ep_mem_low_wat"), 750),
245           "Incorrect intial low wat.");
246     check(epsilon(get_int_stat(h, h1, "ep_mem_high_wat"), 850),
247           "Incorrect initial high wat.");
248     check((get_float_stat(h, h1, "ep_mem_low_wat_percent") == (float)0.75),
249           "Incorrect initial low wat. percent");
250     check((get_float_stat(h, h1, "ep_mem_high_wat_percent") == (float)0.85),
251           "Incorrect initial high wat. percent");
252
253     return SUCCESS;
254 }
255
256 static enum test_result test_whitespace_db(ENGINE_HANDLE *h,
257                                            ENGINE_HANDLE_V1 *h1) {
258     vals.clear();
259     checkeq(ENGINE_SUCCESS,
260             h1->get_stats(h, NULL, NULL, 0, add_stats),
261            "Failed to get stats.");
262
263     std::string dbname;
264     std::string policy;
265     policy = isPersistentBucket(h, h1)
266                      ? vals.find("ep_item_eviction_policy")->second
267                      : "ephemeral";
268     dbname.assign(policy + std::string(WHITESPACE_DB));
269
270     std::string oldparam("dbname=" + vals["ep_dbname"]);
271     std::string newparam("dbname=" + dbname);
272     std::string config = testHarness.get_current_testcase()->cfg;
273     std::string::size_type found = config.find(oldparam);
274     if (found != config.npos) {
275         config.replace(found, oldparam.size(), newparam);
276     }
277     testHarness.reload_engine(&h, &h1,
278                               testHarness.engine_path,
279                               config.c_str(),
280                               true, false);
281     wait_for_warmup_complete(h, h1);
282
283     vals.clear();
284     checkeq(ENGINE_SUCCESS,
285             h1->get_stats(h, NULL, NULL, 0, add_stats),
286            "Failed to get stats.");
287
288     if (vals["ep_dbname"] != dbname) {
289         std::cerr << "Expected dbname = '" << dbname << "'"
290                   << ", got '" << vals["ep_dbname"] << "'" << std::endl;
291         return FAIL;
292     }
293
294     check(access(dbname.c_str(), F_OK) != -1, "I expected the whitespace db to exist");
295     return SUCCESS;
296 }
297
298 static enum test_result test_get_miss(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
299     checkeq(ENGINE_KEY_ENOENT, verify_key(h, h1, "k"), "Expected miss.");
300     return SUCCESS;
301 }
302
303 static enum test_result test_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
304     item *i = NULL;
305     item_info info;
306     uint64_t vb_uuid = 0, high_seqno = 0;
307     const int num_sets = 5, num_keys = 4;
308
309     std::string key_arr[num_keys] = { "dummy_key",
310                                       "checkpoint_start",
311                                       "checkpoint_end",
312                                       "key" };
313
314
315     for (int k = 0; k < num_keys; k++) {
316         for (int j = 0; j < num_sets; j++) {
317             memset(&info, 0, sizeof(info));
318             vb_uuid = get_ull_stat(h, h1, "vb_0:0:id", "failovers");
319             high_seqno = get_ull_stat(h, h1, "vb_0:high_seqno",
320                                       "vbucket-seqno");
321
322             std::string err_str_store("Error setting " + key_arr[k]);
323             checkeq(ENGINE_SUCCESS,
324                     store(h, h1, NULL, OPERATION_SET, key_arr[k].c_str(),
325                           "somevalue", &i),
326                     err_str_store.c_str());
327             h1->release(h, NULL, i);
328
329             std::string err_str_get_item_info("Error getting " + key_arr[k]);
330             checkeq(true, get_item_info(h, h1, &info, key_arr[k].c_str()),
331                   err_str_get_item_info.c_str());
332
333             std::string err_str_vb_uuid("Expected valid vbucket uuid for " +
334                                         key_arr[k]);
335             checkeq(vb_uuid, info.vbucket_uuid, err_str_vb_uuid.c_str());
336
337             std::string err_str_seqno("Expected valid sequence number for " +
338                                         key_arr[k]);
339             checkeq(high_seqno + 1, info.seqno, err_str_seqno.c_str());
340         }
341     }
342
343     if (isPersistentBucket(h, h1)) {
344         wait_for_flusher_to_settle(h, h1);
345
346         std::stringstream error1, error2;
347         error1 << "Expected ep_total_persisted >= num_keys (" << num_keys << ")";
348         error2 << "Expected ep_total_persisted <= num_sets*num_keys ("
349                << num_sets*num_keys << ")";
350
351         // The flusher could of ran > 1 times. We can only assert
352         // that we persisted between num_keys and upto num_keys*num_sets
353         check(get_int_stat(h, h1, "ep_total_persisted") >= num_keys,
354               error1.str().c_str());
355         check(get_int_stat(h, h1, "ep_total_persisted") <= num_sets*num_keys,
356               error2.str().c_str());
357     }
358     return SUCCESS;
359 }
360
361 extern "C" {
362     static void conc_del_set_thread(void *arg) {
363         struct handle_pair *hp = static_cast<handle_pair *>(arg);
364         item *it = NULL;
365
366         for (int i = 0; i < 5000; ++i) {
367             store(hp->h, hp->h1, NULL, OPERATION_ADD,
368                   "key", "somevalue", &it);
369             hp->h1->release(hp->h, NULL, it);
370             checkeq(ENGINE_SUCCESS,
371                     store(hp->h, hp->h1, NULL, OPERATION_SET,
372                           "key", "somevalue", &it),
373                     "Error setting.");
374             hp->h1->release(hp->h, NULL, it);
375             // Ignoring the result here -- we're racing.
376             del(hp->h, hp->h1, "key", 0, 0);
377         }
378     }
379 }
380
381 static enum test_result test_conc_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
382
383     const int n_threads = 8;
384     cb_thread_t threads[n_threads];
385     struct handle_pair hp = {h, h1};
386
387     wait_for_persisted_value(h, h1, "key", "value1");
388
389     for (int i = 0; i < n_threads; i++) {
390         int r = cb_create_thread(&threads[i], conc_del_set_thread, &hp, 0);
391         cb_assert(r == 0);
392     }
393
394     for (int i = 0; i < n_threads; i++) {
395         int r = cb_join_thread(threads[i]);
396         cb_assert(r == 0);
397     }
398
399     if (isWarmupEnabled(h, h1)) {
400         wait_for_flusher_to_settle(h, h1);
401
402         testHarness.reload_engine(&h, &h1,
403                                   testHarness.engine_path,
404                                   testHarness.get_current_testcase()->cfg,
405                                   true, false);
406         wait_for_warmup_complete(h, h1);
407
408         cb_assert(0 == get_int_stat(h, h1, "ep_warmup_dups"));
409     }
410
411     return SUCCESS;
412 }
413
414 struct multi_set_args {
415     ENGINE_HANDLE *h;
416     ENGINE_HANDLE_V1 *h1;
417     std::string prefix;
418     int count;
419 };
420
421 extern "C" {
422     static void multi_set_thread(void *arg) {
423         struct multi_set_args *msa = static_cast<multi_set_args *>(arg);
424
425         for (int i = 0; i < msa->count; i++) {
426             item *it = NULL;
427             std::stringstream s;
428             s << msa->prefix << i;
429             std::string key(s.str());
430             checkeq(ENGINE_SUCCESS,
431                     store(msa->h, msa->h1, NULL, OPERATION_SET,
432                           key.c_str(), "somevalue", &it),
433                     "Set failure!");
434             msa->h1->release(msa->h, NULL, it);
435         }
436     }
437 }
438
439 static enum test_result test_multi_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
440
441     cb_thread_t thread1, thread2;
442     struct multi_set_args msa1, msa2;
443     msa1.h = h;
444     msa1.h1 = h1;
445     msa1.prefix = "ONE_";
446     msa1.count = 50000;
447     cb_assert(cb_create_thread(&thread1, multi_set_thread, &msa1, 0) == 0);
448
449     msa2.h = h;
450     msa2.h1 = h1;
451     msa2.prefix = "TWO_";
452     msa2.count = 50000;
453     cb_assert(cb_create_thread(&thread2, multi_set_thread, &msa2, 0) == 0);
454
455     cb_assert(cb_join_thread(thread1) == 0);
456     cb_assert(cb_join_thread(thread2) == 0);
457
458     wait_for_flusher_to_settle(h, h1);
459
460     checkeq(100000, get_int_stat(h, h1, "curr_items"),
461             "Mismatch in number of items inserted");
462     checkeq(100000, get_int_stat(h, h1, "vb_0:high_seqno", "vbucket-seqno"),
463             "Unexpected high sequence number");
464
465     return SUCCESS;
466 }
467
468 static enum test_result test_set_get_hit(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
469     item *i = NULL;
470     checkeq(ENGINE_SUCCESS,
471             store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i),
472             "store failure");
473     check_key_value(h, h1, "key", "somevalue", 9);
474     h1->release(h, NULL, i);
475     return SUCCESS;
476 }
477
478 static enum test_result test_getl_delete_with_cas(ENGINE_HANDLE *h,
479                                                   ENGINE_HANDLE_V1 *h1) {
480     item *itm = NULL;
481     checkeq(ENGINE_SUCCESS,
482             store(h, h1, NULL, OPERATION_SET, "key", "value", &itm),
483             "Failed to set key");
484     h1->release(h, NULL, itm);
485
486     item* locked = nullptr;
487     checkeq(ENGINE_SUCCESS,
488             getl(h, h1, nullptr, &locked, "key", 0, 15),
489             "Expected getl to succeed on key");
490     item_info info;
491     check(h1->get_item_info(h, nullptr, locked, &info),
492           "Failed to get item info");
493
494     checkeq(ENGINE_SUCCESS, del(h, h1, "key", info.cas, 0), "Expected SUCCESS");
495     h1->release(h, nullptr, locked);
496
497     return SUCCESS;
498 }
499
500 static enum test_result test_getl_delete_with_bad_cas(ENGINE_HANDLE *h,
501                                                       ENGINE_HANDLE_V1 *h1) {
502     item *itm = NULL;
503     checkeq(ENGINE_SUCCESS,
504             store(h, h1, NULL, OPERATION_SET,
505                   "key", "value", &itm),
506             "Failed to set key");
507     h1->release(h, NULL, itm);
508
509     uint64_t cas = last_cas;
510     item* locked = nullptr;
511     checkeq(ENGINE_SUCCESS,
512             getl(h, h1, nullptr, &locked, "key", 0, 15),
513             "Expected getl to succeed on key");
514     h1->release(h, nullptr, locked);
515
516     checkeq(ENGINE_LOCKED_TMPFAIL, del(h, h1, "key", cas, 0), "Expected TMPFAIL");
517
518     return SUCCESS;
519 }
520
521 static enum test_result test_getl_set_del_with_meta(ENGINE_HANDLE *h,
522                                                     ENGINE_HANDLE_V1 *h1) {
523     item *itm = NULL;
524     const char *key = "key";
525     const char *val = "value";
526     const char *newval = "newvalue";
527     checkeq(ENGINE_SUCCESS,
528             store(h, h1, NULL, OPERATION_SET, key, val, &itm),
529             "Failed to set key");
530     h1->release(h, NULL, itm);
531
532     item* locked = nullptr;
533     checkeq(ENGINE_SUCCESS, getl(h, h1, nullptr, &locked, key, 0, 15),
534           "Expected getl to succeed on key");
535     h1->release(h, nullptr, locked);
536
537     check(get_meta(h, h1, key), "Expected to get meta");
538
539     //init some random metadata
540     ItemMetaData itm_meta;
541     itm_meta.revSeqno = 10;
542     itm_meta.cas = 0xdeadbeef;
543     itm_meta.exptime = time(NULL) + 300;
544     itm_meta.flags = 0xdeadbeef;
545
546     //do a set with meta
547     set_with_meta(h, h1, key, strlen(key), newval, strlen(newval), 0,
548                   &itm_meta, last_cas);
549     checkeq(PROTOCOL_BINARY_RESPONSE_LOCKED, last_status.load(),
550           "Expected item to be locked");
551
552     //do a del with meta
553     del_with_meta(h, h1, key, strlen(key), 0, &itm_meta, last_cas);
554     checkeq(PROTOCOL_BINARY_RESPONSE_LOCKED, last_status.load(),
555           "Expected item to be locked");
556     return SUCCESS;
557 }
558
559 static enum test_result test_getl(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
560     const char *key = "k1";
561     uint16_t vbucketId = 0;
562     uint32_t expiration = 25;
563
564     item* locked;
565     checkeq(ENGINE_KEY_ENOENT,
566             getl(h, h1, nullptr, &locked, key, vbucketId, expiration),
567           "expected the key to be missing...");
568
569     item *i = NULL;
570     checkeq(ENGINE_SUCCESS,
571             store(h, h1, NULL, OPERATION_SET, key, "{\"lock\":\"data\"}",
572                   &i, 0, vbucketId, 3600, PROTOCOL_BINARY_DATATYPE_JSON),
573             "Failed to store an item.");
574     h1->release(h, NULL, i);
575
576     /* retry getl, should succeed */
577     checkeq(ENGINE_SUCCESS,
578             getl(h, h1, nullptr, &locked, key, vbucketId, expiration),
579             "Expected to be able to getl on first try");
580
581     item_info info;
582     check(h1->get_item_info(h, nullptr, locked, &info),
583           "Failed to get item info");
584
585     checkeq(std::string{"{\"lock\":\"data\"}"},
586             std::string((const char*)info.value[0].iov_base,
587                         info.value[0].iov_len),
588             "Body was malformed.");
589     checkeq(static_cast<uint8_t>(PROTOCOL_BINARY_DATATYPE_JSON),
590             info.datatype,
591             "Expected datatype to be JSON");
592     h1->release(h, nullptr, locked);
593
594     /* wait 16 seconds */
595     testHarness.time_travel(16);
596
597     /* lock's taken so this should fail */
598     checkeq(ENGINE_TMPFAIL,
599             getl(h, h1, nullptr, &locked, key, vbucketId, expiration),
600             "Expected to fail getl on second try");
601
602     checkne(ENGINE_SUCCESS,
603             store(h, h1, NULL, OPERATION_SET, key, "lockdata2", &i, 0,
604                   vbucketId),
605             "Should have failed to store an item.");
606     h1->release(h, NULL, i);
607
608     /* wait another 10 seconds */
609     testHarness.time_travel(10);
610
611     /* retry set, should succeed */
612     checkeq(ENGINE_SUCCESS,
613             store(h, h1, NULL, OPERATION_SET, key, "lockdata", &i, 0, vbucketId),
614             "Failed to store an item.");
615     h1->release(h, NULL, i);
616
617     /* point to wrong vbucket, to test NOT_MY_VB response */
618     checkeq(ENGINE_NOT_MY_VBUCKET,
619             getl(h, h1, nullptr, &locked, key, 10, expiration),
620             "Should have received not my vbucket response");
621
622     /* acquire lock, should succeed */
623     checkeq(ENGINE_SUCCESS,
624             getl(h, h1, nullptr, &locked, key, vbucketId, expiration),
625             "Aquire lock should have succeeded");
626     check(h1->get_item_info(h, nullptr, locked, &info),
627           "Failed to get item info");
628     checkeq(static_cast<uint8_t>(PROTOCOL_BINARY_RAW_BYTES), info.datatype,
629             "Expected datatype to be RAW BYTES");
630     h1->release(h, nullptr, locked);
631
632     /* try an delete operation which should fail */
633     uint64_t cas = 0;
634     i = NULL;
635
636     checkeq(ENGINE_LOCKED_TMPFAIL, del(h, h1, key, 0, 0), "Delete failed");
637
638
639     /* bug MB 2699 append after getl should fail with ENGINE_TMPFAIL */
640
641     testHarness.time_travel(26);
642
643     char binaryData1[] = "abcdefg\0gfedcba";
644
645     checkeq(ENGINE_SUCCESS,
646             storeCasVb11(h, h1, NULL, OPERATION_SET, key,
647                          binaryData1, sizeof(binaryData1) - 1, 82758, &i, 0, 0),
648             "Failed set.");
649     h1->release(h, NULL, i);
650
651     /* acquire lock, should succeed */
652     checkeq(ENGINE_SUCCESS,
653             getl(h, h1, nullptr, &locked, key, vbucketId, expiration),
654             "Aquire lock should have succeeded");
655     h1->release(h, nullptr, locked);
656
657     /* bug MB 3252 & MB 3354.
658      * 1. Set a key with an expiry value.
659      * 2. Take a lock on the item before it expires
660      * 3. Wait for the item to expire
661      * 4. Perform a CAS operation, should fail
662      * 5. Perform a set operation, should succeed
663      */
664     const char *ekey = "test_expiry";
665     const char *edata = "some test data here.";
666
667     item *it = NULL;
668
669     checkeq(ENGINE_SUCCESS,
670             allocate(h, h1, NULL, &it, ekey, strlen(edata), 0, 2,
671                      PROTOCOL_BINARY_RAW_BYTES, 0),
672             "Allocation Failed");
673
674     check(h1->get_item_info(h, NULL, it, &info),
675           "Failed to get item info");
676
677     memcpy(info.value[0].iov_base, edata, strlen(edata));
678
679     checkeq(ENGINE_SUCCESS,
680             h1->store(h, NULL, it, &cas, OPERATION_SET, DocumentState::Alive),
681            "Failed to Store item");
682     check_key_value(h, h1, ekey, edata, strlen(edata));
683     h1->release(h, NULL, it);
684
685     testHarness.time_travel(3);
686     cas = last_cas;
687
688     /* cas should fail */
689     check(storeCasVb11(h, h1, NULL, OPERATION_CAS, ekey,
690                        binaryData1, sizeof(binaryData1) - 1, 82758, &i, cas, 0)
691           != ENGINE_SUCCESS,
692           "CAS succeeded.");
693     h1->release(h, NULL, i);
694
695     /* but a simple store should succeed */
696     checkeq(ENGINE_SUCCESS,
697             store(h, h1, NULL, OPERATION_SET, ekey, edata, &i, 0, vbucketId),
698             "Failed to store an item.");
699     h1->release(h, NULL, i);
700
701     return SUCCESS;
702 }
703
704 static enum test_result test_unl(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
705
706     const char *key = "k2";
707     uint16_t vbucketId = 0;
708
709     checkeq(ENGINE_SUCCESS,
710             h1->get_stats(h, NULL, NULL, 0, add_stats),
711             "Failed to get stats.");
712
713     std::string eviction_policy;
714     auto itr = vals.find("ep_item_eviction_policy");
715     if (itr != vals.end()) {
716         eviction_policy = itr->second;
717     } else {
718         eviction_policy = "value_only";
719     }
720
721     if (eviction_policy == "full_eviction") {
722         checkeq(ENGINE_TMPFAIL,
723                 unl(h, h1, nullptr, key, vbucketId),
724                 "expected a TMPFAIL");
725     } else {
726         checkeq(ENGINE_KEY_ENOENT,
727                 unl(h, h1, nullptr, key, vbucketId),
728                 "expected the key to be missing...");
729     }
730
731     item *i = NULL;
732     checkeq(ENGINE_SUCCESS,
733             store(h, h1, NULL, OPERATION_SET, key, "lockdata", &i, 0, vbucketId),
734             "Failed to store an item.");
735     h1->release(h, NULL, i);
736
737     /* getl, should succeed */
738     item* locked = nullptr;
739     checkeq(ENGINE_SUCCESS,
740             getl(h, h1, nullptr, &locked, key, vbucketId, 0),
741             "Expected to be able to getl on first try");
742     item_info info;
743     checkeq(true, h1->get_item_info(h, nullptr, locked, &info),
744             "failed to get item info");
745     uint64_t cas = info.cas;
746     h1->release(h, nullptr, locked);
747
748     /* lock's taken unlocking with a random cas value should fail */
749     checkeq(ENGINE_TMPFAIL,
750             unl(h, h1, nullptr, key, vbucketId),
751             "Expected to fail getl on second try");
752
753     checkeq(ENGINE_SUCCESS,
754             unl(h, h1, nullptr, key, vbucketId, cas),
755             "Expected to succed unl with correct cas");
756
757     /* acquire lock, should succeed */
758     checkeq(ENGINE_SUCCESS,
759             getl(h, h1, nullptr, &locked, key, vbucketId, 0),
760             "Lock should work after unlock");
761     h1->release(h, nullptr, locked);
762
763     /* wait 16 seconds */
764     testHarness.time_travel(16);
765
766     /* lock has expired, unl should fail */
767     checkeq(ENGINE_TMPFAIL,
768             unl(h, h1, nullptr, key, vbucketId, last_cas),
769             "Expected to fail unl on lock timeout");
770
771     return SUCCESS;
772 }
773
774 static enum test_result test_unl_nmvb(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
775
776     const char *key = "k2";
777     uint16_t vbucketId = 10;
778
779     checkeq(ENGINE_NOT_MY_VBUCKET,
780             unl(h, h1, nullptr, key, vbucketId),
781           "expected NOT_MY_VBUCKET to unlocking a key in a vbucket we don't own");
782
783     return SUCCESS;
784 }
785
786 static enum test_result test_set_get_hit_bin(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
787     char binaryData[] = "abcdefg\0gfedcba";
788     cb_assert(sizeof(binaryData) != strlen(binaryData));
789
790     item *i = NULL;
791     checkeq(ENGINE_SUCCESS,
792             storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
793                          binaryData, sizeof(binaryData), 82758, &i, 0, 0),
794             "Failed to set.");
795     h1->release(h, NULL, i);
796     check_key_value(h, h1, "key", binaryData, sizeof(binaryData));
797     return SUCCESS;
798 }
799
800 static enum test_result test_set_with_cas_non_existent(ENGINE_HANDLE *h,
801                                                        ENGINE_HANDLE_V1 *h1) {
802     const char *key = "test_expiry_flush";
803     item *i = NULL;
804
805     checkeq(ENGINE_SUCCESS,
806             allocate(h, h1, NULL, &i, key, 10, 0, 0,
807             PROTOCOL_BINARY_RAW_BYTES, 0),
808             "Allocation failed.");
809
810     Item *it = reinterpret_cast<Item*>(i);
811     it->setCas(1234);
812
813     uint64_t cas = 0;
814     checkeq(ENGINE_KEY_ENOENT,
815             h1->store(h, NULL, i, &cas, OPERATION_SET, DocumentState::Alive),
816             "Expected not found");
817     h1->release(h, NULL, i);
818
819     return SUCCESS;
820 }
821
822 static enum test_result test_set_change_flags(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
823     item *i = NULL;
824     checkeq(ENGINE_SUCCESS,
825             store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i),
826             "Failed to set.");
827     h1->release(h, NULL, i);
828
829     item_info info;
830     uint32_t flags = 828258;
831     check(get_item_info(h, h1, &info, "key"), "Failed to get value.");
832     cb_assert(info.flags != flags);
833
834     checkeq(ENGINE_SUCCESS,
835             storeCasVb11(h, h1, NULL, OPERATION_SET, "key",
836                          "newvalue", strlen("newvalue"), flags, &i, 0, 0),
837             "Failed to set again.");
838     h1->release(h, NULL, i);
839
840     check(get_item_info(h, h1, &info, "key"), "Failed to get value.");
841
842     return info.flags == flags ? SUCCESS : FAIL;
843 }
844
845 static enum test_result test_add(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
846     item *i = NULL;
847     item_info info;
848     uint64_t vb_uuid = 0;
849     uint64_t high_seqno = 0;
850
851     memset(&info, 0, sizeof(info));
852
853     vb_uuid = get_ull_stat(h, h1, "vb_0:0:id", "failovers");
854     high_seqno = get_ull_stat(h, h1, "vb_0:high_seqno", "vbucket-seqno");
855
856     checkeq(ENGINE_SUCCESS,
857             store(h, h1, NULL, OPERATION_ADD,"key", "somevalue", &i),
858             "Failed to add value.");
859     h1->release(h, NULL, i);
860
861     check(get_item_info(h, h1, &info, "key"), "Error getting item info");
862     checkeq(vb_uuid, info.vbucket_uuid, "Expected valid vbucket uuid");
863     checkeq(high_seqno + 1, info.seqno, "Expected valid sequence number");
864
865     checkeq(ENGINE_NOT_STORED,
866             store(h, h1, NULL, OPERATION_ADD,"key", "somevalue", &i),
867             "Failed to fail to re-add value.");
868     h1->release(h, NULL, i);
869
870     // This aborts on failure.
871     check_key_value(h, h1, "key", "somevalue", 9);
872
873     // Expiration above was an hour, so let's go to The Future
874     testHarness.time_travel(3800);
875
876     checkeq(ENGINE_SUCCESS,
877             store(h, h1, NULL, OPERATION_ADD,"key", "newvalue", &i),
878             "Failed to add value again.");
879
880     h1->release(h, NULL, i);
881     check_key_value(h, h1, "key", "newvalue", 8);
882     return SUCCESS;
883 }
884
885 static enum test_result test_add_add_with_cas(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
886     item *i = NULL;
887     checkeq(ENGINE_SUCCESS,
888             store(h, h1, NULL, OPERATION_ADD, "key", "somevalue", &i),
889             "Failed set.");
890     check_key_value(h, h1, "key", "somevalue", 9);
891     item_info info;
892     check(h1->get_item_info(h, NULL, i, &info),
893           "Should be able to get info");
894
895     item *i2 = NULL;
896     checkeq(ENGINE_KEY_EEXISTS,
897             store(h, h1, NULL, OPERATION_ADD, "key",
898                   "somevalue", &i2, info.cas),
899             "Should not be able to add the key two times");
900
901     h1->release(h, NULL, i);
902     h1->release(h, NULL, i2);
903     return SUCCESS;
904 }
905
906 static enum test_result test_cas(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
907     item *i = NULL;
908     checkeq(ENGINE_SUCCESS,
909             store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i),
910             "Failed to do initial set.");
911     h1->release(h, NULL, i);
912     check(store(h, h1, NULL, OPERATION_CAS, "key", "failcas", &i) != ENGINE_SUCCESS,
913           "Failed to fail initial CAS.");
914     h1->release(h, NULL, i);
915     check_key_value(h, h1, "key", "somevalue", 9);
916
917     checkeq(ENGINE_SUCCESS,
918             get(h, h1, NULL, &i, "key", 0),
919             "Failed to get value.");
920
921     item_info info;
922     check(h1->get_item_info(h, NULL, i, &info), "Failed to get item info.");
923     h1->release(h, NULL, i);
924
925     checkeq(ENGINE_SUCCESS,
926             store(h, h1, NULL, OPERATION_CAS, "key", "winCas", &i, info.cas),
927             "Failed to store CAS");
928     h1->release(h, NULL, i);
929     check_key_value(h, h1, "key", "winCas", 6);
930
931     uint64_t cval = 99999;
932     checkeq(ENGINE_KEY_ENOENT,
933             store(h, h1, NULL, OPERATION_CAS, "non-existing", "winCas",
934                   &i, cval),
935             "CAS for non-existing key returned the wrong error code");
936     h1->release(h, NULL, i);
937     return SUCCESS;
938 }
939
940 static enum test_result test_replace(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
941     item *i = NULL;
942     item_info info;
943     uint64_t vb_uuid = 0;
944     uint64_t high_seqno = 0;
945
946     memset(&info, 0, sizeof(info));
947
948     check(store(h, h1, NULL, OPERATION_REPLACE,"key", "somevalue", &i) != ENGINE_SUCCESS,
949           "Failed to fail to replace non-existing value.");
950
951     h1->release(h, NULL, i);
952     checkeq(ENGINE_SUCCESS,
953             store(h, h1, NULL, OPERATION_SET,"key", "somevalue", &i),
954             "Failed to set value.");
955     h1->release(h, NULL, i);
956
957     vb_uuid = get_ull_stat(h, h1, "vb_0:0:id", "failovers");
958     high_seqno = get_ull_stat(h, h1, "vb_0:high_seqno", "vbucket-seqno");
959
960     checkeq(ENGINE_SUCCESS,
961             store(h, h1, NULL, OPERATION_REPLACE,"key", "somevalue", &i),
962             "Failed to replace existing value.");
963     h1->release(h, NULL, i);
964
965     check(get_item_info(h, h1, &info, "key"), "Error getting item info");
966
967     checkeq(vb_uuid, info.vbucket_uuid, "Expected valid vbucket uuid");
968     checkeq(high_seqno + 1, info.seqno, "Expected valid sequence number");
969
970     check_key_value(h, h1, "key", "somevalue", 9);
971     return SUCCESS;
972 }
973
974 static enum test_result test_touch(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
975     // Try to touch an unknown item...
976     checkeq(ENGINE_KEY_ENOENT, touch(h, h1, "mykey", 0, 0), "Testing unknown key");
977
978     // illegal vbucket
979     checkeq(ENGINE_NOT_MY_VBUCKET, touch(h, h1, "mykey", 5, 0), "Testing illegal vbucket");
980
981     // Store the item!
982     item *itm = NULL;
983     checkeq(ENGINE_SUCCESS,
984             store(h, h1, NULL, OPERATION_SET, "mykey", "somevalue", &itm),
985             "Failed set.");
986     h1->release(h, NULL, itm);
987
988     check_key_value(h, h1, "mykey", "somevalue", strlen("somevalue"));
989
990     check(get_meta(h, h1, "mykey"), "Get meta failed");
991     checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
992             "Get meta failed");
993
994     uint64_t curr_cas = last_meta.cas;
995     time_t curr_exptime = last_meta.exptime;
996     uint64_t curr_revseqno = last_meta.revSeqno;
997
998     checkeq(ENGINE_SUCCESS,
999             touch(h, h1, "mykey", 0, uint32_t(time(NULL) + 10)),
1000             "touch mykey");
1001     check(last_cas != curr_cas, "touch should have returned an updated CAS");
1002
1003     check(get_meta(h, h1, "mykey"), "Get meta failed");
1004     checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
1005             "Get meta failed");
1006
1007     check(last_meta.cas != curr_cas, "touch should have updated the CAS");
1008     check(last_meta.exptime != curr_exptime, "touch should have updated the expiry time");
1009     check(last_meta.revSeqno == curr_revseqno + 1, "touch should have incremented rev seqno");
1010
1011     // time-travel 9 secs..
1012     testHarness.time_travel(9);
1013
1014     // The item should still exist
1015     check_key_value(h, h1, "mykey", "somevalue", 9);
1016
1017     // time-travel 2 secs..
1018     testHarness.time_travel(2);
1019
1020     // The item should have expired now...
1021     checkeq(ENGINE_KEY_ENOENT,
1022             get(h, h1, NULL, &itm, "mykey", 0), "Item should be gone");
1023     return SUCCESS;
1024 }
1025
1026 static enum test_result test_touch_mb7342(ENGINE_HANDLE *h,
1027                                           ENGINE_HANDLE_V1 *h1) {
1028     const char *key = "MB-7342";
1029     // Store the item!
1030     item *itm = NULL;
1031     checkeq(ENGINE_SUCCESS,
1032             store(h, h1, NULL, OPERATION_SET, key, "v", &itm),
1033             "Failed set.");
1034     h1->release(h, NULL, itm);
1035
1036     checkeq(ENGINE_SUCCESS, touch(h, h1, key, 0, 0), "touch key");
1037
1038     check_key_value(h, h1, key, "v", 1);
1039
1040     // Travel a loong time to see if the object is still there (the default
1041     // store sets an exp time of 3600
1042     testHarness.time_travel(3700);
1043
1044     check_key_value(h, h1, key, "v", 1);
1045
1046     return SUCCESS;
1047 }
1048
1049 static enum test_result test_touch_mb10277(ENGINE_HANDLE *h,
1050                                             ENGINE_HANDLE_V1 *h1) {
1051     const char *key = "MB-10277";
1052     // Store the item!
1053     item *itm = NULL;
1054     checkeq(ENGINE_SUCCESS,
1055             store(h, h1, NULL, OPERATION_SET, key, "v", &itm),
1056             "Failed set.");
1057     h1->release(h, NULL, itm);
1058     wait_for_flusher_to_settle(h, h1);
1059     evict_key(h, h1, key, 0, "Ejected.");
1060
1061     checkeq(ENGINE_SUCCESS,
1062             touch(h, h1, key, 0, 3600), // A new expiration time remains in the same.
1063             "touch key");
1064
1065     return SUCCESS;
1066 }
1067
1068 static enum test_result test_gat(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1069     // Try to gat an unknown item...
1070     auto ret = gat(h, h1, "mykey", 0, 10);
1071     checkeq(ENGINE_KEY_ENOENT, ENGINE_ERROR_CODE(ret.first),
1072             "Testing unknown key");
1073
1074     // illegal vbucket
1075     ret = gat(h, h1, "mykey", 5, 10);
1076     checkeq(ENGINE_NOT_MY_VBUCKET,
1077             ENGINE_ERROR_CODE(ret.first), "Testing illegal vbucket");
1078
1079     // Store the item!
1080     item *itm = NULL;
1081     checkeq(ENGINE_SUCCESS,
1082             store(h, h1, NULL, OPERATION_SET, "mykey", "{\"some\":\"value\"}",
1083                   &itm, 0, 0, 3600, PROTOCOL_BINARY_DATATYPE_JSON),
1084             "Failed set.");
1085     h1->release(h, NULL, itm);
1086
1087     check_key_value(h, h1, "mykey", "{\"some\":\"value\"}",
1088             strlen("{\"some\":\"value\"}"));
1089
1090     ret = gat(h, h1, "mykey", 0, 10);
1091     checkeq(ENGINE_SUCCESS,
1092             ENGINE_ERROR_CODE(ret.first), "gat mykey");
1093
1094     item_info info;
1095     check(h1->get_item_info(h, nullptr, ret.second.get(), &info),
1096           "Getting item info failed");
1097
1098     checkeq(static_cast<uint8_t>(PROTOCOL_BINARY_DATATYPE_JSON),
1099             info.datatype, "Expected datatype to be JSON");
1100
1101     std::string body{static_cast<char*>(info.value[0].iov_base),
1102     info.value[0].iov_len};
1103     check(body.compare(0, sizeof("{\"some\":\"value\"}"),
1104                        "{\"some\":\"value\"}") == 0,
1105           "Invalid data returned");
1106
1107     // time-travel 9 secs..
1108     testHarness.time_travel(9);
1109
1110     // The item should still exist
1111     check_key_value(h, h1, "mykey", "{\"some\":\"value\"}",
1112                     strlen("{\"some\":\"value\"}"));
1113
1114     // time-travel 2 secs..
1115     testHarness.time_travel(2);
1116
1117     // The item should have expired now...
1118     checkeq(ENGINE_KEY_ENOENT,
1119             get(h, h1, NULL, &itm, "mykey", 0), "Item should be gone");
1120     return SUCCESS;
1121 }
1122
1123 static enum test_result test_gat_locked(ENGINE_HANDLE *h,
1124                                         ENGINE_HANDLE_V1 *h1) {
1125     item *itm = NULL;
1126     checkeq(ENGINE_SUCCESS,
1127             store(h, h1, NULL, OPERATION_SET,
1128                   "key", "value", &itm),
1129             "Failed to set key");
1130     h1->release(h, NULL, itm);
1131
1132     item* locked = nullptr;
1133     checkeq(ENGINE_SUCCESS,
1134             getl(h, h1, nullptr, &locked, "key", 0, 15),
1135           "Expected getl to succeed on key");
1136     h1->release(h, nullptr, locked);
1137
1138     auto ret = gat(h, h1, "key", 0, 10);
1139     checkeq(ENGINE_LOCKED, ENGINE_ERROR_CODE(ret.first), "Expected LOCKED");
1140
1141     testHarness.time_travel(16);
1142     ret = gat(h, h1, "key", 0, 10);
1143     checkeq(ENGINE_SUCCESS, ENGINE_ERROR_CODE(ret.first), "Expected success");
1144
1145     testHarness.time_travel(11);
1146     checkeq(ENGINE_KEY_ENOENT,
1147             get(h, h1, NULL, &itm, "key", 0),
1148             "Expected value to be expired");
1149     return SUCCESS;
1150 }
1151
1152 static enum test_result test_touch_locked(ENGINE_HANDLE *h,
1153                                           ENGINE_HANDLE_V1 *h1) {
1154     item *itm = NULL;
1155     checkeq(ENGINE_SUCCESS,
1156             store(h, h1, NULL, OPERATION_SET, "key", "value", &itm),
1157             "Failed to set key");
1158     h1->release(h, NULL, itm);
1159
1160     item* locked = nullptr;
1161     checkeq(ENGINE_SUCCESS,
1162             getl(h, h1, nullptr, &locked, "key", 0, 15),
1163             "Expected getl to succeed on key");
1164     h1->release(h, nullptr, locked);
1165
1166     checkeq(ENGINE_LOCKED, touch(h, h1, "key", 0, 10),
1167             "Expected tmp fail");
1168
1169     testHarness.time_travel(16);
1170     checkeq(ENGINE_SUCCESS, touch(h, h1, "key", 0, 10), "Expected success");
1171
1172     testHarness.time_travel(11);
1173     checkeq(ENGINE_KEY_ENOENT,
1174             get(h, h1, NULL, &itm, "key", 0),
1175             "Expected value to be expired");
1176
1177     return SUCCESS;
1178 }
1179
1180 static enum test_result test_mb5215(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1181     if (!isWarmupEnabled(h, h1)) {
1182         return SKIPPED;
1183     }
1184
1185     item *itm = NULL;
1186     checkeq(ENGINE_SUCCESS,
1187             store(h, h1, NULL, OPERATION_SET, "coolkey", "cooler", &itm),
1188             "Failed set.");
1189     h1->release(h, NULL, itm);
1190
1191     check_key_value(h, h1, "coolkey", "cooler", strlen("cooler"));
1192
1193     // set new exptime to 111
1194     int expTime = time(NULL) + 111;
1195
1196     checkeq(ENGINE_SUCCESS, touch(h, h1, "coolkey", 0, expTime),
1197             "touch coolkey");
1198
1199     //reload engine
1200     testHarness.reload_engine(&h, &h1,
1201                               testHarness.engine_path,
1202                               testHarness.get_current_testcase()->cfg,
1203                               true, false);
1204
1205     wait_for_warmup_complete(h, h1);
1206
1207     //verify persisted expiration time
1208     const char *statkey = "key coolkey 0";
1209     int newExpTime;
1210     checkeq(ENGINE_SUCCESS,
1211             get(h, h1, NULL, &itm, "coolkey", 0),
1212             "Missing key");
1213     h1->release(h, NULL, itm);
1214     newExpTime = get_int_stat(h, h1, "key_exptime", statkey);
1215     checkeq(expTime, newExpTime, "Failed to persist new exptime");
1216
1217     // evict key, touch expiration time, and verify
1218     evict_key(h, h1, "coolkey", 0, "Ejected.");
1219
1220     expTime = time(NULL) + 222;
1221     checkeq(ENGINE_SUCCESS ,touch(h, h1, "coolkey", 0, expTime),
1222             "touch coolkey");
1223
1224     testHarness.reload_engine(&h, &h1,
1225                               testHarness.engine_path,
1226                               testHarness.get_current_testcase()->cfg,
1227                               true, false);
1228     wait_for_warmup_complete(h, h1);
1229
1230     checkeq(ENGINE_SUCCESS,
1231             get(h, h1, NULL, &itm, "coolkey", 0),
1232             "Missing key");
1233     h1->release(h, NULL, itm);
1234     newExpTime = get_int_stat(h, h1, "key_exptime", statkey);
1235     checkeq(expTime, newExpTime, "Failed to persist new exptime");
1236
1237     return SUCCESS;
1238 }
1239
1240 /* Testing functionality to store a value for a deleted item
1241  * and also retrieve the value of a deleted item.
1242  * Need to check:
1243  *
1244  * - Each possible state transition between Alive, Deleted-with-value and
1245  *   Deleted-no-value.
1246  */
1247 static enum test_result test_delete_with_value(ENGINE_HANDLE* h,
1248                                                ENGINE_HANDLE_V1* h1) {
1249     const uint64_t cas_0 = 0;
1250     const uint16_t vbid = 0;
1251     const void* cookie = testHarness.create_cookie();
1252
1253     // Store an initial (not-deleted) value.
1254     checkeq(ENGINE_SUCCESS,
1255             store(h, h1, cookie, OPERATION_SET, "key", "somevalue", nullptr),
1256             "Failed set");
1257     wait_for_flusher_to_settle(h, h1);
1258
1259     checkeq(uint64_t(1),
1260             get_stat<uint64_t>(h, h1, "vb_0:num_items", "vbucket-details 0"),
1261             "Unexpected initial item count");
1262
1263     /* Alive -> Deleted-with-value */
1264     checkeq(ENGINE_SUCCESS,
1265             delete_with_value(h, h1, cookie, cas_0, "key", "deleted"),
1266             "Failed Alive -> Delete-with-value");
1267
1268     checkeq(uint64_t(0),
1269             get_stat<uint64_t>(h, h1, "vb_0:num_items", "vbucket-details 0"),
1270             "Unexpected num_items after Alive -> Delete-with-value");
1271
1272     auto res = get_value(h, h1, cookie, "key", vbid, DocStateFilter::Alive);
1273     checkeq(ENGINE_KEY_ENOENT,
1274             res.first,
1275             "Unexpectedly accessed Deleted-with-value via DocState::Alive");
1276
1277     res = get_value(h, h1, cookie, "key", vbid, DocStateFilter::AliveOrDeleted);
1278     checkeq(ENGINE_SUCCESS,
1279             res.first,
1280             "Failed to fetch Alive -> Delete-with-value");
1281     checkeq(std::string("deleted"), res.second, "Unexpected value (deleted)");
1282
1283     /* Deleted-with-value -> Deleted-with-value (different value). */
1284     checkeq(ENGINE_SUCCESS,
1285             delete_with_value(h, h1, cookie, cas_0, "key", "deleted 2"),
1286             "Failed Deleted-with-value -> Deleted-with-value");
1287
1288     checkeq(uint64_t(0),
1289             get_stat<uint64_t>(h, h1, "vb_0:num_items", "vbucket-details 0"),
1290             "Unexpected num_items after Delete-with-value -> "
1291             "Delete-with-value");
1292
1293     res = get_value(h, h1, cookie, "key", vbid, DocStateFilter::AliveOrDeleted);
1294     checkeq(ENGINE_SUCCESS, res.first, "Failed to fetch key (deleted 2)");
1295     checkeq(std::string("deleted 2"),
1296             res.second,
1297             "Unexpected value (deleted 2)");
1298
1299     /* Delete-with-value -> Alive */
1300     checkeq(ENGINE_SUCCESS,
1301             store(h, h1, cookie, OPERATION_SET, "key", "alive 2", nullptr),
1302             "Failed Delete-with-value -> Alive");
1303     wait_for_flusher_to_settle(h, h1);
1304
1305     checkeq(uint64_t(1),
1306             get_stat<uint64_t>(h, h1, "vb_0:num_items", "vbucket-details 0"),
1307             "Unexpected num_items after Delete-with-value -> Alive");
1308
1309     res = get_value(h, h1, cookie, "key", vbid, DocStateFilter::Alive);
1310     checkeq(ENGINE_SUCCESS,
1311             res.first,
1312             "Failed to fetch Delete-with-value -> Alive via DocState::Alive");
1313     checkeq(std::string("alive 2"), res.second, "Unexpected value (alive 2)");
1314
1315     // Also check via DocState::Deleted
1316     res = get_value(h, h1, cookie, "key", vbid, DocStateFilter::AliveOrDeleted);
1317     checkeq(ENGINE_SUCCESS,
1318             res.first,
1319             "Failed to fetch Delete-with-value -> Alive via DocState::Deleted");
1320     checkeq(std::string("alive 2"),
1321             res.second,
1322             "Unexpected value (alive 2) via DocState::Deleted");
1323
1324     /* Alive -> Deleted-no-value */
1325     checkeq(ENGINE_SUCCESS,
1326             del(h, h1, "key", cas_0, vbid, cookie),
1327             "Failed Alive -> Deleted-no-value");
1328     wait_for_flusher_to_settle(h, h1);
1329
1330     checkeq(uint64_t(0),
1331             get_stat<uint64_t>(h, h1, "vb_0:num_items", "vbucket-details 0"),
1332             "Unexpected num_items after Alive -> Delete-no-value");
1333
1334     res = get_value(h, h1, cookie, "key", vbid, DocStateFilter::Alive);
1335     checkeq(ENGINE_KEY_ENOENT,
1336             res.first,
1337             "Unexpectedly accessed Deleted-no-value via DocState::Alive");
1338
1339     /* Deleted-no-value -> Delete-with-value */
1340     checkeq(ENGINE_SUCCESS,
1341             delete_with_value(h, h1, cookie, cas_0, "key", "deleted 3"),
1342             "Failed delete with value (deleted 2)");
1343
1344     res = get_value(h, h1, cookie, "key", vbid, DocStateFilter::AliveOrDeleted);
1345     checkeq(ENGINE_SUCCESS, res.first, "Failed to fetch key (deleted 3)");
1346     checkeq(std::string("deleted 3"),
1347             res.second,
1348             "Unexpected value (deleted 3)");
1349
1350     testHarness.destroy_cookie(cookie);
1351
1352     return SUCCESS;
1353 }
1354
1355 /* Similar to test_delete_with_value, except also checks that CAS values
1356  */
1357 static enum test_result test_delete_with_value_cas(ENGINE_HANDLE *h,
1358                                                    ENGINE_HANDLE_V1 *h1) {
1359     checkeq(ENGINE_SUCCESS,
1360             store(h, h1, nullptr, OPERATION_SET, "key1", "somevalue", nullptr),
1361             "Failed set");
1362
1363     check(get_meta(h, h1, "key1"), "Get meta failed");
1364     checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS,
1365             last_status.load(), "Get meta failed");
1366
1367     uint64_t curr_revseqno = last_meta.revSeqno;
1368
1369     /* Store a deleted item first with CAS 0 */
1370     checkeq(ENGINE_SUCCESS,
1371             store(h, h1, nullptr, OPERATION_SET, "key1", "deletevalue", nullptr,
1372                   0, 0, 3600, 0x00, DocumentState::Deleted),
1373             "Failed delete with value");
1374
1375     check(get_meta(h, h1, "key1"), "Get meta failed");
1376     checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS,
1377             last_status.load(), "Get meta failed");
1378
1379     checkeq(last_meta.revSeqno, curr_revseqno + 1,
1380             "rev seqno should have incremented");
1381
1382     item *i = nullptr;
1383     checkeq(ENGINE_SUCCESS,
1384             store(h, h1, nullptr, OPERATION_SET, "key2", "somevalue", &i),
1385             "Failed set");
1386
1387     item_info info;
1388     check(h1->get_item_info(h, nullptr, i, &info),
1389           "Getting item info failed");
1390
1391     h1->release(h, nullptr, i);
1392
1393     check(get_meta(h, h1, "key2"), "Get meta failed");
1394     checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS,
1395             last_status.load(), "Get meta failed");
1396
1397     curr_revseqno = last_meta.revSeqno;
1398
1399     /* Store a deleted item with the existing CAS value */
1400     checkeq(ENGINE_SUCCESS,
1401             store(h, h1, nullptr, OPERATION_SET, "key2", "deletevaluewithcas",
1402                   nullptr, info.cas, 0, 3600, 0x00, DocumentState::Deleted),
1403             "Failed delete value with cas");
1404
1405     wait_for_flusher_to_settle(h, h1);
1406
1407     check(get_meta(h, h1, "key2"), "Get meta failed");
1408     checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS,
1409             last_status.load(), "Get meta failed");
1410
1411     checkeq(last_meta.revSeqno, curr_revseqno + 1,
1412             "rev seqno should have incremented");
1413
1414     curr_revseqno = last_meta.revSeqno;
1415
1416     checkeq(ENGINE_SUCCESS,
1417             store(h, h1, nullptr, OPERATION_SET, "key2",
1418                   "newdeletevalue", &i, 0, 0, 3600, 0x00, DocumentState::Deleted),
1419             "Failed delete value with cas");
1420
1421     wait_for_flusher_to_settle(h, h1);
1422
1423     check(h1->get_item_info(h, nullptr, i, &info), "Getting item info failed");
1424     checkeq(int(DocumentState::Deleted),
1425             int(info.document_state),
1426             "Incorrect DocState for deleted item");
1427     checkne(uint64_t(0), info.cas, "Expected non-zero CAS for deleted item");
1428
1429     h1->release(h, nullptr, i);
1430
1431     check(get_meta(h, h1, "key2"), "Get meta failed");
1432     checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS,
1433             last_status.load(), "Get meta failed");
1434
1435     checkeq(last_meta.revSeqno, curr_revseqno + 1,
1436             "rev seqno should have incremented");
1437
1438     curr_revseqno = last_meta.revSeqno;
1439
1440     // Attempt to Delete-with-value using incorrect CAS (should fail)
1441     const uint64_t incorrect_CAS = info.cas + 1;
1442     checkeq(ENGINE_KEY_EEXISTS,
1443             store(h, h1, nullptr, OPERATION_SET, "key2",
1444                   "newdeletevaluewithcas", nullptr, incorrect_CAS, 0, 3600,
1445                   0x00, DocumentState::Deleted),
1446             "Expected KEY_EEXISTS with incorrect CAS");
1447
1448     // Attempt with correct CAS.
1449     checkeq(ENGINE_SUCCESS,
1450             store(h, h1, nullptr, OPERATION_SET, "key2",
1451                   "newdeletevaluewithcas", nullptr, info.cas, 0, 3600, 0x00,
1452                   DocumentState::Deleted), "Failed delete value with cas");
1453
1454     wait_for_flusher_to_settle(h, h1);
1455
1456     checkeq(ENGINE_SUCCESS,
1457             get(h, h1, nullptr, &i, "key2", 0 ,
1458                 DocStateFilter::AliveOrDeleted),
1459                 "Failed to get value");
1460
1461     check(get_meta(h, h1, "key2"), "Get meta failed");
1462     checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS,
1463             last_status.load(), "Get meta failed");
1464
1465     checkeq(last_meta.revSeqno, curr_revseqno + 1,
1466             "rev seqno should have incremented");
1467
1468     check(h1->get_item_info(h, nullptr, i, &info),
1469           "Getting item info failed");
1470     checkeq(int(DocumentState::Deleted),
1471             int(info.document_state),
1472             "Incorrect DocState for deleted item");
1473
1474     checkeq(static_cast<uint8_t>(DocumentState::Deleted),
1475             static_cast<uint8_t>(info.document_state),
1476             "document must be in deleted state");
1477
1478     std::string buf(static_cast<char*>(info.value[0].iov_base),
1479                     info.value[0].iov_len);
1480
1481     checkeq(0, buf.compare("newdeletevaluewithcas"), "Data mismatch");
1482
1483     h1->release(h, nullptr, i);
1484
1485     checkeq(ENGINE_KEY_ENOENT,
1486             get(h, h1, nullptr, &i, "key", 0, DocStateFilter::Alive),
1487                 "Getting value should have failed");
1488
1489     return SUCCESS;
1490 }
1491
1492 static enum test_result test_delete(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1493     item *i = NULL;
1494     // First try to delete something we know to not be there.
1495     checkeq(ENGINE_KEY_ENOENT,
1496             del(h, h1, "key", 0, 0), "Failed to fail initial delete.");
1497     checkeq(ENGINE_SUCCESS,
1498             store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i),
1499             "Failed set.");
1500     Item *it = reinterpret_cast<Item*>(i);
1501     uint64_t orig_cas = it->getCas();
1502     h1->release(h, NULL, i);
1503     check_key_value(h, h1, "key", "somevalue", 9);
1504
1505     uint64_t cas = 0;
1506     uint64_t vb_uuid = 0;
1507     mutation_descr_t mut_info;
1508     uint64_t high_seqno = 0;
1509
1510     memset(&mut_info, 0, sizeof(mut_info));
1511
1512     vb_uuid = get_ull_stat(h, h1, "vb_0:0:id", "failovers");
1513     high_seqno = get_ull_stat(h, h1, "vb_0:high_seqno", "vbucket-seqno");
1514     checkeq(ENGINE_SUCCESS, del(h, h1, "key", &cas, 0, nullptr, &mut_info),
1515             "Failed remove with value.");
1516     check(orig_cas != cas, "Expected CAS to be updated on delete");
1517     checkeq(ENGINE_KEY_ENOENT, verify_key(h, h1, "key"), "Expected missing key");
1518     checkeq(vb_uuid, mut_info.vbucket_uuid, "Expected valid vbucket uuid");
1519     checkeq(high_seqno + 1, mut_info.seqno, "Expected valid sequence number");
1520
1521     // Can I time travel to an expired object and delete it?
1522     checkeq(ENGINE_SUCCESS, store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i),
1523             "Failed set.");
1524     h1->release(h, NULL, i);
1525     testHarness.time_travel(3617);
1526     checkeq(ENGINE_KEY_ENOENT, del(h, h1, "key", 0, 0),
1527             "Did not get ENOENT removing an expired object.");
1528     checkeq(ENGINE_KEY_ENOENT, verify_key(h, h1, "key"), "Expected missing key");
1529
1530     return SUCCESS;
1531 }
1532
1533 static enum test_result test_set_delete(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1534     item *i = NULL;
1535     checkeq(ENGINE_SUCCESS, store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i),
1536             "Failed set.");
1537     h1->release(h, NULL, i);
1538     check_key_value(h, h1, "key", "somevalue", 9);
1539     checkeq(ENGINE_SUCCESS, del(h, h1, "key", 0, 0),
1540             "Failed remove with value.");
1541     checkeq(ENGINE_KEY_ENOENT, verify_key(h, h1, "key"), "Expected missing key");
1542     wait_for_flusher_to_settle(h, h1);
1543     wait_for_stat_to_be(h, h1, "curr_items", 0);
1544     return SUCCESS;
1545 }
1546
1547 static enum test_result test_set_delete_invalid_cas(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1548     item *i = NULL;
1549     checkeq(ENGINE_SUCCESS,
1550             store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i),
1551             "Failed set.");
1552     check_key_value(h, h1, "key", "somevalue", 9);
1553     item_info info;
1554     check(h1->get_item_info(h, NULL, i, &info),
1555           "Should be able to get info");
1556     h1->release(h, NULL, i);
1557
1558     checkeq(ENGINE_KEY_EEXISTS, del(h, h1, "key", info.cas + 1, 0),
1559           "Didn't expect to be able to remove the item with wrong cas");
1560
1561     checkeq(ENGINE_SUCCESS, del(h, h1, "key", info.cas, 0),
1562         "Subsequent delete with correct CAS did not succeed");
1563
1564     return SUCCESS;
1565 }
1566
1567 static enum test_result test_delete_set(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1568     if (!isWarmupEnabled(h, h1)) {
1569         return SKIPPED;
1570     }
1571
1572     wait_for_persisted_value(h, h1, "key", "value1");
1573
1574     checkeq(ENGINE_SUCCESS,
1575             del(h, h1, "key", 0, 0), "Failed remove with value.");
1576
1577     wait_for_persisted_value(h, h1, "key", "value2");
1578
1579     testHarness.reload_engine(&h, &h1,
1580                               testHarness.engine_path,
1581                               testHarness.get_current_testcase()->cfg,
1582                               true, false);
1583     wait_for_warmup_complete(h, h1);
1584
1585     check_key_value(h, h1, "key", "value2", 6);
1586     checkeq(ENGINE_SUCCESS,
1587             del(h, h1, "key", 0, 0), "Failed remove with value.");
1588     wait_for_flusher_to_settle(h, h1);
1589
1590     testHarness.reload_engine(&h, &h1,
1591                               testHarness.engine_path,
1592                               testHarness.get_current_testcase()->cfg,
1593                               true, false);
1594     wait_for_warmup_complete(h, h1);
1595
1596     checkeq(ENGINE_KEY_ENOENT, verify_key(h, h1, "key"), "Expected missing key");
1597
1598     return SUCCESS;
1599 }
1600
1601 static enum test_result test_get_delete_missing_file(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1602     checkeq(ENGINE_SUCCESS,
1603             h1->get_stats(h, NULL, NULL, 0, add_stats),
1604            "Failed to get stats.");
1605
1606     // TODO: This test needs to be skipped for forestdb as backend because
1607     // in that case we don't open and close on every operation. Thus, a get
1608     // after deleting the database file would still result in a SUCCESS.
1609     // In the future, regardless of the storage type, the resulting error
1610     // should be the same.
1611     std::string backend = vals["ep_backend"];
1612     if (backend == "forestdb") {
1613         return SKIPPED;
1614     }
1615     const char *key = "key";
1616     wait_for_persisted_value(h, h1, key, "value2delete");
1617
1618     // whack the db file and directory where the key is stored
1619     std::string dbname = vals["ep_dbname"];
1620     rmdb(dbname.c_str());
1621
1622     item *i = NULL;
1623     ENGINE_ERROR_CODE errorCode = get(h, h1, NULL, &i, key, 0);
1624     h1->release(h, NULL, i);
1625
1626     // ep engine must be unaware of well-being of the db file as long as
1627     // the item is still in the memory
1628     checkeq(ENGINE_SUCCESS, errorCode, "Expected success for get");
1629
1630     i = NULL;
1631     evict_key(h, h1, key);
1632     errorCode = get(h, h1, NULL, &i, key, 0);
1633     h1->release(h, NULL, i);
1634
1635     // ep engine must be now aware of the ill-fated db file where
1636     // the item is supposedly stored
1637     checkeq(ENGINE_TMPFAIL, errorCode, "Expected tmp fail for get");
1638
1639     return SUCCESS;
1640 }
1641
1642 static enum test_result test_bug2509(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1643     for (int j = 0; j < 10000; ++j) {
1644         item *itm = NULL;
1645         checkeq(ENGINE_SUCCESS,
1646                 store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &itm),
1647                 "Failed set.");
1648         h1->release(h, NULL, itm);
1649         usleep(10);
1650         checkeq(ENGINE_SUCCESS, del(h, h1, "key", 0, 0), "Failed remove with value.");
1651         usleep(10);
1652     }
1653
1654     if (isWarmupEnabled(h, h1)) {
1655         // Restart again, to verify we don't have any duplicates.
1656         testHarness.reload_engine(&h, &h1,
1657                                   testHarness.engine_path,
1658                                   testHarness.get_current_testcase()->cfg,
1659                                   true, false);
1660         wait_for_warmup_complete(h, h1);
1661
1662         return get_int_stat(h, h1, "ep_warmup_dups") == 0 ? SUCCESS : FAIL;
1663     }
1664     return SUCCESS;
1665 }
1666
1667 static enum test_result test_bug7023(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1668     std::vector<std::string> keys;
1669     // Make a vbucket mess.
1670     const int nitems = 10000;
1671     const int iterations = 5;
1672     for (int j = 0; j < nitems; ++j) {
1673         keys.push_back("key" + std::to_string(j));
1674     }
1675
1676     std::vector<std::string>::iterator it;
1677     for (int j = 0; j < iterations; ++j) {
1678         check(set_vbucket_state(h, h1, 0, vbucket_state_dead),
1679               "Failed set set vbucket 0 dead.");
1680         vbucketDelete(h, h1, 0);
1681         checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS,
1682                 last_status.load(),
1683                 "Expected vbucket deletion to work.");
1684         check(set_vbucket_state(h, h1, 0, vbucket_state_active),
1685               "Failed set set vbucket 0 active.");
1686         for (it = keys.begin(); it != keys.end(); ++it) {
1687             item *i;
1688             checkeq(ENGINE_SUCCESS,
1689                     store(h, h1, NULL, OPERATION_SET, it->c_str(), it->c_str(), &i),
1690                     "Failed to store a value");
1691             h1->release(h, NULL, i);
1692
1693         }
1694     }
1695     wait_for_flusher_to_settle(h, h1);
1696
1697     if (isWarmupEnabled(h, h1)) {
1698         // Restart again, to verify no data loss.
1699         testHarness.reload_engine(&h, &h1,
1700                                   testHarness.engine_path,
1701                                   testHarness.get_current_testcase()->cfg,
1702                                   true, false);
1703         wait_for_warmup_complete(h, h1);
1704         checkeq(nitems,
1705                 get_int_stat(h, h1, "ep_warmup_value_count", "warmup"),
1706                 "Incorrect items following warmup");
1707     }
1708     return SUCCESS;
1709 }
1710
1711 static enum test_result test_mb3169(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1712     item *i = NULL;
1713     checkeq(ENGINE_SUCCESS,
1714             store(h, h1, NULL, OPERATION_SET, "set", "value", &i, 0, 0),
1715             "Failed to store a value");
1716     h1->release(h, NULL, i);
1717     checkeq(ENGINE_SUCCESS,
1718             store(h, h1, NULL, OPERATION_SET, "delete", "0", &i, 0, 0),
1719             "Failed to store a value");
1720     h1->release(h, NULL, i);
1721     checkeq(ENGINE_SUCCESS,
1722             store(h, h1, NULL, OPERATION_SET, "get", "getvalue", &i, 0, 0),
1723             "Failed to store a value");
1724     h1->release(h, NULL, i);
1725
1726     wait_for_stat_to_be(h, h1, "ep_total_persisted", 3);
1727
1728     evict_key(h, h1, "set", 0, "Ejected.");
1729     evict_key(h, h1, "delete", 0, "Ejected.");
1730     evict_key(h, h1, "get", 0, "Ejected.");
1731
1732     checkeq(3, get_int_stat(h, h1, "ep_num_non_resident"),
1733             "Expected four items to be resident");
1734
1735     checkeq(ENGINE_SUCCESS,
1736             store(h, h1, NULL, OPERATION_SET, "set", "value2", &i, 0, 0),
1737             "Failed to store a value");
1738     h1->release(h, NULL, i);
1739     wait_for_flusher_to_settle(h, h1);
1740
1741     checkeq(2, get_int_stat(h, h1, "ep_num_non_resident"),
1742           "Expected mutation to mark item resident");
1743
1744     checkeq(ENGINE_SUCCESS, del(h, h1, "delete", 0, 0),
1745             "Delete failed");
1746
1747     checkeq(1, get_int_stat(h, h1, "ep_num_non_resident"),
1748             "Expected delete to remove non-resident item");
1749
1750     check_key_value(h, h1, "get", "getvalue", 8);
1751
1752     checkeq(0, get_int_stat(h, h1, "ep_num_non_resident"),
1753             "Expected all items to be resident");
1754     return SUCCESS;
1755 }
1756
1757 static enum test_result test_mb5172(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1758     if (!isWarmupEnabled(h, h1)) {
1759         return SKIPPED;
1760     }
1761
1762     item *i = NULL;
1763     checkeq(ENGINE_SUCCESS,
1764             store(h, h1, NULL, OPERATION_SET, "key-1", "value-1", &i, 0, 0),
1765             "Failed to store a value");
1766     h1->release(h, NULL, i);
1767     checkeq(ENGINE_SUCCESS,
1768             store(h, h1, NULL, OPERATION_SET, "key-2", "value-2", &i, 0, 0),
1769             "Failed to store a value");
1770     h1->release(h, NULL, i);
1771
1772     wait_for_flusher_to_settle(h, h1);
1773
1774     checkeq(0, get_int_stat(h, h1, "ep_num_non_resident"),
1775             "Expected all items to be resident");
1776
1777     // restart the server.
1778     testHarness.reload_engine(&h, &h1,
1779                               testHarness.engine_path,
1780                               testHarness.get_current_testcase()->cfg,
1781                               true, false);
1782
1783     wait_for_warmup_complete(h, h1);
1784     checkeq(0, get_int_stat(h, h1, "ep_num_non_resident"),
1785             "Expected all items to be resident");
1786     return SUCCESS;
1787 }
1788
1789 static enum test_result test_set_vbucket_out_of_range(ENGINE_HANDLE *h,
1790                                                        ENGINE_HANDLE_V1 *h1) {
1791     check(!set_vbucket_state(h, h1, 10000, vbucket_state_active),
1792           "Shouldn't have been able to set vbucket 10000");
1793     return SUCCESS;
1794 }
1795
1796 static enum test_result test_flush(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1797     item *i = NULL;
1798
1799     if (get_bool_stat(h, h1, "ep_flushall_enabled") == false) {
1800         check(set_param(h, h1, protocol_binary_engine_param_flush,
1801                         "flushall_enabled", "true"),
1802               "Set flushall_enabled should have worked");
1803     }
1804     check(get_bool_stat(h, h1, "ep_flushall_enabled"),
1805           "flushall wasn't enabled");
1806
1807     // First try to delete something we know to not be there.
1808     checkeq(ENGINE_KEY_ENOENT,
1809             del(h, h1, "key", 0, 0),
1810             "Failed to fail initial delete.");
1811     checkeq(ENGINE_SUCCESS,
1812             store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i),
1813             "Failed set.");
1814     h1->release(h, NULL, i);
1815     check_key_value(h, h1, "key", "somevalue", 9);
1816
1817     set_degraded_mode(h, h1, NULL, true);
1818     checkeq(ENGINE_SUCCESS,
1819             h1->flush(h, NULL),
1820             "Failed to flush");
1821     set_degraded_mode(h, h1, NULL, false);
1822
1823     checkeq(ENGINE_KEY_ENOENT, verify_key(h, h1, "key"), "Expected missing key");
1824
1825     checkeq(ENGINE_SUCCESS,
1826             store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i),
1827             "Failed post-flush set.");
1828     h1->release(h, NULL, i);
1829     check_key_value(h, h1, "key", "somevalue", 9);
1830
1831     return SUCCESS;
1832 }
1833
1834 static enum test_result test_flush_stats(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1835     item *i = NULL;
1836     int cacheSize = get_int_stat(h, h1, "ep_total_cache_size");
1837     int nonResident = get_int_stat(h, h1, "ep_num_non_resident");
1838
1839     int itemsRemoved = get_int_stat(h, h1, "ep_items_rm_from_checkpoints");
1840     checkeq(ENGINE_SUCCESS,
1841             store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i),
1842             "Failed set.");
1843     h1->release(h, NULL, i);
1844     checkeq(ENGINE_SUCCESS,
1845             store(h, h1, NULL, OPERATION_SET, "key2", "somevalue", &i),
1846             "Failed set.");
1847     h1->release(h, NULL, i);
1848     testHarness.time_travel(65);
1849     wait_for_stat_change(h, h1, "ep_items_rm_from_checkpoints", itemsRemoved);
1850
1851     checkeq(ENGINE_SUCCESS, verify_key(h, h1, "key"), "Expected key");
1852     checkeq(ENGINE_SUCCESS, verify_key(h, h1, "key2"), "Expected key2");
1853
1854     check_key_value(h, h1, "key", "somevalue", 9);
1855     check_key_value(h, h1, "key2", "somevalue", 9);
1856
1857     set_degraded_mode(h, h1, NULL, true);
1858     checkeq(ENGINE_SUCCESS, h1->flush(h, NULL), "Failed to flush");
1859     set_degraded_mode(h, h1, NULL, false);
1860     checkeq(ENGINE_KEY_ENOENT, verify_key(h, h1, "key"), "Expected missing key");
1861     checkeq(ENGINE_KEY_ENOENT, verify_key(h, h1, "key2"), "Expected missing key");
1862
1863     wait_for_flusher_to_settle(h, h1);
1864
1865     int cacheSize2 = get_int_stat(h, h1, "ep_total_cache_size");
1866     int nonResident2 = get_int_stat(h, h1, "ep_num_non_resident");
1867
1868     cb_assert(nonResident2 == nonResident);
1869     cb_assert(cacheSize2 == cacheSize);
1870
1871     return SUCCESS;
1872 }
1873
1874 static enum test_result test_flush_multiv(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1875     item *i = NULL;
1876     check(set_vbucket_state(h, h1, 2, vbucket_state_active),
1877           "Failed to set vbucket state.");
1878     checkeq(ENGINE_SUCCESS,
1879             store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i),
1880             "Failed set.");
1881     h1->release(h, NULL, i);
1882     checkeq(ENGINE_SUCCESS,
1883             store(h, h1, NULL, OPERATION_SET, "key2", "somevalue", &i, 0, 2),
1884             "Failed set in vb2.");
1885     h1->release(h, NULL, i);
1886
1887     checkeq(ENGINE_SUCCESS, verify_key(h, h1, "key"), "Expected key");
1888     checkeq(ENGINE_SUCCESS, verify_key(h, h1, "key2", 2), "Expected key2");
1889
1890     check_key_value(h, h1, "key", "somevalue", 9);
1891     check_key_value(h, h1, "key2", "somevalue", 9, 2);
1892
1893     set_degraded_mode(h, h1, NULL, true);
1894     checkeq(ENGINE_SUCCESS, h1->flush(h, NULL), "Failed to flush");
1895     set_degraded_mode(h, h1, NULL, false);
1896
1897     vals.clear();
1898     checkeq(ENGINE_SUCCESS, h1->get_stats(h, NULL, NULL, 0, add_stats),
1899             "Failed to get stats.");
1900     check(vals.find("ep_flush_all") != vals.end(),
1901           "Failed to get the status of flush_all");
1902
1903     checkeq(ENGINE_KEY_ENOENT, verify_key(h, h1, "key"),
1904             "Expected missing key");
1905     checkeq(ENGINE_KEY_ENOENT, verify_key(h, h1, "key2", 2),
1906             "Expected missing key");
1907
1908     return SUCCESS;
1909 }
1910
1911 static enum test_result test_flush_disabled(ENGINE_HANDLE *h,
1912                                             ENGINE_HANDLE_V1 *h1) {
1913     item *i = NULL;
1914     // start an engine with disabled flush, the flush() should be noop and
1915     // we expect to see the key after flush()
1916
1917     // store a key and check its existence
1918     checkeq(ENGINE_SUCCESS,
1919             store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i),
1920             "Failed set.");
1921     h1->release(h, NULL, i);
1922     check_key_value(h, h1, "key", "somevalue", 9);
1923     // expect error msg engine does not support operation
1924     checkeq(ENGINE_ENOTSUP,
1925             h1->flush(h, NULL),
1926             "Flush should be disabled");
1927     //check the key
1928     checkeq(ENGINE_SUCCESS, verify_key(h, h1, "key"), "Expected key");
1929
1930     // restart engine with flush enabled and redo the test, we expect flush to succeed
1931     if (isWarmupEnabled(h, h1)) {
1932         std::string param = "flushall_enabled=false";
1933         std::string config = testHarness.get_current_testcase()->cfg;
1934         size_t found = config.find(param);
1935         if(found != config.npos) {
1936             config.replace(found, param.size(), "flushall_enabled=true");
1937         }
1938         testHarness.reload_engine(&h, &h1,
1939                                   testHarness.engine_path,
1940                                   config.c_str(),
1941                                   true, false);
1942         wait_for_warmup_complete(h, h1);
1943
1944
1945         set_degraded_mode(h, h1, NULL, true);
1946         checkeq(ENGINE_SUCCESS,
1947                 h1->flush(h, NULL), "Flush should be enabled");
1948         set_degraded_mode(h, h1, NULL, false);
1949
1950         //expect missing key
1951         checkeq(ENGINE_KEY_ENOENT, verify_key(h, h1, "key"), "Expected missing key");
1952     }
1953     return SUCCESS;
1954 }
1955
1956 static enum test_result test_CBD_152(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1957     item *i = NULL;
1958
1959     // turn off flushall_enabled parameter
1960     set_param(h, h1, protocol_binary_engine_param_flush, "flushall_enabled", "false");
1961     checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
1962           "Failed to set flushall_enabled param");
1963
1964     // store a key and check its existence
1965     checkeq(ENGINE_SUCCESS,
1966             store(h, h1, NULL, OPERATION_SET, "key", "somevalue", &i),
1967             "Failed set.");
1968     h1->release(h, NULL, i);
1969
1970     check_key_value(h, h1, "key", "somevalue", 9);
1971     // expect error msg engine does not support operation
1972     checkeq(ENGINE_ENOTSUP, h1->flush(h, NULL), "Flush should be disabled");
1973     //check the key
1974     checkeq(ENGINE_SUCCESS, verify_key(h, h1, "key"), "Expected key");
1975
1976     // turn on flushall_enabled parameter
1977     set_param(h, h1, protocol_binary_engine_param_flush, "flushall_enabled", "true");
1978     checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
1979           "Failed to set flushall_enabled param");
1980     // flush should succeed
1981     set_degraded_mode(h, h1, NULL, true);
1982     checkeq(ENGINE_SUCCESS, h1->flush(h, NULL), "Flush should be enabled");
1983     set_degraded_mode(h, h1, NULL, false);
1984     //expect missing key
1985     check(ENGINE_KEY_ENOENT == verify_key(h, h1, "key"), "Expected missing key");
1986
1987     return SUCCESS;
1988 }
1989
1990 static enum test_result set_max_cas_mb21190(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
1991     uint64_t max_cas = get_ull_stat(h, h1, "vb_0:max_cas", "vbucket-details 0");
1992     std::string max_cas_str = std::to_string(max_cas+1);
1993     set_param(h, h1, protocol_binary_engine_param_vbucket,
1994               "max_cas", max_cas_str.data(), 0);
1995     checkeq(PROTOCOL_BINARY_RESPONSE_SUCCESS, last_status.load(),
1996             "Failed to set_param max_cas");
1997     checkeq(max_cas + 1,
1998             get_ull_stat(h, h1, "vb_0:max_cas", "vbucket-details 0"),
1999             "max_cas didn't change");
2000     set_param(h, h1, protocol_binary_engine_param_vbucket,
2001               "max_cas", max_cas_str.data(), 1);
2002     checkeq(PROTOCOL_BINARY_RESPONSE_NOT_MY_VBUCKET, last_status.load(),
2003             "Expected not my vbucket for vb 1");
2004     set_param(h, h1, protocol_binary_engine_param_vbucket,
2005               "max_cas", "JUNK", 0);
2006     checkeq(PROTOCOL_BINARY_RESPONSE_EINVAL, last_status.load(),
2007             "Expected EINVAL");
2008     return SUCCESS;
2009 }
2010
2011 static enum test_result warmup_mb21769(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2012     if (!isWarmupEnabled(h, h1)) {
2013         return SKIPPED;
2014     }
2015
2016     // Validate some VB data post warmup
2017     // VB 0 will be empty
2018     // VB 1 will not be empty
2019     // VB 2 will not be empty and will have had set_state as the final ops
2020
2021     check(set_vbucket_state(h, h1, 1, vbucket_state_active),
2022           "Failed to set vbucket state for vb1");
2023     check(set_vbucket_state(h, h1, 2, vbucket_state_active),
2024           "Failed to set vbucket state for vb2");
2025
2026     const int num_items = 10;
2027     write_items(h, h1, num_items, 0, "vb1", "value", 0/*expiry*/, 1/*vb*/);
2028     write_items(h, h1, num_items, 0, "vb2", "value", 0/*expiry*/, 2/*vb*/);
2029     wait_for_flusher_to_settle(h, h1);
2030
2031     // flip replica to active to drive more _local writes
2032     check(set_vbucket_state(h, h1, 2, vbucket_state_replica),
2033           "Failed to set vbucket state (replica) for vb2");
2034     wait_for_flusher_to_settle(h, h1);
2035
2036     check(set_vbucket_state(h, h1, 2, vbucket_state_active),
2037           "Failed to set vbucket state (replica) for vb2");
2038     wait_for_flusher_to_settle(h, h1);
2039
2040     // Force a shutdown so the warmup will create failover entries
2041     testHarness.reload_engine(&h, &h1,
2042                               testHarness.engine_path,
2043                               testHarness.get_current_testcase()->cfg,
2044                               true, true);
2045
2046     wait_for_warmup_complete(h, h1);
2047
2048     // values of interested stats for each VB
2049     std::array<uint64_t, 3> high_seqnos = {{0, num_items, num_items}};
2050     std::array<uint64_t, 3> snap_starts = {{0, num_items, num_items}};
2051     std::array<uint64_t, 3> snap_ends = {{0, num_items, num_items}};
2052     // we will check the seqno of the 0th entry of each vbucket's failover table
2053     std::array<uint64_t, 3> failover_entry0 = {{0, num_items, num_items}};
2054
2055     for (uint64_t vb = 0; vb <= 2; vb++) {
2056         std::string vb_prefix = "vb_" + std::to_string(vb) + ":";
2057         std::string high_seqno = vb_prefix + "high_seqno";
2058         std::string snap_start = vb_prefix + "last_persisted_snap_start";
2059         std::string snap_end = vb_prefix + "last_persisted_snap_end";
2060         std::string fail0 = vb_prefix + "0:seq";
2061         std::string vb_group_key = "vbucket-seqno " + std::to_string(vb);
2062         std::string failovers_key = "failovers " + std::to_string(vb);
2063
2064         checkeq(high_seqnos[vb],
2065                 get_ull_stat(h, h1, high_seqno.c_str(), vb_group_key.c_str()),
2066                 std::string("high_seqno incorrect vb:" + std::to_string(vb)).c_str());
2067         checkeq(snap_starts[vb],
2068                 get_ull_stat(h, h1, snap_start.c_str(), vb_group_key.c_str()),
2069                 std::string("snap_start incorrect vb:" + std::to_string(vb)).c_str());
2070         checkeq(snap_ends[vb],
2071                 get_ull_stat(h, h1, snap_end.c_str(), vb_group_key.c_str()),
2072                 std::string("snap_end incorrect vb:" + std::to_string(vb)).c_str());
2073         checkeq(failover_entry0[vb],
2074                 get_ull_stat(h, h1, fail0.c_str(), failovers_key.c_str()),
2075                 std::string("failover table entry 0 is incorrect vb:" + std::to_string(vb)).c_str());
2076     }
2077
2078     return SUCCESS;
2079 }
2080
2081 /**
2082  * Callback from the document API being called after the CAS was assigned
2083  * to the object. We're allowed to modify the content, so let's just change
2084  * the string.
2085  *
2086  * @param info info about the document
2087  */
2088 static uint64_t pre_link_seqno(0);
2089 static void pre_link_doc_callback(item_info& info) {
2090     checkne(uint64_t(0), info.cas, "CAS value should be set");
2091     // mock the actual value so we can see it was changed
2092     memcpy(info.value[0].iov_base, "valuesome", 9);
2093     pre_link_seqno = info.seqno;
2094 }
2095
2096 /**
2097  * Verify that we've hooked into the checkpoint and that the pre-link
2098  * document api method is called.
2099  */
2100 static test_result pre_link_document(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1) {
2101     item_info info;
2102     item* it;
2103
2104     PreLinkFunction function = pre_link_doc_callback;
2105     testHarness.set_pre_link_function(function);
2106     checkeq(ENGINE_SUCCESS,
2107             store(h, h1, nullptr, OPERATION_SET, "key", "somevalue", &it),
2108             "Failed set.");
2109     h1->release(h, nullptr, it);
2110     testHarness.set_pre_link_function({});
2111
2112     // Fetch the value and verify that the callback was called!
2113     checkeq(ENGINE_SUCCESS, get(h, h1, nullptr, &it, "key", 0), "get failed");
2114     check(h1->get_item_info(h, nullptr, it, &info), "Failed to get item info.");
2115     checkeq(0, memcmp(info.value[0].iov_base, "valuesome", 9),
2116            "Expected value to be modified");
2117     checkeq(pre_link_seqno, info.seqno, "Sequence numbers should match");
2118     h1->release(h, nullptr, it);
2119
2120     return SUCCESS;
2121 }
2122
2123 /**
2124  * verify that get_if works as expected
2125  */
2126 static test_result get_if(ENGINE_HANDLE* h, ENGINE_HANDLE_V1* h1) {
2127     item* it;
2128     const std::string key("get_if");
2129
2130     checkeq(ENGINE_SUCCESS,
2131             store(h, h1, nullptr, OPERATION_SET, key.c_str(), "somevalue", &it),
2132             "Failed set.");
2133     h1->release(h, nullptr, it);
2134
2135     if (isPersistentBucket(h, h1)) {
2136         wait_for_flusher_to_settle(h, h1);
2137         evict_key(h, h1, key.c_str(), 0, "Ejected.");
2138     }
2139
2140     auto doc = h1->get_if(h,
2141                           nullptr,
2142                           DocKey(key, testHarness.doc_namespace),
2143                           0,
2144                           [](const item_info&) {
2145                               return true;
2146                           });
2147     check(doc.second, "document should be found");
2148
2149     doc = h1->get_if(h,
2150                      nullptr,
2151                      DocKey(key, testHarness.doc_namespace),
2152                      0,
2153                      [](const item_info&) { return false; });
2154     check(!doc.second, "document should not be found");
2155
2156     doc = h1->get_if(h,
2157                      nullptr,
2158                      DocKey(std::string{"no"}, testHarness.doc_namespace),
2159                      0,
2160                      [](const item_info&) { return true; });
2161     check(!doc.second, "non-existing document should not be found");
2162
2163     checkeq(ENGINE_SUCCESS, del(h, h1, key.c_str(), 0, 0),
2164             "Failed remove with value");
2165
2166     doc = h1->get_if(h,
2167                      nullptr,
2168                      DocKey(key, testHarness.doc_namespace),
2169                      0,
2170                      [](const item_info&) { return true; });
2171     check(!doc.second, "deleted document should not be found");
2172
2173     return SUCCESS;
2174 }
2175
2176 ///////////////////////////////////////////////////////////////////////////////
2177 // Test manifest //////////////////////////////////////////////////////////////
2178 ///////////////////////////////////////////////////////////////////////////////
2179
2180 const char *default_dbname = "./ep_testsuite_basic";
2181
2182 BaseTestCase testsuite_testcases[] = {
2183         TestCase("test alloc limit", test_alloc_limit, test_setup, teardown,
2184                  NULL, prepare, cleanup),
2185         TestCase("test_memory_tracking", test_memory_tracking, test_setup,
2186                  teardown, NULL, prepare, cleanup),
2187         TestCase("test total memory limit", test_memory_limit,
2188                  test_setup, teardown,
2189                  "max_size=10240000;ht_locks=1;ht_size=3;"
2190                  "chk_remover_stime=1;chk_period=60",
2191                  prepare_ep_bucket, cleanup),
2192         TestCase("test max_size - water_mark changes",
2193                  test_max_size_and_water_marks_settings,
2194                  test_setup, teardown,
2195                  "max_size=1000;ht_locks=1;ht_size=3", prepare,
2196                  cleanup),
2197         TestCase("test whitespace dbname", test_whitespace_db,
2198                  test_setup, teardown,
2199                  "dbname=" WHITESPACE_DB ";ht_locks=1;ht_size=3",
2200                  prepare, cleanup),
2201         TestCase("get miss", test_get_miss, test_setup, teardown,
2202                  NULL, prepare, cleanup),
2203         TestCase("set", test_set, test_setup, teardown,
2204                  NULL, prepare, cleanup),
2205         TestCase("concurrent set", test_conc_set, test_setup,
2206                  teardown, NULL, prepare, cleanup),
2207         TestCase("multi set", test_multi_set, test_setup,
2208                  teardown, NULL, prepare, cleanup),
2209         TestCase("set+get hit", test_set_get_hit, test_setup,
2210                  teardown, NULL, prepare, cleanup),
2211         TestCase("test getl then del with cas", test_getl_delete_with_cas,
2212                  test_setup, teardown, NULL, prepare, cleanup),
2213         TestCase("test getl then del with bad cas",
2214                  test_getl_delete_with_bad_cas,
2215                  test_setup, teardown, NULL, prepare, cleanup),
2216         TestCase("test getl then set with meta",
2217                  test_getl_set_del_with_meta,
2218                  test_setup, teardown, NULL, prepare, cleanup),
2219         TestCase("getl", test_getl, test_setup, teardown,
2220                  NULL, prepare, cleanup),
2221         TestCase("unl",  test_unl, test_setup, teardown,
2222                  NULL, prepare, cleanup),
2223         TestCase("unl not my vbucket", test_unl_nmvb,
2224                  test_setup, teardown, NULL, prepare, cleanup),
2225         TestCase("set+get hit (bin)", test_set_get_hit_bin,
2226                  test_setup, teardown, NULL, prepare, cleanup),
2227         TestCase("set with cas non-existent", test_set_with_cas_non_existent,
2228                  test_setup, teardown, NULL, prepare, cleanup),
2229         TestCase("set+change flags", test_set_change_flags,
2230                  test_setup, teardown, NULL, prepare, cleanup),
2231         TestCase("add", test_add, test_setup, teardown, NULL,
2232                  prepare, cleanup),
2233         TestCase("add+add(same cas)", test_add_add_with_cas,
2234                  test_setup, teardown, NULL, prepare, cleanup),
2235         TestCase("cas", test_cas, test_setup, teardown, NULL,
2236                  prepare, cleanup),
2237         TestCase("replace", test_replace, test_setup, teardown,
2238                  NULL, prepare, cleanup),
2239         TestCase("test touch", test_touch, test_setup, teardown,
2240                  NULL, prepare, cleanup),
2241         TestCase("test touch (MB-7342)", test_touch_mb7342, test_setup, teardown,
2242                  NULL, prepare, cleanup),
2243         TestCase("test touch (MB-10277)", test_touch_mb10277, test_setup,
2244                  teardown, NULL, prepare_ep_bucket, cleanup),
2245         TestCase("test gat", test_gat, test_setup, teardown,
2246                  NULL, prepare, cleanup),
2247         TestCase("test locked gat", test_gat_locked,
2248                  test_setup, teardown, NULL, prepare, cleanup),
2249         TestCase("test locked touch", test_touch_locked,
2250                  test_setup, teardown, NULL, prepare, cleanup),
2251         TestCase("test mb5215", test_mb5215, test_setup, teardown,
2252                  NULL, prepare_ep_bucket, cleanup),
2253         TestCase("delete", test_delete, test_setup, teardown,
2254                  NULL, prepare, cleanup),
2255         TestCase("delete with value", test_delete_with_value, test_setup, teardown,
2256                  NULL, prepare, cleanup),
2257         TestCase("delete with value CAS", test_delete_with_value_cas,
2258                  test_setup, teardown, NULL, prepare, cleanup),
2259         TestCase("set/delete", test_set_delete, test_setup,
2260                  teardown, NULL, prepare, cleanup),
2261         TestCase("set/delete (invalid cas)", test_set_delete_invalid_cas,
2262                  test_setup, teardown, NULL, prepare, cleanup),
2263         TestCase("delete/set/delete", test_delete_set, test_setup,
2264                  teardown, NULL, prepare, cleanup),
2265         TestCase("get/delete with missing db file", test_get_delete_missing_file,
2266                  test_setup, teardown, NULL, prepare_ep_bucket, cleanup),
2267         TestCase("retain rowid over a soft delete", test_bug2509,
2268                  test_setup, teardown, NULL, prepare, cleanup),
2269         TestCase("vbucket deletion doesn't affect new data", test_bug7023,
2270                  test_setup, teardown, NULL,
2271                  /* TODO Ephemeral: broken till delete is implemented in seq
2272                   list */
2273                  prepare_skip_broken_under_ephemeral, cleanup),
2274         TestCase("non-resident decrementers", test_mb3169,
2275                  test_setup, teardown, NULL, prepare_ep_bucket, cleanup),
2276         TestCase("resident ratio after warmup", test_mb5172,
2277                  test_setup, teardown, NULL, prepare, cleanup),
2278         TestCase("set vb 10000", test_set_vbucket_out_of_range,
2279                  test_setup, teardown, "max_vbuckets=1024", prepare, cleanup),
2280         TestCase("flush", test_flush, test_setup, teardown,
2281                  NULL,
2282                  /* TODO Ephemeral: FLUSH currently not working*/prepare_skip_broken_under_ephemeral,
2283                  cleanup),
2284         TestCase("flush with stats", test_flush_stats, test_setup, teardown,
2285                  "flushall_enabled=true;chk_remover_stime=1;chk_period=60",
2286                  /* TODO Ephemeral: FLUSH currently not working*/prepare_skip_broken_under_ephemeral,
2287                  cleanup),
2288         TestCase("flush multi vbuckets", test_flush_multiv,
2289                  test_setup, teardown,
2290                  "flushall_enabled=true;max_vbuckets=16;ht_size=7;ht_locks=3",
2291                  /* TODO Ephemeral: FLUSH currently not working*/prepare_skip_broken_under_ephemeral,
2292                  cleanup),
2293         TestCase("flush_disabled", test_flush_disabled, test_setup, teardown,
2294                  "flushall_enabled=false;max_vbuckets=16;ht_size=7;ht_locks=3",
2295                  prepare, cleanup),
2296         TestCase("flushall params", test_CBD_152, test_setup, teardown,
2297                  "flushall_enabled=true;max_vbuckets=16;"
2298                  "ht_size=7;ht_locks=3",
2299                  /* TODO Ephemeral: FLUSH currently not working*/prepare_skip_broken_under_ephemeral,
2300                  cleanup),
2301         TestCase("set max_cas MB21190", set_max_cas_mb21190, test_setup, teardown, nullptr,
2302                  prepare, cleanup),
2303         TestCase("warmup_mb21769", warmup_mb21769, test_setup, teardown, nullptr,
2304                  prepare, cleanup),
2305
2306         TestCase("pre_link_document", pre_link_document, test_setup, teardown, nullptr,
2307                  prepare, cleanup),
2308
2309         TestCase("engine get_if", get_if, test_setup, teardown, nullptr,
2310                  prepare, cleanup),
2311
2312         // sentinel
2313         TestCase(NULL, NULL, NULL, NULL, NULL, prepare, cleanup)
2314 };