MB-23906: Implement delete-with-value with store() instead of delete()
[ep-engine.git] / tests / module_tests / evp_store_rollback_test.cc
1 /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 /*
3  *     Copyright 2016 Couchbase, Inc
4  *
5  *   Licensed under the Apache License, Version 2.0 (the "License");
6  *   you may not use this file except in compliance with the License.
7  *   You may obtain a copy of the License at
8  *
9  *       http://www.apache.org/licenses/LICENSE-2.0
10  *
11  *   Unless required by applicable law or agreed to in writing, software
12  *   distributed under the License is distributed on an "AS IS" BASIS,
13  *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  *   See the License for the specific language governing permissions and
15  *   limitations under the License.
16  */
17
18 /**
19  * Tests for Rollback functionality in EPStore.
20  */
21
22 #include "dcp/dcpconnmap.h"
23 #include "evp_store_single_threaded_test.h"
24 #include "evp_store_test.h"
25 #include "failover-table.h"
26 #include "programs/engine_testapp/mock_server.h"
27 #include "tests/mock/mock_dcp.h"
28 #include "tests/mock/mock_dcp_consumer.h"
29 #include "tests/module_tests/test_helpers.h"
30
31 class RollbackTest : public EPBucketTest,
32                      public ::testing::WithParamInterface<std::string>
33 {
34     void SetUp() override {
35         EPBucketTest::SetUp();
36         // Start vbucket as active to allow us to store items directly to it.
37         store->setVBucketState(vbid, vbucket_state_active, false);
38
39         // For any rollback tests which actually want to rollback, we need
40         // to ensure that we don't rollback more than 50% of the seqno count
41         // as then the VBucket is just cleared (it'll instead expect a resync
42         // from zero.
43         // Therefore create 10 dummy items which we don't otherwise care
44         // about (most of the Rollback test only work with a couple of
45         // "active" items.
46         const auto dummy_elements = size_t{5};
47         for (size_t ii = 1; ii <= dummy_elements; ii++) {
48             auto res = store_item(vbid,
49                                   makeStoredDocKey("dummy" + std::to_string(ii)),
50                                   "dummy");
51             ASSERT_EQ(ii, res.getBySeqno());
52         }
53         ASSERT_EQ(dummy_elements, store->flushVBucket(vbid));
54         initial_seqno = dummy_elements;
55     }
56
57 protected:
58     /*
59      * Fake callback emulating dcp_add_failover_log
60      */
61     static ENGINE_ERROR_CODE fakeDcpAddFailoverLog(vbucket_failover_t* entry,
62                                                    size_t nentries,
63                                                    const void *cookie) {
64         return ENGINE_SUCCESS;
65     }
66
67     /**
68      * Test rollback after deleting an item.
69      * @param flush_before_rollback: Should the vbuckt be flushed to disk just
70      *        before the rollback (i.e. guaranteeing the in-memory state is in sync
71      *        with disk).
72      */
73     void rollback_after_deletion_test(bool flush_before_rollback) {
74         // Setup: Store an item then flush the vBucket (creating a checkpoint);
75         // then delete the item and create a second checkpoint.
76         StoredDocKey a = makeStoredDocKey("key");
77         auto item_v1 = store_item(vbid, a, "1");
78         ASSERT_EQ(initial_seqno + 1, item_v1.getBySeqno());
79         ASSERT_EQ(1, store->flushVBucket(vbid));
80         uint64_t cas = item_v1.getCas();
81         ASSERT_EQ(ENGINE_SUCCESS,
82                   store->deleteItem(a,
83                                     cas,
84                                     vbid,
85                                     /*cookie*/ nullptr,
86                                     /*itemMeta*/ nullptr,
87                                     /*mutation_descr_t*/ nullptr));
88         if (flush_before_rollback) {
89             ASSERT_EQ(1, store->flushVBucket(vbid));
90         }
91         // Sanity-check - item should no longer exist.
92         EXPECT_EQ(ENGINE_KEY_ENOENT,
93                   store->get(a, vbid, nullptr, {}).getStatus());
94
95         // Test - rollback to seqno of item_v1 and verify that the previous value
96         // of the item has been restored.
97         store->setVBucketState(vbid, vbucket_state_replica, false);
98         ASSERT_EQ(ENGINE_SUCCESS, store->rollback(vbid, item_v1.getBySeqno()));
99         auto result = getInternal(
100                 a, vbid, /*cookie*/ nullptr, vbucket_state_replica, {});
101         ASSERT_EQ(ENGINE_SUCCESS, result.getStatus());
102         EXPECT_EQ(item_v1, *result.getValue())
103             << "Fetched item after rollback should match item_v1";
104         delete result.getValue();
105
106         if (!flush_before_rollback) {
107             EXPECT_EQ(0, store->flushVBucket(vbid));
108         }
109     }
110
111     // Test rollback after modifying an item.
112     void rollback_after_mutation_test(bool flush_before_rollback) {
113         // Setup: Store an item then flush the vBucket (creating a checkpoint);
114         // then update the item with a new value and create a second checkpoint.
115         StoredDocKey a = makeStoredDocKey("a");
116         auto item_v1 = store_item(vbid, a, "old");
117         ASSERT_EQ(initial_seqno + 1, item_v1.getBySeqno());
118         ASSERT_EQ(1, store->flushVBucket(vbid));
119
120         auto item2 = store_item(vbid, a, "new");
121         ASSERT_EQ(initial_seqno + 2, item2.getBySeqno());
122
123         StoredDocKey key = makeStoredDocKey("key");
124         store_item(vbid, key, "meh");
125
126         if (flush_before_rollback) {
127             EXPECT_EQ(2, store->flushVBucket(vbid));
128         }
129
130         // Test - rollback to seqno of item_v1 and verify that the previous value
131         // of the item has been restored.
132         store->setVBucketState(vbid, vbucket_state_replica, false);
133         ASSERT_EQ(ENGINE_SUCCESS, store->rollback(vbid, item_v1.getBySeqno()));
134         ASSERT_EQ(item_v1.getBySeqno(), store->getVBucket(vbid)->getHighSeqno());
135
136         // a should have the value of 'old'
137         {
138             auto result = store->get(a, vbid, nullptr, {});
139             ASSERT_EQ(ENGINE_SUCCESS, result.getStatus());
140             EXPECT_EQ(item_v1, *result.getValue())
141                 << "Fetched item after rollback should match item_v1";
142             delete result.getValue();
143         }
144
145         // key should be gone
146         {
147             auto result = store->get(key, vbid, nullptr, {});
148             EXPECT_EQ(ENGINE_KEY_ENOENT, result.getStatus())
149                 << "A key set after the rollback point was found";
150         }
151
152         if (!flush_before_rollback) {
153             // The rollback should of wiped out any keys waiting for persistence
154             EXPECT_EQ(0, store->flushVBucket(vbid));
155         }
156     }
157
158 // This test triggers MSVC 'cl' to assert, a lot of time has been spent trying
159 // to tweak the code so it compiles, but no solution yet. Disabled for VS 2013
160 #if !defined(_MSC_VER) || _MSC_VER != 1800
161     void rollback_to_middle_test(bool flush_before_rollback) {
162         // create some more checkpoints just to see a few iterations
163         // of parts of the rollback function.
164
165         // need to store a certain number of keys because rollback
166         // 'bails' if the rollback is too much.
167         for (int i = 0; i < 6; i++) {
168             store_item(vbid, makeStoredDocKey("key_" + std::to_string(i)), "dontcare");
169         }
170         // the roll back function will rewind disk to key7.
171         auto rollback_item = store_item(vbid, makeStoredDocKey("key7"), "dontcare");
172         ASSERT_EQ(7, store->flushVBucket(vbid));
173
174         // every key past this point will be lost from disk in a mid-point.
175         auto item_v1 = store_item(vbid, makeStoredDocKey("rollback-cp-1"), "keep-me");
176         auto item_v2 = store_item(vbid, makeStoredDocKey("rollback-cp-2"), "rollback to me");
177         store_item(vbid, makeStoredDocKey("rollback-cp-3"), "i'm gone");
178         auto rollback = item_v2.getBySeqno(); // ask to rollback to here.
179         ASSERT_EQ(3, store->flushVBucket(vbid));
180
181         for (int i = 0; i < 3; i++) {
182             store_item(vbid, makeStoredDocKey("anotherkey_" + std::to_string(i)), "dontcare");
183         }
184
185         if (flush_before_rollback) {
186             ASSERT_EQ(3, store->flushVBucket(vbid));
187         }
188
189
190         // Rollback should succeed, but rollback to 0
191         store->setVBucketState(vbid, vbucket_state_replica, false);
192         EXPECT_EQ(ENGINE_SUCCESS, store->rollback(vbid, rollback));
193
194         // These keys should be gone after the rollback
195         for (int i = 0; i < 3; i++) {
196             auto result = store->get(makeStoredDocKey("rollback-cp-" + std::to_string(i)), vbid, nullptr, {});
197             EXPECT_EQ(ENGINE_KEY_ENOENT, result.getStatus())
198                 << "A key set after the rollback point was found";
199         }
200
201         // These keys should be gone after the rollback
202         for (int i = 0; i < 3; i++) {
203             auto result = store->get(makeStoredDocKey("anotherkey_" + std::to_string(i)), vbid, nullptr, {});
204             EXPECT_EQ(ENGINE_KEY_ENOENT, result.getStatus())
205                 << "A key set after the rollback point was found";
206         }
207
208         // Rolled back to the previous checkpoint
209         EXPECT_EQ(rollback_item.getBySeqno(),
210                   store->getVBucket(vbid)->getHighSeqno());
211     }
212 #endif
213
214 protected:
215     int64_t initial_seqno;
216 };
217
218 TEST_P(RollbackTest, RollbackAfterMutation) {
219     rollback_after_mutation_test(/*flush_before_rollbaack*/true);
220 }
221
222 TEST_P(RollbackTest, RollbackAfterMutationNoFlush) {
223     rollback_after_mutation_test(/*flush_before_rollback*/false);
224 }
225
226 TEST_P(RollbackTest, RollbackAfterDeletion) {
227     rollback_after_deletion_test(/*flush_before_rollback*/true);
228 }
229
230 TEST_P(RollbackTest, RollbackAfterDeletionNoFlush) {
231     rollback_after_deletion_test(/*flush_before_rollback*/false);
232 }
233
234 #if !defined(_MSC_VER) || _MSC_VER != 1800
235 TEST_P(RollbackTest, RollbackToMiddleOfAPersistedSnapshot) {
236     rollback_to_middle_test(true);
237 }
238
239 TEST_P(RollbackTest, RollbackToMiddleOfAPersistedSnapshotNoFlush) {
240     rollback_to_middle_test(false);
241 }
242
243 TEST_P(RollbackTest, RollbackToMiddleOfAnUnPersistedSnapshot) {
244     /* need to store a certain number of keys because rollback
245        'bails (rolls back to 0)' if the rollback is too much. */
246     const int numItems = 10;
247     for (int i = 0; i < numItems; i++) {
248         store_item(vbid,
249                    makeStoredDocKey("key_" + std::to_string(i)),
250                    "not rolled back");
251     }
252
253     /* the roll back function will rewind disk to key11. */
254     auto rollback_item =
255             store_item(vbid, makeStoredDocKey("key11"), "rollback pt");
256
257     ASSERT_EQ(numItems + 1, store->flushVBucket(vbid));
258
259     /* Keys to be lost in rollback */
260     auto item_v1 = store_item(
261             vbid, makeStoredDocKey("rollback-cp-1"), "hope to keep till here");
262     /* ask to rollback to here; this item is in a checkpoint and
263        is not persisted */
264     auto rollbackReqSeqno = item_v1.getBySeqno();
265
266     auto item_v2 = store_item(vbid, makeStoredDocKey("rollback-cp-2"), "gone");
267
268     /* do rollback */
269     store->setVBucketState(vbid, vbucket_state_replica, false);
270     EXPECT_EQ(ENGINE_SUCCESS, store->rollback(vbid, rollbackReqSeqno));
271
272     /* confirm that we have rolled back to the disk snapshot */
273     EXPECT_EQ(rollback_item.getBySeqno(),
274               store->getVBucket(vbid)->getHighSeqno());
275
276     /* since we rely only on disk snapshots currently, we must lose the items in
277        the checkpoints */
278     for (int i = 0; i < 2; i++) {
279         auto result =
280                 store->get(makeStoredDocKey("rollback-cp-" + std::to_string(i)),
281                            vbid,
282                            nullptr,
283                            {});
284         EXPECT_EQ(ENGINE_KEY_ENOENT, result.getStatus())
285                 << "A key set after the rollback point was found";
286     }
287 }
288 #endif
289
290 /*
291  * The opencheckpointid of a bucket can be zero after a rollback.
292  * From MB21784 if an opencheckpointid was zero it was assumed that the
293  * vbucket was in backfilling state.  This caused the producer stream
294  * request to be stuck waiting for backfilling to complete.
295  */
296 TEST_P(RollbackTest, MB21784) {
297     // Make the vbucket a replica
298     store->setVBucketState(vbid, vbucket_state_replica, false);
299     // Perform a rollback
300     EXPECT_EQ(ENGINE_SUCCESS, store->rollback(vbid, initial_seqno))
301         << "rollback did not return ENGINE_SUCCESS";
302
303     // Assert the checkpointmanager clear function (called during rollback)
304     // has set the opencheckpointid to zero
305     auto vb = store->getVBucket(vbid);
306     auto& ckpt_mgr = vb->checkpointManager;
307     EXPECT_EQ(0, ckpt_mgr.getOpenCheckpointId()) << "opencheckpointId not zero";
308
309     // Create a new Dcp producer, reserving its cookie.
310     get_mock_server_api()->cookie->reserve(cookie);
311     dcp_producer_t producer = engine->getDcpConnMap().newProducer(
312             cookie, "test_producer", /*notifyOnly*/false, /*isKeyOnly*/false);
313
314     uint64_t rollbackSeqno;
315     auto err = producer->streamRequest(/*flags*/0,
316                                        /*opaque*/0,
317                                        /*vbucket*/vbid,
318                                        /*start_seqno*/0,
319                                        /*end_seqno*/0,
320                                        /*vb_uuid*/0,
321                                        /*snap_start*/0,
322                                        /*snap_end*/0,
323                                        &rollbackSeqno,
324                                        RollbackTest::fakeDcpAddFailoverLog);
325     EXPECT_EQ(ENGINE_SUCCESS, err)
326         << "stream request did not return ENGINE_SUCCESS";
327     // Close stream
328     ASSERT_EQ(ENGINE_SUCCESS, producer->closeStream(/*opaque*/0, vbid));
329     engine->handleDisconnect(cookie);
330 }
331
332 class RollbackDcpTest : public SingleThreadedEPBucketTest {
333 public:
334     RollbackDcpTest()
335         : cookie(create_mock_cookie()),
336           producers(get_dcp_producers(nullptr, nullptr)) {
337     }
338
339     void SetUp() override {
340         SingleThreadedEPBucketTest::SetUp();
341         store->setVBucketState(vbid, vbucket_state_active, false);
342         consumer = new MockDcpConsumer(*engine, cookie, "test_consumer");
343         vb = store->getVBucket(vbid);
344         producers->stream_req = &RollbackDcpTest::streamRequest;
345     }
346
347     void TearDown() override {
348         consumer->closeAllStreams();
349         destroy_mock_cookie(cookie);
350         consumer.reset();
351         vb.reset();
352         SingleThreadedEPBucketTest::TearDown();
353     }
354
355     // build a rollback response command
356     std::unique_ptr<char[]> getRollbackResponse(uint32_t opaque,
357                                                 uint64_t rollbackSeq) const {
358         auto msg = std::make_unique<char[]>(
359                 sizeof(protocol_binary_response_header) + sizeof(uint64_t));
360         auto* p = reinterpret_cast<protocol_binary_response_dcp_stream_req*>(
361                 msg.get());
362
363         p->message.header.response.opcode = PROTOCOL_BINARY_CMD_DCP_STREAM_REQ;
364         p->message.header.response.status =
365                 htons(PROTOCOL_BINARY_RESPONSE_ROLLBACK);
366         p->message.header.response.opaque = opaque;
367         p->message.header.response.bodylen = htonl(sizeof(uint64_t));
368
369         auto* seq = reinterpret_cast<uint64_t*>(p + 1);
370         *seq = htonll(rollbackSeq);
371         return msg;
372     }
373
374     static struct StreamRequestData {
375         bool called;
376         uint32_t opaque;
377         uint16_t vbucket;
378         uint32_t flags;
379         uint64_t start_seqno;
380         uint64_t end_seqno;
381         uint64_t vbucket_uuid;
382         uint64_t snap_start_seqno;
383         uint64_t snap_end_seqno;
384     } streamRequestData;
385
386     static ENGINE_ERROR_CODE streamRequest(const void* cookie,
387                                            uint32_t opaque,
388                                            uint16_t vbucket,
389                                            uint32_t flags,
390                                            uint64_t start_seqno,
391                                            uint64_t end_seqno,
392                                            uint64_t vbucket_uuid,
393                                            uint64_t snap_start_seqno,
394                                            uint64_t snap_end_seqno) {
395         streamRequestData = {true,
396                              opaque,
397                              vbucket,
398                              flags,
399                              start_seqno,
400                              end_seqno,
401                              vbucket_uuid,
402                              snap_start_seqno,
403                              snap_end_seqno};
404
405         return ENGINE_SUCCESS;
406     }
407
408     void stepForStreamRequest(uint64_t startSeqno, uint64_t vbUUID) {
409         while (consumer->step(producers.get()) == ENGINE_WANT_MORE) {
410         }
411         EXPECT_TRUE(streamRequestData.called);
412         EXPECT_EQ(startSeqno, streamRequestData.start_seqno);
413         EXPECT_EQ(vbUUID, streamRequestData.vbucket_uuid);
414         streamRequestData = {};
415     }
416
417     void createItems(int items, int flushes) {
418         // Flush multiple checkpoints of unique keys
419         for (int ii = 0; ii < flushes; ii++) {
420             EXPECT_TRUE(store_items(items,
421                                     vbid,
422                                     {"anykey_" + std::to_string(ii) + "_",
423                                      DocNamespace::DefaultCollection},
424                                     "value"));
425             flush_vbucket_to_disk(vbid, items);
426             // Add an entry for this seqno
427             vb->failovers->createEntry(items * (ii + 1));
428         }
429
430         store->setVBucketState(vbid, vbucket_state_replica, false);
431     }
432
433     uint64_t addStream(int nitems) {
434         consumer->addStream(/*opaque*/ 0, vbid, /*flags*/ 0);
435         // Step consumer to retrieve the first stream request.
436         uint64_t vbUUID = vb->failovers->getLatestEntry().vb_uuid;
437         stepForStreamRequest(nitems, vbUUID);
438         return vbUUID;
439     }
440
441     void responseNoRollback(int nitems,
442                             uint64_t rollbackSeq,
443                             uint64_t previousUUID) {
444         // Now push a reponse to the consumer, saying rollback to 0.
445         // The consumer must ignore the 0 rollback and retry a stream-request
446         // with the next failover entry.
447         auto msg = getRollbackResponse(1 /*opaque*/, rollbackSeq);
448         EXPECT_TRUE(consumer->handleResponse(
449                 reinterpret_cast<protocol_binary_response_header*>(msg.get())));
450
451         // Consumer should of added a StreamRequest with a different vbuuid
452         EXPECT_NE(previousUUID, vb->failovers->getLatestEntry().vb_uuid);
453
454         stepForStreamRequest(nitems, vb->failovers->getLatestEntry().vb_uuid);
455     }
456
457     void responseRollback(uint64_t rollbackSeq) {
458         // Now push a reponse to the consumer, saying rollback to 0.
459         // The consumer must ignore the 0 rollback and retry a stream-request
460         // with the next failover entry.
461         auto msg = getRollbackResponse(1 /*opaque*/, rollbackSeq);
462         EXPECT_TRUE(consumer->handleResponse(
463                 reinterpret_cast<protocol_binary_response_header*>(msg.get())));
464
465         // consumer must of scheduled a RollbackTask (writer task)
466         auto& lpWriteQ = *task_executor->getLpTaskQ()[WRITER_TASK_IDX];
467         ASSERT_EQ(1, lpWriteQ.getFutureQueueSize());
468         runNextTask(lpWriteQ);
469     }
470
471     const void* cookie;
472     SingleThreadedRCPtr<MockDcpConsumer> consumer;
473     std::unique_ptr<dcp_message_producers> producers;
474     VBucketPtr vb;
475 };
476
477 RollbackDcpTest::StreamRequestData RollbackDcpTest::streamRequestData = {};
478
479 /**
480  * Push stream responses to a consumer and test
481  * 1. The first rollback to 0 response is ignored, the consumer requests again
482  *    with new data.
483  * 2. The second rollback to 0 response triggers a rollback to 0.
484  */
485 TEST_F(RollbackDcpTest, test_rollback_zero) {
486     const int items = 40;
487     const int flushes = 1;
488     const int nitems = items * flushes;
489     const int rollbackPoint = 0; // expect final rollback to be to 0
490
491     // Test will create anykey_0_{0..items-1}
492     createItems(items, flushes);
493
494     auto uuid = addStream(nitems);
495
496     responseNoRollback(nitems, 0, uuid);
497
498     // All keys available
499     for (int ii = 0; ii < items; ii++) {
500         std::string key = "anykey_0_" + std::to_string(ii);
501         auto result = store->get(
502                 {key, DocNamespace::DefaultCollection}, vbid, nullptr, {});
503         EXPECT_EQ(ENGINE_SUCCESS, result.getStatus()) << "Problem with " << key;
504         delete result.getValue();
505     }
506
507     responseRollback(rollbackPoint);
508
509     // All keys now gone
510     for (int ii = 0; ii < items; ii++) {
511         std::string key = "anykey_0_" + std::to_string(ii);
512         auto result = store->get(
513                 {key, DocNamespace::DefaultCollection}, vbid, nullptr, {});
514         EXPECT_EQ(ENGINE_KEY_ENOENT, result.getStatus()) << "Problem with "
515                                                          << key;
516     }
517
518     // Expected a rollback to 0 which is a VB reset, so discard the now dead
519     // vb and obtain replacement
520     vb = store->getVBucket(vbid);
521
522     // Rollback complete and will have posted a new StreamRequest
523     stepForStreamRequest(rollbackPoint,
524                          vb->failovers->getLatestEntry().vb_uuid);
525     EXPECT_EQ(rollbackPoint, vb->getHighSeqno()) << "VB hasn't rolled back to "
526                                                  << rollbackPoint;
527 }
528
529 /**
530  * Push stream responses to a consumer and test
531  * 1. The first rollback to 0 response is ignored, the consumer requests again
532  *    with new data.
533  * 2. The second rollback response is non-zero, and the consumer accepts that
534  *    and rolls back to the rollbackPoint and requests a stream for it.
535  */
536 TEST_F(RollbackDcpTest, test_rollback_nonzero) {
537     const int items = 10;
538     const int flushes = 4;
539     const int nitems = items * flushes;
540     const int rollbackPoint = 3 * items; // rollback to 3/4
541
542     // Test will create anykey_{0..flushes-1}_{0..items-1}
543     createItems(items, flushes);
544
545     auto uuid = addStream(nitems);
546
547     responseNoRollback(nitems, 0, uuid);
548
549     // All keys available
550     for (int ii = 0; ii < items; ii++) {
551         for (int ff = 0; ff < flushes; ff++) {
552             std::string key =
553                     "anykey_" + std::to_string(ff) + "_" + std::to_string(ii);
554             auto result = store->get(
555                     {key, DocNamespace::DefaultCollection}, vbid, nullptr, {});
556             EXPECT_EQ(ENGINE_SUCCESS, result.getStatus()) << "Expected to find "
557                                                           << key;
558             delete result.getValue();
559         }
560     }
561
562     responseRollback(rollbackPoint);
563
564     // 3/4 keys available
565     for (int ii = 0; ii < items; ii++) {
566         for (int ff = 0; ff < 3; ff++) {
567             std::string key =
568                     "anykey_" + std::to_string(ff) + "_" + std::to_string(ii);
569             auto result = store->get(
570                     {key, DocNamespace::DefaultCollection}, vbid, nullptr, {});
571             EXPECT_EQ(ENGINE_SUCCESS, result.getStatus()) << "Expected to find "
572                                                           << key;
573             delete result.getValue();
574         }
575     }
576
577     // Final 1/4 were discarded by the rollback
578     for (int ii = 0; ii < items; ii++) {
579         std::string key = "anykey_3_" + std::to_string(ii);
580         auto result = store->get(
581                 {key, DocNamespace::DefaultCollection}, vbid, nullptr, {});
582         EXPECT_EQ(ENGINE_KEY_ENOENT, result.getStatus()) << "Problem with "
583                                                          << key;
584     }
585
586     // Rollback complete and will have posted a new StreamRequest
587     stepForStreamRequest(rollbackPoint,
588                          vb->failovers->getLatestEntry().vb_uuid);
589     EXPECT_EQ(rollbackPoint, vb->getHighSeqno()) << "VB hasn't rolled back to "
590                                                  << rollbackPoint;
591 }
592
593 // Test cases which run in both Full and Value eviction
594 INSTANTIATE_TEST_CASE_P(FullAndValueEviction,
595                         RollbackTest,
596                         ::testing::Values("value_only", "full_eviction"),
597                         [] (const ::testing::TestParamInfo<std::string>& info) {
598                             return info.param;
599                         });