Merge branch 'watson'
[ep-engine.git] / tests / module_tests / evp_store_rollback_test.cc
1 /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 /*
3  *     Copyright 2016 Couchbase, Inc
4  *
5  *   Licensed under the Apache License, Version 2.0 (the "License");
6  *   you may not use this file except in compliance with the License.
7  *   You may obtain a copy of the License at
8  *
9  *       http://www.apache.org/licenses/LICENSE-2.0
10  *
11  *   Unless required by applicable law or agreed to in writing, software
12  *   distributed under the License is distributed on an "AS IS" BASIS,
13  *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  *   See the License for the specific language governing permissions and
15  *   limitations under the License.
16  */
17
18 /**
19  * Tests for Rollback functionality in EPStore.
20  */
21
22 #include "dcp/dcpconnmap.h"
23 #include "evp_store_single_threaded_test.h"
24 #include "evp_store_test.h"
25 #include "failover-table.h"
26 #include "programs/engine_testapp/mock_server.h"
27 #include "tests/mock/mock_dcp.h"
28 #include "tests/mock/mock_dcp_consumer.h"
29 #include "tests/module_tests/test_helpers.h"
30
31 class RollbackTest : public EPBucketTest,
32                      public ::testing::WithParamInterface<std::string>
33 {
34     void SetUp() override {
35         EPBucketTest::SetUp();
36         // Start vbucket as active to allow us to store items directly to it.
37         store->setVBucketState(vbid, vbucket_state_active, false);
38
39         // For any rollback tests which actually want to rollback, we need
40         // to ensure that we don't rollback more than 50% of the seqno count
41         // as then the VBucket is just cleared (it'll instead expect a resync
42         // from zero.
43         // Therefore create 10 dummy items which we don't otherwise care
44         // about (most of the Rollback test only work with a couple of
45         // "active" items.
46         const auto dummy_elements = size_t{5};
47         for (size_t ii = 1; ii <= dummy_elements; ii++) {
48             auto res = store_item(vbid,
49                                   makeStoredDocKey("dummy" + std::to_string(ii)),
50                                   "dummy");
51             ASSERT_EQ(ii, res.getBySeqno());
52         }
53         ASSERT_EQ(dummy_elements, store->flushVBucket(vbid));
54         initial_seqno = dummy_elements;
55     }
56
57 protected:
58     /*
59      * Fake callback emulating dcp_add_failover_log
60      */
61     static ENGINE_ERROR_CODE fakeDcpAddFailoverLog(vbucket_failover_t* entry,
62                                                    size_t nentries,
63                                                    const void *cookie) {
64         return ENGINE_SUCCESS;
65     }
66
67     /**
68      * Test rollback after deleting an item.
69      * @param flush_before_rollback: Should the vbuckt be flushed to disk just
70      *        before the rollback (i.e. guaranteeing the in-memory state is in sync
71      *        with disk).
72      */
73     void rollback_after_deletion_test(bool flush_before_rollback) {
74         // Setup: Store an item then flush the vBucket (creating a checkpoint);
75         // then delete the item and create a second checkpoint.
76         StoredDocKey a = makeStoredDocKey("key");
77         auto item_v1 = store_item(vbid, a, "1");
78         ASSERT_EQ(initial_seqno + 1, item_v1.getBySeqno());
79         ASSERT_EQ(1, store->flushVBucket(vbid));
80         uint64_t cas = item_v1.getCas();
81         ASSERT_EQ(ENGINE_SUCCESS,
82                   store->deleteItem(a,
83                                     cas,
84                                     vbid,
85                                     /*cookie*/ nullptr,
86                                     /*itemMeta*/ nullptr,
87                                     /*mutation_descr_t*/ nullptr));
88         if (flush_before_rollback) {
89             ASSERT_EQ(1, store->flushVBucket(vbid));
90         }
91         // Sanity-check - item should no longer exist.
92         EXPECT_EQ(ENGINE_KEY_ENOENT,
93                   store->get(a, vbid, nullptr, {}).getStatus());
94
95         // Test - rollback to seqno of item_v1 and verify that the previous value
96         // of the item has been restored.
97         store->setVBucketState(vbid, vbucket_state_replica, false);
98         ASSERT_EQ(ENGINE_SUCCESS, store->rollback(vbid, item_v1.getBySeqno()));
99         auto result = getInternal(
100                 a, vbid, /*cookie*/ nullptr, vbucket_state_replica, {});
101         ASSERT_EQ(ENGINE_SUCCESS, result.getStatus());
102         EXPECT_EQ(item_v1, *result.getValue())
103             << "Fetched item after rollback should match item_v1";
104         delete result.getValue();
105
106         if (!flush_before_rollback) {
107             EXPECT_EQ(0, store->flushVBucket(vbid));
108         }
109     }
110
111     // Test rollback after modifying an item.
112     void rollback_after_mutation_test(bool flush_before_rollback) {
113         // Setup: Store an item then flush the vBucket (creating a checkpoint);
114         // then update the item with a new value and create a second checkpoint.
115         StoredDocKey a = makeStoredDocKey("a");
116         auto item_v1 = store_item(vbid, a, "old");
117         ASSERT_EQ(initial_seqno + 1, item_v1.getBySeqno());
118         ASSERT_EQ(1, store->flushVBucket(vbid));
119
120         auto item2 = store_item(vbid, a, "new");
121         ASSERT_EQ(initial_seqno + 2, item2.getBySeqno());
122
123         StoredDocKey key = makeStoredDocKey("key");
124         store_item(vbid, key, "meh");
125
126         if (flush_before_rollback) {
127             EXPECT_EQ(2, store->flushVBucket(vbid));
128         }
129
130         // Test - rollback to seqno of item_v1 and verify that the previous value
131         // of the item has been restored.
132         store->setVBucketState(vbid, vbucket_state_replica, false);
133         ASSERT_EQ(ENGINE_SUCCESS, store->rollback(vbid, item_v1.getBySeqno()));
134         ASSERT_EQ(item_v1.getBySeqno(), store->getVBucket(vbid)->getHighSeqno());
135
136         // a should have the value of 'old'
137         {
138             auto result = store->get(a, vbid, nullptr, {});
139             ASSERT_EQ(ENGINE_SUCCESS, result.getStatus());
140             EXPECT_EQ(item_v1, *result.getValue())
141                 << "Fetched item after rollback should match item_v1";
142             delete result.getValue();
143         }
144
145         // key should be gone
146         {
147             auto result = store->get(key, vbid, nullptr, {});
148             EXPECT_EQ(ENGINE_KEY_ENOENT, result.getStatus())
149                 << "A key set after the rollback point was found";
150         }
151
152         if (!flush_before_rollback) {
153             // The rollback should of wiped out any keys waiting for persistence
154             EXPECT_EQ(0, store->flushVBucket(vbid));
155         }
156     }
157
158 // This test triggers MSVC 'cl' to assert, a lot of time has been spent trying
159 // to tweak the code so it compiles, but no solution yet. Disabled for VS 2013
160 #if !defined(_MSC_VER) || _MSC_VER != 1800
161     void rollback_to_middle_test(bool flush_before_rollback) {
162         // create some more checkpoints just to see a few iterations
163         // of parts of the rollback function.
164
165         // need to store a certain number of keys because rollback
166         // 'bails' if the rollback is too much.
167         for (int i = 0; i < 6; i++) {
168             store_item(vbid, makeStoredDocKey("key_" + std::to_string(i)), "dontcare");
169         }
170         // the roll back function will rewind disk to key7.
171         auto rollback_item = store_item(vbid, makeStoredDocKey("key7"), "dontcare");
172         ASSERT_EQ(7, store->flushVBucket(vbid));
173
174         // every key past this point will be lost from disk in a mid-point.
175         auto item_v1 = store_item(vbid, makeStoredDocKey("rollback-cp-1"), "keep-me");
176         auto item_v2 = store_item(vbid, makeStoredDocKey("rollback-cp-2"), "rollback to me");
177         store_item(vbid, makeStoredDocKey("rollback-cp-3"), "i'm gone");
178         auto rollback = item_v2.getBySeqno(); // ask to rollback to here.
179         ASSERT_EQ(3, store->flushVBucket(vbid));
180
181         for (int i = 0; i < 3; i++) {
182             store_item(vbid, makeStoredDocKey("anotherkey_" + std::to_string(i)), "dontcare");
183         }
184
185         if (flush_before_rollback) {
186             ASSERT_EQ(3, store->flushVBucket(vbid));
187         }
188
189
190         // Rollback should succeed, but rollback to 0
191         store->setVBucketState(vbid, vbucket_state_replica, false);
192         EXPECT_EQ(ENGINE_SUCCESS, store->rollback(vbid, rollback));
193
194         // These keys should be gone after the rollback
195         for (int i = 0; i < 3; i++) {
196             auto result = store->get(makeStoredDocKey("rollback-cp-" + std::to_string(i)), vbid, nullptr, {});
197             EXPECT_EQ(ENGINE_KEY_ENOENT, result.getStatus())
198                 << "A key set after the rollback point was found";
199         }
200
201         // These keys should be gone after the rollback
202         for (int i = 0; i < 3; i++) {
203             auto result = store->get(makeStoredDocKey("anotherkey_" + std::to_string(i)), vbid, nullptr, {});
204             EXPECT_EQ(ENGINE_KEY_ENOENT, result.getStatus())
205                 << "A key set after the rollback point was found";
206         }
207
208         // Rolled back to the previous checkpoint
209         EXPECT_EQ(rollback_item.getBySeqno(),
210                   store->getVBucket(vbid)->getHighSeqno());
211     }
212 #endif
213
214 protected:
215     int64_t initial_seqno;
216 };
217
218 TEST_P(RollbackTest, RollbackAfterMutation) {
219     rollback_after_mutation_test(/*flush_before_rollbaack*/true);
220 }
221
222 TEST_P(RollbackTest, RollbackAfterMutationNoFlush) {
223     rollback_after_mutation_test(/*flush_before_rollback*/false);
224 }
225
226 TEST_P(RollbackTest, RollbackAfterDeletion) {
227     rollback_after_deletion_test(/*flush_before_rollback*/true);
228 }
229
230 TEST_P(RollbackTest, RollbackAfterDeletionNoFlush) {
231     rollback_after_deletion_test(/*flush_before_rollback*/false);
232 }
233
234 #if !defined(_MSC_VER) || _MSC_VER != 1800
235 TEST_P(RollbackTest, RollbackToMiddleOfAPersistedSnapshot) {
236     rollback_to_middle_test(true);
237 }
238
239 TEST_P(RollbackTest, RollbackToMiddleOfAPersistedSnapshotNoFlush) {
240     rollback_to_middle_test(false);
241 }
242
243 TEST_P(RollbackTest, RollbackToMiddleOfAnUnPersistedSnapshot) {
244     /* need to store a certain number of keys because rollback
245        'bails (rolls back to 0)' if the rollback is too much. */
246     const int numItems = 10;
247     for (int i = 0; i < numItems; i++) {
248         store_item(vbid,
249                    makeStoredDocKey("key_" + std::to_string(i)),
250                    "not rolled back");
251     }
252
253     /* the roll back function will rewind disk to key11. */
254     auto rollback_item =
255             store_item(vbid, makeStoredDocKey("key11"), "rollback pt");
256
257     ASSERT_EQ(numItems + 1, store->flushVBucket(vbid));
258
259     /* Keys to be lost in rollback */
260     auto item_v1 = store_item(
261             vbid, makeStoredDocKey("rollback-cp-1"), "hope to keep till here");
262     /* ask to rollback to here; this item is in a checkpoint and
263        is not persisted */
264     auto rollbackReqSeqno = item_v1.getBySeqno();
265
266     auto item_v2 = store_item(vbid, makeStoredDocKey("rollback-cp-2"), "gone");
267
268     /* do rollback */
269     store->setVBucketState(vbid, vbucket_state_replica, false);
270     EXPECT_EQ(ENGINE_SUCCESS, store->rollback(vbid, rollbackReqSeqno));
271
272     /* confirm that we have rolled back to the disk snapshot */
273     EXPECT_EQ(rollback_item.getBySeqno(),
274               store->getVBucket(vbid)->getHighSeqno());
275
276     /* since we rely only on disk snapshots currently, we must lose the items in
277        the checkpoints */
278     for (int i = 0; i < 2; i++) {
279         auto result =
280                 store->get(makeStoredDocKey("rollback-cp-" + std::to_string(i)),
281                            vbid,
282                            nullptr,
283                            {});
284         EXPECT_EQ(ENGINE_KEY_ENOENT, result.getStatus())
285                 << "A key set after the rollback point was found";
286     }
287 }
288 #endif
289
290 /*
291  * The opencheckpointid of a bucket is one after a rollback.
292  */
293 TEST_P(RollbackTest, MB21784) {
294     // Make the vbucket a replica
295     store->setVBucketState(vbid, vbucket_state_replica, false);
296     // Perform a rollback
297     EXPECT_EQ(ENGINE_SUCCESS, store->rollback(vbid, initial_seqno))
298         << "rollback did not return ENGINE_SUCCESS";
299
300     // Assert the checkpointmanager clear function (called during rollback)
301     // has set the opencheckpointid to one
302     auto vb = store->getVBucket(vbid);
303     auto& ckpt_mgr = vb->checkpointManager;
304     EXPECT_EQ(1, ckpt_mgr.getOpenCheckpointId()) << "opencheckpointId not one";
305
306     // Create a new Dcp producer, reserving its cookie.
307     get_mock_server_api()->cookie->reserve(cookie);
308     dcp_producer_t producer = engine->getDcpConnMap().newProducer(
309             cookie, "test_producer", /*notifyOnly*/false, /*isKeyOnly*/false);
310
311     uint64_t rollbackSeqno;
312     auto err = producer->streamRequest(/*flags*/0,
313                                        /*opaque*/0,
314                                        /*vbucket*/vbid,
315                                        /*start_seqno*/0,
316                                        /*end_seqno*/0,
317                                        /*vb_uuid*/0,
318                                        /*snap_start*/0,
319                                        /*snap_end*/0,
320                                        &rollbackSeqno,
321                                        RollbackTest::fakeDcpAddFailoverLog);
322     EXPECT_EQ(ENGINE_SUCCESS, err)
323         << "stream request did not return ENGINE_SUCCESS";
324     // Close stream
325     ASSERT_EQ(ENGINE_SUCCESS, producer->closeStream(/*opaque*/0, vbid));
326     engine->handleDisconnect(cookie);
327 }
328
329 class RollbackDcpTest : public SingleThreadedEPBucketTest {
330 public:
331     RollbackDcpTest()
332         : cookie(create_mock_cookie()),
333           producers(get_dcp_producers(nullptr, nullptr)) {
334     }
335
336     void SetUp() override {
337         SingleThreadedEPBucketTest::SetUp();
338         store->setVBucketState(vbid, vbucket_state_active, false);
339         consumer = new MockDcpConsumer(*engine, cookie, "test_consumer");
340         vb = store->getVBucket(vbid);
341         producers->stream_req = &RollbackDcpTest::streamRequest;
342     }
343
344     void TearDown() override {
345         consumer->closeAllStreams();
346         destroy_mock_cookie(cookie);
347         consumer.reset();
348         vb.reset();
349         SingleThreadedEPBucketTest::TearDown();
350     }
351
352     // build a rollback response command
353     std::unique_ptr<char[]> getRollbackResponse(uint32_t opaque,
354                                                 uint64_t rollbackSeq) const {
355         auto msg = std::make_unique<char[]>(
356                 sizeof(protocol_binary_response_header) + sizeof(uint64_t));
357         auto* p = reinterpret_cast<protocol_binary_response_dcp_stream_req*>(
358                 msg.get());
359
360         p->message.header.response.opcode = PROTOCOL_BINARY_CMD_DCP_STREAM_REQ;
361         p->message.header.response.status =
362                 htons(PROTOCOL_BINARY_RESPONSE_ROLLBACK);
363         p->message.header.response.opaque = opaque;
364         p->message.header.response.bodylen = htonl(sizeof(uint64_t));
365
366         auto* seq = reinterpret_cast<uint64_t*>(p + 1);
367         *seq = htonll(rollbackSeq);
368         return msg;
369     }
370
371     static struct StreamRequestData {
372         bool called;
373         uint32_t opaque;
374         uint16_t vbucket;
375         uint32_t flags;
376         uint64_t start_seqno;
377         uint64_t end_seqno;
378         uint64_t vbucket_uuid;
379         uint64_t snap_start_seqno;
380         uint64_t snap_end_seqno;
381     } streamRequestData;
382
383     static ENGINE_ERROR_CODE streamRequest(const void* cookie,
384                                            uint32_t opaque,
385                                            uint16_t vbucket,
386                                            uint32_t flags,
387                                            uint64_t start_seqno,
388                                            uint64_t end_seqno,
389                                            uint64_t vbucket_uuid,
390                                            uint64_t snap_start_seqno,
391                                            uint64_t snap_end_seqno) {
392         streamRequestData = {true,
393                              opaque,
394                              vbucket,
395                              flags,
396                              start_seqno,
397                              end_seqno,
398                              vbucket_uuid,
399                              snap_start_seqno,
400                              snap_end_seqno};
401
402         return ENGINE_SUCCESS;
403     }
404
405     void stepForStreamRequest(uint64_t startSeqno, uint64_t vbUUID) {
406         while (consumer->step(producers.get()) == ENGINE_WANT_MORE) {
407         }
408         EXPECT_TRUE(streamRequestData.called);
409         EXPECT_EQ(startSeqno, streamRequestData.start_seqno);
410         EXPECT_EQ(vbUUID, streamRequestData.vbucket_uuid);
411         streamRequestData = {};
412     }
413
414     void createItems(int items, int flushes) {
415         // Flush multiple checkpoints of unique keys
416         for (int ii = 0; ii < flushes; ii++) {
417             EXPECT_TRUE(store_items(items,
418                                     vbid,
419                                     {"anykey_" + std::to_string(ii) + "_",
420                                      DocNamespace::DefaultCollection},
421                                     "value"));
422             flush_vbucket_to_disk(vbid, items);
423             // Add an entry for this seqno
424             vb->failovers->createEntry(items * (ii + 1));
425         }
426
427         store->setVBucketState(vbid, vbucket_state_replica, false);
428     }
429
430     uint64_t addStream(int nitems) {
431         consumer->addStream(/*opaque*/ 0, vbid, /*flags*/ 0);
432         // Step consumer to retrieve the first stream request.
433         uint64_t vbUUID = vb->failovers->getLatestEntry().vb_uuid;
434         stepForStreamRequest(nitems, vbUUID);
435         return vbUUID;
436     }
437
438     void responseNoRollback(int nitems,
439                             uint64_t rollbackSeq,
440                             uint64_t previousUUID) {
441         // Now push a reponse to the consumer, saying rollback to 0.
442         // The consumer must ignore the 0 rollback and retry a stream-request
443         // with the next failover entry.
444         auto msg = getRollbackResponse(1 /*opaque*/, rollbackSeq);
445         EXPECT_TRUE(consumer->handleResponse(
446                 reinterpret_cast<protocol_binary_response_header*>(msg.get())));
447
448         // Consumer should of added a StreamRequest with a different vbuuid
449         EXPECT_NE(previousUUID, vb->failovers->getLatestEntry().vb_uuid);
450
451         stepForStreamRequest(nitems, vb->failovers->getLatestEntry().vb_uuid);
452     }
453
454     void responseRollback(uint64_t rollbackSeq) {
455         // Now push a reponse to the consumer, saying rollback to 0.
456         // The consumer must ignore the 0 rollback and retry a stream-request
457         // with the next failover entry.
458         auto msg = getRollbackResponse(1 /*opaque*/, rollbackSeq);
459         EXPECT_TRUE(consumer->handleResponse(
460                 reinterpret_cast<protocol_binary_response_header*>(msg.get())));
461
462         // consumer must of scheduled a RollbackTask (writer task)
463         auto& lpWriteQ = *task_executor->getLpTaskQ()[WRITER_TASK_IDX];
464         ASSERT_EQ(1, lpWriteQ.getFutureQueueSize());
465         runNextTask(lpWriteQ);
466     }
467
468     const void* cookie;
469     SingleThreadedRCPtr<MockDcpConsumer> consumer;
470     std::unique_ptr<dcp_message_producers> producers;
471     VBucketPtr vb;
472 };
473
474 RollbackDcpTest::StreamRequestData RollbackDcpTest::streamRequestData = {};
475
476 /**
477  * Push stream responses to a consumer and test
478  * 1. The first rollback to 0 response is ignored, the consumer requests again
479  *    with new data.
480  * 2. The second rollback to 0 response triggers a rollback to 0.
481  */
482 TEST_F(RollbackDcpTest, test_rollback_zero) {
483     const int items = 40;
484     const int flushes = 1;
485     const int nitems = items * flushes;
486     const int rollbackPoint = 0; // expect final rollback to be to 0
487
488     // Test will create anykey_0_{0..items-1}
489     createItems(items, flushes);
490
491     auto uuid = addStream(nitems);
492
493     responseNoRollback(nitems, 0, uuid);
494
495     // All keys available
496     for (int ii = 0; ii < items; ii++) {
497         std::string key = "anykey_0_" + std::to_string(ii);
498         auto result = store->get(
499                 {key, DocNamespace::DefaultCollection}, vbid, nullptr, {});
500         EXPECT_EQ(ENGINE_SUCCESS, result.getStatus()) << "Problem with " << key;
501         delete result.getValue();
502     }
503
504     responseRollback(rollbackPoint);
505
506     // All keys now gone
507     for (int ii = 0; ii < items; ii++) {
508         std::string key = "anykey_0_" + std::to_string(ii);
509         auto result = store->get(
510                 {key, DocNamespace::DefaultCollection}, vbid, nullptr, {});
511         EXPECT_EQ(ENGINE_KEY_ENOENT, result.getStatus()) << "Problem with "
512                                                          << key;
513     }
514
515     // Expected a rollback to 0 which is a VB reset, so discard the now dead
516     // vb and obtain replacement
517     vb = store->getVBucket(vbid);
518
519     // Rollback complete and will have posted a new StreamRequest
520     stepForStreamRequest(rollbackPoint,
521                          vb->failovers->getLatestEntry().vb_uuid);
522     EXPECT_EQ(rollbackPoint, vb->getHighSeqno()) << "VB hasn't rolled back to "
523                                                  << rollbackPoint;
524 }
525
526 /**
527  * Push stream responses to a consumer and test
528  * 1. The first rollback to 0 response is ignored, the consumer requests again
529  *    with new data.
530  * 2. The second rollback response is non-zero, and the consumer accepts that
531  *    and rolls back to the rollbackPoint and requests a stream for it.
532  */
533 TEST_F(RollbackDcpTest, test_rollback_nonzero) {
534     const int items = 10;
535     const int flushes = 4;
536     const int nitems = items * flushes;
537     const int rollbackPoint = 3 * items; // rollback to 3/4
538
539     // Test will create anykey_{0..flushes-1}_{0..items-1}
540     createItems(items, flushes);
541
542     auto uuid = addStream(nitems);
543
544     responseNoRollback(nitems, 0, uuid);
545
546     // All keys available
547     for (int ii = 0; ii < items; ii++) {
548         for (int ff = 0; ff < flushes; ff++) {
549             std::string key =
550                     "anykey_" + std::to_string(ff) + "_" + std::to_string(ii);
551             auto result = store->get(
552                     {key, DocNamespace::DefaultCollection}, vbid, nullptr, {});
553             EXPECT_EQ(ENGINE_SUCCESS, result.getStatus()) << "Expected to find "
554                                                           << key;
555             delete result.getValue();
556         }
557     }
558
559     responseRollback(rollbackPoint);
560
561     // 3/4 keys available
562     for (int ii = 0; ii < items; ii++) {
563         for (int ff = 0; ff < 3; ff++) {
564             std::string key =
565                     "anykey_" + std::to_string(ff) + "_" + std::to_string(ii);
566             auto result = store->get(
567                     {key, DocNamespace::DefaultCollection}, vbid, nullptr, {});
568             EXPECT_EQ(ENGINE_SUCCESS, result.getStatus()) << "Expected to find "
569                                                           << key;
570             delete result.getValue();
571         }
572     }
573
574     // Final 1/4 were discarded by the rollback
575     for (int ii = 0; ii < items; ii++) {
576         std::string key = "anykey_3_" + std::to_string(ii);
577         auto result = store->get(
578                 {key, DocNamespace::DefaultCollection}, vbid, nullptr, {});
579         EXPECT_EQ(ENGINE_KEY_ENOENT, result.getStatus()) << "Problem with "
580                                                          << key;
581     }
582
583     // Rollback complete and will have posted a new StreamRequest
584     stepForStreamRequest(rollbackPoint,
585                          vb->failovers->getLatestEntry().vb_uuid);
586     EXPECT_EQ(rollbackPoint, vb->getHighSeqno()) << "VB hasn't rolled back to "
587                                                  << rollbackPoint;
588 }
589
590 // Test cases which run in both Full and Value eviction
591 INSTANTIATE_TEST_CASE_P(FullAndValueEviction,
592                         RollbackTest,
593                         ::testing::Values("value_only", "full_eviction"),
594                         [] (const ::testing::TestParamInfo<std::string>& info) {
595                             return info.param;
596                         });