tests/module_tests/kvstore_test.cc

   1 /* -*- Mode: C++; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */
   2 /*
   3  *     Copyright 2016 Couchbase, Inc
   4  *
   5  *   Licensed under the Apache License, Version 2.0 (the "License");
   6  *   you may not use this file except in compliance with the License.
   7  *   You may obtain a copy of the License at
   8  *
   9  *       http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  *   Unless required by applicable law or agreed to in writing, software
  12  *   distributed under the License is distributed on an "AS IS" BASIS,
  13  *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  *   See the License for the specific language governing permissions and
  15  *   limitations under the License.
  16  */
  17
  18 #include "config.h"
  19
  20 #include <platform/dirutils.h>
  21
  22 #include "callbacks.h"
  23 #include "couch-kvstore/couch-kvstore.h"
  24 #include "kvstore.h"
  25 #include "src/internal.h"
  26 #include "tests/module_tests/test_helpers.h"
  27 #include "tests/test_fileops.h"
  28
  29 #include <gmock/gmock.h>
  30 #include <gtest/gtest.h>
  31 #include <kvstore.h>
  32 #include <unordered_map>
  33 #include <vector>
  34
  35 class WriteCallback : public Callback<mutation_result> {
  36 public:
  37     WriteCallback() {}
  38
  39     void callback(mutation_result &result) {
  40
  41     }
  42
  43 };
  44
  45 class StatsCallback : public Callback<kvstats_ctx> {
  46 public:
  47     StatsCallback() {}
  48
  49     void callback(kvstats_ctx &result) {
  50
  51     }
  52
  53 };
  54
  55 class KVStoreTestCacheCallback : public Callback<CacheLookup> {
  56 public:
  57     KVStoreTestCacheCallback(int64_t s, int64_t e, uint16_t vbid) :
  58         start(s), end(e), vb(vbid) { }
  59
  60     void callback(CacheLookup &lookup) {
  61         EXPECT_EQ(vb, lookup.getVBucketId());
  62         EXPECT_LE(start, lookup.getBySeqno());
  63         EXPECT_LE(lookup.getBySeqno(), end);
  64     }
  65
  66 private:
  67     int64_t start;
  68     int64_t end;
  69     uint16_t vb;
  70 };
  71
  72 class GetCallback : public Callback<GetValue> {
  73 public:
  74     GetCallback(ENGINE_ERROR_CODE _expectedErrorCode = ENGINE_SUCCESS) :
  75         expectCompressed(false),
  76         expectedErrorCode(_expectedErrorCode) { }
  77
  78     GetCallback(bool expect_compressed,
  79                 ENGINE_ERROR_CODE _expectedErrorCode = ENGINE_SUCCESS) :
  80         expectCompressed(expect_compressed),
  81         expectedErrorCode(_expectedErrorCode) { }
  82
  83     void callback(GetValue &result) {
  84         EXPECT_EQ(expectedErrorCode, result.getStatus());
  85         if (result.getStatus() == ENGINE_SUCCESS) {
  86             if (expectCompressed) {
  87                 EXPECT_EQ(PROTOCOL_BINARY_DATATYPE_SNAPPY,
  88                           result.getValue()->getDataType());
  89                 result.getValue()->decompressValue();
  90             }
  91
  92             EXPECT_EQ(0,
  93                       strncmp("value",
  94                               result.getValue()->getData(),
  95                               result.getValue()->getNBytes()));
  96             delete result.getValue();
  97         }
  98     }
  99
 100 private:
 101     bool expectCompressed;
 102     ENGINE_ERROR_CODE expectedErrorCode;
 103 };
 104
 105 class BloomFilterCallback : public Callback<std::string&, bool&> {
 106 public:
 107     BloomFilterCallback() {}
 108     void callback(std::string& ra, bool& rb) override {}
 109 };
 110
 111 class ExpiryCallback : public Callback<std::string&, uint64_t&> {
 112 public:
 113     ExpiryCallback() {}
 114     void callback(std::string& ra, uint64_t& rb) override {}
 115 };
 116
 117 /**
 118  * Utility template for generating callbacks for various
 119  * KVStore functions from a lambda/std::function
 120  */
 121 template <typename... RV>
 122 class CustomCallback : public Callback<RV...> {
 123 public:
 124     CustomCallback(std::function<void(RV...)> _cb)
 125         : cb(_cb) {}
 126     CustomCallback()
 127         : cb([](RV... val){}) {}
 128
 129     void callback(RV&...result) {
 130         cb(result...);
 131     }
 132
 133 protected:
 134     std::function<void(RV...)> cb;
 135 };
 136
 137 /**
 138  * Callback that can be given a lambda to use, specifically
 139  * for the Rollback callback
 140  */
 141 class CustomRBCallback : public RollbackCB {
 142 public:
 143     CustomRBCallback(std::function<void(GetValue)> _cb)
 144         : cb(_cb) {}
 145     CustomRBCallback()
 146         : cb([](GetValue val){}) {}
 147
 148     void callback(GetValue &result) {
 149         cb(result);
 150         delete result.getValue();
 151     }
 152
 153 protected:
 154     std::function<void(GetValue)> cb;
 155 };
 156
 157 // Initializes a KVStore
 158 static void initialize_kv_store(KVStore* kvstore) {
 159     std::string failoverLog("");
 160     // simulate the setVbState by incrementing the rev
 161     kvstore->incrementRevision(0);
 162     vbucket_state state(vbucket_state_active, 0, 0, 0, 0, 0, 0, 0, failoverLog);
 163     // simulate the setVbState by incrementing the rev
 164     kvstore->incrementRevision(0);
 165     kvstore->snapshotVBucket(0, state,
 166                             VBStatePersist::VBSTATE_PERSIST_WITHOUT_COMMIT);
 167 }
 168
 169 // Creates and initializes a KVStore with the given config
 170 static std::unique_ptr<KVStore> setup_kv_store(KVStoreConfig& config) {
 171     auto kvstore = KVStoreFactory::create(config);
 172     initialize_kv_store(kvstore.rw.get());
 173     return std::move(kvstore.rw);
 174 }
 175
 176 /* Test callback for stats handling.
 177  * 'cookie' is a std::unordered_map<std::string, std::string) which stats
 178  * are accumulated in.
 179  */
 180 static void add_stat_callback(const char *key, const uint16_t klen,
 181                               const char *val, const uint32_t vlen,
 182                               const void *cookie) {
 183     auto* map = reinterpret_cast<std::map<std::string, std::string>*>(
 184             const_cast<void*>(cookie));
 185     ASSERT_NE(nullptr, map);
 186     map->insert(std::make_pair(std::string(key, klen),
 187                                std::string(val, vlen)));
 188 }
 189
 190 /**
 191  * Test fixture for KVStore tests. Inherited by both ForestDB and
 192  * Couchstore test fixtures.
 193  **/
 194 class KVStoreTest : public ::testing::Test {
 195 protected:
 196     void SetUp() override {
 197         auto* info = ::testing::UnitTest::GetInstance()->current_test_info();
 198         data_dir = std::string(info->test_case_name()) + "_" + info->name() +
 199             ".db";
 200     }
 201
 202     void TearDown() override {
 203         cb::io::rmrf(data_dir);
 204     }
 205
 206     std::string data_dir;
 207 };
 208
 209 /// Test fixture for tests which run on both Couchstore and ForestDB.
 210 class CouchAndForestTest : public KVStoreTest,
 211                            public ::testing::WithParamInterface<std::string> {
 212 };
 213
 214 /// Test fixture for tests which run only on Couchstore.
 215 class CouchKVStoreTest : public KVStoreTest {
 216 };
 217
 218 /* Test basic set / get of a document */
 219 TEST_P(CouchAndForestTest, BasicTest) {
 220     KVStoreConfig config(
 221             1024, 4, data_dir, GetParam(), 0, false /*persistnamespace*/);
 222     auto kvstore = setup_kv_store(config);
 223
 224     kvstore->begin();
 225     StoredDocKey key = makeStoredDocKey("key");
 226     Item item(key, 0, 0, "value", 5);
 227     WriteCallback wc;
 228     kvstore->set(item, wc);
 229
 230     EXPECT_TRUE(kvstore->commit(nullptr /*no collections manifest*/));
 231
 232     GetCallback gc;
 233     kvstore->get(key, 0, gc);
 234 }
 235
 236 TEST_F(CouchKVStoreTest, CompressedTest) {
 237     KVStoreConfig config(
 238             1024, 4, data_dir, "couchdb", 0, false /*persistnamespace*/);
 239     auto kvstore = setup_kv_store(config);
 240
 241     kvstore->begin();
 242
 243     uint8_t datatype = PROTOCOL_BINARY_RAW_BYTES;
 244     WriteCallback wc;
 245     for (int i = 1; i <= 5; i++) {
 246         std::string key("key" + std::to_string(i));
 247         Item item(makeStoredDocKey(key),
 248                   0, 0, "value", 5, &datatype, 1, 0, i);
 249         kvstore->set(item, wc);
 250     }
 251
 252     StatsCallback sc;
 253     kvstore->commit(nullptr /*no collections manifest*/);
 254
 255     std::shared_ptr<Callback<GetValue> > cb(new GetCallback(true/*expectcompressed*/));
 256     std::shared_ptr<Callback<CacheLookup> > cl(new KVStoreTestCacheCallback(1, 5, 0));
 257     ScanContext* scanCtx;
 258     scanCtx = kvstore->initScanContext(cb, cl, 0, 1,
 259                                        DocumentFilter::ALL_ITEMS,
 260                                        ValueFilter::VALUES_COMPRESSED);
 261
 262     ASSERT_NE(nullptr, scanCtx);
 263     EXPECT_EQ(scan_success, kvstore->scan(scanCtx));
 264     kvstore->destroyScanContext(scanCtx);
 265 }
 266
 267 // Verify the stats returned from operations are accurate.
 268 TEST_F(CouchKVStoreTest, StatsTest) {
 269     KVStoreConfig config(
 270             1024, 4, data_dir, "couchdb", 0, false /*persistnamespace*/);
 271     auto kvstore = setup_kv_store(config);
 272
 273     // Perform a transaction with a single mutation (set) in it.
 274     kvstore->begin();
 275     const std::string key{"key"};
 276     const std::string value{"value"};
 277     Item item(makeStoredDocKey(key), 0, 0, value.c_str(), value.size());
 278     WriteCallback wc;
 279     kvstore->set(item, wc);
 280
 281     StatsCallback sc;
 282     EXPECT_TRUE(kvstore->commit(nullptr /*no collections manifest*/));
 283     // Check statistics are correct.
 284     std::map<std::string, std::string> stats;
 285     kvstore->addStats(add_stat_callback, &stats);
 286     EXPECT_EQ("1", stats["rw_0:io_num_write"]);
 287     const size_t io_write_bytes = stoul(stats["rw_0:io_write_bytes"]);
 288     EXPECT_EQ(key.size() + value.size() +
 289               MetaData::getMetaDataSize(MetaData::Version::V1),
 290               io_write_bytes);
 291
 292     // Hard to determine exactly how many bytes should have been written, but
 293     // expect non-zero, and least as many as the actual documents.
 294     const size_t io_total_write_bytes = stoul(stats["rw_0:io_total_write_bytes"]);
 295     EXPECT_GT(io_total_write_bytes, 0);
 296     EXPECT_GE(io_total_write_bytes, io_write_bytes);
 297 }
 298
 299 // Verify the compaction stats returned from operations are accurate.
 300 TEST_F(CouchKVStoreTest, CompactStatsTest) {
 301     KVStoreConfig config(
 302             1, 4, data_dir, "couchdb", 0, false /*persistnamespace*/);
 303     auto kvstore = setup_kv_store(config);
 304
 305     // Perform a transaction with a single mutation (set) in it.
 306     kvstore->begin();
 307     const std::string key{"key"};
 308     const std::string value{"value"};
 309     Item item(makeStoredDocKey(key), 0, 0, value.c_str(), value.size());
 310     WriteCallback wc;
 311     kvstore->set(item, wc);
 312
 313     EXPECT_TRUE(kvstore->commit(nullptr /*no collections manifest*/));
 314
 315     std::shared_ptr<Callback<std::string&, bool&> >
 316         filter(new BloomFilterCallback());
 317     std::shared_ptr<Callback<std::string&, uint64_t&> >
 318         expiry(new ExpiryCallback());
 319
 320     compaction_ctx cctx;
 321     cctx.purge_before_seq = 0;
 322     cctx.purge_before_ts = 0;
 323     cctx.curr_time = 0;
 324     cctx.drop_deletes = 0;
 325     cctx.db_file_id = 0;
 326
 327     EXPECT_TRUE(kvstore->compactDB(&cctx));
 328     // Check statistics are correct.
 329     std::map<std::string, std::string> stats;
 330     kvstore->addStats(add_stat_callback, &stats);
 331     EXPECT_EQ("1", stats["rw_0:io_num_write"]);
 332     const size_t io_write_bytes = stoul(stats["rw_0:io_write_bytes"]);
 333
 334     // Hard to determine exactly how many bytes should have been written, but
 335     // expect non-zero, and at least twice as many as the actual documents for
 336     // the total and once as many for compaction alone.
 337     const size_t io_total_write_bytes = stoul(stats["rw_0:io_total_write_bytes"]);
 338     const size_t io_compaction_write_bytes = stoul(stats["rw_0:io_compaction_write_bytes"]);
 339     EXPECT_GT(io_total_write_bytes, 0);
 340     EXPECT_GT(io_compaction_write_bytes, 0);
 341     EXPECT_GT(io_total_write_bytes, io_compaction_write_bytes);
 342     EXPECT_GE(io_total_write_bytes, io_write_bytes * 2);
 343     EXPECT_GE(io_compaction_write_bytes, io_write_bytes);
 344 }
 345
 346 // Regression test for MB-17517 - ensure that if a couchstore file has a max
 347 // CAS of -1, it is detected and reset to zero when file is loaded.
 348 TEST_F(CouchKVStoreTest, MB_17517MaxCasOfMinus1) {
 349     KVStoreConfig config(
 350             1024, 4, data_dir, "couchdb", 0, false /*persistnamespace*/);
 351     auto kvstore = KVStoreFactory::create(config);
 352     ASSERT_NE(nullptr, kvstore.rw);
 353
 354     // Activate vBucket.
 355     std::string failoverLog("[]");
 356     vbucket_state state(vbucket_state_active, /*ckid*/0, /*maxDelSeqNum*/0,
 357                         /*highSeqno*/0, /*purgeSeqno*/0, /*lastSnapStart*/0,
 358                         /*lastSnapEnd*/0, /*maxCas*/-1, failoverLog);
 359     EXPECT_TRUE(kvstore.rw->snapshotVBucket(
 360             /*vbid*/ 0, state, VBStatePersist::VBSTATE_PERSIST_WITHOUT_COMMIT));
 361     EXPECT_EQ(~0ull, kvstore.rw->listPersistedVbuckets()[0]->maxCas);
 362
 363     // Close the file, then re-open.
 364     kvstore = KVStoreFactory::create(config);
 365     EXPECT_NE(nullptr, kvstore.rw);
 366
 367     // Check that our max CAS was repaired on startup.
 368     EXPECT_EQ(0u, kvstore.rw->listPersistedVbuckets()[0]->maxCas);
 369 }
 370
 371 // Regression test for MB-19430 - ensure that an attempt to get the
 372 // item count from a file which doesn't exist yet propagates the
 373 // error so the caller can detect (and retry as necessary).
 374 TEST_F(CouchKVStoreTest, MB_18580_ENOENT) {
 375     KVStoreConfig config(
 376             1024, 4, data_dir, "couchdb", 0, false /*persistnamespace*/);
 377     // Create a read-only kvstore (which disables item count caching), then
 378     // attempt to get the count from a non-existent vbucket.
 379     auto kvstore = KVStoreFactory::create(config);
 380     ASSERT_NE(nullptr, kvstore.ro);
 381
 382     // Expect to get a system_error (ENOENT)
 383     EXPECT_THROW(kvstore.ro->getDbFileInfo(0), std::system_error);
 384 }
 385
 386 /**
 387  * The CouchKVStoreErrorInjectionTest cases utilise GoogleMock to inject
 388  * errors into couchstore as if they come from the filesystem in order
 389  * to observe how CouchKVStore handles the error and logs it.
 390  *
 391  * The GoogleMock framework allows expectations to be set on how an object
 392  * will be called and how it will respond. Generally we will set a Couchstore
 393  * FileOps instance to return an error code on the 'nth' call as follows:
 394  *
 395  *      EXPECT_CALL(ops, open(_, _, _, _)).Times(AnyNumber());
 396  *      EXPECT_CALL(ops, open(_, _, _, _))
 397  *          .WillOnce(Return(COUCHSTORE_ERROR_OPEN_FILE)).RetiresOnSaturation();
 398  *      EXPECT_CALL(ops, open(_, _, _, _)).Times(n).RetiresOnSaturation();
 399  *
 400  * We will additionally set an expectation on the LoggerMock regarding how it
 401  * will be called by CouchKVStore. In this instance we have set an expectation
 402  * that the logger will be called with a logging level greater than or equal
 403  * to info, and the log message will contain the error string that corresponds
 404  * to `COUCHSTORE_ERROR_OPEN_FILE`.
 405  *
 406  *      EXPECT_CALL(logger, mlog(_, _)).Times(AnyNumber());
 407  *      EXPECT_CALL(logger, mlog(Ge(EXTENSION_LOG_WARNING),
 408  *                               VCE(COUCHSTORE_ERROR_OPEN_FILE))
 409  *      ).Times(1).RetiresOnSaturation();
 410  */
 411
 412 using namespace testing;
 413
 414 /**
 415  * The MockLogger is used to verify that the logger is called with certain
 416  * parameters / messages.
 417  *
 418  * The MockLogger is slightly misleading in that it mocks a function that
 419  * is not on the API of the logger, instead mocking a function that is
 420  * called with the preformatted log message.
 421  */
 422 class MockLogger : public Logger {
 423 public:
 424     MockLogger() {
 425         ON_CALL(*this, mlog(_, _)).WillByDefault(Invoke([](EXTENSION_LOG_LEVEL sev,
 426                                                            const std::string& msg){
 427         }));
 428     }
 429
 430     void vlog(EXTENSION_LOG_LEVEL severity, const char* fmt, va_list va) const override {
 431         mlog(severity, vatos(fmt, va));
 432     }
 433
 434     MOCK_CONST_METHOD2(mlog, void(EXTENSION_LOG_LEVEL severity,
 435                                   const std::string& message));
 436
 437 private:
 438     /**
 439      * Convert fmt cstring and a variadic arguments list to a string
 440      */
 441     static std::string vatos(const char* fmt, va_list va) {
 442         std::vector<char> buffer;
 443         va_list cpy;
 444
 445         // Calculate Size
 446         va_copy(cpy, va);
 447         buffer.resize(vsnprintf(nullptr, 0, fmt, cpy) + 1);
 448         va_end(cpy);
 449
 450         // Write to vector and return as string
 451         vsnprintf(buffer.data(), buffer.size(), fmt, va);
 452         return std::string(buffer.data());
 453     }
 454
 455 };
 456
 457 /**
 458  * VCE: Verify Couchstore Error
 459  *
 460  * This is a GoogleMock matcher which will match against a string
 461  * which has the corresponding message for the passed couchstore
 462  * error code in it. e.g.
 463  *
 464  *     VCE(COUCHSTORE_ERROR_WRITE)
 465  *
 466  * will match against a string which contains 'error writing to file'.
 467  */
 468 MATCHER_P(VCE, value, "is string of %(value)") {
 469     return arg.find(couchstore_strerror(value)) != std::string::npos;
 470 }
 471
 472 /**
 473  * CouchKVStoreErrorInjectionTest is used for tests which verify
 474  * log messages from error injection in couchstore.
 475  */
 476 class CouchKVStoreErrorInjectionTest : public ::testing::Test {
 477 public:
 478     CouchKVStoreErrorInjectionTest()
 479         : data_dir("CouchKVStoreErrorInjectionTest.db"),
 480           ops(create_default_file_ops()),
 481           config(KVStoreConfig(1024,
 482                                4,
 483                                data_dir,
 484                                "couchdb",
 485                                0,
 486                                false /*persistnamespace*/)
 487                          .setLogger(logger)
 488                          .setBuffered(false)) {
 489         try {
 490             cb::io::rmrf(data_dir.c_str());
 491         } catch (std::system_error& e) {
 492             if (e.code() != std::error_code(ENOENT, std::system_category())) {
 493                 throw e;
 494             }
 495         }
 496         kvstore.reset(new CouchKVStore(config, ops));
 497         initialize_kv_store(kvstore.get());
 498     }
 499     ~CouchKVStoreErrorInjectionTest() {
 500         cb::io::rmrf(data_dir.c_str());
 501     }
 502
 503 protected:
 504     void generate_items(size_t count) {
 505         for(unsigned i(0); i < count; i++) {
 506             std::string key("key" + std::to_string(i));
 507             items.push_back(Item(makeStoredDocKey(key), 0, 0, "value", 5,
 508                                  nullptr, 0, 0, i + 1));
 509         }
 510     }
 511
 512     void populate_items(size_t count) {
 513         generate_items(count);
 514         CustomCallback<mutation_result> set_callback;
 515         kvstore->begin();
 516         for(const auto& item: items) {
 517             kvstore->set(item, set_callback);
 518         }
 519         kvstore->commit(nullptr /*no collections manifest*/);
 520     }
 521
 522     vb_bgfetch_queue_t make_bgfetch_queue() {
 523         vb_bgfetch_queue_t itms;
 524         for(const auto& item: items) {
 525             vb_bgfetch_item_ctx_t ctx;
 526             ctx.isMetaOnly = false;
 527             itms[item.getKey()] = std::move(ctx);
 528         }
 529         return itms;
 530     }
 531
 532
 533     const std::string data_dir;
 534
 535     ::testing::NiceMock<MockOps> ops;
 536     ::testing::NiceMock<MockLogger> logger;
 537
 538     KVStoreConfig config;
 539     std::unique_ptr<CouchKVStore> kvstore;
 540     std::vector<Item> items;
 541 };
 542
 543
 544 /**
 545  * Injects error during CouchKVStore::openDB_retry/couchstore_open_db_ex
 546  */
 547 TEST_F(CouchKVStoreErrorInjectionTest, openDB_retry_open_db_ex) {
 548     generate_items(1);
 549     CustomCallback<mutation_result> set_callback;
 550
 551     kvstore->begin();
 552     kvstore->set(items.front(), set_callback);
 553     {
 554         /* Establish Logger expectation */
 555         EXPECT_CALL(logger, mlog(_, _)).Times(AnyNumber());
 556         EXPECT_CALL(logger, mlog(Ge(EXTENSION_LOG_NOTICE),
 557                                  VCE(COUCHSTORE_ERROR_OPEN_FILE))
 558         ).Times(1).RetiresOnSaturation();
 559
 560         /* Establish FileOps expectation */
 561         EXPECT_CALL(ops, open(_, _, _, _)).Times(AnyNumber());
 562         EXPECT_CALL(ops, open(_, _, _, _))
 563             .WillOnce(Return(COUCHSTORE_ERROR_OPEN_FILE)).RetiresOnSaturation();
 564
 565         kvstore->commit(nullptr /*no collections manifest*/);
 566     }
 567 }
 568
 569 /**
 570  * Injects error during CouchKVStore::openDB/couchstore_open_db_ex
 571  */
 572 TEST_F(CouchKVStoreErrorInjectionTest, openDB_open_db_ex) {
 573     generate_items(1);
 574     CustomCallback<mutation_result> set_callback;
 575
 576     kvstore->begin();
 577     kvstore->set(items.front(), set_callback);
 578     {
 579         /* Establish Logger expectation */
 580         EXPECT_CALL(logger, mlog(_, _)).Times(AnyNumber());
 581         EXPECT_CALL(logger, mlog(Ge(EXTENSION_LOG_WARNING),
 582                                  VCE(COUCHSTORE_ERROR_OPEN_FILE))
 583                    ).Times(1).RetiresOnSaturation();
 584
 585         /* Establish FileOps expectation */
 586         EXPECT_CALL(ops, open(_, _, _, _))
 587             .WillRepeatedly(Return(COUCHSTORE_ERROR_OPEN_FILE)).RetiresOnSaturation();
 588
 589         kvstore->commit(nullptr /*no collections manifest*/);
 590     }
 591 }
 592
 593 /**
 594  * Injects error during CouchKVStore::commit/couchstore_save_documents
 595  */
 596 TEST_F(CouchKVStoreErrorInjectionTest, commit_save_documents) {
 597     generate_items(1);
 598     CustomCallback<mutation_result> set_callback;
 599
 600     kvstore->begin();
 601     kvstore->set(items.front(), set_callback);
 602     {
 603         /* Establish Logger expectation */
 604         EXPECT_CALL(logger, mlog(_, _)).Times(AnyNumber());
 605         EXPECT_CALL(logger, mlog(Ge(EXTENSION_LOG_WARNING),
 606                                  VCE(COUCHSTORE_ERROR_WRITE))
 607                    ).Times(1).RetiresOnSaturation();
 608
 609         /* Establish FileOps expectation */
 610         EXPECT_CALL(ops, pwrite(_, _, _, _, _))
 611             .WillOnce(Return(COUCHSTORE_ERROR_WRITE)).RetiresOnSaturation();
 612
 613         kvstore->commit(nullptr /*no collections manifest*/);
 614     }
 615
 616 }
 617
 618 /**
 619  * Injects error during CouchKVStore::commit/couchstore_save_local_document
 620  */
 621 TEST_F(CouchKVStoreErrorInjectionTest, commit_save_local_document) {
 622     generate_items(1);
 623     CustomCallback<mutation_result> set_callback;
 624
 625     kvstore->begin();
 626     kvstore->set(items.front(), set_callback);
 627     {
 628         /* Establish Logger expectation */
 629         EXPECT_CALL(logger, mlog(_, _)).Times(AnyNumber());
 630         EXPECT_CALL(logger, mlog(Ge(EXTENSION_LOG_WARNING),
 631                                  VCE(COUCHSTORE_ERROR_WRITE))
 632                    ).Times(1).RetiresOnSaturation();
 633
 634         /* Establish FileOps expectation */
 635         EXPECT_CALL(ops, pwrite(_, _, _, _, _))
 636             .WillOnce(Return(COUCHSTORE_ERROR_WRITE)).RetiresOnSaturation();
 637         EXPECT_CALL(ops, pwrite(_, _, _, _, _)).Times(6).RetiresOnSaturation();
 638
 639         kvstore->commit(nullptr /*no collections manifest*/);
 640     }
 641
 642 }
 643
 644 /**
 645  * Injects error during CouchKVStore::commit/couchstore_commit
 646  */
 647 TEST_F(CouchKVStoreErrorInjectionTest, commit_commit) {
 648     generate_items(1);
 649     CustomCallback<mutation_result> set_callback;
 650
 651     kvstore->begin();
 652     kvstore->set(items.front(), set_callback);
 653     {
 654         /* Establish Logger expectation */
 655         EXPECT_CALL(logger, mlog(_, _)).Times(AnyNumber());
 656         EXPECT_CALL(logger, mlog(Ge(EXTENSION_LOG_WARNING),
 657                                  VCE(COUCHSTORE_ERROR_WRITE))
 658                    ).Times(1).RetiresOnSaturation();
 659
 660         /* Establish FileOps expectation */
 661         EXPECT_CALL(ops, pwrite(_, _, _, _, _))
 662             .WillOnce(Return(COUCHSTORE_ERROR_WRITE)).RetiresOnSaturation();
 663         EXPECT_CALL(ops, pwrite(_, _, _, _, _)).Times(8).RetiresOnSaturation();
 664
 665         kvstore->commit(nullptr /*no collections manifest*/);
 666     }
 667 }
 668
 669 /**
 670  * Injects error during CouchKVStore::get/couchstore_docinfo_by_id
 671  */
 672 TEST_F(CouchKVStoreErrorInjectionTest, get_docinfo_by_id) {
 673     populate_items(1);
 674     CustomCallback<GetValue> get_callback;
 675     {
 676         /* Establish Logger expectation */
 677         EXPECT_CALL(logger, mlog(_, _)).Times(AnyNumber());
 678         EXPECT_CALL(logger, mlog(Ge(EXTENSION_LOG_WARNING),
 679                                  VCE(COUCHSTORE_ERROR_READ))
 680                    ).Times(1).RetiresOnSaturation();
 681
 682         /* Establish FileOps expectation */
 683         EXPECT_CALL(ops, pread(_, _, _, _, _))
 684             .WillOnce(Return(COUCHSTORE_ERROR_READ)).RetiresOnSaturation();
 685         EXPECT_CALL(ops, pread(_, _, _, _, _)).Times(3).RetiresOnSaturation();
 686         kvstore->get(items.front().getKey(), 0, get_callback);
 687
 688     }
 689 }
 690
 691 /**
 692  * Injects error during CouchKVStore::get/couchstore_open_doc_with_docinfo
 693  */
 694 TEST_F(CouchKVStoreErrorInjectionTest, get_open_doc_with_docinfo) {
 695     populate_items(1);
 696     CustomCallback<GetValue> get_callback;
 697     {
 698         /* Establish Logger expectation */
 699         EXPECT_CALL(logger, mlog(_, _)).Times(AnyNumber());
 700         EXPECT_CALL(logger, mlog(Ge(EXTENSION_LOG_WARNING),
 701                                  VCE(COUCHSTORE_ERROR_READ))
 702                    ).Times(1).RetiresOnSaturation();
 703
 704         /* Establish FileOps expectation */
 705         EXPECT_CALL(ops, pread(_, _, _, _, _))
 706             .WillOnce(Return(COUCHSTORE_ERROR_READ)).RetiresOnSaturation();
 707         EXPECT_CALL(ops, pread(_, _, _, _, _)).Times(5).RetiresOnSaturation();
 708         kvstore->get(items.front().getKey(), 0, get_callback);
 709
 710     }
 711 }
 712
 713 /**
 714  * Injects error during CouchKVStore::getMulti/couchstore_docinfos_by_id
 715  */
 716 TEST_F(CouchKVStoreErrorInjectionTest, getMulti_docinfos_by_id) {
 717     populate_items(1);
 718     vb_bgfetch_queue_t itms(make_bgfetch_queue());
 719     {
 720
 721         /* Establish Logger expectation */
 722         EXPECT_CALL(logger, mlog(_, _)).Times(AnyNumber());
 723         EXPECT_CALL(logger, mlog(Ge(EXTENSION_LOG_WARNING),
 724                                  VCE(COUCHSTORE_ERROR_READ))
 725                    ).Times(1).RetiresOnSaturation();
 726
 727         /* Establish FileOps expectation */
 728         EXPECT_CALL(ops, pread(_, _, _, _, _))
 729             .WillOnce(Return(COUCHSTORE_ERROR_READ)).RetiresOnSaturation();
 730         EXPECT_CALL(ops, pread(_, _, _, _, _)).Times(3).RetiresOnSaturation();
 731         kvstore->getMulti(0, itms);
 732
 733     }
 734 }
 735
 736
 737 /**
 738  * Injects error during CouchKVStore::getMulti/couchstore_open_doc_with_docinfo
 739  */
 740 TEST_F(CouchKVStoreErrorInjectionTest, getMulti_open_doc_with_docinfo) {
 741     populate_items(1);
 742     vb_bgfetch_queue_t itms(make_bgfetch_queue());
 743     {
 744         /* Check preconditions */
 745         ASSERT_EQ(0, kvstore->getKVStoreStat().numGetFailure);
 746
 747         /* Establish FileOps expectation */
 748         EXPECT_CALL(ops, pread(_, _, _, _, _))
 749             .WillOnce(Return(COUCHSTORE_ERROR_READ)).RetiresOnSaturation();
 750         EXPECT_CALL(ops, pread(_, _, _, _, _)).Times(5).RetiresOnSaturation();
 751         kvstore->getMulti(0, itms);
 752
 753         EXPECT_EQ(1, kvstore->getKVStoreStat().numGetFailure);
 754     }
 755 }
 756
 757 /**
 758  * Injects error during CouchKVStore::compactDB/couchstore_compact_db_ex
 759  */
 760 TEST_F(CouchKVStoreErrorInjectionTest, compactDB_compact_db_ex) {
 761     populate_items(1);
 762
 763     compaction_ctx cctx;
 764     cctx.purge_before_seq = 0;
 765     cctx.purge_before_ts = 0;
 766     cctx.curr_time = 0;
 767     cctx.drop_deletes = 0;
 768     cctx.db_file_id = 0;
 769
 770     {
 771         /* Establish Logger expectation */
 772         EXPECT_CALL(logger, mlog(_, _)).Times(AnyNumber());
 773         EXPECT_CALL(logger, mlog(Ge(EXTENSION_LOG_WARNING),
 774                                  VCE(COUCHSTORE_ERROR_OPEN_FILE))
 775                    ).Times(1).RetiresOnSaturation();
 776
 777         /* Establish FileOps expectation */
 778         EXPECT_CALL(ops, open(_, _, _, _))
 779             .WillOnce(Return(COUCHSTORE_ERROR_OPEN_FILE)).RetiresOnSaturation();
 780         EXPECT_CALL(ops, open(_, _, _, _)).Times(1).RetiresOnSaturation();
 781         kvstore->compactDB(&cctx);
 782     }
 783 }
 784
 785 /**
 786  * Injects error during CouchKVStore::getNumItems/couchstore_changes_count
 787  */
 788 TEST_F(CouchKVStoreErrorInjectionTest, getNumItems_changes_count) {
 789     populate_items(1);
 790     {
 791         /* Establish FileOps expectation */
 792         EXPECT_CALL(ops, pread(_, _, _, _, _))
 793             .WillOnce(Return(COUCHSTORE_ERROR_READ)).RetiresOnSaturation();
 794         EXPECT_CALL(ops, pread(_, _, _, _, _)).Times(3).RetiresOnSaturation();
 795         try {
 796             kvstore->getNumItems(0, 0, 100000);
 797             EXPECT_TRUE(false) << "kvstore->getNumItems(0, 0, 100000); should "
 798                                   "have thrown a runtime_error";
 799         } catch (const std::runtime_error& e) {
 800             EXPECT_THAT(std::string(e.what()), VCE(COUCHSTORE_ERROR_READ));
 801         }
 802
 803     }
 804 }
 805
 806 /**
 807  * Injects error during CouchKVStore::reset/couchstore_commit
 808  */
 809 TEST_F(CouchKVStoreErrorInjectionTest, reset_commit) {
 810     populate_items(1);
 811     {
 812         /* Establish Logger expectation */
 813         EXPECT_CALL(logger, mlog(_, _)).Times(AnyNumber());
 814         EXPECT_CALL(logger, mlog(Ge(EXTENSION_LOG_WARNING),
 815                                  VCE(COUCHSTORE_ERROR_READ))
 816                    ).Times(1).RetiresOnSaturation();
 817
 818         /* Establish FileOps expectation */
 819         EXPECT_CALL(ops, sync(_, _))
 820             .WillOnce(Return(COUCHSTORE_ERROR_READ)).RetiresOnSaturation();
 821
 822         kvstore->reset(0);
 823     }
 824 }
 825
 826 /**
 827  * Injects error during CouchKVStore::initScanContext/couchstore_changes_count
 828  */
 829 TEST_F(CouchKVStoreErrorInjectionTest, initScanContext_changes_count) {
 830     populate_items(1);
 831     auto cb(std::make_shared<CustomCallback<GetValue>>());
 832     auto cl(std::make_shared<CustomCallback<CacheLookup>>());
 833     {
 834         /* Establish FileOps expectation */
 835         EXPECT_CALL(ops, pread(_, _, _, _, _))
 836             .WillOnce(Return(COUCHSTORE_ERROR_READ)).RetiresOnSaturation();
 837         EXPECT_CALL(ops, pread(_, _, _, _, _)).Times(3).RetiresOnSaturation();
 838
 839         ScanContext* scanCtx = nullptr;
 840         scanCtx = kvstore->initScanContext(cb, cl, 0, 0,
 841                                            DocumentFilter::ALL_ITEMS,
 842                                            ValueFilter::VALUES_DECOMPRESSED);
 843         EXPECT_EQ(nullptr, scanCtx)
 844                 << "kvstore->initScanContext(cb, cl, 0, 0, "
 845                    "DocumentFilter::ALL_ITEMS, "
 846                    "ValueFilter::VALUES_DECOMPRESSED); should "
 847                    "have returned NULL";
 848
 849         kvstore->destroyScanContext(scanCtx);
 850     }
 851 }
 852
 853 /**
 854  * Injects error during CouchKVStore::scan/couchstore_changes_since
 855  */
 856 TEST_F(CouchKVStoreErrorInjectionTest, scan_changes_since) {
 857     populate_items(1);
 858     auto cb(std::make_shared<CustomCallback<GetValue>>());
 859     auto cl(std::make_shared<CustomCallback<CacheLookup>>());
 860     auto scan_context = kvstore->initScanContext(cb, cl, 0, 0,
 861                                               DocumentFilter::ALL_ITEMS,
 862                                               ValueFilter::VALUES_DECOMPRESSED);
 863     {
 864         /* Establish Logger expectation */
 865         EXPECT_CALL(logger, mlog(_, _)).Times(AnyNumber());
 866         EXPECT_CALL(logger, mlog(Ge(EXTENSION_LOG_WARNING),
 867                                  VCE(COUCHSTORE_ERROR_READ))
 868                    ).Times(1).RetiresOnSaturation();
 869
 870         /* Establish FileOps expectation */
 871         EXPECT_CALL(ops, pread(_, _, _, _, _))
 872             .WillOnce(Return(COUCHSTORE_ERROR_READ)).RetiresOnSaturation();
 873
 874         kvstore->scan(scan_context);
 875     }
 876
 877     kvstore->destroyScanContext(scan_context);
 878 }
 879
 880 /**
 881  * Injects error during CouchKVStore::recordDbDump/couchstore_open_doc_with_docinfo
 882  */
 883 TEST_F(CouchKVStoreErrorInjectionTest, recordDbDump_open_doc_with_docinfo) {
 884     populate_items(1);
 885     auto cb(std::make_shared<CustomCallback<GetValue>>());
 886     auto cl(std::make_shared<CustomCallback<CacheLookup>>());
 887     auto scan_context = kvstore->initScanContext(cb, cl, 0, 0,
 888                                                  DocumentFilter::ALL_ITEMS,
 889                                                  ValueFilter::VALUES_DECOMPRESSED);
 890     {
 891         /* Establish Logger expectation */
 892         EXPECT_CALL(logger, mlog(_, _)).Times(AnyNumber());
 893         EXPECT_CALL(logger, mlog(Ge(EXTENSION_LOG_WARNING),
 894                                  VCE(COUCHSTORE_ERROR_READ))
 895                    ).Times(1).RetiresOnSaturation();
 896
 897         /* Establish FileOps expectation */
 898         EXPECT_CALL(ops, pread(_, _, _, _, _))
 899             .WillOnce(Return(COUCHSTORE_ERROR_READ)).RetiresOnSaturation();
 900         EXPECT_CALL(ops, pread(_, _, _, _, _)).Times(2).RetiresOnSaturation();
 901
 902         kvstore->scan(scan_context);
 903     }
 904
 905     kvstore->destroyScanContext(scan_context);
 906 }
 907
 908 /**
 909  * Injects error during CouchKVStore::rollback/couchstore_changes_count/1
 910  */
 911 TEST_F(CouchKVStoreErrorInjectionTest, rollback_changes_count1) {
 912     generate_items(6);
 913     CustomCallback<mutation_result> set_callback;
 914
 915     for(const auto item: items) {
 916         kvstore->begin();
 917         kvstore->set(item, set_callback);
 918         kvstore->commit(nullptr /*no collections manifest*/);
 919     }
 920
 921     auto rcb(std::make_shared<CustomRBCallback>());
 922     {
 923         /* Establish Logger expectation */
 924         EXPECT_CALL(logger, mlog(_, _)).Times(AnyNumber());
 925         EXPECT_CALL(logger, mlog(Ge(EXTENSION_LOG_WARNING),
 926                                  VCE(COUCHSTORE_ERROR_READ))
 927                    ).Times(1).RetiresOnSaturation();
 928
 929         /* Establish FileOps expectation */
 930         EXPECT_CALL(ops, pread(_, _, _, _, _))
 931             .WillOnce(Return(COUCHSTORE_ERROR_READ)).RetiresOnSaturation();
 932         EXPECT_CALL(ops, pread(_, _, _, _, _)).Times(3).RetiresOnSaturation();
 933
 934         kvstore->rollback(0, 5, rcb);
 935     }
 936 }
 937
 938 /**
 939  * Injects error during CouchKVStore::rollback/couchstore_rewind_header
 940  */
 941 TEST_F(CouchKVStoreErrorInjectionTest, rollback_rewind_header) {
 942     generate_items(6);
 943     CustomCallback<mutation_result> set_callback;
 944
 945     for(const auto item: items) {
 946         kvstore->begin();
 947         kvstore->set(item, set_callback);
 948         kvstore->commit(nullptr /*no collections manifest*/);
 949     }
 950
 951     auto rcb(std::make_shared<CustomRBCallback>());
 952     {
 953         /* Establish Logger expectation */
 954         EXPECT_CALL(logger, mlog(_, _)).Times(AnyNumber());
 955         EXPECT_CALL(logger, mlog(Ge(EXTENSION_LOG_WARNING),
 956                                  VCE(COUCHSTORE_ERROR_DB_NO_LONGER_VALID))
 957                    ).Times(1).RetiresOnSaturation();
 958
 959         /* Establish FileOps expectation */
 960         EXPECT_CALL(ops, pread(_, _, _, _, _))
 961             /* Doing an ALLOC_FAIL as Couchstore will just
 962              * keep rolling back otherwise */
 963             .WillOnce(Return(COUCHSTORE_ERROR_ALLOC_FAIL)).RetiresOnSaturation();
 964         EXPECT_CALL(ops, pread(_, _, _, _, _)).Times(9).RetiresOnSaturation();
 965
 966         kvstore->rollback(0, 5, rcb);
 967     }
 968 }
 969
 970 /**
 971  * Injects error during CouchKVStore::rollback/couchstore_changes_count/2
 972  */
 973 TEST_F(CouchKVStoreErrorInjectionTest, rollback_changes_count2) {
 974     generate_items(6);
 975     CustomCallback<mutation_result> set_callback;
 976
 977     for(const auto item: items) {
 978         kvstore->begin();
 979         kvstore->set(item, set_callback);
 980         kvstore->commit(nullptr /*no collections manifest*/);
 981     }
 982
 983     auto rcb(std::make_shared<CustomRBCallback>());
 984     {
 985         /* Establish Logger expectation */
 986         EXPECT_CALL(logger, mlog(_, _)).Times(AnyNumber());
 987         EXPECT_CALL(logger, mlog(Ge(EXTENSION_LOG_WARNING),
 988                                  VCE(COUCHSTORE_ERROR_READ))
 989                    ).Times(1).RetiresOnSaturation();
 990
 991         /* Establish FileOps expectation */
 992         EXPECT_CALL(ops, pread(_, _, _, _, _))
 993             .WillOnce(Return(COUCHSTORE_ERROR_READ)).RetiresOnSaturation();
 994         EXPECT_CALL(ops, pread(_, _, _, _, _)).Times(11).RetiresOnSaturation();
 995
 996         kvstore->rollback(0, 5, rcb);
 997     }
 998 }
 999
1000 /**
1001  * Injects error during CouchKVStore::readVBState/couchstore_open_local_document
1002  */
1003 TEST_F(CouchKVStoreErrorInjectionTest, readVBState_open_local_document) {
1004     generate_items(6);
1005     CustomCallback<mutation_result> set_callback;
1006
1007     for(const auto item: items) {
1008         kvstore->begin();
1009         kvstore->set(item, set_callback);
1010         kvstore->commit(nullptr /*no collections manifest*/);
1011     }
1012
1013     auto rcb(std::make_shared<CustomRBCallback>());
1014     {
1015         /* Establish Logger expectation */
1016         EXPECT_CALL(logger, mlog(_, _)).Times(AnyNumber());
1017         EXPECT_CALL(logger, mlog(Ge(EXTENSION_LOG_WARNING),
1018                                  VCE(COUCHSTORE_ERROR_READ))
1019                    ).Times(1).RetiresOnSaturation();
1020
1021         /* Establish FileOps expectation */
1022         EXPECT_CALL(ops, pread(_, _, _, _, _))
1023             .WillOnce(Return(COUCHSTORE_ERROR_READ)).RetiresOnSaturation();
1024         EXPECT_CALL(ops, pread(_, _, _, _, _)).Times(20).RetiresOnSaturation();
1025
1026         kvstore->rollback(0, 5, rcb);
1027     }
1028 }
1029
1030 /**
1031  * Injects error during CouchKVStore::getAllKeys/couchstore_all_docs
1032  */
1033 TEST_F(CouchKVStoreErrorInjectionTest, getAllKeys_all_docs) {
1034     populate_items(1);
1035
1036     auto adcb(std::make_shared<CustomCallback<const DocKey&>>());
1037     StoredDocKey start = makeStoredDocKey("");
1038     {
1039         /* Establish Logger expectation */
1040         EXPECT_CALL(logger, mlog(_, _)).Times(AnyNumber());
1041         EXPECT_CALL(logger, mlog(Ge(EXTENSION_LOG_WARNING),
1042                                  VCE(COUCHSTORE_ERROR_READ))
1043                    ).Times(1).RetiresOnSaturation();
1044
1045         /* Establish FileOps expectation */
1046         EXPECT_CALL(ops, pread(_, _, _, _, _))
1047             .WillOnce(Return(COUCHSTORE_ERROR_READ)).RetiresOnSaturation();
1048         EXPECT_CALL(ops, pread(_, _, _, _, _)).Times(3).RetiresOnSaturation();
1049
1050
1051         kvstore->getAllKeys(0, start, 1, adcb);
1052     }
1053 }
1054
1055 /**
1056  * Injects error during CouchKVStore::closeDB/couchstore_close_file
1057  */
1058 TEST_F(CouchKVStoreErrorInjectionTest, closeDB_close_file) {
1059     {
1060         /* Establish Logger expectation */
1061         EXPECT_CALL(logger, mlog(_, _)).Times(AnyNumber());
1062         EXPECT_CALL(logger, mlog(Ge(EXTENSION_LOG_WARNING),
1063                                  VCE(COUCHSTORE_ERROR_FILE_CLOSE))
1064                    ).Times(1).RetiresOnSaturation();
1065
1066         /* Establish FileOps expectation */
1067         EXPECT_CALL(ops, close(_, _)).Times(AnyNumber());
1068         EXPECT_CALL(ops, close(_, _))
1069                 .WillOnce(DoAll(IgnoreResult(Invoke(ops.get_wrapped(),
1070                                                     &FileOpsInterface::close)),
1071                                 Return(COUCHSTORE_ERROR_FILE_CLOSE)))
1072                 .RetiresOnSaturation();
1073
1074         populate_items(1);
1075     }
1076 }
1077
1078 class MockCouchRequest : public CouchRequest {
1079 public:
1080     class MetaData {
1081     public:
1082         MetaData()
1083             : cas(0),
1084               expiry(0),
1085               flags(0),
1086               ext1(0),
1087               ext2(0),
1088               legacyDeleted(0) {
1089         }
1090
1091         uint64_t cas;
1092         uint32_t expiry;
1093         uint32_t flags;
1094         uint8_t ext1;
1095         uint8_t ext2;
1096         uint8_t legacyDeleted; // allow testing via 19byte meta document
1097
1098         static const size_t sizeofV0 = 16;
1099         static const size_t sizeofV1 = 18;
1100         static const size_t sizeofV2 = 19;
1101     };
1102
1103     MockCouchRequest(const Item& it,
1104                      uint64_t rev,
1105                      MutationRequestCallback& cb,
1106                      bool del)
1107         : CouchRequest(it, rev, cb, del, false /*persist namespace*/) {
1108     }
1109
1110     ~MockCouchRequest() {}
1111
1112     // Update what will be written as 'metadata'
1113     void writeMetaData(MetaData& meta, size_t size) {
1114         std::memcpy(dbDocInfo.rev_meta.buf, &meta, size);
1115         dbDocInfo.rev_meta.size = size;
1116     }
1117 };
1118
1119 class MockCouchKVStore : public CouchKVStore {
1120 public:
1121     MockCouchKVStore(KVStoreConfig& config) : CouchKVStore(config) {
1122     }
1123
1124     // Mocks original code but returns the IORequest for fuzzing
1125     MockCouchRequest* setAndReturnRequest(const Item &itm, Callback<mutation_result> &cb) {
1126         if (isReadOnly()) {
1127             throw std::logic_error("MockCouchKVStore::set: Not valid on a read-only "
1128                             "object.");
1129         }
1130         if (!intransaction) {
1131             throw std::invalid_argument("MockCouchKVStore::set: intransaction must be "
1132                             "true to perform a set operation.");
1133         }
1134
1135         bool deleteItem = false;
1136         MutationRequestCallback requestcb;
1137         uint64_t fileRev = dbFileRevMap[itm.getVBucketId()];
1138
1139         // each req will be de-allocated after commit
1140         requestcb.setCb = &cb;
1141         MockCouchRequest *req = new MockCouchRequest(itm, fileRev, requestcb, deleteItem);
1142         pendingReqsQ.push_back(req);
1143         return req;
1144     }
1145 };
1146
1147 //
1148 // Explicitly test couchstore (not valid for ForestDB)
1149 // Intended to ensure we can read and write couchstore files and
1150 // parse metadata we store in them.
1151 //
1152 class CouchstoreTest : public ::testing::Test {
1153 public:
1154     CouchstoreTest()
1155         : data_dir("CouchstoreTest.db"),
1156           vbid(0),
1157           config(KVStoreConfig(1024,
1158                                4,
1159                                data_dir,
1160                                "couchdb",
1161                                0,
1162                                false /*persistnamespace*/)
1163                          .setBuffered(false)) {
1164         try {
1165             cb::io::rmrf(data_dir.c_str());
1166         } catch (std::system_error& e) {
1167             if (e.code() != std::error_code(ENOENT, std::system_category())) {
1168                 throw e;
1169             }
1170         }
1171         kvstore.reset(new MockCouchKVStore(config));
1172         StatsCallback sc;
1173         std::string failoverLog("");
1174         // simulate a setVBState - increment the rev and then persist the
1175         // state
1176         kvstore->incrementRevision(0);
1177         vbucket_state state(vbucket_state_active, 0, 0, 0, 0, 0, 0, 0,
1178                             failoverLog);
1179         // simulate a setVBState - increment the dbFile revision
1180         kvstore->incrementRevision(0);
1181         kvstore->snapshotVBucket(0, state,
1182                                  VBStatePersist::VBSTATE_PERSIST_WITHOUT_COMMIT);
1183     }
1184
1185     ~CouchstoreTest() {
1186         cb::io::rmrf(data_dir.c_str());
1187     }
1188
1189 protected:
1190     std::string data_dir;
1191     std::unique_ptr<MockCouchKVStore> kvstore;
1192     uint16_t vbid;
1193     KVStoreConfig config;
1194 };
1195
1196 template<class T>
1197 class MockedGetCallback : public Callback<T> {
1198     public:
1199         MockedGetCallback() {}
1200
1201         ~MockedGetCallback() {
1202             delete savedValue.getValue();
1203         }
1204
1205         void callback(GetValue& value){
1206             status(value.getStatus());
1207             if (value.getStatus() == ENGINE_SUCCESS) {
1208                 EXPECT_CALL(*this, value("value"));
1209                 cas(value.getValue()->getCas());
1210                 expTime(value.getValue()->getExptime());
1211                 flags(value.getValue()->getFlags());
1212                 datatype(protocol_binary_datatype_t(value.getValue()->getDataType()));
1213                 this->value(std::string(value.getValue()->getData(),
1214                                         value.getValue()->getNBytes()));
1215                 savedValue = value;
1216             }
1217         }
1218
1219         Item* getValue() {
1220             return savedValue.getValue();
1221         }
1222
1223         /*
1224          * Define a number of mock methods that will be invoked by the
1225          * callback method. Functions can then setup expectations of the
1226          * value of each method e.g. expect cas to be -1
1227          */
1228         MOCK_METHOD1_T(status, void(ENGINE_ERROR_CODE));
1229         MOCK_METHOD1_T(cas, void(uint64_t));
1230         MOCK_METHOD1_T(expTime, void(uint32_t));
1231         MOCK_METHOD1_T(flags, void(uint32_t));
1232         MOCK_METHOD1_T(datatype, void(protocol_binary_datatype_t));
1233         MOCK_METHOD1_T(value, void(std::string));
1234     private:
1235         GetValue savedValue;
1236 };
1237
1238 /*
1239  * The overall aim of these tests is to create an Item, write it to disk
1240  * then read it back from disk and look at various fields which are
1241  * built from the couchstore rev_meta feature.
1242  *
1243  * Validation of the Item read from disk is performed by the GetCallback.
1244  * A number of validators can be called upon which compare the disk Item
1245  * against an expected Item.
1246  *
1247  * The MockCouchKVStore exposes some of the internals of the class so we
1248  * can inject custom metadata by using ::setAndReturnRequest instead of ::set
1249  *
1250  */
1251 TEST_F(CouchstoreTest, noMeta) {
1252     StoredDocKey key = makeStoredDocKey("key");
1253     Item item(key, 0, 0, "value", 5);
1254     WriteCallback wc;
1255     kvstore->begin();
1256     auto request = kvstore->setAndReturnRequest(item, wc);
1257
1258     // Now directly mess with the metadata of the value which will be written
1259     MockCouchRequest::MetaData meta;
1260     request->writeMetaData(meta, 0); // no meta!
1261
1262     kvstore->commit(nullptr /*no collections manifest*/);
1263
1264     GetCallback gc(ENGINE_TMPFAIL);
1265     kvstore->get(key, 0, gc);
1266 }
1267
1268 TEST_F(CouchstoreTest, shortMeta) {
1269     StoredDocKey key = makeStoredDocKey("key");
1270     Item item(key, 0, 0, "value", 5);
1271     WriteCallback wc;
1272     kvstore->begin();
1273     auto request = kvstore->setAndReturnRequest(item, wc);
1274
1275     // Now directly mess with the metadata of the value which will be written
1276     MockCouchRequest::MetaData meta;
1277     request->writeMetaData(meta, 4); // not enough meta!
1278     kvstore->commit(nullptr /*no collections manifest*/);
1279
1280     GetCallback gc(ENGINE_TMPFAIL);
1281     kvstore->get(key, 0, gc);
1282 }
1283
1284 TEST_F(CouchstoreTest, testV0MetaThings) {
1285     StoredDocKey key = makeStoredDocKey("key");
1286     // Baseline test, just writes meta things and reads them
1287     // via standard interfaces
1288     // Ensure CAS, exptime and flags are set to something.
1289     Item item(key,
1290               0x01020304/*flags*/, 0xaa00bb11/*expiry*/,
1291               "value", 5,
1292               nullptr, 0,
1293               0xf00fcafe11225566ull);
1294
1295     WriteCallback wc;
1296     kvstore->begin();
1297     kvstore->set(item, wc);
1298     kvstore->commit(nullptr /*no collections manifest*/);
1299
1300     MockedGetCallback<GetValue> gc;
1301     EXPECT_CALL(gc, status(ENGINE_SUCCESS));
1302     EXPECT_CALL(gc, cas(0xf00fcafe11225566ull));
1303     EXPECT_CALL(gc, expTime(0xaa00bb11));
1304     EXPECT_CALL(gc, flags(0x01020304));
1305     EXPECT_CALL(gc, datatype(PROTOCOL_BINARY_RAW_BYTES));
1306     kvstore->get(key, 0, gc);
1307 }
1308
1309 TEST_F(CouchstoreTest, testV1MetaThings) {
1310     // Baseline test, just writes meta things and reads them
1311     // via standard interfaces
1312     // Ensure CAS, exptime and flags are set to something.
1313     uint8_t datatype = PROTOCOL_BINARY_DATATYPE_JSON; //lies, but non-zero
1314     StoredDocKey key = makeStoredDocKey("key");
1315     Item item(key,
1316               0x01020304/*flags*/, 0xaa00bb11,/*expiry*/
1317               "value", 5,
1318               &datatype, 1, /*ext_meta is v1 extension*/
1319               0xf00fcafe11225566ull);
1320     EXPECT_NE(0, datatype); // make sure we writing non-zero
1321     WriteCallback wc;
1322     kvstore->begin();
1323     kvstore->set(item, wc);
1324     kvstore->commit(nullptr /*no collections manifest*/);
1325
1326     MockedGetCallback<GetValue> gc;
1327     EXPECT_CALL(gc, status(ENGINE_SUCCESS));
1328     EXPECT_CALL(gc, cas(0xf00fcafe11225566ull));
1329     EXPECT_CALL(gc, expTime(0xaa00bb11));
1330     EXPECT_CALL(gc, flags(0x01020304));
1331     EXPECT_CALL(gc, datatype(PROTOCOL_BINARY_DATATYPE_JSON));
1332
1333     kvstore->get(key, 0, gc);
1334 }
1335
1336 TEST_F(CouchstoreTest, fuzzV0) {
1337     StoredDocKey key = makeStoredDocKey("key");
1338     Item item(key, 0, 0, "value", 5);
1339     WriteCallback wc;
1340     kvstore->begin();
1341     auto request = kvstore->setAndReturnRequest(item, wc);
1342
1343     // Now directly mess with the metadata of the value which will be written
1344     MockCouchRequest::MetaData meta;
1345     meta.cas = 0xf00fcafe11225566ull;
1346     meta.expiry = 0xaa00bb11;
1347     meta.flags = 0x01020304;
1348     request->writeMetaData(meta, MockCouchRequest::MetaData::sizeofV0);
1349     kvstore->commit(nullptr /*no collections manifest*/);
1350
1351     // CAS is byteswapped when read back
1352     MockedGetCallback<GetValue> gc;
1353     EXPECT_CALL(gc, status(ENGINE_SUCCESS));
1354     EXPECT_CALL(gc, cas(htonll(0xf00fcafe11225566ull)));
1355     EXPECT_CALL(gc, expTime(htonl(0xaa00bb11)));
1356     EXPECT_CALL(gc, flags(0x01020304));
1357     EXPECT_CALL(gc, datatype(PROTOCOL_BINARY_RAW_BYTES));
1358     kvstore->get(key, 0, gc);
1359 }
1360
1361 TEST_F(CouchstoreTest, fuzzV1) {
1362     StoredDocKey key = makeStoredDocKey("key");
1363     Item item(key, 0, 0, "value", 5);
1364     WriteCallback wc;
1365     kvstore->begin();
1366     auto request = kvstore->setAndReturnRequest(item, wc);
1367
1368     // Now directly mess with the metadata of the value which will be written
1369     MockCouchRequest::MetaData meta;
1370     meta.cas = 0xf00fcafe11225566ull;
1371     meta.expiry = 0xaa00bb11;
1372     meta.flags = 0x01020304;
1373     meta.ext1 = 2;
1374     meta.ext2 = 33;
1375     request->writeMetaData(meta, MockCouchRequest::MetaData::sizeofV1);
1376     kvstore->commit(nullptr /*no collections manifest*/);
1377     MockedGetCallback<GetValue> gc;
1378     uint8_t expectedDataType = 33;
1379     EXPECT_CALL(gc, status(ENGINE_SUCCESS));
1380     EXPECT_CALL(gc, cas(htonll(0xf00fcafe11225566ull)));
1381     EXPECT_CALL(gc, expTime(htonl(0xaa00bb11)));
1382     EXPECT_CALL(gc, flags(0x01020304));
1383     EXPECT_CALL(gc, datatype(protocol_binary_datatype_t(expectedDataType)));
1384     kvstore->get(key, 0, gc);
1385 }
1386
1387 TEST_F(CouchstoreTest, testV0WriteReadWriteRead) {
1388     // Ensure CAS, exptime and flags are set to something.
1389     uint8_t datatype = PROTOCOL_BINARY_DATATYPE_JSON; //lies, but non-zero
1390     StoredDocKey key = makeStoredDocKey("key");
1391     Item item(key,
1392               0x01020304/*flags*/, 0xaa00bb11,/*expiry*/
1393               "value", 5,
1394               &datatype, 1, /*ext_meta is v1 extension*/
1395               0xf00fcafe11225566ull);
1396
1397     EXPECT_NE(0, datatype); // make sure we writing non-zero values
1398
1399     // Write an item with forced (valid) V0 meta
1400     MockCouchRequest::MetaData meta;
1401     meta.cas = 0xf00fcafe11225566ull;
1402     meta.expiry = 0xaa00bb11;
1403     meta.flags = 0x01020304;
1404
1405     WriteCallback wc;
1406     kvstore->begin();
1407     auto request = kvstore->setAndReturnRequest(item, wc);
1408
1409     // Force the meta to be V0
1410     request->writeMetaData(meta, MockCouchRequest::MetaData::sizeofV0);
1411
1412     // Commit it
1413     kvstore->commit(nullptr /*no collections manifest*/);
1414
1415     // Read back, are V1 fields sane?
1416     MockedGetCallback<GetValue> gc;
1417     EXPECT_CALL(gc, status(ENGINE_SUCCESS));
1418     EXPECT_CALL(gc, cas(htonll(0xf00fcafe11225566ull)));
1419     EXPECT_CALL(gc, expTime(htonl(0xaa00bb11)));
1420     EXPECT_CALL(gc, flags(0x01020304));
1421     EXPECT_CALL(gc, datatype(protocol_binary_datatype_t(meta.ext2)));
1422     kvstore->get(key, 0, gc);
1423
1424     // Write back the item we read (this will write out V1 meta)
1425     kvstore->begin();
1426     kvstore->set(*gc.getValue(), wc);
1427     kvstore->commit(nullptr /*no collections manifest*/);
1428
1429     // Read back, is conf_res_mode sane?
1430     MockedGetCallback<GetValue> gc2;
1431     EXPECT_CALL(gc2, status(ENGINE_SUCCESS));
1432     EXPECT_CALL(gc2, cas(htonll(0xf00fcafe11225566ull)));
1433     EXPECT_CALL(gc2, expTime(htonl(0xaa00bb11)));
1434     EXPECT_CALL(gc2, flags(0x01020304));
1435     EXPECT_CALL(gc2, datatype(protocol_binary_datatype_t(meta.ext2)));
1436     kvstore->get(key, 0, gc2);
1437 }
1438
1439 TEST_F(CouchstoreTest, testV2WriteRead) {
1440     // Ensure CAS, exptime and flags are set to something.
1441     uint8_t datatype = PROTOCOL_BINARY_DATATYPE_JSON; //lies, but non-zero
1442     StoredDocKey key = makeStoredDocKey("key");
1443     Item item(key,
1444               0x01020304/*flags*/, 0xaa00bb11,/*expiry*/
1445               "value", 5,
1446               &datatype, 1, /*ext_meta is v1 extension*/
1447               0xf00fcafe11225566ull);
1448
1449     EXPECT_NE(0, datatype); // make sure we writing non-zero values
1450
1451     // Write an item with forced (valid) V2 meta
1452     // In 4.6 we removed the extra conflict resolution byte, so be sure we
1453     // operate correctly if a document has V2 meta.
1454     MockCouchRequest::MetaData meta;
1455     meta.cas = 0xf00fcafe11225566ull;
1456     meta.expiry = 0xaa00bb11;
1457     meta.flags = 0x01020304;
1458     meta.ext1 = FLEX_META_CODE;
1459     meta.ext2 = datatype;
1460     meta.legacyDeleted = 0x01;
1461
1462     WriteCallback wc;
1463     kvstore->begin();
1464     auto request = kvstore->setAndReturnRequest(item, wc);
1465
1466     // Force the meta to be V2 (19 bytes)
1467     request->writeMetaData(meta, MockCouchRequest::MetaData::sizeofV2);
1468
1469     // Commit it
1470     kvstore->commit(nullptr /*no collections manifest*/);
1471
1472     // Read back successful, the extra byte will of been dropped.
1473     MockedGetCallback<GetValue> gc;
1474     EXPECT_CALL(gc, status(ENGINE_SUCCESS));
1475     EXPECT_CALL(gc, cas(htonll(0xf00fcafe11225566ull)));
1476     EXPECT_CALL(gc, expTime(htonl(0xaa00bb11)));
1477     EXPECT_CALL(gc, flags(0x01020304));
1478     EXPECT_CALL(gc, datatype(protocol_binary_datatype_t(meta.ext2)));
1479     kvstore->get(key, 0, gc);
1480 }
1481
1482 class CouchKVStoreMetaData : public ::testing::Test {
1483 };
1484
1485 TEST_F(CouchKVStoreMetaData, basic) {
1486     // Lock down the size assumptions.
1487     EXPECT_EQ(16, MetaData::getMetaDataSize(MetaData::Version::V0));
1488     EXPECT_EQ(16 + 2, MetaData::getMetaDataSize(MetaData::Version::V1));
1489     EXPECT_EQ(16 + 2 + 1, MetaData::getMetaDataSize(MetaData::Version::V2));
1490 }
1491
1492 TEST_F(CouchKVStoreMetaData, overlay) {
1493     std::vector<char> data(16);
1494     sized_buf meta;
1495     meta.buf = data.data();
1496     meta.size = data.size();
1497     auto metadata = MetaDataFactory::createMetaData(meta);
1498     EXPECT_EQ(MetaData::Version::V0, metadata->getVersionInitialisedFrom());
1499
1500     data.resize(16 + 2);
1501     meta.buf = data.data();
1502     meta.size = data.size();
1503     metadata = MetaDataFactory::createMetaData(meta);
1504     EXPECT_EQ(MetaData::Version::V1, metadata->getVersionInitialisedFrom());
1505
1506     // Even with a 19 byte (v2) meta, the expectation is we become V1
1507     data.resize(16 + 2 + 1);
1508     meta.buf = data.data();
1509     meta.size = data.size();
1510     metadata = MetaDataFactory::createMetaData(meta);
1511     EXPECT_EQ(MetaData::Version::V1, metadata->getVersionInitialisedFrom());
1512
1513     // Buffers too large and small
1514     data.resize(16 + 2 + 1 + 1);
1515     meta.buf = data.data();
1516     meta.size = data.size();
1517     EXPECT_THROW(MetaDataFactory::createMetaData(meta), std::logic_error);
1518
1519     data.resize(15);
1520     meta.buf = data.data();
1521     meta.size = data.size();
1522     EXPECT_THROW(MetaDataFactory::createMetaData(meta), std::logic_error);
1523 }
1524
1525 TEST_F(CouchKVStoreMetaData, overlayExpands1) {
1526     std::vector<char> data(16);
1527     sized_buf meta;
1528     sized_buf out;
1529     meta.buf = data.data();
1530     meta.size = data.size();
1531
1532     // V0 in yet V1 "moved out"
1533     auto metadata = MetaDataFactory::createMetaData(meta);
1534     EXPECT_EQ(MetaData::Version::V0, metadata->getVersionInitialisedFrom());
1535     out.size = MetaData::getMetaDataSize(MetaData::Version::V1);
1536     out.buf = new char[out.size];
1537     metadata->copyToBuf(out);
1538     EXPECT_EQ(out.size, MetaData::getMetaDataSize(MetaData::Version::V1));
1539
1540     // We created a copy of the metadata so we must cleanup
1541     delete [] out.buf;
1542 }
1543
1544 TEST_F(CouchKVStoreMetaData, overlayExpands2) {
1545     std::vector<char> data(16 + 2);
1546     sized_buf meta;
1547     sized_buf out;
1548     meta.buf = data.data();
1549     meta.size = data.size();
1550
1551     // V1 in V1 "moved out"
1552     auto metadata = MetaDataFactory::createMetaData(meta);
1553     EXPECT_EQ(MetaData::Version::V1, metadata->getVersionInitialisedFrom());
1554     out.size = MetaData::getMetaDataSize(MetaData::Version::V1);
1555     out.buf = new char[out.size];
1556     metadata->copyToBuf(out);
1557     EXPECT_EQ(out.size, MetaData::getMetaDataSize(MetaData::Version::V1));
1558
1559     // We created a copy of the metadata so we must cleanup
1560     delete [] out.buf;
1561 }
1562
1563 TEST_F(CouchKVStoreMetaData, writeToOverlay) {
1564     std::vector<char> data(16);
1565     sized_buf meta;
1566     sized_buf out;
1567     meta.buf = data.data();
1568     meta.size = data.size();
1569
1570     // Test that we can initialise from V0 but still set
1571     // all fields of all versions
1572     auto metadata = MetaDataFactory::createMetaData(meta);
1573     EXPECT_EQ(MetaData::Version::V0, metadata->getVersionInitialisedFrom());
1574
1575     uint64_t cas = 0xf00f00ull;
1576     uint32_t exp = 0xcafe1234;
1577     uint32_t flags = 0xc0115511;
1578     metadata->setCas(cas);
1579     metadata->setExptime(exp);
1580     metadata->setFlags(flags);
1581     metadata->setDataType(PROTOCOL_BINARY_DATATYPE_JSON);
1582
1583     // Check they all read back
1584     EXPECT_EQ(cas, metadata->getCas());
1585     EXPECT_EQ(exp, metadata->getExptime());
1586     EXPECT_EQ(flags, metadata->getFlags());
1587     EXPECT_EQ(FLEX_META_CODE, metadata->getFlexCode());
1588     EXPECT_EQ(PROTOCOL_BINARY_DATATYPE_JSON, metadata->getDataType());
1589
1590     // Now we move the metadata out, this will give back a V1 structure
1591     out.size = MetaData::getMetaDataSize(MetaData::Version::V1);
1592     out.buf = new char[out.size];
1593     metadata->copyToBuf(out);
1594     metadata = MetaDataFactory::createMetaData(out);
1595     EXPECT_EQ(MetaData::Version::V1, metadata->getVersionInitialisedFrom()); // Is it V1?
1596
1597     // All the written fields should be the same
1598     // Check they all read back
1599     EXPECT_EQ(cas, metadata->getCas());
1600     EXPECT_EQ(exp, metadata->getExptime());
1601     EXPECT_EQ(flags, metadata->getFlags());
1602     EXPECT_EQ(FLEX_META_CODE, metadata->getFlexCode());
1603     EXPECT_EQ(PROTOCOL_BINARY_DATATYPE_JSON, metadata->getDataType());
1604     EXPECT_EQ(out.size, MetaData::getMetaDataSize(MetaData::Version::V1));
1605
1606     // We moved the metadata so we must cleanup
1607     delete [] out.buf;
1608 }
1609
1610 //
1611 // Test that assignment operates as expected (we use this in edit_docinfo_hook)
1612 //
1613 TEST_F(CouchKVStoreMetaData, assignment) {
1614     std::vector<char> data(16);
1615     sized_buf meta;
1616     meta.buf = data.data();
1617     meta.size = data.size();
1618     auto metadata = MetaDataFactory::createMetaData(meta);
1619     uint64_t cas = 0xf00f00ull;
1620     uint32_t exp = 0xcafe1234;
1621     uint32_t flags = 0xc0115511;
1622     metadata->setCas(cas);
1623     metadata->setExptime(exp);
1624     metadata->setFlags(flags);
1625     metadata->setDataType( PROTOCOL_BINARY_DATATYPE_JSON);
1626
1627     // Create a second metadata to write into
1628     auto copy = MetaDataFactory::createMetaData();
1629
1630     // Copy overlaid into managed
1631     *copy = *metadata;
1632
1633     // Test that the copy doesn't write to metadata
1634     copy->setExptime(100);
1635     EXPECT_EQ(exp, metadata->getExptime());
1636
1637     EXPECT_EQ(cas, copy->getCas());
1638     EXPECT_EQ(100, copy->getExptime());
1639     EXPECT_EQ(flags, copy->getFlags());
1640     EXPECT_EQ(FLEX_META_CODE, copy->getFlexCode());
1641     EXPECT_EQ(PROTOCOL_BINARY_DATATYPE_JSON, copy->getDataType());
1642
1643     // And a final assignment
1644     auto copy2 = MetaDataFactory::createMetaData();
1645     *copy2 = *copy;
1646
1647     // test that copy2 doesn't update copy
1648     copy2->setCas(99);
1649     EXPECT_NE(99, copy->getCas());
1650
1651     // Yet copy2 did
1652     EXPECT_EQ(99, copy2->getCas());
1653     EXPECT_EQ(100, copy2->getExptime());
1654     EXPECT_EQ(flags, copy2->getFlags());
1655     EXPECT_EQ(FLEX_META_CODE, copy2->getFlexCode());
1656     EXPECT_EQ(PROTOCOL_BINARY_DATATYPE_JSON, copy2->getDataType());
1657 }
1658
1659 #ifdef EP_USE_FORESTDB
1660 // Test cases which run on both Couchstore and ForestDB
1661 INSTANTIATE_TEST_CASE_P(CouchstoreAndForestDB,
1662                         CouchAndForestTest,
1663                         ::testing::Values("couchdb", "forestdb"),
1664                         [] (const ::testing::TestParamInfo<std::string>& info) {
1665                             return info.param;
1666                         });
1667 #else
1668 INSTANTIATE_TEST_CASE_P(CouchstoreAndForestDB,
1669                         CouchAndForestTest,
1670                         ::testing::Values("couchdb"),
1671                         [] (const ::testing::TestParamInfo<std::string>& info) {
1672                             return info.param;
1673                         });
1674 #endif