)]}'
{"id":"kv_engine~113749","triplet_id":"kv_engine~master~I33ecfa78b03b4d2120b5d05f54984b24ce038fd8","project":"kv_engine","branch":"master","hashtags":[],"change_id":"I33ecfa78b03b4d2120b5d05f54984b24ce038fd8","subject":"MB-35458 [SR]: Move SyncWrite completion to bg DurabilityCompletionTask","status":"MERGED","created":"2019-08-23 11:56:52.000000000","updated":"2019-08-27 09:57:26.000000000","submitted":"2019-08-27 09:14:19.000000000","submitter":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"total_comment_count":8,"unresolved_comment_count":0,"has_review_started":true,"submission_id":"113749-1566897259964-d83a7204","meta_rev_id":"c32d0963a751a18f3e1d0e1d9fb9dde4338acc48","_number":113749,"virtual_id_number":113749,"owner":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"actions":{},"labels":{"Verified":{"approved":{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"all":[{"value":0,"_account_id":1000002,"name":"Trond Norbye","display_name":"Trond","email":"trond.norbye@couchbase.com","username":"trond","avatars":[{"url":"https://www.gravatar.com/avatar/1521a1fa74885ab7b9748113afe689cc.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/1521a1fa74885ab7b9748113afe689cc.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/1521a1fa74885ab7b9748113afe689cc.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/1521a1fa74885ab7b9748113afe689cc.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"value":0,"_account_id":1004368,"name":"Richard de Mellow","email":"richard.demellow@couchbase.com","username":"rdemellow","avatars":[{"url":"https://www.gravatar.com/avatar/35c97da872c07458f5e5bc4938b2489f.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/35c97da872c07458f5e5bc4938b2489f.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/35c97da872c07458f5e5bc4938b2489f.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/35c97da872c07458f5e5bc4938b2489f.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"value":0,"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"value":0,"_account_id":1004341,"name":"Ben Huddleston","email":"ben.huddleston@couchbase.com","username":"BenHuddleston","avatars":[{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"value":0,"_account_id":1003828,"name":"James H","email":"james.harrison@couchbase.com","username":"jameseh96","avatars":[{"url":"https://www.gravatar.com/avatar/70d2875f461e17e71a2a16972681c794.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/70d2875f461e17e71a2a16972681c794.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/70d2875f461e17e71a2a16972681c794.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/70d2875f461e17e71a2a16972681c794.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"value":0,"_account_id":1001547,"name":"Dan Owen","email":"owend@couchbase.com","username":"owend","avatars":[{"url":"https://www.gravatar.com/avatar/0761a726bc37606caee7ec6bf566deca.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0761a726bc37606caee7ec6bf566deca.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0761a726bc37606caee7ec6bf566deca.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0761a726bc37606caee7ec6bf566deca.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"tag":"autogenerated:jenkins-gerrit-trigger","value":1,"date":"2019-08-27 09:14:19.000000000","permitted_voting_range":{"min":1,"max":1},"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},{"value":0,"_account_id":1001546,"name":"Jim Walker","email":"jim@couchbase.com","username":"jim","avatars":[{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"value":0,"_account_id":1002319,"name":"Dave Finlay","email":"dave.finlay@couchbase.com","username":"dave-finlay","avatars":[{"url":"https://www.gravatar.com/avatar/aa50dbeeeae1f0a9a777fcf4b80c4fb9.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/aa50dbeeeae1f0a9a777fcf4b80c4fb9.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/aa50dbeeeae1f0a9a777fcf4b80c4fb9.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/aa50dbeeeae1f0a9a777fcf4b80c4fb9.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]}],"values":{"-1":"Fails"," 0":"No score","+1":"Verified"},"description":"","default_value":0},"Code-Review":{"approved":{"_account_id":1004341,"name":"Ben Huddleston","email":"ben.huddleston@couchbase.com","username":"BenHuddleston","avatars":[{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"all":[{"value":0,"_account_id":1000002,"name":"Trond Norbye","display_name":"Trond","email":"trond.norbye@couchbase.com","username":"trond","avatars":[{"url":"https://www.gravatar.com/avatar/1521a1fa74885ab7b9748113afe689cc.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/1521a1fa74885ab7b9748113afe689cc.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/1521a1fa74885ab7b9748113afe689cc.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/1521a1fa74885ab7b9748113afe689cc.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"value":0,"_account_id":1004368,"name":"Richard de Mellow","email":"richard.demellow@couchbase.com","username":"rdemellow","avatars":[{"url":"https://www.gravatar.com/avatar/35c97da872c07458f5e5bc4938b2489f.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/35c97da872c07458f5e5bc4938b2489f.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/35c97da872c07458f5e5bc4938b2489f.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/35c97da872c07458f5e5bc4938b2489f.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"value":0,"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"value":2,"date":"2019-08-27 09:14:19.000000000","permitted_voting_range":{"min":2,"max":2},"_account_id":1004341,"name":"Ben Huddleston","email":"ben.huddleston@couchbase.com","username":"BenHuddleston","avatars":[{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"value":0,"_account_id":1003828,"name":"James H","email":"james.harrison@couchbase.com","username":"jameseh96","avatars":[{"url":"https://www.gravatar.com/avatar/70d2875f461e17e71a2a16972681c794.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/70d2875f461e17e71a2a16972681c794.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/70d2875f461e17e71a2a16972681c794.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/70d2875f461e17e71a2a16972681c794.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"value":0,"_account_id":1001547,"name":"Dan Owen","email":"owend@couchbase.com","username":"owend","avatars":[{"url":"https://www.gravatar.com/avatar/0761a726bc37606caee7ec6bf566deca.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0761a726bc37606caee7ec6bf566deca.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0761a726bc37606caee7ec6bf566deca.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0761a726bc37606caee7ec6bf566deca.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"value":0,"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},{"value":0,"_account_id":1001546,"name":"Jim Walker","email":"jim@couchbase.com","username":"jim","avatars":[{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"value":0,"_account_id":1002319,"name":"Dave Finlay","email":"dave.finlay@couchbase.com","username":"dave-finlay","avatars":[{"url":"https://www.gravatar.com/avatar/aa50dbeeeae1f0a9a777fcf4b80c4fb9.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/aa50dbeeeae1f0a9a777fcf4b80c4fb9.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/aa50dbeeeae1f0a9a777fcf4b80c4fb9.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/aa50dbeeeae1f0a9a777fcf4b80c4fb9.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]}],"values":{"-2":"Do not submit","-1":"I would prefer that you didn\u0027t submit this"," 0":"No score","+1":"Looks good to me, but someone else must approve","+2":"Looks good to me, approved"},"description":"","default_value":0},"Well-Formed":{"all":[{"value":0,"_account_id":1000002,"name":"Trond Norbye","display_name":"Trond","email":"trond.norbye@couchbase.com","username":"trond","avatars":[{"url":"https://www.gravatar.com/avatar/1521a1fa74885ab7b9748113afe689cc.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/1521a1fa74885ab7b9748113afe689cc.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/1521a1fa74885ab7b9748113afe689cc.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/1521a1fa74885ab7b9748113afe689cc.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"value":0,"_account_id":1004368,"name":"Richard de Mellow","email":"richard.demellow@couchbase.com","username":"rdemellow","avatars":[{"url":"https://www.gravatar.com/avatar/35c97da872c07458f5e5bc4938b2489f.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/35c97da872c07458f5e5bc4938b2489f.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/35c97da872c07458f5e5bc4938b2489f.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/35c97da872c07458f5e5bc4938b2489f.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"value":0,"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"value":0,"_account_id":1004341,"name":"Ben Huddleston","email":"ben.huddleston@couchbase.com","username":"BenHuddleston","avatars":[{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"value":0,"_account_id":1003828,"name":"James H","email":"james.harrison@couchbase.com","username":"jameseh96","avatars":[{"url":"https://www.gravatar.com/avatar/70d2875f461e17e71a2a16972681c794.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/70d2875f461e17e71a2a16972681c794.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/70d2875f461e17e71a2a16972681c794.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/70d2875f461e17e71a2a16972681c794.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"value":0,"_account_id":1001547,"name":"Dan Owen","email":"owend@couchbase.com","username":"owend","avatars":[{"url":"https://www.gravatar.com/avatar/0761a726bc37606caee7ec6bf566deca.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0761a726bc37606caee7ec6bf566deca.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0761a726bc37606caee7ec6bf566deca.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0761a726bc37606caee7ec6bf566deca.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"value":0,"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},{"value":0,"_account_id":1001546,"name":"Jim Walker","email":"jim@couchbase.com","username":"jim","avatars":[{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"value":0,"_account_id":1002319,"name":"Dave Finlay","email":"dave.finlay@couchbase.com","username":"dave-finlay","avatars":[{"url":"https://www.gravatar.com/avatar/aa50dbeeeae1f0a9a777fcf4b80c4fb9.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/aa50dbeeeae1f0a9a777fcf4b80c4fb9.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/aa50dbeeeae1f0a9a777fcf4b80c4fb9.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/aa50dbeeeae1f0a9a777fcf4b80c4fb9.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]}],"values":{"-1":"Commit is not well-formed"," 0":"Well-formedness not checked","+1":"Commit is well-formed"},"description":"","default_value":0,"optional":true}},"removable_reviewers":[],"reviewers":{"REVIEWER":[{"_account_id":1000002,"name":"Trond Norbye","display_name":"Trond","email":"trond.norbye@couchbase.com","username":"trond","avatars":[{"url":"https://www.gravatar.com/avatar/1521a1fa74885ab7b9748113afe689cc.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/1521a1fa74885ab7b9748113afe689cc.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/1521a1fa74885ab7b9748113afe689cc.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/1521a1fa74885ab7b9748113afe689cc.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"_account_id":1001546,"name":"Jim Walker","email":"jim@couchbase.com","username":"jim","avatars":[{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"_account_id":1001547,"name":"Dan Owen","email":"owend@couchbase.com","username":"owend","avatars":[{"url":"https://www.gravatar.com/avatar/0761a726bc37606caee7ec6bf566deca.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0761a726bc37606caee7ec6bf566deca.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0761a726bc37606caee7ec6bf566deca.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0761a726bc37606caee7ec6bf566deca.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"_account_id":1002319,"name":"Dave Finlay","email":"dave.finlay@couchbase.com","username":"dave-finlay","avatars":[{"url":"https://www.gravatar.com/avatar/aa50dbeeeae1f0a9a777fcf4b80c4fb9.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/aa50dbeeeae1f0a9a777fcf4b80c4fb9.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/aa50dbeeeae1f0a9a777fcf4b80c4fb9.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/aa50dbeeeae1f0a9a777fcf4b80c4fb9.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"_account_id":1003828,"name":"James H","email":"james.harrison@couchbase.com","username":"jameseh96","avatars":[{"url":"https://www.gravatar.com/avatar/70d2875f461e17e71a2a16972681c794.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/70d2875f461e17e71a2a16972681c794.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/70d2875f461e17e71a2a16972681c794.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/70d2875f461e17e71a2a16972681c794.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"_account_id":1004341,"name":"Ben Huddleston","email":"ben.huddleston@couchbase.com","username":"BenHuddleston","avatars":[{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"_account_id":1004368,"name":"Richard de Mellow","email":"richard.demellow@couchbase.com","username":"rdemellow","avatars":[{"url":"https://www.gravatar.com/avatar/35c97da872c07458f5e5bc4938b2489f.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/35c97da872c07458f5e5bc4938b2489f.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/35c97da872c07458f5e5bc4938b2489f.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/35c97da872c07458f5e5bc4938b2489f.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]}]},"pending_reviewers":{},"reviewer_updates":[{"updated":"2019-08-23 12:02:19.000000000","updated_by":{"_account_id":1000002,"name":"Trond Norbye","display_name":"Trond","email":"trond.norbye@couchbase.com","username":"trond","avatars":[{"url":"https://www.gravatar.com/avatar/1521a1fa74885ab7b9748113afe689cc.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/1521a1fa74885ab7b9748113afe689cc.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/1521a1fa74885ab7b9748113afe689cc.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/1521a1fa74885ab7b9748113afe689cc.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"reviewer":{"_account_id":1000002,"name":"Trond Norbye","display_name":"Trond","email":"trond.norbye@couchbase.com","username":"trond","avatars":[{"url":"https://www.gravatar.com/avatar/1521a1fa74885ab7b9748113afe689cc.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/1521a1fa74885ab7b9748113afe689cc.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/1521a1fa74885ab7b9748113afe689cc.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/1521a1fa74885ab7b9748113afe689cc.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"state":"REVIEWER"},{"updated":"2019-08-23 12:02:19.000000000","updated_by":{"_account_id":1001546,"name":"Jim Walker","email":"jim@couchbase.com","username":"jim","avatars":[{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"reviewer":{"_account_id":1001546,"name":"Jim Walker","email":"jim@couchbase.com","username":"jim","avatars":[{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"state":"REVIEWER"},{"updated":"2019-08-23 12:02:19.000000000","updated_by":{"_account_id":1001547,"name":"Dan Owen","email":"owend@couchbase.com","username":"owend","avatars":[{"url":"https://www.gravatar.com/avatar/0761a726bc37606caee7ec6bf566deca.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0761a726bc37606caee7ec6bf566deca.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0761a726bc37606caee7ec6bf566deca.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0761a726bc37606caee7ec6bf566deca.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"reviewer":{"_account_id":1001547,"name":"Dan Owen","email":"owend@couchbase.com","username":"owend","avatars":[{"url":"https://www.gravatar.com/avatar/0761a726bc37606caee7ec6bf566deca.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0761a726bc37606caee7ec6bf566deca.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0761a726bc37606caee7ec6bf566deca.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0761a726bc37606caee7ec6bf566deca.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"state":"REVIEWER"},{"updated":"2019-08-23 12:02:19.000000000","updated_by":{"_account_id":1002319,"name":"Dave Finlay","email":"dave.finlay@couchbase.com","username":"dave-finlay","avatars":[{"url":"https://www.gravatar.com/avatar/aa50dbeeeae1f0a9a777fcf4b80c4fb9.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/aa50dbeeeae1f0a9a777fcf4b80c4fb9.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/aa50dbeeeae1f0a9a777fcf4b80c4fb9.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/aa50dbeeeae1f0a9a777fcf4b80c4fb9.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"reviewer":{"_account_id":1002319,"name":"Dave Finlay","email":"dave.finlay@couchbase.com","username":"dave-finlay","avatars":[{"url":"https://www.gravatar.com/avatar/aa50dbeeeae1f0a9a777fcf4b80c4fb9.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/aa50dbeeeae1f0a9a777fcf4b80c4fb9.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/aa50dbeeeae1f0a9a777fcf4b80c4fb9.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/aa50dbeeeae1f0a9a777fcf4b80c4fb9.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"state":"REVIEWER"},{"updated":"2019-08-23 12:02:19.000000000","updated_by":{"_account_id":1004368,"name":"Richard de Mellow","email":"richard.demellow@couchbase.com","username":"rdemellow","avatars":[{"url":"https://www.gravatar.com/avatar/35c97da872c07458f5e5bc4938b2489f.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/35c97da872c07458f5e5bc4938b2489f.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/35c97da872c07458f5e5bc4938b2489f.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/35c97da872c07458f5e5bc4938b2489f.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"reviewer":{"_account_id":1004368,"name":"Richard de Mellow","email":"richard.demellow@couchbase.com","username":"rdemellow","avatars":[{"url":"https://www.gravatar.com/avatar/35c97da872c07458f5e5bc4938b2489f.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/35c97da872c07458f5e5bc4938b2489f.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/35c97da872c07458f5e5bc4938b2489f.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/35c97da872c07458f5e5bc4938b2489f.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"state":"REVIEWER"},{"updated":"2019-08-23 15:36:11.000000000","updated_by":{"_account_id":1003828,"name":"James H","email":"james.harrison@couchbase.com","username":"jameseh96","avatars":[{"url":"https://www.gravatar.com/avatar/70d2875f461e17e71a2a16972681c794.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/70d2875f461e17e71a2a16972681c794.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/70d2875f461e17e71a2a16972681c794.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/70d2875f461e17e71a2a16972681c794.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"reviewer":{"_account_id":1003828,"name":"James H","email":"james.harrison@couchbase.com","username":"jameseh96","avatars":[{"url":"https://www.gravatar.com/avatar/70d2875f461e17e71a2a16972681c794.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/70d2875f461e17e71a2a16972681c794.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/70d2875f461e17e71a2a16972681c794.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/70d2875f461e17e71a2a16972681c794.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"state":"REVIEWER"},{"updated":"2019-08-23 19:08:31.000000000","updated_by":{"_account_id":1004341,"name":"Ben Huddleston","email":"ben.huddleston@couchbase.com","username":"BenHuddleston","avatars":[{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"reviewer":{"_account_id":1004341,"name":"Ben Huddleston","email":"ben.huddleston@couchbase.com","username":"BenHuddleston","avatars":[{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"state":"REVIEWER"},{"updated":"2019-08-23 19:24:39.000000000","updated_by":{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"reviewer":{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"state":"REVIEWER"}],"messages":[{"id":"ca16439a3fbb040871311ff37f06cd387b90d6f8","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"date":"2019-08-23 11:56:52.000000000","message":"Uploaded patch set 1.","accounts_in_message":[],"_revision_number":1},{"id":"ff8a9b5d73668ab10517edc4f84915cabd260aa2","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"date":"2019-08-23 12:01:39.000000000","message":"Uploaded patch set 2.","accounts_in_message":[],"_revision_number":2},{"id":"15c65d995aa031c35efdac0dd917ce2d973d0b73","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"date":"2019-08-23 12:01:47.000000000","message":"Patch Set 1: Verified-1\n\nBuild Failed \n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-clang_analyzer-master/19529/ : ABORTED\n\nBuild which was aborted due to a newer patch set being uploaded for the given review. ( http://cv.jenkins.couchbase.com/job/kv_engine-clang_analyzer-master/19529/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-windows-master/22862/ : ABORTED\n\nNo problems were identified. If you know why this problem occurred, please add a suitable Cause for it. ( http://cv.jenkins.couchbase.com/job/kv_engine-windows-master/22862/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-linux-master/21916/ : ABORTED\n\nBuild which was aborted due to a newer patch set being uploaded for the given review. ( http://cv.jenkins.couchbase.com/job/kv_engine-linux-master/21916/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-linux-master-CE/3584/ : ABORTED\n\nBuild which was aborted due to a newer patch set being uploaded for the given review. ( http://cv.jenkins.couchbase.com/job/kv_engine-linux-master-CE/3584/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-threadsanitizer-master/13803/ : ABORTED\n\nNo problems were identified. If you know why this problem occurred, please add a suitable Cause for it. ( http://cv.jenkins.couchbase.com/job/kv_engine-threadsanitizer-master/13803/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.ASan-UBSan/job/master/6542/ : ABORTED\n\nBuild which was aborted due to a newer patch set being uploaded for the given review. ( http://cv.jenkins.couchbase.com/job/kv_engine.ASan-UBSan/job/master/6542/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-clang_format/20519/ : SUCCESS","accounts_in_message":[],"_revision_number":1},{"id":"1c6af9ea6fbd72131eb1f9439a0ec7c5c982a951","author":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"date":"2019-08-23 12:03:19.000000000","message":"Patch Set 2:\n\nNote: re-apply of http://review.couchbase.org/#/c/113141/, after fixing an issue where the last vBucket could be skipped.\n\nTo aid reviewers, the only changes since the original patch are between v1..v2:\n* v1 is the original patch merged as http://review.couchbase.org/#/c/113141/ (with the error)\n* v2 has the fix to not skip the last vBucket.","accounts_in_message":[],"_revision_number":2},{"id":"abd7a5e8fc9b15772249718b77851bcfb2620837","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"date":"2019-08-23 12:44:42.000000000","message":"Patch Set 2: Verified-1\n\nBuild Failed \n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-threadsanitizer-master/13804/ : FAILURE\n\nFailure of a GoogleTest test:\n\nCheckpointRemoverEPTest.CursorDropMemoryFreed (5405 ms) ( http://cv.jenkins.couchbase.com/job/kv_engine-threadsanitizer-master/13804/ )\n\nFailure of a CTest test  30/179 Test  #23: ep-engine_ep_unit_tests ...................................................... ( http://cv.jenkins.couchbase.com/job/kv_engine-threadsanitizer-master/13804/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-clang_analyzer-master/19530/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-windows-master/22863/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-linux-master/21917/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-linux-master-CE/3587/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-clang_format/20520/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv-engine-cv-perf/12833/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.ASan-UBSan/job/master/6543/ : SUCCESS","accounts_in_message":[],"_revision_number":2},{"id":"382371771201137cea9da111b5b4b135b1eff35b","author":{"_account_id":1004341,"name":"Ben Huddleston","email":"ben.huddleston@couchbase.com","username":"BenHuddleston","avatars":[{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"date":"2019-08-23 14:43:58.000000000","message":"Patch Set 2: Code-Review-1\n\n(2 comments)\n\nCode looks good, just comment/test things","accounts_in_message":[],"_revision_number":2},{"id":"6fae9a232cafcff387147108357b0fefe232bb9f","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"date":"2019-08-23 14:48:36.000000000","message":"Patch Set 3: Published edit on patch set 2.","accounts_in_message":[],"_revision_number":3},{"id":"4b28e5e609e825277ae804e47b49fcc7270f32fa","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"date":"2019-08-23 14:48:37.000000000","message":"Patch Set 2:\n\nBuild Failed \n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-threadsanitizer-master/13809/ : ABORTED\n\nBuild which was aborted due to a newer patch set being uploaded for the given review. ( http://cv.jenkins.couchbase.com/job/kv_engine-threadsanitizer-master/13809/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-clang_analyzer-master/19530/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-windows-master/22863/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-linux-master/21917/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-linux-master-CE/3587/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-clang_format/20520/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv-engine-cv-perf/12833/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.ASan-UBSan/job/master/6543/ : SUCCESS","accounts_in_message":[],"_revision_number":2},{"id":"a1b0e070fcbd9335479e492e1bc7375d32aa47b2","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"date":"2019-08-23 15:19:34.000000000","message":"Patch Set 3: Verified-1\n\nBuild Failed \n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-clang_analyzer-master/19534/ : FAILURE\n\nCompile error at /home/couchbase/jenkins/workspace/kv_engine-clang_analyzer-master/kv_engine/engines/ep/tests/module_tests/durability_monitor_test.cc:1004:46:\nerror: expected \";\" after expression\n ( http://cv.jenkins.couchbase.com/job/kv_engine-clang_analyzer-master/19534/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-windows-master/22869/ : FAILURE\n\nCompile error at ..\\kv_engine\\engines\\ep\\tests\\module_tests\\durability_monitor_test.cc(1005): error C2143: syntax error: missing \";\" before \"}\" ( http://cv.jenkins.couchbase.com/job/kv_engine-windows-master/22869/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-linux-master/21921/ : FAILURE\n\nCompile error at /home/couchbase/jenkins/workspace/kv_engine-linux-master/kv_engine/engines/ep/tests/module_tests/durability_monitor_test.cc:1005:5:\nerror: expected ?;? before ?}? token\n ( http://cv.jenkins.couchbase.com/job/kv_engine-linux-master/21921/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-linux-master-CE/3590/ : FAILURE\n\nCompile error at /home/couchbase/jenkins/workspace/kv_engine-linux-master-CE/kv_engine/engines/ep/tests/module_tests/durability_monitor_test.cc:1005:5:\nerror: expected ?;? before ?}? token\n ( http://cv.jenkins.couchbase.com/job/kv_engine-linux-master-CE/3590/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv-engine-cv-perf/12837/ : FAILURE\n\nCompile error at /home/couchbase/jenkins/workspace/kv-engine-cv-perf/kv_engine/engines/ep/tests/module_tests/durability_monitor_test.cc:1005:5:\nerror: expected \";\" before \"}\" token\n ( http://cv.jenkins.couchbase.com/job/kv-engine-cv-perf/12837/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-threadsanitizer-master/13810/ : FAILURE\n\nCompile error at /home/couchbase/jenkins/workspace/kv_engine-threadsanitizer-master/kv_engine/engines/ep/tests/module_tests/durability_monitor_test.cc:1005:5:\nerror: expected ?;? before ?}? token\n ( http://cv.jenkins.couchbase.com/job/kv_engine-threadsanitizer-master/13810/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.ASan-UBSan/job/master/6547/ : FAILURE\n\nCompile error at [2019-08-23T15:02:38.046Z] /home/couchbase/jenkins/workspace/kv_engine.ASan-UBSan_master/kv_engine/engines/ep/tests/module_tests/durability_monitor_test.cc:1005:5:\nerror: expected ?;? before ?}? token\n ( http://cv.jenkins.couchbase.com/job/kv_engine.ASan-UBSan/job/master/6547/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-clang_format/20524/ : SUCCESS","accounts_in_message":[],"_revision_number":3},{"id":"b95630d15f12d5321036f090d1e6bad666630acd","author":{"_account_id":1003828,"name":"James H","email":"james.harrison@couchbase.com","username":"jameseh96","avatars":[{"url":"https://www.gravatar.com/avatar/70d2875f461e17e71a2a16972681c794.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/70d2875f461e17e71a2a16972681c794.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/70d2875f461e17e71a2a16972681c794.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/70d2875f461e17e71a2a16972681c794.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"date":"2019-08-23 15:36:11.000000000","message":"Patch Set 3:\n\n(6 comments)\n\nJust some textual points - as always only for if you happen to do another patch set anyway :)","accounts_in_message":[],"_revision_number":3},{"id":"20ae48b6f35059dbf4123b43127f867ca7b81155","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"date":"2019-08-23 18:57:20.000000000","message":"Patch Set 4: Published edit on patch set 3.","accounts_in_message":[],"_revision_number":4},{"id":"95cacc842499d7b03141a18354a4f148efca2a95","author":{"_account_id":1004341,"name":"Ben Huddleston","email":"ben.huddleston@couchbase.com","username":"BenHuddleston","avatars":[{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"date":"2019-08-23 19:08:31.000000000","message":"Patch Set 4: Code-Review+2","accounts_in_message":[],"_revision_number":4},{"id":"f92fe981b647eb5b907d51d597093a939b9ef16f","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"date":"2019-08-23 19:24:39.000000000","message":"Patch Set 4: Verified+1\n\nBuild Successful \n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-clang_analyzer-master/19540/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-windows-master/22878/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-linux-master/21928/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-linux-master-CE/3597/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-clang_format/20530/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv-engine-cv-perf/12843/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-threadsanitizer-master/13818/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.ASan-UBSan/job/master/6554/ : SUCCESS","accounts_in_message":[],"_revision_number":4},{"id":"9b7e3c3333ba65c9f2e936d843810a72e85fc2ea","tag":"autogenerated:gerrit:merged","author":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"date":"2019-08-27 09:14:19.000000000","message":"Change has been successfully cherry-picked as f6ed489642e80594ff6a9a87a12f3d9c343fccfc by Dave Rigby","accounts_in_message":[],"_revision_number":5},{"id":"0bb233bcbc4a472e35d54559cb01b7eea5049910","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"date":"2019-08-27 09:14:29.000000000","message":"Patch Set 5:\n\nBuild Started http://cv.jenkins.couchbase.com/job/kv_engine-master-post-commit/2101/","accounts_in_message":[],"_revision_number":5},{"id":"c32d0963a751a18f3e1d0e1d9fb9dde4338acc48","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"date":"2019-08-27 09:57:26.000000000","message":"Patch Set 5:\n\nBuild Successful \n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-master-post-commit/2101/ : SUCCESS\u0027","accounts_in_message":[],"_revision_number":5}],"current_revision_number":5,"current_revision":"f6ed489642e80594ff6a9a87a12f3d9c343fccfc","revisions":{"bc3bb8501f574b050b032f484398fd761c0648b0":{"kind":"REWORK","_number":1,"created":"2019-08-23 11:56:52.000000000","uploader":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"ref":"refs/changes/49/113749/1","fetch":{"anonymous http":{"url":"https://review.couchbase.org/kv_engine","ref":"refs/changes/49/113749/1","commands":{"Branch":"git fetch https://review.couchbase.org/kv_engine refs/changes/49/113749/1 \u0026\u0026 git checkout -b change-113749 FETCH_HEAD","Checkout":"git fetch https://review.couchbase.org/kv_engine refs/changes/49/113749/1 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.couchbase.org/kv_engine refs/changes/49/113749/1 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.couchbase.org/kv_engine refs/changes/49/113749/1 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.couchbase.org/kv_engine refs/changes/49/113749/1","Reset To":"git fetch https://review.couchbase.org/kv_engine refs/changes/49/113749/1 \u0026\u0026 git reset --hard FETCH_HEAD"}}},"commit":{"parents":[{"commit":"3ad8757a1838d29b1a2da35dc4c5e7ec0b22e4c8","subject":"Don\u0027t accept cipherlist with no usable ciphers"}],"author":{"name":"Dave Rigby","email":"daver@couchbase.com","date":"2019-08-09 11:55:21.000000000","tz":60},"committer":{"name":"Dave Rigby","email":"daver@couchbase.com","date":"2019-08-23 11:56:31.000000000","tz":60},"subject":"MB-35458 [SR]: Move SyncWrite completion to bg DurabilityCompletionTask","message":"MB-35458 [SR]: Move SyncWrite completion to bg DurabilityCompletionTask\n\nChange how SyncWrites which are Resolved and awaiting Completion are\nhandled, by moving the final VBucket::commit() / abort() into a\nbackground task - DurabilityCompletionTask.\n\n+Background+\n\nThere are two reasons for making this change:\n\na) Performance - specifically latency of front-end worker threads.\n\nBy moving completion into a background task, we reduce the amount of\nwork done on the thread which actually detected the SyncWrite was\nresolved - typically the front-end DCP threads when a DCP_SEQNO_ACK\nis processed.\nGiven that we SEQNO_ACK at the end of Snapshot, A single SEQNO_ACK\ncould result in committing multiple SyncWrites. Committing one\nSyncWrite is similar to a normal front-end Set operation, so there is\npotentially a non-trivial amount of work needed to be done when\ncompleting SyncWrites, which could tie up the front-end thread (causing\nother Connections to have to wait) for a noticable amount of time.\n\nb) Simplification of lock management.\n\nDoing completion in a background task simplifies lock management, for\nexample we avoid lock inversions with earlier locks acquired during\ndcpSeqnoAck when attemping to later call notifySeqnoAvailable when this\nwas done on the original thread.\n\n+Problem+\n\nWhile (a) was the first reason identified for making this change\n(see MB-33092), (b) is the reason this change is being made now. During\ntesting the following lock-order-inversion was seen:\n\n    WARNING: ThreadSanitizer: lock-order-inversion (potential deadlock)\n    Cycle in lock order graph:\n\n    Stream::streamMutex \u003d\u003e StreamContainer::rwlock \u003d\u003e Stream::streamMutex\n\nThe crux of the issue is the processing of DCP_SEQNO_ACKNOWLEDGED\nmessages by the DcpProducer - this acquires the Stream::streamMutex\nbefore calling VBucket::seqnoAcknowledged(), however that function\ncurrently results in VBucket::commit() being called to synchronously\ncomplete the SyncWrite; which in turn must nodify all connected\nreplica that a new seqno is available, requiring\nStreamContainer::rwlock to be acquired:\n\n  Mutex StreamContainer::rwlock acquired here while holding mutex Stream::streamMutex in thread T15:\n    ...\n    #6 StreamContainer\u003cstd::shared_ptr\u003cStream\u003e \u003e::rlock()\n    #7 DcpProducer::notifySeqnoAvailable(Vbid, unsigned long)\n    ...\n    #13 VBucket::commit(...)\n    #14 ActiveDurabilityMonitor::commit(...)\n    #15 ActiveDurabilityMonitor::processCompletedSyncWriteQueue()\n    #16 ActiveDurabilityMonitor::seqnoAckReceived(...)\n    #17 VBucket::seqnoAcknowledged(...)\n    #18 ActiveStream::seqnoAck(...)\n    #19 DcpProducer::seqno_acknowledged(...)\n    ...\n\n  Mutex Stream::streamMutex previously acquired by the same thread here:\n    ...\n    #3 std::lock_guard\u003cstd::mutex\u003e::lock_guard(std::mutex\u0026)\n    #4 ActiveStream::seqnoAck(...)\n    #5 DcpProducer::seqno_acknowledged(...)\n    ...\n\nThis conflicts with the ordering seen when sending items out on the\nDCP connection - inside DcpProducer::step() where the\nStreamContainer::rwlock is acquired first, then ActiveStream::mutex\nacquired later:\n\n  Mutex Stream::streamMutex acquired here while holding mutex StreamContainer::rwlock in thread T15:\n    ...\n    #3 std::lock_guard\u003cstd::mutex\u003e::lock_guard(std::mutex\u0026)\n    #4 ActiveStream::next()\n    #5 DcpProducer::getNextItem()\n    #6 DcpProducer::step(dcp_message_producers*)\n    ...\n\n  Mutex StreamContainer::rwlock previously acquired by the same thread here:\n    #0 pthread_rwlock_rdlock \u003cnull\u003e (libtsan.so.0+0x00000002c98b)\n    ...\n    #4 std::shared_lock\u003ccb::RWLock\u003e::shared_lock(cb::RWLock\u0026)\n    #5 StreamContainer\u003c\u003e::ResumableIterationHandle::ResumableIterationHandle()\n    #6 StreamContainer\u003c\u003e::startResumable()\n    #7 DcpProducer::getNextItem()\n    #8 DcpProducer::step(dcp_message_producers*)\n    ...\n\n+Solution+\n\nThe processing of resolved SyncWrites moved into a new background task.\nInstead of immediately processing them within\nActiveDM::seqnoAckReceived(), that function notifies the new NonIO\nDurabilityCompletionTask that there are SyncWrites waiting for\ncompletion.\n\nDurabilityCompletionTask maintains a bool per vBucket indicating if\nthere are SyncWrites for that vBucket pending completion. When the\ntask is run, for each flag which is true it calls\nVBucket::processResolvedSyncWrites() for the associated VBucket.\n\n+Implementaiton Notes+\n\nCurrently there is just a single DurabilityCompletionTask (per Bucket),\nthis was chosen as 1 task per vBucket (i.e. 1024 per Bucket) would\nbe inefficient for our current background task scheduler (both in terms\nof latency to schedule each task for only one vBucket\u0027s worth of work,\nand in terms of managing that many tasks in the future queue).\n\nHowever, that does _potentially_ mean there\u0027s fewer resources (threads)\navailable to complete SyncWrites on - previously that work could be\ndone concurrently on all frontend threads (~O(num_cpus). Now the same\nwork only has 1 thread available to run on (there\u0027s only a single\nDurabilityCompletionTask).\n\n_If_ this becomes a bottleneck we could look at increasing the number of\nDurabilityCompletionTask - e.g. sharding all vBuckets across multiple\ntasks like flusher / bgfetcher.\n\nChange-Id: I33ecfa78b03b4d2120b5d05f54984b24ce038fd8\n"},"parents_data":[{"branch_name":"refs/heads/master","commit_id":"3ad8757a1838d29b1a2da35dc4c5e7ec0b22e4c8","is_merged_in_target_branch":true,"change_id":"I48a671f66b87887f4d2e244b27990eac0ed83b98","change_number":113701,"patch_set_number":5,"change_status":"MERGED"}],"branch":"refs/heads/master"},"c25470267142833d31f3cbbc39ab109f5c5845cd":{"kind":"REWORK","_number":2,"created":"2019-08-23 12:01:39.000000000","uploader":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"ref":"refs/changes/49/113749/2","fetch":{"anonymous http":{"url":"https://review.couchbase.org/kv_engine","ref":"refs/changes/49/113749/2","commands":{"Branch":"git fetch https://review.couchbase.org/kv_engine refs/changes/49/113749/2 \u0026\u0026 git checkout -b change-113749 FETCH_HEAD","Checkout":"git fetch https://review.couchbase.org/kv_engine refs/changes/49/113749/2 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.couchbase.org/kv_engine refs/changes/49/113749/2 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.couchbase.org/kv_engine refs/changes/49/113749/2 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.couchbase.org/kv_engine refs/changes/49/113749/2","Reset To":"git fetch https://review.couchbase.org/kv_engine refs/changes/49/113749/2 \u0026\u0026 git reset --hard FETCH_HEAD"}}},"commit":{"parents":[{"commit":"3ad8757a1838d29b1a2da35dc4c5e7ec0b22e4c8","subject":"Don\u0027t accept cipherlist with no usable ciphers"}],"author":{"name":"Dave Rigby","email":"daver@couchbase.com","date":"2019-08-09 11:55:21.000000000","tz":60},"committer":{"name":"Dave Rigby","email":"daver@couchbase.com","date":"2019-08-23 12:01:13.000000000","tz":60},"subject":"MB-35458 [SR]: Move SyncWrite completion to bg DurabilityCompletionTask","message":"MB-35458 [SR]: Move SyncWrite completion to bg DurabilityCompletionTask\n\n[[Re-apply after fixing error in DurabilityCompletionTask::run\n   (skipping last vBucket).]]\n\nChange how SyncWrites which are Resolved and awaiting Completion are\nhandled, by moving the final VBucket::commit() / abort() into a\nbackground task - DurabilityCompletionTask.\n\n+Background+\n\nThere are two reasons for making this change:\n\na) Performance - specifically latency of front-end worker threads.\n\nBy moving completion into a background task, we reduce the amount of\nwork done on the thread which actually detected the SyncWrite was\nresolved - typically the front-end DCP threads when a DCP_SEQNO_ACK\nis processed.\nGiven that we SEQNO_ACK at the end of Snapshot, A single SEQNO_ACK\ncould result in committing multiple SyncWrites. Committing one\nSyncWrite is similar to a normal front-end Set operation, so there is\npotentially a non-trivial amount of work needed to be done when\ncompleting SyncWrites, which could tie up the front-end thread (causing\nother Connections to have to wait) for a noticable amount of time.\n\nb) Simplification of lock management.\n\nDoing completion in a background task simplifies lock management, for\nexample we avoid lock inversions with earlier locks acquired during\ndcpSeqnoAck when attemping to later call notifySeqnoAvailable when this\nwas done on the original thread.\n\n+Problem+\n\nWhile (a) was the first reason identified for making this change\n(see MB-33092), (b) is the reason this change is being made now. During\ntesting the following lock-order-inversion was seen:\n\n    WARNING: ThreadSanitizer: lock-order-inversion (potential deadlock)\n    Cycle in lock order graph:\n\n    Stream::streamMutex \u003d\u003e StreamContainer::rwlock \u003d\u003e Stream::streamMutex\n\nThe crux of the issue is the processing of DCP_SEQNO_ACKNOWLEDGED\nmessages by the DcpProducer - this acquires the Stream::streamMutex\nbefore calling VBucket::seqnoAcknowledged(), however that function\ncurrently results in VBucket::commit() being called to synchronously\ncomplete the SyncWrite; which in turn must nodify all connected\nreplica that a new seqno is available, requiring\nStreamContainer::rwlock to be acquired:\n\n  Mutex StreamContainer::rwlock acquired here while holding mutex Stream::streamMutex in thread T15:\n    ...\n    #6 StreamContainer\u003cstd::shared_ptr\u003cStream\u003e \u003e::rlock()\n    #7 DcpProducer::notifySeqnoAvailable(Vbid, unsigned long)\n    ...\n    #13 VBucket::commit(...)\n    #14 ActiveDurabilityMonitor::commit(...)\n    #15 ActiveDurabilityMonitor::processCompletedSyncWriteQueue()\n    #16 ActiveDurabilityMonitor::seqnoAckReceived(...)\n    #17 VBucket::seqnoAcknowledged(...)\n    #18 ActiveStream::seqnoAck(...)\n    #19 DcpProducer::seqno_acknowledged(...)\n    ...\n\n  Mutex Stream::streamMutex previously acquired by the same thread here:\n    ...\n    #3 std::lock_guard\u003cstd::mutex\u003e::lock_guard(std::mutex\u0026)\n    #4 ActiveStream::seqnoAck(...)\n    #5 DcpProducer::seqno_acknowledged(...)\n    ...\n\nThis conflicts with the ordering seen when sending items out on the\nDCP connection - inside DcpProducer::step() where the\nStreamContainer::rwlock is acquired first, then ActiveStream::mutex\nacquired later:\n\n  Mutex Stream::streamMutex acquired here while holding mutex StreamContainer::rwlock in thread T15:\n    ...\n    #3 std::lock_guard\u003cstd::mutex\u003e::lock_guard(std::mutex\u0026)\n    #4 ActiveStream::next()\n    #5 DcpProducer::getNextItem()\n    #6 DcpProducer::step(dcp_message_producers*)\n    ...\n\n  Mutex StreamContainer::rwlock previously acquired by the same thread here:\n    #0 pthread_rwlock_rdlock \u003cnull\u003e (libtsan.so.0+0x00000002c98b)\n    ...\n    #4 std::shared_lock\u003ccb::RWLock\u003e::shared_lock(cb::RWLock\u0026)\n    #5 StreamContainer\u003c\u003e::ResumableIterationHandle::ResumableIterationHandle()\n    #6 StreamContainer\u003c\u003e::startResumable()\n    #7 DcpProducer::getNextItem()\n    #8 DcpProducer::step(dcp_message_producers*)\n    ...\n\n+Solution+\n\nThe processing of resolved SyncWrites moved into a new background task.\nInstead of immediately processing them within\nActiveDM::seqnoAckReceived(), that function notifies the new NonIO\nDurabilityCompletionTask that there are SyncWrites waiting for\ncompletion.\n\nDurabilityCompletionTask maintains a bool per vBucket indicating if\nthere are SyncWrites for that vBucket pending completion. When the\ntask is run, for each flag which is true it calls\nVBucket::processResolvedSyncWrites() for the associated VBucket.\n\n+Implementaiton Notes+\n\nCurrently there is just a single DurabilityCompletionTask (per Bucket),\nthis was chosen as 1 task per vBucket (i.e. 1024 per Bucket) would\nbe inefficient for our current background task scheduler (both in terms\nof latency to schedule each task for only one vBucket\u0027s worth of work,\nand in terms of managing that many tasks in the future queue).\n\nHowever, that does _potentially_ mean there\u0027s fewer resources (threads)\navailable to complete SyncWrites on - previously that work could be\ndone concurrently on all frontend threads (~O(num_cpus). Now the same\nwork only has 1 thread available to run on (there\u0027s only a single\nDurabilityCompletionTask).\n\n_If_ this becomes a bottleneck we could look at increasing the number of\nDurabilityCompletionTask - e.g. sharding all vBuckets across multiple\ntasks like flusher / bgfetcher.\n\nChange-Id: I33ecfa78b03b4d2120b5d05f54984b24ce038fd8\n"},"parents_data":[{"branch_name":"refs/heads/master","commit_id":"3ad8757a1838d29b1a2da35dc4c5e7ec0b22e4c8","is_merged_in_target_branch":true,"change_id":"I48a671f66b87887f4d2e244b27990eac0ed83b98","change_number":113701,"patch_set_number":5,"change_status":"MERGED"}],"branch":"refs/heads/master"},"f137552b4fcee90864e74aa2fd414d7051abb820":{"kind":"REWORK","_number":3,"created":"2019-08-23 14:48:36.000000000","uploader":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"ref":"refs/changes/49/113749/3","fetch":{"anonymous http":{"url":"https://review.couchbase.org/kv_engine","ref":"refs/changes/49/113749/3","commands":{"Branch":"git fetch https://review.couchbase.org/kv_engine refs/changes/49/113749/3 \u0026\u0026 git checkout -b change-113749 FETCH_HEAD","Checkout":"git fetch https://review.couchbase.org/kv_engine refs/changes/49/113749/3 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.couchbase.org/kv_engine refs/changes/49/113749/3 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.couchbase.org/kv_engine refs/changes/49/113749/3 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.couchbase.org/kv_engine refs/changes/49/113749/3","Reset To":"git fetch https://review.couchbase.org/kv_engine refs/changes/49/113749/3 \u0026\u0026 git reset --hard FETCH_HEAD"}}},"commit":{"parents":[{"commit":"3ad8757a1838d29b1a2da35dc4c5e7ec0b22e4c8","subject":"Don\u0027t accept cipherlist with no usable ciphers"}],"author":{"name":"Dave Rigby","email":"daver@couchbase.com","date":"2019-08-09 11:55:21.000000000","tz":60},"committer":{"name":"Dave Rigby","email":"daver@couchbase.com","date":"2019-08-23 14:48:33.000000000","tz":0},"subject":"MB-35458 [SR]: Move SyncWrite completion to bg DurabilityCompletionTask","message":"MB-35458 [SR]: Move SyncWrite completion to bg DurabilityCompletionTask\n\n[[Re-apply after fixing error in DurabilityCompletionTask::run\n   (skipping last vBucket).]]\n\nChange how SyncWrites which are Resolved and awaiting Completion are\nhandled, by moving the final VBucket::commit() / abort() into a\nbackground task - DurabilityCompletionTask.\n\n+Background+\n\nThere are two reasons for making this change:\n\na) Performance - specifically latency of front-end worker threads.\n\nBy moving completion into a background task, we reduce the amount of\nwork done on the thread which actually detected the SyncWrite was\nresolved - typically the front-end DCP threads when a DCP_SEQNO_ACK\nis processed.\nGiven that we SEQNO_ACK at the end of Snapshot, A single SEQNO_ACK\ncould result in committing multiple SyncWrites. Committing one\nSyncWrite is similar to a normal front-end Set operation, so there is\npotentially a non-trivial amount of work needed to be done when\ncompleting SyncWrites, which could tie up the front-end thread (causing\nother Connections to have to wait) for a noticable amount of time.\n\nb) Simplification of lock management.\n\nDoing completion in a background task simplifies lock management, for\nexample we avoid lock inversions with earlier locks acquired during\ndcpSeqnoAck when attemping to later call notifySeqnoAvailable when this\nwas done on the original thread.\n\n+Problem+\n\nWhile (a) was the first reason identified for making this change\n(see MB-33092), (b) is the reason this change is being made now. During\ntesting the following lock-order-inversion was seen:\n\n    WARNING: ThreadSanitizer: lock-order-inversion (potential deadlock)\n    Cycle in lock order graph:\n\n    Stream::streamMutex \u003d\u003e StreamContainer::rwlock \u003d\u003e Stream::streamMutex\n\nThe crux of the issue is the processing of DCP_SEQNO_ACKNOWLEDGED\nmessages by the DcpProducer - this acquires the Stream::streamMutex\nbefore calling VBucket::seqnoAcknowledged(), however that function\ncurrently results in VBucket::commit() being called to synchronously\ncomplete the SyncWrite; which in turn must nodify all connected\nreplica that a new seqno is available, requiring\nStreamContainer::rwlock to be acquired:\n\n  Mutex StreamContainer::rwlock acquired here while holding mutex Stream::streamMutex in thread T15:\n    ...\n    #6 StreamContainer\u003cstd::shared_ptr\u003cStream\u003e \u003e::rlock()\n    #7 DcpProducer::notifySeqnoAvailable(Vbid, unsigned long)\n    ...\n    #13 VBucket::commit(...)\n    #14 ActiveDurabilityMonitor::commit(...)\n    #15 ActiveDurabilityMonitor::processCompletedSyncWriteQueue()\n    #16 ActiveDurabilityMonitor::seqnoAckReceived(...)\n    #17 VBucket::seqnoAcknowledged(...)\n    #18 ActiveStream::seqnoAck(...)\n    #19 DcpProducer::seqno_acknowledged(...)\n    ...\n\n  Mutex Stream::streamMutex previously acquired by the same thread here:\n    ...\n    #3 std::lock_guard\u003cstd::mutex\u003e::lock_guard(std::mutex\u0026)\n    #4 ActiveStream::seqnoAck(...)\n    #5 DcpProducer::seqno_acknowledged(...)\n    ...\n\nThis conflicts with the ordering seen when sending items out on the\nDCP connection - inside DcpProducer::step() where the\nStreamContainer::rwlock is acquired first, then ActiveStream::mutex\nacquired later:\n\n  Mutex Stream::streamMutex acquired here while holding mutex StreamContainer::rwlock in thread T15:\n    ...\n    #3 std::lock_guard\u003cstd::mutex\u003e::lock_guard(std::mutex\u0026)\n    #4 ActiveStream::next()\n    #5 DcpProducer::getNextItem()\n    #6 DcpProducer::step(dcp_message_producers*)\n    ...\n\n  Mutex StreamContainer::rwlock previously acquired by the same thread here:\n    #0 pthread_rwlock_rdlock \u003cnull\u003e (libtsan.so.0+0x00000002c98b)\n    ...\n    #4 std::shared_lock\u003ccb::RWLock\u003e::shared_lock(cb::RWLock\u0026)\n    #5 StreamContainer\u003c\u003e::ResumableIterationHandle::ResumableIterationHandle()\n    #6 StreamContainer\u003c\u003e::startResumable()\n    #7 DcpProducer::getNextItem()\n    #8 DcpProducer::step(dcp_message_producers*)\n    ...\n\n+Solution+\n\nThe processing of resolved SyncWrites moved into a new background task.\nInstead of immediately processing them within\nActiveDM::seqnoAckReceived(), that function notifies the new NonIO\nDurabilityCompletionTask that there are SyncWrites waiting for\ncompletion.\n\nDurabilityCompletionTask maintains a bool per vBucket indicating if\nthere are SyncWrites for that vBucket pending completion. When the\ntask is run, for each flag which is true it calls\nVBucket::processResolvedSyncWrites() for the associated VBucket.\n\n+Implementaiton Notes+\n\nCurrently there is just a single DurabilityCompletionTask (per Bucket),\nthis was chosen as 1 task per vBucket (i.e. 1024 per Bucket) would\nbe inefficient for our current background task scheduler (both in terms\nof latency to schedule each task for only one vBucket\u0027s worth of work,\nand in terms of managing that many tasks in the future queue).\n\nHowever, that does _potentially_ mean there\u0027s fewer resources (threads)\navailable to complete SyncWrites on - previously that work could be\ndone concurrently on all frontend threads (~O(num_cpus). Now the same\nwork only has 1 thread available to run on (there\u0027s only a single\nDurabilityCompletionTask).\n\n_If_ this becomes a bottleneck we could look at increasing the number of\nDurabilityCompletionTask - e.g. sharding all vBuckets across multiple\ntasks like flusher / bgfetcher.\n\nChange-Id: I33ecfa78b03b4d2120b5d05f54984b24ce038fd8\n"},"parents_data":[{"branch_name":"refs/heads/master","commit_id":"3ad8757a1838d29b1a2da35dc4c5e7ec0b22e4c8","is_merged_in_target_branch":true,"change_id":"I48a671f66b87887f4d2e244b27990eac0ed83b98","change_number":113701,"patch_set_number":5,"change_status":"MERGED"}],"branch":"refs/heads/master"},"392f47e6e14d156c5d4ff41aa02a25c68373e24b":{"kind":"REWORK","_number":4,"created":"2019-08-23 18:57:20.000000000","uploader":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"ref":"refs/changes/49/113749/4","fetch":{"anonymous http":{"url":"https://review.couchbase.org/kv_engine","ref":"refs/changes/49/113749/4","commands":{"Branch":"git fetch https://review.couchbase.org/kv_engine refs/changes/49/113749/4 \u0026\u0026 git checkout -b change-113749 FETCH_HEAD","Checkout":"git fetch https://review.couchbase.org/kv_engine refs/changes/49/113749/4 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.couchbase.org/kv_engine refs/changes/49/113749/4 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.couchbase.org/kv_engine refs/changes/49/113749/4 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.couchbase.org/kv_engine refs/changes/49/113749/4","Reset To":"git fetch https://review.couchbase.org/kv_engine refs/changes/49/113749/4 \u0026\u0026 git reset --hard FETCH_HEAD"}}},"commit":{"parents":[{"commit":"3ad8757a1838d29b1a2da35dc4c5e7ec0b22e4c8","subject":"Don\u0027t accept cipherlist with no usable ciphers"}],"author":{"name":"Dave Rigby","email":"daver@couchbase.com","date":"2019-08-09 11:55:21.000000000","tz":60},"committer":{"name":"Dave Rigby","email":"daver@couchbase.com","date":"2019-08-23 18:57:15.000000000","tz":0},"subject":"MB-35458 [SR]: Move SyncWrite completion to bg DurabilityCompletionTask","message":"MB-35458 [SR]: Move SyncWrite completion to bg DurabilityCompletionTask\n\n[[Re-apply after fixing error in DurabilityCompletionTask::run\n   (skipping last vBucket).]]\n\nChange how SyncWrites which are Resolved and awaiting Completion are\nhandled, by moving the final VBucket::commit() / abort() into a\nbackground task - DurabilityCompletionTask.\n\n+Background+\n\nThere are two reasons for making this change:\n\na) Performance - specifically latency of front-end worker threads.\n\nBy moving completion into a background task, we reduce the amount of\nwork done on the thread which actually detected the SyncWrite was\nresolved - typically the front-end DCP threads when a DCP_SEQNO_ACK\nis processed.\nGiven that we SEQNO_ACK at the end of Snapshot, A single SEQNO_ACK\ncould result in committing multiple SyncWrites. Committing one\nSyncWrite is similar to a normal front-end Set operation, so there is\npotentially a non-trivial amount of work needed to be done when\ncompleting SyncWrites, which could tie up the front-end thread (causing\nother Connections to have to wait) for a noticable amount of time.\n\nb) Simplification of lock management.\n\nDoing completion in a background task simplifies lock management, for\nexample we avoid lock inversions with earlier locks acquired during\ndcpSeqnoAck when attemping to later call notifySeqnoAvailable when this\nwas done on the original thread.\n\n+Problem+\n\nWhile (a) was the first reason identified for making this change\n(see MB-33092), (b) is the reason this change is being made now. During\ntesting the following lock-order-inversion was seen:\n\n    WARNING: ThreadSanitizer: lock-order-inversion (potential deadlock)\n    Cycle in lock order graph:\n\n    Stream::streamMutex \u003d\u003e StreamContainer::rwlock \u003d\u003e Stream::streamMutex\n\nThe crux of the issue is the processing of DCP_SEQNO_ACKNOWLEDGED\nmessages by the DcpProducer - this acquires the Stream::streamMutex\nbefore calling VBucket::seqnoAcknowledged(), however that function\ncurrently results in VBucket::commit() being called to synchronously\ncomplete the SyncWrite; which in turn must nodify all connected\nreplica that a new seqno is available, requiring\nStreamContainer::rwlock to be acquired:\n\n  Mutex StreamContainer::rwlock acquired here while holding mutex Stream::streamMutex in thread T15:\n    ...\n    #6 StreamContainer\u003cstd::shared_ptr\u003cStream\u003e \u003e::rlock()\n    #7 DcpProducer::notifySeqnoAvailable(Vbid, unsigned long)\n    ...\n    #13 VBucket::commit(...)\n    #14 ActiveDurabilityMonitor::commit(...)\n    #15 ActiveDurabilityMonitor::processCompletedSyncWriteQueue()\n    #16 ActiveDurabilityMonitor::seqnoAckReceived(...)\n    #17 VBucket::seqnoAcknowledged(...)\n    #18 ActiveStream::seqnoAck(...)\n    #19 DcpProducer::seqno_acknowledged(...)\n    ...\n\n  Mutex Stream::streamMutex previously acquired by the same thread here:\n    ...\n    #3 std::lock_guard\u003cstd::mutex\u003e::lock_guard(std::mutex\u0026)\n    #4 ActiveStream::seqnoAck(...)\n    #5 DcpProducer::seqno_acknowledged(...)\n    ...\n\nThis conflicts with the ordering seen when sending items out on the\nDCP connection - inside DcpProducer::step() where the\nStreamContainer::rwlock is acquired first, then ActiveStream::mutex\nacquired later:\n\n  Mutex Stream::streamMutex acquired here while holding mutex StreamContainer::rwlock in thread T15:\n    ...\n    #3 std::lock_guard\u003cstd::mutex\u003e::lock_guard(std::mutex\u0026)\n    #4 ActiveStream::next()\n    #5 DcpProducer::getNextItem()\n    #6 DcpProducer::step(dcp_message_producers*)\n    ...\n\n  Mutex StreamContainer::rwlock previously acquired by the same thread here:\n    #0 pthread_rwlock_rdlock \u003cnull\u003e (libtsan.so.0+0x00000002c98b)\n    ...\n    #4 std::shared_lock\u003ccb::RWLock\u003e::shared_lock(cb::RWLock\u0026)\n    #5 StreamContainer\u003c\u003e::ResumableIterationHandle::ResumableIterationHandle()\n    #6 StreamContainer\u003c\u003e::startResumable()\n    #7 DcpProducer::getNextItem()\n    #8 DcpProducer::step(dcp_message_producers*)\n    ...\n\n+Solution+\n\nThe processing of resolved SyncWrites moved into a new background task.\nInstead of immediately processing them within\nActiveDM::seqnoAckReceived(), that function notifies the new NonIO\nDurabilityCompletionTask that there are SyncWrites waiting for\ncompletion.\n\nDurabilityCompletionTask maintains a bool per vBucket indicating if\nthere are SyncWrites for that vBucket pending completion. When the\ntask is run, for each flag which is true it calls\nVBucket::processResolvedSyncWrites() for the associated VBucket.\n\n+Implementation Notes+\n\nCurrently there is just a single DurabilityCompletionTask (per Bucket),\nthis was chosen as 1 task per vBucket (i.e. 1024 per Bucket) would\nbe inefficient for our current background task scheduler (both in terms\nof latency to schedule each task for only one vBucket\u0027s worth of work,\nand in terms of managing that many tasks in the future queue).\n\nHowever, that does _potentially_ mean there\u0027s fewer resources (threads)\navailable to complete SyncWrites on - previously that work could be\ndone concurrently on all frontend threads (~O(num_cpus). Now the same\nwork only has 1 thread available to run on (there\u0027s only a single\nDurabilityCompletionTask).\n\n_If_ this becomes a bottleneck we could look at increasing the number of\nDurabilityCompletionTask - e.g. sharding all vBuckets across multiple\ntasks like flusher / bgfetcher.\n\nChange-Id: I33ecfa78b03b4d2120b5d05f54984b24ce038fd8\n"},"parents_data":[{"branch_name":"refs/heads/master","commit_id":"3ad8757a1838d29b1a2da35dc4c5e7ec0b22e4c8","is_merged_in_target_branch":true,"change_id":"I48a671f66b87887f4d2e244b27990eac0ed83b98","change_number":113701,"patch_set_number":5,"change_status":"MERGED"}],"branch":"refs/heads/master"},"f6ed489642e80594ff6a9a87a12f3d9c343fccfc":{"kind":"REWORK","_number":5,"created":"2019-08-27 09:14:19.000000000","uploader":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"ref":"refs/changes/49/113749/5","fetch":{"anonymous http":{"url":"https://review.couchbase.org/kv_engine","ref":"refs/changes/49/113749/5","commands":{"Branch":"git fetch https://review.couchbase.org/kv_engine refs/changes/49/113749/5 \u0026\u0026 git checkout -b change-113749 FETCH_HEAD","Checkout":"git fetch https://review.couchbase.org/kv_engine refs/changes/49/113749/5 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.couchbase.org/kv_engine refs/changes/49/113749/5 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.couchbase.org/kv_engine refs/changes/49/113749/5 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.couchbase.org/kv_engine refs/changes/49/113749/5","Reset To":"git fetch https://review.couchbase.org/kv_engine refs/changes/49/113749/5 \u0026\u0026 git reset --hard FETCH_HEAD"}}},"commit":{"parents":[{"commit":"dd410e5d0db5bf7627e2378e2fc090b8c0ed156d","subject":"MB-35631: Add HCS to \"Sending disk snapshot\" log"}],"author":{"name":"Dave Rigby","email":"daver@couchbase.com","date":"2019-08-09 11:55:21.000000000","tz":60},"committer":{"name":"Dave Rigby","email":"daver@couchbase.com","date":"2019-08-27 09:14:19.000000000","tz":0},"subject":"MB-35458 [SR]: Move SyncWrite completion to bg DurabilityCompletionTask","message":"MB-35458 [SR]: Move SyncWrite completion to bg DurabilityCompletionTask\n\n[[Re-apply after fixing error in DurabilityCompletionTask::run\n   (skipping last vBucket).]]\n\nChange how SyncWrites which are Resolved and awaiting Completion are\nhandled, by moving the final VBucket::commit() / abort() into a\nbackground task - DurabilityCompletionTask.\n\n+Background+\n\nThere are two reasons for making this change:\n\na) Performance - specifically latency of front-end worker threads.\n\nBy moving completion into a background task, we reduce the amount of\nwork done on the thread which actually detected the SyncWrite was\nresolved - typically the front-end DCP threads when a DCP_SEQNO_ACK\nis processed.\nGiven that we SEQNO_ACK at the end of Snapshot, A single SEQNO_ACK\ncould result in committing multiple SyncWrites. Committing one\nSyncWrite is similar to a normal front-end Set operation, so there is\npotentially a non-trivial amount of work needed to be done when\ncompleting SyncWrites, which could tie up the front-end thread (causing\nother Connections to have to wait) for a noticable amount of time.\n\nb) Simplification of lock management.\n\nDoing completion in a background task simplifies lock management, for\nexample we avoid lock inversions with earlier locks acquired during\ndcpSeqnoAck when attemping to later call notifySeqnoAvailable when this\nwas done on the original thread.\n\n+Problem+\n\nWhile (a) was the first reason identified for making this change\n(see MB-33092), (b) is the reason this change is being made now. During\ntesting the following lock-order-inversion was seen:\n\n    WARNING: ThreadSanitizer: lock-order-inversion (potential deadlock)\n    Cycle in lock order graph:\n\n    Stream::streamMutex \u003d\u003e StreamContainer::rwlock \u003d\u003e Stream::streamMutex\n\nThe crux of the issue is the processing of DCP_SEQNO_ACKNOWLEDGED\nmessages by the DcpProducer - this acquires the Stream::streamMutex\nbefore calling VBucket::seqnoAcknowledged(), however that function\ncurrently results in VBucket::commit() being called to synchronously\ncomplete the SyncWrite; which in turn must nodify all connected\nreplica that a new seqno is available, requiring\nStreamContainer::rwlock to be acquired:\n\n  Mutex StreamContainer::rwlock acquired here while holding mutex Stream::streamMutex in thread T15:\n    ...\n    #6 StreamContainer\u003cstd::shared_ptr\u003cStream\u003e \u003e::rlock()\n    #7 DcpProducer::notifySeqnoAvailable(Vbid, unsigned long)\n    ...\n    #13 VBucket::commit(...)\n    #14 ActiveDurabilityMonitor::commit(...)\n    #15 ActiveDurabilityMonitor::processCompletedSyncWriteQueue()\n    #16 ActiveDurabilityMonitor::seqnoAckReceived(...)\n    #17 VBucket::seqnoAcknowledged(...)\n    #18 ActiveStream::seqnoAck(...)\n    #19 DcpProducer::seqno_acknowledged(...)\n    ...\n\n  Mutex Stream::streamMutex previously acquired by the same thread here:\n    ...\n    #3 std::lock_guard\u003cstd::mutex\u003e::lock_guard(std::mutex\u0026)\n    #4 ActiveStream::seqnoAck(...)\n    #5 DcpProducer::seqno_acknowledged(...)\n    ...\n\nThis conflicts with the ordering seen when sending items out on the\nDCP connection - inside DcpProducer::step() where the\nStreamContainer::rwlock is acquired first, then ActiveStream::mutex\nacquired later:\n\n  Mutex Stream::streamMutex acquired here while holding mutex StreamContainer::rwlock in thread T15:\n    ...\n    #3 std::lock_guard\u003cstd::mutex\u003e::lock_guard(std::mutex\u0026)\n    #4 ActiveStream::next()\n    #5 DcpProducer::getNextItem()\n    #6 DcpProducer::step(dcp_message_producers*)\n    ...\n\n  Mutex StreamContainer::rwlock previously acquired by the same thread here:\n    #0 pthread_rwlock_rdlock \u003cnull\u003e (libtsan.so.0+0x00000002c98b)\n    ...\n    #4 std::shared_lock\u003ccb::RWLock\u003e::shared_lock(cb::RWLock\u0026)\n    #5 StreamContainer\u003c\u003e::ResumableIterationHandle::ResumableIterationHandle()\n    #6 StreamContainer\u003c\u003e::startResumable()\n    #7 DcpProducer::getNextItem()\n    #8 DcpProducer::step(dcp_message_producers*)\n    ...\n\n+Solution+\n\nThe processing of resolved SyncWrites moved into a new background task.\nInstead of immediately processing them within\nActiveDM::seqnoAckReceived(), that function notifies the new NonIO\nDurabilityCompletionTask that there are SyncWrites waiting for\ncompletion.\n\nDurabilityCompletionTask maintains a bool per vBucket indicating if\nthere are SyncWrites for that vBucket pending completion. When the\ntask is run, for each flag which is true it calls\nVBucket::processResolvedSyncWrites() for the associated VBucket.\n\n+Implementation Notes+\n\nCurrently there is just a single DurabilityCompletionTask (per Bucket),\nthis was chosen as 1 task per vBucket (i.e. 1024 per Bucket) would\nbe inefficient for our current background task scheduler (both in terms\nof latency to schedule each task for only one vBucket\u0027s worth of work,\nand in terms of managing that many tasks in the future queue).\n\nHowever, that does _potentially_ mean there\u0027s fewer resources (threads)\navailable to complete SyncWrites on - previously that work could be\ndone concurrently on all frontend threads (~O(num_cpus). Now the same\nwork only has 1 thread available to run on (there\u0027s only a single\nDurabilityCompletionTask).\n\n_If_ this becomes a bottleneck we could look at increasing the number of\nDurabilityCompletionTask - e.g. sharding all vBuckets across multiple\ntasks like flusher / bgfetcher.\n\nChange-Id: I33ecfa78b03b4d2120b5d05f54984b24ce038fd8\nReviewed-on: http://review.couchbase.org/113749\nReviewed-by: Ben Huddleston \u003cben.huddleston@couchbase.com\u003e\nTested-by: Build Bot \u003cbuild@couchbase.com\u003e\n"},"parents_data":[{"branch_name":"refs/heads/master","commit_id":"dd410e5d0db5bf7627e2378e2fc090b8c0ed156d","is_merged_in_target_branch":true,"change_id":"I7a0c090029c01861319aa50bba35af21eb42986e","change_number":113755,"patch_set_number":2,"change_status":"MERGED"}],"branch":"refs/heads/master"}},"requirements":[],"submit_records":[],"submit_requirements":[]}