)]}'
{"id":"kv_engine~138373","triplet_id":"kv_engine~master~Iecbff9f3b45fc9e3d385c67f6a6dd32242dc76fe","project":"kv_engine","branch":"master","hashtags":[],"change_id":"Iecbff9f3b45fc9e3d385c67f6a6dd32242dc76fe","subject":"MB-42029: FollyExecPool: Wait for tasks cancelled in unregisterTaskable","status":"MERGED","created":"2020-10-18 20:26:11.000000000","updated":"2020-10-23 11:01:31.000000000","submitted":"2020-10-23 10:00:50.000000000","submitter":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"total_comment_count":0,"unresolved_comment_count":0,"has_review_started":true,"submission_id":"138373-1603447250996-884e4f1c","meta_rev_id":"5497c344493357348ecde53de2c2a674b7426b62","_number":138373,"virtual_id_number":138373,"owner":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"actions":{},"labels":{"Verified":{"approved":{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"all":[{"value":0,"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"tag":"autogenerated:gerrit:merged","value":1,"date":"2020-10-23 10:00:50.000000000","permitted_voting_range":{"min":1,"max":1},"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},{"value":0,"_account_id":1001546,"name":"Jim Walker","email":"jim@couchbase.com","username":"jim","avatars":[{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]}],"values":{"-1":"Fails"," 0":"No score","+1":"Verified"},"description":"","default_value":0},"Code-Review":{"approved":{"_account_id":1001546,"name":"Jim Walker","email":"jim@couchbase.com","username":"jim","avatars":[{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"all":[{"value":0,"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"value":0,"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},{"tag":"autogenerated:gerrit:merged","value":2,"date":"2020-10-23 10:00:50.000000000","permitted_voting_range":{"min":2,"max":2},"_account_id":1001546,"name":"Jim Walker","email":"jim@couchbase.com","username":"jim","avatars":[{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]}],"values":{"-2":"Do not submit","-1":"I would prefer that you didn\u0027t submit this"," 0":"No score","+1":"Looks good to me, but someone else must approve","+2":"Looks good to me, approved"},"description":"","default_value":0},"Well-Formed":{"all":[{"value":0,"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"value":0,"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},{"value":0,"_account_id":1001546,"name":"Jim Walker","email":"jim@couchbase.com","username":"jim","avatars":[{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]}],"values":{"-1":"Commit is not well-formed"," 0":"Well-formedness not checked","+1":"Commit is well-formed"},"description":"","default_value":0,"optional":true}},"removable_reviewers":[{"_account_id":1000002,"name":"Trond Norbye","display_name":"Trond","email":"trond.norbye@couchbase.com","username":"trond","avatars":[{"url":"https://www.gravatar.com/avatar/1521a1fa74885ab7b9748113afe689cc.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/1521a1fa74885ab7b9748113afe689cc.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/1521a1fa74885ab7b9748113afe689cc.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/1521a1fa74885ab7b9748113afe689cc.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"_account_id":1004368,"name":"Richard de Mellow","email":"richard.demellow@couchbase.com","username":"rdemellow","avatars":[{"url":"https://www.gravatar.com/avatar/35c97da872c07458f5e5bc4938b2489f.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/35c97da872c07458f5e5bc4938b2489f.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/35c97da872c07458f5e5bc4938b2489f.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/35c97da872c07458f5e5bc4938b2489f.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"_account_id":1004341,"name":"Ben Huddleston","email":"ben.huddleston@couchbase.com","username":"BenHuddleston","avatars":[{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"_account_id":1003828,"name":"James H","email":"james.harrison@couchbase.com","username":"jameseh96","avatars":[{"url":"https://www.gravatar.com/avatar/70d2875f461e17e71a2a16972681c794.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/70d2875f461e17e71a2a16972681c794.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/70d2875f461e17e71a2a16972681c794.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/70d2875f461e17e71a2a16972681c794.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"_account_id":1001547,"name":"Dan Owen","email":"owend@couchbase.com","username":"owend","avatars":[{"url":"https://www.gravatar.com/avatar/0761a726bc37606caee7ec6bf566deca.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0761a726bc37606caee7ec6bf566deca.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0761a726bc37606caee7ec6bf566deca.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0761a726bc37606caee7ec6bf566deca.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"_account_id":1004059,"name":"Paolo Cocchi","email":"paolo.cocchi@couchbase.com","username":"paolococchi","avatars":[{"url":"https://www.gravatar.com/avatar/3367cfbc4fbf29e6d6b3a5397e41849d.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/3367cfbc4fbf29e6d6b3a5397e41849d.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/3367cfbc4fbf29e6d6b3a5397e41849d.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/3367cfbc4fbf29e6d6b3a5397e41849d.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"_account_id":1002319,"name":"Dave Finlay","email":"dave.finlay@couchbase.com","username":"dave-finlay","avatars":[{"url":"https://www.gravatar.com/avatar/aa50dbeeeae1f0a9a777fcf4b80c4fb9.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/aa50dbeeeae1f0a9a777fcf4b80c4fb9.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/aa50dbeeeae1f0a9a777fcf4b80c4fb9.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/aa50dbeeeae1f0a9a777fcf4b80c4fb9.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]}],"reviewers":{"REVIEWER":[{"_account_id":1000002,"name":"Trond Norbye","display_name":"Trond","email":"trond.norbye@couchbase.com","username":"trond","avatars":[{"url":"https://www.gravatar.com/avatar/1521a1fa74885ab7b9748113afe689cc.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/1521a1fa74885ab7b9748113afe689cc.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/1521a1fa74885ab7b9748113afe689cc.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/1521a1fa74885ab7b9748113afe689cc.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"_account_id":1001546,"name":"Jim Walker","email":"jim@couchbase.com","username":"jim","avatars":[{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"_account_id":1001547,"name":"Dan Owen","email":"owend@couchbase.com","username":"owend","avatars":[{"url":"https://www.gravatar.com/avatar/0761a726bc37606caee7ec6bf566deca.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0761a726bc37606caee7ec6bf566deca.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0761a726bc37606caee7ec6bf566deca.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0761a726bc37606caee7ec6bf566deca.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"_account_id":1002319,"name":"Dave Finlay","email":"dave.finlay@couchbase.com","username":"dave-finlay","avatars":[{"url":"https://www.gravatar.com/avatar/aa50dbeeeae1f0a9a777fcf4b80c4fb9.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/aa50dbeeeae1f0a9a777fcf4b80c4fb9.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/aa50dbeeeae1f0a9a777fcf4b80c4fb9.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/aa50dbeeeae1f0a9a777fcf4b80c4fb9.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"_account_id":1003828,"name":"James H","email":"james.harrison@couchbase.com","username":"jameseh96","avatars":[{"url":"https://www.gravatar.com/avatar/70d2875f461e17e71a2a16972681c794.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/70d2875f461e17e71a2a16972681c794.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/70d2875f461e17e71a2a16972681c794.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/70d2875f461e17e71a2a16972681c794.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"_account_id":1004059,"name":"Paolo Cocchi","email":"paolo.cocchi@couchbase.com","username":"paolococchi","avatars":[{"url":"https://www.gravatar.com/avatar/3367cfbc4fbf29e6d6b3a5397e41849d.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/3367cfbc4fbf29e6d6b3a5397e41849d.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/3367cfbc4fbf29e6d6b3a5397e41849d.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/3367cfbc4fbf29e6d6b3a5397e41849d.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"_account_id":1004341,"name":"Ben Huddleston","email":"ben.huddleston@couchbase.com","username":"BenHuddleston","avatars":[{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},{"_account_id":1004368,"name":"Richard de Mellow","email":"richard.demellow@couchbase.com","username":"rdemellow","avatars":[{"url":"https://www.gravatar.com/avatar/35c97da872c07458f5e5bc4938b2489f.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/35c97da872c07458f5e5bc4938b2489f.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/35c97da872c07458f5e5bc4938b2489f.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/35c97da872c07458f5e5bc4938b2489f.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]}]},"pending_reviewers":{},"reviewer_updates":[{"updated":"2020-10-18 21:07:21.000000000","updated_by":{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"reviewer":{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"state":"REVIEWER"},{"updated":"2020-10-21 11:11:25.000000000","updated_by":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"reviewer":{"_account_id":1002319,"name":"Dave Finlay","email":"dave.finlay@couchbase.com","username":"dave-finlay","avatars":[{"url":"https://www.gravatar.com/avatar/aa50dbeeeae1f0a9a777fcf4b80c4fb9.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/aa50dbeeeae1f0a9a777fcf4b80c4fb9.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/aa50dbeeeae1f0a9a777fcf4b80c4fb9.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/aa50dbeeeae1f0a9a777fcf4b80c4fb9.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"state":"REVIEWER"},{"updated":"2020-10-21 11:11:25.000000000","updated_by":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"reviewer":{"_account_id":1001546,"name":"Jim Walker","email":"jim@couchbase.com","username":"jim","avatars":[{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"state":"REVIEWER"},{"updated":"2020-10-21 11:11:25.000000000","updated_by":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"reviewer":{"_account_id":1004059,"name":"Paolo Cocchi","email":"paolo.cocchi@couchbase.com","username":"paolococchi","avatars":[{"url":"https://www.gravatar.com/avatar/3367cfbc4fbf29e6d6b3a5397e41849d.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/3367cfbc4fbf29e6d6b3a5397e41849d.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/3367cfbc4fbf29e6d6b3a5397e41849d.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/3367cfbc4fbf29e6d6b3a5397e41849d.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"state":"REVIEWER"},{"updated":"2020-10-21 11:11:25.000000000","updated_by":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"reviewer":{"_account_id":1001547,"name":"Dan Owen","email":"owend@couchbase.com","username":"owend","avatars":[{"url":"https://www.gravatar.com/avatar/0761a726bc37606caee7ec6bf566deca.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0761a726bc37606caee7ec6bf566deca.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0761a726bc37606caee7ec6bf566deca.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0761a726bc37606caee7ec6bf566deca.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"state":"REVIEWER"},{"updated":"2020-10-21 11:11:25.000000000","updated_by":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"reviewer":{"_account_id":1003828,"name":"James H","email":"james.harrison@couchbase.com","username":"jameseh96","avatars":[{"url":"https://www.gravatar.com/avatar/70d2875f461e17e71a2a16972681c794.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/70d2875f461e17e71a2a16972681c794.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/70d2875f461e17e71a2a16972681c794.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/70d2875f461e17e71a2a16972681c794.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"state":"REVIEWER"},{"updated":"2020-10-21 11:11:25.000000000","updated_by":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"reviewer":{"_account_id":1004341,"name":"Ben Huddleston","email":"ben.huddleston@couchbase.com","username":"BenHuddleston","avatars":[{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/ccb3e7e3b52c3d14e759d1b4c0507a3b.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"state":"REVIEWER"},{"updated":"2020-10-21 11:11:25.000000000","updated_by":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"reviewer":{"_account_id":1004368,"name":"Richard de Mellow","email":"richard.demellow@couchbase.com","username":"rdemellow","avatars":[{"url":"https://www.gravatar.com/avatar/35c97da872c07458f5e5bc4938b2489f.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/35c97da872c07458f5e5bc4938b2489f.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/35c97da872c07458f5e5bc4938b2489f.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/35c97da872c07458f5e5bc4938b2489f.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"state":"REVIEWER"},{"updated":"2020-10-21 11:11:25.000000000","updated_by":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"reviewer":{"_account_id":1000002,"name":"Trond Norbye","display_name":"Trond","email":"trond.norbye@couchbase.com","username":"trond","avatars":[{"url":"https://www.gravatar.com/avatar/1521a1fa74885ab7b9748113afe689cc.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/1521a1fa74885ab7b9748113afe689cc.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/1521a1fa74885ab7b9748113afe689cc.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/1521a1fa74885ab7b9748113afe689cc.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"state":"REVIEWER"}],"messages":[{"id":"db996bd8d07eae72bf99ac252e461d3c74a264eb","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"date":"2020-10-18 20:26:11.000000000","message":"Uploaded patch set 1.","accounts_in_message":[],"_revision_number":1},{"id":"2d6492570a3bb36f2771c21d7fae41ebeb997706","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"date":"2020-10-18 21:07:21.000000000","message":"Patch Set 1: Verified-1\n\nBuild Failed \n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-clang_format_9/3650/ : FAILURE\n\nNo problems were identified. If you know why this problem occurred, please add a suitable Cause for it. ( http://cv.jenkins.couchbase.com/job/kv_engine-clang_format_9/3650/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-windows-master/31183/ : FAILURE\n\nFailure of GoogleTest \"TransportProtocols/TtlTest.Set/Mcbp\":\n\n\u003cpre\u003e\n[ RUN      ] TransportProtocols/TtlTest.Set/Mcbp\nFailed building socket: connect() failed (127.0.0.1 50332): No connection could be made because the target machine actively refused it.\n..\\kv_engine\\tests\\testapp\\testapp.cc(638): error: Failed\nFailed to connect socket to 127.0.0.1:50332\n..\\kv_engine\\tests\\testapp\\testapp.cc(206): error: Expected: ((SOCKET)(~0)) !\u003d (sock), actual: 18446744073709551615 vs 18446744073709551615\nFailed building socket: connect() failed (127.0.0.1 50332): No connection could be made because the target machine actively refused it.\nunknown file: error: C++ exception with description \"Failed to connect to: localhost:50332: No connection could be made because the target machine actively refused it.\" thrown in SetUp().\n[  FAILED  ] TransportProtocols/TtlTest.Set/Mcbp, where GetParam() \u003d Mcbp (2322 ms)\nTransportProtocols/TtlTest.Set/Mcbp\n\u003c/pre\u003e\n ( http://cv.jenkins.couchbase.com/job/kv_engine-windows-master/31183/ )\n\nTimeout of a CTest test 393/393 Test #390: cluster_test ..................................................................................................................} ( http://cv.jenkins.couchbase.com/job/kv_engine-windows-master/31183/ )\n\nFailure of a CTest test 251/393 Test #258: ep_testsuite_dcp.full_eviction.magma .......................................................................................... ( http://cv.jenkins.couchbase.com/job/kv_engine-windows-master/31183/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.macos/job/master/5514/ : FAILURE\n\nFailure of a CTest test [2020-10-18T20:36:16.240Z]   1/407 Test #259: ep_testsuite_dcp.full_eviction.comp_active .................................................................. ( http://cv.jenkins.couchbase.com/job/kv_engine.macos/job/master/5514/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.linux/job/master/6555/ : FAILURE\n\nFailure of GoogleTest \"TransportProtocols/SaslTest.UnknownUserSCRAM_SHA512/Mcbp\":\n\n\u003cpre\u003e\n[ RUN      ] TransportProtocols/SaslTest.UnknownUserSCRAM_SHA512/Mcbp\n[2020-10-18T20:51:57.634Z] ../kv_engine/tests/testapp/testapp_sasl.cc:64: Failure\n[2020-10-18T20:51:57.634Z] Expected: connection.deleteBucket(bucket3) doesn\"t throw an exception.\n[2020-10-18T20:51:57.634Z]   Actual: it throws.\n[2020-10-18T20:51:57.634Z] [  FAILED  ] TransportProtocols/SaslTest.UnknownUserSCRAM_SHA512/Mcbp, where GetParam() \u003d Mcbp (41 ms)\nTransportProtocols/SaslTest.UnknownUserSCRAM_SHA512/Mcbp\n\u003c/pre\u003e\n ( http://cv.jenkins.couchbase.com/job/kv_engine.linux/job/master/6555/ )\n\nFailure of a CTest test [2020-10-18T20:47:42.579Z]   2/410 Test #260: ep_testsuite_dcp.value_eviction.comp_active ................................................................. ( http://cv.jenkins.couchbase.com/job/kv_engine.linux/job/master/6555/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.threadsanitizer/job/master/6712/ : FAILURE\n\nThreadSanitizer issue: data race /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/follytsan-prefix/src/follytsan/folly/io/async/EventBase.cpp:664 in folly::EventBase::runInEventBaseThreadAndWait(folly::Function\u003cvoid ()\u003e)  ( http://cv.jenkins.couchbase.com/job/kv_engine.threadsanitizer/job/master/6712/ )\n\nFailure of a CTest test [2020-10-18T20:31:37.960Z]   3/388 Test #243: ep_testsuite_dcp.value_eviction.comp_passive ................................................................ ( http://cv.jenkins.couchbase.com/job/kv_engine.threadsanitizer/job/master/6712/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.ASan-UBSan/job/master/14350/ : FAILURE\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-clang_tidy/5121/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-clang_analyzer-master/26920/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.linux-CE/job/master/6437/ : SUCCESS","accounts_in_message":[],"_revision_number":1},{"id":"b3b2e1bdc43fe1d05f223cc4d9cc21a0ca8b76c4","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"date":"2020-10-19 11:42:19.000000000","message":"Uploaded patch set 2.","accounts_in_message":[],"_revision_number":2},{"id":"1fc6f1e705a133f1f3b3bae54187d636149855fd","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"date":"2020-10-19 12:49:53.000000000","message":"Patch Set 2: Verified-1\n\nBuild Failed \n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-windows-master/31194/ : FAILURE\n\nFailure of GoogleTest \"TransportProtocols/SaslTest.UnknownUserPlain/Mcbp\":\n\n\u003cpre\u003e\n[ RUN      ] TransportProtocols/SaslTest.UnknownUserPlain/Mcbp\n[       OK ] TransportProtocols/SaslTest.UnknownUserPlain/Mcbp (77 ms)\n[ RUN      ] TransportProtocols/SaslTest.UnknownUserPlain/McbpSsl\nWARNING: Logging before InitGoogleLogging() is written to STDERR\nW1019 04:58:52.778748 98144 HazptrDomain.h:671] Using the default inline executor for asynchronous reclamation may be susceptible to deadlock if the current thread happens to hold a resource needed by the deleter of a reclaimable object\n..\\kv_engine\\tests\\testapp\\testapp_sasl.cc(64): error: Expected: connection.deleteBucket(bucket3) doesn\"t throw an exception.\n  Actual: it throws.\n[  FAILED  ] TransportProtocols/SaslTest.UnknownUserPlain/McbpSsl, where GetParam() \u003d McbpSsl (66 ms)\nTransportProtocols/SaslTest.UnknownUserPlain/Mcbp\n\u003c/pre\u003e\n ( http://cv.jenkins.couchbase.com/job/kv_engine-windows-master/31194/ )\n\nTimeout of a CTest test 392/393 Test #387: memcached_testapp.ep.TransportProtocols/XattrNoDocDurabilityTest ..............................................................} ( http://cv.jenkins.couchbase.com/job/kv_engine-windows-master/31194/ )\n\nFailure of a CTest test 251/393 Test #254: ep_testsuite_dcp.full_eviction.comp_active .................................................................................... ( http://cv.jenkins.couchbase.com/job/kv_engine-windows-master/31194/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.macos/job/master/5526/ : FAILURE\n\nFailure of GoogleTest \"TransportProtocols/SaslTest.UnknownUserSCRAM_SHA1/Mcbp\":\n\n\u003cpre\u003e\n[ RUN      ] TransportProtocols/SaslTest.UnknownUserSCRAM_SHA1/Mcbp\n[2020-10-19T11:58:37.314Z] [       OK ] TransportProtocols/SaslTest.UnknownUserSCRAM_SHA1/Mcbp (23 ms)\n[2020-10-19T11:58:37.314Z] [ RUN      ] TransportProtocols/SaslTest.UnknownUserSCRAM_SHA1/McbpSsl\n[2020-10-19T11:58:37.314Z] ../kv_engine/tests/testapp/testapp_sasl.cc:64: Failure\n[2020-10-19T11:58:37.314Z] Expected: connection.deleteBucket(bucket3) doesn\"t throw an exception.\n[2020-10-19T11:58:37.314Z]   Actual: it throws.\n[2020-10-19T11:58:37.314Z] [  FAILED  ] TransportProtocols/SaslTest.UnknownUserSCRAM_SHA1/McbpSsl, where GetParam() \u003d McbpSsl (24 ms)\nTransportProtocols/SaslTest.UnknownUserSCRAM_SHA1/Mcbp\n\u003c/pre\u003e\n ( http://cv.jenkins.couchbase.com/job/kv_engine.macos/job/master/5526/ )\n\nFailure of a CTest test [2020-10-19T11:58:37.314Z]  96/407 Test #387: memcached_testapp.ep.TransportProtocols/SaslTest ............................................................ ( http://cv.jenkins.couchbase.com/job/kv_engine.macos/job/master/5526/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.linux/job/master/6568/ : FAILURE\n\nFailure of GoogleTest \"TransportProtocols/SaslTest.TestSaslMixFrom_PLAIN/Mcbp\":\n\n\u003cpre\u003e\n[ RUN      ] TransportProtocols/SaslTest.TestSaslMixFrom_PLAIN/Mcbp\n[2020-10-19T12:11:29.033Z] ../kv_engine/tests/testapp/testapp_sasl.cc:64: Failure\n[2020-10-19T12:11:29.033Z] Expected: connection.deleteBucket(bucket3) doesn\"t throw an exception.\n[2020-10-19T12:11:29.033Z]   Actual: it throws.\n[2020-10-19T12:11:29.033Z] [  FAILED  ] TransportProtocols/SaslTest.TestSaslMixFrom_PLAIN/Mcbp, where GetParam() \u003d Mcbp (66 ms)\nTransportProtocols/SaslTest.TestSaslMixFrom_PLAIN/Mcbp\n\u003c/pre\u003e\n ( http://cv.jenkins.couchbase.com/job/kv_engine.linux/job/master/6568/ )\n\nFailure of a CTest test [2020-10-19T12:08:58.038Z]  22/410 Test #266: ep_testsuite_dcp.full_eviction.rocksdb ...................................................................... ( http://cv.jenkins.couchbase.com/job/kv_engine.linux/job/master/6568/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.linux-CE/job/master/6449/ : FAILURE\n\nFailure of GoogleTest \"TransportProtocols/SaslTest.UnknownUserPlain/Mcbp\":\n\n\u003cpre\u003e\n[ RUN      ] TransportProtocols/SaslTest.UnknownUserPlain/Mcbp\n[2020-10-19T12:01:33.258Z] ../kv_engine/tests/testapp/testapp_sasl.cc:64: Failure\n[2020-10-19T12:01:33.259Z] Expected: connection.deleteBucket(bucket3) doesn\"t throw an exception.\n[2020-10-19T12:01:33.259Z]   Actual: it throws.\n[2020-10-19T12:01:33.259Z] [  FAILED  ] TransportProtocols/SaslTest.UnknownUserPlain/Mcbp, where GetParam() \u003d Mcbp (35 ms)\nTransportProtocols/SaslTest.UnknownUserPlain/Mcbp\n\u003c/pre\u003e\n ( http://cv.jenkins.couchbase.com/job/kv_engine.linux-CE/job/master/6449/ )\n\nFailure of a CTest test [2020-10-19T11:58:21.301Z]   1/397 Test #254: ep_testsuite_dcp.full_eviction.comp_passive ................................................................. ( http://cv.jenkins.couchbase.com/job/kv_engine.linux-CE/job/master/6449/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.ASan-UBSan/job/master/14362/ : FAILURE\n\nUndefined Sanitizer: conf...../kv_engine/engines/ep/src/vb_count_visitor.h:97:42: runtime error: 1.84467e+21 is outside the range of representable values of type \"unsigned long\" ( http://cv.jenkins.couchbase.com/job/kv_engine.ASan-UBSan/job/master/14362/ )\n\nFailure of a CTest test [2020-10-19T11:50:42.619Z] 230/393 Test #230: ep_testsuite.value_eviction.comp_passive ....................................................................Child aborted ( http://cv.jenkins.couchbase.com/job/kv_engine.ASan-UBSan/job/master/14362/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.threadsanitizer/job/master/6724/ : FAILURE\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-clang_format_9/3662/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-clang_tidy/5133/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-clang_analyzer-master/26931/ : SUCCESS","accounts_in_message":[],"_revision_number":2},{"id":"b2e887fb657f34bcae0aab75fa79b5053b3b8e4f","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"date":"2020-10-19 13:13:45.000000000","message":"Uploaded patch set 3.","accounts_in_message":[],"_revision_number":3},{"id":"9b699ae91a82e316204855055ce12a659f96422f","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"date":"2020-10-19 13:30:47.000000000","message":"Uploaded patch set 4.","accounts_in_message":[],"_revision_number":4},{"id":"127c27798268cf0395ffcb931a6cd6b7416ed098","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"date":"2020-10-19 13:30:54.000000000","message":"Patch Set 3: Verified-1\n\nBuild Failed \n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-windows-master/31202/ : ABORTED\n\nNo problems were identified. If you know why this problem occurred, please add a suitable Cause for it. ( http://cv.jenkins.couchbase.com/job/kv_engine-windows-master/31202/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.ASan-UBSan/job/master/14377/ : ABORTED\n\nBuild which was aborted due to a newer patch set being uploaded for the given review. ( http://cv.jenkins.couchbase.com/job/kv_engine.ASan-UBSan/job/master/14377/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.threadsanitizer/job/master/6734/ : ABORTED\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.linux-CE/job/master/6459/ : ABORTED\n\nBuild which was aborted due to a newer patch set being uploaded for the given review. ( http://cv.jenkins.couchbase.com/job/kv_engine.linux-CE/job/master/6459/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.linux/job/master/6578/ : ABORTED\n\nBuild which was aborted due to a newer patch set being uploaded for the given review. ( http://cv.jenkins.couchbase.com/job/kv_engine.linux/job/master/6578/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.macos/job/master/5535/ : SUCCESS","accounts_in_message":[],"_revision_number":3},{"id":"81b96968c75d32fef8fec223cc4c4c3e4d2af41f","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"date":"2020-10-19 14:03:02.000000000","message":"Uploaded patch set 5.","accounts_in_message":[],"_revision_number":5},{"id":"74b441eea9cab879f7f321f52e490b51a19cb60b","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"date":"2020-10-19 14:03:14.000000000","message":"Patch Set 4: Verified-1\n\nBuild Failed \n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-windows-master/31205/ : ABORTED\n\nBuild which was aborted due to a newer patch set being uploaded for the given review. ( http://cv.jenkins.couchbase.com/job/kv_engine-windows-master/31205/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.threadsanitizer/job/master/6737/ : ABORTED\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.linux-CE/job/master/6462/ : ABORTED\n\nBuild which was aborted due to a newer patch set being uploaded for the given review. ( http://cv.jenkins.couchbase.com/job/kv_engine.linux-CE/job/master/6462/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-clang_format_9/3670/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-clang_tidy/5141/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-clang_analyzer-master/26939/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.macos/job/master/5538/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.linux/job/master/6581/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.ASan-UBSan/job/master/14380/ : SUCCESS","accounts_in_message":[],"_revision_number":4},{"id":"4901e875a6144134a0eb952ba6cc98c2b9632fcf","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"date":"2020-10-19 15:22:45.000000000","message":"Patch Set 5: Verified-1\n\nBuild Failed \n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.linux/job/master/6584/ : FAILURE\n\nFailure of GoogleTest \"EphemeralOrPersistent/STExpiryPagerTest.ExpiredItemsDeleted/persistentMagma_value_only\":\n\n\u003cpre\u003e\n[ RUN      ] EphemeralOrPersistent/STExpiryPagerTest.ExpiredItemsDeleted/persistentMagma_value_only\n[2020-10-19T14:30:00.791Z] ../kv_engine/engines/ep/tests/module_tests/item_pager_test.cc:1292: Failure\n[2020-10-19T14:30:00.791Z] Expected equality of these values:\n[2020-10-19T14:30:00.791Z]   ENGINE_SUCCESS\n[2020-10-19T14:30:00.791Z]     Which is: 0\n[2020-10-19T14:30:00.791Z]   getKeyFn(key_2)\n[2020-10-19T14:30:00.791Z]     Which is: 1\n[2020-10-19T14:30:00.791Z] Key with TTL:20 should still exist.\n[2020-10-19T14:30:00.791Z] ../kv_engine/engines/ep/tests/module_tests/item_pager_test.cc:1299: Failure\n[2020-10-19T14:30:00.791Z] Expected equality of these values:\n[2020-10-19T14:30:00.791Z]   2\n[2020-10-19T14:30:00.791Z]   engine-\u003egetVBucket(vbid)-\u003egetNumItems()\n[2020-10-19T14:30:00.791Z]     Which is: 1\n[2020-10-19T14:30:00.791Z] Should still have 2 items after time-travelling\n[2020-10-19T14:30:00.791Z] [  FAILED  ] EphemeralOrPersistent/STExpiryPagerTest.ExpiredItemsDeleted/persistentMagma_value_only, where GetParam() \u003d (\"persistentMagma\", \"value_only\") (15365 ms)\nEphemeralOrPersistent/STExpiryPagerTest.ExpiredItemsDeleted/persistentMagma_value_only\n\u003c/pre\u003e\n ( http://cv.jenkins.couchbase.com/job/kv_engine.linux/job/master/6584/ )\n\nFailure of a CTest test [2020-10-19T14:30:00.791Z]   1/410 Test #219: ep-engine_ep_unit_tests.EphemeralOrPersistent/STExpiryPagerTest ............................................. ( http://cv.jenkins.couchbase.com/job/kv_engine.linux/job/master/6584/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.ASan-UBSan/job/master/14384/ : FAILURE\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.threadsanitizer/job/master/6741/ : FAILURE\n\nFailure of an engine_testapp test:\n\n\u003cpre\u003e\nRunning [0007/0047]: multi set...Exceeded maximum wait time of 60000000us waiting for stat \"ep_queue_size\" to be 0 (last value:5052) - aborting.\n\u003c/pre\u003e\n ( http://cv.jenkins.couchbase.com/job/kv_engine.threadsanitizer/job/master/6741/ )\n\nThreadSanitizer issue: data race (/home/couchbase/jenkins/workspace/kv_engine.threadsanitizer_master/install/lib/libtsan.so.0+0x27806) in free  ( http://cv.jenkins.couchbase.com/job/kv_engine.threadsanitizer/job/master/6741/ )\n\nFailure of a CTest test [2020-10-19T14:24:06.719Z]  30/388 Test #247: ep_testsuite_dcp.ephemeral.comp_passive ..................................................................... ( http://cv.jenkins.couchbase.com/job/kv_engine.threadsanitizer/job/master/6741/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-clang_format_9/3674/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-clang_tidy/5145/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-clang_analyzer-master/26943/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-windows-master/31208/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.macos/job/master/5541/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.linux-CE/job/master/6465/ : SUCCESS","accounts_in_message":[],"_revision_number":5},{"id":"e7158f33de84597c49a2517a04f3a0812b4638ee","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"date":"2020-10-19 17:09:06.000000000","message":"Patch Set 5:\n\nBuild Failed \n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.ASan-UBSan/job/master/14392/ : FAILURE\n\nUndefined Sanitizer: 7.3.0/bits/invoke.h:73:14: runtime error: member call on address 0x61700000a180 which does not point to an object of type \"EPBucket\" ( http://cv.jenkins.couchbase.com/job/kv_engine.ASan-UBSan/job/master/14392/ )\n\nFailure of a CTest test [2020-10-19T16:08:08.435Z]   1/393 Test #244: ep_testsuite_basic.value_eviction.magma .....................................................................Child aborted ( http://cv.jenkins.couchbase.com/job/kv_engine.ASan-UBSan/job/master/14392/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-clang_format_9/3674/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-clang_tidy/5145/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-clang_analyzer-master/26943/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-windows-master/31208/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.macos/job/master/5541/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.linux/job/master/6592/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.linux-CE/job/master/6465/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.threadsanitizer/job/master/6750/ : SUCCESS","accounts_in_message":[],"_revision_number":5},{"id":"41fc217c1f350be6c02768500f1c847d395ccfd7","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"date":"2020-10-20 11:30:38.000000000","message":"Patch Set 5:\n\nBuild Failed \n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.ASan-UBSan/job/master/14422/ : FAILURE\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-clang_format_9/3674/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-clang_tidy/5145/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-clang_analyzer-master/26943/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-windows-master/31208/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.macos/job/master/5541/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.linux/job/master/6592/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.linux-CE/job/master/6465/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.threadsanitizer/job/master/6750/ : SUCCESS","accounts_in_message":[],"_revision_number":5},{"id":"a8b0aa1e6736513d0732187eb05d34d0b68d4ddc","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"date":"2020-10-20 14:17:22.000000000","message":"Uploaded patch set 6.","accounts_in_message":[],"_revision_number":6},{"id":"7eb463e835a2ae7b78eb92cc4f8b54fbd665827e","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"date":"2020-10-20 14:31:11.000000000","message":"Uploaded patch set 7.","accounts_in_message":[],"_revision_number":7},{"id":"9bd42f5fb42f3018f93d6e6cd1efc259413a9f66","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"date":"2020-10-20 14:31:14.000000000","message":"Patch Set 6: Verified-1\n\nBuild Failed \n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-windows-master/31256/ : ABORTED\n\nNo problems were identified. If you know why this problem occurred, please add a suitable Cause for it. ( http://cv.jenkins.couchbase.com/job/kv_engine-windows-master/31256/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.macos/job/master/5589/ : ABORTED\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.linux/job/master/6637/ : ABORTED\n\nBuild which was aborted due to a newer patch set being uploaded for the given review. ( http://cv.jenkins.couchbase.com/job/kv_engine.linux/job/master/6637/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.threadsanitizer/job/master/6795/ : ABORTED\n\nBuild which was aborted due to a newer patch set being uploaded for the given review. ( http://cv.jenkins.couchbase.com/job/kv_engine.threadsanitizer/job/master/6795/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.linux-CE/job/master/6516/ : ABORTED\n\nBuild which was aborted due to a newer patch set being uploaded for the given review. ( http://cv.jenkins.couchbase.com/job/kv_engine.linux-CE/job/master/6516/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.ASan-UBSan/job/master/14438/ : ABORTED\n\nBuild which was aborted due to a newer patch set being uploaded for the given review. ( http://cv.jenkins.couchbase.com/job/kv_engine.ASan-UBSan/job/master/14438/ )","accounts_in_message":[],"_revision_number":6},{"id":"3541a02715b7e720d80cf401b34aba0527069e87","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"date":"2020-10-20 16:10:50.000000000","message":"Uploaded patch set 8.","accounts_in_message":[],"_revision_number":8},{"id":"99ad48548cfbb3d88a2f824d9b85285ae183f54b","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"date":"2020-10-20 16:10:55.000000000","message":"Patch Set 7: Verified-1\n\nBuild Failed \n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-clang_analyzer-master/26988/ : ABORTED\n\nBuild which was aborted due to a newer patch set being uploaded for the given review. ( http://cv.jenkins.couchbase.com/job/kv_engine-clang_analyzer-master/26988/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.macos/job/master/5591/ : ABORTED\n\nBuild which was aborted due to a newer patch set being uploaded for the given review. ( http://cv.jenkins.couchbase.com/job/kv_engine.macos/job/master/5591/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.threadsanitizer/job/master/6798/ : ABORTED\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.ASan-UBSan/job/master/14441/ : ABORTED\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.linux-CE/job/master/6518/ : ABORTED\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-clang_format_9/3719/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-clang_tidy/5190/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-windows-master/31258/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.linux/job/master/6639/ : SUCCESS","accounts_in_message":[],"_revision_number":7},{"id":"b4a01782d18d6b658dfc4e10d777614430b5344c","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"date":"2020-10-20 16:36:26.000000000","message":"Uploaded patch set 9: Patch Set 8 was rebased.","accounts_in_message":[],"_revision_number":9},{"id":"8ccfdfe8d819629ba33a2d53708606d635a419b3","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"date":"2020-10-20 16:36:33.000000000","message":"Patch Set 8: Verified-1\n\nBuild Failed \n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.linux-CE/job/master/6525/ : ABORTED\n\nBuild which was aborted due to a newer patch set being uploaded for the given review. ( http://cv.jenkins.couchbase.com/job/kv_engine.linux-CE/job/master/6525/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.threadsanitizer/job/master/6805/ : ABORTED\n\nBuild which was aborted due to a newer patch set being uploaded for the given review. ( http://cv.jenkins.couchbase.com/job/kv_engine.threadsanitizer/job/master/6805/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.macos/job/master/5598/ : ABORTED\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.linux/job/master/6645/ : ABORTED\n\nBuild which was aborted due to a newer patch set being uploaded for the given review. ( http://cv.jenkins.couchbase.com/job/kv_engine.linux/job/master/6645/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.ASan-UBSan/job/master/14448/ : ABORTED\n\nBuild which was aborted due to a newer patch set being uploaded for the given review. ( http://cv.jenkins.couchbase.com/job/kv_engine.ASan-UBSan/job/master/14448/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-windows-master/31264/ : SUCCESS","accounts_in_message":[],"_revision_number":8},{"id":"35437da18ab2731b9d63c53fc5794b1a56d9e12d","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"date":"2020-10-20 17:33:27.000000000","message":"Patch Set 9: Verified-1\n\nBuild Failed \n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.linux-CE/job/master/6527/ : FAILURE\n\nFailure of GoogleTest \"EphemeralOrPersistent/STItemPagerTest.ActiveEvictedIfReplicaEvictionInsufficient/persistent_value_only\":\n\n\u003cpre\u003e\n[ RUN      ] EphemeralOrPersistent/STItemPagerTest.ActiveEvictedIfReplicaEvictionInsufficient/persistent_value_only\n[2020-10-20T17:21:38.328Z] ../kv_engine/engines/ep/tests/module_tests/item_pager_test.cc:1130: Failure\n[2020-10-20T17:21:38.328Z] Expected: (activeRR) \u003c (100), actual: 100 vs 100\n[2020-10-20T17:21:38.328Z] [  FAILED  ] EphemeralOrPersistent/STItemPagerTest.ActiveEvictedIfReplicaEvictionInsufficient/persistent_value_only, where GetParam() \u003d (\"persistent\", \"value_only\") (68178 ms)\nEphemeralOrPersistent/STItemPagerTest.ActiveEvictedIfReplicaEvictionInsufficient/persistent_value_only\n\u003c/pre\u003e\n ( http://cv.jenkins.couchbase.com/job/kv_engine.linux-CE/job/master/6527/ )\n\nFailure of a CTest test [2020-10-20T17:21:38.328Z]   1/397 Test #214: ep-engine_ep_unit_tests.EphemeralOrPersistent/STItemPagerTest ............................................... ( http://cv.jenkins.couchbase.com/job/kv_engine.linux-CE/job/master/6527/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.ASan-UBSan/job/master/14450/ : FAILURE\n\nAddressSanitizer issue: odr-violation: global \"cb_new_hook\" at ../platform/src/cb_malloc_arena.cc:30:22 ( http://cv.jenkins.couchbase.com/job/kv_engine.ASan-UBSan/job/master/14450/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-clang_format_9/3726/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-clang_tidy/5197/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-clang_analyzer-master/26995/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-windows-master/31266/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.linux/job/master/6647/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.macos/job/master/5600/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.threadsanitizer/job/master/6807/ : SUCCESS","accounts_in_message":[],"_revision_number":9},{"id":"5b1ede8901de0045b86d03d1e768b2676da4d526","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"date":"2020-10-20 19:16:07.000000000","message":"Uploaded patch set 10: Patch Set 9 was rebased.","accounts_in_message":[],"_revision_number":10},{"id":"027ec69d662ab6f344bce45e94607a6e9632581d","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"date":"2020-10-20 19:54:15.000000000","message":"Patch Set 10: Verified-1\n\nBuild Failed \n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.linux-CE/job/master/6533/ : FAILURE\n\nFailure of GoogleTest \"EphemeralOrPersistent/STItemPagerTest.ActiveEvictedIfReplicaEvictionInsufficient/persistent_full_eviction\":\n\n\u003cpre\u003e\n[ RUN      ] EphemeralOrPersistent/STItemPagerTest.ActiveEvictedIfReplicaEvictionInsufficient/persistent_full_eviction\n[2020-10-20T19:40:22.828Z] ../kv_engine/engines/ep/tests/module_tests/item_pager_test.cc:1130: Failure\n[2020-10-20T19:40:22.828Z] Expected: (activeRR) \u003c (100), actual: 100 vs 100\n[2020-10-20T19:40:22.828Z] [  FAILED  ] EphemeralOrPersistent/STItemPagerTest.ActiveEvictedIfReplicaEvictionInsufficient/persistent_full_eviction, where GetParam() \u003d (\"persistent\", \"full_eviction\") (25747 ms)\nEphemeralOrPersistent/STItemPagerTest.ActiveEvictedIfReplicaEvictionInsufficient/persistent_full_eviction\n\u003c/pre\u003e\n ( http://cv.jenkins.couchbase.com/job/kv_engine.linux-CE/job/master/6533/ )\n\nFailure of a CTest test [2020-10-20T19:40:22.827Z] 231/397 Test #214: ep-engine_ep_unit_tests.EphemeralOrPersistent/STItemPagerTest ............................................... ( http://cv.jenkins.couchbase.com/job/kv_engine.linux-CE/job/master/6533/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-clang_format_9/3730/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-clang_tidy/5201/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-clang_analyzer-master/26999/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-windows-master/31271/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.macos/job/master/5605/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.linux/job/master/6652/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.ASan-UBSan/job/master/14455/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.threadsanitizer/job/master/6813/ : SUCCESS","accounts_in_message":[],"_revision_number":10},{"id":"72454df98b8495c224200ced42368dce4ee23a93","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"date":"2020-10-20 20:31:02.000000000","message":"Patch Set 10: Verified+1\n\nBuild Successful \n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-clang_format_9/3730/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-clang_tidy/5201/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-clang_analyzer-master/26999/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-windows-master/31271/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.macos/job/master/5605/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.linux/job/master/6652/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.ASan-UBSan/job/master/14455/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.threadsanitizer/job/master/6813/ : SUCCESS\n\nhttp://cv.jenkins.couchbase.com/job/kv_engine.linux-CE/job/master/6534/ : SUCCESS","accounts_in_message":[],"_revision_number":10},{"id":"43099e1f94abef3f79bc7eed903c43aaf03b6027","author":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"date":"2020-10-22 08:59:46.000000000","message":"Patch Set 10:\n\nping for review","accounts_in_message":[],"_revision_number":10},{"id":"3dc702a0b58cf3939202b31821cf7c8ce7a6d619","author":{"_account_id":1001546,"name":"Jim Walker","email":"jim@couchbase.com","username":"jim","avatars":[{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"date":"2020-10-23 09:21:23.000000000","message":"Patch Set 10: Code-Review+2","accounts_in_message":[],"_revision_number":10},{"id":"422d1378dfda0117c3fea6428ea821c399380340","tag":"autogenerated:gerrit:merged","author":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"date":"2020-10-23 10:00:50.000000000","message":"Change has been successfully cherry-picked as c370cd5ec3a018dccbf4857a97453b53d8bc6ec6 by Dave Rigby","accounts_in_message":[],"_revision_number":11},{"id":"b04c9ae11125c38a00591917f75af947411171ab","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"date":"2020-10-23 10:39:17.000000000","message":"Patch Set 11:\n\nBuild Started http://cv.jenkins.couchbase.com/job/kv_engine-master-post-commit/5354/ (2/2)","accounts_in_message":[],"_revision_number":11},{"id":"5497c344493357348ecde53de2c2a674b7426b62","tag":"autogenerated:jenkins-gerrit-trigger","author":{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"date":"2020-10-23 11:01:31.000000000","message":"Patch Set 11:\n\nBuild Failed \n\nhttp://cv.jenkins.couchbase.com/job/kv_engine-master-post-commit/5354/ : FAILURE\n\nCMake error: /home/couchbase/jenkins/workspace/kv_engine-master-post-commit/tlm/cmake/Modules/go-install.cmake:98 (MESSAGE) ( http://cv.jenkins.couchbase.com/job/kv_engine-master-post-commit/5354/ )\n\nhttp://cv.jenkins.couchbase.com/job/kv-engine-master-perf/3926/ : SUCCESS\u0027","accounts_in_message":[],"_revision_number":11}],"current_revision_number":11,"current_revision":"c370cd5ec3a018dccbf4857a97453b53d8bc6ec6","revisions":{"279eea9cc433d33955ca2add85a494be824a20b3":{"kind":"REWORK","_number":1,"created":"2020-10-18 20:26:11.000000000","uploader":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"ref":"refs/changes/73/138373/1","fetch":{"anonymous http":{"url":"https://review.couchbase.org/kv_engine","ref":"refs/changes/73/138373/1","commands":{"Branch":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/1 \u0026\u0026 git checkout -b change-138373 FETCH_HEAD","Checkout":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/1 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/1 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/1 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.couchbase.org/kv_engine refs/changes/73/138373/1","Reset To":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/1 \u0026\u0026 git reset --hard FETCH_HEAD"}}},"commit":{"parents":[{"commit":"6b0d3157a700b1a128f2a6296382e24dc190dc63","subject":"Cleanup: Remove else after return"}],"author":{"name":"Dave Rigby","email":"daver@couchbase.com","date":"2020-09-17 14:58:51.000000000","tz":0},"committer":{"name":"Dave Rigby","email":"daver@couchbase.com","date":"2020-10-18 20:25:39.000000000","tz":0},"subject":"MB-42029_GlobalTask_dtor_in_CPU_thread_SQUASHED","message":"MB-42029_GlobalTask_dtor_in_CPU_thread_SQUASHED\n\nSquashed commits on MB-42029_GlobalTask_dtor_in_CPU_thread up to ea0d27f\n\nChange-Id: Iecbff9f3b45fc9e3d385c67f6a6dd32242dc76fe\n"},"parents_data":[{"branch_name":"refs/heads/master","commit_id":"6b0d3157a700b1a128f2a6296382e24dc190dc63","is_merged_in_target_branch":true,"change_id":"Ibdb64068625c205b3b5dba08e104733714006347","change_number":138038,"patch_set_number":3,"change_status":"MERGED"}],"branch":"refs/heads/master"},"700b36e4e0eecfd10cb9e4b35b902c5d6c00facf":{"kind":"REWORK","_number":2,"created":"2020-10-19 11:42:19.000000000","uploader":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"ref":"refs/changes/73/138373/2","fetch":{"anonymous http":{"url":"https://review.couchbase.org/kv_engine","ref":"refs/changes/73/138373/2","commands":{"Branch":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/2 \u0026\u0026 git checkout -b change-138373 FETCH_HEAD","Checkout":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/2 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/2 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/2 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.couchbase.org/kv_engine refs/changes/73/138373/2","Reset To":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/2 \u0026\u0026 git reset --hard FETCH_HEAD"}}},"commit":{"parents":[{"commit":"6e2878c24667921eaf76df4d8dd51c1b3c97460f","subject":"DEBUG: Include tid in log pattern"}],"author":{"name":"Dave Rigby","email":"daver@couchbase.com","date":"2020-09-17 14:58:51.000000000","tz":0},"committer":{"name":"Dave Rigby","email":"daver@couchbase.com","date":"2020-10-19 11:41:27.000000000","tz":0},"subject":"MB-42029_GlobalTask_dtor_in_CPU_thread_SQUASHED","message":"MB-42029_GlobalTask_dtor_in_CPU_thread_SQUASHED\n\nSquashed commits on MB-42029_GlobalTask_dtor_in_CPU_thread up to\nea0d27, plus fix for attempting to wake task after cancel.\n\nChange-Id: Iecbff9f3b45fc9e3d385c67f6a6dd32242dc76fe\n"},"parents_data":[{"branch_name":"refs/heads/master","commit_id":"6e2878c24667921eaf76df4d8dd51c1b3c97460f","is_merged_in_target_branch":false,"change_id":"I6b451ba21c124cedc86d7653791bbb892c00b803","change_number":133975,"patch_set_number":9,"change_status":"ABANDONED"}],"branch":"refs/heads/master"},"914e5e825fee99221b2ff1cc1bb8bcae0078ac70":{"kind":"REWORK","_number":3,"created":"2020-10-19 13:13:45.000000000","uploader":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"ref":"refs/changes/73/138373/3","fetch":{"anonymous http":{"url":"https://review.couchbase.org/kv_engine","ref":"refs/changes/73/138373/3","commands":{"Branch":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/3 \u0026\u0026 git checkout -b change-138373 FETCH_HEAD","Checkout":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/3 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/3 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/3 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.couchbase.org/kv_engine refs/changes/73/138373/3","Reset To":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/3 \u0026\u0026 git reset --hard FETCH_HEAD"}}},"commit":{"parents":[{"commit":"6b0d3157a700b1a128f2a6296382e24dc190dc63","subject":"Cleanup: Remove else after return"}],"author":{"name":"Dave Rigby","email":"daver@couchbase.com","date":"2020-09-17 14:58:51.000000000","tz":0},"committer":{"name":"Dave Rigby","email":"daver@couchbase.com","date":"2020-10-19 13:12:46.000000000","tz":0},"subject":"MB-42029: FollyExecPool: Wait for tasks cancelled in unregisterTaskable","message":"MB-42029: FollyExecPool: Wait for tasks cancelled in unregisterTaskable\n\n+Issue+\n\nWhen enabling FollyExecutorPool by default, TSan reports the following\nrace when running ./ep-engine_ep_unit_tests\n\"--gtest_filter\u003dDurabilityRespondAmbiguousTest.*\":\n\nAuxIO thread:\n    Previous atomic write of size 8 at 0x7b74000020a0 by thread T8:\n    #0 __tsan_atomic64_fetch_sub \u003cnull\u003e (libtsan.so.0+0x000000060890)\n    ...\n    #4 ~HashTable kv_engine/engines/ep/src/hash_table.cc:161 (ep-engine_ep_unit_tests+0x00000122cae1)\n    #5 ~VBucket kv_engine/engines/ep/src/vbucket.cc:286 (ep-engine_ep_unit_tests+0x0000012b3af4)\n    #6 ~EPVBucket kv_engine/engines/ep/src/ep_vb.cc:101 (ep-engine_ep_unit_tests+0x0000011af5e1)\n    ...\n    #10 ~VBucketMemoryDeletionTask kv_engine/engines/ep/src/vbucketdeletiontask.cc:45 (ep-engine_ep_unit_tests+0x0000012e4530)\n    ...\n    #17 std::__shared_ptr\u003cGlobalTask\u003e::reset() /usr/local/include/c++/7.3.0/bits/shared_ptr_base.h:1235 (ep-engine_ep_unit_tests+0x000001221e75)\n    #18 FollyExecutorPool::TaskProxy::~TaskProxy()::{lambda()#1}::operator()() kv_engine/engines/ep/src/folly_executorpool.cc:80 (ep-engine_ep_unit_tests+0x000001221e75)\n\nMain thread:\n    Write of size 8 at 0x7b74000020a0 by main thread:\n    #0 free \u003cnull\u003e (libtsan.so.0+0x000000027806)\n    ...\n    #6 CoreStore\u003c...\u003e::~CoreStore() platform/include/platform/corestore.h:50 (ep-engine_ep_unit_tests+0x0000012988b1)\n    #7 ~EPStats kv_engine/engines/ep/src/stats.cc:132 (ep-engine_ep_unit_tests+0x0000012988b1)\n    #8 ~EventuallyPersistentEngine kv_engine/engines/ep/src/ep_engine.cc:6593 (ep-engine_ep_unit_tests+0x0000011e3bb5)\n    ...\n    #12 DurabilityRespondAmbiguousTest_RespondAmbiguousNotificationDeadLock_Test::TestBody() kv_engine/engines/ep/tests/module_tests/evp_store_durability_test.cc:2350 (ep-engine_ep_unit_tests+0x000000bd3642)\n\nThe crux of this issue seems to be that a background AuxIO task run\nvia the FollyExecutorPool is deleting a VBucket object concurrently\nwith the main thread deleting an EPStats object.\n\n+Background+\n\nDetails of how CB3ExecutorPool and FollyExecutorPool implement\n{{unregisterTaskable()}}, which I believe is what leads to this\nproblem.\n\nCB3ExecutorPool:\n\nDuring CB3ExecutorPool::unregisterTaskable():\n\n1. CB3ExecutorPool::_stopTaskGroup() is called and will wait for\n   VBucketMemoryDeletionTask to run.\n\n2. When VBucketMemoryDeletionTask::run() is call it returns false.\n\n3. CB3ExecutorThread will then synchronously call\n   CB3ExecutorThread::cancelCurrentTask() -\u003e\n   CB3ExecutorPool::cancel(). That removes all Cb3ExecutorPool-owned\n   references to task, and hence will run VBucketMemoryDeletionTask\n   dtor.\n\n4. VBucketMemoryDeletionTask dtor frees the VBucket object.\n\nAs such, by the time CB3ExecutorPool::unregisterTaskable() returns the\nVBucket is *guaranteed* to have been freed.\n\nFollyExecutorPool:\n\nDuring FollyExecutorPool::unregisterTaskable():\n\n1. All tasks scheduled to run in future (owned by IO thread EventBase)\n   are either cancelled (if allowed), or woken to run asap on CPU pool.\n\n2. All tasks waiting to, or currently running on CPU pool are waited\n   for by polling for taskOwners to no longer contain any tasks for\n   the given taskable.\n\n3. (On the CPU threads) Each queued task is run, on completion\n   rescheduleTaskAfterRun is called to add work to the IO thread\n   EventBase to decide when to reschedule, or (in this case) to\n   actualy cancel the task.\n\n4. (On the IO thread) FollyExecutorPool::rescheduleTaskAfterRun is\n   called, for cancelled tasks this calls State::cancelTask() which\n   removes the task from taskOwners - at which point TaskProxy dtor\n   runs, which schedules _another_ task on CPU pool to actually delete\n   the GlobalTask.\n\nThe problem here is that the TaskProxy is removed from taskOwners and\ndeletes at (4) on the IO thread; however the GlobalTask destruction is\ndeferred to later execution on a CPU thread. As such,\nFollyExecutorPool::unregisterTaskable() can see taskOwners having no\ntasks left for the taskable (and hence return) _before_ the VBucket\nobject is deleted.\n\nNote the deferred deletion at (5) was added to avoid potentially large\namounts of work being done on the IO thread - we aim to minimise work\ndone here as it can impact the scheduling of other tasks.\n\n+Solution+\n\nIf the TaskProxy removal from taskOwners is deferred until _after_ the\nGlobalTask shared ownership is released, then unregisterTaskable()\nwill no longer return until all GlobalTask references inside\nFollyExecutorPool have been released.\n\nTo achieve this changes the ownership model in FollyExecutorPool are\nneeded:\n\n1. Don\u0027t immediately remove TaskProxy from taskOwners in cancelTask().\n   Instead:\n   a) Mark it as cancelled, by setting the GlobalTask ptr to null,\n   b) Schedule an asynchronous task to release its GlobalTask\n      shared_ptr.\n\n2. This new async task (setup in resetTaskPtrViaCpuPool) releases the\n   TaskProxy\u0027s shared ownership on GlobalTask, then schedules an (IO\n   thread) completion task to finally remove the TaskProxy from\n   taskOwners.\n\nunregisterTaskable() is unchanged - it still waits for the taskOwner\nmap for the given Taskable become empty; however given the above\nchanges that only happens once the GlobalTask reference has been\nreleased.\n\nChange-Id: Iecbff9f3b45fc9e3d385c67f6a6dd32242dc76fe\n"},"parents_data":[{"branch_name":"refs/heads/master","commit_id":"6b0d3157a700b1a128f2a6296382e24dc190dc63","is_merged_in_target_branch":true,"change_id":"Ibdb64068625c205b3b5dba08e104733714006347","change_number":138038,"patch_set_number":3,"change_status":"MERGED"}],"branch":"refs/heads/master"},"33114dae63631bedf7a386e97d36f380e88f06e1":{"kind":"REWORK","_number":4,"created":"2020-10-19 13:30:47.000000000","uploader":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"ref":"refs/changes/73/138373/4","fetch":{"anonymous http":{"url":"https://review.couchbase.org/kv_engine","ref":"refs/changes/73/138373/4","commands":{"Branch":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/4 \u0026\u0026 git checkout -b change-138373 FETCH_HEAD","Checkout":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/4 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/4 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/4 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.couchbase.org/kv_engine refs/changes/73/138373/4","Reset To":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/4 \u0026\u0026 git reset --hard FETCH_HEAD"}}},"commit":{"parents":[{"commit":"6b0d3157a700b1a128f2a6296382e24dc190dc63","subject":"Cleanup: Remove else after return"}],"author":{"name":"Dave Rigby","email":"daver@couchbase.com","date":"2020-09-17 14:58:51.000000000","tz":0},"committer":{"name":"Dave Rigby","email":"daver@couchbase.com","date":"2020-10-19 13:23:12.000000000","tz":0},"subject":"MB-42029: FollyExecPool: Wait for tasks cancelled in unregisterTaskable","message":"MB-42029: FollyExecPool: Wait for tasks cancelled in unregisterTaskable\n\n+Issue+\n\nWhen enabling FollyExecutorPool by default, TSan reports the following\nrace when running ./ep-engine_ep_unit_tests\n\"--gtest_filter\u003dDurabilityRespondAmbiguousTest.*\":\n\nAuxIO thread:\n    Previous atomic write of size 8 at 0x7b74000020a0 by thread T8:\n    #0 __tsan_atomic64_fetch_sub \u003cnull\u003e (libtsan.so.0+0x000000060890)\n    ...\n    #4 ~HashTable kv_engine/engines/ep/src/hash_table.cc:161 (ep-engine_ep_unit_tests+0x00000122cae1)\n    #5 ~VBucket kv_engine/engines/ep/src/vbucket.cc:286 (ep-engine_ep_unit_tests+0x0000012b3af4)\n    #6 ~EPVBucket kv_engine/engines/ep/src/ep_vb.cc:101 (ep-engine_ep_unit_tests+0x0000011af5e1)\n    ...\n    #10 ~VBucketMemoryDeletionTask kv_engine/engines/ep/src/vbucketdeletiontask.cc:45 (ep-engine_ep_unit_tests+0x0000012e4530)\n    ...\n    #17 std::__shared_ptr\u003cGlobalTask\u003e::reset() /usr/local/include/c++/7.3.0/bits/shared_ptr_base.h:1235 (ep-engine_ep_unit_tests+0x000001221e75)\n    #18 FollyExecutorPool::TaskProxy::~TaskProxy()::{lambda()#1}::operator()() kv_engine/engines/ep/src/folly_executorpool.cc:80 (ep-engine_ep_unit_tests+0x000001221e75)\n\nMain thread:\n    Write of size 8 at 0x7b74000020a0 by main thread:\n    #0 free \u003cnull\u003e (libtsan.so.0+0x000000027806)\n    ...\n    #6 CoreStore\u003c...\u003e::~CoreStore() platform/include/platform/corestore.h:50 (ep-engine_ep_unit_tests+0x0000012988b1)\n    #7 ~EPStats kv_engine/engines/ep/src/stats.cc:132 (ep-engine_ep_unit_tests+0x0000012988b1)\n    #8 ~EventuallyPersistentEngine kv_engine/engines/ep/src/ep_engine.cc:6593 (ep-engine_ep_unit_tests+0x0000011e3bb5)\n    ...\n    #12 DurabilityRespondAmbiguousTest_RespondAmbiguousNotificationDeadLock_Test::TestBody() kv_engine/engines/ep/tests/module_tests/evp_store_durability_test.cc:2350 (ep-engine_ep_unit_tests+0x000000bd3642)\n\nThe crux of this issue seems to be that a background AuxIO task run\nvia the FollyExecutorPool is deleting a VBucket object concurrently\nwith the main thread deleting an EPStats object.\n\n+Background+\n\nDetails of how CB3ExecutorPool and FollyExecutorPool implement\n{{unregisterTaskable()}}, which I believe is what leads to this\nproblem.\n\nCB3ExecutorPool:\n\nDuring CB3ExecutorPool::unregisterTaskable():\n\n1. CB3ExecutorPool::_stopTaskGroup() is called and will wait for\n   VBucketMemoryDeletionTask to run.\n\n2. When VBucketMemoryDeletionTask::run() is call it returns false.\n\n3. CB3ExecutorThread will then synchronously call\n   CB3ExecutorThread::cancelCurrentTask() -\u003e\n   CB3ExecutorPool::cancel(). That removes all Cb3ExecutorPool-owned\n   references to task, and hence will run VBucketMemoryDeletionTask\n   dtor.\n\n4. VBucketMemoryDeletionTask dtor frees the VBucket object.\n\nAs such, by the time CB3ExecutorPool::unregisterTaskable() returns the\nVBucket is *guaranteed* to have been freed.\n\nFollyExecutorPool:\n\nDuring FollyExecutorPool::unregisterTaskable():\n\n1. All tasks scheduled to run in future (owned by IO thread EventBase)\n   are either cancelled (if allowed), or woken to run asap on CPU pool.\n\n2. All tasks waiting to, or currently running on CPU pool are waited\n   for by polling for taskOwners to no longer contain any tasks for\n   the given taskable.\n\n3. (On the CPU threads) Each queued task is run, on completion\n   rescheduleTaskAfterRun is called to add work to the IO thread\n   EventBase to decide when to reschedule, or (in this case) to\n   actualy cancel the task.\n\n4. (On the IO thread) FollyExecutorPool::rescheduleTaskAfterRun is\n   called, for cancelled tasks this calls State::cancelTask() which\n   removes the task from taskOwners - at which point TaskProxy dtor\n   runs, which schedules _another_ task on CPU pool to actually delete\n   the GlobalTask.\n\nThe problem here is that the TaskProxy is removed from taskOwners and\ndeletes at (4) on the IO thread; however the GlobalTask destruction is\ndeferred to later execution on a CPU thread. As such,\nFollyExecutorPool::unregisterTaskable() can see taskOwners having no\ntasks left for the taskable (and hence return) _before_ the VBucket\nobject is deleted.\n\nNote the deferred deletion at (5) was added to avoid potentially large\namounts of work being done on the IO thread - we aim to minimise work\ndone here as it can impact the scheduling of other tasks.\n\n+Solution+\n\nIf the TaskProxy removal from taskOwners is deferred until _after_ the\nGlobalTask shared ownership is released, then unregisterTaskable()\nwill no longer return until all GlobalTask references inside\nFollyExecutorPool have been released.\n\nTo achieve this changes the ownership model in FollyExecutorPool are\nneeded:\n\n1. Don\u0027t immediately remove TaskProxy from taskOwners in cancelTask().\n   Instead:\n   a) Mark it as cancelled, by setting the GlobalTask ptr to null,\n   b) Schedule an asynchronous task to release its GlobalTask\n      shared_ptr.\n\n2. This new async task (setup in resetTaskPtrViaCpuPool) releases the\n   TaskProxy\u0027s shared ownership on GlobalTask, then schedules an (IO\n   thread) completion task to finally remove the TaskProxy from\n   taskOwners.\n\nunregisterTaskable() is unchanged - it still waits for the taskOwner\nmap for the given Taskable become empty; however given the above\nchanges that only happens once the GlobalTask reference has been\nreleased.\n\nChange-Id: Iecbff9f3b45fc9e3d385c67f6a6dd32242dc76fe\n"},"parents_data":[{"branch_name":"refs/heads/master","commit_id":"6b0d3157a700b1a128f2a6296382e24dc190dc63","is_merged_in_target_branch":true,"change_id":"Ibdb64068625c205b3b5dba08e104733714006347","change_number":138038,"patch_set_number":3,"change_status":"MERGED"}],"branch":"refs/heads/master"},"44248091bf826789ff63252517e72bad92d770c5":{"kind":"REWORK","_number":5,"created":"2020-10-19 14:03:02.000000000","uploader":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"ref":"refs/changes/73/138373/5","fetch":{"anonymous http":{"url":"https://review.couchbase.org/kv_engine","ref":"refs/changes/73/138373/5","commands":{"Branch":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/5 \u0026\u0026 git checkout -b change-138373 FETCH_HEAD","Checkout":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/5 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/5 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/5 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.couchbase.org/kv_engine refs/changes/73/138373/5","Reset To":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/5 \u0026\u0026 git reset --hard FETCH_HEAD"}}},"commit":{"parents":[{"commit":"6b0d3157a700b1a128f2a6296382e24dc190dc63","subject":"Cleanup: Remove else after return"}],"author":{"name":"Dave Rigby","email":"daver@couchbase.com","date":"2020-09-17 14:58:51.000000000","tz":0},"committer":{"name":"Dave Rigby","email":"daver@couchbase.com","date":"2020-10-19 14:02:02.000000000","tz":0},"subject":"MB-42029: FollyExecPool: Wait for tasks cancelled in unregisterTaskable","message":"MB-42029: FollyExecPool: Wait for tasks cancelled in unregisterTaskable\n\n+Issue+\n\nWhen enabling FollyExecutorPool by default, TSan reports the following\nrace when running ./ep-engine_ep_unit_tests\n\"--gtest_filter\u003dDurabilityRespondAmbiguousTest.*\":\n\nAuxIO thread:\n    Previous atomic write of size 8 at 0x7b74000020a0 by thread T8:\n    #0 __tsan_atomic64_fetch_sub \u003cnull\u003e (libtsan.so.0+0x000000060890)\n    ...\n    #4 ~HashTable kv_engine/engines/ep/src/hash_table.cc:161 (ep-engine_ep_unit_tests+0x00000122cae1)\n    #5 ~VBucket kv_engine/engines/ep/src/vbucket.cc:286 (ep-engine_ep_unit_tests+0x0000012b3af4)\n    #6 ~EPVBucket kv_engine/engines/ep/src/ep_vb.cc:101 (ep-engine_ep_unit_tests+0x0000011af5e1)\n    ...\n    #10 ~VBucketMemoryDeletionTask kv_engine/engines/ep/src/vbucketdeletiontask.cc:45 (ep-engine_ep_unit_tests+0x0000012e4530)\n    ...\n    #17 std::__shared_ptr\u003cGlobalTask\u003e::reset() /usr/local/include/c++/7.3.0/bits/shared_ptr_base.h:1235 (ep-engine_ep_unit_tests+0x000001221e75)\n    #18 FollyExecutorPool::TaskProxy::~TaskProxy()::{lambda()#1}::operator()() kv_engine/engines/ep/src/folly_executorpool.cc:80 (ep-engine_ep_unit_tests+0x000001221e75)\n\nMain thread:\n    Write of size 8 at 0x7b74000020a0 by main thread:\n    #0 free \u003cnull\u003e (libtsan.so.0+0x000000027806)\n    ...\n    #6 CoreStore\u003c...\u003e::~CoreStore() platform/include/platform/corestore.h:50 (ep-engine_ep_unit_tests+0x0000012988b1)\n    #7 ~EPStats kv_engine/engines/ep/src/stats.cc:132 (ep-engine_ep_unit_tests+0x0000012988b1)\n    #8 ~EventuallyPersistentEngine kv_engine/engines/ep/src/ep_engine.cc:6593 (ep-engine_ep_unit_tests+0x0000011e3bb5)\n    ...\n    #12 DurabilityRespondAmbiguousTest_RespondAmbiguousNotificationDeadLock_Test::TestBody() kv_engine/engines/ep/tests/module_tests/evp_store_durability_test.cc:2350 (ep-engine_ep_unit_tests+0x000000bd3642)\n\nThe crux of this issue seems to be that a background AuxIO task run\nvia the FollyExecutorPool is deleting a VBucket object concurrently\nwith the main thread deleting an EPStats object.\n\n+Background+\n\nDetails of how CB3ExecutorPool and FollyExecutorPool implement\n{{unregisterTaskable()}}, which I believe is what leads to this\nproblem.\n\nCB3ExecutorPool:\n\nDuring CB3ExecutorPool::unregisterTaskable():\n\n1. CB3ExecutorPool::_stopTaskGroup() is called and will wait for\n   VBucketMemoryDeletionTask to run.\n\n2. When VBucketMemoryDeletionTask::run() is call it returns false.\n\n3. CB3ExecutorThread will then synchronously call\n   CB3ExecutorThread::cancelCurrentTask() -\u003e\n   CB3ExecutorPool::cancel(). That removes all Cb3ExecutorPool-owned\n   references to task, and hence will run VBucketMemoryDeletionTask\n   dtor.\n\n4. VBucketMemoryDeletionTask dtor frees the VBucket object.\n\nAs such, by the time CB3ExecutorPool::unregisterTaskable() returns the\nVBucket is *guaranteed* to have been freed.\n\nFollyExecutorPool:\n\nDuring FollyExecutorPool::unregisterTaskable():\n\n1. All tasks scheduled to run in future (owned by IO thread EventBase)\n   are either cancelled (if allowed), or woken to run asap on CPU pool.\n\n2. All tasks waiting to, or currently running on CPU pool are waited\n   for by polling for taskOwners to no longer contain any tasks for\n   the given taskable.\n\n3. (On the CPU threads) Each queued task is run, on completion\n   rescheduleTaskAfterRun is called to add work to the IO thread\n   EventBase to decide when to reschedule, or (in this case) to\n   actualy cancel the task.\n\n4. (On the IO thread) FollyExecutorPool::rescheduleTaskAfterRun is\n   called, for cancelled tasks this calls State::cancelTask() which\n   removes the task from taskOwners - at which point TaskProxy dtor\n   runs, which schedules _another_ task on CPU pool to actually delete\n   the GlobalTask.\n\nThe problem here is that the TaskProxy is removed from taskOwners and\ndeletes at (4) on the IO thread; however the GlobalTask destruction is\ndeferred to later execution on a CPU thread. As such,\nFollyExecutorPool::unregisterTaskable() can see taskOwners having no\ntasks left for the taskable (and hence return) _before_ the VBucket\nobject is deleted.\n\nNote the deferred deletion at (5) was added to avoid potentially large\namounts of work being done on the IO thread - we aim to minimise work\ndone here as it can impact the scheduling of other tasks.\n\n+Solution+\n\nIf the TaskProxy removal from taskOwners is deferred until _after_ the\nGlobalTask shared ownership is released, then unregisterTaskable()\nwill no longer return until all GlobalTask references inside\nFollyExecutorPool have been released.\n\nTo achieve this changes the ownership model in FollyExecutorPool are\nneeded:\n\n1. Don\u0027t immediately remove TaskProxy from taskOwners in cancelTask().\n   Instead:\n   a) Mark it as cancelled, by setting the GlobalTask ptr to null,\n   b) Schedule an asynchronous task to release its GlobalTask\n      shared_ptr.\n\n2. This new async task (setup in resetTaskPtrViaCpuPool) releases the\n   TaskProxy\u0027s shared ownership on GlobalTask, then schedules an (IO\n   thread) completion task to finally remove the TaskProxy from\n   taskOwners.\n\nunregisterTaskable() is unchanged - it still waits for the taskOwner\nmap for the given Taskable become empty; however given the above\nchanges that only happens once the GlobalTask reference has been\nreleased.\n\nChange-Id: Iecbff9f3b45fc9e3d385c67f6a6dd32242dc76fe\n"},"parents_data":[{"branch_name":"refs/heads/master","commit_id":"6b0d3157a700b1a128f2a6296382e24dc190dc63","is_merged_in_target_branch":true,"change_id":"Ibdb64068625c205b3b5dba08e104733714006347","change_number":138038,"patch_set_number":3,"change_status":"MERGED"}],"branch":"refs/heads/master"},"dbf8b211f32c459eb926df29f46b907f98a624d3":{"kind":"REWORK","_number":6,"created":"2020-10-20 14:17:22.000000000","uploader":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"ref":"refs/changes/73/138373/6","fetch":{"anonymous http":{"url":"https://review.couchbase.org/kv_engine","ref":"refs/changes/73/138373/6","commands":{"Branch":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/6 \u0026\u0026 git checkout -b change-138373 FETCH_HEAD","Checkout":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/6 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/6 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/6 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.couchbase.org/kv_engine refs/changes/73/138373/6","Reset To":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/6 \u0026\u0026 git reset --hard FETCH_HEAD"}}},"commit":{"parents":[{"commit":"6b0d3157a700b1a128f2a6296382e24dc190dc63","subject":"Cleanup: Remove else after return"}],"author":{"name":"Dave Rigby","email":"daver@couchbase.com","date":"2020-09-17 14:58:51.000000000","tz":0},"committer":{"name":"Dave Rigby","email":"daver@couchbase.com","date":"2020-10-20 14:17:09.000000000","tz":60},"subject":"MB-42029: FollyExecPool: Wait for tasks cancelled in unregisterTaskable","message":"MB-42029: FollyExecPool: Wait for tasks cancelled in unregisterTaskable\n\n+Issue+\n\nWhen enabling FollyExecutorPool by default, TSan reports the following\nrace when running ./ep-engine_ep_unit_tests\n\"--gtest_filter\u003dDurabilityRespondAmbiguousTest.*\":\n\nAuxIO thread:\n    Previous atomic write of size 8 at 0x7b74000020a0 by thread T8:\n    #0 __tsan_atomic64_fetch_sub \u003cnull\u003e (libtsan.so.0+0x000000060890)\n    ...\n    #4 ~HashTable kv_engine/engines/ep/src/hash_table.cc:161 (ep-engine_ep_unit_tests+0x00000122cae1)\n    #5 ~VBucket kv_engine/engines/ep/src/vbucket.cc:286 (ep-engine_ep_unit_tests+0x0000012b3af4)\n    #6 ~EPVBucket kv_engine/engines/ep/src/ep_vb.cc:101 (ep-engine_ep_unit_tests+0x0000011af5e1)\n    ...\n    #10 ~VBucketMemoryDeletionTask kv_engine/engines/ep/src/vbucketdeletiontask.cc:45 (ep-engine_ep_unit_tests+0x0000012e4530)\n    ...\n    #17 std::__shared_ptr\u003cGlobalTask\u003e::reset() /usr/local/include/c++/7.3.0/bits/shared_ptr_base.h:1235 (ep-engine_ep_unit_tests+0x000001221e75)\n    #18 FollyExecutorPool::TaskProxy::~TaskProxy()::{lambda()#1}::operator()() kv_engine/engines/ep/src/folly_executorpool.cc:80 (ep-engine_ep_unit_tests+0x000001221e75)\n\nMain thread:\n    Write of size 8 at 0x7b74000020a0 by main thread:\n    #0 free \u003cnull\u003e (libtsan.so.0+0x000000027806)\n    ...\n    #6 CoreStore\u003c...\u003e::~CoreStore() platform/include/platform/corestore.h:50 (ep-engine_ep_unit_tests+0x0000012988b1)\n    #7 ~EPStats kv_engine/engines/ep/src/stats.cc:132 (ep-engine_ep_unit_tests+0x0000012988b1)\n    #8 ~EventuallyPersistentEngine kv_engine/engines/ep/src/ep_engine.cc:6593 (ep-engine_ep_unit_tests+0x0000011e3bb5)\n    ...\n    #12 DurabilityRespondAmbiguousTest_RespondAmbiguousNotificationDeadLock_Test::TestBody() kv_engine/engines/ep/tests/module_tests/evp_store_durability_test.cc:2350 (ep-engine_ep_unit_tests+0x000000bd3642)\n\nThe crux of this issue seems to be that a background AuxIO task run\nvia the FollyExecutorPool is deleting a VBucket object concurrently\nwith the main thread deleting an EPStats object.\n\n+Background+\n\nDetails of how CB3ExecutorPool and FollyExecutorPool implement\n{{unregisterTaskable()}}, which I believe is what leads to this\nproblem.\n\nCB3ExecutorPool:\n\nDuring CB3ExecutorPool::unregisterTaskable():\n\n1. CB3ExecutorPool::_stopTaskGroup() is called and will wait for\n   VBucketMemoryDeletionTask to run.\n\n2. When VBucketMemoryDeletionTask::run() is call it returns false.\n\n3. CB3ExecutorThread will then synchronously call\n   CB3ExecutorThread::cancelCurrentTask() -\u003e\n   CB3ExecutorPool::cancel(). That removes all Cb3ExecutorPool-owned\n   references to task, and hence will run VBucketMemoryDeletionTask\n   dtor.\n\n4. VBucketMemoryDeletionTask dtor frees the VBucket object.\n\nAs such, by the time CB3ExecutorPool::unregisterTaskable() returns the\nVBucket is *guaranteed* to have been freed.\n\nFollyExecutorPool:\n\nDuring FollyExecutorPool::unregisterTaskable():\n\n1. All tasks scheduled to run in future (owned by IO thread EventBase)\n   are either cancelled (if allowed), or woken to run asap on CPU pool.\n\n2. All tasks waiting to, or currently running on CPU pool are waited\n   for by polling for taskOwners to no longer contain any tasks for\n   the given taskable.\n\n3. (On the CPU threads) Each queued task is run, on completion\n   rescheduleTaskAfterRun is called to add work to the IO thread\n   EventBase to decide when to reschedule, or (in this case) to\n   actualy cancel the task.\n\n4. (On the IO thread) FollyExecutorPool::rescheduleTaskAfterRun is\n   called, for cancelled tasks this calls State::cancelTask() which\n   removes the task from taskOwners - at which point TaskProxy dtor\n   runs, which schedules _another_ task on CPU pool to actually delete\n   the GlobalTask.\n\nThe problem here is that the TaskProxy is removed from taskOwners and\ndeletes at (4) on the IO thread; however the GlobalTask destruction is\ndeferred to later execution on a CPU thread. As such,\nFollyExecutorPool::unregisterTaskable() can see taskOwners having no\ntasks left for the taskable (and hence return) _before_ the VBucket\nobject is deleted.\n\nNote the deferred deletion at (5) was added to avoid potentially large\namounts of work being done on the IO thread - we aim to minimise work\ndone here as it can impact the scheduling of other tasks.\n\n+Solution+\n\nIf the TaskProxy removal from taskOwners is deferred until _after_ the\nGlobalTask shared ownership is released, then unregisterTaskable()\nwill no longer return until all GlobalTask references inside\nFollyExecutorPool have been released.\n\nTo achieve this changes the ownership model in FollyExecutorPool are\nneeded:\n\n1. Don\u0027t immediately remove TaskProxy from taskOwners in cancelTask().\n   Instead:\n   a) Mark it as cancelled, by setting the GlobalTask ptr to null,\n   b) Schedule an asynchronous task to release its GlobalTask\n      shared_ptr.\n\n2. This new async task (setup in resetTaskPtrViaCpuPool) releases the\n   TaskProxy\u0027s shared ownership on GlobalTask, then schedules an (IO\n   thread) completion task to finally remove the TaskProxy from\n   taskOwners.\n\nunregisterTaskable() is unchanged - it still waits for the taskOwner\nmap for the given Taskable become empty; however given the above\nchanges that only happens once the GlobalTask reference has been\nreleased.\n\nChange-Id: Iecbff9f3b45fc9e3d385c67f6a6dd32242dc76fe\n"},"parents_data":[{"branch_name":"refs/heads/master","commit_id":"6b0d3157a700b1a128f2a6296382e24dc190dc63","is_merged_in_target_branch":true,"change_id":"Ibdb64068625c205b3b5dba08e104733714006347","change_number":138038,"patch_set_number":3,"change_status":"MERGED"}],"branch":"refs/heads/master"},"dfd83d4c4d6c7cb66a21c3fd5f15cd9dbfdda876":{"kind":"REWORK","_number":7,"created":"2020-10-20 14:31:11.000000000","uploader":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"ref":"refs/changes/73/138373/7","fetch":{"anonymous http":{"url":"https://review.couchbase.org/kv_engine","ref":"refs/changes/73/138373/7","commands":{"Branch":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/7 \u0026\u0026 git checkout -b change-138373 FETCH_HEAD","Checkout":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/7 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/7 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/7 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.couchbase.org/kv_engine refs/changes/73/138373/7","Reset To":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/7 \u0026\u0026 git reset --hard FETCH_HEAD"}}},"commit":{"parents":[{"commit":"6b0d3157a700b1a128f2a6296382e24dc190dc63","subject":"Cleanup: Remove else after return"}],"author":{"name":"Dave Rigby","email":"daver@couchbase.com","date":"2020-09-17 14:58:51.000000000","tz":0},"committer":{"name":"Dave Rigby","email":"daver@couchbase.com","date":"2020-10-20 14:31:03.000000000","tz":60},"subject":"MB-42029: FollyExecPool: Wait for tasks cancelled in unregisterTaskable","message":"MB-42029: FollyExecPool: Wait for tasks cancelled in unregisterTaskable\n\n+Issue+\n\nWhen enabling FollyExecutorPool by default, TSan reports the following\nrace when running ./ep-engine_ep_unit_tests\n\"--gtest_filter\u003dDurabilityRespondAmbiguousTest.*\":\n\nAuxIO thread:\n    Previous atomic write of size 8 at 0x7b74000020a0 by thread T8:\n    #0 __tsan_atomic64_fetch_sub \u003cnull\u003e (libtsan.so.0+0x000000060890)\n    ...\n    #4 ~HashTable kv_engine/engines/ep/src/hash_table.cc:161 (ep-engine_ep_unit_tests+0x00000122cae1)\n    #5 ~VBucket kv_engine/engines/ep/src/vbucket.cc:286 (ep-engine_ep_unit_tests+0x0000012b3af4)\n    #6 ~EPVBucket kv_engine/engines/ep/src/ep_vb.cc:101 (ep-engine_ep_unit_tests+0x0000011af5e1)\n    ...\n    #10 ~VBucketMemoryDeletionTask kv_engine/engines/ep/src/vbucketdeletiontask.cc:45 (ep-engine_ep_unit_tests+0x0000012e4530)\n    ...\n    #17 std::__shared_ptr\u003cGlobalTask\u003e::reset() /usr/local/include/c++/7.3.0/bits/shared_ptr_base.h:1235 (ep-engine_ep_unit_tests+0x000001221e75)\n    #18 FollyExecutorPool::TaskProxy::~TaskProxy()::{lambda()#1}::operator()() kv_engine/engines/ep/src/folly_executorpool.cc:80 (ep-engine_ep_unit_tests+0x000001221e75)\n\nMain thread:\n    Write of size 8 at 0x7b74000020a0 by main thread:\n    #0 free \u003cnull\u003e (libtsan.so.0+0x000000027806)\n    ...\n    #6 CoreStore\u003c...\u003e::~CoreStore() platform/include/platform/corestore.h:50 (ep-engine_ep_unit_tests+0x0000012988b1)\n    #7 ~EPStats kv_engine/engines/ep/src/stats.cc:132 (ep-engine_ep_unit_tests+0x0000012988b1)\n    #8 ~EventuallyPersistentEngine kv_engine/engines/ep/src/ep_engine.cc:6593 (ep-engine_ep_unit_tests+0x0000011e3bb5)\n    ...\n    #12 DurabilityRespondAmbiguousTest_RespondAmbiguousNotificationDeadLock_Test::TestBody() kv_engine/engines/ep/tests/module_tests/evp_store_durability_test.cc:2350 (ep-engine_ep_unit_tests+0x000000bd3642)\n\nThe crux of this issue seems to be that a background AuxIO task run\nvia the FollyExecutorPool is deleting a VBucket object concurrently\nwith the main thread deleting an EPStats object.\n\n+Background+\n\nDetails of how CB3ExecutorPool and FollyExecutorPool implement\n{{unregisterTaskable()}}, which I believe is what leads to this\nproblem.\n\nCB3ExecutorPool:\n\nDuring CB3ExecutorPool::unregisterTaskable():\n\n1. CB3ExecutorPool::_stopTaskGroup() is called and will wait for\n   VBucketMemoryDeletionTask to run.\n\n2. When VBucketMemoryDeletionTask::run() is call it returns false.\n\n3. CB3ExecutorThread will then synchronously call\n   CB3ExecutorThread::cancelCurrentTask() -\u003e\n   CB3ExecutorPool::cancel(). That removes all Cb3ExecutorPool-owned\n   references to task, and hence will run VBucketMemoryDeletionTask\n   dtor.\n\n4. VBucketMemoryDeletionTask dtor frees the VBucket object.\n\nAs such, by the time CB3ExecutorPool::unregisterTaskable() returns the\nVBucket is *guaranteed* to have been freed.\n\nFollyExecutorPool:\n\nDuring FollyExecutorPool::unregisterTaskable():\n\n1. All tasks scheduled to run in future (owned by IO thread EventBase)\n   are either cancelled (if allowed), or woken to run asap on CPU pool.\n\n2. All tasks waiting to, or currently running on CPU pool are waited\n   for by polling for taskOwners to no longer contain any tasks for\n   the given taskable.\n\n3. (On the CPU threads) Each queued task is run, on completion\n   rescheduleTaskAfterRun is called to add work to the IO thread\n   EventBase to decide when to reschedule, or (in this case) to\n   actualy cancel the task.\n\n4. (On the IO thread) FollyExecutorPool::rescheduleTaskAfterRun is\n   called, for cancelled tasks this calls State::cancelTask() which\n   removes the task from taskOwners - at which point TaskProxy dtor\n   runs, which schedules _another_ task on CPU pool to actually delete\n   the GlobalTask.\n\nThe problem here is that the TaskProxy is removed from taskOwners and\ndeletes at (4) on the IO thread; however the GlobalTask destruction is\ndeferred to later execution on a CPU thread. As such,\nFollyExecutorPool::unregisterTaskable() can see taskOwners having no\ntasks left for the taskable (and hence return) _before_ the VBucket\nobject is deleted.\n\nNote the deferred deletion at (5) was added to avoid potentially large\namounts of work being done on the IO thread - we aim to minimise work\ndone here as it can impact the scheduling of other tasks.\n\n+Solution+\n\nIf the TaskProxy removal from taskOwners is deferred until _after_ the\nGlobalTask shared ownership is released, then unregisterTaskable()\nwill no longer return until all GlobalTask references inside\nFollyExecutorPool have been released.\n\nTo achieve this changes the ownership model in FollyExecutorPool are\nneeded:\n\n1. Don\u0027t immediately remove TaskProxy from taskOwners in cancelTask().\n   Instead:\n   a) Mark it as cancelled, by setting the GlobalTask ptr to null,\n   b) Schedule an asynchronous task to release its GlobalTask\n      shared_ptr.\n\n2. This new async task (setup in resetTaskPtrViaCpuPool) releases the\n   TaskProxy\u0027s shared ownership on GlobalTask, then schedules an (IO\n   thread) completion task to finally remove the TaskProxy from\n   taskOwners.\n\nunregisterTaskable() is unchanged - it still waits for the taskOwner\nmap for the given Taskable become empty; however given the above\nchanges that only happens once the GlobalTask reference has been\nreleased.\n\nChange-Id: Iecbff9f3b45fc9e3d385c67f6a6dd32242dc76fe\n"},"parents_data":[{"branch_name":"refs/heads/master","commit_id":"6b0d3157a700b1a128f2a6296382e24dc190dc63","is_merged_in_target_branch":true,"change_id":"Ibdb64068625c205b3b5dba08e104733714006347","change_number":138038,"patch_set_number":3,"change_status":"MERGED"}],"branch":"refs/heads/master"},"42a28df6b8cf1805bc1b6c82d7c5895d933d28b4":{"kind":"REWORK","_number":8,"created":"2020-10-20 16:10:50.000000000","uploader":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"ref":"refs/changes/73/138373/8","fetch":{"anonymous http":{"url":"https://review.couchbase.org/kv_engine","ref":"refs/changes/73/138373/8","commands":{"Branch":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/8 \u0026\u0026 git checkout -b change-138373 FETCH_HEAD","Checkout":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/8 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/8 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/8 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.couchbase.org/kv_engine refs/changes/73/138373/8","Reset To":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/8 \u0026\u0026 git reset --hard FETCH_HEAD"}}},"commit":{"parents":[{"commit":"6b0d3157a700b1a128f2a6296382e24dc190dc63","subject":"Cleanup: Remove else after return"}],"author":{"name":"Dave Rigby","email":"daver@couchbase.com","date":"2020-09-17 14:58:51.000000000","tz":0},"committer":{"name":"Dave Rigby","email":"daver@couchbase.com","date":"2020-10-20 16:02:05.000000000","tz":60},"subject":"MB-42029: FollyExecPool: Wait for tasks cancelled in unregisterTaskable","message":"MB-42029: FollyExecPool: Wait for tasks cancelled in unregisterTaskable\n\n+Issue+\n\nWhen enabling FollyExecutorPool by default, TSan reports the following\nrace when running ./ep-engine_ep_unit_tests\n\"--gtest_filter\u003dDurabilityRespondAmbiguousTest.*\":\n\nAuxIO thread:\n    Previous atomic write of size 8 at 0x7b74000020a0 by thread T8:\n    #0 __tsan_atomic64_fetch_sub \u003cnull\u003e (libtsan.so.0+0x000000060890)\n    ...\n    #4 ~HashTable kv_engine/engines/ep/src/hash_table.cc:161 (ep-engine_ep_unit_tests+0x00000122cae1)\n    #5 ~VBucket kv_engine/engines/ep/src/vbucket.cc:286 (ep-engine_ep_unit_tests+0x0000012b3af4)\n    #6 ~EPVBucket kv_engine/engines/ep/src/ep_vb.cc:101 (ep-engine_ep_unit_tests+0x0000011af5e1)\n    ...\n    #10 ~VBucketMemoryDeletionTask kv_engine/engines/ep/src/vbucketdeletiontask.cc:45 (ep-engine_ep_unit_tests+0x0000012e4530)\n    ...\n    #17 std::__shared_ptr\u003cGlobalTask\u003e::reset() /usr/local/include/c++/7.3.0/bits/shared_ptr_base.h:1235 (ep-engine_ep_unit_tests+0x000001221e75)\n    #18 FollyExecutorPool::TaskProxy::~TaskProxy()::{lambda()#1}::operator()() kv_engine/engines/ep/src/folly_executorpool.cc:80 (ep-engine_ep_unit_tests+0x000001221e75)\n\nMain thread:\n    Write of size 8 at 0x7b74000020a0 by main thread:\n    #0 free \u003cnull\u003e (libtsan.so.0+0x000000027806)\n    ...\n    #6 CoreStore\u003c...\u003e::~CoreStore() platform/include/platform/corestore.h:50 (ep-engine_ep_unit_tests+0x0000012988b1)\n    #7 ~EPStats kv_engine/engines/ep/src/stats.cc:132 (ep-engine_ep_unit_tests+0x0000012988b1)\n    #8 ~EventuallyPersistentEngine kv_engine/engines/ep/src/ep_engine.cc:6593 (ep-engine_ep_unit_tests+0x0000011e3bb5)\n    ...\n    #12 DurabilityRespondAmbiguousTest_RespondAmbiguousNotificationDeadLock_Test::TestBody() kv_engine/engines/ep/tests/module_tests/evp_store_durability_test.cc:2350 (ep-engine_ep_unit_tests+0x000000bd3642)\n\nThe crux of this issue seems to be that a background AuxIO task run\nvia the FollyExecutorPool is deleting a VBucket object concurrently\nwith the main thread deleting an EPStats object.\n\n+Background+\n\nDetails of how CB3ExecutorPool and FollyExecutorPool implement\n{{unregisterTaskable()}}, which I believe is what leads to this\nproblem.\n\nCB3ExecutorPool:\n\nDuring CB3ExecutorPool::unregisterTaskable():\n\n1. CB3ExecutorPool::_stopTaskGroup() is called and will wait for\n   VBucketMemoryDeletionTask to run.\n\n2. When VBucketMemoryDeletionTask::run() is call it returns false.\n\n3. CB3ExecutorThread will then synchronously call\n   CB3ExecutorThread::cancelCurrentTask() -\u003e\n   CB3ExecutorPool::cancel(). That removes all Cb3ExecutorPool-owned\n   references to task, and hence will run VBucketMemoryDeletionTask\n   dtor.\n\n4. VBucketMemoryDeletionTask dtor frees the VBucket object.\n\nAs such, by the time CB3ExecutorPool::unregisterTaskable() returns the\nVBucket is *guaranteed* to have been freed.\n\nFollyExecutorPool:\n\nDuring FollyExecutorPool::unregisterTaskable():\n\n1. All tasks scheduled to run in future (owned by IO thread EventBase)\n   are either cancelled (if allowed), or woken to run asap on CPU pool.\n\n2. All tasks waiting to, or currently running on CPU pool are waited\n   for by polling for taskOwners to no longer contain any tasks for\n   the given taskable.\n\n3. (On the CPU threads) Each queued task is run, on completion\n   rescheduleTaskAfterRun is called to add work to the IO thread\n   EventBase to decide when to reschedule, or (in this case) to\n   actualy cancel the task.\n\n4. (On the IO thread) FollyExecutorPool::rescheduleTaskAfterRun is\n   called, for cancelled tasks this calls State::cancelTask() which\n   removes the task from taskOwners - at which point TaskProxy dtor\n   runs, which schedules _another_ task on CPU pool to actually delete\n   the GlobalTask.\n\nThe problem here is that the TaskProxy is removed from taskOwners and\ndeletes at (4) on the IO thread; however the GlobalTask destruction is\ndeferred to later execution on a CPU thread. As such,\nFollyExecutorPool::unregisterTaskable() can see taskOwners having no\ntasks left for the taskable (and hence return) _before_ the VBucket\nobject is deleted.\n\nNote the deferred deletion at (5) was added to avoid potentially large\namounts of work being done on the IO thread - we aim to minimise work\ndone here as it can impact the scheduling of other tasks.\n\n+Solution+\n\nIf the TaskProxy removal from taskOwners is deferred until _after_ the\nGlobalTask shared ownership is released, then unregisterTaskable()\nwill no longer return until all GlobalTask references inside\nFollyExecutorPool have been released.\n\nTo achieve this changes the ownership model in FollyExecutorPool are\nneeded:\n\n1. Don\u0027t immediately remove TaskProxy from taskOwners in cancelTask().\n   Instead:\n   a) Mark it as cancelled, by setting the GlobalTask ptr to null,\n   b) Schedule an asynchronous task to release its GlobalTask\n      shared_ptr.\n\n2. This new async task (setup in resetTaskPtrViaCpuPool) releases the\n   TaskProxy\u0027s shared ownership on GlobalTask, then schedules an (IO\n   thread) completion task to finally remove the TaskProxy from\n   taskOwners.\n\nunregisterTaskable() is unchanged - it still waits for the taskOwner\nmap for the given Taskable become empty; however given the above\nchanges that only happens once the GlobalTask reference has been\nreleased.\n\nChange-Id: Iecbff9f3b45fc9e3d385c67f6a6dd32242dc76fe\n"},"parents_data":[{"branch_name":"refs/heads/master","commit_id":"6b0d3157a700b1a128f2a6296382e24dc190dc63","is_merged_in_target_branch":true,"change_id":"Ibdb64068625c205b3b5dba08e104733714006347","change_number":138038,"patch_set_number":3,"change_status":"MERGED"}],"branch":"refs/heads/master"},"36bd1a847a625e7fd2ef3bd45aa0dbe373ad330f":{"kind":"TRIVIAL_REBASE","_number":9,"created":"2020-10-20 16:36:26.000000000","uploader":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"ref":"refs/changes/73/138373/9","fetch":{"anonymous http":{"url":"https://review.couchbase.org/kv_engine","ref":"refs/changes/73/138373/9","commands":{"Branch":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/9 \u0026\u0026 git checkout -b change-138373 FETCH_HEAD","Checkout":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/9 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/9 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/9 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.couchbase.org/kv_engine refs/changes/73/138373/9","Reset To":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/9 \u0026\u0026 git reset --hard FETCH_HEAD"}}},"commit":{"parents":[{"commit":"1cd0855ee1c43d1a20259271340241c071deb665","subject":"Optimize McdTestappTest/MB_10114 under TSan"}],"author":{"name":"Dave Rigby","email":"daver@couchbase.com","date":"2020-09-17 14:58:51.000000000","tz":0},"committer":{"name":"Dave Rigby","email":"daver@couchbase.com","date":"2020-10-20 16:32:07.000000000","tz":0},"subject":"MB-42029: FollyExecPool: Wait for tasks cancelled in unregisterTaskable","message":"MB-42029: FollyExecPool: Wait for tasks cancelled in unregisterTaskable\n\n+Issue+\n\nWhen enabling FollyExecutorPool by default, TSan reports the following\nrace when running ./ep-engine_ep_unit_tests\n\"--gtest_filter\u003dDurabilityRespondAmbiguousTest.*\":\n\nAuxIO thread:\n    Previous atomic write of size 8 at 0x7b74000020a0 by thread T8:\n    #0 __tsan_atomic64_fetch_sub \u003cnull\u003e (libtsan.so.0+0x000000060890)\n    ...\n    #4 ~HashTable kv_engine/engines/ep/src/hash_table.cc:161 (ep-engine_ep_unit_tests+0x00000122cae1)\n    #5 ~VBucket kv_engine/engines/ep/src/vbucket.cc:286 (ep-engine_ep_unit_tests+0x0000012b3af4)\n    #6 ~EPVBucket kv_engine/engines/ep/src/ep_vb.cc:101 (ep-engine_ep_unit_tests+0x0000011af5e1)\n    ...\n    #10 ~VBucketMemoryDeletionTask kv_engine/engines/ep/src/vbucketdeletiontask.cc:45 (ep-engine_ep_unit_tests+0x0000012e4530)\n    ...\n    #17 std::__shared_ptr\u003cGlobalTask\u003e::reset() /usr/local/include/c++/7.3.0/bits/shared_ptr_base.h:1235 (ep-engine_ep_unit_tests+0x000001221e75)\n    #18 FollyExecutorPool::TaskProxy::~TaskProxy()::{lambda()#1}::operator()() kv_engine/engines/ep/src/folly_executorpool.cc:80 (ep-engine_ep_unit_tests+0x000001221e75)\n\nMain thread:\n    Write of size 8 at 0x7b74000020a0 by main thread:\n    #0 free \u003cnull\u003e (libtsan.so.0+0x000000027806)\n    ...\n    #6 CoreStore\u003c...\u003e::~CoreStore() platform/include/platform/corestore.h:50 (ep-engine_ep_unit_tests+0x0000012988b1)\n    #7 ~EPStats kv_engine/engines/ep/src/stats.cc:132 (ep-engine_ep_unit_tests+0x0000012988b1)\n    #8 ~EventuallyPersistentEngine kv_engine/engines/ep/src/ep_engine.cc:6593 (ep-engine_ep_unit_tests+0x0000011e3bb5)\n    ...\n    #12 DurabilityRespondAmbiguousTest_RespondAmbiguousNotificationDeadLock_Test::TestBody() kv_engine/engines/ep/tests/module_tests/evp_store_durability_test.cc:2350 (ep-engine_ep_unit_tests+0x000000bd3642)\n\nThe crux of this issue seems to be that a background AuxIO task run\nvia the FollyExecutorPool is deleting a VBucket object concurrently\nwith the main thread deleting an EPStats object.\n\n+Background+\n\nDetails of how CB3ExecutorPool and FollyExecutorPool implement\n{{unregisterTaskable()}}, which I believe is what leads to this\nproblem.\n\nCB3ExecutorPool:\n\nDuring CB3ExecutorPool::unregisterTaskable():\n\n1. CB3ExecutorPool::_stopTaskGroup() is called and will wait for\n   VBucketMemoryDeletionTask to run.\n\n2. When VBucketMemoryDeletionTask::run() is call it returns false.\n\n3. CB3ExecutorThread will then synchronously call\n   CB3ExecutorThread::cancelCurrentTask() -\u003e\n   CB3ExecutorPool::cancel(). That removes all Cb3ExecutorPool-owned\n   references to task, and hence will run VBucketMemoryDeletionTask\n   dtor.\n\n4. VBucketMemoryDeletionTask dtor frees the VBucket object.\n\nAs such, by the time CB3ExecutorPool::unregisterTaskable() returns the\nVBucket is *guaranteed* to have been freed.\n\nFollyExecutorPool:\n\nDuring FollyExecutorPool::unregisterTaskable():\n\n1. All tasks scheduled to run in future (owned by IO thread EventBase)\n   are either cancelled (if allowed), or woken to run asap on CPU pool.\n\n2. All tasks waiting to, or currently running on CPU pool are waited\n   for by polling for taskOwners to no longer contain any tasks for\n   the given taskable.\n\n3. (On the CPU threads) Each queued task is run, on completion\n   rescheduleTaskAfterRun is called to add work to the IO thread\n   EventBase to decide when to reschedule, or (in this case) to\n   actualy cancel the task.\n\n4. (On the IO thread) FollyExecutorPool::rescheduleTaskAfterRun is\n   called, for cancelled tasks this calls State::cancelTask() which\n   removes the task from taskOwners - at which point TaskProxy dtor\n   runs, which schedules _another_ task on CPU pool to actually delete\n   the GlobalTask.\n\nThe problem here is that the TaskProxy is removed from taskOwners and\ndeletes at (4) on the IO thread; however the GlobalTask destruction is\ndeferred to later execution on a CPU thread. As such,\nFollyExecutorPool::unregisterTaskable() can see taskOwners having no\ntasks left for the taskable (and hence return) _before_ the VBucket\nobject is deleted.\n\nNote the deferred deletion at (5) was added to avoid potentially large\namounts of work being done on the IO thread - we aim to minimise work\ndone here as it can impact the scheduling of other tasks.\n\n+Solution+\n\nIf the TaskProxy removal from taskOwners is deferred until _after_ the\nGlobalTask shared ownership is released, then unregisterTaskable()\nwill no longer return until all GlobalTask references inside\nFollyExecutorPool have been released.\n\nTo achieve this changes the ownership model in FollyExecutorPool are\nneeded:\n\n1. Don\u0027t immediately remove TaskProxy from taskOwners in cancelTask().\n   Instead:\n   a) Mark it as cancelled, by setting the GlobalTask ptr to null,\n   b) Schedule an asynchronous task to release its GlobalTask\n      shared_ptr.\n\n2. This new async task (setup in resetTaskPtrViaCpuPool) releases the\n   TaskProxy\u0027s shared ownership on GlobalTask, then schedules an (IO\n   thread) completion task to finally remove the TaskProxy from\n   taskOwners.\n\nunregisterTaskable() is unchanged - it still waits for the taskOwner\nmap for the given Taskable become empty; however given the above\nchanges that only happens once the GlobalTask reference has been\nreleased.\n\nChange-Id: Iecbff9f3b45fc9e3d385c67f6a6dd32242dc76fe\n"},"parents_data":[{"branch_name":"refs/heads/master","commit_id":"1cd0855ee1c43d1a20259271340241c071deb665","is_merged_in_target_branch":false,"change_id":"I87fc2157c83ef0717378b627c6cb4ed5d0cd2ecb","change_number":138555,"patch_set_number":1,"change_status":"MERGED"}],"branch":"refs/heads/master"},"428c9c46d67e7b5048d6aaf198bc1a5873bf4513":{"kind":"TRIVIAL_REBASE","_number":10,"created":"2020-10-20 19:16:07.000000000","uploader":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"ref":"refs/changes/73/138373/10","fetch":{"anonymous http":{"url":"https://review.couchbase.org/kv_engine","ref":"refs/changes/73/138373/10","commands":{"Branch":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/10 \u0026\u0026 git checkout -b change-138373 FETCH_HEAD","Checkout":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/10 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/10 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/10 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.couchbase.org/kv_engine refs/changes/73/138373/10","Reset To":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/10 \u0026\u0026 git reset --hard FETCH_HEAD"}}},"commit":{"parents":[{"commit":"d335b8dbb7ab8b3a891d18277f48882326df05e3","subject":"Optimize McdTestappTest/MB_10114 under TSan"}],"author":{"name":"Dave Rigby","email":"daver@couchbase.com","date":"2020-09-17 14:58:51.000000000","tz":0},"committer":{"name":"Dave Rigby","email":"daver@couchbase.com","date":"2020-10-20 19:15:18.000000000","tz":0},"subject":"MB-42029: FollyExecPool: Wait for tasks cancelled in unregisterTaskable","message":"MB-42029: FollyExecPool: Wait for tasks cancelled in unregisterTaskable\n\n+Issue+\n\nWhen enabling FollyExecutorPool by default, TSan reports the following\nrace when running ./ep-engine_ep_unit_tests\n\"--gtest_filter\u003dDurabilityRespondAmbiguousTest.*\":\n\nAuxIO thread:\n    Previous atomic write of size 8 at 0x7b74000020a0 by thread T8:\n    #0 __tsan_atomic64_fetch_sub \u003cnull\u003e (libtsan.so.0+0x000000060890)\n    ...\n    #4 ~HashTable kv_engine/engines/ep/src/hash_table.cc:161 (ep-engine_ep_unit_tests+0x00000122cae1)\n    #5 ~VBucket kv_engine/engines/ep/src/vbucket.cc:286 (ep-engine_ep_unit_tests+0x0000012b3af4)\n    #6 ~EPVBucket kv_engine/engines/ep/src/ep_vb.cc:101 (ep-engine_ep_unit_tests+0x0000011af5e1)\n    ...\n    #10 ~VBucketMemoryDeletionTask kv_engine/engines/ep/src/vbucketdeletiontask.cc:45 (ep-engine_ep_unit_tests+0x0000012e4530)\n    ...\n    #17 std::__shared_ptr\u003cGlobalTask\u003e::reset() /usr/local/include/c++/7.3.0/bits/shared_ptr_base.h:1235 (ep-engine_ep_unit_tests+0x000001221e75)\n    #18 FollyExecutorPool::TaskProxy::~TaskProxy()::{lambda()#1}::operator()() kv_engine/engines/ep/src/folly_executorpool.cc:80 (ep-engine_ep_unit_tests+0x000001221e75)\n\nMain thread:\n    Write of size 8 at 0x7b74000020a0 by main thread:\n    #0 free \u003cnull\u003e (libtsan.so.0+0x000000027806)\n    ...\n    #6 CoreStore\u003c...\u003e::~CoreStore() platform/include/platform/corestore.h:50 (ep-engine_ep_unit_tests+0x0000012988b1)\n    #7 ~EPStats kv_engine/engines/ep/src/stats.cc:132 (ep-engine_ep_unit_tests+0x0000012988b1)\n    #8 ~EventuallyPersistentEngine kv_engine/engines/ep/src/ep_engine.cc:6593 (ep-engine_ep_unit_tests+0x0000011e3bb5)\n    ...\n    #12 DurabilityRespondAmbiguousTest_RespondAmbiguousNotificationDeadLock_Test::TestBody() kv_engine/engines/ep/tests/module_tests/evp_store_durability_test.cc:2350 (ep-engine_ep_unit_tests+0x000000bd3642)\n\nThe crux of this issue seems to be that a background AuxIO task run\nvia the FollyExecutorPool is deleting a VBucket object concurrently\nwith the main thread deleting an EPStats object.\n\n+Background+\n\nDetails of how CB3ExecutorPool and FollyExecutorPool implement\n{{unregisterTaskable()}}, which I believe is what leads to this\nproblem.\n\nCB3ExecutorPool:\n\nDuring CB3ExecutorPool::unregisterTaskable():\n\n1. CB3ExecutorPool::_stopTaskGroup() is called and will wait for\n   VBucketMemoryDeletionTask to run.\n\n2. When VBucketMemoryDeletionTask::run() is call it returns false.\n\n3. CB3ExecutorThread will then synchronously call\n   CB3ExecutorThread::cancelCurrentTask() -\u003e\n   CB3ExecutorPool::cancel(). That removes all Cb3ExecutorPool-owned\n   references to task, and hence will run VBucketMemoryDeletionTask\n   dtor.\n\n4. VBucketMemoryDeletionTask dtor frees the VBucket object.\n\nAs such, by the time CB3ExecutorPool::unregisterTaskable() returns the\nVBucket is *guaranteed* to have been freed.\n\nFollyExecutorPool:\n\nDuring FollyExecutorPool::unregisterTaskable():\n\n1. All tasks scheduled to run in future (owned by IO thread EventBase)\n   are either cancelled (if allowed), or woken to run asap on CPU pool.\n\n2. All tasks waiting to, or currently running on CPU pool are waited\n   for by polling for taskOwners to no longer contain any tasks for\n   the given taskable.\n\n3. (On the CPU threads) Each queued task is run, on completion\n   rescheduleTaskAfterRun is called to add work to the IO thread\n   EventBase to decide when to reschedule, or (in this case) to\n   actualy cancel the task.\n\n4. (On the IO thread) FollyExecutorPool::rescheduleTaskAfterRun is\n   called, for cancelled tasks this calls State::cancelTask() which\n   removes the task from taskOwners - at which point TaskProxy dtor\n   runs, which schedules _another_ task on CPU pool to actually delete\n   the GlobalTask.\n\nThe problem here is that the TaskProxy is removed from taskOwners and\ndeletes at (4) on the IO thread; however the GlobalTask destruction is\ndeferred to later execution on a CPU thread. As such,\nFollyExecutorPool::unregisterTaskable() can see taskOwners having no\ntasks left for the taskable (and hence return) _before_ the VBucket\nobject is deleted.\n\nNote the deferred deletion at (5) was added to avoid potentially large\namounts of work being done on the IO thread - we aim to minimise work\ndone here as it can impact the scheduling of other tasks.\n\n+Solution+\n\nIf the TaskProxy removal from taskOwners is deferred until _after_ the\nGlobalTask shared ownership is released, then unregisterTaskable()\nwill no longer return until all GlobalTask references inside\nFollyExecutorPool have been released.\n\nTo achieve this changes the ownership model in FollyExecutorPool are\nneeded:\n\n1. Don\u0027t immediately remove TaskProxy from taskOwners in cancelTask().\n   Instead:\n   a) Mark it as cancelled, by setting the GlobalTask ptr to null,\n   b) Schedule an asynchronous task to release its GlobalTask\n      shared_ptr.\n\n2. This new async task (setup in resetTaskPtrViaCpuPool) releases the\n   TaskProxy\u0027s shared ownership on GlobalTask, then schedules an (IO\n   thread) completion task to finally remove the TaskProxy from\n   taskOwners.\n\nunregisterTaskable() is unchanged - it still waits for the taskOwner\nmap for the given Taskable become empty; however given the above\nchanges that only happens once the GlobalTask reference has been\nreleased.\n\nChange-Id: Iecbff9f3b45fc9e3d385c67f6a6dd32242dc76fe\n"},"parents_data":[{"branch_name":"refs/heads/master","commit_id":"d335b8dbb7ab8b3a891d18277f48882326df05e3","is_merged_in_target_branch":false,"change_id":"I87fc2157c83ef0717378b627c6cb4ed5d0cd2ecb","change_number":138555,"patch_set_number":2,"change_status":"MERGED"}],"branch":"refs/heads/master"},"c370cd5ec3a018dccbf4857a97453b53d8bc6ec6":{"kind":"REWORK","_number":11,"created":"2020-10-23 10:00:50.000000000","uploader":{"_account_id":1000966,"name":"Dave Rigby","email":"daver@couchbase.com","username":"drigby","avatars":[{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/514e75a8d75cc1fcdb22433d445ae8f1.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"ref":"refs/changes/73/138373/11","fetch":{"anonymous http":{"url":"https://review.couchbase.org/kv_engine","ref":"refs/changes/73/138373/11","commands":{"Branch":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/11 \u0026\u0026 git checkout -b change-138373 FETCH_HEAD","Checkout":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/11 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/11 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/11 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.couchbase.org/kv_engine refs/changes/73/138373/11","Reset To":"git fetch https://review.couchbase.org/kv_engine refs/changes/73/138373/11 \u0026\u0026 git reset --hard FETCH_HEAD"}}},"commit":{"parents":[{"commit":"d541aaa9c9e6c44fdb004aec1578bfdcd10d4b47","subject":"MB-41747: Reset the cached couch handle"}],"author":{"name":"Dave Rigby","email":"daver@couchbase.com","date":"2020-09-17 14:58:51.000000000","tz":0},"committer":{"name":"Dave Rigby","email":"daver@couchbase.com","date":"2020-10-23 10:00:50.000000000","tz":0},"subject":"MB-42029: FollyExecPool: Wait for tasks cancelled in unregisterTaskable","message":"MB-42029: FollyExecPool: Wait for tasks cancelled in unregisterTaskable\n\n+Issue+\n\nWhen enabling FollyExecutorPool by default, TSan reports the following\nrace when running ./ep-engine_ep_unit_tests\n\"--gtest_filter\u003dDurabilityRespondAmbiguousTest.*\":\n\nAuxIO thread:\n    Previous atomic write of size 8 at 0x7b74000020a0 by thread T8:\n    #0 __tsan_atomic64_fetch_sub \u003cnull\u003e (libtsan.so.0+0x000000060890)\n    ...\n    #4 ~HashTable kv_engine/engines/ep/src/hash_table.cc:161 (ep-engine_ep_unit_tests+0x00000122cae1)\n    #5 ~VBucket kv_engine/engines/ep/src/vbucket.cc:286 (ep-engine_ep_unit_tests+0x0000012b3af4)\n    #6 ~EPVBucket kv_engine/engines/ep/src/ep_vb.cc:101 (ep-engine_ep_unit_tests+0x0000011af5e1)\n    ...\n    #10 ~VBucketMemoryDeletionTask kv_engine/engines/ep/src/vbucketdeletiontask.cc:45 (ep-engine_ep_unit_tests+0x0000012e4530)\n    ...\n    #17 std::__shared_ptr\u003cGlobalTask\u003e::reset() /usr/local/include/c++/7.3.0/bits/shared_ptr_base.h:1235 (ep-engine_ep_unit_tests+0x000001221e75)\n    #18 FollyExecutorPool::TaskProxy::~TaskProxy()::{lambda()#1}::operator()() kv_engine/engines/ep/src/folly_executorpool.cc:80 (ep-engine_ep_unit_tests+0x000001221e75)\n\nMain thread:\n    Write of size 8 at 0x7b74000020a0 by main thread:\n    #0 free \u003cnull\u003e (libtsan.so.0+0x000000027806)\n    ...\n    #6 CoreStore\u003c...\u003e::~CoreStore() platform/include/platform/corestore.h:50 (ep-engine_ep_unit_tests+0x0000012988b1)\n    #7 ~EPStats kv_engine/engines/ep/src/stats.cc:132 (ep-engine_ep_unit_tests+0x0000012988b1)\n    #8 ~EventuallyPersistentEngine kv_engine/engines/ep/src/ep_engine.cc:6593 (ep-engine_ep_unit_tests+0x0000011e3bb5)\n    ...\n    #12 DurabilityRespondAmbiguousTest_RespondAmbiguousNotificationDeadLock_Test::TestBody() kv_engine/engines/ep/tests/module_tests/evp_store_durability_test.cc:2350 (ep-engine_ep_unit_tests+0x000000bd3642)\n\nThe crux of this issue seems to be that a background AuxIO task run\nvia the FollyExecutorPool is deleting a VBucket object concurrently\nwith the main thread deleting an EPStats object.\n\n+Background+\n\nDetails of how CB3ExecutorPool and FollyExecutorPool implement\n{{unregisterTaskable()}}, which I believe is what leads to this\nproblem.\n\nCB3ExecutorPool:\n\nDuring CB3ExecutorPool::unregisterTaskable():\n\n1. CB3ExecutorPool::_stopTaskGroup() is called and will wait for\n   VBucketMemoryDeletionTask to run.\n\n2. When VBucketMemoryDeletionTask::run() is call it returns false.\n\n3. CB3ExecutorThread will then synchronously call\n   CB3ExecutorThread::cancelCurrentTask() -\u003e\n   CB3ExecutorPool::cancel(). That removes all Cb3ExecutorPool-owned\n   references to task, and hence will run VBucketMemoryDeletionTask\n   dtor.\n\n4. VBucketMemoryDeletionTask dtor frees the VBucket object.\n\nAs such, by the time CB3ExecutorPool::unregisterTaskable() returns the\nVBucket is *guaranteed* to have been freed.\n\nFollyExecutorPool:\n\nDuring FollyExecutorPool::unregisterTaskable():\n\n1. All tasks scheduled to run in future (owned by IO thread EventBase)\n   are either cancelled (if allowed), or woken to run asap on CPU pool.\n\n2. All tasks waiting to, or currently running on CPU pool are waited\n   for by polling for taskOwners to no longer contain any tasks for\n   the given taskable.\n\n3. (On the CPU threads) Each queued task is run, on completion\n   rescheduleTaskAfterRun is called to add work to the IO thread\n   EventBase to decide when to reschedule, or (in this case) to\n   actualy cancel the task.\n\n4. (On the IO thread) FollyExecutorPool::rescheduleTaskAfterRun is\n   called, for cancelled tasks this calls State::cancelTask() which\n   removes the task from taskOwners - at which point TaskProxy dtor\n   runs, which schedules _another_ task on CPU pool to actually delete\n   the GlobalTask.\n\nThe problem here is that the TaskProxy is removed from taskOwners and\ndeletes at (4) on the IO thread; however the GlobalTask destruction is\ndeferred to later execution on a CPU thread. As such,\nFollyExecutorPool::unregisterTaskable() can see taskOwners having no\ntasks left for the taskable (and hence return) _before_ the VBucket\nobject is deleted.\n\nNote the deferred deletion at (5) was added to avoid potentially large\namounts of work being done on the IO thread - we aim to minimise work\ndone here as it can impact the scheduling of other tasks.\n\n+Solution+\n\nIf the TaskProxy removal from taskOwners is deferred until _after_ the\nGlobalTask shared ownership is released, then unregisterTaskable()\nwill no longer return until all GlobalTask references inside\nFollyExecutorPool have been released.\n\nTo achieve this changes the ownership model in FollyExecutorPool are\nneeded:\n\n1. Don\u0027t immediately remove TaskProxy from taskOwners in cancelTask().\n   Instead:\n   a) Mark it as cancelled, by setting the GlobalTask ptr to null,\n   b) Schedule an asynchronous task to release its GlobalTask\n      shared_ptr.\n\n2. This new async task (setup in resetTaskPtrViaCpuPool) releases the\n   TaskProxy\u0027s shared ownership on GlobalTask, then schedules an (IO\n   thread) completion task to finally remove the TaskProxy from\n   taskOwners.\n\nunregisterTaskable() is unchanged - it still waits for the taskOwner\nmap for the given Taskable become empty; however given the above\nchanges that only happens once the GlobalTask reference has been\nreleased.\n\nChange-Id: Iecbff9f3b45fc9e3d385c67f6a6dd32242dc76fe\nReviewed-on: http://review.couchbase.org/c/kv_engine/+/138373\nTested-by: Build Bot \u003cbuild@couchbase.com\u003e\nReviewed-by: Jim Walker \u003cjim@couchbase.com\u003e\n"},"parents_data":[{"branch_name":"refs/heads/master","commit_id":"d541aaa9c9e6c44fdb004aec1578bfdcd10d4b47","is_merged_in_target_branch":true,"change_id":"I5f4809d9b5e266bb02a3c011e7316162201bab54","change_number":138740,"patch_set_number":3,"change_status":"MERGED"}],"branch":"refs/heads/master"}},"requirements":[],"submit_records":[{"status":"CLOSED","labels":[{"label":"Verified","status":"OK","applied_by":{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]}},{"label":"Code-Review","status":"OK","applied_by":{"_account_id":1001546,"name":"Jim Walker","email":"jim@couchbase.com","username":"jim","avatars":[{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/a06928c0be234da34bb6fc731e04a0d7.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]}},{"label":"Well-Formed","status":"MAY"}]},{"status":"CLOSED"}],"submit_requirements":[{"name":"Verified","status":"SATISFIED","is_legacy":true,"submittability_expression_result":{"expression":"label:Verified\u003dCustom-Rule","fulfilled":true,"status":"PASS","passing_atoms":["label:Verified\u003dCustom-Rule"],"failing_atoms":[]}},{"name":"Code-Review","status":"SATISFIED","is_legacy":true,"submittability_expression_result":{"expression":"label:Code-Review\u003dCustom-Rule","fulfilled":true,"status":"PASS","passing_atoms":["label:Code-Review\u003dCustom-Rule"],"failing_atoms":[]}},{"name":"Custom-Rule","status":"SATISFIED","is_legacy":true,"submittability_expression_result":{"expression":"rule:Custom-Rule","fulfilled":true,"status":"PASS","passing_atoms":["Custom-Rule"],"failing_atoms":[]}}]}