)]}'
{"id":"libcouchbase~244745","triplet_id":"libcouchbase~master~I83c8c95a1b279082cd750d8363664f14a78aa0c5","project":"libcouchbase","branch":"master","hashtags":[],"change_id":"I83c8c95a1b279082cd750d8363664f14a78aa0c5","subject":"CCBC-1702: requeue ops when vbmap briefly has no master","status":"MERGED","created":"2026-05-08 00:04:55.000000000","updated":"2026-05-11 20:03:10.000000000","submitted":"2026-05-11 20:03:10.000000000","submitter":{"_account_id":1000201,"name":"Sergey Avseyev","email":"sergey.avseyev@gmail.com","username":"avsej","avatars":[{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"total_comment_count":0,"unresolved_comment_count":0,"has_review_started":true,"submission_id":"244745","meta_rev_id":"3adc4647594165700e4c4023c81febc4dd14e636","_number":244745,"virtual_id_number":244745,"owner":{"_account_id":1000201,"name":"Sergey Avseyev","email":"sergey.avseyev@gmail.com","username":"avsej","avatars":[{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"actions":{},"labels":{"Verified":{"approved":{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"all":[{"tag":"autogenerated:gerrit:merged","value":1,"date":"2026-05-11 20:03:10.000000000","permitted_voting_range":{"min":1,"max":1},"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},{"value":0,"_account_id":1000201,"name":"Sergey Avseyev","email":"sergey.avseyev@gmail.com","username":"avsej","avatars":[{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]}],"values":{"-1":"Fails"," 0":"No score","+1":"Verified"},"description":"","default_value":0},"Code-Review":{"approved":{"_account_id":1000201,"name":"Sergey Avseyev","email":"sergey.avseyev@gmail.com","username":"avsej","avatars":[{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"all":[{"value":0,"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},{"tag":"autogenerated:gerrit:merged","value":2,"date":"2026-05-11 20:03:10.000000000","permitted_voting_range":{"min":2,"max":2},"_account_id":1000201,"name":"Sergey Avseyev","email":"sergey.avseyev@gmail.com","username":"avsej","avatars":[{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]}],"values":{"-2":"Do not submit","-1":"I would prefer that you didn\u0027t submit this"," 0":"No score","+1":"Looks good to me, but someone else must approve","+2":"Looks good to me, approved"},"description":"","default_value":0},"Well-Formed":{"all":[{"value":0,"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},{"value":0,"_account_id":1000201,"name":"Sergey Avseyev","email":"sergey.avseyev@gmail.com","username":"avsej","avatars":[{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]}],"values":{"-1":"Commit is not well-formed"," 0":"Well-formedness not checked","+1":"Commit is well-formed"},"description":"","default_value":0,"optional":true}},"removable_reviewers":[],"reviewers":{"REVIEWER":[{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},{"_account_id":1000201,"name":"Sergey Avseyev","email":"sergey.avseyev@gmail.com","username":"avsej","avatars":[{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]}]},"pending_reviewers":{},"reviewer_updates":[{"updated":"2026-05-08 00:05:05.000000000","updated_by":{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"reviewer":{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"state":"CC"},{"updated":"2026-05-08 01:28:02.000000000","updated_by":{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"reviewer":{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"state":"REVIEWER"}],"messages":[{"id":"d54c955117f88d1531d490f9c29c9629b3d87a67","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":1000201,"name":"Sergey Avseyev","email":"sergey.avseyev@gmail.com","username":"avsej","avatars":[{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"date":"2026-05-08 00:04:55.000000000","message":"Uploaded patch set 1.","accounts_in_message":[],"_revision_number":1},{"id":"712b954c0702eeca5ff838a053196c3cc6c32d45","author":{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"date":"2026-05-08 00:05:05.000000000","message":"Patch Set 1:\n\nBuild Started https://sdk.jenkins.couchbase.com/job/c-cpp/job/lcb/job/lcb-gerrit-trigger/2252/","accounts_in_message":[],"_revision_number":1},{"id":"41174c51848c347bec4a6da3b7c15396cb6375d1","author":{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"date":"2026-05-08 01:28:02.000000000","message":"Patch Set 1: Verified+1\n\nBuild Successful \n\nhttps://sdk.jenkins.couchbase.com/job/c-cpp/job/lcb/job/lcb-gerrit-trigger/2252/ : SUCCESS","accounts_in_message":[],"_revision_number":1},{"id":"26d0e16eea916733903ae5d4e64ce49a2c0a129f","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":1000201,"name":"Sergey Avseyev","email":"sergey.avseyev@gmail.com","username":"avsej","avatars":[{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"date":"2026-05-08 19:36:45.000000000","message":"Uploaded patch set 2.\n\nOutdated Votes:\n* Verified+1 (copy condition: \"changekind:NO_CHANGE OR changekind:NO_CODE_CHANGE\")\n","accounts_in_message":[],"_revision_number":2},{"id":"aa11584ad11a56583aaa13e5ba516fcfe819aede","author":{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"date":"2026-05-08 19:36:56.000000000","message":"Patch Set 2:\n\nBuild Started https://sdk.jenkins.couchbase.com/job/c-cpp/job/lcb/job/lcb-gerrit-trigger/2254/","accounts_in_message":[],"_revision_number":2},{"id":"4aef4770d5827cbba6cc728f64b1b4b793ff86f7","author":{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"date":"2026-05-08 21:00:05.000000000","message":"Patch Set 2: Verified+1\n\nBuild Successful \n\nhttps://sdk.jenkins.couchbase.com/job/c-cpp/job/lcb/job/lcb-gerrit-trigger/2254/ : SUCCESS","accounts_in_message":[],"_revision_number":2},{"id":"181afb5efcb07ad66f872f52f9be2ea7f1662e70","tag":"autogenerated:gerrit:newPatchSet","author":{"_account_id":1000201,"name":"Sergey Avseyev","email":"sergey.avseyev@gmail.com","username":"avsej","avatars":[{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"date":"2026-05-11 17:21:37.000000000","message":"Uploaded patch set 3.\n\nOutdated Votes:\n* Verified+1 (copy condition: \"changekind:NO_CHANGE OR changekind:NO_CODE_CHANGE\")\n","accounts_in_message":[],"_revision_number":3},{"id":"a6e3602b555439af6a2c28ccd580aeb296cc192d","author":{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"date":"2026-05-11 17:21:45.000000000","message":"Patch Set 3:\n\nBuild Started https://sdk.jenkins.couchbase.com/job/c-cpp/job/lcb/job/lcb-gerrit-trigger/2261/","accounts_in_message":[],"_revision_number":3},{"id":"eaec90dbad25ca23375be06f64dfe3d2c7f8df44","author":{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]},"date":"2026-05-11 18:55:43.000000000","message":"Patch Set 3: Verified+1\n\nBuild Successful \n\nhttps://sdk.jenkins.couchbase.com/job/c-cpp/job/lcb/job/lcb-gerrit-trigger/2261/ : SUCCESS","accounts_in_message":[],"_revision_number":3},{"id":"e18b0f8b099b6fd71fb9313ff439fd3fe951f6ec","author":{"_account_id":1000201,"name":"Sergey Avseyev","email":"sergey.avseyev@gmail.com","username":"avsej","avatars":[{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"date":"2026-05-11 20:03:06.000000000","message":"Patch Set 3: Code-Review+2","accounts_in_message":[],"_revision_number":3},{"id":"3adc4647594165700e4c4023c81febc4dd14e636","tag":"autogenerated:gerrit:merged","author":{"_account_id":1000201,"name":"Sergey Avseyev","email":"sergey.avseyev@gmail.com","username":"avsej","avatars":[{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"date":"2026-05-11 20:03:10.000000000","message":"Change has been successfully cherry-picked as faf66a339124fdb6d00e07b094326d3239feb419","accounts_in_message":[],"_revision_number":4}],"current_revision_number":4,"current_revision":"faf66a339124fdb6d00e07b094326d3239feb419","revisions":{"646314981b557f47646f6a81ebd43501545bcc3d":{"kind":"REWORK","_number":1,"created":"2026-05-08 00:04:55.000000000","uploader":{"_account_id":1000201,"name":"Sergey Avseyev","email":"sergey.avseyev@gmail.com","username":"avsej","avatars":[{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"ref":"refs/changes/45/244745/1","fetch":{"anonymous http":{"url":"https://review.couchbase.org/libcouchbase","ref":"refs/changes/45/244745/1","commands":{"Branch":"git fetch https://review.couchbase.org/libcouchbase refs/changes/45/244745/1 \u0026\u0026 git checkout -b change-244745 FETCH_HEAD","Checkout":"git fetch https://review.couchbase.org/libcouchbase refs/changes/45/244745/1 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.couchbase.org/libcouchbase refs/changes/45/244745/1 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.couchbase.org/libcouchbase refs/changes/45/244745/1 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.couchbase.org/libcouchbase refs/changes/45/244745/1","Reset To":"git fetch https://review.couchbase.org/libcouchbase refs/changes/45/244745/1 \u0026\u0026 git reset --hard FETCH_HEAD"}}},"commit":{"parents":[{"commit":"3a66a6e38b4161b29bb4a4259300fc7aa4c8165a","subject":"CCBC-1699: TLS + rebalance/failover: fix UAF crash"}],"author":{"name":"Sergey Avseyev","email":"sergey.avseyev@gmail.com","date":"2026-05-08 00:02:37.000000000","tz":-420},"committer":{"name":"Sergey Avseyev","email":"sergey.avseyev@gmail.com","date":"2026-05-08 00:04:39.000000000","tz":-420},"subject":"CCBC-1702: requeue ops when vbmap briefly has no master","message":"CCBC-1702: requeue ops when vbmap briefly has no master\n\nDuring replace_config(), mcreq_queue_take_pipelines() blanks\ncq-\u003enpipelines to 0 before mcreq_queue_add_pipelines() reinstalls the\nnew array. Any retryq tick that fires inside that window sees\nsrvix \u003e\u003d cq-\u003enpipelines for every op and -- with the historical\ndefault of LCB_RETRY_ON_MISSINGNODE \u003d 0 -- fails the op with\nLCB_ERR_NO_MATCHING_SERVER even though a healthy map is about to be\ninstalled. The same code path also fires legitimately when a vbucket\nhas no master mapped immediately after a failover, before a replica\nis promoted.\n\nThis is not a CBS 6.0-specific bug. The race exists on every server\nversion. CBS 6.0 just exposes it more reliably because no\nPROTOCOL_BINARY_FEATURE_DUPLEX means no server-driven config push,\nno idle inbound read watcher, and config arrives later via bgpoll\n(default 2.5 s) -- giving retryq more chances to fire inside the\nreplace window. On newer servers the duplex push path drives a\nsingle atomic config replace from the read callback, hitting the\nsame race far less often.\n\nEmpirical confirmation from SDKD situational tests vs CBS 6.0.5 + TLS:\nzero LCB_ERR_TIMEOUT errors observed across both build-1515 baseline\n(3a66a6e3) and build-1522 (CCBC-1701 keepalive on top), confirming\nthe timeout-detection hypothesis was off-target. Errors are dominated\nby LCB_ERR_NO_MATCHING_SERVER from this code path and downstream\nLCB_ERR_DOCUMENT_NOT_FOUND from retried ops landing on a\npartially-converged map.\n\nFix: flip the default of LCB_RETRY_ON_MISSINGNODE from 0 to\nLCB_RETRY_CMDS_ALL. Packets whose vbucket has no mapped master are\nrequeued back into the retry queue rather than failed immediately;\nthe op deadline (default 2.5 s) still bounds how long we keep\nretrying. If the map never recovers within the deadline, the op\nfails with the original error (preserved as origerr) -- same\nuser-visible code, just not from a scheduling coincidence.\n\nThe opt-out path is unchanged: callers who want the old fail-fast\nbehavior can set LCB_CNTL_RETRYMODE / connection-string\n\"retry\u003dmissingnode\u003d0\".\n\nTests\n-----\n\ntests/iotests/t_netfail.cc::testRetryOnMissingNodeAfterMapRepair\nexercises the failure path without iptables. Stops the config\nmonitor, sets vbuckets[vb].servers[0] \u003d -1 to force lcbvb_vbmaster()\nto return -1 (the precise condition retryq.cc:264 trips on),\nschedules a 200 ms timer that restores the master, and issues an\nlcb_get with a 2 s deadline. Verified to fail on gerrit/master with\nLCB_ERR_NO_MATCHING_SERVER in \u003c500 ms; passes with the new default\nbecause the op stays in retryq across ticks and dispatches once the\nmap is repaired.\n\nThe pre-existing testNegativeIndex still passes -- the op now waits\nthe full 500 ms op_timeout instead of failing fast, but the assertion\n(LCB_ERR_NO_MATCHING_SERVER, preserved as origerr) is unchanged.\n\nChange-Id: I83c8c95a1b279082cd750d8363664f14a78aa0c5\n"},"parents_data":[{"branch_name":"refs/heads/master","commit_id":"3a66a6e38b4161b29bb4a4259300fc7aa4c8165a","is_merged_in_target_branch":true,"change_id":"I92f2ecef0f5cca170874c524457f49c38b8525b8","change_number":244520,"patch_set_number":3,"change_status":"MERGED"}],"branch":"refs/heads/master"},"6add39e6881129eab7cf8c93c48638e083e0b4a6":{"kind":"REWORK","_number":2,"created":"2026-05-08 19:36:45.000000000","uploader":{"_account_id":1000201,"name":"Sergey Avseyev","email":"sergey.avseyev@gmail.com","username":"avsej","avatars":[{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"ref":"refs/changes/45/244745/2","fetch":{"anonymous http":{"url":"https://review.couchbase.org/libcouchbase","ref":"refs/changes/45/244745/2","commands":{"Branch":"git fetch https://review.couchbase.org/libcouchbase refs/changes/45/244745/2 \u0026\u0026 git checkout -b change-244745 FETCH_HEAD","Checkout":"git fetch https://review.couchbase.org/libcouchbase refs/changes/45/244745/2 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.couchbase.org/libcouchbase refs/changes/45/244745/2 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.couchbase.org/libcouchbase refs/changes/45/244745/2 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.couchbase.org/libcouchbase refs/changes/45/244745/2","Reset To":"git fetch https://review.couchbase.org/libcouchbase refs/changes/45/244745/2 \u0026\u0026 git reset --hard FETCH_HEAD"}}},"commit":{"parents":[{"commit":"3a66a6e38b4161b29bb4a4259300fc7aa4c8165a","subject":"CCBC-1699: TLS + rebalance/failover: fix UAF crash"}],"author":{"name":"Sergey Avseyev","email":"sergey.avseyev@gmail.com","date":"2026-05-08 00:02:37.000000000","tz":-420},"committer":{"name":"Sergey Avseyev","email":"sergey.avseyev@gmail.com","date":"2026-05-08 19:35:30.000000000","tz":-420},"subject":"CCBC-1702: requeue ops when vbmap briefly has no master","message":"CCBC-1702: requeue ops when vbmap briefly has no master\n\nThe historical default LCB_RETRY_ON_MISSINGNODE \u003d 0 made retryq fail\nops with LCB_ERR_NO_MATCHING_SERVER whenever lcbvb_vbmaster() returned\n-1 or returned an index \u003e\u003d cq-\u003enpipelines for the requested vbucket.\nThat happens whenever the cluster\u0027s view of the vbmap is in transition\n-- e.g., immediately after a failover, before a replica is promoted,\nor during the brief window when the cmdq is in the middle of swapping\nin a new pipeline array via replace_config().\n\nThis is not a CBS 6.0-specific bug. The race exists on every server\nversion. CBS 6.0 just exposes it more reliably because no\nPROTOCOL_BINARY_FEATURE_DUPLEX means no server-driven config push, no\nidle inbound read watcher, and config arrives later via bgpoll\n(default 2.5 s) -- giving retryq more chances to fire inside the\nreplace window. On newer servers the duplex push path drives a single\natomic config replace from the read callback, hitting the same race\nfar less often.\n\nEmpirical confirmation from SDKD situational tests vs CBS 6.0.5 + TLS:\nzero LCB_ERR_TIMEOUT errors observed across the baseline (build-1515)\nor the CCBC-1701 keepalive build (build-1522), confirming the\ntimeout-detection hypothesis was off-target. Errors are dominated by\nLCB_ERR_NO_MATCHING_SERVER from this code path and downstream\nLCB_ERR_DOCUMENT_NOT_FOUND from retried ops landing on a\npartially-converged map. With the fixes here, build-1523 reports\nNO_MATCHING_SERVER count 363 -\u003e 0 across the entire suite, and\nLCB_ERR_DOCUMENT_NOT_FOUND drops from 260 to 239.\n\nFixes applied\n-------------\n\n1) Flip LCB_RETRY_ON_MISSINGNODE default from 0 to LCB_RETRY_CMDS_ALL.\n   Packets whose vbucket has no mapped master are requeued back into\n   the retry queue rather than failed immediately; the op deadline\n   (default 2.5 s) still bounds how long we keep retrying. If the map\n   never recovers within the deadline, the op fails with the original\n   error (preserved as origerr) -- same user-visible code, just not\n   from a scheduling coincidence. Opt-out is unchanged via cntl /\n   connection-string \\\"retry\u003dmissingnode\u003d0\\\".\n\n2) Hold a ConfigInfo ref across Server::handle_nmv. The pre-fix code\n   read instance-\u003ecmdq.config (a raw lcbvb_CONFIG*) without a\n   refcount, so cccp_update() inside the handler -- or any other\n   config-replace path that fires in a nested event-loop frame --\n   could decref the old ConfigInfo and free its lcbvb_CONFIG while\n   lcbvb_nmv_remap_ex() was mid-deref. Build-1523 hit this as a\n   SIGSEGV in lcbvb_nmv_remap_ex during FoRecoverDelta scenarios when\n   the (1) requeue change kept ops in flight long enough to reach\n   handle_nmv across a config replacement. The latent UAF was\n   reachable on master too, just rarer; the requeue fix surfaced it.\n   Hold the ref at function entry, decref on every return path.\n\n3) Defensive zeroing in lcbvb_destroy(). After freeing the contained\n   pointers, NULL them out before freeing the struct. Any remaining\n   latent UAF on lcbvb_CONFIG (through cmdq.config, a captured\n   lcbvb_CONFIG*, or a stale Server) now NULL-derefs deterministically\n   at the offending field instead of reading whatever the next\n   allocator hands out. Cheap insurance.\n\n4) Atomic swap in replace_config(). Replaced the\n   take_pipelines/build/add_pipelines pattern -- which left the cmdq\n   with pipelines\u003dNULL and npipelines\u003d0 across an arbitrary number of\n   synchronous Server constructor allocs -- with a single tight swap\n   block at the end of replace_config(). cq-\u003epipelines, cq-\u003enpipelines,\n   cq-\u003e_npipelines_ex, cq-\u003escheds, and cq-\u003econfig are now updated as\n   one sequence; the cmdq is never observably in a half-installed\n   state. This is structural defense-in-depth: while the old window\n   was not directly observable to retryq (single-threaded event loop),\n   any future synchronous reader added inside replace_config() would\n   no longer see an inconsistent cmdq.\n\nTests\n-----\n\ntests/iotests/t_netfail.cc adds two cases. Both reproduce the failure\nmode without iptables.\n\ntestRetryOnMissingNodeAfterMapRepair: stops the config monitor, sets\nvbuckets[vb].servers[0] \u003d -1 to force lcbvb_vbmaster() to return -1\n(the precise condition retryq.cc:264 trips on), schedules a 200 ms\ntimer that restores the master in place, and issues an lcb_get with\na 2 s deadline. On gerrit/master (MISSINGNODE \u003d 0) retryq fails the\nop at the first 10 ms tick with LCB_ERR_NO_MATCHING_SERVER. With (1)\napplied, the op stays in retryq across ticks; once the timer\nrestores the map, the next tick dispatches it on the correct\npipeline and the GET succeeds.\n\ntestConfigReplaceMidRetry: same setup, but instead of restoring the\nlive vbc in place, snapshots the vbc to JSON via lcbvb_save_json(),\nloads it into a brand-new lcbvb_CONFIG, wraps it in a ConfigInfo, and\ncalls lcb_update_vbconfig() from the timer. This exercises the full\nreplace_config() path: cur_configinfo is swapped, replace_config()\nmoves pipelines, the old ConfigInfo is decref\u0027d, and the old\nlcbvb_CONFIG is freed via lcbvb_destroy(). Without (4) the cmdq\nwould have been visibly inconsistent during the swap; without (2)\nand (3) any captured raw lcbvb_CONFIG* across the call would UAF on\nthe freed memory. With all four fixes, the post-swap retryq tick\ndispatches on the new pipelines and the GET succeeds.\n\nThe pre-existing testNegativeIndex still passes; the op now waits\nthe full 500 ms op_timeout before failing with NO_MATCHING_SERVER\n(preserved as origerr) instead of failing fast.\n\nVerified locally on libev/libevent IO plugins: full unit-tests suite\n(194/194, modulo 1 environment-only Behavior.PluginDefaults skip)\ngreen.\n\nFiles touched: src/settings.{cc,h}, src/mcserver/mcserver.cc,\nsrc/newconfig.cc, src/vbucket/vbucket.c, tests/iotests/t_netfail.cc.\n\nChange-Id: I83c8c95a1b279082cd750d8363664f14a78aa0c5\n"},"parents_data":[{"branch_name":"refs/heads/master","commit_id":"3a66a6e38b4161b29bb4a4259300fc7aa4c8165a","is_merged_in_target_branch":true,"change_id":"I92f2ecef0f5cca170874c524457f49c38b8525b8","change_number":244520,"patch_set_number":3,"change_status":"MERGED"}],"branch":"refs/heads/master"},"b76aa893adfae576421b2f9c5f28a000278fd3cb":{"kind":"REWORK","_number":3,"created":"2026-05-11 17:21:37.000000000","uploader":{"_account_id":1000201,"name":"Sergey Avseyev","email":"sergey.avseyev@gmail.com","username":"avsej","avatars":[{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"ref":"refs/changes/45/244745/3","fetch":{"anonymous http":{"url":"https://review.couchbase.org/libcouchbase","ref":"refs/changes/45/244745/3","commands":{"Branch":"git fetch https://review.couchbase.org/libcouchbase refs/changes/45/244745/3 \u0026\u0026 git checkout -b change-244745 FETCH_HEAD","Checkout":"git fetch https://review.couchbase.org/libcouchbase refs/changes/45/244745/3 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.couchbase.org/libcouchbase refs/changes/45/244745/3 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.couchbase.org/libcouchbase refs/changes/45/244745/3 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.couchbase.org/libcouchbase refs/changes/45/244745/3","Reset To":"git fetch https://review.couchbase.org/libcouchbase refs/changes/45/244745/3 \u0026\u0026 git reset --hard FETCH_HEAD"}}},"commit":{"parents":[{"commit":"3a66a6e38b4161b29bb4a4259300fc7aa4c8165a","subject":"CCBC-1699: TLS + rebalance/failover: fix UAF crash"}],"author":{"name":"Sergey Avseyev","email":"sergey.avseyev@gmail.com","date":"2026-05-08 00:02:37.000000000","tz":-420},"committer":{"name":"Sergey Avseyev","email":"sergey.avseyev@gmail.com","date":"2026-05-11 17:21:21.000000000","tz":-420},"subject":"CCBC-1702: requeue ops when vbmap briefly has no master","message":"CCBC-1702: requeue ops when vbmap briefly has no master\n\nThe historical default LCB_RETRY_ON_MISSINGNODE \u003d 0 made retryq fail\nops with LCB_ERR_NO_MATCHING_SERVER whenever lcbvb_vbmaster() returned\n-1 or returned an index \u003e\u003d cq-\u003enpipelines for the requested vbucket.\nThat happens whenever the cluster\u0027s view of the vbmap is in transition\n-- e.g., immediately after a failover, before a replica is promoted,\nor during the brief window when the cmdq is in the middle of swapping\nin a new pipeline array via replace_config().\n\nThis is not a CBS 6.0-specific bug. The race exists on every server\nversion. CBS 6.0 just exposes it more reliably because no\nPROTOCOL_BINARY_FEATURE_DUPLEX means no server-driven config push, no\nidle inbound read watcher, and config arrives later via bgpoll\n(default 2.5 s) -- giving retryq more chances to fire inside the\nreplace window. On newer servers the duplex push path drives a single\natomic config replace from the read callback, hitting the same race\nfar less often.\n\nEmpirical confirmation from SDKD situational tests vs CBS 6.0.5 + TLS:\nzero LCB_ERR_TIMEOUT errors observed across the baseline (build-1515)\nor the CCBC-1701 keepalive build (build-1522), confirming the\ntimeout-detection hypothesis was off-target. Errors are dominated by\nLCB_ERR_NO_MATCHING_SERVER from this code path and downstream\nLCB_ERR_DOCUMENT_NOT_FOUND from retried ops landing on a\npartially-converged map. With the fixes here, build-1523 reports\nNO_MATCHING_SERVER count 363 -\u003e 0 across the entire suite.\n\nFixes applied\n-------------\n\n1) Flip LCB_RETRY_ON_MISSINGNODE default from 0 to LCB_RETRY_CMDS_ALL.\n   Packets whose vbucket has no mapped master are requeued back into\n   the retry queue rather than failed immediately; the op deadline\n   (default 2.5 s) still bounds how long we keep retrying. If the map\n   never recovers within the deadline, the op fails with the original\n   error (preserved as origerr) -- same user-visible code, just not\n   from a scheduling coincidence. Opt-out is unchanged via cntl /\n   connection-string \"retry\u003dmissingnode\u003d0\".\n\n2) Hold a ConfigInfo ref across Server::handle_nmv. The pre-fix code\n   read instance-\u003ecmdq.config (a raw lcbvb_CONFIG*) without a\n   refcount, so cccp_update() inside the handler -- or any other\n   config-replace path that fires in a nested event-loop frame --\n   could decref the old ConfigInfo and free its lcbvb_CONFIG while\n   lcbvb_nmv_remap_ex() was mid-deref. Build-1523 hit this as a\n   SIGSEGV in lcbvb_nmv_remap_ex during FoRecoverDelta scenarios when\n   the (1) requeue change kept ops in flight long enough to reach\n   handle_nmv across a config replacement. The latent UAF was\n   reachable on master too, just rarer; the requeue fix surfaced it.\n   Hold the ref at function entry, decref on every return path.\n\n3) Defensive zeroing in lcbvb_destroy(). After freeing the contained\n   pointers, NULL them out before freeing the struct. Any remaining\n   latent UAF on lcbvb_CONFIG (through cmdq.config, a captured\n   lcbvb_CONFIG*, or a stale Server) now NULL-derefs deterministically\n   at the offending field instead of reading whatever the next\n   allocator hands out. Cheap insurance.\n\n4) Atomic swap in replace_config(). Replaced the\n   take_pipelines/build/add_pipelines pattern -- which left the cmdq\n   with pipelines\u003dNULL and npipelines\u003d0 across an arbitrary number of\n   synchronous Server constructor allocs -- with a single tight swap\n   block at the end of replace_config(). cq-\u003epipelines, cq-\u003enpipelines,\n   cq-\u003e_npipelines_ex, cq-\u003escheds, and cq-\u003econfig are now updated as\n   one sequence; the cmdq is never observably in a half-installed\n   state. This is structural defense-in-depth.\n\n5) Bounds guard in lcb_vbguess_remap(). After PS2 still reproduced\n   exactly one SIGSEGV in FoRecoverDelta-SUBDOC at the first deref of\n   cfg-\u003evbuckets[vbid] (build-1524), with cfg \u003d LCBT_VBCONFIG(instance)\n   pointing at a freed-and-poisoned lcbvb_CONFIG (vbuckets\u003dNULL via\n   (3)). The handle_nmv ref guard in (2) pins cur_configinfo but the\n   deref is on cmdq.config; there is at least one SUBDOC code path\n   where the two diverge or where cmdq.config briefly points at a\n   stale vbc. Until that path is fully traced, gate the deref:\n   reject NULL cfg, NULL cfg-\u003evbuckets, or vbid out of [0, cfg-\u003envb).\n   On reject, the op falls through lcb_kv_should_retry -\u003e\n   mcreq_renew_packet -\u003e retryq-\u003enmvadd, which is the same path as a\n   legitimate \"no remap currently available\" and waits for the next\n   config to arrive. Logged at WARN so the rejection is visible in\n   production logs.\n\nTests\n-----\n\ntests/iotests/t_netfail.cc adds two cases. Both reproduce the failure\nmode without iptables.\n\ntestRetryOnMissingNodeAfterMapRepair: stops the config monitor, sets\nvbuckets[vb].servers[0] \u003d -1 to force lcbvb_vbmaster() to return -1\n(the precise condition retryq.cc:264 trips on), schedules a 200 ms\ntimer that restores the master in place, and issues an lcb_get with\na 2 s deadline. On gerrit/master (MISSINGNODE \u003d 0) retryq fails the\nop at the first 10 ms tick with LCB_ERR_NO_MATCHING_SERVER. With (1)\napplied, the op stays in retryq across ticks; once the timer\nrestores the map, the next tick dispatches it on the correct\npipeline and the GET succeeds.\n\ntestConfigReplaceMidRetry: same setup, but instead of restoring the\nlive vbc in place, snapshots the vbc to JSON via lcbvb_save_json(),\nloads it into a brand-new lcbvb_CONFIG, wraps it in a ConfigInfo, and\ncalls lcb_update_vbconfig() from the timer. This exercises the full\nreplace_config() path: cur_configinfo is swapped, replace_config()\nmoves pipelines, the old ConfigInfo is decref\u0027d, and the old\nlcbvb_CONFIG is freed via lcbvb_destroy(). Without (4) the cmdq\nwould have been visibly inconsistent during the swap; without (2)\nand (3) any captured raw lcbvb_CONFIG* across the call would UAF on\nthe freed memory. With all five fixes, the post-swap retryq tick\ndispatches on the new pipelines and the GET succeeds.\n\nThe pre-existing testNegativeIndex still passes; the op now waits\nthe full 500 ms op_timeout before failing with NO_MATCHING_SERVER\n(preserved as origerr) instead of failing fast.\n\nVerified locally on libev/libevent IO plugins: full unit-tests suite\n(194/194, modulo 1 environment-only Behavior.PluginDefaults skip)\ngreen.\n\nFiles touched: src/settings.{cc,h}, src/mcserver/mcserver.cc,\nsrc/newconfig.cc, src/vbucket/vbucket.c, tests/iotests/t_netfail.cc.\n\nChange-Id: I83c8c95a1b279082cd750d8363664f14a78aa0c5\n"},"parents_data":[{"branch_name":"refs/heads/master","commit_id":"3a66a6e38b4161b29bb4a4259300fc7aa4c8165a","is_merged_in_target_branch":true,"change_id":"I92f2ecef0f5cca170874c524457f49c38b8525b8","change_number":244520,"patch_set_number":3,"change_status":"MERGED"}],"branch":"refs/heads/master"},"faf66a339124fdb6d00e07b094326d3239feb419":{"kind":"NO_CODE_CHANGE","_number":4,"created":"2026-05-11 20:03:10.000000000","uploader":{"_account_id":1000201,"name":"Sergey Avseyev","email":"sergey.avseyev@gmail.com","username":"avsej","avatars":[{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]},"ref":"refs/changes/45/244745/4","fetch":{"anonymous http":{"url":"https://review.couchbase.org/libcouchbase","ref":"refs/changes/45/244745/4","commands":{"Branch":"git fetch https://review.couchbase.org/libcouchbase refs/changes/45/244745/4 \u0026\u0026 git checkout -b change-244745 FETCH_HEAD","Checkout":"git fetch https://review.couchbase.org/libcouchbase refs/changes/45/244745/4 \u0026\u0026 git checkout FETCH_HEAD","Cherry Pick":"git fetch https://review.couchbase.org/libcouchbase refs/changes/45/244745/4 \u0026\u0026 git cherry-pick FETCH_HEAD","Format Patch":"git fetch https://review.couchbase.org/libcouchbase refs/changes/45/244745/4 \u0026\u0026 git format-patch -1 --stdout FETCH_HEAD","Pull":"git pull https://review.couchbase.org/libcouchbase refs/changes/45/244745/4","Reset To":"git fetch https://review.couchbase.org/libcouchbase refs/changes/45/244745/4 \u0026\u0026 git reset --hard FETCH_HEAD"}}},"commit":{"parents":[{"commit":"3a66a6e38b4161b29bb4a4259300fc7aa4c8165a","subject":"CCBC-1699: TLS + rebalance/failover: fix UAF crash"}],"author":{"name":"Sergey Avseyev","email":"sergey.avseyev@gmail.com","date":"2026-05-08 00:02:37.000000000","tz":-420},"committer":{"name":"Sergey Avseyev","email":"sergey.avseyev@gmail.com","date":"2026-05-11 20:03:10.000000000","tz":0},"subject":"CCBC-1702: requeue ops when vbmap briefly has no master","message":"CCBC-1702: requeue ops when vbmap briefly has no master\n\nThe historical default LCB_RETRY_ON_MISSINGNODE \u003d 0 made retryq fail\nops with LCB_ERR_NO_MATCHING_SERVER whenever lcbvb_vbmaster() returned\n-1 or returned an index \u003e\u003d cq-\u003enpipelines for the requested vbucket.\nThat happens whenever the cluster\u0027s view of the vbmap is in transition\n-- e.g., immediately after a failover, before a replica is promoted,\nor during the brief window when the cmdq is in the middle of swapping\nin a new pipeline array via replace_config().\n\nThis is not a CBS 6.0-specific bug. The race exists on every server\nversion. CBS 6.0 just exposes it more reliably because no\nPROTOCOL_BINARY_FEATURE_DUPLEX means no server-driven config push, no\nidle inbound read watcher, and config arrives later via bgpoll\n(default 2.5 s) -- giving retryq more chances to fire inside the\nreplace window. On newer servers the duplex push path drives a single\natomic config replace from the read callback, hitting the same race\nfar less often.\n\nEmpirical confirmation from SDKD situational tests vs CBS 6.0.5 + TLS:\nzero LCB_ERR_TIMEOUT errors observed across the baseline (build-1515)\nor the CCBC-1701 keepalive build (build-1522), confirming the\ntimeout-detection hypothesis was off-target. Errors are dominated by\nLCB_ERR_NO_MATCHING_SERVER from this code path and downstream\nLCB_ERR_DOCUMENT_NOT_FOUND from retried ops landing on a\npartially-converged map. With the fixes here, build-1523 reports\nNO_MATCHING_SERVER count 363 -\u003e 0 across the entire suite.\n\nFixes applied\n-------------\n\n1) Flip LCB_RETRY_ON_MISSINGNODE default from 0 to LCB_RETRY_CMDS_ALL.\n   Packets whose vbucket has no mapped master are requeued back into\n   the retry queue rather than failed immediately; the op deadline\n   (default 2.5 s) still bounds how long we keep retrying. If the map\n   never recovers within the deadline, the op fails with the original\n   error (preserved as origerr) -- same user-visible code, just not\n   from a scheduling coincidence. Opt-out is unchanged via cntl /\n   connection-string \"retry\u003dmissingnode\u003d0\".\n\n2) Hold a ConfigInfo ref across Server::handle_nmv. The pre-fix code\n   read instance-\u003ecmdq.config (a raw lcbvb_CONFIG*) without a\n   refcount, so cccp_update() inside the handler -- or any other\n   config-replace path that fires in a nested event-loop frame --\n   could decref the old ConfigInfo and free its lcbvb_CONFIG while\n   lcbvb_nmv_remap_ex() was mid-deref. Build-1523 hit this as a\n   SIGSEGV in lcbvb_nmv_remap_ex during FoRecoverDelta scenarios when\n   the (1) requeue change kept ops in flight long enough to reach\n   handle_nmv across a config replacement. The latent UAF was\n   reachable on master too, just rarer; the requeue fix surfaced it.\n   Hold the ref at function entry, decref on every return path.\n\n3) Defensive zeroing in lcbvb_destroy(). After freeing the contained\n   pointers, NULL them out before freeing the struct. Any remaining\n   latent UAF on lcbvb_CONFIG (through cmdq.config, a captured\n   lcbvb_CONFIG*, or a stale Server) now NULL-derefs deterministically\n   at the offending field instead of reading whatever the next\n   allocator hands out. Cheap insurance.\n\n4) Atomic swap in replace_config(). Replaced the\n   take_pipelines/build/add_pipelines pattern -- which left the cmdq\n   with pipelines\u003dNULL and npipelines\u003d0 across an arbitrary number of\n   synchronous Server constructor allocs -- with a single tight swap\n   block at the end of replace_config(). cq-\u003epipelines, cq-\u003enpipelines,\n   cq-\u003e_npipelines_ex, cq-\u003escheds, and cq-\u003econfig are now updated as\n   one sequence; the cmdq is never observably in a half-installed\n   state. This is structural defense-in-depth.\n\n5) Bounds guard in lcb_vbguess_remap(). After PS2 still reproduced\n   exactly one SIGSEGV in FoRecoverDelta-SUBDOC at the first deref of\n   cfg-\u003evbuckets[vbid] (build-1524), with cfg \u003d LCBT_VBCONFIG(instance)\n   pointing at a freed-and-poisoned lcbvb_CONFIG (vbuckets\u003dNULL via\n   (3)). The handle_nmv ref guard in (2) pins cur_configinfo but the\n   deref is on cmdq.config; there is at least one SUBDOC code path\n   where the two diverge or where cmdq.config briefly points at a\n   stale vbc. Until that path is fully traced, gate the deref:\n   reject NULL cfg, NULL cfg-\u003evbuckets, or vbid out of [0, cfg-\u003envb).\n   On reject, the op falls through lcb_kv_should_retry -\u003e\n   mcreq_renew_packet -\u003e retryq-\u003enmvadd, which is the same path as a\n   legitimate \"no remap currently available\" and waits for the next\n   config to arrive. Logged at WARN so the rejection is visible in\n   production logs.\n\nTests\n-----\n\ntests/iotests/t_netfail.cc adds two cases. Both reproduce the failure\nmode without iptables.\n\ntestRetryOnMissingNodeAfterMapRepair: stops the config monitor, sets\nvbuckets[vb].servers[0] \u003d -1 to force lcbvb_vbmaster() to return -1\n(the precise condition retryq.cc:264 trips on), schedules a 200 ms\ntimer that restores the master in place, and issues an lcb_get with\na 2 s deadline. On gerrit/master (MISSINGNODE \u003d 0) retryq fails the\nop at the first 10 ms tick with LCB_ERR_NO_MATCHING_SERVER. With (1)\napplied, the op stays in retryq across ticks; once the timer\nrestores the map, the next tick dispatches it on the correct\npipeline and the GET succeeds.\n\ntestConfigReplaceMidRetry: same setup, but instead of restoring the\nlive vbc in place, snapshots the vbc to JSON via lcbvb_save_json(),\nloads it into a brand-new lcbvb_CONFIG, wraps it in a ConfigInfo, and\ncalls lcb_update_vbconfig() from the timer. This exercises the full\nreplace_config() path: cur_configinfo is swapped, replace_config()\nmoves pipelines, the old ConfigInfo is decref\u0027d, and the old\nlcbvb_CONFIG is freed via lcbvb_destroy(). Without (4) the cmdq\nwould have been visibly inconsistent during the swap; without (2)\nand (3) any captured raw lcbvb_CONFIG* across the call would UAF on\nthe freed memory. With all five fixes, the post-swap retryq tick\ndispatches on the new pipelines and the GET succeeds.\n\nThe pre-existing testNegativeIndex still passes; the op now waits\nthe full 500 ms op_timeout before failing with NO_MATCHING_SERVER\n(preserved as origerr) instead of failing fast.\n\nVerified locally on libev/libevent IO plugins: full unit-tests suite\n(194/194, modulo 1 environment-only Behavior.PluginDefaults skip)\ngreen.\n\nFiles touched: src/settings.{cc,h}, src/mcserver/mcserver.cc,\nsrc/newconfig.cc, src/vbucket/vbucket.c, tests/iotests/t_netfail.cc.\n\nChange-Id: I83c8c95a1b279082cd750d8363664f14a78aa0c5\nReviewed-on: https://review.couchbase.org/c/libcouchbase/+/244745\nTested-by: Build Bot \u003cbuild@couchbase.com\u003e\nReviewed-by: Sergey Avseyev \u003csergey.avseyev@gmail.com\u003e\n"},"parents_data":[{"branch_name":"refs/heads/master","commit_id":"3a66a6e38b4161b29bb4a4259300fc7aa4c8165a","is_merged_in_target_branch":true,"change_id":"I92f2ecef0f5cca170874c524457f49c38b8525b8","change_number":244520,"patch_set_number":3,"change_status":"MERGED"}],"branch":"refs/heads/master"}},"requirements":[],"submit_records":[{"rule_name":"gerrit~DefaultSubmitRule","status":"CLOSED","labels":[{"label":"Verified","status":"OK","applied_by":{"_account_id":1000011,"name":"Build Bot","email":"build@couchbase.com","username":"buildbot","avatars":[{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/0ba35a43c05d6329d9b2136bab05a733.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}],"tags":["SERVICE_USER"]}},{"label":"Code-Review","status":"OK","applied_by":{"_account_id":1000201,"name":"Sergey Avseyev","email":"sergey.avseyev@gmail.com","username":"avsej","avatars":[{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d32","height":32},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d56","height":56},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d100","height":100},{"url":"https://www.gravatar.com/avatar/dd70f9d8cc5f9ee488d68e7a787ba526.jpg?d\u003didenticon\u0026r\u003dpg\u0026s\u003d120","height":120}]}},{"label":"Well-Formed","status":"MAY"}]}],"submit_requirements":[{"name":"Verified","status":"SATISFIED","is_legacy":true,"submittability_expression_result":{"expression":"label:Verified\u003dMAX -label:Verified\u003dMIN","fulfilled":true,"status":"PASS","passing_atoms":["label:Verified\u003dMAX","-label:Verified\u003dMIN"],"failing_atoms":[]}},{"name":"Code-Review","status":"SATISFIED","is_legacy":true,"submittability_expression_result":{"expression":"label:Code-Review\u003dMAX -label:Code-Review\u003dMIN","fulfilled":true,"status":"PASS","passing_atoms":["label:Code-Review\u003dMAX","-label:Code-Review\u003dMIN"],"failing_atoms":[]}}]}