0

I have a three nodes replica and I prefer a 10.0.0.5 as a primary. I have just realized during a performance test that 10.0.0.2 is primary instead. I thought it should have step out automatically because it has lower priority. I ran step out manually but it is still primary. Any idea what have I configured incorrectly?

rs_bud:PRIMARY> version();
4.4.8

rs_bud:PRIMARY> rs.conf();
{
        "_id" : "rs_bud",
        "version" : 2,
        "term" : 13,
        "protocolVersion" : NumberLong(1),
        "writeConcernMajorityJournalDefault" : true,
        "members" : [
                {
                        "_id" : 0,
                        "host" : "10.0.0.4:27017",
                        "arbiterOnly" : false,
                        "buildIndexes" : true,
                        "hidden" : false,
                        "priority" : 2,
                        "tags" : {},
                        "slaveDelay" : NumberLong(0),
                        "votes" : 1
                },
                {
                        "_id" : 1,
                        "host" : "10.0.0.5:27017",
                        "arbiterOnly" : false,
                        "buildIndexes" : true,
                        "hidden" : false,
                        "priority" : 1,
                        "tags" : {},
                        "slaveDelay" : NumberLong(0),
                        "votes" : 1
                },
                {
                        "_id" : 2,
                        "host" : "10.0.0.2:27017",
                        "arbiterOnly" : false,
                        "buildIndexes" : true,
                        "hidden" : false,
                        "priority" : 5,
                        "tags" : {},
                        "slaveDelay" : NumberLong(0),
                        "votes" : 1
                }
        ],
        "settings" : {
                "chainingAllowed" : true,
                "heartbeatIntervalMillis" : 2000,
                "heartbeatTimeoutSecs" : 10,
                "electionTimeoutMillis" : 10000,
                "catchUpTimeoutMillis" : -1,
                "catchUpTakeoverDelayMillis" : 30000,
                "getLastErrorModes" : {

                },
                "getLastErrorDefaults" : {
                        "w" : 1,
                        "wtimeout" : 0
                },
                "replicaSetId" : ObjectId("612bc65e0d5e82841e5c9182")
        }
}

rs_bud:PRIMARY> rs.status()
{
        "set" : "rs_bud",
        "date" : ISODate("2021-11-28T19:10:41.915Z"),
        "myState" : 1,
        "term" : NumberLong(11),
        "syncSourceHost" : "",
        "syncSourceId" : -1,
        "heartbeatIntervalMillis" : NumberLong(2000),
        "majorityVoteCount" : 2,
        "writeMajorityCount" : 2,
        "votingMembersCount" : 3,
        "writableVotingMembersCount" : 3,
        "optimes" : {
                "lastCommittedOpTime" : {
                        "ts" : Timestamp(1638126639, 1),
                        "t" : NumberLong(11)
                },
                "lastCommittedWallTime" : ISODate("2021-11-28T19:10:39.580Z"),
                "readConcernMajorityOpTime" : {
                        "ts" : Timestamp(1638126639, 1),
                        "t" : NumberLong(11)
                },
                "readConcernMajorityWallTime" : ISODate("2021-11-28T19:10:39.580Z"),
                "appliedOpTime" : {
                        "ts" : Timestamp(1638126639, 1),
                        "t" : NumberLong(11)
                },
                "durableOpTime" : {
                        "ts" : Timestamp(1638126639, 1),
                        "t" : NumberLong(11)
                },
                "lastAppliedWallTime" : ISODate("2021-11-28T19:10:39.580Z"),
                "lastDurableWallTime" : ISODate("2021-11-28T19:10:39.580Z")
        },
        "lastStableRecoveryTimestamp" : Timestamp(1638126589, 1),
        "electionCandidateMetrics" : {
                "lastElectionReason" : "priorityTakeover",
                "lastElectionDate" : ISODate("2021-10-12T11:22:33.609Z"),
                "electionTerm" : NumberLong(11),
                "lastCommittedOpTimeAtElection" : {
                        "ts" : Timestamp(1634037753, 1),
                        "t" : NumberLong(10)
                },
                "lastSeenOpTimeAtElection" : {
                        "ts" : Timestamp(1634037753, 1),
                        "t" : NumberLong(10)
                },
                "numVotesNeeded" : 2,
                "priorityAtElection" : 5,
                "electionTimeoutMillis" : NumberLong(10000),
                "priorPrimaryMemberId" : 0,
                "numCatchUpOps" : NumberLong(0),
                "newTermStartDate" : ISODate("2021-10-12T11:22:33.620Z"),
                "wMajorityWriteAvailabilityDate" : ISODate("2021-10-12T11:22:33.719Z")
        },
        "electionParticipantMetrics" : {
                "votedForCandidate" : false,
                "electionTerm" : NumberLong(10),
                "lastVoteDate" : ISODate("2021-10-12T11:22:20.240Z"),
                "electionCandidateMemberId" : 0,
                "voteReason" : "candidate's data is staler than mine. candidate's last applied OpTime: { ts: Timestamp(1634037726, 1), t: 9 }, my last applied OpTime: { ts: Timestamp(1634037736, 1), t: 9 }",
                "lastAppliedOpTimeAtElection" : {
                        "ts" : Timestamp(1634037736, 1),
                        "t" : NumberLong(9)
                },
                "maxAppliedOpTimeInSet" : {
                        "ts" : Timestamp(1634037736, 1),
                        "t" : NumberLong(9)
                },
                "priorityAtElection" : 5
        },
        "members" : [
                {
                        "_id" : 0,
                        "name" : "10.0.0.4:27017",
                        "health" : 1,
                        "state" : 2,
                        "stateStr" : "SECONDARY",
                        "uptime" : 4088900,
                        "optime" : {
                                "ts" : Timestamp(1638126639, 1),
                                "t" : NumberLong(11)
                        },
                        "optimeDurable" : {
                                "ts" : Timestamp(1638126639, 1),
                                "t" : NumberLong(11)
                        },
                        "optimeDate" : ISODate("2021-11-28T19:10:39Z"),
                        "optimeDurableDate" : ISODate("2021-11-28T19:10:39Z"),
                        "lastHeartbeat" : ISODate("2021-11-28T19:10:40.826Z"),
                        "lastHeartbeatRecv" : ISODate("2021-11-28T19:10:40.095Z"),
                        "pingMs" : NumberLong(0),
                        "lastHeartbeatMessage" : "",
                        "syncSourceHost" : "10.0.0.2:27017",
                        "syncSourceId" : 2,
                        "infoMessage" : "",
                        "configVersion" : 2,
                        "configTerm" : 11
                },
                {
                        "_id" : 1,
                        "name" : "10.0.0.5:27017",
                        "health" : 1,
                        "state" : 2,
                        "stateStr" : "SECONDARY",
                        "uptime" : 6051201,
                        "optime" : {
                                "ts" : Timestamp(1638126639, 1),
                                "t" : NumberLong(11)
                        },
                        "optimeDurable" : {
                                "ts" : Timestamp(1638126639, 1),
                                "t" : NumberLong(11)
                        },
                        "optimeDate" : ISODate("2021-11-28T19:10:39Z"),
                        "optimeDurableDate" : ISODate("2021-11-28T19:10:39Z"),
                        "lastHeartbeat" : ISODate("2021-11-28T19:10:41.206Z"),
                        "lastHeartbeatRecv" : ISODate("2021-11-28T19:10:41.510Z"),
                        "pingMs" : NumberLong(0),
                        "lastHeartbeatMessage" : "",
                        "syncSourceHost" : "10.0.0.2:27017",
                        "syncSourceId" : 2,
                        "infoMessage" : "",
                        "configVersion" : 2,
                        "configTerm" : 11
                },
                {
                        "_id" : 2,
                        "name" : "10.0.0.2:27017",
                        "health" : 1,
                        "state" : 1,
                        "stateStr" : "PRIMARY",
                        "uptime" : 6051201,
                        "optime" : {
                                "ts" : Timestamp(1638126639, 1),
                                "t" : NumberLong(11)
                        },
                        "optimeDate" : ISODate("2021-11-28T19:10:39Z"),
                        "syncSourceHost" : "",
                        "syncSourceId" : -1,
                        "infoMessage" : "",
                        "electionTime" : Timestamp(1634037753, 2),
                        "electionDate" : ISODate("2021-10-12T11:22:33Z"),
                        "configVersion" : 2,
                        "configTerm" : 11,
                        "self" : true,
                        "lastHeartbeatMessage" : ""
                }
        ],
        "ok" : 1,
        "$clusterTime" : {
                "clusterTime" : Timestamp(1638126639, 1),
                "signature" : {
                        "hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
                        "keyId" : NumberLong(0)
                }
        },
        "operationTime" : Timestamp(1638126639, 1)
}

rs_bud:PRIMARY> rs.stepDown(120)
{
        "ok" : 1,
        "$clusterTime" : {
                "clusterTime" : Timestamp(1638126759, 1),
                "signature" : {
                        "hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
                        "keyId" : NumberLong(0)
                }
        },
        "operationTime" : Timestamp(1638126759, 1)
}

rs_bud:PRIMARY> rs.status()
{
        "set" : "rs_bud",
        "date" : ISODate("2021-11-28T19:32:15.609Z"),
        "myState" : 1,
        "term" : NumberLong(13),
        "syncSourceHost" : "",
        "syncSourceId" : -1,
        "heartbeatIntervalMillis" : NumberLong(2000),
        "majorityVoteCount" : 2,
        "writeMajorityCount" : 2,
        "votingMembersCount" : 3,
        "writableVotingMembersCount" : 3,
        "optimes" : {
                "lastCommittedOpTime" : {
                        "ts" : Timestamp(1638127934, 1),
                        "t" : NumberLong(13)
                },
                "lastCommittedWallTime" : ISODate("2021-11-28T19:32:14.137Z"),
                "readConcernMajorityOpTime" : {
                        "ts" : Timestamp(1638127934, 1),
                        "t" : NumberLong(13)
                },
                "readConcernMajorityWallTime" : ISODate("2021-11-28T19:32:14.137Z"),
                "appliedOpTime" : {
                        "ts" : Timestamp(1638127934, 1),
                        "t" : NumberLong(13)
                },
                "durableOpTime" : {
                        "ts" : Timestamp(1638127934, 1),
                        "t" : NumberLong(13)
                },
                "lastAppliedWallTime" : ISODate("2021-11-28T19:32:14.137Z"),
                "lastDurableWallTime" : ISODate("2021-11-28T19:32:14.137Z")
        },
        "lastStableRecoveryTimestamp" : Timestamp(1638127904, 1),
        "electionCandidateMetrics" : {
                "lastElectionReason" : "priorityTakeover",
                "lastElectionDate" : ISODate("2021-11-28T19:14:44.084Z"),
                "electionTerm" : NumberLong(13),
                "lastCommittedOpTimeAtElection" : {
                        "ts" : Timestamp(1638126883, 1),
                        "t" : NumberLong(12)
                },
                "lastSeenOpTimeAtElection" : {
                        "ts" : Timestamp(1638126883, 1),
                        "t" : NumberLong(12)
                },
                "numVotesNeeded" : 2,
                "priorityAtElection" : 5,
                "electionTimeoutMillis" : NumberLong(10000),
                "priorPrimaryMemberId" : 0,
                "numCatchUpOps" : NumberLong(0),
                "newTermStartDate" : ISODate("2021-11-28T19:14:44.097Z"),
                "wMajorityWriteAvailabilityDate" : ISODate("2021-11-28T19:14:45.099Z")
        },
        "electionParticipantMetrics" : {
                "votedForCandidate" : true,
                "electionTerm" : NumberLong(12),
                "lastVoteDate" : ISODate("2021-11-28T19:12:43.830Z"),
                "electionCandidateMemberId" : 0,
                "voteReason" : "",
                "lastAppliedOpTimeAtElection" : {
                        "ts" : Timestamp(1638126759, 1),
                        "t" : NumberLong(11)
                },
                "maxAppliedOpTimeInSet" : {
                        "ts" : Timestamp(1638126759, 1),
                        "t" : NumberLong(11)
                },
                "priorityAtElection" : 5
        },
        "members" : [
                {
                        "_id" : 0,
                        "name" : "10.0.0.4:27017",
                        "health" : 1,
                        "state" : 2,
                        "stateStr" : "SECONDARY",
                        "uptime" : 4090193,
                        "optime" : {
                                "ts" : Timestamp(1638127934, 1),
                                "t" : NumberLong(13)
                        },
                        "optimeDurable" : {
                                "ts" : Timestamp(1638127934, 1),
                                "t" : NumberLong(13)
                        },
                        "optimeDate" : ISODate("2021-11-28T19:32:14Z"),
                        "optimeDurableDate" : ISODate("2021-11-28T19:32:14Z"),
                        "lastHeartbeat" : ISODate("2021-11-28T19:32:14.634Z"),
                        "lastHeartbeatRecv" : ISODate("2021-11-28T19:32:13.662Z"),
                        "pingMs" : NumberLong(0),
                        "lastHeartbeatMessage" : "",
                        "syncSourceHost" : "10.0.0.2:27017",
                        "syncSourceId" : 2,
                        "infoMessage" : "",
                        "configVersion" : 2,
                        "configTerm" : 13
                },
                {
                        "_id" : 1,
                        "name" : "10.0.0.5:27017",
                        "health" : 1,
                        "state" : 2,
                        "stateStr" : "SECONDARY",
                        "uptime" : 6052494,
                        "optime" : {
                                "ts" : Timestamp(1638127934, 1),
                                "t" : NumberLong(13)
                        },
                        "optimeDurable" : {
                                "ts" : Timestamp(1638127934, 1),
                                "t" : NumberLong(13)
                        },
                        "optimeDate" : ISODate("2021-11-28T19:32:14Z"),
                        "optimeDurableDate" : ISODate("2021-11-28T19:32:14Z"),
                        "lastHeartbeat" : ISODate("2021-11-28T19:32:14.847Z"),
                        "lastHeartbeatRecv" : ISODate("2021-11-28T19:32:14.933Z"),
                        "pingMs" : NumberLong(0),
                        "lastHeartbeatMessage" : "",
                        "syncSourceHost" : "10.0.0.4:27017",
                        "syncSourceId" : 0,
                        "infoMessage" : "",
                        "configVersion" : 2,
                        "configTerm" : 13
                },
                {
                        "_id" : 2,
                        "name" : "10.0.0.2:27017",
                        "health" : 1,
                        "state" : 1,
                        "stateStr" : "PRIMARY",
                        "uptime" : 6052495,
                        "optime" : {
                                "ts" : Timestamp(1638127934, 1),
                                "t" : NumberLong(13)
                        },
                        "optimeDate" : ISODate("2021-11-28T19:32:14Z"),
                        "syncSourceHost" : "",
                        "syncSourceId" : -1,
                        "infoMessage" : "",
                        "electionTime" : Timestamp(1638126884, 1),
                        "electionDate" : ISODate("2021-11-28T19:14:44Z"),
                        "configVersion" : 2,
                        "configTerm" : 13,
                        "self" : true,
                        "lastHeartbeatMessage" : ""
                }
        ],
        "ok" : 1,
        "$clusterTime" : {
                "clusterTime" : Timestamp(1638127934, 1),
                "signature" : {
                        "hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
                        "keyId" : NumberLong(0)
                }
        },
        "operationTime" : Timestamp(1638127934, 1)
}

Update - reconfiguration procedure

cfg["members"][0]["priority"] = 10;
cfg["members"][1]["priority"] = 5;
cfg["members"][2]["priority"] = 1;
rs.reconfig(cfg);
{
        "ok" : 1,
        "$clusterTime" : {
                "clusterTime" : Timestamp(1638213674, 1),
                "signature" : {
                        "hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),
                        "keyId" : NumberLong(0)
                }
        },
        "operationTime" : Timestamp(1638213674, 1)
}

rs.stepDown(20)

1 Answer 1

3

Answer is priority... Higher priority wins. So, "host" : "10.0.0.2:27017", with "priority" : 5 wins the election against "host" : "10.0.0.5:27017", "priority" : 1.

Solution is give priority 5 to node 10.0.0.5 and priority 1 to that 10.0.0.2

1
  • Oh, so the priority has an opposite meaning. I will try later. Thanks Nov 29, 2021 at 11:19

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Not the answer you're looking for? Browse other questions tagged or ask your own question.