enhance: clean channel node info in meta store (milvus-io#32988)

issue: milvus-io#32910
see also: milvus-io#32911
when channel exclusive mode is enabled, replica will record channel node
info in meta store, and if the balance policy changes, which means
channel exclusive mode is disabled, we should clean up the channel node
info in meta store, and stop to balance node between channels.

Signed-off-by: Wei Liu <wei.liu@zilliz.com>

Author: weiliu1031

configs/milvus.yaml

@@ -297,6 +297,7 @@ queryCoord:
   checkNodeSessionInterval: 60 # the interval(in seconds) of check querynode cluster session
   gracefulStopTimeout: 5 # seconds. force stop node without graceful stop
   enableStoppingBalance: true # whether enable stopping balance
+  channelExclusiveNodeFactor: 4 # the least node number for enable channel's exclusive mode
   cleanExcludeSegmentInterval: 60 # the time duration of clean pipeline exclude segment which used for filter invalid data, in seconds
   ip:  # if not specified, use the first unicastable address
   port: 19531

internal/querycoordv2/meta/replica.go

@@ -230,9 +230,11 @@ func (replica *mutableReplica) tryBalanceNodeForChannel() {
 		return
 	}
 
+	balancePolicy := paramtable.Get().QueryCoordCfg.Balancer.GetValue()
+	enableChannelExclusiveMode := balancePolicy == ChannelLevelScoreBalancerName
 	channelExclusiveFactor := paramtable.Get().QueryCoordCfg.ChannelExclusiveNodeFactor.GetAsInt()
-	// to do: if query node scale in happens, and the condition does not meet, should we exit channel's exclusive mode?
-	if len(replica.rwNodes) < len(channelNodeInfos)*channelExclusiveFactor {
+	// if balance policy or node count doesn't match condition, clean up channel node info
+	if !enableChannelExclusiveMode || len(replica.rwNodes) < len(channelNodeInfos)*channelExclusiveFactor {
 		for name := range replica.replicaPB.GetChannelNodeInfos() {
 			replica.replicaPB.ChannelNodeInfos[name] = &querypb.ChannelNodeInfo{}
 		}

internal/querycoordv2/meta/replica_test.go

@@ -180,6 +180,9 @@ func (suite *ReplicaSuite) testRead(r *Replica) {
 }
 
 func (suite *ReplicaSuite) TestChannelExclusiveMode() {
+	paramtable.Get().Save(paramtable.Get().QueryCoordCfg.Balancer.Key, ChannelLevelScoreBalancerName)
+	defer paramtable.Get().Reset(paramtable.Get().QueryCoordCfg.Balancer.Key)
+
 	r := newReplica(&querypb.Replica{
 		ID:            1,
 		CollectionID:  2,