enhance: update milvus.yaml (milvus-io#31832)

See milvus-io#32168

---------

Signed-off-by: Ted Xu <ted.xu@zilliz.com>

Author: tedxu

cmd/tools/config-docs-generator/main.go

@@ -12,8 +12,10 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
-var inputFile = "configs/milvus.yaml"
-var outputPath = os.Getenv("PWD")
+var (
+	inputFile  = "configs/milvus.yaml"
+	outputPath = os.Getenv("PWD")
+)
 
 func main() {
 	flag.StringVar(&inputFile, "i", inputFile, "input file")
@@ -168,13 +170,13 @@ For the convenience of maintenance, Milvus classifies its configurations into %s
 
 `
 	const fileName = "system_configuration.md"
-	var fileContent = head
+	fileContent := head
 	for _, sec := range secs {
 		fileContent += sec.systemConfiguratinContent()
 		sectionFileContent := sec.sectionPageContent()
-		os.WriteFile(filepath.Join(outputPath, sec.fileName()), []byte(sectionFileContent), 0644)
+		os.WriteFile(filepath.Join(outputPath, sec.fileName()), []byte(sectionFileContent), 0o644)
 	}
-	err := os.WriteFile(filepath.Join(outputPath, fileName), []byte(fileContent), 0644)
+	err := os.WriteFile(filepath.Join(outputPath, fileName), []byte(fileContent), 0o644)
 	return errors.Wrapf(err, "writefile %s", fileName)
 }
 

cmd/tools/config/generate.go

@@ -3,8 +3,9 @@ package main
 import (
 	"encoding/csv"
 	"fmt"
-	"os"
+	"io"
 	"reflect"
+	"sort"
 	"strings"
 
 	"github.com/samber/lo"
@@ -47,6 +48,13 @@ func collect() []DocContent {
 	return result
 }
 
+func quoteIfNeeded(s string) string {
+	if strings.ContainsAny(s, "[],{}") {
+		return fmt.Sprintf("\"%s\"", s)
+	}
+	return s
+}
+
 func collectRecursive(params *paramtable.ComponentParam, data *[]DocContent, val *reflect.Value) {
 	if val.Kind() != reflect.Struct {
 		return
@@ -63,27 +71,34 @@ func collectRecursive(params *paramtable.ComponentParam, data *[]DocContent, val
 			log.Debug("got key", zap.String("key", item.Key), zap.Any("value", defaultValue), zap.String("variable", val.Type().Field(j).Name))
 			*data = append(*data, DocContent{item.Key, defaultValue, item.Version, refreshable, item.Export, item.Doc})
 			for _, fk := range item.FallbackKeys {
+				defaultValue = params.GetWithDefault(fk, item.DefaultValue)
 				log.Debug("got fallback key", zap.String("key", fk), zap.Any("value", defaultValue), zap.String("variable", val.Type().Field(j).Name))
 				*data = append(*data, DocContent{fk, defaultValue, item.Version, refreshable, item.Export, item.Doc})
 			}
 		} else if t == "paramtable.ParamGroup" {
 			item := subVal.Interface().(paramtable.ParamGroup)
 			log.Debug("got key", zap.String("key", item.KeyPrefix), zap.String("variable", val.Type().Field(j).Name))
 			refreshable := tag.Get("refreshable")
-			*data = append(*data, DocContent{item.KeyPrefix, "", item.Version, refreshable, item.Export, item.Doc})
+
+			// Sort group items to stablize the output order
+			m := item.GetValue()
+			keys := make([]string, 0, len(m))
+			for k := range m {
+				keys = append(keys, k)
+			}
+			sort.Strings(keys)
+			for _, key := range keys {
+				value := m[key]
+				log.Debug("got group entry", zap.String("key", key), zap.String("value", value))
+				*data = append(*data, DocContent{fmt.Sprintf("%s%s", item.KeyPrefix, key), quoteIfNeeded(value), item.Version, refreshable, item.Export, ""})
+			}
 		} else {
 			collectRecursive(params, data, &subVal)
 		}
 	}
 }
 
-func WriteCsv() {
-	f, err := os.Create("configs.csv")
-	defer f.Close()
-	if err != nil {
-		log.Error("create file failed", zap.Error(err))
-		os.Exit(-2)
-	}
+func WriteCsv(f io.Writer) {
 	w := csv.NewWriter(f)
 	w.Write([]string{"key", "defaultValue", "sinceVersion", "refreshable", "exportToUser", "comment"})
 
@@ -101,7 +116,7 @@ type YamlGroup struct {
 }
 
 type YamlMarshaller struct {
-	writer *os.File
+	writer io.Writer
 	groups []YamlGroup
 	data   []DocContent
 }
@@ -142,19 +157,19 @@ func (m *YamlMarshaller) writeYamlRecursive(data []DocContent, level int) {
 		isDisabled := slices.Contains(disabledGroups, strings.Split(content.key, ".")[0])
 		if strings.Count(content.key, ".") == level {
 			if isDisabled {
-				m.writer.WriteString("# ")
+				io.WriteString(m.writer, "# ")
 			}
 			m.writeContent(key, content.defaultValue, content.comment, level)
 			continue
 		}
 		extra, ok := extraHeaders[key]
 		if ok {
-			m.writer.WriteString(extra + "\n")
+			io.WriteString(m.writer, extra+"\n")
 		}
 		if isDisabled {
-			m.writer.WriteString("# ")
+			io.WriteString(m.writer, "# ")
 		}
-		m.writer.WriteString(fmt.Sprintf("%s%s:\n", strings.Repeat(" ", level*2), key))
+		io.WriteString(m.writer, fmt.Sprintf("%s%s:\n", strings.Repeat(" ", level*2), key))
 		m.writeYamlRecursive(contents, level+1)
 	}
 }
@@ -163,27 +178,20 @@ func (m *YamlMarshaller) writeContent(key, value, comment string, level int) {
 	if strings.Contains(comment, "\n") {
 		multilines := strings.Split(comment, "\n")
 		for _, line := range multilines {
-			m.writer.WriteString(fmt.Sprintf("%s# %s\n", strings.Repeat(" ", level*2), line))
+			io.WriteString(m.writer, fmt.Sprintf("%s# %s\n", strings.Repeat(" ", level*2), line))
 		}
-		m.writer.WriteString(fmt.Sprintf("%s%s: %s\n", strings.Repeat(" ", level*2), key, value))
+		io.WriteString(m.writer, fmt.Sprintf("%s%s: %s\n", strings.Repeat(" ", level*2), key, value))
 	} else if comment != "" {
-		m.writer.WriteString(fmt.Sprintf("%s%s: %s # %s\n", strings.Repeat(" ", level*2), key, value, comment))
+		io.WriteString(m.writer, fmt.Sprintf("%s%s: %s # %s\n", strings.Repeat(" ", level*2), key, value, comment))
 	} else {
-		m.writer.WriteString(fmt.Sprintf("%s%s: %s\n", strings.Repeat(" ", level*2), key, value))
+		io.WriteString(m.writer, fmt.Sprintf("%s%s: %s\n", strings.Repeat(" ", level*2), key, value))
 	}
 }
 
-func WriteYaml() {
-	f, err := os.Create("milvus.yaml")
-	defer f.Close()
-	if err != nil {
-		log.Error("create file failed", zap.Error(err))
-		os.Exit(-2)
-	}
-
+func WriteYaml(w io.Writer) {
 	result := collect()
 
-	f.WriteString(`# Licensed to the LF AI & Data foundation under one
+	io.WriteString(w, `# Licensed to the LF AI & Data foundation under one
 # or more contributor license agreements. See the NOTICE file
 # distributed with this work for additional information
 # regarding copyright ownership. The ASF licenses this file
@@ -207,6 +215,13 @@ func WriteYaml() {
 		{
 			name: "metastore",
 		},
+		{
+			name: "tikv",
+			header: `
+# Related configuration of tikv, used to store Milvus metadata.
+# Notice that when TiKV is enabled for metastore, you still need to have etcd for service discovery.
+# TiKV is a good option when the metadata size requires better horizontal scalability.`,
+		},
 		{
 			name: "localStorage",
 		},
@@ -215,15 +230,19 @@ func WriteYaml() {
 			header: `
 # Related configuration of MinIO/S3/GCS or any other service supports S3 API, which is responsible for data persistence for Milvus.
 # We refer to the storage service as MinIO/S3 in the following description for simplicity.`,
+		},
+		{
+			name: "mq",
+			header: `
+# Milvus supports four MQ: rocksmq(based on RockDB), natsmq(embedded nats-server), Pulsar and Kafka.
+# You can change your mq by setting mq.type field.
+# If you don't set mq.type field as default, there is a note about enabling priority if we config multiple mq in this file.
+# 1. standalone(local) mode: rocksmq(default) > natsmq > Pulsar > Kafka
+# 2. cluster mode:  Pulsar(default) > Kafka (rocksmq and natsmq is unsupported in cluster mode)`,
 		},
 		{
 			name: "pulsar",
 			header: `
-# Milvus supports three MQ: rocksmq(based on RockDB), Pulsar and Kafka, which should be reserved in config what you use.
-# There is a note about enabling priority if we config multiple mq in this file
-# 1. standalone(local) mode: rocksmq(default) > Pulsar > Kafka
-# 2. cluster mode:  Pulsar(default) > Kafka (rocksmq is unsupported)
-
 # Related configuration of pulsar, used to manage Milvus logs of recent mutation operations, output streaming log, and provide log publish-subscribe services.`,
 		},
 		{
@@ -234,6 +253,12 @@ func WriteYaml() {
 		{
 			name: "rocksmq",
 		},
+		{
+			name: "natsmq",
+			header: `
+# natsmq configuration.
+# more detail: https://docs.nats.io/running-a-nats-service/configuration`,
+		},
 		{
 			name:   "rootCoord",
 			header: "\n# Related configuration of rootCoord, used to handle data definition language (DDL) and data control language (DCL) requests",
@@ -294,8 +319,18 @@ func WriteYaml() {
 		{
 			name: "trace",
 		},
+		{
+			name: "gpu",
+			header: `
+#when using GPU indexing, Milvus will utilize a memory pool to avoid frequent memory allocation and deallocation.
+#here, you can set the size of the memory occupied by the memory pool, with the unit being MB.
+#note that there is a possibility of Milvus crashing when the actual memory demand exceeds the value set by maxMemSize.
+#if initMemSize and MaxMemSize both set zero,
+#milvus will automatically initialize half of the available GPU memory,
+#maxMemSize will the whole available GPU memory.`,
+		},
 	}
-	marshller := YamlMarshaller{f, groups, result}
+	marshller := YamlMarshaller{w, groups, result}
 	marshller.writeYamlRecursive(lo.Filter(result, func(d DocContent, _ int) bool {
 		return d.exportToUser
 	}), 0)

cmd/tools/config/generate_test.go

@@ -0,0 +1,49 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"go.uber.org/zap"
+
+	"github.com/milvus-io/milvus/pkg/log"
+	"github.com/milvus-io/milvus/pkg/util/paramtable"
+)
+
+// Assert the milvus.yaml file is consistent to paramtable
+//
+//	 Please be noted that milvus.yaml is generated by code, so don't edit it directly, instead, change the code in paramtable
+//		and run `make milvus-tools && ./bin/tools/config gen-yaml && mv milvus.yaml configs/milvus.yaml`.
+func TestYamlFile(t *testing.T) {
+	w := bytes.Buffer{}
+	WriteYaml(&w)
+
+	base := paramtable.NewBaseTable()
+	f, err := os.Open(fmt.Sprintf("%s/%s", base.GetConfigDir(), "milvus.yaml"))
+	assert.NoError(t, err, "expecting configs/milvus.yaml")
+	defer f.Close()
+	fileScanner := bufio.NewScanner(f)
+	codeScanner := bufio.NewScanner(&w)
+	for fileScanner.Scan() && codeScanner.Scan() {
+		if fileScanner.Text() != codeScanner.Text() {
+			assert.FailNow(t, fmt.Sprintf("configs/milvus.yaml is not consistent with paramtable, file: [%s], code: [%s]. Do not edit milvus.yaml directly.",
+				fileScanner.Text(), codeScanner.Text()))
+		}
+		log.Error("", zap.Any("file", fileScanner.Text()), zap.Any("code", codeScanner.Text()))
+	}
+}

cmd/tools/config/main.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"os"
 
+	"go.uber.org/zap"
+
 	"github.com/milvus-io/milvus/pkg/log"
 )
 
@@ -22,9 +24,21 @@ func main() {
 	}
 	switch args[1] {
 	case generateCsv:
-		WriteCsv()
+		f, err := os.Create("configs.csv")
+		defer f.Close()
+		if err != nil {
+			log.Error("create file failed", zap.Error(err))
+			os.Exit(-2)
+		}
+		WriteCsv(f)
 	case generateYaml:
-		WriteYaml()
+		f, err := os.Create("milvus.yaml")
+		defer f.Close()
+		if err != nil {
+			log.Error("create file failed", zap.Error(err))
+			os.Exit(-2)
+		}
+		WriteYaml(f)
 	case showYaml:
 		var f string
 		if len(args) == 2 {