fix:add json stats version because previous change milvus-io#46130 (milvus-io#46467)

milvus-io#42533

Signed-off-by: luzhang <luzhang@zilliz.com>
Co-authored-by: luzhang <luzhang@zilliz.com>

Author: zhagnlu

internal/core/src/common/Consts.h

@@ -56,7 +56,8 @@ const char HINTS[] = "hints";
 const char JSON_KEY_INDEX_LOG_ROOT_PATH[] = "json_key_index_log";
 const char NGRAM_LOG_ROOT_PATH[] = "ngram_log";
 constexpr const char* JSON_STATS_ROOT_PATH = "json_stats";
-constexpr const char* JSON_STATS_DATA_FORMAT_VERSION = "2";
+// Version 3: metadata moved to separate meta.json file (instead of parquet metadata)
+constexpr const char* JSON_STATS_DATA_FORMAT_VERSION = "3";
 constexpr const char* JSON_STATS_SHARED_INDEX_PATH = "shared_key_index";
 constexpr const char* JSON_STATS_SHREDDING_DATA_PATH = "shredding_data";
 constexpr const char* JSON_STATS_META_FILE_NAME = "meta.json";

internal/datanode/index/task_stats.go

@@ -569,9 +569,11 @@ func (st *statsTask) createJSONKeyStats(ctx context.Context,
 	)
 
 	if jsonKeyStatsDataFormat != common.JSONStatsDataFormatVersion {
-		log.Info("create json key index failed dataformat invalid")
+		log.Warn("create json key index failed dataformat invalid", zap.Int64("dataformat version", jsonKeyStatsDataFormat),
+			zap.Int64("code version", common.JSONStatsDataFormatVersion))
 		return nil
 	}
+
 	fieldBinlogs := lo.GroupBy(insertBinlogs, func(binlog *datapb.FieldBinlog) int64 {
 		return binlog.GetFieldID()
 	})

internal/querynodev2/segments/segment.go

@@ -1227,13 +1227,15 @@ func (s *LocalSegment) LoadJSONKeyIndex(ctx context.Context, jsonKeyStats *datap
 		return nil
 	}
 
+	// for compatibility, we only support load data format version equal to the current data format version
+	// if the data format version is less than the current version, wait for trigger a stats task again
 	if jsonKeyStats.GetJsonKeyStatsDataFormat() != common.JSONStatsDataFormatVersion {
-		log.Ctx(ctx).Info("load json key index failed dataformat invalid", zap.Int64("dataformat", jsonKeyStats.GetJsonKeyStatsDataFormat()), zap.Int64("field id", jsonKeyStats.GetFieldID()), zap.Any("json key logs", jsonKeyStats))
+		log.Ctx(ctx).Warn("load json key index failed dataformat invalid", zap.Int64("dataformat", jsonKeyStats.GetJsonKeyStatsDataFormat()), zap.Int64("field id", jsonKeyStats.GetFieldID()), zap.Any("json key logs", jsonKeyStats))
 		return nil
 	}
 
 	log.Ctx(ctx).Info("load json key index", zap.Int64("field id", jsonKeyStats.GetFieldID()), zap.Any("json key logs", jsonKeyStats))
-	if info, ok := s.fieldJSONStats[jsonKeyStats.GetFieldID()]; ok && info.GetDataFormatVersion() >= common.JSONStatsDataFormatVersion {
+	if _, ok := s.fieldJSONStats[jsonKeyStats.GetFieldID()]; ok {
 		log.Warn("JsonKeyIndexStats already loaded", zap.Int64("field id", jsonKeyStats.GetFieldID()), zap.Any("json key logs", jsonKeyStats))
 		return nil
 	}

pkg/common/common.go

@@ -134,7 +134,8 @@ const (
 )
 
 const (
-	JSONStatsDataFormatVersion = 2
+	// Version 3: metadata moved to separate meta.json file (instead of parquet metadata)
+	JSONStatsDataFormatVersion = 3
 )
 
 // Search, Index parameter keys