feat: Geospatial Data Type and GIS Function support for milvus (milvus-io#44547)

issue: milvus-io#43427

This pr's main goal is merge milvus-io#37417 to milvus 2.5 without conflicts.

# Main Goals

1. Create and describe collections with geospatial type
2. Insert geospatial data into the insert binlog
3. Load segments containing geospatial data into memory
4. Enable query and search can display  geospatial data
5. Support using GIS funtions like ST_EQUALS in query
6. Support R-Tree index for geometry type

# Solution

1. **Add Type**: Modify the Milvus core by adding a Geospatial type in
both the C++ and Go code layers, defining the Geospatial data structure
and the corresponding interfaces.
2. **Dependency Libraries**: Introduce necessary geospatial data
processing libraries. In the C++ source code, use Conan package
management to include the GDAL library. In the Go source code, add the
go-geom library to the go.mod file.
3. **Protocol Interface**: Revise the Milvus protocol to provide
mechanisms for Geospatial message serialization and deserialization.
4. **Data Pipeline**: Facilitate interaction between the client and
proxy using the WKT format for geospatial data. The proxy will convert
all data into WKB format for downstream processing, providing column
data interfaces, segment encapsulation, segment loading, payload
writing, and cache block management.
5. **Query Operators**: Implement simple display and support for filter
queries. Initially, focus on filtering based on spatial relationships
for a single column of geospatial literal values, providing parsing and
execution for query expressions.Now only support brutal search
7. **Client Modification**: Enable the client to handle user input for
geospatial data and facilitate end-to-end testing.Check the modification
in pymilvus.

---------

Signed-off-by: Yinwei Li <yinwei.li@zilliz.com>
Signed-off-by: Cai Zhang <cai.zhang@zilliz.com>
Co-authored-by: ZhuXi <150327960+Yinwei-Yu@users.noreply.github.com>

Author: xiaocai2333

client/column/columns.go

@@ -212,6 +212,9 @@ func FieldDataColumn(fd *schemapb.FieldData, begin, end int) (Column, error) {
 	case schemapb.DataType_JSON:
 		return parseScalarData(fd.GetFieldName(), fd.GetScalars().GetJsonData().GetData(), begin, end, validData, NewColumnJSONBytes, NewNullableColumnJSONBytes)
 
+	case schemapb.DataType_Geometry:
+		return parseScalarData(fd.GetFieldName(), fd.GetScalars().GetGeometryWktData().GetData(), begin, end, validData, NewColumnGeometryWKT, NewNullableColumnGeometryWKT)
+
 	case schemapb.DataType_FloatVector:
 		vectors := fd.GetVectors()
 		x, ok := vectors.GetData().(*schemapb.VectorField_FloatVector)

client/column/conversion.go

@@ -117,7 +117,8 @@ func values2FieldData[T any](values []T, fieldType entity.FieldType, dim int) *s
 		entity.FieldTypeInt64,
 		entity.FieldTypeVarChar,
 		entity.FieldTypeString,
-		entity.FieldTypeJSON:
+		entity.FieldTypeJSON,
+		entity.FieldTypeGeometry:
 		fd.Field = &schemapb.FieldData_Scalars{
 			Scalars: values2Scalars(values, fieldType), // scalars,
 		}
@@ -199,6 +200,12 @@ func values2Scalars[T any](values []T, fieldType entity.FieldType) *schemapb.Sca
 				Data: data,
 			},
 		}
+	case entity.FieldTypeGeometry:
+		var strVals []string
+		strVals, ok = any(values).([]string)
+		scalars.Data = &schemapb.ScalarField_GeometryWktData{
+			GeometryWktData: &schemapb.GeometryWktArray{Data: strVals},
+		}
 	}
 	// shall not be accessed
 	if !ok {

client/column/geometry.go

@@ -0,0 +1,91 @@
+package column
+
+import (
+	"github.com/cockroachdb/errors"
+
+	"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
+	"github.com/milvus-io/milvus/client/v2/entity"
+)
+
+type ColumnGeometryWKT struct {
+	*genericColumnBase[string]
+}
+
+// Name returns column name.
+func (c *ColumnGeometryWKT) Name() string {
+	return c.name
+}
+
+// Type returns column entity.FieldType.
+func (c *ColumnGeometryWKT) Type() entity.FieldType {
+	return entity.FieldTypeGeometry
+}
+
+// Len returns column values length.
+func (c *ColumnGeometryWKT) Len() int {
+	return len(c.values)
+}
+
+func (c *ColumnGeometryWKT) Slice(start, end int) Column {
+	l := c.Len()
+	if start > l {
+		start = l
+	}
+	if end == -1 || end > l {
+		end = l
+	}
+	return &ColumnGeometryWKT{
+		genericColumnBase: c.genericColumnBase.slice(start, end),
+	}
+}
+
+// Get returns value at index as interface{}.
+func (c *ColumnGeometryWKT) Get(idx int) (interface{}, error) {
+	if idx < 0 || idx >= c.Len() {
+		return nil, errors.New("index out of range")
+	}
+	return c.values[idx], nil
+}
+
+func (c *ColumnGeometryWKT) GetAsString(idx int) (string, error) {
+	return c.ValueByIdx(idx)
+}
+
+// FieldData return column data mapped to schemapb.FieldData.
+func (c *ColumnGeometryWKT) FieldData() *schemapb.FieldData {
+	fd := c.genericColumnBase.FieldData()
+	return fd
+}
+
+// ValueByIdx returns value of the provided index.
+func (c *ColumnGeometryWKT) ValueByIdx(idx int) (string, error) {
+	if idx < 0 || idx >= c.Len() {
+		return "", errors.New("index out of range")
+	}
+	return c.values[idx], nil
+}
+
+// AppendValue append value into column.
+func (c *ColumnGeometryWKT) AppendValue(i interface{}) error {
+	s, ok := i.(string)
+	if !ok {
+		return errors.New("expect geometry WKT type(string)")
+	}
+	c.values = append(c.values, s)
+	return nil
+}
+
+// Data returns column data.
+func (c *ColumnGeometryWKT) Data() []string {
+	return c.values
+}
+
+func NewColumnGeometryWKT(name string, values []string) *ColumnGeometryWKT {
+	return &ColumnGeometryWKT{
+		genericColumnBase: &genericColumnBase[string]{
+			name:      name,
+			fieldType: entity.FieldTypeGeometry,
+			values:    values,
+		},
+	}
+}

client/column/geometry_test.go

@@ -0,0 +1,76 @@
+package column
+
+import (
+	"fmt"
+	"math/rand"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/suite"
+
+	"github.com/milvus-io/milvus/client/v2/entity"
+)
+
+type ColumnGeometryWKTSuite struct {
+	suite.Suite
+}
+
+func (s *ColumnGeometryWKTSuite) SetupSuite() {
+	rand.Seed(time.Now().UnixNano())
+}
+
+func (s *ColumnGeometryWKTSuite) TestAttrMethods() {
+	columnName := fmt.Sprintf("column_Geometrywkt_%d", rand.Int())
+	columnLen := 8 + rand.Intn(10)
+
+	v := make([]string, columnLen)
+	column := NewColumnGeometryWKT(columnName, v)
+
+	s.Run("test_meta", func() {
+		ft := entity.FieldTypeGeometry
+		s.Equal("Geometry", ft.Name())
+		s.Equal("Geometry", ft.String())
+		pbName, pbType := ft.PbFieldType()
+		s.Equal("Geometry", pbName)
+		s.Equal("Geometry", pbType)
+	})
+
+	s.Run("test_column_attribute", func() {
+		s.Equal(columnName, column.Name())
+		s.Equal(entity.FieldTypeGeometry, column.Type())
+		s.Equal(columnLen, column.Len())
+		s.EqualValues(v, column.Data())
+	})
+
+	s.Run("test_column_field_data", func() {
+		fd := column.FieldData()
+		s.NotNil(fd)
+		s.Equal(fd.GetFieldName(), columnName)
+	})
+
+	s.Run("test_column_valuer_by_idx", func() {
+		_, err := column.ValueByIdx(-1)
+		s.Error(err)
+		_, err = column.ValueByIdx(columnLen)
+		s.Error(err)
+		for i := 0; i < columnLen; i++ {
+			v, err := column.ValueByIdx(i)
+			s.NoError(err)
+			s.Equal(column.values[i], v)
+		}
+	})
+
+	s.Run("test_append_value", func() {
+		item := "POINT (30.123 -10.456)"
+		err := column.AppendValue(item)
+		s.NoError(err)
+		s.Equal(columnLen+1, column.Len())
+		val, err := column.ValueByIdx(columnLen)
+		s.NoError(err)
+		s.Equal(item, val)
+	})
+}
+
+func TestColumnGeometryWKT(t *testing.T) {
+	suite.Run(t, new(ColumnGeometryWKTSuite))
+}

client/column/nullable.go

@@ -18,17 +18,18 @@ package column
 
 var (
 	// scalars
-	NewNullableColumnBool        NullableColumnCreateFunc[bool, *ColumnBool]         = NewNullableColumnCreator(NewColumnBool).New
-	NewNullableColumnInt8        NullableColumnCreateFunc[int8, *ColumnInt8]         = NewNullableColumnCreator(NewColumnInt8).New
-	NewNullableColumnInt16       NullableColumnCreateFunc[int16, *ColumnInt16]       = NewNullableColumnCreator(NewColumnInt16).New
-	NewNullableColumnInt32       NullableColumnCreateFunc[int32, *ColumnInt32]       = NewNullableColumnCreator(NewColumnInt32).New
-	NewNullableColumnInt64       NullableColumnCreateFunc[int64, *ColumnInt64]       = NewNullableColumnCreator(NewColumnInt64).New
-	NewNullableColumnVarChar     NullableColumnCreateFunc[string, *ColumnVarChar]    = NewNullableColumnCreator(NewColumnVarChar).New
-	NewNullableColumnString      NullableColumnCreateFunc[string, *ColumnString]     = NewNullableColumnCreator(NewColumnString).New
-	NewNullableColumnFloat       NullableColumnCreateFunc[float32, *ColumnFloat]     = NewNullableColumnCreator(NewColumnFloat).New
-	NewNullableColumnDouble      NullableColumnCreateFunc[float64, *ColumnDouble]    = NewNullableColumnCreator(NewColumnDouble).New
-	NewNullableColumnTimestamptz NullableColumnCreateFunc[int64, *ColumnTimestamptz] = NewNullableColumnCreator(NewColumnTimestamptz).New
-	NewNullableColumnJSONBytes   NullableColumnCreateFunc[[]byte, *ColumnJSONBytes]  = NewNullableColumnCreator(NewColumnJSONBytes).New
+	NewNullableColumnBool        NullableColumnCreateFunc[bool, *ColumnBool]          = NewNullableColumnCreator(NewColumnBool).New
+	NewNullableColumnInt8        NullableColumnCreateFunc[int8, *ColumnInt8]          = NewNullableColumnCreator(NewColumnInt8).New
+	NewNullableColumnInt16       NullableColumnCreateFunc[int16, *ColumnInt16]        = NewNullableColumnCreator(NewColumnInt16).New
+	NewNullableColumnInt32       NullableColumnCreateFunc[int32, *ColumnInt32]        = NewNullableColumnCreator(NewColumnInt32).New
+	NewNullableColumnInt64       NullableColumnCreateFunc[int64, *ColumnInt64]        = NewNullableColumnCreator(NewColumnInt64).New
+	NewNullableColumnVarChar     NullableColumnCreateFunc[string, *ColumnVarChar]     = NewNullableColumnCreator(NewColumnVarChar).New
+	NewNullableColumnString      NullableColumnCreateFunc[string, *ColumnString]      = NewNullableColumnCreator(NewColumnString).New
+	NewNullableColumnFloat       NullableColumnCreateFunc[float32, *ColumnFloat]      = NewNullableColumnCreator(NewColumnFloat).New
+	NewNullableColumnDouble      NullableColumnCreateFunc[float64, *ColumnDouble]     = NewNullableColumnCreator(NewColumnDouble).New
+	NewNullableColumnTimestamptz NullableColumnCreateFunc[int64, *ColumnTimestamptz]  = NewNullableColumnCreator(NewColumnTimestamptz).New
+	NewNullableColumnJSONBytes   NullableColumnCreateFunc[[]byte, *ColumnJSONBytes]   = NewNullableColumnCreator(NewColumnJSONBytes).New
+	NewNullableColumnGeometryWKT NullableColumnCreateFunc[string, *ColumnGeometryWKT] = NewNullableColumnCreator(NewColumnGeometryWKT).New
 	// array
 	NewNullableColumnBoolArray    NullableColumnCreateFunc[[]bool, *ColumnBoolArray]      = NewNullableColumnCreator(NewColumnBoolArray).New
 	NewNullableColumnInt8Array    NullableColumnCreateFunc[[]int8, *ColumnInt8Array]      = NewNullableColumnCreator(NewColumnInt8Array).New

client/entity/field.go

@@ -56,6 +56,8 @@ func (t FieldType) Name() string {
 		return "Array"
 	case FieldTypeJSON:
 		return "JSON"
+	case FieldTypeGeometry:
+		return "Geometry"
 	case FieldTypeBinaryVector:
 		return "BinaryVector"
 	case FieldTypeFloatVector:
@@ -98,6 +100,8 @@ func (t FieldType) String() string {
 		return "Array"
 	case FieldTypeJSON:
 		return "JSON"
+	case FieldTypeGeometry:
+		return "Geometry"
 	case FieldTypeBinaryVector:
 		return "[]byte"
 	case FieldTypeFloatVector:
@@ -138,6 +142,8 @@ func (t FieldType) PbFieldType() (string, string) {
 		return "VarChar", "string"
 	case FieldTypeJSON:
 		return "JSON", "JSON"
+	case FieldTypeGeometry:
+		return "Geometry", "Geometry"
 	case FieldTypeBinaryVector:
 		return "[]byte", ""
 	case FieldTypeFloatVector:
@@ -181,6 +187,8 @@ const (
 	FieldTypeArray FieldType = 22
 	// FieldTypeJSON field type JSON
 	FieldTypeJSON FieldType = 23
+	// FieldTypeGeometry field type Geometry
+	FieldTypeGeometry FieldType = 24
 	// FieldTypeBinaryVector field type binary vector
 	FieldTypeBinaryVector FieldType = 100
 	// FieldTypeFloatVector field type float vector

client/index/common.go

@@ -67,4 +67,5 @@ const (
 	Sorted   IndexType = "STL_SORT"
 	Inverted IndexType = "INVERTED"
 	BITMAP   IndexType = "BITMAP"
+	RTREE    IndexType = "RTREE"
 )

client/index/rtree.go

@@ -0,0 +1,70 @@
+// Licensed to the LF AI & Data foundation under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package index
+
+var _ Index = rtreeIndex{}
+
+// rtreeIndex represents an RTree index for geometry fields
+type rtreeIndex struct {
+	baseIndex
+}
+
+func (idx rtreeIndex) Params() map[string]string {
+	params := map[string]string{
+		IndexTypeKey: string(RTREE),
+	}
+	return params
+}
+
+// NewRTreeIndex creates a new RTree index with default parameters
+func NewRTreeIndex() Index {
+	return rtreeIndex{
+		baseIndex: baseIndex{
+			indexType: RTREE,
+		},
+	}
+}
+
+// NewRTreeIndexWithParams creates a new RTree index with custom parameters
+func NewRTreeIndexWithParams() Index {
+	return rtreeIndex{
+		baseIndex: baseIndex{
+			indexType: RTREE,
+		},
+	}
+}
+
+// RTreeIndexBuilder provides a fluent API for building RTree indexes
+type RTreeIndexBuilder struct {
+	index rtreeIndex
+}
+
+// NewRTreeIndexBuilder creates a new RTree index builder
+func NewRTreeIndexBuilder() *RTreeIndexBuilder {
+	return &RTreeIndexBuilder{
+		index: rtreeIndex{
+			baseIndex: baseIndex{
+				indexType: RTREE,
+			},
+		},
+	}
+}
+
+// Build returns the constructed RTree index
+func (b *RTreeIndexBuilder) Build() Index {
+	return b.index
+}