[test]Add method to analyze chaos test result (milvus-io#26724)

Signed-off-by: zhuwenxing <wenxing.zhu@zilliz.com>

Author: zhuwenxing

tests/benchmark/requirements.txt

@@ -2,12 +2,12 @@
 # --extra-index-url https://test.pypi.org/simple/
 # pymilvus==2.0.0rc3.dev8
 
-grpcio==1.37.1
+grpcio==1.53.0
 grpcio-testing==1.37.1
 grpcio-tools==1.37.1
 
 pandas==1.1.5
-scipy==1.3.1
+scipy==1.10.0
 scikit-learn==0.19.1
 h5py==2.7.1
 # influxdb==5.2.2

tests/python_client/chaos/checker.py

@@ -9,8 +9,9 @@ def pytest_addoption(parser):
     parser.addoption("--target_number", action="store", default="1", help="target_number")
     parser.addoption("--chaos_duration", action="store", default="1m", help="chaos_duration")
     parser.addoption("--chaos_interval", action="store", default="10s", help="chaos_interval")
-    parser.addoption("--request_duration", action="store", default="3m", help="request_duration")
-    parser.addoption("--is_check", action="store", type=bool, default=False, help="is_check")   
+    parser.addoption("--request_duration", action="store", default="5m", help="request_duration")
+    parser.addoption("--is_check", action="store", type=bool, default=False, help="is_check")
+    parser.addoption("--wait_signal", action="store", type=bool, default=True, help="wait_signal")
 
 
 @pytest.fixture
@@ -56,3 +57,8 @@ def request_duration(request):
 @pytest.fixture
 def is_check(request):
     return request.config.getoption("--is_check")
+
+
+@pytest.fixture
+def wait_signal(request):
+    return request.config.getoption("--wait_signal")

tests/python_client/chaos/conftest.py

@@ -22,4 +22,5 @@
 WAIT_PER_OP = 10  # time to wait in seconds between operations
 CHAOS_DURATION = 120  # chaos duration time in seconds
 DEFAULT_INDEX_PARAM = {"index_type": "HNSW", "metric_type": "L2", "params": {"M": 48, "efConstruction": 500}}
-DEFAULT_SEARCH_PARAM = {"metric_type": "L2", "params": {"ef": 64}}
+DEFAULT_SEARCH_PARAM = {"metric_type": "L2", "params": {"ef": 64}}
+CHAOS_INFO_SAVE_PATH = "/tmp/ci_logs/chaos_info.json"

tests/python_client/chaos/constants.py

@@ -0,0 +1,5 @@
+
+# for test result anaylszer
+prettytable==3.8.0
+pyarrow==11.0.0
+fastparquet==2023.7.0

tests/python_client/chaos/requirements.txt

@@ -3,12 +3,14 @@
 import time
 from time import sleep
 from pathlib import Path
+import json
 from pymilvus import connections
 from common.cus_resource_opts import CustomResourceOperations as CusResource
 from common.milvus_sys import MilvusSys
-import logging as log
+from utils.util_log import test_log as log
+from datetime import datetime
 from utils.util_k8s import wait_pods_ready, get_milvus_instance_name, get_milvus_deploy_tool
-from utils.util_common import update_key_value, update_key_name, gen_experiment_config
+from utils.util_common import update_key_value, update_key_name, gen_experiment_config, wait_signal_to_apply_chaos
 import constants
 
 
@@ -54,9 +56,17 @@ def teardown(self):
         chaos_res.delete(meta_name, raise_ex=False)
         sleep(2)
 
-    def test_chaos_apply(self, chaos_type, target_component, target_number, chaos_duration, chaos_interval):
+    def test_chaos_apply(self, chaos_type, target_component, target_number, chaos_duration, chaos_interval, wait_signal):
         # start the monitor threads to check the milvus ops
         log.info("*********************Chaos Test Start**********************")
+        if wait_signal:
+            log.info("need wait signal to start chaos")
+            ready_for_chaos = wait_signal_to_apply_chaos()
+            if not ready_for_chaos:
+                log.info("did not get the signal to apply chaos")
+                raise Exception
+            else:
+                log.info("get the signal to apply chaos")
         log.info(connections.get_connection_addr('default'))
         release_name = self.release_name
         chaos_config = gen_experiment_config(
@@ -88,6 +98,7 @@ def test_chaos_apply(self, chaos_type, target_component, target_number, chaos_du
                                 version=constants.CHAOS_VERSION,
                                 namespace=constants.CHAOS_NAMESPACE)
         chaos_res.create(chaos_config)
+        create_time = datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S.%f')
         log.info("chaos injected")
         res = chaos_res.list_all()
         chaos_list = [r['metadata']['name'] for r in res['items']]
@@ -97,6 +108,7 @@ def test_chaos_apply(self, chaos_type, target_component, target_number, chaos_du
         sleep(chaos_duration)
         # delete chaos
         chaos_res.delete(meta_name)
+        delete_time = datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S.%f')
         log.info("chaos deleted")
         res = chaos_res.list_all()
         chaos_list = [r['metadata']['name'] for r in res['items']]
@@ -114,6 +126,18 @@ def test_chaos_apply(self, chaos_type, target_component, target_number, chaos_du
         log.info("all pods are ready")
         pods_ready_time = time.time() - t0
         log.info(f"pods ready time: {pods_ready_time}")
+        recovery_time = datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S.%f')
+        event_records = {
+            "chaos_type": chaos_type,
+            "target_component": target_component,
+            "meta_name": meta_name,
+            "create_time": create_time,
+            "delete_time": delete_time,
+            "recovery_time": recovery_time
+        }
+        # save event records to json file
+        with open(constants.CHAOS_INFO_SAVE_PATH, 'w') as f:
+            json.dump(event_records, f)
         # reconnect to test the service healthy
         start_time = time.time()
         end_time = start_time + 120
@@ -125,5 +149,6 @@ def test_chaos_apply(self, chaos_type, target_component, target_number, chaos_du
                 log.error(e)
                 sleep(2)
         recovery_time = time.time() - start_time
-        log.info(f"recovery time: {recovery_time}")
+        log.info(f"recovery time from pod ready to can be connected: {recovery_time}")
+
         log.info("*********************Chaos Test Completed**********************")

tests/python_client/chaos/test_chaos_apply.py

@@ -1,4 +1,4 @@
-import threading
+import time
 import pytest
 import json
 from time import sleep
@@ -9,40 +9,20 @@
                            SearchChecker,
                            QueryChecker,
                            DeleteChecker,
-                           Op)
-from common.cus_resource_opts import CustomResourceOperations as CusResource
+                           Op,
+                           ResultAnalyzer
+                           )
+from utils.util_k8s import wait_pods_ready, get_milvus_instance_name
 from utils.util_log import test_log as log
 from chaos import chaos_commons as cc
 from common import common_func as cf
+from common.milvus_sys import MilvusSys
 from chaos.chaos_commons import assert_statistic
 from common.common_type import CaseLabel
 from chaos import constants
 from delayed_assert import expect, assert_expectations
 
 
-def assert_statistic(checkers, expectations={}):
-    for k in checkers.keys():
-        # expect succ if no expectations
-        succ_rate = checkers[k].succ_rate()
-        total = checkers[k].total()
-        average_time = checkers[k].average_time
-        if 'compact' in str(k):
-            log.info("skip compact check")
-            log.info(
-                f"Expect Succ: {str(k)} succ rate {succ_rate}, total: {total}, average time: {average_time:.4f}")
-            continue           
-        if expectations.get(k, '') == constants.FAIL:
-            log.info(
-                f"Expect Fail: {str(k)} succ rate {succ_rate}, total: {total}, average time: {average_time:.4f}")
-            expect(succ_rate < 0.49 or total < 2,
-                   f"Expect Fail: {str(k)} succ rate {succ_rate}, total: {total}, average time: {average_time:.4f}")
-        else:
-            log.info(
-                f"Expect Succ: {str(k)} succ rate {succ_rate}, total: {total}, average time: {average_time:.4f}")
-            expect(succ_rate > 0.90 and total > 2,
-                   f"Expect Succ: {str(k)} succ rate {succ_rate}, total: {total}, average time: {average_time:.4f}")
-
-
 def get_all_collections():
     try:
         with open("/tmp/ci_logs/all_collections.json", "r") as f:
@@ -70,7 +50,7 @@ class TestBase:
 class TestOperations(TestBase):
 
     @pytest.fixture(scope="function", autouse=True)
-    def connection(self, host, port, user, password):
+    def connection(self, host, port, user, password, milvus_ns):
         if user and password:
             # log.info(f"connect to {host}:{port} with user {user} and password {password}")
             connections.connect('default', host=host, port=port, user=user, password=password, secure=True)
@@ -82,7 +62,10 @@ def connection(self, host, port, user, password):
         self.host = host
         self.port = port
         self.user = user
-        self.password = password        
+        self.password = password
+        self.milvus_sys = MilvusSys(alias='default')
+        self.milvus_ns = milvus_ns
+        self.release_name = get_milvus_instance_name(self.milvus_ns, milvus_sys=self.milvus_sys)
 
     def init_health_checkers(self, collection_name=None):
         c_name = collection_name
@@ -91,7 +74,7 @@ def init_health_checkers(self, collection_name=None):
             Op.flush: FlushChecker(collection_name=c_name),
             Op.search: SearchChecker(collection_name=c_name),
             Op.query: QueryChecker(collection_name=c_name),
-            Op.compact:CompactChecker(collection_name=c_name),
+            Op.compact: CompactChecker(collection_name=c_name),
             Op.delete: DeleteChecker(collection_name=c_name),
         }
         self.health_checkers = checkers
@@ -107,11 +90,14 @@ def test_operations(self, request_duration, is_check, collection_name):
         # start the monitor threads to check the milvus ops
         log.info("*********************Test Start**********************")
         log.info(connections.get_connection_addr('default'))
+        # event_records = EventRecords()
         c_name = collection_name if collection_name else cf.gen_unique_str("Checker_")
+        # event_records.insert("init_health_checkers", "start")
         self.init_health_checkers(collection_name=c_name)
+        # event_records.insert("init_health_checkers", "finished")
         cc.start_monitor_threads(self.health_checkers)
         log.info("*********************Load Start**********************")
-        request_duration = request_duration.replace("h","*3600+").replace("m","*60+").replace("s","")
+        request_duration = request_duration.replace("h", "*3600+").replace("m", "*60+").replace("s", "")
         if request_duration[-1] == "+":
             request_duration = request_duration[:-1]
         request_duration = eval(request_duration)
@@ -120,7 +106,11 @@ def test_operations(self, request_duration, is_check, collection_name):
             for k, v in self.health_checkers.items():
                 v.check_result()
                 # log.info(v.check_result())
+        wait_pods_ready(self.milvus_ns, f"app.kubernetes.io/instance={self.release_name}")
+        time.sleep(60)
+        ra = ResultAnalyzer()
+        ra.get_stage_success_rate()
         if is_check:
             assert_statistic(self.health_checkers)
-            assert_expectations()        
+            assert_expectations()
         log.info("*********************Chaos Test Completed**********************")

tests/python_client/chaos/testcases/test_concurrent_operation.py

@@ -11,11 +11,15 @@
                            IndexChecker,
                            DeleteChecker,
                            DropChecker,
-                           Op)
+                           Op,
+                           EventRecords,
+                           ResultAnalyzer
+                           )
 from utils.util_log import test_log as log
 from utils.util_k8s import wait_pods_ready, get_milvus_instance_name
 from chaos import chaos_commons as cc
 from common.common_type import CaseLabel
+from common.milvus_sys import MilvusSys
 from chaos.chaos_commons import assert_statistic
 from chaos import constants
 from delayed_assert import assert_expectations
@@ -50,6 +54,7 @@ def connection(self, host, port, user, password, milvus_ns):
         self.port = port
         self.user = user
         self.password = password
+        self.milvus_sys = MilvusSys(alias='default')
         self.milvus_ns = milvus_ns
         self.release_name = get_milvus_instance_name(self.milvus_ns, milvus_sys=self.milvus_sys)
 
@@ -72,8 +77,11 @@ def test_operations(self, request_duration, is_check):
         # start the monitor threads to check the milvus ops
         log.info("*********************Test Start**********************")
         log.info(connections.get_connection_addr('default'))
+        event_records = EventRecords()
         c_name = None
+        event_records.insert("init_health_checkers", "start")
         self.init_health_checkers(collection_name=c_name)
+        event_records.insert("init_health_checkers", "finished")
         cc.start_monitor_threads(self.health_checkers)
         log.info("*********************Load Start**********************")
         # wait request_duration
@@ -83,6 +91,9 @@ def test_operations(self, request_duration, is_check):
         request_duration = eval(request_duration)
         for i in range(10):
             sleep(request_duration // 10)
+            # add an event so that the chaos can start to apply
+            if i == 3:
+                event_records.insert("init_chaos", "ready")
             for k, v in self.health_checkers.items():
                 v.check_result()
         if is_check:
@@ -91,4 +102,9 @@ def test_operations(self, request_duration, is_check):
         # wait all pod ready
         wait_pods_ready(self.milvus_ns, f"app.kubernetes.io/instance={self.release_name}")
         time.sleep(60)
+        for k, v in self.health_checkers.items():
+            v.pause()
+        ra = ResultAnalyzer()
+        ra.get_stage_success_rate()
+        ra.show_result_table()
         log.info("*********************Chaos Test Completed**********************")

tests/python_client/chaos/testcases/test_single_request_operation.py

@@ -1,5 +1,10 @@
 --extra-index-url https://test.pypi.org/simple/
 docker==5.0.0
-grpcio==1.37.1
+grpcio==1.53.0
 grpcio-tools==1.37.1
-pymilvus==2.0.0rc8
+pymilvus==2.0.0rc8
+
+# for test result anaylszer
+prettytable==3.8.0
+pyarrow==11.0.0
+fastparquet==2023.7.0

tests/python_client/deploy/requirements.txt

@@ -39,9 +39,15 @@ Faker==19.2.0
 h5py==3.8.0
 
 # for log
-loguru==0.6.0
+loguru==0.7.0
 
 # util
 psutil==5.9.4
+pandas==1.5.3
 # for standby test
 etcd-sdk-python==0.0.2
+
+# for test result anaylszer
+prettytable==3.8.0
+pyarrow==11.0.0
+fastparquet==2023.7.0