first refactor

Author: JoyboyWang

.DS_Store

@@ -7,7 +7,7 @@
 
 def ClearDir(dirpath):
     if os.path.exists(dirpath):
-        print("正在删除.....", dirpath)
+        print("Deleting...", dirpath)
         shutil.rmtree(path=dirpath)
     os.makedirs(dirpath)
 
@@ -31,7 +31,7 @@ def GetPointSets(pts_size,
                  half_space_dist_,
                  dim,
                  gauss_delta_,
-                 _means=np.array([])):  # 获得最初的point set（包含num_clusters个簇，每个簇
+                 _means=np.array([])):  # generate gaussian clusters with specified parameters
     _points = []
     _labels = []
     # 生成一个多维高斯分布
@@ -62,17 +62,12 @@ def GetPointSets(pts_size,
 def GenDistubIds(pts_size, keep_ratio):
     ids = range(0, pts_size)
 
-    # keep_ids 相似性的点
-    # dist_ids 不相似的点
     keep_ids = random.sample(ids, int(keep_ratio * pts_size))
     keep_ids.sort()
     dist_ids = [i for i in ids if i not in keep_ids]
 
     return keep_ids, dist_ids
 
-    # keep_ids, dist_ids = GenDistubIds(pts_size, keep_ratio)
-
-
 # disturb given points
 def DisturbPoints(inputs, dim, keep_ids, dist_ids, disturb_dist, HARD_MOVE):
     output = inputs
@@ -91,8 +86,6 @@ def DisturbPoints(inputs, dim, keep_ids, dist_ids, disturb_dist, HARD_MOVE):
 def DisturbClusters(inputs, dim, labels, disturb_label, pts_size, disturb_dist,
                     HARD_MOVE):
     ids = range(0, pts_size)
-    # keep_ids 相似性的点
-    # dist_ids 不相似的点
     dist_ids = [i for i in range(len(labels)) if labels[i] == disturb_label]
     keep_ids = [i for i in ids if i not in dist_ids]
 
@@ -128,8 +121,6 @@ def DisturbPointSets(inputs, labels, disturb_label_num, keep_ratio):
 
     output = inputs
     ids = range(0, pts_size)
-    # keep_ids 相似性的点
-    # dist_ids 不相似的点
     disturb_labels = random.sample(range(num_clusters), disturb_label_num)
 
     num_each_cluster = int((1. - keep_ratio) * pts_size / disturb_label_num)
@@ -194,23 +185,19 @@ def overlapClusters(inputs, pts_size, dim, means, labels, merge_labels):
     ids = range(0, pts_size)
 
     merge_mean = np.zeros((dim))
-    # 计算出要合并的几个簇中心的中心
+    # the center of several clusters
     for label in merge_labels:
         merge_mean += means[label]
     merge_mean /= len(merge_labels)
 
     for id in ids:
         # move the cluster center to the same location
         if labels[id] in merge_labels:
-            # print(output[id, :].shape)
-            # print(merge_mean.shape)
-            # print(means[labels[id], :].shape)
             output[id, :] += merge_mean - means[labels[id], :].reshape(dim, )
 
     return output
 
 
-# 缩小整个类簇而不改变knn
 def scaleCluster(inputs,
                  pts_size,
                  dim,
@@ -228,7 +215,6 @@ def scaleCluster(inputs,
                                   pts_size=pts_size,
                                   dim=dim)
 
-    # 将所有点围绕该中心进行缩放,并移动到
     shrink_ids = [id for id in ids if labels[id] == shrink_label]
     for id in shrink_ids:
         output[id] = new_center + (output[id] - scale_center) * scale_factor
@@ -312,63 +298,3 @@ def DistOfEdges(dists, indices):
 def writeInfo(filepath, info):
     with open(filepath, 'w', encoding='utf-8') as f:
         json.dump(info, f)
-
-
-# # Amplify the cluster
-# def DiffuseCluster(inputs, labels, disturb_label):
-#     output = inputs
-#     assert(disturb_label>=0 and disturb_label<=num_clusters)
-
-#     # compute the centroid of the cluster
-#     disturb_ids = [i for i in range(len(labels)) if labels[i] == disturb_label]
-#     disturb_inputs = [inputs[id] for id in disturb_ids]
-#     disturb_inputs = np.array(disturb_ids)
-
-#     centroid = np.sum(disturb_ids, 0) / disturb_ids.shape(0)
-
-#     # for each point, compute the vector
-#     for i in disturb_inputs.shape[0]:
-#         # each point go through the vector
-#         output[i] +=
-
-#     return output
-
-# # random select keeping edges. NEVER USE IT
-# def DisturbEdges(inputs, kd_tree, keep_ratio=0.7):
-#     outputs = inputs
-#     dists, indices = kd_tree.query(
-#         inputs, k=k_closest_count)  # 一口气对所有points构建knn
-#     edge_size = indices.shape[0]*(k_closest_count-1)
-
-#     np.arange(0, points.shape[0])
-#     ids = range(0, pts_size)
-
-#     # keep_edges 相似性的边
-#     keep_edges = []
-#     while len(keep_edges) != edge_size*keep_ratio:
-#         keep_ids_0 = np.random.randint(0, pts_size)
-#         keep_ids_1 = np.random.randint(0, pts_size)
-#         # ensure no self-loop and duplicate
-#         if keep_ids_0 == keep_ids_1 or [keep_ids_0, keep_ids_1] in keep_edges:
-#             continue
-#         keep_edges.append([keep_ids_0, keep_ids_1])
-
-#     keep_edges.sort()
-#     # dist_edges 不相似的边
-#     dist_edges = []
-#     for i in range(len(indices)):
-#         for j in indices[i]:
-#             # print(i)
-#             # print(j)
-#             if [i, j] not in keep_edges:
-#                 dist_edges.append([i, j])
-
-#     # disturb edge endpoints
-#     for [i, j] in dist_edges:
-#         _moveVec_i = np.random.uniform(-0.5, 0.5, (dim))
-#         _moveVec_j = np.random.uniform(-0.5, 0.5, (dim))
-
-#         outputs[i] += _moveVec_i
-#         outputs[j] += _moveVec_j
-
-#     return outputs, keep_edges

codes/.DS_Store

@@ -1,11 +1,8 @@
-#!/usr/bin/env python3
 # encoding = utf-8
 import os
 import sys
 import numpy as np
 from sklearn.neighbors import KDTree
-import shutil
-import random
 import json
 import time
 
@@ -17,7 +14,7 @@
 import dg_utils
 
 
-# read point sets and configuration from file.
+# read high dimensional dataset from file
 def ReadPointSets(filepath, norm_data):
     _points = []
     _labels = []
@@ -42,7 +39,6 @@ def ReadPointSets(filepath, norm_data):
             _flag = True
 
             _points.append(_point)
-            # label
             _labels.append(items[-1])
 
     p = np.array(_points)
@@ -57,19 +53,20 @@ def ReadPointSets(filepath, norm_data):
 def sav_graph(filepath, points, labels, k_closest_count):
     tree = KDTree(points)
     dists, indices = tree.query(points,
-                                k=k_closest_count + 1)  # 一口气对所有points构建knn
+                                k=k_closest_count + 1)  # build knn graph from points
 
-    with open(filepath, 'w') as file:  # 打开新文件fm_{0}.txt
-        # first write in the number of nodes
-        file.write(str(points.shape[0]) + "\n")  # 写入当前点
+    with open(filepath, 'w') as file:
+        # write in the number of nodes
+        file.write(str(points.shape[0]) + "\n")
 
+        # write each point
         for i in range(points.shape[0]):
-            file.write(str(i) + "\t")  # 写入当前点
+            file.write(str(i) + "\t")
             count = 0
             for t in range(indices[i].shape[0]):
-                if indices[i][t] == i:  # 是否包含自身
+                if indices[i][t] == i:
                     continue
-                if count == k_closest_count:  # 只写入Indices中前k-1个点
+                if count == k_closest_count:  # write the k_closest_count neighbors
                     break
                 # write incient point and corresponding distance
                 file.write(str(indices[i][t]) + "\t" + str(dists[i][t]) + "\t")
@@ -80,7 +77,7 @@ def sav_graph(filepath, points, labels, k_closest_count):
 if __name__ == '__main__':
     argv = sys.argv
     assert (len(argv) == 2)
-    config_path = argv[1]  #"../../config/config_0.json"
+    config_path = argv[1]
 
     start = time.perf_counter()
     with open(config_path, 'r') as f:
@@ -91,7 +88,7 @@ def sav_graph(filepath, points, labels, k_closest_count):
         graph_path = config.graph_dir
         dg_utils.ClearDir(graph_path)
 
-        k_closest_count = config.k_closest_count  #min(3*perplexity, pts_size)   # K近邻的个数+1（虽然是K=4，但由于包含自身，实际为K-1邻近）
+        k_closest_count = config.k_closest_count
 
         raw_files = []
         for filename in os.listdir(input_dir):
@@ -100,17 +97,16 @@ def sav_graph(filepath, points, labels, k_closest_count):
             if ext == ".txt":
                 raw_files.append(raw_file)
 
-        # size_dims = []
         for filepath in raw_files:
             data_id = dg_utils.GetGraphIDFromPath(filepath)
-            print("当前处理: " + str(data_id) + " Graph")
+            print("Processing graph " + str(data_id))
 
             cur_points, labels, pts_size, dim = ReadPointSets(filepath,
                                                               norm_data=True)
             print((pts_size, dim))
 
             sav_graph(os.path.join(graph_path, "g_{}.txt".format(data_id)),
-                      cur_points, labels, k_closest_count)  # 打开新文件fm_{1}.txt
+                      cur_points, labels, k_closest_count)
 
     elapsed = (time.perf_counter() - start)
     print("Total time for building knn graph:", elapsed)

codes/graphBuild/dg_utils.py

@@ -0,0 +1,44 @@
+QMAKE_MAC_SDK.macosx.Path = /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX11.1.sdk
+QMAKE_MAC_SDK.macosx.PlatformPath = /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform
+QMAKE_MAC_SDK.macosx.SDKVersion = 11.1
+QMAKE_MAC_SDK.macx-clang.macosx.QMAKE_CC = /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang
+QMAKE_MAC_SDK.macx-clang.macosx.QMAKE_CXX = /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang++
+QMAKE_MAC_SDK.macx-clang.macosx.QMAKE_FIX_RPATH = \
+    /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/install_name_tool \
+    -id
+QMAKE_MAC_SDK.macx-clang.macosx.QMAKE_AR = \
+    /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/ar \
+    cq
+QMAKE_MAC_SDK.macx-clang.macosx.QMAKE_RANLIB = \
+    /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/ranlib \
+    -s
+QMAKE_MAC_SDK.macx-clang.macosx.QMAKE_LINK = /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang++
+QMAKE_MAC_SDK.macx-clang.macosx.QMAKE_LINK_SHLIB = /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang++
+QMAKE_MAC_SDK.macx-clang.macosx.QMAKE_ACTOOL = /Applications/Xcode.app/Contents/Developer/usr/bin/actool
+QMAKE_MAC_SDK.macx-clang.macosx.QMAKE_LINK_C = /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang
+QMAKE_MAC_SDK.macx-clang.macosx.QMAKE_LINK_C_SHLIB = /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang
+QMAKE_CXX.QT_COMPILER_STDCXX = 199711L
+QMAKE_CXX.QMAKE_APPLE_CC = 6000
+QMAKE_CXX.QMAKE_APPLE_CLANG_MAJOR_VERSION = 12
+QMAKE_CXX.QMAKE_APPLE_CLANG_MINOR_VERSION = 0
+QMAKE_CXX.QMAKE_APPLE_CLANG_PATCH_VERSION = 0
+QMAKE_CXX.QMAKE_GCC_MAJOR_VERSION = 4
+QMAKE_CXX.QMAKE_GCC_MINOR_VERSION = 2
+QMAKE_CXX.QMAKE_GCC_PATCH_VERSION = 1
+QMAKE_CXX.COMPILER_MACROS = \
+    QT_COMPILER_STDCXX \
+    QMAKE_APPLE_CC \
+    QMAKE_APPLE_CLANG_MAJOR_VERSION \
+    QMAKE_APPLE_CLANG_MINOR_VERSION \
+    QMAKE_APPLE_CLANG_PATCH_VERSION \
+    QMAKE_GCC_MAJOR_VERSION \
+    QMAKE_GCC_MINOR_VERSION \
+    QMAKE_GCC_PATCH_VERSION
+QMAKE_CXX.INCDIRS = \
+    /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/include/c++/v1 \
+    /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/clang/12.0.0/include \
+    /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX11.1.sdk/usr/include \
+    /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/include
+QMAKE_CXX.LIBDIRS = /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX11.1.sdk/usr/lib
+QMAKE_XCODE_DEVELOPER_PATH = /Applications/Xcode.app/Contents/Developer
+QMAKE_XCODE_VERSION = 12.3