Renew milvus2-sdk-node to fit the latest practices & rename watch.js …

…to watcher.js (#49)
shotit · Jul 9, 2023 · 150d0c7 · 150d0c7
2 parents e3a182d + 94670aa
commit 150d0c7
Show file tree

Hide file tree

Showing 7 changed files with 249 additions and 361 deletions.
diff --git a/ecosystem.config.json b/ecosystem.config.json
@@ -16,7 +16,7 @@
     },
     {
       "name": "shotit-watcher",
-      "script": "watch.js",
+      "script": "watcher.js",
       "instances": 1,
       "autorestart": true,
       "exec_mode": "fork"

diff --git a/loader.js b/loader.js
@@ -30,24 +30,18 @@ const initializeMilvusCollection = async () => {
         name: "cl_ha",
         description: "Dynamic fields for LIRE Solr",
         data_type: 101, // DataType.FloatVector
-        type_params: {
-          dim: "100",
-        },
+        dim: 100,
       },
       // {
       //   name: "cl_hi",
-      //   data_type: 21, //DataType.VARCHAR
-      //   type_params: {
-      //     max_length: "200",
-      //   },
+      //   data_type: 21, //DataType.VarChar
+      //   max_length: 200,
       //   description: "Metric Spaces Indexing",
       // },
       {
         name: "id",
-        data_type: 21, //DataType.VARCHAR
-        type_params: {
-          max_length: "500",
-        },
+        data_type: 21, //DataType.VarChar
+        max_length: 500,
         description: "${imdbID}/${fileName}/${time}",
       },
       {
@@ -61,8 +55,8 @@ const initializeMilvusCollection = async () => {
 
   const fallBack = async () => {
     try {
-      await milvusClient.collectionManager.releaseCollection({ collection_name: "shotit" });
-      await milvusClient.collectionManager.createCollection(params);
+      await milvusClient.releaseCollection({ collection_name: "shotit" });
+      await milvusClient.createCollection(params);
       console.log('collection_name: "shotit" ensured');
       milvusClient.closeConnection();
     } catch (error) {
@@ -220,20 +214,20 @@ const messageHandle = async (data) => {
       // Insert at a batch of 2 thousand each time, if more than that
       let loopCount = jsonData.length / 2000;
       if (loopCount <= 1) {
-        await milvusClient.dataManager.insert({
+        await milvusClient.insert({
           collection_name: "shotit",
           fields_data: jsonData,
         });
       } else {
         for (let i = 0; i < Math.ceil(loopCount); i++) {
           if (i === Math.ceil(loopCount) - 1) {
-            await milvusClient.dataManager.insert({
+            await milvusClient.insert({
               collection_name: "shotit",
               fields_data: jsonData.slice(i * 2000),
             });
             break;
           }
-          await milvusClient.dataManager.insert({
+          await milvusClient.insert({
             collection_name: "shotit",
             fields_data: jsonData.slice(i * 2000, i * 2000 + 2000),
           });
@@ -248,7 +242,7 @@ const messageHandle = async (data) => {
 
       // let loopCount = jsonData.length / 10000;
       // if (loopCount <= 1) {
-      //   await milvusClient.dataManager.insert({
+      //   await milvusClient.insert({
       //     collection_name: "shotit",
       //     fields_data: jsonData,
       //   });
@@ -263,7 +257,7 @@ const messageHandle = async (data) => {
       //   }
       //   await Promise.all(
       //     batchList.map(async (batch) => {
-      //       await milvusClient.dataManager.insert({
+      //       await milvusClient.insert({
       //         collection_name: "shotit",
       //         fields_data: batch,
       //       });
@@ -275,7 +269,7 @@ const messageHandle = async (data) => {
 
       startTime = performance.now();
       console.log("Flush begins", startTime);
-      await milvusClient.dataManager.flushSync({ collection_names: ["shotit"] });
+      await milvusClient.flushSync({ collection_names: ["shotit"] });
       console.log("Flush done", performance.now() - startTime);
 
       const index_params = {
@@ -286,7 +280,7 @@ const messageHandle = async (data) => {
 
       startTime = performance.now();
       console.log("Index begins", startTime);
-      await milvusClient.indexManager.createIndex({
+      await milvusClient.createIndex({
         collection_name: "shotit",
         field_name: "cl_ha",
         extra_params: index_params,
@@ -302,7 +296,7 @@ const messageHandle = async (data) => {
       // console.log("Load begins", startTime);
       // // Sync trick to prevent gRPC overload so that the follwing large-volume insert
       // // operation would not cause "Error: 14 UNAVAILABLE: Connection dropped"
-      // await milvusClient.collectionManager.loadCollectionSync({
+      // await milvusClient.loadCollectionSync({
       //   collection_name: "shotit",
       // });
       // console.log("Load done", performance.now() - startTime);
@@ -345,7 +339,7 @@ const closeHandle = async () => {
     startTime = performance.now();
     const milvusClient = new MilvusClient(MILVUS_URL);
     console.log("Flush begins", startTime);
-    await milvusClient.dataManager.flushSync({ collection_names: ["shotit"] });
+    await milvusClient.flushSync({ collection_names: ["shotit"] });
     console.log("Flush done", performance.now() - startTime);
     milvusClient.closeConnection();
   });

diff --git a/package.json b/package.json
@@ -27,7 +27,7 @@
   },
   "homepage": "https://github.com/shotit/shotit-worker#readme",
   "dependencies": {
-    "@zilliz/milvus2-sdk-node": "2.1.2",
+    "@zilliz/milvus2-sdk-node": "2.2.18",
     "body-parser": "^1.20.0",
     "chokidar": "^3.5.3",
     "dotenv": "^16.3.1",

diff --git a/searcher.js b/searcher.js
@@ -60,7 +60,7 @@ const getNormalizedCharCodesVector = (str, length = 100, base = 1) => {
 const search = async (hash) => {
   const milvusClient = new MilvusClient(MILVUS_URL);
 
-  await milvusClient.collectionManager.loadCollectionSync({
+  await milvusClient.loadCollectionSync({
     collection_name: "shotit",
   });
 
@@ -81,7 +81,7 @@ const search = async (hash) => {
 
   console.log(normalizedCharCodesVector);
 
-  const results = await milvusClient.dataManager.search({
+  const results = await milvusClient.search({
     collection_name: "shotit",
     expr: "",
     vectors: [normalizedCharCodesVector],

diff --git a/watcher.Dockerfile b/watcher.Dockerfile
@@ -7,5 +7,5 @@ ENV NODE_ENV=production
 WORKDIR /app
 COPY ["package.json", "yarn.lock*", "./"]
 RUN yarn install --frozen-lockfile --production
-COPY watch.js ./
-CMD [ "node", "watch.js" ]
+COPY watcher.js ./
+CMD [ "node", "watcher.js" ]
diff --git a/watch.js → watcher.js b/watch.js → watcher.js