Supplement local development guide

shotit · Apr 22, 2023 · 960e0bc · 960e0bc
1 parent e44f84d
commit 960e0bc
Show file tree

Hide file tree

Showing 7 changed files with 489 additions and 30 deletions.
diff --git a/.env.example b/.env.example
@@ -4,9 +4,9 @@ TRACE_ALGO=cl
 # Database setting
 SOLA_DB_HOST=127.0.0.1
 SOLA_DB_PORT=3306
-SOLA_DB_USER=sola
-SOLA_DB_PWD=sola
+SOLA_DB_USER=root
 SOLA_DB_NAME=sola
+SOLA_DB_PWD=sola
 
 # Redis setting
 REDIS_HOST=127.0.0.1
@@ -38,3 +38,8 @@ EMAIL_PASS=
 DISCORD_URL=
 TELEGRAM_ID=
 TELEGRAM_URL=
+
+# Shotit dependency services
+SEARCHER_URL=http://127.0.0.1:19531
+SORTER_URL=http://127.0.0.1:19532
+MILVUS_URL=127.0.0.1:19530
diff --git a/.gitignore b/.gitignore
@@ -102,3 +102,7 @@ dist
 
 # TernJS port file
 .tern-port
+
+# milvus specific files 
+volumes/
+milvus.yaml
diff --git a/Dockerfile b/Dockerfile
@@ -9,6 +9,6 @@ RUN apt-get update && apt-get install -y ffmpeg
 ENV NODE_ENV=production
 WORKDIR /app
 COPY ["package.json", "yarn.lock*", "./"]
-RUN yarn install --production
+RUN yarn install --frozen-lockfile --production
 COPY . .
 CMD [ "node", "server.js" ]
diff --git a/README.md b/README.md
@@ -1,16 +1,13 @@
 # shotit-api
 
-[![License](https://img.shields.io/github/license/soruly/shotit-api.svg?style=flat-square)](https://github.com/soruly/shotit-api/blob/master/LICENSE)
-[![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/soruly/shotit-api/docker-image.yml?style=flat-square)](https://github.com/soruly/shotit-api/actions)
-[![Codecov](https://img.shields.io/codecov/c/github/soruly/shotit-api?style=flat-square&token=8C25WLSEDJ)](https://codecov.io/gh/soruly/shotit-api)
-[![Docker](https://img.shields.io/docker/pulls/soruly/shotit-api?style=flat-square)](https://hub.docker.com/r/soruly/shotit-api)
-[![Docker Image Size](https://img.shields.io/docker/image-size/soruly/shotit-api/latest?style=flat-square)](https://hub.docker.com/r/soruly/shotit-api)
-[![Discord](https://img.shields.io/discord/437578425767559188.svg?style=flat-square)](https://discord.gg/K9jn6Kj)
+[![License](https://img.shields.io/github/license/shotit/shotit-api.svg?style=flat-square)](https://github.com/shotit/shotit-api/blob/master/LICENSE)
+[![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/shotit/shotit-api/docker-image.yml?style=flat-square)](https://github.com/shotit/shotit-api/actions)
+[![Codecov](https://img.shields.io/codecov/c/github/shotit/shotit-api?style=flat-square&token=8C25WLSEDJ)](https://codecov.io/gh/shotit/shotit-api)
+[![Docker](https://img.shields.io/docker/pulls/shotit/shotit-api?style=flat-square)](https://hub.docker.com/r/shotit/shotit-api)
+[![Docker Image Size](https://img.shields.io/docker/image-size/shotit/shotit-api/v0.9.1?style=flat-square)](https://hub.docker.com/r/shotit/shotit-api)
 
 The ultimate brain of [shotit](https://github.com/shotit/shotit), in charge of task coordination.
 
-[API Docs](https://soruly.github.io/shotit-api/)
-
 ### Features
 
 - serve image search request
@@ -22,39 +19,54 @@ The ultimate brain of [shotit](https://github.com/shotit/shotit), in charge of t
 
 ### Prerequisites
 
-- Node.js 14.x
+- Node.js 16.x, 18.x
 - mariaDB 10.4.x
 - redis
-- [liresolr](https://github.com/soruly/liresolr)
+- [liresolr](https://github.com/Leslie-Wong-H/liresolr)
+- [milvus-standalone](https://github.com/milvus-io/milvus)
+- [milvus-minio](https://github.com/milvus-io/milvus)
+- [milvus-etcd](https://github.com/milvus-io/etcd)
+- [shotit-searcher](https://github.com/shotit/shotit-worker)
+- [shotit-sorter(optional)](https://github.com/shotit/shotit-sorter)
 - g++, cmake (if you need to compile OpenCV)
 
-### Install
+### Local Development Guide
 
-Install Prerequisites first, then:
+Install:
 
 ```
-git clone https://github.com/soruly/shotit-api.git
+git clone https://github.com/shotit/shotit-api.git
 cd shotit-api
-npm install
+yarn install
 ```
 
-### Configuration
+Install Prerequisites by docker compose first, [docker-desktop](https://www.docker.com/products/docker-desktop/) required:
 
 - Copy `.env.example` to `.env`
-- Edit `.env` as appropriate for your setup
+- Edit `.env` as appropriate for your setup, as is for the first time.
+- Copy `milvus.yaml.example` to `milvus.yaml`
+- Edit `milvus.yaml` as appropriate for your setup, as is for the first time.
+
+```
+(Windows or Mac):
+docker compose up -d
+(Linux):
+docker-compose up -d
+```
 
 ### Start server
 
 You can use [pm2](https://pm2.keymetrics.io/) to run this in background in cluster mode.
 
-Use below commands to start / restart / stop server.
+Use below commands to start / restart / stop / log server.
 
 ```
-npm run start
-npm run stop
-npm run reload
-npm run restart
-npm run delete
+yarn start
+yarn stop
+yarn reload
+yarn restart
+yarn delete
+yarn logs
 ```
 
 To change the number of nodejs instances, edit ecosystem.config.json
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -2,6 +2,7 @@ version: "3.2"
 
 services:
   mariadb:
+    container_name: shotit-mariadb
     image: mariadb:latest
     restart: unless-stopped
     environment:
@@ -13,6 +14,7 @@ services:
 
 
   redis:
+    container_name: shotit-redis
     image: redis:latest
     ports:
       - ${REDIS_PORT}:6379
@@ -21,15 +23,112 @@ services:
 
 
   liresolr:
-    image: ghcr.io/soruly/liresolr:latest
+    container_name: shotit-liresolr
+    image: ghcr.io/leslie-wong-h/liresolr:latest
     command: solr-precreate cl_0 /opt/solr/server/solr/configsets/liresolr
     restart: unless-stopped
     ports:
-      - 18983:8983
+      - 8983:8983
     volumes:
-      - type: bind
-        source: /var/solr
-        target: /var/solr
+      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/mycores:/var/solr
+    networks:
+      shotit_net:
+
+
+  searcher:
+    container_name: shotit-searcher
+    image: ghcr.io/shotit/shotit-worker-searcher:v0.9.1
+    restart: unless-stopped
+    ports:
+      - 19531:19531
+    environment:
+      - TRACE_ALGO=cl
+      - SOLA_SOLR_LIST=http://172.17.0.1:8983/solr/
+      - SEARCHER_URL=http://172.17.0.1:19531
+      - MILVUS_URL=172.17.0.1:19530
+    networks:
+      shotit_net:
+
+
+  sorter:
+    container_name: shotit-sorter
+    image: ghcr.io/shotit/shotit-sorter:v0.9.1
+    restart: unless-stopped
+    ports:
+      - 19532:19532
+    networks:
+      shotit_net:
+
+
+  etcd:
+    container_name: milvus-etcd
+    image: quay.io/coreos/etcd:v3.5.0
+    restart: unless-stopped
+    environment:
+      - ETCD_AUTO_COMPACTION_MODE=revision
+      - ETCD_AUTO_COMPACTION_RETENTION=1000
+      - ETCD_QUOTA_BACKEND_BYTES=4294967296
+      - ETCD_SNAPSHOT_COUNT=50000
+    volumes:
+      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/etcd:/etcd
+    command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd
+    ports:
+      - "2379:2379"
+    networks:
+      shotit_net:
+
+
+  minio:
+    container_name: milvus-minio
+    image: minio/minio:RELEASE.2022-03-17T06-34-49Z
+    restart: unless-stopped
+    environment:
+      - MINIO_ROOT_USER=minioadmin
+      - MINIO_ROOT_PASSWORD=minioadmin
+      - MINIO_ETCD_ENDPOINTS=http://172.17.0.1:2379
+    volumes:
+      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/minio:/minio_data
+    # Local Minio Server
+    command: minio server /minio_data --console-address ":9001"
+    # Remote Minio Gateway
+    # command: minio gateway s3 ${AWS_ENDPOINT_URL} --console-address ":9001"
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    depends_on:
+      - "etcd"
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "curl",
+          "-f",
+          "http://localhost:9000/minio/health/live"
+        ]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+    networks:
+      shotit_net:
+
+
+  standalone:
+    container_name: milvus-standalone
+    image: milvusdb/milvus:v2.2.2
+    restart: unless-stopped
+    command: [ "milvus", "run", "standalone" ]
+    environment:
+      ETCD_ENDPOINTS: etcd:2379
+      MINIO_ADDRESS: minio:9000
+    volumes:
+      - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/milvus:/var/lib/milvus
+      - ${DOCKER_VOLUME_DIRECTORY:-.}/milvus.yaml:/milvus/configs/milvus.yaml
+    ports:
+      - "19530:19530"
+      - "9091:9091"
+    depends_on:
+      - "etcd"
+      - "minio"
     networks:
       shotit_net: