diff --git a/.env.example b/.env.example index 15990de..3505b2b 100644 --- a/.env.example +++ b/.env.example @@ -4,9 +4,9 @@ TRACE_ALGO=cl # Database setting SOLA_DB_HOST=127.0.0.1 SOLA_DB_PORT=3306 -SOLA_DB_USER=sola -SOLA_DB_PWD=sola +SOLA_DB_USER=root SOLA_DB_NAME=sola +SOLA_DB_PWD=sola # Redis setting REDIS_HOST=127.0.0.1 @@ -38,3 +38,8 @@ EMAIL_PASS= DISCORD_URL= TELEGRAM_ID= TELEGRAM_URL= + +# Shotit dependency services +SEARCHER_URL=http://127.0.0.1:19531 +SORTER_URL=http://127.0.0.1:19532 +MILVUS_URL=127.0.0.1:19530 \ No newline at end of file diff --git a/.gitignore b/.gitignore index 6704566..7d301d2 100644 --- a/.gitignore +++ b/.gitignore @@ -102,3 +102,7 @@ dist # TernJS port file .tern-port + +# milvus specific files +volumes/ +milvus.yaml \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index ff52a4b..928ec0a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,6 +9,6 @@ RUN apt-get update && apt-get install -y ffmpeg ENV NODE_ENV=production WORKDIR /app COPY ["package.json", "yarn.lock*", "./"] -RUN yarn install --production +RUN yarn install --frozen-lockfile --production COPY . . CMD [ "node", "server.js" ] diff --git a/README.md b/README.md index c3d1a6f..cc07f7e 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,13 @@ # shotit-api -[![License](https://img.shields.io/github/license/soruly/shotit-api.svg?style=flat-square)](https://github.com/soruly/shotit-api/blob/master/LICENSE) -[![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/soruly/shotit-api/docker-image.yml?style=flat-square)](https://github.com/soruly/shotit-api/actions) -[![Codecov](https://img.shields.io/codecov/c/github/soruly/shotit-api?style=flat-square&token=8C25WLSEDJ)](https://codecov.io/gh/soruly/shotit-api) -[![Docker](https://img.shields.io/docker/pulls/soruly/shotit-api?style=flat-square)](https://hub.docker.com/r/soruly/shotit-api) -[![Docker Image Size](https://img.shields.io/docker/image-size/soruly/shotit-api/latest?style=flat-square)](https://hub.docker.com/r/soruly/shotit-api) -[![Discord](https://img.shields.io/discord/437578425767559188.svg?style=flat-square)](https://discord.gg/K9jn6Kj) +[![License](https://img.shields.io/github/license/shotit/shotit-api.svg?style=flat-square)](https://github.com/shotit/shotit-api/blob/master/LICENSE) +[![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/shotit/shotit-api/docker-image.yml?style=flat-square)](https://github.com/shotit/shotit-api/actions) +[![Codecov](https://img.shields.io/codecov/c/github/shotit/shotit-api?style=flat-square&token=8C25WLSEDJ)](https://codecov.io/gh/shotit/shotit-api) +[![Docker](https://img.shields.io/docker/pulls/shotit/shotit-api?style=flat-square)](https://hub.docker.com/r/shotit/shotit-api) +[![Docker Image Size](https://img.shields.io/docker/image-size/shotit/shotit-api/v0.9.1?style=flat-square)](https://hub.docker.com/r/shotit/shotit-api) The ultimate brain of [shotit](https://github.com/shotit/shotit), in charge of task coordination. -[API Docs](https://soruly.github.io/shotit-api/) - ### Features - serve image search request @@ -22,39 +19,54 @@ The ultimate brain of [shotit](https://github.com/shotit/shotit), in charge of t ### Prerequisites -- Node.js 14.x +- Node.js 16.x, 18.x - mariaDB 10.4.x - redis -- [liresolr](https://github.com/soruly/liresolr) +- [liresolr](https://github.com/Leslie-Wong-H/liresolr) +- [milvus-standalone](https://github.com/milvus-io/milvus) +- [milvus-minio](https://github.com/milvus-io/milvus) +- [milvus-etcd](https://github.com/milvus-io/etcd) +- [shotit-searcher](https://github.com/shotit/shotit-worker) +- [shotit-sorter(optional)](https://github.com/shotit/shotit-sorter) - g++, cmake (if you need to compile OpenCV) -### Install +### Local Development Guide -Install Prerequisites first, then: +Install: ``` -git clone https://github.com/soruly/shotit-api.git +git clone https://github.com/shotit/shotit-api.git cd shotit-api -npm install +yarn install ``` -### Configuration +Install Prerequisites by docker compose first, [docker-desktop](https://www.docker.com/products/docker-desktop/) required: - Copy `.env.example` to `.env` -- Edit `.env` as appropriate for your setup +- Edit `.env` as appropriate for your setup, as is for the first time. +- Copy `milvus.yaml.example` to `milvus.yaml` +- Edit `milvus.yaml` as appropriate for your setup, as is for the first time. + +``` +(Windows or Mac): +docker compose up -d +(Linux): +docker-compose up -d +``` ### Start server You can use [pm2](https://pm2.keymetrics.io/) to run this in background in cluster mode. -Use below commands to start / restart / stop server. +Use below commands to start / restart / stop / log server. ``` -npm run start -npm run stop -npm run reload -npm run restart -npm run delete +yarn start +yarn stop +yarn reload +yarn restart +yarn delete +yarn logs ``` To change the number of nodejs instances, edit ecosystem.config.json diff --git a/docker-compose.yml b/docker-compose.yml index 5d66478..5c2a55a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,6 +2,7 @@ version: "3.2" services: mariadb: + container_name: shotit-mariadb image: mariadb:latest restart: unless-stopped environment: @@ -13,6 +14,7 @@ services: redis: + container_name: shotit-redis image: redis:latest ports: - ${REDIS_PORT}:6379 @@ -21,15 +23,112 @@ services: liresolr: - image: ghcr.io/soruly/liresolr:latest + container_name: shotit-liresolr + image: ghcr.io/leslie-wong-h/liresolr:latest command: solr-precreate cl_0 /opt/solr/server/solr/configsets/liresolr restart: unless-stopped ports: - - 18983:8983 + - 8983:8983 volumes: - - type: bind - source: /var/solr - target: /var/solr + - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/mycores:/var/solr + networks: + shotit_net: + + + searcher: + container_name: shotit-searcher + image: ghcr.io/shotit/shotit-worker-searcher:v0.9.1 + restart: unless-stopped + ports: + - 19531:19531 + environment: + - TRACE_ALGO=cl + - SOLA_SOLR_LIST=http://172.17.0.1:8983/solr/ + - SEARCHER_URL=http://172.17.0.1:19531 + - MILVUS_URL=172.17.0.1:19530 + networks: + shotit_net: + + + sorter: + container_name: shotit-sorter + image: ghcr.io/shotit/shotit-sorter:v0.9.1 + restart: unless-stopped + ports: + - 19532:19532 + networks: + shotit_net: + + + etcd: + container_name: milvus-etcd + image: quay.io/coreos/etcd:v3.5.0 + restart: unless-stopped + environment: + - ETCD_AUTO_COMPACTION_MODE=revision + - ETCD_AUTO_COMPACTION_RETENTION=1000 + - ETCD_QUOTA_BACKEND_BYTES=4294967296 + - ETCD_SNAPSHOT_COUNT=50000 + volumes: + - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/etcd:/etcd + command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd + ports: + - "2379:2379" + networks: + shotit_net: + + + minio: + container_name: milvus-minio + image: minio/minio:RELEASE.2022-03-17T06-34-49Z + restart: unless-stopped + environment: + - MINIO_ROOT_USER=minioadmin + - MINIO_ROOT_PASSWORD=minioadmin + - MINIO_ETCD_ENDPOINTS=http://172.17.0.1:2379 + volumes: + - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/minio:/minio_data + # Local Minio Server + command: minio server /minio_data --console-address ":9001" + # Remote Minio Gateway + # command: minio gateway s3 ${AWS_ENDPOINT_URL} --console-address ":9001" + ports: + - "9000:9000" + - "9001:9001" + depends_on: + - "etcd" + healthcheck: + test: + [ + "CMD", + "curl", + "-f", + "http://localhost:9000/minio/health/live" + ] + interval: 30s + timeout: 20s + retries: 3 + networks: + shotit_net: + + + standalone: + container_name: milvus-standalone + image: milvusdb/milvus:v2.2.2 + restart: unless-stopped + command: [ "milvus", "run", "standalone" ] + environment: + ETCD_ENDPOINTS: etcd:2379 + MINIO_ADDRESS: minio:9000 + volumes: + - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/milvus:/var/lib/milvus + - ${DOCKER_VOLUME_DIRECTORY:-.}/milvus.yaml:/milvus/configs/milvus.yaml + ports: + - "19530:19530" + - "9091:9091" + depends_on: + - "etcd" + - "minio" networks: shotit_net: diff --git a/milvus.yaml.example b/milvus.yaml.example new file mode 100644 index 0000000..c4b7a4f --- /dev/null +++ b/milvus.yaml.example @@ -0,0 +1,338 @@ +# Licensed to the LF AI & Data foundation under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Related configuration of etcd, used to store Milvus metadata. +etcd: + endpoints: + - localhost:2379 + rootPath: by-dev # The root path where data is stored in etcd + metaSubPath: meta # metaRootPath = rootPath + '/' + metaSubPath + kvSubPath: kv # kvRootPath = rootPath + '/' + kvSubPath + log: + # path is one of: + # - "default" as os.Stderr, + # - "stderr" as os.Stderr, + # - "stdout" as os.Stdout, + # - file path to append server logs to. + # please adjust in embedded Milvus: /tmp/milvus/logs/etcd.log + path: stdout + level: info # Only supports debug, info, warn, error, panic, or fatal. Default 'info'. + use: + # please adjust in embedded Milvus: true + embed: false # Whether to enable embedded Etcd (an in-process EtcdServer). + data: + # Embedded Etcd only. + # please adjust in embedded Milvus: /tmp/milvus/etcdData/ + dir: default.etcd + ssl: + enabled: false # Whether to support ETCD secure connection mode + tlsCert: /path/to/etcd-client.pem # path to your cert file + tlsKey: /path/to/etcd-client-key.pem # path to your key file + tlsCACert: /path/to/ca.pem # path to your CACert file + # TLS min version + # Optional values: 1.0, 1.1, 1.2, 1.3。 + # We recommend using version 1.2 and above + tlsMinVersion: 1.3 + +# please adjust in embedded Milvus: /tmp/milvus/data/ +localStorage: + path: /var/lib/milvus/data/ + +# Related configuration of minio, which is responsible for data persistence for Milvus. +minio: + address: localhost # Address of MinIO/S3 + port: 9000 # Port of MinIO/S3 + accessKeyID: minioadmin # accessKeyID of MinIO/S3 + secretAccessKey: minioadmin # MinIO/S3 encryption string + useSSL: false # Access to MinIO/S3 with SSL + bucketName: "a-bucket" # Bucket name in MinIO/S3 + rootPath: files # The root path where the message is stored in MinIO/S3 + # Whether to use AWS IAM role to access S3 instead of access/secret keys + # For more infomation, refer to https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html + useIAM: false + # Custom endpoint for fetch IAM role credentials. + # Leave it empty if you want to use AWS default endpoint + iamEndpoint: "" + +# Milvus supports three MQ: rocksmq(based on RockDB), Pulsar and Kafka, which should be reserved in config what you use. +# There is a note about enabling priority if we config multiple mq in this file +# 1. standalone(local) mode: rockskmq(default) > Pulsar > Kafka +# 2. cluster mode: Pulsar(default) > Kafka (rocksmq is unsupported) + +# Related configuration of pulsar, used to manage Milvus logs of recent mutation operations, output streaming log, and provide log publish-subscribe services. +pulsar: + address: localhost # Address of pulsar + port: 6650 # Port of pulsar + webport: 80 # Web port of pulsar, if you connect direcly without proxy, should use 8080 + maxMessageSize: 5242880 # 5 * 1024 * 1024 Bytes, Maximum size of each message in pulsar. + +# If you want to enable kafka, needs to comment the pulsar configs +#kafka: +# brokerList: localhost1:9092,localhost2:9092,localhost3:9092 +# saslUsername: username +# saslPassword: password +# saslMechanisms: PLAIN +# securityProtocol: SASL_SSL + +rocksmq: + # please adjust in embedded Milvus: /tmp/milvus/rdb_data + path: /var/lib/milvus/rdb_data # The path where the message is stored in rocksmq + rocksmqPageSize: 134217728 # 128 MB, 1 * 128 * 1024 * 1024 bytes, The size of each page of messages in rocksmq + retentionTimeInMinutes: 60 # 1 hour, 1 * 1 * 60 minutes, The retention time of the message in rocksmq. + retentionSizeInMB: 1024 # 1 GB, 1 * 1024 MB, The retention size of the message in rocksmq. + compactionInterval: 3600 # 1 hour, trigger rocksdb compaction every hour to remove deleted data + lrucacheratio: 0.06 # rocksdb cache memory ratio + +# Related configuration of rootCoord, used to handle data definition language (DDL) and data control language (DCL) requests +rootCoord: + address: localhost + port: 53100 + + dmlChannelNum: 256 # The number of dml channels created at system startup + maxPartitionNum: 4096 # Maximum number of partitions in a collection + minSegmentSizeToEnableIndex: 1024 # It's a threshold. When the segment size is less than this value, the segment will not be indexed + + # (in seconds) Duration after which an import task will expire (be killed). Default 900 seconds (15 minutes). + # Note: If default value is to be changed, change also the default in: internal/util/paramtable/component_param.go + importTaskExpiration: 900 + # (in seconds) Milvus will keep the record of import tasks for at least `importTaskRetention` seconds. Default 86400 + # seconds (24 hours). + # Note: If default value is to be changed, change also the default in: internal/util/paramtable/component_param.go + importTaskRetention: 86400 + # (in seconds) Check an import task's segment loading state in queryNodes every `importSegmentStateCheckInterval` + # seconds. Default 10 seconds. + # Note: If default value is to be changed, change also the default in: internal/util/paramtable/component_param.go + importSegmentStateCheckInterval: 10 + # (in seconds) Maximum time to wait for segments in a single import task to be loaded in queryNodes. + # Default 60 seconds (1 minute). + # Note: If default value is to be changed, change also the default in: internal/util/paramtable/component_param.go + importSegmentStateWaitLimit: 60 + # (in seconds) Check the building status of a task's segments' indices every `importIndexCheckInterval` seconds. + # Default 10 seconds. + # Note: If default value is to be changed, change also the default in: internal/util/paramtable/component_param.go + importIndexCheckInterval: 10 + # (in seconds) Maximum time to wait for indices to be built on a single import task's segments. + # Default 600 seconds (10 minutes). + # Note: If default value is to be changed, change also the default in: internal/util/paramtable/component_param.go + importIndexWaitLimit: 600 + +# Related configuration of proxy, used to validate client requests and reduce the returned results. +proxy: + port: 19530 + internalPort: 19529 + http: + enabled: true # Whether to enable the http server + debug_mode: false # Whether to enable http server debug mode + + timeTickInterval: 200 # ms, the interval that proxy synchronize the time tick + msgStream: + timeTick: + bufSize: 512 + maxNameLength: 255 # Maximum length of name for a collection or alias + maxFieldNum: 256 # Maximum number of fields in a collection + maxDimension: 32768 # Maximum dimension of a vector + maxShardNum: 256 # Maximum number of shards in a collection + maxTaskNum: 1024 # max task number of proxy task queue + # please adjust in embedded Milvus: false + ginLogging: true # Whether to produce gin logs. + + +# Related configuration of queryCoord, used to manage topology and load balancing for the query nodes, and handoff from growing segments to sealed segments. +queryCoord: + address: localhost + port: 19531 + autoHandoff: true # Enable auto handoff + autoBalance: true # Enable auto balance + overloadedMemoryThresholdPercentage: 90 # The threshold percentage that memory overload + balanceIntervalSeconds: 60 + memoryUsageMaxDifferencePercentage: 30 + +# Related configuration of queryNode, used to run hybrid search between vector and scalar data. +queryNode: + cacheSize: 32 # GB, default 32 GB, `cacheSize` is the memory used for caching data for faster query. The `cacheSize` must be less than system memory size. + port: 21123 + loadMemoryUsageFactor: 3 # The multiply factor of calculating the memory usage while loading segments + + stats: + publishInterval: 1000 # Interval for querynode to report node information (milliseconds) + dataSync: + flowGraph: + maxQueueLength: 1024 # Maximum length of task queue in flowgraph + maxParallelism: 1024 # Maximum number of tasks executed in parallel in the flowgraph + # Segcore will divide a segment into multiple chunks to enbale small index + segcore: + chunkRows: 1024 # The number of vectors in a chunk. + # Note: we have disabled segment small index since @2022.05.12. So below related configurations won't work. + # We won't create small index for growing segments and search on these segments will directly use bruteforce scan. + smallIndex: + nlist: 128 # small index nlist, recommend to set sqrt(chunkRows), must smaller than chunkRows/8 + nprobe: 16 # nprobe to search small index, based on your accuracy requirement, must smaller than nlist + cache: + enabled: true + memoryLimit: 2147483648 # 2 GB, 2 * 1024 *1024 *1024 + + scheduler: + receiveChanSize: 10240 + unsolvedQueueSize: 10240 + # maxReadConcurrentRatio is the concurrency ratio of read task (search task and query task). + # Max read concurrency would be the value of `runtime.NumCPU * maxReadConcurrentRatio`. + # It defaults to 2.0, which means max read concurrency would be the value of runtime.NumCPU * 2. + # Max read concurrency must greater than or equal to 1, and less than or equal to runtime.NumCPU * 100. + maxReadConcurrentRatio: 2.0 # (0, 100] + cpuRatio: 10.0 # ratio used to estimate read task cpu usage. + + grouping: + enabled: true + maxNQ: 1000 + topKMergeRatio: 10.0 + +indexCoord: + address: localhost + port: 31000 + + gc: + interval: 600 # gc interval in seconds + +indexNode: + port: 21121 + + scheduler: + buildParallel: 1 + +dataCoord: + address: localhost + port: 13333 + enableCompaction: true # Enable data segment compression + enableGarbageCollection: true + + segment: + maxSize: 512 # Maximum size of a segment in MB + sealProportion: 0.25 # It's the minimum proportion for a segment which can be sealed + assignmentExpiration: 2000 # The time of the assignment expiration in ms + maxLife: 86400 # The max lifetime of segment in seconds, 24*60*60 + + compaction: + enableAutoCompaction: true + + gc: + interval: 3600 # gc interval in seconds + missingTolerance: 86400 # file meta missing tolerance duration in seconds, 60*24 + dropTolerance: 86400 # file belongs to dropped entity tolerance duration in seconds, 60*24 + + +dataNode: + port: 21124 + + dataSync: + flowGraph: + maxQueueLength: 1024 # Maximum length of task queue in flowgraph + maxParallelism: 1024 # Maximum number of tasks executed in parallel in the flowgraph + flush: + # Max buffer size to flush for a single segment. + insertBufSize: 16777216 # Bytes, 16 MB + +# Configures the system log output. +log: + level: debug # Only supports debug, info, warn, error, panic, or fatal. Default 'info'. + file: + # please adjust in embedded Milvus: /tmp/milvus/logs + rootPath: "" # default to stdout, stderr + maxSize: 300 # MB + maxAge: 10 # Maximum time for log retention in day. + maxBackups: 20 + format: text # text/json + +grpc: + log: + level: WARNING + + serverMaxRecvSize: 2147483647 # math.MaxInt32 + serverMaxSendSize: 2147483647 # math.MaxInt32 + clientMaxRecvSize: 104857600 # 100 MB, 100 * 1024 * 1024 + clientMaxSendSize: 104857600 # 100 MB, 100 * 1024 * 1024 + + client: + dialTimeout: 5000 + keepAliveTime: 10000 + keepAliveTimeout: 20000 + maxMaxAttempts: 5 + initialBackOff: 1.0 + maxBackoff: 60.0 + backoffMultiplier: 2.0 + +# Configure the proxy tls enable. +tls: + serverPemPath: configs/cert/server.pem + serverKeyPath: configs/cert/server.key + caPemPath: configs/cert/ca.pem + + +common: + # Channel name generation rule: ${namePrefix}-${ChannelIdx} + chanNamePrefix: + cluster: "by-dev" + rootCoordTimeTick: "rootcoord-timetick" + rootCoordStatistics: "rootcoord-statistics" + rootCoordDml: "rootcoord-dml" + rootCoordDelta: "rootcoord-delta" + search: "search" + searchResult: "searchResult" + queryTimeTick: "queryTimeTick" + queryNodeStats: "query-node-stats" + # Cmd for loadIndex, flush, etc... + cmd: "cmd" + dataCoordStatistic: "datacoord-statistics-channel" + dataCoordTimeTick: "datacoord-timetick-channel" + dataCoordSegmentInfo: "segment-info-channel" + + # Sub name generation rule: ${subNamePrefix}-${NodeID} + subNamePrefix: + rootCoordSubNamePrefix: "rootCoord" + proxySubNamePrefix: "proxy" + queryNodeSubNamePrefix: "queryNode" + dataNodeSubNamePrefix: "dataNode" + dataCoordSubNamePrefix: "dataCoord" + + defaultPartitionName: "_default" # default partition name for a collection + defaultIndexName: "_default_idx" # default index name + retentionDuration: 432000 # 5 days in seconds + entityExpiration: -1 # Entity expiration in seconds, CAUTION make sure entityExpiration >= retentionDuration and -1 means never expire + + gracefulTime: 5000 # milliseconds. it represents the interval (in ms) by which the request arrival time needs to be subtracted in the case of Bounded Consistency. + + # Default value: auto + # Valid values: [auto, avx512, avx2, avx, sse4_2] + # This configuration is only used by querynode and indexnode, it selects CPU instruction set for Searching and Index-building. + simdType: auto + indexSliceSize: 16 # MB + + # please adjust in embedded Milvus: local + storageType: minio + + security: + authorizationEnabled: false + # tls mode values [0, 1, 2] + # 0 is close, 1 is one-way authentication, 2 is two-way authentication. + tlsMode: 0 + + mem_purge_ratio: 0.2 # in Linux os, if memory-fragmentation-size >= used-memory * ${mem_purge_ratio}, then do `malloc_trim` + +# diskQuota is used to target local minio, not rdb_data +# quotaAndLimits: +# limitWriting: +# diskProtection: +# enabled: true +# diskQuota: 8192 \ No newline at end of file diff --git a/package.json b/package.json index c51461e..2919fdf 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "restart": "pm2 restart ecosystem.config.json", "reload": "pm2 reload ecosystem.config.json", "delete": "pm2 delete ecosystem.config.json", + "logs": "pm2 logs", "prettier": "prettier", "format": "prettier --write \"**/*.js\"", "lint": "prettier --check \"**/*.js\"",