.github/workflows/daily.yml

@@ -27,6 +27,7 @@ jobs:
             docker compose run --rm --no-deps coa-updater | ts "[%Y-%m-%d %H:%M:%S]"
 
       - name: Run stats-updater
+        continue-on-error: true
         uses: appleboy/ssh-action@v1.0.3
         with:
           host: ${{ secrets.PRODUCTION_SSH_HOST }}
@@ -41,6 +42,7 @@ jobs:
             docker compose run --rm --no-deps stats-updater
 
       - name: Run cover-updater
+        continue-on-error: true
         uses: appleboy/ssh-action@v1.0.3
         with:
           host: ${{ secrets.PRODUCTION_SSH_HOST }}
@@ -55,6 +57,7 @@ jobs:
             docker compose run --rm --no-deps cover-updater
 
       - name: Run duck-estimator
+        continue-on-error: true
         uses: appleboy/ssh-action@v1.0.3
         with:
           host: ${{ secrets.PRODUCTION_SSH_HOST }}
@@ -79,7 +82,7 @@ jobs:
           script_stop: true
           envs: REMOTE_ROOT
           script: |
-            cd $REMOTE_ROOT
+            cd $REMOTE_ROOT/packages/api
             docker compose run --rm --no-deps sprite-names-updater
 
       - name: Run subscription-handler

.github/workflows/deploy.yml

@@ -47,10 +47,10 @@ jobs:
         with:
           version: 9.8.0
 
-      - name: Use Node.js 20
+      - name: Use Node.js 22
         uses: actions/setup-node@v4
         with:
-          node-version: 20
+          node-version: 22
           cache: "pnpm"
 
       - uses: oven-sh/setup-bun@v1
@@ -73,17 +73,13 @@ jobs:
       - name: Retrieve pre-build files
         run: |
           pnpm i -g turbo
-          turbo -F "...[$lastSuccessfulCommit]" -F '!~dumili' -F '!~dumili-api' prod:transfer-files-pre
-
-      - name: Build apps
-        run: |
-          turbo -F "...[$lastSuccessfulCommit]" -F '!~dumili' -F '!~dumili-api' build
+          turbo -F '!~dumili' -F '!~dumili-api' prod:transfer-files-pre
 
       - uses: docker/setup-buildx-action@v3
 
-      - name: Build and push Docker images
+      - name: Build apps
         run: |
-          turbo -F "...[$lastSuccessfulCommit]" -F '!~ci' -F '!~dumili' -F '!~dumili-api' prod:build-docker
+          turbo -F "...[$lastSuccessfulCommit]" -F '!~ci' -F '!~dumili' -F '!~dumili-api' build prod:build-docker
 
       - name: Send post-build files
         run: |

.gitignore

@@ -1,3 +1,4 @@
+!.gitkeep
 .DS_Store
 .eslintcache
 .eslintrc-auto-import.json
@@ -11,18 +12,17 @@
 **/.eslintcache
 **/.turbo/
 **/.venv
+**/*.js.map
 **/auto-imports.d.ts
+**/bundle.mjs
 **/components.d.ts
+**/index.js
+/vendor/
 apps/edges/edges
 !apps/edges/edges/.gitkeep
-/vendor/
-**/bundle.mjs
-**/index.js
-**/*.js.map
 dist
 dist-ssr
 local.properties
 node_modules
 output-metadata.json
 packages/api/pastec-*.dat
-!.gitkeep

.husky/pre-commit

@@ -1 +1,4 @@
+export NVM_DIR="$HOME/.nvm"
+[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
+nvm use
 pnpm lint

.nvmrc

@@ -1 +1 @@
-v20
+v22

README.md

@@ -19,6 +19,6 @@ Related projects :
 
 ### Prerequisites
 
-- [Node.js](https://nodejs.org/en/) 20
+- [Node.js](https://nodejs.org/en/) 22
 - [Docker](https://www.docker.com/)
 - Some Unix or WSL2 environment

apps/coa-updater/package.json

@@ -1,21 +1,21 @@
 {
   "name": "~coa-updater",
   "scripts": {
-    "build": "bun build --target bun --sourcemap index.ts --outfile bundle.mjs",
+    "build": "bun build --target bun index.ts --outfile bundle.mjs",
     "prod:transfer-files-post": "pnpm -F '~ci' prod:transfer-files apps/coa-updater/docker-compose.yml:@apps/coa-updater/docker-compose.yml"
   },
   "keywords": [],
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "csv-generate": "^4.4.1",
-    "csv-parse": "^5.5.6",
+    "csv-generate": "^4.4.2",
+    "csv-parse": "^5.6.0",
     "mariadb": "^3.4.0"
   },
   "devDependencies": {
     "@types/bun": "latest",
-    "@types/node": "^20.17.6",
-    "typescript": "^5.6.3"
+    "@types/node": "^22.10.2",
+    "typescript": "^5.7.2"
   },
   "module": "index.ts",
   "type": "module"

apps/cover-updater/scripts/import-covers.sh

@@ -5,4 +5,4 @@ COVER_LIST_EXPORT_FILE=/home/covers.csv
 echo "Exporting covers from the COA database..."
 mysql -h ${MYSQL_COA_HOST} -uroot -p${MYSQL_ROOT_PASSWORD} ${MYSQL_COA_DATABASE} </home/scripts/sql/get-coa-covers.sql | sed 's/\t/,/g' >${COVER_LIST_EXPORT_FILE} &&
     echo "Importing covers into the covers database..." &&
-    mysql -uroot -p${MYSQL_ROOT_PASSWORD} -h ${MYSQL_COVER_INFO_HOST} --local-infile=1 ${MYSQL_COVER_INFO_DATABASE} -e "LOAD DATA LOCAL INFILE \"${COVER_LIST_EXPORT_FILE}\" IGNORE INTO TABLE covers FIELDS TERMINATED BY ',' IGNORE 1 LINES (issuecode,sitecode,url);"
+    mysql -uroot -p${MYSQL_ROOT_PASSWORD} -h ${MYSQL_COVER_INFO_HOST} --local-infile=1 ${MYSQL_COVER_INFO_DATABASE} -e "LOAD DATA LOCAL INFILE \"${COVER_LIST_EXPORT_FILE}\" IGNORE INTO TABLE covers FIELDS TERMINATED BY ',' IGNORE 1 LINES (issuecode,sitecode,url)"

apps/cover-updater/scripts/sql/get-coa-covers.sql

@@ -11,7 +11,7 @@
     (
       SELECT MIN(position)
       FROM inducks_entry e
-      WHERE entry.issuecode = e.issuecode AND position NOT LIKE 'p%'
+      WHERE entry.issuecode = e.issuecode
       GROUP BY e.issuecode
     )
   AND issuecode != '';

apps/duck-estimator/package.json

@@ -2,34 +2,34 @@
   "name": "~duck-estimator",
   "version": "1.0.0",
   "engines": {
-    "node": ">=20.0.0 <21.0.0"
+    "node": ">=22.0.0 <23.0.0"
   },
   "author": {
     "email": "brunoperel@gmail.com",
     "name": "Bruno Perel"
   },
   "scripts": {
-    "build": "bun build --target node --sourcemap index.ts --outdir dist && mv dist/index.js bundle.mjs && rm -rf dist",
+    "build": "bun build --target node index.ts --outdir dist && mv dist/index.js bundle.mjs && rm -rf dist",
     "start": "bun --inspect run --watch index.ts",
     "prod:build-docker": "REPO_NAME=ghcr.io/bperel/duck-estimator pnpm -F '~ci' prod:build-docker -f apps/duck-estimator/Dockerfile",
     "prod:transfer-files-pre": "pnpm -F '~ci' prod:transfer-files @apps/duck-estimator/.env.local:apps/duck-estimator/.env",
     "prod:transfer-files-post": "pnpm -F '~ci' prod:transfer-files apps/duck-estimator/docker-compose.yml:@apps/duck-estimator/docker-compose.yml"
   },
   "dependencies": {
     "bedetheque-scraper": "^3.2.0",
-    "csv-parse": "^5.5.6",
+    "csv-parse": "^5.6.0",
     "csv-writer": "^1.6.0",
-    "dotenv": "^16.4.5",
-    "electron": "^32.2.3",
+    "dotenv": "^16.4.7",
+    "electron": "^32.2.7",
     "playwright-firefox": "~1.47.2",
     "yargs": "^17.7.2",
     "~prisma-schemas": "workspace:*"
   },
   "devDependencies": {
-    "@types/node": "^20.17.6",
+    "@types/node": "^22.10.2",
     "@types/yargs": "^17.0.33",
     "tsc-alias": "^1.8.10",
     "tsx": "^4.19.2",
-    "typescript": "^5.6.3"
+    "typescript": "^5.7.2"
   }
 }

apps/dumili/api/Dockerfile

@@ -1,9 +1,9 @@
-FROM node:20-slim
+FROM node:22-slim
 LABEL org.opencontainers.image.authors="Bruno Perel"
 WORKDIR /app
 
 COPY packages/prisma-schemas/client_coa/libquery_engine-debian-openssl-3.0.x.so.node /tmp/prisma-engines/
-COPY apps/dumili/api/client_dumili/libquery_engine-debian-openssl-3.0.x.so.node /tmp/prisma-engines/
+COPY apps/dumili/api/prisma/client_dumili/libquery_engine-debian-openssl-3.0.x.so.node /tmp/prisma-engines/
 
 COPY apps/dumili/api/.env /app/
 COPY apps/dumili/api/bundle.mjs /app/

apps/dumili/api/docker-compose-dev.yml

@@ -12,6 +12,7 @@ services:
       - "8006:8081"
     volumes:
       - /root/.paddleocr
+      - ./paddleocr/server.py:/server.py
     build:
       context: ../../..
       dockerfile: apps/dumili/api/paddleocr/Dockerfile

apps/dumili/api/index.ts

@@ -39,17 +39,12 @@ export type SessionDataWithIndexationId = {
   indexationId: string;
 };
 export type SessionData = { user: SessionUser };
-export class ServerWithData<Data extends object> extends Server<
-  Record<string, never>,
-  Record<string, never>,
-  Record<string, never>,
-  Data
-> {}
 
 export type NamespaceWithData<
   Services extends EventsMap,
+  ServerSentEvents extends EventsMap = object,
   Data extends object = object,
-> = Namespace<Services, Record<string, never>, Record<string, never>, Data>;
+> = Namespace<Services, ServerSentEvents, Record<string, never>, Data>;
 
 export const prisma = new PrismaClient();
 

apps/dumili/api/package.json

@@ -3,34 +3,31 @@
   "type": "module",
   "version": "1.0.0",
   "engines": {
-    "node": ">=20.0.0 <21.0.0"
+    "node": ">=22.0.0 <23.0.0"
   },
   "scripts": {
     "clean": "rm -rf dist",
-    "build": "bun build --target node --sourcemap index.ts --outfile bundle.mjs",
+    "build": "bun build --target node index.ts --outfile bundle.mjs",
     "prisma-pull-generate": "prisma db pull && prisma generate",
     "prisma-generate": "prisma generate",
     "prisma-migrate": "prisma migrate deploy",
-    "dev:setup": "docker compose -f docker-compose-dev.yml up --force-recreate -d",
-    "dev:blocking": "bash ../../../packages/prisma-schemas/wait-until-db-ready.bash && prisma migrate deploy && prisma generate",
-    "dev": "concurrently --kill-others-on-fail -n typecheck,bun \"tsc --noEmit\" \"bun --inspect run --hot index.ts\"",
+    "dev": "docker compose -f docker-compose-dev.yml up --force-recreate -d && bash ../../../packages/prisma-schemas/wait-until-db-ready.bash && prisma migrate deploy && prisma generate && concurrently --kill-others-on-fail -n docker-compose,typecheck \"docker compose -f docker-compose-dev.yml logs -f\" \"tsc --noEmit\"",
     "prod:deploy": "DIR=apps/dumili/api pnpm -F '~ci' prod:docker-compose-up",
     "prod-build-docker-api": "REPO_NAME=ghcr.io/bperel/dumili-api pnpm -F '~ci' prod:build-docker -f apps/dumili/api/Dockerfile",
-    "prod-build-docker-kumiko": "REPO_NAME=ghcr.io/bperel/kumiko pnpm -F '~ci' prod:build-docker -f apps/dumili/kumiko/Dockerfile",
-    "prod-build-docker-paddleocr": "REPO_NAME=ghcr.io/bperel/paddleocr pnpm -F '~ci' prod:build-docker -f apps/dumili/paddleocr/Dockerfile",
+    "prod-build-docker-kumiko": "REPO_NAME=ghcr.io/bperel/kumiko pnpm -F '~ci' prod:build-docker -f apps/dumili/api/kumiko/Dockerfile",
+    "prod-build-docker-paddleocr": "REPO_NAME=ghcr.io/bperel/paddleocr pnpm -F '~ci' prod:build-docker -f apps/dumili/api/paddleocr/Dockerfile",
     "prod:build-docker": "pnpm run '/^prod-build-docker-/'",
     "prod:transfer-files-pre": "pnpm -F '~ci' prod:transfer-files @apps/dumili/api/.env.prod.local:apps/dumili/api/.env",
     "prod:transfer-files-post": "pnpm -F '~ci' prod:transfer-files apps/dumili/api/docker-compose.yml:@apps/dumili/api/docker-compose.yml"
   },
   "dependencies": {
-    "@prisma/client": "^5.22.0",
-    "@sentry/node": "^8.37.1",
-    "axios": "^1.7.7",
+    "@prisma/client": "^6.1.0",
+    "@sentry/node": "^8.47.0",
+    "axios": "^1.7.9",
     "busboy": "^1.6.0",
     "cloudinary": "^1.41.3",
-    "dotenv": "^16.4.5",
+    "dotenv": "^16.4.7",
     "jsonwebtoken": "^9.0.2",
-    "pdf2pic": "^3.1.3",
     "sharp": "^0.33.5",
     "socket.io": "^4.8.1",
     "~api": "workspace:*",
@@ -43,9 +40,9 @@
     "@types/busboy": "^1.5.4",
     "@types/i18n": "^0.13.12",
     "@types/jsonwebtoken": "^9.0.7",
-    "@types/node": "^20.17.6",
+    "@types/node": "^22.10.2",
     "concurrently": "^9.1.0",
-    "prisma": "^5.22.0",
-    "typescript": "^5.6.3"
+    "prisma": "^6.1.0",
+    "typescript": "^5.7.2"
   }
 }

apps/dumili/api/paddleocr/Dockerfile

@@ -8,7 +8,7 @@ RUN cd openssl-1.1.1v && ./config && make -j$(nproc) && make test
 
 FROM python:3.9-slim
 
-RUN apt update && apt install -y --no-install-recommends libgl1-mesa-glx libgomp1 libglib2.0-0 gcc python3-dev && apt clean
+RUN apt update && apt install -y --no-install-recommends libgl1-mesa-glx libgomp1 libglib2.0-0 gcc python3-dev patch && apt clean
 
 COPY --from=libs /home/openssl-1.1.1v/libcrypto.so.1.1 /lib/
 COPY --from=libs /home/openssl-1.1.1v/libssl.so.1.1 /lib/

apps/dumili/api/paddleocr/requirements.txt

@@ -1,2 +1,2 @@
-paddleocr>=2.7.0.2
-paddlepaddle>=2.6.0
+paddleocr>=2.8.1
+paddlepaddle>=3.0.0b1

apps/dumili/api/paddleocr/server.py

@@ -4,10 +4,8 @@
 utils.run_check()
 from paddleocr import PaddleOCR
 from http.server import BaseHTTPRequestHandler, HTTPServer
-import random
 import json
 import os
-import base64
 
 # select countrycode, GROUP_CONCAT(languagecode ORDER BY entries_with_language DESC) AS languages
 # from (select countrycode, coalesce(inducks_entry.languagecode, ip.languagecode) as languagecode, count(*) as entries_with_language
@@ -332,42 +330,29 @@
 
 class PaddleOCRRequestHandler(BaseHTTPRequestHandler):
     def do_POST(self):
-        content_length = int(self.headers['Content-Length']) 
-        base64Text = self.rfile.read(content_length)
+        content_length = int(self.headers['Content-Length'])
+        post_data = json.loads(self.rfile.read(content_length))
+        url = post_data['url']
+        language = post_data['language']
 
-        file_name = ''.join((random.choice('abcdefghi') for i in range(5))) + '.png'
-        try:
-            file_content = base64.b64decode(base64Text)
-            with open(file_name,"wb") as f:
-                f.write(file_content)
-        except Exception as e:
-            print(str(e))
-
-        result = None
-        # result = ocr_languages['french'].ocr(file_name, cls=True)
-        # result = result[0]
-        if result is None:
-            os.remove(file_name)
-            self.send_response(200)
-            self.send_header('Content-type', 'application/json')
-            self.end_headers()
-            self.wfile.write(json.dumps([]).encode())
-            return
-
-        boxes = [line[0] for line in result]
-        texts = [line[1][0] for line in result]
-        scores = [line[1][1] for line in result]
+        os.remove("tmp.jpg") if os.path.exists("tmp.jpg") else None
+        result = ocr_languages[language].ocr(url, cls=True)
+        result = result[0]
 
         converted_data = []
-        for i in range(len(boxes)):
-            converted_item = {
-                "box": boxes[i],
-                "text": texts[i],
-                "confidence": scores[i]
-            }
-            converted_data.append(converted_item)
+        if result is not None:
+          boxes = [line[0] for line in result]
+          texts = [line[1][0] for line in result]
+          scores = [line[1][1] for line in result]
+
+          for i in range(len(boxes)):
+              converted_item = {
+                  "box": boxes[i],
+                  "text": texts[i],
+                  "confidence": scores[i]
+              }
+              converted_data.append(converted_item)
 
-        os.remove(file_name)
         self.send_response(200)
         self.send_header('Content-type', 'application/json')
         self.end_headers()