UBERF-9747

Signed-off-by: Andrey Sobolev <[email protected]>

Author: haiodo

.vscode/launch.json

@@ -103,7 +103,8 @@
         "UPLOAD_URL": "/files",
         "AI_BOT_URL": "http://localhost:4010",
         "STATS_URL": "http://huly.local:4900",
-        "QUEUE_CONFIG": "localhost:19092"
+        "QUEUE_CONFIG": "localhost:19092",
+        "RATE_LIMIT_MAX": "25000"
       },
       "runtimeArgs": ["--nolazy", "-r", "ts-node/register"],
       "runtimeVersion": "20",
@@ -120,9 +121,9 @@
       "args": ["src/__start.ts"],
       "env": {
         "FULLTEXT_URL": "http://localhost:4710",
-        // "DB_URL": "mongodb://localhost:27018",
+        "DB_URL": "mongodb://localhost:27018",
         // "DB_URL": "postgresql://postgres:example@localhost:5432",
-        "DB_URL": "postgresql://[email protected]:26258/defaultdb?sslmode=disable",
+        // "DB_URL": "postgresql://[email protected]:26258/defaultdb?sslmode=disable",
         // "GREEN_URL": "http://huly.local:6767?token=secret",
         "SERVER_PORT": "3334",
         "METRICS_CONSOLE": "false",
@@ -183,7 +184,7 @@
       "request": "launch",
       "args": ["src/__start.ts"],
       "env": {
-        "MONGO_URL": "mongodb://localhost:27017",
+        // "MONGO_URL": "mongodb://localhost:27017",
         // "DB_URL": "mongodb://localhost:27017",
         "DB_URL": "postgresql://[email protected]:26257/defaultdb?sslmode=disable",
         "SERVER_SECRET": "secret",
@@ -217,12 +218,13 @@
       "args": ["src/__start.ts"],
       "env": {
         // "MONGO_URL": "mongodb://localhost:27018",
-        // "DB_URL": "mongodb://localhost:27018",
+        "DB_URL": "mongodb://localhost:27018",
         // "DB_URL": "postgresql://postgres:example@localhost:5432",
-        "DB_URL": "postgresql://[email protected]:26258/defaultdb?sslmode=disable",
+        // "DB_URL": "postgresql://[email protected]:26258/defaultdb?sslmode=disable",
         "SERVER_SECRET": "secret",
-        "REGION_INFO": "|Mongo;pg|Postgres;cockroach|CockroachDB",
-        "TRANSACTOR_URL": "ws://transactor:3334;ws://localhost:3334",
+        "QUEUE_CONFIG": "huly.local:19092",
+        "REGION_INFO": "|America;europe|Europe",
+        "TRANSACTOR_URL": "ws://huly.local:3334;ws://huly.local:3334,ws://huly.local:3335;ws://huly.local:3335;europe",
         "ACCOUNTS_URL": "http://localhost:3003",
         "ACCOUNT_PORT": "3003",
         "FRONT_URL": "http://localhost:8083",

common/config/rush/command-line.json

@@ -4,7 +4,6 @@
  */
 {
   "$schema": "https://developer.microsoft.com/json-schemas/rush/v5/command-line.schema.json",
-
   "phases": [
     {
       "name": "_phase:build",

common/scripts/rush_docker.sh

@@ -0,0 +1,3 @@
+export RUSH_ALLOW_UNSUPPORTED_NODEJS=1
+rush docker:build -p 20 \
+  --to @hcengineering/pod-server --to @hcengineering/pod-front --to @hcengineering/prod --to @hcengineering/pod-account --to @hcengineering/pod-workspace --to @hcengineering/pod-collaborator --to @hcengineering/tool --to @hcengineering/pod-print --to @hcengineering/pod-sign --to @hcengineering/pod-analytics-collector --to @hcengineering/rekoni-service --to @hcengineering/pod-ai-bot --to @hcengineering/import-tool --to @hcengineering/pod-stats --to @hcengineering/pod-fulltext --to @hcengineering/pod-love --to @hcengineering/pod-mail --to @hcengineering/pod-datalake --to @hcengineering/pod-inbound-mail --to @hcengineering/pod-export --to @hcengineering/pod-msg2file

dev/doc-import-tool/src/import.ts

@@ -47,7 +47,7 @@ export default async function importExtractedFile (
 
   try {
     console.log(`Connected to ${transactorUrl}`)
-    const txops = new TxOperations(connection, core.account.System)
+    const txops = new TxOperations(connection, core.account.System, workspaceId)
 
     try {
       const docId = await createDocument(txops, extractedFile, config)

dev/import-tool/src/index.ts

@@ -95,7 +95,7 @@ export function importTool (): void {
 
     console.log('Connecting to Transactor URL: ', selectedWs.endpoint)
     const connection = await createClient(selectedWs.endpoint, selectedWs.token)
-    const client = new TxOperations(connection, socialId)
+    const client = new TxOperations(connection, socialId, selectedWs.workspace)
     const fileUploader = new FrontFileUploader(
       getFrontUrl(),
       selectedWs.workspace,

dev/prod/src/platform.ts

@@ -68,7 +68,7 @@ import { uploaderId } from '@hcengineering/uploader'
 import { mediaId } from '@hcengineering/media'
 import recorder, { recorderId } from '@hcengineering/recorder'
 import { viewId } from '@hcengineering/view'
-import workbench, { workbenchId } from '@hcengineering/workbench'
+import workbench, { workbenchId, workbenchAppsId } from '@hcengineering/workbench'
 import { mailId } from '@hcengineering/mail'
 import { chatId } from '@hcengineering/chat'
 import github, { githubId } from '@hcengineering/github'
@@ -148,6 +148,7 @@ import { preferenceId } from '@hcengineering/preference'
 import { uiId } from '@hcengineering/ui/src/plugin'
 import { configureAnalytics } from './analytics'
 
+
 export interface Config {
   ACCOUNTS_URL: string
   UPLOAD_URL: string
@@ -487,6 +488,7 @@ export async function configurePlatform() {
     uiPlugin.metadata.Routes,
     new Map([
       [workbenchId, workbench.component.WorkbenchApp],
+      [workbenchAppsId, workbench.component.WorkbenchApps],
       [loginId, login.component.LoginApp],
       [onboardId, onboard.component.OnboardApp],
       [githubId, github.component.ConnectApp],

dev/tool/src/benchmark.ts

@@ -410,7 +410,7 @@ export function benchmarkWorker (): void {
 
       if (msg.options.mode === 'find-all') {
         const benchmarkPersonId = (core.account.System + '_benchmark') as PersonId
-        const opt = new TxOperations(connection, benchmarkPersonId)
+        const opt = new TxOperations(connection, benchmarkPersonId, msg.workspaceId)
         parentPort?.postMessage({
           type: 'operate',
           workId: msg.workId
@@ -551,7 +551,7 @@ export async function stressBenchmark (transactor: string, mode: StressBenchmark
 export async function testFindAll (endpoint: string, workspace: WorkspaceUuid, account: PersonUuid): Promise<void> {
   const connection = await connect(endpoint, workspace, account)
   try {
-    const client = new TxOperations(connection, core.account.System)
+    const client = new TxOperations(connection, core.account.System, workspace)
     const start = platformNow()
     const res = await client.findAll(
       recruit.class.Applicant,
@@ -576,7 +576,7 @@ export async function generateWorkspaceData (
   email: string
 ): Promise<void> {
   const connection = await connect(endpoint, workspace)
-  const client = new TxOperations(connection, core.account.System)
+  const client = new TxOperations(connection, core.account.System, workspace)
   try {
     const emailSocialString = buildSocialIdString({ type: SocialIdType.EMAIL, value: email })
     const person = await getPersonBySocialKey(client, emailSocialString)

dev/tool/src/clean.ts

@@ -86,11 +86,10 @@ export async function cleanWorkspace (
   opt: { recruit: boolean, tracker: boolean, removedTx: boolean }
 ): Promise<void> {
   const connection = (await connect(transactorUrl, workspaceId, undefined, {
-    mode: 'backup',
-    model: 'upgrade'
+    mode: 'backup'
   })) as unknown as CoreClient & BackupClient
   try {
-    const ops = new TxOperations(connection, core.account.System)
+    const ops = new TxOperations(connection, core.account.System, workspaceId)
 
     const hierarchy = ops.getHierarchy()
 
@@ -164,6 +163,7 @@ export async function cleanWorkspace (
       client.close()
     }
   } catch (err: any) {
+    // TODO: Add force-close
     console.trace(err)
   } finally {
     await connection.close()
@@ -232,8 +232,7 @@ export async function cleanRemovedTransactions (workspaceId: WorkspaceUuid, tran
 
 export async function optimizeModel (workspaceId: WorkspaceUuid, transactorUrl: string): Promise<void> {
   const connection = (await connect(transactorUrl, workspaceId, undefined, {
-    mode: 'backup',
-    model: 'upgrade'
+    mode: 'backup'
   })) as unknown as CoreClient & BackupClient
   try {
     let count = 0
@@ -298,6 +297,8 @@ export async function optimizeModel (workspaceId: WorkspaceUuid, transactorUrl:
   } catch (err: any) {
     console.trace(err)
   } finally {
+    // TODO: Add force-close
+    await connection.sendForceClose()
     await connection.close()
   }
 }
@@ -307,7 +308,7 @@ export async function cleanArchivedSpaces (workspaceId: WorkspaceUuid, transacto
   })) as unknown as CoreClient & BackupClient
   try {
     const count = 0
-    const ops = new TxOperations(connection, core.account.System)
+    const ops = new TxOperations(connection, core.account.System, workspaceId)
     while (true) {
       const spaces = await connection.findAll(core.class.Space, { archived: true }, { limit: 1000 })
       if (spaces.length === 0) {
@@ -475,7 +476,7 @@ export async function fixSkills (
     // fix skills with + and -
     if (step === '3') {
       console.log('STEP 3')
-      const ops = new TxOperations(connection, core.account.System)
+      const ops = new TxOperations(connection, core.account.System, workspaceId)
       const regex = /\S+(?:[-+]\S+)+/g
       const tagsToClean = (await connection.findAll(tags.class.TagElement, {
         category: {
@@ -532,7 +533,7 @@ export async function fixSkills (
         }
       })) as TagElement[]
       goodTags = goodTags.sort((a, b) => b.title.length - a.title.length).filter((t) => t.title.length > 2)
-      const ops = new TxOperations(connection, core.account.System)
+      const ops = new TxOperations(connection, core.account.System, workspaceId)
       const tagsToClean = (await connection.findAll(tags.class.TagElement, {
         category: {
           $in: ['recruit:category:Other', 'document:category:Other', 'tracker:category:Other'] as Ref<TagCategory>[]
@@ -669,8 +670,7 @@ export async function restoreRecruitingTaskTypes (
   transactorUrl: string
 ): Promise<void> {
   const connection = (await connect(transactorUrl, workspaceId, undefined, {
-    mode: 'backup',
-    model: 'upgrade'
+    mode: 'backup'
   })) as unknown as CoreClient & BackupClient
   const client = getMongoClient(mongoUrl)
   try {
@@ -772,6 +772,7 @@ export async function restoreRecruitingTaskTypes (
         statusCategories.sort(compareCategories)
 
         const createTxNew: TxCreateDoc<TaskType> = {
+          _uuid: workspaceId,
           _id: generateId(),
           _class: core.class.TxCreateDoc,
           space: core.space.Tx,
@@ -822,7 +823,9 @@ export async function restoreRecruitingTaskTypes (
   } catch (err: any) {
     console.trace(err)
   } finally {
+    // TODO: Add force-close
     client.close()
+    await connection.sendForceClose()
     await connection.close()
   }
 }
@@ -833,8 +836,7 @@ export async function restoreHrTaskTypesFromUpdates (
   transactorUrl: string
 ): Promise<void> {
   const connection = (await connect(transactorUrl, workspaceId, undefined, {
-    mode: 'backup',
-    model: 'upgrade'
+    mode: 'backup'
   })) as unknown as CoreClient & BackupClient
   const client = getMongoClient(mongoUrl)
   try {
@@ -927,6 +929,7 @@ export async function restoreHrTaskTypesFromUpdates (
         const ofClassClass = hierarchy.getClass(recruit.class.Applicant)
 
         await db.collection<TxCreateDoc<Doc>>(DOMAIN_TX).insertOne({
+          _uuid: workspaceId,
           _id: generateId(),
           _class: core.class.TxCreateDoc,
           space: core.space.Tx,
@@ -946,6 +949,7 @@ export async function restoreHrTaskTypesFromUpdates (
         })
 
         createTaskTypeTx = {
+          _uuid: workspaceId,
           _id: generateId(),
           _class: core.class.TxCreateDoc,
           space: core.space.Tx,
@@ -980,6 +984,7 @@ export async function restoreHrTaskTypesFromUpdates (
       const ofClassClass = hierarchy.getClass(recruit.class.Vacancy)
 
       await db.collection<TxCreateDoc<Doc>>(DOMAIN_TX).insertOne({
+        _uuid: workspaceId,
         _id: generateId(),
         _class: core.class.TxCreateDoc,
         space: core.space.Tx,
@@ -999,6 +1004,7 @@ export async function restoreHrTaskTypesFromUpdates (
       })
 
       const createProjectTypeTx: TxCreateDoc<ProjectType> = {
+        _uuid: workspaceId,
         _id: generateId(),
         _class: core.class.TxCreateDoc,
         space: core.space.Tx,
@@ -1031,6 +1037,7 @@ export async function restoreHrTaskTypesFromUpdates (
     console.trace(err)
   } finally {
     client.close()
+    await connection.sendForceClose()
     await connection.close()
   }
 }

dev/tool/src/configuration.ts

@@ -41,7 +41,7 @@ export async function changeConfiguration (
     }
     const enable = (cmd.enable ?? '').trim().split(',')
     console.log('enable', enable)
-    const ops = new TxFactory(core.account.ConfigUser)
+    const ops = new TxFactory(core.account.ConfigUser, workspaceId)
     if (enable.length > 0) {
       const p = config.filter((it) => enable.includes(it.pluginId) || enable.includes('*'))
       for (const pp of p) {

dev/tool/src/db.ts

@@ -96,7 +96,7 @@ async function moveWorkspace (
     const token = generateToken(systemAccountUuid, wsId, { service: 'tool' })
     const endpoint = await getTransactorEndpoint(token, 'external')
     const connection = (await connect(endpoint, wsId, undefined, {
-      model: 'upgrade'
+      mode: 'backup'
     })) as unknown as Client & BackupClient
     for (const collection of collections) {
       const domain = translateDomain(collection.collectionName)

dev/tool/src/index.ts

@@ -36,6 +36,7 @@ import {
 } from '@hcengineering/server-backup'
 import serverClientPlugin, { getAccountClient } from '@hcengineering/server-client'
 import {
+  createBackupPipeline,
   registerAdapterFactory,
   registerDestroyFactory,
   registerServerPlugins,
@@ -84,8 +85,12 @@ import {
   shutdownPostgres
 } from '@hcengineering/postgres'
 import {
+  createDummyStorageAdapter,
   QueueTopic,
   workspaceEvents,
+  wrapPipeline,
+  type Pipeline,
+  type PipelineFactory,
   type QueueWorkspaceMessage,
   type StorageAdapter
 } from '@hcengineering/server-core'
@@ -1155,6 +1160,16 @@ export function devTool (
 
           const workspaceStorage: StorageAdapter | undefined =
             storageConfig !== undefined ? buildStorageFromConfig(storageConfig) : undefined
+
+          const { dbUrl, txes } = prepareTools()
+
+          const pipelineFactory: PipelineFactory = createBackupPipeline(toolCtx, dbUrl, txes, {
+            externalStorage: workspaceStorage ?? createDummyStorageAdapter(),
+            usePassedCtx: true
+          })
+
+          let pipeline: Pipeline | undefined
+
           await restore(toolCtx, await getWorkspaceTransactorEndpoint(workspace), wsIds, storage, {
             date: parseInt(date ?? '-1'),
             merge: cmd.merge,
@@ -1163,7 +1178,13 @@ export function devTool (
             include: cmd.include === '*' ? undefined : new Set(cmd.include.split(';')),
             skip: new Set(cmd.skip.split(';')),
             storageAdapter: workspaceStorage,
-            historyFile: cmd.historyFile
+            historyFile: cmd.historyFile,
+            getConnection: async () => {
+              if (pipeline === undefined) {
+                pipeline = await pipelineFactory(toolCtx, wsIds, () => {}, null, null)
+              }
+              return wrapPipeline(toolCtx, pipeline, wsIds)
+            }
           })
           const queue = getPlatformQueue('tool', ws.region)
           const wsProducer = queue.getProducer<QueueWorkspaceMessage>(toolCtx, QueueTopic.Workspace)

models/board/src/migration.ts

@@ -14,7 +14,7 @@
 //
 
 import { boardId, type Card } from '@hcengineering/board'
-import { TxOperations } from '@hcengineering/core'
+import { type TxOperations } from '@hcengineering/core'
 import {
   createOrUpdate,
   tryMigrate,
@@ -96,8 +96,7 @@ export const boardOperation: MigrateOperation = {
       {
         state: 'board0001',
         func: async (client) => {
-          const ops = new TxOperations(client, core.account.System)
-          await createDefaults(ops)
+          await createDefaults(client)
         }
       }
     ])