Merge remote-tracking branch 'origin/develop' into staging

Signed-off-by: Andrey Sobolev <[email protected]>

Author: haiodo

.github/issue_template.md

@@ -2,7 +2,7 @@
 A clear and concise description of the issue.
 
 ### Your environment
-* Version of huly
+* Version of Huly
 * Browser (and version)
 * Your operating system (and version)
 

.vscode/launch.json

@@ -95,9 +95,11 @@
         "ACCOUNT_PORT": "3000",
         "FRONT_URL": "http://localhost:8080",
         "SES_URL": "",
+        // "WS_LIVENESS_DAYS": "1",
         "MINIO_ACCESS_KEY": "minioadmin",
         "MINIO_SECRET_KEY": "minioadmin",
-        "MINIO_ENDPOINT": "localhost"
+        "MINIO_ENDPOINT": "localhost",
+        // "DISABLE_SIGNUP": "true",
         // "INIT_SCRIPT_URL": "https://raw.githubusercontent.com/hcengineering/init/main/script.yaml",
         // "INIT_WORKSPACE": "onboarding",
       },

dev/docker-compose.yaml

@@ -83,8 +83,10 @@ services:
       - RESERVED_DB_NAMES=telegram,gmail,github
       - MODEL_ENABLED=*
       - LAST_NAME_FIRST=true
+      # - WS_LIVENESS_DAYS=1
       - ACCOUNTS_URL=http://host.docker.internal:3000
       - BRANDING_PATH=/var/cfg/branding.json
+      # - DISABLE_SIGNUP=true
       # - INIT_SCRIPT_URL=https://raw.githubusercontent.com/hcengineering/init/main/script.yaml
       # - INIT_WORKSPACE=onboarding      
     restart: unless-stopped
@@ -197,6 +199,7 @@ services:
       - DESKTOP_UPDATES_URL=https://dist.huly.io
       - DESKTOP_UPDATES_CHANNEL=dev
       - BRANDING_URL=http://host.docker.internal:8087/branding.json
+      # - DISABLE_SIGNUP=true
     restart: unless-stopped
   transactor:
     image: hardcoreeng/transactor

dev/tool/src/db.ts

@@ -54,6 +54,10 @@ export async function moveFromMongoToPG (
   client.close()
 }
 
+function escapeBackticks (str: string): string {
+  return str.replaceAll("'", "''")
+}
+
 async function moveWorkspace (
   accountDb: AccountDB,
   mongo: MongoClient,
@@ -77,32 +81,38 @@ async function moveWorkspace (
     for (const collection of collections) {
       const cursor = collection.find()
       const domain = translateDomain(collection.collectionName)
+      const current = await pgClient.query(`SELECT _id FROM ${domain} WHERE "workspaceId" = $1`, [ws.workspace])
+      const currentIds = new Set(current.rows.map((r) => r._id))
       console.log('move domain', domain)
+      const docs: Doc[] = []
       while (true) {
-        const doc = (await cursor.next()) as Doc | null
-        if (doc === null) break
-        try {
-          const converted = convertDoc(doc, ws.workspaceName ?? ws.workspace)
-          await retryTxn(pgClient, async (client) => {
-            await client.query(
-              `INSERT INTO ${domain} (_id, "workspaceId", _class, "createdBy", "modifiedBy", "modifiedOn", "createdOn", space, "attachedTo", data) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)`,
-              [
-                converted._id,
-                converted.workspaceId,
-                converted._class,
-                converted.createdBy ?? converted.modifiedBy,
-                converted.modifiedBy,
-                converted.modifiedOn,
-                converted.createdOn ?? converted.modifiedOn,
-                converted.space,
-                converted.attachedTo,
-                converted.data
-              ]
-            )
-          })
-        } catch (err) {
-          console.log('error when move doc', doc._id, doc._class, err)
-          continue
+        while (docs.length < 50000) {
+          const doc = (await cursor.next()) as Doc | null
+          if (doc === null) break
+          if (currentIds.has(doc._id)) continue
+          docs.push(doc)
+        }
+        if (docs.length === 0) break
+        while (docs.length > 0) {
+          const part = docs.splice(0, 500)
+          const vals = part
+            .map((doc) => {
+              const d = convertDoc(doc, ws.workspace)
+              return `('${d._id}', '${d.workspaceId}', '${d._class}', '${d.createdBy ?? d.modifiedBy}', '${d.modifiedBy}', ${d.modifiedOn}, ${d.createdOn ?? d.modifiedOn}, '${d.space}', ${
+                d.attachedTo != null ? `'${d.attachedTo}'` : 'NULL'
+              }, '${escapeBackticks(JSON.stringify(d.data))}')`
+            })
+            .join(', ')
+          try {
+            await retryTxn(pgClient, async (client) => {
+              await client.query(
+                `INSERT INTO ${translateDomain(domain)} (_id, "workspaceId", _class, "createdBy", "modifiedBy", "modifiedOn", "createdOn", space, "attachedTo", data) VALUES ${vals}`
+              )
+            })
+          } catch (err) {
+            console.log('error when move doc to', domain, err)
+            continue
+          }
         }
       }
     }

dev/tool/src/index.ts

@@ -1598,7 +1598,7 @@ export function devTool (
       throw new Error('mongodbUri is not set')
     }
 
-    await withDatabase(dbUrl, async (db) => {
+    await withDatabase(mongodbUri, async (db) => {
       const workspaces = await listWorkspacesRaw(db)
       workspaces.sort((a, b) => b.lastVisit - a.lastVisit)
       await moveFromMongoToPG(
@@ -1617,7 +1617,7 @@ export function devTool (
       throw new Error('mongodbUri is not set')
     }
 
-    await withDatabase(dbUrl, async (db) => {
+    await withDatabase(mongodbUri, async (db) => {
       const workspaceInfo = await getWorkspaceById(db, workspace)
       if (workspaceInfo === null) {
         throw new Error(`workspace ${workspace} not found`)

dev/tool/src/workspace.ts

@@ -112,7 +112,7 @@ export async function recreateElastic (
     const db = getWorkspaceMongoDB(_client, workspaceId)
     await db
       .collection(DOMAIN_DOC_INDEX_STATE)
-      .updateMany({ _class: core.class.DocIndexState }, { $set: { stages: {} } })
+      .updateMany({ _class: core.class.DocIndexState }, { $set: { stages: {}, needIndex: true } })
     await connection.sendForceClose()
   } finally {
     client.close()

models/contact/src/index.ts

@@ -955,7 +955,8 @@ export function createModel (builder: Builder): void {
   )
 
   // Allow to use fuzzy search for mixins
-  builder.mixin(contact.class.Contact, core.class.Class, core.mixin.FullTextSearchContext, {
+  builder.createDoc(core.class.FullTextSearchContext, core.space.Model, {
+    toClass: contact.class.Contact,
     fullTextSummary: true
   })
 

models/controlled-documents/src/index.ts

@@ -549,7 +549,8 @@ export function createModel (builder: Builder): void {
     func: documents.function.GetAllDocumentStates
   })
 
-  builder.mixin(documents.class.Document, core.class.Class, core.mixin.FullTextSearchContext, {
+  builder.createDoc(core.class.FullTextSearchContext, core.space.Model, {
+    toClass: documents.class.Document,
     fullTextSummary: true,
     childProcessingAllowed: true
   })
@@ -886,11 +887,13 @@ export function defineNotifications (builder: Builder): void {
 }
 
 export function defineSearch (builder: Builder): void {
-  builder.mixin(documents.class.Document, core.class.Class, core.mixin.FullTextSearchContext, {
+  builder.createDoc(core.class.FullTextSearchContext, core.space.Model, {
+    toClass: documents.class.Document,
     parentPropagate: true
   })
 
-  builder.mixin(documents.class.DocumentMeta, core.class.Class, core.mixin.FullTextSearchContext, {
+  builder.createDoc(core.class.FullTextSearchContext, core.space.Model, {
+    toClass: documents.class.DocumentMeta,
     fullTextSummary: true,
     childProcessingAllowed: true,
     propagate: []

models/core/src/core.ts

@@ -361,8 +361,10 @@ export class TDocIndexState extends TDoc implements DocIndexState {
     generationId?: string
 }
 
-@MMixin(core.mixin.FullTextSearchContext, core.class.Class)
-export class TFullTextSearchContext extends TClass implements FullTextSearchContext {}
+@Model(core.class.FullTextSearchContext, core.class.Doc, DOMAIN_MODEL)
+export class TFullTextSearchContext extends TDoc implements FullTextSearchContext {
+  toClass!: Ref<Class<Doc<Space>>>
+}
 
 @MMixin(core.mixin.ConfigurationElement, core.class.Class)
 export class TConfigurationElement extends TClass implements ConfigurationElement {

models/core/src/index.ts

@@ -309,7 +309,8 @@ export function createModel (builder: Builder): void {
     ]
   })
 
-  builder.mixin(core.class.Space, core.class.Class, core.mixin.FullTextSearchContext, {
+  builder.createDoc(core.class.FullTextSearchContext, core.space.Model, {
+    toClass: core.class.Space,
     childProcessingAllowed: false
   })