Support export to JSON, CSV (#8193)

Signed-off-by: Anna Khismatullina <[email protected]>

Author: akhismat

.vscode/launch.json

@@ -798,5 +798,24 @@
       "outputCapture": "std",
       "cwd": "${workspaceRoot}/dev/import-tool"
     },
+    {
+      "name": "Debug Export service",
+      "type": "node",
+      "request": "launch",
+      "args": ["src/index.ts"],
+      "env": {
+        "PORT": "4009",
+        "SECRET": "secret",
+        "ACCOUNTS_URL": "http://localhost:3000",
+        "MINIO_ENDPOINT": "localhost",
+        "MINIO_ACCESS_KEY": "minioadmin",
+        "MINIO_SECRET_KEY": "minioadmin",
+        "TRANSACTOR_URL": "ws://localhost:3333",
+        "SERVICE_ID": "export-service"
+      },
+      "runtimeArgs": ["--nolazy", "-r", "ts-node/register"],
+      "sourceMaps": true,
+      "cwd": "${workspaceRoot}/services/export/pod-export"
+    }
   ]
 }

common/config/rush/command-line.json

@@ -246,7 +246,7 @@
       "summary": "Build docker with platform",
       "description": "use to build all docker containers required for platform",
       "safeForSimultaneousRushProcesses": true,
-      "shellCommand": "rush docker:build -p 20 --to @hcengineering/pod-server --to @hcengineering/pod-front --to @hcengineering/prod --to @hcengineering/pod-account --to @hcengineering/pod-workspace --to @hcengineering/pod-collaborator --to @hcengineering/tool --to @hcengineering/pod-print --to @hcengineering/pod-sign --to @hcengineering/pod-analytics-collector --to @hcengineering/rekoni-service --to @hcengineering/pod-ai-bot --to @hcengineering/import-tool --to @hcengineering/pod-stats --to @hcengineering/pod-fulltext --to @hcengineering/pod-love --to @hcengineering/green --to @hcengineering/pod-mail --to @hcengineering/pod-datalake --to @hcengineering/pod-hook"
+      "shellCommand": "rush docker:build -p 20 --to @hcengineering/pod-server --to @hcengineering/pod-front --to @hcengineering/prod --to @hcengineering/pod-account --to @hcengineering/pod-workspace --to @hcengineering/pod-collaborator --to @hcengineering/tool --to @hcengineering/pod-print --to @hcengineering/pod-sign --to @hcengineering/pod-analytics-collector --to @hcengineering/rekoni-service --to @hcengineering/pod-ai-bot --to @hcengineering/import-tool --to @hcengineering/pod-stats --to @hcengineering/pod-fulltext --to @hcengineering/pod-love --to @hcengineering/green --to @hcengineering/pod-mail --to @hcengineering/pod-datalake --to @hcengineering/pod-hook --to @hcengineering/pod-export"
     },
     {
       "commandKind": "global",

common/config/rush/pnpm-lock.yaml

@@ -373,6 +373,7 @@ export async function configurePlatform (): Promise<void> {
   configureNotifications()
 
   setMetadata(setting.metadata.BackupUrl, config.BACKUP_URL ?? '')
+  setMetadata(setting.metadata.ExportUrl, config.EXPORT_URL ?? '')
 
   if (config.INITIAL_URL !== '') {
     setLocationStorageKey('uberflow_child')

desktop/src/ui/platform.ts

@@ -41,6 +41,7 @@ export interface Config {
   STREAM_URL?: string
   BACKUP_URL?: string
   PUBLIC_SCHEDULE_URL?: string
+  EXPORT_URL?: string
 }
 
 export interface Branding {

desktop/src/ui/types.ts

@@ -440,6 +440,24 @@ services:
 #      - ACCOUNTS_URL=http://huly.local:3000
 #      - SERVICE_ID=telegram-bot-service
 #      - STATS_URL=http://huly.local:4900
+  export:
+    image: hardcoreeng/export
+    extra_hosts:
+      - 'huly.local:host-gateway'
+    links:
+      - mongodb
+      - minio
+      - stats
+    ports:
+      - 4009:4009
+    environment:
+      - PORT=4009
+      - SECRET=secret
+      - SERVICE_ID=export-service
+      - DB_URL=${MONGO_URL}
+      - STATS_URL=http://huly.local:4900
+      - STORAGE_CONFIG=${STORAGE_CONFIG}
+      - ACCOUNTS_URL=http://huly.local:3000
 volumes:
   db:
   dbpg:

dev/docker-compose.yaml

@@ -17,11 +17,11 @@ import {
   ClickupImporter,
   defaultDocumentPreprocessors,
   DocumentConverter,
-  FrontFileUploader,
-  importNotion,
-  UnifiedFormatImporter,
   type DocumentConverterOptions,
   type FileUploader,
+  FrontFileUploader,
+  HulyFormatImporter,
+  importNotion,
   type Logger
 } from '@hcengineering/importer'
 import { setMetadata } from '@hcengineering/platform'
@@ -164,7 +164,7 @@ export function importTool (): void {
     .action(async (dir: string, cmd) => {
       const { workspace, user, password } = cmd
       await authorize(user, password, workspace, async (client, uploader) => {
-        const importer = new UnifiedFormatImporter(client, uploader, new ConsoleLogger())
+        const importer = new HulyFormatImporter(client, uploader, new ConsoleLogger())
         await importer.importFolder(dir)
       })
     })

dev/import-tool/src/index.ts

@@ -20,5 +20,6 @@
   "STATS_URL": "http://localhost:4900",
   "PASSWORD_STRICTNESS": "none",
   "STREAM_URL": "http://localhost:1080/recording",
-  "PUBLIC_SCHEDULE_URL": "http://localhost:8060"
+  "PUBLIC_SCHEDULE_URL": "http://localhost:8060",
+  "EXPORT_URL": "http://localhost:4009/export"
 }

dev/prod/public/config.json

@@ -176,6 +176,7 @@ export interface Config {
   BACKUP_URL?: string
   STREAM_URL?: string
   PUBLIC_SCHEDULE_URL?: string
+  EXPORT_URL?: string
 }
 
 export interface Branding {
@@ -560,6 +561,7 @@ export async function configurePlatform() {
   setMetadata(workbench.metadata.DefaultSpecial, myBranding.defaultSpecial ?? 'issues')
 
   setMetadata(setting.metadata.BackupUrl, config.BACKUP_URL ?? '')
+  setMetadata(setting.metadata.ExportUrl, config.EXPORT_URL ?? '')
 
   initThemeStore()
 }

dev/prod/src/platform.ts

@@ -336,6 +336,20 @@ export function createModel (builder: Builder): void {
     },
     setting.ids.InviteSettings
   )
+  builder.createDoc(
+    setting.class.WorkspaceSettingCategory,
+    core.space.Model,
+    {
+      name: 'export',
+      label: setting.string.Export,
+      icon: setting.icon.Export,
+      component: setting.component.Export,
+      group: 'settings-editor',
+      role: AccountRole.User,
+      order: 4800
+    },
+    setting.ids.Export
+  )
   // Currently remove Support item from settings
   // builder.createDoc(
   //   setting.class.SettingsCategory,

models/setting/src/index.ts

@@ -18,7 +18,7 @@ import { markupToMarkdown } from '@hcengineering/text-markdown'
 import { mkdir, readdir, readFile, writeFile } from 'fs/promises'
 import * as yaml from 'js-yaml'
 import { basename, dirname, extname, join, relative } from 'path'
-import { UnifiedControlledDocumentHeader, UnifiedDocumentTemplateHeader } from '../huly/unified'
+import { HulyControlledDocumentHeader, HulyDocumentTemplateHeader } from '../huly/huly'
 
 export interface DocumentConverterOptions {
   outputPath: string
@@ -32,7 +32,7 @@ export interface DocumentState {
   path: string
   root: string
   markup: MarkupNode
-  header?: UnifiedControlledDocumentHeader | UnifiedDocumentTemplateHeader
+  header?: HulyControlledDocumentHeader | HulyDocumentTemplateHeader
 }
 
 export interface DocumentPreprocessorOptions<T> {