feat(bulk-import): implement template execution for bulk-import (#1266)

* feat(bulk-import): implement bulk import with help of scaffolder template

Signed-off-by: Oleksandr Andriienko <[email protected]>

* feat(bulk-import): code cleanup

Signed-off-by: Oleksandr Andriienko <[email protected]>

* feat(bulk-import): cleanup deps

Signed-off-by: Oleksandr Andriienko <[email protected]>

* feat(bulk-import): fix some of the failing tests

Signed-off-by: Patrick <[email protected]>

* feat(bulk-import): implement some gitlab specific stuff

Signed-off-by: Oleksandr Andriienko <[email protected]>

* feat(bulk-import): simplify the starting template

Signed-off-by: Patrick <[email protected]>

* feat(bulk-import): swap branch descriptions

Signed-off-by: Patrick <[email protected]>

* feat(bulk-import): fix remaining failing tests

Signed-off-by: Patrick <[email protected]>

* feat(bulk-import): run yarn dedupe

Signed-off-by: Patrick <[email protected]>

* feat(bulk-import): remove unused file

Signed-off-by: Oleksandr Andriienko <[email protected]>

* feat(bulk-import): fix link bug, use entityRef in config

Signed-off-by: Oleksandr Andriienko <[email protected]>

* feat(bulk-import): implement task localization

Signed-off-by: Oleksandr Andriienko <[email protected]>

* feat(bulk-import): fix compilation

Signed-off-by: Oleksandr Andriienko <[email protected]>

* feat(bulk-import): hide dao types, update api.md

Signed-off-by: Oleksandr Andriienko <[email protected]>

* feat(bulk-import): update sqlite dependency for node 22

Signed-off-by: Oleksandr Andriienko <[email protected]>

* feat(bulk-import): fix report again

Signed-off-by: Oleksandr Andriienko <[email protected]>

* feat(bulk-import): cleanup

Signed-off-by: Oleksandr Andriienko <[email protected]>

* feat(bulk-import): decrease code duplication

Signed-off-by: Oleksandr Andriienko <[email protected]>

* feat(bulk-import): decrease code duplication

Signed-off-by: Oleksandr Andriienko <[email protected]>

* feat(bulk-import): skip duplication for fixtures

Signed-off-by: Oleksandr Andriienko <[email protected]>

* feat(bulk-import): fix sonarcloud warnings

Signed-off-by: Oleksandr Andriienko <[email protected]>

* feat(bulk-import): fix some sonarcloud warnings

Signed-off-by: Oleksandr Andriienko <[email protected]>

* feat(bulk-import): fix duplicated DB migration

Signed-off-by: Oleksandr Andriienko <[email protected]>

* feat(bulk-import): remove confusing template

Signed-off-by: Oleksandr Andriienko <[email protected]>

* feat(bulk-import): fix compilation

Signed-off-by: Oleksandr Andriienko <[email protected]>

---------

Signed-off-by: Oleksandr Andriienko <[email protected]>
Signed-off-by: Patrick <[email protected]>
Co-authored-by: Patrick <[email protected]>

Author: AndrienkoAleksandr

.sonarcloud.properties

@@ -1 +1 @@
-sonar.cpd.exclusions=workspaces/*/plugins/*/**/*.test.*, workspaces/*/plugins/*/src/translations/*.ts
+sonar.cpd.exclusions=workspaces/*/plugins/*/**/*.test.*, workspaces/*/plugins/*/src/translations/*.ts, workspaces/*/plugins/*/__fixtures__/*

workspaces/bulk-import/.eslintignore

@@ -1,3 +1,4 @@
 playwright.config.ts
 dist/
-dist-types/
+dist-types/
+plugins/bulk-import-backend/src/generated/openapi.d.ts

workspaces/bulk-import/app-config.yaml

@@ -32,6 +32,9 @@ backend:
     client: better-sqlite3
     connection: ':memory:'
   # workingDirectory: /tmp # Use this to configure a working directory for the scaffolder, defaults to the OS temp-dir
+  reading:
+    allow:
+      - host: raw.githubusercontent.com
 
 integrations:
   gitlab:
@@ -92,9 +95,24 @@ catalog:
     - type: file
       target: ../../examples/entities.yaml
 
-    # Local example template
+    # Local example templates
     - type: file
-      target: ../../examples/template/template.yaml
+      target: ../../examples/template/get-default-branch.yaml
+      rules:
+        - allow: [Template]
+
+    - type: file
+      target: ../../examples/template/create-repo-with-catalog-info.yaml
+      rules:
+        - allow: [Template]
+
+    - type: file
+      target: ../../examples/template/create-pr-with-catalog-info.yaml
+      rules:
+        - allow: [Template]
+
+    - type: file
+      target: ../../examples/template/import-components-from-existing-repo.yaml
       rules:
         - allow: [Template]
 

workspaces/bulk-import/examples/template/content/catalog-info.yaml

@@ -1,8 +1,10 @@
 apiVersion: backstage.io/v1alpha1
 kind: Component
 metadata:
-  name: ${{ values.name | dump }}
+  name: ${{ values.name }}
+  annotations:
+    github.com/project-slug: ${{ values.organization }}/${{ values.name }}
 spec:
-  type: service
-  owner: user:guest
-  lifecycle: experimental
+  type: other
+  owner: ${{ values.organization }}
+  lifecycle: unknown

workspaces/bulk-import/examples/template/content/package.json

@@ -0,0 +1,59 @@
+apiVersion: scaffolder.backstage.io/v1beta3
+kind: Template
+metadata:
+  name: create-pr-with-catalog-info
+  title: Propose catalog-info.yaml in the pull request
+  description: Creates a catalog-info.yaml in the given GitHub repo.
+
+spec:
+  owner: user:default/guest
+  type: service
+
+  parameters:
+    - title: Provide repository details
+      required:
+        - repoUrl
+      properties:
+        repoUrl:
+          type: string
+          title: Repository URL (Backstage format)
+          description: 'e.g. github.com?owner=Org&repo=repoName'
+        organization:
+          type: string
+          title: Owner of the Repository
+        branchName:
+          type: string
+          title: The branch to add the catalog entity to
+        targetBranchName:
+          type: string
+          title: The branch to target the PR to
+        name:
+          type: string
+          title: Name of the repository
+
+  steps:
+    - id: fetch-base
+      name: Fetch Base
+      action: fetch:template
+      input:
+        url: ./content
+        values:
+          name: ${{ parameters.name }}
+          organization: ${{ parameters.organization }}
+
+    - id: publish
+      name: Create PR to add catalog-info.yaml
+      action: publish:github:pull-request
+      input:
+        repoUrl: ${{ parameters.repoUrl }}
+        branchName: ${{ parameters.branchName }}
+        targetBranchName: ${{ parameters.targetBranchName }}
+        title: 'Add catalog-info.yaml'
+        description: 'This PR adds a default catalog-info.yaml for Backstage.'
+
+    - id: register
+      name: Register catalog-info.yaml
+      action: catalog:register
+      input:
+        optional: true
+        repoContentsUrl: https://github.com/${{ parameters.organization }}/${{ parameters.name }}/blob/${{ parameters.branchName }}/catalog-info.yaml

workspaces/bulk-import/examples/template/create-pr-with-catalog-info.yaml

@@ -6,7 +6,7 @@
     "node": "18 || 20"
   },
   "scripts": {
-    "start": "backstage-cli repo start",
+    "start": "NODE_OPTIONS=--no-node-snapshot backstage-cli repo start",
     "tsc": "tsc",
     "tsc:full": "tsc --skipLibCheck true --incremental false",
     "build:all": "backstage-cli repo build --all",

workspaces/bulk-import/examples/template/template.yaml

@@ -48,6 +48,7 @@ import { TechDocsAddons } from '@backstage/plugin-techdocs-react';
 import { UserSettingsPage } from '@backstage/plugin-user-settings';
 import { BulkImportPage } from '@red-hat-developer-hub/backstage-plugin-bulk-import';
 import { bulkImportTranslations } from '@red-hat-developer-hub/backstage-plugin-bulk-import/alpha';
+import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
 import { getThemes } from '@redhat-developer/red-hat-developer-hub-theme';
 import { Navigate, Route } from 'react-router-dom';
 import { apis } from './apis';
@@ -98,6 +99,8 @@ const app = createApp({
   themes: getThemes(),
 });
 
+const queryClient = new QueryClient();
+
 const routes = (
   <FlatRoutes>
     <Route path="/" element={<Navigate to="catalog" />} />
@@ -133,7 +136,14 @@ const routes = (
     <Route path="/settings" element={<UserSettingsPage />} />
     <Route path="/catalog-graph" element={<CatalogGraphPage />} />
     <Route path="/bulk-import" element={<Navigate to="repositories" />} />
-    <Route path="/bulk-import/repositories" element={<BulkImportPage />} />
+    <Route
+      path="/bulk-import/repositories"
+      element={
+        <QueryClientProvider client={queryClient}>
+          <BulkImportPage />
+        </QueryClientProvider>
+      }
+    />
   </FlatRoutes>
 );
 

workspaces/bulk-import/package.json

@@ -37,15 +37,17 @@
     "@backstage/plugin-permission-node": "^0.10.3",
     "@backstage/plugin-proxy-backend": "^0.6.5",
     "@backstage/plugin-scaffolder-backend": "^2.2.0",
+    "@backstage/plugin-scaffolder-backend-module-github": "^0.8.2",
     "@backstage/plugin-search-backend": "^2.0.5",
     "@backstage/plugin-search-backend-module-catalog": "^0.3.7",
     "@backstage/plugin-search-backend-module-pg": "^0.5.47",
     "@backstage/plugin-search-backend-module-techdocs": "^0.4.5",
     "@backstage/plugin-search-backend-node": "^1.3.14",
     "@backstage/plugin-techdocs-backend": "^2.0.5",
     "@red-hat-developer-hub/backstage-plugin-bulk-import-backend": "workspace:^",
+    "@roadiehq/scaffolder-backend-module-utils": "^3.6.0",
     "app": "link:../app",
-    "better-sqlite3": "^9.0.0",
+    "better-sqlite3": "^12.4.1",
     "node-gyp": "^10.0.0",
     "pg": "^8.11.3",
     "winston": "^3.2.1"