From f84521149afc49b8353f334ddd83580526b09e7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Chopin?= Date: Fri, 17 Jan 2025 16:58:05 +0100 Subject: [PATCH] feat(database): add support for multiple database migrations directories (#423) --- docs/content/1.docs/2.features/database.md | 99 ++++++++++++++++--- docs/nuxt.config.ts | 5 + docs/pages/index.vue | 2 +- playground/layers/auth/nuxt.config.ts | 1 + .../database/migrations/0001_create-users.sql | 5 + .../cms/db-migrations/0001_create-pages.sql | 5 + playground/modules/cms/index.ts | 14 +++ playground/nuxt.config.ts | 11 +++ playground/server/database/queries/admin.sql | 6 ++ src/features.ts | 33 ++++++- src/module.ts | 25 ++--- .../database/server/plugins/migrations.dev.ts | 10 +- .../server/utils/migrations/helpers.ts | 69 +++++++++++-- .../server/utils/migrations/migrations.ts | 40 ++++++-- .../server/utils/migrations/remote.ts | 86 ++++++++++------ src/types/module.ts | 15 +++ src/utils/build.ts | 15 ++- 17 files changed, 350 insertions(+), 91 deletions(-) create mode 100644 playground/layers/auth/nuxt.config.ts create mode 100644 playground/layers/auth/server/database/migrations/0001_create-users.sql create mode 100644 playground/modules/cms/db-migrations/0001_create-pages.sql create mode 100644 playground/modules/cms/index.ts create mode 100644 playground/server/database/queries/admin.sql diff --git a/docs/content/1.docs/2.features/database.md b/docs/content/1.docs/2.features/database.md index 92e2d22a..8ae8c737 100644 --- a/docs/content/1.docs/2.features/database.md +++ b/docs/content/1.docs/2.features/database.md @@ -241,11 +241,63 @@ This method can have poorer performance (prepared statements can be reused in so ## Database Migrations -Database migrations provide version control for your database schema. They track changes and ensure consistent schema evolution across all environments through incremental updates. +Database migrations provide version control for your database schema. They track changes and ensure consistent schema evolution across all environments through incremental updates. NuxtHub supports SQL migration files (`.sql`). + +### Migrations Directories + +NuxtHub scans the `server/database/migrations` directory for migrations **for each [Nuxt layer](https://nuxt.com/docs/getting-started/layers)**. + +If you need to scan additional migrations directories, you can specify them in your `nuxt.config.ts` file. + +```ts [nuxt.config.ts] +export default defineNuxtConfig({ + hub: { + // Array of additional migration directories to scan + databaseMigrationsDirs: [ + 'my-module/db-migrations/' + ] + } +}) +``` +::note +NuxtHub will scan both `server/database/migrations` and `my-module/db-migrations` directories for `.sql` files. +:: + +If you want more control to the migrations directories or you are working on a [Nuxt module](https://nuxt.com/docs/guide/going-further/modules), you can use the `hub:database:migrations:dirs` hook: + +::code-group +```ts [modules/auth/index.ts] +import { createResolver, defineNuxtModule } from 'nuxt/kit' + +export default defineNuxtModule({ + meta: { + name: 'my-auth-module' + }, + setup(options, nuxt) { + const { resolve } = createResolver(import.meta.url) + + nuxt.hook('hub:database:migrations:dirs', (dirs) => { + dirs.push(resolve('db-migrations')) + }) + } +}) +``` +```sql [modules/auth/db-migrations/0001_create-users.sql] +CREATE TABLE IF NOT EXISTS users ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + email TEXT NOT NULL +); +``` +:: + +::tip +All migrations files are copied to the `.data/hub/database/migrations` directory when you run Nuxt. This consolidated view helps you track all migrations and enables you to use `npx nuxthub database migrations ` commands. +:: ### Automatic Application -SQL migrations in `server/database/migrations/*.sql` are automatically applied when you: +All `.sql` files in the database migrations directories are automatically applied when you: - Start the development server (`npx nuxt dev` or [`npx nuxt dev --remote`](/docs/getting-started/remote-storage)) - Preview builds locally ([`npx nuxthub preview`](/changelog/nuxthub-preview)) - Deploy via [`npx nuxthub deploy`](/docs/getting-started/deploy#nuxthub-cli) or [Cloudflare Pages CI](/docs/getting-started/deploy#cloudflare-pages-ci) @@ -275,7 +327,6 @@ Migration files are created in `server/database/migrations/`. After creation, add your SQL queries to modify the database schema. - ::note{to="/docs/recipes/drizzle#npm-run-dbgenerate"} With [Drizzle ORM](/docs/recipes/drizzle), migrations are automatically created when you run `npx drizzle-kit generate`. :: @@ -327,23 +378,39 @@ NUXT_HUB_PROJECT_URL= NUXT_HUB_PROJECT_SECRET_KEY= nuxthub database ``` :: -### Migrating from Drizzle ORM +### Post-Migration Queries -Since NuxtHub doesn't recognize previously applied Drizzle ORM migrations (stored in `__drizzle_migrations`), it will attempt to rerun all migrations in `server/database/migrations/*.sql`. To prevent this: +::important +This feature is for advanced use cases. As the queries are run after the migrations process (see [Automatic Application](#automatic-application)), you want to make sure your queries are idempotent. +:: -1. Mark existing migrations as applied in each environment: +Sometimes you need to run additional queries after migrations are applied without tracking them in the migrations table. - ```bash [Terminal] - # Local environment - npx nuxthub database migrations mark-all-applied +NuxtHub provides the `hub:database:queries:paths` hook for this purpose: - # Preview environment - npx nuxthub database migrations mark-all-applied --preview +::code-group +```ts [modules/admin/index.ts] +import { createResolver, defineNuxtModule } from 'nuxt/kit' - # Production environment - npx nuxthub database migrations mark-all-applied --production - ``` +export default defineNuxtModule({ + meta: { + name: 'my-auth-module' + }, + setup(options, nuxt) { + const { resolve } = createResolver(import.meta.url) -2. Remove `server/plugins/database.ts` as it's no longer needed. + nuxt.hook('hub:database:queries:paths', (queries) => { + // Add SQL files to run after migrations + queries.push(resolve('./db-queries/seed-admin.sql')) + }) + } +}) +``` +```sql [modules/admin/db-queries/seed-admin.sql] +INSERT OR IGNORE INTO admin_users (id, email, password) VALUES (1, 'admin@nuxt.com', 'admin'); +``` +:: -That's it! You can keep using `npx drizzle-kit generate` to generate migrations when updating your Drizzle ORM schema. +::note +These queries run after all migrations are applied but are not tracked in the `_hub_migrations` table. Use this for operations that should run when deploying your project. +:: diff --git a/docs/nuxt.config.ts b/docs/nuxt.config.ts index 1cc642dc..e16b3f75 100644 --- a/docs/nuxt.config.ts +++ b/docs/nuxt.config.ts @@ -17,6 +17,11 @@ export default defineNuxtConfig({ devtools: { enabled: true }, + content: { + highlight: { + langs: ['sql'] + } + }, routeRules: { '/': { prerender: true }, '/api/search.json': { prerender: true }, diff --git a/docs/pages/index.vue b/docs/pages/index.vue index b9a26bd5..5847519f 100644 --- a/docs/pages/index.vue +++ b/docs/pages/index.vue @@ -98,7 +98,7 @@ onMounted(() => { /> - Used and loved by 7K+ developers and teams. + Used and loved by 8K+ developers and teams. diff --git a/playground/layers/auth/nuxt.config.ts b/playground/layers/auth/nuxt.config.ts new file mode 100644 index 00000000..268da7f8 --- /dev/null +++ b/playground/layers/auth/nuxt.config.ts @@ -0,0 +1 @@ +export default defineNuxtConfig({}) diff --git a/playground/layers/auth/server/database/migrations/0001_create-users.sql b/playground/layers/auth/server/database/migrations/0001_create-users.sql new file mode 100644 index 00000000..7a2878c1 --- /dev/null +++ b/playground/layers/auth/server/database/migrations/0001_create-users.sql @@ -0,0 +1,5 @@ +CREATE TABLE IF NOT EXISTS users ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + email TEXT NOT NULL, + password TEXT NOT NULL +); diff --git a/playground/modules/cms/db-migrations/0001_create-pages.sql b/playground/modules/cms/db-migrations/0001_create-pages.sql new file mode 100644 index 00000000..54612bd5 --- /dev/null +++ b/playground/modules/cms/db-migrations/0001_create-pages.sql @@ -0,0 +1,5 @@ +CREATE TABLE IF NOT EXISTS pages ( + id INTEGER PRIMARY KEY, + title TEXT NOT NULL, + content TEXT NOT NULL +); diff --git a/playground/modules/cms/index.ts b/playground/modules/cms/index.ts new file mode 100644 index 00000000..867e6b3e --- /dev/null +++ b/playground/modules/cms/index.ts @@ -0,0 +1,14 @@ +import { createResolver, defineNuxtModule } from 'nuxt/kit' + +export default defineNuxtModule({ + meta: { + name: 'my-auth-module' + }, + setup(options, nuxt) { + const { resolve } = createResolver(import.meta.url) + + nuxt.hook('hub:database:migrations:dirs', (dirs) => { + dirs.push(resolve('db-migrations')) + }) + } +}) diff --git a/playground/nuxt.config.ts b/playground/nuxt.config.ts index 2d15fda6..6619337a 100644 --- a/playground/nuxt.config.ts +++ b/playground/nuxt.config.ts @@ -1,6 +1,9 @@ // import { encodeHost } from 'ufo' +import { createResolver } from 'nuxt/kit' import module from '../src/module' +const resolver = createResolver(import.meta.url) + export default defineNuxtConfig({ modules: [ '@nuxt/ui', @@ -55,6 +58,14 @@ export default defineNuxtConfig({ } // projectUrl: ({ branch }) => branch === 'main' ? 'https://playground.nuxt.dev' : `https://${encodeHost(branch).replace(/\//g, '-')}.playground-to39.pages.dev` }, + hooks: { + 'hub:database:migrations:dirs': (dirs) => { + dirs.push('my-module/database/migrations') + }, + 'hub:database:queries:paths': (queries) => { + queries.push(resolver.resolve('server/database/queries/admin.sql')) + } + }, basicAuth: { enabled: process.env.NODE_ENV === 'production', diff --git a/playground/server/database/queries/admin.sql b/playground/server/database/queries/admin.sql new file mode 100644 index 00000000..7374dc86 --- /dev/null +++ b/playground/server/database/queries/admin.sql @@ -0,0 +1,6 @@ +CREATE TABLE IF NOT EXISTS admin_users ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + email TEXT NOT NULL UNIQUE, + password TEXT NOT NULL +); +INSERT OR IGNORE INTO admin_users (id, email, password) VALUES (1, 'admin@nuxt.com', 'admin'); diff --git a/src/features.ts b/src/features.ts index 5919aa59..e2be0af6 100644 --- a/src/features.ts +++ b/src/features.ts @@ -1,5 +1,6 @@ import { execSync } from 'node:child_process' import { pathToFileURL } from 'node:url' +import { mkdir } from 'node:fs/promises' import { isWindows } from 'std-env' import type { Nuxt } from '@nuxt/schema' import { join } from 'pathe' @@ -9,6 +10,7 @@ import { defu } from 'defu' import { $fetch } from 'ofetch' import { addDevToolsCustomTabs } from './utils/devtools' import { getCloudflareAccessHeaders } from './runtime/utils/cloudflareAccess' +import { copyDatabaseMigrationsToHubDir, copyDatabaseQueriesToHubDir } from './runtime/database/server/utils/migrations/helpers' const log = logger.withTag('nuxt:hub') const { resolve, resolvePath } = createResolver(import.meta.url) @@ -57,11 +59,25 @@ export interface HubConfig { } & Record } - migrationsPath?: string + dir?: string + databaseMigrationsDirs?: string[] + databaseQueriesPaths?: string[] openAPIRoute?: string } -export function setupBase(nuxt: Nuxt, hub: HubConfig) { +export async function setupBase(nuxt: Nuxt, hub: HubConfig) { + // Create the hub.dir directory + hub.dir = join(nuxt.options.rootDir, hub.dir!) + try { + await mkdir(hub.dir, { recursive: true }) + } catch (e: any) { + if (e.errno === -17) { + // File already exists + } else { + throw e + } + } + // Add Server scanning addServerScanDir(resolve('./runtime/base/server')) addServerImportsDir([resolve('./runtime/base/server/utils'), resolve('./runtime/base/server/utils/migrations')]) @@ -196,12 +212,19 @@ export async function setupCache(nuxt: Nuxt) { addServerScanDir(resolve('./runtime/cache/server')) } -export function setupDatabase(nuxt: Nuxt, hub: HubConfig) { - // Keep track of the path to migrations - hub.migrationsPath = join(nuxt.options.rootDir, 'server/database/migrations') +export async function setupDatabase(nuxt: Nuxt, hub: HubConfig) { // Add Server scanning addServerScanDir(resolve('./runtime/database/server')) addServerImportsDir(resolve('./runtime/database/server/utils')) + nuxt.hook('modules:done', async () => { + // Call hub:database:migrations:dirs hook + await nuxt.callHook('hub:database:migrations:dirs', hub.databaseMigrationsDirs!) + // Copy all migrations files to the hub.dir directory + await copyDatabaseMigrationsToHubDir(hub) + // Call hub:database:migrations:queries hook + await nuxt.callHook('hub:database:queries:paths', hub.databaseQueriesPaths!) + await copyDatabaseQueriesToHubDir(hub) + }) } export function setupKV(_nuxt: Nuxt) { diff --git a/src/module.ts b/src/module.ts index eb97cd04..9a5439bc 100644 --- a/src/module.ts +++ b/src/module.ts @@ -1,4 +1,4 @@ -import { mkdir, writeFile, readFile } from 'node:fs/promises' +import { writeFile, readFile } from 'node:fs/promises' import { argv } from 'node:process' import { defineNuxtModule, createResolver, logger, installModule, addServerHandler, addServerPlugin } from '@nuxt/kit' import { join } from 'pathe' @@ -38,6 +38,7 @@ export default defineNuxtModule({ let remoteArg = parseArgs(argv, { remote: { type: 'string' } }).remote as string remoteArg = (remoteArg === '' ? 'true' : remoteArg) const runtimeConfig = nuxt.options.runtimeConfig + const databaseMigrationsDirs = nuxt.options._layers?.map(layer => join(layer.config.serverDir!, 'database/migrations')).filter(Boolean) const hub = defu(runtimeConfig.hub || {}, options, { // Self-hosted project projectUrl: process.env.NUXT_HUB_PROJECT_URL || '', @@ -60,6 +61,9 @@ export default defineNuxtModule({ database: false, kv: false, vectorize: {}, + // Database Migrations + databaseMigrationsDirs, + databaseQueriesPaths: [], // Other options version, env: process.env.NUXT_HUB_ENV || 'production', @@ -102,14 +106,14 @@ export default defineNuxtModule({ }) } - setupBase(nuxt, hub as HubConfig) + await setupBase(nuxt, hub as HubConfig) hub.openapi && setupOpenAPI(nuxt, hub as HubConfig) hub.ai && await setupAI(nuxt, hub as HubConfig) hub.analytics && setupAnalytics(nuxt) hub.blob && setupBlob(nuxt) hub.browser && await setupBrowser(nuxt) hub.cache && await setupCache(nuxt) - hub.database && setupDatabase(nuxt, hub as HubConfig) + hub.database && await setupDatabase(nuxt, hub as HubConfig) hub.kv && setupKV(nuxt) Object.keys(hub.vectorize!).length && setupVectorize(nuxt, hub as HubConfig) @@ -196,17 +200,6 @@ export default defineNuxtModule({ log.info(`Using local storage from \`${hub.dir}\``) } - // Create the hub.dir directory - const hubDir = join(rootDir, hub.dir) - try { - await mkdir(hubDir, { recursive: true }) - } catch (e: any) { - if (e.errno === -17) { - // File already exists - } else { - throw e - } - } const workspaceDir = await findWorkspaceDir(rootDir) // Add it to .gitignore const gitignorePath = join(workspaceDir, '.gitignore') @@ -219,11 +212,11 @@ export default defineNuxtModule({ // const needWrangler = Boolean(hub.analytics || hub.blob || hub.database || hub.kv || Object.keys(hub.bindings.hyperdrive).length > 0) if (needWrangler) { // Generate the wrangler.toml file - const wranglerPath = join(hubDir, './wrangler.toml') + const wranglerPath = join(hub.dir, './wrangler.toml') await writeFile(wranglerPath, generateWrangler(nuxt, hub as HubConfig), 'utf-8') // @ts-expect-error cloudflareDev is not typed here nuxt.options.nitro.cloudflareDev = { - persistDir: hubDir, + persistDir: hub.dir, configPath: wranglerPath, silent: true } diff --git a/src/runtime/database/server/plugins/migrations.dev.ts b/src/runtime/database/server/plugins/migrations.dev.ts index 5b388cf2..0e51c8ad 100644 --- a/src/runtime/database/server/plugins/migrations.dev.ts +++ b/src/runtime/database/server/plugins/migrations.dev.ts @@ -1,6 +1,6 @@ -import { applyRemoteMigrations } from '../utils/migrations/remote' +import { applyRemoteDatabaseMigrations, applyRemoteDatabaseQueries } from '../utils/migrations/remote' import { hubHooks } from '../../../base/server/utils/hooks' -import { applyMigrations } from '../utils/migrations/migrations' +import { applyDatabaseMigrations, applyDatabaseQueries } from '../utils/migrations/migrations' import { useRuntimeConfig, defineNitroPlugin } from '#imports' export default defineNitroPlugin(async () => { @@ -11,9 +11,11 @@ export default defineNitroPlugin(async () => { hubHooks.hookOnce('bindings:ready', async () => { if (hub.remote && hub.projectKey) { // linked to a NuxtHub project - await applyRemoteMigrations(hub) + await applyRemoteDatabaseMigrations(hub) + await applyRemoteDatabaseQueries(hub) } else { // local dev & self hosted - await applyMigrations(hub) + await applyDatabaseMigrations(hub) + await applyDatabaseQueries(hub) } await hubHooks.callHookParallel('database:migrations:done') diff --git a/src/runtime/database/server/utils/migrations/helpers.ts b/src/runtime/database/server/utils/migrations/helpers.ts index dfb0423a..ea2cf120 100644 --- a/src/runtime/database/server/utils/migrations/helpers.ts +++ b/src/runtime/database/server/utils/migrations/helpers.ts @@ -1,29 +1,83 @@ +import { readFile } from 'node:fs/promises' import { createStorage } from 'unstorage' import fsDriver from 'unstorage/drivers/fs' +import overlayDriver from 'unstorage/drivers/overlay' +import { join } from 'pathe' import type { HubConfig } from '../../../../../features' -export function useMigrationsStorage(hub: HubConfig) { +// #region Migrations +export function useDatabaseMigrationsStorage(hub: HubConfig) { + // .data/hub/database/migrations return createStorage({ driver: fsDriver({ - base: hub.migrationsPath, - ignore: ['.DS_Store'] + base: join(hub.dir!, 'database/migrations') }) }) } -export async function getMigrationFiles(hub: HubConfig) { - const fileKeys = await useMigrationsStorage(hub).getKeys() +export async function getDatabaseMigrationFiles(hub: HubConfig) { + const fileKeys = await useDatabaseMigrationsStorage(hub).getKeys() return fileKeys.filter(file => file.endsWith('.sql')) } -export const CreateMigrationsTableQuery = `CREATE TABLE IF NOT EXISTS _hub_migrations ( +export async function copyDatabaseMigrationsToHubDir(hub: HubConfig) { + const srcStorage = createStorage({ + driver: overlayDriver({ + layers: hub.databaseMigrationsDirs!.map(dir => fsDriver({ + base: dir, + ignore: ['.DS_Store'] + })) + }) + }) + const destStorage = useDatabaseMigrationsStorage(hub) + await destStorage.clear() + const migrationFiles = (await srcStorage.getKeys()).filter(file => file.endsWith('.sql')) + await Promise.all(migrationFiles.map(async (file) => { + const sql = await srcStorage.getItem(file) + await destStorage.setItem(file, sql) + })) +} + +export const CreateDatabaseMigrationsTableQuery = `CREATE TABLE IF NOT EXISTS _hub_migrations ( id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT UNIQUE, applied_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL );` -export const AppliedMigrationsQuery = 'select "id", "name", "applied_at" from "_hub_migrations" order by "_hub_migrations"."id"' +export const AppliedDatabaseMigrationsQuery = 'select "id", "name", "applied_at" from "_hub_migrations" order by "_hub_migrations"."id"' +// #endregion + +// #region Queries +export function useDatabaseQueriesStorage(hub: HubConfig) { + // .data/hub/database/migrations + return createStorage({ + driver: fsDriver({ + base: join(hub.dir!, 'database/queries') + }) + }) +} +export async function getDatabaseQueryFiles(hub: HubConfig) { + const fileKeys = await useDatabaseQueriesStorage(hub).getKeys() + return fileKeys.filter(file => file.endsWith('.sql')) +} + +export async function copyDatabaseQueriesToHubDir(hub: HubConfig) { + const destStorage = useDatabaseQueriesStorage(hub) + await destStorage.clear() + + await Promise.all(hub.databaseQueriesPaths!.map(async (path) => { + try { + const filename = path.split('/').pop()! + const sql = await readFile(path, 'utf-8') + await destStorage.setItem(filename, sql) + } catch (error: any) { + console.error(`Failed to read database query file ${path}: ${error.message}`) + } + })) +} +// #endregion +// #region Utils export function splitSqlQueries(sqlFileContent: string): string[] { const queries = [] // Track whether we're inside a string literal @@ -91,3 +145,4 @@ export function splitSqlQueries(sqlFileContent: string): string[] { return query.replace(/;+$/, ';') }) } +// #endregion diff --git a/src/runtime/database/server/utils/migrations/migrations.ts b/src/runtime/database/server/utils/migrations/migrations.ts index 5a81834e..9e61276e 100644 --- a/src/runtime/database/server/utils/migrations/migrations.ts +++ b/src/runtime/database/server/utils/migrations/migrations.ts @@ -1,17 +1,17 @@ import log from 'consola' import { hubDatabase } from '../database' import type { HubConfig } from '../../../../../features' -import { AppliedMigrationsQuery, CreateMigrationsTableQuery, getMigrationFiles, splitSqlQueries, useMigrationsStorage } from './helpers' +import { AppliedDatabaseMigrationsQuery, CreateDatabaseMigrationsTableQuery, getDatabaseMigrationFiles, getDatabaseQueryFiles, splitSqlQueries, useDatabaseMigrationsStorage, useDatabaseQueriesStorage } from './helpers' // Apply migrations during local development and self-hosted remote development. // See src/utils/migrations/remote.ts for applying migrations on remote development (linked projects) and Pages CI deployments -export async function applyMigrations(hub: HubConfig) { - const migrationsStorage = useMigrationsStorage(hub) +export async function applyDatabaseMigrations(hub: HubConfig) { + const migrationsStorage = useDatabaseMigrationsStorage(hub) const db = hubDatabase() - await db.prepare(CreateMigrationsTableQuery).all() - const appliedMigrations = (await db.prepare(AppliedMigrationsQuery).all()).results - const localMigrations = (await getMigrationFiles(hub)).map(fileName => fileName.replace('.sql', '')) + await db.prepare(CreateDatabaseMigrationsTableQuery).all() + const appliedMigrations = (await db.prepare(AppliedDatabaseMigrationsQuery).all()).results + const localMigrations = (await getDatabaseMigrationFiles(hub)).map(fileName => fileName.replace('.sql', '')) const pendingMigrations = localMigrations.filter(localName => !appliedMigrations.find(({ name }) => name === localName)) if (!pendingMigrations.length) return log.success('Database migrations up to date') @@ -26,13 +26,37 @@ export async function applyMigrations(hub: HubConfig) { try { await db.batch(queries.map(q => db.prepare(q))) } catch (error: any) { - log.error(`Failed to apply migration \`./server/database/migrations/${migration}.sql\`\n`, error?.message) + log.error(`Failed to apply migration \`.data/hub/database/migrations/${migration}.sql\`\n`, error?.message) if (error?.message?.includes('already exists')) { log.info('If your database already contains the migration, run `npx nuxthub database migrations mark-all-applied` to mark all migrations as applied.') } break } - log.success(`Database migration \`./server/database/migrations/${migration}.sql\` applied`) + log.success(`Database migration \`.data/hub/database/migrations/${migration}.sql\` applied`) + } +} + +// Apply migrations during local development and self-hosted remote development. +// See src/utils/migrations/remote.ts for applying migrations on remote development (linked projects) and Pages CI deployments +export async function applyDatabaseQueries(hub: HubConfig) { + const queriesStorage = useDatabaseQueriesStorage(hub) + const db = hubDatabase() + + const queriesPaths = await getDatabaseQueryFiles(hub) + if (!queriesPaths.length) return log.success('No database queries to apply') + + for (const queryPath of queriesPaths) { + const sql = await queriesStorage.getItem(queryPath) + if (!sql) continue + const queries = splitSqlQueries(sql) + try { + await db.batch(queries.map(q => db.prepare(q))) + } catch (error: any) { + log.error(`Failed to apply query \`.data/hub/database/queries/${queryPath}\`\n`, error?.message) + break + } + + log.success(`Database query \`.data/hub/database/queries/${queryPath}\` applied`) } } diff --git a/src/runtime/database/server/utils/migrations/remote.ts b/src/runtime/database/server/utils/migrations/remote.ts index 60214068..8610ff1a 100644 --- a/src/runtime/database/server/utils/migrations/remote.ts +++ b/src/runtime/database/server/utils/migrations/remote.ts @@ -1,18 +1,41 @@ import log from 'consola' import { $fetch } from 'ofetch' import type { HubConfig } from '../../../../../features' -import { AppliedMigrationsQuery, CreateMigrationsTableQuery, getMigrationFiles, useMigrationsStorage } from './helpers' +import { AppliedDatabaseMigrationsQuery, CreateDatabaseMigrationsTableQuery, getDatabaseMigrationFiles, getDatabaseQueryFiles, useDatabaseMigrationsStorage, useDatabaseQueriesStorage } from './helpers' -export async function applyRemoteMigrations(hub: HubConfig) { - const srcStorage = useMigrationsStorage(hub) +export async function queryRemoteDatabase(hub: HubConfig, query: string) { + return await $fetch, success: boolean, meta: object }>>(`/api/projects/${hub.projectKey}/database/${hub.env}/query`, { + baseURL: hub.url, + method: 'POST', + headers: { + authorization: `Bearer ${process.env.NUXT_HUB_PROJECT_DEPLOY_TOKEN || hub.userToken}` + }, + body: { query } + }) +} + +// #region Remote migrations + +export async function fetchRemoteDatabaseMigrations(hub: HubConfig) { + const res = await queryRemoteDatabase<{ id: number, name: string, applied_at: string }>(hub, AppliedDatabaseMigrationsQuery).catch((error) => { + if (error.response?._data?.message.includes('no such table')) { + return [] + } + throw error + }) + return res[0]?.results ?? [] +} + +export async function applyRemoteDatabaseMigrations(hub: HubConfig) { + const migrationsStorage = useDatabaseMigrationsStorage(hub) let appliedMigrations = [] try { - appliedMigrations = await fetchRemoteMigrations(hub) + appliedMigrations = await fetchRemoteDatabaseMigrations(hub) } catch (error: any) { log.error(`Could not fetch applied migrations: ${error.response?._data?.message}`) return false } - const localMigrations = (await getMigrationFiles(hub)).map(fileName => fileName.replace('.sql', '')) + const localMigrations = (await getDatabaseMigrationFiles(hub)).map(fileName => fileName.replace('.sql', '')) const pendingMigrations = localMigrations.filter(localName => !appliedMigrations.find(({ name }) => name === localName)) if (!pendingMigrations.length) { @@ -21,51 +44,56 @@ export async function applyRemoteMigrations(hub: HubConfig) { } for (const migration of pendingMigrations) { - let query = await srcStorage.getItem(`${migration}.sql`) + let query = await migrationsStorage.getItem(`${migration}.sql`) if (!query) continue if (query.replace(/\s$/, '').at(-1) !== ';') query += ';' // ensure previous statement ended before running next query query += ` - ${CreateMigrationsTableQuery} + ${CreateDatabaseMigrationsTableQuery} INSERT INTO _hub_migrations (name) values ('${migration}'); ` try { await queryRemoteDatabase(hub, query) } catch (error: any) { - log.error(`Failed to apply migration \`./server/database/migrations/${migration}.sql\`: ${error.response?._data?.message}`) + log.error(`Failed to apply migration \`${migration}.sql\`: ${error.response?._data?.message}`) if (error.response?._data?.message?.includes('already exists')) { log.info(`To mark all migrations as already applied, run: \`npx nuxthub database migrations mark-all-applied --${hub.env}\``) } return false } - log.success(`Database migration \`./server/database/migrations/${migration}.sql\` applied`) + log.success(`Database migration \`${migration}.sql\` applied`) log.success('Database migrations up to date') return true } } -export async function queryRemoteDatabase(hub: HubConfig, query: string) { - return await $fetch, success: boolean, meta: object }>>(`/api/projects/${hub.projectKey}/database/${hub.env}/query`, { - baseURL: hub.url, - method: 'POST', - headers: { - authorization: `Bearer ${process.env.NUXT_HUB_PROJECT_DEPLOY_TOKEN || hub.userToken}` - }, - body: { query } - }) -} +// #endregion +// #region Remote Queries -export async function createRemoteMigrationsTable(hub: HubConfig) { - await queryRemoteDatabase(hub, CreateMigrationsTableQuery) -} +export async function applyRemoteDatabaseQueries(hub: HubConfig) { + const queriesStorage = useDatabaseQueriesStorage(hub) + const queriesPaths = await getDatabaseQueryFiles(hub) + if (!queriesPaths.length) { + log.success('No database queries to apply') + return true + } -export async function fetchRemoteMigrations(hub: HubConfig) { - const res = await queryRemoteDatabase<{ id: number, name: string, applied_at: string }>(hub, AppliedMigrationsQuery).catch((error) => { - if (error.response?._data?.message.includes('no such table')) { - return [] + for (const queryPath of queriesPaths) { + let query = await queriesStorage.getItem(queryPath) + if (!query) continue + if (query.replace(/\s$/, '').at(-1) !== ';') query += ';' // ensure previous statement ended before running next query + + try { + await queryRemoteDatabase(hub, query) + } catch (error: any) { + log.error(`Failed to apply query \`${queryPath}\`: ${error.response?._data?.message}`) + return false } - throw error - }) - return res[0]?.results ?? [] + + log.success(`Database query \`${queryPath}\` applied`) + return true + } } + +// #endregion diff --git a/src/types/module.ts b/src/types/module.ts index d109d3cb..163cf18c 100644 --- a/src/types/module.ts +++ b/src/types/module.ts @@ -1,3 +1,18 @@ +export interface ModuleHooks { + /** + * Add directories to the database migrations. + * @param dirs - The path of the migrations directories to add. + * @returns void | Promise + */ + 'hub:database:migrations:dirs': (dirs: string[]) => void | Promise + /** + * Add queries to run after the database migrations are applied but are not tracked in the _hub_migrations table. + * @param queries - The path of the SQL queries paths to add. + * @returns void | Promise + */ + 'hub:database:queries:paths': (queries: string[]) => void | Promise +} + export interface ModuleOptions { /** * Set `true` to enable AI for the project. diff --git a/src/utils/build.ts b/src/utils/build.ts index 4f11aac6..171a6e8d 100644 --- a/src/utils/build.ts +++ b/src/utils/build.ts @@ -1,10 +1,10 @@ import { writeFile, cp } from 'node:fs/promises' import { logger } from '@nuxt/kit' -import { join } from 'pathe' +import { join, resolve } from 'pathe' import { $fetch } from 'ofetch' import type { Nuxt } from '@nuxt/schema' import type { HubConfig } from '../features' -import { applyRemoteMigrations } from '../runtime/database/server/utils/migrations/remote' +import { applyRemoteDatabaseMigrations, applyRemoteDatabaseQueries } from '../runtime/database/server/utils/migrations/remote' const log = logger.withTag('nuxt:hub') @@ -98,10 +98,15 @@ export function addBuildHooks(nuxt: Nuxt, hub: HubConfig) { }) // Apply migrations if database is enabled if (hub.database) { - const migrationsApplied = await applyRemoteMigrations(hub) + const migrationsApplied = await applyRemoteDatabaseMigrations(hub) if (!migrationsApplied) { process.exit(1) } + + const queriesApplied = await applyRemoteDatabaseQueries(hub) + if (!queriesApplied) { + process.exit(1) + } } }) }) @@ -124,8 +129,8 @@ export function addBuildHooks(nuxt: Nuxt, hub: HubConfig) { if (hub.database) { try { - await cp(join(nitro.options.rootDir, 'server/database/migrations'), join(nitro.options.output.dir, 'database/migrations'), { recursive: true }) - log.info('Database migrations included in build') + await cp(resolve(nitro.options.rootDir, hub.dir!, 'database'), resolve(nitro.options.output.dir, 'database'), { recursive: true }) + log.info('Database migrations and queries included in build') } catch (error: unknown) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') { log.info('Skipping bundling database migrations - no migrations found')