From 2aee11daf4a7fb07b6d53e59e002fedd28907192 Mon Sep 17 00:00:00 2001 From: Edy Silva Date: Sat, 11 Jan 2025 22:16:02 -0300 Subject: [PATCH] sqlite, test: expose sqlite online backup api --- src/node_sqlite.cc | 296 +++++++++++++++++++++++++++ src/node_sqlite.h | 2 + test/parallel/test-sqlite-backup.mjs | 83 ++++++++ 3 files changed, 381 insertions(+) create mode 100644 test/parallel/test-sqlite-backup.mjs diff --git a/src/node_sqlite.cc b/src/node_sqlite.cc index 0b8f7ef2a21763..0002e1c2e9aaa8 100644 --- a/src/node_sqlite.cc +++ b/src/node_sqlite.cc @@ -8,6 +8,7 @@ #include "node_errors.h" #include "node_mem-inl.h" #include "sqlite3.h" +#include "threadpoolwork-inl.h" #include "util-inl.h" #include @@ -29,6 +30,7 @@ using v8::FunctionCallback; using v8::FunctionCallbackInfo; using v8::FunctionTemplate; using v8::Global; +using v8::HandleScope; using v8::Int32; using v8::Integer; using v8::Isolate; @@ -40,6 +42,7 @@ using v8::NewStringType; using v8::Null; using v8::Number; using v8::Object; +using v8::Promise; using v8::SideEffectType; using v8::String; using v8::TryCatch; @@ -81,6 +84,24 @@ inline MaybeLocal CreateSQLiteError(Isolate* isolate, return e; } +inline MaybeLocal CreateSQLiteError(Isolate* isolate, int errcode) { + const char* errstr = sqlite3_errstr(errcode); + Local js_errmsg; + Local e; + Environment* env = Environment::GetCurrent(isolate); + if (!String::NewFromUtf8(isolate, errstr).ToLocal(&js_errmsg) || + !CreateSQLiteError(isolate, errstr).ToLocal(&e) || + e->Set(isolate->GetCurrentContext(), + env->errcode_string(), + Integer::New(isolate, errcode)) + .IsNothing() || + e->Set(isolate->GetCurrentContext(), env->errstr_string(), js_errmsg) + .IsNothing()) { + return MaybeLocal(); + } + return e; +} + inline MaybeLocal CreateSQLiteError(Isolate* isolate, sqlite3* db) { int errcode = sqlite3_extended_errcode(db); const char* errstr = sqlite3_errstr(errcode); @@ -128,6 +149,171 @@ inline void THROW_ERR_SQLITE_ERROR(Isolate* isolate, int errcode) { isolate->ThrowException(error); } +class BackupJob : public ThreadPoolWork { + public: + explicit BackupJob(Environment* env, + DatabaseSync* source, + Local resolver, + std::string source_db, + std::string destination_name, + std::string dest_db, + int pages, + Local progressFunc) + : ThreadPoolWork(env, "node_sqlite3.BackupJob"), + env_(env), + source_(source), + source_db_(source_db), + destination_name_(destination_name), + dest_db_(dest_db), + pages_(pages) { + resolver_.Reset(env->isolate(), resolver); + progressFunc_.Reset(env->isolate(), progressFunc); + } + + void ScheduleBackup() { + Isolate* isolate = env()->isolate(); + HandleScope handle_scope(isolate); + + backup_status_ = sqlite3_open(destination_name_.c_str(), &pDest_); + + Local resolver = + Local::New(env()->isolate(), resolver_); + + Local e = Local(); + + if (backup_status_ != SQLITE_OK) { + CreateSQLiteError(isolate, pDest_).ToLocal(&e); + + Cleanup(); + + resolver->Reject(env()->context(), e).ToChecked(); + + return; + } + + pBackup_ = sqlite3_backup_init( + pDest_, dest_db_.c_str(), source_->Connection(), source_db_.c_str()); + + if (pBackup_ == nullptr) { + CreateSQLiteError(isolate, pDest_).ToLocal(&e); + + sqlite3_close(pDest_); + + resolver->Reject(env()->context(), e).ToChecked(); + + return; + } + + this->ScheduleWork(); + } + + void DoThreadPoolWork() override { + backup_status_ = sqlite3_backup_step(pBackup_, pages_); + + const char* errstr = sqlite3_errstr(backup_status_); + } + + void AfterThreadPoolWork(int status) override { + HandleScope handle_scope(env()->isolate()); + + if (resolver_.IsEmpty()) { + Cleanup(); + + return; + } + + Local resolver = + Local::New(env()->isolate(), resolver_); + + if (!(backup_status_ == SQLITE_OK || backup_status_ == SQLITE_DONE || + backup_status_ == SQLITE_BUSY || backup_status_ == SQLITE_LOCKED)) { + Local e = Local(); + + CreateSQLiteError(env()->isolate(), backup_status_).ToLocal(&e); + + Cleanup(); + + resolver->Reject(env()->context(), e).ToChecked(); + + return; + } + + int total_pages = sqlite3_backup_pagecount(pBackup_); + int remaining_pages = sqlite3_backup_remaining(pBackup_); + + if (remaining_pages != 0) { + Local fn = + Local::New(env()->isolate(), progressFunc_); + + if (!fn.IsEmpty()) { + Local argv[] = { + Integer::New(env()->isolate(), total_pages), + Integer::New(env()->isolate(), remaining_pages), + }; + + TryCatch try_catch(env()->isolate()); + fn->Call(env()->context(), Null(env()->isolate()), 2, argv) + .FromMaybe(Local()); + + if (try_catch.HasCaught()) { + Cleanup(); + + resolver->Reject(env()->context(), try_catch.Exception()).ToChecked(); + + return; + } + } + + // There's still work to do + this->ScheduleWork(); + + return; + } + + Local message = + String::NewFromUtf8( + env()->isolate(), "Backup completed", NewStringType::kNormal) + .ToLocalChecked(); + + Local e = Local(); + CreateSQLiteError(env()->isolate(), pDest_).ToLocal(&e); + + Cleanup(); + + if (backup_status_ == SQLITE_OK) { + resolver->Resolve(env()->context(), message).ToChecked(); + } else { + resolver->Reject(env()->context(), e).ToChecked(); + } + } + + private: + void Cleanup() { + if (pBackup_) { + sqlite3_backup_finish(pBackup_); + } + + if (pDest_) { + backup_status_ = sqlite3_errcode(pDest_); + sqlite3_close(pDest_); + } + } + + // https://github.com/nodejs/node/blob/649da3b8377e030ea7b9a1bc0308451e26e28740/src/crypto/crypto_keygen.h#L126 + int backup_status_; + Environment* env() const { return env_; } + sqlite3* pDest_; + sqlite3_backup* pBackup_; + Environment* env_; + DatabaseSync* source_; + Global resolver_; + Global progressFunc_; + std::string source_db_; + std::string destination_name_; + std::string dest_db_; + int pages_; +}; + class UserDefinedFunction { public: explicit UserDefinedFunction(Environment* env, @@ -533,6 +719,115 @@ void DatabaseSync::Exec(const FunctionCallbackInfo& args) { CHECK_ERROR_OR_THROW(env->isolate(), db->connection_, r, SQLITE_OK, void()); } +// database.backup(destination, { sourceDb, targetDb, rate, progress: (total, +// remaining) => {} ) +void DatabaseSync::Backup(const FunctionCallbackInfo& args) { + Environment* env = Environment::GetCurrent(args); + + if (!args[0]->IsString()) { + THROW_ERR_INVALID_ARG_TYPE( + env->isolate(), "The \"destination\" argument must be a string."); + return; + } + + int rate = 100; + std::string source_db = "main"; + std::string dest_db = "main"; + + DatabaseSync* db; + ASSIGN_OR_RETURN_UNWRAP(&db, args.This()); + + THROW_AND_RETURN_ON_BAD_STATE(env, !db->IsOpen(), "database is not open"); + + Utf8Value destFilename(env->isolate(), args[0].As()); + Local progressFunc = Local(); + + if (args.Length() > 1) { + if (!args[1]->IsObject()) { + THROW_ERR_INVALID_ARG_TYPE(env->isolate(), + "The \"options\" argument must be an object."); + return; + } + + Local options = args[1].As(); + Local progress_string = + FIXED_ONE_BYTE_STRING(env->isolate(), "progress"); + Local rate_string = FIXED_ONE_BYTE_STRING(env->isolate(), "rate"); + Local target_db_string = + FIXED_ONE_BYTE_STRING(env->isolate(), "targetDb"); + Local source_db_string = + FIXED_ONE_BYTE_STRING(env->isolate(), "sourceDb"); + + Local rateValue = + options->Get(env->context(), rate_string).ToLocalChecked(); + + if (!rateValue->IsUndefined()) { + if (!rateValue->IsInt32()) { + THROW_ERR_INVALID_ARG_TYPE( + env->isolate(), + "The \"options.rate\" argument must be an integer."); + return; + } + + rate = rateValue.As()->Value(); + } + + Local sourceDbValue = + options->Get(env->context(), source_db_string).ToLocalChecked(); + + if (!sourceDbValue->IsUndefined()) { + if (!sourceDbValue->IsString()) { + THROW_ERR_INVALID_ARG_TYPE( + env->isolate(), + "The \"options.sourceDb\" argument must be a string."); + return; + } + + source_db = + Utf8Value(env->isolate(), sourceDbValue.As()).ToString(); + } + + Local targetDbValue = + options->Get(env->context(), target_db_string).ToLocalChecked(); + + if (!targetDbValue->IsUndefined()) { + if (!targetDbValue->IsString()) { + THROW_ERR_INVALID_ARG_TYPE( + env->isolate(), + "The \"options.targetDb\" argument must be a string."); + return; + } + + dest_db = + Utf8Value(env->isolate(), targetDbValue.As()).ToString(); + } + + Local progressValue = + options->Get(env->context(), progress_string).ToLocalChecked(); + + if (!progressValue->IsUndefined()) { + if (!progressValue->IsFunction()) { + THROW_ERR_INVALID_ARG_TYPE( + env->isolate(), + "The \"options.progress\" argument must be a function."); + return; + } + + progressFunc = progressValue.As(); + } + } + + Local resolver = Promise::Resolver::New(env->context()) + .ToLocalChecked() + .As(); + + args.GetReturnValue().Set(resolver->GetPromise()); + + BackupJob* job = new BackupJob( + env, db, resolver, source_db, *destFilename, dest_db, rate, progressFunc); + job->ScheduleBackup(); +} + void DatabaseSync::CustomFunction(const FunctionCallbackInfo& args) { DatabaseSync* db; ASSIGN_OR_RETURN_UNWRAP(&db, args.This()); @@ -1718,6 +2013,7 @@ static void Initialize(Local target, SetProtoMethod(isolate, db_tmpl, "close", DatabaseSync::Close); SetProtoMethod(isolate, db_tmpl, "prepare", DatabaseSync::Prepare); SetProtoMethod(isolate, db_tmpl, "exec", DatabaseSync::Exec); + SetProtoMethod(isolate, db_tmpl, "backup", DatabaseSync::Backup); SetProtoMethod(isolate, db_tmpl, "function", DatabaseSync::CustomFunction); SetProtoMethod( isolate, db_tmpl, "createSession", DatabaseSync::CreateSession); diff --git a/src/node_sqlite.h b/src/node_sqlite.h index e78aa39abb3ba5..4b70be6b14ec95 100644 --- a/src/node_sqlite.h +++ b/src/node_sqlite.h @@ -57,6 +57,7 @@ class DatabaseSync : public BaseObject { static void Close(const v8::FunctionCallbackInfo& args); static void Prepare(const v8::FunctionCallbackInfo& args); static void Exec(const v8::FunctionCallbackInfo& args); + static void Backup(const v8::FunctionCallbackInfo& args); static void CustomFunction(const v8::FunctionCallbackInfo& args); static void CreateSession(const v8::FunctionCallbackInfo& args); static void ApplyChangeset(const v8::FunctionCallbackInfo& args); @@ -81,6 +82,7 @@ class DatabaseSync : public BaseObject { bool enable_load_extension_; sqlite3* connection_; + std::set backups_; std::set sessions_; std::unordered_set statements_; diff --git a/test/parallel/test-sqlite-backup.mjs b/test/parallel/test-sqlite-backup.mjs new file mode 100644 index 00000000000000..4043345f9718fe --- /dev/null +++ b/test/parallel/test-sqlite-backup.mjs @@ -0,0 +1,83 @@ +import common from '../common/index.js'; +import tmpdir from '../common/tmpdir.js'; +import { join } from 'path'; +import { DatabaseSync } from 'node:sqlite'; +import { test } from 'node:test'; +import { writeFileSync } from 'fs'; + +let cnt = 0; + +tmpdir.refresh(); + +function nextDb() { + return join(tmpdir.path, `database-${cnt++}.db`); +} + +function makeSourceDb() { + const database = new DatabaseSync(':memory:'); + + database.exec(` + CREATE TABLE data( + key INTEGER PRIMARY KEY, + value TEXT + ) STRICT + `); + + const insert = database.prepare('INSERT INTO data (key, value) VALUES (?, ?)'); + + for (let i = 1; i <= 2; i++) { + insert.run(i, `value-${i}`); + } + + return database; +} + +test('throws exception when trying to start backup from a closed database', async (t) => { + t.assert.rejects(async () => { + const database = new DatabaseSync(':memory:'); + + database.close(); + + await database.backup('backup.db'); + }, common.expectsError({ + code: 'ERR_INVALID_STATE', + message: 'database is not open' + })); +}); + +test('database backup', async (t) => { + const progressFn = t.mock.fn(); + const database = makeSourceDb(); + const destDb = nextDb(); + + await database.backup(destDb, { + rate: 1, + progress: progressFn, + }); + + const backup = new DatabaseSync(destDb); + const rows = backup.prepare('SELECT * FROM data').all(); + + // The source database has two pages - using the default page size -, so the progress function should be called once (the last call is not made since + // the promise resolves) + t.assert.strictEqual(progressFn.mock.calls.length, 1); + t.assert.deepStrictEqual(rows, [ + { __proto__: null, key: 1, value: 'value-1' }, + { __proto__: null, key: 2, value: 'value-2' }, + ]); +}); + +test('database backup fails when dest file is not writable', (t) => { + const readonlyDestDb = nextDb(); + writeFileSync(readonlyDestDb, '', { mode: 0o444 }); + + const database = makeSourceDb(); + + t.assert.rejects(async () => { + await database.backup(readonlyDestDb); + }, common.expectsError({ + code: 'ERR_SQLITE_ERROR', + message: 'attempt to write a readonly database' + })); +}); +