Add SQLite port (sqlite3) (#17297)

The SQLite port adds `-sUSE_SQLITE3`. Internally it uses
sqlite amalgamation (single file distribution). The port creates
`libsqlite3.a` (non-threaded) or `libsqlite3-mt.a` (multi-threaded).

Author: tiran

embuilder.py

@@ -61,6 +61,8 @@
     'crt1_proxy_main',
     'libunwind-except',
     'libnoexit',
+    'sqlite3',
+    'sqlite3-mt',
 ]
 
 # Additional tasks on top of MINIMAL_TASKS that are necessary for PIC testing on
@@ -103,6 +105,7 @@
     'sdl2_image_png': ('sdl2_image', {'SDL2_IMAGE_FORMATS': ["png"]}),
     'sdl2_image_jpg': ('sdl2_image', {'SDL2_IMAGE_FORMATS': ["jpg"]}),
     'libpng-mt': ('libpng', {'USE_PTHREADS': 1}),
+    'sqlite3-mt': ('sqlite3', {'USE_PTHREADS': 1}),
 }
 
 PORTS = sorted(list(ports.ports_by_name.keys()) + list(PORT_VARIANTS.keys()))

src/settings.js

@@ -1498,6 +1498,10 @@ var SDL2_IMAGE_FORMATS = [];
 // [link]
 var SDL2_MIXER_FORMATS = ["ogg"];
 
+// 1 = use sqlite3 from emscripten-ports
+// [link]
+var USE_SQLITE3 = false;
+
 // If true, the current build is performed for the Emscripten test harness.
 // [other]
 var IN_TEST_HARNESS = false;

tests/test_core.py

@@ -6665,38 +6665,20 @@ def test_freetype(self):
   @no_asan('local count too large for VMs')
   @no_ubsan('local count too large for VMs')
   @is_slow_test
-  def test_sqlite(self):
-    self.set_setting('EXPORTED_FUNCTIONS', ['_main', '_sqlite3_open', '_sqlite3_close', '_sqlite3_exec', '_sqlite3_free'])
-    if '-g' in self.emcc_args:
-      print("disabling inlining") # without registerize (which -g disables), we generate huge amounts of code
-      self.set_setting('INLINING_LIMIT')
-
-    # newer clang has a warning for implicit conversions that lose information,
-    # which happens in sqlite (see #9138)
-    self.emcc_args += ['-Wno-implicit-int-float-conversion']
-    # newer clang warns about "suspicious concatenation of string literals in an
-    # array initialization; did you mean to separate the elements with a comma?"
-    self.emcc_args += ['-Wno-string-concatenation']
-    # ignore unknown flags, which lets the above flags be used on github CI
-    # before the LLVM change rolls in (the same LLVM change that adds the
-    # warning also starts to warn on it)
-    self.emcc_args += ['-Wno-unknown-warning-option']
-    self.emcc_args += ['-Wno-pointer-bool-conversion']
-
-    self.emcc_args += ['-I' + test_file('third_party/sqlite')]
-
-    src = '''
-       #define SQLITE_DISABLE_LFS
-       #define LONGDOUBLE_TYPE double
-       #define SQLITE_INT64_TYPE long long int
-       #define SQLITE_THREADSAFE 0
-    '''
-    src += read_file(test_file('third_party/sqlite/sqlite3.c'))
-    src += read_file(test_file('sqlite/benchmark.c'))
-    self.do_run(src,
-                read_file(test_file('sqlite/benchmark.txt')),
-                includes=[test_file('sqlite')],
-                force_c=True)
+  @parameterized({
+    '': (False,),
+    'pthreads': (True,),
+  })
+  def test_sqlite(self, use_pthreads):
+    if use_pthreads:
+      self.set_setting('USE_PTHREADS')
+      self.setup_node_pthreads()
+    self.emcc_args += ['-sUSE_SQLITE3']
+    self.do_run_from_file(
+      test_file('sqlite/benchmark.c'),
+      test_file('sqlite/benchmark.txt'),
+      force_c=True
+    )
 
   @needs_make('mingw32-make')
   @is_slow_test

tools/ports/sqlite3.py

@@ -0,0 +1,93 @@
+# Copyright 2022 The Emscripten Authors.  All rights reserved.
+# Emscripten is available under two separate licenses, the MIT license and the
+# University of Illinois/NCSA Open Source License.  Both these licenses can be
+# found in the LICENSE file.
+
+import os
+import shutil
+import logging
+
+# sqlite amalgamation download URL uses relase year and tag
+# 2022  and (3, 38, 5) -> '/2022/sqlite-amalgamation-3380500.zip'
+VERSION = (3, 39, 0)
+VERSION_YEAR = 2022
+HASH = 'cbaf4adb3e404d9aa403b34f133c5beca5f641ae1e23f84dbb021da1fb9efdc7c56b5922eb533ae5cb6d26410ac60cb3f026085591bc83ebc1c225aed0cf37ca'
+
+deps = []
+
+
+def needed(settings):
+  return settings.USE_SQLITE3
+
+
+def get_lib_name(settings):
+  return 'libsqlite3' + ('-mt' if settings.USE_PTHREADS else '') + '.a'
+
+
+def get(ports, settings, shared):
+  release = f'sqlite-amalgamation-{VERSION[0]}{VERSION[1]:02}{VERSION[2]:02}00'
+  # TODO: Fetch the file from an emscripten-hosted mirror.
+  ports.fetch_project('sqlite3', f'https://www.sqlite.org/{VERSION_YEAR}/{release}.zip', release, sha512hash=HASH)
+
+  def create(final):
+    logging.info('building port: libsqlite3')
+
+    source_path = os.path.join(ports.get_dir(), 'sqlite3', release)
+    dest_path = os.path.join(ports.get_build_dir(), 'sqlite3')
+
+    shutil.rmtree(dest_path, ignore_errors=True)
+    shutil.copytree(source_path, dest_path)
+
+    ports.install_headers(dest_path)
+
+    # flags are based on sqlite-autoconf output.
+    # SQLITE_HAVE_ZLIB is only used by shell.c
+    flags = [
+      '-DSTDC_HEADERS=1',
+      '-DHAVE_SYS_TYPES_H=1',
+      '-DHAVE_SYS_STAT_H=1',
+      '-DHAVE_STDLIB_H=1',
+      '-DHAVE_STRING_H=1',
+      '-DHAVE_MEMORY_H=1',
+      '-DHAVE_STRINGS_H=1',
+      '-DHAVE_INTTYPES_H=1',
+      '-DHAVE_STDINT_H=1',
+      '-DHAVE_UNISTD_H=1',
+      '-DHAVE_FDATASYNC=1',
+      '-DHAVE_USLEEP=1',
+      '-DHAVE_LOCALTIME_R=1',
+      '-DHAVE_GMTIME_R=1',
+      '-DHAVE_DECL_STRERROR_R=1',
+      '-DHAVE_STRERROR_R=1',
+      '-DHAVE_POSIX_FALLOCATE=1',
+      '-DSQLITE_OMIT_LOAD_EXTENSION=1',
+      '-DSQLITE_ENABLE_MATH_FUNCTIONS=1',
+      '-DSQLITE_ENABLE_FTS4=1',
+      '-DSQLITE_ENABLE_FTS5=1',
+      '-DSQLITE_ENABLE_RTREE=1',
+      '-DSQLITE_ENABLE_GEOPOLY=1',
+      '-DSQLITE_OMIT_POPEN=1',
+    ]
+    if settings.USE_PTHREADS:
+      flags += [
+        '-sUSE_PTHREADS',
+        '-DSQLITE_THREADSAFE=1',
+      ]
+    else:
+      flags += ['-DSQLITE_THREADSAFE=0']
+
+    ports.build_port(dest_path, final, flags=flags, exclude_files=['shell.c'])
+
+  return [shared.Cache.get_lib(get_lib_name(settings), create, what='port')]
+
+
+def clear(ports, settings, shared):
+  shared.Cache.erase_lib(get_lib_name(settings))
+
+
+def process_args(ports):
+  return []
+
+
+def show():
+  return 'sqlite (USE_SQLITE3=1); public domain)'

tools/settings.py

@@ -47,6 +47,7 @@
     'USE_FREETYPE',
     'SDL2_MIXER_FORMATS',
     'SDL2_IMAGE_FORMATS',
+    'USE_SQLITE3',
 }
 
 # Subset of settings that apply at compile time.