Use packaged native core and remote demo dataset

README.md

@@ -233,19 +233,35 @@ For more information regarding launching and using Ladybug Explorer, please refe
 npm i
 ```
 
-#### Download and compile Ladybug
+#### Install Ladybug native bindings
 
-```bash
-git submodule update --init --recursive
-npm run build-lbug
-```
+The `@ladybugdb/core` dependency installs the native module for your platform
+through its npm install script and optional platform packages.
 
 #### Generate grammar files
 
+The generated TypeScript parser is checked into `src/utils/CypherParser`, so
+you do not need to regenerate it for normal development. Regenerate it only
+when Ladybug's Cypher grammar changes.
+
+To update the grammar:
+
+1. Check out the Ladybug source tree at the version you want to match. For
+   release updates, use the commit or tag that corresponds to the
+   `@ladybugdb/core` version in `package.json`.
+2. Make the Ladybug checkout available at `./ladybug`. The generator currently
+   reads `ladybug/scripts/antlr4/keywordhandler.py`,
+   `ladybug/src/antlr4/Cypher.g4`, and `ladybug/src/antlr4/keywords.txt`.
+3. Run the generator:
+
 ```bash
 npm run generate-grammar
 ```
 
+4. Review and commit the regenerated files under `src/utils/CypherParser`.
+   The temporary `ladybug` checkout is only an input to generation and should
+   not be committed.
+
 #### Fetch datasets
 
 ```bash

datasets.json

@@ -7,7 +7,20 @@
       "name": "Tutorial Dataset",
       "path": "demo-db/csv",
       "isProduction": true,
-      "description": "Data used in Create Your First Graph (https://docs.ladybugdb.com/get-started/) tutorial."
+      "description": "Data used in Create Your First Graph (https://docs.ladybugdb.com/get-started/) tutorial.",
+      "source": {
+        "type": "url",
+        "baseUrl": "https://huggingface.co/datasets/ladybugdb/demo-db/resolve/main",
+        "files": [
+          "city.csv",
+          "city.csv.gzip",
+          "copy.cypher",
+          "follows.csv",
+          "lives-in.csv",
+          "schema.cypher",
+          "user.csv"
+        ]
+      }
     },
     {
       "name": "TinySNB",

package.json

@@ -11,11 +11,10 @@
     "serve-prod": "NODE_ENV=production LBUG_WASM=false node src/server/index.js",
     "build": "LBUG_IN_MEMORY=true LBUG_WASM=false NODE_ENV=production vue-cli-service build",
     "lint": "vue-cli-service lint",
-    "build-lbug": "cd ladybug/tools/nodejs_api/ && npm i && cd ../.. && make nodejs NUM_THREADS=$(node -e \"console.log(require('os').cpus().length)\")",
     "fetch-datasets": "node src/server/utils/FetchDatasets.js",
     "generate-grammar-prod": "python3 ladybug/scripts/antlr4/keywordhandler.py ladybug/src/antlr4/Cypher.g4 ladybug/src/antlr4/keywords.txt ./Cypher.g4 /tmp/header.h && rm /tmp/header.h && rm -rf src/utils/CypherParser/ && npx antlr4ng-cli -o src/utils/CypherParser -Dlanguage=TypeScript -no-visitor -no-listener Cypher.g4 && rm Cypher.g4 && cd src/utils/CypherParser && mv CypherParser.ts CypherParserOld.ts && grep -v \"notify\" CypherParserOld.ts > CypherParser.ts && rm CypherParserOld.ts",
     "generate-grammar": "python3 ladybug/scripts/antlr4/keywordhandler.py ladybug/src/antlr4/Cypher.g4 ladybug/src/antlr4/keywords.txt ./Cypher.g4 /tmp/header.h && rm /tmp/header.h && rm -rf src/utils/CypherParser/ && npx antlr4ng-cli -o src/utils/CypherParser -Dlanguage=TypeScript -no-visitor -no-listener Cypher.g4 && rm Cypher.g4 && cd src/utils/CypherParser && mv CypherParser.ts CypherParserOld.ts && grep -v \"notify\" CypherParserOld.ts > CypherParser.ts && rm CypherParserOld.ts",
-    "clean": "rm -rf node_modules dist datasets src/utils/CypherParser && git submodule deinit --all",
+    "clean": "rm -rf node_modules dist datasets src/utils/CypherParser",
     "eslint": "eslint --ext .js,.vue src",
     "eslint-fix": "eslint --ext .js,.vue src --fix"
   },
@@ -37,10 +36,10 @@
   "devDependencies": {
     "@babel/core": "^7.12.16",
     "@babel/plugin-transform-private-methods": "^7.27.1",
-    "@vue/cli-plugin-babel": "~5.0.0",
-    "@vue/cli-plugin-eslint": "^5.0.8",
-    "@vue/cli-plugin-typescript": "~5.0.0",
-    "@vue/cli-service": "~5.0.0",
+    "@vue/cli-plugin-babel": "5.0.9",
+    "@vue/cli-plugin-eslint": "5.0.9",
+    "@vue/cli-plugin-typescript": "5.0.9",
+    "@vue/cli-service": "5.0.9",
     "antlr4ng-cli": "^1.0.0",
     "copy-webpack-plugin": "^12.0.2",
     "eslint": "^8.55.0",

src/server/Datasets.js

@@ -46,6 +46,16 @@ const getDatasetPath = (dataset) => {
   return path.resolve(path.join(base, datasetPath));
 };
 
+const resolveCopyPath = (line, datasetPath) => {
+  return line.replace(
+    /(\bFROM\s+)(["'])(?!\/|[A-Za-z]:|https?:\/\/)([^"']+)\2/i,
+    (_, prefix, quote, filePath) => {
+      const resolvedPath = path.join(datasetPath, filePath).replaceAll("\\", "/");
+      return `${prefix}${quote}${resolvedPath}${quote}`;
+    }
+  );
+};
+
 router.get("/", async (_, res) => {
   await getDatasetsToShow();
   return res.send(Object.keys(DATASETS_TO_SHOW));
@@ -139,7 +149,7 @@ router.get("/:dataset/copy", async (req, res) => {
       .map((line) => line.trim())
       .filter((line) => line.length > 0)
       .map((line) => {
-        return line.replace("dataset/", base + "/")
+        return resolveCopyPath(line.replace("dataset/", base + "/"), datasetPath);
       });
     commands = ddls.concat(copyCommands);
   } catch (err) {

src/server/utils/Database.js

@@ -10,22 +10,7 @@ const CONSTANTS = require("./Constants");
 const MODES = CONSTANTS.MODES;
 const READ_WRITE_MODE = MODES.READ_WRITE;
 
-let lbug;
-if (process.env.NODE_ENV !== "production") {
-  const lbugPath = path.join(
-    __dirname,
-    "..",
-    "..",
-    "..",
-    "ladybug",
-    "tools",
-    "nodejs_api",
-    "build/"
-  );
-  lbug = require(lbugPath);
-} else {
-  lbug = require("@ladybugdb/core");
-}
+const lbug = require("@ladybugdb/core");
 const os = require("os");
 
 class Database {

src/server/utils/FetchDatasets.js

@@ -1,6 +1,7 @@
 const { spawn } = require("child_process");
 const path = require("path");
 const fs = require("fs/promises");
+const https = require("https");
 
 const tempPath = path.join(__dirname, "..", "..", "..", "temp");
 const datasetsPath = path.join(__dirname, "..", "..", "..", "datasets");
@@ -16,22 +17,9 @@ const deleteFolderRecursive = async (path) => {
   }
 };
 
-(async () => {
-  console.log("Reading config file from", configPath);
-  const config = JSON.parse(await fs.readFile(configPath, "utf-8"));
-  console.log("Deleting temp and datasets folders");
-  await deleteFolderRecursive(tempPath);
-  await deleteFolderRecursive(datasetsPath);
-  console.log("Cloning repository");
-  await new Promise((resolve, reject) => {
-    const gitProcess = spawn("git", [
-      "clone",
-      "-n",
-      "--depth=1",
-      "--filter=tree:0",
-      config.repository,
-      tempPath,
-    ]);
+const runGit = (args, options = {}) => {
+  return new Promise((resolve, reject) => {
+    const gitProcess = spawn("git", args, options);
     gitProcess.stdout.on("data", (data) => {
       console.log(data.toString());
     });
@@ -42,57 +30,107 @@ const deleteFolderRecursive = async (path) => {
       if (code === 0) {
         resolve();
       } else {
-        reject("Error cloning repository");
+        reject(new Error(`Git command failed: git ${args.join(" ")}`));
       }
     });
   });
+};
+
+const downloadFile = async (url, destination) => {
+  await fs.mkdir(path.dirname(destination), { recursive: true });
+  await new Promise((resolve, reject) => {
+    https.get(url, (response) => {
+      if ([301, 302, 303, 307, 308].includes(response.statusCode)) {
+        const nextUrl = new URL(response.headers.location, url).toString();
+        response.resume();
+        downloadFile(nextUrl, destination).then(resolve, reject);
+        return;
+      }
+      if (response.statusCode !== 200) {
+        response.resume();
+        reject(new Error(`Failed to download ${url}: HTTP ${response.statusCode}`));
+        return;
+      }
+
+      const chunks = [];
+      response.on("data", (chunk) => chunks.push(chunk));
+      response.on("end", async () => {
+        try {
+          await fs.writeFile(destination, Buffer.concat(chunks));
+          resolve();
+        } catch (err) {
+          reject(err);
+        }
+      });
+    }).on("error", reject);
+  });
+};
+
+const fetchUrlDataset = async (dataset) => {
+  const datasetPath = path.join(datasetsPath, dataset.path);
+  const baseUrl = dataset.source.baseUrl.replace(/\/$/, "");
+  console.log(`Downloading ${dataset.name} from ${baseUrl}`);
+  for (const file of dataset.source.files) {
+    const url = `${baseUrl}/${dataset.path}/${file}`;
+    const destination = path.join(datasetPath, file);
+    console.log(`\t - ${url}`);
+    await downloadFile(url, destination);
+  }
+};
+
+const fetchGitDatasets = async (config, datasets) => {
+  if (datasets.length === 0) {
+    return;
+  }
+
+  console.log("Cloning repository");
+  await runGit([
+    "clone",
+    "-n",
+    "--depth=1",
+    "--filter=tree:0",
+    config.repository,
+    tempPath,
+  ]);
+
   console.log("Gathering datasets paths");
-  const datasetPaths = config.datasets.map((d) => {
+  const datasetPaths = datasets.map((d) => {
     console.log(`\t - ${d.name} at ${d.path}`);
     return path.join(config.datasetsRoot, d.path);
   });
+
   console.log("Initializing sparse checkout");
-  await new Promise((resolve, reject) => {
-    const gitProcess = spawn(
-      "git",
-      ["sparse-checkout", "set", "--no-cone", ...datasetPaths],
-      { cwd: tempPath }
-    );
-    gitProcess.stdout.on("data", (data) => {
-      console.log(data.toString());
-    });
-    gitProcess.stderr.on("data", (data) => {
-      console.error(data.toString());
-    });
-    gitProcess.on("close", (code) => {
-      if (code === 0) {
-        resolve();
-      } else {
-        reject("Error initializing sparse checkout");
-      }
-    });
+  await runGit(["sparse-checkout", "set", "--no-cone", ...datasetPaths], {
+    cwd: tempPath,
   });
+
   console.log("Pulling repository");
-  await new Promise((resolve, reject) => {
-    const gitProcess = spawn("git", ["checkout", config.commitHash], {
-      cwd: tempPath,
-    });
-    gitProcess.stdout.on("data", (data) => {
-      console.log(data.toString());
-    });
-    gitProcess.stderr.on("data", (data) => {
-      console.error(data.toString());
-    });
-    gitProcess.on("close", (code) => {
-      if (code === 0) {
-        resolve();
-      } else {
-        reject("Error pulling repository");
-      }
-    });
-  });
+  await runGit(["checkout", config.commitHash], { cwd: tempPath });
+
   console.log("Moving datasets");
-  await fs.rename(path.join(tempPath, config.datasetsRoot), datasetsPath);
+  for (const dataset of datasets) {
+    const source = path.join(tempPath, config.datasetsRoot, dataset.path);
+    const destination = path.join(datasetsPath, dataset.path);
+    await fs.mkdir(path.dirname(destination), { recursive: true });
+    await fs.rename(source, destination);
+  }
+};
+
+(async () => {
+  console.log("Reading config file from", configPath);
+  const config = JSON.parse(await fs.readFile(configPath, "utf-8"));
+  console.log("Deleting temp and datasets folders");
+  await deleteFolderRecursive(tempPath);
+  await deleteFolderRecursive(datasetsPath);
+
+  const urlDatasets = config.datasets.filter((d) => d.source?.type === "url");
+  const gitDatasets = config.datasets.filter((d) => !d.source);
+
+  for (const dataset of urlDatasets) {
+    await fetchUrlDataset(dataset);
+  }
+  await fetchGitDatasets(config, gitDatasets);
+
   console.log("Deleting temp folder");
   await deleteFolderRecursive(tempPath);
   console.log("Done");