Skip to content

fix: add dependencies for pg vector store #312

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 10 commits into from
Sep 24, 2024
Merged

fix: add dependencies for pg vector store #312

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
f1d185a
fix: add dependencies for pg vector store
marcusschiesser Sep 23, 2024
809f88a
add coderabbit config
marcusschiesser Sep 24, 2024
418b180
add deps check
marcusschiesser Sep 24, 2024
5146713
fix qdrant
marcusschiesser Sep 24, 2024
8690ba2
fixed all vectordbs
marcusschiesser Sep 24, 2024
cc62e6c
fix astra and pg
marcusschiesser Sep 24, 2024
6351281
update changeset
marcusschiesser Sep 24, 2024
87e4cf2
fix express
marcusschiesser Sep 24, 2024
6e248a2
revert pgvectorstore
marcusschiesser Sep 24, 2024
fe645fa
fix type error for no-files
marcusschiesser Sep 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/strong-wasps-nail.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"create-llama": patch
---

Add dependencies for PGVectorStore
6 changes: 6 additions & 0 deletions .coderabbit.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# coderabbit.yml
reviews:
path_instructions:
- path: "templates/**"
instructions: |
For files under the `templates` folder, do not report 'Missing Dependencies Detected' errors.
97 changes: 97 additions & 0 deletions e2e/resolve_ts_dependencies.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
import { expect, test } from "@playwright/test";
import { exec } from "child_process";
import fs from "fs";
import path from "path";
import util from "util";
import { TemplateFramework, TemplateVectorDB } from "../helpers/types";
import { createTestDir, runCreateLlama } from "./utils";

const execAsync = util.promisify(exec);

const templateFramework: TemplateFramework = process.env.FRAMEWORK
? (process.env.FRAMEWORK as TemplateFramework)
: "nextjs";
const dataSource: string = process.env.DATASOURCE
? process.env.DATASOURCE
: "--example-file";

if (
templateFramework == "nextjs" ||
templateFramework == "express" // test is only relevant for TS projects
) {
// vectorDBs combinations to test
const vectorDbs: TemplateVectorDB[] = [
"mongo",
"pg",
"qdrant",
"pinecone",
"milvus",
"astra",
"chroma",
"llamacloud",
"weaviate",
];

test.describe("Test resolve TS dependencies", () => {
for (const vectorDb of vectorDbs) {
const optionDescription = `vectorDb: ${vectorDb}, dataSource: ${dataSource}`;

test(`options: ${optionDescription}`, async () => {
const cwd = await createTestDir();

const result = await runCreateLlama(
cwd,
"streaming",
templateFramework,
dataSource,
vectorDb,
3000, // port
8000, // externalPort
"none", // postInstallAction
undefined, // ui
templateFramework === "nextjs" ? "" : "--no-frontend", // appType
undefined, // llamaCloudProjectName
undefined, // llamaCloudIndexName
);
const name = result.projectName;

// Check if the app folder exists
const appDir = path.join(cwd, name);
const dirExists = fs.existsSync(appDir);
expect(dirExists).toBeTruthy();

// Install dependencies using pnpm
try {
const { stderr: installStderr } = await execAsync(
"pnpm install --prefer-offline",
{
cwd: appDir,
},
);
expect(installStderr).toBeFalsy();
} catch (error) {
console.error("Error installing dependencies:", error);
throw error;
}

// Run tsc type check and capture the output
try {
const { stdout, stderr } = await execAsync(
"pnpm exec tsc -b --diagnostics",
{
cwd: appDir,
},
);
// Check if there's any error output
expect(stderr).toBeFalsy();

// Log the stdout for debugging purposes
console.log("TypeScript type-check output:", stdout);
} catch (error) {
console.error("Error running tsc:", error);
throw error;
}
});
}
});
}
6 changes: 4 additions & 2 deletions e2e/utils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,11 +105,11 @@ export async function runCreateLlama(
},
});
appProcess.stderr?.on("data", (data) => {
console.log(data.toString());
console.error(data.toString());
});
appProcess.on("exit", (code) => {
if (code !== 0 && code !== null) {
throw new Error(`create-llama command was failed!`);
throw new Error(`create-llama command failed with exit code ${code}`);
}
});

Expand All @@ -121,6 +121,8 @@ export async function runCreateLlama(
port,
externalPort,
);
} else if (postInstallAction === "dependencies") {
await waitForProcess(appProcess, 1000 * 60); // wait 1 min for dependencies to be resolved
} else {
// wait 10 seconds for create-llama to exit
await waitForProcess(appProcess, 1000 * 10);
Expand Down
37 changes: 36 additions & 1 deletion helpers/typescript.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,7 @@ export const installTSTemplate = async ({
framework,
ui,
observability,
vectorDb,
});

if (postInstallAction === "runApp" || postInstallAction === "dependencies") {
Expand All @@ -200,9 +201,16 @@ async function updatePackageJson({
framework,
ui,
observability,
vectorDb,
}: Pick<
InstallTemplateArgs,
"root" | "appName" | "dataSources" | "framework" | "ui" | "observability"
| "root"
| "appName"
| "dataSources"
| "framework"
| "ui"
| "observability"
| "vectorDb"
> & {
relativeEngineDestPath: string;
}): Promise<any> {
Expand Down Expand Up @@ -249,6 +257,33 @@ async function updatePackageJson({
};
}

if (vectorDb === "pg") {
packageJson.dependencies = {
...packageJson.dependencies,
pg: "^8.12.0",
};
}

if (vectorDb === "qdrant") {
packageJson.dependencies = {
...packageJson.dependencies,
"@qdrant/js-client-rest": "^1.11.0",
};
}
if (vectorDb === "mongo") {
packageJson.dependencies = {
...packageJson.dependencies,
mongodb: "^6.7.0",
};
}

if (vectorDb === "milvus") {
packageJson.dependencies = {
...packageJson.dependencies,
"@zilliz/milvus2-sdk-node": "^2.4.6",
};
}

if (observability === "traceloop") {
packageJson.dependencies = {
...packageJson.dependencies,
Expand Down
4 changes: 2 additions & 2 deletions templates/components/vectordbs/typescript/astra/generate.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
/* eslint-disable turbo/no-undeclared-env-vars */
import * as dotenv from "dotenv";
import { VectorStoreIndex, storageContextFromDefaults } from "llamaindex";
import { AstraDBVectorStore } from "llamaindex/storage/vectorStore/AstraDBVectorStore";
import { AstraDBVectorStore } from "llamaindex/vector-store/AstraDBVectorStore";
import { getDocuments } from "./loader";
import { initSettings } from "./settings";
import { checkRequiredEnvVars } from "./shared";
Expand All @@ -15,7 +15,7 @@ async function loadAndIndex() {
// create vector store and a collection
const collectionName = process.env.ASTRA_DB_COLLECTION!;
const vectorStore = new AstraDBVectorStore();
await vectorStore.create(collectionName, {
await vectorStore.createAndConnect(collectionName, {
vector: {
dimension: parseInt(process.env.EMBEDDING_DIM!),
metric: "cosine",
Expand Down
2 changes: 1 addition & 1 deletion templates/components/vectordbs/typescript/astra/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/* eslint-disable turbo/no-undeclared-env-vars */
import { VectorStoreIndex } from "llamaindex";
import { AstraDBVectorStore } from "llamaindex/storage/vectorStore/AstraDBVectorStore";
import { AstraDBVectorStore } from "llamaindex/vector-store/AstraDBVectorStore";
import { checkRequiredEnvVars } from "./shared";

export async function getDataSource(params?: any) {
Expand Down
4 changes: 2 additions & 2 deletions templates/components/vectordbs/typescript/chroma/generate.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
/* eslint-disable turbo/no-undeclared-env-vars */
import * as dotenv from "dotenv";
import { VectorStoreIndex, storageContextFromDefaults } from "llamaindex";
import { ChromaVectorStore } from "llamaindex/storage/vectorStore/ChromaVectorStore";
import { ChromaVectorStore } from "llamaindex/vector-store/ChromaVectorStore";
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

💡 Codebase verification

Missing Dependency: llamaindex is not listed in package.json.

The ChromaVectorStore is being imported from the llamaindex package in multiple TypeScript files, but llamaindex is not listed under dependencies, devDependencies, or peerDependencies in package.json.

  • Action Required: Add llamaindex to the appropriate dependency section in package.json to ensure it is correctly installed and available during runtime.
Analysis chain

Verify the updated import path for ChromaVectorStore.

The import path for ChromaVectorStore has been updated. This change likely reflects a reorganization in the llamaindex library structure.

To ensure this change is correct and consistent, please run the following script:

This will help confirm that:

  1. The import path is consistent across the project.
  2. The llamaindex package version is up-to-date and supports this new import path.
Scripts executed

The following scripts were executed for the analysis:

Script:

#!/bin/bash
# Description: Verify the import path for ChromaVectorStore across the project

# Test: Search for ChromaVectorStore imports
rg --type typescript "import.*ChromaVectorStore.*from.*llamaindex"

# Test: Check if the llamaindex package is up-to-date
if [ -f "package.json" ]; then
  jq '.dependencies.llamaindex' package.json
fi

Length of output: 177

Script:

#!/bin/bash
# Description: Verify the import path for ChromaVectorStore across the project and check all dependency sections for llamaindex

# Test: Search for ChromaVectorStore imports using wildcard pattern
rg "import.*ChromaVectorStore.*from.*llamaindex" --glob "*.ts"

# Test: Check if the llamaindex package is listed in dependencies, devDependencies, or peerDependencies
if [ -f "package.json" ]; then
  echo "Dependencies:"
  jq '.dependencies.llamaindex' package.json
  echo "DevDependencies:"
  jq '.devDependencies.llamaindex' package.json
  echo "PeerDependencies:"
  jq '.peerDependencies.llamaindex' package.json
fi

Length of output: 637

import { getDocuments } from "./loader";
import { initSettings } from "./settings";
import { checkRequiredEnvVars } from "./shared";
Expand All @@ -16,7 +16,7 @@ async function loadAndIndex() {
const chromaUri = `http://${process.env.CHROMA_HOST}:${process.env.CHROMA_PORT}`;

const vectorStore = new ChromaVectorStore({
collectionName: process.env.CHROMA_COLLECTION,
collectionName: process.env.CHROMA_COLLECTION!,
chromaClientParams: { path: chromaUri },
});

Expand Down
4 changes: 2 additions & 2 deletions templates/components/vectordbs/typescript/chroma/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
/* eslint-disable turbo/no-undeclared-env-vars */
import { VectorStoreIndex } from "llamaindex";
import { ChromaVectorStore } from "llamaindex/storage/vectorStore/ChromaVectorStore";
import { ChromaVectorStore } from "llamaindex/vector-store/ChromaVectorStore";
import { checkRequiredEnvVars } from "./shared";

export async function getDataSource(params?: any) {
checkRequiredEnvVars();
const chromaUri = `http://${process.env.CHROMA_HOST}:${process.env.CHROMA_PORT}`;

const store = new ChromaVectorStore({
collectionName: process.env.CHROMA_COLLECTION,
collectionName: process.env.CHROMA_COLLECTION!,
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🛠️ Refactor suggestion

Consider a more robust approach for handling environment variables.

While the non-null assertion operator (!) resolves TypeScript errors, it might lead to runtime errors if the CHROMA_COLLECTION environment variable is not set. Although checkRequiredEnvVars() is called earlier, which might validate the presence of required environment variables, it's generally safer to handle potential undefined values explicitly.

Consider using a default value or throwing a more descriptive error:

collectionName: process.env.CHROMA_COLLECTION ?? throw new Error("CHROMA_COLLECTION environment variable is not set"),

This approach ensures type safety and provides a clear error message if the environment variable is missing.

chromaClientParams: { path: chromaUri },
});

Expand Down
1 change: 0 additions & 1 deletion templates/components/vectordbs/typescript/llamacloud/queryFilter.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ export function generateFilters(documentIds: string[]): MetadataFilters {
// public documents don't have the "private" field or it's set to "false"
const publicDocumentsFilter: MetadataFilter = {
key: "private",
value: null,
operator: "is_empty",
};

Expand Down
2 changes: 1 addition & 1 deletion templates/components/vectordbs/typescript/milvus/generate.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
/* eslint-disable turbo/no-undeclared-env-vars */
import * as dotenv from "dotenv";
import { VectorStoreIndex, storageContextFromDefaults } from "llamaindex";
import { MilvusVectorStore } from "llamaindex/storage/vectorStore/MilvusVectorStore";
import { MilvusVectorStore } from "llamaindex/vector-store/MilvusVectorStore";
import { getDocuments } from "./loader";
import { initSettings } from "./settings";
import { checkRequiredEnvVars, getMilvusClient } from "./shared";
Expand Down
2 changes: 1 addition & 1 deletion templates/components/vectordbs/typescript/milvus/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { VectorStoreIndex } from "llamaindex";
import { MilvusVectorStore } from "llamaindex/storage/vectorStore/MilvusVectorStore";
import { MilvusVectorStore } from "llamaindex/vector-store/MilvusVectorStore";
import { checkRequiredEnvVars, getMilvusClient } from "./shared";

export async function getDataSource(params?: any) {
Expand Down
2 changes: 1 addition & 1 deletion templates/components/vectordbs/typescript/mongo/generate.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
/* eslint-disable turbo/no-undeclared-env-vars */
import * as dotenv from "dotenv";
import { storageContextFromDefaults, VectorStoreIndex } from "llamaindex";
import { MongoDBAtlasVectorSearch } from "llamaindex/storage/vectorStore/MongoDBAtlasVectorStore";
import { MongoDBAtlasVectorSearch } from "llamaindex/vector-store/MongoDBAtlasVectorStore";
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

💡 Codebase verification

Import path points to a non-existent directory.

The new import path llamaindex/vector-store/MongoDBAtlasVectorStore does not exist. The directory llamaindex/vector-store was not found, which may cause import errors.

Analysis chain

LGTM! Verify the new import path.

The change to the import path for MongoDBAtlasVectorSearch looks good. This update likely reflects a restructuring in the llamaindex library, moving the MongoDBAtlasVectorStore from storage/vectorStore to vector-store.

To ensure this change is correct and functional, please run the following verification steps:

  1. Ensure the llamaindex package is up-to-date.
  2. Verify that the code compiles without any import-related errors.
  3. If possible, run a quick test to confirm that the MongoDBAtlasVectorSearch functionality still works as expected with this new import path.

This script will check if llamaindex is installed and attempt to compile the TypeScript file to catch any potential import errors.

Scripts executed

The following scripts were executed for the analysis:

Script:

#!/bin/bash
# Description: Verify the new import path for MongoDBAtlasVectorSearch

# Test: Check if the new import path exists in the llamaindex package
npm list llamaindex | grep -q "llamaindex" && echo "llamaindex is installed" || echo "llamaindex is not installed"

# Note: The following command assumes TypeScript is set up in the project
# If TypeScript is not set up, you may need to adjust the command accordingly
npx tsc --noEmit templates/components/vectordbs/typescript/mongo/generate.ts

Length of output: 328

Script:

#!/bin/bash
# Description: Verify the new import path for MongoDBAtlasVectorSearch using fd and ast-grep

# Check if MongoDBAtlasVectorStore exists in the new path
fd "MongoDBAtlasVectorStore" ./llamaindex/vector-store --type f

# Verify the import statement in generate.ts
ast-grep --lang typescript --pattern $'import { MongoDBAtlasVectorSearch } from "llamaindex/vector-store/MongoDBAtlasVectorStore";' templates/components/vectordbs/typescript/mongo/generate.ts

Length of output: 523

import { MongoClient } from "mongodb";
import { getDocuments } from "./loader";
import { initSettings } from "./settings";
Expand Down
2 changes: 1 addition & 1 deletion templates/components/vectordbs/typescript/mongo/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/* eslint-disable turbo/no-undeclared-env-vars */
import { VectorStoreIndex } from "llamaindex";
import { MongoDBAtlasVectorSearch } from "llamaindex/storage/vectorStore/MongoDBAtlasVectorStore";
import { MongoDBAtlasVectorSearch } from "llamaindex/vector-store/MongoDBAtlasVectorStore";
import { MongoClient } from "mongodb";
import { checkRequiredEnvVars, POPULATED_METADATA_FIELDS } from "./shared";

Expand Down
7 changes: 5 additions & 2 deletions templates/components/vectordbs/typescript/pg/generate.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
/* eslint-disable turbo/no-undeclared-env-vars */
import * as dotenv from "dotenv";
import { VectorStoreIndex, storageContextFromDefaults } from "llamaindex";
import { PGVectorStore } from "llamaindex/storage/vectorStore/PGVectorStore";
import {
PGVectorStore,
VectorStoreIndex,
storageContextFromDefaults,
} from "llamaindex";
import { getDocuments } from "./loader";
import { initSettings } from "./settings";
import {
Expand Down
3 changes: 1 addition & 2 deletions templates/components/vectordbs/typescript/pg/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
/* eslint-disable turbo/no-undeclared-env-vars */
import { VectorStoreIndex } from "llamaindex";
import { PGVectorStore } from "llamaindex/storage/vectorStore/PGVectorStore";
import { PGVectorStore, VectorStoreIndex } from "llamaindex";
import {
PGVECTOR_SCHEMA,
PGVECTOR_TABLE,
Expand Down
2 changes: 1 addition & 1 deletion templates/components/vectordbs/typescript/pinecone/generate.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
/* eslint-disable turbo/no-undeclared-env-vars */
import * as dotenv from "dotenv";
import { VectorStoreIndex, storageContextFromDefaults } from "llamaindex";
import { PineconeVectorStore } from "llamaindex/storage/vectorStore/PineconeVectorStore";
import { PineconeVectorStore } from "llamaindex/vector-store/PineconeVectorStore";
import { getDocuments } from "./loader";
import { initSettings } from "./settings";
import { checkRequiredEnvVars } from "./shared";
Expand Down
2 changes: 1 addition & 1 deletion templates/components/vectordbs/typescript/pinecone/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/* eslint-disable turbo/no-undeclared-env-vars */
import { VectorStoreIndex } from "llamaindex";
import { PineconeVectorStore } from "llamaindex/storage/vectorStore/PineconeVectorStore";
import { PineconeVectorStore } from "llamaindex/vector-store/PineconeVectorStore";
import { checkRequiredEnvVars } from "./shared";

export async function getDataSource(params?: any) {
Expand Down
2 changes: 1 addition & 1 deletion templates/components/vectordbs/typescript/qdrant/generate.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
/* eslint-disable turbo/no-undeclared-env-vars */
import * as dotenv from "dotenv";
import { VectorStoreIndex, storageContextFromDefaults } from "llamaindex";
import { QdrantVectorStore } from "llamaindex/storage/vectorStore/QdrantVectorStore";
import { QdrantVectorStore } from "llamaindex/vector-store/QdrantVectorStore";
import { getDocuments } from "./loader";
import { initSettings } from "./settings";
import { checkRequiredEnvVars, getQdrantClient } from "./shared";
Expand Down
2 changes: 1 addition & 1 deletion templates/components/vectordbs/typescript/qdrant/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import * as dotenv from "dotenv";
import { VectorStoreIndex } from "llamaindex";
import { QdrantVectorStore } from "llamaindex/storage/vectorStore/QdrantVectorStore";
import { QdrantVectorStore } from "llamaindex/vector-store/QdrantVectorStore";
import { checkRequiredEnvVars, getQdrantClient } from "./shared";

dotenv.config();
Expand Down
2 changes: 1 addition & 1 deletion templates/components/vectordbs/typescript/weaviate/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import * as dotenv from "dotenv";
import { VectorStoreIndex } from "llamaindex";
import { WeaviateVectorStore } from "llamaindex/storage/vectorStore/WeaviateVectorStore";
import { WeaviateVectorStore } from "llamaindex/vector-store/WeaviateVectorStore";
import { checkRequiredEnvVars, DEFAULT_INDEX_NAME } from "./shared";

dotenv.config();
Expand Down
Loading