Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Unit tests for multiple languages and the build prompt. #186

Open
wants to merge 50 commits into
base: main
Choose a base branch
from
Open
Changes from 1 commit
Commits
Show all changes
50 commits
Select commit Hold shift + click to select a range
f98d8f2
Merge branch 'plandex-ai:main' into feature/evals
ZanzyTHEbar Jun 26, 2024
33d472e
chore: Add advanced evaluation methods for test cases
ZanzyTHEbar Jun 28, 2024
203e209
chore: Update test cases with new evaluation methods
ZanzyTHEbar Jun 28, 2024
6c34630
chore: migrate tests from verify to yaml file
ZanzyTHEbar Jul 2, 2024
b7c610b
chore: Update test file paths for tests in fix directory
ZanzyTHEbar Jul 2, 2024
67798fd
chore: Update test file paths for tests in fix directory
ZanzyTHEbar Jul 2, 2024
935db3a
chore: setup multi-language tests for build dir
ZanzyTHEbar Jul 2, 2024
f91e53a
chore: begin migrating language tests to individual files
ZanzyTHEbar Jul 10, 2024
aeb4d8b
chore: continue migrating out languages tests to individual files
ZanzyTHEbar Jul 11, 2024
05424f3
chore: js and python tests
ZanzyTHEbar Jul 11, 2024
ee22522
chore: Add error handling to `processData`, update TODO comment, and …
ZanzyTHEbar Jul 11, 2024
a0e053f
chore: Add pdx line nums to tests
ZanzyTHEbar Jul 11, 2024
d0541ce
chore: cleaning up tests in build dir
ZanzyTHEbar Jul 16, 2024
6b0dfe9
chore: update md files for connection and data models
ZanzyTHEbar Jul 16, 2024
899a696
chore: update md files for network and sort function
ZanzyTHEbar Jul 16, 2024
5fbeb6f
chore: update md file user handler
ZanzyTHEbar Jul 16, 2024
5be03ff
chore: update md file user management js test
ZanzyTHEbar Jul 16, 2024
655b928
chore: Add error handling to `processData`, update TODO comment, and …
ZanzyTHEbar Jul 16, 2024
5bdb12b
chore: implement base-line asset files for fix test directory
ZanzyTHEbar Jul 19, 2024
1374559
chore: fix asset generation logic
ZanzyTHEbar Jul 19, 2024
016a87f
chore: minor updates to some of the assets
ZanzyTHEbar Jul 19, 2024
04d34ad
chore: migrate removal go assets to new file format
ZanzyTHEbar Jul 19, 2024
97782f4
chore: add assets for verify tests
ZanzyTHEbar Jul 19, 2024
407e02b
chore: finishing up verify
ZanzyTHEbar Jul 26, 2024
91c88d3
chore: split diffs and changes into separate dirs
ZanzyTHEbar Jul 29, 2024
4aa73a8
chore: finish java code for verify
ZanzyTHEbar Jul 29, 2024
8362643
chore: Refactor Java code and fix TODO comments
ZanzyTHEbar Jul 29, 2024
1c0982a
chore: finish up python code for verify test
ZanzyTHEbar Jul 29, 2024
22e2f10
chore: begin preparring to write tests
ZanzyTHEbar Jul 29, 2024
b0b9c7a
chore: begin preparring to write tests
ZanzyTHEbar Jul 29, 2024
de581c7
chore: finalizing the fix directory
ZanzyTHEbar Aug 7, 2024
24ff033
chore: finish assests for fix test dir
ZanzyTHEbar Aug 7, 2024
654ab7f
chore: Add line nums to fix post files
ZanzyTHEbar Aug 7, 2024
52f5900
chore: Add line nums to fix post files
ZanzyTHEbar Aug 7, 2024
01318c1
chore: Add line nums to fix post files
ZanzyTHEbar Aug 7, 2024
8c97e97
Merge branch 'plandex-ai:main' into feature/evals
ZanzyTHEbar Aug 7, 2024
205a32f
Merge pull request #3 from ZanzyTHEbar/feature/evals
ZanzyTHEbar Aug 7, 2024
568e717
chore: finish tests for the fix directory
ZanzyTHEbar Aug 7, 2024
54c5cfd
chore: implement the go tests for verify directory
ZanzyTHEbar Aug 7, 2024
365d310
chore: update test names to include the language identifier
ZanzyTHEbar Aug 7, 2024
f013921
chore: add all language tests for verify dir
ZanzyTHEbar Aug 7, 2024
74ef66f
chore: rename tests in fix dir to include language id
ZanzyTHEbar Aug 7, 2024
f8af6fc
chore: add js language tests to fix
ZanzyTHEbar Aug 7, 2024
5d913ea
chore: add html and java tests to fix
ZanzyTHEbar Aug 7, 2024
7842c4f
Merge pull request #4 from ZanzyTHEbar/feature/evals
ZanzyTHEbar Aug 13, 2024
eff36ce
chore: write post asset code files for build
ZanzyTHEbar Aug 15, 2024
e788158
chore: setup initial build test structure
ZanzyTHEbar Aug 15, 2024
2f94f01
Merge pull request #5 from ZanzyTHEbar/feature/evals
ZanzyTHEbar Aug 15, 2024
05f2319
chore: build tests complete
ZanzyTHEbar Aug 16, 2024
598e26b
Merge pull request #6 from ZanzyTHEbar/feature/evals
ZanzyTHEbar Aug 16, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
chore: write post asset code files for build
ZanzyTHEbar committed Aug 15, 2024
commit eff36ceb73b67db0ab1bd1f883be9d24c47364f0
245 changes: 122 additions & 123 deletions test/evals/promptfoo-poc/build/assets/go/code/build.post.go
Original file line number Diff line number Diff line change
@@ -1,123 +1,122 @@
pdx-1: package cmd
pdx-2:
pdx-3: import (
pdx-4: "fmt"
pdx-5: "path/filepath"
pdx-6: "plandex/api"
pdx-7: "plandex/auth"
pdx-8: "plandex/lib"
pdx-9: "plandex/term"
pdx-10: "strconv"
pdx-11: "strings"
pdx-12:
pdx-13: "github.com/plandex/plandex/shared"
pdx-14: "github.com/spf13/cobra"
pdx-15: )
pdx-16:
pdx-17: func parseRange(arg string) ([]int, error) {
pdx-18: var indices []int
pdx-19: parts := strings.Split(arg, "-")
pdx-20: if len(parts) == 2 {
pdx-21: start, err := strconv.Atoi(parts[0])
pdx-22: if err != nil {
pdx-23: return nil, err
pdx-24: }
pdx-25: end, err := strconv.Atoi(parts[1])
pdx-26: if err != nil {
pdx-27: return nil, err
pdx-28: }
pdx-29: for i := start; i <= end; i++ {
pdx-30: indices = append(indices, i)
pdx-31: }
pdx-32: } else {
pdx-33: index, err := strconv.Atoi(arg)
pdx-34: if err != nil {
pdx-35: return nil, err
pdx-36: }
pdx-37: indices = append(indices, index)
pdx-38: }
pdx-39: return indices, nil
pdx-40: }
pdx-41:
pdx-42: func contextRm(cmd *cobra.Command, args []string) {
pdx-43: auth.MustResolveAuthWithOrg()
pdx-44: lib.MustResolveProject()
pdx-45:
pdx-46: if lib.CurrentPlanId == "" {
pdx-47: fmt.Println("🤷‍♂️ No current plan")
pdx-48: return
pdx-49: }
pdx-50:
pdx-51: term.StartSpinner("")
pdx-52: contexts, err := api.Client.ListContext(lib.CurrentPlanId, lib.CurrentBranch)
pdx-53:
pdx-54: if err != nil {
pdx-55: term.OutputErrorAndExit("Error retrieving context: %v", err)
pdx-56: }
pdx-57:
pdx-58: deleteIds := map[string]bool{}
pdx-59:
pdx-60: for _, arg := range args {
pdx-61: indices, err := parseRange(arg)
pdx-62: if err != nil {
pdx-63: term.OutputErrorAndExit("Error parsing range: %v", err)
pdx-64: }
pdx-65:
pdx-66: for _, index := range indices {
pdx-67: if index > 0 && index <= len(contexts) {
pdx-68: context := contexts[index-1]
pdx-69: deleteIds[context.Id] = true
pdx-70: }
pdx-71: }
pdx-72: }
pdx-73:
pdx-74: for i, context := range contexts {
pdx-75: for _, id := range args {
pdx-76: if fmt.Sprintf("%d", i+1) == id || context.Name == id || context.FilePath == id || context.Url == id {
pdx-77: deleteIds[context.Id] = true
pdx-78: break
pdx-79: } else if context.FilePath != "" {
pdx-80: // Check if id is a glob pattern
pdx-81: matched, err := filepath.Match(id, context.FilePath)
pdx-82: if err != nil {
pdx-83: term.OutputErrorAndExit("Error matching glob pattern: %v", err)
pdx-84: }
pdx-85: if matched {
pdx-86: deleteIds[context.Id] = true
pdx-87: break
pdx-88: }
pdx-89:
pdx-90: // Check if id is a parent directory
pdx-91: parentDir := context.FilePath
pdx-92: for parentDir != "." && parentDir != "/" && parentDir != "" {
pdx-93: if parentDir == id {
pdx-94: deleteIds[context.Id] = true
pdx-95: break
pdx-96: }
pdx-97: parentDir = filepath.Dir(parentDir) // Move up one directory
pdx-98: }
pdx-99: }
pdx-100: }
pdx-101: }
pdx-102:
pdx-103: if len(deleteIds) > 0 {
pdx-104: res, err := api.Client.DeleteContext(lib.CurrentPlanId, lib.CurrentBranch, shared.DeleteContextRequest{
pdx-105: Ids: deleteIds,
pdx-106: })
pdx-107: term.StopSpinner()
pdx-108:
pdx-109: if err != nil {
pdx-110: term.OutputErrorAndExit("Error deleting context: %v", err)
pdx-111: }
pdx-112:
pdx-113: fmt.Println("✅ " + res.Msg)
pdx-114: } else {
pdx-115: term.StopSpinner()
pdx-116: fmt.Println("🤷‍♂️ No context removed")
pdx-117: }
pdx-118: }
pdx-119:
pdx-120: func init() {
pdx-121: RootCmd.AddCommand(contextRmCmd)
pdx-122: }
pdx-123:
package cmd

import (
"fmt"
"path/filepath"
"plandex/api"
"plandex/auth"
"plandex/lib"
"plandex/term"
"strconv"
"strings"

"github.com/plandex/plandex/shared"
"github.com/spf13/cobra"
)

func parseRange(arg string) ([]int, error) {
var indices []int
parts := strings.Split(arg, "-")
if len(parts) == 2 {
start, err := strconv.Atoi(parts[0])
if err != nil {
return nil, err
}
end, err := strconv.Atoi(parts[1])
if err != nil {
return nil, err
}
for i := start; i <= end; i++ {
indices = append(indices, i)
}
} else {
index, err := strconv.Atoi(arg)
if err != nil {
return nil, err
}
indices = append(indices, index)
}
return indices, nil
}

func contextRm(cmd *cobra.Command, args []string) {
auth.MustResolveAuthWithOrg()
lib.MustResolveProject()

if lib.CurrentPlanId == "" {
fmt.Println("🤷‍♂️ No current plan")
return
}

term.StartSpinner("")
contexts, err := api.Client.ListContext(lib.CurrentPlanId, lib.CurrentBranch)

if err != nil {
term.OutputErrorAndExit("Error retrieving context: %v", err)
}

deleteIds := map[string]bool{}

for _, arg := range args {
indices, err := parseRange(arg)
if err != nil {
term.OutputErrorAndExit("Error parsing range: %v", err)
}

for _, index := range indices {
if index > 0 && index <= len(contexts) {
context := contexts[index-1]
deleteIds[context.Id] = true
}
}
}

for i, context := range contexts {
for _, id := range args {
if fmt.Sprintf("%d", i+1) == id || context.Name == id || context.FilePath == id || context.Url == id {
deleteIds[context.Id] = true
break
} else if context.FilePath != "" {
// Check if id is a glob pattern
matched, err := filepath.Match(id, context.FilePath)
if err != nil {
term.OutputErrorAndExit("Error matching glob pattern: %v", err)
}
if matched {
deleteIds[context.Id] = true
break
}

// Check if id is a parent directory
parentDir := context.FilePath
for parentDir != "." && parentDir != "/" && parentDir != "" {
if parentDir == id {
deleteIds[context.Id] = true
break
}
parentDir = filepath.Dir(parentDir) // Move up one directory
}
}
}
}

if len(deleteIds) > 0 {
res, err := api.Client.DeleteContext(lib.CurrentPlanId, lib.CurrentBranch, shared.DeleteContextRequest{
Ids: deleteIds,
})
term.StopSpinner()

if err != nil {
term.OutputErrorAndExit("Error deleting context: %v", err)
}

fmt.Println("✅ " + res.Msg)
} else {
term.StopSpinner()
fmt.Println("🤷‍♂️ No context removed")
}
}

func init() {
RootCmd.AddCommand(contextRmCmd)
}
42 changes: 42 additions & 0 deletions test/evals/promptfoo-poc/build/assets/go/code/connection.post.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
package database

import (
"database/sql"
"log"
)

var db *sql.DB

func init() {
// Initialize database connection

var err error
for i := 0; i < 3; i++ {
db, err = sql.Open("postgres", "user=postgres password=postgres dbname=postgres sslmode=disable")
if err != nil {
log.Println("Error connecting to database: ", err)
time.Sleep(5 * time.Second)
continue
}
err = db.Ping()
if err != nil {
log.Println("Error connecting to database: ", err)
time.Sleep(5 * time.Second)
continue
}
log.Println("Connected to database")
break
}
}

func GetConnection() *sql.DB {
return db
}


func CloseConnection() {
if db != nil {
db.Close()
log.Println("Database connection closed")
}
}
23 changes: 23 additions & 0 deletions test/evals/promptfoo-poc/build/assets/go/code/data_models.post.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
package models

type Post struct {
ID string
Content string
Author string
CreatedAt time.Time
Comments []*Comment
}

type Comment struct {
ID string
Content string
Author string
CreatedAt time.Time
}

type User struct {
ID string
Username string
Email string
CreatedAt time.Time
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
package network

func InitializeNetwork() {
// Initialize default network settings

// Check if environment variables are set
if ipAddress := os.Getenv("IP_ADDRESS"); ipAddress != "" {
ConfigureIP(ipAddress)
}
if subnetMask := os.Getenv("SUBNET_MASK"); subnetMask != "" {
ConfigureSubnet(subnetMask)
}
if gateway := os.Getenv("GATEWAY"); gateway != "" {
ConfigureGateway(gateway)
}
if dnsServer := os.Getenv("DNS_SERVER"); dnsServer != "" {
ConfigureDNS(dnsServer)
}
if domain := os.Getenv("DOMAIN"); domain != "" {
ConfigureDomain(domain)
}
if proxy := os.Getenv("PROXY"); proxy != "" {
ConfigureProxy(proxy)
}
if port := os.Getenv("PORT"); port != "" {
ConfigurePort(port)
}
// Verify network connectivity
VerifyConnectivity()
}

func ConfigureIP(ipAddress string) {
// Code to configure IP address
}

func VerifyConnectivity() {
// Ping a set of predefined addresses

// Ping the default gateway
if err := ping(""); err != nil {
log.Fatal("Failed to ping the default gateway")
}

// Ping the DNS server
if err := ping(""); err != nil {
log.Fatal("Failed to ping the DNS server")
}

// Ping a public domain
if err := ping(""); err != nil {
log.Fatal("Failed to ping a public domain")
}

// Ping a public IP address
if err := ping(""); err != nil {
log.Fatal("Failed to ping a public IP address")
}

log.Println("Network configuration successful")
}
18 changes: 18 additions & 0 deletions test/evals/promptfoo-poc/build/assets/go/code/server_setup.post.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
package server

import (
"log"
"net/http"
"time"
)

func init() { log.Printf("Server starting at %s...", time.Now()) }

func handler(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("Hello, world!"))
log.Printf("Request: %s %s", r.Method, r.RequestURI)
}

func apiHandler(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("API is live"))
}
Loading