From d170bb7c049645c16afa183df4731e13b5da72b5 Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Thu, 11 Jul 2024 14:40:27 -0400 Subject: [PATCH 01/29] seperate the filtering from the middleware changes Signed-off-by: Dave Lee --- core/config/backend_config.go | 64 ++++++++++++++++++++++ core/config/backend_config_filter.go | 35 ++++++++++++ core/config/backend_config_loader.go | 20 +++++++ core/config/backend_config_test.go | 72 +++++++++++++++++++++++++ core/http/ctx/fiber.go | 4 +- core/http/endpoints/localai/welcome.go | 2 +- core/http/endpoints/openai/assistant.go | 2 +- core/http/endpoints/openai/list.go | 38 ++++++------- core/http/routes/ui.go | 6 +-- core/services/list_models.go | 62 ++++++++++----------- 10 files changed, 244 insertions(+), 61 deletions(-) create mode 100644 core/config/backend_config_filter.go diff --git a/core/config/backend_config.go b/core/config/backend_config.go index 1e647cebeef1..45cb6cdf8439 100644 --- a/core/config/backend_config.go +++ b/core/config/backend_config.go @@ -402,3 +402,67 @@ func (c *BackendConfig) Validate() bool { func (c *BackendConfig) HasTemplate() bool { return c.TemplateConfig.Completion != "" || c.TemplateConfig.Edit != "" || c.TemplateConfig.Chat != "" || c.TemplateConfig.ChatMessage != "" } + +type BackendConfigUsecases int + +const ( + FLAG_ANY BackendConfigUsecases = 0b00000000 + FLAG_CHAT BackendConfigUsecases = 0b00000001 + FLAG_COMPLETION BackendConfigUsecases = 0b00000010 + FLAG_EDIT BackendConfigUsecases = 0b00000100 + FLAG_EMBEDDINGS BackendConfigUsecases = 0b00001000 + FLAG_RERANK BackendConfigUsecases = 0b00010000 + FLAG_IMAGE BackendConfigUsecases = 0b00100000 + FLAG_TRANSCRIPT BackendConfigUsecases = 0b01000000 + FLAG_TTS BackendConfigUsecases = 0b10000000 + + // Common Subsets + FLAG_LLM BackendConfigUsecases = FLAG_CHAT & FLAG_COMPLETION & FLAG_EDIT +) + +func (c *BackendConfig) HasUsecases(u BackendConfigUsecases) bool { + if (u & FLAG_CHAT) == FLAG_CHAT { + if c.TemplateConfig.Chat == "" && c.TemplateConfig.ChatMessage == "" { + return false + } + } + if (u & FLAG_COMPLETION) == FLAG_COMPLETION { + if c.TemplateConfig.Completion == "" { + return false + } + } + if (u & FLAG_EDIT) == FLAG_EDIT { + if c.TemplateConfig.Edit == "" { + return false + } + } + if (u & FLAG_EMBEDDINGS) == FLAG_EMBEDDINGS { + if !c.Embeddings { + return false + } + } + if (u & FLAG_IMAGE) == FLAG_IMAGE { + // This one feels _really_ bad. Need to experiment but it's better than nothing at all? + if (c.Backend != "diffusers" || c.Diffusers.PipelineType != "") && c.Backend != "tinydream" || c.Backend != "stablediffusion" { + return false + } + } + if (u & FLAG_RERANK) == FLAG_RERANK { + if c.Backend != "rerankers" { + return false + } + } + if (u & FLAG_TRANSCRIPT) == FLAG_TRANSCRIPT { + if c.Backend != "whisper" { + return false + } + } + if (u & FLAG_TTS) == FLAG_TTS { + // This one feels _really_ bad. Need to reach out to TTS experts to find something salient here. + if c.Backend != "transformer-musicgen" && c.Backend != "piper" && c.Backend != "parler-tts" { + return false + } + } + + return true +} diff --git a/core/config/backend_config_filter.go b/core/config/backend_config_filter.go new file mode 100644 index 000000000000..f1eb24883b0c --- /dev/null +++ b/core/config/backend_config_filter.go @@ -0,0 +1,35 @@ +package config + +import "regexp" + +type BackendConfigFilterFn func(string, *BackendConfig) bool + +func NoFilterFn(_ string, _ *BackendConfig) bool { return true } + +func BuildNameFilterFn(filter string) (BackendConfigFilterFn, error) { + if filter == "" { + return NoFilterFn, nil + } + rxp, err := regexp.Compile(filter) + if err != nil { + return nil, err + } + return func(name string, config *BackendConfig) bool { + if config != nil { + return rxp.MatchString(config.Name) + } + return rxp.MatchString(name) + }, nil +} + +func BuildUsecaseFilterFn(usecases BackendConfigUsecases) BackendConfigFilterFn { + if usecases == FLAG_ANY { + return NoFilterFn + } + return func(name string, config *BackendConfig) bool { + if config == nil { + return false // TODO: Potentially make this a param, for now, no known usecase to include + } + return config.HasUsecases(usecases) + } +} diff --git a/core/config/backend_config_loader.go b/core/config/backend_config_loader.go index 283dac52bd7d..0787aa525184 100644 --- a/core/config/backend_config_loader.go +++ b/core/config/backend_config_loader.go @@ -201,6 +201,26 @@ func (bcl *BackendConfigLoader) GetAllBackendConfigs() []BackendConfig { return res } +func (bcl *BackendConfigLoader) GetBackendConfigsByFilter(filter BackendConfigFilterFn) []BackendConfig { + bcl.Lock() + defer bcl.Unlock() + var res []BackendConfig + + if filter == nil { + filter = NoFilterFn + } + + for n, v := range bcl.configs { + if filter(n, &v) { + res = append(res, v) + } + } + + // TODO: I don't think this one needs to Sort on name... but we'll see what breaks. + + return res +} + func (bcl *BackendConfigLoader) RemoveBackendConfig(m string) { bcl.Lock() defer bcl.Unlock() diff --git a/core/config/backend_config_test.go b/core/config/backend_config_test.go index da2459332873..f89d47cd1d31 100644 --- a/core/config/backend_config_test.go +++ b/core/config/backend_config_test.go @@ -61,4 +61,76 @@ parameters: Expect(config.Validate()).To(BeTrue()) }) }) + It("Properly handles backend usecase matching", func() { + + a := BackendConfig{ + Name: "a", + } + Expect(a.HasUsecases(FLAG_ANY)).To(BeFalse()) + + b := BackendConfig{ + Name: "b", + Backend: "stablediffusion", + } + Expect(b.HasUsecases(FLAG_ANY)).To(BeTrue()) + Expect(b.HasUsecases(FLAG_IMAGE)).To(BeTrue()) + Expect(b.HasUsecases(FLAG_CHAT)).To(BeFalse()) + + c := BackendConfig{ + Name: "c", + Backend: "llama-cpp", + TemplateConfig: TemplateConfig{ + Chat: "chat", + }, + } + Expect(c.HasUsecases(FLAG_ANY)).To(BeTrue()) + Expect(c.HasUsecases(FLAG_IMAGE)).To(BeFalse()) + Expect(c.HasUsecases(FLAG_COMPLETION)).To(BeFalse()) + Expect(c.HasUsecases(FLAG_CHAT)).To(BeTrue()) + + d := BackendConfig{ + Name: "d", + Backend: "llama-cpp", + TemplateConfig: TemplateConfig{ + Chat: "chat", + Completion: "completion", + }, + } + Expect(d.HasUsecases(FLAG_ANY)).To(BeTrue()) + Expect(d.HasUsecases(FLAG_IMAGE)).To(BeFalse()) + Expect(d.HasUsecases(FLAG_COMPLETION)).To(BeTrue()) + Expect(d.HasUsecases(FLAG_CHAT)).To(BeTrue()) + + e := BackendConfig{ + Name: "e", + Backend: "llama-cpp", + TemplateConfig: TemplateConfig{ + Completion: "completion", + }, + Embeddings: true, + } + + Expect(e.HasUsecases(FLAG_ANY)).To(BeTrue()) + Expect(e.HasUsecases(FLAG_IMAGE)).To(BeFalse()) + Expect(e.HasUsecases(FLAG_COMPLETION)).To(BeTrue()) + Expect(e.HasUsecases(FLAG_CHAT)).To(BeFalse()) + Expect(e.HasUsecases(FLAG_EMBEDDINGS)).To(BeTrue()) + + f := BackendConfig{ + Name: "f", + Backend: "piper", + } + Expect(f.HasUsecases(FLAG_ANY)).To(BeTrue()) + Expect(f.HasUsecases(FLAG_TTS)).To(BeTrue()) + Expect(f.HasUsecases(FLAG_CHAT)).To(BeFalse()) + + g := BackendConfig{ + Name: "g", + Backend: "whisper", + } + Expect(g.HasUsecases(FLAG_ANY)).To(BeTrue()) + Expect(g.HasUsecases(FLAG_TRANSCRIPT)).To(BeTrue()) + Expect(g.HasUsecases(FLAG_TTS)).To(BeFalse()) + + }) }) diff --git a/core/http/ctx/fiber.go b/core/http/ctx/fiber.go index 9405984711ae..28a35ac4c1d5 100644 --- a/core/http/ctx/fiber.go +++ b/core/http/ctx/fiber.go @@ -21,12 +21,12 @@ func ModelFromContext(ctx *fiber.Ctx, cl *config.BackendConfigLoader, loader *mo } // Set model from bearer token, if available - bearer := strings.TrimLeft(ctx.Get("authorization"), "Bearer ") + bearer := strings.TrimLeft(ctx.Get("authorization"), "Bear ") // Reduced duplicate characters of Bearer bearerExists := bearer != "" && loader.ExistsInModelPath(bearer) // If no model was specified, take the first available if modelInput == "" && !bearerExists && firstModel { - models, _ := services.ListModels(cl, loader, "", true) + models, _ := services.ListModels(cl, loader, config.NoFilterFn, services.SKIP_IF_CONFIGURED) if len(models) > 0 { modelInput = models[0] log.Debug().Msgf("No model specified, using: %s", modelInput) diff --git a/core/http/endpoints/localai/welcome.go b/core/http/endpoints/localai/welcome.go index b9c7a57308bd..98a9b14e658e 100644 --- a/core/http/endpoints/localai/welcome.go +++ b/core/http/endpoints/localai/welcome.go @@ -13,7 +13,7 @@ import ( func WelcomeEndpoint(appConfig *config.ApplicationConfig, cl *config.BackendConfigLoader, ml *model.ModelLoader, modelStatus func() (map[string]string, map[string]string)) func(*fiber.Ctx) error { return func(c *fiber.Ctx) error { - models, _ := services.ListModels(cl, ml, "", true) + models, _ := services.ListModels(cl, ml, config.NoFilterFn, services.SKIP_IF_CONFIGURED) backendConfigs := cl.GetAllBackendConfigs() galleryConfigs := map[string]*gallery.Config{} diff --git a/core/http/endpoints/openai/assistant.go b/core/http/endpoints/openai/assistant.go index ba2ebcdee50a..30afbde5dd1e 100644 --- a/core/http/endpoints/openai/assistant.go +++ b/core/http/endpoints/openai/assistant.go @@ -216,7 +216,7 @@ func filterAssistantsAfterID(assistants []Assistant, id string) []Assistant { func modelExists(cl *config.BackendConfigLoader, ml *model.ModelLoader, modelName string) (found bool) { found = false - models, err := services.ListModels(cl, ml, "", true) + models, err := services.ListModels(cl, ml, config.NoFilterFn, services.SKIP_IF_CONFIGURED) if err != nil { return } diff --git a/core/http/endpoints/openai/list.go b/core/http/endpoints/openai/list.go index d446b10008d8..80dcb3e49442 100644 --- a/core/http/endpoints/openai/list.go +++ b/core/http/endpoints/openai/list.go @@ -18,32 +18,32 @@ func ListModelsEndpoint(bcl *config.BackendConfigLoader, ml *model.ModelLoader) filter := c.Query("filter") // By default, exclude any loose files that are already referenced by a configuration file. - excludeConfigured := c.QueryBool("excludeConfigured", true) + var policy services.LooseFilePolicy + if c.QueryBool("excludeConfigured", true) { + policy = services.SKIP_IF_CONFIGURED + } else { + policy = services.ALWAYS_INCLUDE // This replicates current behavior. TODO: give more options to the user? + } + + filterFn, err := config.BuildNameFilterFn(filter) + if err != nil { + return err + } - dataModels, err := modelList(bcl, ml, filter, excludeConfigured) + modelNames, err := services.ListModels(bcl, ml, filterFn, policy) if err != nil { return err } + + // Map from a slice of names to a slice of OpenAIModel response objects + dataModels := []schema.OpenAIModel{} + for _, m := range modelNames { + dataModels = append(dataModels, schema.OpenAIModel{ID: m, Object: "model"}) + } + return c.JSON(schema.ModelsDataResponse{ Object: "list", Data: dataModels, }) } } - -func modelList(bcl *config.BackendConfigLoader, ml *model.ModelLoader, filter string, excludeConfigured bool) ([]schema.OpenAIModel, error) { - - models, err := services.ListModels(bcl, ml, filter, excludeConfigured) - if err != nil { - return nil, err - } - - dataModels := []schema.OpenAIModel{} - - // Then iterate through the loose files: - for _, m := range models { - dataModels = append(dataModels, schema.OpenAIModel{ID: m, Object: "model"}) - } - - return dataModels, nil -} diff --git a/core/http/routes/ui.go b/core/http/routes/ui.go index 33706944fa2e..09bce9e17728 100644 --- a/core/http/routes/ui.go +++ b/core/http/routes/ui.go @@ -269,7 +269,7 @@ func RegisterUIRoutes(app *fiber.App, // Show the Chat page app.Get("/chat/:model", auth, func(c *fiber.Ctx) error { - backendConfigs, _ := services.ListModels(cl, ml, "", true) + backendConfigs, _ := services.ListModels(cl, ml, config.NoFilterFn, services.SKIP_IF_CONFIGURED) summary := fiber.Map{ "Title": "LocalAI - Chat with " + c.Params("model"), @@ -284,7 +284,7 @@ func RegisterUIRoutes(app *fiber.App, }) app.Get("/talk/", auth, func(c *fiber.Ctx) error { - backendConfigs, _ := services.ListModels(cl, ml, "", true) + backendConfigs, _ := services.ListModels(cl, ml, config.NoFilterFn, services.SKIP_IF_CONFIGURED) if len(backendConfigs) == 0 { // If no model is available redirect to the index which suggests how to install models @@ -305,7 +305,7 @@ func RegisterUIRoutes(app *fiber.App, app.Get("/chat/", auth, func(c *fiber.Ctx) error { - backendConfigs, _ := services.ListModels(cl, ml, "", true) + backendConfigs, _ := services.ListModels(cl, ml, config.NoFilterFn, services.SKIP_IF_CONFIGURED) if len(backendConfigs) == 0 { // If no model is available redirect to the index which suggests how to install models diff --git a/core/services/list_models.go b/core/services/list_models.go index 4b578e2579fa..c310ac150beb 100644 --- a/core/services/list_models.go +++ b/core/services/list_models.go @@ -1,55 +1,47 @@ package services import ( - "regexp" - "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/pkg/model" ) -func ListModels(bcl *config.BackendConfigLoader, ml *model.ModelLoader, filter string, excludeConfigured bool) ([]string, error) { +type LooseFilePolicy int - models, err := ml.ListFilesInModelPath() - if err != nil { - return nil, err - } +const ( + SKIP_IF_CONFIGURED LooseFilePolicy = iota + SKIP_ALWAYS + ALWAYS_INCLUDE + LOOSE_ONLY +) - var mm map[string]interface{} = map[string]interface{}{} +func ListModels(bcl *config.BackendConfigLoader, ml *model.ModelLoader, filter config.BackendConfigFilterFn, looseFilePolicy LooseFilePolicy) ([]string, error) { - dataModels := []string{} + var skipMap map[string]interface{} = map[string]interface{}{} - var filterFn func(name string) bool + dataModels := []string{} - // If filter is not specified, do not filter the list by model name - if filter == "" { - filterFn = func(_ string) bool { return true } - } else { - // If filter _IS_ specified, we compile it to a regex which is used to create the filterFn - rxp, err := regexp.Compile(filter) - if err != nil { - return nil, err - } - filterFn = func(name string) bool { - return rxp.MatchString(name) + // Start with known configurations + if looseFilePolicy != LOOSE_ONLY { + for _, c := range bcl.GetBackendConfigsByFilter(filter) { + if looseFilePolicy == SKIP_IF_CONFIGURED { + skipMap[c.Model] = nil + } + dataModels = append(dataModels, c.Name) } } - // Start with the known configurations - for _, c := range bcl.GetAllBackendConfigs() { - if excludeConfigured { - mm[c.Model] = nil - } + // Then iterate through the loose files if requested. + if looseFilePolicy != SKIP_ALWAYS { - if filterFn(c.Name) { - dataModels = append(dataModels, c.Name) + models, err := ml.ListFilesInModelPath() + if err != nil { + return nil, err } - } - - // Then iterate through the loose files: - for _, m := range models { - // And only adds them if they shouldn't be skipped. - if _, exists := mm[m]; !exists && filterFn(m) { - dataModels = append(dataModels, m) + for _, m := range models { + // And only adds them if they shouldn't be skipped. + if _, exists := skipMap[m]; !exists && filter(m, nil) { + dataModels = append(dataModels, m) + } } } From d5d683707152f7f7a82a17e9668e26a8ff946722 Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Thu, 11 Jul 2024 14:45:47 -0400 Subject: [PATCH 02/29] merge with 2772, since ListModels now has an easy way to include loose files Signed-off-by: Dave Lee --- core/http/endpoints/localai/welcome.go | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/core/http/endpoints/localai/welcome.go b/core/http/endpoints/localai/welcome.go index 39fa7926c265..bbec03116b1b 100644 --- a/core/http/endpoints/localai/welcome.go +++ b/core/http/endpoints/localai/welcome.go @@ -13,7 +13,7 @@ import ( func WelcomeEndpoint(appConfig *config.ApplicationConfig, cl *config.BackendConfigLoader, ml *model.ModelLoader, modelStatus func() (map[string]string, map[string]string)) func(*fiber.Ctx) error { return func(c *fiber.Ctx) error { - models, _ := services.ListModels(cl, ml, config.NoFilterFn, services.SKIP_IF_CONFIGURED) + models, _ := services.ListModels(cl, ml, config.NoFilterFn, services.ALWAYS_INCLUDE) backendConfigs := cl.GetAllBackendConfigs() galleryConfigs := map[string]*gallery.Config{} @@ -29,18 +29,10 @@ func WelcomeEndpoint(appConfig *config.ApplicationConfig, // Get model statuses to display in the UI the operation in progress processingModels, taskTypes := modelStatus() - modelsWithoutConfig := []string{} - - for _, m := range models { - if _, ok := galleryConfigs[m]; !ok { - modelsWithoutConfig = append(modelsWithoutConfig, m) - } - } - summary := fiber.Map{ "Title": "LocalAI API - " + internal.PrintableVersion(), "Version": internal.PrintableVersion(), - "Models": modelsWithoutConfig, + "Models": models, "ModelsConfig": backendConfigs, "GalleryConfig": galleryConfigs, "IsP2PEnabled": p2p.IsP2PEnabled(), From 688cddc5bcc4226f068e54e863fa2abf80922fb0 Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Thu, 11 Jul 2024 15:10:50 -0400 Subject: [PATCH 03/29] fix backwards bool, add documentation to explain why Signed-off-by: Dave Lee --- core/config/backend_config_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/config/backend_config_test.go b/core/config/backend_config_test.go index f89d47cd1d31..a733aa545bd8 100644 --- a/core/config/backend_config_test.go +++ b/core/config/backend_config_test.go @@ -66,7 +66,7 @@ parameters: a := BackendConfig{ Name: "a", } - Expect(a.HasUsecases(FLAG_ANY)).To(BeFalse()) + Expect(a.HasUsecases(FLAG_ANY)).To(BeTrue()) // FLAG_ANY just means the config _exists_ essentially. b := BackendConfig{ Name: "b", From d9443cfdeee34912807e158263e0a54b6ab9f626 Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Thu, 11 Jul 2024 15:57:24 -0400 Subject: [PATCH 04/29] fix another backwards bool Signed-off-by: Dave Lee --- core/config/backend_config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/config/backend_config.go b/core/config/backend_config.go index 45cb6cdf8439..747beea2a6fc 100644 --- a/core/config/backend_config.go +++ b/core/config/backend_config.go @@ -443,7 +443,7 @@ func (c *BackendConfig) HasUsecases(u BackendConfigUsecases) bool { } if (u & FLAG_IMAGE) == FLAG_IMAGE { // This one feels _really_ bad. Need to experiment but it's better than nothing at all? - if (c.Backend != "diffusers" || c.Diffusers.PipelineType != "") && c.Backend != "tinydream" || c.Backend != "stablediffusion" { + if (c.Backend != "diffusers" && c.Diffusers.PipelineType != "") || c.Backend != "tinydream" || c.Backend != "stablediffusion" { return false } } From 41c1ecc1193708c6eb7877733337c59a5d98cee5 Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Thu, 11 Jul 2024 16:00:03 -0400 Subject: [PATCH 05/29] fix another backwards bool Signed-off-by: Dave Lee --- core/config/backend_config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/config/backend_config.go b/core/config/backend_config.go index 747beea2a6fc..4dcaad7a45fe 100644 --- a/core/config/backend_config.go +++ b/core/config/backend_config.go @@ -443,7 +443,7 @@ func (c *BackendConfig) HasUsecases(u BackendConfigUsecases) bool { } if (u & FLAG_IMAGE) == FLAG_IMAGE { // This one feels _really_ bad. Need to experiment but it's better than nothing at all? - if (c.Backend != "diffusers" && c.Diffusers.PipelineType != "") || c.Backend != "tinydream" || c.Backend != "stablediffusion" { + if (c.Backend != "diffusers" && c.Diffusers.PipelineType != "") && (c.Backend != "tinydream") && (c.Backend != "stablediffusion") { return false } } From d2699d250e4050f200b17cd73bebd2b14bfd6df3 Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Thu, 11 Jul 2024 22:45:14 -0400 Subject: [PATCH 06/29] clean Signed-off-by: Dave Lee --- core/config/backend_config.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/config/backend_config.go b/core/config/backend_config.go index 4dcaad7a45fe..8c586c1f9d1f 100644 --- a/core/config/backend_config.go +++ b/core/config/backend_config.go @@ -3,6 +3,7 @@ package config import ( "os" "regexp" + "slices" "strings" "github.com/mudler/LocalAI/core/schema" @@ -442,8 +443,7 @@ func (c *BackendConfig) HasUsecases(u BackendConfigUsecases) bool { } } if (u & FLAG_IMAGE) == FLAG_IMAGE { - // This one feels _really_ bad. Need to experiment but it's better than nothing at all? - if (c.Backend != "diffusers" && c.Diffusers.PipelineType != "") && (c.Backend != "tinydream") && (c.Backend != "stablediffusion") { + if (c.Backend == "diffusers" && c.Diffusers.PipelineType != "") || ((c.Backend != "tinydream") || (c.Backend != "stablediffusion")) { return false } } @@ -458,8 +458,8 @@ func (c *BackendConfig) HasUsecases(u BackendConfigUsecases) bool { } } if (u & FLAG_TTS) == FLAG_TTS { - // This one feels _really_ bad. Need to reach out to TTS experts to find something salient here. - if c.Backend != "transformer-musicgen" && c.Backend != "piper" && c.Backend != "parler-tts" { + ttsBackends := []string{"piper", "transformer-musicgen", "parler-tts"} + if !slices.Contains(ttsBackends, c.Backend) { return false } } From 380282df250d3779db4631e636bce4da33a3c946 Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Thu, 11 Jul 2024 23:27:31 -0400 Subject: [PATCH 07/29] pay attention while coding Signed-off-by: Dave Lee --- core/config/backend_config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/config/backend_config.go b/core/config/backend_config.go index 8c586c1f9d1f..ad2f43d8487b 100644 --- a/core/config/backend_config.go +++ b/core/config/backend_config.go @@ -443,7 +443,7 @@ func (c *BackendConfig) HasUsecases(u BackendConfigUsecases) bool { } } if (u & FLAG_IMAGE) == FLAG_IMAGE { - if (c.Backend == "diffusers" && c.Diffusers.PipelineType != "") || ((c.Backend != "tinydream") || (c.Backend != "stablediffusion")) { + if (c.Backend == "diffusers" && c.Diffusers.PipelineType != "") && ((c.Backend != "tinydream") || (c.Backend != "stablediffusion")) { return false } } From 865aad46a56ed32b7198f57def7692e7e5a30d06 Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Thu, 11 Jul 2024 23:28:18 -0400 Subject: [PATCH 08/29] pay attention while coding Signed-off-by: Dave Lee --- core/config/backend_config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/config/backend_config.go b/core/config/backend_config.go index ad2f43d8487b..ec0f420b85c4 100644 --- a/core/config/backend_config.go +++ b/core/config/backend_config.go @@ -443,7 +443,7 @@ func (c *BackendConfig) HasUsecases(u BackendConfigUsecases) bool { } } if (u & FLAG_IMAGE) == FLAG_IMAGE { - if (c.Backend == "diffusers" && c.Diffusers.PipelineType != "") && ((c.Backend != "tinydream") || (c.Backend != "stablediffusion")) { + if (c.Backend == "diffusers" && c.Diffusers.PipelineType != "") && ((c.Backend != "tinydream") && (c.Backend != "stablediffusion")) { return false } } From fd92e810b8a0221459e922516e8a766dba5ed80e Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Fri, 12 Jul 2024 13:09:10 -0400 Subject: [PATCH 09/29] fix Signed-off-by: Dave Lee --- core/config/backend_config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/config/backend_config.go b/core/config/backend_config.go index ec0f420b85c4..dcc66cce3005 100644 --- a/core/config/backend_config.go +++ b/core/config/backend_config.go @@ -443,7 +443,7 @@ func (c *BackendConfig) HasUsecases(u BackendConfigUsecases) bool { } } if (u & FLAG_IMAGE) == FLAG_IMAGE { - if (c.Backend == "diffusers" && c.Diffusers.PipelineType != "") && ((c.Backend != "tinydream") && (c.Backend != "stablediffusion")) { + if !((c.Backend == "diffusers" && c.Diffusers.PipelineType != "") && ((c.Backend != "tinydream") && (c.Backend != "stablediffusion"))) { return false } } From bbd56bd5d64277a1ce98d59618d56385ab13b207 Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Fri, 12 Jul 2024 13:44:08 -0400 Subject: [PATCH 10/29] fix Signed-off-by: Dave Lee --- core/config/backend_config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/config/backend_config.go b/core/config/backend_config.go index dcc66cce3005..31004317e457 100644 --- a/core/config/backend_config.go +++ b/core/config/backend_config.go @@ -443,7 +443,7 @@ func (c *BackendConfig) HasUsecases(u BackendConfigUsecases) bool { } } if (u & FLAG_IMAGE) == FLAG_IMAGE { - if !((c.Backend == "diffusers" && c.Diffusers.PipelineType != "") && ((c.Backend != "tinydream") && (c.Backend != "stablediffusion"))) { + if !((c.Backend == "diffusers" && c.Diffusers.PipelineType != "") && ((c.Backend != "tinydream") || (c.Backend != "stablediffusion"))) { return false } } From ffe1006ac3094c7b0e6dbd98c58ff336d4c40425 Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Fri, 12 Jul 2024 13:45:50 -0400 Subject: [PATCH 11/29] bad merge Signed-off-by: Dave Lee --- core/config/backend_config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/config/backend_config.go b/core/config/backend_config.go index 31004317e457..e02f9433d2ea 100644 --- a/core/config/backend_config.go +++ b/core/config/backend_config.go @@ -443,7 +443,7 @@ func (c *BackendConfig) HasUsecases(u BackendConfigUsecases) bool { } } if (u & FLAG_IMAGE) == FLAG_IMAGE { - if !((c.Backend == "diffusers" && c.Diffusers.PipelineType != "") && ((c.Backend != "tinydream") || (c.Backend != "stablediffusion"))) { + if !((c.Backend == "diffusers" && c.Diffusers.PipelineType != "") || ((c.Backend != "tinydream") && (c.Backend != "stablediffusion"))) { return false } } From cffb1489d80d65de38cf57e243dbd641c8773a65 Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Fri, 12 Jul 2024 15:56:05 -0400 Subject: [PATCH 12/29] fix Signed-off-by: Dave Lee --- core/config/backend_config.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/core/config/backend_config.go b/core/config/backend_config.go index e02f9433d2ea..8f2bdbedddd0 100644 --- a/core/config/backend_config.go +++ b/core/config/backend_config.go @@ -443,9 +443,15 @@ func (c *BackendConfig) HasUsecases(u BackendConfigUsecases) bool { } } if (u & FLAG_IMAGE) == FLAG_IMAGE { - if !((c.Backend == "diffusers" && c.Diffusers.PipelineType != "") || ((c.Backend != "tinydream") && (c.Backend != "stablediffusion"))) { + imageBackends := []string{"diffusers", "tinydream", "stablediffusion"} + if !slices.Contains(imageBackends, c.Backend) { return false } + + if c.Backend == "diffusers" && c.Diffusers.PipelineType == "" { + return false + } + } if (u & FLAG_RERANK) == FLAG_RERANK { if c.Backend != "rerankers" { From 474a417c013c9259a483e499ba4c89379e422bd9 Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Sun, 28 Jul 2024 13:56:37 -0400 Subject: [PATCH 13/29] fix pointer change Signed-off-by: Dave Lee --- core/config/backend_config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/config/backend_config.go b/core/config/backend_config.go index cc288af68c54..7fe9c28148f0 100644 --- a/core/config/backend_config.go +++ b/core/config/backend_config.go @@ -442,7 +442,7 @@ func (c *BackendConfig) HasUsecases(u BackendConfigUsecases) bool { } } if (u & FLAG_EMBEDDINGS) == FLAG_EMBEDDINGS { - if !c.Embeddings { + if c.Embeddings == nil || !*c.Embeddings { return false } } From 794c35999655bd65dfff2af83b65ed2da5a071af Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Sun, 28 Jul 2024 21:23:40 -0400 Subject: [PATCH 14/29] fix merge error in a test Signed-off-by: Dave Lee --- core/config/backend_config_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/config/backend_config_test.go b/core/config/backend_config_test.go index a733aa545bd8..8e88b4fc8fd6 100644 --- a/core/config/backend_config_test.go +++ b/core/config/backend_config_test.go @@ -101,13 +101,14 @@ parameters: Expect(d.HasUsecases(FLAG_COMPLETION)).To(BeTrue()) Expect(d.HasUsecases(FLAG_CHAT)).To(BeTrue()) + trueValue := true e := BackendConfig{ Name: "e", Backend: "llama-cpp", TemplateConfig: TemplateConfig{ Completion: "completion", }, - Embeddings: true, + Embeddings: &trueValue, } Expect(e.HasUsecases(FLAG_ANY)).To(BeTrue()) From 5c1b603068729d595e94a7fa8d6c35ead0a1e28f Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Thu, 1 Aug 2024 11:13:30 -0400 Subject: [PATCH 15/29] ALWAYS_INCLUDE ==> SKIP_IF_CONFIGURED see #3107 and #3011 Signed-off-by: Dave Lee --- core/http/endpoints/localai/welcome.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/http/endpoints/localai/welcome.go b/core/http/endpoints/localai/welcome.go index bbec03116b1b..98a9b14e658e 100644 --- a/core/http/endpoints/localai/welcome.go +++ b/core/http/endpoints/localai/welcome.go @@ -13,7 +13,7 @@ import ( func WelcomeEndpoint(appConfig *config.ApplicationConfig, cl *config.BackendConfigLoader, ml *model.ModelLoader, modelStatus func() (map[string]string, map[string]string)) func(*fiber.Ctx) error { return func(c *fiber.Ctx) error { - models, _ := services.ListModels(cl, ml, config.NoFilterFn, services.ALWAYS_INCLUDE) + models, _ := services.ListModels(cl, ml, config.NoFilterFn, services.SKIP_IF_CONFIGURED) backendConfigs := cl.GetAllBackendConfigs() galleryConfigs := map[string]*gallery.Config{} From a16790a9864922cd3c5b11235619620b140964d8 Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Sat, 24 Aug 2024 13:08:22 -0400 Subject: [PATCH 16/29] update for SoundGeneration Signed-off-by: Dave Lee --- core/config/backend_config.go | 29 ++++++++++++++++++++--------- core/config/backend_config_test.go | 9 +++++++++ 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/core/config/backend_config.go b/core/config/backend_config.go index 5b2ed3442dd5..bfd17d1351be 100644 --- a/core/config/backend_config.go +++ b/core/config/backend_config.go @@ -414,20 +414,25 @@ func (c *BackendConfig) HasTemplate() bool { type BackendConfigUsecases int const ( - FLAG_ANY BackendConfigUsecases = 0b00000000 - FLAG_CHAT BackendConfigUsecases = 0b00000001 - FLAG_COMPLETION BackendConfigUsecases = 0b00000010 - FLAG_EDIT BackendConfigUsecases = 0b00000100 - FLAG_EMBEDDINGS BackendConfigUsecases = 0b00001000 - FLAG_RERANK BackendConfigUsecases = 0b00010000 - FLAG_IMAGE BackendConfigUsecases = 0b00100000 - FLAG_TRANSCRIPT BackendConfigUsecases = 0b01000000 - FLAG_TTS BackendConfigUsecases = 0b10000000 + FLAG_ANY BackendConfigUsecases = 0b000000000 + FLAG_CHAT BackendConfigUsecases = 0b000000001 + FLAG_COMPLETION BackendConfigUsecases = 0b000000010 + FLAG_EDIT BackendConfigUsecases = 0b000000100 + FLAG_EMBEDDINGS BackendConfigUsecases = 0b000001000 + FLAG_RERANK BackendConfigUsecases = 0b000010000 + FLAG_IMAGE BackendConfigUsecases = 0b000100000 + FLAG_TRANSCRIPT BackendConfigUsecases = 0b001000000 + FLAG_TTS BackendConfigUsecases = 0b010000000 + FLAG_SOUND_GENERATION BackendConfigUsecases = 0b100000000 // Common Subsets FLAG_LLM BackendConfigUsecases = FLAG_CHAT & FLAG_COMPLETION & FLAG_EDIT ) +// HasUsecases examines a BackendConfig and determines which endpoints have a chance of success. +// This is a **heuristic based** function, as the backend in question may not be loaded yet. +// In the future, we may wish to consider storing this information in the config directly (which could get out of date) +// or alternatively interrogating the backends (not always loaded) or loading the "implemented services" earlier with external backend registry? func (c *BackendConfig) HasUsecases(u BackendConfigUsecases) bool { if (u & FLAG_CHAT) == FLAG_CHAT { if c.TemplateConfig.Chat == "" && c.TemplateConfig.ChatMessage == "" { @@ -477,5 +482,11 @@ func (c *BackendConfig) HasUsecases(u BackendConfigUsecases) bool { } } + if (u & FLAG_SOUND_GENERATION) == FLAG_SOUND_GENERATION { + if c.Backend != "transformer-musicgen" { + return false + } + } + return true } diff --git a/core/config/backend_config_test.go b/core/config/backend_config_test.go index 8e88b4fc8fd6..38e76e4ca102 100644 --- a/core/config/backend_config_test.go +++ b/core/config/backend_config_test.go @@ -133,5 +133,14 @@ parameters: Expect(g.HasUsecases(FLAG_TRANSCRIPT)).To(BeTrue()) Expect(g.HasUsecases(FLAG_TTS)).To(BeFalse()) + h := BackendConfig{ + Name: "h", + Backend: "transformers-musicgen", + } + Expect(h.HasUsecases(FLAG_ANY)).To(BeTrue()) + Expect(h.HasUsecases(FLAG_TRANSCRIPT)).To(BeFalse()) + Expect(h.HasUsecases(FLAG_TTS)).To(BeTrue()) + Expect(h.HasUsecases(FLAG_SOUND_GENERATION)).To(BeTrue()) + }) }) From 7cb71c71be8152ac30a9f495052118e15b24e885 Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Sat, 24 Aug 2024 13:12:30 -0400 Subject: [PATCH 17/29] missed file Signed-off-by: Dave Lee --- core/config/backend_config.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/config/backend_config.go b/core/config/backend_config.go index bfd17d1351be..4f194b994b49 100644 --- a/core/config/backend_config.go +++ b/core/config/backend_config.go @@ -433,6 +433,9 @@ const ( // This is a **heuristic based** function, as the backend in question may not be loaded yet. // In the future, we may wish to consider storing this information in the config directly (which could get out of date) // or alternatively interrogating the backends (not always loaded) or loading the "implemented services" earlier with external backend registry? +// +// In its current state, this function should ideally check for properties of the config like templates, rather than the direct backend name checks for the lower half. +// This avoids the maintenance burden of updating this list for each new backend - but unfortunately, that's the best option for some services currently. func (c *BackendConfig) HasUsecases(u BackendConfigUsecases) bool { if (u & FLAG_CHAT) == FLAG_CHAT { if c.TemplateConfig.Chat == "" && c.TemplateConfig.ChatMessage == "" { From 66f78261c1361f551cde7567fe88d60897aaec92 Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Sun, 25 Aug 2024 01:56:08 -0400 Subject: [PATCH 18/29] re-add fix lost in DCO force push Signed-off-by: Dave Lee --- core/config/backend_config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/config/backend_config.go b/core/config/backend_config.go index 4f194b994b49..e35d02e607ea 100644 --- a/core/config/backend_config.go +++ b/core/config/backend_config.go @@ -479,14 +479,14 @@ func (c *BackendConfig) HasUsecases(u BackendConfigUsecases) bool { } } if (u & FLAG_TTS) == FLAG_TTS { - ttsBackends := []string{"piper", "transformer-musicgen", "parler-tts"} + ttsBackends := []string{"piper", "transformers-musicgen", "parler-tts"} if !slices.Contains(ttsBackends, c.Backend) { return false } } if (u & FLAG_SOUND_GENERATION) == FLAG_SOUND_GENERATION { - if c.Backend != "transformer-musicgen" { + if c.Backend != "transformers-musicgen" { return false } } From 9cb2d0ca927e9bbeed68f30f949acd9591a78d6b Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Fri, 6 Sep 2024 13:33:21 -0400 Subject: [PATCH 19/29] add cli command Signed-off-by: Dave Lee --- core/cli/util.go | 38 +++++++++++++++++++++++++++++++++-- core/config/backend_config.go | 16 +++++++++++++++ 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/core/cli/util.go b/core/cli/util.go index b3e545d869e3..57b8ad9efeaa 100644 --- a/core/cli/util.go +++ b/core/cli/util.go @@ -15,8 +15,9 @@ import ( ) type UtilCMD struct { - GGUFInfo GGUFInfoCMD `cmd:"" name:"gguf-info" help:"Get information about a GGUF file"` - HFScan HFScanCMD `cmd:"" name:"hf-scan" help:"Checks installed models for known security issues. WARNING: this is a best-effort feature and may not catch everything!"` + GGUFInfo GGUFInfoCMD `cmd:"" name:"gguf-info" help:"Get information about a GGUF file"` + HFScan HFScanCMD `cmd:"" name:"hf-scan" help:"Checks installed models for known security issues. WARNING: this is a best-effort feature and may not catch everything!"` + UsecaseHeuristic UsecaseHeuristicCMD `cmd:"" name:"usecase-heuristic" help:"Checks a specific model config and prints what usecase LocalAI will offer for it."` } type GGUFInfoCMD struct { @@ -30,6 +31,11 @@ type HFScanCMD struct { ToScan []string `arg:""` } +type UsecaseHeuristicCMD struct { + ConfigName string `name:"The config file to check"` + ModelsPath string `env:"LOCALAI_MODELS_PATH,MODELS_PATH" type:"path" default:"${basepath}/models" help:"Path containing models used for inferencing" group:"storage"` +} + func (u *GGUFInfoCMD) Run(ctx *cliContext.Context) error { if u.Args == nil || len(u.Args) == 0 { return fmt.Errorf("no GGUF file provided") @@ -99,3 +105,31 @@ func (hfscmd *HFScanCMD) Run(ctx *cliContext.Context) error { return nil } } + +func (uhcmd *UsecaseHeuristicCMD) Run(ctx *cliContext.Context) error { + if len(uhcmd.ConfigName) == 0 { + log.Error().Msg("ConfigName is a required parameter") + return fmt.Errorf("config name is a required parameter") + } + if len(uhcmd.ModelsPath) == 0 { + log.Error().Msg("ModelsPath is a required parameter") + return fmt.Errorf("model path is a required parameter") + } + bcl := config.NewBackendConfigLoader(uhcmd.ModelsPath) + err := bcl.LoadBackendConfig(uhcmd.ConfigName) + if err != nil { + log.Error().Err(err).Str("ConfigName", uhcmd.ConfigName).Msg("error while loading backend") + return err + } + bc, exists := bcl.GetBackendConfig(uhcmd.ConfigName) + if !exists { + log.Error().Str("ConfigName", uhcmd.ConfigName).Msg("ConfigName not found") + } + for name, uc := range config.GetAllBackendConfigUsecases() { + if bc.HasUsecases(uc) { + log.Info().Str("Usecase", name) + } + } + log.Info().Msg("---") + return nil +} diff --git a/core/config/backend_config.go b/core/config/backend_config.go index e35d02e607ea..1e6eeeb8b3b4 100644 --- a/core/config/backend_config.go +++ b/core/config/backend_config.go @@ -429,6 +429,22 @@ const ( FLAG_LLM BackendConfigUsecases = FLAG_CHAT & FLAG_COMPLETION & FLAG_EDIT ) +func GetAllBackendConfigUsecases() map[string]BackendConfigUsecases { + return map[string]BackendConfigUsecases{ + "FLAG_ANY": FLAG_ANY, + "FLAG_CHAT": FLAG_CHAT, + "FLAG_COMPLETION": FLAG_COMPLETION, + "FLAG_EDIT": FLAG_EDIT, + "FLAG_EMBEDDINGS": FLAG_EMBEDDINGS, + "FLAG_RERANK": FLAG_RERANK, + "FLAG_IMAGE": FLAG_IMAGE, + "FLAG_TRANSCRIPT": FLAG_TRANSCRIPT, + "FLAG_TTS": FLAG_TTS, + "FLAG_SOUND_GENERATION": FLAG_SOUND_GENERATION, + "FLAG_LLM": FLAG_LLM, + } +} + // HasUsecases examines a BackendConfig and determines which endpoints have a chance of success. // This is a **heuristic based** function, as the backend in question may not be loaded yet. // In the future, we may wish to consider storing this information in the config directly (which could get out of date) From 67d6bf2179bbfdea681631d2a856142a2fe42d7e Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Thu, 26 Sep 2024 00:33:19 -0400 Subject: [PATCH 20/29] experimental workflow fix to unbreak Signed-off-by: Dave Lee --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2af3fd002ec7..52ba18249ec9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -185,7 +185,7 @@ jobs: - name: Test run: | LOCALAI_MODELS_DIR=$PWD/models LOCALAI_IMAGE_TAG=test LOCALAI_IMAGE=local-ai-aio \ - make run-e2e-aio + make prepare run-e2e-aio - name: Setup tmate session if tests fail if: ${{ failure() }} uses: mxschmitt/action-tmate@v3.18 From d68dcca129ea2dc3ddd0c47f4cade13074c5fd6d Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Thu, 26 Sep 2024 04:06:50 -0400 Subject: [PATCH 21/29] revert Signed-off-by: Dave Lee --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 52ba18249ec9..2af3fd002ec7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -185,7 +185,7 @@ jobs: - name: Test run: | LOCALAI_MODELS_DIR=$PWD/models LOCALAI_IMAGE_TAG=test LOCALAI_IMAGE=local-ai-aio \ - make prepare run-e2e-aio + make run-e2e-aio - name: Setup tmate session if tests fail if: ${{ failure() }} uses: mxschmitt/action-tmate@v3.18 From 0e77ac2407bbc12c6fa4e7afd730e0783170bd79 Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Fri, 27 Sep 2024 18:56:20 -0400 Subject: [PATCH 22/29] allow manually adding usecases Signed-off-by: Dave Lee --- core/config/backend_config.go | 54 +++++++++++++++++++++++------- core/config/backend_config_test.go | 11 ++++++ 2 files changed, 53 insertions(+), 12 deletions(-) diff --git a/core/config/backend_config.go b/core/config/backend_config.go index b10add192472..5bbdd3e7d31b 100644 --- a/core/config/backend_config.go +++ b/core/config/backend_config.go @@ -9,6 +9,7 @@ import ( "github.com/mudler/LocalAI/core/schema" "github.com/mudler/LocalAI/pkg/downloader" "github.com/mudler/LocalAI/pkg/functions" + "gopkg.in/yaml.v3" ) const ( @@ -28,13 +29,15 @@ type BackendConfig struct { schema.PredictionOptions `yaml:"parameters"` Name string `yaml:"name"` - F16 *bool `yaml:"f16"` - Threads *int `yaml:"threads"` - Debug *bool `yaml:"debug"` - Roles map[string]string `yaml:"roles"` - Embeddings *bool `yaml:"embeddings"` - Backend string `yaml:"backend"` - TemplateConfig TemplateConfig `yaml:"template"` + F16 *bool `yaml:"f16"` + Threads *int `yaml:"threads"` + Debug *bool `yaml:"debug"` + Roles map[string]string `yaml:"roles"` + Embeddings *bool `yaml:"embeddings"` + Backend string `yaml:"backend"` + TemplateConfig TemplateConfig `yaml:"template"` + KnownUsecaseStrings []string `yaml:"known_usecases"` + KnownUsecases BackendConfigUsecases `yaml:"-"` PromptStrings, InputStrings []string `yaml:"-"` InputToken [][]int `yaml:"-"` @@ -195,6 +198,17 @@ type TemplateConfig struct { JoinChatMessagesByCharacter *string `yaml:"join_chat_messages_by_character"` } +func (c *BackendConfig) UnmarshalYAML(value *yaml.Node) error { + type BCAlias BackendConfig + var aux BCAlias + if err := value.Decode(&aux); err != nil { + return err + } + *c = BackendConfig(aux) + c.KnownUsecases = GetUsecasesFromYAML(c.KnownUsecaseStrings) + return nil +} + func (c *BackendConfig) SetFunctionCallString(s string) { c.functionCallString = s } @@ -446,14 +460,30 @@ func GetAllBackendConfigUsecases() map[string]BackendConfigUsecases { } } +func GetUsecasesFromYAML(input []string) BackendConfigUsecases { + result := FLAG_ANY + flags := GetAllBackendConfigUsecases() + for _, str := range input { + flag, exists := flags["FLAG_"+strings.ToUpper(str)] + if exists { + result &= flag + } + } + return result +} + // HasUsecases examines a BackendConfig and determines which endpoints have a chance of success. -// This is a **heuristic based** function, as the backend in question may not be loaded yet. -// In the future, we may wish to consider storing this information in the config directly (which could get out of date) -// or alternatively interrogating the backends (not always loaded) or loading the "implemented services" earlier with external backend registry? -// +func (c *BackendConfig) HasUsecases(u BackendConfigUsecases) bool { + if (u & c.KnownUsecases) == c.KnownUsecases { + return true + } + return c.GuessUsecases(u) +} + +// GuessUsecases is a **heuristic based** function, as the backend in question may not be loaded yet, and the config may not record what it's useful at. // In its current state, this function should ideally check for properties of the config like templates, rather than the direct backend name checks for the lower half. // This avoids the maintenance burden of updating this list for each new backend - but unfortunately, that's the best option for some services currently. -func (c *BackendConfig) HasUsecases(u BackendConfigUsecases) bool { +func (c *BackendConfig) GuessUsecases(u BackendConfigUsecases) bool { if (u & FLAG_CHAT) == FLAG_CHAT { if c.TemplateConfig.Chat == "" && c.TemplateConfig.ChatMessage == "" { return false diff --git a/core/config/backend_config_test.go b/core/config/backend_config_test.go index 38e76e4ca102..26c5133382e6 100644 --- a/core/config/backend_config_test.go +++ b/core/config/backend_config_test.go @@ -142,5 +142,16 @@ parameters: Expect(h.HasUsecases(FLAG_TTS)).To(BeTrue()) Expect(h.HasUsecases(FLAG_SOUND_GENERATION)).To(BeTrue()) + i := BackendConfig{ + Name: "i", + Backend: "whisper", + KnownUsecaseStrings: []string{"chat", "COMPLETION"}, + } + Expect(i.HasUsecases(FLAG_ANY)).To(BeTrue()) + Expect(i.HasUsecases(FLAG_TRANSCRIPT)).To(BeTrue()) + Expect(i.HasUsecases(FLAG_TTS)).To(BeFalse()) + Expect(e.HasUsecases(FLAG_COMPLETION)).To(BeTrue()) + Expect(e.HasUsecases(FLAG_CHAT)).To(BeTrue()) + }) }) From 258c9bdb5f75d8e13c8a9295f47e1755c10b8951 Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Fri, 27 Sep 2024 19:55:52 -0400 Subject: [PATCH 23/29] fix Signed-off-by: Dave Lee --- core/config/backend_config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/config/backend_config.go b/core/config/backend_config.go index 5bbdd3e7d31b..6c4de82a61a4 100644 --- a/core/config/backend_config.go +++ b/core/config/backend_config.go @@ -466,7 +466,7 @@ func GetUsecasesFromYAML(input []string) BackendConfigUsecases { for _, str := range input { flag, exists := flags["FLAG_"+strings.ToUpper(str)] if exists { - result &= flag + result |= flag } } return result From 204d6a174d4482fdbcd09cb8f1f5fd9a89963e74 Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Fri, 27 Sep 2024 20:50:10 -0400 Subject: [PATCH 24/29] fix Signed-off-by: Dave Lee --- core/config/backend_config.go | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/core/config/backend_config.go b/core/config/backend_config.go index 6c4de82a61a4..096becdba114 100644 --- a/core/config/backend_config.go +++ b/core/config/backend_config.go @@ -29,15 +29,15 @@ type BackendConfig struct { schema.PredictionOptions `yaml:"parameters"` Name string `yaml:"name"` - F16 *bool `yaml:"f16"` - Threads *int `yaml:"threads"` - Debug *bool `yaml:"debug"` - Roles map[string]string `yaml:"roles"` - Embeddings *bool `yaml:"embeddings"` - Backend string `yaml:"backend"` - TemplateConfig TemplateConfig `yaml:"template"` - KnownUsecaseStrings []string `yaml:"known_usecases"` - KnownUsecases BackendConfigUsecases `yaml:"-"` + F16 *bool `yaml:"f16"` + Threads *int `yaml:"threads"` + Debug *bool `yaml:"debug"` + Roles map[string]string `yaml:"roles"` + Embeddings *bool `yaml:"embeddings"` + Backend string `yaml:"backend"` + TemplateConfig TemplateConfig `yaml:"template"` + KnownUsecaseStrings []string `yaml:"known_usecases"` + KnownUsecases *BackendConfigUsecases `yaml:"-"` PromptStrings, InputStrings []string `yaml:"-"` InputToken [][]int `yaml:"-"` @@ -460,7 +460,10 @@ func GetAllBackendConfigUsecases() map[string]BackendConfigUsecases { } } -func GetUsecasesFromYAML(input []string) BackendConfigUsecases { +func GetUsecasesFromYAML(input []string) *BackendConfigUsecases { + if len(input) == 0 { + return nil + } result := FLAG_ANY flags := GetAllBackendConfigUsecases() for _, str := range input { @@ -469,12 +472,12 @@ func GetUsecasesFromYAML(input []string) BackendConfigUsecases { result |= flag } } - return result + return &result } // HasUsecases examines a BackendConfig and determines which endpoints have a chance of success. func (c *BackendConfig) HasUsecases(u BackendConfigUsecases) bool { - if (u & c.KnownUsecases) == c.KnownUsecases { + if (c.KnownUsecases != nil) && ((u & *c.KnownUsecases) == *c.KnownUsecases) { return true } return c.GuessUsecases(u) From 0b43fad8da4e282760a3d3232c7ad6e7d7d634b2 Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Sat, 28 Sep 2024 00:18:29 -0400 Subject: [PATCH 25/29] fix Signed-off-by: Dave Lee --- core/config/backend_config_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/config/backend_config_test.go b/core/config/backend_config_test.go index 26c5133382e6..6347828b883a 100644 --- a/core/config/backend_config_test.go +++ b/core/config/backend_config_test.go @@ -150,8 +150,8 @@ parameters: Expect(i.HasUsecases(FLAG_ANY)).To(BeTrue()) Expect(i.HasUsecases(FLAG_TRANSCRIPT)).To(BeTrue()) Expect(i.HasUsecases(FLAG_TTS)).To(BeFalse()) - Expect(e.HasUsecases(FLAG_COMPLETION)).To(BeTrue()) - Expect(e.HasUsecases(FLAG_CHAT)).To(BeTrue()) + Expect(i.HasUsecases(FLAG_COMPLETION)).To(BeTrue()) + Expect(i.HasUsecases(FLAG_CHAT)).To(BeTrue()) }) }) From f68d54ec10507f4a9887a14805ef726693d1a0eb Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Sat, 28 Sep 2024 21:38:09 -0400 Subject: [PATCH 26/29] test change Signed-off-by: Dave Lee --- core/config/backend_config_test.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/core/config/backend_config_test.go b/core/config/backend_config_test.go index 6347828b883a..8531b9bacae7 100644 --- a/core/config/backend_config_test.go +++ b/core/config/backend_config_test.go @@ -19,12 +19,17 @@ var _ = Describe("Test cases for config related functions", func() { `backend: "../foo-bar" name: "foo" parameters: - model: "foo-bar"`) + model: "foo-bar" +known_usecases: +- chat +- COMPLETION +`) Expect(err).ToNot(HaveOccurred()) config, err := readBackendConfigFromFile(tmp.Name()) Expect(err).To(BeNil()) Expect(config).ToNot(BeNil()) Expect(config.Validate()).To(BeFalse()) + Expect(config.KnownUsecases).ToNot(BeNil()) }) It("Test Validate", func() { tmp, err := os.CreateTemp("", "config.yaml") From 7589128fafbf6b4034d78885c7de44244b11d1b6 Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Sat, 28 Sep 2024 22:00:43 -0400 Subject: [PATCH 27/29] test fix Signed-off-by: Dave Lee --- core/config/backend_config_test.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/core/config/backend_config_test.go b/core/config/backend_config_test.go index 8531b9bacae7..317ff73a70d6 100644 --- a/core/config/backend_config_test.go +++ b/core/config/backend_config_test.go @@ -147,10 +147,12 @@ parameters: Expect(h.HasUsecases(FLAG_TTS)).To(BeTrue()) Expect(h.HasUsecases(FLAG_SOUND_GENERATION)).To(BeTrue()) + knownUsecases := FLAG_CHAT & FLAG_COMPLETION i := BackendConfig{ - Name: "i", - Backend: "whisper", - KnownUsecaseStrings: []string{"chat", "COMPLETION"}, + Name: "i", + Backend: "whisper", + // Earlier test checks parsing, this just needs to set final values + KnownUsecases: &knownUsecases, } Expect(i.HasUsecases(FLAG_ANY)).To(BeTrue()) Expect(i.HasUsecases(FLAG_TRANSCRIPT)).To(BeTrue()) From 45c809c90709c5c0d7918480bb681282f16fc905 Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Sat, 28 Sep 2024 23:17:18 -0400 Subject: [PATCH 28/29] oops Signed-off-by: Dave Lee --- core/config/backend_config_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/config/backend_config_test.go b/core/config/backend_config_test.go index 317ff73a70d6..04eacb7ec627 100644 --- a/core/config/backend_config_test.go +++ b/core/config/backend_config_test.go @@ -147,7 +147,7 @@ parameters: Expect(h.HasUsecases(FLAG_TTS)).To(BeTrue()) Expect(h.HasUsecases(FLAG_SOUND_GENERATION)).To(BeTrue()) - knownUsecases := FLAG_CHAT & FLAG_COMPLETION + knownUsecases := FLAG_CHAT | FLAG_COMPLETION i := BackendConfig{ Name: "i", Backend: "whisper", From d4224e35b1dd133ab4eb2b40a73db718b91afba9 Mon Sep 17 00:00:00 2001 From: Dave Lee Date: Sun, 29 Sep 2024 02:31:01 -0400 Subject: [PATCH 29/29] oops Signed-off-by: Dave Lee --- core/config/backend_config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/config/backend_config.go b/core/config/backend_config.go index 096becdba114..8db94f7cded0 100644 --- a/core/config/backend_config.go +++ b/core/config/backend_config.go @@ -477,7 +477,7 @@ func GetUsecasesFromYAML(input []string) *BackendConfigUsecases { // HasUsecases examines a BackendConfig and determines which endpoints have a chance of success. func (c *BackendConfig) HasUsecases(u BackendConfigUsecases) bool { - if (c.KnownUsecases != nil) && ((u & *c.KnownUsecases) == *c.KnownUsecases) { + if (c.KnownUsecases != nil) && ((u & *c.KnownUsecases) == u) { return true } return c.GuessUsecases(u)