feat: more provider status (#48)

The CR status field will now report the provider's health and
configuration.


Approved-by: VaishnaviHire

Approved-by: mfleader

Author: leseb

.github/workflows/run-e2e-test.yml

@@ -86,3 +86,21 @@ jobs:
       - name: Run e2e tests
         run: |
           make test-e2e
+
+      - name: Get logs
+        if: ${{ always() }}
+        run: |
+          kubectl -n llama-stack-test get all -o yaml > all.log
+          kubectl -n llama-stack-k8s-operator-system logs deployment.apps/llama-stack-k8s-operator-controller-manager > controller-manager.log
+          kubectl -n llama-stack-test describe all > all-describe.log
+          kubectl -n llama-stack-test describe events > events.log
+          kubectl get llamastackdistributions --all-namespaces -o yaml > llamastackdistributions.log
+
+      - name: Upload all logs to artifacts
+        if: ${{ always() }}
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: logs-${{ github.run_id }}-${{ github.run_attempt }}
+          path: |
+            *.log
+          retention-days: 1

Makefile

@@ -301,7 +301,7 @@ api-docs: crd-ref-docs ## Creates API docs using https://github.com/elastic/crd-
 	mkdir -p docs
 	$(CRD_REF_DOCS) --source-path ./ --output-path $(API_DOCS_PATH) --renderer markdown --config ./crd-ref-docs.config.yaml
 	@# Combined command to remove .io links, ensure a trailing newline, and collapse multiple blank lines.
-	@sed -i.bak -e '/\.io\/[^v][^1].*)/d' -e '/^$$/N;/^\n$$/D' $(API_DOCS_PATH)
+	@sed -i.bak -e  '/^$$/N;/^\n$$/D' $(API_DOCS_PATH)
 	@# BSD sed doesn't generate trailing newlines, so no need to remove them.
 	@# BSD sed does not have a '--version' flag, so we need to check for it.
 	@if sed --version >/dev/null 2>&1; then \

api/v1alpha1/llamastackdistribution_types.go

@@ -2008,12 +2008,28 @@ spec:
                       properties:
                         api:
                           type: string
+                        config:
+                          x-kubernetes-preserve-unknown-fields: true
+                        health:
+                          description: HealthStatus represents the health status of
+                            a provider
+                          properties:
+                            message:
+                              type: string
+                            status:
+                              type: string
+                          required:
+                          - message
+                          - status
+                          type: object
                         provider_id:
                           type: string
                         provider_type:
                           type: string
                       required:
                       - api
+                      - config
+                      - health
                       - provider_id
                       - provider_type
                       type: object

api/v1alpha1/zz_generated.deepcopy.go

@@ -22,6 +22,8 @@ _Appears in:_
 | --- | --- | --- | --- |
 | `name` _string_ |  | llama-stack |  |
 | `port` _integer_ |  |  |  |
+| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#resourcerequirements-v1-core)_ |  |  |  |
+| `env` _[EnvVar](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#envvar-v1-core) array_ |  |  |  |
 
 #### DistributionConfig
 
@@ -59,6 +61,7 @@ _Appears in:_
 | `kind` _string_ | `LlamaStackDistribution` | | |
 | `kind` _string_ | Kind is a string value representing the REST resource this object represents.<br />Servers may infer this from the endpoint the client submits requests to.<br />Cannot be updated.<br />In CamelCase.<br />More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds |  |  |
 | `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.<br />Servers should convert recognized schemas to the latest internal value, and<br />may reject unrecognized values.<br />More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources |  |  |
+| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. |  |  |
 | `spec` _[LlamaStackDistributionSpec](#llamastackdistributionspec)_ |  |  |  |
 | `status` _[LlamaStackDistributionStatus](#llamastackdistributionstatus)_ |  |  |  |
 
@@ -72,6 +75,7 @@ LlamaStackDistributionList contains a list of LlamaStackDistribution.
 | `kind` _string_ | `LlamaStackDistributionList` | | |
 | `kind` _string_ | Kind is a string value representing the REST resource this object represents.<br />Servers may infer this from the endpoint the client submits requests to.<br />Cannot be updated.<br />In CamelCase.<br />More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds |  |  |
 | `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.<br />Servers should convert recognized schemas to the latest internal value, and<br />may reject unrecognized values.<br />More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources |  |  |
+| `metadata` _[ListMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#listmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. |  |  |
 | `items` _[LlamaStackDistribution](#llamastackdistribution) array_ |  |  |  |
 
 #### LlamaStackDistributionSpec
@@ -108,6 +112,20 @@ _Appears in:_
 
 | Field | Description | Default | Validation |
 | --- | --- | --- | --- |
+| `volumes` _[Volume](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#volume-v1-core) array_ |  |  |  |
+| `volumeMounts` _[VolumeMount](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#volumemount-v1-core) array_ |  |  |  |
+
+#### ProviderHealthStatus
+
+HealthStatus represents the health status of a provider
+
+_Appears in:_
+- [ProviderInfo](#providerinfo)
+
+| Field | Description | Default | Validation |
+| --- | --- | --- | --- |
+| `status` _string_ |  |  |  |
+| `message` _string_ |  |  |  |
 
 #### ProviderInfo
 
@@ -121,6 +139,8 @@ _Appears in:_
 | `api` _string_ |  |  |  |
 | `provider_id` _string_ |  |  |  |
 | `provider_type` _string_ |  |  |  |
+| `config` _[JSON](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#json-v1-apiextensions-k8s-io)_ |  |  |  |
+| `health` _[ProviderHealthStatus](#providerhealthstatus)_ |  |  |  |
 
 #### ServerSpec
 
@@ -145,4 +165,5 @@ _Appears in:_
 
 | Field | Description | Default | Validation |
 | --- | --- | --- | --- |
+| `size` _[Quantity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#quantity-resource-api)_ | Size is the size of the persistent volume claim created for holding persistent data of the llama-stack server |  |  |
 | `mountPath` _string_ | MountPath is the path where the storage will be mounted in the container |  |  |

config/crd/bases/llamastack.io_llamastackdistributions.yaml

@@ -2017,12 +2017,28 @@ spec:
                       properties:
                         api:
                           type: string
+                        config:
+                          x-kubernetes-preserve-unknown-fields: true
+                        health:
+                          description: HealthStatus represents the health status of
+                            a provider
+                          properties:
+                            message:
+                              type: string
+                            status:
+                              type: string
+                          required:
+                          - message
+                          - status
+                          type: object
                         provider_id:
                           type: string
                         provider_type:
                           type: string
                       required:
                       - api
+                      - config
+                      - health
                       - provider_id
                       - provider_type
                       type: object

docs/api-overview.md

@@ -3,11 +3,13 @@ package e2e
 
 import (
 	"context"
+	"os"
 	"testing"
 	"time"
 
 	"github.com/llamastack/llama-stack-k8s-operator/api/v1alpha1"
 	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v2"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
@@ -202,9 +204,22 @@ func testDistributionStatus(t *testing.T, llsdistributionCR *v1alpha1.LlamaStack
 			return false, nil
 		}
 
+		// Verify that providers have config and health info
+		if len(updatedDistribution.Status.DistributionConfig.Providers) == 0 {
+			return false, nil
+		}
+
 		return true, nil
 	})
-	require.NoError(t, err, "Failed to wait for distribution status update")
+	if err != nil {
+		// Get the final state to print on error
+		finalDistribution := &v1alpha1.LlamaStackDistribution{}
+		TestEnv.Client.Get(TestEnv.Ctx, client.ObjectKey{
+			Namespace: llsdistributionCR.Namespace,
+			Name:      llsdistributionCR.Name,
+		}, finalDistribution)
+		require.NoError(t, err, "Failed to wait for distribution status update", finalDistribution.Status)
+	}
 
 	// Get final state and verify
 	updatedDistribution := &v1alpha1.LlamaStackDistribution{}
@@ -220,6 +235,39 @@ func testDistributionStatus(t *testing.T, llsdistributionCR *v1alpha1.LlamaStack
 	require.Equal(t, updatedDistribution.Spec.Server.Distribution.Name,
 		updatedDistribution.Status.DistributionConfig.ActiveDistribution,
 		"Active distribution should match the spec")
+
+	// Verify provider config and health
+	require.NotEmpty(t, updatedDistribution.Status.DistributionConfig.Providers,
+		"Providers should be populated")
+
+	// Verify that each provider has config and health info
+	for _, provider := range updatedDistribution.Status.DistributionConfig.Providers {
+		require.NotEmpty(t, provider.API, "Provider should have API info")
+		require.NotEmpty(t, provider.ProviderID, "Provider should have ProviderID info")
+		require.NotEmpty(t, provider.ProviderType, "Provider should have ProviderType info")
+		require.NotNil(t, provider.Config, "Provider should have config info")
+		// If Ollama test it returns OK status
+		if provider.ProviderID == "ollama" {
+			require.Equal(t, "OK", provider.Health.Status, "Provider should have OK health status")
+		}
+		// Check that status is one of the allowed values
+		require.Contains(t, []string{"OK", "Error", "Not Implemented"}, provider.Health.Status, "Provider health status should be one of: OK, Error, Not Implemented")
+		// There is no message for OK status
+		if provider.Health.Status != "OK" {
+			require.NotEmpty(t, provider.Health.Message, "Provider should have health message")
+		}
+		require.NotEmpty(t, provider.Config, "Provider config should not be empty")
+	}
+
+	// Write the final distribution status to a file for CI to collect
+	yaml, err := yaml.Marshal(updatedDistribution)
+	if err != nil {
+		t.Fatalf("Failed to marshal distribution: %v", err)
+	}
+	// Weak - do this better to write to a temp file and then move it to the right place at the
+	// repo's root so the CI agent can collect it
+	err = os.WriteFile("../../distribution.log", yaml, 0644)
+	require.NoError(t, err)
 }
 
 func testPVCConfiguration(t *testing.T, distribution *v1alpha1.LlamaStackDistribution) {

release/operator.yaml

@@ -10,9 +10,19 @@ func TestE2E(t *testing.T) {
 	// Run validation tests
 	t.Run("validation", TestValidationSuite)
 
-	// // Run creation tests
-	t.Run("creation", TestCreationSuite)
+	// Track if creation tests passed
+	creationFailed := false
 
-	// Run deletion tests if not skipped
-	t.Run("deletion", TestDeletionSuite)
+	// Run creation tests
+	t.Run("creation", func(t *testing.T) {
+		TestCreationSuite(t)
+		creationFailed = t.Failed()
+	})
+
+	// Run deletion tests only if creation passed
+	if !creationFailed {
+		t.Run("deletion", TestDeletionSuite)
+	} else {
+		t.Log("Skipping deletion tests due to creation test failures")
+	}
 }