Add validation testing framework with testing.Short() guards

.github/workflows/ci.yml

@@ -34,6 +34,9 @@ jobs:
     - name: Run checks (lint, vet, fmt-check, test)
       run: make check
 
+    # Note: Validation tests with real cloud providers run in separate workflows
+    # See .github/workflows/validation-*.yml for provider-specific validation tests
+
     - name: Run security scan
       run: make security
       continue-on-error: true

.github/workflows/validation-lambdalabs.yml

@@ -0,0 +1,58 @@
+name: LambdaLabs Validation Tests
+
+on:
+  schedule:
+    # Run daily at 2 AM UTC
+    - cron: '0 2 * * *'
+  workflow_dispatch:
+    # Allow manual triggering
+  pull_request:
+    paths:
+      - 'internal/lambdalabs/**'
+      - 'pkg/v1/**'
+    branches: [ main ]
+
+jobs:
+  lambdalabs-validation:
+    name: LambdaLabs Provider Validation
+    runs-on: ubuntu-latest
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request'
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Go
+      uses: actions/setup-go@v4
+      with:
+        go-version: '1.23.0'
+
+    - name: Cache Go modules
+      uses: actions/cache@v4
+      with:
+        path: |
+          ~/.cache/go-build
+          ~/go/pkg/mod
+        key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+        restore-keys: |
+          ${{ runner.os }}-go-
+
+    - name: Install dependencies
+      run: make deps
+
+    - name: Run LambdaLabs validation tests
+      env:
+        LAMBDALABS_API_KEY: ${{ secrets.LAMBDALABS_API_KEY }}
+        TEST_PRIVATE_KEY_BASE64: ${{ secrets.TEST_PRIVATE_KEY_BASE64 }}
+        TEST_PUBLIC_KEY_BASE64: ${{ secrets.TEST_PUBLIC_KEY_BASE64 }}
+        VALIDATION_TEST: true
+      run: |
+        cd internal/lambdalabs
+        go test -v -short=false -timeout=30m ./...
+
+    - name: Upload test results
+      uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: lambdalabs-validation-results
+        path: |
+          internal/lambdalabs/coverage.out

.gitignore

@@ -0,0 +1,2 @@
+.env
+__debug_bin*

.vscode/settings.json

@@ -0,0 +1,29 @@
+{
+    "go.useLanguageServer": true,
+    "gopls": {
+        "gofumpt": true
+    },
+    "[go]": {
+        "editor.formatOnSave": true,
+        "editor.codeActionsOnSave": {
+            "source.organizeImports": "explicit"
+        }
+    },
+    "[go.mod]": {
+        "editor.formatOnSave": true,
+        "editor.codeActionsOnSave": {
+            "source.organizeImports": "explicit"
+        }
+    },
+    "go.lintTool": "golangci-lint",
+    "go.lintFlags": [
+        "--fast"
+    ],
+    "go.testFlags": [
+        "-race",
+        "-v",
+        "-count=1",
+        "-timeout=30m"
+    ],
+    "go.testEnvFile": "${workspaceFolder}/.env"
+}

Makefile

@@ -3,10 +3,16 @@
 
 # Variables
 BINARY_NAME=compute
-MODULE_NAME=github.com/brevdev/compute
+MODULE_NAME=github.com/brevdev/cloud
 BUILD_DIR=build
 COVERAGE_DIR=coverage
 
+# Load environment variables from .env file if it exists
+ifneq (,$(wildcard .env))
+    include .env
+    export
+endif
+
 # Go related variables
 GOCMD=go
 GOBUILD=$(GOCMD) build
@@ -59,6 +65,18 @@ build-windows:
 .PHONY: test
 test:
 	@echo "Running tests..."
+	$(GOTEST) -v -short ./...
+
+# Run validation tests
+.PHONY: test-validation
+test-validation:
+	@echo "Running validation tests..."
+	$(GOTEST) -v -short=false ./...
+
+# Run all tests including validation
+.PHONY: test-all
+test-all:
+	@echo "Running all tests..."
 	$(GOTEST) -v ./...
 
 # Run tests with coverage
@@ -181,7 +199,9 @@ help:
 	@echo "Available targets:"
 	@echo "  build          - Build the project"
 	@echo "  build-all      - Build for Linux, macOS, and Windows"
-	@echo "  test           - Run tests"
+	@echo "  test           - Run tests (with -short flag)"
+	@echo "  test-validation - Run validation tests (without -short flag)"
+	@echo "  test-all       - Run all tests including validation"
 	@echo "  test-coverage  - Run tests with coverage report"
 	@echo "  test-race      - Run tests with race detection"
 	@echo "  bench          - Run benchmarks"
@@ -197,4 +217,4 @@ help:
 	@echo "  docs           - Generate documentation"
 	@echo "  check          - Run all checks (lint, vet, fmt-check, test)"
 	@echo "  install-tools  - Install development tools"
-	@echo "  help           - Show this help message" 
+	@echo "  help           - Show this help message"  

docs/VALIDATION_TESTING.md

@@ -0,0 +1,135 @@
+# Validation Testing
+
+This document describes the validation testing framework for cloud provider implementations.
+
+## Overview
+
+Validation tests verify that cloud provider implementations correctly implement the SDK interfaces by making real API calls to cloud providers. These tests are separate from unit tests and require actual cloud credentials.
+
+## Running Validation Tests
+
+### Locally
+
+```bash
+# Skip validation tests (default)
+make test
+
+# Run validation tests
+make test-validation
+
+# Run all tests
+make test-all
+```
+
+### Environment Variables
+
+Each provider requires specific environment variables:
+
+- **LambdaLabs**: `LAMBDALABS_API_KEY`
+
+### CI/CD
+
+Validation tests run automatically:
+- Daily via scheduled workflows
+- On pull requests when labeled with `run-validation`
+- Manually via workflow dispatch
+
+## Adding New Providers
+
+1. Create validation test file: `internal/{provider}/v1/validation_test.go`
+2. Use the shared validation package with provider-specific configuration:
+
+```go
+func TestValidationFunctions(t *testing.T) {
+    apiKey := os.Getenv("YOUR_PROVIDER_API_KEY")
+    if apiKey == "" {
+        t.Skip("YOUR_PROVIDER_API_KEY not set, skipping YourProvider validation tests")
+    }
+
+    config := validation.ProviderConfig{
+        Credential: NewYourProviderCredential("validation-test", apiKey),
+    }
+    validation.RunValidationSuite(t, config)
+}
+```
+
+3. Create CI workflow: `.github/workflows/validation-{provider}.yml`
+4. Add environment variables to CI secrets
+5. Update this documentation
+
+## Shared Validation Package
+
+The validation tests use a shared package at `internal/validation/` that provides:
+- `RunValidationSuite()` - Tests all validation functions from pkg/v1/
+- `RunInstanceLifecycleValidation()` - Tests instance lifecycle operations
+- `ProviderConfig` - Configuration for provider-specific setup using CloudCredential
+
+The `ProviderConfig` uses the existing `CloudCredential` interface which acts as a factory for `CloudClient` instances. The provider name is automatically obtained from the credential's `GetCloudProviderID()` method. This approach eliminates code duplication and ensures consistent validation testing across all providers while leveraging the existing credential abstraction.
+
+## Test Structure
+
+Validation tests use `testing.Short()` guards:
+
+```go
+func TestValidation(t *testing.T) {
+    if testing.Short() {
+        t.Skip("Skipping validation tests in short mode")
+    }
+    // ... validation logic
+}
+```
+
+This ensures validation tests only run when explicitly requested.
+
+## Validation Functions Tested
+
+The framework tests all validation functions from the SDK:
+
+### Instance Management
+- `ValidateCreateInstance` - Tests instance creation with timing and attribute validation
+- `ValidateListCreatedInstance` - Tests instance listing and filtering
+- `ValidateTerminateInstance` - Tests instance termination
+- `ValidateMergeInstanceForUpdate` - Tests instance update merging logic
+
+### Instance Types
+- `ValidateGetInstanceTypes` - Tests instance type retrieval and filtering
+- `ValidateRegionalInstanceTypes` - Tests regional instance type filtering
+- `ValidateStableInstanceTypeIDs` - Tests instance type ID stability
+
+### Locations
+- `ValidateGetLocations` - Tests location retrieval and availability
+
+## Security Considerations
+
+- Validation tests use real cloud credentials stored as GitHub secrets
+- Tests create and destroy real cloud resources
+- Proper cleanup is implemented to avoid resource leaks
+- Tests are designed to be cost-effective and use minimal resources
+
+## Troubleshooting
+
+### Common Issues
+
+1. **Missing credentials**: Ensure environment variables are set
+2. **Quota limits**: Tests may skip if quota is exceeded
+3. **Resource availability**: Tests adapt to available instance types and locations
+4. **Network timeouts**: Tests use appropriate timeouts for cloud operations
+
+### Debugging
+
+```bash
+# Run specific validation test
+go test -v -short=false -run TestValidationFunctions ./internal/lambdalabs/v1/
+
+# Run with verbose output
+go test -v -short=false -timeout=20m ./internal/lambdalabs/v1/
+```
+
+## Contributing
+
+When adding new validation functions:
+
+1. Add the validation function to the appropriate `pkg/v1/*.go` file
+2. Add corresponding test in `internal/{provider}/v1/validation_test.go`
+3. Ensure proper cleanup and error handling
+4. Update this documentation

docs/example-dot-env

@@ -0,0 +1,3 @@
+LAMBDALABS_API_KEY=secret_my-api-key_**********
+TEST_PRIVATE_KEY_BASE64=LS0tLS1CRUdJTiBSU0EgUFJJVk...
+TEST_PUBLIC_KEY_BASE64=LS0tLS1CRUdJTiBQVUJMSUMgS0V...

go.mod

@@ -7,7 +7,7 @@ toolchain go1.23.2
 require (
 	github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b
 	github.com/bojanz/currency v1.3.1
-	github.com/brevdev/compute v0.0.0-20250805004716-bc4fe363e0ea
+	github.com/google/go-cmp v0.7.0
 	github.com/google/uuid v1.6.0
 	github.com/jarcoal/httpmock v1.4.0
 	github.com/nebius/gosdk v0.0.0-20250731090238-d96c0d4a5930
@@ -16,19 +16,22 @@ require (
 
 require (
 	buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.31.0-20231030212536-12f9cba37c9d.2 // indirect
+	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/cockroachdb/apd/v3 v3.2.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/gliderlabs/ssh v0.3.8 // indirect
 	github.com/gofrs/flock v0.12.1 // indirect
 	github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	golang.org/x/crypto v0.41.0 // indirect
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
-	golang.org/x/net v0.38.0 // indirect
-	golang.org/x/sync v0.12.0 // indirect
-	golang.org/x/sys v0.31.0 // indirect
-	golang.org/x/text v0.23.0 // indirect
+	golang.org/x/net v0.42.0 // indirect
+	golang.org/x/sync v0.16.0 // indirect
+	golang.org/x/sys v0.35.0 // indirect
+	golang.org/x/text v0.28.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect
 	google.golang.org/grpc v1.64.1 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect

go.sum

@@ -2,10 +2,10 @@ buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.31.0-2023103021253
 buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.31.0-20231030212536-12f9cba37c9d.2/go.mod h1:xafc+XIsTxTy76GJQ1TKgvJWsSugFBqMaN27WhUblew=
 github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b h1:mimo19zliBX/vSQ6PWWSL9lK8qwHozUj03+zLoEB8O0=
 github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b/go.mod h1:fvzegU4vN3H1qMT+8wDmzjAcDONcgo2/SZ/TyfdUOFs=
+github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
+github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
 github.com/bojanz/currency v1.3.1 h1:3BUAvy/5hU/Pzqg5nrQslVihV50QG+A2xKPoQw1RKH4=
 github.com/bojanz/currency v1.3.1/go.mod h1:jNoZiJyRTqoU5DFoa+n+9lputxPUDa8Fz8BdDrW06Go=
-github.com/brevdev/compute v0.0.0-20250805004716-bc4fe363e0ea h1:U+mj2Q4lYMMkCuflMzFeIzf0tiASimf8/juGhcAT3DY=
-github.com/brevdev/compute v0.0.0-20250805004716-bc4fe363e0ea/go.mod h1:rxhy3+lWmdnVABBys6l+Z+rVDeKa5nyy0avoQkTmTFw=
 github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/cockroachdb/apd/v3 v3.2.1 h1:U+8j7t0axsIgvQUqthuNm82HIrYXodOV2iWLWtEaIwg=
@@ -14,14 +14,16 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c=
+github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU=
 github.com/gofrs/flock v0.12.1 h1:MTLVXXHf8ekldpJk3AKicLij9MdwOWkZ+a/jHHZby9E=
 github.com/gofrs/flock v0.12.1/go.mod h1:9zxTsyu5xtJ9DK+1tFZyibEV7y3uwDxPPfbxeeHCoD0=
 github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI=
 github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
-github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 h1:pRhl55Yx1eC7BZ1N+BBWwnKaMyD8uC+34TLdndZMAKk=
@@ -52,16 +54,26 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4=
+golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc=
 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g=
 golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
 golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8=
 golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
+golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs=
+golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8=
 golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw=
 golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
+golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=
+golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
 golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik=
 golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI=
+golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
 golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY=
 golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4=
+golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
+golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY=