Skip to content

Commit effec05

Browse files
Copilotdciangot
Copilot
and
dciangot
authored
Add GitHub Actions e2e test workflow using Dagger
Agent-Logs-Url: https://github.com/interlink-hq/vk-test-set/sessions/81a6df2f-5438-464e-b624-02310eb51c92 Co-authored-by: dciangot <4144326+dciangot@users.noreply.github.com>
1 parent 3917e09 commit effec05

11 files changed

Lines changed: 420 additions & 0 deletions

File tree

‎.github/workflows/e2e.yaml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
name: e2e-tests
2+
3+
on:
4+
push: {}
5+
pull_request: {}
6+
7+
jobs:
8+
test:
9+
name: e2e
10+
runs-on: ubuntu-latest
11+
steps:
12+
- name: Checkout
13+
uses: actions/checkout@v4
14+
with:
15+
submodules: recursive
16+
- name: E2E Test
17+
uses: dagger/dagger-for-github@v7
18+
with:
19+
workdir: ci
20+
verb: call
21+
args: -s --name e2e new-interlink test stdout
22+
cloud-token: ${{ secrets.DAGGER_CLOUD_TOKEN }}
23+
version: "0.19.2"

‎ci/.gitattributes‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
/dagger.gen.go linguist-generated
2+
/internal/dagger/** linguist-generated
3+
/internal/querybuilder/** linguist-generated
4+
/internal/telemetry/** linguist-generated

‎ci/.gitignore‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
/dagger.gen.go
2+
/internal/dagger
3+
/internal/querybuilder
4+
/internal/telemetry
5+
/.env

‎ci/dagger.json‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
{
2+
"name": "vk-test-set",
3+
"engineVersion": "v0.19.2",
4+
"sdk": {
5+
"source": "go"
6+
},
7+
"dependencies": [
8+
{
9+
"name": "k3s",
10+
"source": "github.com/marcosnils/daggerverse/k3s@k3s/v0.1.10",
11+
"pin": "28eea1fcf3b6ecb38a628186107760acd717442f"
12+
}
13+
]
14+
}

‎ci/go.mod‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
module dagger/vk-test-set
2+
3+
go 1.24.0

‎ci/go.sum‎

Whitespace-only changes.

‎ci/main.go‎

Lines changed: 232 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,232 @@
1+
// A Dagger module to run vk-test-set end-to-end tests against interLink components
2+
//
3+
// Visit the interLink documentation for more info: https://interlink-hq.github.io/interLink/docs/intro/
4+
5+
package main
6+
7+
import (
8+
"context"
9+
"dagger/vk-test-set/internal/dagger"
10+
"fmt"
11+
"time"
12+
)
13+
14+
// vkConfigMap is the ConfigMap that configures virtual-kubelet to connect to the interLink service
15+
const vkConfigMap = `apiVersion: v1
16+
kind: ConfigMap
17+
metadata:
18+
name: virtual-kubelet-config
19+
namespace: interlink
20+
data:
21+
InterLinkConfig.yaml: |
22+
InterlinkURL: "http://interlink"
23+
InterlinkPort: "3000"
24+
VerboseLogging: true
25+
ErrorsOnlyLogging: false
26+
ServiceAccount: "virtual-kubelet"
27+
Namespace: interlink
28+
VKTokenFile: ""
29+
Resources:
30+
CPU: "100"
31+
Memory: "128Gi"
32+
Pods: "100"
33+
HTTP:
34+
Insecure: true
35+
KubeletHTTP:
36+
Insecure: true
37+
`
38+
39+
// VkTestSet is the Dagger module for vk-test-set e2e tests
40+
type VkTestSet struct {
41+
Name string
42+
VirtualKubeletRef string
43+
InterlinkRef string
44+
PluginRef string
45+
}
46+
47+
// New initializes the Dagger module
48+
func New(
49+
name string,
50+
// +optional
51+
// +default="ghcr.io/interlink-hq/interlink/virtual-kubelet-inttw:0.6.0"
52+
virtualKubeletRef string,
53+
// +optional
54+
// +default="ghcr.io/interlink-hq/interlink/interlink:0.6.0"
55+
interlinkRef string,
56+
// +optional
57+
// +default="ghcr.io/interlink-hq/interlink-sidecar-slurm/interlink-sidecar-slurm:0.5.0"
58+
pluginRef string,
59+
) *VkTestSet {
60+
return &VkTestSet{
61+
Name: name,
62+
VirtualKubeletRef: virtualKubeletRef,
63+
InterlinkRef: interlinkRef,
64+
PluginRef: pluginRef,
65+
}
66+
}
67+
68+
// NewInterlink sets up a k3s cluster with the interLink API, slurm mock plugin, and virtual-kubelet
69+
func (m *VkTestSet) NewInterlink(
70+
ctx context.Context,
71+
// +optional
72+
// +defaultPath="./manifests"
73+
manifests *dagger.Directory,
74+
// +optional
75+
// +defaultPath="./manifests/interlink-config.yaml"
76+
interlinkConfig *dagger.File,
77+
// +optional
78+
// +defaultPath="./manifests/plugin-config.yaml"
79+
pluginConfig *dagger.File,
80+
) (*VkTestSet, error) {
81+
// Start the slurm plugin service (SHARED_FS=true enables mock mode without real SLURM)
82+
pluginEndpoint, err := dag.Container().From(m.PluginRef).
83+
WithFile("/etc/interlink/InterLinkConfig.yaml", pluginConfig).
84+
WithEnvVariable("BUST", time.Now().String()).
85+
WithEnvVariable("SLURMCONFIGPATH", "/etc/interlink/InterLinkConfig.yaml").
86+
WithEnvVariable("SHARED_FS", "true").
87+
WithExposedPort(4000).
88+
AsService(dagger.ContainerAsServiceOpts{
89+
UseEntrypoint: true,
90+
InsecureRootCapabilities: true,
91+
}).Start(ctx)
92+
if err != nil {
93+
return nil, err
94+
}
95+
96+
// Start the interLink API service, bound to the plugin
97+
interlinkEndpoint, err := dag.Container().From(m.InterlinkRef).
98+
WithFile("/etc/interlink/InterLinkConfig.yaml", interlinkConfig).
99+
WithEnvVariable("BUST", time.Now().String()).
100+
WithServiceBinding("plugin", pluginEndpoint).
101+
WithEnvVariable("INTERLINKCONFIGPATH", "/etc/interlink/InterLinkConfig.yaml").
102+
WithExposedPort(3000).
103+
AsService(dagger.ContainerAsServiceOpts{
104+
UseEntrypoint: true,
105+
InsecureRootCapabilities: true,
106+
}).Start(ctx)
107+
if err != nil {
108+
return nil, err
109+
}
110+
111+
// Set up k3s cluster with the interLink service accessible as "interlink"
112+
K3s := dag.K3S(m.Name).With(func(k *dagger.K3S) *dagger.K3S {
113+
return k.WithContainer(
114+
k.Container().
115+
WithEnvVariable("BUST", time.Now().String()).
116+
WithServiceBinding("interlink", interlinkEndpoint),
117+
)
118+
})
119+
120+
_, err = K3s.Server().Start(ctx)
121+
if err != nil {
122+
return nil, err
123+
}
124+
125+
time.Sleep(60 * time.Second) // wait for k3s to be ready
126+
127+
kubeConfig := K3s.Config(dagger.K3SConfigOpts{Local: false})
128+
129+
// Build VK deployment YAML with the configured image reference
130+
vkDeploymentYAML := fmt.Sprintf(`apiVersion: apps/v1
131+
kind: Deployment
132+
metadata:
133+
name: virtual-kubelet
134+
namespace: interlink
135+
labels:
136+
nodeName: virtual-kubelet
137+
spec:
138+
replicas: 1
139+
selector:
140+
matchLabels:
141+
nodeName: virtual-kubelet
142+
template:
143+
metadata:
144+
labels:
145+
nodeName: virtual-kubelet
146+
spec:
147+
hostNetwork: true
148+
automountServiceAccountToken: true
149+
serviceAccountName: virtual-kubelet
150+
containers:
151+
- name: inttw-vk
152+
image: "%s"
153+
imagePullPolicy: Always
154+
env:
155+
- name: NODENAME
156+
value: virtual-kubelet
157+
- name: KUBELET_PORT
158+
value: "10251"
159+
- name: POD_IP
160+
valueFrom:
161+
fieldRef:
162+
fieldPath: status.podIP
163+
- name: CONFIGPATH
164+
value: "/etc/interlink/InterLinkConfig.yaml"
165+
volumeMounts:
166+
- name: config
167+
mountPath: /etc/interlink/InterLinkConfig.yaml
168+
subPath: InterLinkConfig.yaml
169+
volumes:
170+
- name: config
171+
configMap:
172+
name: virtual-kubelet-config
173+
`, m.VirtualKubeletRef)
174+
175+
// Deploy virtual-kubelet and its supporting resources into k3s
176+
_, err = dag.Container().From("bitnamilegacy/kubectl:1.33-debian-12").
177+
WithUser("root").
178+
WithMountedFile("/.kube/config", kubeConfig).
179+
WithEnvVariable("KUBECONFIG", "/.kube/config").
180+
WithEnvVariable("BUST", time.Now().String()).
181+
WithDirectory("/manifests", manifests).
182+
WithNewFile("/manifests/virtual-kubelet-config.yaml", vkConfigMap).
183+
WithNewFile("/manifests/virtual-kubelet-deployment.yaml", vkDeploymentYAML).
184+
WithExec([]string{"bash", "-c", `
185+
kubectl create namespace interlink --dry-run=client -o yaml | kubectl apply -f - &&
186+
kubectl apply -f /manifests/service-account.yaml &&
187+
kubectl apply -f /manifests/virtual-kubelet-config.yaml &&
188+
kubectl apply -f /manifests/virtual-kubelet-deployment.yaml &&
189+
kubectl wait --for=condition=Available deployment/virtual-kubelet -n interlink --timeout=300s &&
190+
kubectl wait --for=condition=Ready node/virtual-kubelet --timeout=300s
191+
`}).
192+
Sync(ctx)
193+
if err != nil {
194+
return nil, err
195+
}
196+
197+
return m, nil
198+
}
199+
200+
// Test installs the vk-test-set and runs the pytest suite against the running cluster
201+
func (m *VkTestSet) Test(
202+
ctx context.Context,
203+
// +optional
204+
// +defaultPath=".."
205+
testSet *dagger.Directory,
206+
// +optional
207+
// +defaultPath="./manifests"
208+
manifests *dagger.Directory,
209+
) (*dagger.Container, error) {
210+
c := dag.Container().From("bitnamilegacy/kubectl:1.33-debian-12").
211+
WithUser("root").
212+
WithExec([]string{"mkdir", "-p", "/opt/user"}).
213+
WithExec([]string{"chown", "-R", "1001:0", "/opt/user"}).
214+
WithExec([]string{"apt", "update"}).
215+
WithExec([]string{"apt", "install", "-y", "curl", "python3", "python3-pip", "python3-venv", "git"}).
216+
WithMountedFile("/.kube/config", dag.K3S(m.Name).Config(dagger.K3SConfigOpts{Local: false})).
217+
WithExec([]string{"chown", "1001:0", "/.kube/config"}).
218+
WithUser("1001").
219+
WithDirectory("/manifests", manifests).
220+
WithDirectory("/opt/user/vk-test-set", testSet, dagger.ContainerWithDirectoryOpts{Owner: "1001:0"}).
221+
WithEntrypoint([]string{"kubectl"}).
222+
WithExec([]string{"bash", "-c", "cp /manifests/vktest_config.yaml /opt/user/vk-test-set/vktest_config.yaml"}).
223+
WithWorkdir("/opt/user/vk-test-set").
224+
WithExec([]string{"bash", "-c", "kubectl get csr -o name | xargs -r kubectl certificate approve"}).
225+
WithExec([]string{"bash", "-c", "python3 -m venv .venv && source .venv/bin/activate && pip3 install -e ./"})
226+
227+
result := c.
228+
WithExec([]string{"bash", "-c", "kubectl get csr -o name | xargs -r kubectl certificate approve"}).
229+
WithExec([]string{"bash", "-c", "source .venv/bin/activate && export KUBECONFIG=/.kube/config && pytest -v -k 'not rclone and not limits and not stress and not multi-init and not fail'"})
230+
231+
return result, nil
232+
}

‎ci/manifests/interlink-config.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
InterlinkAddress: "http://0.0.0.0"
2+
InterlinkPort: "3000"
3+
SidecarURL: "http://plugin"
4+
SidecarPort: "4000"
5+
VerboseLogging: true
6+
ErrorsOnlyLogging: false
7+
DataRootFolder: "~/.interlink"

‎ci/manifests/plugin-config.yaml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
InterlinkURL: "http://interlink"
2+
InterlinkPort: "3000"
3+
SidecarURL: "http://0.0.0.0"
4+
SidecarPort: "4000"
5+
VerboseLogging: true
6+
ErrorsOnlyLogging: false
7+
DataRootFolder: "/home/ubuntu/.interlink/"
8+
ExportPodData: true
9+
SbatchPath: "/usr/bin/sbatch"
10+
ScancelPath: "/usr/bin/scancel"
11+
SqueuePath: "/usr/bin/squeue"
12+
CommandPrefix: ""
13+
SingularityPrefix: ""
14+
Namespace: "vk"
15+
Tsocks: false
16+
TsocksPath: "$WORK/tsocks-1.8beta5+ds1/libtsocks.so"
17+
TsocksLoginNode: "login01"
18+
BashPath: /bin/bash
19+
EnableProbes: true

0 commit comments

Comments
 (0)