Skip to content

Commit 24cea1b

Browse files
tenzen-ysaileshd1402
tenzen-y
authored and
saileshd1402
committed
KEP-2170: Implement runtime framework (kubeflow#2248)
* KEP-2170: Implement runtime framework interfaces Signed-off-by: Yuki Iwai <[email protected]> * Remove grep dependency Signed-off-by: Yuki Iwai <[email protected]> * KEP-2170: Implement ValidateObjects interface to the runtime framework Signed-off-by: Yuki Iwai <[email protected]> * KEP-2170: Expose the TrainingRuntime and ClusterTrainingRuntime Kind Signed-off-by: Yuki Iwai <[email protected]> * KEP-2170: Remove unneeded scheme field from the internal TrainingRuntime Signed-off-by: Yuki Iwai <[email protected]> * Rephrase the error message Signed-off-by: Yuki Iwai <[email protected]> * Distinguish TrainingRuntime and ClusterTrainingRuntime when creating indexes for the TrainJobs Signed-off-by: Yuki Iwai <[email protected]> * Propagate the TrainJob labels and annotations to the JobSet Signed-off-by: Yuki Iwai <[email protected]> * Remove PodAnnotations from the runtime info Signed-off-by: Yuki Iwai <[email protected]> * Implement TrainingRuntime ReplicatedJob validation Signed-off-by: Yuki Iwai <[email protected]> * Add TODO comments Signed-off-by: Yuki Iwai <[email protected]> * Replace queueSuspendedTrainJob with queueSuspendedTrainJobs Signed-off-by: Yuki Iwai <[email protected]> --------- Signed-off-by: Yuki Iwai <[email protected]> Signed-off-by: sailesh duddupudi <[email protected]>
1 parent 4ff5052 commit 24cea1b

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+3521
-227
lines changed

‎.github/workflows/unittests.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ jobs:
3737
3838
- name: Run Go test for v2
3939
run: |
40+
make testv2
4041
make test-integrationv2 ENVTEST_K8S_VERSION=${{ matrix.kubernetes-version }}
4142
4243
- name: Coveralls report

‎Makefile‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,10 @@ test: envtest
8080
test-integrationv2: envtest
8181
KUBEBUILDER_ASSETS="$(shell setup-envtest use $(ENVTEST_K8S_VERSION) -p path)" go test ./test/... -coverprofile cover.out
8282

83+
.PHONY: testv2
84+
testv2:
85+
go test ./pkg/controller.v2/... ./pkg/runtime.v2/... ./pkg/webhook.v2/... ./pkg/util.v2/... -coverprofile cover.out
86+
8387
envtest:
8488
ifndef HAS_SETUP_ENVTEST
8589
go install sigs.k8s.io/controller-runtime/tools/setup-envtest@bf15e44028f908c790721fc8fe67c7bf2d06a611 # v0.17.2

‎cmd/training-operator.v2alpha1/main.go‎

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ limitations under the License.
1717
package main
1818

1919
import (
20+
"context"
2021
"crypto/tls"
2122
"errors"
2223
"flag"
@@ -25,7 +26,7 @@ import (
2526

2627
zaplog "go.uber.org/zap"
2728
"go.uber.org/zap/zapcore"
28-
"k8s.io/apimachinery/pkg/runtime"
29+
apiruntime "k8s.io/apimachinery/pkg/runtime"
2930
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
3031
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
3132
ctrl "sigs.k8s.io/controller-runtime"
@@ -34,22 +35,25 @@ import (
3435
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
3536
"sigs.k8s.io/controller-runtime/pkg/webhook"
3637
jobsetv1alpha2 "sigs.k8s.io/jobset/api/jobset/v1alpha2"
38+
schedulerpluginsv1alpha1 "sigs.k8s.io/scheduler-plugins/apis/scheduling/v1alpha1"
3739

3840
kubeflowv2 "github.com/kubeflow/training-operator/pkg/apis/kubeflow.org/v2alpha1"
3941
"github.com/kubeflow/training-operator/pkg/cert"
4042
controllerv2 "github.com/kubeflow/training-operator/pkg/controller.v2"
43+
runtimecore "github.com/kubeflow/training-operator/pkg/runtime.v2/core"
4144
webhookv2 "github.com/kubeflow/training-operator/pkg/webhook.v2"
4245
)
4346

4447
var (
45-
scheme = runtime.NewScheme()
48+
scheme = apiruntime.NewScheme()
4649
setupLog = ctrl.Log.WithName("setup")
4750
)
4851

4952
func init() {
5053
utilruntime.Must(clientgoscheme.AddToScheme(scheme))
5154
utilruntime.Must(kubeflowv2.AddToScheme(scheme))
5255
utilruntime.Must(jobsetv1alpha2.AddToScheme(scheme))
56+
utilruntime.Must(schedulerpluginsv1alpha1.AddToScheme(scheme))
5357
}
5458

5559
func main() {
@@ -127,27 +131,34 @@ func main() {
127131
os.Exit(1)
128132
}
129133

134+
ctx := ctrl.SetupSignalHandler()
135+
130136
setupProbeEndpoints(mgr, certsReady)
131137
// Set up controllers using goroutines to start the manager quickly.
132-
go setupControllers(mgr, certsReady)
138+
go setupControllers(ctx, mgr, certsReady)
133139

134140
setupLog.Info("Starting manager")
135-
if err = mgr.Start(ctrl.SetupSignalHandler()); err != nil {
141+
if err = mgr.Start(ctx); err != nil {
136142
setupLog.Error(err, "Could not run manager")
137143
os.Exit(1)
138144
}
139145
}
140146

141-
func setupControllers(mgr ctrl.Manager, certsReady <-chan struct{}) {
147+
func setupControllers(ctx context.Context, mgr ctrl.Manager, certsReady <-chan struct{}) {
142148
setupLog.Info("Waiting for certificate generation to complete")
143149
<-certsReady
144150
setupLog.Info("Certs ready")
145151

146-
if failedCtrlName, err := controllerv2.SetupControllers(mgr); err != nil {
152+
runtimes, err := runtimecore.New(ctx, mgr.GetClient(), mgr.GetFieldIndexer())
153+
if err != nil {
154+
setupLog.Error(err, "Could not initialize runtimes")
155+
os.Exit(1)
156+
}
157+
if failedCtrlName, err := controllerv2.SetupControllers(mgr, runtimes); err != nil {
147158
setupLog.Error(err, "Could not create controller", "controller", failedCtrlName)
148159
os.Exit(1)
149160
}
150-
if failedWebhook, err := webhookv2.Setup(mgr); err != nil {
161+
if failedWebhook, err := webhookv2.Setup(mgr, runtimes); err != nil {
151162
setupLog.Error(err, "Could not create webhook", "webhook", failedWebhook)
152163
os.Exit(1)
153164
}

‎go.mod‎

Lines changed: 33 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -3,49 +3,50 @@ module github.com/kubeflow/training-operator
33
go 1.22
44

55
require (
6-
github.com/go-logr/logr v1.4.1
6+
github.com/go-logr/logr v1.4.2
77
github.com/google/go-cmp v0.6.0
8-
github.com/onsi/ginkgo/v2 v2.17.1
9-
github.com/onsi/gomega v1.32.0
8+
github.com/onsi/ginkgo/v2 v2.19.0
9+
github.com/onsi/gomega v1.33.1
1010
github.com/open-policy-agent/cert-controller v0.10.1
1111
github.com/prometheus/client_golang v1.18.0
12-
github.com/sirupsen/logrus v1.9.0
12+
github.com/sirupsen/logrus v1.9.3
1313
github.com/stretchr/testify v1.9.0
1414
go.uber.org/zap v1.27.0
15-
k8s.io/api v0.29.3
16-
k8s.io/apimachinery v0.29.3
17-
k8s.io/client-go v0.29.3
18-
k8s.io/code-generator v0.29.3
19-
k8s.io/klog/v2 v2.110.1
15+
k8s.io/api v0.29.5
16+
k8s.io/apimachinery v0.29.5
17+
k8s.io/client-go v0.29.5
18+
k8s.io/code-generator v0.29.5
19+
k8s.io/klog/v2 v2.120.1
2020
k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00
21-
k8s.io/utils v0.0.0-20230726121419-3b25d923346b
21+
k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0
2222
sigs.k8s.io/controller-runtime v0.17.3
2323
sigs.k8s.io/jobset v0.5.2
24+
sigs.k8s.io/kueue v0.6.3
2425
sigs.k8s.io/scheduler-plugins v0.28.9
2526
sigs.k8s.io/yaml v1.4.0
2627
volcano.sh/apis v1.9.0
2728
)
2829

2930
require (
3031
github.com/beorn7/perks v1.0.1 // indirect
31-
github.com/cespare/xxhash/v2 v2.2.0 // indirect
32+
github.com/cespare/xxhash/v2 v2.3.0 // indirect
3233
github.com/davecgh/go-spew v1.1.1 // indirect
33-
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
34-
github.com/evanphx/json-patch v5.6.0+incompatible // indirect
34+
github.com/emicklei/go-restful/v3 v3.12.1 // indirect
35+
github.com/evanphx/json-patch v5.9.0+incompatible // indirect
3536
github.com/evanphx/json-patch/v5 v5.8.0 // indirect
3637
github.com/fsnotify/fsnotify v1.7.0 // indirect
3738
github.com/go-logr/zapr v1.3.0 // indirect
38-
github.com/go-openapi/jsonpointer v0.19.6 // indirect
39-
github.com/go-openapi/jsonreference v0.20.2 // indirect
40-
github.com/go-openapi/swag v0.22.3 // indirect
41-
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
39+
github.com/go-openapi/jsonpointer v0.21.0 // indirect
40+
github.com/go-openapi/jsonreference v0.21.0 // indirect
41+
github.com/go-openapi/swag v0.23.0 // indirect
42+
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
4243
github.com/gogo/protobuf v1.3.2 // indirect
4344
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
4445
github.com/golang/protobuf v1.5.4 // indirect
4546
github.com/google/gnostic-models v0.6.8 // indirect
4647
github.com/google/gofuzz v1.2.0 // indirect
47-
github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
48-
github.com/google/uuid v1.3.1 // indirect
48+
github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect
49+
github.com/google/uuid v1.6.0 // indirect
4950
github.com/imdario/mergo v0.3.16 // indirect
5051
github.com/josharian/intern v1.0.0 // indirect
5152
github.com/json-iterator/go v1.1.12 // indirect
@@ -56,30 +57,29 @@ require (
5657
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
5758
github.com/pkg/errors v0.9.1 // indirect
5859
github.com/pmezard/go-difflib v1.0.0 // indirect
59-
github.com/prometheus/client_model v0.5.0 // indirect
60+
github.com/prometheus/client_model v0.6.1 // indirect
6061
github.com/prometheus/common v0.45.0 // indirect
6162
github.com/prometheus/procfs v0.12.0 // indirect
6263
github.com/spf13/pflag v1.0.5 // indirect
6364
go.uber.org/atomic v1.11.0 // indirect
6465
go.uber.org/multierr v1.11.0 // indirect
65-
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
66-
golang.org/x/mod v0.16.0 // indirect
67-
golang.org/x/net v0.23.0 // indirect
68-
golang.org/x/oauth2 v0.12.0 // indirect
69-
golang.org/x/sys v0.18.0 // indirect
70-
golang.org/x/term v0.18.0 // indirect
71-
golang.org/x/text v0.14.0 // indirect
72-
golang.org/x/time v0.3.0 // indirect
73-
golang.org/x/tools v0.19.0 // indirect
66+
golang.org/x/exp v0.0.0-20240530194437-404ba88c7ed0 // indirect
67+
golang.org/x/mod v0.17.0 // indirect
68+
golang.org/x/net v0.25.0 // indirect
69+
golang.org/x/oauth2 v0.20.0 // indirect
70+
golang.org/x/sys v0.20.0 // indirect
71+
golang.org/x/term v0.20.0 // indirect
72+
golang.org/x/text v0.15.0 // indirect
73+
golang.org/x/time v0.5.0 // indirect
74+
golang.org/x/tools v0.21.0 // indirect
7475
gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
75-
google.golang.org/appengine v1.6.7 // indirect
76-
google.golang.org/protobuf v1.33.0 // indirect
76+
google.golang.org/protobuf v1.34.1 // indirect
7777
gopkg.in/inf.v0 v0.9.1 // indirect
7878
gopkg.in/yaml.v2 v2.4.0 // indirect
7979
gopkg.in/yaml.v3 v3.0.1 // indirect
8080
k8s.io/apiextensions-apiserver v0.29.2 // indirect
81-
k8s.io/component-base v0.29.2 // indirect
82-
k8s.io/gengo v0.0.0-20230829151522-9cce18d56c01 // indirect
81+
k8s.io/component-base v0.29.5 // indirect
82+
k8s.io/gengo v0.0.0-20240404160639-a0386bf69313 // indirect
8383
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
8484
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
8585
)

0 commit comments

Comments
 (0)