Skip to content

Commit b0a18be

Browse files
frewilhelmfrewilhelm
committed
storage migration to OCI artifacts
1 parent 455055a commit b0a18be

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

68 files changed

+3268
-1784
lines changed

‎.github/workflows/tests.yaml

Lines changed: 132 additions & 132 deletions
Original file line numberDiff line numberDiff line change
@@ -62,135 +62,135 @@ jobs:
6262
bin/k8s
6363
key: ${{ steps.cache-k8s-restore.outputs.cache-primary-key }}
6464

65-
e2e-tests:
66-
runs-on: large_runner
67-
steps:
68-
- name: Self Hosted Runner Post Job Cleanup Action
69-
uses: TooMuch4U/[email protected]
70-
- name: Checkout
71-
uses: actions/checkout@v4
72-
- name: Setup Go
73-
uses: actions/setup-go@v5
74-
with:
75-
go-version-file: '${{ github.workspace }}/go.mod'
76-
cache: false
77-
78-
- name: Get Go environment
79-
run: |
80-
echo "go_cache=$(go env GOCACHE)" >> $GITHUB_ENV
81-
echo "go_modcache=$(go env GOMODCACHE)" >> $GITHUB_ENV
82-
- name: Set up cache
83-
# https://github.com/actions/setup-go/issues/358 - cache is not working with setup-go for multiple jobs
84-
uses: actions/cache@v4
85-
with:
86-
path: |
87-
${{ env.go_cache }}
88-
${{ env.go_modcache }}
89-
bin/k8s
90-
key: ${{ env.cache_name }}-${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}-${{ hashFiles('**/go.mod') }}
91-
restore-keys: |
92-
${{ env.cache_name }}-${{ runner.os }}-go-
93-
env:
94-
cache_name: integration-test
95-
96-
- name: Create k8s Kind Cluster
97-
uses: helm/kind-action@v1
98-
with:
99-
config: test/e2e/config/kind-config.yaml
100-
101-
- name: Install internal image registry
102-
run: |
103-
kubectl apply -f test/e2e/config/image-registry.yaml
104-
kubectl wait pod -l app=registry --for condition=Ready --timeout 5m
105-
kubectl wait pod -l app=protected-registry1 --for condition=Ready --timeout 5m
106-
kubectl wait pod -l app=protected-registry2 --for condition=Ready --timeout 5m
107-
108-
- name: Install external CRDs
109-
run: kubectl apply --server-side -k https://github.com/openfluxcd/artifact//config/crd?ref=v0.1.1
110-
111-
- name: Checkout helm-controller
112-
uses: actions/checkout@v4
113-
with:
114-
repository: openfluxcd/helm-controller
115-
path: helm-controller
116-
117-
# TODO: Create helm-controller image in public repository to omit rebuilds
118-
- name: Install helm-controller
119-
env:
120-
IMG: localhost:31000/helm-controller:latest
121-
run: |
122-
make -C helm-controller docker-build
123-
make -C helm-controller docker-push
124-
make -C helm-controller install
125-
make -C helm-controller deploy
126-
kubectl wait deployment.apps/helm-controller --for condition=Available --namespace helm-system --timeout 5m
127-
kubectl logs --tail -1 -l app=helm-controller -n helm-system -f --ignore-errors &> helm-controller.log &
128-
129-
- name: Checkout kustomize-controller
130-
uses: actions/checkout@v4
131-
with:
132-
repository: openfluxcd/kustomize-controller
133-
path: kustomize-controller
134-
135-
# TODO: Create kustomize-controller image in public repository to omit rebuilds
136-
- name: Install kustomize-controller
137-
env:
138-
IMG: localhost:31000/kustomize-controller:latest
139-
run: |
140-
make -C kustomize-controller docker-build
141-
make -C kustomize-controller docker-push
142-
make -C kustomize-controller install
143-
make -C kustomize-controller deploy
144-
kubectl wait deployment.apps/kustomize-controller --for condition=Available --namespace kustomize-system --timeout 5m
145-
kubectl logs --tail -1 -l app=kustomize-controller -n kustomize-system -f --ignore-errors &> kustomize-controller.log &
146-
147-
# TODO: Replace once the release with the 'skipDigestGeneration' field in the component constructor is available
148-
# uses: open-component-model/ocm-setup-action@main
149-
# with:
150-
# version: v0.19.0-rc.1
151-
- name: Set up cache for ocm (temporarily)
152-
uses: actions/cache@v4
153-
with:
154-
path: |
155-
ocm/bin
156-
key: ${{ env.cache_name }}-${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}-${{ hashFiles('**/go.mod') }}
157-
restore-keys: |
158-
${{ env.cache_name }}-${{ runner.os }}-go-
159-
env:
160-
cache_name: dummy-cache
161-
- name: Checkout OCM (temporarily)
162-
uses: actions/checkout@v4
163-
with:
164-
repository: open-component-model/ocm
165-
path: ocm
166-
- name: Build OCM (temporarily)
167-
run: |
168-
make -C ocm bin/ocm
169-
echo "${{ github.workspace }}/ocm/bin" >> "$GITHUB_PATH"
170-
171-
- name: Run e2e test
172-
env:
173-
RESOURCE_TIMEOUT: 5m
174-
HELM_CHART: ghcr.io/stefanprodan/charts/podinfo:6.7.1
175-
IMAGE_REFERENCE: ghcr.io/stefanprodan/podinfo:6.7.1
176-
CONTROLLER_LOG_PATH: ./ocm-k8s-toolkit-controller.log
177-
IMAGE_REGISTRY_URL: http://localhost:31000
178-
INTERNAL_IMAGE_REGISTRY_URL: http://registry-internal.default.svc.cluster.local:5000
179-
PROTECTED_REGISTRY_URL: http://localhost:31001
180-
INTERNAL_PROTECTED_REGISTRY_URL: http://protected-registry1-internal.default.svc.cluster.local:5001
181-
PROTECTED_REGISTRY_URL2: http://localhost:31002
182-
INTERNAL_PROTECTED_REGISTRY_URL2: http://protected-registry2-internal.default.svc.cluster.local:5002
183-
run: make test-e2e
184-
185-
- name: Publish logs on failure
186-
if: failure()
187-
uses: actions/upload-artifact@v4
188-
with:
189-
name: controller-logs
190-
# Currently, it is planned that the integration tests runs on every commit on a PR. Therefore, we could
191-
# produce a lot of logs. To note clutter the storage, the retention-days are reduced to 1.
192-
retention-days: 1
193-
path: |
194-
helm-controller.log
195-
kustomize-controller.log
196-
ocm-k8s-toolkit-controller.log
65+
# e2e-tests:
66+
# runs-on: large_runner
67+
# steps:
68+
# - name: Self Hosted Runner Post Job Cleanup Action
69+
# uses: TooMuch4U/[email protected]
70+
# - name: Checkout
71+
# uses: actions/checkout@v4
72+
# - name: Setup Go
73+
# uses: actions/setup-go@v5
74+
# with:
75+
# go-version-file: '${{ github.workspace }}/go.mod'
76+
# cache: false
77+
#
78+
# - name: Get Go environment
79+
# run: |
80+
# echo "go_cache=$(go env GOCACHE)" >> $GITHUB_ENV
81+
# echo "go_modcache=$(go env GOMODCACHE)" >> $GITHUB_ENV
82+
# - name: Set up cache
83+
# # https://github.com/actions/setup-go/issues/358 - cache is not working with setup-go for multiple jobs
84+
# uses: actions/cache@v4
85+
# with:
86+
# path: |
87+
# ${{ env.go_cache }}
88+
# ${{ env.go_modcache }}
89+
# bin/k8s
90+
# key: ${{ env.cache_name }}-${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}-${{ hashFiles('**/go.mod') }}
91+
# restore-keys: |
92+
# ${{ env.cache_name }}-${{ runner.os }}-go-
93+
# env:
94+
# cache_name: integration-test
95+
#
96+
# - name: Create k8s Kind Cluster
97+
# uses: helm/kind-action@v1
98+
# with:
99+
# config: test/e2e/config/kind-config.yaml
100+
#
101+
# - name: Install internal image registry
102+
# run: |
103+
# kubectl apply -f test/e2e/config/image-registry.yaml
104+
# kubectl wait pod -l app=registry --for condition=Ready --timeout 5m
105+
# kubectl wait pod -l app=protected-registry1 --for condition=Ready --timeout 5m
106+
# kubectl wait pod -l app=protected-registry2 --for condition=Ready --timeout 5m
107+
#
108+
# - name: Install external CRDs
109+
# run: kubectl apply --server-side -k https://github.com/openfluxcd/artifact//config/crd?ref=v0.1.1
110+
#
111+
# - name: Checkout helm-controller
112+
# uses: actions/checkout@v4
113+
# with:
114+
# repository: openfluxcd/helm-controller
115+
# path: helm-controller
116+
#
117+
# # TODO: Create helm-controller image in public repository to omit rebuilds
118+
# - name: Install helm-controller
119+
# env:
120+
# IMG: localhost:31000/helm-controller:latest
121+
# run: |
122+
# make -C helm-controller docker-build
123+
# make -C helm-controller docker-push
124+
# make -C helm-controller install
125+
# make -C helm-controller deploy
126+
# kubectl wait deployment.apps/helm-controller --for condition=Available --namespace helm-system --timeout 5m
127+
# kubectl logs --tail -1 -l app=helm-controller -n helm-system -f --ignore-errors &> helm-controller.log &
128+
#
129+
# - name: Checkout kustomize-controller
130+
# uses: actions/checkout@v4
131+
# with:
132+
# repository: openfluxcd/kustomize-controller
133+
# path: kustomize-controller
134+
#
135+
# # TODO: Create kustomize-controller image in public repository to omit rebuilds
136+
# - name: Install kustomize-controller
137+
# env:
138+
# IMG: localhost:31000/kustomize-controller:latest
139+
# run: |
140+
# make -C kustomize-controller docker-build
141+
# make -C kustomize-controller docker-push
142+
# make -C kustomize-controller install
143+
# make -C kustomize-controller deploy
144+
# kubectl wait deployment.apps/kustomize-controller --for condition=Available --namespace kustomize-system --timeout 5m
145+
# kubectl logs --tail -1 -l app=kustomize-controller -n kustomize-system -f --ignore-errors &> kustomize-controller.log &
146+
#
147+
# # TODO: Replace once the release with the 'skipDigestGeneration' field in the component constructor is available
148+
# # uses: open-component-model/ocm-setup-action@main
149+
# # with:
150+
# # version: v0.19.0-rc.1
151+
# - name: Set up cache for ocm (temporarily)
152+
# uses: actions/cache@v4
153+
# with:
154+
# path: |
155+
# ocm/bin
156+
# key: ${{ env.cache_name }}-${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}-${{ hashFiles('**/go.mod') }}
157+
# restore-keys: |
158+
# ${{ env.cache_name }}-${{ runner.os }}-go-
159+
# env:
160+
# cache_name: dummy-cache
161+
# - name: Checkout OCM (temporarily)
162+
# uses: actions/checkout@v4
163+
# with:
164+
# repository: open-component-model/ocm
165+
# path: ocm
166+
# - name: Build OCM (temporarily)
167+
# run: |
168+
# make -C ocm bin/ocm
169+
# echo "${{ github.workspace }}/ocm/bin" >> "$GITHUB_PATH"
170+
#
171+
# - name: Run e2e test
172+
# env:
173+
# RESOURCE_TIMEOUT: 5m
174+
# HELM_CHART: ghcr.io/stefanprodan/charts/podinfo:6.7.1
175+
# IMAGE_REFERENCE: ghcr.io/stefanprodan/podinfo:6.7.1
176+
# CONTROLLER_LOG_PATH: ./ocm-k8s-toolkit-controller.log
177+
# IMAGE_REGISTRY_URL: http://localhost:31000
178+
# INTERNAL_IMAGE_REGISTRY_URL: http://registry-internal.default.svc.cluster.local:5000
179+
# PROTECTED_REGISTRY_URL: http://localhost:31001
180+
# INTERNAL_PROTECTED_REGISTRY_URL: http://protected-registry1-internal.default.svc.cluster.local:5001
181+
# PROTECTED_REGISTRY_URL2: http://localhost:31002
182+
# INTERNAL_PROTECTED_REGISTRY_URL2: http://protected-registry2-internal.default.svc.cluster.local:5002
183+
# run: make test-e2e
184+
#
185+
# - name: Publish logs on failure
186+
# if: failure()
187+
# uses: actions/upload-artifact@v4
188+
# with:
189+
# name: controller-logs
190+
# # Currently, it is planned that the integration tests runs on every commit on a PR. Therefore, we could
191+
# # produce a lot of logs. To note clutter the storage, the retention-days are reduced to 1.
192+
# retention-days: 1
193+
# path: |
194+
# helm-controller.log
195+
# kustomize-controller.log
196+
# ocm-k8s-toolkit-controller.log

‎Makefile

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,10 @@ else
1010
GOBIN=$(shell go env GOBIN)
1111
endif
1212

13+
OS ?= $(shell go env GOOS)
14+
ARCH ?= $(shell go env GOARCH)
15+
16+
1317
# CONTAINER_TOOL defines the container tool to be used for building images.
1418
# Be aware that the target commands are only tested with Docker which is
1519
# scaffolded by default. However, you might want to replace it to use other
@@ -64,7 +68,7 @@ vet: ## Run go vet against code.
6468
go vet ./...
6569

6670
.PHONY: test
67-
test: manifests generate envtest ## Run tests.
71+
test: manifests generate envtest zot-registry ## Run tests.
6872
KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out
6973

7074
# Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors.
@@ -178,6 +182,7 @@ KUSTOMIZE_VERSION ?= v5.4.1
178182
CONTROLLER_TOOLS_VERSION ?= v0.16.0
179183
ENVTEST_VERSION ?= release-0.18
180184
GOLANGCI_LINT_VERSION ?= v1.61.0
185+
ZOT_VERSION ?= v2.1.2
181186

182187
## ZOT OCI Registry
183188
ZOT_VERSION ?= v2.1.2
@@ -213,6 +218,12 @@ deploy-cert-manager: ## Deploy cert-manager to the K8s cluster specified in ~/.k
213218
undeploy-cert-manager: ## Undeploy cert-manager from the K8s cluster specified in ~/.kube/config.
214219
$(KUBECTL) delete --ignore-not-found=$(IGNORE_NOT_FOUND) -f $(CERT-MANAGER_YAML)
215220

221+
.PHONY: zot-registry
222+
zot-registry: $(LOCALBIN) # Download zot registry binary locally if necessary.
223+
wget "https://github.com/project-zot/zot/releases/download/$(ZOT_VERSION)/zot-$(OS)-$(ARCH)-minimal" \
224+
-O $(LOCALBIN)/zot-registry \
225+
&& chmod u+x $(LOCALBIN)/zot-registry
226+
216227
.PHONY: golangci-lint
217228
golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary.
218229
$(GOLANGCI_LINT): $(LOCALBIN)

‎PROJECT

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,4 +71,13 @@ resources:
7171
kind: Replication
7272
path: github.com/open-component-model/ocm-k8s-toolkit/api/v1alpha1
7373
version: v1alpha1
74-
version: "3"
74+
- api:
75+
crdVersion: v1
76+
namespaced: true
77+
controller: true
78+
domain: ocm.software
79+
group: delivery
80+
kind: Snapshot
81+
path: github.com/open-component-model/ocm-k8s-toolkit/api/v1alpha1
82+
version: v1alpha1
83+
version: "3"

‎api/v1alpha1/common_types.go

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,3 +113,15 @@ type ResourceInfo struct {
113113
// +required
114114
Digest string `json:"digest,omitempty"`
115115
}
116+
117+
type BlobInfo struct {
118+
// Digest is the digest of the blob in the form of '<algorithm>:<checksum>'.
119+
Digest string `json:"digest"`
120+
121+
// Tag/Version of the blob
122+
Tag string `json:"tag"`
123+
124+
// Size is the number of bytes of the blob.
125+
// Can be used to determine how to file should be handled when downloaded (memory/disk)
126+
Size int64 `json:"size"`
127+
}

‎api/v1alpha1/component_types.go

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -100,11 +100,11 @@ type ComponentStatus struct {
100100
// +optional
101101
Conditions []metav1.Condition `json:"conditions,omitempty"`
102102

103-
// ArtifactRef references the generated artifact containing a list of
103+
// SnapshotRef references the generated snapshot containing a list of
104104
// component descriptors. This list can be used by other controllers to
105105
// avoid re-downloading (and potentially also re-verifying) the components.
106106
// +optional
107-
ArtifactRef corev1.LocalObjectReference `json:"artifactRef,omitempty"`
107+
SnapshotRef corev1.LocalObjectReference `json:"snapshotRef,omitempty"`
108108

109109
// Component specifies the concrete version of the component that was
110110
// fetched after based on the semver constraints during the last successful
@@ -180,6 +180,10 @@ func (in *Component) GetVerifications() []Verification {
180180
return in.Spec.Verify
181181
}
182182

183+
func (in *Component) GetSnapshotName() string {
184+
return in.Status.SnapshotRef.Name
185+
}
186+
183187
// +kubebuilder:object:root=true
184188

185189
// ComponentList contains a list of Component.

0 commit comments

Comments
 (0)