Skip to content

Add support for container build compression #5964

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion plugins/nf-wave/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ dependencies {
api 'org.apache.commons:commons-lang3:3.12.0'
api 'com.google.code.gson:gson:2.10.1'
api 'org.yaml:snakeyaml:2.2'
api 'io.seqera:wave-api:0.15.1'
api 'io.seqera:wave-api:0.16.0'
api 'io.seqera:wave-utils:0.15.1'

testImplementation(testFixtures(project(":nextflow")))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ package io.seqera.wave.plugin
import groovy.transform.CompileStatic
import groovy.transform.EqualsAndHashCode
import groovy.transform.ToString
import io.seqera.wave.api.BuildCompression
import io.seqera.wave.api.ImageNameStrategy
import io.seqera.wave.api.PackagesSpec
import io.seqera.wave.api.ScanLevel
Expand Down Expand Up @@ -154,4 +155,9 @@ class SubmitContainerTokenRequest {
*/
List<ScanLevel> scanLevels

/**
* Model build compression option
*/
BuildCompression buildCompression

}
Original file line number Diff line number Diff line change
Expand Up @@ -230,7 +230,8 @@ class WaveClient {
dryRun: ContainerInspectMode.dryRun(),
mirror: config.mirrorMode(),
scanMode: config.scanMode(),
scanLevels: config.scanAllowedLevels()
scanLevels: config.scanAllowedLevels(),
buildCompression: config.buildCompression()
)
}

Expand All @@ -257,7 +258,8 @@ class WaveClient {
dryRun: ContainerInspectMode.dryRun(),
mirror: config.mirrorMode(),
scanMode: config.scanMode(),
scanLevels: config.scanAllowedLevels()
scanLevels: config.scanAllowedLevels(),
buildCompression: config.buildCompression()
)
return sendRequest(request)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ package io.seqera.wave.plugin.config
import groovy.transform.CompileStatic
import groovy.transform.ToString
import groovy.util.logging.Slf4j
import io.seqera.wave.api.BuildCompression
import io.seqera.wave.api.ScanLevel
import io.seqera.wave.api.ScanMode
import io.seqera.wave.config.CondaOpts
Expand Down Expand Up @@ -53,6 +54,7 @@ class WaveConfig {
final private Boolean mirrorMode
final private ScanMode scanMode
final private List<ScanLevel> scanAllowedLevels
final private BuildCompression buildCompression

WaveConfig(Map opts, Map<String,String> env=System.getenv()) {
this.enabled = opts.enabled
Expand All @@ -72,6 +74,7 @@ class WaveConfig {
this.buildMaxDuration = opts.navigate('build.maxDuration', '40m') as Duration
this.scanMode = opts.navigate('scan.mode') as ScanMode
this.scanAllowedLevels = parseScanLevels(opts.navigate('scan.allowedLevels'))
this.buildCompression = parseCompression(opts.navigate('build.compression') as Map)
// some validation
validateConfig()
}
Expand Down Expand Up @@ -102,6 +105,8 @@ class WaveConfig {

Duration buildMaxDuration() { buildMaxDuration }

BuildCompression buildCompression() { buildCompression }

private void validateConfig() {
def scheme= FileHelper.getUrlProtocol(endpoint)
if( scheme !in ['http','https'] )
Expand Down Expand Up @@ -191,4 +196,17 @@ class WaveConfig {
}
throw new IllegalArgumentException("Invalid value for 'wave.scan.levels' setting - offending value: $value; type: ${value.getClass().getName()}")
}

protected BuildCompression parseCompression(Map opts) {
if( !opts )
return null
final result = new BuildCompression()
if( opts.mode )
result.mode = BuildCompression.Mode.valueOf(opts.mode.toString().toLowerCase())
if( opts.level )
result.level = opts.level as Integer
if( opts.force )
result.force = opts.force as Boolean
return result
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ import com.sun.net.httpserver.HttpServer
import groovy.json.JsonOutput
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import io.seqera.wave.api.BuildCompression
import io.seqera.wave.api.BuildStatusResponse
import io.seqera.wave.api.ContainerStatus
import io.seqera.wave.api.ContainerStatusResponse
Expand Down Expand Up @@ -281,6 +282,27 @@ class WaveClientTest extends Specification {
req.timestamp instanceof String
}

def 'should create request object with build compression' () {
given:
def session = Mock(Session) { getConfig() >> [wave:[build:[compression:[mode:'estargz', level:11]]]]}
def IMAGE = 'foo:latest'
def wave = new WaveClient(session)

when:
def req = wave.makeRequest(WaveAssets.fromImage(IMAGE))
then:
req.containerImage == IMAGE
!req.containerPlatform
!req.containerFile
!req.condaFile
!req.containerConfig.layers
and:
req.buildCompression == new BuildCompression().withMode(BuildCompression.Mode.estargz).withLevel(11)
and:
req.fingerprint == 'bd2cb4b32df41f2d290ce2366609f2ad'
req.timestamp instanceof String
}

def 'should create request object with dry-run mode' () {
given:
ContainerInspectMode.activate(true)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

package io.seqera.wave.plugin.config

import io.seqera.wave.api.BuildCompression
import io.seqera.wave.api.ScanLevel
import io.seqera.wave.api.ScanMode
import nextflow.util.Duration
Expand Down Expand Up @@ -185,7 +186,7 @@ class WaveConfigTest extends Specification {
given:
def config = new WaveConfig([enabled: true])
expect:
config.toString() == 'WaveConfig(enabled:true, endpoint:https://wave.seqera.io, containerConfigUrl:[], tokensCacheMaxDuration:30m, condaOpts:CondaOpts(mambaImage=mambaorg/micromamba:1.5.10-noble; basePackages=conda-forge::procps-ng, commands=null), strategy:[container, dockerfile, conda], bundleProjectResources:null, buildRepository:null, cacheRepository:null, retryOpts:RetryOpts(delay:450ms, maxDelay:1m 30s, maxAttempts:10, jitter:0.25), httpClientOpts:HttpOpts(), freezeMode:null, preserveFileTimestamp:null, buildMaxDuration:40m, mirrorMode:null, scanMode:null, scanAllowedLevels:null)'
config.toString() == 'WaveConfig(enabled:true, endpoint:https://wave.seqera.io, containerConfigUrl:[], tokensCacheMaxDuration:30m, condaOpts:CondaOpts(mambaImage=mambaorg/micromamba:1.5.10-noble; basePackages=conda-forge::procps-ng, commands=null), strategy:[container, dockerfile, conda], bundleProjectResources:null, buildRepository:null, cacheRepository:null, retryOpts:RetryOpts(delay:450ms, maxDelay:1m 30s, maxAttempts:10, jitter:0.25), httpClientOpts:HttpOpts(), freezeMode:null, preserveFileTimestamp:null, buildMaxDuration:40m, mirrorMode:null, scanMode:null, scanAllowedLevels:null, buildCompression:null)'
}

def 'should not allow invalid setting' () {
Expand Down Expand Up @@ -257,7 +258,24 @@ class WaveConfigTest extends Specification {
'low,high' | List.of(ScanLevel.LOW,ScanLevel.HIGH)
'LOW, HIGH' | List.of(ScanLevel.LOW,ScanLevel.HIGH)
['medium','high'] | List.of(ScanLevel.MEDIUM,ScanLevel.HIGH)
}

def 'should validate build compression' () {
expect:
new WaveConfig(build: [compression: COMPRESSION]).buildCompression() == EXPECTED
where:
COMPRESSION | EXPECTED
null | null
[mode:'gzip'] | BuildCompression.gzip
[mode:'estargz'] | BuildCompression.estargz
and:
[mode:'gzip', level: 1] | new BuildCompression().withMode(BuildCompression.Mode.gzip).withLevel(1)
[mode:'estargz', level: 2]| new BuildCompression().withMode(BuildCompression.Mode.estargz).withLevel(2)
[mode:'zstd', level: 3] | new BuildCompression().withMode(BuildCompression.Mode.zstd).withLevel(3)
and:
[mode:'gzip', level: 1, force:true] | new BuildCompression().withMode(BuildCompression.Mode.gzip).withLevel(1).withForce(true)
[mode:'estargz', level: 2, force:true ] | new BuildCompression().withMode(BuildCompression.Mode.estargz).withLevel(2).withForce(true)
[mode:'zstd', level: 3,force:true] | new BuildCompression().withMode(BuildCompression.Mode.zstd).withLevel(3).withForce(true)
}

}