diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bf55878..e755f97 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,16 +15,18 @@ on: jobs: build: - name: Build and Test runs-on: ubuntu-latest + strategy: + matrix: + java: ['11', '16', '17' ] + name: Java ${{ matrix.java }} Build and Test steps: - name: Set up java - uses: actions/setup-java@v1 + uses: actions/setup-java@v2 with: - java-version: 11.0.2 - java-package: jdk - architecture: x64 + distribution: 'temurin' + java-version: ${{ matrix.java }} - name: Set up python uses: actions/setup-python@v2 with: @@ -39,7 +41,7 @@ jobs: - name: Build with Gradle run: ./gradlew build -x test - name: Test - run: ./gradlew check + run: ./gradlew --info check - name: Publish Unit Test Results uses: EnricoMi/publish-unit-test-result-action@v1.23 if: always() diff --git a/.github/workflows/cwl_compliance.yml b/.github/workflows/cwl_compliance.yml index 966b2ed..f931ec5 100644 --- a/.github/workflows/cwl_compliance.yml +++ b/.github/workflows/cwl_compliance.yml @@ -19,11 +19,10 @@ jobs: steps: - name: Set up java - uses: actions/setup-java@v1 + uses: actions/setup-java@v2 with: - java-version: 11.0.2 - java-package: jdk - architecture: x64 + distribution: 'temurin' + java-version: '17' - name: Set up python uses: actions/setup-python@v2 with: diff --git a/.gitignore b/.gitignore index 83e5feb..ffb8e40 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,12 @@ build/ .settings *.iws .DS_Store +.vscode +# Ignore yarn +src/main/frontend/yarn-v1.22.0/ + +# Xenonflow logs log/ db/ xenon-flow.log diff --git a/build.gradle b/build.gradle index f8645e5..4238081 100644 --- a/build.gradle +++ b/build.gradle @@ -4,7 +4,7 @@ plugins { id 'idea' id 'eclipse' id 'application' - id 'org.springframework.boot' version '2.2.4.RELEASE' + id 'org.springframework.boot' version '2.3.12.RELEASE' id 'io.spring.dependency-management' version '1.0.7.RELEASE' id 'jacoco' id "com.github.node-gradle.node" version "2.2.2" @@ -15,7 +15,7 @@ sourceCompatibility = JavaVersion.VERSION_11 targetCompatibility = JavaVersion.VERSION_11 application { - mainClassName = 'nl.esciencecenter.computeservice.rest.Application' + mainClass = 'nl.esciencecenter.computeservice.rest.Application' } @@ -43,26 +43,27 @@ dependencies { implementation group: 'nl.esciencecenter.xenon.adaptors', name: 'xenon-adaptors-cloud', version: '3.0.2' //Spring(boot) - implementation group: 'org.springframework.boot', name: 'spring-boot-starter-web', version: '2.2.4.RELEASE' - implementation group: 'org.springframework.boot', name: 'spring-boot-starter', version: '2.2.4.RELEASE' - implementation group: 'org.springframework.boot', name: 'spring-boot-starter-tomcat', version: '2.2.4.RELEASE' - implementation group: 'org.springframework.boot', name: 'spring-boot-devtools', version: '2.2.4.RELEASE' + implementation group: 'org.springframework.boot', name: 'spring-boot-starter-web', version: '2.3.12.RELEASE' + implementation group: 'org.springframework.boot', name: 'spring-boot-starter', version: '2.3.12.RELEASE' + implementation group: 'org.springframework.boot', name: 'spring-boot-starter-tomcat', version: '2.3.12.RELEASE' + implementation group: 'org.springframework.boot', name: 'spring-boot-devtools', version: '2.3.12.RELEASE' + implementation group: 'org.springframework.boot', name: 'spring-boot-starter-validation', version: '2.3.12.RELEASE' implementation group: 'io.springfox', name: 'springfox-boot-starter', version: '3.0.0' //Jackson, match version used in Spring - implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.9.8' - implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-yaml', version: '2.9.8' - implementation group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version: '2.9.8' + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.11.4' + implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-yaml', version: '2.11.4' + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version: '2.11.4' //compile group: 'com.fasterxml.jackson.datatype', name: 'jackson-datatype-jsr310' implementation group: 'javax.validation', name: 'validation-api', version: '2.0.1.Final' //JPA, hibernate and h2 database persistance libraries - implementation group: 'org.springframework.boot', name: 'spring-boot-starter-data-jpa', version: '2.2.4.RELEASE' + implementation group: 'org.springframework.boot', name: 'spring-boot-starter-data-jpa', version: '2.3.12.RELEASE' runtimeOnly group: 'com.h2database', name: 'h2', version: '1.4.199' // Security - implementation group: 'org.springframework.boot', name: 'spring-boot-starter-security', version: '2.2.4.RELEASE' + implementation group: 'org.springframework.boot', name: 'spring-boot-starter-security', version: '2.3.12.RELEASE' // Admin Interface @@ -75,18 +76,20 @@ dependencies { testImplementation group: 'org.hamcrest', name:'hamcrest', version:'2.2' testImplementation group: 'org.hamcrest', name:'hamcrest-library', version:'2.2' testImplementation group: 'junit', name:'junit', version:'4.13' - testImplementation group: 'org.springframework.boot', name: 'spring-boot-starter-test', version: '2.2.4.RELEASE' + testImplementation group: 'org.springframework.boot', name: 'spring-boot-starter-test', version: '2.3.12.RELEASE' + + runtimeOnly group: 'org.springframework.boot', name: 'spring-boot-properties-migrator', version: '2.3.12.RELEASE' } bootJar { - baseName = 'xenonflow' + baseName = 'xenonflow' } // Include some more files in the distribution zip // Don't use include an extra directory in the dist zip. distributions { boot { - baseName = 'xenonflow' + distributionBaseName = 'xenonflow' contents { from(['config/config.yml', 'config/application.properties']) { into '/config' diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 442d913..e750102 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.8.3-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.3-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/src/main/java/nl/esciencecenter/computeservice/service/staging/BaseStagingObject.java b/src/main/java/nl/esciencecenter/computeservice/service/staging/BaseStagingObject.java index ebd4a17..2a0c997 100644 --- a/src/main/java/nl/esciencecenter/computeservice/service/staging/BaseStagingObject.java +++ b/src/main/java/nl/esciencecenter/computeservice/service/staging/BaseStagingObject.java @@ -2,7 +2,7 @@ import org.commonwl.cwl.Parameter; -public class BaseStagingObject implements StagingObject { +public abstract class BaseStagingObject implements StagingObject { protected long bytesCopied = 0; protected String copyId; diff --git a/src/main/java/nl/esciencecenter/computeservice/service/staging/CommandScriptStagingObject.java b/src/main/java/nl/esciencecenter/computeservice/service/staging/CommandScriptStagingObject.java index 1eca07d..9030e0e 100644 --- a/src/main/java/nl/esciencecenter/computeservice/service/staging/CommandScriptStagingObject.java +++ b/src/main/java/nl/esciencecenter/computeservice/service/staging/CommandScriptStagingObject.java @@ -1,13 +1,32 @@ package nl.esciencecenter.computeservice.service.staging; -import org.commonwl.cwl.Parameter; +import java.util.Set; +import org.slf4j.Logger; + +import nl.esciencecenter.xenon.XenonException; +import nl.esciencecenter.xenon.filesystems.FileSystem; import nl.esciencecenter.xenon.filesystems.Path; +import nl.esciencecenter.xenon.filesystems.PosixFilePermission; public class CommandScriptStagingObject extends StringToFileStagingObject { - public CommandScriptStagingObject(String source, Path targetPath, Parameter parameter) { - super(source, targetPath, parameter); + public CommandScriptStagingObject(String source, Path targetPath) { + super(source, targetPath, null); } + public String stage(Logger jobLogger, FileSystem sourceFileSystem, FileSystem targetFileSystem, + Path sourceDirectory, Path targetDirectory) throws XenonException { + String id = super.stage(jobLogger, sourceFileSystem, targetFileSystem, sourceDirectory, targetDirectory); + + Path targetPath = targetDirectory.resolve(getTargetPath()); + + Set permissions = targetFileSystem.getAttributes(targetPath).getPermissions(); + permissions.add(PosixFilePermission.OWNER_EXECUTE); + permissions.add(PosixFilePermission.GROUP_EXECUTE); + permissions.add(PosixFilePermission.OTHERS_EXECUTE); + targetFileSystem.setPosixFilePermissions(targetPath, permissions); + + return id; + } } diff --git a/src/main/java/nl/esciencecenter/computeservice/service/staging/CwlFileStagingObject.java b/src/main/java/nl/esciencecenter/computeservice/service/staging/CwlFileStagingObject.java index c820328..08d6d4d 100644 --- a/src/main/java/nl/esciencecenter/computeservice/service/staging/CwlFileStagingObject.java +++ b/src/main/java/nl/esciencecenter/computeservice/service/staging/CwlFileStagingObject.java @@ -1,33 +1,17 @@ package nl.esciencecenter.computeservice.service.staging; -import org.commonwl.cwl.Parameter; +import org.slf4j.Logger; +import nl.esciencecenter.xenon.XenonException; +import nl.esciencecenter.xenon.filesystems.FileSystem; import nl.esciencecenter.xenon.filesystems.Path; -public class CwlFileStagingObject extends BaseStagingObject { - private Path sourcePath; - private Path targetPath; +public class CwlFileStagingObject extends FileStagingObject { + protected FileSystem cwlFileSystem; - public CwlFileStagingObject(Path sourcePath, Path targetPath, Parameter parameter) { - super(parameter); - this.sourcePath = sourcePath; - this.targetPath = targetPath; - } - - public Path getSourcePath() { - return sourcePath; - } - - public void setSourcePath(Path sourcePath) { - this.sourcePath = sourcePath; - } - - public Path getTargetPath() { - return targetPath; - } - - public void setTargetPath(Path targetPath) { - this.targetPath = targetPath; + public CwlFileStagingObject(Path sourcePath, Path targetPath, FileSystem cwlFileSystem) { + super(sourcePath, targetPath, null); + this.cwlFileSystem = cwlFileSystem; } @Override @@ -35,4 +19,9 @@ public String toString() { return "CwlFileStagingObject [sourcePath=" + sourcePath + ", targetPath=" + targetPath + ", parameter=" + parameter + "]"; } + + public String stage(Logger jobLogger, FileSystem sourceFileSystem, FileSystem targetFileSystem, + Path sourceDirectory, Path targetDirectory) throws XenonException { + return super.stage(jobLogger, cwlFileSystem, targetFileSystem, sourceDirectory, targetDirectory); + } } \ No newline at end of file diff --git a/src/main/java/nl/esciencecenter/computeservice/service/staging/DirectoryStagingObject.java b/src/main/java/nl/esciencecenter/computeservice/service/staging/DirectoryStagingObject.java index 2175bdf..67d90a9 100644 --- a/src/main/java/nl/esciencecenter/computeservice/service/staging/DirectoryStagingObject.java +++ b/src/main/java/nl/esciencecenter/computeservice/service/staging/DirectoryStagingObject.java @@ -1,7 +1,11 @@ package nl.esciencecenter.computeservice.service.staging; import org.commonwl.cwl.Parameter; +import org.slf4j.Logger; +import nl.esciencecenter.xenon.XenonException; +import nl.esciencecenter.xenon.filesystems.CopyMode; +import nl.esciencecenter.xenon.filesystems.FileSystem; import nl.esciencecenter.xenon.filesystems.Path; public class DirectoryStagingObject extends BaseStagingObject implements FileOrDirectoryStagingObject { @@ -35,4 +39,24 @@ public String toString() { return "DirectoryStagingObject [sourcePath=" + sourcePath + ", targetPath=" + targetPath + ", parameter=" + parameter + "]"; } + + public String stage(Logger jobLogger, FileSystem sourceFileSystem, FileSystem targetFileSystem, + Path sourceDirectory, Path targetDirectory) throws XenonException { + Path sourcePath = getSourcePath(); + if (!sourcePath.isAbsolute()) { + sourcePath = sourceDirectory.resolve(getSourcePath()).toAbsolutePath(); + } + Path targetPath = targetDirectory.resolve(getTargetPath()); + Path targetDir = targetPath.getParent(); + if (!targetFileSystem.exists(targetDir)) { + targetFileSystem.createDirectories(targetDir); + } + + jobLogger.info("Copying from " + sourcePath + " to " + targetPath); + String copyId = sourceFileSystem.copy(sourcePath, targetFileSystem, targetPath, CopyMode.REPLACE, + true); + + setCopyId(copyId); + return copyId; + } } diff --git a/src/main/java/nl/esciencecenter/computeservice/service/staging/FileStagingObject.java b/src/main/java/nl/esciencecenter/computeservice/service/staging/FileStagingObject.java index f7b7bc7..4c41116 100644 --- a/src/main/java/nl/esciencecenter/computeservice/service/staging/FileStagingObject.java +++ b/src/main/java/nl/esciencecenter/computeservice/service/staging/FileStagingObject.java @@ -1,12 +1,16 @@ package nl.esciencecenter.computeservice.service.staging; import org.commonwl.cwl.Parameter; +import org.slf4j.Logger; +import nl.esciencecenter.xenon.XenonException; +import nl.esciencecenter.xenon.filesystems.CopyMode; +import nl.esciencecenter.xenon.filesystems.FileSystem; import nl.esciencecenter.xenon.filesystems.Path; public class FileStagingObject extends BaseStagingObject implements FileOrDirectoryStagingObject { - private Path sourcePath; - private Path targetPath; + protected Path sourcePath; + protected Path targetPath; public FileStagingObject(Path sourcePath, Path targetPath, Parameter parameter) { super(parameter); @@ -35,4 +39,24 @@ public String toString() { return "FileStagingObject [sourcePath=" + sourcePath + ", targetPath=" + targetPath + ", parameter=" + parameter + "]"; } + + public String stage(Logger jobLogger, FileSystem sourceFileSystem, FileSystem targetFileSystem, + Path sourceDirectory, Path targetDirectory) throws XenonException { + Path sourcePath = getSourcePath(); + if (!sourcePath.isAbsolute()) { + sourcePath = sourceDirectory.resolve(getSourcePath()).toAbsolutePath(); + } + Path targetPath = targetDirectory.resolve(getTargetPath()); + Path targetDir = targetPath.getParent(); + if (!targetFileSystem.exists(targetDir)) { + targetFileSystem.createDirectories(targetDir); + } + + jobLogger.info("Copying from " + sourcePath + " to " + targetPath); + String copyId = sourceFileSystem.copy(sourcePath, targetFileSystem, targetPath, CopyMode.REPLACE, + false); + + setCopyId(copyId); + return copyId; + } } \ No newline at end of file diff --git a/src/main/java/nl/esciencecenter/computeservice/service/staging/FileToMapStagingObject.java b/src/main/java/nl/esciencecenter/computeservice/service/staging/FileToMapStagingObject.java deleted file mode 100644 index 7ebdcfd..0000000 --- a/src/main/java/nl/esciencecenter/computeservice/service/staging/FileToMapStagingObject.java +++ /dev/null @@ -1,38 +0,0 @@ -package nl.esciencecenter.computeservice.service.staging; - -import org.commonwl.cwl.Parameter; - -import nl.esciencecenter.xenon.filesystems.Path; - -public class FileToMapStagingObject extends BaseStagingObject { - private Path sourcePath; - private String targetString; - - public FileToMapStagingObject(Path sourcePath, String targetString, Parameter parameter) { - super(parameter); - this.sourcePath = sourcePath; - this.targetString = targetString; - } - - public Path getSourcePath() { - return sourcePath; - } - - public void setSourcePath(Path sourcePath) { - this.sourcePath = sourcePath; - } - - public String getTargetString() { - return targetString; - } - - public void setTargetString(String targetString) { - this.targetString = targetString; - } - - @Override - public String toString() { - return "FileToMapStagingObject [sourcePath=" + sourcePath + ", targetString=" + targetString - + ", parameter=" + parameter + "]"; - } -} diff --git a/src/main/java/nl/esciencecenter/computeservice/service/staging/FileToStringStagingObject.java b/src/main/java/nl/esciencecenter/computeservice/service/staging/FileToStringStagingObject.java deleted file mode 100644 index d92b051..0000000 --- a/src/main/java/nl/esciencecenter/computeservice/service/staging/FileToStringStagingObject.java +++ /dev/null @@ -1,38 +0,0 @@ -package nl.esciencecenter.computeservice.service.staging; - -import org.commonwl.cwl.Parameter; - -import nl.esciencecenter.xenon.filesystems.Path; - -public class FileToStringStagingObject extends BaseStagingObject { - private Path sourcePath; - private String targetString; - - public FileToStringStagingObject(Path sourcePath, String targetString, Parameter parameter) { - super(parameter); - this.sourcePath = sourcePath; - this.targetString = targetString; - } - - public Path getSourcePath() { - return sourcePath; - } - - public void setSourcePath(Path sourcePath) { - this.sourcePath = sourcePath; - } - - public String getTargetString() { - return targetString; - } - - public void setTargetString(String targetString) { - this.targetString = targetString; - } - - @Override - public String toString() { - return "FileToStringStagingObject [sourcePath=" + sourcePath + ", targetString=" + targetString - + ", parameter=" + parameter + "]"; - } -} \ No newline at end of file diff --git a/src/main/java/nl/esciencecenter/computeservice/service/staging/StagingManifestFactory.java b/src/main/java/nl/esciencecenter/computeservice/service/staging/StagingManifestFactory.java index 1f456bc..60c0ddb 100644 --- a/src/main/java/nl/esciencecenter/computeservice/service/staging/StagingManifestFactory.java +++ b/src/main/java/nl/esciencecenter/computeservice/service/staging/StagingManifestFactory.java @@ -47,12 +47,12 @@ public static StagingManifest createStagingInManifest(Job job, FileSystem cwlFil } if (cwlCommandScript == null) { - manifest.add(new CommandScriptStagingObject("#!/usr/bin/env bash\n\ncwltool $@", new Path("cwlcommand"), null)); + manifest.add(new CommandScriptStagingObject("#!/usr/bin/env bash\n\ncwltool $@", new Path("cwlcommand"))); } else { - manifest.add(new CommandScriptStagingObject(cwlCommandScript, new Path("cwlcommand"), null)); + manifest.add(new CommandScriptStagingObject(cwlCommandScript, new Path("cwlcommand"))); } - manifest.add(new CwlFileStagingObject(wfd.localPath, wfd.workflowBaseName, null)); + manifest.add(new CwlFileStagingObject(wfd.localPath, wfd.workflowBaseName, cwlFileSystem)); addSubWorkflowsToManifest(wfd.workflow, manifest, wfd.workflowBasePath, cwlFileSystem, jobLogger); WorkflowBinding newInput = addInputToManifest(job, wfd.workflow, manifest, jobLogger); @@ -117,7 +117,7 @@ public static void addSubWorkflowsToManifest(Workflow workflow, StagingManifest localPath = workflowBasePath.resolve(path); } Path remotePath = path; - manifest.add(new CwlFileStagingObject(localPath, remotePath, null)); + manifest.add(new CwlFileStagingObject(localPath, remotePath, fileSystem)); } } diff --git a/src/main/java/nl/esciencecenter/computeservice/service/staging/StagingObject.java b/src/main/java/nl/esciencecenter/computeservice/service/staging/StagingObject.java index 5f82ea6..58c38d9 100644 --- a/src/main/java/nl/esciencecenter/computeservice/service/staging/StagingObject.java +++ b/src/main/java/nl/esciencecenter/computeservice/service/staging/StagingObject.java @@ -1,6 +1,11 @@ package nl.esciencecenter.computeservice.service.staging; import org.commonwl.cwl.Parameter; +import org.slf4j.Logger; + +import nl.esciencecenter.xenon.XenonException; +import nl.esciencecenter.xenon.filesystems.FileSystem; +import nl.esciencecenter.xenon.filesystems.Path; public interface StagingObject { @@ -14,4 +19,7 @@ public interface StagingObject { public void setParameter(Parameter parameter); public Parameter getParameter(); + + public String stage(Logger jobLogger, FileSystem sourceFileSystem, FileSystem targetFileSystem, + Path sourceDirectory, Path targetDirectory) throws XenonException; } \ No newline at end of file diff --git a/src/main/java/nl/esciencecenter/computeservice/service/staging/StringToFileStagingObject.java b/src/main/java/nl/esciencecenter/computeservice/service/staging/StringToFileStagingObject.java index b31c4e4..d4b1956 100644 --- a/src/main/java/nl/esciencecenter/computeservice/service/staging/StringToFileStagingObject.java +++ b/src/main/java/nl/esciencecenter/computeservice/service/staging/StringToFileStagingObject.java @@ -1,12 +1,17 @@ package nl.esciencecenter.computeservice.service.staging; +import java.io.PrintWriter; + import org.commonwl.cwl.Parameter; +import org.slf4j.Logger; +import nl.esciencecenter.xenon.XenonException; +import nl.esciencecenter.xenon.filesystems.FileSystem; import nl.esciencecenter.xenon.filesystems.Path; public class StringToFileStagingObject extends BaseStagingObject { - private String sourceString; - private Path targetPath; + protected String sourceString; + protected Path targetPath; public StringToFileStagingObject(String source, Path targetPath, Parameter parameter) { super(parameter); @@ -38,4 +43,27 @@ public String toString() { + ", parameter=" + parameter + "]"; } + + public String stage(Logger jobLogger, FileSystem sourceFileSystem, FileSystem targetFileSystem, + Path sourceDirectory, Path targetDirectory) throws XenonException { + Path targetPath = targetDirectory.resolve(getTargetPath()); + Path targetDir = targetPath.getParent(); + + if (!targetFileSystem.exists(targetDir)) { + targetFileSystem.createDirectories(targetDir); + } + + jobLogger.info("Writing string to: " + targetPath); + String contents = getSourceString(); + + jobLogger.debug("Input contents: " + contents); + + PrintWriter out = new PrintWriter(targetFileSystem.writeToFile(targetPath)); + out.write(contents); + out.close(); + + setBytesCopied(contents.length()); + + return null; + } } \ No newline at end of file diff --git a/src/main/java/nl/esciencecenter/computeservice/service/staging/XenonStager.java b/src/main/java/nl/esciencecenter/computeservice/service/staging/XenonStager.java index e312f50..f2c45ae 100644 --- a/src/main/java/nl/esciencecenter/computeservice/service/staging/XenonStager.java +++ b/src/main/java/nl/esciencecenter/computeservice/service/staging/XenonStager.java @@ -1,7 +1,6 @@ package nl.esciencecenter.computeservice.service.staging; import java.io.IOException; -import java.io.PrintWriter; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; @@ -9,7 +8,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; import org.commonwl.cwl.CwlException; import org.commonwl.cwl.Parameter; @@ -28,11 +26,9 @@ import nl.esciencecenter.computeservice.service.XenonMonitor; import nl.esciencecenter.computeservice.service.XenonService; import nl.esciencecenter.xenon.XenonException; -import nl.esciencecenter.xenon.filesystems.CopyMode; import nl.esciencecenter.xenon.filesystems.CopyStatus; import nl.esciencecenter.xenon.filesystems.FileSystem; import nl.esciencecenter.xenon.filesystems.Path; -import nl.esciencecenter.xenon.filesystems.PosixFilePermission; import nl.esciencecenter.computeservice.utils.FileSystemUtils; public abstract class XenonStager { @@ -43,6 +39,10 @@ public abstract class XenonStager { protected XenonService xenonService; private HashMap copyMap; + /** + * Data class that holds references to the manifest + * and all ids created for staging this job. + */ private static class StagingJob { private final StagingManifest manifest; private final List copyIds; @@ -56,6 +56,9 @@ public StagingJob(StagingManifest manifest, List copyIds, List c } } + /** + * Data class for staging jobs out that also holds the exitcode + */ private static final class StagingOutJob extends StagingJob { private final int exitcode; @@ -105,106 +108,32 @@ public Pair,List> doStaging(StagingManifest manifest) throw } Path sourceDirectory = sourceFileSystem.getWorkingDirectory().toAbsolutePath(); - Path cwlDirectory = cwlFileSystem.getWorkingDirectory().toAbsolutePath(); List stagingIds = new LinkedList(); List cwlFileIds = new LinkedList(); // Copy all the files there for (StagingObject stageObject : manifest) { if (stageObject instanceof CwlFileStagingObject) { - CwlFileStagingObject object = (CwlFileStagingObject) stageObject; - Path sourcePath = object.getSourcePath(); - if (!sourcePath.isAbsolute()) { - sourcePath = cwlDirectory.resolve(object.getSourcePath()).toAbsolutePath(); - } - Path targetPath = targetDirectory.resolve(object.getTargetPath()); - Path targetDir = targetPath.getParent(); - if (!targetFileSystem.exists(targetDir)) { - targetFileSystem.createDirectories(targetDir); - } - - jobLogger.info("Copying from " + sourcePath + " to " + targetPath); - String copyId = cwlFileSystem.copy(sourcePath, targetFileSystem, targetPath, CopyMode.REPLACE, - false); - stageObject.setCopyId(copyId); + String copyId = stageObject.stage(jobLogger, cwlFileSystem, targetFileSystem, sourceDirectory, targetDirectory); cwlFileIds.add(copyId); - }else if (stageObject instanceof FileStagingObject) { - FileStagingObject object = (FileStagingObject) stageObject; - Path sourcePath = object.getSourcePath(); - if (!sourcePath.isAbsolute()) { - sourcePath = sourceDirectory.resolve(object.getSourcePath()).toAbsolutePath(); - } - Path targetPath = targetDirectory.resolve(object.getTargetPath()); - Path targetDir = targetPath.getParent(); - if (!targetFileSystem.exists(targetDir)) { - targetFileSystem.createDirectories(targetDir); - } - - jobLogger.info("Copying from " + sourcePath + " to " + targetPath); - String copyId = sourceFileSystem.copy(sourcePath, targetFileSystem, targetPath, CopyMode.REPLACE, - false); - stageObject.setCopyId(copyId); - stagingIds.add(copyId); } else if (stageObject instanceof StringToFileStagingObject) { - StringToFileStagingObject object = (StringToFileStagingObject) stageObject; - Path targetPath = targetDirectory.resolve(object.getTargetPath()); - Path targetDir = targetPath.getParent(); - - if (!targetFileSystem.exists(targetDir)) { - targetFileSystem.createDirectories(targetDir); - } - - jobLogger.info("Writing string to: " + targetPath); - String contents = object.getSourceString(); - - jobLogger.debug("Input contents: " + contents); - - PrintWriter out = new PrintWriter(targetFileSystem.writeToFile(targetPath)); - out.write(contents); - out.close(); - - stageObject.setBytesCopied(contents.length()); - - if (stageObject instanceof CommandScriptStagingObject) { - Set permissions = targetFileSystem.getAttributes(targetPath).getPermissions(); - permissions.add(PosixFilePermission.OWNER_EXECUTE); - permissions.add(PosixFilePermission.GROUP_EXECUTE); - permissions.add(PosixFilePermission.OTHERS_EXECUTE); - targetFileSystem.setPosixFilePermissions(targetPath, permissions); - } - } else if (stageObject instanceof DirectoryStagingObject) { - DirectoryStagingObject object = (DirectoryStagingObject) stageObject; - Path sourcePath = object.getSourcePath(); - if (!sourcePath.isAbsolute()) { - sourcePath = sourceDirectory.resolve(object.getSourcePath()).toAbsolutePath(); - } - Path targetPath = targetDirectory.resolve(object.getTargetPath()); - Path targetDir = targetPath.getParent(); - - if (!targetFileSystem.exists(targetDir)) { - targetFileSystem.createDirectories(targetDir); - } - - jobLogger.info("Copying from " + sourcePath + " to " + targetPath); - String copyId = sourceFileSystem.copy(sourcePath, targetFileSystem, targetPath, CopyMode.REPLACE, - true); - stageObject.setCopyId(copyId); + stageObject.stage(jobLogger, sourceFileSystem, targetFileSystem, sourceDirectory, targetDirectory); + } else { + String copyId = stageObject.stage(jobLogger, sourceFileSystem, targetFileSystem, sourceDirectory, targetDirectory); stagingIds.add(copyId); } } return Pair.of(stagingIds, cwlFileIds); } + public void updateStaging() { + // This for loop is using an iterator because we remove items while looping for(Iterator> stagingEntries=copyMap.entrySet().iterator(); stagingEntries.hasNext();) { Map.Entry entry = stagingEntries.next(); String jobId = entry.getKey(); StagingJob stagingJob = entry.getValue(); - - StagingManifest manifest = stagingJob.manifest; - List copyIds = stagingJob.copyIds; - List cwlFileIds = stagingJob.cwlFileIds; Logger jobLogger = LoggerFactory.getLogger("jobs." + jobId); Optional j = repository.findById(jobId); @@ -219,6 +148,11 @@ public void updateStaging() { FileSystem sourceFileSystem = getSourceFileSystem(); FileSystem cwlFileSystem = getCwlFileSystem(); + StagingManifest manifest = stagingJob.manifest; + List copyIds = stagingJob.copyIds; + List cwlFileIds = stagingJob.cwlFileIds; + + // Check if the job has been cancelled if (job.getInternalState().isCancellationActive() || job.getInternalState().isDeletionActive()) { for (String id: copyIds) { diff --git a/src/test/java/nl/esciencecenter/computeservice/rest/CwlResultTest.java b/src/test/java/nl/esciencecenter/computeservice/rest/CwlResultTest.java index 778668d..1c52619 100644 --- a/src/test/java/nl/esciencecenter/computeservice/rest/CwlResultTest.java +++ b/src/test/java/nl/esciencecenter/computeservice/rest/CwlResultTest.java @@ -54,6 +54,10 @@ public class CwlResultTest { @After public void deleteJob() throws Exception { + if (xenonService.getConfig().getSourceFilesystemConfig().shouldClearOnJobDone()) { + // Make sure we do not throw away the input if the testDeleteInput test fails + xenonService.getConfig().getSourceFilesystemConfig().setClearOnJobDone(false); + } for (Job job : CwlTestUtils.getCreated()) { mockMvc.perform( delete(job.getUri()) diff --git a/src/test/resources/config/config.yml b/src/test/resources/config/config.yml index 8884be8..acfad7d 100755 --- a/src/test/resources/config/config.yml +++ b/src/test/resources/config/config.yml @@ -17,7 +17,7 @@ ComputeResources: local: cwlCommand: | #!/usr/bin/env bash - + cwltool $@ maxTime: 20 scheduler: