performance tuning

Author: danv61

vcell-client/src/main/java/cbit/vcell/client/ClientSimManager.java

@@ -277,7 +277,7 @@ public void run(Hashtable<String, Object> hashTable) throws Exception {
 			if(failure) {	// just open some dialog for now; eventually we'll have some unobtrusive visual notification
 				PopupGenerator.showErrorDialog(ClientSimManager.this.getDocumentWindowManager(), "PostProcessing failed");
 			} else {
-				PopupGenerator.showInfoDialog(ClientSimManager.this.getDocumentWindowManager(), "PostProcessing successful");
+//				PopupGenerator.showInfoDialog(ClientSimManager.this.getDocumentWindowManager(), "PostProcessing successful");
 			}
 		}
 	};

vcell-server/src/main/java/cbit/vcell/message/server/htc/slurm/SlurmProxy.java

@@ -419,10 +419,10 @@ public String getPostProcessCommands() {
 
 	}
 
-	private static int getRoundedMemoryLimit(long memPerTaskMB) {
-		int rawLimit = (int)(memPerTaskMB * 0.9);
-		// Round down to nearest 100 MB
-		return (rawLimit / 100) * 100;
+	private static int roundUpToBlock(long memPerTaskMB, int blockSizeMB) {
+		long block = blockSizeMB; // promote to long for safe math
+		long rounded = ((memPerTaskMB + block - 1) / block) * block;
+		return (int) rounded;
 	}
 	private static String extractUser(ExecutableCommand.Container commandSet) {
 		for (ExecutableCommand ec: commandSet.getExecCommands()) {
@@ -661,9 +661,14 @@ String  generateLangevinBatchScript(String jobName, ExecutableCommand.Container
 		SolverDescription solverDescription = std.getSolverDescription();
 		MemLimitResults memoryMBAllowed = HtcProxy.getMemoryLimit(vcellUserid, simID, solverDescription, memSizeMB, simTask.isPowerUser());
 
-		int timeoutPerTaskSeconds = 28800;		// 8 hours TODO: do we hardcode this? Should it be part of LangevinSimulationOptions?
+		// TODO: do we hardcode these? Should it be part of LangevinSimulationOptions? Or, even better, properties?
+		int timeoutPerTaskSeconds = 86400;			// seconds. 24 hours
+		long hardbBtchMemoryLimitPerTask = 1024;	// MB. we hard limit mem to 1G for langevin batch jobs
+		int blockSizeMB = 256; 						// MB. SLURM memory allocation granularity
 		String slurmJobTimeout = computeSlurmTimeLimit(totalNumberOfJobs, numberOfConcurrentTasks, timeoutPerTaskSeconds);
-		int javaMemXmx = getRoundedMemoryLimit(memoryMBAllowed.getMemLimit());
+		long batchMemoryLimitPerTask = memoryMBAllowed.getMemLimit();
+		batchMemoryLimitPerTask = Math.min(batchMemoryLimitPerTask, hardbBtchMemoryLimitPerTask);
+		int javaMemXmx = roundUpToBlock(batchMemoryLimitPerTask, blockSizeMB) + blockSizeMB;	// add extra block for overhead
 
 		// -------------------------------------------------------------
 

vcell-server/src/test/resources/slurm_fixtures/langevin/V_TEST2_999999999_0_0.slurm.sub

@@ -9,7 +9,7 @@
 #SBATCH --cpus-per-task=1
 #SBATCH --mem-per-cpu=4096M
 #SBATCH --nodes=1
-#SBATCH --time=52:49:00		# timeout for the entire job
+#SBATCH --time=6-14:24:00		# timeout for the entire job
 #SBATCH --no-kill
 #SBATCH --no-requeue
 
@@ -22,7 +22,7 @@ set +e
 USERID=danv
 SIM_ID=999999999
 TOTAL_JOBS=8            # to be set by generator to lso.getTotalNumberOfJobs()
-JOB_TIMEOUT_SECONDS=28800  # per-job timeout (seconds), adjust per generator
+JOB_TIMEOUT_SECONDS=86400  # per-job timeout (seconds), adjust per generator
 LOG_FILE="/share/apps/vcell3/htclogs/V_TEST2_999999999_0_.submit.log"
 MESSAGING_CONFIG_FILE="/share/apps/vcell3/users/danv/SimID_999999999_0_.langevinMessagingConfig"
 
@@ -94,7 +94,7 @@ container_bindings+="--bind /share/apps/vcell12/users:/share/apps/vcell12/users
 container_bindings+="--bind /share/apps/vcell3/htclogs:/htclogs "
 container_bindings+="--bind /scratch/vcell:/solvertmp "
 
-container_env="--env java_mem_Xmx=3600M "
+container_env="--env java_mem_Xmx=1280M "
 container_env+="--env jmshost_sim_internal=k8s-wn-01.cam.uchc.edu "
 container_env+="--env jmsport_sim_internal=31618 "
 container_env+="--env jmsrestport_sim_internal=30163 "