Merge in expected travel time changes to current

build.gradle

@@ -30,6 +30,7 @@ plugins {
     id 'com.bmuschko.docker-java-application' version '9.4.0'
     id "cz.alenkacz.gradle.scalafmt" version "1.16.2"
     id "java-library"
+    id 'scala'
 }
 
 apply plugin: 'java'
@@ -44,7 +45,8 @@ apply plugin: 'ManifestClasspath'
 apply plugin: 'scalafmt'
 
 group = 'beam'
-version = '1.0.beta'
+version = '1.0.beta.1'
+version = '0.9.13-beta-v6'
 
 description = """"""
 
@@ -145,13 +147,20 @@ configurations.all {
         // we need this in order to get Elki library working fine.
         force 'net.jafama:jafama:2.3.2'
         // we need this in order to get Elki library working fine.
-        force 'it.unimi.dsi:fastutil:8.5.4'
+        force 'it.unimi.dsi:fastutil:8.5.12'
 
         force 'org.apache.logging.log4j:log4j-core:2.15.0'
         force 'org.apache.logging.log4j:log4j-api:2.15.0'
+        force 'org.apache.parquet:parquet-avro:1.12.3'
+        force 'org.apache.parquet:parquet-hadoop:1.12.3'
+        force 'org.apache.parquet:parquet-common:1.12.3'
+        force 'org.apache.parquet:parquet-column:1.12.3'
+        force 'org.apache.parquet:parquet-encoding:1.12.3'
     }
     exclude group: "log4j", module: "log4j"
     exclude group: "org.matsim", module: "matsim"
+    exclude group: 'org.slf4j', module: 'slf4j-log4j12'
+    exclude group: 'org.slf4j', module: 'slf4j-reload4j'
 }
 
 dependencies {
@@ -167,6 +176,13 @@ dependencies {
     implementation group: 'com.google.inject.extensions', name: 'guice-multibindings', version: '4.1.0'
     implementation group: 'org.apache.commons', name: 'commons-collections4', version: '4.4'
     implementation group: 'org.apache.commons', name: 'commons-math3', version: '3.6.1'
+    implementation 'org.apache.commons:commons-lang3:3.12.0'
+    implementation 'commons-io:commons-io:2.11.0'
+    implementation 'org.apache.commons:commons-csv:1.8'
+    implementation 'org.apache.commons:commons-text:1.10.0'
+    implementation 'commons-lang:commons-lang:2.6'
+    implementation 'org.glassfish:javax.el:3.0.0'
+    implementation 'javax.el:javax.el-api:3.0.0'
     implementation group: 'org.apache.httpcomponents', name: 'fluent-hc', version: '4.5.13'
 
     // Apache 2.0
@@ -208,7 +224,7 @@ dependencies {
 
     implementation(group: 'org.matsim.contrib', name: 'decongestion', version: '12.0')
 
-    implementation(group: 'com.github.LBNL-UCB-STI.matsim', name: 'matsim', version: '12.0-beam-8') {
+    implementation(group: 'com.github.LBNL-UCB-STI.matsim', name: 'matsim', version: '12.0-beam-9') {
         exclude group: 'log4j', module: 'log4j'
     }
 
@@ -229,6 +245,8 @@ dependencies {
     implementation("com.github.LBNL-UCB-STI:jsprit-wrapper:v0.5.1")
     implementation("com.github.LBNL-UCB-STI:omx-java:v2.0.2")
 
+    implementation group: 'dev.zarr', name: 'jzarr', version: '0.4.2'
+
     testImplementation group: 'junit', name: 'junit', version: '4.8'
     testImplementation group: 'org.mockito', name: 'mockito-inline', version: '2.27.0'
     testImplementation group: "org.mockito", name: "mockito-core", version: "2.28.2"
@@ -281,12 +299,12 @@ dependencies {
     // https://mvnrepository.com/artifact/io.circe/circe-parser_2.12
     implementation group: 'io.circe', name: "circe-parser_${scalaBinaryVersion}", version: circeBinaryVersion
 
-    implementation group: 'com.typesafe.play', name: "play-json_${scalaBinaryVersion}", version: '2.6.3'
+    implementation group: 'com.typesafe.play', name: "play-json_${scalaBinaryVersion}", version: '2.8.1'
 
     implementation(group: 'com.github.romix.akka', name: "akka-kryo-serialization_${scalaBinaryVersion}", version: '0.5.2') {
         exclude group: 'com.esotericsoftware', module: 'kryo'
     }
-    implementation group: 'com.esotericsoftware', name: 'kryo', version: '4.0.2'
+    implementation group: 'com.esotericsoftware', name: 'kryo', version: '4.0.3'
 
     implementation "com.github.vagmcs:optimus_${scalaBinaryVersion}:3.1.0"
     implementation "com.github.vagmcs:optimus-solver-oj_${scalaBinaryVersion}:3.1.0"
@@ -328,16 +346,19 @@ dependencies {
     implementation group: 'com.zaxxer', name: 'nuprocess', version: '1.2.4'
 
     def parquet = "1.12.3"
-    implementation group: 'org.apache.parquet', name: 'parquet-hadoop', version: parquet
-    implementation group: 'org.apache.parquet', name: 'parquet-avro', version: parquet
-    implementation(group: 'org.apache.hadoop', name: 'hadoop-client', version: '2.7.3') {
-        exclude group: 'org.slf4j', module: 'slf4j-log4j12'
+    // Parquet dependencies with shading
+    implementation 'org.apache.parquet:parquet-avro:1.12.3'
+    implementation 'org.apache.parquet:parquet-hadoop:1.12.3'
+    implementation 'org.apache.hadoop:hadoop-common:3.3.4'
+    implementation 'org.apache.hadoop:hadoop-mapreduce-client-core:3.3.4'
+    implementation('org.apache.hadoop:hadoop-client:3.3.6') {
         // Exclude `ASM` because it is binary incompatible with the one which is gotten from `com.conveyal:kryo-tools`: `org.ow2.asm:asm:5.0.4`
         exclude group: 'asm', module: 'asm'
         // Exclude jsp-api because it contains an older version of javax.el (Expression Language) which is incompatible
         // with one that Hibernate-validator uses
         exclude group: 'javax.servlet.jsp', module: 'jsp-api'
     }
+    implementation 'it.unimi.dsi:fastutil:8.5.12'
 
     implementation(group: 'com.lihaoyi', name: "sourcecode_${scalaBinaryVersion}", version: '0.1.9')
 
@@ -792,6 +813,9 @@ tasks.register('createDockerfile', Dockerfile) {
     addFile 'entrypoint.sh', 'entrypoint.sh'
     runCommand('chmod +x /app/entrypoint.sh')
     entryPoint('/app/entrypoint.sh')
+
+    mainClassName = 'beam.sim.RunBeam' // Replace with the desired main class
+
 }
 
 ext.getDockerTag = { ->
@@ -816,17 +840,30 @@ ext.getDockerRepository = { ->
     }
 }
 
-tasks.register('buildImageWithoutTags', DockerBuildImage) { dependsOn createDockerfile }
+tasks.register('buildImageWithoutTags', DockerBuildImage) {
+    dependsOn createDockerfile
+    platform = 'linux/amd64'
+}
 
 // one can build a docker image with command ./gradlew -Ptag=beammodel/beam:0.9.12 buildImage
 tasks.register('buildImage', DockerTagImage) {
     description 'Builds the Docker BEAM image, all code and test data are included into the image.'
-    group 'Docker'
+    group 'Docker '
 
     dependsOn buildImageWithoutTags
 
+    // Ensure platform compatibility
+    inputs.property("platform", "linux/amd64")
+
     tag = getDockerTag()
     repository = getDockerRepository()
     println("The new image will be tagged '$getDockerRepository:$getDockerTag'. To change use 'tag' project parameter, i.e. '-Ptag=<value>'")
     targetImageId buildImageWithoutTags.getImageId()
-}
+
+    // Force OCI format compatibility
+    doFirst {
+        if (!project.hasProperty('tag')) {
+            throw new GradleException("Please specify a tag using -Ptag=repository:version")
+        }
+    }
+}

src/main/R/freight/freight-processing.R

@@ -30,40 +30,46 @@ isCav <- function(x) {
   return(x >= 4)
 }
 
+### Test
+# work_folder <- normalizePath("~/Workspace/Data/Scenarios/sfbay/validation_data/BEAM")
+# speed <- readCsv(pp(work_folder, "/sfbay_residential_simpl_network.csv.gz"))
+# ggplot(speed, aes(x=speed_beam-speed_npmrds)) + geom_histogram() + xlim(0, 100)
+
+
 # emission
-emfac_sf_file <- normalizePath('~/Workspace/Models/emfac/2018/SF_2018_Annual_fleet_data_population_20240311153419.csv')
-emfac_sf <- readCsv(emfac_sf_file)
-emfac_sf_normalized <- emfac_sf[,.(sum_population=(sum(population))),by=.(vehicle_class, fuel)]
-emfac_sf_normalized$share_population <- emfac_sf_normalized$sum_population/sum(emfac_sf_normalized$sum_population)
+# emfac_sf_file <- normalizePath('~/Workspace/Models/emfac/2018/SF_2018_Annual_fleet_data_population_20240311153419.csv')
+# emfac_sf <- readCsv(emfac_sf_file)
+# emfac_sf_normalized <- emfac_sf[,.(sum_population=(sum(population))),by=.(vehicle_class, fuel)]
+# emfac_sf_normalized$share_population <- emfac_sf_normalized$sum_population/sum(emfac_sf_normalized$sum_population)
 
 ###
 
 # Create a sample data frame
-data <- data.frame(
-  Name = c("John", "Jane", "Bob", "Alice", "John", "Jane", "Bob", "Alice"),
-  Age = c(25, 30, 35, 40, 27, 32, 38, 42),
-  City = c("New York", "London", "Paris", "Tokyo", "New York", "London", "Paris", "Tokyo"),
-  Gender = c("M", "F", "M", "F", "M", "F", "M", "F")
-)
+# data <- data.frame(
+#   Name = c("John", "Jane", "Bob", "Alice", "John", "Jane", "Bob", "Alice"),
+#   Age = c(25, 30, 35, 40, 27, 32, 38, 42),
+#   City = c("New York", "London", "Paris", "Tokyo", "New York", "London", "Paris", "Tokyo"),
+#   Gender = c("M", "F", "M", "F", "M", "F", "M", "F")
+# )
 
 # Load the required packages
-library(dplyr)
-library(stringr)
+# library(dplyr)
+# library(stringr)
 
 # Group the data frame by 'City' and 'Gender', and concatenate 'Name' into a new column
-grouped <- data %>%
-  group_by(City, Gender) %>%
-  summarise(Names = str_c(Name, collapse = ", "))
-
-print(grouped)
+# grouped <- data %>%
+#   group_by(City, Gender) %>%
+#   summarise(Names = str_c(Name, collapse = ", "))
+# 
+# print(grouped)
 
 ### RouteE
-
-work_folder <- normalizePath("~/Workspace/Data/FREIGHT/seattle")
-household <- readCsv(pp(work_folder, "/households.csv.gz"))
-ggplot(household, aes(x=income/1000)) + geom_histogram() + xlim(0, 100)
+# work_folder <- normalizePath("~/Workspace/Data/FREIGHT/seattle")
+# household <- readCsv(pp(work_folder, "/households.csv.gz"))
+# ggplot(household, aes(x=income/1000)) + geom_histogram() + xlim(0, 100)
 
 
+##
 work_folder <- normalizePath("~/Workspace/Data/FREIGHT/seattle")
 geo <- geojson_sf(pp(work_folder, "/validation/npmrds/Seattle_counties.geojson"))
 

src/main/R/freight/freight-smart2-plotting.R

@@ -356,6 +356,7 @@ dgb2b_runs <-
     demand_growth_runs_dir,
     "all_b2b_growth"
   )
+dgb2b_runs_2018 <- readCsv(pp(demand_growth_runs_dir,"2018_base/0.events.csv.gz"))
 dgb2b_runs <- format_path_traversals(dgb2b_runs)
 
 dgb2b_summary <- dgb2b_runs[,