From 001d575f77fa6eeb53c95d4ee847d4ca445d7f6b Mon Sep 17 00:00:00 2001 From: Matt Rolchigo Date: Thu, 15 May 2025 15:29:11 -0400 Subject: [PATCH 1/3] In-memory Finch-ExaCA coupling with multiple Finch simulations --- bin/run.cpp | 94 +++++++++++- bin/runCoupled.cpp | 283 ++++++++++++++++++++++++++++------- src/CAinputdata.hpp | 2 +- src/CAinputs.hpp | 134 ++++++++++------- src/CAtemperature.hpp | 198 ++++++++++++++---------- src/runCA.hpp | 268 +++++++++++++-------------------- unit_test/tstTemperature.hpp | 166 +++++++++++--------- unit_test/tstUpdate.hpp | 98 +++++++++++- 8 files changed, 812 insertions(+), 431 deletions(-) diff --git a/bin/run.cpp b/bin/run.cpp index fa7c4f64..f1e5379e 100644 --- a/bin/run.cpp +++ b/bin/run.cpp @@ -45,6 +45,7 @@ int main(int argc, char *argv[]) { std::string input_file = argv[1]; // Read input file Inputs inputs(id, input_file); + std::string simulation_type = inputs.simulation_type; // Setup local and global grids, decomposing domain (needed to construct temperature) Grid grid(inputs.simulation_type, id, np, inputs.domain.number_of_layers, inputs.domain, inputs.substrate, @@ -52,7 +53,98 @@ int main(int argc, char *argv[]) { // Temperature fields characterized by data in this structure Temperature temperature(grid, inputs.temperature, inputs.print.store_solidification_start); - runExaCA(id, np, inputs, timers, grid, temperature); + // Material response function + InterfacialResponseFunction irf(inputs.domain.deltat, grid.deltax, inputs.irf); + + // Read temperature data if necessary + if (simulation_type == "FromFile") + temperature.readTemperatureData(id, grid, 0); + // Initialize the temperature fields for the simulation type of interest + if ((simulation_type == "Directional") || (simulation_type == "SingleGrain")) + temperature.initialize(id, simulation_type, grid, inputs.domain.deltat); + else if (simulation_type == "Spot") + temperature.initialize(id, grid, irf.freezingRange(), inputs.domain.deltat, inputs.domain.spot_radius); + else if ((simulation_type == "FromFile") || (simulation_type == "FromFinch")) + temperature.initialize(0, id, grid, irf.freezingRange(), inputs.domain.deltat, simulation_type); + MPI_Barrier(MPI_COMM_WORLD); + + // Initialize grain orientations + Orientation orientation(id, inputs.grain_orientation_file, false); + MPI_Barrier(MPI_COMM_WORLD); + + // Initialize cell types, grain IDs, and layer IDs + CellData celldata(grid, inputs.substrate, inputs.print.store_melt_pool_edge); + if (simulation_type == "Directional") + celldata.initSubstrate(id, grid, inputs.rng_seed); + else if (simulation_type == "SingleGrain") + celldata.initSubstrate(id, grid); + else + celldata.initSubstrate(id, grid, inputs.rng_seed, temperature.number_of_solidification_events); + MPI_Barrier(MPI_COMM_WORLD); + + // Variables characterizing the active cell region within each rank's grid, including buffers for ghost node + // data (fixed size) and the steering vector/steering vector size on host/device + Interface interface(id, grid.domain_size, inputs.substrate.init_oct_size); + MPI_Barrier(MPI_COMM_WORLD); + + // Nucleation data structure, containing views of nuclei locations, time steps, and ids, and nucleation + // event counters - initialized with an estimate on the number of nuclei in the layer Without knowing + // estimated_nuclei_this_rank_this_layer yet, initialize nucleation data structures to estimated sizes, + // resize inside of placeNuclei when the number of nuclei per rank is known + int estimated_nuclei_this_rank_this_layer = + inputs.nucleation.n_max * pow(grid.deltax, 3) * grid.domain_size; + Nucleation nucleation(estimated_nuclei_this_rank_this_layer, inputs.nucleation); + // Fill in nucleation data structures, and assign nucleation undercooling values to potential nucleation + // events Potential nucleation grains are only associated with liquid cells in layer 0 - they will be + // initialized for each successive layer when layer 0 is complete + nucleation.placeNuclei(temperature, inputs.rng_seed, 0, grid, id); + + // Initialize printing struct from inputs + Print print(grid, np, inputs.print); + + // End of initialization + timers.stopInit(); + MPI_Barrier(MPI_COMM_WORLD); + + int cycle = 0; + timers.startRun(); + + // Run ExaCA to model solidification of each layer + for (int layernumber = 0; layernumber < grid.number_of_layers; layernumber++) { + timers.startLayer(); + runExaCALayer(id, np, layernumber, cycle, inputs, timers, grid, temperature, irf, orientation, celldata, + interface, nucleation, print, simulation_type); + + if (layernumber != grid.number_of_layers - 1) { + // Initialize new temperature field data for layer "layernumber + 1" + // TODO: reorganize these temperature functions calls into a temperature.init_next_layer as done + // with the substrate If the next layer's temperature data isn't already stored, it should be read + if ((simulation_type == "FromFile") && (inputs.temperature.layerwise_temp_read)) + temperature.readTemperatureData(id, grid, layernumber + 1); + MPI_Barrier(MPI_COMM_WORLD); + // Initialize next layer's temperature data + temperature.initialize(layernumber + 1, id, grid, irf.freezingRange(), inputs.domain.deltat, + simulation_type); + // Reset solidification event counter of all cells to zeros for the next layer, resizing to number + // of cells associated with the next layer, and get the subview for undercooling + temperature.resetLayerEventsUndercooling(grid); + + // Initialize next layer of the simulation + initExaCALayer(id, np, layernumber, cycle, inputs, grid, temperature, orientation, celldata, + interface, nucleation, print, simulation_type); + timers.stopLayer(layernumber); + } + else { + MPI_Barrier(MPI_COMM_WORLD); + timers.stopLayer(); + } + } + timers.stopRun(); + MPI_Barrier(MPI_COMM_WORLD); + + // Print ExaCA end-of-run data + finalizeExaCA(id, np, cycle, inputs, timers, grid, temperature, irf, orientation, celldata, interface, + nucleation, print, simulation_type); } } // Finalize Kokkos diff --git a/bin/runCoupled.cpp b/bin/runCoupled.cpp index 88de6733..ae8baf4c 100644 --- a/bin/runCoupled.cpp +++ b/bin/runCoupled.cpp @@ -14,57 +14,90 @@ #include #include -void runCoupled(int id, int np, Finch::Inputs finch_inputs, Inputs exaca_inputs) { - // Set up Finch +// Run Finch and store solidification event data for use by ExaCA. Either called for a specific layer, or for a series +// of layers where the return view stores time-temperature histories for layers l=0,1,2... where first_value[l] is the +// index of the first event and last_value[l]-1 is the index of the last event associated with layer l +Kokkos::View +getFinchData(const int id, const int np, const int first_finch_simulation, const int num_finch_simulations, + const int number_of_layers, Inputs exaca_inputs, std::array &exaca_low_corner, + std::array &exaca_high_corner, std::vector &first_value_finch, + std::vector &last_value_finch) { + using exec_space = Kokkos::DefaultExecutionSpace; using memory_space = exec_space::memory_space; + Kokkos::View input_temperature_data( + Kokkos::ViewAllocateWithoutInitializing("finch_temperature_data"), 0); + // Run Finch, storing solidification data, domain bounds, and the locations in the view where each layer's data + // starts and ends + const int last_finch_simulation = first_finch_simulation + num_finch_simulations; + for (int finch_input_file_num = first_finch_simulation; finch_input_file_num < last_finch_simulation; + finch_input_file_num++) { + // Setup Finch simulation + Finch::Inputs finch_inputs(MPI_COMM_WORLD, exaca_inputs.temperature.temp_paths[finch_input_file_num]); - // Create timers - Timers timers(id); - timers.startHeatTransfer(); - - // initialize a moving beam - Finch::MovingBeam beam(finch_inputs.source.scan_path_file); - - // Define boundary condition details. - std::array bc_types = {"adiabatic", "adiabatic", "adiabatic", - "adiabatic", "adiabatic", "adiabatic"}; - - // create the global mesh - Finch::Grid finch_grid(MPI_COMM_WORLD, finch_inputs.space.cell_size, - finch_inputs.space.global_low_corner, finch_inputs.space.global_high_corner, - finch_inputs.space.ranks_per_dim, bc_types, - finch_inputs.space.initial_temperature); - - // Run Finch heat transfer - Finch::Layer app(finch_inputs, finch_grid); - auto fd = Finch::createSolver(finch_inputs, finch_grid); - app.run(exec_space(), finch_inputs, finch_grid, beam, fd); - - // Start of ExaCA init time. - timers.stopHeatTransfer(); - timers.startInit(); - - // Setup local and global grids, decomposing domain (needed to construct temperature) - // ExaCA either uses the same bounds as Finch, or a bounding box that only includes the regions that underwent - // melting and solidification - std::array exaca_low_corner, exaca_high_corner; - if (exaca_inputs.temperature.trim_unmelted_region) { - exaca_low_corner = app.getLowerSolidificationDataBounds(); - exaca_high_corner = app.getUpperSolidificationDataBounds(); - } - else { - exaca_low_corner = finch_inputs.space.global_low_corner; - exaca_high_corner = finch_inputs.space.global_high_corner; + // initialize a moving beam + Finch::MovingBeam beam(finch_inputs.source.scan_path_file); + + // Define boundary condition details. + std::array bc_types = {"adiabatic", "adiabatic", "adiabatic", + "adiabatic", "adiabatic", "adiabatic"}; + + // create the global mesh + Finch::Grid finch_grid(MPI_COMM_WORLD, finch_inputs.space.cell_size, + finch_inputs.space.global_low_corner, + finch_inputs.space.global_high_corner, finch_inputs.space.ranks_per_dim, + bc_types, finch_inputs.space.initial_temperature); + + // Run Finch heat transfer + Finch::Layer app(finch_inputs, finch_grid); + auto fd = Finch::createSolver(finch_inputs, finch_grid); + app.run(exec_space(), finch_inputs, finch_grid, beam, fd); + + // Bounds of Finch domain for this layer stored for use by ExaCA - overwrite with bounds of Finch events if trim + // options were specified + std::array exaca_low_corner_layer = finch_inputs.space.global_low_corner; + std::array exaca_high_corner_layer = finch_inputs.space.global_high_corner; + std::array finch_low_corner_layer = app.getLowerSolidificationDataBounds(MPI_COMM_WORLD); + std::array finch_high_corner_layer = app.getUpperSolidificationDataBounds(MPI_COMM_WORLD); + if (exaca_inputs.temperature.trim_unmelted_region_xy) { + exaca_low_corner_layer[0] = finch_low_corner_layer[0]; + exaca_high_corner_layer[0] = finch_high_corner_layer[0]; + exaca_low_corner_layer[1] = finch_low_corner_layer[1]; + exaca_high_corner_layer[1] = finch_high_corner_layer[1]; + } + if (exaca_inputs.temperature.trim_unmelted_region_z) { + exaca_low_corner_layer[2] = finch_low_corner_layer[2]; + exaca_high_corner_layer[2] = finch_high_corner_layer[2]; + } + + // Extend overall Finch bounds if necessary + if (finch_input_file_num == 0) { + for (int i = 0; i < 3; i++) { + exaca_low_corner[i] = exaca_low_corner_layer[i]; + exaca_high_corner[i] = exaca_high_corner_layer[i]; + } + } + else { + for (int i = 0; i < 3; i++) { + exaca_low_corner[i] = std::min(exaca_low_corner_layer[i], exaca_low_corner[i]); + exaca_high_corner[i] = std::min(exaca_high_corner_layer[i], exaca_high_corner[i]); + } + } + + // Append this layer's solidification data to input_temperature_data + app.appendSolidificationData(input_temperature_data, first_value_finch, last_value_finch, finch_input_file_num, + num_finch_simulations); + // If performing multiple finch simulations during initialization, fill first/last_value_finch with values for + // repeated data + if (num_finch_simulations > 1) { + for (int repeated_layer = num_finch_simulations; repeated_layer < number_of_layers; repeated_layer++) { + const int repeated_file = repeated_layer % num_finch_simulations; + first_value_finch[repeated_layer] = first_value_finch[repeated_file]; + last_value_finch[repeated_layer] = last_value_finch[repeated_file]; + } + } } - Grid exaca_grid(id, np, exaca_inputs.domain, exaca_inputs.substrate, exaca_inputs.temperature, - finch_inputs.space.cell_size, exaca_low_corner, exaca_high_corner); - // Temperature fields characterized by data in this structure - Temperature temperature(id, np, exaca_grid, exaca_inputs.temperature, app.getSolidificationData(), - exaca_inputs.print.store_solidification_start); - - // Now run ExaCA - runExaCA(id, np, exaca_inputs, timers, exaca_grid, temperature); + return input_temperature_data; } int main(int argc, char *argv[]) { @@ -77,6 +110,8 @@ int main(int argc, char *argv[]) { MPI_Comm_size(MPI_COMM_WORLD, &np); // Get individual process ID MPI_Comm_rank(MPI_COMM_WORLD, &id); + using exec_space = Kokkos::DefaultExecutionSpace; + using memory_space = exec_space::memory_space; if (id == 0) { std::cout << "ExaCA version: " << version() << " \nExaCA commit: " << gitCommitHash() @@ -84,21 +119,155 @@ int main(int argc, char *argv[]) { Kokkos::DefaultExecutionSpace().print_configuration(std::cout); std::cout << "Number of MPI ranks = " << np << std::endl; } - if (argc < 3) { - throw std::runtime_error("Error: Must provide path to Finch and ExaCA input files on the command line."); - } - else { + std::string exaca_input_file; + if (argc < 2) + throw std::runtime_error("Error: Must provide path to input file on the command line."); + else if (argc == 2) + exaca_input_file = argv[1]; + else + exaca_input_file = argv[2]; + + // Setup ExaCA simulation details + Inputs inputs(id, exaca_input_file); + + // Without Finch input file on the command line, ensure that at least 1 Finch input file is listed in the ExaCA + // input file. Otherwise store Finch input filename given on the command line within the ExaCA inputs struct + if ((argc == 2) && (inputs.temperature.temp_files_in_series == 0)) + throw std::runtime_error( + "Error: Finch input file must be given on the command line if absent from the ExaCA input file"); + else if (argc > 2) { std::string finch_input_file = argv[1]; - std::string exaca_input_file = argv[2]; + inputs.temperature.temp_files_in_series = 1; + inputs.temperature.temp_paths.push_back(finch_input_file); + if (id == 0) + std::cout << "Warning: Ability to specify Finch input file on the command line is deprecated and will " + "be removed in a future release, please use the FinchInputFile input list in the " + "TemperatureData portion of the ExaCA input file to specify desired Finch input file(s)" + << std::endl; + } + + // Create timers + Timers timers(id); + + // ExaCA either uses the same bounds as Finch, or a bounding box that only includes the regions that underwent + // melting and solidification + std::array exaca_low_corner, exaca_high_corner; + // Should Finch simulations for all layers be performed and time-temperature history data stored prior to + // running ExaCA, or should each Finch simulation be run one at a time between ExaCA-simulated layers? + int num_finch_simulations, first_finch_simulation; + if (inputs.temperature.layerwise_temp_read) + num_finch_simulations = 1; + else + num_finch_simulations = inputs.temperature.temp_files_in_series; + // Store values denoting which portions of input_temperature_data belong to which simulated layer in Finch + std::vector first_value_finch(inputs.domain.number_of_layers), + last_value_finch(inputs.domain.number_of_layers); + // View for time-temperature history data from Finch - run either just the first layer, or run all Finch + // simulations and store results + timers.startHeatTransfer(); + Kokkos::View input_temperature_data = + getFinchData(id, np, 0, num_finch_simulations, inputs.domain.number_of_layers, inputs, exaca_low_corner, + exaca_high_corner, first_value_finch, last_value_finch); + MPI_Barrier(MPI_COMM_WORLD); + timers.stopHeatTransfer(); + + // Start of ExaCA init time. + timers.startInit(); + // Initialize ExaCA grid, using bounds of input Finch data + Grid grid(id, np, inputs.domain, inputs.substrate, inputs.temperature, inputs.domain.deltax, exaca_low_corner, + exaca_high_corner); + + // Material response function + InterfacialResponseFunction irf(inputs.domain.deltat, grid.deltax, inputs.irf); + + // Temperature fields characterized by data in this structure - rearrange data in input_temperature_data to + // match ExaCA's domain decomposition. Store which temperature data is associated with which layers + Temperature temperature(id, np, grid, inputs.temperature, input_temperature_data, + first_value_finch, last_value_finch, + inputs.print.store_solidification_start); + temperature.initialize(0, id, grid, irf.freezingRange(), inputs.domain.deltat, "FromFinch"); + MPI_Barrier(MPI_COMM_WORLD); + + // Initialize grain orientations + Orientation orientation(id, inputs.grain_orientation_file, false); + MPI_Barrier(MPI_COMM_WORLD); + + // Initialize cell types, grain IDs, and layer IDs + CellData celldata(grid, inputs.substrate, inputs.print.store_melt_pool_edge); + celldata.initSubstrate(id, grid, inputs.rng_seed, temperature.number_of_solidification_events); + MPI_Barrier(MPI_COMM_WORLD); - // Setup Finch simulation - Finch::Inputs finch_inputs(MPI_COMM_WORLD, finch_input_file); + // Variables characterizing the active cell region within each rank's grid, including buffers for ghost node + // data (fixed size) and the steering vector/steering vector size on host/device + Interface interface(id, grid.domain_size, inputs.substrate.init_oct_size); + MPI_Barrier(MPI_COMM_WORLD); - // Setup ExaCA simulation details - Inputs inputs(id, exaca_input_file); + // Nucleation data structure, containing views of nuclei locations, time steps, and ids, and nucleation event + // counters - initialized with an estimate on the number of nuclei in the layer Without knowing + // estimated_nuclei_this_rank_this_layer yet, initialize nucleation data structures to estimated sizes, resize + // inside of placeNuclei when the number of nuclei per rank is known + int estimated_nuclei_this_rank_this_layer = inputs.nucleation.n_max * pow(grid.deltax, 3) * grid.domain_size; + Nucleation nucleation(estimated_nuclei_this_rank_this_layer, inputs.nucleation); + // Fill in nucleation data structures, and assign nucleation undercooling values to potential nucleation events + // Potential nucleation grains are only associated with liquid cells in layer 0 - they will be initialized for + // each successive layer when layer 0 is complete + nucleation.placeNuclei(temperature, inputs.rng_seed, 0, grid, id); - runCoupled(id, np, finch_inputs, inputs); + // Initialize printing struct from inputs + Print print(grid, np, inputs.print); + + // End of initialization + timers.stopInit(); + MPI_Barrier(MPI_COMM_WORLD); + + int cycle = 0; + timers.startRun(); + + // Run ExaCA to model solidification of each layer + for (int layernumber = 0; layernumber < grid.number_of_layers; layernumber++) { + timers.startLayer(); + runExaCALayer(id, np, layernumber, cycle, inputs, timers, grid, temperature, irf, orientation, celldata, + interface, nucleation, print, "FromFinch"); + + if (layernumber != grid.number_of_layers - 1) { + + // Initialize new temperature field data for layer "layernumber + 1" - this is either stored in the + // temperature struct, or comes from a new Finch simulation + if (inputs.temperature.layerwise_temp_read) { + timers.startHeatTransfer(); + input_temperature_data = + getFinchData(id, np, layernumber + 1, num_finch_simulations, inputs.domain.number_of_layers, + inputs, exaca_low_corner, exaca_high_corner, first_value_finch, last_value_finch); + timers.stopHeatTransfer(); + temperature.copyTemperatureData(id, np, layernumber + 1, grid, input_temperature_data, + first_value_finch, last_value_finch); + } + MPI_Barrier(MPI_COMM_WORLD); + // Initialize next layer's temperature data + temperature.initialize(layernumber + 1, id, grid, irf.freezingRange(), inputs.domain.deltat, + "FromFinch"); + + // Reset solidification event counter of all cells to zeros for the next layer, resizing to number of + // cells associated with the next layer, and get the subview for undercooling (TODO: part of + // temperature.initialize in the future) + temperature.resetLayerEventsUndercooling(grid); + + // Initialize next layer of the simulation + initExaCALayer(id, np, layernumber, cycle, inputs, grid, temperature, orientation, celldata, interface, + nucleation, print, "FromFinch"); + timers.stopLayer(layernumber); + } + else { + MPI_Barrier(MPI_COMM_WORLD); + timers.stopLayer(); + } } + timers.stopRun(); + MPI_Barrier(MPI_COMM_WORLD); + + // Print ExaCA end-of-run data + finalizeExaCA(id, np, cycle, inputs, timers, grid, temperature, irf, orientation, celldata, interface, + nucleation, print, "FromFinch"); } // Finalize Kokkos & MPI diff --git a/src/CAinputdata.hpp b/src/CAinputdata.hpp index e783b82a..b94de262 100644 --- a/src/CAinputdata.hpp +++ b/src/CAinputdata.hpp @@ -71,7 +71,7 @@ struct TemperatureInputs { double G = 0, R = 0; double init_undercooling = 0.0; // Used for FromFinch and FromFile problem types with translated temperature data - bool trim_unmelted_region = false; + bool trim_unmelted_region_xy = false, trim_unmelted_region_z = false; bool use_fixed_x_bounds = false, use_fixed_y_bounds = false; std::vector temperature_x_bounds = {std::numeric_limits::lowest(), std::numeric_limits::max()}; diff --git a/src/CAinputs.hpp b/src/CAinputs.hpp index 9a5882ea..70c4175f 100644 --- a/src/CAinputs.hpp +++ b/src/CAinputs.hpp @@ -111,15 +111,60 @@ struct Inputs { // Temperature inputs: if ((simulation_type == "FromFile") || (simulation_type == "FromFinch")) { - if (simulation_type == "FromFile") { - if ((input_data["TemperatureData"].contains("HeatTransferCellSize")) && (id == 0)) - std::cout << "Note: Heat transport data cell size is no longer an input used in ExaCA, temperature " - "data must be at the same resolution as the CA cell size" - << std::endl; - // Read all temperature files at once (default), or one at a time? - if (input_data["TemperatureData"].contains("LayerwiseTempRead")) { - temperature.layerwise_temp_read = input_data["TemperatureData"]["LayerwiseTempRead"]; + // Read/store all temperature files at once (default), or one at a time? + if (input_data["TemperatureData"].contains("LayerwiseTempRead")) + temperature.layerwise_temp_read = input_data["TemperatureData"]["LayerwiseTempRead"]; + // Specified region in X and Y for simulation + if (input_data["TemperatureData"].contains("XRegion")) { + temperature.temperature_x_bounds[0] = input_data["TemperatureData"]["XRegion"][0]; + temperature.temperature_x_bounds[1] = input_data["TemperatureData"]["XRegion"][1]; + temperature.use_fixed_x_bounds = true; + } + if (input_data["TemperatureData"].contains("YRegion")) { + temperature.temperature_y_bounds[0] = input_data["TemperatureData"]["YRegion"][0]; + temperature.temperature_y_bounds[1] = input_data["TemperatureData"]["YRegion"][1]; + temperature.use_fixed_y_bounds = true; + } + if ((input_data["TemperatureData"].contains("HeatTransferCellSize")) && (id == 0)) + std::cout << "Note: Heat transport data cell size is no longer an input used in ExaCA, temperature " + "data must be at the same resolution as the CA cell size" + << std::endl; + // Optional translation/mirroring of input temperature data + if (input_data["TemperatureData"].contains("TranslationCount")) { + temperature.number_of_copies = input_data["TemperatureData"]["TranslationCount"]; + // Include the original in the total number of copies (number of translations + 1) + temperature.number_of_copies++; + std::string offset_direction = input_data["TemperatureData"]["OffsetDirection"]; + // Offsets are given in cells (for consistency with layer height) and microseconds (for consistency + // with time step) but stored in meters and seconds for consistency with how cell size and time step + // are treated + if (offset_direction == "X") { + // X offset from input file, Y defaults to 0 + temperature.x_offset = input_data["TemperatureData"]["SpatialOffset"]; } + else if (offset_direction == "Y") { + // Y offset from input file, X defaults to 0 + temperature.y_offset = input_data["TemperatureData"]["SpatialOffset"]; + } + else + throw std::runtime_error("Error: OffsetDirection must be either X or Y"); + temperature.x_offset = temperature.x_offset * domain.deltax; + temperature.y_offset = temperature.y_offset * domain.deltax; + temperature.temporal_offset = input_data["TemperatureData"]["TemporalOffset"]; + temperature.temporal_offset = temperature.temporal_offset * pow(10, -6); + + // For the direction not used to offset copies of the temperature data, should the data be mirrored + // on every other pass (i.e., bidirectional scanning of a laser)? + bool alternating_direction = input_data["TemperatureData"]["AlternatingDirection"]; + if (alternating_direction) { + // False (default) for offset direction, true for other direction + if (offset_direction == "X") + temperature.mirror_y = true; + else + temperature.mirror_x = true; + } + } + if (simulation_type == "FromFile") { // Get the paths/number of/names of the temperature data files used temperature.temp_files_in_series = input_data["TemperatureData"]["TemperatureFiles"].size(); if (temperature.temp_files_in_series == 0) @@ -130,61 +175,34 @@ struct Inputs { temperature.temp_paths.push_back(input_data["TemperatureData"]["TemperatureFiles"][filename]); } } - // See if temperature data translation instructions are given - if (input_data.contains("TemperatureData")) { + else if ((simulation_type == "FromFinch") && (input_data.contains("TemperatureData"))) { + // For FromFinch problems where the Finch input file was not given on the command line, read input + // filename(s) from the ExaCA input file + if (input_data["TemperatureData"].contains("FinchInputFiles")) { + temperature.temp_files_in_series = input_data["TemperatureData"]["FinchInputFiles"].size(); + if (temperature.temp_files_in_series == 0) + throw std::runtime_error("Error: FinchInputFiles must contain at least one entry"); + else { + for (int filename = 0; filename < temperature.temp_files_in_series; filename++) + temperature.temp_paths.push_back( + input_data["TemperatureData"]["FinchInputFiles"][filename]); + } + } // Option to trim bounds of Finch data around the region that underwent solidification. If not given, // defaults to false // As an alternative to the TrimUnmeltedRegion option, fixed X or Y bounds can be given to trim the // input data - if ((input_data["TemperatureData"].contains("TrimUnmeltedRegion")) && (simulation_type == "FromFinch")) - temperature.trim_unmelted_region = input_data["TemperatureData"]["TrimUnmeltedRegion"]; - else { - if (input_data["TemperatureData"].contains("XRegion")) { - temperature.temperature_x_bounds[0] = input_data["TemperatureData"]["XRegion"][0]; - temperature.temperature_x_bounds[1] = input_data["TemperatureData"]["XRegion"][1]; - temperature.use_fixed_x_bounds = true; - } - if (input_data["TemperatureData"].contains("YRegion")) { - temperature.temperature_y_bounds[0] = input_data["TemperatureData"]["YRegion"][0]; - temperature.temperature_y_bounds[1] = input_data["TemperatureData"]["YRegion"][1]; - temperature.use_fixed_y_bounds = true; - } - } - // Optional translation/mirroring of input temperature data - if (input_data["TemperatureData"].contains("TranslationCount")) { - temperature.number_of_copies = input_data["TemperatureData"]["TranslationCount"]; - // Include the original in the total number of copies (number of translations + 1) - temperature.number_of_copies++; - std::string offset_direction = input_data["TemperatureData"]["OffsetDirection"]; - // Offsets are given in cells (for consistency with layer height) and microseconds (for consistency - // with time step) but stored in meters and seconds for consistency with how cell size and time step - // are treated - if (offset_direction == "X") { - // X offset from input file, Y defaults to 0 - temperature.x_offset = input_data["TemperatureData"]["SpatialOffset"]; - } - else if (offset_direction == "Y") { - // Y offset from input file, X defaults to 0 - temperature.y_offset = input_data["TemperatureData"]["SpatialOffset"]; - } - else - throw std::runtime_error("Error: OffsetDirection must be either X or Y"); - temperature.x_offset = temperature.x_offset * domain.deltax; - temperature.y_offset = temperature.y_offset * domain.deltax; - temperature.temporal_offset = input_data["TemperatureData"]["TemporalOffset"]; - temperature.temporal_offset = temperature.temporal_offset * pow(10, -6); - - // For the direction not used to offset copies of the temperature data, should the data be mirrored - // on every other pass (i.e., bidirectional scanning of a laser)? - bool alternating_direction = input_data["TemperatureData"]["AlternatingDirection"]; - if (alternating_direction) { - // False (default) for offset direction, true for other direction - if (offset_direction == "X") - temperature.mirror_y = true; - else - temperature.mirror_x = true; - } + if ((input_data["TemperatureData"].contains("TrimUnmeltedRegion"))) { + if (id == 0) + std::cout << "Warning: TrimUnmeltedRegion option is deprecated and will be removed in a future " + "release, please use TrimUnmeltedRegionLateral or TrimUnmeltedRegionBuild options" + << std::endl; + temperature.trim_unmelted_region_xy = input_data["TemperatureData"]["TrimUnmeltedRegion"]; } + if ((input_data["TemperatureData"].contains("TrimUnmeltedRegionLateral"))) + temperature.trim_unmelted_region_xy = input_data["TemperatureData"]["TrimUnmeltedRegionLateral"]; + if ((input_data["TemperatureData"].contains("TrimUnmeltedRegionBuild"))) + temperature.trim_unmelted_region_z = input_data["TemperatureData"]["TrimUnmeltedRegionBuild"]; } } else { diff --git a/src/CAtemperature.hpp b/src/CAtemperature.hpp index 678c7561..e5304eab 100644 --- a/src/CAtemperature.hpp +++ b/src/CAtemperature.hpp @@ -100,7 +100,8 @@ struct Temperature { // Constructor using in-memory temperature data from external source (input_temperature_data) Temperature(const int id, const int np, const Grid &grid, TemperatureInputs inputs, - view_type_coupled input_temperature_data, const bool store_solidification_start = false) + view_type_coupled input_temperature_data, std::vector first_value_finch, + std::vector last_value_finch, const bool store_solidification_start = false) : max_solidification_events( view_type_int(Kokkos::ViewAllocateWithoutInitializing("number_of_layers"), grid.number_of_layers)) , liquidus_time( @@ -118,7 +119,7 @@ struct Temperature { , _store_solidification_start(store_solidification_start) , _inputs(inputs) { - copyTemperatureData(id, np, grid, input_temperature_data); + copyTemperatureData(id, np, 0, grid, input_temperature_data, first_value_finch, last_value_finch); if (_store_solidification_start) { // Default init starting undercooling in cells to zero undercooling_solidification_start_all_layers = @@ -153,79 +154,114 @@ struct Temperature { } // Copy data from external source onto the appropriate MPI ranks - void copyTemperatureData(const int id, const int np, const Grid &grid, view_type_coupled input_temperature_data, - const int resize_padding = 100) { + void copyTemperatureData(const int id, const int np, const int first_layer_init, const Grid &grid, + view_type_coupled input_temperature_data, std::vector first_value_finch, + std::vector last_value_finch, const int resize_padding = 100) { // Take first num_temperature_components columns of input_temperature_data - int finch_data_size = input_temperature_data.extent(0); int finch_temp_components = input_temperature_data.extent(1); - std::cout << "Rank " << id << " has " << finch_data_size << " events from the Finch simulation" << std::endl; - - // First, store data with Y coordinates in bounds for this rank in raw_temperature_data - int temperature_point_counter = 0; - extractTemperatureData(input_temperature_data, raw_temperature_data, temperature_point_counter, grid); - - // Communication pattern - sending to right, receiving from left - int left, right; - if (id == 0) - left = np - 1; - else - left = id - 1; - if (id == np - 1) - right = 0; + // Number of layers of Finch data to copy into temperature struct at once + int num_layers_init; + if (_inputs.layerwise_temp_read) + num_layers_init = 1; else - right = id + 1; - - // Send and recieve data so each rank parses all finch data points - int send_data_size = finch_data_size; - for (int i = 0; i < np - 1; i++) { - // Get size for sending/receiving - int recv_data_size; - MPI_Request send_request_size, recv_request_size; - MPI_Isend(&send_data_size, 1, MPI_INT, right, 0, MPI_COMM_WORLD, &send_request_size); - MPI_Irecv(&recv_data_size, 1, MPI_INT, left, 0, MPI_COMM_WORLD, &recv_request_size); - MPI_Wait(&send_request_size, MPI_STATUS_IGNORE); - MPI_Wait(&recv_request_size, MPI_STATUS_IGNORE); - // Allocate view for received data - view_type_coupled finch_data_recv(Kokkos::ViewAllocateWithoutInitializing("finch_data_recv"), - recv_data_size, finch_temp_components); - - // Send data to the right, recieve data from the left - if needed, increase size of data stored on this rank - // to accomodate received data - MPI_Request send_request_data, recv_request_data; - MPI_Isend(input_temperature_data.data(), send_data_size * finch_temp_components, MPI_DOUBLE, right, 1, - MPI_COMM_WORLD, &send_request_data); - MPI_Irecv(finch_data_recv.data(), recv_data_size * finch_temp_components, MPI_DOUBLE, left, 1, - MPI_COMM_WORLD, &recv_request_data); - int current_size = raw_temperature_data.extent(0); - if (temperature_point_counter + _inputs.number_of_copies * recv_data_size >= current_size) - Kokkos::resize(raw_temperature_data, - temperature_point_counter + _inputs.number_of_copies * recv_data_size + resize_padding, - num_temperature_components); - MPI_Wait(&send_request_data, MPI_STATUS_IGNORE); - MPI_Wait(&recv_request_data, MPI_STATUS_IGNORE); - - // Unpack the appropriate received data into raw_temperature_data - extractTemperatureData(finch_data_recv, raw_temperature_data, temperature_point_counter, grid); - - // Replace send buffer with the received data - input_temperature_data = finch_data_recv; - send_data_size = recv_data_size; + num_layers_init = _inputs.temp_files_in_series; + int last_layer_init = first_layer_init + num_layers_init; + int temperature_point_counter = 0; + for (int layer = first_layer_init; layer < last_layer_init; layer++) { + + // input_temperature_data contains data for one or multiple Finch simulations of various layers - which are + // tracked via the first_value_finch and last_value_finch vectors. The first_value and last_value vectors + // track, on each ExaCA MPI rank, which portions of the data are associated with which layer + const int layer_start = first_value_finch[layer]; + const int layer_end = last_value_finch[layer]; + std::pair finch_layer_range = std::make_pair(layer_start, layer_end); + auto input_temperature_data_layer = Kokkos::subview(input_temperature_data, finch_layer_range, Kokkos::ALL); + + // First, store data with Y coordinates in bounds for this rank in raw_temperature_data + const int temperature_point_counter_start = temperature_point_counter; + extractTemperatureData(input_temperature_data_layer, raw_temperature_data, temperature_point_counter, grid); + + // Communication pattern - sending to right, receiving from left + int left, right; + if (id == 0) + left = np - 1; + else + left = id - 1; + if (id == np - 1) + right = 0; + else + right = id + 1; + + int finch_data_size = input_temperature_data_layer.extent(0); + std::cout << "Rank " << id << " has " << finch_data_size << " events from Finch simulation number " << layer + << ", corresponding to events " << layer_start << " through " << layer_end - 1 << std::endl; + // Send and recieve data so each rank parses all finch data points + view_type_coupled finch_data_send(Kokkos::ViewAllocateWithoutInitializing("finch_data_send"), + layer_end - layer_start, finch_temp_components); + Kokkos::deep_copy(finch_data_send, input_temperature_data_layer); + int send_data_size = finch_data_size; + for (int i = 0; i < np - 1; i++) { + // Get size for sending/receiving + int recv_data_size; + MPI_Request send_request_size, recv_request_size; + MPI_Isend(&send_data_size, 1, MPI_INT, right, 0, MPI_COMM_WORLD, &send_request_size); + MPI_Irecv(&recv_data_size, 1, MPI_INT, left, 0, MPI_COMM_WORLD, &recv_request_size); + MPI_Wait(&send_request_size, MPI_STATUS_IGNORE); + MPI_Wait(&recv_request_size, MPI_STATUS_IGNORE); + // Allocate view for received data + view_type_coupled finch_data_recv(Kokkos::ViewAllocateWithoutInitializing("finch_data_recv"), + recv_data_size, finch_temp_components); + + // Send data to the right, recieve data from the left - if needed, increase size of data stored on this + // rank to accomodate received data + MPI_Request send_request_data, recv_request_data; + MPI_Isend(finch_data_send.data(), send_data_size * finch_temp_components, MPI_DOUBLE, right, 1, + MPI_COMM_WORLD, &send_request_data); + MPI_Irecv(finch_data_recv.data(), recv_data_size * finch_temp_components, MPI_DOUBLE, left, 1, + MPI_COMM_WORLD, &recv_request_data); + int current_size = raw_temperature_data.extent(0); + if (temperature_point_counter + _inputs.number_of_copies * recv_data_size >= current_size) + Kokkos::resize(raw_temperature_data, + temperature_point_counter + _inputs.number_of_copies * recv_data_size + + resize_padding, + num_temperature_components); + MPI_Wait(&send_request_data, MPI_STATUS_IGNORE); + MPI_Wait(&recv_request_data, MPI_STATUS_IGNORE); + + // Unpack the appropriate received data into raw_temperature_data + extractTemperatureData(finch_data_recv, raw_temperature_data, temperature_point_counter, grid); + + // Replace send buffer with the received data + Kokkos::realloc(finch_data_send, recv_data_size, finch_temp_components); + Kokkos::deep_copy(finch_data_send, finch_data_recv); + send_data_size = recv_data_size; + } + first_value(layer) = temperature_point_counter_start; + last_value(layer) = temperature_point_counter; + std::cout << "Rank " << id << " has " << temperature_point_counter - temperature_point_counter_start + << " events to simulate with ExaCA from Finch simulation number " << layer << std::endl; } // Resize with the number of temperature data points on this rank now known Kokkos::resize(raw_temperature_data, temperature_point_counter, num_temperature_components); - for (int n = 0; n < grid.number_of_layers; n++) { - first_value(n) = 0; - last_value(n) = temperature_point_counter; - } - std::cout << "Rank " << id << " has " << temperature_point_counter << " events to simulate with ExaCA" - << std::endl; + + // If reading multiple files during initialization, associated each layer with the appropriate portion of the + // Finch data + if (!(_inputs.layerwise_temp_read)) + fillFirstLastValue(grid.number_of_layers, _inputs.temp_files_in_series); } // Get temperature data from Finch and store as raw temperatures. template void extractTemperatureData(const SrcViewType &temp_src, DstViewType &temp_dst, int &temp_count, const Grid grid) { int data_size = temp_src.extent(0); + // Ensure that dst view has sufficient capacity to store values from temp_src + int max_num_pts_extracted = data_size * _inputs.number_of_copies; + int current_dst_view_size = temp_dst.extent(0); + int free_space_dst_view = temp_dst.extent(0) - temp_count; + if (max_num_pts_extracted > free_space_dst_view) + Kokkos::resize(temp_dst, current_dst_view_size + max_num_pts_extracted - free_space_dst_view, + num_temperature_components); for (int n = 0; n < data_size; n++) { for (int l = 0; l < _inputs.number_of_copies; l++) { // Consider each translated point in either X or Y, optionally mirroring the point in the non-translated @@ -362,6 +398,24 @@ struct Temperature { } } + // For simulations where temperature data is read during ExaCA initialization and reused for multiple layers, fill + // out first_value and last_value to associate each layer with the correct temperature data + void fillFirstLastValue(const int number_of_layers, const int temp_files_in_series) { + for (int layer_read_count = temp_files_in_series; layer_read_count < number_of_layers; layer_read_count++) { + if (temp_files_in_series == 1) { + // All layers have the same temperature data + first_value(layer_read_count) = first_value(layer_read_count - 1); + last_value(layer_read_count) = last_value(layer_read_count - 1); + } + else { + // All layers have different temperature data but in a repeating pattern + int repeated_file = layer_read_count % temp_files_in_series; + first_value(layer_read_count) = first_value(repeated_file); + last_value(layer_read_count) = last_value(repeated_file); + } + } + } + // Read in temperature data from files, stored in the host view raw_temperature_data, with the appropriate MPI ranks // storing the appropriate data void readTemperatureData(int id, const Grid &grid, int layernumber) { @@ -402,24 +456,8 @@ struct Temperature { } // End loop over all files read for all layers Kokkos::resize(raw_temperature_data, number_of_temperature_data_points, num_temperature_components); // Determine start values for each layer's data within raw_temperature_data, if all layers were read - if (!(_inputs.layerwise_temp_read)) { - if (grid.number_of_layers > _inputs.temp_files_in_series) { - for (int layer_read_count = _inputs.temp_files_in_series; layer_read_count < grid.number_of_layers; - layer_read_count++) { - if (_inputs.temp_files_in_series == 1) { - // All layers have the same temperature data - first_value(layer_read_count) = first_value(layer_read_count - 1); - last_value(layer_read_count) = last_value(layer_read_count - 1); - } - else { - // All layers have different temperature data but in a repeating pattern - int repeated_file = (layer_read_count) % _inputs.temp_files_in_series; - first_value(layer_read_count) = first_value(repeated_file); - last_value(layer_read_count) = last_value(repeated_file); - } - } - } - } + if (!(_inputs.layerwise_temp_read)) + fillFirstLastValue(grid.number_of_layers, _inputs.temp_files_in_series); } // Initialize temperature data with a fixed thermal gradient in Z (can also be zero) for Directional/SingleGrain diff --git a/src/runCA.hpp b/src/runCA.hpp index e0d1550e..ed12edb7 100644 --- a/src/runCA.hpp +++ b/src/runCA.hpp @@ -13,180 +13,124 @@ #include #include +// Run ExaCA using thermal data for a given problem type template -void runExaCA(int id, int np, Inputs inputs, Timers timers, Grid grid, Temperature temperature) { +void runExaCALayer(int id, int np, int layernumber, int &cycle, Inputs inputs, Timers &timers, Grid &grid, + Temperature temperature, InterfacialResponseFunction irf, + Orientation &orientation, CellData &celldata, + Interface &interface, Nucleation &nucleation, Print &print, + std::string simulation_type) { // Run on the default space. using memory_space = MemorySpace; - std::string simulation_type = inputs.simulation_type; - - // Material response function - InterfacialResponseFunction irf(inputs.domain.deltat, grid.deltax, inputs.irf); - - // Read temperature data if necessary - if (simulation_type == "FromFile") - temperature.readTemperatureData(id, grid, 0); - // Initialize the temperature fields for the simulation type of interest - if ((simulation_type == "Directional") || (simulation_type == "SingleGrain")) - temperature.initialize(id, simulation_type, grid, inputs.domain.deltat); - else if (simulation_type == "Spot") - temperature.initialize(id, grid, irf.freezingRange(), inputs.domain.deltat, inputs.domain.spot_radius); - else if ((simulation_type == "FromFile") || (simulation_type == "FromFinch")) - temperature.initialize(0, id, grid, irf.freezingRange(), inputs.domain.deltat, simulation_type); - MPI_Barrier(MPI_COMM_WORLD); + int x_switch = 0; + + // Loop continues until all liquid cells claimed by solid grains, and no solid cells undergo remelting + do { + + // Start of time step - optional print current state of ExaCA simulation (up to and including the current + // layer's data) + print.printIntralayer(id, np, layernumber, inputs.domain.deltat, cycle, grid, celldata, temperature, interface, + orientation); + cycle++; + + // Cells with a successful nucleation event are marked and added to a steering vector, later dealt with in + // cellCapture + timers.startNucleation(); + nucleation.nucleateGrain(cycle, grid, celldata, interface); + timers.stopNucleation(); + + // Cells that have a successful nucleation event, and other cells that are at the solid-liquid interface are + // added to a steering vector. Logic in fillSteeringVector_NoRemelt is a simpified version of + // fillSteeringVector_Remelt + timers.startSV(); + if ((simulation_type == "Directional") || (simulation_type == "SingleGrain")) + fillSteeringVector_NoRemelt(cycle, grid, celldata, temperature, interface); + else + fillSteeringVector_Remelt(cycle, grid, celldata, temperature, interface); + timers.stopSV(); + + // Iterate over the steering vector to perform active cell creation and capture operations, and if needed, + // melting of cells that have gone above the liquidus. Also places halo cell data into send buffers, later + // checking the MPI buffers to ensure that all appropriate interface updates in the halo regions were + // recorded + timers.startCapture(); + cellCapture(cycle, np, grid, irf, celldata, temperature, interface, orientation); + checkBuffers(id, cycle, grid, celldata, interface, orientation.n_grain_orientations); + timers.stopCapture(); + + if (np > 1) { + // Update ghost nodes + timers.startComm(); + haloUpdate(cycle, id, grid, celldata, interface, orientation); + timers.stopComm(); + } - // Initialize grain orientations - Orientation orientation(id, inputs.grain_orientation_file, false); - MPI_Barrier(MPI_COMM_WORLD); + // Check on progress of solidification simulation of the current layer, setting x_switch = 1 if complete + if ((cycle % 1000 == 0) && (simulation_type != "SingleGrain")) { + intermediateOutputAndCheck(id, np, cycle, grid, nucleation.successful_nucleation_counter, x_switch, + celldata, temperature, inputs.simulation_type, layernumber, orientation, print, + inputs.domain.deltat, interface); + } + else if (simulation_type == "SingleGrain") { + intermediateOutputAndCheck(id, cycle, grid, x_switch, celldata.cell_type); + } - // Initialize cell types, grain IDs, and layer IDs - CellData celldata(grid, inputs.substrate, inputs.print.store_melt_pool_edge); - if (simulation_type == "Directional") - celldata.initSubstrate(id, grid, inputs.rng_seed); - else if (simulation_type == "SingleGrain") - celldata.initSubstrate(id, grid); - else - celldata.initSubstrate(id, grid, inputs.rng_seed, temperature.number_of_solidification_events); - MPI_Barrier(MPI_COMM_WORLD); + } while (x_switch == 0); - // Variables characterizing the active cell region within each rank's grid, including buffers for ghost node data - // (fixed size) and the steering vector/steering vector size on host/device - Interface interface(id, grid.domain_size, inputs.substrate.init_oct_size); - MPI_Barrier(MPI_COMM_WORLD); + // Reset intralayer print counter and print time series file for previous layer's intralayer data (if needed) + print.resetIntralayer(id, layernumber); - // Nucleation data structure, containing views of nuclei locations, time steps, and ids, and nucleation event - // counters - initialized with an estimate on the number of nuclei in the layer Without knowing - // estimated_nuclei_this_rank_this_layer yet, initialize nucleation data structures to estimated sizes, resize - // inside of placeNuclei when the number of nuclei per rank is known - int estimated_nuclei_this_rank_this_layer = inputs.nucleation.n_max * pow(grid.deltax, 3) * grid.domain_size; - Nucleation nucleation(estimated_nuclei_this_rank_this_layer, inputs.nucleation); - // Fill in nucleation data structures, and assign nucleation undercooling values to potential nucleation events - // Potential nucleation grains are only associated with liquid cells in layer 0 - they will be initialized for each - // successive layer when layer 0 is complete - nucleation.placeNuclei(temperature, inputs.rng_seed, 0, grid, id); - - // Initialize printing struct from inputs - Print print(grid, np, inputs.print); - - // End of initialization - timers.stopInit(); - MPI_Barrier(MPI_COMM_WORLD); + if (layernumber != grid.number_of_layers - 1) { + MPI_Barrier(MPI_COMM_WORLD); - int cycle = 0; - timers.startRun(); - for (int layernumber = 0; layernumber < grid.number_of_layers; layernumber++) { - - int x_switch = 0; - timers.startLayer(); - - // Loop continues until all liquid cells claimed by solid grains, and no solid cells undergo remelting - do { - - // Start of time step - optional print current state of ExaCA simulation (up to and including the current - // layer's data) - print.printIntralayer(id, np, layernumber, inputs.domain.deltat, cycle, grid, celldata, temperature, - interface, orientation); - cycle++; - - // Cells with a successful nucleation event are marked and added to a steering vector, later dealt with in - // cellCapture - timers.startNucleation(); - nucleation.nucleateGrain(cycle, grid, celldata, interface); - timers.stopNucleation(); - - // Cells that have a successful nucleation event, and other cells that are at the solid-liquid interface are - // added to a steering vector. Logic in fillSteeringVector_NoRemelt is a simpified version of - // fillSteeringVector_Remelt - timers.startSV(); - if ((simulation_type == "Directional") || (simulation_type == "SingleGrain")) - fillSteeringVector_NoRemelt(cycle, grid, celldata, temperature, interface); - else - fillSteeringVector_Remelt(cycle, grid, celldata, temperature, interface); - timers.stopSV(); - - // Iterate over the steering vector to perform active cell creation and capture operations, and if needed, - // melting of cells that have gone above the liquidus. Also places halo cell data into send buffers, later - // checking the MPI buffers to ensure that all appropriate interface updates in the halo regions were - // recorded - timers.startCapture(); - cellCapture(cycle, np, grid, irf, celldata, temperature, interface, orientation); - checkBuffers(id, cycle, grid, celldata, interface, orientation.n_grain_orientations); - timers.stopCapture(); - - if (np > 1) { - // Update ghost nodes - timers.startComm(); - haloUpdate(cycle, id, grid, celldata, interface, orientation); - timers.stopComm(); - } - - // Check on progress of solidification simulation of the current layer, setting x_switch = 1 if complete - if ((cycle % 1000 == 0) && (simulation_type != "SingleGrain")) { - intermediateOutputAndCheck(id, np, cycle, grid, nucleation.successful_nucleation_counter, x_switch, - celldata, temperature, inputs.simulation_type, layernumber, orientation, - print, inputs.domain.deltat, interface); - } - else if (simulation_type == "SingleGrain") { - intermediateOutputAndCheck(id, cycle, grid, x_switch, celldata.cell_type); - } - - } while (x_switch == 0); - - // Reset intralayer print counter and print time series file for previous layer's intralayer data (if needed) - print.resetIntralayer(id, layernumber); - - if (layernumber != grid.number_of_layers - 1) { - MPI_Barrier(MPI_COMM_WORLD); - - // Optional print current state of ExaCA - print.printInterlayer(id, np, layernumber, inputs.domain.deltat, cycle, grid, celldata, temperature, - interface, orientation); - - // Determine new active cell domain size and offset from bottom of global domain - grid.initNextLayer(id, simulation_type, layernumber + 1); - - // Initialize new temperature field data for layer "layernumber + 1" - // TODO: reorganize these temperature functions calls into a temperature.init_next_layer as done with the - // substrate - // If the next layer's temperature data isn't already stored, it should be read - if ((simulation_type == "FromFile") && (inputs.temperature.layerwise_temp_read)) - temperature.readTemperatureData(id, grid, layernumber + 1); - MPI_Barrier(MPI_COMM_WORLD); - // Initialize next layer's temperature data - temperature.initialize(layernumber + 1, id, grid, irf.freezingRange(), inputs.domain.deltat, - simulation_type); - - // Reset solidification event counter of all cells to zeros for the next layer, resizing to number of cells - // associated with the next layer, and get the subview for undercooling - temperature.resetLayerEventsUndercooling(grid); - // Resize and zero all view data relating to the active region from the last layer, in preparation for the - // next layer - interface.initNextLayer(grid.domain_size); - MPI_Barrier(MPI_COMM_WORLD); - - // Sets up views, powder layer (if necessary), and cell types for the next layer of a multilayer problem - celldata.initNextLayer(layernumber + 1, id, grid, inputs.rng_seed, - temperature.number_of_solidification_events); - - // Initialize potential nucleation event data for next layer "layernumber + 1" - // Views containing nucleation data will be resized to the possible number of nuclei on a given MPI rank for - // the next layer - nucleation.resetNucleiCounters(); // start counters at 0 - nucleation.placeNuclei(temperature, inputs.rng_seed, layernumber + 1, grid, id); - - x_switch = 0; - MPI_Barrier(MPI_COMM_WORLD); - cycle = 0; - timers.stopLayer(layernumber); - } - else { - MPI_Barrier(MPI_COMM_WORLD); - timers.stopLayer(); - } + // Optional print current state of ExaCA + print.printInterlayer(id, np, layernumber, inputs.domain.deltat, cycle, grid, celldata, temperature, interface, + orientation); + + // If more layers to simulate, determine new active cell domain size and offset from bottom of global domain + grid.initNextLayer(id, simulation_type, layernumber + 1); + } +} + +// Initialize structs for the next layer of a multilayer ExaCA simulation +template +void initExaCALayer(int id, int np, int layernumber, int &cycle, Inputs inputs, Grid &grid, + Temperature temperature, Orientation &orientation, + CellData &celldata, Interface &interface, + Nucleation &nucleation, Print &print, std::string simulation_type) { + + if (layernumber != grid.number_of_layers - 1) { + // Resize and zero all view data relating to the active region from the last layer, in preparation for the + // next layer + interface.initNextLayer(grid.domain_size); + MPI_Barrier(MPI_COMM_WORLD); + + // Sets up views, powder layer (if necessary), and cell types for the next layer of a multilayer problem + celldata.initNextLayer(layernumber + 1, id, grid, inputs.rng_seed, temperature.number_of_solidification_events); + + // Initialize potential nucleation event data for next layer "layernumber + 1" + // Views containing nucleation data will be resized to the possible number of nuclei on a given MPI rank for + // the next layer + nucleation.resetNucleiCounters(); // start counters at 0 + nucleation.placeNuclei(temperature, inputs.rng_seed, layernumber + 1, grid, id); + + // If not on the last layer, reset the time step counter to 0 (if on the last layer, the time step is stored and + // printed in the log file) + cycle = 0; } - timers.stopRun(); MPI_Barrier(MPI_COMM_WORLD); +} + +// Finalize timer values, print end-of-run output to the console/files, and print the log file +template +void finalizeExaCA(int id, int np, int cycle, Inputs inputs, Timers timers, Grid &grid, + Temperature temperature, InterfacialResponseFunction irf, + Orientation &orientation, CellData &celldata, + Interface &interface, Nucleation &nucleation, Print &print, + std::string simulation_type) { + timers.startOutput(); // Collect and print specified final fields to output files diff --git a/unit_test/tstTemperature.hpp b/unit_test/tstTemperature.hpp index 834fe749..ea53ca82 100644 --- a/unit_test/tstTemperature.hpp +++ b/unit_test/tstTemperature.hpp @@ -304,56 +304,54 @@ void testInit_UnidirectionalGradient(const std::string simulation_type, const do // Check temperature initialization for translated/mirrored FromFile or FromFinch data template void checkTemperatureResults(Inputs &inputs, Grid &grid, Temperature &temperature, - const int number_of_copies, const int id, const int np, const bool mirror_x) { - const int expected_num_data_points = grid.nx * number_of_copies * np; - std::vector expected_data_point_found(expected_num_data_points, false); - for (int n = 0; n < expected_num_data_points; n++) { - // X,Y,Z the cell would be placed at - const int coord_x = temperature.getTempCoordX(n, grid.x_min, grid.deltax); - const int coord_y = temperature.getTempCoordY(n, grid.y_min, grid.deltax, grid.y_offset); - const int coord_z = temperature.getTempCoordZ(n, grid.deltax, grid.layer_height, 0, grid.z_min_layer); - const int index = grid.get1DIndex(coord_x, coord_y, coord_z); - if (!expected_data_point_found[index]) - expected_data_point_found[index] = true; - else { - std::string error = "Error: Rank " + std::to_string(id) + " has more than one data point at location (" + - std::to_string(coord_x) + "," + std::to_string(coord_y) + "," + std::to_string(coord_z); - throw std::runtime_error(error); + const int number_of_copies, const int id, const int np, const bool mirror_x, + const int number_of_layers, const int num_layers_with_points) { + + // Should have equal numbers of points associated with each layer - if initializing only 1 layer at a time, the data + // from the other layers should not have been stored + const int expected_num_data_points = grid.nx * number_of_copies * np * num_layers_with_points; + EXPECT_EQ(temperature.raw_temperature_data.extent(0), expected_num_data_points); + std::vector expected_num_points_per_cell(grid.domain_size, 0); + for (int layernumber = 0; layernumber < num_layers_with_points; layernumber++) { + for (int n = 0; n < grid.nx * number_of_copies * np; n++) { + int point_num = layernumber * grid.nx * number_of_copies * np + n; + // X,Y,Z the cell would be placed at + const int coord_x = temperature.getTempCoordX(point_num, grid.x_min, grid.deltax); + const int coord_y = temperature.getTempCoordY(point_num, grid.y_min, grid.deltax, grid.y_offset); + const int coord_z = + temperature.getTempCoordZ(point_num, grid.deltax, grid.layer_height, layernumber, grid.z_min_layer); + const int index = grid.get1DIndex(coord_x, coord_y, coord_z); + expected_num_points_per_cell[index]++; + // Make sure the cell is associated with the correct temperature data from "input_temperature_data" + // TM for even numbered translations (or when mirror_x is false) is equal to the X coordinate plus any + // applied temporal offset in Y related to its Y coordinate. If mirrored, odd numbered translations will use + // (nx - coord_x) in place of coord_x in these calculations. TL should be (1 * layernumber) larger than TM, + // and the cooling rate is proportional to 100 times the MPI rank the point was initalized on (in turn equal + // to its Z coordinate) + const bool x_coord_mirrored = ((mirror_x) && (coord_y % 2 == 1)); + double loc_along_scan; + if (x_coord_mirrored) + loc_along_scan = static_cast((grid.nx - 1) - coord_x); + else + loc_along_scan = static_cast(coord_x); + const double expected_tm = + loc_along_scan + static_cast(coord_y) * inputs.temperature.temporal_offset; + const double expected_tl = expected_tm + 1.0 * (layernumber + 1); + const double expected_cr = 100.0 * static_cast(coord_z + 1); + EXPECT_DOUBLE_EQ(expected_tm, temperature.raw_temperature_data(point_num, 3)); + EXPECT_DOUBLE_EQ(expected_tl, temperature.raw_temperature_data(point_num, 4)); + EXPECT_DOUBLE_EQ(expected_cr, temperature.raw_temperature_data(point_num, 5)); } - // Make sure the cell is associated with the correct temperature data from "input_temperature_data" - // TM for even numbered translations (or when mirror_x is false) is equal to the X coordinate plus any applied - // temporal offset in Y related to its Y coordinate. If mirrored, odd numbered translations will use (nx - - // coord_x) in place of coord_x in these calculations. TL should be one larger than TM, and the cooling rate is - // proportional to 100 times the MPI rank the point was initalized on (in turn equal to its Z coordinate) - const bool x_coord_mirrored = ((mirror_x) && (coord_y % 2 == 1)); - double loc_along_scan; - if (x_coord_mirrored) - loc_along_scan = static_cast((grid.nx - 1) - coord_x); - else - loc_along_scan = static_cast(coord_x); - const double expected_tm = loc_along_scan + static_cast(coord_y) * inputs.temperature.temporal_offset; - const double expected_tl = expected_tm + 1.0; - const double expected_cr = 100.0 * static_cast(coord_z + 1); - EXPECT_DOUBLE_EQ(expected_tm, temperature.raw_temperature_data(n, 3)); - EXPECT_DOUBLE_EQ(expected_tl, temperature.raw_temperature_data(n, 4)); - EXPECT_DOUBLE_EQ(expected_cr, temperature.raw_temperature_data(n, 5)); } - // Make sure one data point was present in each expected cell - for (int n = 0; n < expected_num_data_points; n++) { - const int coord_x = temperature.getTempCoordX(n, grid.x_min, grid.deltax); - const int coord_y = temperature.getTempCoordY(n, grid.y_min, grid.deltax, grid.y_offset); - const int coord_z = temperature.getTempCoordZ(n, grid.deltax, grid.layer_height, 0, grid.z_min_layer); - const int index = grid.get1DIndex(coord_x, coord_y, coord_z); - if (!expected_data_point_found[index]) { - if (!expected_data_point_found[index]) - expected_data_point_found[index] = true; - else { - std::string error = "Error: Rank " + std::to_string(id) + - " has more than one data point at location (" + std::to_string(coord_x) + "," + - std::to_string(coord_y) + "," + std::to_string(coord_z); - throw std::runtime_error(error); - } + // Make sure one data point was present in each expected cell per each layer of data stored. With number_of_copies = + // 1, data will only be present at Y = 0 locally, and each copy is translated 1 cell in Y + for (int index = 0; index < grid.domain_size; index++) { + const int coord_x = grid.getCoordX(index); + const int coord_y = grid.getCoordY(index); + const int coord_z = grid.getCoordZ(index); + if ((coord_y < number_of_copies) && (coord_z < np)) { + EXPECT_EQ(expected_num_points_per_cell[index], num_layers_with_points); } } } @@ -390,7 +388,8 @@ void initTestGrid(const int id, const int np, Grid &grid) { } // Test storing temperature data from Finch on the correct ExaCA ranks, optionally mirroring and translating the data -void testInitTemperatureFromFinch(const bool mirror_x, const int number_of_copies) { +void testInitTemperatureFromFinch(const bool mirror_x, const int number_of_copies, const bool layerwise_temp_init, + const int number_of_layers) { using memory_space = TEST_MEMSPACE; using view_type_coupled = Kokkos::View; @@ -403,35 +402,53 @@ void testInitTemperatureFromFinch(const bool mirror_x, const int number_of_copie // Default inputs struct - manually set temperature values for test Inputs inputs; + inputs.temperature.layerwise_temp_read = layerwise_temp_init; + inputs.temperature.temp_files_in_series = number_of_layers; // Default grid struct - manually set up domain for test Grid grid; initTestInputs(mirror_x, number_of_copies, inputs); initTestGrid(id, np, grid); - - // Create dummy Finch temperature data stored on various ranks, to be mapped to ExaCA ranks - view_type_coupled input_temperature_data(Kokkos::ViewAllocateWithoutInitializing("FinchData"), grid.nx * np, 6); - - // Each rank has Finch data at a given Z location, at all relevant X values and at every third Y coordinate (so each - // ExaCA rank after rearranging the data will only own data from one of the Y coordinates). Number of data points - // proportional to the number of MPI ranks + grid.number_of_layers = number_of_layers; + + // Create dummy Finch temperature data stored on various ranks, to be mapped to ExaCA ranks. Each layer has nx * np + // locations with data + const int locations_with_data = np * grid.nx * number_of_layers; + int num_layers_with_points; + if (layerwise_temp_init) + num_layers_with_points = 1; + else + num_layers_with_points = number_of_layers; + view_type_coupled input_temperature_data(Kokkos::ViewAllocateWithoutInitializing("FinchData"), locations_with_data, + 6); + std::vector first_value, last_value; int finch_counter = 0; - for (int coord_x = 0; coord_x < grid.nx; coord_x++) { - for (int coord_y = 0; coord_y < np; coord_y++) { - input_temperature_data(finch_counter, 0) = grid.x_min + grid.deltax * coord_x; - input_temperature_data(finch_counter, 1) = grid.y_min + 3.0 * coord_y * grid.deltax; - input_temperature_data(finch_counter, 2) = grid.z_min + id * grid.deltax; - input_temperature_data(finch_counter, 3) = static_cast(coord_x); - input_temperature_data(finch_counter, 4) = static_cast(coord_x) + 1.0; - input_temperature_data(finch_counter, 5) = 100.0 * (id + 1); - finch_counter++; + for (int layernumber = 0; layernumber < num_layers_with_points; layernumber++) { + first_value.push_back(finch_counter); + // Each rank has Finch data at a given Z location, at all relevant X values and at every third Y coordinate (so + // each ExaCA rank after rearranging the data will only own data from one of the Y coordinates). Number of data + // points proportional to the number of MPI ranks and varies based on the layer number. Liquidus times are + // offset from the melting times by a value proportional to the layer number + for (int coord_x = 0; coord_x < grid.nx; coord_x++) { + for (int coord_y = 0; coord_y < np; coord_y++) { + input_temperature_data(finch_counter, 0) = grid.x_min + grid.deltax * coord_x; + input_temperature_data(finch_counter, 1) = grid.y_min + 3.0 * coord_y * grid.deltax; + input_temperature_data(finch_counter, 2) = grid.z_min + id * grid.deltax; + input_temperature_data(finch_counter, 3) = static_cast(coord_x); + input_temperature_data(finch_counter, 4) = static_cast(coord_x) + 1.0 * (layernumber + 1); + input_temperature_data(finch_counter, 5) = 100.0 * (id + 1); + finch_counter++; + } } + last_value.push_back(finch_counter); } // Construct temperature views - Temperature temperature(id, np, grid, inputs.temperature, input_temperature_data); + Temperature temperature(id, np, grid, inputs.temperature, input_temperature_data, first_value, + last_value); // Check that the right ranks have the right temperature data - checkTemperatureResults(inputs, grid, temperature, number_of_copies, id, np, mirror_x); + checkTemperatureResults(inputs, grid, temperature, number_of_copies, id, np, mirror_x, number_of_layers, + num_layers_with_points); } // Test storing temperature data from a file on the correct ExaCA ranks, using the same process at the FromFinch test @@ -481,7 +498,8 @@ void testInitTemperatureFromFile(const bool mirror_x, const int number_of_copies temperature.readTemperatureData(id, grid, 0); // Check that the right ranks have the right temperature data - checkTemperatureResults(inputs, grid, temperature, number_of_copies, id, np, mirror_x); + checkTemperatureResults(inputs, grid, temperature, number_of_copies, id, np, mirror_x, grid.number_of_layers, + grid.number_of_layers); } //---------------------------------------------------------------------------// @@ -504,9 +522,17 @@ TEST(TEST_CATEGORY, temperature) { testInit_UnidirectionalGradient("Directional", 1000000, 2); testInit_UnidirectionalGradient("SingleGrain", 0, 2); testInit_UnidirectionalGradient("SingleGrain", 1000000, 2); - testInitTemperatureFromFinch(false, 1); - testInitTemperatureFromFinch(false, 3); - testInitTemperatureFromFinch(true, 3); + // Coupled Finch-ExaCA simulation init tests: mirror_x (true/false), number_of_copies, layerwise_temp_init + // (true/false), number_of_layers + std::vector mirror_x_vals = {false, false, true, false, false}; + std::vector number_of_copies_vals = {1, 3, 3, 1, 1}; + std::vector layerwise_temp_init_vals = {false, false, false, false, true}; + std::vector number_of_layers_finch_vals = {1, 1, 1, 2, 2}; + num_vals = mirror_x_vals.size(); + for (int test_count = 0; test_count < num_vals; test_count++) { + testInitTemperatureFromFinch(mirror_x_vals[test_count], number_of_copies_vals[test_count], + layerwise_temp_init_vals[test_count], number_of_layers_finch_vals[test_count]); + } testInitTemperatureFromFile(false, 1); testInitTemperatureFromFile(false, 3); testInitTemperatureFromFile(true, 3); diff --git a/unit_test/tstUpdate.hpp b/unit_test/tstUpdate.hpp index 181f8dad..0d94d0f3 100644 --- a/unit_test/tstUpdate.hpp +++ b/unit_test/tstUpdate.hpp @@ -44,11 +44,58 @@ void testSmallDirS() { inputs.temperature); // Temperature fields characterized by data in this structure Temperature temperature(grid, inputs.temperature, inputs.print.store_solidification_start); + temperature.initialize(id, "Directional", grid, inputs.domain.deltat); + MPI_Barrier(MPI_COMM_WORLD); // Run SmallDirS problem and check volume fraction of nucleated grains with 1% tolerance of expected value (to // account for the non-deterministic nature of the cell capture) - runExaCA(id, np, inputs, timers, grid, temperature); + // Material response function + InterfacialResponseFunction irf(inputs.domain.deltat, grid.deltax, inputs.irf); + + // Initialize grain orientations + Orientation orientation(id, inputs.grain_orientation_file, false); + MPI_Barrier(MPI_COMM_WORLD); + + // Initialize cell types, grain IDs, and layer IDs + CellData celldata(grid, inputs.substrate, inputs.print.store_melt_pool_edge); + celldata.initSubstrate(id, grid, inputs.rng_seed); + MPI_Barrier(MPI_COMM_WORLD); + + // Variables characterizing the active cell region within each rank's grid, including buffers for ghost node data + // (fixed size) and the steering vector/steering vector size on host/device + Interface interface(id, grid.domain_size, inputs.substrate.init_oct_size); + MPI_Barrier(MPI_COMM_WORLD); + + // Nucleation data structure, containing views of nuclei locations, time steps, and ids, and nucleation event + // counters - initialized with an estimate on the number of nuclei in the layer Without knowing + // estimated_nuclei_this_rank_this_layer yet, initialize nucleation data structures to estimated sizes, resize + // inside of placeNuclei when the number of nuclei per rank is known + int estimated_nuclei_this_rank_this_layer = inputs.nucleation.n_max * pow(grid.deltax, 3) * grid.domain_size; + Nucleation nucleation(estimated_nuclei_this_rank_this_layer, inputs.nucleation); + // Fill in nucleation data structures, and assign nucleation undercooling values to potential nucleation events + // Potential nucleation grains are only associated with liquid cells in layer 0 - they will be initialized for each + // successive layer when layer 0 is complete + nucleation.placeNuclei(temperature, inputs.rng_seed, 0, grid, id); + + // Initialize printing struct from inputs + Print print(grid, np, inputs.print); + + // End of initialization + timers.stopInit(); + MPI_Barrier(MPI_COMM_WORLD); + + int cycle = 0; + timers.startRun(); + + runExaCALayer(id, np, 0, cycle, inputs, timers, grid, temperature, irf, orientation, celldata, interface, + nucleation, print, "Directional"); + + timers.stopRun(); + MPI_Barrier(MPI_COMM_WORLD); + // Print ExaCA end-of-run data + finalizeExaCA(id, np, cycle, inputs, timers, grid, temperature, irf, orientation, celldata, interface, nucleation, + print, "Directional"); // MPI barrier to ensure that log file has been written MPI_Barrier(MPI_COMM_WORLD); std::string log_file = "TestProblemSmallDirS.json"; @@ -76,9 +123,56 @@ void testSmallEquiaxedGrain() { inputs.temperature); // Temperature fields characterized by data in this structure Temperature temperature(grid, inputs.temperature, inputs.print.store_solidification_start); + temperature.initialize(id, "SingleGrain", grid, inputs.domain.deltat); // Run Small equiaxed grain problem and check time step at which the grain reaches the domain edge - runExaCA(id, np, inputs, timers, grid, temperature); + // Material response function + InterfacialResponseFunction irf(inputs.domain.deltat, grid.deltax, inputs.irf); + + // Initialize grain orientations + Orientation orientation(id, inputs.grain_orientation_file, false); + MPI_Barrier(MPI_COMM_WORLD); + + // Initialize cell types, grain IDs, and layer IDs + CellData celldata(grid, inputs.substrate, inputs.print.store_melt_pool_edge); + celldata.initSubstrate(id, grid); + MPI_Barrier(MPI_COMM_WORLD); + + // Variables characterizing the active cell region within each rank's grid, including buffers for ghost node data + // (fixed size) and the steering vector/steering vector size on host/device + Interface interface(id, grid.domain_size, inputs.substrate.init_oct_size); + MPI_Barrier(MPI_COMM_WORLD); + + // Nucleation data structure, containing views of nuclei locations, time steps, and ids, and nucleation event + // counters - initialized with an estimate on the number of nuclei in the layer Without knowing + // estimated_nuclei_this_rank_this_layer yet, initialize nucleation data structures to estimated sizes, resize + // inside of placeNuclei when the number of nuclei per rank is known + int estimated_nuclei_this_rank_this_layer = inputs.nucleation.n_max * pow(grid.deltax, 3) * grid.domain_size; + Nucleation nucleation(estimated_nuclei_this_rank_this_layer, inputs.nucleation); + // Fill in nucleation data structures, and assign nucleation undercooling values to potential nucleation events + // Potential nucleation grains are only associated with liquid cells in layer 0 - they will be initialized for each + // successive layer when layer 0 is complete + nucleation.placeNuclei(temperature, inputs.rng_seed, 0, grid, id); + + // Initialize printing struct from inputs + Print print(grid, np, inputs.print); + + // End of initialization + timers.stopInit(); + MPI_Barrier(MPI_COMM_WORLD); + + int cycle = 0; + timers.startRun(); + + runExaCALayer(id, np, 0, cycle, inputs, timers, grid, temperature, irf, orientation, celldata, interface, + nucleation, print, "SingleGrain"); + + timers.stopRun(); + MPI_Barrier(MPI_COMM_WORLD); + + // Print ExaCA end-of-run data + finalizeExaCA(id, np, cycle, inputs, timers, grid, temperature, irf, orientation, celldata, interface, nucleation, + print, "SingleGrain"); // MPI barrier to ensure that log file has been written MPI_Barrier(MPI_COMM_WORLD); From 2e727013f8794ef25516ad531af7dc4153346d94 Mon Sep 17 00:00:00 2001 From: Matt Rolchigo Date: Thu, 5 Jun 2025 17:17:23 -0400 Subject: [PATCH 2/3] Cleanup to get rid of warnings --- bin/run.cpp | 6 ++-- bin/runCoupled.cpp | 10 +++--- src/runCA.hpp | 16 +++------ unit_test/test_harness_m.cmake | 59 ++++++++++++++++++++++++++++++++++ unit_test/tstTemperature.hpp | 11 +++---- unit_test/tstUpdate.hpp | 6 ++-- 6 files changed, 76 insertions(+), 32 deletions(-) create mode 100644 unit_test/test_harness_m.cmake diff --git a/bin/run.cpp b/bin/run.cpp index f1e5379e..203d2508 100644 --- a/bin/run.cpp +++ b/bin/run.cpp @@ -130,8 +130,7 @@ int main(int argc, char *argv[]) { temperature.resetLayerEventsUndercooling(grid); // Initialize next layer of the simulation - initExaCALayer(id, np, layernumber, cycle, inputs, grid, temperature, orientation, celldata, - interface, nucleation, print, simulation_type); + initExaCALayer(id, layernumber, cycle, inputs, grid, temperature, celldata, interface, nucleation); timers.stopLayer(layernumber); } else { @@ -143,8 +142,7 @@ int main(int argc, char *argv[]) { MPI_Barrier(MPI_COMM_WORLD); // Print ExaCA end-of-run data - finalizeExaCA(id, np, cycle, inputs, timers, grid, temperature, irf, orientation, celldata, interface, - nucleation, print, simulation_type); + finalizeExaCA(id, np, cycle, inputs, timers, grid, temperature, orientation, celldata, interface, print); } } // Finalize Kokkos diff --git a/bin/runCoupled.cpp b/bin/runCoupled.cpp index ae8baf4c..ad40466e 100644 --- a/bin/runCoupled.cpp +++ b/bin/runCoupled.cpp @@ -18,7 +18,7 @@ // of layers where the return view stores time-temperature histories for layers l=0,1,2... where first_value[l] is the // index of the first event and last_value[l]-1 is the index of the last event associated with layer l Kokkos::View -getFinchData(const int id, const int np, const int first_finch_simulation, const int num_finch_simulations, +getFinchData(const int, const int, const int first_finch_simulation, const int num_finch_simulations, const int number_of_layers, Inputs exaca_inputs, std::array &exaca_low_corner, std::array &exaca_high_corner, std::vector &first_value_finch, std::vector &last_value_finch) { @@ -154,7 +154,7 @@ int main(int argc, char *argv[]) { std::array exaca_low_corner, exaca_high_corner; // Should Finch simulations for all layers be performed and time-temperature history data stored prior to // running ExaCA, or should each Finch simulation be run one at a time between ExaCA-simulated layers? - int num_finch_simulations, first_finch_simulation; + int num_finch_simulations; if (inputs.temperature.layerwise_temp_read) num_finch_simulations = 1; else @@ -253,8 +253,7 @@ int main(int argc, char *argv[]) { temperature.resetLayerEventsUndercooling(grid); // Initialize next layer of the simulation - initExaCALayer(id, np, layernumber, cycle, inputs, grid, temperature, orientation, celldata, interface, - nucleation, print, "FromFinch"); + initExaCALayer(id, layernumber, cycle, inputs, grid, temperature, celldata, interface, nucleation); timers.stopLayer(layernumber); } else { @@ -266,8 +265,7 @@ int main(int argc, char *argv[]) { MPI_Barrier(MPI_COMM_WORLD); // Print ExaCA end-of-run data - finalizeExaCA(id, np, cycle, inputs, timers, grid, temperature, irf, orientation, celldata, interface, - nucleation, print, "FromFinch"); + finalizeExaCA(id, np, cycle, inputs, timers, grid, temperature, orientation, celldata, interface, print); } // Finalize Kokkos & MPI diff --git a/src/runCA.hpp b/src/runCA.hpp index ed12edb7..65c66e82 100644 --- a/src/runCA.hpp +++ b/src/runCA.hpp @@ -21,9 +21,6 @@ void runExaCALayer(int id, int np, int layernumber, int &cycle, Inputs inputs, T Interface &interface, Nucleation &nucleation, Print &print, std::string simulation_type) { - // Run on the default space. - using memory_space = MemorySpace; - int x_switch = 0; // Loop continues until all liquid cells claimed by solid grains, and no solid cells undergo remelting @@ -96,10 +93,9 @@ void runExaCALayer(int id, int np, int layernumber, int &cycle, Inputs inputs, T // Initialize structs for the next layer of a multilayer ExaCA simulation template -void initExaCALayer(int id, int np, int layernumber, int &cycle, Inputs inputs, Grid &grid, - Temperature temperature, Orientation &orientation, - CellData &celldata, Interface &interface, - Nucleation &nucleation, Print &print, std::string simulation_type) { +void initExaCALayer(int id, int layernumber, int &cycle, Inputs inputs, Grid &grid, + Temperature temperature, CellData &celldata, + Interface &interface, Nucleation &nucleation) { if (layernumber != grid.number_of_layers - 1) { // Resize and zero all view data relating to the active region from the last layer, in preparation for the @@ -126,10 +122,8 @@ void initExaCALayer(int id, int np, int layernumber, int &cycle, Inputs inputs, // Finalize timer values, print end-of-run output to the console/files, and print the log file template void finalizeExaCA(int id, int np, int cycle, Inputs inputs, Timers timers, Grid &grid, - Temperature temperature, InterfacialResponseFunction irf, - Orientation &orientation, CellData &celldata, - Interface &interface, Nucleation &nucleation, Print &print, - std::string simulation_type) { + Temperature temperature, Orientation &orientation, + CellData &celldata, Interface &interface, Print &print) { timers.startOutput(); diff --git a/unit_test/test_harness_m.cmake b/unit_test/test_harness_m.cmake new file mode 100644 index 00000000..197f23c0 --- /dev/null +++ b/unit_test/test_harness_m.cmake @@ -0,0 +1,59 @@ +# --------------------------------------------------------------------------## +# Create main tests (all tests use Kokkos) +# --------------------------------------------------------------------------## +macro(ExaCA_add_tests) + cmake_parse_arguments(EXACA_UNIT_TEST "MPI" "PACKAGE" "NAMES" ${ARGN}) + set(EXACA_UNIT_TEST_MPIEXEC_NUMPROCS 1) + if(EXACA_UNIT_TEST_MPI) + foreach(_np 2 4) + if(MPIEXEC_MAX_NUMPROCS GREATER_EQUAL ${_np}) + list(APPEND EXACA_UNIT_TEST_MPIEXEC_NUMPROCS ${_np}) + endif() + endforeach() + endif() + set(EXACA_UNIT_TEST_NUMTHREADS 1) + foreach(_nt 2 4) + if(MPIEXEC_MAX_NUMPROCS GREATER_EQUAL ${_nt}) + list(APPEND EXACA_UNIT_TEST_NUMTHREADS ${_nt}) + endif() + endforeach() + set(EXACA_UNIT_TEST_MAIN ${TEST_HARNESS_DIR}/unit_test_main.cpp) + foreach(_device ${EXACA_TEST_DEVICES}) + set(_dir ${CMAKE_CURRENT_BINARY_DIR}/${_device}) + file(MAKE_DIRECTORY ${_dir}) + foreach(_test ${EXACA_UNIT_TEST_NAMES}) + set(_file ${_dir}/tst${_test}_${_device}.cpp) + file(WRITE ${_file} "#include \n" + "#include \n") + set(_target ExaCA_${_test}_test_${_device}) + add_executable(${_target} ${_file} ${EXACA_UNIT_TEST_MAIN}) + target_include_directories(${_target} PRIVATE ${_dir} ${TEST_HARNESS_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}) + target_link_libraries(${_target} ${EXACA_UNIT_TEST_PACKAGE} + ${gtest_target}) + + foreach(_np ${EXACA_UNIT_TEST_MPIEXEC_NUMPROCS}) + # FIXME: remove PTHREAD + if(_device STREQUAL PTHREAD + OR _device STREQUAL THREADS + OR _device STREQUAL OPENMP) + foreach(_thread ${EXACA_UNIT_TEST_NUMTHREADS}) + add_test( + NAME ${_target}_np_${_np}_nt_${_thread} + COMMAND + ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${_np} + ${MPIEXEC_PREFLAGS} $ + ${MPIEXEC_POSTFLAGS} ${gtest_args} --kokkos-threads=${_thread}) + endforeach() + else() + add_test( + NAME ${_target}_np_${_np} + COMMAND + ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${_np} + ${MPIEXEC_PREFLAGS} $ + ${MPIEXEC_POSTFLAGS} ${gtest_args}) + endif() + endforeach() + endforeach() + endforeach() +endmacro() diff --git a/unit_test/tstTemperature.hpp b/unit_test/tstTemperature.hpp index ea53ca82..a55e2529 100644 --- a/unit_test/tstTemperature.hpp +++ b/unit_test/tstTemperature.hpp @@ -304,8 +304,8 @@ void testInit_UnidirectionalGradient(const std::string simulation_type, const do // Check temperature initialization for translated/mirrored FromFile or FromFinch data template void checkTemperatureResults(Inputs &inputs, Grid &grid, Temperature &temperature, - const int number_of_copies, const int id, const int np, const bool mirror_x, - const int number_of_layers, const int num_layers_with_points) { + const int number_of_copies, const int, const int np, const bool mirror_x, + const int num_layers_with_points) { // Should have equal numbers of points associated with each layer - if initializing only 1 layer at a time, the data // from the other layers should not have been stored @@ -347,7 +347,6 @@ void checkTemperatureResults(Inputs &inputs, Grid &grid, Temperature Date: Thu, 5 Jun 2025 17:37:04 -0400 Subject: [PATCH 3/3] getSolidificationDataBounds in Finch not working with communicator? --- bin/runCoupled.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/runCoupled.cpp b/bin/runCoupled.cpp index ad40466e..aa9b3ee2 100644 --- a/bin/runCoupled.cpp +++ b/bin/runCoupled.cpp @@ -57,8 +57,8 @@ getFinchData(const int, const int, const int first_finch_simulation, const int n // options were specified std::array exaca_low_corner_layer = finch_inputs.space.global_low_corner; std::array exaca_high_corner_layer = finch_inputs.space.global_high_corner; - std::array finch_low_corner_layer = app.getLowerSolidificationDataBounds(MPI_COMM_WORLD); - std::array finch_high_corner_layer = app.getUpperSolidificationDataBounds(MPI_COMM_WORLD); + std::array finch_low_corner_layer = app.getLowerSolidificationDataBounds(); + std::array finch_high_corner_layer = app.getUpperSolidificationDataBounds(); if (exaca_inputs.temperature.trim_unmelted_region_xy) { exaca_low_corner_layer[0] = finch_low_corner_layer[0]; exaca_high_corner_layer[0] = finch_high_corner_layer[0];