From 9cfff4614a466826a84dd94cb4addfa500729780 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Tue, 29 Apr 2025 11:12:30 +0200 Subject: [PATCH 1/2] SWMR functionality --- hdf5/examples/swmr.rs | 39 +++++++++++++++++++++++++++++++++++++++ hdf5/src/hl/dataset.rs | 18 ++++++++++++++++-- hdf5/src/hl/file.rs | 16 +++++++++++++--- 3 files changed, 68 insertions(+), 5 deletions(-) create mode 100644 hdf5/examples/swmr.rs diff --git a/hdf5/examples/swmr.rs b/hdf5/examples/swmr.rs new file mode 100644 index 00000000..d97b595a --- /dev/null +++ b/hdf5/examples/swmr.rs @@ -0,0 +1,39 @@ +use hdf5_metno as hdf5; + +fn reader() { + let file = hdf5::File::open_as("swmr.h5", hdf5::OpenMode::ReadSWMR).unwrap(); + println!("Reader: Opened file"); + let var = file.dataset("foo").unwrap(); + + for _ in 0..5 { + var.refresh().unwrap(); + let shape = var.shape(); + println!("Reader: Got shape: {shape:?}"); + // If reading one should use the shape directly without + // using the convenience read_2d etc. functions which + // might get confused if the shape is changed during reading + std::thread::sleep(std::time::Duration::from_secs(5)); + } +} + +fn main() { + let file = + hdf5::File::with_options().with_fapl(|fapl| fapl.libver_v110()).create("swmr.h5").unwrap(); + + let var = file.new_dataset::().shape((0.., 5)).create("foo").unwrap(); + + file.start_swmr().unwrap(); + println!("Writer: Wrote file"); + + let thread = std::thread::spawn(|| reader()); + + for i in 0..5 { + var.resize((i + 1, 5)).unwrap(); + var.write_slice(&[i, i, i, i, i], (i, 0..)).unwrap(); + var.flush().unwrap(); + println!("Writer: Wrote {i}"); + std::thread::sleep(std::time::Duration::from_secs(5)); + } + + thread.join().unwrap(); +} diff --git a/hdf5/src/hl/dataset.rs b/hdf5/src/hl/dataset.rs index 77807c53..f8f28127 100644 --- a/hdf5/src/hl/dataset.rs +++ b/hdf5/src/hl/dataset.rs @@ -5,8 +5,8 @@ use ndarray::{self, ArrayView}; use hdf5_sys::h5::HADDR_UNDEF; use hdf5_sys::h5d::{ - H5Dcreate2, H5Dcreate_anon, H5Dget_access_plist, H5Dget_create_plist, H5Dget_offset, - H5Dset_extent, + H5Dcreate2, H5Dcreate_anon, H5Dflush, H5Dget_access_plist, H5Dget_create_plist, H5Dget_offset, + H5Drefresh, H5Dset_extent, }; use hdf5_sys::h5l::H5Ldelete; use hdf5_sys::h5p::H5P_DEFAULT; @@ -154,6 +154,20 @@ impl Dataset { pub fn filters(&self) -> Vec { self.dcpl().map_or(Vec::default(), |pl| pl.filters()) } + + /// Flush the dataset metadata from the metadata cache to the file + pub fn flush(&self) -> Result<()> { + let id = self.id(); + h5call!(H5Dflush(id))?; + Ok(()) + } + + /// Refresh metadata items assosicated with the dataset + pub fn refresh(&self) -> Result<()> { + let id = self.id(); + h5call!(H5Drefresh(id))?; + Ok(()) + } } pub struct Maybe(Option); diff --git a/hdf5/src/hl/file.rs b/hdf5/src/hl/file.rs index 81ad839d..dea3f1d3 100644 --- a/hdf5/src/hl/file.rs +++ b/hdf5/src/hl/file.rs @@ -5,8 +5,9 @@ use std::path::Path; use hdf5_sys::h5f::{ H5Fclose, H5Fcreate, H5Fflush, H5Fget_access_plist, H5Fget_create_plist, H5Fget_filesize, - H5Fget_freespace, H5Fget_intent, H5Fget_obj_count, H5Fget_obj_ids, H5Fopen, H5F_ACC_DEFAULT, - H5F_ACC_EXCL, H5F_ACC_RDONLY, H5F_ACC_RDWR, H5F_ACC_TRUNC, H5F_SCOPE_LOCAL, + H5Fget_freespace, H5Fget_intent, H5Fget_obj_count, H5Fget_obj_ids, H5Fopen, + H5Fstart_swmr_write, H5F_ACC_DEFAULT, H5F_ACC_EXCL, H5F_ACC_RDONLY, H5F_ACC_RDWR, + H5F_ACC_SWMR_READ, H5F_ACC_TRUNC, H5F_SCOPE_LOCAL, }; use crate::hl::plist::{ @@ -20,6 +21,8 @@ use crate::internal_prelude::*; pub enum OpenMode { /// Open a file as read-only, file must exist. Read, + /// Open a file as read-only in SWMR mode, file must exist. + ReadSWMR, /// Open a file as read/write, file must exist. ReadWrite, /// Create a file, truncate if exists. @@ -178,6 +181,12 @@ impl File { pub fn fcpl(&self) -> Result { self.create_plist() } + + pub fn start_swmr(&self) -> Result<()> { + let id = self.id(); + h5call!(H5Fstart_swmr_write(id))?; + Ok(()) + } } /// File builder allowing to customize file access/creation property lists. @@ -231,6 +240,7 @@ impl FileBuilder { )?; let flags = match mode { OpenMode::Read => H5F_ACC_RDONLY, + OpenMode::ReadSWMR => H5F_ACC_RDONLY | H5F_ACC_SWMR_READ, OpenMode::ReadWrite => H5F_ACC_RDWR, OpenMode::Create => H5F_ACC_TRUNC, OpenMode::CreateExcl | OpenMode::Append => H5F_ACC_EXCL, @@ -239,7 +249,7 @@ impl FileBuilder { h5lock!({ let fapl = self.fapl.finish()?; match mode { - OpenMode::Read | OpenMode::ReadWrite => { + OpenMode::Read | OpenMode::ReadWrite | OpenMode::ReadSWMR => { File::from_id(h5try!(H5Fopen(fname_ptr, flags, fapl.id()))) } _ => { From 00c1a94150587ad23ede226ca523e6488e71d13f Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Tue, 29 Apr 2025 11:47:49 +0200 Subject: [PATCH 2/2] Add backwards compatibility --- CHANGELOG.md | 4 +++- hdf5/examples/swmr.rs | 38 ++++++++++++++++++++++++-------------- hdf5/src/hl/dataset.rs | 8 ++++++-- hdf5/src/hl/file.rs | 18 ++++++++++++++---- 4 files changed, 47 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9cd5b8f8..a64fa5fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,13 @@ # Changelog -## hdf5 unreleased ## hdf5-types unreleased ## hdf5-derive unreleased ## hdf5-sys unreleased ## hdf5-src unreleased +## hdf5 unreleased +- Added support for Single Writer Multiple Readers (SWMR) (breaking change, OpenMode has extra variant) + ## hdf5-types v0.10.1 Release date: Mar 19, 2025 - Fixed deref of null ptr in `VarLenAscii`/`VarLenUnicode` diff --git a/hdf5/examples/swmr.rs b/hdf5/examples/swmr.rs index d97b595a..b2912f6e 100644 --- a/hdf5/examples/swmr.rs +++ b/hdf5/examples/swmr.rs @@ -1,5 +1,7 @@ +#[cfg(feature = "1.10.2")] use hdf5_metno as hdf5; +#[cfg(feature = "1.10.2")] fn reader() { let file = hdf5::File::open_as("swmr.h5", hdf5::OpenMode::ReadSWMR).unwrap(); println!("Reader: Opened file"); @@ -17,23 +19,31 @@ fn reader() { } fn main() { - let file = - hdf5::File::with_options().with_fapl(|fapl| fapl.libver_v110()).create("swmr.h5").unwrap(); + #[cfg(not(feature = "1.10.2"))] + println!("This examples requires hdf5 >= 1.10.2 to enable SWMR and set libver_bounds"); - let var = file.new_dataset::().shape((0.., 5)).create("foo").unwrap(); + #[cfg(feature = "1.10.2")] + { + let file = hdf5::File::with_options() + .with_fapl(|fapl| fapl.libver_v110()) + .create("swmr.h5") + .unwrap(); - file.start_swmr().unwrap(); - println!("Writer: Wrote file"); + let var = file.new_dataset::().shape((0.., 5)).create("foo").unwrap(); - let thread = std::thread::spawn(|| reader()); + file.start_swmr().unwrap(); + println!("Writer: Wrote file"); - for i in 0..5 { - var.resize((i + 1, 5)).unwrap(); - var.write_slice(&[i, i, i, i, i], (i, 0..)).unwrap(); - var.flush().unwrap(); - println!("Writer: Wrote {i}"); - std::thread::sleep(std::time::Duration::from_secs(5)); - } + let thread = std::thread::spawn(|| reader()); - thread.join().unwrap(); + for i in 0..5 { + var.resize((i + 1, 5)).unwrap(); + var.write_slice(&[i, i, i, i, i], (i, 0..)).unwrap(); + var.flush().unwrap(); + println!("Writer: Wrote {i}"); + std::thread::sleep(std::time::Duration::from_secs(5)); + } + + thread.join().unwrap(); + } } diff --git a/hdf5/src/hl/dataset.rs b/hdf5/src/hl/dataset.rs index f8f28127..24f3b160 100644 --- a/hdf5/src/hl/dataset.rs +++ b/hdf5/src/hl/dataset.rs @@ -5,9 +5,11 @@ use ndarray::{self, ArrayView}; use hdf5_sys::h5::HADDR_UNDEF; use hdf5_sys::h5d::{ - H5Dcreate2, H5Dcreate_anon, H5Dflush, H5Dget_access_plist, H5Dget_create_plist, H5Dget_offset, - H5Drefresh, H5Dset_extent, + H5Dcreate2, H5Dcreate_anon, H5Dget_access_plist, H5Dget_create_plist, H5Dget_offset, + H5Dset_extent, }; +#[cfg(feature = "1.10.0")] +use hdf5_sys::h5d::{H5Dflush, H5Drefresh}; use hdf5_sys::h5l::H5Ldelete; use hdf5_sys::h5p::H5P_DEFAULT; use hdf5_sys::h5z::H5Z_filter_t; @@ -156,6 +158,7 @@ impl Dataset { } /// Flush the dataset metadata from the metadata cache to the file + #[cfg(feature = "1.10.0")] pub fn flush(&self) -> Result<()> { let id = self.id(); h5call!(H5Dflush(id))?; @@ -163,6 +166,7 @@ impl Dataset { } /// Refresh metadata items assosicated with the dataset + #[cfg(feature = "1.10.0")] pub fn refresh(&self) -> Result<()> { let id = self.id(); h5call!(H5Drefresh(id))?; diff --git a/hdf5/src/hl/file.rs b/hdf5/src/hl/file.rs index dea3f1d3..3dd59f53 100644 --- a/hdf5/src/hl/file.rs +++ b/hdf5/src/hl/file.rs @@ -5,10 +5,11 @@ use std::path::Path; use hdf5_sys::h5f::{ H5Fclose, H5Fcreate, H5Fflush, H5Fget_access_plist, H5Fget_create_plist, H5Fget_filesize, - H5Fget_freespace, H5Fget_intent, H5Fget_obj_count, H5Fget_obj_ids, H5Fopen, - H5Fstart_swmr_write, H5F_ACC_DEFAULT, H5F_ACC_EXCL, H5F_ACC_RDONLY, H5F_ACC_RDWR, - H5F_ACC_SWMR_READ, H5F_ACC_TRUNC, H5F_SCOPE_LOCAL, + H5Fget_freespace, H5Fget_intent, H5Fget_obj_count, H5Fget_obj_ids, H5Fopen, H5F_ACC_DEFAULT, + H5F_ACC_EXCL, H5F_ACC_RDONLY, H5F_ACC_RDWR, H5F_ACC_TRUNC, H5F_SCOPE_LOCAL, }; +#[cfg(feature = "1.10.0")] +use hdf5_sys::h5f::{H5Fstart_swmr_write, H5F_ACC_SWMR_READ}; use crate::hl::plist::{ file_access::{FileAccess, FileAccessBuilder}, @@ -18,10 +19,12 @@ use crate::internal_prelude::*; /// File opening mode. #[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[cfg_attr(not(feature = "1.10.0"), non_exhaustive)] pub enum OpenMode { /// Open a file as read-only, file must exist. Read, /// Open a file as read-only in SWMR mode, file must exist. + #[cfg(feature = "1.10.0")] ReadSWMR, /// Open a file as read/write, file must exist. ReadWrite, @@ -182,6 +185,8 @@ impl File { self.create_plist() } + #[cfg(feature = "1.10.0")] + /// Mark this file as ready for opening as SWMR pub fn start_swmr(&self) -> Result<()> { let id = self.id(); h5call!(H5Fstart_swmr_write(id))?; @@ -240,18 +245,23 @@ impl FileBuilder { )?; let flags = match mode { OpenMode::Read => H5F_ACC_RDONLY, + #[cfg(feature = "1.10.0")] OpenMode::ReadSWMR => H5F_ACC_RDONLY | H5F_ACC_SWMR_READ, OpenMode::ReadWrite => H5F_ACC_RDWR, OpenMode::Create => H5F_ACC_TRUNC, OpenMode::CreateExcl | OpenMode::Append => H5F_ACC_EXCL, + #[cfg(not(feature = "1.10.0"))] + _ => unreachable!(), }; let fname_ptr = filename.as_ptr(); h5lock!({ let fapl = self.fapl.finish()?; match mode { - OpenMode::Read | OpenMode::ReadWrite | OpenMode::ReadSWMR => { + OpenMode::Read | OpenMode::ReadWrite => { File::from_id(h5try!(H5Fopen(fname_ptr, flags, fapl.id()))) } + #[cfg(feature = "1.10.0")] + OpenMode::ReadSWMR => File::from_id(h5try!(H5Fopen(fname_ptr, flags, fapl.id()))), _ => { let fcpl = self.fcpl.finish()?; File::from_id(h5try!(H5Fcreate(fname_ptr, flags, fcpl.id(), fapl.id())))