diff --git a/acquire-common b/acquire-common index 753b26b..4444f5c 160000 --- a/acquire-common +++ b/acquire-common @@ -1 +1 @@ -Subproject commit 753b26b1fa5b137bad5e3f4871005d62d51a1a44 +Subproject commit 4444f5c95f699e22b47e9a6e321359016faa03e6 diff --git a/drivers.json b/drivers.json index 81db5a8..713cc83 100644 --- a/drivers.json +++ b/drivers.json @@ -1,7 +1,7 @@ { - "acquire-driver-zarr": "0.1.9", + "acquire-driver-zarr": "0.1.10", "acquire-driver-egrabber": "0.1.5", "acquire-driver-hdcam": "0.1.9", "acquire-driver-spinnaker": "0.1.1", - "acquire-driver-pvcam": "0.1.0" + "acquire-driver-pvcam": "0.1.1" } diff --git a/python/acquire/acquire.pyi b/python/acquire/acquire.pyi index 74b1b79..b743a43 100644 --- a/python/acquire/acquire.pyi +++ b/python/acquire/acquire.pyi @@ -29,6 +29,7 @@ class AvailableData: class Camera: identifier: Optional[DeviceIdentifier] settings: CameraProperties + def __init__(self, *args: None, **kwargs: Any) -> None: ... def dict(self) -> Dict[str, Any]: ... @@ -43,6 +44,7 @@ class CameraCapabilities: supported_pixel_types: List[SampleType] digital_lines: DigitalLineCapabilities triggers: TriggerCapabilities + def dict(self) -> Dict[str, Any]: ... @final @@ -56,35 +58,52 @@ class CameraProperties: shape: Tuple[int, int] input_triggers: InputTriggers output_triggers: OutputTriggers + def __init__(self, *args: None, **kwargs: Any) -> None: ... def dict(self) -> Dict[str, Any]: ... @final class Capabilities: video: Tuple[VideoStreamCapabilities, VideoStreamCapabilities] + def __init__(self, *args: None, **kwargs: Any) -> None: ... def dict(self) -> Dict[str, Any]: ... @final -class ChunkDims: - width: int - height: int - planes: int - def dict(self) -> Dict[str, Any]: ... +class DimensionType: + """The storage dimension type. -@final -class ChunkingCapabilities: - is_supported: bool - width: Property - height: Property - planes: Property - def dict(self) -> Dict[str, Any]: ... + Space: spatial dimension. + Channel: color channel dimension. + Time: time dimension. + Other: other dimension. + + When downsampling, Space and Time dimensions are downsampled by the same factor. + Channel and Other dimensions are not downsampled. + + This value is also reflected in the dimension metadata of an OME-Zarr dataset. + """ + + Space: ClassVar[DimensionType] + Channel: ClassVar[DimensionType] + Time: ClassVar[DimensionType] + Other: ClassVar[DimensionType] + + def __init__(self, *args: None, **kwargs: Any) -> None: ... + def __eq__(self, other: object) -> bool: ... + def __ge__(self, other: object) -> bool: ... + def __gt__(self, other: object) -> bool: ... + def __int__(self) -> int: ... + def __le__(self, other: object) -> bool: ... + def __lt__(self, other: object) -> bool: ... + def __ne__(self, other: object) -> bool: ... @final class DeviceIdentifier: id: Tuple[int, int] kind: DeviceKind name: str + def __init__(self, *args: None, **kwargs: Any) -> None: ... def dict(self) -> Dict[str, Any]: ... @staticmethod @@ -103,6 +122,7 @@ class DeviceKind: Signals: ClassVar[DeviceKind] StageAxis: ClassVar[DeviceKind] Storage: ClassVar[DeviceKind] + def __init__(self, *args: None, **kwargs: Any) -> None: ... def __eq__(self, other: object) -> bool: ... def __ge__(self, other: object) -> bool: ... @@ -131,6 +151,7 @@ class DeviceState: AwaitingConfiguration: ClassVar[DeviceState] Armed: ClassVar[DeviceState] Running: ClassVar[DeviceState] + def __eq__(self, other: object) -> bool: ... def __ge__(self, other: object) -> bool: ... def __gt__(self, other: object) -> bool: ... @@ -143,12 +164,14 @@ class DeviceState: class DigitalLineCapabilities: line_count: int names: Tuple[str, str, str, str, str, str, str, str] + def dict(self) -> Dict[str, Any]: ... @final class Direction: Backward: ClassVar[Direction] Forward: ClassVar[Direction] + def __eq__(self, other: object) -> bool: ... def __ge__(self, other: object) -> bool: ... def __gt__(self, other: object) -> bool: ... @@ -162,17 +185,14 @@ class InputTriggers: acquisition_start: Trigger exposure: Trigger frame_start: Trigger - def dict(self) -> Dict[str, Any]: ... -@final -class MultiscaleCapabilities: - is_supported: bool def dict(self) -> Dict[str, Any]: ... @final class OffsetShapeCapabilities: x: Property y: Property + def dict(self) -> Dict[str, Any]: ... @final @@ -180,6 +200,7 @@ class OutputTriggers: exposure: Trigger frame_start: Trigger trigger_wait: Trigger + def dict(self) -> Dict[str, Any]: ... @final @@ -187,6 +208,7 @@ class PID: derivative: float integral: float proportional: float + def __init__(self, *args: None, **kwargs: Any) -> None: ... def dict(self) -> Dict[str, Any]: ... @@ -196,6 +218,7 @@ class Property: low: float high: float kind: PropertyType + def __init__(self, *args: None, **kwargs: Any) -> None: ... def dict(self) -> Dict[str, Any]: ... @@ -205,6 +228,7 @@ class PropertyType: FloatingPrecision: ClassVar[PropertyType] Enum: ClassVar[PropertyType] String: ClassVar[PropertyType] + def __eq__(self, other: object) -> bool: ... def __ge__(self, other: object) -> bool: ... def __gt__(self, other: object) -> bool: ... @@ -216,6 +240,7 @@ class PropertyType: @final class Properties: video: Tuple[VideoStream, VideoStream] + def __init__(self, *args: None, **kwargs: Any) -> None: ... def dict(self) -> Dict[str, Any]: ... @@ -237,6 +262,7 @@ class Runtime: class SampleRateHz: numerator: int denominator: int + def __init__(self, *args: None, **kwargs: Any) -> None: ... def dict(self) -> Dict[str, Any]: ... @@ -250,6 +276,7 @@ class SampleType: U10: ClassVar[SampleType] U12: ClassVar[SampleType] U14: ClassVar[SampleType] + def __eq__(self, other: object) -> bool: ... def __ge__(self, other: object) -> bool: ... def __gt__(self, other: object) -> bool: ... @@ -258,25 +285,11 @@ class SampleType: def __lt__(self, other: object) -> bool: ... def __ne__(self, other: object) -> bool: ... -@final -class ShardDims: - width: int - height: int - planes: int - def dict(self) -> Dict[str, Any]: ... - -@final -class ShardingCapabilities: - is_supported: bool - width: Property - height: Property - planes: Property - def dict(self) -> Dict[str, Any]: ... - @final class SignalIOKind: Input: ClassVar[SignalIOKind] Output: ClassVar[SignalIOKind] + def __eq__(self, other: object) -> bool: ... def __ge__(self, other: object) -> bool: ... def __gt__(self, other: object) -> bool: ... @@ -289,6 +302,7 @@ class SignalIOKind: class SignalType: Analog: ClassVar[SignalType] Digital: ClassVar[SignalType] + def __eq__(self, other: object) -> bool: ... def __ge__(self, other: object) -> bool: ... def __gt__(self, other: object) -> bool: ... @@ -301,13 +315,25 @@ class SignalType: class Storage: identifier: Optional[DeviceIdentifier] settings: StorageProperties + def dict(self) -> Dict[str, Any]: ... @final class StorageCapabilities: - chunk_dims_px: ChunkingCapabilities - shard_dims_chunks: ShardingCapabilities - multiscale: MultiscaleCapabilities + chunking_is_supported: bool + sharding_is_supported: bool + multiscale_is_supported: bool + + def dict(self) -> Dict[str, Any]: ... + +@final +class StorageDimension: + name: str + kind: DimensionType + array_size_px: int + chunk_size_px: int + shard_size_chunks: int + def dict(self) -> Dict[str, Any]: ... @final @@ -316,9 +342,9 @@ class StorageProperties: filename: Optional[str] first_frame_id: int pixel_scale_um: Tuple[float, float] - chunk_dims_px: ChunkDims - shard_dims_chunks: ShardDims + acquisition_dimensions: List[StorageDimension] enable_multiscale: bool + def dict(self) -> Dict[str, Any]: ... @final @@ -327,6 +353,7 @@ class Trigger: enable: bool line: int kind: SignalIOKind + def __init__(self, *args: None, **kwargs: Any) -> None: ... def dict(self) -> Dict[str, Any]: ... @@ -335,6 +362,7 @@ class TriggerCapabilities: acquisition_start: TriggerInputOutputCapabilities exposure: TriggerInputOutputCapabilities frame_start: TriggerInputOutputCapabilities + def dict(self) -> Dict[str, Any]: ... @final @@ -345,6 +373,7 @@ class TriggerEdge: AnyEdge: ClassVar[TriggerEdge] LevelLow: ClassVar[TriggerEdge] LevelHigh: ClassVar[TriggerEdge] + def __eq__(self, other: object) -> bool: ... def __ge__(self, other: object) -> bool: ... def __gt__(self, other: object) -> bool: ... @@ -357,6 +386,7 @@ class TriggerEdge: class TriggerInputOutputCapabilities: input: int output: int + def dict(self) -> Dict[str, Any]: ... @final @@ -368,12 +398,14 @@ class VideoFrame: class VideoFrameMetadata: frame_id: int timestamps: VideoFrameTimestamps + def dict(self) -> Dict[str, Any]: ... @final class VideoFrameTimestamps: hardware: int acq_thread: int + def dict(self) -> Dict[str, Any]: ... @final @@ -382,6 +414,7 @@ class VideoStream: storage: Storage max_frame_count: int frame_average_count: int + def dict(self) -> Dict[str, Any]: ... @final @@ -390,12 +423,14 @@ class VideoStreamCapabilities: storage: StorageCapabilities max_frame_count: Property frame_average_count: Property + def dict(self) -> Dict[str, Any]: ... @final class VoltageRange: mn: float mx: float + @overload def __init__(self) -> None: ... @overload diff --git a/src/lib.rs b/src/lib.rs index ec6f344..7f9fee4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -80,9 +80,6 @@ fn acquire(py: Python, m: &PyModule) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; m.add_class::()?; m.add_class::()?; @@ -91,8 +88,8 @@ fn acquire(py: Python, m: &PyModule) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; - m.add_class::()?; - m.add_class::()?; + m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/src/storage.rs b/src/storage.rs index 3de14f9..9af321c 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -1,7 +1,8 @@ use crate::{ capi, - components::{macros::impl_plain_old_dict, Property}, + components::macros::{cvt, impl_plain_old_dict}, }; +use anyhow::{anyhow, Result}; use pyo3::prelude::*; use serde::{Deserialize, Serialize}; use std::{ @@ -11,40 +12,88 @@ use std::{ }; #[pyclass] -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct ChunkDims { - #[pyo3(get, set)] - #[serde(default)] - width: u32, +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub enum DimensionType { + Space, + Channel, + Time, + Other, +} + +impl Default for DimensionType { + fn default() -> Self { + DimensionType::Space + } +} + +cvt!(DimensionType => capi::DimensionType, + Space => DimensionType_DimensionType_Space, + Channel => DimensionType_DimensionType_Channel, + Time => DimensionType_DimensionType_Time, + Other => DimensionType_DimensionType_Other +); +#[pyclass] +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StorageDimension { #[pyo3(get, set)] #[serde(default)] - height: u32, + pub(crate) name: Option, #[pyo3(get, set)] #[serde(default)] - planes: u32, -} - -impl_plain_old_dict!(ChunkDims); + pub(crate) kind: DimensionType, -#[pyclass] -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct ShardDims { #[pyo3(get, set)] #[serde(default)] - width: u32, + pub(crate) array_size_px: u32, #[pyo3(get, set)] #[serde(default)] - height: u32, + pub(crate) chunk_size_px: u32, #[pyo3(get, set)] #[serde(default)] - planes: u32, + pub(crate) shard_size_chunks: u32, +} + +impl Default for StorageDimension { + fn default() -> Self { + Self { + name: Default::default(), + kind: Default::default(), + array_size_px: Default::default(), + chunk_size_px: Default::default(), + shard_size_chunks: Default::default(), + } + } } -impl_plain_old_dict!(ShardDims); +impl_plain_old_dict!(StorageDimension); + +impl TryFrom for StorageDimension { + type Error = anyhow::Error; + + fn try_from(value: capi::StorageDimension) -> Result { + let name = if value.name.nbytes == 0 { + None + } else { + Some( + unsafe { CStr::from_ptr(value.name.str_) } + .to_str()? + .to_owned(), + ) + }; + + Ok(Self { + name, + kind: value.kind.try_into()?, + array_size_px: value.array_size_px, + chunk_size_px: value.chunk_size_px, + shard_size_chunks: value.shard_size_chunks, + }) + } +} #[pyclass] #[derive(Debug, Clone, Serialize, Deserialize)] @@ -66,10 +115,7 @@ pub struct StorageProperties { pub(crate) pixel_scale_um: (f64, f64), #[pyo3(get, set)] - pub(crate) chunk_dims_px: Py, - - #[pyo3(get, set)] - pub(crate) shard_dims_chunks: Py, + pub(crate) acquisition_dimensions: Vec>, #[pyo3(get, set)] pub(crate) enable_multiscale: bool, @@ -79,15 +125,12 @@ impl_plain_old_dict!(StorageProperties); impl Default for StorageProperties { fn default() -> Self { - let chunk_dims_px = Python::with_gil(|py| Py::new(py, ChunkDims::default()).unwrap()); - let shard_dims_chunks = Python::with_gil(|py| Py::new(py, ShardDims::default()).unwrap()); Self { filename: Default::default(), external_metadata_json: Default::default(), first_frame_id: Default::default(), pixel_scale_um: Default::default(), - chunk_dims_px, - shard_dims_chunks, + acquisition_dimensions: Default::default(), enable_multiscale: Default::default(), } } @@ -116,140 +159,31 @@ impl TryFrom for StorageProperties { ) }; - let chunk_dims_px = Python::with_gil(|py| { - Py::new( - py, - ChunkDims { - width: value.chunk_dims_px.width, - height: value.chunk_dims_px.height, - planes: value.chunk_dims_px.planes, - }, - ) - .unwrap() - }); - - let shard_dims_chunks = Python::with_gil(|py| { - Py::new( - py, - ShardDims { - width: value.shard_dims_chunks.width, - height: value.shard_dims_chunks.height, - planes: value.shard_dims_chunks.planes, - }, - ) - .unwrap() - }); + let mut acquisition_dimensions: Vec> = Default::default(); + for i in 0..value.acquisition_dimensions.size { + acquisition_dimensions.push(Python::with_gil(|py| { + Py::new( + py, + StorageDimension::try_from(unsafe { + *value.acquisition_dimensions.data.add(i) + }) + .unwrap(), + ) + .unwrap() + })); + } Ok(Self { filename, first_frame_id: value.first_frame_id, external_metadata_json, pixel_scale_um: (value.pixel_scale_um.x, value.pixel_scale_um.y), - chunk_dims_px, - shard_dims_chunks, + acquisition_dimensions, enable_multiscale: (value.enable_multiscale == 1), }) } } -impl TryFrom<&StorageProperties> for capi::StorageProperties { - type Error = anyhow::Error; - - fn try_from(value: &StorageProperties) -> Result { - let mut out: capi::StorageProperties = unsafe { std::mem::zeroed() }; - // Careful: x needs to live long enough - let x = if let Some(filename) = &value.filename { - Some(CString::new(filename.as_str())?) - } else { - None - }; - let (filename, bytes_of_filename) = if let Some(ref x) = x { - (x.as_ptr(), x.to_bytes_with_nul().len()) - } else { - (null(), 0) - }; - - // Careful: y needs to live long enough - let y = if let Some(metadata) = &value.external_metadata_json { - Some(CString::new(metadata.as_str())?) - } else { - None - }; - let (metadata, bytes_of_metadata) = if let Some(ref y) = y { - (y.as_ptr(), y.to_bytes_with_nul().len()) - } else { - (null(), 0) - }; - - let chunk_dims_px = Python::with_gil(|py| -> PyResult<_> { - let chunk_dims_px: ChunkDims = value.chunk_dims_px.extract(py)?; - Ok(chunk_dims_px) - })?; - - let shard_dims_chunks = Python::with_gil(|py| -> PyResult<_> { - let shard_dims_chunks: ShardDims = value.shard_dims_chunks.extract(py)?; - Ok(shard_dims_chunks) - })?; - - // This copies the string into a buffer owned by the return value. - if !unsafe { - capi::storage_properties_init( - &mut out, - value.first_frame_id, - filename, - bytes_of_filename as _, - metadata, - bytes_of_metadata as _, - capi::PixelScale { - x: value.pixel_scale_um.0, - y: value.pixel_scale_um.1, - }, - ) == 1 - } { - Err(anyhow::anyhow!("Failed acquire api status check")) - } else if !unsafe { - capi::storage_properties_set_chunking_props( - &mut out, - chunk_dims_px.width, - chunk_dims_px.height, - chunk_dims_px.planes, - ) == 1 - } { - Err(anyhow::anyhow!("Failed acquire api status check")) - } else if !unsafe { - capi::storage_properties_set_sharding_props( - &mut out, - shard_dims_chunks.width, - shard_dims_chunks.height, - shard_dims_chunks.planes, - ) == 1 - } { - Err(anyhow::anyhow!("Failed acquire api status check")) - } else if !unsafe { - capi::storage_properties_set_enable_multiscale(&mut out, value.enable_multiscale as u8) - == 1 - } { - Err(anyhow::anyhow!("Failed acquire api status check")) - } else { - Ok(out) - } - } -} - -impl Default for capi::StorageProperties { - fn default() -> Self { - Self { - filename: Default::default(), - first_frame_id: Default::default(), - external_metadata_json: Default::default(), - pixel_scale_um: Default::default(), - chunk_dims_px: Default::default(), - shard_dims_chunks: Default::default(), - enable_multiscale: Default::default(), - } - } -} - impl Default for capi::String { fn default() -> Self { Self { @@ -269,78 +203,6 @@ impl Default for capi::PixelScale { } } -impl Default for capi::StorageProperties_storage_properties_chunking_s { - fn default() -> Self { - Self { - width: Default::default(), - height: Default::default(), - planes: Default::default(), - } - } -} - -impl Default for capi::StorageProperties_storage_properties_sharding_s { - fn default() -> Self { - Self { - width: Default::default(), - height: Default::default(), - planes: Default::default(), - } - } -} - -impl TryFrom for ChunkDims { - type Error = anyhow::Error; - - fn try_from( - value: capi::StorageProperties_storage_properties_chunking_s, - ) -> Result { - Ok(ChunkDims { - width: value.width, - height: value.height, - planes: value.planes, - }) - } -} - -impl TryFrom<&ChunkDims> for capi::StorageProperties_storage_properties_chunking_s { - type Error = anyhow::Error; - - fn try_from(value: &ChunkDims) -> Result { - Ok(capi::StorageProperties_storage_properties_chunking_s { - width: value.width, - height: value.height, - planes: value.planes, - }) - } -} - -impl TryFrom for ShardDims { - type Error = anyhow::Error; - - fn try_from( - value: capi::StorageProperties_storage_properties_sharding_s, - ) -> Result { - Ok(ShardDims { - width: value.width, - height: value.height, - planes: value.planes, - }) - } -} - -impl TryFrom<&ShardDims> for capi::StorageProperties_storage_properties_sharding_s { - type Error = anyhow::Error; - - fn try_from(value: &ShardDims) -> Result { - Ok(capi::StorageProperties_storage_properties_sharding_s { - width: value.width, - height: value.height, - planes: value.planes, - }) - } -} - impl Default for capi::ImageShape_image_dims_s { fn default() -> Self { Self { @@ -380,118 +242,28 @@ impl Display for capi::String { } } -/// StorageCapabilities::ChunkingCapabilities -#[pyclass] -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ChunkingCapabilities { - #[pyo3(get)] - is_supported: bool, - - #[pyo3(get)] - width: Py, - - #[pyo3(get)] - height: Py, - - #[pyo3(get)] - planes: Py, -} - -impl_plain_old_dict!(ChunkingCapabilities); - -impl Default for ChunkingCapabilities { - fn default() -> Self { - let width = Python::with_gil(|py| Py::new(py, Property::default()).unwrap()); - let height = Python::with_gil(|py| Py::new(py, Property::default()).unwrap()); - let planes = Python::with_gil(|py| Py::new(py, Property::default()).unwrap()); - Self { - is_supported: Default::default(), - width, - height, - planes, - } - } -} - -/// StorageCapabilities::ShardingCapabilities -#[pyclass] -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ShardingCapabilities { - #[pyo3(get)] - is_supported: bool, - - #[pyo3(get)] - width: Py, - - #[pyo3(get)] - height: Py, - - #[pyo3(get)] - planes: Py, -} - -impl_plain_old_dict!(ShardingCapabilities); - -impl Default for ShardingCapabilities { - fn default() -> Self { - let width = Python::with_gil(|py| Py::new(py, Property::default()).unwrap()); - let height = Python::with_gil(|py| Py::new(py, Property::default()).unwrap()); - let planes = Python::with_gil(|py| Py::new(py, Property::default()).unwrap()); - Self { - is_supported: Default::default(), - width, - height, - planes, - } - } -} - -/// StorageCapabilities::MultiscaleCapabilities -#[pyclass] -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct MultiscaleCapabilities { - #[pyo3(get)] - is_supported: bool, -} - -impl_plain_old_dict!(MultiscaleCapabilities); - -impl Default for MultiscaleCapabilities { - fn default() -> Self { - Self { - is_supported: Default::default(), - } - } -} - /// StorageCapabilities #[pyclass] #[derive(Debug, Clone, Serialize, Deserialize)] pub struct StorageCapabilities { #[pyo3(get)] - chunk_dims_px: Py, + chunking_is_supported: bool, #[pyo3(get)] - shard_dims_chunks: Py, + sharding_is_supported: bool, #[pyo3(get)] - multiscale: Py, + multiscale_is_supported: bool, } impl_plain_old_dict!(StorageCapabilities); impl Default for StorageCapabilities { fn default() -> Self { - let chunk_dims_px = - Python::with_gil(|py| Py::new(py, ChunkingCapabilities::default()).unwrap()); - let shard_dims_chunks = - Python::with_gil(|py| Py::new(py, ShardingCapabilities::default()).unwrap()); - let multiscale = - Python::with_gil(|py| Py::new(py, MultiscaleCapabilities::default()).unwrap()); Self { - chunk_dims_px, - shard_dims_chunks, - multiscale, + chunking_is_supported: Default::default(), + sharding_is_supported: Default::default(), + multiscale_is_supported: Default::default(), } } } @@ -500,140 +272,150 @@ impl TryFrom for StorageCapabilities { type Error = anyhow::Error; fn try_from(value: capi::StoragePropertyMetadata) -> Result { - let chunk_dims_px = Python::with_gil(|py| -> PyResult<_> { - let width: Property = value.chunk_dims_px.width.try_into()?; - let height: Property = value.chunk_dims_px.height.try_into()?; - let planes: Property = value.chunk_dims_px.planes.try_into()?; - let chunking = ChunkingCapabilities { - is_supported: (value.chunk_dims_px.is_supported == 1), - width: Py::new(py, width)?, - height: Py::new(py, height)?, - planes: Py::new(py, planes)?, - }; - Py::new(py, chunking) - })?; - - let shard_dims_chunks = Python::with_gil(|py| -> PyResult<_> { - let width: Property = value.shard_dims_chunks.width.try_into()?; - let height: Property = value.shard_dims_chunks.height.try_into()?; - let planes: Property = value.shard_dims_chunks.planes.try_into()?; - let sharding = ShardingCapabilities { - is_supported: (value.shard_dims_chunks.is_supported == 1), - width: Py::new(py, width)?, - height: Py::new(py, height)?, - planes: Py::new(py, planes)?, - }; - Py::new(py, sharding) - })?; - - let multiscale = Python::with_gil(|py| -> PyResult<_> { - let multiscale = MultiscaleCapabilities { - is_supported: (value.multiscale.is_supported == 1), - }; - Py::new(py, multiscale) - })?; - Ok(Self { - chunk_dims_px, - shard_dims_chunks, - multiscale, + chunking_is_supported: value.chunking_is_supported == 1, + sharding_is_supported: value.sharding_is_supported == 1, + multiscale_is_supported: value.multiscale_is_supported == 1, }) } } -/// capi -impl Default for capi::StoragePropertyMetadata_storage_property_metadata_chunking_s { +/// capi::StorageProperties +impl Default for capi::StorageProperties { fn default() -> Self { Self { - is_supported: Default::default(), - width: Default::default(), - height: Default::default(), - planes: Default::default(), + filename: Default::default(), + first_frame_id: Default::default(), + external_metadata_json: Default::default(), + pixel_scale_um: Default::default(), + acquisition_dimensions: Default::default(), + enable_multiscale: Default::default(), } } } -impl TryFrom<&ChunkingCapabilities> - for capi::StoragePropertyMetadata_storage_property_metadata_chunking_s -{ +impl TryFrom<&StorageProperties> for capi::StorageProperties { type Error = anyhow::Error; - fn try_from(value: &ChunkingCapabilities) -> Result { - let (width, height, planes) = Python::with_gil(|py| -> PyResult<_> { - let width: Property = value.width.extract(py)?; - let height: Property = value.height.extract(py)?; - let planes: Property = value.planes.extract(py)?; - Ok((width, height, planes)) - })?; + fn try_from(value: &StorageProperties) -> Result { + let mut out: capi::StorageProperties = unsafe { std::mem::zeroed() }; + // Careful: x needs to live long enough + let x = if let Some(filename) = &value.filename { + Some(CString::new(filename.as_str())?) + } else { + None + }; + let (filename, bytes_of_filename) = if let Some(ref x) = x { + (x.as_ptr(), x.to_bytes_with_nul().len()) + } else { + (null(), 0) + }; - Ok(Self { - is_supported: value.is_supported as u8, - width: (&width).try_into()?, - height: (&height).try_into()?, - planes: (&planes).try_into()?, - }) + // Careful: y needs to live long enough + let y = if let Some(metadata) = &value.external_metadata_json { + Some(CString::new(metadata.as_str())?) + } else { + None + }; + let (metadata, bytes_of_metadata) = if let Some(ref y) = y { + (y.as_ptr(), y.to_bytes_with_nul().len()) + } else { + (null(), 0) + }; + + // This copies the string into a buffer owned by the return value. + if !unsafe { + capi::storage_properties_init( + &mut out, + value.first_frame_id, + filename, + bytes_of_filename as _, + metadata, + bytes_of_metadata as _, + capi::PixelScale { + x: value.pixel_scale_um.0, + y: value.pixel_scale_um.1, + }, + value.acquisition_dimensions.len() as u8, + ) == 1 + } { + Err(anyhow::anyhow!("Failed to initialize storage properties.")) + } else if !unsafe { + capi::storage_properties_set_enable_multiscale(&mut out, value.enable_multiscale as u8) + == 1 + } { + Err(anyhow::anyhow!("Failed acquire api status check")) + } else { + // initialize each dimension separately + for (i, pydim) in value.acquisition_dimensions.iter().enumerate() { + let dim = Python::with_gil(|py| -> PyResult<_> { + let storage_dim: StorageDimension = pydim.extract(py)?; + Ok(storage_dim) + })?; + + // Careful: x needs to live long enough + let x = if let Some(name) = &dim.name { + Some(CString::new(name.as_str())?) + } else { + None + }; + let (name, bytes_of_name) = if let Some(ref x) = x { + (x.as_ptr(), x.to_bytes_with_nul().len()) + } else { + (null(), 0) + }; + + if !unsafe { + capi::storage_properties_set_dimension( + &mut out, + i.try_into().unwrap(), + name, + bytes_of_name, + dim.kind.into(), + dim.array_size_px, + dim.chunk_size_px, + dim.shard_size_chunks, + ) == 1 + } { + return Err(anyhow::anyhow!("Failed to set storage dimension.")); + } + } + + Ok(out) + } } } -impl Default for capi::StoragePropertyMetadata_storage_property_metadata_sharding_s { +/// capi::StorageProperties_storage_properties_dimensions_s +impl Default for capi::StorageProperties_storage_properties_dimensions_s { fn default() -> Self { Self { - is_supported: Default::default(), - width: Default::default(), - height: Default::default(), - planes: Default::default(), + data: null_mut(), + size: Default::default(), } } } -impl TryFrom<&ShardingCapabilities> - for capi::StoragePropertyMetadata_storage_property_metadata_sharding_s -{ - type Error = anyhow::Error; - - fn try_from(value: &ShardingCapabilities) -> Result { - let (width, height, planes) = Python::with_gil(|py| -> PyResult<_> { - let width: Property = value.width.extract(py)?; - let height: Property = value.height.extract(py)?; - let planes: Property = value.planes.extract(py)?; - Ok((width, height, planes)) - })?; - - Ok(Self { - is_supported: value.is_supported as u8, - width: (&width).try_into()?, - height: (&height).try_into()?, - planes: (&planes).try_into()?, - }) - } -} - -impl Default for capi::StoragePropertyMetadata_storage_property_metadata_multiscale_s { +/// capi::StorageDimension +impl Default for capi::StorageDimension { fn default() -> Self { Self { - is_supported: Default::default(), + name: Default::default(), + kind: Default::default(), + array_size_px: Default::default(), + chunk_size_px: Default::default(), + shard_size_chunks: Default::default(), } } } -impl TryFrom<&MultiscaleCapabilities> - for capi::StoragePropertyMetadata_storage_property_metadata_multiscale_s -{ - type Error = anyhow::Error; - - fn try_from(value: &MultiscaleCapabilities) -> Result { - Ok(Self { - is_supported: value.is_supported as u8, - }) - } -} - +/// capi::StoragePropertyMetadata impl Default for capi::StoragePropertyMetadata { fn default() -> Self { Self { - chunk_dims_px: Default::default(), - shard_dims_chunks: Default::default(), - multiscale: Default::default(), + chunking_is_supported: Default::default(), + sharding_is_supported: Default::default(), + multiscale_is_supported: Default::default(), } } } @@ -642,19 +424,10 @@ impl TryFrom<&StorageCapabilities> for capi::StoragePropertyMetadata { type Error = anyhow::Error; fn try_from(value: &StorageCapabilities) -> Result { - let (chunk_dims_px, shard_dims_chunks, multiscale) = - Python::with_gil(|py| -> PyResult<_> { - let chunk_dims_px: ChunkingCapabilities = value.chunk_dims_px.extract(py)?; - let shard_dims_chunks: ShardingCapabilities = - value.shard_dims_chunks.extract(py)?; - let multiscale: MultiscaleCapabilities = value.multiscale.extract(py)?; - Ok((chunk_dims_px, shard_dims_chunks, multiscale)) - })?; - Ok(Self { - chunk_dims_px: (&chunk_dims_px).try_into()?, - shard_dims_chunks: (&shard_dims_chunks).try_into()?, - multiscale: (&multiscale).try_into()?, + chunking_is_supported: value.chunking_is_supported as u8, + sharding_is_supported: value.sharding_is_supported as u8, + multiscale_is_supported: value.multiscale_is_supported as u8, }) } } diff --git a/tests/test_basic.py b/tests/test_basic.py index cbca4ea..cc28799 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -19,10 +19,10 @@ def runtime(): def test_version(): - assert isinstance(acquire.__version__, str) - # this will fail if pip install -e . has not been run - # so feel free to remove this line if it's not what you want to test - assert acquire.__version__ != "uninstalled" + assert isinstance(acquire.__version__, str) + # this will fail if pip install -e . has not been run + # so feel free to remove this line if it's not what you want to test + assert acquire.__version__ != "uninstalled" def test_set(): @@ -512,31 +512,21 @@ def test_simulated_camera_capabilities( @pytest.mark.parametrize( - ("descriptor", "extension", "chunking", "sharding", "multiscale"), + ("descriptor", "chunking", "sharding", "multiscale"), [ - ("raw", "bin", None, None, False), - ("trash", "", None, None, False), - ("tiff", "tif", None, None, False), - ("tiff-json", "tif", None, None, False), - ( - "zarr", - "zarr", - { - "width": {"low": 32, "high": 65535}, - "height": {"low": 32, "high": 65535}, - "planes": {"low": 32, "high": 65535}, - }, - None, - True, - ), + ("raw", False, False, False), + ("trash", False, False, False), + ("tiff", False, False, False), + ("tiff-json", False, False, False), + ("zarr", True, False, True), + ("zarrv3", True, True, False), ], ) def test_storage_capabilities( runtime: Runtime, descriptor: str, - extension: str, - chunking: Optional[Dict[str, Any]], - sharding: Optional[Dict[str, Any]], + chunking: bool, + sharding: bool, multiscale: bool, ): dm = runtime.device_manager() @@ -547,58 +537,15 @@ def test_storage_capabilities( p.video[0].storage.settings.external_metadata_json = json.dumps( {"hello": "world"} ) # for tiff-json - p.video[0].max_frame_count = 1000 + p.video[0].max_frame_count = 1 runtime.set_configuration(p) c = runtime.get_capabilities() storage = c.video[0].storage - chunk_dims_px = storage.chunk_dims_px - - assert chunk_dims_px.width.kind == PropertyType.FixedPrecision - assert chunk_dims_px.height.kind == PropertyType.FixedPrecision - assert chunk_dims_px.planes.kind == PropertyType.FixedPrecision - - if chunking is None: - assert chunk_dims_px.is_supported is False - assert chunk_dims_px.width.low == chunk_dims_px.width.high == 0.0 - assert chunk_dims_px.height.low == chunk_dims_px.height.high == 0.0 - assert chunk_dims_px.planes.low == chunk_dims_px.planes.high == 0.0 - else: - assert chunk_dims_px.is_supported is True - assert chunk_dims_px.width.low == chunking["width"]["low"] - assert chunk_dims_px.width.high == chunking["width"]["high"] - assert chunk_dims_px.height.low == chunking["height"]["low"] - assert chunk_dims_px.height.high == chunking["height"]["high"] - assert chunk_dims_px.planes.low == chunking["planes"]["low"] - assert chunk_dims_px.planes.high == chunking["planes"]["high"] - - shard_dims_chunks = storage.shard_dims_chunks - - assert shard_dims_chunks.width.kind == PropertyType.FixedPrecision - assert shard_dims_chunks.height.kind == PropertyType.FixedPrecision - assert shard_dims_chunks.planes.kind == PropertyType.FixedPrecision - - if sharding is None: - assert shard_dims_chunks.is_supported is False - assert shard_dims_chunks.width.low == 0.0 - assert shard_dims_chunks.width.high == 0.0 - - assert shard_dims_chunks.height.low == 0.0 - assert shard_dims_chunks.height.high == 0.0 - - assert shard_dims_chunks.planes.low == 0.0 - assert shard_dims_chunks.planes.high == 0.0 - else: - assert shard_dims_chunks.is_supported is True - assert shard_dims_chunks.width.low == chunking["width"]["low"] - assert shard_dims_chunks.width.high == chunking["width"]["high"] - assert shard_dims_chunks.height.low == chunking["height"]["low"] - assert shard_dims_chunks.height.high == chunking["height"]["high"] - assert shard_dims_chunks.planes.low == chunking["planes"]["low"] - assert shard_dims_chunks.planes.high == chunking["planes"]["high"] - - assert storage.multiscale.is_supported == multiscale + assert storage.chunking_is_supported == chunking + assert storage.sharding_is_supported == sharding + assert storage.multiscale_is_supported == multiscale def test_invalidated_frame(runtime: Runtime): diff --git a/tests/test_zarr.py b/tests/test_zarr.py index b14638b..c084374 100644 --- a/tests/test_zarr.py +++ b/tests/test_zarr.py @@ -22,6 +22,84 @@ def runtime(): yield acquire.Runtime() +def test_set_acquisition_dimensions( + runtime: Runtime, request: pytest.FixtureRequest +): + dm = runtime.device_manager() + props = runtime.get_configuration() + props.video[0].camera.identifier = dm.select( + DeviceKind.Camera, ".*empty.*" + ) + props.video[0].camera.settings.shape = (64, 48) + + props.video[0].storage.identifier = dm.select(DeviceKind.Storage, "Zarr") + props.video[0].storage.settings.filename = f"{request.node.name}.zarr" + props.video[0].max_frame_count = 32 + + # configure storage dimensions + dimension_x = acquire.StorageDimension( + name="x", kind="Space", array_size_px=64, chunk_size_px=64 + ) + assert dimension_x.shard_size_chunks == 0 + + dimension_y = acquire.StorageDimension( + name="y", kind="Space", array_size_px=48, chunk_size_px=48 + ) + assert dimension_y.shard_size_chunks == 0 + + dimension_t = acquire.StorageDimension( + name="t", kind="Time", array_size_px=32, chunk_size_px=32 + ) + assert dimension_t.shard_size_chunks == 0 + + props.video[0].storage.settings.acquisition_dimensions = [ + dimension_x, + dimension_y, + dimension_t, + ] + assert len(props.video[0].storage.settings.acquisition_dimensions) == 3 + + # sleep(10) + + # set and test + props = runtime.set_configuration(props) + assert len(props.video[0].storage.settings.acquisition_dimensions) == 3 + + assert ( + props.video[0].storage.settings.acquisition_dimensions[0].name + == dimension_x.name + ) + assert ( + props.video[0].storage.settings.acquisition_dimensions[0].kind + == dimension_x.kind + ) + assert ( + props.video[0].storage.settings.acquisition_dimensions[0].array_size_px + == dimension_x.array_size_px + ) + assert ( + props.video[0].storage.settings.acquisition_dimensions[0].chunk_size_px + == dimension_x.chunk_size_px + ) + + assert ( + props.video[0].storage.settings.acquisition_dimensions[2].name + == dimension_t.name + ) + assert ( + props.video[0].storage.settings.acquisition_dimensions[2].kind + == dimension_t.kind + ) + assert ( + props.video[0].storage.settings.acquisition_dimensions[2].array_size_px + == dimension_t.array_size_px + ) + assert ( + props.video[0].storage.settings.acquisition_dimensions[2].chunk_size_px + == dimension_t.chunk_size_px + ) + + def test_write_external_metadata_to_zarr( runtime: Runtime, request: pytest.FixtureRequest ): @@ -37,9 +115,28 @@ def test_write_external_metadata_to_zarr( metadata = {"hello": "world"} p.video[0].storage.settings.external_metadata_json = json.dumps(metadata) p.video[0].storage.settings.pixel_scale_um = (0.5, 4) - p.video[0].storage.settings.chunk_dims_px.width = 33 - p.video[0].storage.settings.chunk_dims_px.height = 47 - p.video[0].storage.settings.chunk_dims_px.planes = 4 + + # configure storage dimensions + dimension_x = acquire.StorageDimension( + name="x", kind="Space", array_size_px=33, chunk_size_px=33 + ) + assert dimension_x.shard_size_chunks == 0 + + dimension_y = acquire.StorageDimension( + name="y", kind="Space", array_size_px=47, chunk_size_px=47 + ) + assert dimension_y.shard_size_chunks == 0 + + dimension_z = acquire.StorageDimension( + name="z", kind="Space", array_size_px=0, chunk_size_px=4 + ) + assert dimension_z.shard_size_chunks == 0 + + p.video[0].storage.settings.acquisition_dimensions = [ + dimension_x, + dimension_y, + dimension_z, + ] p = runtime.set_configuration(p) @@ -65,7 +162,6 @@ def test_write_external_metadata_to_zarr( image_data = multi_scale_image_node.data[0] assert image_data.shape == ( p.video[0].max_frame_count, - 1, p.video[0].camera.settings.shape[1], p.video[0].camera.settings.shape[0], ) @@ -74,13 +170,13 @@ def test_write_external_metadata_to_zarr( axes = multi_scale_image_metadata["axes"] axis_names = tuple(a["name"] for a in axes) - assert axis_names == ("t", "c", "y", "x") + assert axis_names == ("z", "y", "x") axis_types = tuple(a["type"] for a in axes) - assert axis_types == ("time", "channel", "space", "space") + assert axis_types == ("space", "space", "space") axis_units = tuple(a.get("unit") for a in axes) - assert axis_units == (None, None, "micrometer", "micrometer") + assert axis_units == (None, "micrometer", "micrometer") # We only have one multi-scale level and one transform. transform = multi_scale_image_metadata["coordinateTransformations"][0][0] @@ -123,6 +219,35 @@ def test_write_compressed_zarr( p.video[0].storage.settings.filename = filename metadata = {"foo": "bar"} p.video[0].storage.settings.external_metadata_json = json.dumps(metadata) + + # configure storage dimensions + dimension_x = acquire.StorageDimension( + name="x", kind="Space", array_size_px=64, chunk_size_px=64 + ) + assert dimension_x.shard_size_chunks == 0 + + dimension_y = acquire.StorageDimension( + name="y", kind="Space", array_size_px=48, chunk_size_px=48 + ) + assert dimension_y.shard_size_chunks == 0 + + dimension_c = acquire.StorageDimension( + name="c", kind="Channel", array_size_px=1, chunk_size_px=1 + ) + assert dimension_c.shard_size_chunks == 0 + + dimension_t = acquire.StorageDimension( + name="t", kind="Time", array_size_px=0, chunk_size_px=70 + ) + assert dimension_t.shard_size_chunks == 0 + + p.video[0].storage.settings.acquisition_dimensions = [ + dimension_x, + dimension_y, + dimension_c, + dimension_t, + ] + runtime.set_configuration(p) runtime.start() @@ -186,9 +311,27 @@ def test_write_zarr_with_chunking( p.video[0].storage.settings.filename = f"{request.node.name}.zarr" p.video[0].max_frame_count = number_of_frames - p.video[0].storage.settings.chunk_dims_px.width = 1920 // 2 - p.video[0].storage.settings.chunk_dims_px.height = 1080 // 2 - p.video[0].storage.settings.chunk_dims_px.planes = 64 + # configure storage dimensions + dimension_x = acquire.StorageDimension( + name="x", kind="Space", array_size_px=1920, chunk_size_px=960 + ) + assert dimension_x.shard_size_chunks == 0 + + dimension_y = acquire.StorageDimension( + name="y", kind="Space", array_size_px=1080, chunk_size_px=540 + ) + assert dimension_y.shard_size_chunks == 0 + + dimension_t = acquire.StorageDimension( + name="t", kind="Time", array_size_px=0, chunk_size_px=64 + ) + assert dimension_t.shard_size_chunks == 0 + + p.video[0].storage.settings.acquisition_dimensions = [ + dimension_x, + dimension_y, + dimension_t, + ] runtime.set_configuration(p) @@ -198,11 +341,10 @@ def test_write_zarr_with_chunking( group = zarr.open(p.video[0].storage.settings.filename) data = group["0"] - assert data.chunks == (64, 1, 1080 // 2, 1920 // 2) + assert data.chunks == (64, 540, 960) assert data.shape == ( number_of_frames, - 1, p.video[0].camera.settings.shape[1], p.video[0].camera.settings.shape[0], ) @@ -233,14 +375,27 @@ def test_write_zarr_multiscale( p.video[0].storage.settings.pixel_scale_um = (1, 1) p.video[0].max_frame_count = 100 - p.video[0].storage.settings.chunk_dims_px.width = ( - p.video[0].camera.settings.shape[0] // 3 + # configure storage dimensions + dimension_x = acquire.StorageDimension( + name="x", kind="Space", array_size_px=1920, chunk_size_px=640 + ) + assert dimension_x.shard_size_chunks == 0 + + dimension_y = acquire.StorageDimension( + name="y", kind="Space", array_size_px=1080, chunk_size_px=360 ) - p.video[0].storage.settings.chunk_dims_px.height = ( - p.video[0].camera.settings.shape[1] // 3 + assert dimension_y.shard_size_chunks == 0 + + dimension_t = acquire.StorageDimension( + name="t", kind="Time", array_size_px=0, chunk_size_px=64 ) - p.video[0].storage.settings.chunk_dims_px.planes = 64 + assert dimension_t.shard_size_chunks == 0 + p.video[0].storage.settings.acquisition_dimensions = [ + dimension_x, + dimension_y, + dimension_t, + ] p.video[0].storage.settings.enable_multiscale = True runtime.set_configuration(p) @@ -257,11 +412,11 @@ def test_write_zarr_multiscale( ] assert len(data) == 3 - image = data[0][0, 0, :, :].compute() # convert dask array to numpy array + image = data[0][0, :, :].compute() # convert dask array to numpy array for d in data: assert ( - np.linalg.norm(image - d[0, 0, :, :].compute()) == 0 + np.linalg.norm(image - d[0, :, :].compute()) == 0 ) # validate against the same method from scikit-image image = downscale_local_mean(image, (2, 2)).astype(np.uint8) @@ -299,9 +454,36 @@ def test_write_zarr_v3( p.video[0].storage.settings.filename = f"{request.node.name}.zarr" p.video[0].max_frame_count = number_of_frames - p.video[0].storage.settings.chunk_dims_px.width = 1920 // 2 - p.video[0].storage.settings.chunk_dims_px.height = 1080 // 2 - p.video[0].storage.settings.chunk_dims_px.planes = 64 + # configure storage dimensions + dimension_x = acquire.StorageDimension( + name="x", + kind="Space", + array_size_px=1920, + chunk_size_px=960, + shard_size_chunks=2, + ) + + dimension_y = acquire.StorageDimension( + name="y", + kind="Space", + array_size_px=1080, + chunk_size_px=540, + shard_size_chunks=2, + ) + + dimension_t = acquire.StorageDimension( + name="t", + kind="Time", + array_size_px=0, + chunk_size_px=64, + shard_size_chunks=1, + ) + + p.video[0].storage.settings.acquisition_dimensions = [ + dimension_x, + dimension_y, + dimension_t, + ] runtime.set_configuration(p) @@ -312,11 +494,10 @@ def test_write_zarr_v3( group = zarr.open(store=store, mode="r") data = group["0"] - assert data.chunks == (64, 1, 1080 // 2, 1920 // 2) + assert data.chunks == (64, 540, 960) assert data.shape == ( number_of_frames, - 1, p.video[0].camera.settings.shape[1], p.video[0].camera.settings.shape[0], ) @@ -342,6 +523,26 @@ def test_metadata_with_trailing_whitespace( p.video[0].storage.settings.external_metadata_json = ( json.dumps(metadata) + " " ) + + # configure storage dimensions + dimension_x = acquire.StorageDimension( + name="x", kind="Space", array_size_px=64, chunk_size_px=64 + ) + + dimension_y = acquire.StorageDimension( + name="y", kind="Space", array_size_px=48, chunk_size_px=48 + ) + + dimension_t = acquire.StorageDimension( + name="t", kind="Time", array_size_px=0, chunk_size_px=64 + ) + + p.video[0].storage.settings.acquisition_dimensions = [ + dimension_x, + dimension_y, + dimension_t, + ] + runtime.set_configuration(p) runtime.start()