From 6a5ebd12893dd16739515238a84f1a1314822925 Mon Sep 17 00:00:00 2001 From: Rhys Thomas Date: Thu, 20 Aug 2026 17:47:37 +0100 Subject: [PATCH 1/5] Add Oxford h5oina support --- defdap/file_readers.py | 88 ++++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 1 + 2 files changed, 89 insertions(+) diff --git a/defdap/file_readers.py b/defdap/file_readers.py index d743238..a3de6a4 100644 --- a/defdap/file_readers.py +++ b/defdap/file_readers.py @@ -19,6 +19,7 @@ from abc import ABC, abstractmethod import pathlib import re +import h5py from typing import TextIO, Dict, List, Callable, Any, Type, Optional @@ -61,6 +62,7 @@ def get_loader(data_type: str, file_name: pathlib.Path) -> 'Type[EBSDDataLoader] data_type = { '.crc': 'oxfordbinary', '.cpr': 'oxfordbinary', + '.h5oina': 'oxfordh5', '.ctf': 'oxfordtext', '.ang': 'edaxang', }.get(file_name.suffix, 'oxfordbinary') @@ -70,6 +72,7 @@ def get_loader(data_type: str, file_name: pathlib.Path) -> 'Type[EBSDDataLoader] loader = { 'oxfordbinary': OxfordBinaryLoader, 'oxfordtext': OxfordTextLoader, + 'oxfordh5': Oxfordh5Loader, 'edaxang': EdaxAngLoader, 'pythondict': PythonDictLoader, }[data_type] @@ -234,6 +237,91 @@ def parse_phase() -> Phase: self.check_data() +class Oxfordh5Loader(EBSDDataLoader): + def load(self, file_name: pathlib.Path) -> None: + """Read an Oxford Instruments ``.h5oina`` orientation file. + + Parameters + ---------- + file_name : pathlib.Path + Path to file. + + """ + # open data file and read in metadata + if not file_name.is_file(): + raise FileNotFoundError(f"Cannot open file {file_name}") + + file = h5py.File(file_name) + + header = file['1']['EBSD']['Header'] + data = file['1']['EBSD']['Data'] + + x_dim = int(header['X Cells'][0]) + y_dim = int(header['Y Cells'][0]) + shape = (y_dim, x_dim) + self.loaded_metadata['shape'] = shape + + self.loaded_metadata['step_size'] = float(header['X Step'][0]) + + ## Check this is acquisition orientataion from ctf + self.loaded_metadata['acquisition_rotation'] = Quat.from_euler_angles(*header['Specimen Orientation Euler'][0]) + + for phase_data in header['Phases'].values(): + + phase = Phase( + phase_data['Phase Name'][0].decode(), + phase_data['Laue Group'][0], + phase_data['Space Group'][0], + np.concatenate([ + phase_data['Lattice Dimensions'][0], + phase_data['Lattice Angles'][0] + ])) + + self.loaded_metadata['phases'].append(phase) + + self.check_metadata() + + # Data also available: Bands, Detector Distance, Error, Pattern Center X, Pattern Center Y + + self.loaded_data.add( + 'band_contrast', np.array(data['Band Contrast']).reshape(shape), + unit='', type='map', order=0, + plot_params={ + 'plot_colour_bar': True, + 'cmap': 'gray', + 'clabel': 'Band contrast', + } + ) + self.loaded_data.add( + 'band_slope', np.array(data['Band Slope']).reshape(shape), + unit='', type='map', order=0, + plot_params={ + 'plot_colour_bar': True, + 'cmap': 'gray', + 'clabel': 'Band slope', + } + ) + self.loaded_data.add( + 'mean_angular_deviation', np.array(data['Mean Angular Deviation']).reshape(shape), + unit='', type='map', order=0, + plot_params={ + 'plot_colour_bar': True, + 'clabel': 'Mean angular deviation', + } + ) + self.loaded_data.add( + 'pattern_quality', np.array(data['Pattern Quality']).reshape(shape), + unit='', type='map', order=0, + plot_params={ + 'plot_colour_bar': True, + 'clabel': 'Pattern quality', + } + ) + self.loaded_data.phase = np.array(data['Phase']).reshape(shape) + self.loaded_data.euler_angle = data['Euler'][:].reshape(shape + (3,)).transpose((2, 0, 1)) + + self.check_data() + class EdaxAngLoader(EBSDDataLoader): def load(self, file_name: pathlib.Path) -> None: """ Read an EDAX .ang file. diff --git a/pyproject.toml b/pyproject.toml index 6ef948c..72d36aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,7 @@ dependencies = [ "matplotlib_scalebar", "networkx", "numba", + "h5py" ] [project.urls] From 83e52d40c9698f01fffdd15ea54be681ac641a57 Mon Sep 17 00:00:00 2001 From: Michael Atkinson Date: Fri, 21 Aug 2026 09:46:38 +0100 Subject: [PATCH 2/5] feat: Add Oxford h5oina support --- defdap/file_readers.py | 68 +++++++++++++++++++++++++----------------- 1 file changed, 41 insertions(+), 27 deletions(-) diff --git a/defdap/file_readers.py b/defdap/file_readers.py index a3de6a4..504a2cb 100644 --- a/defdap/file_readers.py +++ b/defdap/file_readers.py @@ -13,16 +13,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -import numpy as np -from numpy.lib.recfunctions import structured_to_unstructured -import pandas as pd from abc import ABC, abstractmethod import pathlib import re -import h5py - from typing import TextIO, Dict, List, Callable, Any, Type, Optional +import h5py +import numpy as np +from numpy.lib.recfunctions import structured_to_unstructured +import pandas as pd + from defdap.crystal import Phase from defdap.quat import Quat from defdap.utils import Datastore @@ -57,7 +57,9 @@ def __init__(self) -> None: self.data_format = None @staticmethod - def get_loader(data_type: str, file_name: pathlib.Path) -> 'Type[EBSDDataLoader]': + def get_loader( + data_type: str, file_name: pathlib.Path + ) -> 'Type[EBSDDataLoader]': if data_type is None: data_type = { '.crc': 'oxfordbinary', @@ -256,18 +258,15 @@ def load(self, file_name: pathlib.Path) -> None: header = file['1']['EBSD']['Header'] data = file['1']['EBSD']['Data'] - x_dim = int(header['X Cells'][0]) - y_dim = int(header['Y Cells'][0]) - shape = (y_dim, x_dim) + shape = (int(header['Y Cells'][0]), int(header['X Cells'][0])) self.loaded_metadata['shape'] = shape - self.loaded_metadata['step_size'] = float(header['X Step'][0]) - ## Check this is acquisition orientataion from ctf - self.loaded_metadata['acquisition_rotation'] = Quat.from_euler_angles(*header['Specimen Orientation Euler'][0]) + self.loaded_metadata['acquisition_rotation'] = Quat.from_euler_angles( + *header['Specimen Orientation Euler'][0] + ) for phase_data in header['Phases'].values(): - phase = Phase( phase_data['Phase Name'][0].decode(), phase_data['Laue Group'][0], @@ -276,13 +275,12 @@ def load(self, file_name: pathlib.Path) -> None: phase_data['Lattice Dimensions'][0], phase_data['Lattice Angles'][0] ])) - self.loaded_metadata['phases'].append(phase) - - self.check_metadata() - # Data also available: Bands, Detector Distance, Error, Pattern Center X, Pattern Center Y + self.check_metadata() + # Data also available: Bands, Detector Distance, Error, Pattern Center + # X, Pattern Center Y self.loaded_data.add( 'band_contrast', np.array(data['Band Contrast']).reshape(shape), unit='', type='map', order=0, @@ -302,7 +300,8 @@ def load(self, file_name: pathlib.Path) -> None: } ) self.loaded_data.add( - 'mean_angular_deviation', np.array(data['Mean Angular Deviation']).reshape(shape), + 'mean_angular_deviation', + np.array(data['Mean Angular Deviation']).reshape(shape), unit='', type='map', order=0, plot_params={ 'plot_colour_bar': True, @@ -310,7 +309,8 @@ def load(self, file_name: pathlib.Path) -> None: } ) self.loaded_data.add( - 'pattern_quality', np.array(data['Pattern Quality']).reshape(shape), + 'pattern_quality', + np.array(data['Pattern Quality']).reshape(shape), unit='', type='map', order=0, plot_params={ 'plot_colour_bar': True, @@ -318,10 +318,13 @@ def load(self, file_name: pathlib.Path) -> None: } ) self.loaded_data.phase = np.array(data['Phase']).reshape(shape) - self.loaded_data.euler_angle = data['Euler'][:].reshape(shape + (3,)).transpose((2, 0, 1)) + self.loaded_data.euler_angle = ( + data['Euler'][:].reshape(shape + (3,)).transpose((2, 0, 1)) + ) self.check_data() + class EdaxAngLoader(EBSDDataLoader): def load(self, file_name: pathlib.Path) -> None: """ Read an EDAX .ang file. @@ -415,7 +418,9 @@ def load(self, file_name: pathlib.Path) -> None: ) add_phase = 1 if data['phase'].min() == 0 else 0 self.loaded_data.phase = data['phase'].reshape(shape) + add_phase - self.loaded_data['phase', 'plot_params']['vmax'] = len(self.loaded_metadata['phases']) + self.loaded_data['phase', 'plot_params']['vmax'] = len( + self.loaded_metadata['phases'] + ) # flatten the structured dtype euler_angle = structured_to_unstructured( @@ -512,8 +517,10 @@ def parse_line(line: str, group_dict: Dict) -> None: group_name = group_pat.match(line.strip()).group(1) group_dict = dict() - read_until_string(cpr_file, '[', comment_char=comment_char, - line_process=lambda l: parse_line(l, group_dict)) + read_until_string( + cpr_file, '[', comment_char=comment_char, + line_process=lambda l: parse_line(l, group_dict) + ) metadata[group_name] = group_dict # Create phase objects and move metadata to object metadata dict @@ -637,7 +644,8 @@ def load_oxford_crc(self, file_name: pathlib.Path) -> None: data[['ph1', 'phi', 'ph2']].reshape(shape)).transpose((2, 0, 1)) if self.loaded_metadata['edx']['Count'] > 0: - EDXFields = [key for key in data.dtype.fields.keys() if key.startswith('EDX')] + EDXFields = [key for key in data.dtype.fields.keys() + if key.startswith('EDX')] for field in EDXFields: self.loaded_data.add( field, @@ -678,7 +686,9 @@ def load(self, data_dict: Dict[str, Any]) -> None: unit='', type='map', order=0 ) self.loaded_data.phase = data_dict['phase'] - self.loaded_data['phase', 'plot_params']['vmax'] = len(self.loaded_metadata['phases']) + self.loaded_data['phase', 'plot_params']['vmax'] = len( + self.loaded_metadata['phases'] + ) self.loaded_data.euler_angle = data_dict['euler_angle'] self.check_data() @@ -907,8 +917,12 @@ def load(self, file_name: pathlib.Path) -> None: # if y descending, flip if np.all(np.diff(data['y'][:,0])) > 0: - self.loaded_data.coordinate = np.array([data['x'][::-1], data['y'][::-1]]) - self.loaded_data.displacement = np.array([data['u'][::-1], data['v'][::-1]]) + self.loaded_data.coordinate = np.array( + [data['x'][::-1], data['y'][::-1]] + ) + self.loaded_data.displacement = np.array( + [data['u'][::-1], data['v'][::-1]] + ) else: self.loaded_data.coordinate = np.array([data['x'], data['y']]) self.loaded_data.displacement = np.array([data['u'], data['v']]) From da9b2e254d5acd212ed2f24fd2d7b1c5fca23f51 Mon Sep 17 00:00:00 2001 From: Rhys Thomas Date: Fri, 21 Aug 2026 14:54:55 +0100 Subject: [PATCH 3/5] feat: Support for processed dataset in h5oina --- defdap/file_readers.py | 120 ++++++++++++++++++++++++++--------------- 1 file changed, 76 insertions(+), 44 deletions(-) diff --git a/defdap/file_readers.py b/defdap/file_readers.py index 504a2cb..9285a24 100644 --- a/defdap/file_readers.py +++ b/defdap/file_readers.py @@ -19,6 +19,7 @@ from typing import TextIO, Dict, List, Callable, Any, Type, Optional import h5py +from isort import file import numpy as np from numpy.lib.recfunctions import structured_to_unstructured import pandas as pd @@ -240,33 +241,50 @@ def parse_phase() -> Phase: class Oxfordh5Loader(EBSDDataLoader): - def load(self, file_name: pathlib.Path) -> None: + def load(self, file_name: pathlib.Path, dataset = None) -> None: """Read an Oxford Instruments ``.h5oina`` orientation file. Parameters ---------- file_name : pathlib.Path Path to file. + dataset : str (raw or processed), optional + Dataset to load. If None, defaults to raw data. """ - # open data file and read in metadata + # Open data file and read in metadata if not file_name.is_file(): raise FileNotFoundError(f"Cannot open file {file_name}") file = h5py.File(file_name) - header = file['1']['EBSD']['Header'] - data = file['1']['EBSD']['Data'] - - shape = (int(header['Y Cells'][0]), int(header['X Cells'][0])) + # This header contains all the information in the map that does not change with processing + raw_header = file['1']['EBSD']['Header'] + shape = (int(raw_header['Y Cells'][0]), int(raw_header['X Cells'][0])) self.loaded_metadata['shape'] = shape - self.loaded_metadata['step_size'] = float(header['X Step'][0]) - ## Check this is acquisition orientataion from ctf + self.loaded_metadata['step_size'] = float(raw_header['X Step'][0]) self.loaded_metadata['acquisition_rotation'] = Quat.from_euler_angles( - *header['Specimen Orientation Euler'][0] + *raw_header['Specimen Orientation Euler'][0] ) - for phase_data in header['Phases'].values(): + # Check if `Data Processing` dataset exists in the h5 + if 'Data' in file['1']['Data Processing'] and dataset is None: + print('\n\t' + 'Multiple datasets in h5 file, defaulting to raw data.') + print('\tProcessed data can be accessed by passing `processed` to the `dataset` argument.') + + # Handle `raw` or `processed` selection + if dataset is None or dataset is 'raw': + root = file['1']['EBSD'] + if dataset is 'processed': + if 'Data Processing' not in file['1']: + raise ValueError('No processed data in h5 file.') + elif 'Data' not in file['1']['Data Processing']: + raise ValueError('No processed data in h5 file.') + else: + root = file['1']['Data Processing'] + + # Phase data from relevant dataset + for phase_data in root['Header']['Phases'].values(): phase = Phase( phase_data['Phase Name'][0].decode(), phase_data['Laue Group'][0], @@ -279,47 +297,61 @@ def load(self, file_name: pathlib.Path) -> None: self.check_metadata() - # Data also available: Bands, Detector Distance, Error, Pattern Center - # X, Pattern Center Y - self.loaded_data.add( - 'band_contrast', np.array(data['Band Contrast']).reshape(shape), - unit='', type='map', order=0, - plot_params={ - 'plot_colour_bar': True, - 'cmap': 'gray', - 'clabel': 'Band contrast', - } - ) - self.loaded_data.add( - 'band_slope', np.array(data['Band Slope']).reshape(shape), - unit='', type='map', order=0, - plot_params={ - 'plot_colour_bar': True, - 'cmap': 'gray', - 'clabel': 'Band slope', - } - ) - self.loaded_data.add( - 'mean_angular_deviation', - np.array(data['Mean Angular Deviation']).reshape(shape), - unit='', type='map', order=0, - plot_params={ - 'plot_colour_bar': True, - 'clabel': 'Mean angular deviation', - } - ) - self.loaded_data.add( + # Some data is only avaiable and relevant for the raw data, for example band contrast + if dataset is 'raw': + self.loaded_data.add( + 'band_contrast', np.array(root['Data']['Band Contrast']).reshape(shape), + unit='', type='map', order=0, + plot_params={ + 'plot_colour_bar': True, + 'cmap': 'gray', + 'clabel': 'Band contrast', + } + ) + self.loaded_data.add( + 'band_slope', np.array(root['Data']['Band Slope']).reshape(shape), + unit='', type='map', order=0, + plot_params={ + 'plot_colour_bar': True, + 'cmap': 'gray', + 'clabel': 'Band slope', + } + ) + self.loaded_data.add( + 'mean_angular_deviation', + np.array(root['Data']['Mean Angular Deviation']).reshape(shape), + unit='', type='map', order=0, + plot_params={ + 'plot_colour_bar': True, + 'clabel': 'Mean angular deviation', + } + ) + self.loaded_data.add( + 'pattern_quality', + np.array(root['Data']['Pattern Quality']).reshape(shape), + unit='', type='map', order=0, + plot_params={ + 'plot_colour_bar': True, + 'clabel': 'Pattern quality', + } + ) + + # If pattern matching is performed, the cross correlation coefficient is useful + if dataset is 'processed' and 'Pattern Matching' in root: + self.loaded_data.add( 'pattern_quality', - np.array(data['Pattern Quality']).reshape(shape), + np.array(root['Pattern Matching']['Data']['Cross Correlation Coefficient']).reshape(shape), unit='', type='map', order=0, plot_params={ 'plot_colour_bar': True, - 'clabel': 'Pattern quality', + 'clabel': 'Cross Correlation Coefficient', } ) - self.loaded_data.phase = np.array(data['Phase']).reshape(shape) + + # Get Euler angles from relevant dataset + self.loaded_data.phase = np.array(root['Data']['Phase']).reshape(shape) self.loaded_data.euler_angle = ( - data['Euler'][:].reshape(shape + (3,)).transpose((2, 0, 1)) + root['Data']['Euler'][:].reshape(shape + (3,)).transpose((2, 0, 1)) ) self.check_data() From e8ecfb4a4ec89e47aa6d585bb515212f6bb08eaf Mon Sep 17 00:00:00 2001 From: Rhys Thomas Date: Fri, 21 Aug 2026 14:58:26 +0100 Subject: [PATCH 4/5] fix: code typo --- defdap/file_readers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/defdap/file_readers.py b/defdap/file_readers.py index 9285a24..a2b57e7 100644 --- a/defdap/file_readers.py +++ b/defdap/file_readers.py @@ -19,7 +19,6 @@ from typing import TextIO, Dict, List, Callable, Any, Type, Optional import h5py -from isort import file import numpy as np from numpy.lib.recfunctions import structured_to_unstructured import pandas as pd From df899d67e0697bf018e829b899483bfccf6a22bb Mon Sep 17 00:00:00 2001 From: Michael Atkinson Date: Fri, 21 Aug 2026 16:25:29 +0100 Subject: [PATCH 5/5] refactor: tidy h5 file reader --- defdap/file_readers.py | 59 +++++++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/defdap/file_readers.py b/defdap/file_readers.py index a2b57e7..8b9539e 100644 --- a/defdap/file_readers.py +++ b/defdap/file_readers.py @@ -257,7 +257,8 @@ def load(self, file_name: pathlib.Path, dataset = None) -> None: file = h5py.File(file_name) - # This header contains all the information in the map that does not change with processing + # This header contains all the information in the map that does not + # change with processing raw_header = file['1']['EBSD']['Header'] shape = (int(raw_header['Y Cells'][0]), int(raw_header['X Cells'][0])) self.loaded_metadata['shape'] = shape @@ -268,19 +269,21 @@ def load(self, file_name: pathlib.Path, dataset = None) -> None: # Check if `Data Processing` dataset exists in the h5 if 'Data' in file['1']['Data Processing'] and dataset is None: - print('\n\t' + 'Multiple datasets in h5 file, defaulting to raw data.') - print('\tProcessed data can be accessed by passing `processed` to the `dataset` argument.') + print('\n\tMultiple datasets in h5 file, defaulting to raw data.') + print( + '\tProcessed data can be accessed by passing `processed` to ' + 'the `dataset` argument.' + ) # Handle `raw` or `processed` selection - if dataset is None or dataset is 'raw': + if dataset is None or dataset == 'raw': root = file['1']['EBSD'] - if dataset is 'processed': + if dataset == 'processed': if 'Data Processing' not in file['1']: raise ValueError('No processed data in h5 file.') - elif 'Data' not in file['1']['Data Processing']: + if 'Data' not in file['1']['Data Processing']: raise ValueError('No processed data in h5 file.') - else: - root = file['1']['Data Processing'] + root = file['1']['Data Processing'] # Phase data from relevant dataset for phase_data in root['Header']['Phases'].values(): @@ -296,10 +299,13 @@ def load(self, file_name: pathlib.Path, dataset = None) -> None: self.check_metadata() - # Some data is only avaiable and relevant for the raw data, for example band contrast - if dataset is 'raw': + # Some data is only available and relevant for the raw data, for + # example band contrast + if dataset == 'raw': + raw_data = root['Data'] self.loaded_data.add( - 'band_contrast', np.array(root['Data']['Band Contrast']).reshape(shape), + 'band_contrast', + np.array(raw_data['Band Contrast']).reshape(shape), unit='', type='map', order=0, plot_params={ 'plot_colour_bar': True, @@ -308,7 +314,8 @@ def load(self, file_name: pathlib.Path, dataset = None) -> None: } ) self.loaded_data.add( - 'band_slope', np.array(root['Data']['Band Slope']).reshape(shape), + 'band_slope', + np.array(raw_data['Band Slope']).reshape(shape), unit='', type='map', order=0, plot_params={ 'plot_colour_bar': True, @@ -318,7 +325,7 @@ def load(self, file_name: pathlib.Path, dataset = None) -> None: ) self.loaded_data.add( 'mean_angular_deviation', - np.array(root['Data']['Mean Angular Deviation']).reshape(shape), + np.array(raw_data['Mean Angular Deviation']).reshape(shape), unit='', type='map', order=0, plot_params={ 'plot_colour_bar': True, @@ -327,7 +334,7 @@ def load(self, file_name: pathlib.Path, dataset = None) -> None: ) self.loaded_data.add( 'pattern_quality', - np.array(root['Data']['Pattern Quality']).reshape(shape), + np.array(raw_data['Pattern Quality']).reshape(shape), unit='', type='map', order=0, plot_params={ 'plot_colour_bar': True, @@ -335,17 +342,21 @@ def load(self, file_name: pathlib.Path, dataset = None) -> None: } ) - # If pattern matching is performed, the cross correlation coefficient is useful - if dataset is 'processed' and 'Pattern Matching' in root: + # If pattern matching is performed, the cross correlation coefficient + # is useful + if dataset == 'processed' and 'Pattern Matching' in root: + pattern_data = root['Pattern Matching']['Data'] self.loaded_data.add( - 'pattern_quality', - np.array(root['Pattern Matching']['Data']['Cross Correlation Coefficient']).reshape(shape), - unit='', type='map', order=0, - plot_params={ - 'plot_colour_bar': True, - 'clabel': 'Cross Correlation Coefficient', - } - ) + 'pattern_quality', + np.array( + pattern_data['Cross Correlation Coefficient'] + ).reshape(shape), + unit='', type='map', order=0, + plot_params={ + 'plot_colour_bar': True, + 'clabel': 'Cross Correlation Coefficient', + } + ) # Get Euler angles from relevant dataset self.loaded_data.phase = np.array(root['Data']['Phase']).reshape(shape)