diff --git a/apread/apreader.py b/apread/apreader.py index c1ec566..2a64945 100644 --- a/apread/apreader.py +++ b/apread/apreader.py @@ -38,7 +38,7 @@ class APReader: """ Groups: List[Group] - def __init__(self, path, verbose=False, filterData=False): + def __init__(self, path, verbose=False, filterData=False, fastload=True): """Creates a new APReader based on a .binary file (path). Args: @@ -48,6 +48,7 @@ def __init__(self, path, verbose=False, filterData=False): """ self.verbose = verbose self.filterData = filterData + self.fastload = fastload self.filepath = path self.fileName = os.path.splitext(os.path.basename(path))[0] self.Channels = [] @@ -255,7 +256,7 @@ def read(self): for i in range(self.numChannels): # create new channel on top of reader #! be careful with current stream position - channel = Channel(reader, self.fileName, self.verbose, self.filterData) + channel = Channel(reader, self.fileName, self.verbose, self.filterData, self.fastload) if not channel.broken and channel.length > 0: self.Channels.append(channel) diff --git a/apread/binaryReader.py b/apread/binaryReader.py index 894b28b..a2c47cb 100644 --- a/apread/binaryReader.py +++ b/apread/binaryReader.py @@ -51,7 +51,7 @@ def read_chars(self, size, encoding: str = "utf-8"): return ret.decode(encoding) def read_char(self, encoding: str = "ascii"): return struct.unpack(self.endian + "c", self.read(1))[0].decode(encoding) - def read_single(self) -> int: + def read_byte(self) -> int: return struct.unpack(self.endian + "b", self.read(1))[0] def read_int16(self) -> int: @@ -60,6 +60,9 @@ def read_int16(self) -> int: def read_int32(self) -> int: return struct.unpack(self.endian + "i", self.read(4))[0] + def read_float(self) -> float: + return struct.unpack(self.endian + "f", self.read(4))[0] + def read_double(self) -> float: return struct.unpack(self.endian + "d", self.read(8))[0] # Aliases diff --git a/apread/entries.py b/apread/entries.py index 5441c9d..19afd57 100644 --- a/apread/entries.py +++ b/apread/entries.py @@ -14,7 +14,7 @@ # progress from tqdm import tqdm from apread.loader import Loader - +import numpy as np # filtering from scipy.signal import lfilter @@ -33,17 +33,18 @@ class Channel: If there is more than one channel having the same amount of entries, every channel will get the same reference to the time channel. """ - data: List[float] + # data: List[float] verbose: bool # Defines if data should be filtered. filterData: bool - def __init__(self, reader: BinaryReader, fileName='unknown', verbose=False, filterData=False): + def __init__(self, reader: BinaryReader, fileName='unknown', verbose=False, filterData=False, fastload=True): """ Creates the Channel. Uses a reader (BinaryReader) to read the data from the file accessed by "APReader.__init__". """ + self.fastload = fastload # defines, if the apreader should output verbose debug messages self.verbose = verbose # defines, wether read data should be filtered @@ -80,14 +81,23 @@ def __init__(self, reader: BinaryReader, fileName='unknown', verbose=False, filt self.dw = reader.read_int16() # time of reading self.time = reader.read_double() - # extended channel header (mostly unused) - self.header = reader.read(reader.read_int32()) + # extended channel header + self.nHdrBytes = reader.read_int32() + self.extHeader = self.readExtHeader(reader) + + precDict = {0:8, 1:4, 2:2} # key: Attribute "Exportformat", value: precision in bytes + try: + self.precision = precDict[self.extHeader['ExportFormat']] + except KeyError: + print('Unexpected value of attribute "ExportFormat" in the extended header of channel {}. Assuming double precision.'.format(self.Name)) + self.precision = 8 + # linearization mode self.lmode = reader.read_char() # user scale self.scale = reader.read_char() # unknown points - self.npoi = reader.read_single() + self.npoi = reader.read_byte() # readaway for i in range(self.npoi): reader.read_double() @@ -101,8 +111,80 @@ def __init__(self, reader: BinaryReader, fileName='unknown', verbose=False, filt # flag to indicate that everything is fine self.broken = False + + def readExtHeader(self, rdr: BinaryReader): + """ + Reads the extended header of this Channel. + + NOTE: The catman binary files use byte padding, which means that all + values are stored at byte addresses which are integer multiples of their + width in bytes (i.e. doubles are stored on addresses divisible by eight, + floats on addresses divisible by four etc.) + + See the link below for more info: + https://stackoverflow.com/questions/4306186/structure-padding-and-packing + For this reason, I've added three bytes of padding before the attribute + 'NominalRange', which is a float. + + """ + pos0 = rdr.tell() # In general not a multiple of eight, which is unexpected! + exthdr = {} + exthdr['T0'] = rdr.read_double() # (pos0+) 8 + exthdr['dt'] = rdr.read_double() # 16 + exthdr['SensorType'] = rdr.read_int16() # 18 + exthdr['SupplyVoltage'] = rdr.read_int16() # 20 + + exthdr['FiltChar'] = rdr.read_int16() # 22 + exthdr['FiltFreq'] = rdr.read_int16() # 24 + exthdr['TareVal'] = rdr.read_float() # 28 + exthdr['ZeroVal'] = rdr.read_float() # 32 + exthdr['MeasRange'] = rdr.read_float() # 36 + exthdr['InChar'] = [rdr.read_float() for i in range(4)] # 40, 44, 48, 52 + + exthdr['SerNo'] = rdr.read_string(32) # 84 + exthdr['PhysUnit'] = rdr.read_string(8) # 92 + exthdr['NativeUnit'] = rdr.read_string(8) # 100 + + exthdr['Slot'] = rdr.read_int16() # 102 + exthdr['SubSlot'] = rdr.read_int16() # 104 + exthdr['AmpType'] = rdr.read_int16() # 106 + exthdr['APType'] = rdr.read_int16() # 108 + exthdr['kFactor'] = rdr.read_float() # 112 + exthdr['bFactor'] = rdr.read_float() # 116 + + exthdr['MeasSig'] = rdr.read_int16() # 118 + exthdr['AmpInput'] = rdr.read_int16() # 120 + exthdr['HPFilt'] = rdr.read_int16() # 122 + exthdr['OLImportInfo'] = rdr.read_byte() # 123 + exthdr['ScaleType'] = rdr.read_byte() # 124 + exthdr['SoftwareTareVal'] = rdr.read_float() # 128 + exthdr['WriteProtected'] = rdr.read_byte() # 129 + padding = rdr.read_string(3) # 132 + + exthdr['NominalRange'] = rdr.read_float() # 136 + exthdr['CLCFactor'] = rdr.read_float() # 140 + exthdr['ExportFormat'] = rdr.read_byte() # 141 + reserve = rdr.read_string(7) # 148 + # reserve = rdr.read_string(10) + posN = rdr.tell() + + if (posN-pos0) != self.nHdrBytes: + print(""" + WARNING: + The number of bytes read in the extended header of the channel + '{}' + doesn't match its declared length. + This probably means that the hardcoded format definition in the method + 'Channel.readExtHeader' is no longer valid and must be revised. + Leaving the extended header as-is and resetting the read position of + the binary reader. Assuming double precision for the data. + """.format(self.Name)) + rdr.seek(pos0 + self.nHdrBytes) + exthdr['ExportFormat'] = 0 + + return exthdr def readData(self): """ @@ -114,12 +196,34 @@ def readData(self): # if something was wrong previously, nothing will happen here if self.broken: return - - # initialize data - self.data = [] - # read all channel data - for i in tqdm(range(self.length), leave=False): - self.data.append(self.reader.read_double()) + + if self.fastload: + # The data is stored channelwise. We therefore only need to pass pointers to the first and last byte. + if self.precision == 8 or self.precision == 4: + self.data = np.fromfile(self.reader.buf, dtype=np.dtype('f{}'.format(self.precision)), count=self.length) + elif self.precision == 2: + MinValue = self.reader.read_double() + MaxValue = self.reader.read_double() + sf = (MaxValue - MinValue)/32767 # scale factor + self.data = np.fromfile(self.reader.buf, dtype=np.dtype('u2'), count=self.length)*sf + MinValue + else: + # initialize data + self.data = [] + + # read all channel data + if self.precision == 8: + for i in tqdm(range(self.length), leave=False): + self.data.append(self.reader.read_double()) + elif self.precision == 4: + for i in tqdm(range(self.length), leave=False): + self.data.append(self.reader.read_float()) + elif self.precision == 2: + MinValue = self.reader.read_double() + MaxValue = self.reader.read_double() + sf = (MaxValue - MinValue)/32767 # scale factor + + for i in tqdm(range(self.length), leave=False): + self.data.append(self.reader.read_int16()*sf + MinValue) # filter data if self.filterData: diff --git a/test/2byteJob1_2022_04_12_13_32_10.TST b/test/2byteJob1_2022_04_12_13_32_10.TST new file mode 100644 index 0000000..1d0735e --- /dev/null +++ b/test/2byteJob1_2022_04_12_13_32_10.TST @@ -0,0 +1,27 @@ +CATMAN TEST FILE +PATH=C:\Users\sommer\Documents\HBM\catmanEasy\DATA\2022\Precision 4byte vs 8byte +Job name=Job1 +Data file with path=C:\Users\sommer\Documents\HBM\catmanEasy\DATA\2022\Precision 4byte vs 8byte\2byteJob1_2022_04_12_13_32_10.bin +Data file=2byteJob1_2022_04_12_13_32_10.bin +DAQ project=Precision 4byte vs 8byte.MEP +File comment= +Time zone=Mitteleuropäische Sommerzeit (UTC+2:00) +Bearbeiter= +Abteilung= +Kommentar= +Default sample rate=300 Hz +Slow sample rate=10 Hz +Fast sample rate=4800 Hz +Number of samples default sample rate=100 +Number of samples slow sample rate=0 +Number of samples fast sample rate=0 +Saving mode=AUTO +Master/Slave mode=No +Start mode=Manual +Start time=2022/04/12 13:32:09 +Stop mode=Number of samples +Stop time=2022/04/12 13:32:10 +catmanEasy/AP version=5.5.2.8 +Numerical precision=2 Byte Integer +Firmware MX840A=4.40.6.0 +DATAFILE=C:\Users\sommer\Documents\HBM\catmanEasy\DATA\2022\Precision 4byte vs 8byte\2byteJob1_2022_04_12_13_32_10.bin diff --git a/test/2byteJob1_2022_04_12_13_32_10.bin b/test/2byteJob1_2022_04_12_13_32_10.bin new file mode 100644 index 0000000..5639eb7 Binary files /dev/null and b/test/2byteJob1_2022_04_12_13_32_10.bin differ diff --git a/test/4byteJob1_2022_04_12_13_31_47.TST b/test/4byteJob1_2022_04_12_13_31_47.TST new file mode 100644 index 0000000..0b101af --- /dev/null +++ b/test/4byteJob1_2022_04_12_13_31_47.TST @@ -0,0 +1,27 @@ +CATMAN TEST FILE +PATH=C:\Users\sommer\Documents\HBM\catmanEasy\DATA\2022\Precision 4byte vs 8byte +Job name=Job1 +Data file with path=C:\Users\sommer\Documents\HBM\catmanEasy\DATA\2022\Precision 4byte vs 8byte\4byteJob1_2022_04_12_13_31_47.bin +Data file=4byteJob1_2022_04_12_13_31_47.bin +DAQ project=Precision 4byte vs 8byte.MEP +File comment= +Time zone=Mitteleuropäische Sommerzeit (UTC+2:00) +Bearbeiter= +Abteilung= +Kommentar= +Default sample rate=300 Hz +Slow sample rate=10 Hz +Fast sample rate=4800 Hz +Number of samples default sample rate=100 +Number of samples slow sample rate=0 +Number of samples fast sample rate=0 +Saving mode=AUTO +Master/Slave mode=No +Start mode=Manual +Start time=2022/04/12 13:31:47 +Stop mode=Number of samples +Stop time=2022/04/12 13:31:47 +catmanEasy/AP version=5.5.2.8 +Numerical precision=4 Byte Float +Firmware MX840A=4.40.6.0 +DATAFILE=C:\Users\sommer\Documents\HBM\catmanEasy\DATA\2022\Precision 4byte vs 8byte\4byteJob1_2022_04_12_13_31_47.bin diff --git a/test/4byteJob1_2022_04_12_13_31_47.bin b/test/4byteJob1_2022_04_12_13_31_47.bin new file mode 100644 index 0000000..78c4f0e Binary files /dev/null and b/test/4byteJob1_2022_04_12_13_31_47.bin differ diff --git a/test/8byteJob1_2022_04_12_13_31_23.TST b/test/8byteJob1_2022_04_12_13_31_23.TST new file mode 100644 index 0000000..e561e6d --- /dev/null +++ b/test/8byteJob1_2022_04_12_13_31_23.TST @@ -0,0 +1,27 @@ +CATMAN TEST FILE +PATH=C:\Users\sommer\Documents\HBM\catmanEasy\DATA\2022\Precision 4byte vs 8byte +Job name=Job1 +Data file with path=C:\Users\sommer\Documents\HBM\catmanEasy\DATA\2022\Precision 4byte vs 8byte\8byteJob1_2022_04_12_13_31_23.bin +Data file=8byteJob1_2022_04_12_13_31_23.bin +DAQ project=Precision 4byte vs 8byte.MEP +File comment= +Time zone=Mitteleuropäische Sommerzeit (UTC+2:00) +Bearbeiter= +Abteilung= +Kommentar= +Default sample rate=300 Hz +Slow sample rate=10 Hz +Fast sample rate=4800 Hz +Number of samples default sample rate=100 +Number of samples slow sample rate=0 +Number of samples fast sample rate=0 +Saving mode=AUTO +Master/Slave mode=No +Start mode=Manual +Start time=2022/04/12 13:31:23 +Stop mode=Number of samples +Stop time=2022/04/12 13:31:23 +catmanEasy/AP version=5.5.2.8 +Numerical precision=8 Byte Float +Firmware MX840A=4.40.6.0 +DATAFILE=C:\Users\sommer\Documents\HBM\catmanEasy\DATA\2022\Precision 4byte vs 8byte\8byteJob1_2022_04_12_13_31_23.bin diff --git a/test/8byteJob1_2022_04_12_13_31_23.bin b/test/8byteJob1_2022_04_12_13_31_23.bin new file mode 100644 index 0000000..b34b350 Binary files /dev/null and b/test/8byteJob1_2022_04_12_13_31_23.bin differ diff --git a/test/catmanBinaryFormat.xls b/test/catmanBinaryFormat.xls new file mode 100644 index 0000000..fcfb107 Binary files /dev/null and b/test/catmanBinaryFormat.xls differ