Commit 9a310e45 authored by Javier Quinteros's avatar Javier Quinteros
Browse files

First version of a TDMS class with a very basic metadata iterator

Still needs to be tested.
parent 22351ff7
...@@ -22,218 +22,245 @@ import struct ...@@ -22,218 +22,245 @@ import struct
import datetime import datetime
HEADERLEN = 28 class TDMS(object):
def __init__(self, filename, loglevel='INFO'):
kTocMetaData = 1 << 1 logs = logging.getLogger('OpenFile')
kTocNewObjList = 1 << 2 logs.setLevel(loglevel)
kTocRawData = 1 << 3
kTocInterleavedData = 1 << 5 # Initialization of local variables
kTocBigEndian = 1 << 6 HEADERLEN = 28
kTocDAQmxRawData = 1 << 7 kTocMetaData = 1 << 1
kTocNewObjList = 1 << 2
FF64b = 0xFFFFFFFFFFFFFFFF kTocRawData = 1 << 3
FF32b = 0xFFFFFFFF kTocInterleavedData = 1 << 5
kTocBigEndian = 1 << 6
kTocDAQmxRawData = 1 << 7
def main(): self.__FF64b = 0xFFFFFFFFFFFFFFFF
# Check verbosity in the output self.__FF32b = 0xFFFFFFFF
msg = 'Read and convert waveforms generated by a DAS system.'
parser = argparse.ArgumentParser(description=msg) self.__data2mask = {
parser.add_argument('-l', '--loglevel', 0: ('c', 1), # tdsTypeVoid
help='Verbosity in the output.', 1: ('b', 1), # tdsTypeI8
choices=['CRITICAL', 'ERROR', 'WARNING', 'INFO', 2: ('h', 2), # tdsTypeI16
'DEBUG'], 3: ('i', 4), # tdsTypeI32
default='WARNING') 4: ('q', 8), # tdsTypeI64
parser.add_argument('filename', help='File name to read and process.') 5: ('b', 1), # tdsTypeU8
6: ('h', 2), # tdsTypeU16
args = parser.parse_args() 7: ('i', 4), # tdsTypeU32
logging.basicConfig(level=args.loglevel) 8: ('q', 8), # tdsTypeU64
9: ('f', 4), # tdsTypeSingleFloat
logs = logging.getLogger('OpenFile') 10: ('d', 8), # tdsTypeDoubleFloat
logs.setLevel(args.loglevel) 0x20: ('I', 4), # tdsTypeString
0x21: ('?', 1), # tdsTypeBoolean
with open(args.filename, 'rb') as fi: 0x44: ('Qq', 16) # tdsTypeTimeStamp
leadin = fi.read(HEADERLEN) }
# tdsTypeFixedPoint = 0x4F,
# tdsTypeComplexSingleFloat = 0x08000c,
# tdsTypeComplexDoubleFloat = 0x10000d,
# tdsTypeDAQmxRawData = 0xFFFFFFFF
if input is None:
logs.error('No input file was specified!')
self.__fi = open(filename, 'rb')
leadin = self.__fi.read(HEADERLEN)
(tag, ToCmask) = struct.unpack('<4si', leadin[:8]) (tag, ToCmask) = struct.unpack('<4si', leadin[:8])
self.metadata = bool(ToCmask & kTocMetaData)
self.newObjects = bool(ToCmask & kTocNewObjList)
self.rawData = bool(ToCmask & kTocRawData)
self.interleavedData = bool(ToCmask & kTocInterleavedData)
self.DAQmxRawData = bool(ToCmask & kTocDAQmxRawData)
# All input from now on will be formatted by this # All input from now on will be formatted by this
endian = '>' if ToCmask & kTocBigEndian else '<' self.__endian = '>' if ToCmask & kTocBigEndian else '<'
(version, segmentOffset, dataOffset) = struct.unpack('%ciQQ' % endian, leadin[8:])
if tag.decode() != 'TDSm': if tag.decode() != 'TDSm':
raise Exception('Tag is not TDSm!') raise Exception('Tag is not TDSm!')
(version, self.__segmentOffset, self.__dataOffset) = \
struct.unpack('%ciQQ' % self.__endian, leadin[8:])
logs.info((tag, ToCmask, version, self.__segmentOffset, self.__dataOffset))
if version != 4713: if version != 4713:
logs.warning('Version number is not 4713!') logs.warning('Version number is not 4713!')
logs.info('Metadata: ' + ('yes' if ToCmask & kTocMetaData else 'no')) if self.__segmentOffset == self.__FF64b:
logs.info('Object list: ' + ('yes' if ToCmask & kTocNewObjList else 'no'))
logs.info('Raw data: ' + ('yes' if ToCmask & kTocRawData else 'no'))
logs.info('Interleaved data: ' + ('yes' if ToCmask & kTocInterleavedData else 'no'))
logs.info('BigEndian: ' + ('yes' if ToCmask & kTocBigEndian else 'no'))
logs.info('DAQmx raw data: ' + ('yes' if ToCmask & kTocDAQmxRawData else 'no'))
if segmentOffset == FF64b:
logs.error('Severe problem while writing data (crash, power outage)') logs.error('Severe problem while writing data (crash, power outage)')
if ((ToCmask & kTocMetaData) and not dataOffset): if self.metadata and not self.__dataOffset:
logs.error('Flag indicates Metadata but its length is 0!') logs.error('Flag indicates Metadata but its length is 0!')
if (ToCmask & kTocDAQmxRawData): if self.DAQmxRawData:
logs.warning('DAQmx raw data is still not supported!') logs.warning('DAQmx raw data is still not supported!')
logs.info((tag, ToCmask, version, segmentOffset, dataOffset)) # Absolute offsets
self.__segmentOffset += HEADERLEN
self.__dataOffset += HEADERLEN
logs.info('Metadata: ' + ('yes' if self.metadata else 'no'))
logs.info('Object list: ' + ('yes' if self.newObjects else 'no'))
logs.info('Raw data: ' + ('yes' if self.rawData else 'no'))
logs.info('Interleaved data: ' + ('yes' if self.interleavedData else 'no'))
logs.info('BigEndian: ' + ('yes' if self.__endian == '<' else 'no'))
logs.info('DAQmx raw data: ' + ('yes' if self.DAQmxRawData else 'no'))
def __iter__(self):
return self
def __next__(self):
return self.__next__metadata__()
def __next__metadata__(self):
logs = logging.getLogger('Iterate Metadata')
# Metadata # Metadata
# Number of objects (unsigned int - 32b) # Number of objects (unsigned int - 32b)
numObjects = struct.unpack('%cI' % endian, fi.read(4))[0] numObjects = struct.unpack('%cI' % self.__endian, self.__fi.read(4))[0]
logs.info('Number of objects in metadata: %s' % numObjects) logs.info('Number of objects in metadata: %s' % numObjects)
numChannels = 0 numChannels = 0
chunkSize = 0 chunkSize = 0
for obj in range(numObjects): for obj in range(numObjects):
channelSize = 0 channelSize = 0
objPath = readstring(fi, endian) objPath = self.__readstring()
logs.debug('Object %s: %s' % (obj, objPath)) logs.info('Object %s: %s' % (obj, objPath))
rawDataIdx = struct.unpack('%cI' % endian, fi.read(4))[0] rawDataIdx = struct.unpack('%cI' % self.__endian, self.__fi.read(4))[0]
if rawDataIdx == FF32b: if rawDataIdx == self.__FF32b:
logs.info('No raw data assigned to this segment') logs.info('No raw data assigned to this segment')
readproperties(fi, endian) self.__readproperties()
continue
elif not rawDataIdx: elif not rawDataIdx:
logs.info('Raw data index in this segment matches the index the same object had in the previous segment') logs.info('Raw data index in this segment matches the index the same object had in the previous segment')
else: else:
numChannels = numChannels + 1 numChannels += 1
# There is raw data! # There is raw data!
sizeBytes = None sizeBytes = None
datatype, arraylen, numValues = struct.unpack('%cIIQ' % endian, fi.read(16)) datatype, arraylen, numValues = struct.unpack('%cIIQ' % self.__endian, self.__fi.read(16))
if datatype == 0x20: if datatype == 0x20:
sizeBytes = struct.unpack('%cQ' % endian, fi.read(8))[0] sizeBytes = struct.unpack('%cQ' % self.__endian, self.__fi.read(8))[0]
if arraylen != 1: if arraylen != 1:
logs.error('Array length MUST be 1! Actual value: %s' % arraylen) logs.error('Array length MUST be 1! Actual value: %s' % arraylen)
# logs.debug('obj %s; datatype: %s; numValues: %s; size: %s' % (obj, datatype, numValues, logs.debug('obj %s; datatype: %s; numValues: %s; size: %s' % (obj, datatype, numValues,
# data2mask[datatype][1]*numValues)) self.__data2mask[datatype][1]*numValues))
channelSize = channelSize + data2mask[datatype][1]*numValues channelSize = self.__data2mask[datatype][1]*numValues
readproperties(fi, endian) self.__readproperties()
logs.debug('channelSize: %s bytes' % channelSize) # logs.debug('channelSize: %s bytes' % channelSize)
chunkSize = chunkSize + channelSize chunkSize = chunkSize + channelSize
samples = int((segmentOffset - dataOffset)/numChannels/data2mask[datatype][1]) # print(self.__segmentOffset, self.__dataOffset, numChannels, self.__data2mask[datatype][1])
logs.info('chunkSize: %s bytes' % chunkSize)
logs.info('Total chunks size: %s' % (segmentOffset - dataOffset)) # samples = int((self.__segmentOffset - self.__dataOffset)/numChannels/self.__data2mask[datatype][1])
logs.info('Length of channel: %d' % ((segmentOffset - dataOffset)/numChannels/data2mask[datatype][1])) # logs.info('chunkSize: %s bytes' % chunkSize)
# New or changed objects # logs.info('Total chunks size: %s' % (self.__segmentOffset - self.__dataOffset))
newObjects = struct.unpack('%cI' % endian, fi.read(4))[0] # logs.info('Length of channel: %d' % ((self.__segmentOffset - self.__dataOffset)/numChannels/self.__data2mask[datatype][1]))
# # New or changed objects
# Got to the beginning of the raw data # newObjects = struct.unpack('%cI' % self.__endian, self.__fi.read(4))[0]
fi.seek(dataOffset + HEADERLEN, 0) #
result = readdata(fi, data2mask[datatype][0], data2mask[datatype][1], samples, ToCmask & kTocInterleavedData, # # Loop through channels
numChannels, endian) # for ch in range(1):
# result = self.__readdata(self.__data2mask[datatype][0], self.__data2mask[datatype][1], samples,
print(result) # numChannels, channel=ch)
def __readstring(self):
def readdata(fi, datatype, datasize, samples, interleaved, numChannels, endian='<'): # logs = logging.getLogger('readstring')
if not interleaved: strlen = struct.unpack('%cI' % self.__endian, self.__fi.read(4))
# Read from first channel # logs.debug('String of length %s' % strlen)
result = struct.unpack('%c%s' % (endian, datatype*samples), fi.read(datasize*samples)) return self.__fi.read(strlen[0]).decode()
result = list(result)
else: def __readvalue(self):
result = list() logs = logging.getLogger('readvalue')
for ch in range(samples):
# Read from first channel (0) datatype = self.__readdatatype()
result.append(struct.unpack('%c%s' % (endian, datatype*numChannels), fi.read(datasize*numChannels))[0]) logs.debug('datatype: 0x%x' % datatype)
return result # Consider cases which need another read
# 0x20 is a string. Read again!
if datatype == 0x20:
def readproperties(fi, endian='<'): return self.__readstring()
logs = logging.getLogger('readproperties')
(mask, numBytes) = self.__data2mask[datatype]
numProps = struct.unpack('%cI' % endian, fi.read(4))[0] logs.debug('Mask: %s; Bytes: %s' % (mask, numBytes))
logs.debug('%s properties' % numProps) # This instruction returns a tuple. Needed for timestamps
for prop in range(numProps): result = struct.unpack('%c%s' % (self.__endian, mask), self.__fi.read(numBytes))
propStr = readstring(fi, endian)
logs.debug('Prop: %s' % propStr) # 0x44 is a timestamp. Read again!
value = readvalue(fi, endian) if datatype == 0x44:
logs.debug(value) result = self.__tup2time(*result)
else:
data2mask = { # Disassemble the tuple if not a timestamp
0: ('c', 1), # tdsTypeVoid result = result[0]
1: ('b', 1), # tdsTypeI8
2: ('h', 2), # tdsTypeI16 logs.debug('result: %s' % result)
3: ('i', 4), # tdsTypeI32 return result
4: ('q', 8), # tdsTypeI64
5: ('b', 1), # tdsTypeU8 def __readproperties(self):
6: ('h', 2), # tdsTypeU16 logs = logging.getLogger('readproperties')
7: ('i', 4), # tdsTypeU32
8: ('q', 8), # tdsTypeU64 numProps = struct.unpack('%cI' % self.__endian, self.__fi.read(4))[0]
9: ('f', 4), # tdsTypeSingleFloat logs.debug('%s properties' % numProps)
10: ('d', 8), # tdsTypeDoubleFloat for prop in range(numProps):
0x20: ('I', 4), # tdsTypeString propStr = self.__readstring()
0x21: ('?', 1), # tdsTypeBoolean logs.debug('Prop: %s' % propStr)
0x44: ('Qq', 16) # tdsTypeTimeStamp value = self.__readvalue()
} logs.debug(value)
# tdsTypeFixedPoint = 0x4F,
# tdsTypeComplexSingleFloat = 0x08000c, def __readdatatype(self):
# tdsTypeComplexDoubleFloat = 0x10000d, return struct.unpack('%cI' % self.__endian, self.__fi.read(4))[0]
# tdsTypeDAQmxRawData = 0xFFFFFFFF
def __tup2time(self, fraction, seconds):
logs = logging.getLogger('tup2time')
def readdatatype(fi, endian='<'): logs.debug('seconds: %s' % seconds)
return struct.unpack('%cI' % endian, fi.read(4))[0] logs.debug('fraction: %s' % fraction)
dt1904 = datetime.datetime(1904, 1, 1)
def readvalue(fi, endian='<'): delta = seconds + fraction * 2**(-64)
logs = logging.getLogger('readvalue') result = dt1904 + datetime.timedelta(seconds=delta)
logs.debug('Date-time %s' % result)
datatype = readdatatype(fi, endian) return result
logs.debug('datatype: 0x%x' % datatype)
def __readdata(self, datatype, datasize, samples, numChannels, channel=0):
# Consider cases which need another read if not self.interleavedData:
# 0x20 is a string. Read again! # Seek where the channel starts
if datatype == 0x20: self.__fi.seek(self.__dataOffset + datasize*samples*channel, 0)
return readstring(fi, endian) result = struct.unpack('%c%s' % (self.__endian, datatype*samples), self.__fi.read(datasize*samples))
# result = list(result)
(mask, numBytes) = data2mask[datatype] else:
logs.debug('Mask: %s; Bytes: %s' % (mask, numBytes)) # Seek where the raw data starts
# This instruction returns a tuple. Needed for timestamps self.__fi.seek(self.__dataOffset, 0)
result = struct.unpack('%c%s' % (endian, mask), fi.read(numBytes)) result = list()
for ch in range(samples):
# 0x44 is a timestamp. Read again! # Read from all channels and select the specific one with an index (channel)
if datatype == 0x44: result.append(struct.unpack('%c%s' % (self.__endian, datatype*numChannels), self.__fi.read(datasize*numChannels))[channel])
result = tup2time(*result)
else: return result
# Disassemble the tuple if not a timestamp
result = result[0]
def main():
logs.debug('result: %s' % result) # Check verbosity in the output
return result msg = 'Read and convert waveforms generated by a DAS system.'
parser = argparse.ArgumentParser(description=msg)
parser.add_argument('-l', '--loglevel',
def tup2time(fraction, seconds): help='Verbosity in the output.',
logs = logging.getLogger('tup2time') choices=['CRITICAL', 'ERROR', 'WARNING', 'INFO',
logs.debug('seconds: %s' % seconds) 'DEBUG'],
logs.debug('fraction: %s' % fraction) default='WARNING')
parser.add_argument('filename', help='File name to read and process.')
dt1904 = datetime.datetime(1904, 1, 1)
delta = seconds + fraction * 2**(-64) args = parser.parse_args()
result = dt1904 + datetime.timedelta(seconds=delta) logging.basicConfig(level=args.loglevel)
logs.debug('Date-time %s' % result)
return result logs = logging.getLogger('OpenFile')
logs.setLevel(args.loglevel)
TDMS(args.filename)
def readstring(fi, endian='<'):
# logs = logging.getLogger('readstring')
strlen = struct.unpack('%cI' % endian, fi.read(4))
# logs.debug('String of length %s' % strlen)
return fi.read(strlen[0]).decode()
if __name__ == '__main__': if __name__ == '__main__':
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment