Commit 0623123c authored by Javier Quinteros's avatar Javier Quinteros
Browse files

Read metadata when file is opened. Iterator reads from object attribute.

parent 615b034e
......@@ -23,12 +23,22 @@ import datetime
class TDMS(object):
# def __exit__(self, exc_type, exc_val, exc_tb):
# if self.__fi is not None:
# self.__fi.close()
def __init__(self, filename, iterate='D', loglevel='INFO'):
logs = logging.getLogger('OpenFile')
logs.setLevel(loglevel)
# What should we iterate? D: Data; M: Metadata
self.iterate = iterate
# Datatype of each channel
self.__datatypes = dict()
# Initialization of local variables
HEADERLEN = 28
self.__HEADERLEN = 28
kTocMetaData = 1 << 1
kTocNewObjList = 1 << 2
kTocRawData = 1 << 3
......@@ -61,16 +71,14 @@ class TDMS(object):
self.__fi = open(filename, 'rb')
leadin = self.__fi.read(HEADERLEN)
leadin = self.__fi.read(self.__HEADERLEN)
(tag, ToCmask) = struct.unpack('<4si', leadin[:8])
self.metadata = bool(ToCmask & kTocMetaData)
self.newObjects = bool(ToCmask & kTocNewObjList)
self.rawData = bool(ToCmask & kTocRawData)
self.interleavedData = bool(ToCmask & kTocInterleavedData)
self.DAQmxRawData = bool(ToCmask & kTocDAQmxRawData)
self.iterate = iterate
self.hasmetadata = bool(ToCmask & kTocMetaData)
self.hasnewObjects = bool(ToCmask & kTocNewObjList)
self.hasrawData = bool(ToCmask & kTocRawData)
self.hasInterleavedData = bool(ToCmask & kTocInterleavedData)
self.hasDAQmxRawData = bool(ToCmask & kTocDAQmxRawData)
# All input from now on will be formatted by this
self.__endian = '>' if ToCmask & kTocBigEndian else '<'
......@@ -89,99 +97,110 @@ class TDMS(object):
if self.__segmentOffset == self.__FF64b:
logs.error('Severe problem while writing data (crash, power outage)')
if self.metadata and not self.__dataOffset:
if self.hasmetadata and not self.__dataOffset:
logs.error('Flag indicates Metadata but its length is 0!')
if self.DAQmxRawData:
if self.hasDAQmxRawData:
logs.warning('DAQmx raw data is still not supported!')
# Absolute offsets
self.__segmentOffset += HEADERLEN
self.__dataOffset += HEADERLEN
self.__segmentOffset += self.__HEADERLEN
self.__dataOffset += self.__HEADERLEN
logs.info('Metadata: ' + ('yes' if self.metadata else 'no'))
logs.info('Object list: ' + ('yes' if self.newObjects else 'no'))
logs.info('Raw data: ' + ('yes' if self.rawData else 'no'))
logs.info('Interleaved data: ' + ('yes' if self.interleavedData else 'no'))
logs.info('Metadata: ' + ('yes' if self.hasmetadata else 'no'))
logs.info('Object list: ' + ('yes' if self.hasnewObjects else 'no'))
logs.info('Raw data: ' + ('yes' if self.hasrawData else 'no'))
logs.info('Interleaved data: ' + ('yes' if self.hasInterleavedData else 'no'))
logs.info('BigEndian: ' + ('yes' if self.__endian == '<' else 'no'))
logs.info('DAQmx raw data: ' + ('yes' if self.DAQmxRawData else 'no'))
def __iter__(self):
if self.iterate:
return self.__iter_metadata__()
else:
return self.__iter_data__()
def __iter_data__(self):
# Metadata
logs = logging.getLogger('Iterate Data')
# samples = int((self.__segmentOffset - self.__dataOffset)/numChannels/self.__data2mask[datatype][1])
# logs.info('chunkSize: %s bytes' % chunkSize)
# logs.info('Total chunks size: %s' % (self.__segmentOffset - self.__dataOffset))
# logs.info('Length of channel: %d' % ((self.__segmentOffset - self.__dataOffset)/numChannels/self.__data2mask[datatype][1]))
# # New or changed objects
# newObjects = struct.unpack('%cI' % self.__endian, self.__fi.read(4))[0]
#
# # Loop through channels
# for ch in range(1):
# result = self.__readdata(self.__data2mask[datatype][0], self.__data2mask[datatype][1], samples,
# numChannels, channel=ch)
logs.info('DAQmx raw data: ' + ('yes' if self.hasDAQmxRawData else 'no'))
# Dictionary to save the metadata defined in the file
self.metadata = dict()
self.readMetadata()
def __iter_metadata__(self):
def readMetadata(self):
# Metadata
logs = logging.getLogger('Iterate Metadata')
logs = logging.getLogger('Read Metadata')
self.__fi.seek(self.__HEADERLEN, 0)
# Number of objects (unsigned int - 32b)
numObjects = struct.unpack('%cI' % self.__endian, self.__fi.read(4))[0]
logs.info('Number of objects in metadata: %s' % numObjects)
numChannels = 0
chunkSize = 0
# chunkSize = 0
for obj in range(numObjects):
channelSize = 0
# channelSize = 0
objPath = self.__readstring()
logs.debug('Object %s: %s' % (obj, objPath))
result = {'path': objPath}
self.metadata[obj] = {'path': objPath}
rawDataIdx = struct.unpack('%cI' % self.__endian, self.__fi.read(4))[0]
if rawDataIdx == self.__FF32b:
logs.debug('No raw data assigned to this segment')
result['data'] = False
yield self.__readproperties(result)
self.metadata[obj]['data'] = False
self.__readproperties(self.metadata[obj])
continue
elif not rawDataIdx:
logs.debug('Raw data index in this segment matches the index the same object had in the previous segment')
else:
result['data'] = True
self.metadata[obj]['data'] = True
numChannels += 1
# There is raw data!
sizeBytes = None
datatype, arraylen, result['numValues'] = \
struct.unpack('%cIIQ' % self.__endian, self.__fi.read(16))
datatype, arraylen, numValues = struct.unpack('%cIIQ' % self.__endian, self.__fi.read(16))
if datatype == 0x20:
result['sizeBytes'] = struct.unpack('%cQ' % self.__endian, self.__fi.read(8))[0]
self.metadata[obj]['sizeBytes'] = struct.unpack('%cQ' % self.__endian, self.__fi.read(8))[0]
if arraylen != 1:
logs.error('Array length MUST be 1! Actual value: %s' % arraylen)
result['datatype'] = self.__data2mask[datatype][0]
self.metadata[obj]['datatype'] = self.__data2mask[datatype][0]
# logs.debug('obj %s; datatype: %s; numValues: %s; size: %s' % (obj, datatype, result['numValues'],
# self.__data2mask[datatype][1]*result['numValues']))
channelSize = self.__data2mask[datatype][1]*result['numValues']
self.__readproperties()
yield result
# channelSize = self.__data2mask[datatype][1]*numValues
self.__readproperties(self.metadata[obj])
# logs.debug('channelSize: %s bytes' % channelSize)
chunkSize = chunkSize + channelSize
self.samples = int((self.__segmentOffset - self.__dataOffset) / numChannels / self.__data2mask[datatype][1])
logs.info('Samples: %s' % self.samples)
# logs.debug('channelSize: %s bytes' % channelSize)
# chunkSize = chunkSize + channelSize
# logs.info('chunkSize: %s bytes' % chunkSize)
# print(self.__segmentOffset, self.__dataOffset, numChannels, self.__data2mask[datatype][1])
def __iter__(self):
if self.iterate:
return self.__iter_metadata__()
else:
return self.__iter_data__()
def __iter_data__(self):
# Metadata
logs = logging.getLogger('Iterate Data')
logs.info('Total chunks size: %s' % (self.__segmentOffset - self.__dataOffset))
logs.info('Length of channel: %d' % ((self.__segmentOffset - self.__dataOffset)/numChannels/self.__data2mask[datatype][1]))
# New or changed objects
newObjects = struct.unpack('%cI' % self.__endian, self.__fi.read(4))[0]
# Loop through channels
for ch in range(1):
result = self.__readdata(self.__data2mask[datatype][0], self.__data2mask[datatype][1], samples,
numChannels, channel=ch)
def __iter_metadata__(self):
# Metadata
logs = logging.getLogger('Iterate Metadata')
for ch in self.metadata:
yield self.metadata[ch]
# print(self.__segmentOffset, self.__dataOffset, numChannels, self.__data2mask[datatype][1])
def __readstring(self):
# logs = logging.getLogger('readstring')
......@@ -279,7 +298,8 @@ def main():
logs = logging.getLogger('OpenFile')
logs.setLevel(args.loglevel)
for ch in TDMS(args.filename, iterate='M'):
td = TDMS(args.filename, iterate='M')
for ch in td:
print(ch)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment