Commit fd5b7803 authored by Javier Quinteros's avatar Javier Quinteros

Major refactor the TDMS class

Only the "with t:" functionality remains. To be tested...
parent d5ee944d
......@@ -28,6 +28,18 @@ class TDMS(object):
if self.__fi is not None:
self.__fi.close()
def __enter__(self):
"""Method which allows to use the syntax 'with object:' and use it inside"""
# self.__select_file()
# Create a buffer space to store the signal coefficients to be
# convoluted during the decimation
for channel in range(self.__chstart, self.__chstop + 1, self.__chstep):
logging.debug('Create empty buffers')
self.__buffer[channel] = None
return self
def __init__(self, filename, directory='.', chstart=0, chstop=None, chstep=1,
starttime=None, endtime=None, iterate='D', decimate=1, firfilter='fir235',
loglevel='INFO'):
......@@ -56,22 +68,23 @@ class TDMS(object):
:param loglevel: Verbosity in the output
:type loglevel: str
"""
# Log level
self.__loglevel = loglevel
logs = logging.getLogger('Init TDMS')
logs.setLevel(loglevel)
PROJECT_DIR = os.path.dirname(__file__)
project_dir = os.path.dirname(__file__)
# Decimation factor
self.__decimate = decimate
# Log level
self.__loglevel = loglevel
# Channel from and to
self.__chstart = chstart
self.__chstop = chstop
self.__chstep = chstep
# Timewindow selection
# Time window selection
self.__twstart = starttime
self.__twend = endtime
......@@ -87,8 +100,11 @@ class TDMS(object):
# Name of file
self.__filename = filename
# FIXME directory is not properly used and is giving an Error if !='.'
self.__directory = directory
# List of available files in the directory
self.__available = list()
for file in sorted(os.listdir(directory)):
......@@ -100,6 +116,7 @@ class TDMS(object):
if self.__twstart is None:
self.__twstart = dt
# Set the time window selection to the minimum datetime found
if self.__twstart < self.__available[0]['dt']:
self.__twstart = self.__available[0]['dt']
......@@ -149,8 +166,9 @@ class TDMS(object):
0x44: ('Qq', 16) # tdsTypeTimeStamp
}
# Read filter to decimate
auxfilter = list()
with open(os.path.join(PROJECT_DIR, 'data/filters/%s.txt' % firfilter)) as fin:
with open(os.path.join(project_dir, 'data/filters/%s.txt' % firfilter)) as fin:
for line in fin.readlines():
auxfilter.append(float(line))
......@@ -161,6 +179,19 @@ class TDMS(object):
# tdsTypeComplexDoubleFloat = 0x10000d,
# tdsTypeDAQmxRawData = 0xFFFFFFFF
# Other variables to be read from the headers
self.__hasInterleavedData = None
self.__endian = None
self.__datatype = None
self.__datatypesize = None
self.numchannels = None
self.__samples = None
self.__samplestart = None
self.__sampleend = None
self.__samplecur = None
self.__select_file()
def __select_file(self):
logs = logging.getLogger('Select file')
logs.setLevel(self.__loglevel)
......@@ -192,8 +223,10 @@ class TDMS(object):
self.endtime = None
self.metadata = dict()
# Save the handle of the open file to be processed
self.__fi = open(filename, 'rb')
# Read headers
leadin = self.__fi.read(self.__HEADERLEN)
(tag, ToCmask) = struct.unpack('<4si', leadin[:8])
......@@ -204,11 +237,15 @@ class TDMS(object):
kTocBigEndian = 1 << 6
kTocDAQmxRawData = 1 << 7
self.hasmetadata = bool(ToCmask & kTocMetaData)
self.hasnewObjects = bool(ToCmask & kTocNewObjList)
self.hasrawData = bool(ToCmask & kTocRawData)
self.hasInterleavedData = bool(ToCmask & kTocInterleavedData)
self.hasDAQmxRawData = bool(ToCmask & kTocDAQmxRawData)
# self.hasmetadata = bool(ToCmask & kTocMetaData)
hasmetadata = bool(ToCmask & kTocMetaData)
# self.hasnewObjects = bool(ToCmask & kTocNewObjList)
hasnewObjects = bool(ToCmask & kTocNewObjList)
# self.hasrawData = bool(ToCmask & kTocRawData)
hasrawData = bool(ToCmask & kTocRawData)
self.__hasInterleavedData = bool(ToCmask & kTocInterleavedData)
# self.hasDAQmxRawData = bool(ToCmask & kTocDAQmxRawData)
hasDAQmxRawData = bool(ToCmask & kTocDAQmxRawData)
# All input from now on will be formatted by this
self.__endian = '>' if ToCmask & kTocBigEndian else '<'
......@@ -227,44 +264,32 @@ class TDMS(object):
if self.__segmentOffset == self.__FF64b:
logs.error('Severe problem while writing data (crash, power outage)')
if self.hasmetadata and not self.__dataOffset:
if hasmetadata and not self.__dataOffset:
logs.error('Flag indicates Metadata but its length is 0!')
if self.hasDAQmxRawData:
if hasDAQmxRawData:
logs.warning('DAQmx raw data is still not supported!')
# Absolute offsets
self.__segmentOffset += self.__HEADERLEN
self.__dataOffset += self.__HEADERLEN
logs.debug('Metadata: ' + ('yes' if self.hasmetadata else 'no'))
logs.debug('Object list: ' + ('yes' if self.hasnewObjects else 'no'))
logs.debug('Raw data: ' + ('yes' if self.hasrawData else 'no'))
logs.debug('Interleaved data: ' + ('yes' if self.hasInterleavedData else 'no'))
logs.debug('Metadata: ' + ('yes' if hasmetadata else 'no'))
logs.debug('Object list: ' + ('yes' if hasnewObjects else 'no'))
logs.debug('Raw data: ' + ('yes' if hasrawData else 'no'))
logs.debug('Interleaved data: ' + ('yes' if self.__hasInterleavedData else 'no'))
logs.debug('BigEndian: ' + ('yes' if self.__endian == '<' else 'no'))
logs.debug('DAQmx raw data: ' + ('yes' if self.hasDAQmxRawData else 'no'))
logs.debug('DAQmx raw data: ' + ('yes' if hasDAQmxRawData else 'no'))
self.readMetadata()
self.__readmetadata()
def resetcurrenttime(self):
def __resetcurrenttime(self):
self.__twstart = self.__origstarttime
self.__twend = self.__origendtime
self.__currentfile = None
self.__select_file()
def __enter__(self):
"""Method which allows to use the syntax 'with object:' and use it inside"""
self.__select_file()
# Create a buffer space to store the signal coefficients to be
# convoluted during the decimation
for channel in range(self.__chstart, self.__chstop + 1, self.__chstep):
logging.debug('Create empty buffers')
self.__buffer[channel] = None
return self
def readMetadata(self):
def __readmetadata(self):
# Metadata
logs = logging.getLogger('Read Metadata')
# handler = logging.StreamHandler(sys.stdout)
......@@ -272,22 +297,23 @@ class TDMS(object):
self.__fi.seek(self.__HEADERLEN, 0)
# Number of objects (unsigned int - 32b)
numObjects = struct.unpack('%cI' % self.__endian, self.__fi.read(4))[0]
logs.debug('Number of objects in metadata: %s' % numObjects)
numobjects = struct.unpack('%cI' % self.__endian, self.__fi.read(4))[0]
logs.debug('Number of objects in metadata: %s' % numobjects)
numChannels = 0
datatype = None
numchannels = 0
# chunkSize = 0
for obj in range(numObjects):
for obj in range(numobjects):
# channelSize = 0
objPath = self.__readstring()
objpath = self.__readstring()
# logs.debug('Object %s: %s' % (obj, objPath))
self.metadata[obj] = {'path': objPath}
self.metadata[obj] = {'path': objpath}
rawDataIdx = struct.unpack('%cI' % self.__endian, self.__fi.read(4))[0]
rawdataidx = struct.unpack('%cI' % self.__endian, self.__fi.read(4))[0]
if rawDataIdx == self.__FF32b:
logs.debug('No raw data assigned to this segment')
if rawdataidx == self.__FF32b:
logs.debug('No raw data assigned to segment %s' % obj)
self.metadata[obj]['data'] = False
self.__readproperties(self.metadata[obj])
......@@ -305,18 +331,18 @@ class TDMS(object):
continue
elif not rawDataIdx:
elif not rawdataidx:
logs.debug('Raw data index in this segment matches the index the same object had in the previous '
'segment')
else:
self.metadata[obj]['data'] = True
self.metadata[obj]['id'] = numChannels
numChannels += 1
self.metadata[obj]['id'] = numchannels
numchannels += 1
# There is raw data!
sizeBytes = None
datatype, arraylen, numValues = struct.unpack('%cIIQ' % self.__endian, self.__fi.read(16))
# sizeBytes = None
datatype, arraylen, numvalues = struct.unpack('%cIIQ' % self.__endian, self.__fi.read(16))
if datatype == 0x20:
self.metadata[obj]['sizeBytes'] = struct.unpack('%cQ' % self.__endian, self.__fi.read(8))[0]
......@@ -331,20 +357,24 @@ class TDMS(object):
self.__readproperties(self.metadata[obj])
# Set the data type as numpy expects it
if datatype is None:
raise Exception('datatype definition not found in any channel!')
if self.__data2mask[datatype][0] == 'h':
self.datatype = '%ci2' % self.__endian
self.__datatype = '%ci2' % self.__endian
elif self.__data2mask[datatype][0] == 'f':
self.datatype = '%cf4' % self.__endian
self.__datatype = '%cf4' % self.__endian
else:
raise Exception('Data type not supported! (%s)' % self.__data2mask[datatype][0])
self.datatypesize = self.__data2mask[datatype][1]
self.numChannels = numChannels
self.samples = int((self.__segmentOffset - self.__dataOffset) / numChannels / self.datatypesize)
self.__datatypesize = self.__data2mask[datatype][1]
self.numchannels = numchannels
self.__samples = int((self.__segmentOffset - self.__dataOffset) / numchannels / self.__datatypesize)
# Calculate endtime based on the number of samples declared and the sampling rate
self.endtime = self.starttime + datetime.timedelta(seconds=(self.samples-1)/self.sampling_rate)
self.endtime = self.starttime + datetime.timedelta(seconds=(self.__samples-1)/self.sampling_rate)
# Sample to start extraction from based on the initial datetime of the file (__twstart)
self.__samplestart = max(floor((self.__twstart - self.starttime).total_seconds() * self.sampling_rate), 0)
# Should I readjust __twstart to align it exactly with the time of the samples?
# print(self.__twstart, self.starttime + datetime.timedelta(seconds=self.__samplestart/self.sampling_rate))
......@@ -352,26 +382,29 @@ class TDMS(object):
self.__samplecur = self.__samplestart
# Open end or beyond this file: read until the last sample
if (self.__twend is None) or (self.__twend >= self.endtime):
self.__sampleend = self.samples-1
self.__sampleend = self.__samples-1
else:
# Otherwise calculate which one is the last sample to read
self.__sampleend = ceil((self.__twend - self.starttime).total_seconds() * self.sampling_rate)
# print(self.__twend, self.starttime, (self.__twend - self.starttime).total_seconds(), self.__sampleend)
logs.debug('Samples: %s' % self.samples)
logs.debug('Samples: %s' % self.__samples)
logs.debug('Samples selected: %s-%s' % (self.__samplestart, self.__sampleend))
logs.debug('Total chunks size: %s' % (self.__segmentOffset - self.__dataOffset))
logs.debug('Length of channel: %d' % ((self.__segmentOffset - self.__dataOffset)/numChannels/self.__data2mask[datatype][1]))
channellength = ((self.__segmentOffset - self.__dataOffset)/numchannels/self.__data2mask[datatype][1])
logs.debug('Length of channel: %d' % channellength)
if self.__chstart >= numChannels:
logs.error('Cannot export from channel %s. Only %s channels present.' % (self.__chstart, numChannels))
if self.__chstart >= numchannels:
logs.error('Cannot export from channel %s. Only %s channels present.' % (self.__chstart, numchannels))
raise IndexError
if self.__chstop is None:
self.__chstop = numChannels-1
elif self.__chstop >= numChannels:
logs.warning('Resetting chstart to %s' % (numChannels-1))
self.__chstop = numChannels-1
self.__chstop = numchannels-1
elif self.__chstop >= numchannels:
logs.warning('Resetting chstop to %s' % (numchannels-1))
self.__chstop = numchannels-1
# New or changed objects
newObjects = struct.unpack('%cI' % self.__endian, self.__fi.read(4))[0]
......@@ -514,9 +547,9 @@ class TDMS(object):
logs = logging.getLogger('Iterate Data')
# Data is stored with values from one channel in a continuous array
if not self.hasInterleavedData:
if not self.__hasInterleavedData:
for ch in range(self.__chstart, self.__chstop + 1, self.__chstep):
self.resetcurrenttime()
self.__resetcurrenttime()
while (self.__twend is None) or (self.__twstart < self.__twend):
# Loop through channels
......@@ -665,29 +698,29 @@ class TDMS(object):
numsamples = self.__sampleend - self.__samplestart + 1
# numSamples = min(self.__sampleend - self.__samplecur + 1, self.__MAXSAMPLES)
if not self.hasInterleavedData:
if not self.__hasInterleavedData:
for ch in channels:
# Seek where the channel starts and add the offset to the first
# sample to read based in the time window selection
# self.__fi.seek(self.__dataOffset + self.datatypesize*self.samples*channel + self.__samplestart, 0)
self.__fi.seek(self.__dataOffset + self.datatypesize*self.samples*ch + self.__samplecur, 0)
# self.__fi.seek(self.__dataOffset + self.__datatypesize*self.__samples*channel + self.__samplestart, 0)
self.__fi.seek(self.__dataOffset + self.__datatypesize*self.__samples*ch + self.__samplecur, 0)
# Read all selected data from the channel in one step
result[ch] = np.fromfile(self.__fi, dtype=self.datatype, count=numsamples)
result[ch] = np.fromfile(self.__fi, dtype=self.__datatype, count=numsamples)
return result
# Seek where the raw data starts and add the offset to the first
# sample to read based in the time window selection
# self.__fi.seek(self.__dataOffset + self.__samplestart*self.datatypesize*self.numChannels, 0)
self.__fi.seek(self.__dataOffset + self.__samplecur*self.datatypesize*self.numChannels, 0)
# self.__fi.seek(self.__dataOffset + self.__samplestart*self.__datatypesize*self.numchannels, 0)
self.__fi.seek(self.__dataOffset + self.__samplecur*self.__datatypesize*self.numchannels, 0)
# Reserve the data for the result
for ch in channels:
result[ch] = np.zeros((numsamples,), dtype=self.datatype)
result[ch] = np.zeros((numsamples,), dtype=self.__datatype)
for sample in range(numsamples):
# Read from all channels and select the specific one with an index (channel)
allchannels = np.fromfile(self.__fi, dtype=self.datatype, count=self.numChannels)
allchannels = np.fromfile(self.__fi, dtype=self.__datatype, count=self.numchannels)
for ch in channels:
result[ch][sample] = allchannels[ch]
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment