Commit 8b9d1286 authored by Javier Quinteros's avatar Javier Quinteros

Return data in blocks of no more than 30000 components and other bugfix.

Longer data blocks were giving a segfault.
Bug when end time window was undefined (None). Comparison failing.
parent be5b5a7a
......@@ -91,6 +91,7 @@ class TDMS(object):
# Initialization of local variables
self.__HEADERLEN = 28
self.__MAXSAMPLES = 30000
self.__FF64b = 0xFFFFFFFFFFFFFFFF
self.__FF32b = 0xFFFFFFFF
......@@ -129,9 +130,16 @@ class TDMS(object):
break
else:
raise Exception('Data not available in the specified time window')
elif self.__currentfile >= len(self.__available):
logs.debug('Last file already processed')
# No more data to iterate
raise IndexError
else:
filename = os.path.join(self.__directory, self.__available[self.__currentfile]['name'])
self.__twstart = self.__available[self.__currentfile]['dt']
if (self.__twend is not None) and (self.__twstart > self.__twend):
logs.debug('Start is greater than end. %s %s' % (self.__twstart, self.__twend))
raise IndexError
logs.info('Opening %s; Startime: %s' % (self.__available[self.__currentfile]['name'], self.__twstart))
# Reset some properties before opening the new file
......@@ -275,6 +283,8 @@ class TDMS(object):
self.endtime = self.starttime + datetime.timedelta(seconds=(self.samples-1)/self.sampling_rate)
self.__samplestart = max(floor((self.__twstart - self.starttime).total_seconds() * self.sampling_rate), 0)
self.__samplecur = self.__samplestart
if (self.__twend is None) or (self.__twend >= self.endtime):
self.__sampleend = self.samples-1
else:
......@@ -288,7 +298,7 @@ class TDMS(object):
if self.__chstart >= numChannels:
logs.error('Cannot export from channel %s. Only %s channels present.' % (self.__chstart, numChannels))
raise Exception
raise IndexError
if self.__chstop is None:
self.__chstop = numChannels-1
......@@ -317,21 +327,29 @@ class TDMS(object):
else:
raise Exception('Encoding type not supported to export in MiniSEED!')
while self.__twstart < self.__twend:
while (self.__twend is None) or (self.__twstart < self.__twend):
# Loop through channels
for ch in range(self.__chstart, self.__chstop+1, self.__chstep):
data = self.__readdata(channel=ch)
stats = {'network': 'XX', 'station': '%05d' % ch, 'location': '',
'channel': 'FH1', 'npts': len(data),
'sampling_rate': self.sampling_rate,
'starttime': UTCDateTime(self.__twstart),
'mseed': {'byteorder': self.__endian,
'reclen': 512}}
yield (data, stats)
while self.__samplecur < self.__sampleend:
for ch in range(self.__chstart, self.__chstop+1, self.__chstep):
data = self.__readdata(channel=ch)
stats = {'network': 'XX', 'station': '%05d' % ch, 'location': '',
'channel': 'FH1', 'npts': len(data),
'sampling_rate': self.sampling_rate,
'starttime': UTCDateTime(self.__twstart),
'mseed': {'byteorder': self.__endian,
'reclen': 512}}
logs.debug('Data length: %d; First component: %s' % (len(data), data[0]))
yield (data, stats)
# Read data in blocks of size self.__MAXSAMPLES
self.__samplecur += self.__MAXSAMPLES
# No more data in this file. Skip to the next one.
self.__currentfile += 1
self.__select_file()
try:
self.__select_file()
except IndexError:
return
def __iter_metadata__(self):
# Metadata
......@@ -401,11 +419,13 @@ class TDMS(object):
return result
def __readdata(self, channel=0):
numSamples = self.__sampleend - self.__samplestart + 1
# numSamples = self.__sampleend - self.__samplestart + 1
numSamples = min(self.__sampleend - self.__samplecur + 1, self.__MAXSAMPLES)
if not self.hasInterleavedData:
# Seek where the channel starts and add the offset to the first
# sample to read based in the time window selection
self.__fi.seek(self.__dataOffset + self.datatypesize*self.samples*channel + self.__samplestart, 0)
# self.__fi.seek(self.__dataOffset + self.datatypesize*self.samples*channel + self.__samplestart, 0)
self.__fi.seek(self.__dataOffset + self.datatypesize*self.samples*channel + self.__samplecur, 0)
# Read all selected data from the channel in one step
result = np.fromfile(self.__fi, dtype=self.datatype, count=numSamples)
......@@ -413,7 +433,8 @@ class TDMS(object):
else:
# Seek where the raw data starts and add the offset to the first
# sample to read based in the time window selection
self.__fi.seek(self.__dataOffset + self.__samplestart*self.datatypesize*self.numChannels, 0)
# self.__fi.seek(self.__dataOffset + self.__samplestart*self.datatypesize*self.numChannels, 0)
self.__fi.seek(self.__dataOffset + self.__samplecur*self.datatypesize*self.numChannels, 0)
# Reserve the data for the result
result = np.zeros((numSamples,), dtype=self.datatype)
for ch in range(numSamples):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment