Commit 30e3963b authored by Marius Kriegerowski's avatar Marius Kriegerowski

cleanup, use windowlength in seconds

parent 641bee38
......@@ -48,7 +48,7 @@ class DataGenerator(Object):
fn_stations = String.T()
need_same_dimensions = True # bin mir nicht sicher, ob das sein muss...
effective_deltat = Float.T(optional=True)
n_samples_max = Int.T(default=3500, help='everything longer than this will be dropped')
sample_length = Float.T(help='length [s] of data window')
reference_target = Target.T(
default=Target(
codes=('', 'NKC', '', 'SHZ'),
......@@ -130,16 +130,17 @@ class DataGenerator(Object):
def generate_output_types(self):
return tf.float32, tf.float32
def write(self):
def write(self, directory):
'''Write data to TFRecordDataset using `self.writer`.'''
writer = tf.python_io.TFRecordWriter(directory)
for ydata, label in self.generate():
ex = tf.train.Example(
features=tf.train.Features(
feature={
'data': _BytesFeature(ydata.tobytes()),
'label': _Int64Feature(label),
'label': _BytesFeature(num.array(label).tobytes()),
}))
self.writer.write(ex.SerializeToString())
writer.write(ex.SerializeToString())
class PileData(DataGenerator):
......@@ -154,6 +155,9 @@ class PileData(DataGenerator):
self.data_pile = pile.make_pile(self.data_path, fileformat=self.data_format)
if self.data_pile.is_empty():
sys.exit('Data pile is empty!')
self.deltat_want = self.deltat_want or min(self.data_pile.deltats.keys())
self.n_samples_max = int(self.sample_length/self.deltat_want)
markers = marker.load_markers(self.fn_markers)
marker.associate_phases_to_events(markers)
......@@ -170,7 +174,6 @@ class PileData(DataGenerator):
self.markers = list(markers_by_nsl.values())[0]
self.markers.sort(key=lambda x: x.tmin)
self.deltat_want = self.deltat_want or min(self.data_pile.deltats.keys())
self.channels = list(self.data_pile.nslc_ids.keys())
self.tensor_shape = (len(self.channels), self.n_samples_max)
......@@ -235,11 +238,13 @@ class OnTheFlyData(DataGenerator):
stations = load_stations(self.fn_stations)
self.targets = synthi.guess_targets_from_stations(
stations, quantity=self.quantity)
self.store = self.gf_engine.get_store() # uses default store
self.n_samples_max = int(self.sample_length/self.store.config.deltat)
self.tensor_shape = (len(self.targets), self.n_samples_max)
def make_data_chunk(self, source, results, store):
def make_data_chunk(self, source, results):
ydata_stacked = self.get_raw_data_chunk()
tref = store.t(
tref = self.store.t(
self.onset_phase, (
source.depth,
source.distance_to(self.reference_target))
......@@ -268,21 +273,7 @@ class OnTheFlyData(DataGenerator):
sources=sources,
targets=self.targets)
store = self.gf_engine.get_store() # uses default store
for isource, source in enumerate(response.request.sources):
chunk = self.make_data_chunk(source, response.results_list[isource], store)
chunk = self.make_data_chunk(source, response.results_list[isource])
yield chunk, self.extract_labels(source)
def write(self, directory):
'''Write data to TFRecordDataset using `self.writer`.'''
writer = tf.python_io.TFRecordWriter(directory)
for ydata, label in self.generate():
ex = tf.train.Example(
features=tf.train.Features(
feature={
'data': _BytesFeature(ydata.tobytes()),
'label': _BytesFeature(num.array(label).tobytes()),
}))
writer.write(ex.SerializeToString())
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment