cleanup, use windowlength in seconds

pinky/src/data.py

@@ -48,7 +48,7 @@ class DataGenerator(Object):
     fn_stations = String.T()
     need_same_dimensions = True # bin mir nicht sicher, ob das sein muss...
     effective_deltat = Float.T(optional=True)
-    n_samples_max = Int.T(default=3500, help='everything longer than this will be dropped')
+    sample_length = Float.T(help='length [s] of data window')
     reference_target = Target.T(
         default=Target(
             codes=('', 'NKC', '', 'SHZ'),
@@ -130,16 +130,17 @@ class DataGenerator(Object):
     def generate_output_types(self):
         return tf.float32, tf.float32
 
-    def write(self):
+    def write(self, directory):
         '''Write data to TFRecordDataset using `self.writer`.'''
+        writer = tf.python_io.TFRecordWriter(directory)
         for ydata, label in self.generate():
             ex = tf.train.Example(
                 features=tf.train.Features(
                     feature={
                         'data': _BytesFeature(ydata.tobytes()),
-                        'label': _Int64Feature(label),
+                        'label': _BytesFeature(num.array(label).tobytes()),
                     }))
-            self.writer.write(ex.SerializeToString())
+            writer.write(ex.SerializeToString())
 
 
 class PileData(DataGenerator):
@@ -154,6 +155,9 @@ class PileData(DataGenerator):
         self.data_pile = pile.make_pile(self.data_path, fileformat=self.data_format)
         if self.data_pile.is_empty():
             sys.exit('Data pile is empty!')
+
+        self.deltat_want = self.deltat_want or min(self.data_pile.deltats.keys())
+        self.n_samples_max = int(self.sample_length/self.deltat_want)
         markers = marker.load_markers(self.fn_markers)
         marker.associate_phases_to_events(markers)
 
@@ -170,7 +174,6 @@ class PileData(DataGenerator):
         self.markers = list(markers_by_nsl.values())[0]
         self.markers.sort(key=lambda x: x.tmin)
 
-        self.deltat_want = self.deltat_want or min(self.data_pile.deltats.keys())
         self.channels = list(self.data_pile.nslc_ids.keys())
         self.tensor_shape = (len(self.channels), self.n_samples_max)
 
@@ -235,11 +238,13 @@ class OnTheFlyData(DataGenerator):
         stations = load_stations(self.fn_stations)
         self.targets = synthi.guess_targets_from_stations(
             stations, quantity=self.quantity)
+        self.store = self.gf_engine.get_store()  # uses default store
+        self.n_samples_max = int(self.sample_length/self.store.config.deltat)
         self.tensor_shape = (len(self.targets), self.n_samples_max)
 
-    def make_data_chunk(self, source, results, store):
+    def make_data_chunk(self, source, results):
         ydata_stacked = self.get_raw_data_chunk()
-        tref = store.t(
+        tref = self.store.t(
             self.onset_phase, (
                 source.depth,
                 source.distance_to(self.reference_target))
@@ -268,21 +273,7 @@ class OnTheFlyData(DataGenerator):
             sources=sources,
             targets=self.targets)
 
-        store = self.gf_engine.get_store()  # uses default store
-
         for isource, source in enumerate(response.request.sources):
-            chunk = self.make_data_chunk(source, response.results_list[isource], store)
+            chunk = self.make_data_chunk(source, response.results_list[isource])
             yield chunk, self.extract_labels(source)
 
-    def write(self, directory):
-        '''Write data to TFRecordDataset using `self.writer`.'''
-        writer = tf.python_io.TFRecordWriter(directory)
-        for ydata, label in self.generate():
-            ex = tf.train.Example(
-                features=tf.train.Features(
-                    feature={
-                        'data': _BytesFeature(ydata.tobytes()),
-                        'label': _BytesFeature(num.array(label).tobytes()),
-                    }))
-            writer.write(ex.SerializeToString())
-