Commit 8d5f44a0 authored by Marius Kriegerowski's avatar Marius Kriegerowski

fix hp optimizer

parent 454cac54
from .data import *
from .tf_util import *
from .util import delete_if_exists
from .optimize import Optimizer
import tensorflow as tf
......@@ -13,18 +14,6 @@ import shutil
logger = logging.getLogger('pinky.model')
def delete_if_exists(dirname):
if os.path.exists(dirname):
logger.info('deleting directory: %s' % dirname)
shutil.rmtree(dirname)
def unlist_dict(d):
'''horrible hack to make the strange scikit-optimize list-of-list shit work
'''
for k, v in d.items():
if isinstance(v, list) and len(v) == 1:
d[k] = v[0]
class Model(Object):
......@@ -32,6 +21,7 @@ class Model(Object):
data_generator = DataGeneratorBase.T()
dropout_rate = Float.T(optional=True)
batch_size = Int.T(default=10)
n_epochs = Int.T(default=1)
outdir = String.T(default='/tmp/dnn-seis')
auto_clear = Bool.T(default=True)
summary_outdir= String.T(default='summary')
......@@ -39,8 +29,7 @@ class Model(Object):
shuffle_size = Int.T(
optional=True, help='if set, shuffle examples at given buffer size.')
def __init__(self, tf_config=None, debug=False, **kwargs):
print(kwargs)
def __init__(self, tf_config=None, **kwargs):
super().__init__(**kwargs)
if self.auto_clear:
......@@ -48,7 +37,7 @@ class Model(Object):
delete_if_exists(self.outdir)
self.tf_config = tf_config
self.debug = debug
self.debug = logger.getEffectiveLevel() == logging.DEBUG
self.sess = tf.Session(config=tf_config)
# initializer = tf.truncated_normal_initializer(
......@@ -57,11 +46,14 @@ class Model(Object):
mean=0.0, stddev=0.1)
def generate_input(self):
'''
Generates data and labels
'''
dataset = self.data_generator.get_dataset()
dataset = dataset.batch(self.batch_size)
if self.shuffle_size:
dataset = dataset.shuffle(buffer_size=self.shuffle_size)
# dataset = dataset.repeat()
dataset = dataset.repeat(count=self.n_epochs)
dataset = dataset.prefetch(buffer_size=self.batch_size)
return dataset.make_one_shot_iterator().get_next()
......@@ -71,8 +63,9 @@ class Model(Object):
'''
CNN along horizontal axis
:param n_filters: number of filters
:param cross_channel_kernel: convolution kernel size accross channels
:param n_filters:
:param kernel_width: convolution kernel size accross time axis
(Needs some debugging and checking)
'''
......@@ -82,34 +75,33 @@ class Model(Object):
cross_channel_kernel = n_channels
with tf.variable_scope('conv_layer%s' %name):
input = tf.layers.conv2d(
inputs=input,
filters=n_filters,
kernel_size=(cross_channel_kernel, kernel_width), # use identity (1) along channels
activation=tf.nn.relu,
bias_initializer=self.initializer,
name=name)
name=name+'conv2d')
input = tf.layers.batch_normalization(input, training=training)
input = tf.layers.max_pooling2d(
input,
pool_size=(2, 2), # (height, width)
strides=(1, 1),
pool_size=(cross_channel_kernel, kernel_width), # (height, width)
strides=(1, 2),
name=name+'max_pooling2d',
)
if self.debug:
# super expensive!!
logging.warn('Debug mode enables super expensive summaries.')
tf.summary.image(
'post-%s' % name, tf.split(
input, num_or_size_splits=n_filters, axis=-1)[0])
variable_summaries(input, name)
return input
def model(self, features, labels, mode, params):
unlist_dict(params)
training = bool(mode == tf.estimator.ModeKeys.TRAIN)
n_filters = 32
......@@ -118,9 +110,10 @@ class Model(Object):
# tf.summary.image('input', features)
# conv = self.time_axis_cnn(features, n_filters, None, kernel_width=3, name='conv1',
# training=training)
conv = self.time_axis_cnn(features, n_filters, 1, kernel_width=1, name='conv1',
training=training)
conv = self.time_axis_cnn(conv, n_filters*2, 1, kernel_width=1, name='conv2',
conv = self.time_axis_cnn(conv, n_filters*2, 1, kernel_width=2, name='conv2',
training=training)
conv = self.time_axis_cnn(conv, n_filters*4, 2, kernel_width=1, name='conv3',
training=training)
......@@ -177,7 +170,6 @@ class Model(Object):
)
def train(self, params=None):
print(params)
params = params or {}
with self.sess as default:
......
from .util import delete_if_exists
from skopt import gp_minimize
from skopt.space import Real, Categorical, Integer
from pyrocko.guts import Object, Int, Float, List, Tuple, String
def to_skopt_real(x, name, prior):
return Real(low=x[0], high=x[1], prior=prior, name=name)
......@@ -11,7 +11,8 @@ def to_skopt_real(x, name, prior):
class Optimizer(Object):
learning_rate = Tuple.T(3, Float.T(), default=(1e-3, 1e-5, 1e-4)) # low, high, default
n_calls = Int.T(default=50)
n_calls = Int.T(default=50, help='number of test sets')
log_path = String.T(default='./logs/')
path_best = String.T(default='winner')
def __init__(self, **kwargs):
......@@ -26,15 +27,19 @@ class Optimizer(Object):
print(self.dimensions)
def evaluate(self, *args):
def evaluate(self, args):
''' wrapper to parse gp_minimize args to model.train'''
print(args)
args = dict(zip(['learning_rate',], args))
self.model.outdir = self.log_dir_name(args)
print(args)
return self.model.train(args)['loss']
def optimize(self, model):
self.model = model
if self.model.auto_clear:
delete_if_exists(self.log_path)
default_parameters = [self.learning_rate[-1]]
gp_minimize(
......@@ -45,13 +50,17 @@ class Optimizer(Object):
x0=default_parameters,
)
def log_dir_name(self, learning_rate):
def log_dir_name(self, params):
# The dir-name for the TensorBoard log-dir.
s = "./logs/lr_{0:.0e}_layers"
# Insert all the hyper-parameters in the dir-name.
log_dir = s.format(learning_rate)
placeholders = '{}_{}_' * len(params)
identifiers = []
for k, v in params.items():
identifiers.append(k[0:3])
identifiers.append(v)
placeholders = placeholders.format(*identifiers)
log_dir = self.log_path + placeholders
return log_dir
......
import os
import shutil
import logging
logger = logging.getLogger()
def delete_if_exists(dirname):
if os.path.exists(dirname):
logger.info('deleting directory: %s' % dirname)
shutil.rmtree(dirname)
def nsl(tr):
return tr.nslc_id[:3]
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment