I would like to transfer a parameter from a callback function to a custom regularizer function.
The follwing callback function will calculate a regularizer value from a confusion matrix:
class call_evaluator(keras.callbacks.Callback):
def __init__(self):
self.regularizer = sym_regularizer()
def on_epoch_end(self, batch, logs=None):
y_pred = tf.cast(tf.math.argmax(model.predict(x_train), axis=1), tf.float32)
y_true = np.argmax(y_train, axis=1)
con_mat = tf.math.confusion_matrix(y_pred, y_true)
diag_sum = tf.linalg.trace(con_mat)
mat_sum = tf.reduce_sum(con_mat)
buffer = tf.math.sqrt(mat_sum / diag_sum)
buffer = buffer.numpy()
self.regularizer.set_penalty(buffer)
The calculated value is used in the following regularizer function:
class sym_regularizer(regularizers.Regularizer):
def __init__(self, sym_penalty=10.0):
# with K.name_scope(self.__class__.__name__):
# self.sym_penalty = K.variable(sym_penalty, name='sym_penalty')
# self.val_sym_penalty = sym_penalty
self.sym_penalty = K.variable(sym_penalty, name='sym_penalty')
self.val_sym_penalty = sym_penalty
def set_penalty(self, sym_penalty):
K.set_value(self.sym_penalty, sym_penalty)
self.val_sym_penalty = sym_penalty
tf.print("self.val_sym_penalty = ", self.val_sym_penalty)
def __call__(self, weights):
regularization = 0
regularization += K.sum(1e-3 * K.square(weights)) + self.val_sym_penalty
return regularization
def get_config(self):
return {'sym_penalty': float(K.get_value(self.sym_penalty))}
The model i use looks like this:
model = tf.keras.models.Sequential()
model.add(tf.keras.layers.Dense(30,
kernel_regularizer=sym_regularizer(),
activation=tf.nn.tanh,
input_shape=(x_train.shape[1],)))
model.add(tf.keras.layers.Dense(10,
kernel_regularizer=sym_regularizer(),
activation=tf.nn.tanh))
model.add(tf.keras.layers.Dense(4,
kernel_regularizer=sym_regularizer(),
activation=tf.nn.softmax))
optimizer = tf.keras.optimizers.SGD(learning_rate=1e-3)
model.compile(loss='categorical_crossentropy', metrics=['accuracy'])
model.fit(x_train,
y_train,
batch_size=100000,
epochs=100,
verbose=1,
callbacks=[call_evaluator()])
The code will successfully access the update set_penalty(self, sym_penalty) in the regularizer function and set the sym_penalty variable. Unfortunately the value will not be updated in the _ call _ part of the function.
The code is based on the following sources:
Keras with activity_regularizer that is updated every iteration
Change keras regularizer during training / dynamic regularization
I'm not able to find the error and i have problems to fully understand the code itself.
