Hands-on 08: Model Compression#
! pip install --user --quiet tensorflow-model-optimization
from tensorflow.keras.utils import to_categorical
from sklearn.datasets import fetch_openml
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder, StandardScaler
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
seed = 42
np.random.seed(seed)
import tensorflow as tf
tf.random.set_seed(seed)
2024-03-31 18:32:06.398355: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: SSE4.1 SSE4.2 AVX AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
Fetch the jet tagging dataset from Open ML#
data = fetch_openml("hls4ml_lhc_jets_hlf", parser="auto")
X, y = data["data"], data["target"]
le = LabelEncoder()
y_onehot = le.fit_transform(y)
y_onehot = to_categorical(y_onehot, 5)
classes = le.classes_
X_train_val, X_test, y_train_val, y_test = train_test_split(X, y_onehot, test_size=0.2, random_state=42)
scaler = StandardScaler()
X_train_val = scaler.fit_transform(X_train_val)
X_test = scaler.transform(X_test)
Now construct a model#
We’ll use the same architecture as in part 1: 3 hidden layers with 64, then 32, then 32 neurons. Each layer will use relu activation.
Add an output layer with 5 neurons (one for each class), then finish with Softmax activation.
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Activation, BatchNormalization
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.regularizers import l1
from callbacks import all_callbacks
model = Sequential()
model.add(Dense(64, input_shape=(16,), name="fc1", kernel_initializer="lecun_uniform"))
model.add(Activation(activation="relu", name="relu1"))
model.add(Dense(32, name="fc2", kernel_initializer="lecun_uniform"))
model.add(Activation(activation="relu", name="relu2"))
model.add(Dense(32, name="fc3", kernel_initializer="lecun_uniform"))
model.add(Activation(activation="relu", name="relu3"))
model.add(Dense(5, name="output", kernel_initializer="lecun_uniform"))
model.add(Activation(activation="softmax", name="softmax"))
Train the unpruned model#
adam = Adam(learning_rate=0.0001)
model.compile(optimizer=adam, loss=["categorical_crossentropy"], metrics=["accuracy"])
callbacks = all_callbacks(
stop_patience=1000,
lr_factor=0.5,
lr_patience=10,
lr_epsilon=0.000001,
lr_cooldown=2,
lr_minimum=0.0000001,
outputDir="unpruned_model",
)
model.fit(
X_train_val, y_train_val, batch_size=1024, epochs=30, validation_split=0.25, shuffle=True, callbacks=callbacks.callbacks
)
Epoch 1/30
1/487 [..............................] - ETA: 3:55 - loss: 1.7058 - accuracy: 0.2178
28/487 [>.............................] - ETA: 0s - loss: 1.6825 - accuracy: 0.2551
55/487 [==>...........................] - ETA: 0s - loss: 1.6401 - accuracy: 0.2813
83/487 [====>.........................] - ETA: 0s - loss: 1.6013 - accuracy: 0.3059
111/487 [=====>........................] - ETA: 0s - loss: 1.5680 - accuracy: 0.3293
139/487 [=======>......................] - ETA: 0s - loss: 1.5405 - accuracy: 0.3496
168/487 [=========>....................] - ETA: 0s - loss: 1.5149 - accuracy: 0.3689
197/487 [===========>..................] - ETA: 0s - loss: 1.4926 - accuracy: 0.3859
225/487 [============>.................] - ETA: 0s - loss: 1.4721 - accuracy: 0.4007
253/487 [==============>...............] - ETA: 0s - loss: 1.4518 - accuracy: 0.4134
282/487 [================>.............] - ETA: 0s - loss: 1.4333 - accuracy: 0.4240
311/487 [==================>...........] - ETA: 0s - loss: 1.4148 - accuracy: 0.4343
340/487 [===================>..........] - ETA: 0s - loss: 1.3974 - accuracy: 0.4429
369/487 [=====================>........] - ETA: 0s - loss: 1.3813 - accuracy: 0.4503
398/487 [=======================>......] - ETA: 0s - loss: 1.3660 - accuracy: 0.4570
426/487 [=========================>....] - ETA: 0s - loss: 1.3519 - accuracy: 0.4629
455/487 [===========================>..] - ETA: 0s - loss: 1.3384 - accuracy: 0.4686
483/487 [============================>.] - ETA: 0s - loss: 1.3259 - accuracy: 0.4741
Epoch 1: val_loss improved from inf to 1.11796, saving model to unpruned_model/model_best.h5
Epoch 1: saving model to unpruned_model/model_last.h5
487/487 [==============================] - 2s 2ms/step - loss: 1.3246 - accuracy: 0.4747 - val_loss: 1.1180 - val_accuracy: 0.5692 - lr: 1.0000e-04
Epoch 2/30
1/487 [..............................] - ETA: 1s - loss: 1.1151 - accuracy: 0.5352
29/487 [>.............................] - ETA: 0s - loss: 1.1074 - accuracy: 0.5723
57/487 [==>...........................] - ETA: 0s - loss: 1.1041 - accuracy: 0.5819
86/487 [====>.........................] - ETA: 0s - loss: 1.0992 - accuracy: 0.5872
/home/runner/miniconda3/envs/phys139/lib/python3.10/site-packages/keras/src/engine/training.py:3103: UserWarning: You are saving your model as an HDF5 file via `model.save()`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')`.
saving_api.save_model(
114/487 [======>.......................] - ETA: 0s - loss: 1.0953 - accuracy: 0.5919
142/487 [=======>......................] - ETA: 0s - loss: 1.0899 - accuracy: 0.5959
170/487 [=========>....................] - ETA: 0s - loss: 1.0846 - accuracy: 0.5992
199/487 [===========>..................] - ETA: 0s - loss: 1.0803 - accuracy: 0.6023
227/487 [============>.................] - ETA: 0s - loss: 1.0758 - accuracy: 0.6051
255/487 [==============>...............] - ETA: 0s - loss: 1.0710 - accuracy: 0.6079
283/487 [================>.............] - ETA: 0s - loss: 1.0672 - accuracy: 0.6101
311/487 [==================>...........] - ETA: 0s - loss: 1.0629 - accuracy: 0.6128
339/487 [===================>..........] - ETA: 0s - loss: 1.0592 - accuracy: 0.6149
368/487 [=====================>........] - ETA: 0s - loss: 1.0547 - accuracy: 0.6175
397/487 [=======================>......] - ETA: 0s - loss: 1.0513 - accuracy: 0.6194
425/487 [=========================>....] - ETA: 0s - loss: 1.0470 - accuracy: 0.6220
454/487 [==========================>...] - ETA: 0s - loss: 1.0430 - accuracy: 0.6242
483/487 [============================>.] - ETA: 0s - loss: 1.0393 - accuracy: 0.6265
Epoch 2: val_loss improved from 1.11796 to 0.97694, saving model to unpruned_model/model_best.h5
Epoch 2: saving model to unpruned_model/model_last.h5
487/487 [==============================] - 1s 2ms/step - loss: 1.0388 - accuracy: 0.6269 - val_loss: 0.9769 - val_accuracy: 0.6629 - lr: 1.0000e-04
Epoch 3/30
1/487 [..............................] - ETA: 1s - loss: 0.9408 - accuracy: 0.6855
30/487 [>.............................] - ETA: 0s - loss: 0.9691 - accuracy: 0.6671
59/487 [==>...........................] - ETA: 0s - loss: 0.9645 - accuracy: 0.6689
87/487 [====>.........................] - ETA: 0s - loss: 0.9620 - accuracy: 0.6696
116/487 [======>.......................] - ETA: 0s - loss: 0.9583 - accuracy: 0.6716
145/487 [=======>......................] - ETA: 0s - loss: 0.9571 - accuracy: 0.6722
174/487 [=========>....................] - ETA: 0s - loss: 0.9556 - accuracy: 0.6731
203/487 [===========>..................] - ETA: 0s - loss: 0.9535 - accuracy: 0.6739
232/487 [=============>................] - ETA: 0s - loss: 0.9505 - accuracy: 0.6754
261/487 [===============>..............] - ETA: 0s - loss: 0.9483 - accuracy: 0.6764
289/487 [================>.............] - ETA: 0s - loss: 0.9466 - accuracy: 0.6770
318/487 [==================>...........] - ETA: 0s - loss: 0.9441 - accuracy: 0.6780
346/487 [====================>.........] - ETA: 0s - loss: 0.9409 - accuracy: 0.6793
375/487 [======================>.......] - ETA: 0s - loss: 0.9386 - accuracy: 0.6802
404/487 [=======================>......] - ETA: 0s - loss: 0.9368 - accuracy: 0.6810
433/487 [=========================>....] - ETA: 0s - loss: 0.9342 - accuracy: 0.6821
461/487 [===========================>..] - ETA: 0s - loss: 0.9324 - accuracy: 0.6829
Epoch 3: val_loss improved from 0.97694 to 0.89573, saving model to unpruned_model/model_best.h5
Epoch 3: saving model to unpruned_model/model_last.h5
487/487 [==============================] - 1s 2ms/step - loss: 0.9305 - accuracy: 0.6840 - val_loss: 0.8957 - val_accuracy: 0.7006 - lr: 1.0000e-04
Epoch 4/30
1/487 [..............................] - ETA: 1s - loss: 0.9546 - accuracy: 0.6621
30/487 [>.............................] - ETA: 0s - loss: 0.8929 - accuracy: 0.6980
58/487 [==>...........................] - ETA: 0s - loss: 0.8888 - accuracy: 0.7006
87/487 [====>.........................] - ETA: 0s - loss: 0.8851 - accuracy: 0.7019
116/487 [======>.......................] - ETA: 0s - loss: 0.8816 - accuracy: 0.7025
144/487 [=======>......................] - ETA: 0s - loss: 0.8798 - accuracy: 0.7043
173/487 [=========>....................] - ETA: 0s - loss: 0.8794 - accuracy: 0.7049
202/487 [===========>..................] - ETA: 0s - loss: 0.8773 - accuracy: 0.7057
231/487 [=============>................] - ETA: 0s - loss: 0.8770 - accuracy: 0.7056
260/487 [===============>..............] - ETA: 0s - loss: 0.8746 - accuracy: 0.7066
289/487 [================>.............] - ETA: 0s - loss: 0.8735 - accuracy: 0.7074
318/487 [==================>...........] - ETA: 0s - loss: 0.8722 - accuracy: 0.7077
347/487 [====================>.........] - ETA: 0s - loss: 0.8703 - accuracy: 0.7083
376/487 [======================>.......] - ETA: 0s - loss: 0.8684 - accuracy: 0.7091
404/487 [=======================>......] - ETA: 0s - loss: 0.8664 - accuracy: 0.7098
433/487 [=========================>....] - ETA: 0s - loss: 0.8656 - accuracy: 0.7101
462/487 [===========================>..] - ETA: 0s - loss: 0.8639 - accuracy: 0.7106
Epoch 4: val_loss improved from 0.89573 to 0.84167, saving model to unpruned_model/model_best.h5
Epoch 4: saving model to unpruned_model/model_last.h5
487/487 [==============================] - 1s 2ms/step - loss: 0.8625 - accuracy: 0.7112 - val_loss: 0.8417 - val_accuracy: 0.7183 - lr: 1.0000e-04
Epoch 5/30
1/487 [..............................] - ETA: 1s - loss: 0.8021 - accuracy: 0.7158
30/487 [>.............................] - ETA: 0s - loss: 0.8238 - accuracy: 0.7240
58/487 [==>...........................] - ETA: 0s - loss: 0.8292 - accuracy: 0.7222
87/487 [====>.........................] - ETA: 0s - loss: 0.8329 - accuracy: 0.7215
116/487 [======>.......................] - ETA: 0s - loss: 0.8310 - accuracy: 0.7215
145/487 [=======>......................] - ETA: 0s - loss: 0.8277 - accuracy: 0.7230
174/487 [=========>....................] - ETA: 0s - loss: 0.8289 - accuracy: 0.7222
203/487 [===========>..................] - ETA: 0s - loss: 0.8282 - accuracy: 0.7222
231/487 [=============>................] - ETA: 0s - loss: 0.8268 - accuracy: 0.7225
260/487 [===============>..............] - ETA: 0s - loss: 0.8259 - accuracy: 0.7224
289/487 [================>.............] - ETA: 0s - loss: 0.8248 - accuracy: 0.7226
317/487 [==================>...........] - ETA: 0s - loss: 0.8240 - accuracy: 0.7225
346/487 [====================>.........] - ETA: 0s - loss: 0.8229 - accuracy: 0.7226
375/487 [======================>.......] - ETA: 0s - loss: 0.8221 - accuracy: 0.7227
404/487 [=======================>......] - ETA: 0s - loss: 0.8216 - accuracy: 0.7229
433/487 [=========================>....] - ETA: 0s - loss: 0.8210 - accuracy: 0.7229
462/487 [===========================>..] - ETA: 0s - loss: 0.8204 - accuracy: 0.7231
Epoch 5: val_loss improved from 0.84167 to 0.81128, saving model to unpruned_model/model_best.h5
Epoch 5: saving model to unpruned_model/model_last.h5
487/487 [==============================] - 1s 2ms/step - loss: 0.8205 - accuracy: 0.7230 - val_loss: 0.8113 - val_accuracy: 0.7256 - lr: 1.0000e-04
Epoch 6/30
1/487 [..............................] - ETA: 1s - loss: 0.8810 - accuracy: 0.7051
30/487 [>.............................] - ETA: 0s - loss: 0.8134 - accuracy: 0.7225
59/487 [==>...........................] - ETA: 0s - loss: 0.8070 - accuracy: 0.7263
88/487 [====>.........................] - ETA: 0s - loss: 0.8074 - accuracy: 0.7249
116/487 [======>.......................] - ETA: 0s - loss: 0.8062 - accuracy: 0.7263
144/487 [=======>......................] - ETA: 0s - loss: 0.8058 - accuracy: 0.7257
172/487 [=========>....................] - ETA: 0s - loss: 0.8054 - accuracy: 0.7256
201/487 [===========>..................] - ETA: 0s - loss: 0.8039 - accuracy: 0.7263
230/487 [=============>................] - ETA: 0s - loss: 0.8042 - accuracy: 0.7264
259/487 [==============>...............] - ETA: 0s - loss: 0.8035 - accuracy: 0.7265
288/487 [================>.............] - ETA: 0s - loss: 0.8029 - accuracy: 0.7267
316/487 [==================>...........] - ETA: 0s - loss: 0.8014 - accuracy: 0.7273
345/487 [====================>.........] - ETA: 0s - loss: 0.8006 - accuracy: 0.7274
373/487 [=====================>........] - ETA: 0s - loss: 0.7997 - accuracy: 0.7276
402/487 [=======================>......] - ETA: 0s - loss: 0.7994 - accuracy: 0.7277
430/487 [=========================>....] - ETA: 0s - loss: 0.7988 - accuracy: 0.7279
459/487 [===========================>..] - ETA: 0s - loss: 0.7977 - accuracy: 0.7280
Epoch 6: val_loss improved from 0.81128 to 0.79376, saving model to unpruned_model/model_best.h5
Epoch 6: saving model to unpruned_model/model_last.h5
487/487 [==============================] - 1s 2ms/step - loss: 0.7970 - accuracy: 0.7281 - val_loss: 0.7938 - val_accuracy: 0.7294 - lr: 1.0000e-04
Epoch 7/30
1/487 [..............................] - ETA: 1s - loss: 0.8029 - accuracy: 0.7109
30/487 [>.............................] - ETA: 0s - loss: 0.7931 - accuracy: 0.7253
59/487 [==>...........................] - ETA: 0s - loss: 0.7880 - accuracy: 0.7281
88/487 [====>.........................] - ETA: 0s - loss: 0.7862 - accuracy: 0.7297
117/487 [======>.......................] - ETA: 0s - loss: 0.7883 - accuracy: 0.7291
145/487 [=======>......................] - ETA: 0s - loss: 0.7883 - accuracy: 0.7292
174/487 [=========>....................] - ETA: 0s - loss: 0.7878 - accuracy: 0.7291
203/487 [===========>..................] - ETA: 0s - loss: 0.7871 - accuracy: 0.7295
232/487 [=============>................] - ETA: 0s - loss: 0.7851 - accuracy: 0.7302
261/487 [===============>..............] - ETA: 0s - loss: 0.7848 - accuracy: 0.7305
289/487 [================>.............] - ETA: 0s - loss: 0.7845 - accuracy: 0.7303
317/487 [==================>...........] - ETA: 0s - loss: 0.7843 - accuracy: 0.7303
346/487 [====================>.........] - ETA: 0s - loss: 0.7837 - accuracy: 0.7306
375/487 [======================>.......] - ETA: 0s - loss: 0.7831 - accuracy: 0.7309
404/487 [=======================>......] - ETA: 0s - loss: 0.7833 - accuracy: 0.7310
433/487 [=========================>....] - ETA: 0s - loss: 0.7828 - accuracy: 0.7312
462/487 [===========================>..] - ETA: 0s - loss: 0.7825 - accuracy: 0.7312
Epoch 7: val_loss improved from 0.79376 to 0.78167, saving model to unpruned_model/model_best.h5
Epoch 7: saving model to unpruned_model/model_last.h5
487/487 [==============================] - 1s 2ms/step - loss: 0.7823 - accuracy: 0.7312 - val_loss: 0.7817 - val_accuracy: 0.7319 - lr: 1.0000e-04
Epoch 8/30
1/487 [..............................] - ETA: 1s - loss: 0.8051 - accuracy: 0.7178
30/487 [>.............................] - ETA: 0s - loss: 0.7759 - accuracy: 0.7321
58/487 [==>...........................] - ETA: 0s - loss: 0.7821 - accuracy: 0.7300
87/487 [====>.........................] - ETA: 0s - loss: 0.7762 - accuracy: 0.7324
116/487 [======>.......................] - ETA: 0s - loss: 0.7776 - accuracy: 0.7319
145/487 [=======>......................] - ETA: 0s - loss: 0.7766 - accuracy: 0.7320
173/487 [=========>....................] - ETA: 0s - loss: 0.7752 - accuracy: 0.7325
202/487 [===========>..................] - ETA: 0s - loss: 0.7747 - accuracy: 0.7328
230/487 [=============>................] - ETA: 0s - loss: 0.7750 - accuracy: 0.7327
258/487 [==============>...............] - ETA: 0s - loss: 0.7744 - accuracy: 0.7331
286/487 [================>.............] - ETA: 0s - loss: 0.7740 - accuracy: 0.7331
314/487 [==================>...........] - ETA: 0s - loss: 0.7730 - accuracy: 0.7333
342/487 [====================>.........] - ETA: 0s - loss: 0.7729 - accuracy: 0.7334
370/487 [=====================>........] - ETA: 0s - loss: 0.7724 - accuracy: 0.7335
398/487 [=======================>......] - ETA: 0s - loss: 0.7721 - accuracy: 0.7334
426/487 [=========================>....] - ETA: 0s - loss: 0.7724 - accuracy: 0.7333
454/487 [==========================>...] - ETA: 0s - loss: 0.7715 - accuracy: 0.7337
483/487 [============================>.] - ETA: 0s - loss: 0.7714 - accuracy: 0.7336
Epoch 8: val_loss improved from 0.78167 to 0.77204, saving model to unpruned_model/model_best.h5
Epoch 8: saving model to unpruned_model/model_last.h5
487/487 [==============================] - 1s 2ms/step - loss: 0.7715 - accuracy: 0.7336 - val_loss: 0.7720 - val_accuracy: 0.7339 - lr: 1.0000e-04
Epoch 9/30
1/487 [..............................] - ETA: 1s - loss: 0.7961 - accuracy: 0.7324
29/487 [>.............................] - ETA: 0s - loss: 0.7626 - accuracy: 0.7355
58/487 [==>...........................] - ETA: 0s - loss: 0.7714 - accuracy: 0.7327
86/487 [====>.........................] - ETA: 0s - loss: 0.7729 - accuracy: 0.7329
114/487 [======>.......................] - ETA: 0s - loss: 0.7704 - accuracy: 0.7332
142/487 [=======>......................] - ETA: 0s - loss: 0.7684 - accuracy: 0.7335
171/487 [=========>....................] - ETA: 0s - loss: 0.7674 - accuracy: 0.7341
200/487 [===========>..................] - ETA: 0s - loss: 0.7664 - accuracy: 0.7343
228/487 [=============>................] - ETA: 0s - loss: 0.7657 - accuracy: 0.7345
257/487 [==============>...............] - ETA: 0s - loss: 0.7658 - accuracy: 0.7344
285/487 [================>.............] - ETA: 0s - loss: 0.7657 - accuracy: 0.7346
314/487 [==================>...........] - ETA: 0s - loss: 0.7646 - accuracy: 0.7349
343/487 [====================>.........] - ETA: 0s - loss: 0.7647 - accuracy: 0.7349
372/487 [=====================>........] - ETA: 0s - loss: 0.7648 - accuracy: 0.7349
401/487 [=======================>......] - ETA: 0s - loss: 0.7642 - accuracy: 0.7352
429/487 [=========================>....] - ETA: 0s - loss: 0.7637 - accuracy: 0.7354
458/487 [===========================>..] - ETA: 0s - loss: 0.7629 - accuracy: 0.7357
486/487 [============================>.] - ETA: 0s - loss: 0.7627 - accuracy: 0.7355
Epoch 9: val_loss improved from 0.77204 to 0.76470, saving model to unpruned_model/model_best.h5
Epoch 9: saving model to unpruned_model/model_last.h5
487/487 [==============================] - 1s 2ms/step - loss: 0.7628 - accuracy: 0.7355 - val_loss: 0.7647 - val_accuracy: 0.7356 - lr: 1.0000e-04
Epoch 10/30
1/487 [..............................] - ETA: 1s - loss: 0.7394 - accuracy: 0.7236
30/487 [>.............................] - ETA: 0s - loss: 0.7629 - accuracy: 0.7339
59/487 [==>...........................] - ETA: 0s - loss: 0.7618 - accuracy: 0.7342
87/487 [====>.........................] - ETA: 0s - loss: 0.7622 - accuracy: 0.7347
115/487 [======>.......................] - ETA: 0s - loss: 0.7604 - accuracy: 0.7349
143/487 [=======>......................] - ETA: 0s - loss: 0.7600 - accuracy: 0.7349
171/487 [=========>....................] - ETA: 0s - loss: 0.7600 - accuracy: 0.7346
199/487 [===========>..................] - ETA: 0s - loss: 0.7593 - accuracy: 0.7350
228/487 [=============>................] - ETA: 0s - loss: 0.7580 - accuracy: 0.7357
256/487 [==============>...............] - ETA: 0s - loss: 0.7579 - accuracy: 0.7357
284/487 [================>.............] - ETA: 0s - loss: 0.7579 - accuracy: 0.7361
313/487 [==================>...........] - ETA: 0s - loss: 0.7573 - accuracy: 0.7363
342/487 [====================>.........] - ETA: 0s - loss: 0.7573 - accuracy: 0.7364
370/487 [=====================>........] - ETA: 0s - loss: 0.7572 - accuracy: 0.7363
398/487 [=======================>......] - ETA: 0s - loss: 0.7563 - accuracy: 0.7366
427/487 [=========================>....] - ETA: 0s - loss: 0.7563 - accuracy: 0.7367
455/487 [===========================>..] - ETA: 0s - loss: 0.7562 - accuracy: 0.7367
484/487 [============================>.] - ETA: 0s - loss: 0.7557 - accuracy: 0.7369
Epoch 10: val_loss improved from 0.76470 to 0.75796, saving model to unpruned_model/model_best.h5
Epoch 10: saving model to unpruned_model/model_last.h5
487/487 [==============================] - 1s 2ms/step - loss: 0.7556 - accuracy: 0.7369 - val_loss: 0.7580 - val_accuracy: 0.7370 - lr: 1.0000e-04
Epoch 11/30
1/487 [..............................] - ETA: 1s - loss: 0.7464 - accuracy: 0.7549
30/487 [>.............................] - ETA: 0s - loss: 0.7434 - accuracy: 0.7414
58/487 [==>...........................] - ETA: 0s - loss: 0.7425 - accuracy: 0.7413
87/487 [====>.........................] - ETA: 0s - loss: 0.7470 - accuracy: 0.7397
116/487 [======>.......................] - ETA: 0s - loss: 0.7485 - accuracy: 0.7383
144/487 [=======>......................] - ETA: 0s - loss: 0.7479 - accuracy: 0.7388
172/487 [=========>....................] - ETA: 0s - loss: 0.7478 - accuracy: 0.7392
201/487 [===========>..................] - ETA: 0s - loss: 0.7494 - accuracy: 0.7387
230/487 [=============>................] - ETA: 0s - loss: 0.7495 - accuracy: 0.7384
259/487 [==============>...............] - ETA: 0s - loss: 0.7493 - accuracy: 0.7388
288/487 [================>.............] - ETA: 0s - loss: 0.7498 - accuracy: 0.7384
317/487 [==================>...........] - ETA: 0s - loss: 0.7495 - accuracy: 0.7383
346/487 [====================>.........] - ETA: 0s - loss: 0.7494 - accuracy: 0.7384
375/487 [======================>.......] - ETA: 0s - loss: 0.7499 - accuracy: 0.7382
404/487 [=======================>......] - ETA: 0s - loss: 0.7493 - accuracy: 0.7383
433/487 [=========================>....] - ETA: 0s - loss: 0.7497 - accuracy: 0.7382
461/487 [===========================>..] - ETA: 0s - loss: 0.7493 - accuracy: 0.7383
Epoch 11: val_loss improved from 0.75796 to 0.75196, saving model to unpruned_model/model_best.h5
Epoch 11: saving model to unpruned_model/model_last.h5
487/487 [==============================] - 1s 2ms/step - loss: 0.7493 - accuracy: 0.7384 - val_loss: 0.7520 - val_accuracy: 0.7381 - lr: 1.0000e-04
Epoch 12/30
1/487 [..............................] - ETA: 1s - loss: 0.7676 - accuracy: 0.7285
30/487 [>.............................] - ETA: 0s - loss: 0.7527 - accuracy: 0.7357
59/487 [==>...........................] - ETA: 0s - loss: 0.7435 - accuracy: 0.7393
88/487 [====>.........................] - ETA: 0s - loss: 0.7436 - accuracy: 0.7389
116/487 [======>.......................] - ETA: 0s - loss: 0.7460 - accuracy: 0.7382
144/487 [=======>......................] - ETA: 0s - loss: 0.7457 - accuracy: 0.7389
173/487 [=========>....................] - ETA: 0s - loss: 0.7444 - accuracy: 0.7396
201/487 [===========>..................] - ETA: 0s - loss: 0.7443 - accuracy: 0.7394
229/487 [=============>................] - ETA: 0s - loss: 0.7434 - accuracy: 0.7394
258/487 [==============>...............] - ETA: 0s - loss: 0.7432 - accuracy: 0.7397
286/487 [================>.............] - ETA: 0s - loss: 0.7436 - accuracy: 0.7396
314/487 [==================>...........] - ETA: 0s - loss: 0.7433 - accuracy: 0.7398
343/487 [====================>.........] - ETA: 0s - loss: 0.7439 - accuracy: 0.7396
372/487 [=====================>........] - ETA: 0s - loss: 0.7434 - accuracy: 0.7398
401/487 [=======================>......] - ETA: 0s - loss: 0.7442 - accuracy: 0.7394
430/487 [=========================>....] - ETA: 0s - loss: 0.7437 - accuracy: 0.7394
458/487 [===========================>..] - ETA: 0s - loss: 0.7436 - accuracy: 0.7394
486/487 [============================>.] - ETA: 0s - loss: 0.7437 - accuracy: 0.7395
Epoch 12: val_loss improved from 0.75196 to 0.74687, saving model to unpruned_model/model_best.h5
Epoch 12: saving model to unpruned_model/model_last.h5
487/487 [==============================] - 1s 2ms/step - loss: 0.7437 - accuracy: 0.7395 - val_loss: 0.7469 - val_accuracy: 0.7397 - lr: 1.0000e-04
Epoch 13/30
1/487 [..............................] - ETA: 1s - loss: 0.7159 - accuracy: 0.7607
30/487 [>.............................] - ETA: 0s - loss: 0.7341 - accuracy: 0.7415
58/487 [==>...........................] - ETA: 0s - loss: 0.7366 - accuracy: 0.7402
87/487 [====>.........................] - ETA: 0s - loss: 0.7382 - accuracy: 0.7397
114/487 [======>.......................] - ETA: 0s - loss: 0.7383 - accuracy: 0.7403
143/487 [=======>......................] - ETA: 0s - loss: 0.7374 - accuracy: 0.7408
171/487 [=========>....................] - ETA: 0s - loss: 0.7379 - accuracy: 0.7410
200/487 [===========>..................] - ETA: 0s - loss: 0.7380 - accuracy: 0.7409
229/487 [=============>................] - ETA: 0s - loss: 0.7384 - accuracy: 0.7406
258/487 [==============>...............] - ETA: 0s - loss: 0.7387 - accuracy: 0.7406
286/487 [================>.............] - ETA: 0s - loss: 0.7371 - accuracy: 0.7412
315/487 [==================>...........] - ETA: 0s - loss: 0.7379 - accuracy: 0.7409
344/487 [====================>.........] - ETA: 0s - loss: 0.7377 - accuracy: 0.7408
373/487 [=====================>........] - ETA: 0s - loss: 0.7382 - accuracy: 0.7408
401/487 [=======================>......] - ETA: 0s - loss: 0.7380 - accuracy: 0.7411
429/487 [=========================>....] - ETA: 0s - loss: 0.7383 - accuracy: 0.7410
458/487 [===========================>..] - ETA: 0s - loss: 0.7385 - accuracy: 0.7410
486/487 [============================>.] - ETA: 0s - loss: 0.7387 - accuracy: 0.7407
Epoch 13: val_loss improved from 0.74687 to 0.74226, saving model to unpruned_model/model_best.h5
Epoch 13: saving model to unpruned_model/model_last.h5
487/487 [==============================] - 1s 2ms/step - loss: 0.7387 - accuracy: 0.7407 - val_loss: 0.7423 - val_accuracy: 0.7406 - lr: 1.0000e-04
Epoch 14/30
1/487 [..............................] - ETA: 1s - loss: 0.7706 - accuracy: 0.7275
30/487 [>.............................] - ETA: 0s - loss: 0.7340 - accuracy: 0.7451
58/487 [==>...........................] - ETA: 0s - loss: 0.7340 - accuracy: 0.7444
86/487 [====>.........................] - ETA: 0s - loss: 0.7315 - accuracy: 0.7446
115/487 [======>.......................] - ETA: 0s - loss: 0.7350 - accuracy: 0.7427
143/487 [=======>......................] - ETA: 0s - loss: 0.7351 - accuracy: 0.7429
171/487 [=========>....................] - ETA: 0s - loss: 0.7354 - accuracy: 0.7426
199/487 [===========>..................] - ETA: 0s - loss: 0.7357 - accuracy: 0.7423
227/487 [============>.................] - ETA: 0s - loss: 0.7346 - accuracy: 0.7426
255/487 [==============>...............] - ETA: 0s - loss: 0.7349 - accuracy: 0.7423
284/487 [================>.............] - ETA: 0s - loss: 0.7343 - accuracy: 0.7426
313/487 [==================>...........] - ETA: 0s - loss: 0.7350 - accuracy: 0.7421
342/487 [====================>.........] - ETA: 0s - loss: 0.7348 - accuracy: 0.7422
370/487 [=====================>........] - ETA: 0s - loss: 0.7341 - accuracy: 0.7425
398/487 [=======================>......] - ETA: 0s - loss: 0.7340 - accuracy: 0.7423
426/487 [=========================>....] - ETA: 0s - loss: 0.7339 - accuracy: 0.7422
454/487 [==========================>...] - ETA: 0s - loss: 0.7344 - accuracy: 0.7420
482/487 [============================>.] - ETA: 0s - loss: 0.7343 - accuracy: 0.7420
Epoch 14: val_loss improved from 0.74226 to 0.73791, saving model to unpruned_model/model_best.h5
Epoch 14: saving model to unpruned_model/model_last.h5
487/487 [==============================] - 1s 2ms/step - loss: 0.7343 - accuracy: 0.7419 - val_loss: 0.7379 - val_accuracy: 0.7424 - lr: 1.0000e-04
Epoch 15/30
1/487 [..............................] - ETA: 1s - loss: 0.7538 - accuracy: 0.7285
30/487 [>.............................] - ETA: 0s - loss: 0.7305 - accuracy: 0.7436
57/487 [==>...........................] - ETA: 0s - loss: 0.7331 - accuracy: 0.7425
85/487 [====>.........................] - ETA: 0s - loss: 0.7336 - accuracy: 0.7428
113/487 [=====>........................] - ETA: 0s - loss: 0.7321 - accuracy: 0.7425
141/487 [=======>......................] - ETA: 0s - loss: 0.7313 - accuracy: 0.7428
169/487 [=========>....................] - ETA: 0s - loss: 0.7311 - accuracy: 0.7428
197/487 [===========>..................] - ETA: 0s - loss: 0.7313 - accuracy: 0.7428
225/487 [============>.................] - ETA: 0s - loss: 0.7312 - accuracy: 0.7429
253/487 [==============>...............] - ETA: 0s - loss: 0.7317 - accuracy: 0.7426
281/487 [================>.............] - ETA: 0s - loss: 0.7323 - accuracy: 0.7425
309/487 [==================>...........] - ETA: 0s - loss: 0.7332 - accuracy: 0.7421
338/487 [===================>..........] - ETA: 0s - loss: 0.7328 - accuracy: 0.7423
366/487 [=====================>........] - ETA: 0s - loss: 0.7323 - accuracy: 0.7425
394/487 [=======================>......] - ETA: 0s - loss: 0.7321 - accuracy: 0.7425
422/487 [========================>.....] - ETA: 0s - loss: 0.7316 - accuracy: 0.7426
451/487 [==========================>...] - ETA: 0s - loss: 0.7306 - accuracy: 0.7429
479/487 [============================>.] - ETA: 0s - loss: 0.7305 - accuracy: 0.7429
Epoch 15: val_loss improved from 0.73791 to 0.73434, saving model to unpruned_model/model_best.h5
Epoch 15: saving model to unpruned_model/model_last.h5
487/487 [==============================] - 1s 2ms/step - loss: 0.7305 - accuracy: 0.7429 - val_loss: 0.7343 - val_accuracy: 0.7430 - lr: 1.0000e-04
Epoch 16/30
1/487 [..............................] - ETA: 1s - loss: 0.6995 - accuracy: 0.7646
29/487 [>.............................] - ETA: 0s - loss: 0.7365 - accuracy: 0.7407
57/487 [==>...........................] - ETA: 0s - loss: 0.7359 - accuracy: 0.7413
85/487 [====>.........................] - ETA: 0s - loss: 0.7323 - accuracy: 0.7421
114/487 [======>.......................] - ETA: 0s - loss: 0.7329 - accuracy: 0.7417
143/487 [=======>......................] - ETA: 0s - loss: 0.7330 - accuracy: 0.7417
172/487 [=========>....................] - ETA: 0s - loss: 0.7305 - accuracy: 0.7427
200/487 [===========>..................] - ETA: 0s - loss: 0.7297 - accuracy: 0.7427
228/487 [=============>................] - ETA: 0s - loss: 0.7292 - accuracy: 0.7431
257/487 [==============>...............] - ETA: 0s - loss: 0.7291 - accuracy: 0.7431
285/487 [================>.............] - ETA: 0s - loss: 0.7291 - accuracy: 0.7431
313/487 [==================>...........] - ETA: 0s - loss: 0.7286 - accuracy: 0.7433
342/487 [====================>.........] - ETA: 0s - loss: 0.7287 - accuracy: 0.7432
371/487 [=====================>........] - ETA: 0s - loss: 0.7281 - accuracy: 0.7435
400/487 [=======================>......] - ETA: 0s - loss: 0.7277 - accuracy: 0.7435
428/487 [=========================>....] - ETA: 0s - loss: 0.7279 - accuracy: 0.7435
457/487 [===========================>..] - ETA: 0s - loss: 0.7276 - accuracy: 0.7435
485/487 [============================>.] - ETA: 0s - loss: 0.7272 - accuracy: 0.7437
Epoch 16: val_loss improved from 0.73434 to 0.73180, saving model to unpruned_model/model_best.h5
Epoch 16: saving model to unpruned_model/model_last.h5
487/487 [==============================] - 1s 2ms/step - loss: 0.7271 - accuracy: 0.7437 - val_loss: 0.7318 - val_accuracy: 0.7431 - lr: 1.0000e-04
Epoch 17/30
1/487 [..............................] - ETA: 1s - loss: 0.7373 - accuracy: 0.7412
30/487 [>.............................] - ETA: 0s - loss: 0.7211 - accuracy: 0.7463
58/487 [==>...........................] - ETA: 0s - loss: 0.7251 - accuracy: 0.7452
86/487 [====>.........................] - ETA: 0s - loss: 0.7276 - accuracy: 0.7437
114/487 [======>.......................] - ETA: 0s - loss: 0.7288 - accuracy: 0.7435
142/487 [=======>......................] - ETA: 0s - loss: 0.7268 - accuracy: 0.7444
171/487 [=========>....................] - ETA: 0s - loss: 0.7241 - accuracy: 0.7455
199/487 [===========>..................] - ETA: 0s - loss: 0.7237 - accuracy: 0.7454
227/487 [============>.................] - ETA: 0s - loss: 0.7226 - accuracy: 0.7457
256/487 [==============>...............] - ETA: 0s - loss: 0.7227 - accuracy: 0.7453
285/487 [================>.............] - ETA: 0s - loss: 0.7236 - accuracy: 0.7449
313/487 [==================>...........] - ETA: 0s - loss: 0.7242 - accuracy: 0.7445
341/487 [====================>.........] - ETA: 0s - loss: 0.7242 - accuracy: 0.7445
370/487 [=====================>........] - ETA: 0s - loss: 0.7243 - accuracy: 0.7446
399/487 [=======================>......] - ETA: 0s - loss: 0.7243 - accuracy: 0.7448
427/487 [=========================>....] - ETA: 0s - loss: 0.7241 - accuracy: 0.7448
455/487 [===========================>..] - ETA: 0s - loss: 0.7244 - accuracy: 0.7446
483/487 [============================>.] - ETA: 0s - loss: 0.7243 - accuracy: 0.7446
Epoch 17: val_loss improved from 0.73180 to 0.72860, saving model to unpruned_model/model_best.h5
Epoch 17: saving model to unpruned_model/model_last.h5
487/487 [==============================] - 1s 2ms/step - loss: 0.7241 - accuracy: 0.7447 - val_loss: 0.7286 - val_accuracy: 0.7441 - lr: 1.0000e-04
Epoch 18/30
1/487 [..............................] - ETA: 1s - loss: 0.7246 - accuracy: 0.7393
30/487 [>.............................] - ETA: 0s - loss: 0.7183 - accuracy: 0.7446
59/487 [==>...........................] - ETA: 0s - loss: 0.7196 - accuracy: 0.7452
87/487 [====>.........................] - ETA: 0s - loss: 0.7196 - accuracy: 0.7457
115/487 [======>.......................] - ETA: 0s - loss: 0.7201 - accuracy: 0.7462
143/487 [=======>......................] - ETA: 0s - loss: 0.7211 - accuracy: 0.7457
172/487 [=========>....................] - ETA: 0s - loss: 0.7217 - accuracy: 0.7457
200/487 [===========>..................] - ETA: 0s - loss: 0.7218 - accuracy: 0.7455
229/487 [=============>................] - ETA: 0s - loss: 0.7217 - accuracy: 0.7457
257/487 [==============>...............] - ETA: 0s - loss: 0.7209 - accuracy: 0.7456
286/487 [================>.............] - ETA: 0s - loss: 0.7210 - accuracy: 0.7458
315/487 [==================>...........] - ETA: 0s - loss: 0.7209 - accuracy: 0.7458
344/487 [====================>.........] - ETA: 0s - loss: 0.7202 - accuracy: 0.7459
372/487 [=====================>........] - ETA: 0s - loss: 0.7198 - accuracy: 0.7461
400/487 [=======================>......] - ETA: 0s - loss: 0.7202 - accuracy: 0.7459
429/487 [=========================>....] - ETA: 0s - loss: 0.7204 - accuracy: 0.7458
458/487 [===========================>..] - ETA: 0s - loss: 0.7209 - accuracy: 0.7456
486/487 [============================>.] - ETA: 0s - loss: 0.7215 - accuracy: 0.7454
Epoch 18: val_loss improved from 0.72860 to 0.72637, saving model to unpruned_model/model_best.h5
Epoch 18: saving model to unpruned_model/model_last.h5
487/487 [==============================] - 1s 2ms/step - loss: 0.7215 - accuracy: 0.7454 - val_loss: 0.7264 - val_accuracy: 0.7448 - lr: 1.0000e-04
Epoch 19/30
1/487 [..............................] - ETA: 1s - loss: 0.7302 - accuracy: 0.7432
30/487 [>.............................] - ETA: 0s - loss: 0.7204 - accuracy: 0.7472
59/487 [==>...........................] - ETA: 0s - loss: 0.7177 - accuracy: 0.7468
88/487 [====>.........................] - ETA: 0s - loss: 0.7204 - accuracy: 0.7461
116/487 [======>.......................] - ETA: 0s - loss: 0.7193 - accuracy: 0.7462
145/487 [=======>......................] - ETA: 0s - loss: 0.7191 - accuracy: 0.7462
174/487 [=========>....................] - ETA: 0s - loss: 0.7184 - accuracy: 0.7466
203/487 [===========>..................] - ETA: 0s - loss: 0.7180 - accuracy: 0.7465
232/487 [=============>................] - ETA: 0s - loss: 0.7186 - accuracy: 0.7463
260/487 [===============>..............] - ETA: 0s - loss: 0.7186 - accuracy: 0.7464
288/487 [================>.............] - ETA: 0s - loss: 0.7193 - accuracy: 0.7461
316/487 [==================>...........] - ETA: 0s - loss: 0.7188 - accuracy: 0.7464
344/487 [====================>.........] - ETA: 0s - loss: 0.7189 - accuracy: 0.7465
373/487 [=====================>........] - ETA: 0s - loss: 0.7189 - accuracy: 0.7463
401/487 [=======================>......] - ETA: 0s - loss: 0.7190 - accuracy: 0.7463
429/487 [=========================>....] - ETA: 0s - loss: 0.7193 - accuracy: 0.7462
457/487 [===========================>..] - ETA: 0s - loss: 0.7186 - accuracy: 0.7464
485/487 [============================>.] - ETA: 0s - loss: 0.7190 - accuracy: 0.7463
Epoch 19: val_loss improved from 0.72637 to 0.72382, saving model to unpruned_model/model_best.h5
Epoch 19: saving model to unpruned_model/model_last.h5
487/487 [==============================] - 1s 2ms/step - loss: 0.7191 - accuracy: 0.7462 - val_loss: 0.7238 - val_accuracy: 0.7457 - lr: 1.0000e-04
Epoch 20/30
1/487 [..............................] - ETA: 1s - loss: 0.7290 - accuracy: 0.7441
30/487 [>.............................] - ETA: 0s - loss: 0.7071 - accuracy: 0.7518
59/487 [==>...........................] - ETA: 0s - loss: 0.7152 - accuracy: 0.7478
88/487 [====>.........................] - ETA: 0s - loss: 0.7156 - accuracy: 0.7473
117/487 [======>.......................] - ETA: 0s - loss: 0.7155 - accuracy: 0.7474
146/487 [=======>......................] - ETA: 0s - loss: 0.7150 - accuracy: 0.7478
174/487 [=========>....................] - ETA: 0s - loss: 0.7168 - accuracy: 0.7471
203/487 [===========>..................] - ETA: 0s - loss: 0.7174 - accuracy: 0.7467
232/487 [=============>................] - ETA: 0s - loss: 0.7181 - accuracy: 0.7465
261/487 [===============>..............] - ETA: 0s - loss: 0.7189 - accuracy: 0.7460
290/487 [================>.............] - ETA: 0s - loss: 0.7184 - accuracy: 0.7463
319/487 [==================>...........] - ETA: 0s - loss: 0.7174 - accuracy: 0.7465
348/487 [====================>.........] - ETA: 0s - loss: 0.7169 - accuracy: 0.7467
377/487 [======================>.......] - ETA: 0s - loss: 0.7167 - accuracy: 0.7467
405/487 [=======================>......] - ETA: 0s - loss: 0.7172 - accuracy: 0.7468
433/487 [=========================>....] - ETA: 0s - loss: 0.7168 - accuracy: 0.7468
460/487 [===========================>..] - ETA: 0s - loss: 0.7171 - accuracy: 0.7468
Epoch 20: val_loss improved from 0.72382 to 0.72174, saving model to unpruned_model/model_best.h5
Epoch 20: saving model to unpruned_model/model_last.h5
487/487 [==============================] - 1s 2ms/step - loss: 0.7170 - accuracy: 0.7468 - val_loss: 0.7217 - val_accuracy: 0.7463 - lr: 1.0000e-04
Epoch 21/30
1/487 [..............................] - ETA: 1s - loss: 0.7633 - accuracy: 0.7344
29/487 [>.............................] - ETA: 0s - loss: 0.7212 - accuracy: 0.7437
58/487 [==>...........................] - ETA: 0s - loss: 0.7151 - accuracy: 0.7463
87/487 [====>.........................] - ETA: 0s - loss: 0.7140 - accuracy: 0.7474
115/487 [======>.......................] - ETA: 0s - loss: 0.7141 - accuracy: 0.7473
144/487 [=======>......................] - ETA: 0s - loss: 0.7157 - accuracy: 0.7472
173/487 [=========>....................] - ETA: 0s - loss: 0.7162 - accuracy: 0.7470
202/487 [===========>..................] - ETA: 0s - loss: 0.7158 - accuracy: 0.7470
231/487 [=============>................] - ETA: 0s - loss: 0.7162 - accuracy: 0.7468
260/487 [===============>..............] - ETA: 0s - loss: 0.7173 - accuracy: 0.7461
289/487 [================>.............] - ETA: 0s - loss: 0.7167 - accuracy: 0.7465
317/487 [==================>...........] - ETA: 0s - loss: 0.7161 - accuracy: 0.7467
346/487 [====================>.........] - ETA: 0s - loss: 0.7164 - accuracy: 0.7467
375/487 [======================>.......] - ETA: 0s - loss: 0.7164 - accuracy: 0.7467
404/487 [=======================>......] - ETA: 0s - loss: 0.7160 - accuracy: 0.7468
433/487 [=========================>....] - ETA: 0s - loss: 0.7155 - accuracy: 0.7471
462/487 [===========================>..] - ETA: 0s - loss: 0.7153 - accuracy: 0.7471
Epoch 21: val_loss improved from 0.72174 to 0.71981, saving model to unpruned_model/model_best.h5
Epoch 21: saving model to unpruned_model/model_last.h5
487/487 [==============================] - 1s 2ms/step - loss: 0.7150 - accuracy: 0.7473 - val_loss: 0.7198 - val_accuracy: 0.7463 - lr: 1.0000e-04
Epoch 22/30
1/487 [..............................] - ETA: 1s - loss: 0.7174 - accuracy: 0.7393
30/487 [>.............................] - ETA: 0s - loss: 0.7103 - accuracy: 0.7482
58/487 [==>...........................] - ETA: 0s - loss: 0.7169 - accuracy: 0.7461
86/487 [====>.........................] - ETA: 0s - loss: 0.7200 - accuracy: 0.7445
114/487 [======>.......................] - ETA: 0s - loss: 0.7182 - accuracy: 0.7459
143/487 [=======>......................] - ETA: 0s - loss: 0.7177 - accuracy: 0.7463
172/487 [=========>....................] - ETA: 0s - loss: 0.7168 - accuracy: 0.7467
201/487 [===========>..................] - ETA: 0s - loss: 0.7152 - accuracy: 0.7467
229/487 [=============>................] - ETA: 0s - loss: 0.7146 - accuracy: 0.7475
258/487 [==============>...............] - ETA: 0s - loss: 0.7138 - accuracy: 0.7479
287/487 [================>.............] - ETA: 0s - loss: 0.7136 - accuracy: 0.7477
315/487 [==================>...........] - ETA: 0s - loss: 0.7129 - accuracy: 0.7481
344/487 [====================>.........] - ETA: 0s - loss: 0.7135 - accuracy: 0.7479
372/487 [=====================>........] - ETA: 0s - loss: 0.7140 - accuracy: 0.7478
400/487 [=======================>......] - ETA: 0s - loss: 0.7145 - accuracy: 0.7476
428/487 [=========================>....] - ETA: 0s - loss: 0.7138 - accuracy: 0.7479
455/487 [===========================>..] - ETA: 0s - loss: 0.7134 - accuracy: 0.7480
483/487 [============================>.] - ETA: 0s - loss: 0.7131 - accuracy: 0.7481
Epoch 22: val_loss improved from 0.71981 to 0.71822, saving model to unpruned_model/model_best.h5
Epoch 22: saving model to unpruned_model/model_last.h5
487/487 [==============================] - 1s 2ms/step - loss: 0.7131 - accuracy: 0.7480 - val_loss: 0.7182 - val_accuracy: 0.7471 - lr: 1.0000e-04
Epoch 23/30
1/487 [..............................] - ETA: 1s - loss: 0.7189 - accuracy: 0.7383
30/487 [>.............................] - ETA: 0s - loss: 0.7068 - accuracy: 0.7513
59/487 [==>...........................] - ETA: 0s - loss: 0.7100 - accuracy: 0.7501
88/487 [====>.........................] - ETA: 0s - loss: 0.7117 - accuracy: 0.7491
116/487 [======>.......................] - ETA: 0s - loss: 0.7122 - accuracy: 0.7487
145/487 [=======>......................] - ETA: 0s - loss: 0.7121 - accuracy: 0.7490
173/487 [=========>....................] - ETA: 0s - loss: 0.7102 - accuracy: 0.7497
202/487 [===========>..................] - ETA: 0s - loss: 0.7100 - accuracy: 0.7496
231/487 [=============>................] - ETA: 0s - loss: 0.7100 - accuracy: 0.7497
259/487 [==============>...............] - ETA: 0s - loss: 0.7107 - accuracy: 0.7495
288/487 [================>.............] - ETA: 0s - loss: 0.7098 - accuracy: 0.7497
317/487 [==================>...........] - ETA: 0s - loss: 0.7105 - accuracy: 0.7493
346/487 [====================>.........] - ETA: 0s - loss: 0.7104 - accuracy: 0.7491
374/487 [======================>.......] - ETA: 0s - loss: 0.7108 - accuracy: 0.7488
403/487 [=======================>......] - ETA: 0s - loss: 0.7113 - accuracy: 0.7485
432/487 [=========================>....] - ETA: 0s - loss: 0.7112 - accuracy: 0.7486
461/487 [===========================>..] - ETA: 0s - loss: 0.7112 - accuracy: 0.7487
Epoch 23: val_loss improved from 0.71822 to 0.71635, saving model to unpruned_model/model_best.h5
Epoch 23: saving model to unpruned_model/model_last.h5
487/487 [==============================] - 1s 2ms/step - loss: 0.7115 - accuracy: 0.7485 - val_loss: 0.7163 - val_accuracy: 0.7477 - lr: 1.0000e-04
Epoch 24/30
1/487 [..............................] - ETA: 1s - loss: 0.6994 - accuracy: 0.7666
29/487 [>.............................] - ETA: 0s - loss: 0.7142 - accuracy: 0.7479
56/487 [==>...........................] - ETA: 0s - loss: 0.7128 - accuracy: 0.7472
84/487 [====>.........................] - ETA: 0s - loss: 0.7116 - accuracy: 0.7482
112/487 [=====>........................] - ETA: 0s - loss: 0.7097 - accuracy: 0.7485
141/487 [=======>......................] - ETA: 0s - loss: 0.7092 - accuracy: 0.7487
169/487 [=========>....................] - ETA: 0s - loss: 0.7077 - accuracy: 0.7493
198/487 [===========>..................] - ETA: 0s - loss: 0.7092 - accuracy: 0.7487
227/487 [============>.................] - ETA: 0s - loss: 0.7092 - accuracy: 0.7491
256/487 [==============>...............] - ETA: 0s - loss: 0.7099 - accuracy: 0.7490
284/487 [================>.............] - ETA: 0s - loss: 0.7090 - accuracy: 0.7493
312/487 [==================>...........] - ETA: 0s - loss: 0.7091 - accuracy: 0.7492
340/487 [===================>..........] - ETA: 0s - loss: 0.7095 - accuracy: 0.7492
369/487 [=====================>........] - ETA: 0s - loss: 0.7095 - accuracy: 0.7492
398/487 [=======================>......] - ETA: 0s - loss: 0.7091 - accuracy: 0.7495
427/487 [=========================>....] - ETA: 0s - loss: 0.7092 - accuracy: 0.7494
456/487 [===========================>..] - ETA: 0s - loss: 0.7095 - accuracy: 0.7492
485/487 [============================>.] - ETA: 0s - loss: 0.7099 - accuracy: 0.7490
Epoch 24: val_loss improved from 0.71635 to 0.71453, saving model to unpruned_model/model_best.h5
Epoch 24: saving model to unpruned_model/model_last.h5
487/487 [==============================] - 1s 2ms/step - loss: 0.7098 - accuracy: 0.7491 - val_loss: 0.7145 - val_accuracy: 0.7478 - lr: 1.0000e-04
Epoch 25/30
1/487 [..............................] - ETA: 1s - loss: 0.6934 - accuracy: 0.7568
30/487 [>.............................] - ETA: 0s - loss: 0.7114 - accuracy: 0.7477
59/487 [==>...........................] - ETA: 0s - loss: 0.7120 - accuracy: 0.7485
88/487 [====>.........................] - ETA: 0s - loss: 0.7114 - accuracy: 0.7487
117/487 [======>.......................] - ETA: 0s - loss: 0.7109 - accuracy: 0.7485
146/487 [=======>......................] - ETA: 0s - loss: 0.7103 - accuracy: 0.7485
175/487 [=========>....................] - ETA: 0s - loss: 0.7099 - accuracy: 0.7491
204/487 [===========>..................] - ETA: 0s - loss: 0.7092 - accuracy: 0.7492
233/487 [=============>................] - ETA: 0s - loss: 0.7096 - accuracy: 0.7489
262/487 [===============>..............] - ETA: 0s - loss: 0.7106 - accuracy: 0.7484
291/487 [================>.............] - ETA: 0s - loss: 0.7102 - accuracy: 0.7487
319/487 [==================>...........] - ETA: 0s - loss: 0.7110 - accuracy: 0.7482
348/487 [====================>.........] - ETA: 0s - loss: 0.7105 - accuracy: 0.7486
377/487 [======================>.......] - ETA: 0s - loss: 0.7096 - accuracy: 0.7489
406/487 [========================>.....] - ETA: 0s - loss: 0.7100 - accuracy: 0.7487
435/487 [=========================>....] - ETA: 0s - loss: 0.7093 - accuracy: 0.7489
464/487 [===========================>..] - ETA: 0s - loss: 0.7086 - accuracy: 0.7492
Epoch 25: val_loss improved from 0.71453 to 0.71329, saving model to unpruned_model/model_best.h5
Epoch 25: saving model to unpruned_model/model_last.h5
487/487 [==============================] - 1s 2ms/step - loss: 0.7083 - accuracy: 0.7494 - val_loss: 0.7133 - val_accuracy: 0.7480 - lr: 1.0000e-04
Epoch 26/30
1/487 [..............................] - ETA: 1s - loss: 0.7471 - accuracy: 0.7207
30/487 [>.............................] - ETA: 0s - loss: 0.7052 - accuracy: 0.7507
58/487 [==>...........................] - ETA: 0s - loss: 0.7049 - accuracy: 0.7508
87/487 [====>.........................] - ETA: 0s - loss: 0.7090 - accuracy: 0.7481
116/487 [======>.......................] - ETA: 0s - loss: 0.7100 - accuracy: 0.7478
145/487 [=======>......................] - ETA: 0s - loss: 0.7111 - accuracy: 0.7478
174/487 [=========>....................] - ETA: 0s - loss: 0.7093 - accuracy: 0.7486
202/487 [===========>..................] - ETA: 0s - loss: 0.7087 - accuracy: 0.7489
231/487 [=============>................] - ETA: 0s - loss: 0.7078 - accuracy: 0.7493
260/487 [===============>..............] - ETA: 0s - loss: 0.7071 - accuracy: 0.7496
288/487 [================>.............] - ETA: 0s - loss: 0.7075 - accuracy: 0.7495
316/487 [==================>...........] - ETA: 0s - loss: 0.7070 - accuracy: 0.7496
345/487 [====================>.........] - ETA: 0s - loss: 0.7072 - accuracy: 0.7496
374/487 [======================>.......] - ETA: 0s - loss: 0.7072 - accuracy: 0.7497
403/487 [=======================>......] - ETA: 0s - loss: 0.7077 - accuracy: 0.7496
432/487 [=========================>....] - ETA: 0s - loss: 0.7071 - accuracy: 0.7498
461/487 [===========================>..] - ETA: 0s - loss: 0.7064 - accuracy: 0.7502
Epoch 26: val_loss improved from 0.71329 to 0.71199, saving model to unpruned_model/model_best.h5
Epoch 26: saving model to unpruned_model/model_last.h5
487/487 [==============================] - 1s 2ms/step - loss: 0.7068 - accuracy: 0.7500 - val_loss: 0.7120 - val_accuracy: 0.7489 - lr: 1.0000e-04
Epoch 27/30
1/487 [..............................] - ETA: 1s - loss: 0.7231 - accuracy: 0.7529
30/487 [>.............................] - ETA: 0s - loss: 0.7056 - accuracy: 0.7495
58/487 [==>...........................] - ETA: 0s - loss: 0.7038 - accuracy: 0.7502
87/487 [====>.........................] - ETA: 0s - loss: 0.7070 - accuracy: 0.7499
115/487 [======>.......................] - ETA: 0s - loss: 0.7073 - accuracy: 0.7496
144/487 [=======>......................] - ETA: 0s - loss: 0.7060 - accuracy: 0.7497
172/487 [=========>....................] - ETA: 0s - loss: 0.7063 - accuracy: 0.7499
201/487 [===========>..................] - ETA: 0s - loss: 0.7059 - accuracy: 0.7496
230/487 [=============>................] - ETA: 0s - loss: 0.7051 - accuracy: 0.7503
258/487 [==============>...............] - ETA: 0s - loss: 0.7046 - accuracy: 0.7505
286/487 [================>.............] - ETA: 0s - loss: 0.7046 - accuracy: 0.7504
315/487 [==================>...........] - ETA: 0s - loss: 0.7054 - accuracy: 0.7501
344/487 [====================>.........] - ETA: 0s - loss: 0.7054 - accuracy: 0.7500
373/487 [=====================>........] - ETA: 0s - loss: 0.7054 - accuracy: 0.7500
402/487 [=======================>......] - ETA: 0s - loss: 0.7058 - accuracy: 0.7498
431/487 [=========================>....] - ETA: 0s - loss: 0.7059 - accuracy: 0.7500
460/487 [===========================>..] - ETA: 0s - loss: 0.7056 - accuracy: 0.7501
Epoch 27: val_loss improved from 0.71199 to 0.71041, saving model to unpruned_model/model_best.h5
Epoch 27: saving model to unpruned_model/model_last.h5
487/487 [==============================] - 1s 2ms/step - loss: 0.7054 - accuracy: 0.7503 - val_loss: 0.7104 - val_accuracy: 0.7490 - lr: 1.0000e-04
Epoch 28/30
1/487 [..............................] - ETA: 1s - loss: 0.7551 - accuracy: 0.7295
30/487 [>.............................] - ETA: 0s - loss: 0.7025 - accuracy: 0.7523
59/487 [==>...........................] - ETA: 0s - loss: 0.7031 - accuracy: 0.7531
87/487 [====>.........................] - ETA: 0s - loss: 0.7028 - accuracy: 0.7522
115/487 [======>.......................] - ETA: 0s - loss: 0.7035 - accuracy: 0.7514
143/487 [=======>......................] - ETA: 0s - loss: 0.7040 - accuracy: 0.7513
172/487 [=========>....................] - ETA: 0s - loss: 0.7034 - accuracy: 0.7512
201/487 [===========>..................] - ETA: 0s - loss: 0.7035 - accuracy: 0.7511
230/487 [=============>................] - ETA: 0s - loss: 0.7037 - accuracy: 0.7511
258/487 [==============>...............] - ETA: 0s - loss: 0.7039 - accuracy: 0.7510
287/487 [================>.............] - ETA: 0s - loss: 0.7033 - accuracy: 0.7513
315/487 [==================>...........] - ETA: 0s - loss: 0.7032 - accuracy: 0.7513
343/487 [====================>.........] - ETA: 0s - loss: 0.7033 - accuracy: 0.7513
372/487 [=====================>........] - ETA: 0s - loss: 0.7042 - accuracy: 0.7508
401/487 [=======================>......] - ETA: 0s - loss: 0.7038 - accuracy: 0.7510
429/487 [=========================>....] - ETA: 0s - loss: 0.7040 - accuracy: 0.7508
457/487 [===========================>..] - ETA: 0s - loss: 0.7039 - accuracy: 0.7507
486/487 [============================>.] - ETA: 0s - loss: 0.7039 - accuracy: 0.7507
Epoch 28: val_loss improved from 0.71041 to 0.70892, saving model to unpruned_model/model_best.h5
Epoch 28: saving model to unpruned_model/model_last.h5
487/487 [==============================] - 1s 2ms/step - loss: 0.7039 - accuracy: 0.7507 - val_loss: 0.7089 - val_accuracy: 0.7495 - lr: 1.0000e-04
Epoch 29/30
1/487 [..............................] - ETA: 1s - loss: 0.7498 - accuracy: 0.7275
29/487 [>.............................] - ETA: 0s - loss: 0.6990 - accuracy: 0.7520
57/487 [==>...........................] - ETA: 0s - loss: 0.6965 - accuracy: 0.7535
85/487 [====>.........................] - ETA: 0s - loss: 0.7007 - accuracy: 0.7522
114/487 [======>.......................] - ETA: 0s - loss: 0.7012 - accuracy: 0.7523
142/487 [=======>......................] - ETA: 0s - loss: 0.7021 - accuracy: 0.7518
171/487 [=========>....................] - ETA: 0s - loss: 0.7030 - accuracy: 0.7513
199/487 [===========>..................] - ETA: 0s - loss: 0.7034 - accuracy: 0.7508
226/487 [============>.................] - ETA: 0s - loss: 0.7040 - accuracy: 0.7508
255/487 [==============>...............] - ETA: 0s - loss: 0.7044 - accuracy: 0.7507
284/487 [================>.............] - ETA: 0s - loss: 0.7044 - accuracy: 0.7503
313/487 [==================>...........] - ETA: 0s - loss: 0.7044 - accuracy: 0.7503
342/487 [====================>.........] - ETA: 0s - loss: 0.7032 - accuracy: 0.7509
371/487 [=====================>........] - ETA: 0s - loss: 0.7029 - accuracy: 0.7510
399/487 [=======================>......] - ETA: 0s - loss: 0.7030 - accuracy: 0.7510
427/487 [=========================>....] - ETA: 0s - loss: 0.7026 - accuracy: 0.7512
456/487 [===========================>..] - ETA: 0s - loss: 0.7028 - accuracy: 0.7511
485/487 [============================>.] - ETA: 0s - loss: 0.7026 - accuracy: 0.7512
Epoch 29: val_loss improved from 0.70892 to 0.70775, saving model to unpruned_model/model_best.h5
Epoch 29: saving model to unpruned_model/model_last.h5
487/487 [==============================] - 1s 2ms/step - loss: 0.7026 - accuracy: 0.7512 - val_loss: 0.7077 - val_accuracy: 0.7497 - lr: 1.0000e-04
Epoch 30/30
1/487 [..............................] - ETA: 1s - loss: 0.7154 - accuracy: 0.7471
30/487 [>.............................] - ETA: 0s - loss: 0.7020 - accuracy: 0.7512
59/487 [==>...........................] - ETA: 0s - loss: 0.7019 - accuracy: 0.7519
88/487 [====>.........................] - ETA: 0s - loss: 0.7010 - accuracy: 0.7517
117/487 [======>.......................] - ETA: 0s - loss: 0.7033 - accuracy: 0.7509
145/487 [=======>......................] - ETA: 0s - loss: 0.7039 - accuracy: 0.7505
174/487 [=========>....................] - ETA: 0s - loss: 0.7031 - accuracy: 0.7506
202/487 [===========>..................] - ETA: 0s - loss: 0.7033 - accuracy: 0.7505
231/487 [=============>................] - ETA: 0s - loss: 0.7030 - accuracy: 0.7505
260/487 [===============>..............] - ETA: 0s - loss: 0.7012 - accuracy: 0.7511
289/487 [================>.............] - ETA: 0s - loss: 0.7009 - accuracy: 0.7512
317/487 [==================>...........] - ETA: 0s - loss: 0.7009 - accuracy: 0.7512
345/487 [====================>.........] - ETA: 0s - loss: 0.7007 - accuracy: 0.7513
374/487 [======================>.......] - ETA: 0s - loss: 0.7002 - accuracy: 0.7517
402/487 [=======================>......] - ETA: 0s - loss: 0.7001 - accuracy: 0.7518
430/487 [=========================>....] - ETA: 0s - loss: 0.7002 - accuracy: 0.7519
459/487 [===========================>..] - ETA: 0s - loss: 0.7007 - accuracy: 0.7518
Epoch 30: val_loss improved from 0.70775 to 0.70633, saving model to unpruned_model/model_best.h5
Epoch 30: saving model to unpruned_model/model_last.h5
487/487 [==============================] - 1s 2ms/step - loss: 0.7013 - accuracy: 0.7517 - val_loss: 0.7063 - val_accuracy: 0.7505 - lr: 1.0000e-04
<keras.src.callbacks.History at 0x7f700c932710>
Train the pruned model#
This time we’ll use the Tensorflow model optimization sparsity to train a sparse model (forcing many weights to ‘0’). In this instance, the target sparsity is 75%
from tensorflow_model_optimization.python.core.sparsity.keras import prune, pruning_callbacks, pruning_schedule
from tensorflow_model_optimization.sparsity.keras import strip_pruning
pruned_model = Sequential()
pruned_model.add(Dense(64, input_shape=(16,), name="fc1", kernel_initializer="lecun_uniform", kernel_regularizer=l1(0.0001)))
pruned_model.add(Activation(activation="relu", name="relu1"))
pruned_model.add(Dense(32, name="fc2", kernel_initializer="lecun_uniform", kernel_regularizer=l1(0.0001)))
pruned_model.add(Activation(activation="relu", name="relu2"))
pruned_model.add(Dense(32, name="fc3", kernel_initializer="lecun_uniform", kernel_regularizer=l1(0.0001)))
pruned_model.add(Activation(activation="relu", name="relu3"))
pruned_model.add(Dense(5, name="output", kernel_initializer="lecun_uniform", kernel_regularizer=l1(0.0001)))
pruned_model.add(Activation(activation="softmax", name="softmax"))
pruning_params = {"pruning_schedule": pruning_schedule.ConstantSparsity(0.75, begin_step=2000, frequency=100)}
pruned_model = prune.prune_low_magnitude(pruned_model, **pruning_params)
---------------------------------------------------------------------------
ModuleNotFoundError Traceback (most recent call last)
Cell In[7], line 1
----> 1 from tensorflow_model_optimization.python.core.sparsity.keras import prune, pruning_callbacks, pruning_schedule
2 from tensorflow_model_optimization.sparsity.keras import strip_pruning
4 pruned_model = Sequential()
ModuleNotFoundError: No module named 'tensorflow_model_optimization'
We’ll use the same settings as before: Adam optimizer with categorical crossentropy loss.
The callbacks will decay the learning rate and save the model into a directory pruned_model.
adam = Adam(lr=0.0001)
pruned_model.compile(optimizer=adam, loss=["categorical_crossentropy"], metrics=["accuracy"])
callbacks = all_callbacks(
stop_patience=1000,
lr_factor=0.5,
lr_patience=10,
lr_epsilon=0.000001,
lr_cooldown=2,
lr_minimum=0.0000001,
outputDir="pruned_model",
)
callbacks.callbacks.append(pruning_callbacks.UpdatePruningStep())
pruned_model.fit(
X_train_val,
y_train_val,
batch_size=1024,
epochs=30,
validation_split=0.25,
shuffle=True,
callbacks=callbacks.callbacks,
verbose=0,
)
# Save the model again but with the pruning 'stripped' to use the regular layer types
pruned_model = strip_pruning(pruned_model)
pruned_model.save("pruned_model/model_best.h5")
Check sparsity#
Make a quick check that the model was indeed trained sparse. We’ll just make a histogram of the weights of the 1st layer, and hopefully observe a large peak in the bin containing ‘0’. Note logarithmic y axis.
bins = np.arange(-2, 2, 0.04)
w_unpruned = model.layers[0].weights[0].numpy().flatten()
w_pruned = pruned_model.layers[0].weights[0].numpy().flatten()
plt.figure(figsize=(7, 7))
plt.hist(w_unpruned, bins=bins, alpha=0.7, label="Unpruned layer 1")
plt.hist(w_pruned, bins=bins, alpha=0.7, label="Pruned layer 1")
plt.xlabel("Weight value")
plt.ylabel("Number of weights")
plt.semilogy()
plt.legend()
print(f"Sparsity of unpruned model layer 1: {np.sum(w_unpruned==0)*100/np.size(w_unpruned)}% zeros")
print(f"Sparsity of pruned model layer 1: {np.sum(w_pruned==0)*100/np.size(w_pruned)}% zeros")
plt.show()
Check performance#
How does this 75% sparse model compare against the unpruned model? Let’s report the accuracy and make a ROC curve. The pruned model is shown with solid lines, the unpruned model is shown with dashed lines.
import plotting
import matplotlib.pyplot as plt
from sklearn.metrics import accuracy_score
from tensorflow.keras.models import load_model
unpruned_model = load_model("unpruned_model/model_best.h5")
y_ref = unpruned_model.predict(X_test, verbose=0)
y_prune = pruned_model.predict(X_test, verbose=0)
print("Accuracy unpruned: {}".format(accuracy_score(np.argmax(y_test, axis=1), np.argmax(y_ref, axis=1))))
print("Accuracy pruned: {}".format(accuracy_score(np.argmax(y_test, axis=1), np.argmax(y_prune, axis=1))))
fig, ax = plt.subplots(figsize=(9, 9))
_ = plotting.make_roc(y_test, y_ref, classes)
plt.gca().set_prop_cycle(None) # reset the colors
_ = plotting.make_roc(y_test, y_prune, classes, linestyle="--")
from matplotlib.lines import Line2D
lines = [Line2D([0], [0], ls="-"), Line2D([0], [0], ls="--")]
from matplotlib.legend import Legend
leg = Legend(ax, lines, labels=["Unpruned", "Pruned"], loc="lower right", frameon=False)
ax.add_artist(leg)
plt.show()