Cainvas

Kannada letter Identification

Credit: AITS Cainvas Community

Photo by Alexandra on Dribbble

In [1]:
import time
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.metrics import accuracy_score
import tensorflow as tf
from tensorflow.keras import datasets, layers, models
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import ReduceLROnPlateau
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.preprocessing.image import ImageDataGenerator

Unzipping data

In [2]:
!wget https://cainvas-static.s3.amazonaws.com/media/user_data/SiddharthGan/Kannada-MNIST.zip

!unzip -qo Kannada-MNIST.zip

# zip folder is not needed anymore
!rm Kannada-MNIST.zip
--2021-09-15 11:45:46--  https://cainvas-static.s3.amazonaws.com/media/user_data/SiddharthGan/Kannada-MNIST.zip
Resolving cainvas-static.s3.amazonaws.com (cainvas-static.s3.amazonaws.com)... 52.219.158.59
Connecting to cainvas-static.s3.amazonaws.com (cainvas-static.s3.amazonaws.com)|52.219.158.59|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 13014944 (12M) [application/zip]
Saving to: ‘Kannada-MNIST.zip’

Kannada-MNIST.zip   100%[===================>]  12.41M  --.-KB/s    in 0.1s    

2021-09-15 11:45:46 (115 MB/s) - ‘Kannada-MNIST.zip’ saved [13014944/13014944]

Data Preprocess

In [3]:
train_set = pd.read_csv('train.csv')
test_set = pd.read_csv("test.csv")
dig_set = pd.read_csv("Dig-MNIST.csv")
In [4]:
train_df=pd.read_csv("train.csv")
test_df=pd.read_csv("test.csv")

x_train=train_df.drop(["label"], axis=1).values.astype('float32')
y_train=train_df["label"].values.astype("int32")

x_test=test_df.drop(["id"],axis=1).values.astype('float32')
In [5]:
test_df.columns
Out[5]:
Index(['id', 'pixel0', 'pixel1', 'pixel2', 'pixel3', 'pixel4', 'pixel5',
       'pixel6', 'pixel7', 'pixel8',
       ...
       'pixel774', 'pixel775', 'pixel776', 'pixel777', 'pixel778', 'pixel779',
       'pixel780', 'pixel781', 'pixel782', 'pixel783'],
      dtype='object', length=785)
In [6]:
x_train=x_train.reshape(x_train.shape[0], 28, 28)/255.0
x_test=x_test.reshape(x_test.shape[0], 28, 28)/255.0

Train-Test Split

In [7]:
from sklearn.model_selection import train_test_split
x_train,x_val,y_train,y_val=train_test_split(x_train, y_train, test_size=0.2)
In [8]:
print(x_train.shape)
print(y_train.shape)
print(x_val.shape)
print(y_val.shape)
print(x_test.shape)
(48000, 28, 28)
(48000,)
(12000, 28, 28)
(12000,)
(5000, 28, 28)
In [9]:
x_train=x_train.reshape(x_train.shape[0], 28,28,1)
x_test=x_test.reshape(x_test.shape[0],28,28,1)
x_val=x_val.reshape(x_val.shape[0],28,28,1)

Model Architecture

In [10]:
from tensorflow.keras.layers import Conv2D, MaxPooling2D, Input, Dense, Activation, ReLU, Flatten, Dropout, BatchNormalization
from tensorflow.keras.models import Model

X=Input(shape=[28,28,1])
x=Conv2D(16, (3,3), strides=1, padding="same", name="conv1")(X)
x=BatchNormalization(momentum=0.1, epsilon=1e-5, gamma_initializer="uniform", name="batch1")(x)
x=Activation('relu',name='relu1')(x)
x=Dropout(0.1)(x)

x=Conv2D(32, (3,3), strides=1, padding="same", name="conv2")(x)
x=BatchNormalization(momentum=0.15, epsilon=1e-5, gamma_initializer="uniform", name="batch2")(x)
x=Activation('relu',name='relu2')(x)
x=Dropout(0.15)(x)
x=MaxPooling2D(pool_size=2, strides=2, padding="same", name="max2")(x)

x=Conv2D(64, (5,5), strides=1, padding ="same", name="conv3")(x)
x=BatchNormalization(momentum=0.17, epsilon=1e-5, gamma_initializer="uniform", name="batch3")(x)
x=Activation('relu', name="relu3")(x)
x=MaxPooling2D(pool_size=2, strides=2, padding="same", name="max3")(x)

x=Conv2D(64, (5,5), strides=1, padding="same", name="conv4")(x)
x=BatchNormalization(momentum=0.15, epsilon=1e-5, gamma_initializer="uniform", name="batch4")(x)
x=Activation('relu', name="relu4")(x)
x=Dropout(0.17)(x)

x=Conv2D(32, (3,3), strides=1, padding="same", name="conv5")(x)
x=BatchNormalization(momentum=0.15, epsilon=1e-5, gamma_initializer="uniform", name="batch5")(x)
x=Activation('relu', name='relu5')(x)
#x=MaxPooling2D(pool_size=2, strides=2, padding="same", name="max5")(x)
x=Dropout(0.2)(x)

x=Conv2D(16, (3,3), strides=1, padding="same", name="conv6")(x)
x=BatchNormalization(momentum=0.15, epsilon=1e-5, gamma_initializer="uniform", name="batch6" )(x)
x=Activation('relu', name="relu6")(x)
x=Dropout(0.05)(x)
#x=MaxPooling2D(pool_size=2, strides=2, padding="same", name="max6")(x)

x=Flatten()(x)
x=Dense(50, name="Dense1")(x)
x=Activation('relu', name='relu7')(x)
x=Dropout(0.05)(x)
x=Dense(25, name="Dense2")(x)
x=Activation('relu', name='relu8')(x)
x=Dropout(0.03)(x)
x=Dense(10, name="Dense3")(x)
x=Activation('softmax')(x)

model=Model(inputs=X, outputs=x)

print(model.summary())
Model: "functional_1"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
input_1 (InputLayer)         [(None, 28, 28, 1)]       0         
_________________________________________________________________
conv1 (Conv2D)               (None, 28, 28, 16)        160       
_________________________________________________________________
batch1 (BatchNormalization)  (None, 28, 28, 16)        64        
_________________________________________________________________
relu1 (Activation)           (None, 28, 28, 16)        0         
_________________________________________________________________
dropout (Dropout)            (None, 28, 28, 16)        0         
_________________________________________________________________
conv2 (Conv2D)               (None, 28, 28, 32)        4640      
_________________________________________________________________
batch2 (BatchNormalization)  (None, 28, 28, 32)        128       
_________________________________________________________________
relu2 (Activation)           (None, 28, 28, 32)        0         
_________________________________________________________________
dropout_1 (Dropout)          (None, 28, 28, 32)        0         
_________________________________________________________________
max2 (MaxPooling2D)          (None, 14, 14, 32)        0         
_________________________________________________________________
conv3 (Conv2D)               (None, 14, 14, 64)        51264     
_________________________________________________________________
batch3 (BatchNormalization)  (None, 14, 14, 64)        256       
_________________________________________________________________
relu3 (Activation)           (None, 14, 14, 64)        0         
_________________________________________________________________
max3 (MaxPooling2D)          (None, 7, 7, 64)          0         
_________________________________________________________________
conv4 (Conv2D)               (None, 7, 7, 64)          102464    
_________________________________________________________________
batch4 (BatchNormalization)  (None, 7, 7, 64)          256       
_________________________________________________________________
relu4 (Activation)           (None, 7, 7, 64)          0         
_________________________________________________________________
dropout_2 (Dropout)          (None, 7, 7, 64)          0         
_________________________________________________________________
conv5 (Conv2D)               (None, 7, 7, 32)          18464     
_________________________________________________________________
batch5 (BatchNormalization)  (None, 7, 7, 32)          128       
_________________________________________________________________
relu5 (Activation)           (None, 7, 7, 32)          0         
_________________________________________________________________
dropout_3 (Dropout)          (None, 7, 7, 32)          0         
_________________________________________________________________
conv6 (Conv2D)               (None, 7, 7, 16)          4624      
_________________________________________________________________
batch6 (BatchNormalization)  (None, 7, 7, 16)          64        
_________________________________________________________________
relu6 (Activation)           (None, 7, 7, 16)          0         
_________________________________________________________________
dropout_4 (Dropout)          (None, 7, 7, 16)          0         
_________________________________________________________________
flatten (Flatten)            (None, 784)               0         
_________________________________________________________________
Dense1 (Dense)               (None, 50)                39250     
_________________________________________________________________
relu7 (Activation)           (None, 50)                0         
_________________________________________________________________
dropout_5 (Dropout)          (None, 50)                0         
_________________________________________________________________
Dense2 (Dense)               (None, 25)                1275      
_________________________________________________________________
relu8 (Activation)           (None, 25)                0         
_________________________________________________________________
dropout_6 (Dropout)          (None, 25)                0         
_________________________________________________________________
Dense3 (Dense)               (None, 10)                260       
_________________________________________________________________
activation (Activation)      (None, 10)                0         
=================================================================
Total params: 223,297
Trainable params: 222,849
Non-trainable params: 448
_________________________________________________________________
None

Training the model

In [11]:
from tensorflow.keras.preprocessing. image import ImageDataGenerator
from tensorflow.keras.callbacks import ModelCheckpoint
from tensorflow.keras.optimizers import Adam

checkpoint=ModelCheckpoint('best.h5', monitor='val_acc', verbose=1, save_best_only=True, mode="max")

epochs=20
lr=0.002
optimizer=Adam(lr=lr, decay=lr/(epochs*1.5))

model.compile(optimizer=optimizer, loss="sparse_categorical_crossentropy",metrics=["accuracy"])

datagenerator=ImageDataGenerator( rotation_range=9, zoom_range=0.25, width_shift_range=0.25, height_shift_range=0.25)

datagenerator.fit(x_train)
batch_size=32
history=model.fit_generator(datagenerator.flow(x_train, y_train, batch_size=batch_size), epochs=epochs, validation_data=(x_val, y_val), verbose=2,steps_per_epoch=x_train.shape[0]//batch_size, callbacks=[checkpoint])
WARNING:tensorflow:From <ipython-input-11-46ddd9518f2c>:17: Model.fit_generator (from tensorflow.python.keras.engine.training) is deprecated and will be removed in a future version.
Instructions for updating:
Please use Model.fit, which supports generators.
Epoch 1/20
WARNING:tensorflow:Can save best model only with val_acc available, skipping.
1500/1500 - 12s - loss: 0.4146 - accuracy: 0.8557 - val_loss: 0.0531 - val_accuracy: 0.9833
Epoch 2/20
WARNING:tensorflow:Can save best model only with val_acc available, skipping.
1500/1500 - 11s - loss: 0.1308 - accuracy: 0.9613 - val_loss: 0.0644 - val_accuracy: 0.9827
Epoch 3/20
WARNING:tensorflow:Can save best model only with val_acc available, skipping.
1500/1500 - 11s - loss: 0.1034 - accuracy: 0.9702 - val_loss: 0.0316 - val_accuracy: 0.9914
Epoch 4/20
WARNING:tensorflow:Can save best model only with val_acc available, skipping.
1500/1500 - 11s - loss: 0.0892 - accuracy: 0.9738 - val_loss: 0.0293 - val_accuracy: 0.9919
Epoch 5/20
WARNING:tensorflow:Can save best model only with val_acc available, skipping.
1500/1500 - 11s - loss: 0.0782 - accuracy: 0.9775 - val_loss: 0.0197 - val_accuracy: 0.9943
Epoch 6/20
WARNING:tensorflow:Can save best model only with val_acc available, skipping.
1500/1500 - 11s - loss: 0.0715 - accuracy: 0.9789 - val_loss: 0.0165 - val_accuracy: 0.9958
Epoch 7/20
WARNING:tensorflow:Can save best model only with val_acc available, skipping.
1500/1500 - 11s - loss: 0.0679 - accuracy: 0.9803 - val_loss: 0.0157 - val_accuracy: 0.9962
Epoch 8/20
WARNING:tensorflow:Can save best model only with val_acc available, skipping.
1500/1500 - 11s - loss: 0.0613 - accuracy: 0.9823 - val_loss: 0.0212 - val_accuracy: 0.9941
Epoch 9/20
WARNING:tensorflow:Can save best model only with val_acc available, skipping.
1500/1500 - 11s - loss: 0.0567 - accuracy: 0.9837 - val_loss: 0.0180 - val_accuracy: 0.9954
Epoch 10/20
WARNING:tensorflow:Can save best model only with val_acc available, skipping.
1500/1500 - 11s - loss: 0.0552 - accuracy: 0.9833 - val_loss: 0.0156 - val_accuracy: 0.9951
Epoch 11/20
WARNING:tensorflow:Can save best model only with val_acc available, skipping.
1500/1500 - 11s - loss: 0.0516 - accuracy: 0.9842 - val_loss: 0.0165 - val_accuracy: 0.9954
Epoch 12/20
WARNING:tensorflow:Can save best model only with val_acc available, skipping.
1500/1500 - 11s - loss: 0.0497 - accuracy: 0.9852 - val_loss: 0.0138 - val_accuracy: 0.9966
Epoch 13/20
WARNING:tensorflow:Can save best model only with val_acc available, skipping.
1500/1500 - 11s - loss: 0.0467 - accuracy: 0.9866 - val_loss: 0.0131 - val_accuracy: 0.9964
Epoch 14/20
WARNING:tensorflow:Can save best model only with val_acc available, skipping.
1500/1500 - 11s - loss: 0.0485 - accuracy: 0.9861 - val_loss: 0.0151 - val_accuracy: 0.9958
Epoch 15/20
WARNING:tensorflow:Can save best model only with val_acc available, skipping.
1500/1500 - 11s - loss: 0.0465 - accuracy: 0.9859 - val_loss: 0.0125 - val_accuracy: 0.9964
Epoch 16/20
WARNING:tensorflow:Can save best model only with val_acc available, skipping.
1500/1500 - 11s - loss: 0.0425 - accuracy: 0.9872 - val_loss: 0.0169 - val_accuracy: 0.9956
Epoch 17/20
WARNING:tensorflow:Can save best model only with val_acc available, skipping.
1500/1500 - 11s - loss: 0.0395 - accuracy: 0.9876 - val_loss: 0.0125 - val_accuracy: 0.9967
Epoch 18/20
WARNING:tensorflow:Can save best model only with val_acc available, skipping.
1500/1500 - 11s - loss: 0.0419 - accuracy: 0.9875 - val_loss: 0.0139 - val_accuracy: 0.9967
Epoch 19/20
WARNING:tensorflow:Can save best model only with val_acc available, skipping.
1500/1500 - 11s - loss: 0.0383 - accuracy: 0.9883 - val_loss: 0.0137 - val_accuracy: 0.9963
Epoch 20/20
WARNING:tensorflow:Can save best model only with val_acc available, skipping.
1500/1500 - 11s - loss: 0.0397 - accuracy: 0.9876 - val_loss: 0.0147 - val_accuracy: 0.9958

Accuracy Loss Graphs

In [12]:
plt.plot(history.history['accuracy'], label='accuracy')
plt.plot(history.history['val_accuracy'], label = 'val_accuracy')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.legend(loc='lower right')
Out[12]:
<matplotlib.legend.Legend at 0x7f40cf4fa5f8>
In [13]:
plt.plot(history.history['loss'], label='loss')
plt.plot(history.history['val_loss'], label = 'val_loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend(loc='lower right')
Out[13]:
<matplotlib.legend.Legend at 0x7f40cf43e080>

Result and Predictions

In [14]:
results=model.predict(x_test)
In [15]:
results=np.argmax(results, axis=1)
In [16]:
results
Out[16]:
array([3, 0, 2, ..., 1, 6, 3])
In [17]:
model.save('model.h5')

Deep CC

In [18]:
!deepCC model.h5
[INFO]
Reading [keras model] 'model.h5'
[SUCCESS]
Saved 'model_deepC/model.onnx'
[INFO]
Reading [onnx model] 'model_deepC/model.onnx'
[INFO]
Model info:
  ir_vesion : 5
  doc       : 
[WARNING]
[ONNX]: graph-node conv1's attribute auto_pad has no meaningful data.
[WARNING]
[ONNX]: graph-node conv2's attribute auto_pad has no meaningful data.
[WARNING]
[ONNX]: graph-node conv3's attribute auto_pad has no meaningful data.
[WARNING]
[ONNX]: graph-node conv4's attribute auto_pad has no meaningful data.
[WARNING]
[ONNX]: graph-node conv5's attribute auto_pad has no meaningful data.
[WARNING]
[ONNX]: graph-node conv6's attribute auto_pad has no meaningful data.
[WARNING]
[ONNX]: terminal (input/output) input_1's shape is less than 1. Changing it to 1.
[WARNING]
[ONNX]: terminal (input/output) activation's shape is less than 1. Changing it to 1.
[INFO]
Running DNNC graph sanity check ...
[SUCCESS]
Passed sanity check.
[INFO]
Writing C++ file 'model_deepC/model.cpp'
[INFO]
deepSea model files are ready in 'model_deepC/' 
[RUNNING COMMAND]
g++ -std=c++11 -O3 -fno-rtti -fno-exceptions -I. -I/opt/tljh/user/lib/python3.7/site-packages/deepC-0.13-py3.7-linux-x86_64.egg/deepC/include -isystem /opt/tljh/user/lib/python3.7/site-packages/deepC-0.13-py3.7-linux-x86_64.egg/deepC/packages/eigen-eigen-323c052e1731 "model_deepC/model.cpp" -D_AITS_MAIN -o "model_deepC/model.exe"
[RUNNING COMMAND]
size "model_deepC/model.exe"
   text	   data	    bss	    dec	    hex	filename
1075765	   3784	    760	1080309	 107bf5	model_deepC/model.exe
[SUCCESS]
Saved model as executable "model_deepC/model.exe"