Cainvas
Model Files
mnist_model.onnx
Model
ONNX
deepSea Compiled Models
mnist_model.exe
deepSea
Ubuntu

PyTorch MNIST Vision App

Credit: AITS Cainvas Community

mnist.gif

Photo by Denis Dmitriev on YouTube

  1. Use MNIST Dataset,
  2. To Train NN model with PyTorch and
  3. Compile NN model with deepC

To run a Code Cell you can click on the ⏯ Run button in the Navigation Bar above or type Shift + Enter

In [1]:
%pylab inline
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.utils.data.dataloader as dataloader
import torch.optim as optim

from torch.utils.data import TensorDataset
from torch.autograd import Variable
from torchvision import transforms
from torchvision.datasets import MNIST

SEED = 1

# CUDA?
cuda = torch.cuda.is_available()

# For reproducibility
torch.manual_seed(SEED)

if cuda:
    torch.cuda.manual_seed(SEED)
Populating the interactive namespace from numpy and matplotlib

Dataset

Download, divide and inspect MNIST dataset

In [2]:
!wget -N 'https://cainvas-static.s3.amazonaws.com/media/user_data/cainvas-admin/MNIST.zip'
!unzip -o MNIST.zip
!rm MNIST.zip
--2020-08-26 15:37:33--  https://cainvas-static.s3.amazonaws.com/media/user_data/cainvas-admin/MNIST.zip
Resolving cainvas-static.s3.amazonaws.com (cainvas-static.s3.amazonaws.com)... 52.219.66.80
Connecting to cainvas-static.s3.amazonaws.com (cainvas-static.s3.amazonaws.com)|52.219.66.80|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 34990163 (33M) [application/zip]
Saving to: ‘MNIST.zip’

MNIST.zip           100%[===================>]  33.37M  61.9MB/s    in 0.5s    

2020-08-26 15:37:34 (61.9 MB/s) - ‘MNIST.zip’ saved [34990163/34990163]

Archive:  MNIST.zip
  inflating: MNIST/data/MNIST/processed/test.pt  
  inflating: MNIST/data/MNIST/processed/training.pt  
  inflating: MNIST/data/MNIST/raw/t10k-images-idx3-ubyte  
  inflating: MNIST/data/MNIST/raw/t10k-images-idx3-ubyte.gz  
  inflating: MNIST/data/MNIST/raw/t10k-labels-idx1-ubyte  
 extracting: MNIST/data/MNIST/raw/t10k-labels-idx1-ubyte.gz  
  inflating: MNIST/data/MNIST/raw/train-images-idx3-ubyte  
  inflating: MNIST/data/MNIST/raw/train-images-idx3-ubyte.gz  
  inflating: MNIST/data/MNIST/raw/train-labels-idx1-ubyte  
  inflating: MNIST/data/MNIST/raw/train-labels-idx1-ubyte.gz  
In [3]:
MNIST_dataset = 'MNIST/data'
train = MNIST(MNIST_dataset, train=True, download=True, transform=transforms.Compose([
    transforms.ToTensor(), # ToTensor does min-max normalization. 
]), )

test = MNIST(MNIST_dataset, train=False, download=True, transform=transforms.Compose([
    transforms.ToTensor(), # ToTensor does min-max normalization. 
]), )

# Create DataLoader
dataloader_args = dict(shuffle=True, batch_size=256,num_workers=4, pin_memory=True) if cuda else dict(shuffle=True, batch_size=64)
train_loader = dataloader.DataLoader(train, **dataloader_args)
test_loader = dataloader.DataLoader(test, **dataloader_args)
In [4]:
train_data = train.train_data
train_data = train.transform(train_data.numpy())
# print(train_data[:, 0, :].shape)
# for px in train_data[:, 0, :]:
#   print(px, ' ')

print('[Train]')
print(' - Numpy Shape:', train.train_data.cpu().numpy().shape)
print(' - Tensor Shape:', train.train_data.size())
print(' - min:', torch.min(train_data))
print(' - max:', torch.max(train_data))
print(' - mean:', torch.mean(train_data))
print(' - std:', torch.std(train_data))
print(' - var:', torch.var(train_data))
/opt/tljh/user/lib/python3.7/site-packages/torchvision/datasets/mnist.py:53: UserWarning: train_data has been renamed data
  warnings.warn("train_data has been renamed data")
[Train]
 - Numpy Shape: (60000, 28, 28)
 - Tensor Shape: torch.Size([60000, 28, 28])
 - min: tensor(0.)
 - max: tensor(1.)
 - mean: tensor(0.1306)
 - std: tensor(0.3081)
 - var: tensor(0.0949)
In [5]:
from PIL import Image
img_data = train_data.numpy()[:, 1, :]
imshow(img_data)
#print(type(img_data))
#img = Image.fromarray(img_data, 'RGB')
#img.show()
#print(img)
#print(data)
#for px in data.flatten():
#    print(px, ' ', end='')
Out[5]:
<matplotlib.image.AxesImage at 0x7fb51882b630>

design NN Model

In [6]:
# One hidden Layer NN
class Model(nn.Module):
    def __init__(self):
        batch_size = 100
        super(Model, self).__init__()
        self.fc = nn.Linear(784, batch_size)
        self.fc2 = nn.Linear(batch_size, 10)

    def forward(self, x):
        x = x.view((-1, 784))
        h = F.relu(self.fc(x))
        h = self.fc2(h)
        return F.log_softmax(h)

model = Model()
if cuda:
    model.cuda() # CUDA!
optimizer = optim.Adam(model.parameters(), lr=1e-3)

Draw NN Model

Talk about PDF Book by Michael Nielsen / Dec 2019

Train

Training time:

  • CPU, about 1 minute and 30 seconds
  • GPU, about 10 seconds
In [7]:
EPOCHS = 10
losses = []

model.train()
for epoch in range(EPOCHS):
    for batch_idx, (data, target) in enumerate(train_loader):
        # Get Samples
        data, target = Variable(data), Variable(target)
        
        if cuda:
            data, target = data.cuda(), target.cuda()
        
        # Init
        optimizer.zero_grad()

        # Predict
        y_pred = model(data) 

        # Calculate loss
        loss = F.cross_entropy(y_pred, target)
        losses.append(loss.cpu().data)
        # Backpropagation
        loss.backward()
        optimizer.step()
        
        
        # Display
        if batch_idx % 100 == 1:
            print('\r Train Epoch: {}/{} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
                epoch+1,
                EPOCHS,
                batch_idx * len(data), 
                len(train_loader.dataset),
                100. * batch_idx / len(train_loader), 
                loss.cpu().data), 
                end='')
    # Eval
    evaluate_x = Variable(test_loader.dataset.data.type_as(torch.FloatTensor()))
    evaluate_y = Variable(test_loader.dataset.targets)
    if cuda:
        evaluate_x, evaluate_y = evaluate_x.cuda(), evaluate_y.cuda()

    model.eval()
    output = model(evaluate_x)
    pred = output.data.max(1)[1]
    d = pred.eq(evaluate_y.data).cpu()
    accuracy = d.sum().item()/d.size().numel()
    
    print('\r Train Epoch: {}/{} [{}/{} ({:.0f}%)]\tLoss: {:.6f}\t Test Accuracy: {:.4f}%'.format(
        epoch+1,
        EPOCHS,
        len(train_loader.dataset), 
        len(train_loader.dataset),
        100. * batch_idx / len(train_loader), 
        loss.cpu().data,
        accuracy*100,
        end=''))
/opt/tljh/user/lib/python3.7/site-packages/ipykernel_launcher.py:13: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.
  del sys.path[0]
 Train Epoch: 1/10 [60000/60000 (100%)]	Loss: 0.174932	 Test Accuracy: 94.2300%
 Train Epoch: 2/10 [60000/60000 (100%)]	Loss: 0.055013	 Test Accuracy: 95.5500%
 Train Epoch: 3/10 [60000/60000 (100%)]	Loss: 0.041712	 Test Accuracy: 96.4900%
 Train Epoch: 4/10 [60000/60000 (100%)]	Loss: 0.063259	 Test Accuracy: 96.7200%
 Train Epoch: 5/10 [60000/60000 (100%)]	Loss: 0.254731	 Test Accuracy: 97.1300%
 Train Epoch: 6/10 [60000/60000 (100%)]	Loss: 0.016007	 Test Accuracy: 97.3600%
 Train Epoch: 7/10 [60000/60000 (100%)]	Loss: 0.003279	 Test Accuracy: 97.2900%
 Train Epoch: 8/10 [60000/60000 (100%)]	Loss: 0.049192	 Test Accuracy: 97.4300%
 Train Epoch: 9/10 [60000/60000 (100%)]	Loss: 0.222993	 Test Accuracy: 97.2200%
 Train Epoch: 10/10 [60000/60000 (100%)]	Loss: 0.006774	 Test Accuracy: 97.7100%
In [8]:
plot(losses)
Out[8]:
[<matplotlib.lines.Line2D at 0x7fb513958438>]

Evaluate

In [9]:
evaluate_x = Variable(test_loader.dataset.data.type_as(torch.FloatTensor()))
evaluate_y = Variable(test_loader.dataset.targets)
if cuda:
    evaluate_x, evaluate_y = evaluate_x.cuda(), evaluate_y.cuda()

model.eval()
output = model(evaluate_x)
pred = output.data.max(1)[1]
d = pred.eq(evaluate_y.data).cpu()
accuracy = d.sum().item()/d.size().numel()

print('Accuracy:', accuracy*100.0)
Accuracy: 97.71
/opt/tljh/user/lib/python3.7/site-packages/ipykernel_launcher.py:13: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.
  del sys.path[0]

Save Model

In [10]:
dummy_input = torch.randn(train_data.shape)
torch.onnx.export(model, (dummy_input), "./mnist_model.onnx", verbose=True)
graph(%0 : Float(28, 60000, 28),
      %fc.weight : Float(100, 784),
      %fc.bias : Float(100),
      %fc2.weight : Float(10, 100),
      %fc2.bias : Float(10)):
  %5 : Tensor = onnx::Constant[value=  -1  784 [ Variable[CPULongType]{2} ]](), scope: Model
  %6 : Float(60000, 784) = onnx::Reshape(%0, %5), scope: Model # <ipython-input-6-283560e84c3e>:10:0
  %7 : Float(60000, 100) = onnx::Gemm[alpha=1, beta=1, transB=1](%6, %fc.weight, %fc.bias), scope: Model/Linear[fc] # /opt/tljh/user/lib/python3.7/site-packages/torch/nn/functional.py:1369:0
  %8 : Float(60000, 100) = onnx::Relu(%7), scope: Model # /opt/tljh/user/lib/python3.7/site-packages/torch/nn/functional.py:913:0
  %9 : Float(60000, 10) = onnx::Gemm[alpha=1, beta=1, transB=1](%8, %fc2.weight, %fc2.bias), scope: Model/Linear[fc2] # /opt/tljh/user/lib/python3.7/site-packages/torch/nn/functional.py:1369:0
  %10 : Float(60000, 10) = onnx::LogSoftmax[axis=1](%9), scope: Model # /opt/tljh/user/lib/python3.7/site-packages/torch/nn/functional.py:1316:0
  return (%10)

/opt/tljh/user/lib/python3.7/site-packages/ipykernel_launcher.py:13: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.
  del sys.path[0]

Compile Model with deepC

In [11]:
!deepCC mnist_model.onnx
reading onnx model from file  mnist_model.onnx
Model info:
  ir_vesion :  4 
  doc       : 
running DNNC graph sanity check ... passed.
Writing C++ file  mnist_model_deepC/mnist_model.cpp
INFO (ONNX): model files are ready in dir mnist_model_deepC
g++ -O3 -I. -I/opt/tljh/user/lib/python3.7/site-packages/deepC-0.13-py3.7-linux-x86_64.egg/deepC/include -isystem /opt/tljh/user/lib/python3.7/site-packages/deepC-0.13-py3.7-linux-x86_64.egg/deepC/packages/eigen-eigen-323c052e1731 mnist_model_deepC/mnist_model.cpp -o mnist_model_deepC/mnist_model.exe
Model executable  mnist_model_deepC/mnist_model.exe

Test Model Binary

Use Model

In [12]:
!./mnist.deepC/mnist.exe
/bin/sh: ./mnist.deepC/mnist.exe: No such file or directory

Display an Image

In [13]:
img_data = data[1][0].numpy();
np.savetxt('img.data', img_data.flatten())
In [14]:
img = Image.fromarray(img_data, 'RGB')
#print(img)
img.show()
imshow(img_data)
Out[14]:
<matplotlib.image.AxesImage at 0x7fb513ac7e48>

Run mode prediction

on the image shown above

In [15]:
!./mnist_model_deepC/mnist_model.exe img.data
reading file img.data.
writing file 10.out.
In [16]:
nn_out = np.loadtxt('10.out')
print ("Model prediction is DIGIT : ", np.argmax(nn_out))
Model prediction is DIGIT :  3