mnist_tensorflow.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "mnist_tensorflow_linear.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python2",
      "display_name": "Python 2"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZMrwNrEalWEd",
        "colab_type": "text"
      },
      "source": [
        "## Comparisons for Adam, AdamW, LAMB"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8Vjw5_PkNWV8",
        "colab_type": "code",
        "outputId": "19c1bdbc-9c50-4bdb-a5a8-d206d8cd68db",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 411
        }
      },
      "source": [
        "# -*- coding: utf-8 -*-\n",
        "\"\"\"Mnist Classifier Using Tensorflow.ipynb\n",
        "\n",
        "Automatically generated by Colaboratory.\n",
        "\n",
        "Original file is located at\n",
        "    https://colab.research.google.com/drive/1M0Xh1wX5mfqE9Y22VJoa6o61vhFs-OHY\n",
        "\n",
        "This Code is to build a classifier for MNIST dataset in Tensorflow\n",
        "\"\"\"\n",
        "\n",
        "# import all packages on top \n",
        "import tensorflow as tf\n",
        "from tensorflow.examples.tutorials.mnist import input_data\n",
        "from optimization import AdamWeightDecayOptimizer, LAMBOptimizer\n",
        "\n",
        "tf.set_random_seed(12345)\n",
        "\n",
        "\n",
        "\"\"\"one is on and rest are off , used for multiclass classification for one class output will be 1, and for others it be zero\"\"\"\n",
        "\n",
        "# Read and create dataset \n",
        "\n",
        "mnist = input_data.read_data_sets('/tmp/data',one_hot=True)\n",
        "\n",
        "# Define model variables \n",
        "n_nodes_hl1 = 256\n",
        "n_nodes_hl2 = 256\n",
        "n_nodes_hl3 = 256\n",
        "\n",
        "n_classes = 10\n",
        "batch_size = 256\n",
        "\n",
        "# Place holder to store values while passing through graph \n",
        "# float value, Height of row is None and width will be 784\n",
        "x = tf.placeholder('float',[None,784])\n",
        "y = tf.placeholder('float')\n",
        "\n",
        "\n",
        "keep_rate = 0.8\n",
        "keep_prob = tf.placeholder(tf.float32)\n"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From <ipython-input-1-ac98d4ff8190>:23: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
            "WARNING:tensorflow:From /usr/local/lib/python2.7/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please write your own downloading logic.\n",
            "WARNING:tensorflow:From /usr/local/lib/python2.7/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use tf.data to implement this functionality.\n",
            "Extracting /tmp/data/train-images-idx3-ubyte.gz\n",
            "WARNING:tensorflow:From /usr/local/lib/python2.7/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use tf.data to implement this functionality.\n",
            "Extracting /tmp/data/train-labels-idx1-ubyte.gz\n",
            "WARNING:tensorflow:From /usr/local/lib/python2.7/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use tf.one_hot on tensors.\n",
            "Extracting /tmp/data/t10k-images-idx3-ubyte.gz\n",
            "Extracting /tmp/data/t10k-labels-idx1-ubyte.gz\n",
            "WARNING:tensorflow:From /usr/local/lib/python2.7/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:290: __init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o13984hY8hjQ",
        "colab_type": "text"
      },
      "source": [
        "### Define MLP"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-SBtYpWm7pqF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Method to define model \n",
        "def mlp(data):\n",
        "  # (input_data * weight)  + bias\n",
        "  hidden_1_layer = {'weights': tf.Variable(tf.random_normal([784, n_nodes_hl1])),\n",
        "                    'biases': tf.Variable(tf.random_normal([n_nodes_hl1]))}\n",
        "  hidden_2_layer = {'weights': tf.Variable(tf.random_normal([n_nodes_hl1, n_nodes_hl2])),\n",
        "                    'biases': tf.Variable(tf.random_normal([n_nodes_hl2]))}\n",
        "  hidden_3_layer = {'weights': tf.Variable(tf.random_normal([n_nodes_hl2, n_nodes_hl3])),\n",
        "                    'biases': tf.Variable(tf.random_normal([n_nodes_hl3]))}\n",
        "  output_layer = {'weights': tf.Variable(tf.random_normal([n_nodes_hl3, n_classes])),\n",
        "                    'biases': tf.Variable(tf.random_normal([n_classes]))}\n",
        "\n",
        "  #input_data * weight +b\n",
        "  l1 = tf.add(tf.matmul(data,hidden_1_layer['weights']), hidden_1_layer['biases'])\n",
        "  l1 = tf.nn.relu(l1)\n",
        "\n",
        "  l2 = tf.add(tf.matmul(l1,hidden_2_layer['weights']),hidden_2_layer['biases'])\n",
        "  l2 = tf.nn.relu(l2)\n",
        "\n",
        "  l3 = tf.add(tf.matmul(l2,hidden_3_layer['weights']),hidden_3_layer['biases'])\n",
        "  l3 = tf.nn.relu(l3)\n",
        "\n",
        "  output = tf.matmul(l3,output_layer['weights'])+  output_layer['biases']\n",
        "\n",
        "  return output"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ecSCBHT38l0f",
        "colab_type": "text"
      },
      "source": [
        "### Define CNN"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TfFVBDNWNebk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def conv2d(x, W):\n",
        "  return tf.nn.conv2d(x, W, strides=[1,1,1,1], padding='SAME')\n",
        "\n",
        "def maxpool2d(x):\n",
        "  return tf.nn.max_pool(x, ksize=[1,2,2,1], strides=[1,2,2,1], padding='SAME')\n",
        "\n",
        "#\n",
        "def cnn(x):\n",
        "  weights = {'W_conv1':tf.Variable(tf.random_normal([5,5,1,32])),\n",
        "             'W_conv2':tf.Variable(tf.random_normal([5,5,32,64])),\n",
        "             'W_fc':tf.Variable(tf.random_normal([7*7*64,1024])),\n",
        "             'out':tf.Variable(tf.random_normal([1024, n_classes]))}\n",
        "\n",
        "  biases = {'b_conv1':tf.Variable(tf.random_normal([32])),\n",
        "             'b_conv2':tf.Variable(tf.random_normal([64])),\n",
        "             'b_fc':tf.Variable(tf.random_normal([1024])),\n",
        "             'out':tf.Variable(tf.random_normal([n_classes]))}\n",
        "\n",
        "  x = tf.reshape(x, shape=[-1, 28, 28, 1])\n",
        "\n",
        "  conv1 = tf.nn.relu(conv2d(x, weights['W_conv1']) + biases['b_conv1'])\n",
        "  conv1 = maxpool2d(conv1)\n",
        "  \n",
        "  conv2 = tf.nn.relu(conv2d(conv1, weights['W_conv2']) + biases['b_conv2'])\n",
        "  conv2 = maxpool2d(conv2)\n",
        "\n",
        "  fc = tf.reshape(conv2,[-1, 7*7*64])\n",
        "  fc = tf.nn.relu(tf.matmul(fc, weights['W_fc'])+biases['b_fc'])\n",
        "  fc = tf.nn.dropout(fc, keep_rate)\n",
        "\n",
        "  output = tf.matmul(fc, weights['out'])+biases['out']\n",
        "\n",
        "  return output"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j6HqCh0S8n-r",
        "colab_type": "text"
      },
      "source": [
        "### Define Bi-RNN"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dGDNZwCaNkxN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def birnn(x, cell_type=tf.nn.rnn_cell.LSTMCell):\n",
        "  timesteps = 28\n",
        "  x = tf.reshape(x, [-1, timesteps ,timesteps])\n",
        "  # Define weights\n",
        "  weights = {\n",
        "      # Hidden layer weights => 2*n_hidden because of forward + backward cells\n",
        "      'out': tf.Variable(tf.random_normal([2*128, 10]))\n",
        "  }\n",
        "  biases = {\n",
        "      'out': tf.Variable(tf.random_normal([10]))\n",
        "  }\n",
        "  # Prepare data shape to match `rnn` function requirements\n",
        "  # Current data input shape: (batch_size, timesteps, n_input)\n",
        "  # Required shape: 'timesteps' tensors list of shape (batch_size, num_input)\n",
        "\n",
        "  # Unstack to get a list of 'timesteps' tensors of shape (batch_size, num_input)\n",
        "  x = tf.unstack(x, timesteps, 1)\n",
        "\n",
        "  # Define rnn cells with tensorflow\n",
        "  fw_cell = cell_type(128)\n",
        "  bw_cell = cell_type(128)\n",
        "\n",
        "  # Get cell output\n",
        "  try:\n",
        "      outputs, _, _ = tf.nn.static_bidirectional_rnn(fw_cell, bw_cell, x,\n",
        "                                            dtype=tf.float32)\n",
        "  except Exception: # Old TensorFlow version only returns outputs not states\n",
        "      outputs = tf.nn.static_bidirectional_rnn(fw_cell, bw_cell, x,\n",
        "                                      dtype=tf.float32)\n",
        "\n",
        "  # Linear activation, using rnn inner loop last output\n",
        "  output = tf.matmul(outputs[-1], weights['out']) + biases['out']\n",
        "\n",
        "  return output"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "u1MXNQqC8sSm",
        "colab_type": "text"
      },
      "source": [
        "### Define Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7eLxAyL7Ngsu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Method to intialize model and train using mnist data and test model function\n",
        "def run_training(neural_net, cell_type=None):\n",
        "  print(\"-\"*50)\n",
        "  print(\"Current Neural Network: \", neural_net)\n",
        "  if cell_type is not None:\n",
        "    prediction = neural_net(x, cell_type)\n",
        "  else:\n",
        "    prediction = neural_net(x)\n",
        "  cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=prediction, labels=y))\n",
        "\n",
        "  # Adam optimizer does has learning rate parameter\n",
        "  optimizers = [0,0,0]\n",
        "  optimizers[0] = tf.train.AdamOptimizer(0.001)\n",
        "  optimizers[1] = AdamWeightDecayOptimizer(0.001)\n",
        "  optimizers[2] = LAMBOptimizer(0.001)\n",
        "\n",
        "  #cycles for feed forward and backprop error\n",
        "  nm_epochs = 100\n",
        "  \n",
        "  #\n",
        "  for opt in optimizers:\n",
        "    print(\"Current Optimizer: \", opt)\n",
        "    optimizer = opt.minimize(cost)\n",
        "    with tf.Session() as sess:\n",
        "      sess.run(tf.global_variables_initializer())\n",
        "      for epoch in range(nm_epochs):\n",
        "        epoch_loss = 0\n",
        "        for _ in range(int(mnist.train.num_examples/batch_size)):\n",
        "          epoch_x,epoch_y = mnist.train.next_batch(batch_size)\n",
        "          _ ,c = sess.run([optimizer, cost], feed_dict= {x:epoch_x, y:epoch_y})\n",
        "          epoch_loss += c\n",
        "        if epoch % 10 == 0:\n",
        "          print(\"Epoch\", epoch,' completed out of ', nm_epochs , \"Loss:\" , epoch_loss)\n",
        "      correct = tf.equal(tf.argmax(prediction, 1), tf.argmax(y, 1))\n",
        "      accuracy =  tf.reduce_mean(tf.cast(correct, 'float'))\n",
        "      print(\"Accuracuy\" ,accuracy.eval({x:mnist.test.images, y:mnist.test.labels}))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QIvsUlKUujGl",
        "colab_type": "code",
        "outputId": "eebfac43-8100-408b-b407-669aad3109b7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 870
        }
      },
      "source": [
        "run_training(mlp)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--------------------------------------------------\n",
            "('Current Neural Network: ', <function mlp at 0x7f47d0476938>)\n",
            "WARNING:tensorflow:From /usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Colocations handled automatically by placer.\n",
            "WARNING:tensorflow:From <ipython-input-5-5141f6246f5a>:8: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "\n",
            "Future major versions of TensorFlow will allow gradients to flow\n",
            "into the labels input on backprop by default.\n",
            "\n",
            "See `tf.nn.softmax_cross_entropy_with_logits_v2`.\n",
            "\n",
            "('Current Optimizer: ', <tensorflow.python.training.adam.AdamOptimizer object at 0x7f47cf877350>)\n",
            "('Epoch', 0, ' completed out of ', 100, 'Loss:', 462588.7238769531)\n",
            "('Epoch', 10, ' completed out of ', 100, 'Loss:', 9729.408418655396)\n",
            "('Epoch', 20, ' completed out of ', 100, 'Loss:', 738.5969310670625)\n",
            "('Epoch', 30, ' completed out of ', 100, 'Loss:', 459.3601968849771)\n",
            "('Epoch', 40, ' completed out of ', 100, 'Loss:', 471.4661360885366)\n",
            "('Epoch', 50, ' completed out of ', 100, 'Loss:', 379.70835684174835)\n",
            "('Epoch', 60, ' completed out of ', 100, 'Loss:', 279.8096264439516)\n",
            "('Epoch', 70, ' completed out of ', 100, 'Loss:', 173.5695101992579)\n",
            "('Epoch', 80, ' completed out of ', 100, 'Loss:', 166.38060583505307)\n",
            "('Epoch', 90, ' completed out of ', 100, 'Loss:', 188.7352870301138)\n",
            "('Accuracuy', 0.9585)\n",
            "('Current Optimizer: ', <optimization.AdamWeightDecayOptimizer object at 0x7f47cf8779d0>)\n",
            "('Epoch', 0, ' completed out of ', 100, 'Loss:', 190649.23259735107)\n",
            "('Epoch', 10, ' completed out of ', 100, 'Loss:', 2878.6353784427047)\n",
            "('Epoch', 20, ' completed out of ', 100, 'Loss:', 416.1524757401712)\n",
            "('Epoch', 30, ' completed out of ', 100, 'Loss:', 316.22468386124797)\n",
            "('Epoch', 40, ' completed out of ', 100, 'Loss:', 272.85802937137623)\n",
            "('Epoch', 50, ' completed out of ', 100, 'Loss:', 251.35715805226835)\n",
            "('Epoch', 60, ' completed out of ', 100, 'Loss:', 195.12688246010725)\n",
            "('Epoch', 70, ' completed out of ', 100, 'Loss:', 114.36987461710063)\n",
            "('Epoch', 80, ' completed out of ', 100, 'Loss:', 147.20930432339196)\n",
            "('Epoch', 90, ' completed out of ', 100, 'Loss:', 240.02615648071952)\n",
            "('Accuracuy', 0.954)\n",
            "('Current Optimizer: ', <optimization.LAMBOptimizer object at 0x7f47cf877450>)\n",
            "('Epoch', 0, ' completed out of ', 100, 'Loss:', 264173.44596862793)\n",
            "('Epoch', 10, ' completed out of ', 100, 'Loss:', 934.651543200016)\n",
            "('Epoch', 20, ' completed out of ', 100, 'Loss:', 128.19198402661831)\n",
            "('Epoch', 30, ' completed out of ', 100, 'Loss:', 26.81428903153818)\n",
            "('Epoch', 40, ' completed out of ', 100, 'Loss:', 7.426490208568408)\n",
            "('Epoch', 50, ' completed out of ', 100, 'Loss:', 2.5576100588409645)\n",
            "('Epoch', 60, ' completed out of ', 100, 'Loss:', 1.5739839647412737)\n",
            "('Epoch', 70, ' completed out of ', 100, 'Loss:', 1.518773596124447)\n",
            "('Epoch', 80, ' completed out of ', 100, 'Loss:', 1.3437223225546404)\n",
            "('Epoch', 90, ' completed out of ', 100, 'Loss:', 1.0773548603369818)\n",
            "('Accuracuy', 0.9788)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XfErV39JwZgu",
        "colab_type": "code",
        "outputId": "58fbc2cf-ae8d-4f15-c9c8-987cb8d25e0b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 734
        }
      },
      "source": [
        "run_training(cnn)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--------------------------------------------------\n",
            "('Current Neural Network: ', <function cnn at 0x7f47d0476848>)\n",
            "WARNING:tensorflow:From <ipython-input-3-438c3928472c>:29: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
            "('Current Optimizer: ', <tensorflow.python.training.adam.AdamOptimizer object at 0x7f47c1101510>)\n",
            "('Epoch', 0, ' completed out of ', 100, 'Loss:', 1790345.1608886719)\n",
            "('Epoch', 10, ' completed out of ', 100, 'Loss:', 19208.704095840454)\n",
            "('Epoch', 20, ' completed out of ', 100, 'Loss:', 4100.823376274595)\n",
            "('Epoch', 30, ' completed out of ', 100, 'Loss:', 1961.4625253677368)\n",
            "('Epoch', 40, ' completed out of ', 100, 'Loss:', 1414.5663056151243)\n",
            "('Epoch', 50, ' completed out of ', 100, 'Loss:', 1202.5498754279688)\n",
            "('Epoch', 60, ' completed out of ', 100, 'Loss:', 921.100043296814)\n",
            "('Epoch', 70, ' completed out of ', 100, 'Loss:', 674.768424814567)\n",
            "('Epoch', 80, ' completed out of ', 100, 'Loss:', 463.47949127200974)\n",
            "('Epoch', 90, ' completed out of ', 100, 'Loss:', 404.9293518103659)\n",
            "('Accuracuy', 0.9873)\n",
            "('Current Optimizer: ', <optimization.AdamWeightDecayOptimizer object at 0x7f47c1101f50>)\n",
            "('Epoch', 0, ' completed out of ', 100, 'Loss:', 676103.1452331543)\n",
            "('Epoch', 10, ' completed out of ', 100, 'Loss:', 7637.875990390778)\n",
            "('Epoch', 20, ' completed out of ', 100, 'Loss:', 2091.3434143066406)\n",
            "('Epoch', 30, ' completed out of ', 100, 'Loss:', 987.7616066932678)\n",
            "('Epoch', 40, ' completed out of ', 100, 'Loss:', 651.5578809679878)\n",
            "('Epoch', 50, ' completed out of ', 100, 'Loss:', 606.37118004472)\n",
            "('Epoch', 60, ' completed out of ', 100, 'Loss:', 433.2797651798463)\n",
            "('Epoch', 70, ' completed out of ', 100, 'Loss:', 307.54644107818604)\n",
            "('Epoch', 80, ' completed out of ', 100, 'Loss:', 312.9413251876831)\n",
            "('Epoch', 90, ' completed out of ', 100, 'Loss:', 155.1624584197998)\n",
            "('Accuracuy', 0.9867)\n",
            "('Current Optimizer: ', <optimization.LAMBOptimizer object at 0x7f47c10939d0>)\n",
            "('Epoch', 0, ' completed out of ', 100, 'Loss:', 1053981.727470398)\n",
            "('Epoch', 10, ' completed out of ', 100, 'Loss:', 324.16113834816747)\n",
            "('Epoch', 20, ' completed out of ', 100, 'Loss:', 1.0996046921354719)\n",
            "('Epoch', 30, ' completed out of ', 100, 'Loss:', 2.6460204066243023)\n",
            "('Epoch', 40, ' completed out of ', 100, 'Loss:', 1.190299780049827)\n",
            "('Epoch', 50, ' completed out of ', 100, 'Loss:', 0.6116361551103182)\n",
            "('Epoch', 60, ' completed out of ', 100, 'Loss:', 0.4394193086118321)\n",
            "('Epoch', 70, ' completed out of ', 100, 'Loss:', 0.3804983706213534)\n",
            "('Epoch', 80, ' completed out of ', 100, 'Loss:', 0.26819803793932806)\n",
            "('Epoch', 90, ' completed out of ', 100, 'Loss:', 0.3742796789856584)\n",
            "('Accuracuy', 0.9913)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UMQ70eenwcVU",
        "colab_type": "code",
        "outputId": "2419bd30-21e4-46b0-b496-d2218ea0fd2d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 836
        }
      },
      "source": [
        "run_training(birnn, cell_type=tf.nn.rnn_cell.BasicRNNCell)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--------------------------------------------------\n",
            "('Current Neural Network: ', <function birnn at 0x7f47d0476d70>)\n",
            "WARNING:tensorflow:From <ipython-input-4-26585241078a>:20: __init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "This class is equivalent as tf.keras.layers.SimpleRNNCell, and will be replaced by that in Tensorflow 2.0.\n",
            "WARNING:tensorflow:From <ipython-input-4-26585241078a>:26: static_bidirectional_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use `keras.layers.Bidirectional(keras.layers.RNN(cell, unroll=True))`, which is equivalent to this API\n",
            "WARNING:tensorflow:From /usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn.py:1565: static_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use `keras.layers.RNN(cell, unroll=True)`, which is equivalent to this API\n",
            "('Current Optimizer: ', <tensorflow.python.training.adam.AdamOptimizer object at 0x7f47c0932a90>)\n",
            "('Epoch', 0, ' completed out of ', 100, 'Loss:', 136.50434014201164)\n",
            "('Epoch', 10, ' completed out of ', 100, 'Loss:', 18.987429643049836)\n",
            "('Epoch', 20, ' completed out of ', 100, 'Loss:', 13.061466319486499)\n",
            "('Epoch', 30, ' completed out of ', 100, 'Loss:', 10.232898156158626)\n",
            "('Epoch', 40, ' completed out of ', 100, 'Loss:', 7.890659255674109)\n",
            "('Epoch', 50, ' completed out of ', 100, 'Loss:', 6.273008614080027)\n",
            "('Epoch', 60, ' completed out of ', 100, 'Loss:', 7.026612656190991)\n",
            "('Epoch', 70, ' completed out of ', 100, 'Loss:', 8.890961387893185)\n",
            "('Epoch', 80, ' completed out of ', 100, 'Loss:', 3.8347834403975867)\n",
            "('Epoch', 90, ' completed out of ', 100, 'Loss:', 5.542974240903277)\n",
            "('Accuracuy', 0.9705)\n",
            "('Current Optimizer: ', <optimization.AdamWeightDecayOptimizer object at 0x7f47c0825f90>)\n",
            "('Epoch', 0, ' completed out of ', 100, 'Loss:', 157.7732678502798)\n",
            "('Epoch', 10, ' completed out of ', 100, 'Loss:', 17.7433260679245)\n",
            "('Epoch', 20, ' completed out of ', 100, 'Loss:', 12.32123020105064)\n",
            "('Epoch', 30, ' completed out of ', 100, 'Loss:', 10.150779344839975)\n",
            "('Epoch', 40, ' completed out of ', 100, 'Loss:', 8.794615186285228)\n",
            "('Epoch', 50, ' completed out of ', 100, 'Loss:', 6.729872531024739)\n",
            "('Epoch', 60, ' completed out of ', 100, 'Loss:', 4.4811526230187155)\n",
            "('Epoch', 70, ' completed out of ', 100, 'Loss:', 5.317337812623009)\n",
            "('Epoch', 80, ' completed out of ', 100, 'Loss:', 3.9421464445767924)\n",
            "('Epoch', 90, ' completed out of ', 100, 'Loss:', 6.052418288425542)\n",
            "('Accuracuy', 0.9775)\n",
            "('Current Optimizer: ', <optimization.LAMBOptimizer object at 0x7f47c08543d0>)\n",
            "('Epoch', 0, ' completed out of ', 100, 'Loss:', 231.1219124495983)\n",
            "('Epoch', 10, ' completed out of ', 100, 'Loss:', 20.253984335809946)\n",
            "('Epoch', 20, ' completed out of ', 100, 'Loss:', 12.353220720775425)\n",
            "('Epoch', 30, ' completed out of ', 100, 'Loss:', 8.519891723524779)\n",
            "('Epoch', 40, ' completed out of ', 100, 'Loss:', 6.687333207344636)\n",
            "('Epoch', 50, ' completed out of ', 100, 'Loss:', 5.144277607323602)\n",
            "('Epoch', 60, ' completed out of ', 100, 'Loss:', 4.60678640846163)\n",
            "('Epoch', 70, ' completed out of ', 100, 'Loss:', 3.687171082245186)\n",
            "('Epoch', 80, ' completed out of ', 100, 'Loss:', 3.226053055957891)\n",
            "('Epoch', 90, ' completed out of ', 100, 'Loss:', 2.971006609965116)\n",
            "('Accuracuy', 0.9802)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pL9j52wKwqCA",
        "colab_type": "code",
        "outputId": "6ae256e2-1e73-476f-d7b3-405a16923789",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 717
        }
      },
      "source": [
        "run_training(birnn, cell_type=tf.nn.rnn_cell.GRUCell)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--------------------------------------------------\n",
            "('Current Neural Network: ', <function birnn at 0x7f47d0476d70>)\n",
            "WARNING:tensorflow:From <ipython-input-4-26585241078a>:20: __init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "This class is equivalent as tf.keras.layers.GRUCell, and will be replaced by that in Tensorflow 2.0.\n",
            "('Current Optimizer: ', <tensorflow.python.training.adam.AdamOptimizer object at 0x7f482b2f2710>)\n",
            "('Epoch', 0, ' completed out of ', 100, 'Loss:', 155.9565221220255)\n",
            "('Epoch', 10, ' completed out of ', 100, 'Loss:', 6.130548636429012)\n",
            "('Epoch', 20, ' completed out of ', 100, 'Loss:', 2.336274054774549)\n",
            "('Epoch', 30, ' completed out of ', 100, 'Loss:', 1.1914153546094894)\n",
            "('Epoch', 40, ' completed out of ', 100, 'Loss:', 0.43508785866106336)\n",
            "('Epoch', 50, ' completed out of ', 100, 'Loss:', 2.353516362994924)\n",
            "('Epoch', 60, ' completed out of ', 100, 'Loss:', 0.35459679920313647)\n",
            "('Epoch', 70, ' completed out of ', 100, 'Loss:', 0.00540988779835061)\n",
            "('Epoch', 80, ' completed out of ', 100, 'Loss:', 0.0019388693119992695)\n",
            "('Epoch', 90, ' completed out of ', 100, 'Loss:', 0.0006927909780074515)\n",
            "('Accuracuy', 0.99)\n",
            "('Current Optimizer: ', <optimization.AdamWeightDecayOptimizer object at 0x7f45e94dddd0>)\n",
            "('Epoch', 0, ' completed out of ', 100, 'Loss:', 85.7605186663568)\n",
            "('Epoch', 20, ' completed out of ', 100, 'Loss:', 1.361588410436525)\n",
            "('Epoch', 30, ' completed out of ', 100, 'Loss:', 0.7022913843393326)\n",
            "('Epoch', 40, ' completed out of ', 100, 'Loss:', 1.3056564743455965)\n",
            "('Epoch', 50, ' completed out of ', 100, 'Loss:', 0.018342056664096162)\n",
            "('Epoch', 60, ' completed out of ', 100, 'Loss:', 0.0036456268392157654)\n",
            "('Epoch', 70, ' completed out of ', 100, 'Loss:', 0.0013102748230267025)\n",
            "('Epoch', 80, ' completed out of ', 100, 'Loss:', 0.0004917663637229452)\n",
            "('Epoch', 90, ' completed out of ', 100, 'Loss:', 0.00015542565545700882)\n",
            "('Accuracuy', 0.9905)\n",
            "('Current Optimizer: ', <optimization.LAMBOptimizer object at 0x7f45e94d2a10>)\n",
            "('Epoch', 0, ' completed out of ', 100, 'Loss:', 340.7324674129486)\n",
            "('Epoch', 10, ' completed out of ', 100, 'Loss:', 11.69379351194948)\n",
            "('Epoch', 20, ' completed out of ', 100, 'Loss:', 5.067263924982399)\n",
            "('Epoch', 30, ' completed out of ', 100, 'Loss:', 2.6601956194499508)\n",
            "('Epoch', 40, ' completed out of ', 100, 'Loss:', 1.7036024106491823)\n",
            "('Epoch', 50, ' completed out of ', 100, 'Loss:', 1.0944334780069767)\n",
            "('Epoch', 60, ' completed out of ', 100, 'Loss:', 0.90041142849077)\n",
            "('Epoch', 70, ' completed out of ', 100, 'Loss:', 0.7445322177591152)\n",
            "('Epoch', 80, ' completed out of ', 100, 'Loss:', 0.5586020087976067)\n",
            "('Epoch', 90, ' completed out of ', 100, 'Loss:', 0.4754122560170799)\n",
            "('Accuracuy', 0.9888)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y0Nf-o6PwuHj",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 734
        },
        "outputId": "f175032c-3f8a-4a7a-c3f3-a75081617f25"
      },
      "source": [
        "run_training(birnn, cell_type=tf.nn.rnn_cell.LSTMCell)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--------------------------------------------------\n",
            "('Current Neural Network: ', <function birnn at 0x7f47d0476d70>)\n",
            "WARNING:tensorflow:From <ipython-input-4-26585241078a>:20: __init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "This class is equivalent as tf.keras.layers.LSTMCell, and will be replaced by that in Tensorflow 2.0.\n",
            "('Current Optimizer: ', <tensorflow.python.training.adam.AdamOptimizer object at 0x7f45e4a4f250>)\n",
            "('Epoch', 0, ' completed out of ', 100, 'Loss:', 136.01194302737713)\n",
            "('Epoch', 10, ' completed out of ', 100, 'Loss:', 6.592356271110475)\n",
            "('Epoch', 20, ' completed out of ', 100, 'Loss:', 3.2185868460219353)\n",
            "('Epoch', 30, ' completed out of ', 100, 'Loss:', 1.2057269491488114)\n",
            "('Epoch', 40, ' completed out of ', 100, 'Loss:', 1.0393099158100085)\n",
            "('Epoch', 50, ' completed out of ', 100, 'Loss:', 1.2134482465698966)\n",
            "('Epoch', 60, ' completed out of ', 100, 'Loss:', 0.7256674864838715)\n",
            "('Epoch', 70, ' completed out of ', 100, 'Loss:', 0.7495522389508551)\n",
            "('Epoch', 80, ' completed out of ', 100, 'Loss:', 0.00742054059264774)\n",
            "('Epoch', 90, ' completed out of ', 100, 'Loss:', 0.002136187549126589)\n",
            "('Accuracuy', 0.9912)\n",
            "('Current Optimizer: ', <optimization.AdamWeightDecayOptimizer object at 0x7f482b2f2690>)\n",
            "('Epoch', 0, ' completed out of ', 100, 'Loss:', 85.6300259232521)\n",
            "('Epoch', 10, ' completed out of ', 100, 'Loss:', 4.055555629427545)\n",
            "('Epoch', 20, ' completed out of ', 100, 'Loss:', 2.0720697779906914)\n",
            "('Epoch', 30, ' completed out of ', 100, 'Loss:', 1.523650611488847)\n",
            "('Epoch', 40, ' completed out of ', 100, 'Loss:', 0.900932673484931)\n",
            "('Epoch', 50, ' completed out of ', 100, 'Loss:', 0.6298316949723812)\n",
            "('Epoch', 60, ' completed out of ', 100, 'Loss:', 1.7031326444703154)\n",
            "('Epoch', 70, ' completed out of ', 100, 'Loss:', 0.17483498998080904)\n",
            "('Epoch', 80, ' completed out of ', 100, 'Loss:', 0.0032450484458195206)\n",
            "('Epoch', 90, ' completed out of ', 100, 'Loss:', 0.0011984165734588714)\n",
            "('Accuracuy', 0.9892)\n",
            "('Current Optimizer: ', <optimization.LAMBOptimizer object at 0x7f45e425ad10>)\n",
            "('Epoch', 0, ' completed out of ', 100, 'Loss:', 347.70945006608963)\n",
            "('Epoch', 10, ' completed out of ', 100, 'Loss:', 17.764954648911953)\n",
            "('Epoch', 20, ' completed out of ', 100, 'Loss:', 7.933466560672969)\n",
            "('Epoch', 30, ' completed out of ', 100, 'Loss:', 4.093579387059435)\n",
            "('Epoch', 40, ' completed out of ', 100, 'Loss:', 2.7274060727795586)\n",
            "('Epoch', 50, ' completed out of ', 100, 'Loss:', 1.718623186869081)\n",
            "('Epoch', 60, ' completed out of ', 100, 'Loss:', 1.419074394434574)\n",
            "('Epoch', 70, ' completed out of ', 100, 'Loss:', 1.319914322317345)\n",
            "('Epoch', 80, ' completed out of ', 100, 'Loss:', 1.0161797366017709)\n",
            "('Epoch', 90, ' completed out of ', 100, 'Loss:', 0.9066535502497572)\n",
            "('Accuracuy', 0.9876)\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}