diff --git a/notebooks/Introduction_to_tensorflow.ipynb b/notebooks/Introduction_to_tensorflow.ipynb
index dcdfe40..0644f08 100644
--- a/notebooks/Introduction_to_tensorflow.ipynb
+++ b/notebooks/Introduction_to_tensorflow.ipynb
@@ -83,18 +83,9 @@
},
{
"cell_type": "code",
- "execution_count": 1,
+ "execution_count": null,
"metadata": {},
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "/home/antreasantoniou/miniconda2/envs/mlp/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: compiletime version 3.5 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.6\n",
- " return f(*args, **kwds)\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"import tensorflow as tf"
]
@@ -116,7 +107,7 @@
},
{
"cell_type": "code",
- "execution_count": 12,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@@ -135,7 +126,7 @@
},
{
"cell_type": "code",
- "execution_count": 13,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@@ -152,7 +143,7 @@
},
{
"cell_type": "code",
- "execution_count": 14,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@@ -172,7 +163,7 @@
},
{
"cell_type": "code",
- "execution_count": 15,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@@ -189,7 +180,7 @@
},
{
"cell_type": "code",
- "execution_count": 16,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@@ -210,7 +201,7 @@
},
{
"cell_type": "code",
- "execution_count": 17,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@@ -234,7 +225,7 @@
},
{
"cell_type": "code",
- "execution_count": 18,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@@ -254,7 +245,7 @@
},
{
"cell_type": "code",
- "execution_count": 19,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@@ -271,18 +262,9 @@
},
{
"cell_type": "code",
- "execution_count": 20,
+ "execution_count": null,
"metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "['inputs', 'targets']\n",
- "['inputs', 'targets']\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"import data_providers as data_providers\n",
"train_data = data_providers.EMNISTDataProvider('train', batch_size=50, flatten=True)\n",
@@ -304,36 +286,9 @@
},
{
"cell_type": "code",
- "execution_count": 22,
+ "execution_count": null,
"metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "End of epoch 1: running error average = 1.18\n",
- "End of epoch 2: running error average = 1.18\n",
- "End of epoch 3: running error average = 1.17\n",
- "End of epoch 4: running error average = 1.17\n",
- "End of epoch 5: running error average = 1.16\n",
- "End of epoch 6: running error average = 1.16\n",
- "End of epoch 7: running error average = 1.16\n",
- "End of epoch 8: running error average = 1.15\n",
- "End of epoch 9: running error average = 1.15\n",
- "End of epoch 10: running error average = 1.15\n",
- "End of epoch 11: running error average = 1.14\n",
- "End of epoch 12: running error average = 1.14\n",
- "End of epoch 13: running error average = 1.14\n",
- "End of epoch 14: running error average = 1.14\n",
- "End of epoch 15: running error average = 1.14\n",
- "End of epoch 16: running error average = 1.13\n",
- "End of epoch 17: running error average = 1.13\n",
- "End of epoch 18: running error average = 1.13\n",
- "End of epoch 19: running error average = 1.13\n",
- "End of epoch 20: running error average = 1.13\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"num_epoch = 20\n",
"for e in range(num_epoch):\n",
@@ -356,7 +311,7 @@
},
{
"cell_type": "code",
- "execution_count": 23,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@@ -390,18 +345,9 @@
},
{
"cell_type": "code",
- "execution_count": 24,
+ "execution_count": null,
"metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Train data: Error=1.14 Accuracy=0.69\n",
- "Valid data: Error=1.29 Accuracy=0.66\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"print('Train data: Error={0:.2f} Accuracy={1:.2f}'\n",
" .format(*get_error_and_accuracy(train_data)))\n",
@@ -424,17 +370,9 @@
},
{
"cell_type": "code",
- "execution_count": 25,
+ "execution_count": null,
"metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Number of operations in graph: 198\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"default_graph = tf.get_default_graph()\n",
"print('Number of operations in graph: {0}'\n",
@@ -450,7 +388,7 @@
},
{
"cell_type": "code",
- "execution_count": 26,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@@ -472,7 +410,7 @@
},
{
"cell_type": "code",
- "execution_count": 28,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@@ -505,7 +443,7 @@
},
{
"cell_type": "code",
- "execution_count": 29,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@@ -543,192 +481,12 @@
},
{
"cell_type": "code",
- "execution_count": 30,
+ "execution_count": null,
"metadata": {},
- "outputs": [
- {
- "data": {
- "text/html": [
- "\n",
- " \n",
- " "
- ],
- "text/plain": [
- ""
- ]
- },
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
+ "outputs": [],
"source": [
"show_graph(graph)"
]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "To highlight how using name scopes can be very helpful in making these graph visualisations more interpretable, running the cell below will create a corresponding visualisation for the graph created in the first exercise, which contains the same operations but without the name scope groupings."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 31,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/html": [
- "\n",
- " \n",
- " "
- ],
- "text/plain": [
- ""
- ]
- },
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "show_graph(tf.get_default_graph())"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "A common problem when doing gradient based training of complex models is how to monitor progress during training. In the `mlp` framework we used last semester we included some basic logging functionality for recording training statistics such as training and validation set error and classificaton accuracy at the end of each epoch. By printing the log output this allowed basic monitoring of how training was proceeding. However due to the noisiness of the the training procedures the raw values printed were often difficult to interpret. After a training run we often plotted training curves to allow better visualisation of how the run went but this could only be done after a run was completed and required a lot of boilerplate code to be written (or copied and pasted...).\n",
- "\n",
- "TensorFlow [*summary* operations](https://www.tensorflow.org/api_docs/python/summary/) are designed to help deal with this issue. Summary operations can be added to the graph to allow summary statistics to be computed and serialized to event files. These event files can then be loaded in TensorBoard *during training* to allow continuous graphing of for example the training and validation set error during training. As well as summary operations for monitoring [scalar](https://www.tensorflow.org/api_docs/python/summary/generation_of_summaries_#scalar) values such as errors or accuracies, TensorFlow also includes summary operations for monitoring [histograms](https://www.tensorflow.org/api_docs/python/summary/generation_of_summaries_#histogram) of tensor quanties (e.g. the distribution of a set of weight parameters), displaying [images](https://www.tensorflow.org/api_docs/python/summary/generation_of_summaries_#image) (for example for checking if random augmentations being applied to image inputs are producing reasonable outputs) and even playing back [audio](https://www.tensorflow.org/api_docs/python/summary/generation_of_summaries_#audio).\n",
- "\n",
- "The cell below adds two simple scalar summary operations to our new graph for monitoring the error and classification accuracy. While we can keep references to all of the summary ops we add to a graph and make sure to run them all individually in the session during training, as with variable initialisation, TensorFlow provides a convenience method to avoid having to write a lot of boilerplate code like this. The `tf.summary.merge_all()` function returns an merged op corresponding to all of the summary ops that have been added to the current default graph. We can then just run this one merged op in our session to generate all the summaries we have added."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 32,
- "metadata": {},
- "outputs": [],
- "source": [
- "with graph.as_default():\n",
- " tf.summary.scalar('error', error)\n",
- " tf.summary.scalar('accuracy', accuracy)\n",
- " summary_op = tf.summary.merge_all()"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "In addition to the (merged) summary operation, we also need to define a *summary writer* object(s) to specify where the summaries should be written to on disk. The `tf.summary.FileWriter` class constructor takes a `logdir` as its first argument which should specify the path to a directory where event files should be written to. In the code below the log directory is specified as a local directory `tf-log` plus a timestamp based sub-directory within this to keep event files corresponding to different runs separated. The `FileWriter` constructor also accepts an optional `graph` argument which we here set to the graph we just populated with summaries. We construct separate writer objects for summaries on the training and validation datasets."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 33,
- "metadata": {},
- "outputs": [],
- "source": [
- "import os\n",
- "import datetime\n",
- "timestamp = datetime.datetime.now().strftime(\"%Y-%m-%d_%H-%M-%S\")\n",
- "train_writer = tf.summary.FileWriter(os.path.join('tf-log', timestamp, 'train'), graph=graph)\n",
- "valid_writer = tf.summary.FileWriter(os.path.join('tf-log', timestamp, 'valid'), graph=graph)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "The final step in using summaries is to run the merged summary op at the appropriate points in training and to add the outputs of the run summary operations to the writers. Here we evaluate the summary op on each training dataset batch and after every 100th batch evaluate the summary op on the whole validation dataset, writing the outputs of each to the relevant writers.\n",
- "\n",
- "If you run the cell below, you should be able to visualise the resulting training run summaries by launching TensorBoard within a shell with\n",
- "\n",
- "```bash\n",
- "tensorboard --logdir=[path/to/tf-log]\n",
- "```\n",
- "\n",
- "where `[path/to/tf-log]` is replaced with the path to the `tf-log` directory specified abovem and then opening the URL `localhost:6006` in a browser."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 34,
- "metadata": {},
- "outputs": [],
- "source": [
- "with graph.as_default():\n",
- " init = tf.global_variables_initializer()\n",
- "sess = tf.InteractiveSession(graph=graph)\n",
- "num_epoch = 5\n",
- "valid_inputs = valid_data.inputs\n",
- "valid_targets = valid_data.to_one_of_k(valid_data.targets)\n",
- "sess.run(init)\n",
- "for e in range(num_epoch):\n",
- " for b, (input_batch, target_batch) in enumerate(train_data):\n",
- " _, summary = sess.run(\n",
- " [train_step, summary_op],\n",
- " feed_dict={inputs: input_batch, targets: target_batch})\n",
- " train_writer.add_summary(summary, e * train_data.num_batches + b)\n",
- " if b % 100 == 0:\n",
- " valid_summary = sess.run(\n",
- " summary_op, feed_dict={inputs: valid_inputs, targets: valid_targets})\n",
- " valid_writer.add_summary(valid_summary, e * train_data.num_batches + b)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "That completes our basic introduction to TensorFlow. If you want more to explore more of TensorFlow before beginning your project for this semester, you may wish to go through some of the [official tutorials](https://www.tensorflow.org/tutorials/) or some of the many sites with unofficial tutorials e.g. the series of notebooks [here](https://github.com/aymericdamien/TensorFlow-Examples). If you have time you may also wish to have a go at the optional exercise below."
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Optional exercise: multiple layer EMNIST classifier using `contrib` modules\n",
- "\n",
- "As well as the core officially supported codebase, TensorFlow is distributed with a series of contributed modules under [`tensorflow.contrib`](https://www.tensorflow.org/api_docs/python/tf/contrib). These tend to provide higher level interfaces for constructing and running common forms of computational graphs which can allow models to be constructed with much more concise code. The interfaces of the `contrib` modules tend to be less stable than the core TensorFlow Python interface and they are also more restricted in the sorts of models that can be created. Therefore it is worthwhile to also be familiar with constructing models with the operations available in the core TensorFlow codebase; you can also often mix and match use of 'native' TensorFlow and functions from `contrib` modules.\n",
- "\n",
- "As an optional extension exercise, construct a deep EMNIST classifier model, either using TensorFlow operations directly as above or using one (or more) of the higher level interfaces defined in `contrib` modules such as [`tensorflow.contrib.learn`](https://www.tensorflow.org/tutorials/tflearn/), [`tensorflow.contrib.layers`](https://www.tensorflow.org/api_docs/python/tf/layers) or [`tensorflow.contrib.slim`](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/slim). You should choose an appropriate model architecture (number and width of layers) and choice of activation function based on your experience fitting models from last semester.\n",
- "\n",
- "As well as exploring the use of the interfaces in `contrib` modules you may wish to explore the more advanced optimizers available in [`tensorflow.train`](https://www.tensorflow.org/api_docs/python/tf/train) - such as [`tensorflow.train.AdamOptimizer`](https://www.tensorflow.org/api_docs/python/tf/train/AdamOptimizer) and [`tensorflow.train.RMSPropOptimizer`](https://www.tensorflow.org/api_docs/python/tf/train/RMSPropOptimizer) corresponding to the adaptive learning rules implemented in the second coursework last semester."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": true
- },
- "outputs": [],
- "source": [
- ""
- ]
}
],
"metadata": {
@@ -741,16 +499,16 @@
"language_info": {
"codemirror_mode": {
"name": "ipython",
- "version": 3.0
+ "version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.6.3"
+ "version": "3.6.2"
}
},
"nbformat": 4,
- "nbformat_minor": 0
-}
\ No newline at end of file
+ "nbformat_minor": 1
+}