anton/mlpractical
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #36 from pswietojanski/master

Browse Source 
adding some fixes
This commit is contained in:
Pawel Swietojanski 2015-12-13 20:09:56 +00:00
commit 162e28b498
3 changed files with 163 additions and 158 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

304
04_Regularisation_solution.ipynb
View File

@ -794,7 +794,7 @@
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": 2,
"metadata": {
"collapsed": false,
"scrolled": true
@ -808,158 +808,158 @@
"INFO:root:Training started...\n",
"INFO:mlp.optimisers:Epoch 0: Training cost (ce) for initial model is 2.624. Accuracy is 8.60%\n",
"INFO:mlp.optimisers:Epoch 0: Validation cost (ce) for initial model is 2.554. Accuracy is 9.84%\n",
"INFO:mlp.optimisers:Epoch 1: Training cost (ce) is 3.491. Accuracy is 47.90%\n",
"INFO:mlp.optimisers:Epoch 1: Validation cost (ce) is 0.969. Accuracy is 65.87%\n",
"INFO:mlp.optimisers:Epoch 1: Took 11 seconds. Training speed 220 pps. Validation speed 1466 pps.\n",
"INFO:mlp.optimisers:Epoch 2: Training cost (ce) is 1.039. Accuracy is 66.50%\n",
"INFO:mlp.optimisers:Epoch 2: Validation cost (ce) is 0.494. Accuracy is 85.09%\n",
"INFO:mlp.optimisers:Epoch 2: Took 11 seconds. Training speed 216 pps. Validation speed 1506 pps.\n",
"INFO:mlp.optimisers:Epoch 3: Training cost (ce) is 0.788. Accuracy is 76.10%\n",
"INFO:mlp.optimisers:Epoch 3: Validation cost (ce) is 0.429. Accuracy is 87.25%\n",
"INFO:mlp.optimisers:Epoch 3: Took 12 seconds. Training speed 205 pps. Validation speed 1484 pps.\n",
"INFO:mlp.optimisers:Epoch 4: Training cost (ce) is 0.770. Accuracy is 76.20%\n",
"INFO:mlp.optimisers:Epoch 4: Validation cost (ce) is 0.401. Accuracy is 88.32%\n",
"INFO:mlp.optimisers:Epoch 4: Took 10 seconds. Training speed 291 pps. Validation speed 1471 pps.\n",
"INFO:mlp.optimisers:Epoch 5: Training cost (ce) is 0.667. Accuracy is 79.10%\n",
"INFO:mlp.optimisers:Epoch 5: Validation cost (ce) is 0.440. Accuracy is 87.54%\n",
"INFO:mlp.optimisers:Epoch 5: Took 11 seconds. Training speed 263 pps. Validation speed 1411 pps.\n",
"INFO:mlp.optimisers:Epoch 6: Training cost (ce) is 0.687. Accuracy is 80.80%\n",
"INFO:mlp.optimisers:Epoch 6: Validation cost (ce) is 0.427. Accuracy is 88.10%\n",
"INFO:mlp.optimisers:Epoch 6: Took 11 seconds. Training speed 209 pps. Validation speed 1541 pps.\n",
"INFO:mlp.optimisers:Epoch 7: Training cost (ce) is 0.624. Accuracy is 82.50%\n",
"INFO:mlp.optimisers:Epoch 7: Validation cost (ce) is 0.412. Accuracy is 88.51%\n",
"INFO:mlp.optimisers:Epoch 7: Took 12 seconds. Training speed 194 pps. Validation speed 1546 pps.\n",
"INFO:mlp.optimisers:Epoch 8: Training cost (ce) is 0.581. Accuracy is 82.90%\n",
"INFO:mlp.optimisers:Epoch 8: Validation cost (ce) is 0.436. Accuracy is 87.98%\n",
"INFO:mlp.optimisers:Epoch 8: Took 12 seconds. Training speed 233 pps. Validation speed 1335 pps.\n",
"INFO:mlp.optimisers:Epoch 9: Training cost (ce) is 0.509. Accuracy is 83.30%\n",
"INFO:mlp.optimisers:Epoch 9: Validation cost (ce) is 0.382. Accuracy is 89.88%\n",
"INFO:mlp.optimisers:Epoch 9: Took 11 seconds. Training speed 245 pps. Validation speed 1458 pps.\n",
"INFO:mlp.optimisers:Epoch 10: Training cost (ce) is 0.488. Accuracy is 84.70%\n",
"INFO:mlp.optimisers:Epoch 10: Validation cost (ce) is 0.404. Accuracy is 89.79%\n",
"INFO:mlp.optimisers:Epoch 10: Took 12 seconds. Training speed 191 pps. Validation speed 1506 pps.\n",
"INFO:mlp.optimisers:Epoch 11: Training cost (ce) is 0.516. Accuracy is 84.60%\n",
"INFO:mlp.optimisers:Epoch 11: Validation cost (ce) is 0.394. Accuracy is 89.56%\n",
"INFO:mlp.optimisers:Epoch 11: Took 10 seconds. Training speed 288 pps. Validation speed 1551 pps.\n",
"INFO:mlp.optimisers:Epoch 12: Training cost (ce) is 0.414. Accuracy is 86.80%\n",
"INFO:mlp.optimisers:Epoch 12: Validation cost (ce) is 0.419. Accuracy is 89.70%\n",
"INFO:mlp.optimisers:Epoch 12: Took 10 seconds. Training speed 286 pps. Validation speed 1486 pps.\n",
"INFO:mlp.optimisers:Epoch 13: Training cost (ce) is 0.520. Accuracy is 85.50%\n",
"INFO:mlp.optimisers:Epoch 13: Validation cost (ce) is 0.482. Accuracy is 88.08%\n",
"INFO:mlp.optimisers:Epoch 13: Took 12 seconds. Training speed 204 pps. Validation speed 1515 pps.\n",
"INFO:mlp.optimisers:Epoch 14: Training cost (ce) is 0.423. Accuracy is 87.70%\n",
"INFO:mlp.optimisers:Epoch 14: Validation cost (ce) is 0.412. Accuracy is 89.93%\n",
"INFO:mlp.optimisers:Epoch 14: Took 11 seconds. Training speed 229 pps. Validation speed 1525 pps.\n",
"INFO:mlp.optimisers:Epoch 15: Training cost (ce) is 0.439. Accuracy is 88.10%\n",
"INFO:mlp.optimisers:Epoch 15: Validation cost (ce) is 0.465. Accuracy is 89.30%\n",
"INFO:mlp.optimisers:Epoch 15: Took 11 seconds. Training speed 225 pps. Validation speed 1452 pps.\n",
"INFO:mlp.optimisers:Epoch 16: Training cost (ce) is 0.460. Accuracy is 87.30%\n",
"INFO:mlp.optimisers:Epoch 16: Validation cost (ce) is 0.392. Accuracy is 90.99%\n",
"INFO:mlp.optimisers:Epoch 16: Took 12 seconds. Training speed 202 pps. Validation speed 1527 pps.\n",
"INFO:mlp.optimisers:Epoch 17: Training cost (ce) is 0.427. Accuracy is 87.60%\n",
"INFO:mlp.optimisers:Epoch 17: Validation cost (ce) is 0.434. Accuracy is 90.41%\n",
"INFO:mlp.optimisers:Epoch 17: Took 11 seconds. Training speed 242 pps. Validation speed 1479 pps.\n",
"INFO:mlp.optimisers:Epoch 18: Training cost (ce) is 0.398. Accuracy is 89.20%\n",
"INFO:mlp.optimisers:Epoch 18: Validation cost (ce) is 0.376. Accuracy is 91.62%\n",
"INFO:mlp.optimisers:Epoch 18: Took 11 seconds. Training speed 206 pps. Validation speed 1543 pps.\n",
"INFO:mlp.optimisers:Epoch 19: Training cost (ce) is 0.342. Accuracy is 88.60%\n",
"INFO:mlp.optimisers:Epoch 19: Validation cost (ce) is 0.422. Accuracy is 91.39%\n",
"INFO:mlp.optimisers:Epoch 19: Took 10 seconds. Training speed 303 pps. Validation speed 1534 pps.\n",
"INFO:mlp.optimisers:Epoch 20: Training cost (ce) is 0.385. Accuracy is 89.60%\n",
"INFO:mlp.optimisers:Epoch 20: Validation cost (ce) is 0.397. Accuracy is 91.78%\n",
"INFO:mlp.optimisers:Epoch 20: Took 11 seconds. Training speed 242 pps. Validation speed 1486 pps.\n",
"INFO:mlp.optimisers:Epoch 21: Training cost (ce) is 0.343. Accuracy is 89.60%\n",
"INFO:mlp.optimisers:Epoch 21: Validation cost (ce) is 0.498. Accuracy is 90.57%\n",
"INFO:mlp.optimisers:Epoch 21: Took 11 seconds. Training speed 229 pps. Validation speed 1508 pps.\n",
"INFO:mlp.optimisers:Epoch 22: Training cost (ce) is 0.406. Accuracy is 88.60%\n",
"INFO:mlp.optimisers:Epoch 22: Validation cost (ce) is 0.378. Accuracy is 92.42%\n",
"INFO:mlp.optimisers:Epoch 22: Took 11 seconds. Training speed 226 pps. Validation speed 1499 pps.\n",
"INFO:mlp.optimisers:Epoch 23: Training cost (ce) is 0.315. Accuracy is 91.60%\n",
"INFO:mlp.optimisers:Epoch 23: Validation cost (ce) is 0.434. Accuracy is 91.37%\n",
"INFO:mlp.optimisers:Epoch 23: Took 12 seconds. Training speed 202 pps. Validation speed 1449 pps.\n",
"INFO:mlp.optimisers:Epoch 24: Training cost (ce) is 0.409. Accuracy is 90.00%\n",
"INFO:mlp.optimisers:Epoch 24: Validation cost (ce) is 0.480. Accuracy is 90.29%\n",
"INFO:mlp.optimisers:Epoch 24: Took 11 seconds. Training speed 209 pps. Validation speed 1529 pps.\n",
"INFO:mlp.optimisers:Epoch 25: Training cost (ce) is 0.328. Accuracy is 91.00%\n",
"INFO:mlp.optimisers:Epoch 25: Validation cost (ce) is 0.446. Accuracy is 91.48%\n",
"INFO:mlp.optimisers:Epoch 25: Took 12 seconds. Training speed 215 pps. Validation speed 1443 pps.\n",
"INFO:mlp.optimisers:Epoch 26: Training cost (ce) is 0.309. Accuracy is 91.40%\n",
"INFO:mlp.optimisers:Epoch 26: Validation cost (ce) is 0.477. Accuracy is 91.10%\n",
"INFO:mlp.optimisers:Epoch 26: Took 11 seconds. Training speed 301 pps. Validation speed 1342 pps.\n",
"INFO:mlp.optimisers:Epoch 27: Training cost (ce) is 0.351. Accuracy is 90.00%\n",
"INFO:mlp.optimisers:Epoch 27: Validation cost (ce) is 0.487. Accuracy is 91.21%\n",
"INFO:mlp.optimisers:Epoch 27: Took 10 seconds. Training speed 279 pps. Validation speed 1502 pps.\n",
"INFO:mlp.optimisers:Epoch 28: Training cost (ce) is 0.284. Accuracy is 92.70%\n",
"INFO:mlp.optimisers:Epoch 28: Validation cost (ce) is 0.463. Accuracy is 92.24%\n",
"INFO:mlp.optimisers:Epoch 28: Took 11 seconds. Training speed 264 pps. Validation speed 1311 pps.\n",
"INFO:mlp.optimisers:Epoch 29: Training cost (ce) is 0.360. Accuracy is 91.70%\n",
"INFO:mlp.optimisers:Epoch 29: Validation cost (ce) is 0.456. Accuracy is 92.37%\n",
"INFO:mlp.optimisers:Epoch 29: Took 11 seconds. Training speed 268 pps. Validation speed 1326 pps.\n",
"INFO:mlp.optimisers:Epoch 30: Training cost (ce) is 0.298. Accuracy is 92.30%\n",
"INFO:mlp.optimisers:Epoch 30: Validation cost (ce) is 0.466. Accuracy is 92.34%\n",
"INFO:mlp.optimisers:Epoch 30: Took 11 seconds. Training speed 242 pps. Validation speed 1473 pps.\n",
"INFO:mlp.optimisers:Epoch 31: Training cost (ce) is 0.316. Accuracy is 92.30%\n",
"INFO:mlp.optimisers:Epoch 31: Validation cost (ce) is 0.516. Accuracy is 91.59%\n",
"INFO:mlp.optimisers:Epoch 31: Took 10 seconds. Training speed 251 pps. Validation speed 1558 pps.\n",
"INFO:mlp.optimisers:Epoch 32: Training cost (ce) is 0.308. Accuracy is 92.50%\n",
"INFO:mlp.optimisers:Epoch 32: Validation cost (ce) is 0.481. Accuracy is 92.51%\n",
"INFO:mlp.optimisers:Epoch 32: Took 11 seconds. Training speed 255 pps. Validation speed 1421 pps.\n",
"INFO:mlp.optimisers:Epoch 33: Training cost (ce) is 0.270. Accuracy is 93.60%\n",
"INFO:mlp.optimisers:Epoch 33: Validation cost (ce) is 0.520. Accuracy is 91.91%\n",
"INFO:mlp.optimisers:Epoch 33: Took 10 seconds. Training speed 294 pps. Validation speed 1484 pps.\n",
"INFO:mlp.optimisers:Epoch 34: Training cost (ce) is 0.359. Accuracy is 92.60%\n",
"INFO:mlp.optimisers:Epoch 34: Validation cost (ce) is 0.665. Accuracy is 89.27%\n",
"INFO:mlp.optimisers:Epoch 34: Took 11 seconds. Training speed 263 pps. Validation speed 1397 pps.\n",
"INFO:mlp.optimisers:Epoch 35: Training cost (ce) is 0.423. Accuracy is 91.50%\n",
"INFO:mlp.optimisers:Epoch 35: Validation cost (ce) is 0.548. Accuracy is 92.05%\n",
"INFO:mlp.optimisers:Epoch 35: Took 11 seconds. Training speed 226 pps. Validation speed 1464 pps.\n",
"INFO:mlp.optimisers:Epoch 36: Training cost (ce) is 0.329. Accuracy is 92.40%\n",
"INFO:mlp.optimisers:Epoch 36: Validation cost (ce) is 0.528. Accuracy is 92.26%\n",
"INFO:mlp.optimisers:Epoch 36: Took 11 seconds. Training speed 248 pps. Validation speed 1511 pps.\n",
"INFO:mlp.optimisers:Epoch 37: Training cost (ce) is 0.342. Accuracy is 92.40%\n",
"INFO:mlp.optimisers:Epoch 37: Validation cost (ce) is 0.506. Accuracy is 92.50%\n",
"INFO:mlp.optimisers:Epoch 37: Took 11 seconds. Training speed 223 pps. Validation speed 1486 pps.\n",
"INFO:mlp.optimisers:Epoch 38: Training cost (ce) is 0.313. Accuracy is 92.60%\n",
"INFO:mlp.optimisers:Epoch 38: Validation cost (ce) is 0.599. Accuracy is 90.90%\n",
"INFO:mlp.optimisers:Epoch 38: Took 10 seconds. Training speed 268 pps. Validation speed 1515 pps.\n",
"INFO:mlp.optimisers:Epoch 39: Training cost (ce) is 0.291. Accuracy is 92.40%\n",
"INFO:mlp.optimisers:Epoch 39: Validation cost (ce) is 0.622. Accuracy is 91.38%\n",
"INFO:mlp.optimisers:Epoch 39: Took 11 seconds. Training speed 259 pps. Validation speed 1462 pps.\n",
"INFO:mlp.optimisers:Epoch 40: Training cost (ce) is 0.339. Accuracy is 92.90%\n",
"INFO:mlp.optimisers:Epoch 40: Validation cost (ce) is 0.525. Accuracy is 92.40%\n",
"INFO:mlp.optimisers:Epoch 40: Took 10 seconds. Training speed 303 pps. Validation speed 1421 pps.\n",
"INFO:mlp.optimisers:Epoch 41: Training cost (ce) is 0.317. Accuracy is 93.00%\n",
"INFO:mlp.optimisers:Epoch 41: Validation cost (ce) is 0.653. Accuracy is 90.91%\n",
"INFO:mlp.optimisers:Epoch 41: Took 10 seconds. Training speed 277 pps. Validation speed 1464 pps.\n",
"INFO:mlp.optimisers:Epoch 42: Training cost (ce) is 0.349. Accuracy is 93.20%\n",
"INFO:mlp.optimisers:Epoch 42: Validation cost (ce) is 0.529. Accuracy is 92.87%\n",
"INFO:mlp.optimisers:Epoch 42: Took 11 seconds. Training speed 278 pps. Validation speed 1447 pps.\n",
"INFO:mlp.optimisers:Epoch 43: Training cost (ce) is 0.278. Accuracy is 94.00%\n",
"INFO:mlp.optimisers:Epoch 43: Validation cost (ce) is 0.557. Accuracy is 92.57%\n",
"INFO:mlp.optimisers:Epoch 43: Took 11 seconds. Training speed 216 pps. Validation speed 1497 pps.\n",
"INFO:mlp.optimisers:Epoch 44: Training cost (ce) is 0.340. Accuracy is 93.10%\n",
"INFO:mlp.optimisers:Epoch 44: Validation cost (ce) is 0.640. Accuracy is 91.63%\n",
"INFO:mlp.optimisers:Epoch 44: Took 11 seconds. Training speed 228 pps. Validation speed 1548 pps.\n",
"INFO:mlp.optimisers:Epoch 45: Training cost (ce) is 0.367. Accuracy is 92.20%\n",
"INFO:mlp.optimisers:Epoch 45: Validation cost (ce) is 0.613. Accuracy is 91.58%\n",
"INFO:mlp.optimisers:Epoch 45: Took 11 seconds. Training speed 251 pps. Validation speed 1504 pps.\n",
"INFO:mlp.optimisers:Epoch 46: Training cost (ce) is 0.257. Accuracy is 95.40%\n",
"INFO:mlp.optimisers:Epoch 46: Validation cost (ce) is 0.610. Accuracy is 91.76%\n",
"INFO:mlp.optimisers:Epoch 46: Took 11 seconds. Training speed 237 pps. Validation speed 1493 pps.\n",
"INFO:mlp.optimisers:Epoch 47: Training cost (ce) is 0.282. Accuracy is 94.40%\n",
"INFO:mlp.optimisers:Epoch 47: Validation cost (ce) is 0.628. Accuracy is 91.62%\n",
"INFO:mlp.optimisers:Epoch 47: Took 10 seconds. Training speed 302 pps. Validation speed 1555 pps.\n",
"INFO:mlp.optimisers:Epoch 48: Training cost (ce) is 0.284. Accuracy is 94.50%\n",
"INFO:mlp.optimisers:Epoch 48: Validation cost (ce) is 0.556. Accuracy is 92.37%\n",
"INFO:mlp.optimisers:Epoch 48: Took 10 seconds. Training speed 266 pps. Validation speed 1488 pps.\n",
"INFO:mlp.optimisers:Epoch 49: Training cost (ce) is 0.263. Accuracy is 93.10%\n",
"INFO:mlp.optimisers:Epoch 49: Validation cost (ce) is 0.608. Accuracy is 92.42%\n",
"INFO:mlp.optimisers:Epoch 49: Took 12 seconds. Training speed 206 pps. Validation speed 1479 pps.\n",
"INFO:mlp.optimisers:Epoch 50: Training cost (ce) is 0.280. Accuracy is 94.30%\n",
"INFO:mlp.optimisers:Epoch 50: Validation cost (ce) is 0.628. Accuracy is 92.29%\n",
"INFO:mlp.optimisers:Epoch 50: Took 11 seconds. Training speed 237 pps. Validation speed 1479 pps.\n",
"INFO:mlp.optimisers:Epoch 1: Training cost (ce) is 3.224. Accuracy is 43.80%\n",
"INFO:mlp.optimisers:Epoch 1: Validation cost (ce) is 0.633. Accuracy is 81.88%\n",
"INFO:mlp.optimisers:Epoch 1: Took 10 seconds. Training speed 273 pps. Validation speed 1678 pps.\n",
"INFO:mlp.optimisers:Epoch 2: Training cost (ce) is 0.957. Accuracy is 68.80%\n",
"INFO:mlp.optimisers:Epoch 2: Validation cost (ce) is 0.523. Accuracy is 84.89%\n",
"INFO:mlp.optimisers:Epoch 2: Took 10 seconds. Training speed 247 pps. Validation speed 1687 pps.\n",
"INFO:mlp.optimisers:Epoch 3: Training cost (ce) is 0.797. Accuracy is 73.50%\n",
"INFO:mlp.optimisers:Epoch 3: Validation cost (ce) is 0.613. Accuracy is 80.22%\n",
"INFO:mlp.optimisers:Epoch 3: Took 10 seconds. Training speed 237 pps. Validation speed 1692 pps.\n",
"INFO:mlp.optimisers:Epoch 4: Training cost (ce) is 0.730. Accuracy is 74.80%\n",
"INFO:mlp.optimisers:Epoch 4: Validation cost (ce) is 0.442. Accuracy is 86.47%\n",
"INFO:mlp.optimisers:Epoch 4: Took 10 seconds. Training speed 242 pps. Validation speed 1687 pps.\n",
"INFO:mlp.optimisers:Epoch 5: Training cost (ce) is 0.596. Accuracy is 79.60%\n",
"INFO:mlp.optimisers:Epoch 5: Validation cost (ce) is 0.410. Accuracy is 87.96%\n",
"INFO:mlp.optimisers:Epoch 5: Took 10 seconds. Training speed 239 pps. Validation speed 1710 pps.\n",
"INFO:mlp.optimisers:Epoch 6: Training cost (ce) is 0.598. Accuracy is 80.20%\n",
"INFO:mlp.optimisers:Epoch 6: Validation cost (ce) is 0.388. Accuracy is 88.27%\n",
"INFO:mlp.optimisers:Epoch 6: Took 10 seconds. Training speed 248 pps. Validation speed 1681 pps.\n",
"INFO:mlp.optimisers:Epoch 7: Training cost (ce) is 0.513. Accuracy is 84.10%\n",
"INFO:mlp.optimisers:Epoch 7: Validation cost (ce) is 0.426. Accuracy is 87.53%\n",
"INFO:mlp.optimisers:Epoch 7: Took 10 seconds. Training speed 245 pps. Validation speed 1687 pps.\n",
"INFO:mlp.optimisers:Epoch 8: Training cost (ce) is 0.451. Accuracy is 85.20%\n",
"INFO:mlp.optimisers:Epoch 8: Validation cost (ce) is 0.383. Accuracy is 88.44%\n",
"INFO:mlp.optimisers:Epoch 8: Took 10 seconds. Training speed 255 pps. Validation speed 1681 pps.\n",
"INFO:mlp.optimisers:Epoch 9: Training cost (ce) is 0.507. Accuracy is 82.70%\n",
"INFO:mlp.optimisers:Epoch 9: Validation cost (ce) is 0.380. Accuracy is 89.12%\n",
"INFO:mlp.optimisers:Epoch 9: Took 10 seconds. Training speed 266 pps. Validation speed 1718 pps.\n",
"INFO:mlp.optimisers:Epoch 10: Training cost (ce) is 0.464. Accuracy is 83.70%\n",
"INFO:mlp.optimisers:Epoch 10: Validation cost (ce) is 0.333. Accuracy is 90.46%\n",
"INFO:mlp.optimisers:Epoch 10: Took 10 seconds. Training speed 245 pps. Validation speed 1698 pps.\n",
"INFO:mlp.optimisers:Epoch 11: Training cost (ce) is 0.399. Accuracy is 87.40%\n",
"INFO:mlp.optimisers:Epoch 11: Validation cost (ce) is 0.334. Accuracy is 90.56%\n",
"INFO:mlp.optimisers:Epoch 11: Took 10 seconds. Training speed 246 pps. Validation speed 1672 pps.\n",
"INFO:mlp.optimisers:Epoch 12: Training cost (ce) is 0.422. Accuracy is 86.30%\n",
"INFO:mlp.optimisers:Epoch 12: Validation cost (ce) is 0.348. Accuracy is 89.54%\n",
"INFO:mlp.optimisers:Epoch 12: Took 10 seconds. Training speed 239 pps. Validation speed 1687 pps.\n",
"INFO:mlp.optimisers:Epoch 13: Training cost (ce) is 0.396. Accuracy is 86.20%\n",
"INFO:mlp.optimisers:Epoch 13: Validation cost (ce) is 0.348. Accuracy is 89.37%\n",
"INFO:mlp.optimisers:Epoch 13: Took 10 seconds. Training speed 236 pps. Validation speed 1692 pps.\n",
"INFO:mlp.optimisers:Epoch 14: Training cost (ce) is 0.360. Accuracy is 88.40%\n",
"INFO:mlp.optimisers:Epoch 14: Validation cost (ce) is 0.354. Accuracy is 89.56%\n",
"INFO:mlp.optimisers:Epoch 14: Took 10 seconds. Training speed 248 pps. Validation speed 1675 pps.\n",
"INFO:mlp.optimisers:Epoch 15: Training cost (ce) is 0.362. Accuracy is 87.20%\n",
"INFO:mlp.optimisers:Epoch 15: Validation cost (ce) is 0.351. Accuracy is 89.66%\n",
"INFO:mlp.optimisers:Epoch 15: Took 10 seconds. Training speed 252 pps. Validation speed 1695 pps.\n",
"INFO:mlp.optimisers:Epoch 16: Training cost (ce) is 0.325. Accuracy is 88.40%\n",
"INFO:mlp.optimisers:Epoch 16: Validation cost (ce) is 0.322. Accuracy is 90.95%\n",
"INFO:mlp.optimisers:Epoch 16: Took 10 seconds. Training speed 256 pps. Validation speed 1684 pps.\n",
"INFO:mlp.optimisers:Epoch 17: Training cost (ce) is 0.351. Accuracy is 88.40%\n",
"INFO:mlp.optimisers:Epoch 17: Validation cost (ce) is 0.334. Accuracy is 90.02%\n",
"INFO:mlp.optimisers:Epoch 17: Took 10 seconds. Training speed 246 pps. Validation speed 1684 pps.\n",
"INFO:mlp.optimisers:Epoch 18: Training cost (ce) is 0.277. Accuracy is 91.80%\n",
"INFO:mlp.optimisers:Epoch 18: Validation cost (ce) is 0.329. Accuracy is 90.35%\n",
"INFO:mlp.optimisers:Epoch 18: Took 10 seconds. Training speed 265 pps. Validation speed 1698 pps.\n",
"INFO:mlp.optimisers:Epoch 19: Training cost (ce) is 0.297. Accuracy is 89.50%\n",
"INFO:mlp.optimisers:Epoch 19: Validation cost (ce) is 0.308. Accuracy is 91.28%\n",
"INFO:mlp.optimisers:Epoch 19: Took 10 seconds. Training speed 245 pps. Validation speed 1678 pps.\n",
"INFO:mlp.optimisers:Epoch 20: Training cost (ce) is 0.295. Accuracy is 90.20%\n",
"INFO:mlp.optimisers:Epoch 20: Validation cost (ce) is 0.319. Accuracy is 90.59%\n",
"INFO:mlp.optimisers:Epoch 20: Took 10 seconds. Training speed 246 pps. Validation speed 1689 pps.\n",
"INFO:mlp.optimisers:Epoch 21: Training cost (ce) is 0.289. Accuracy is 90.10%\n",
"INFO:mlp.optimisers:Epoch 21: Validation cost (ce) is 0.298. Accuracy is 91.54%\n",
"INFO:mlp.optimisers:Epoch 21: Took 10 seconds. Training speed 251 pps. Validation speed 1689 pps.\n",
"INFO:mlp.optimisers:Epoch 22: Training cost (ce) is 0.248. Accuracy is 92.10%\n",
"INFO:mlp.optimisers:Epoch 22: Validation cost (ce) is 0.315. Accuracy is 90.82%\n",
"INFO:mlp.optimisers:Epoch 22: Took 10 seconds. Training speed 250 pps. Validation speed 1689 pps.\n",
"INFO:mlp.optimisers:Epoch 23: Training cost (ce) is 0.268. Accuracy is 90.40%\n",
"INFO:mlp.optimisers:Epoch 23: Validation cost (ce) is 0.309. Accuracy is 90.99%\n",
"INFO:mlp.optimisers:Epoch 23: Took 10 seconds. Training speed 242 pps. Validation speed 1712 pps.\n",
"INFO:mlp.optimisers:Epoch 24: Training cost (ce) is 0.257. Accuracy is 90.70%\n",
"INFO:mlp.optimisers:Epoch 24: Validation cost (ce) is 0.300. Accuracy is 91.57%\n",
"INFO:mlp.optimisers:Epoch 24: Took 10 seconds. Training speed 248 pps. Validation speed 1684 pps.\n",
"INFO:mlp.optimisers:Epoch 25: Training cost (ce) is 0.240. Accuracy is 91.30%\n",
"INFO:mlp.optimisers:Epoch 25: Validation cost (ce) is 0.297. Accuracy is 91.10%\n",
"INFO:mlp.optimisers:Epoch 25: Took 10 seconds. Training speed 246 pps. Validation speed 1692 pps.\n",
"INFO:mlp.optimisers:Epoch 26: Training cost (ce) is 0.242. Accuracy is 91.10%\n",
"INFO:mlp.optimisers:Epoch 26: Validation cost (ce) is 0.297. Accuracy is 91.43%\n",
"INFO:mlp.optimisers:Epoch 26: Took 10 seconds. Training speed 277 pps. Validation speed 1681 pps.\n",
"INFO:mlp.optimisers:Epoch 27: Training cost (ce) is 0.228. Accuracy is 93.10%\n",
"INFO:mlp.optimisers:Epoch 27: Validation cost (ce) is 0.289. Accuracy is 92.07%\n",
"INFO:mlp.optimisers:Epoch 27: Took 10 seconds. Training speed 245 pps. Validation speed 1687 pps.\n",
"INFO:mlp.optimisers:Epoch 28: Training cost (ce) is 0.241. Accuracy is 92.60%\n",
"INFO:mlp.optimisers:Epoch 28: Validation cost (ce) is 0.320. Accuracy is 90.39%\n",
"INFO:mlp.optimisers:Epoch 28: Took 10 seconds. Training speed 246 pps. Validation speed 1698 pps.\n",
"INFO:mlp.optimisers:Epoch 29: Training cost (ce) is 0.227. Accuracy is 92.20%\n",
"INFO:mlp.optimisers:Epoch 29: Validation cost (ce) is 0.283. Accuracy is 91.96%\n",
"INFO:mlp.optimisers:Epoch 29: Took 10 seconds. Training speed 234 pps. Validation speed 1715 pps.\n",
"INFO:mlp.optimisers:Epoch 30: Training cost (ce) is 0.217. Accuracy is 93.20%\n",
"INFO:mlp.optimisers:Epoch 30: Validation cost (ce) is 0.299. Accuracy is 91.30%\n",
"INFO:mlp.optimisers:Epoch 30: Took 10 seconds. Training speed 268 pps. Validation speed 1730 pps.\n",
"INFO:mlp.optimisers:Epoch 31: Training cost (ce) is 0.191. Accuracy is 93.60%\n",
"INFO:mlp.optimisers:Epoch 31: Validation cost (ce) is 0.277. Accuracy is 92.34%\n",
"INFO:mlp.optimisers:Epoch 31: Took 10 seconds. Training speed 254 pps. Validation speed 1692 pps.\n",
"INFO:mlp.optimisers:Epoch 32: Training cost (ce) is 0.212. Accuracy is 93.40%\n",
"INFO:mlp.optimisers:Epoch 32: Validation cost (ce) is 0.287. Accuracy is 91.60%\n",
"INFO:mlp.optimisers:Epoch 32: Took 11 seconds. Training speed 215 pps. Validation speed 1710 pps.\n",
"INFO:mlp.optimisers:Epoch 33: Training cost (ce) is 0.170. Accuracy is 94.30%\n",
"INFO:mlp.optimisers:Epoch 33: Validation cost (ce) is 0.302. Accuracy is 91.77%\n",
"INFO:mlp.optimisers:Epoch 33: Took 10 seconds. Training speed 244 pps. Validation speed 1724 pps.\n",
"INFO:mlp.optimisers:Epoch 34: Training cost (ce) is 0.206. Accuracy is 93.30%\n",
"INFO:mlp.optimisers:Epoch 34: Validation cost (ce) is 0.277. Accuracy is 92.13%\n",
"INFO:mlp.optimisers:Epoch 34: Took 11 seconds. Training speed 213 pps. Validation speed 1701 pps.\n",
"INFO:mlp.optimisers:Epoch 35: Training cost (ce) is 0.191. Accuracy is 93.80%\n",
"INFO:mlp.optimisers:Epoch 35: Validation cost (ce) is 0.286. Accuracy is 92.08%\n",
"INFO:mlp.optimisers:Epoch 35: Took 10 seconds. Training speed 253 pps. Validation speed 1678 pps.\n",
"INFO:mlp.optimisers:Epoch 36: Training cost (ce) is 0.191. Accuracy is 92.00%\n",
"INFO:mlp.optimisers:Epoch 36: Validation cost (ce) is 0.289. Accuracy is 91.89%\n",
"INFO:mlp.optimisers:Epoch 36: Took 11 seconds. Training speed 194 pps. Validation speed 1687 pps.\n",
"INFO:mlp.optimisers:Epoch 37: Training cost (ce) is 0.190. Accuracy is 92.70%\n",
"INFO:mlp.optimisers:Epoch 37: Validation cost (ce) is 0.280. Accuracy is 92.15%\n",
"INFO:mlp.optimisers:Epoch 37: Took 11 seconds. Training speed 192 pps. Validation speed 1678 pps.\n",
"INFO:mlp.optimisers:Epoch 38: Training cost (ce) is 0.200. Accuracy is 93.50%\n",
"INFO:mlp.optimisers:Epoch 38: Validation cost (ce) is 0.317. Accuracy is 91.17%\n",
"INFO:mlp.optimisers:Epoch 38: Took 11 seconds. Training speed 186 pps. Validation speed 1684 pps.\n",
"INFO:mlp.optimisers:Epoch 39: Training cost (ce) is 0.198. Accuracy is 93.30%\n",
"INFO:mlp.optimisers:Epoch 39: Validation cost (ce) is 0.301. Accuracy is 91.26%\n",
"INFO:mlp.optimisers:Epoch 39: Took 11 seconds. Training speed 194 pps. Validation speed 1678 pps.\n",
"INFO:mlp.optimisers:Epoch 40: Training cost (ce) is 0.159. Accuracy is 94.90%\n",
"INFO:mlp.optimisers:Epoch 40: Validation cost (ce) is 0.277. Accuracy is 92.25%\n",
"INFO:mlp.optimisers:Epoch 40: Took 11 seconds. Training speed 212 pps. Validation speed 1667 pps.\n",
"INFO:mlp.optimisers:Epoch 41: Training cost (ce) is 0.150. Accuracy is 94.70%\n",
"INFO:mlp.optimisers:Epoch 41: Validation cost (ce) is 0.265. Accuracy is 92.79%\n",
"INFO:mlp.optimisers:Epoch 41: Took 11 seconds. Training speed 195 pps. Validation speed 1710 pps.\n",
"INFO:mlp.optimisers:Epoch 42: Training cost (ce) is 0.147. Accuracy is 95.20%\n",
"INFO:mlp.optimisers:Epoch 42: Validation cost (ce) is 0.268. Accuracy is 92.54%\n",
"INFO:mlp.optimisers:Epoch 42: Took 10 seconds. Training speed 238 pps. Validation speed 1684 pps.\n",
"INFO:mlp.optimisers:Epoch 43: Training cost (ce) is 0.171. Accuracy is 93.90%\n",
"INFO:mlp.optimisers:Epoch 43: Validation cost (ce) is 0.281. Accuracy is 91.75%\n",
"INFO:mlp.optimisers:Epoch 43: Took 10 seconds. Training speed 229 pps. Validation speed 1713 pps.\n",
"INFO:mlp.optimisers:Epoch 44: Training cost (ce) is 0.123. Accuracy is 95.50%\n",
"INFO:mlp.optimisers:Epoch 44: Validation cost (ce) is 0.276. Accuracy is 92.50%\n",
"INFO:mlp.optimisers:Epoch 44: Took 11 seconds. Training speed 210 pps. Validation speed 1678 pps.\n",
"INFO:mlp.optimisers:Epoch 45: Training cost (ce) is 0.136. Accuracy is 95.30%\n",
"INFO:mlp.optimisers:Epoch 45: Validation cost (ce) is 0.288. Accuracy is 91.73%\n",
"INFO:mlp.optimisers:Epoch 45: Took 11 seconds. Training speed 185 pps. Validation speed 1675 pps.\n",
"INFO:mlp.optimisers:Epoch 46: Training cost (ce) is 0.178. Accuracy is 93.60%\n",
"INFO:mlp.optimisers:Epoch 46: Validation cost (ce) is 0.286. Accuracy is 92.01%\n",
"INFO:mlp.optimisers:Epoch 46: Took 11 seconds. Training speed 213 pps. Validation speed 1672 pps.\n",
"INFO:mlp.optimisers:Epoch 47: Training cost (ce) is 0.139. Accuracy is 95.40%\n",
"INFO:mlp.optimisers:Epoch 47: Validation cost (ce) is 0.262. Accuracy is 92.84%\n",
"INFO:mlp.optimisers:Epoch 47: Took 11 seconds. Training speed 186 pps. Validation speed 1687 pps.\n",
"INFO:mlp.optimisers:Epoch 48: Training cost (ce) is 0.141. Accuracy is 95.30%\n",
"INFO:mlp.optimisers:Epoch 48: Validation cost (ce) is 0.265. Accuracy is 92.80%\n",
"INFO:mlp.optimisers:Epoch 48: Took 11 seconds. Training speed 199 pps. Validation speed 1675 pps.\n",
"INFO:mlp.optimisers:Epoch 49: Training cost (ce) is 0.120. Accuracy is 95.50%\n",
"INFO:mlp.optimisers:Epoch 49: Validation cost (ce) is 0.261. Accuracy is 92.97%\n",
"INFO:mlp.optimisers:Epoch 49: Took 11 seconds. Training speed 206 pps. Validation speed 1684 pps.\n",
"INFO:mlp.optimisers:Epoch 50: Training cost (ce) is 0.123. Accuracy is 96.10%\n",
"INFO:mlp.optimisers:Epoch 50: Validation cost (ce) is 0.272. Accuracy is 92.63%\n",
"INFO:mlp.optimisers:Epoch 50: Took 10 seconds. Training speed 248 pps. Validation speed 1670 pps.\n",
"INFO:root:Testing the model on test set:\n",
"INFO:root:MNIST test set accuracy is 91.75 %, cost (ce) is 0.624\n"
"INFO:root:MNIST test set accuracy is 91.94 %, cost (ce) is 0.291\n"
]
}
],

15
mlp/layers.py
View File

@ -113,18 +113,18 @@ class MLP(object):
if p_inp < 1:
d_inp = self.rng.binomial(1, p_inp, size=x.shape)
self.activations[0] = p_inp_scaler*d_inp*x
self.activations[0] = p_inp_scaler*d_inp*x #it's OK to scale the inputs by p_inp_scaler here
self.activations[1] = self.layers[0].fprop(self.activations[0])
for i in xrange(1, len(self.layers)):
d_hid = 1
if p_hid < 1:
d_hid = self.rng.binomial(1, p_hid, size=self.activations[i].shape)
self.activations[i] *= p_hid_scaler * d_hid
self.activations[i+1] = self.layers[i].fprop(self.activations[i])
self.activations[i] *= d_hid #but not the hidden activations, since the non-linearity grad *may* explicitly depend on them
self.activations[i+1] = self.layers[i].fprop(p_hid_scaler*self.activations[i])
return self.activations[-1]
def bprop(self, cost_grad):
def bprop(self, cost_grad, dp_scheduler=None):
"""
:param cost_grad: matrix -- grad of the cost w.r.t y
:return: None, the deltas are kept in the model
@ -144,10 +144,15 @@ class MLP(object):
self.deltas[top_layer_idx], ograds = self.layers[top_layer_idx - 1].\
bprop_cost(self.activations[top_layer_idx], cost_grad, self.cost)
p_hid_scaler = 1.0
if dp_scheduler is not None:
p_inp, p_hid = dp_scheduler.get_rate()
p_hid_scaler /= p_hid
# then back-prop through remaining layers
for i in xrange(top_layer_idx - 1, 0, -1):
self.deltas[i], ograds = self.layers[i - 1].\
bprop(self.activations[i], ograds)
bprop(self.activations[i], ograds*p_hid_scaler)
def add_layer(self, layer):
self.layers.append(layer)

2
mlp/optimisers.py
View File

@ -124,7 +124,7 @@ class SGDOptimiser(Optimiser):
cost_grad = model.cost.grad(y, t)
# do backward pass through the model
model.bprop(cost_grad)
model.bprop(cost_grad, self.dp_scheduler)
#update the model, here we iterate over layers
#and then over each parameter in the layer