From 98c3bb1b1581a927910823bef7f35c954cea7aef Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Tue, 7 Mar 2023 10:19:51 -0500 Subject: [PATCH] updates files --- docs/02-Quickstart.md | 110 +++++++-------- docs/03-Tensors.md | 8 +- docs/04-Data.md | 13 +- docs/06-BuildModel.md | 60 ++++---- docs/07-Autograd.md | 23 ++- docs/08-Optimization.md | 227 +++++++++++++++--------------- docs/09-SaveLoad.md | 7 +- docs/docs/04-Data_20_1.png | Bin 0 -> 8447 bytes docs/docs/04-Data_5_0.png | Bin 0 -> 26472 bytes tutorials/04-Data.ipynb | 240 +++++++++++++++++++++++++++----- tutorials/07-Autograd.ipynb | 235 ++++++++++++++++++++++++++----- tutorials/08-Optimization.ipynb | 200 ++++++++++++++++++++++---- tutorials/09-SaveLoad.ipynb | 69 ++++++--- 13 files changed, 844 insertions(+), 348 deletions(-) create mode 100644 docs/docs/04-Data_20_1.png create mode 100644 docs/docs/04-Data_5_0.png diff --git a/docs/02-Quickstart.md b/docs/02-Quickstart.md index c529ed9..4edd486 100644 --- a/docs/02-Quickstart.md +++ b/docs/02-Quickstart.md @@ -229,78 +229,78 @@ print("Done!") Epoch 1 ------------------------------- - loss: 2.300704 [ 0/60000] - loss: 2.294491 [ 6400/60000] - loss: 2.270792 [12800/60000] - loss: 2.270757 [19200/60000] - loss: 2.246651 [25600/60000] - loss: 2.223734 [32000/60000] - loss: 2.230299 [38400/60000] - loss: 2.197789 [44800/60000] - loss: 2.186385 [51200/60000] - loss: 2.171854 [57600/60000] + loss: 2.300994 [ 0/60000] + loss: 2.289627 [ 6400/60000] + loss: 2.278757 [12800/60000] + loss: 2.273481 [19200/60000] + loss: 2.260533 [25600/60000] + loss: 2.230715 [32000/60000] + loss: 2.240870 [38400/60000] + loss: 2.210235 [44800/60000] + loss: 2.205794 [51200/60000] + loss: 2.179301 [57600/60000] Test Error: - Accuracy: 40.4%, Avg loss: 2.158354 + Accuracy: 42.7%, Avg loss: 2.175595 Epoch 2 ------------------------------- - loss: 2.157282 [ 0/60000] - loss: 2.157837 [ 6400/60000] - loss: 2.098653 [12800/60000] - loss: 2.123712 [19200/60000] - loss: 2.070209 [25600/60000] - loss: 2.017735 [32000/60000] - loss: 2.044564 [38400/60000] - loss: 1.971302 [44800/60000] - loss: 1.963748 [51200/60000] - loss: 1.920766 [57600/60000] + loss: 2.179688 [ 0/60000] + loss: 2.170581 [ 6400/60000] + loss: 2.125383 [12800/60000] + loss: 2.134987 [19200/60000] + loss: 2.104071 [25600/60000] + loss: 2.039638 [32000/60000] + loss: 2.065766 [38400/60000] + loss: 1.994649 [44800/60000] + loss: 1.991123 [51200/60000] + loss: 1.927214 [57600/60000] Test Error: - Accuracy: 55.5%, Avg loss: 1.902382 + Accuracy: 56.1%, Avg loss: 1.929943 Epoch 3 ------------------------------- - loss: 1.919148 [ 0/60000] - loss: 1.903148 [ 6400/60000] - loss: 1.782882 [12800/60000] - loss: 1.834309 [19200/60000] - loss: 1.722989 [25600/60000] - loss: 1.676954 [32000/60000] - loss: 1.698752 [38400/60000] - loss: 1.602475 [44800/60000] - loss: 1.614792 [51200/60000] - loss: 1.532669 [57600/60000] + loss: 1.957387 [ 0/60000] + loss: 1.929036 [ 6400/60000] + loss: 1.825893 [12800/60000] + loss: 1.850506 [19200/60000] + loss: 1.775094 [25600/60000] + loss: 1.708617 [32000/60000] + loss: 1.727947 [38400/60000] + loss: 1.628896 [44800/60000] + loss: 1.653404 [51200/60000] + loss: 1.548985 [57600/60000] Test Error: - Accuracy: 61.7%, Avg loss: 1.533873 + Accuracy: 60.7%, Avg loss: 1.570322 Epoch 4 ------------------------------- - loss: 1.585873 [ 0/60000] - loss: 1.560321 [ 6400/60000] - loss: 1.407954 [12800/60000] - loss: 1.488211 [19200/60000] - loss: 1.364034 [25600/60000] - loss: 1.362447 [32000/60000] - loss: 1.370802 [38400/60000] - loss: 1.302972 [44800/60000] - loss: 1.327800 [51200/60000] - loss: 1.235748 [57600/60000] + loss: 1.634544 [ 0/60000] + loss: 1.598077 [ 6400/60000] + loss: 1.457816 [12800/60000] + loss: 1.511364 [19200/60000] + loss: 1.425202 [25600/60000] + loss: 1.398494 [32000/60000] + loss: 1.412483 [38400/60000] + loss: 1.328141 [44800/60000] + loss: 1.371268 [51200/60000] + loss: 1.270080 [57600/60000] Test Error: - Accuracy: 63.4%, Avg loss: 1.260575 + Accuracy: 63.2%, Avg loss: 1.298073 Epoch 5 ------------------------------- - loss: 1.331637 [ 0/60000] - loss: 1.313866 [ 6400/60000] - loss: 1.153163 [12800/60000] - loss: 1.257744 [19200/60000] - loss: 1.137783 [25600/60000] - loss: 1.162715 [32000/60000] - loss: 1.172138 [38400/60000] - loss: 1.120971 [44800/60000] - loss: 1.149632 [51200/60000] - loss: 1.069323 [57600/60000] + loss: 1.375485 [ 0/60000] + loss: 1.353134 [ 6400/60000] + loss: 1.197045 [12800/60000] + loss: 1.282228 [19200/60000] + loss: 1.185837 [25600/60000] + loss: 1.195442 [32000/60000] + loss: 1.213788 [38400/60000] + loss: 1.140980 [44800/60000] + loss: 1.188507 [51200/60000] + loss: 1.102179 [57600/60000] Test Error: - Accuracy: 64.6%, Avg loss: 1.093657 + Accuracy: 64.6%, Avg loss: 1.124997 Done! diff --git a/docs/03-Tensors.md b/docs/03-Tensors.md index d87b048..19d68fd 100644 --- a/docs/03-Tensors.md +++ b/docs/03-Tensors.md @@ -73,8 +73,8 @@ print(f"Random Tensor: \n {x_rand} \n") [1, 1]]) Random Tensor: - tensor([[0.0504, 0.9505], - [0.6485, 0.6105]]) + tensor([[0.7786, 0.0142], + [0.3120, 0.9157]]) @@ -97,8 +97,8 @@ print(f"Zeros Tensor: \n {zeros_tensor}") ``` Random Tensor: - tensor([[0.6582, 0.2838, 0.1244], - [0.1692, 0.0394, 0.2638]]) + tensor([[0.7263, 0.5640, 0.3222], + [0.9226, 0.3125, 0.3739]]) Ones Tensor: tensor([[1., 1., 1.], diff --git a/docs/04-Data.md b/docs/04-Data.md index 8cd12e9..e133ffd 100644 --- a/docs/04-Data.md +++ b/docs/04-Data.md @@ -1,8 +1,3 @@ -```python -%matplotlib inline -``` - - [Learn the Basics](intro.html) || [Quickstart](quickstart_tutorial.html) || [Tensors](tensorqs_tutorial.html) || @@ -49,6 +44,8 @@ We load the [FashionMNIST Dataset](https://pytorch.org/vision/stable/datasets.ht ```python +%matplotlib inline + import torch from torch.utils.data import Dataset from torchvision import datasets @@ -106,7 +103,7 @@ plt.show() -![png](../docs/04-Data_files/../docs/04-Data_6_0.png) +![png](../docs/04-Data_files/../docs/04-Data_5_0.png) @@ -268,11 +265,11 @@ print(f"Label: {label}") -![png](../docs/04-Data_files/../docs/04-Data_21_1.png) +![png](../docs/04-Data_files/../docs/04-Data_20_1.png) - Label: 1 + Label: 3 -------------- diff --git a/docs/06-BuildModel.md b/docs/06-BuildModel.md index 92a7fca..6ed386e 100644 --- a/docs/06-BuildModel.md +++ b/docs/06-BuildModel.md @@ -115,7 +115,7 @@ y_pred = pred_probab.argmax(1) print(f"Predicted class: {y_pred}") ``` - Predicted class: tensor([9]) + Predicted class: tensor([1]) -------------- @@ -191,30 +191,26 @@ hidden1 = nn.ReLU()(hidden1) print(f"After ReLU: {hidden1}") ``` - Before ReLU: tensor([[-5.5712e-01, 4.1135e-01, -7.4510e-03, -5.4891e-02, 7.3538e-02, - 4.6617e-01, 5.3287e-01, 7.2283e-02, -3.7471e-01, -3.9285e-01, - -6.7889e-01, 2.1088e-01, 1.8742e-01, 4.0150e-01, -5.6422e-02, - -4.8977e-02, -1.6230e-01, 3.0556e-01, -7.1455e-01, -6.6180e-02], - [-4.2601e-01, 6.2487e-01, -5.9415e-02, 2.3934e-02, 3.9810e-01, - 3.2441e-01, 7.0026e-01, -1.2423e-01, -5.2260e-01, -1.7234e-01, - -5.5835e-01, 2.2128e-01, 2.7830e-01, 2.4191e-01, -7.7681e-02, - -2.4954e-01, 1.5836e-01, 1.9990e-01, -1.1715e-01, -3.2138e-01], - [-4.9225e-01, 4.1050e-01, -1.5492e-01, 8.9106e-03, 3.5985e-01, - 3.1355e-01, 6.2615e-01, -1.9053e-04, -5.7080e-01, -1.7064e-01, - -6.5802e-01, 3.3700e-01, 4.5726e-01, 3.1022e-01, -4.0316e-01, - -3.8029e-01, -1.2243e-01, 3.6732e-01, -5.6789e-01, -9.4490e-02]], - grad_fn=) + Before ReLU: tensor([[-0.6535, 0.0475, 0.2762, 0.2739, 0.3857, 0.1837, -0.1904, -0.3036, + -0.0609, -0.2871, 0.0446, 0.2365, -0.2100, 0.3802, 0.1994, -0.4515, + 0.1591, 0.1378, 0.1966, -0.0231], + [-0.7906, 0.0717, 0.3879, 0.0195, 0.2133, 0.4331, 0.1080, -0.3002, + -0.0044, -0.3400, 0.2174, 0.4808, -0.1150, 0.2409, 0.3484, -0.0483, + 0.3890, 0.1460, 0.1570, 0.1086], + [-0.8346, 0.3771, 0.3634, -0.3699, 0.5272, -0.2396, -0.4630, -0.0269, + -0.0439, -0.4653, 0.1175, 0.4506, -0.1127, 0.1764, 0.1627, 0.0395, + 0.4420, 0.1518, 0.0156, 0.0423]], grad_fn=) - After ReLU: tensor([[0.0000, 0.4113, 0.0000, 0.0000, 0.0735, 0.4662, 0.5329, 0.0723, 0.0000, - 0.0000, 0.0000, 0.2109, 0.1874, 0.4015, 0.0000, 0.0000, 0.0000, 0.3056, - 0.0000, 0.0000], - [0.0000, 0.6249, 0.0000, 0.0239, 0.3981, 0.3244, 0.7003, 0.0000, 0.0000, - 0.0000, 0.0000, 0.2213, 0.2783, 0.2419, 0.0000, 0.0000, 0.1584, 0.1999, - 0.0000, 0.0000], - [0.0000, 0.4105, 0.0000, 0.0089, 0.3599, 0.3136, 0.6262, 0.0000, 0.0000, - 0.0000, 0.0000, 0.3370, 0.4573, 0.3102, 0.0000, 0.0000, 0.0000, 0.3673, - 0.0000, 0.0000]], grad_fn=) + After ReLU: tensor([[0.0000, 0.0475, 0.2762, 0.2739, 0.3857, 0.1837, 0.0000, 0.0000, 0.0000, + 0.0000, 0.0446, 0.2365, 0.0000, 0.3802, 0.1994, 0.0000, 0.1591, 0.1378, + 0.1966, 0.0000], + [0.0000, 0.0717, 0.3879, 0.0195, 0.2133, 0.4331, 0.1080, 0.0000, 0.0000, + 0.0000, 0.2174, 0.4808, 0.0000, 0.2409, 0.3484, 0.0000, 0.3890, 0.1460, + 0.1570, 0.1086], + [0.0000, 0.3771, 0.3634, 0.0000, 0.5272, 0.0000, 0.0000, 0.0000, 0.0000, + 0.0000, 0.1175, 0.4506, 0.0000, 0.1764, 0.1627, 0.0395, 0.4420, 0.1518, + 0.0156, 0.0423]], grad_fn=) ### nn.Sequential @@ -281,23 +277,23 @@ for name, param in model.named_parameters(): ) - Layer: linear_relu_stack.0.weight | Size: torch.Size([512, 784]) | Values : tensor([[ 0.0211, 0.0168, 0.0334, ..., -0.0151, -0.0033, 0.0032], - [-0.0022, 0.0293, -0.0090, ..., -0.0044, -0.0147, -0.0251]], + Layer: linear_relu_stack.0.weight | Size: torch.Size([512, 784]) | Values : tensor([[ 0.0007, 0.0351, 0.0290, ..., 0.0157, -0.0041, -0.0052], + [ 0.0163, -0.0053, 0.0237, ..., -0.0294, 0.0200, 0.0072]], grad_fn=) - Layer: linear_relu_stack.0.bias | Size: torch.Size([512]) | Values : tensor([0.0128, 0.0086], grad_fn=) + Layer: linear_relu_stack.0.bias | Size: torch.Size([512]) | Values : tensor([-0.0143, -0.0101], grad_fn=) - Layer: linear_relu_stack.2.weight | Size: torch.Size([512, 512]) | Values : tensor([[-0.0165, -0.0068, -0.0016, ..., -0.0098, 0.0119, 0.0326], - [ 0.0330, -0.0306, -0.0129, ..., -0.0371, -0.0291, -0.0273]], + Layer: linear_relu_stack.2.weight | Size: torch.Size([512, 512]) | Values : tensor([[-0.0091, 0.0016, 0.0303, ..., 0.0147, 0.0108, 0.0114], + [-0.0018, 0.0363, -0.0248, ..., -0.0332, 0.0185, 0.0011]], grad_fn=) - Layer: linear_relu_stack.2.bias | Size: torch.Size([512]) | Values : tensor([ 0.0024, -0.0164], grad_fn=) + Layer: linear_relu_stack.2.bias | Size: torch.Size([512]) | Values : tensor([0.0409, 0.0064], grad_fn=) - Layer: linear_relu_stack.4.weight | Size: torch.Size([10, 512]) | Values : tensor([[ 0.0046, 0.0249, 0.0123, ..., 0.0352, -0.0170, 0.0232], - [ 0.0038, 0.0283, 0.0235, ..., -0.0416, 0.0304, 0.0217]], + Layer: linear_relu_stack.4.weight | Size: torch.Size([10, 512]) | Values : tensor([[ 0.0349, -0.0004, 0.0420, ..., -0.0023, 0.0277, 0.0173], + [ 0.0015, -0.0185, 0.0072, ..., -0.0159, -0.0068, 0.0271]], grad_fn=) - Layer: linear_relu_stack.4.bias | Size: torch.Size([10]) | Values : tensor([0.0118, 0.0417], grad_fn=) + Layer: linear_relu_stack.4.bias | Size: torch.Size([10]) | Values : tensor([-0.0129, 0.0260], grad_fn=) diff --git a/docs/07-Autograd.md b/docs/07-Autograd.md index f8e1eee..5580131 100644 --- a/docs/07-Autograd.md +++ b/docs/07-Autograd.md @@ -1,8 +1,3 @@ -```python -%matplotlib inline -``` - - [Learn the Basics](intro.html) || [Quickstart](quickstart_tutorial.html) || [Tensors](tensorqs_tutorial.html) || @@ -31,6 +26,8 @@ PyTorch in the following manner: ```python +%matplotlib inline + import torch x = torch.ones(5) # input tensor @@ -77,8 +74,8 @@ print(f"Gradient function for z = {z.grad_fn}") print(f"Gradient function for loss = {loss.grad_fn}") ``` - Gradient function for z = - Gradient function for loss = + Gradient function for z = + Gradient function for loss = ## Computing Gradients @@ -101,12 +98,12 @@ print(w.grad) print(b.grad) ``` - tensor([[0.3244, 0.2353, 0.0700], - [0.3244, 0.2353, 0.0700], - [0.3244, 0.2353, 0.0700], - [0.3244, 0.2353, 0.0700], - [0.3244, 0.2353, 0.0700]]) - tensor([0.3244, 0.2353, 0.0700]) + tensor([[0.1692, 0.2790, 0.2088], + [0.1692, 0.2790, 0.2088], + [0.1692, 0.2790, 0.2088], + [0.1692, 0.2790, 0.2088], + [0.1692, 0.2790, 0.2088]]) + tensor([0.1692, 0.2790, 0.2088])

Note

- We can only obtain the ``grad`` properties for the leaf diff --git a/docs/08-Optimization.md b/docs/08-Optimization.md index aeb9aa0..3367728 100644 --- a/docs/08-Optimization.md +++ b/docs/08-Optimization.md @@ -1,8 +1,3 @@ -```python -%matplotlib inline -``` - - [Learn the Basics](intro.html) || [Quickstart](quickstart_tutorial.html) || [Tensors](tensorqs_tutorial.html) || @@ -28,6 +23,8 @@ and [Build Model](buildmodel_tutorial.html). ```python +%matplotlib inline + import torch from torch import nn from torch.utils.data import DataLoader @@ -209,153 +206,153 @@ print("Done!") Epoch 1 ------------------------------- - loss: 2.310308 [ 64/60000] - loss: 2.291682 [ 6464/60000] - loss: 2.282847 [12864/60000] - loss: 2.278148 [19264/60000] - loss: 2.259573 [25664/60000] - loss: 2.246842 [32064/60000] - loss: 2.237948 [38464/60000] - loss: 2.221490 [44864/60000] - loss: 2.215676 [51264/60000] - loss: 2.186174 [57664/60000] + loss: 2.299604 [ 64/60000] + loss: 2.281797 [ 6464/60000] + loss: 2.269583 [12864/60000] + loss: 2.255457 [19264/60000] + loss: 2.240205 [25664/60000] + loss: 2.213762 [32064/60000] + loss: 2.215705 [38464/60000] + loss: 2.184422 [44864/60000] + loss: 2.175044 [51264/60000] + loss: 2.137501 [57664/60000] Test Error: - Accuracy: 50.1%, Avg loss: 2.185173 + Accuracy: 53.1%, Avg loss: 2.138075 Epoch 2 ------------------------------- - loss: 2.192464 [ 64/60000] - loss: 2.176265 [ 6464/60000] - loss: 2.138019 [12864/60000] - loss: 2.155484 [19264/60000] - loss: 2.096774 [25664/60000] - loss: 2.064352 [32064/60000] - loss: 2.073422 [38464/60000] - loss: 2.019561 [44864/60000] - loss: 2.018754 [51264/60000] - loss: 1.944076 [57664/60000] + loss: 2.153558 [ 64/60000] + loss: 2.139259 [ 6464/60000] + loss: 2.081727 [12864/60000] + loss: 2.085114 [19264/60000] + loss: 2.046907 [25664/60000] + loss: 1.977491 [32064/60000] + loss: 2.007782 [38464/60000] + loss: 1.928677 [44864/60000] + loss: 1.934681 [51264/60000] + loss: 1.844566 [57664/60000] Test Error: - Accuracy: 56.9%, Avg loss: 1.951974 + Accuracy: 59.0%, Avg loss: 1.855136 Epoch 3 ------------------------------- - loss: 1.979550 [ 64/60000] - loss: 1.944613 [ 6464/60000] - loss: 1.850896 [12864/60000] - loss: 1.885921 [19264/60000] - loss: 1.766024 [25664/60000] - loss: 1.721881 [32064/60000] - loss: 1.732149 [38464/60000] - loss: 1.646069 [44864/60000] - loss: 1.663508 [51264/60000] - loss: 1.542335 [57664/60000] + loss: 1.898872 [ 64/60000] + loss: 1.859855 [ 6464/60000] + loss: 1.745800 [12864/60000] + loss: 1.771856 [19264/60000] + loss: 1.671929 [25664/60000] + loss: 1.624660 [32064/60000] + loss: 1.646571 [38464/60000] + loss: 1.553838 [44864/60000] + loss: 1.585847 [51264/60000] + loss: 1.463247 [57664/60000] Test Error: - Accuracy: 60.8%, Avg loss: 1.575167 + Accuracy: 62.0%, Avg loss: 1.491152 Epoch 4 ------------------------------- - loss: 1.641383 [ 64/60000] - loss: 1.597785 [ 6464/60000] - loss: 1.460881 [12864/60000] - loss: 1.522893 [19264/60000] - loss: 1.394849 [25664/60000] - loss: 1.381750 [32064/60000] - loss: 1.389999 [38464/60000] - loss: 1.324359 [44864/60000] - loss: 1.359623 [51264/60000] - loss: 1.242349 [57664/60000] + loss: 1.570554 [ 64/60000] + loss: 1.524995 [ 6464/60000] + loss: 1.381242 [12864/60000] + loss: 1.440385 [19264/60000] + loss: 1.325888 [25664/60000] + loss: 1.331313 [32064/60000] + loss: 1.343411 [38464/60000] + loss: 1.273921 [44864/60000] + loss: 1.314914 [51264/60000] + loss: 1.204072 [57664/60000] Test Error: - Accuracy: 63.2%, Avg loss: 1.281596 + Accuracy: 63.9%, Avg loss: 1.234092 Epoch 5 ------------------------------- - loss: 1.364956 [ 64/60000] - loss: 1.337699 [ 6464/60000] - loss: 1.179997 [12864/60000] - loss: 1.276043 [19264/60000] - loss: 1.145318 [25664/60000] - loss: 1.163051 [32064/60000] - loss: 1.179221 [38464/60000] - loss: 1.127842 [44864/60000] - loss: 1.170320 [51264/60000] - loss: 1.072596 [57664/60000] + loss: 1.318503 [ 64/60000] + loss: 1.292388 [ 6464/60000] + loss: 1.131896 [12864/60000] + loss: 1.229624 [19264/60000] + loss: 1.102847 [25664/60000] + loss: 1.138407 [32064/60000] + loss: 1.157674 [38464/60000] + loss: 1.099932 [44864/60000] + loss: 1.145054 [51264/60000] + loss: 1.048841 [57664/60000] Test Error: - Accuracy: 64.8%, Avg loss: 1.102368 + Accuracy: 65.2%, Avg loss: 1.074347 Epoch 6 ------------------------------- - loss: 1.181124 [ 64/60000] - loss: 1.175671 [ 6464/60000] - loss: 0.999543 [12864/60000] - loss: 1.125861 [19264/60000] - loss: 0.994338 [25664/60000] - loss: 1.020635 [32064/60000] - loss: 1.052101 [38464/60000] - loss: 1.005876 [44864/60000] - loss: 1.050259 [51264/60000] - loss: 0.969423 [57664/60000] + loss: 1.147973 [ 64/60000] + loss: 1.144627 [ 6464/60000] + loss: 0.967731 [12864/60000] + loss: 1.098405 [19264/60000] + loss: 0.965783 [25664/60000] + loss: 1.007831 [32064/60000] + loss: 1.040992 [38464/60000] + loss: 0.989532 [44864/60000] + loss: 1.033878 [51264/60000] + loss: 0.949742 [57664/60000] Test Error: - Accuracy: 65.8%, Avg loss: 0.989962 + Accuracy: 66.5%, Avg loss: 0.970729 Epoch 7 ------------------------------- - loss: 1.055653 [ 64/60000] - loss: 1.073796 [ 6464/60000] - loss: 0.878792 [12864/60000] - loss: 1.027988 [19264/60000] - loss: 0.902191 [25664/60000] - loss: 0.923560 [32064/60000] - loss: 0.970771 [38464/60000] - loss: 0.927402 [44864/60000] - loss: 0.969056 [51264/60000] - loss: 0.901827 [57664/60000] + loss: 1.027588 [ 64/60000] + loss: 1.047764 [ 6464/60000] + loss: 0.855220 [12864/60000] + loss: 1.011105 [19264/60000] + loss: 0.879051 [25664/60000] + loss: 0.915307 [32064/60000] + loss: 0.963445 [38464/60000] + loss: 0.917342 [44864/60000] + loss: 0.956093 [51264/60000] + loss: 0.882487 [57664/60000] Test Error: - Accuracy: 66.8%, Avg loss: 0.914991 + Accuracy: 67.8%, Avg loss: 0.899503 Epoch 8 ------------------------------- - loss: 0.964512 [ 64/60000] - loss: 1.004631 [ 6464/60000] - loss: 0.793878 [12864/60000] - loss: 0.959500 [19264/60000] - loss: 0.842306 [25664/60000] - loss: 0.854395 [32064/60000] - loss: 0.914801 [38464/60000] - loss: 0.875149 [44864/60000] - loss: 0.910963 [51264/60000] - loss: 0.853945 [57664/60000] + loss: 0.938215 [ 64/60000] + loss: 0.979911 [ 6464/60000] + loss: 0.774328 [12864/60000] + loss: 0.949241 [19264/60000] + loss: 0.821273 [25664/60000] + loss: 0.847455 [32064/60000] + loss: 0.908044 [38464/60000] + loss: 0.868443 [44864/60000] + loss: 0.899046 [51264/60000] + loss: 0.833970 [57664/60000] Test Error: - Accuracy: 67.8%, Avg loss: 0.861828 + Accuracy: 69.0%, Avg loss: 0.847812 Epoch 9 ------------------------------- - loss: 0.895530 [ 64/60000] - loss: 0.953656 [ 6464/60000] - loss: 0.731293 [12864/60000] - loss: 0.908750 [19264/60000] - loss: 0.800252 [25664/60000] - loss: 0.803487 [32064/60000] - loss: 0.873069 [38464/60000] - loss: 0.838708 [44864/60000] - loss: 0.867891 [51264/60000] - loss: 0.817475 [57664/60000] + loss: 0.869330 [ 64/60000] + loss: 0.928490 [ 6464/60000] + loss: 0.713798 [12864/60000] + loss: 0.903194 [19264/60000] + loss: 0.780315 [25664/60000] + loss: 0.796318 [32064/60000] + loss: 0.865808 [38464/60000] + loss: 0.834232 [44864/60000] + loss: 0.856279 [51264/60000] + loss: 0.797004 [57664/60000] Test Error: - Accuracy: 68.9%, Avg loss: 0.821918 + Accuracy: 70.2%, Avg loss: 0.808560 Epoch 10 ------------------------------- - loss: 0.841097 [ 64/60000] - loss: 0.913210 [ 6464/60000] - loss: 0.683007 [12864/60000] - loss: 0.869649 [19264/60000] - loss: 0.768555 [25664/60000] - loss: 0.764901 [32064/60000] - loss: 0.839639 [38464/60000] - loss: 0.811697 [44864/60000] - loss: 0.834432 [51264/60000] - loss: 0.788075 [57664/60000] + loss: 0.814243 [ 64/60000] + loss: 0.887317 [ 6464/60000] + loss: 0.666916 [12864/60000] + loss: 0.867729 [19264/60000] + loss: 0.749378 [25664/60000] + loss: 0.757221 [32064/60000] + loss: 0.831676 [38464/60000] + loss: 0.808831 [44864/60000] + loss: 0.822820 [51264/60000] + loss: 0.767592 [57664/60000] Test Error: - Accuracy: 70.1%, Avg loss: 0.790321 + Accuracy: 71.5%, Avg loss: 0.777352 Done! diff --git a/docs/09-SaveLoad.md b/docs/09-SaveLoad.md index d07b56e..7502494 100644 --- a/docs/09-SaveLoad.md +++ b/docs/09-SaveLoad.md @@ -1,8 +1,3 @@ -```python -%matplotlib inline -``` - - [Learn the Basics](intro.html) || [Quickstart](quickstart_tutorial.html) || [Tensors](tensorqs_tutorial.html) || @@ -20,6 +15,8 @@ In this section we will look at how to persist model state with saving, loading ```python +%matplotlib inline + import torch import torchvision.models as models ``` diff --git a/docs/docs/04-Data_20_1.png b/docs/docs/04-Data_20_1.png new file mode 100644 index 0000000000000000000000000000000000000000..5b12ec7390c1c3557f7dd62b75f76e5208064b61 GIT binary patch literal 8447 zcmbuF2{@E{-^cIDsiQ(iD}@#mveekdT9hSgmPD9TjGeme%cxY2vSc@7B1?prY}u*f zl*$-n%N}ZMA^Xnz8@=b8_kEw|ywCHTb6vT*s+s%#FTely_xXPR1?p<6vTWttiXaG! zy4o2%1lf=XKaYOg41ZI)gSrcT!@8Wk=%Vjr<>GE}!xGW5aB;SGaJw6x#!xo*eSh!Bz7Ln5cOB;WGuGwsm} z{;Bwo@kv`#-p)@#9puwLpKMDy{L|s+%MZ3nU%MG~Xpc=|NQ;udK;&WJC#~a~b1Ds2 zD?~d8UhM-%irHzd*?U_9GWYt<3}Xk-ALQz$n2?*B>(ggD=0--t*nGjBl1LqlE&uC=rC+0LWd$98QM*zsc#H!Je;?UY<|bF+Z3 zu$H`+>5@rWV^n?M&ZCj>@nc^5keg0VlaiGDx3E>vB_CzqQA&I`a_H0Mvxk2|>7jnh z$=IF2Dz(JnVOvxsT*SQ0n#OzdXdEWNcb16NH!{N0G-JhHk;6n*crTA)7T zm0X~hcYXhiq-f%sH!7oDrAEcY#aS%_bh`P3`x_P(miyKHS$mLEbMvoeqK z0s;bAEomt!Pct%xb4Ryr-MWi4;L@IMjx9)+YN_wVnGsPn&v^IOd()$j8z)^n-jZuz zV36PX5J4Vk##PEXetLNeCQ$M*h|%@lF3Y3pDfMRUKE5uEGt{B`-|Z&jl?vdF31 zwlVA^dG8mtwzks&ip!^?qN3i@Xf4sgmmyT^x|)*Dq7XWj|vsy>$uU>7cW}K#WO9JYKc5+^VReq-ty@W8gy!JRX{C~h{2OE zJTmrUOQgS zhNh+`&MASv{pv9m?faD(O?B$2plhBxcOTmylQ7kr8s~Ttzq-`3M#8Wu%s9eMmg;we{(DeqjOmXiUwC4s<`@*+*V*0gT**@N~5W{xgW@3DLRTP zO-3|90fBeqcQl$LzHg!kdqmxTGs{VQZf-7>7#_}!(woxt>Dy&Qv;Snj{@D`!?`C?Altos-l+mH7NQ~Q zU%a>hF*PbJ<2Nud$;i(ax3sbvu2>ptY)Dq6lsG6W`}a+FWd4BE%jbJC2Xo0@7-`9B zY{|MbvpiLonUQgBC%>kymR9ZVLeXQ#bfphC8h>bP?C?@UZfa-LY8(?4)tT$|rlk=ajgXs6jdZ^0k5@y}i;6B#uzoa$l ztd62+z*jt|C0&!3eWHM}X$xyChQNPkI)eGG5EY%YGS=;D zEXDT`h_dpmg!`B&%FmFA@aZfk-x_lIvuEP?v5F-}l>RpC2wl^`Aq#tCA}8; zrxM~YgjKD!wl>CG_Bd1yn7hCJy{7;5iTgifP{ovLrm3eFj@=n}#%DA$CFNH#*7?&- z?9Q!ssYI zVzCVUo$q^4aii$EDT6xx+Jb3bZhsvF?V7Bvr8QGPsZ(6*B&Sm-QtR+QeE}ghEmCvX2%^!(0E@ z%M&s6AzZTJSKn-Z^5hBNkaho?;H>yLIh~qlVSN)5(&FrBCYgMIOU~nY)|+j}pc0kO z0EeT59Z|Jy&M-9cSta7*8}9EG%C~J69Bs=}1rZ@hI*^8khbty7Z$Qi*EHqb_mC1ei z^2KZdTQ`+|hyZMR6`g%~wZrcrr3!8(9lA)ntcuHhsxwhZy%u;S=VFcC`ST6j${P_EC5J_eW{Se}MMJ~pVSiTu zz!)tWx##^$jmx{Fs}alXL#{>(?YP3#ChATao%7VkviM z@$#8RR73LRM1+JI0na2ZzeKuj)CjMGmnbg#kn%n2w+|1QRQMDx&WxmO;r4i}wS^T? z^P#gdj+KPv`^f0(g@uLi6f8;~e7zUf)E}ytEn2y(7b-XTv#p(-1Qc5N%52BhyGjmH zwy-=qzAaN_K88Ri`la{zh=t2mg1OLA5AJUFvC45ZpT5AOR3e>jh0*~70>MBRX| zs^4wJ_WhDnvW=soBp&**6%4bAP0^nA^C4m8bzNMJI&MO@JVxzw@v!bu}zbR<;0Dbwn_zpZk z;>KsSER%xaj1J8N1v8KjVUVv&6%7aB6A=r+%$}Z}?`vvG>5{K5*UxcrZD3duU>E~~ z6gA$YXU~q;#!2bGG8uKfNK8CE;ZfGS5gFueFS|22IXOL9BWkxaJL(0~D*)EkdL&~A zdTZ2K@%ZtnUgsjqiK!M{fwrfL$CSoTZKll+;H`U0ZHv7<*-;|I)U z7P2lC5|TIcuEoiD=3>z-N^SE28JD-PmpNCfNtnVew>T_1|IGrLPRcVv$XP9|U~FTd z6Aq>OvnjyQ&=eG0SZW7CETwB28b)FD&CMNYKI7g!fT}iaxt4S)oEG2F%+f1k(*KWb z|JMz)am@X~UB9buf==MU;yTin!7xbTdBLmId^c>Hx9st4Ge-x2uL(?QnH;{`NWEVju+rv?tpXmRa2o(D=-=7*MRYwjybC^!8 z@Li%3I~+@EVIdh7GnL^h7slKx?Pv#N-55tj!FMqhvjR=y;mSNQvLG=LGK3WH^h8X@ z3kM8-ac|AUVcb))JdDjp8t7Km0xUd6=;B6;G$O<>6~N6eDB_ zYfC+#!VDg!Q`dV5#33XMa5MwJ!s|&SPL!WJ6sRoZ5pJC~HnSeErpZAu(y6slJ@&L+ zyLW>utP2Hfz?;^J5~07RKnLeK(a@1YriKt|P&(nXQ;E=tW7lar2-oZUk-RY7s8dT^ zNARiuHYq$PKrM%QMpl*t{v-wi^TKt+%P+iWj{3t!UKLCPh6v`ss3hvDg{hsv0JDRK zzU0vF84s^W1;;Zp+F=BUS24C_6Ee7{$!B0{`s{~od#vZix*(c>a~p93MGbMdp4?hY zom?_REY<)@?Rof600{|)_I%cras}kziCFVqZ|Gqr^5%I~!H2_so*nDTRFB|)Hzk*4 zTCArLBhviLIFDD|t~J{btZ)kjM+xBedOE5C^x)AmD3N&aIKMil*yWok8ifYg85w$x zj*c}TZ@Mtw7TP0O%DlU{xH!Zag(e8nVo z9u9b#6gelFdyh&u^i-_m>ZswZtOVg}V4s9XIQQ>2gq3C9$)8{QGpP8zp>x)+ZvVhw z#cq#&mb*%c(XF0?Y-21-t@BkES&*O4zknLnjGOspj!B#Ss>saC?6Kskx=a7P_eyFvjk!ciXSML~dpu z`Q#Zt;I(5-h&yzNbcTI(;n9vsAT&ulV+`wXO{hz}wP_0pL$G4P`Yz^UG1vRfz(3r{ z37>v<6wut**qBa*hD{|h){(xw^XJ!B&S`5yl=wGH-nyw;)=CIs?Zvx|Fa*SEo#|YL zLW(a$@N1pKL()a%sLZo=>MUyb9tWTh)1j%VY)f@P?w!{IAVT2a;Gk1M#=jHeX0vU7 zOELge-Wp}=Bq-1|$GgjGi3N78B+L}O7KC14&jTX3Ps^3Q2cq9Y6MhV)5%dglNg}48 zpny(=24+V)4(^0JQB#+aqxs(UkBs8qr6#`Ll3wt0x)&})Q#v0WQl=8uS%^cCPzH&} zNiB6oYyKY0&J&ki=fCyRsZi+*_q*Aj;g(O}NhA^#MDLTNq$CN$4lKs>_mm1)DSogT z7TV{_dq#{ySXm@u3VR5pSnxBKp}Raj?N22_x5q8za!S}nV$s3+L%5~)j0A|GS;UkS zUexO2a|=2ZEJiIydjBe$8~%XBZ~aoAdH1ee*3{S&U=r@1IZVQ$sjNH-owjknv4T5( z%r|)xyuUToe>393lFBY9BGPjEhi&=ZbB??B?8!(^R|l7sL7_MmbWIO68X6n7<=VI9 zKGE93f(&wMI%#_R6OAf(23gBFEiH$*pEe>T&zz=7y2tlK@B)lHZeC1EIt4A7~UN+BBuT&RB78@ktS6h-xNkr)Vv7≻!9D|;zfHS~gt$~{S^ zZct4tv)MU@NP*r7hxSiMpJhV2?oilG@~y)p9okd4WZkR*LR&i}IfDg{98m`2YviM_ zZ@VBkKaR~He#CPyMq!hDHA>}b$^HBHBZ)-ME=@oiK|w)?lYH6Ok$MTD`A(DPmv5PS z4e@2+D$Su^fTU(*WPE_h2&l_=&pzY!{(54xy+8xJFMRO>pscv_r&C>Mo&4w|Tvzal zz%9dYb0(iUw3DnBigBI_IzTgo90D{s-?2*uQazOhYXkjM&4zm{`*(pR+1G<6y;>eA zvK#6sq`YwKS`6OIgbXUgp54z01y1Q)m>Q@huA_t0WP5*Wb5j#!DlUy$QbY_R3moby zEo3+X2}Wgyi@OZ!qI8J5r105u{-aUi5)yDs7dUwAV_)w2hntgsjbQ5+EX*E{?C(|l z7G*wCI=Dj&FEd#)glE^T+-bW}O)lr6=DT(QlaWn!k&?tiOMr|K86A*;@)^u`&oAOSjkR3_Y&j$xVSi12$rE|)L4au zRQI3L(J?xE_O&bCYV^G{YmD;&kzQ%jU#=IWU=D1ln=&vmN@Mi-!Rw<&zA{}=x}n_i z;vU~V**O9Xc4q11Qhxu!oqp%Qk$&go!q2_I%)bTt%!W$PhIQm;A|bjMNtAwly$C+X zT;0naQpFeOc0i_U|4wMhs!C=M4VCQq-SMnnQbp+UO?RSo0vL=w5T!!>UI|_B&yFJsu4Y|OzN~`jSE&!Dg zobehxJ-t3JwbAT~3Ui>ZrVPW(?D^$?&OIVhY`rrf{=8ZV3gyiUIhHlitOqQXg#JvA z+EQi7Y*IH4q%dK5%9yTKbGdu4DK0x_x-yh=?$@0<(wyh3y@%7&${>p>;l8IIUe=29 zEB|sGMe|}vq+$7$=|(El)Az zsfXsd`Ynso@%=p;AJzmh4p)WDGmf6vcnXh?uwh+^kWLz^iIXO+)vdlc<)0UTg zw8+xu!Iy@Y3@kW5=|9LvTc{thYN&{&^t8X4DywO6wFHYS?8klL%AGf%JQT1dS`6|e z5Gty|id(ztSk&SE%&jINFE395LJrGo*Q}wplO*h-tHKIGe?vtGx!v2)t|6bI_Umeq zQ-=;^-vJ-+G$kdah7g_(z2NK7_^}kfwN*}d4_twovArfzy1CvKUoQst8Wa>H;z>9T zx*I3y@B!{W8;4s&yG9ugD+;H1De#_Qo*C=9l&li0XJKKHIRl&sVchNH^Tn;uV3Y$` z1vrCCxbI_NYZ%SR)lpnhGNY)d=u*P|@b7;5zKAOU)`KzJf>!fJSTL}AZNm#=Hxx|A zrA^h-kLn%E)}eYi9UM`L|!Tb^EPO@iU6jLY^cXvfH~ zP7Tz}0roOpZg}Csi{k^dM|SH+rB#+S&_#h)lNpG0rDts=Ato%W7uKs|sUeimG(7rq z*Q>8>dF9sP!!wY-YiMXNqKPk%>((>W1XjCWy~G}_3PT1wW4fc4n)G!%A0Z1HVegE$J_q*rj@%Uel$90)&nC6`C_w#u#uh;YazF=gaJNq}W z-=<8NGFxw(&aNp_e)7YAFaJ6ne`jip`xX2`Ik9E`iQP`ePPiX(IXXrE(23&?PA42} z4=;8*>O!}5a#WI6mS4AKvCWAS$LVSc3bg61qgy7rfBbQ4&NkOS&KYmrB6+Pa zIWFbm@Wlk#i?K>we}-Cb-6*M>`qSKmfS*+UI1{jIPekS4KmT;<@?YHXF-7g3o$oAF zqc$%-H8PSdpR%J`tNpXDLBDV5;gAfslLF5ee3o}nH8+yy;xDFHPd_U?Wy(p}6{Gk+ z5}V%7z;FKgn>PNBuF>GsDN{C_eu_W$SZDzr%9OwxU#3l&a{TABESbfN-`yHFV>=wM zkY4pJy)MmKI#WI*B;>YLf#1V}Pfq7(X}r7d{Mq!kS+hdY)7O7$Dk#_)t+uzPr>AZB z593*<+lHsKB`#dOuX3=L8l<^=N0|JzWjjJE12XMvI(JGfn)2fQfwWutkt!cNV?NiV z=^lH1eTl4Y;MJjCLGjCA+4kWtC(}6P8G9@(FOVWv1S^H&VWDEp1qZe7XJ5SSf{rF~+eAc_u zZQ52YMa$+%ODj0#b>pGDJ43B`dtH&*oSfTvr7_>T_-L5L@<=2xpzu3o*mVCL-E*Vr`PtFNCb zZr{6iPtCvX;Zcsid?T9{`SZ^|zr+_zq?%c0Ee$zHXnKy)`yA_DBaeW1YvBgBJn=Lr+bc zQQhV}tuo`mgFk1^oEiM?-MhBht1EXmKb@s;_3yubzHs3}aBrEmm8IX%(Qb{Dt)0QL zy3L)rU7>hO7XktNX&TV&bd1&t4k@3*It5K{h_ejyaPcc?%aVD=#m9dq-c~^WCW#rS+Gz3y;6(TF}zm{PLQK<;sCZ z?-Y&h=kwp**fgy!*FEKHTh)$NuU<8$7D|P@e!cU+gM+8)>1CP^j}|ZT93K++HkLg& z^z7Edhr90RM=r<7x#m04U)0l5eEY_Y8^el*8r^?ue0{Uor6Z%})#kaHOOGBs+Vi!- za0g4vw>i@3(cSK{xA?&7;Z*Y#S_n#QsWo&FL?li;5de!RHw_{`f(kKh5uQ&G` z{uF;ZAz>RQNZM-bI*rFJdVXR4Z9D}TnV!BHt72Ma!}47BuF&(+ep$oLHJ08<8cy^$ZDECWSkjmhIiU*X7gguvfxX zK1;|GAm5tp)FdAwXBg@?He7UXZ{ku74Rf5P<FE5OH`E&G(p#cB`GLs2Sa0)Yql#1G8gUheglFy_)(GC;+IjuYCzlwE`$Fx! z%oI79#(0*WsjT0`*nqdpha~gthg(PQL^}5T^wUpyEdTMkUX3!doD+BP=ic7lVXx5{ zX64td^E^`7J`^z})2T_zP$y4&VyLXGyQ4Ps*{t7w>v7EOV2gD>efpQe+O;L(s*dK& z8X1{Jd|t!0jA?y!xZ)Z@l+J=TZ;ZOg_wU7n&hP1Q=_=Qi?wdD%e*V3CtIB#coW_Rx zVgft{8r-dzg$3Olar~e8%Xe;wR+y2Xq+7FV-@ee{>$sFDco{K*Pfg2oYhK}cn90Um zyVi1HTVIZkAlh9+sJr_9fw<1DYBQ(44<9~^`s)4pj^$-&?BkR7yU3q6wmKADS@t&B zLj4o2I&)CD##}Z=AX&56MFyAXZS0m=MZ$Ex-J3il$qeqKyAXeaGUp8~h-hyyuJa!!c z@8yk6XO^m{?E7ijv}GGM?3eNzJXBOxmeRR@^mAI7j?R(#ENA5y|If9l^AcTdm2dLfZ@-c27X0MQj7tG8n_7rOTFWam;R6WLqA4Hr=NDukoI+f<4+bdrd5s zDOs0Z?cik=Bc_u-C7<`@uLF4=-Qk|)g2m&*UKerB11~K@#4<1_b?*NB*+EE1Z~OLn z!Tu9H*)I$HgGaT)$G3(A2QN`57=0KJ;834s-00Q5*0VWSmjpvgpY9iJ0&>=t%E~@V z;ZHABj#hKMvgOjEA`Vl$r9`#q>brLb5wnbim_GnM>mgc4cI%09nc4OmUAH>FWo;^oI6%YT7kKBFhQJ2yDvrV>t9)Gtn zT+Q{~ean2)QtgS+w!y@dN2{hzo%*)G-yd)J_3H&f!P0(%n{W_zjb$P6nz0J@ChqHb zvw3b2uGi{Blaxh5!Amxu+lKe#Q1|ex|I=T_{dIF2Q3Whpym%vC(9%EueBkA+r*|Q* zpKGt|BL8Ucu+WhE`BDGmCih0q^UGIkurCahbScvu*yiAHe`nN&^5I$`*{v^eKvrJ6 zcFhHuBBru;#`Nj0&I((upmEcKYpnb&_N7=_SPV)o6ct^$`TU$~4#H^_O)|}xClri?C%WF|GAT{`%_(LPXEAx!Nzs2AULP zCgf!hZjC*^JehH{y*jB%>9q@;-a^jy%a_~2*ByN&>aTE8>1acaYt`8UetwVg5Y8WZ zbRQPg^h#fEU%m6(Txt0-t&#f-X>Ci#yzZ^I_hHsCo?HLgyxCsE5LdM2;_BqfpDN;a z96WdfA!=iFij_h|ykS^v?S9)gCz~!m$?o`QdM_d2a$cUs(1%M3M-brViHNM|Dwue* zR8G#U7w6Q)bF8oCXzhbTE`^fr+XgxtNR)Bw;4UKt7w+XAl8h7{4aTlH@KA?*B9b`Y zzTFj0q2CGE)_3`%xMS9)r_Y|X#9orWf?V|qWyq1X%DeN##aH8-sdsd}pxjN)-BG&8 z)Wiv&nHkjd$TwHjZ`4CUQL%j1oH_lbnT{QrV_z#QkzJXENbVlg!oa?H^XBE@dvu{d z`idjGH!e|CRkbF!HNrkmg~s&=4i2{5v*&8f-Q99s*@Bo}gju|k`fPgs_}GZIqN1Xu zjZH#WaPWJkql191rqfO-xNq$7;isti6)VGW+pZ^AY>J zcgGv_k=h5z%F1u(_OSEq+qWB~CkAyDmMwcRYu2n@udg59C+$ytdWp&G>&$6SR`a;_ z_>(&-*_kuDckSAh_sDm|i&PU~VPPixZg$rdlCi?AW4hJcJ3pg9@XhamtV*SdH!4{1(~98bf`CxB|x>)Jv`9Wb^Fd8uO&;CSbBNo zs5;hf=t9QW;Oq39BLBGe_qA$MPKt#pSjh`poCwYvA8uDZHtna!b$_Ec4@||WI*@w! z>}Z=w!Idvxjt`BGbhQ-8v996?54}GxZMXjX6#UVq?rA5f&!0bkSt#wl#(%t*b^E~s zqbpag9QpX><~rB5Jr}j6SSu%gtNHua|4Rd~;Zw4uNoR+ckkCy>Wb`OSsd@8aTWwTb zCqBiqDx(mYMEC`xJ(nWlD97KO%+b%C_5?Svt=F$~Utelw(Y1`}nGdSl;^Q(`XjD7- z^@=MCkoi?Ps!omfTWy*`*z!X&q&yyrD6CjvL~ryINV0-Tq`BFj);&>QcZj_~edAEt0hp1&|km=aK=yoa?+gDPT@13dm=vaxE za(q@+t%yI0q^c-HlDoXpt$pu>or79lnVvD*Zy#^f)tJxhUb*{D0tN4J9fziW35S^D zv3KvahzcZH>L}viZhCs_HZ%J9>NTiX;#x^bC8Drq%^rGs*Vx62zbOlL-poeNG)3mG z$=2%ke{HQceDdzgd0PDx2fvvfsB&zXnwn_|5>%>T*)o|c(a|(US5tviNqOKBB(D-m z4JUibx!?fet2{TehrUh%B0cMU77RD@b?JtxS7w@J^d3{_$^+9K3SL)v6Yt7(s2ThXAZ5h&9KKo0iPN|GWgLaRX z-YtuI$u3Z3-HZPG(8x%8M*N*S_bA+6NevI965BVA2kdvF_TYG&IFTMPUtAo&A0MAc zQFrhB+VQwe=trJm$JWx3fi6`}nR=Ioa=d*_GAAo(#P->_*_^?)s>CXZBiM$6Uw zG9umL9Ea%7zF6ja-KB7|Q&Cha$LobG=Xh+|bSfe)+M3P8}}$D`hJ=+ zoO^7y-LJp z`?wm8)4SJi{7Shm7nzkZRT~ppZTPe2NNl*#dh7P>DsJk~GqzTSkSsTAp1*C%Uh&(9 z)Ti^>y9LwxOanWtZ}=wHZT)H^yHP~oGOX-azrW!Qvm48T`t^je+|s4bMJAA&Dx+?0J}<^+CpUih zaF|-po{lb-;cO=Np!yea2?>O~=tS=9+ATWUw{K^3qvcW7^-dZ46qQllIL$xj7L9@g z#qmJvA5nq#$qJg9@vNU-&i3toA&g@y^Z_Y!4;`5eCtS@sL;0Y(>h9gUncW{h+9>Oe z1mcJ8q9eIDF%V+SODK`fFa8HP=l^}1oNS3_Dx6K?I}VWu!unX78W+)v)?bv5^u3p! z9+}2-G%B#Kkw-u&S8WSb*}g?bT5+92?Vl``p?KHg*r4ImXeBn?C@x8v_B9}28(LDu z)^cF*Ea%#t@1SoKFV7%> z+WO6nO`NQbTqCNw>Y39%3aWFYG(PsyZ3TB|3+f<;FuLh|KCRyi70Hd%&yD$J4b`{w z=2OkvH!cvGIct^@hbBo!{YHM>ita-Tx5ux3yxvGEp4Gu=R&DZ0QTy2PfwRHtGQ7vY13{{<&tioD>P9ZuWiv~YMoYnVNI-bWKx{*2;+h4>8&K7 zvE^GvM}4RoYuDM@z@=YPffEcML*2!HjdkRm!hYqhi|D-QO!(hw^CP) zi;b;}!r2qyo1T9V@TeNmv{Is@w^l{y#-n%N`?BA{I*E(4s2#ZGOlRKU{El-Y+ z2i5RuZX-JM$|wfocj_qm{+>@+M!D%i(kYLI??otieYK(bbk=9v8N_X6GPB~X?v zTgJ(vRbEzE^+YA~sWf+kFfZ#@+VA=Z-4PWxP7QSxvlUmH8ldHfh?6|i(PtyOI(-bc zqA{;d5S8yeAjXIEWH-krF3FI0JsK%XjBj6k-fvGj^1eKYEtYwO1&3(OOen*@2wze{Z~G2X4AvgB$LLIG^pPm8679%lDDw_1fIn z^yH;x1_lO!ihe5fM~2Pa&?t!V(WUL8r^|Ok!%UGPsX;8LYCpldJ!m z%KHCa%Wu^K39=-{m9*0M3vnt8hOP&4)2xF~ZiS~OEz>5cKpPp~X~sFz+YJ&H;&rmI zL2FW(^r1rS@k0!r<2os6X(z^r66cA^x-H^yH<(*eX%$;n)wwtn-eIyHFmHd+Mc1a7 z%zXPtc~d6Z*F?&J0|z)BI1QB&ypncf^>o%zl&KMEG@c4;fQ~L7uc!PN*Zt6e1L4RS z%Dmbe2?;A$14<}lsrCH_u17@JF}lbr#S=Epta3N+{-z}V+jo3Z<^8*vbxifqDs=YB z*61xbS=y@BkeBmwk}PuF@Rs6?)u(sd>DZt|+HhRxN4;Uy%67To+0}i?t^t=mHaM|6 zos)LdA&E#J+-gham=>la&EK$i@#FZcfW{XzGnQ#)62f@<*+2^4NwlZxj?s@XIyw;|Js6;Y95rHv$96&V@XUNOGqbD>4k569KP zjk$-?I*zqMJ6Gh;;@0>y6a;J|t&_7qXG>*&N$Y!YS{^4rNY>6(o0}j_TXQkMue7$* zn)dnvLIb;wv)YC?N@3CB4x|qh_CFd@irGpnF6Xc%)J|+GnMsm&Z%>a;<6JvKbMq)g zM|IV?QW`c4{LlW9a*k@3GkOaI4atHOLFsHf1Qp+9K-PTSIwDQlP(j-rs3dbg`F0GbKD6hV$9i2OVVm{X;o%+D_qa6jWpn=Wq zbo4JxgybyBhaYmTtn3#h|LTp04&79YiHNw@3YB>>i$|Xsk1FGi(?(K#1$Hc}wWjGt zbO;p)E){h?DnFd1x;L4cMkq>=a!5y4SDb%o(b^u*33lo%u1zQ;rUXi^TgN(%xU$_X z^qOg$PXmx~@AA9`I%y2NnlQUk#3*Eu4brr{)3ncmEe=HkHaiG~=j8$8|<1_GQargM#kSxclY&wVXyp1#h#Q zsmcO0uxS3;IsN%o-@oR#9HwjsTZZn|t@kL90yu*Z4D4F22fkJ7O{Xp@&t(~D*X4Mn zWPSLJI*OXth@Kv}FKNDoPS@*-4>H5U4_sf{T0BLvwF~U`VJBPq=g?>t0oJG8UaL?j zlr}b6*lLl?{8CNt{z0GLzcBm9g@uI^DO|AtCKGj;-fVWDds19gwR3W+Yny%hn|@FF z$2Uu;8WhR_v>1^=5|r3h6u=3*HudjSzuzpuCmV#NZKx|D6d?2uUf|wh+P@@+E8Tf2 z*G42tlc2ZuH*$D>8(48ppqjKcqj@l*18E=EjivT0g9&MExH1RHHOxvf+t&V&2dOhjqQaP4 zlAYnWOejC{-h&4Z_65##Re;6Put-P#p>D8g~c&O2`3eo--o^tbg;ym7+jbO;I@8`PBLmM`as)KupL z_-WNup9@7gF;&f>^=#R{|2n$qwyKyimG^&*#Ee4CjBAxZ|Jktb`{(VOVav%L+$b(M zY(Kw-!(OmpK_zEN^l@!=@bBa4-TbeKB?A*T&r9vfRCa2t7s+h6ZZ2-IKveWDu98xX z;|c{Cm%%TGsH5;XchRAsBsoqiRe}Ebpp1gDO@*Wr60zpnrkCrp1!>G<-Ez9#n$FJ7 zb_U2Dids&M8p;9(&O+^fk&OO{wHEnWZ`7RQ;_%+&dCa?GHyg5Yk;m-EZ@@e!xGkrK?DNlu2oo^rc>mT#yQ}Jpd%`)tc zG8D|%J4?zf4QYOUEcx%175{kJ&hlC>%gR(at5>bUsb=F)N2F$i3NKt3eJ5_}lqlPh z2xTSCv}x0-Bxq~&&`Xz%3|L3)fFyxbDM4h-j>l_8goSM1-nvj$=B~n^mH$ZA?C)xF zV%V!*S=8OdsT^OiW(~SZ)ikXWAK!dZlbG^YH1z6KNI(6O3l}tddme0;s?Iq`3sFcf zJ=`(UsZ(eaN%?-q`hV_hb91u@A3~~~!HE-JKdD8DL7r|uF)`(2PE|JDnyRs3L#;rt zYVF#+nMk*ye5SbSK00)8W;ayxL<-5&9#E|#;!skUWX_h-fKmpV)~E3Ys?;1W-v_=U zZq!kj$wqX%R@a<4@#iGf-9-3sAC2gE4*ofFW=uJLuCKkOOvI_`-nEF{*Ggp~eE1J` zrNn{j7F9ZyKq(GWQ01t*w%uw?x34J`DS++?pLcaiUMx`vAlPmBvG*1atqPhA+wmV~^pNHq#GO}AYi{AQ%o7vy zdRp`5=H?kN10K8+)-Hg^A;@FLp6gayWYlVKv>Vv{k@5~`5J&2 z@h#b>cDp{bPiNN@mAh_wf1X}WeBG|R-arX0pTS#FESP!W;o&{7btN-9YP(*AtQ|F| zF3;HZUckypIouYGrk(s-J7;atv5R`CJ0bchE=cPNu|(mC+=f@O4rm6y2NtC4d^v-w zC_;81H4%5`&dzX|g$oxdOQ{}xwIKV;(^<=wEZKxstr;Fv7@YG126t2-Ce z|MSY#CdpTMP6ZbGlFe5_Vr-)pOpJSHk96j?5PHV5w^c9RAi8<3|KJ(oM^*))*A6-X zu^xT>Qj%(X2H&6WIj=ps>5f74D%JYqo0$S0=onan-#k{Ba?KBm*5Mb_Bcl0e} zG|!Qa%od!>uYI&>rPsNZaS^Z)MV0NY5~B@yTn;}jm|P$EoR2bkJ9dPi4cs#^BkPWJ zSxgUTnOCww8Zu4(6Zw1g?K71wIR5^xa7F4ljp2`RoWjgF6oj(GyIxQ)m1Hp9NA&2Ayg-p@EtUTz*dk%1E{D)KU%G7 z^_n&J{mi#+Jtw?qQ7f(KQNG%~UAsOkTe{Q(C+~isKFrfg7cbuH-cZ(?2OE(yUx;uM z9Dc(6`(Ixb7w5wVt=zG5=T`_CcAvcmzr43}c1~42_F6Y zcPV-x?*?!;f%-8Tb@PR~U?lZ-NB`8em6lR|dqmaI z1T{8<;gn5hrZ-mJ-JM8!y1e0cBdl6q##j$=x$a4KF5cyxB0KX^AgJEOKO}Fi1Jp-nJdXrpW=pE=(YeLx(j|r{FtVkosj0+Ng-g-%<=NaFNQuoYEma#z5gG{4 z;{*3-5WQ=ouvdmHFSRRi||V%896iw>?79J#TM=j%W#iE(CO^eUaqLvot1}9 z6e{Q4aXP?*(Xxrrpr2>Vs2NPb!;q1wvf9<;)64uFT7izv`1n9Uv5A%MGQ6jrcW35a z45;)l5DG?}@QS22n1*kOuFXSPu|?+ye5k^XfxGcB^U8XUKpt0iNpGqG;>R_OiwFn^ z2x;s2rF1XK`d7~Q!gukOU7*hu!%Sb(D<}&CY%$c(eB&4XheO`gE9Yg+ZPomY>cCqy~i@^mwv2Bnu zYD#x%vYb9gVg(^MkOa2b+TOiyp1q4ehIk=W6JoAvuAgg-8-0595n2M*!@MAP6+T$J zZ-KS?Zv|9B^Q}YgK@J9R2i-skiVUe`aJGtW>4lKQrln~Go9!qHjGkI*P*H`U_FCu& zS743UR^JadcPUy{WioZf?;?u{8WbPnwVgaySVdO|^*}CI)=3mQ%cXt3oMK2&cFE^| z-9_+_Nvso;;j47piX$JMU)Z&O{|ekGP;tW9AQ(n{y-%u{@a!v1-0Pjs-?u8Tl=65t zmE!T`hDVJ1t~8ZX^r8?$t(A~Cu4s>aGJJ@zcNY}La{Y*xgWdd=Co`lqWDCMb-B46i z6ihw}AX?xlXP&glolDtH*v0cw@WYjPOFPk<~XNCk3alFmXN(o(Vld)yWaf-|RD|U!R$cJ607{5~(WA z2RUIE$X=_eYEnXYP-ZvSOZ}3&W&7<5(~?f%>5A~DP5=ELbt)d+1I1Lg9T-|w6v{Sn zK8)hs^!1Qfv;-t|H}r#Z66F_+bvRW;B_`**5@8__MD!vDiSzs3Ys4oC;|+Y*)IS(C zD{7y#X;+Y|N^%CNfgAdt&o>m|BPj2t%gD<5YMn#m6`eG}165Qw%jM-u1ri;{dXYu# zN?|@x8FQtz>IGvpGps92(~>s%Tb^fY5hExHzgZzt_WJc6dX8I%9U}!CR|)Vi6%I`> zgp~1*Ys39jgttMMcB=$URaF$K4KY3#2D?&Jbk+<*lvgD~#a82o8Sc1jCD{n0P|*aJ z-Q>BYn*kV1nX*x5vZIXpAHeXZKIdJgeA#4a8I#lT<^3~`2RNmOIM;`DY9~UjTw!!i zpZ&F<2`w$00mowETASzA>eB@Q!;8Rb=zxbm!tN8xO->yh{Nmh;Lw# z&&zaD<-o7M+xolcy5}Nu(S*`ZpMHINGq^Xf4zr0oQZh8$p2F+~-E)tEN(B@NjeD#0 zbA5INd~B&VIaeq8QvB~yP}<^0vRLYFpN%qyF8!#*^2_6uvw-|~Myfn~ap&H>?G1Mt z7Hi4KJQa-9Iz~8(^FFNRC5ZA8gexU!Yl^@Z-fn%((ryxAfgX-XLqSOO;d7XF1}H*+ zNQl|gKDenk-cziA`?D<^)`+qIwD)>WhLD_$j781~MGpVN2Yj*0OZ|-j0RXDS_;q!4 zb_RBK_Y}vGCUHMUz;`JrpLdqWYxyGnUvKz-dpU2aX1z;)>*(PGwaO(OK1aBTvGE~_ znvz*zYIRj&Lzd<9aZ`Jx8Szc1e;pVDpDJ`qhPo!kov5Q_{^MzQgU=?dV%xfhw-p`D zrXeRM!MT+a%2%(d?jA=V73Cvd?xBOKV{|LqS0_*qr~XmzvgrabzNsKNU|Zt8Bt|!i zYGnahRv3pDPr6Cf_S0p29^&Kw&qD?OrPYS$I3@ly%Xm3>Xt><$7b@~kRob@97ZKTS zUv`u#J~ys>>wz1NEd(H$v@ClOt)7W=HRNVOp_zwhY)nsAlmqg=i;kS)$grmx>*%0U zo=^1wQ~QsvxkEl%r(_5K60;kbS=kykFh1#QDYX+XMXbQML{vQT^@fId&KB)CXAoTk z-jL55hNx0y)cp6~b6^pp`vQ}^P2yZD>T-tCCH(ul4CRh**>X0g#O;hnl%;7Q5IgwE z%C^HQlj+OOUBAL$w&NsEk58Pu0-UvMecI{*#U1Jp628V@LEhxHsM9jlTGvQupm> zi3omrceReQtCn-uCcS(GUWAsm3A)=d3Id3hU{L8NPQMqdY+paE%D~uI8CM?q2BN{U zFbh#H(nWljUL-RSJ^%(g1ETLciu=b#pLzfOyju$kqI^8v-E=4)hrdAz zq&ZX^Z}je(ESmo0Xrm|w$y847t1`j$dC5X*ze^#mgks~QK(CYa;eQ`nInGWrXe6jP6k}MdlvYr63B2_%6FbuS$#M4pRbED6 z)7fY($cZBV5N9wcv+-W5jm1+^V@IU9)5?F5{kF=P*y|&ucys`~z)}SLKw+F#i`V?1eL3yF_dYc;)nx@{GunqC< zAZSP9){F2n8{8rzdNT{gN9>@?8A(t;7WIzB1 zT6O&zl)wZE5-ymF_EvmVJfhbZRqU3diYoyFp~#6`ZyVd1H`dSNWT`ydqSK6Hgiu-I zx$cKvv{+xfQYEXj0VVdXhW^UkhN67a!q9+iXm}No2Hln8(Rp(QAzM{NDOFM0%ly)l zOW(eIyIWHt>M^RgDw?Tkbkb(?_N$oP#CP16a`7vcPwvkCM=WM9ms%^8j830BG zCA1mbuu*1%2cOCDK!*cQX8AjI(^}< z&|b_+h%(0LhQ*dZNswgszLw?+E_U>8rM5fN1oH6inc5AHh&LAq7>l)X-nftOIM~}= z^yGqZFVGH>eMqe$d@%0u5ota{F4THAE&Hw@Z2=wn2k3|f$leHK?z+d0-GPtgUzc+= zqw_s5%e$0_8;kQ7RM^Gt4&V9o$KtesXVfKaS+mh#RLPF*<#Fc~{rF5(+_MeB;;2j@ zW&v1Q=#8En$1t$*-rm*9a$v1RGMVd&ii?#vd_Es8MKC#L`a@!dIp1}cT z{Dp*pXOsN%^y&8~rka;Yn6+_Wsd7^lOMgKLA*b`r`T8F${`mT;k;taetqs+gAEg>tY0|~56h1oBK5^{G*QE4g4 z#uVM$a(2JkmahhPzhZrdqx%)~yTf?mxAcgknG$U>%DO#GFga50LB<5-qio!`o z*0qak7jXLYPt;Keko)Kn%J#O(ITEA|iUBC5iDfoBVQ7sqF+d077m!ivI5LRD_MB<+1*{woRL5 zhBt%}db&_Lp06JDosvBu#L>Mz$2C!LyxnRdr+Us-ovz1>3m1&(XyNg+Ww+@RP;#p~ zR>!*6)#Z4kizo5a^fLmt^87}#H)XjwjBA7?C7O7r(9-8nc&WBswot|*dig=x!)^!v zw8MB~8D)HQ&Df{oX|8YV?US@lNbqtRjVA`_mSEcsG~98G=anW{2vcPNnlUW_aYowMN>5beWeA9xi8Qe_ytY5C zrPfiRv+U&pW>SW^IP(xM$gR^xRbIPX@!eOu#=PvD5>~I?$Pf>_k#FtRG?2yiCF~Lt zHuX|aP)f$wyRW^1-uU$NJq`T>1rrVUu|we+o(~ltxp!GA$3x8F4B}0vwC;>pUnDZY zYZ|{pjO;fe1%n^JxzRNA(fC$HVG0Z4Lzg-2XGTYwJ*Lds*u#O}eaW3|B_v)AbnxO^ z|2}&*+}-X|a=8ogcBMp8v0i(^!t&ftquB>sK(3*Qk+X9aH#;BP?=_lzH^0L-u_SUQ zFGn^*kd`mCR=0hDZb!M=v@M^bYsEUOKN;#KX*YysG*l(F)HX&!jvCC9j#7um#_WdN zd5=O0HpNHi>HZ^S?Dn5*2-;H$l3VMY3ne*&aN-bVc`C41N{Ha5IF7M41TOcCBySJI z-Zqtz@ZnLlAFSxewO!Bd7K5UqrQHOX)u840l*&_N4r#It#SjnfVN_+mO=!+BaluZjjg;jyG=KwC zNpd|hP!=H}C9LPY48>%hDB*9wME1Tr!Ei|ZUZeBjYOdGRtj+BYFOfUv;^M+^2DBo+ zeuI*~Nm7TwyT^|=Qu8oB3sh`zMpnQ!5TXDRz3j_kG`}@G73gcJK6s*tL8bReoFw+3 zqLid$(mx?&vg0BShKSW8pKN4M3$rkn$=2zYS&I@Vq(#lqr{BqSSN^f8=`@;X11=iB zE_$HbW>?zt3(U2;ydaEFu#?z@**0a?xpjGW&}Ou;;S>yw`XoqZXx`f_&fIn2z};4m zb}vN~PGq3~*(d3j-s$nL$Reiyp7o?poZ=9sQ>+j@7 zx*pU{X7^-u%~eTK-={+c!#N(v>$ni5T&)?CAUG-WFrE^~p9~tr4OO(H85eOu*wa^fZvJiO50;6n_I5aKQZ{(PnnTE}3Gg2Dhrw>>J| zb)`KIx-ouo!y617qD-*8PQaaa^+JSVbnoDzJSE+z7Di{31u%cteH-JL6g3FhRT(we z3t=wLgdXEJY)jREEniYz9g#H4r2Ru?hprh*g)7j9J6%@hy6Srs?SEp4Q=-<>7@P4r z5j3MNB2KDo-6WoSkSZ=7r^#BYEB+*w86qT9SAfIhC@wyK-n_#MtT;%dB(O#)WoT@y z8Sb!fOAdgRRUGfgY(jES9PM;3*iC4RqYNwO7Ypr5W4BylbQ3q=Tdf13@qX(9VM0wi z@7T1dbr&oilm-0W+OdRFyPpstS`&oH&u3lp&?*CBlbs{EZZH zZ2!61@r0C^7%I#%VZ!=NPGVqY4iEmUP43;X$w8e7WG*`cUykXtQpWtG|CmVo*D7Jnr^`14I8mAP=SMn%->eZP+t`8@&_>BGf)5e;-8fj^5s1u+9P2@UAhz(c(}7 z4y`d`-|wFzwn0}iR|?@_DJB4L2^BDfcr7n451_@n`{^z%+eOJxD7GXJt^?*CvDEP_ z21K<(5roLkc^#w46PQ2R?YsHCwIb$u$h_Yg%e?)LS?{NNqTat{$kN)iXAibsOy3~T zuHiS0_p5$-d;5|`U*+z%D6YwDbQ{^4@rsNWo1|E}fC>y(p+yjB0yEO1TZq#QtHr=V zTFM~qMvm6W;rqzvF0d3ay@3oyW|E2Z9XLg%*5nRVpHRQG#*YQh*r5N&;Kb1ADdV zrTTE04a~e$bA0G4QBi~(acO=jRA4#n`11dI@x{x6nuCDeNRpa9S85IRSFA(rhOzb4 zt3xnWX^z%k_U6r-GA%TPxaYp z*z2K(5i>YSTj3U+QX2ShYBzcw1;`=F7azrmtsiW9dl1o(zzw!&#r{5jeg|r)7LWtl zomP(mlXGE$6k`L%QsBc8>uqJo@o{dFgQ~a=2tL^iLZ(59$?DzznbNrdabOuZlm3h~tm!UAgbGC^1{skIo4 zK?vrIcj@SCWr!?+?z#>`9sC|(5^K@DJUY=<39~g+-eMJ*TLqc;5=htz@Bks0(tX+G zH=xe%G9nM2xVI_!{SnX*)bZEm?I}VVB#HU(K&3DZK_72;2Vq+kYN+A-vJah089 zjCkl{Tu?Kay~nWl4%mmD^;D>{N67~z1NQi!kExl7RGl7inpr9$oFD%3&XK<18X=i4Bv4r zEX*4Lv=JObUUTyi1vND;4+*OVUxrK=7q%E+AQfZB2|@rudYX%>{4M`;hleImkyo#_ zD`;vqv}3-@1fKU;7TMN;N$B>fL=)G~!~ztSlImOt#HtW7<+}?312BOz)T*(yB?ULg z65Dr}RI*#324}BrOu{z_nodTF)BNLGB{X%Ryi^wA?t3AM^=Wi9*&9ugBGn$RVJW}I z4}<<$9bkR*sHe;a&56<87=dS_dRHF*LO>?QzFhsu1U~R)LD!C-+Uk*Q^H8T^H_3$n zWRK5%ZB@mQ*kE7+z5Mmvd-r+>=7PcR*ML<Z$!JVd#(xYf7iyceCKj%HnL9qiAzufchL}=`6Bj;VbQ}lpidcb>Njet139ZAOFBu6S=a`Ju$T_5G(u>i$P3%^&f{fdp0_!Qp-M}yxj(A3hwhm zPQ*vVb6}5q-G6rb^mYplbTdcX=_`@y9|ZXxo@gZ%1lH$l`CVzEge+0j(R!@gv^{_|YU}in8R?)N+O=M!tTM^&S2ijIcp~(NP$z*KlEXV_j7X%uV57 z|CDE>8UpXs3`{btC`7IpqfAF|5ag200n%{}La) z!c#IbdN-g243g~9pve0<&*)8Nu6Ic+sO66@N+n-I8;?;=*`G>n&fV> zx+TU38GzJ0A`>HJ3Y2#a8EBNTfx?^UX3L@H`+mmPS&A+nyNa0hA8HGJzYqY4a&ke5 zKUgC?R>Y-3&E@OX?K(F{auv9s8<>?TLjJcGyqFpv{OPYVFDJ%(XZ9a4^MS){BZ63gw8;tLdYvdkl*uawIxz`K8_dkUbn_53VpP#P16n~MI%<&d?f2ZPK2GjHS?9 zJpi)~kY}Z^YSk-(sYa;K+Q`9dM6SIjorz1Bf*Q~gt$SEh)MpF-vA#wuI|yojTvgqG z*N{)$!Ndddq;eSDNEUg}-<=R^8?mnqO@CYv)W(6>xVU^c^v{74MU!1G`q%%wIqZy~ zx8>M}(s7(%1cEQ>KQu7=4b~@HLY(h(w%qFw>|}~F7>qrcfW2OcV6R3&gpzl^19cSA zsaWT9oH*M5>Iqw%*^MPY-v*2(g(ZAyOcN577IBRnGPlQAhJ$XTwX)hLKU6sMZoa~`BhrmJMs@6*b(u$Q3*jvM^YzyB8Db2o?z`5xh1T)4V2YSNT$6O&yYSaajl z_`nlTj#Le-XDT^>lK><|Rg8CiA+6)ity}O!2JcKlCX+5Q+i{D|>C* zO$>I$y>G+t)n`}BV2dGH!!a{sjvftohJM6)MGp4j+;6Ri6UiCGa>dFhrLzXlxZMbz zi9Gr2=6O_~Z(bbD%-9%ntL+t`alUhE)gm*fX)QMMrxIi*cmnp2ehJx&$Z*c_DkI39^jRgOCukDU}^>)lH2EFQ4Srgr$SvBD#grfn+cNOnEM)+qt3_oa6%GQXy-}ar5H4?2~ zAZO!qQ&&Q>0Wq{8Y;e%SRlHOl*+ZE#In8P#I5s&C%A}++52P|Fn4JJnDtwr%){#g`qYw-=)m7ToL9$>&F18qo@n_7Dz|bt+7j_pC zOCz`XWQWpUajqY@rFC#Ru@~k}MxU+sTGAW8tvAPfyKBq9#9QQ)`Ob)%>uGyG8^%8%{*Gq-FRCS||L z%#XbK?3CjnY=Y)-U}&fJlD!MYi*UGoeUJ+642WOjOp9U!Y(w3s=r}S2v>Xhm;}?NX zzhI;lW#KM5cBnc0V|Lr5y|^TtNI`}MuF_=eKU~3Lbfe6_OF^}_o}+%^H!90vSuAWxS?-3gOY%9}N-)eAnbWcJ z!*_Vg*hberI}1ls)w=#>-W-K z&Fn1l*@0Utqj9?vd1|h0M;K06>k0p6BLRZa=mG;E8JeAQu`rqu4FJiMzYpOgpkc>SYexclfh70VWS8fg&LsAkHm*)Z++I7I%uI^@@2C2V?-{NCtbOQnjJl+<=y|m%O8X;VaJE235aSc75|z~^>|XlN zr2lU0rsQU_Gb&^f!g4$rP1q*;xEVTMtNnSuid079R8_V0G9@T)1sLvLJcM)GTitKY z6>KEM0(N+UJjW1H3FdGaisNwd_mzCZtr2+dQZ|fX1<9Cb$y+i?OqM5zYwwDbHFFp} zV9yQsGZLLUzzC+FH%4wMqab35@bkU;khc4@&|50ULsR1nY7W%LwSJ#-=u1viMUx?r zNE*>=ubkWv3s{;!OC8JqzsAn|r^!2x15#%Unzz0M{omrU*R7NRBDJYGUaFii(2`zhnpZen_ z{sSaDZ9m8R^M1dcUL5UUMXmRJtqm!m#_LvHg$AbXM11ivtFfY3WDvRP{Esq;1|F0u zt@d@8Ii5PY?A@T$_VZE6B1Jg%g$I z7FbjdwHL(~gxCt%shp>Q>kkl;z%iV$8ZA?#~pp0HDM@^!H9B6E*12_^CaB-l}ln)};C+Tw<@j@SStlJUHa-Y2_&SU~0^8??iGF6z(<%@yxG-6(N#-WkO6#Rm90VOkAJ zgIMHZ;#uEbR17kXkj5Z7xhrD>-}vQqxfxFC?`o#DvDPZ+je-%CxDX1aEbI$fI~G@u6qZsJfPp7w;I)BHjxi2!|+t{HC58ZckAUUQGieytOa z#L8}je0EFb80|cFVoX~3^7I^?lCamL%sH8KAI8jKhJfd%Aw29(qO)t@yyX=U7ymwE z4$%No2H?EF<+O)nJJI|5VNu>z1K|6#Iv)wTD=#;2VY?JN&Y+H)radVZV zC@%Jh=dYCKS)Ms_SZb37HS6C(yOyo?*)KX*T&fN6vV7gwic}=7{Dak;XG9S8(LjF5 z@00?Tq8Wu)sUzi3IDr*@hdofk8)-cf;D%k>s8g1ggW1GU5SBlhxy%BbH>*tcd2H2X za)l>Z-ENM2pk~eoUlnlkC_3sDm0B3uvu+O-H$^@4xcBGr3b9DVQ^0ygf*(2JS!cV9 z_h`#EUb&omKCs$R<*%TS5Nl=wSXrg!lD;ZBNBYA_Z%iR^>deeXW?|;`wv`YMd^LA0 zyPSR_s1f^43=F;R67so3*MBN9U(<8%p@#dJ@yT*D@}Yu#FuESy&+reCHvg05hJ>5K zP6@>X9~+8sS$4wPN#+{6r`-MBfeR2>^0?5iK=fg5k>d1+qnF5?dB*?dFYG%qJY{KX zYs2m4H^cH<5P#wFk>$jJu6lS~%ZM4N^88MS(k>>%ZZVSj`V%Ab5hP}#i3^>f_t>4< z2ZNy)D=uETgf|wb$k;TV(T`tNAekP| z7DOO1cNGgrNDie+gqtiC}lZf=e?t9j_n=`C@F(#9C!Nykzx z;k%CwNf&`Gku8q0S^MR@C&3*>&<1rwOZ}6k@Q=NJr-km`VQ+?H2lr=7auVwGZaWaR z4XlO?VvNsnEh9c@H@1-7keRysD{t@Nu_jU95Q7A<_Z{oN!N{52m-`|cp*CuOdDGfJ z0QpG-Bo}2%Nr0TfGo??E49l{qOuXcQ4dosQ@2M^k=N5sQ#hS#$#4|Vvc0w1sOVXKe zXzG9IN`$h4aLq_X@&o5G&RBg`!EKDH6EcE3fb#gj`o@sT;X e|4;-+Cmo`K@AK6iQG@tP4&QGGU0<{=>h!

Note

You can set the value of ``requires_grad`` when creating a\n tensor, or later by using ``x.requires_grad_(True)`` method.

\n\n" + "

Note

You can set the value of ``requires_grad`` when creating a\n", + " tensor, or later by using ``x.requires_grad_(True)`` method.

\n", + "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "A function that we apply to tensors to construct computational graph is\nin fact an object of class ``Function``. This object knows how to\ncompute the function in the *forward* direction, and also how to compute\nits derivative during the *backward propagation* step. A reference to\nthe backward propagation function is stored in ``grad_fn`` property of a\ntensor. You can find more information of ``Function`` [in the\ndocumentation](https://pytorch.org/docs/stable/autograd.html#function)_.\n\n\n" + "A function that we apply to tensors to construct computational graph is\n", + "in fact an object of class ``Function``. This object knows how to\n", + "compute the function in the *forward* direction, and also how to compute\n", + "its derivative during the *backward propagation* step. A reference to\n", + "the backward propagation function is stored in ``grad_fn`` property of a\n", + "tensor. You can find more information of ``Function`` [in the\n", + "documentation](https://pytorch.org/docs/stable/autograd.html#function)_.\n", + "\n", + "\n" ] }, { @@ -58,14 +101,25 @@ }, "outputs": [], "source": [ - "print(f\"Gradient function for z = {z.grad_fn}\")\nprint(f\"Gradient function for loss = {loss.grad_fn}\")" + "print(f\"Gradient function for z = {z.grad_fn}\")\n", + "print(f\"Gradient function for loss = {loss.grad_fn}\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Computing Gradients\n\nTo optimize weights of parameters in the neural network, we need to\ncompute the derivatives of our loss function with respect to parameters,\nnamely, we need $\\frac{\\partial loss}{\\partial w}$ and\n$\\frac{\\partial loss}{\\partial b}$ under some fixed values of\n``x`` and ``y``. To compute those derivatives, we call\n``loss.backward()``, and then retrieve the values from ``w.grad`` and\n``b.grad``:\n\n\n" + "## Computing Gradients\n", + "\n", + "To optimize weights of parameters in the neural network, we need to\n", + "compute the derivatives of our loss function with respect to parameters,\n", + "namely, we need $\\frac{\\partial loss}{\\partial w}$ and\n", + "$\\frac{\\partial loss}{\\partial b}$ under some fixed values of\n", + "``x`` and ``y``. To compute those derivatives, we call\n", + "``loss.backward()``, and then retrieve the values from ``w.grad`` and\n", + "``b.grad``:\n", + "\n", + "\n" ] }, { @@ -76,21 +130,42 @@ }, "outputs": [], "source": [ - "loss.backward()\nprint(w.grad)\nprint(b.grad)" + "loss.backward()\n", + "print(w.grad)\n", + "print(b.grad)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "

Note

- We can only obtain the ``grad`` properties for the leaf\n nodes of the computational graph, which have ``requires_grad`` property\n set to ``True``. For all other nodes in our graph, gradients will not be\n available.\n - We can only perform gradient calculations using\n ``backward`` once on a given graph, for performance reasons. If we need\n to do several ``backward`` calls on the same graph, we need to pass\n ``retain_graph=True`` to the ``backward`` call.

\n\n\n" + "

Note

- We can only obtain the ``grad`` properties for the leaf\n", + " nodes of the computational graph, which have ``requires_grad`` property\n", + " set to ``True``. For all other nodes in our graph, gradients will not be\n", + " available.\n", + " - We can only perform gradient calculations using\n", + " ``backward`` once on a given graph, for performance reasons. If we need\n", + " to do several ``backward`` calls on the same graph, we need to pass\n", + " ``retain_graph=True`` to the ``backward`` call.

\n", + "\n", + "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Disabling Gradient Tracking\n\nBy default, all tensors with ``requires_grad=True`` are tracking their\ncomputational history and support gradient computation. However, there\nare some cases when we do not need to do that, for example, when we have\ntrained the model and just want to apply it to some input data, i.e. we\nonly want to do *forward* computations through the network. We can stop\ntracking computations by surrounding our computation code with\n``torch.no_grad()`` block:\n\n\n" + "## Disabling Gradient Tracking\n", + "\n", + "By default, all tensors with ``requires_grad=True`` are tracking their\n", + "computational history and support gradient computation. However, there\n", + "are some cases when we do not need to do that, for example, when we have\n", + "trained the model and just want to apply it to some input data, i.e. we\n", + "only want to do *forward* computations through the network. We can stop\n", + "tracking computations by surrounding our computation code with\n", + "``torch.no_grad()`` block:\n", + "\n", + "\n" ] }, { @@ -101,14 +176,22 @@ }, "outputs": [], "source": [ - "z = torch.matmul(x, w)+b\nprint(z.requires_grad)\n\nwith torch.no_grad():\n z = torch.matmul(x, w)+b\nprint(z.requires_grad)" + "z = torch.matmul(x, w)+b\n", + "print(z.requires_grad)\n", + "\n", + "with torch.no_grad():\n", + " z = torch.matmul(x, w)+b\n", + "print(z.requires_grad)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Another way to achieve the same result is to use the ``detach()`` method\non the tensor:\n\n\n" + "Another way to achieve the same result is to use the ``detach()`` method\n", + "on the tensor:\n", + "\n", + "\n" ] }, { @@ -119,28 +202,88 @@ }, "outputs": [], "source": [ - "z = torch.matmul(x, w)+b\nz_det = z.detach()\nprint(z_det.requires_grad)" + "z = torch.matmul(x, w)+b\n", + "z_det = z.detach()\n", + "print(z_det.requires_grad)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "There are reasons you might want to disable gradient tracking:\n - To mark some parameters in your neural network as **frozen parameters**.\n - To **speed up computations** when you are only doing forward pass, because computations on tensors that do\n not track gradients would be more efficient.\n\n" + "There are reasons you might want to disable gradient tracking:\n", + " - To mark some parameters in your neural network as **frozen parameters**.\n", + " - To **speed up computations** when you are only doing forward pass, because computations on tensors that do\n", + " not track gradients would be more efficient.\n", + "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## More on Computational Graphs\nConceptually, autograd keeps a record of data (tensors) and all executed\noperations (along with the resulting new tensors) in a directed acyclic\ngraph (DAG) consisting of\n[Function](https://pytorch.org/docs/stable/autograd.html#torch.autograd.Function)_\nobjects. In this DAG, leaves are the input tensors, roots are the output\ntensors. By tracing this graph from roots to leaves, you can\nautomatically compute the gradients using the chain rule.\n\nIn a forward pass, autograd does two things simultaneously:\n\n- run the requested operation to compute a resulting tensor\n- maintain the operation\u2019s *gradient function* in the DAG.\n\nThe backward pass kicks off when ``.backward()`` is called on the DAG\nroot. ``autograd`` then:\n\n- computes the gradients from each ``.grad_fn``,\n- accumulates them in the respective tensor\u2019s ``.grad`` attribute\n- using the chain rule, propagates all the way to the leaf tensors.\n\n

Note

**DAGs are dynamic in PyTorch**\n An important thing to note is that the graph is recreated from scratch; after each\n ``.backward()`` call, autograd starts populating a new graph. This is\n exactly what allows you to use control flow statements in your model;\n you can change the shape, size and operations at every iteration if\n needed.

\n\n" + "## More on Computational Graphs\n", + "Conceptually, autograd keeps a record of data (tensors) and all executed\n", + "operations (along with the resulting new tensors) in a directed acyclic\n", + "graph (DAG) consisting of\n", + "[Function](https://pytorch.org/docs/stable/autograd.html#torch.autograd.Function)_\n", + "objects. In this DAG, leaves are the input tensors, roots are the output\n", + "tensors. By tracing this graph from roots to leaves, you can\n", + "automatically compute the gradients using the chain rule.\n", + "\n", + "In a forward pass, autograd does two things simultaneously:\n", + "\n", + "- run the requested operation to compute a resulting tensor\n", + "- maintain the operation’s *gradient function* in the DAG.\n", + "\n", + "The backward pass kicks off when ``.backward()`` is called on the DAG\n", + "root. ``autograd`` then:\n", + "\n", + "- computes the gradients from each ``.grad_fn``,\n", + "- accumulates them in the respective tensor’s ``.grad`` attribute\n", + "- using the chain rule, propagates all the way to the leaf tensors.\n", + "\n", + "

Note

**DAGs are dynamic in PyTorch**\n", + " An important thing to note is that the graph is recreated from scratch; after each\n", + " ``.backward()`` call, autograd starts populating a new graph. This is\n", + " exactly what allows you to use control flow statements in your model;\n", + " you can change the shape, size and operations at every iteration if\n", + " needed.

\n", + "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Optional Reading: Tensor Gradients and Jacobian Products\n\nIn many cases, we have a scalar loss function, and we need to compute\nthe gradient with respect to some parameters. However, there are cases\nwhen the output function is an arbitrary tensor. In this case, PyTorch\nallows you to compute so-called **Jacobian product**, and not the actual\ngradient.\n\nFor a vector function $\\vec{y}=f(\\vec{x})$, where\n$\\vec{x}=\\langle x_1,\\dots,x_n\\rangle$ and\n$\\vec{y}=\\langle y_1,\\dots,y_m\\rangle$, a gradient of\n$\\vec{y}$ with respect to $\\vec{x}$ is given by **Jacobian\nmatrix**:\n\n\\begin{align}J=\\left(\\begin{array}{ccc}\n \\frac{\\partial y_{1}}{\\partial x_{1}} & \\cdots & \\frac{\\partial y_{1}}{\\partial x_{n}}\\\\\n \\vdots & \\ddots & \\vdots\\\\\n \\frac{\\partial y_{m}}{\\partial x_{1}} & \\cdots & \\frac{\\partial y_{m}}{\\partial x_{n}}\n \\end{array}\\right)\\end{align}\n\nInstead of computing the Jacobian matrix itself, PyTorch allows you to\ncompute **Jacobian Product** $v^T\\cdot J$ for a given input vector\n$v=(v_1 \\dots v_m)$. This is achieved by calling ``backward`` with\n$v$ as an argument. The size of $v$ should be the same as\nthe size of the original tensor, with respect to which we want to\ncompute the product:\n\n\n" + "## Optional Reading: Tensor Gradients and Jacobian Products\n", + "\n", + "In many cases, we have a scalar loss function, and we need to compute\n", + "the gradient with respect to some parameters. However, there are cases\n", + "when the output function is an arbitrary tensor. In this case, PyTorch\n", + "allows you to compute so-called **Jacobian product**, and not the actual\n", + "gradient.\n", + "\n", + "For a vector function $\\vec{y}=f(\\vec{x})$, where\n", + "$\\vec{x}=\\langle x_1,\\dots,x_n\\rangle$ and\n", + "$\\vec{y}=\\langle y_1,\\dots,y_m\\rangle$, a gradient of\n", + "$\\vec{y}$ with respect to $\\vec{x}$ is given by **Jacobian\n", + "matrix**:\n", + "\n", + "\\begin{align}J=\\left(\\begin{array}{ccc}\n", + " \\frac{\\partial y_{1}}{\\partial x_{1}} & \\cdots & \\frac{\\partial y_{1}}{\\partial x_{n}}\\\\\n", + " \\vdots & \\ddots & \\vdots\\\\\n", + " \\frac{\\partial y_{m}}{\\partial x_{1}} & \\cdots & \\frac{\\partial y_{m}}{\\partial x_{n}}\n", + " \\end{array}\\right)\\end{align}\n", + "\n", + "Instead of computing the Jacobian matrix itself, PyTorch allows you to\n", + "compute **Jacobian Product** $v^T\\cdot J$ for a given input vector\n", + "$v=(v_1 \\dots v_m)$. This is achieved by calling ``backward`` with\n", + "$v$ as an argument. The size of $v$ should be the same as\n", + "the size of the original tensor, with respect to which we want to\n", + "compute the product:\n", + "\n", + "\n" ] }, { @@ -151,35 +294,61 @@ }, "outputs": [], "source": [ - "inp = torch.eye(4, 5, requires_grad=True)\nout = (inp+1).pow(2).t()\nout.backward(torch.ones_like(out), retain_graph=True)\nprint(f\"First call\\n{inp.grad}\")\nout.backward(torch.ones_like(out), retain_graph=True)\nprint(f\"\\nSecond call\\n{inp.grad}\")\ninp.grad.zero_()\nout.backward(torch.ones_like(out), retain_graph=True)\nprint(f\"\\nCall after zeroing gradients\\n{inp.grad}\")" + "inp = torch.eye(4, 5, requires_grad=True)\n", + "out = (inp+1).pow(2).t()\n", + "out.backward(torch.ones_like(out), retain_graph=True)\n", + "print(f\"First call\\n{inp.grad}\")\n", + "out.backward(torch.ones_like(out), retain_graph=True)\n", + "print(f\"\\nSecond call\\n{inp.grad}\")\n", + "inp.grad.zero_()\n", + "out.backward(torch.ones_like(out), retain_graph=True)\n", + "print(f\"\\nCall after zeroing gradients\\n{inp.grad}\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Notice that when we call ``backward`` for the second time with the same\nargument, the value of the gradient is different. This happens because\nwhen doing ``backward`` propagation, PyTorch **accumulates the\ngradients**, i.e. the value of computed gradients is added to the\n``grad`` property of all leaf nodes of computational graph. If you want\nto compute the proper gradients, you need to zero out the ``grad``\nproperty before. In real-life training an *optimizer* helps us to do\nthis.\n\n" + "Notice that when we call ``backward`` for the second time with the same\n", + "argument, the value of the gradient is different. This happens because\n", + "when doing ``backward`` propagation, PyTorch **accumulates the\n", + "gradients**, i.e. the value of computed gradients is added to the\n", + "``grad`` property of all leaf nodes of computational graph. If you want\n", + "to compute the proper gradients, you need to zero out the ``grad``\n", + "property before. In real-life training an *optimizer* helps us to do\n", + "this.\n", + "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "

Note

Previously we were calling ``backward()`` function without\n parameters. This is essentially equivalent to calling\n ``backward(torch.tensor(1.0))``, which is a useful way to compute the\n gradients in case of a scalar-valued function, such as loss during\n neural network training.

\n\n\n" + "

Note

Previously we were calling ``backward()`` function without\n", + " parameters. This is essentially equivalent to calling\n", + " ``backward(torch.tensor(1.0))``, which is a useful way to compute the\n", + " gradients in case of a scalar-valued function, such as loss during\n", + " neural network training.

\n", + "\n", + "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "--------------\n\n\n" + "--------------\n", + "\n", + "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Further Reading\n- [Autograd Mechanics](https://pytorch.org/docs/stable/notes/autograd.html)\n\n" + "### Further Reading\n", + "- [Autograd Mechanics](https://pytorch.org/docs/stable/notes/autograd.html)\n", + "\n" ] } ], @@ -204,4 +373,4 @@ }, "nbformat": 4, "nbformat_minor": 0 -} \ No newline at end of file +} diff --git a/tutorials/08-Optimization.ipynb b/tutorials/08-Optimization.ipynb index b68272f..4a76e9e 100644 --- a/tutorials/08-Optimization.ipynb +++ b/tutorials/08-Optimization.ipynb @@ -1,21 +1,31 @@ { "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "%matplotlib inline" - ] - }, { "cell_type": "markdown", "metadata": {}, "source": [ - "\n[Learn the Basics](intro.html) ||\n[Quickstart](quickstart_tutorial.html) ||\n[Tensors](tensorqs_tutorial.html) ||\n[Datasets & DataLoaders](data_tutorial.html) ||\n[Transforms](transforms_tutorial.html) ||\n[Build Model](buildmodel_tutorial.html) ||\n[Autograd](autogradqs_tutorial.html) ||\n**Optimization** ||\n[Save & Load Model](saveloadrun_tutorial.html)\n\n# Optimizing Model Parameters\n\nNow that we have a model and data it's time to train, validate and test our model by optimizing its parameters on\nour data. Training a model is an iterative process; in each iteration the model makes a guess about the output, calculates\nthe error in its guess (*loss*), collects the derivatives of the error with respect to its parameters (as we saw in\nthe [previous section](autograd_tutorial.html)), and **optimizes** these parameters using gradient descent. For a more\ndetailed walkthrough of this process, check out this video on [backpropagation from 3Blue1Brown](https://www.youtube.com/watch?v=tIeHLnjs5U8)_.\n\n## Prerequisite Code\nWe load the code from the previous sections on [Datasets & DataLoaders](data_tutorial.html)\nand [Build Model](buildmodel_tutorial.html).\n" + "\n", + "[Learn the Basics](intro.html) ||\n", + "[Quickstart](quickstart_tutorial.html) ||\n", + "[Tensors](tensorqs_tutorial.html) ||\n", + "[Datasets & DataLoaders](data_tutorial.html) ||\n", + "[Transforms](transforms_tutorial.html) ||\n", + "[Build Model](buildmodel_tutorial.html) ||\n", + "[Autograd](autogradqs_tutorial.html) ||\n", + "**Optimization** ||\n", + "[Save & Load Model](saveloadrun_tutorial.html)\n", + "\n", + "# Optimizing Model Parameters\n", + "\n", + "Now that we have a model and data it's time to train, validate and test our model by optimizing its parameters on\n", + "our data. Training a model is an iterative process; in each iteration the model makes a guess about the output, calculates\n", + "the error in its guess (*loss*), collects the derivatives of the error with respect to its parameters (as we saw in\n", + "the [previous section](autograd_tutorial.html)), and **optimizes** these parameters using gradient descent. For a more\n", + "detailed walkthrough of this process, check out this video on [backpropagation from 3Blue1Brown](https://www.youtube.com/watch?v=tIeHLnjs5U8)_.\n", + "\n", + "## Prerequisite Code\n", + "We load the code from the previous sections on [Datasets & DataLoaders](data_tutorial.html)\n", + "and [Build Model](buildmodel_tutorial.html).\n" ] }, { @@ -26,14 +36,67 @@ }, "outputs": [], "source": [ - "import torch\nfrom torch import nn\nfrom torch.utils.data import DataLoader\nfrom torchvision import datasets\nfrom torchvision.transforms import ToTensor\n\ntraining_data = datasets.FashionMNIST(\n root=\"data\",\n train=True,\n download=True,\n transform=ToTensor()\n)\n\ntest_data = datasets.FashionMNIST(\n root=\"data\",\n train=False,\n download=True,\n transform=ToTensor()\n)\n\ntrain_dataloader = DataLoader(training_data, batch_size=64)\ntest_dataloader = DataLoader(test_data, batch_size=64)\n\nclass NeuralNetwork(nn.Module):\n def __init__(self):\n super(NeuralNetwork, self).__init__()\n self.flatten = nn.Flatten()\n self.linear_relu_stack = nn.Sequential(\n nn.Linear(28*28, 512),\n nn.ReLU(),\n nn.Linear(512, 512),\n nn.ReLU(),\n nn.Linear(512, 10),\n )\n\n def forward(self, x):\n x = self.flatten(x)\n logits = self.linear_relu_stack(x)\n return logits\n\nmodel = NeuralNetwork()" + "%matplotlib inline\n", + "\n", + "import torch\n", + "from torch import nn\n", + "from torch.utils.data import DataLoader\n", + "from torchvision import datasets\n", + "from torchvision.transforms import ToTensor\n", + "\n", + "training_data = datasets.FashionMNIST(\n", + " root=\"data\",\n", + " train=True,\n", + " download=True,\n", + " transform=ToTensor()\n", + ")\n", + "\n", + "test_data = datasets.FashionMNIST(\n", + " root=\"data\",\n", + " train=False,\n", + " download=True,\n", + " transform=ToTensor()\n", + ")\n", + "\n", + "train_dataloader = DataLoader(training_data, batch_size=64)\n", + "test_dataloader = DataLoader(test_data, batch_size=64)\n", + "\n", + "class NeuralNetwork(nn.Module):\n", + " def __init__(self):\n", + " super(NeuralNetwork, self).__init__()\n", + " self.flatten = nn.Flatten()\n", + " self.linear_relu_stack = nn.Sequential(\n", + " nn.Linear(28*28, 512),\n", + " nn.ReLU(),\n", + " nn.Linear(512, 512),\n", + " nn.ReLU(),\n", + " nn.Linear(512, 10),\n", + " )\n", + "\n", + " def forward(self, x):\n", + " x = self.flatten(x)\n", + " logits = self.linear_relu_stack(x)\n", + " return logits\n", + "\n", + "model = NeuralNetwork()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Hyperparameters\n\nHyperparameters are adjustable parameters that let you control the model optimization process.\nDifferent hyperparameter values can impact model training and convergence rates\n([read more](https://pytorch.org/tutorials/beginner/hyperparameter_tuning_tutorial.html)_ about hyperparameter tuning)\n\nWe define the following hyperparameters for training:\n - **Number of Epochs** - the number times to iterate over the dataset\n - **Batch Size** - the number of data samples propagated through the network before the parameters are updated\n - **Learning Rate** - how much to update models parameters at each batch/epoch. Smaller values yield slow learning speed, while large values may result in unpredictable behavior during training.\n\n\n" + "## Hyperparameters\n", + "\n", + "Hyperparameters are adjustable parameters that let you control the model optimization process.\n", + "Different hyperparameter values can impact model training and convergence rates\n", + "([read more](https://pytorch.org/tutorials/beginner/hyperparameter_tuning_tutorial.html)_ about hyperparameter tuning)\n", + "\n", + "We define the following hyperparameters for training:\n", + " - **Number of Epochs** - the number times to iterate over the dataset\n", + " - **Batch Size** - the number of data samples propagated through the network before the parameters are updated\n", + " - **Learning Rate** - how much to update models parameters at each batch/epoch. Smaller values yield slow learning speed, while large values may result in unpredictable behavior during training.\n", + "\n", + "\n" ] }, { @@ -44,14 +107,40 @@ }, "outputs": [], "source": [ - "learning_rate = 1e-3\nbatch_size = 64\nepochs = 5" + "learning_rate = 1e-3\n", + "batch_size = 64\n", + "epochs = 5" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Optimization Loop\n\nOnce we set our hyperparameters, we can then train and optimize our model with an optimization loop. Each\niteration of the optimization loop is called an **epoch**.\n\nEach epoch consists of two main parts:\n - **The Train Loop** - iterate over the training dataset and try to converge to optimal parameters.\n - **The Validation/Test Loop** - iterate over the test dataset to check if model performance is improving.\n\nLet's briefly familiarize ourselves with some of the concepts used in the training loop. Jump ahead to\nsee the `full-impl-label` of the optimization loop.\n\n### Loss Function\n\nWhen presented with some training data, our untrained network is likely not to give the correct\nanswer. **Loss function** measures the degree of dissimilarity of obtained result to the target value,\nand it is the loss function that we want to minimize during training. To calculate the loss we make a\nprediction using the inputs of our given data sample and compare it against the true data label value.\n\nCommon loss functions include [nn.MSELoss](https://pytorch.org/docs/stable/generated/torch.nn.MSELoss.html#torch.nn.MSELoss) (Mean Square Error) for regression tasks, and\n[nn.NLLLoss](https://pytorch.org/docs/stable/generated/torch.nn.NLLLoss.html#torch.nn.NLLLoss) (Negative Log Likelihood) for classification.\n[nn.CrossEntropyLoss](https://pytorch.org/docs/stable/generated/torch.nn.CrossEntropyLoss.html#torch.nn.CrossEntropyLoss) combines ``nn.LogSoftmax`` and ``nn.NLLLoss``.\n\nWe pass our model's output logits to ``nn.CrossEntropyLoss``, which will normalize the logits and compute the prediction error.\n\n" + "## Optimization Loop\n", + "\n", + "Once we set our hyperparameters, we can then train and optimize our model with an optimization loop. Each\n", + "iteration of the optimization loop is called an **epoch**.\n", + "\n", + "Each epoch consists of two main parts:\n", + " - **The Train Loop** - iterate over the training dataset and try to converge to optimal parameters.\n", + " - **The Validation/Test Loop** - iterate over the test dataset to check if model performance is improving.\n", + "\n", + "Let's briefly familiarize ourselves with some of the concepts used in the training loop. Jump ahead to\n", + "see the `full-impl-label` of the optimization loop.\n", + "\n", + "### Loss Function\n", + "\n", + "When presented with some training data, our untrained network is likely not to give the correct\n", + "answer. **Loss function** measures the degree of dissimilarity of obtained result to the target value,\n", + "and it is the loss function that we want to minimize during training. To calculate the loss we make a\n", + "prediction using the inputs of our given data sample and compare it against the true data label value.\n", + "\n", + "Common loss functions include [nn.MSELoss](https://pytorch.org/docs/stable/generated/torch.nn.MSELoss.html#torch.nn.MSELoss) (Mean Square Error) for regression tasks, and\n", + "[nn.NLLLoss](https://pytorch.org/docs/stable/generated/torch.nn.NLLLoss.html#torch.nn.NLLLoss) (Negative Log Likelihood) for classification.\n", + "[nn.CrossEntropyLoss](https://pytorch.org/docs/stable/generated/torch.nn.CrossEntropyLoss.html#torch.nn.CrossEntropyLoss) combines ``nn.LogSoftmax`` and ``nn.NLLLoss``.\n", + "\n", + "We pass our model's output logits to ``nn.CrossEntropyLoss``, which will normalize the logits and compute the prediction error.\n", + "\n" ] }, { @@ -62,14 +151,22 @@ }, "outputs": [], "source": [ - "# Initialize the loss function\nloss_fn = nn.CrossEntropyLoss()" + "# Initialize the loss function\n", + "loss_fn = nn.CrossEntropyLoss()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Optimizer\n\nOptimization is the process of adjusting model parameters to reduce model error in each training step. **Optimization algorithms** define how this process is performed (in this example we use Stochastic Gradient Descent).\nAll optimization logic is encapsulated in the ``optimizer`` object. Here, we use the SGD optimizer; additionally, there are many [different optimizers](https://pytorch.org/docs/stable/optim.html)\navailable in PyTorch such as ADAM and RMSProp, that work better for different kinds of models and data.\n\nWe initialize the optimizer by registering the model's parameters that need to be trained, and passing in the learning rate hyperparameter.\n\n" + "### Optimizer\n", + "\n", + "Optimization is the process of adjusting model parameters to reduce model error in each training step. **Optimization algorithms** define how this process is performed (in this example we use Stochastic Gradient Descent).\n", + "All optimization logic is encapsulated in the ``optimizer`` object. Here, we use the SGD optimizer; additionally, there are many [different optimizers](https://pytorch.org/docs/stable/optim.html)\n", + "available in PyTorch such as ADAM and RMSProp, that work better for different kinds of models and data.\n", + "\n", + "We initialize the optimizer by registering the model's parameters that need to be trained, and passing in the learning rate hyperparameter.\n", + "\n" ] }, { @@ -87,14 +184,22 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Inside the training loop, optimization happens in three steps:\n * Call ``optimizer.zero_grad()`` to reset the gradients of model parameters. Gradients by default add up; to prevent double-counting, we explicitly zero them at each iteration.\n * Backpropagate the prediction loss with a call to ``loss.backward()``. PyTorch deposits the gradients of the loss w.r.t. each parameter.\n * Once we have our gradients, we call ``optimizer.step()`` to adjust the parameters by the gradients collected in the backward pass.\n\n" + "Inside the training loop, optimization happens in three steps:\n", + " * Call ``optimizer.zero_grad()`` to reset the gradients of model parameters. Gradients by default add up; to prevent double-counting, we explicitly zero them at each iteration.\n", + " * Backpropagate the prediction loss with a call to ``loss.backward()``. PyTorch deposits the gradients of the loss w.r.t. each parameter.\n", + " * Once we have our gradients, we call ``optimizer.step()`` to adjust the parameters by the gradients collected in the backward pass.\n", + "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "\n## Full Implementation\nWe define ``train_loop`` that loops over our optimization code, and ``test_loop`` that\nevaluates the model's performance against our test data.\n\n" + "\n", + "## Full Implementation\n", + "We define ``train_loop`` that loops over our optimization code, and ``test_loop`` that\n", + "evaluates the model's performance against our test data.\n", + "\n" ] }, { @@ -105,14 +210,46 @@ }, "outputs": [], "source": [ - "def train_loop(dataloader, model, loss_fn, optimizer):\n size = len(dataloader.dataset)\n for batch, (X, y) in enumerate(dataloader):\n # Compute prediction and loss\n pred = model(X)\n loss = loss_fn(pred, y)\n\n # Backpropagation\n optimizer.zero_grad()\n loss.backward()\n optimizer.step()\n\n if batch % 100 == 0:\n loss, current = loss.item(), (batch + 1) * len(X)\n print(f\"loss: {loss:>7f} [{current:>5d}/{size:>5d}]\")\n\n\ndef test_loop(dataloader, model, loss_fn):\n size = len(dataloader.dataset)\n num_batches = len(dataloader)\n test_loss, correct = 0, 0\n\n with torch.no_grad():\n for X, y in dataloader:\n pred = model(X)\n test_loss += loss_fn(pred, y).item()\n correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n\n test_loss /= num_batches\n correct /= size\n print(f\"Test Error: \\n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \\n\")" + "def train_loop(dataloader, model, loss_fn, optimizer):\n", + " size = len(dataloader.dataset)\n", + " for batch, (X, y) in enumerate(dataloader):\n", + " # Compute prediction and loss\n", + " pred = model(X)\n", + " loss = loss_fn(pred, y)\n", + "\n", + " # Backpropagation\n", + " optimizer.zero_grad()\n", + " loss.backward()\n", + " optimizer.step()\n", + "\n", + " if batch % 100 == 0:\n", + " loss, current = loss.item(), (batch + 1) * len(X)\n", + " print(f\"loss: {loss:>7f} [{current:>5d}/{size:>5d}]\")\n", + "\n", + "\n", + "def test_loop(dataloader, model, loss_fn):\n", + " size = len(dataloader.dataset)\n", + " num_batches = len(dataloader)\n", + " test_loss, correct = 0, 0\n", + "\n", + " with torch.no_grad():\n", + " for X, y in dataloader:\n", + " pred = model(X)\n", + " test_loss += loss_fn(pred, y).item()\n", + " correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n", + "\n", + " test_loss /= num_batches\n", + " correct /= size\n", + " print(f\"Test Error: \\n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \\n\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "We initialize the loss function and optimizer, and pass it to ``train_loop`` and ``test_loop``.\nFeel free to increase the number of epochs to track the model's improving performance.\n\n" + "We initialize the loss function and optimizer, and pass it to ``train_loop`` and ``test_loop``.\n", + "Feel free to increase the number of epochs to track the model's improving performance.\n", + "\n" ] }, { @@ -123,14 +260,27 @@ }, "outputs": [], "source": [ - "loss_fn = nn.CrossEntropyLoss()\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n\nepochs = 10\nfor t in range(epochs):\n print(f\"Epoch {t+1}\\n-------------------------------\")\n train_loop(train_dataloader, model, loss_fn, optimizer)\n test_loop(test_dataloader, model, loss_fn)\nprint(\"Done!\")" + "loss_fn = nn.CrossEntropyLoss()\n", + "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n", + "\n", + "epochs = 10\n", + "for t in range(epochs):\n", + " print(f\"Epoch {t+1}\\n-------------------------------\")\n", + " train_loop(train_dataloader, model, loss_fn, optimizer)\n", + " test_loop(test_dataloader, model, loss_fn)\n", + "print(\"Done!\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Further Reading\n- [Loss Functions](https://pytorch.org/docs/stable/nn.html#loss-functions)\n- [torch.optim](https://pytorch.org/docs/stable/optim.html)\n- [Warmstart Training a Model](https://pytorch.org/tutorials/recipes/recipes/warmstarting_model_using_parameters_from_a_different_model.html)\n\n\n" + "## Further Reading\n", + "- [Loss Functions](https://pytorch.org/docs/stable/nn.html#loss-functions)\n", + "- [torch.optim](https://pytorch.org/docs/stable/optim.html)\n", + "- [Warmstart Training a Model](https://pytorch.org/tutorials/recipes/recipes/warmstarting_model_using_parameters_from_a_different_model.html)\n", + "\n", + "\n" ] } ], @@ -155,4 +305,4 @@ }, "nbformat": 4, "nbformat_minor": 0 -} \ No newline at end of file +} diff --git a/tutorials/09-SaveLoad.ipynb b/tutorials/09-SaveLoad.ipynb index d13e7e7..c6d3d72 100644 --- a/tutorials/09-SaveLoad.ipynb +++ b/tutorials/09-SaveLoad.ipynb @@ -1,21 +1,23 @@ { "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "%matplotlib inline" - ] - }, { "cell_type": "markdown", "metadata": {}, "source": [ - "\n[Learn the Basics](intro.html) ||\n[Quickstart](quickstart_tutorial.html) ||\n[Tensors](tensorqs_tutorial.html) ||\n[Datasets & DataLoaders](data_tutorial.html) ||\n[Transforms](transforms_tutorial.html) ||\n[Build Model](buildmodel_tutorial.html) ||\n[Autograd](autogradqs_tutorial.html) ||\n[Optimization](optimization_tutorial.html) ||\n**Save & Load Model**\n\n# Save and Load the Model\n\nIn this section we will look at how to persist model state with saving, loading and running model predictions.\n" + "\n", + "[Learn the Basics](intro.html) ||\n", + "[Quickstart](quickstart_tutorial.html) ||\n", + "[Tensors](tensorqs_tutorial.html) ||\n", + "[Datasets & DataLoaders](data_tutorial.html) ||\n", + "[Transforms](transforms_tutorial.html) ||\n", + "[Build Model](buildmodel_tutorial.html) ||\n", + "[Autograd](autogradqs_tutorial.html) ||\n", + "[Optimization](optimization_tutorial.html) ||\n", + "**Save & Load Model**\n", + "\n", + "# Save and Load the Model\n", + "\n", + "In this section we will look at how to persist model state with saving, loading and running model predictions.\n" ] }, { @@ -26,14 +28,21 @@ }, "outputs": [], "source": [ - "import torch\nimport torchvision.models as models" + "%matplotlib inline\n", + "\n", + "import torch\n", + "import torchvision.models as models" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Saving and Loading Model Weights\nPyTorch models store the learned parameters in an internal\nstate dictionary, called ``state_dict``. These can be persisted via the ``torch.save``\nmethod:\n\n" + "## Saving and Loading Model Weights\n", + "PyTorch models store the learned parameters in an internal\n", + "state dictionary, called ``state_dict``. These can be persisted via the ``torch.save``\n", + "method:\n", + "\n" ] }, { @@ -44,14 +53,17 @@ }, "outputs": [], "source": [ - "model = models.vgg16(pretrained=True)\ntorch.save(model.state_dict(), 'model_weights.pth')" + "model = models.vgg16(pretrained=True)\n", + "torch.save(model.state_dict(), 'model_weights.pth')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "To load model weights, you need to create an instance of the same model first, and then load the parameters\nusing ``load_state_dict()`` method.\n\n" + "To load model weights, you need to create an instance of the same model first, and then load the parameters\n", + "using ``load_state_dict()`` method.\n", + "\n" ] }, { @@ -62,21 +74,28 @@ }, "outputs": [], "source": [ - "model = models.vgg16() # we do not specify pretrained=True, i.e. do not load default weights\nmodel.load_state_dict(torch.load('model_weights.pth'))\nmodel.eval()" + "model = models.vgg16() # we do not specify pretrained=True, i.e. do not load default weights\n", + "model.load_state_dict(torch.load('model_weights.pth'))\n", + "model.eval()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "

Note

be sure to call ``model.eval()`` method before inferencing to set the dropout and batch normalization layers to evaluation mode. Failing to do this will yield inconsistent inference results.

\n\n" + "

Note

be sure to call ``model.eval()`` method before inferencing to set the dropout and batch normalization layers to evaluation mode. Failing to do this will yield inconsistent inference results.

\n", + "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Saving and Loading Models with Shapes\nWhen loading model weights, we needed to instantiate the model class first, because the class\ndefines the structure of a network. We might want to save the structure of this class together with\nthe model, in which case we can pass ``model`` (and not ``model.state_dict()``) to the saving function:\n\n" + "## Saving and Loading Models with Shapes\n", + "When loading model weights, we needed to instantiate the model class first, because the class\n", + "defines the structure of a network. We might want to save the structure of this class together with\n", + "the model, in which case we can pass ``model`` (and not ``model.state_dict()``) to the saving function:\n", + "\n" ] }, { @@ -94,7 +113,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We can then load the model like this:\n\n" + "We can then load the model like this:\n", + "\n" ] }, { @@ -112,14 +132,17 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "

Note

This approach uses Python [pickle](https://docs.python.org/3/library/pickle.html) module when serializing the model, thus it relies on the actual class definition to be available when loading the model.

\n\n" + "

Note

This approach uses Python [pickle](https://docs.python.org/3/library/pickle.html) module when serializing the model, thus it relies on the actual class definition to be available when loading the model.

\n", + "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Related Tutorials\n[Saving and Loading a General Checkpoint in PyTorch](https://pytorch.org/tutorials/recipes/recipes/saving_and_loading_a_general_checkpoint.html)\n\n" + "## Related Tutorials\n", + "[Saving and Loading a General Checkpoint in PyTorch](https://pytorch.org/tutorials/recipes/recipes/saving_and_loading_a_general_checkpoint.html)\n", + "\n" ] } ], @@ -144,4 +167,4 @@ }, "nbformat": 4, "nbformat_minor": 0 -} \ No newline at end of file +}