{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "66951a8c", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "torch : 1.10.1\n", "pytorch_lightning: 1.6.0.dev0\n", "torchmetrics : 0.6.2\n", "matplotlib : 3.3.4\n", "coral_pytorch : 1.2.0\n", "\n" ] } ], "source": [ "%load_ext watermark\n", "%watermark -p torch,pytorch_lightning,torchmetrics,matplotlib,coral_pytorch" ] }, { "cell_type": "code", "execution_count": 2, "id": "60a1d929", "metadata": {}, "outputs": [], "source": [ "%load_ext pycodestyle_magic\n", "%flake8_on --ignore W291,W293,E703" ] }, { "cell_type": "markdown", "id": "6dc0efd6", "metadata": {}, "source": [ "      \n", "\n", "# Cross entropy baseline model for ordinal regression and deep learning -- cement strength dataset" ] }, { "cell_type": "markdown", "id": "47f22067", "metadata": {}, "source": [ "This is a regular cross entropy classifier as a baseline for comparison with ordinal regression methods." ] }, { "cell_type": "markdown", "id": "6c528557", "metadata": {}, "source": [ "## General settings and hyperparameters" ] }, { "cell_type": "markdown", "id": "2527be84", "metadata": {}, "source": [ "- Here, we specify some general hyperparameter values and general settings\n", "- Note that for small datatsets, it is not necessary and better not to use multiple workers as it can sometimes cause issues with too many open files in PyTorch" ] }, { "cell_type": "code", "execution_count": 3, "id": "37d461ef", "metadata": {}, "outputs": [], "source": [ "BATCH_SIZE = 128\n", "NUM_EPOCHS = 200\n", "LEARNING_RATE = 0.005\n", "NUM_WORKERS = 0\n", "\n", "DATA_BASEPATH = \".\"" ] }, { "cell_type": "markdown", "id": "ddfe5704", "metadata": {}, "source": [ "## Implementing a `MultiLayerPerceptron` using PyTorch Lightning's `LightningModule`" ] }, { "cell_type": "markdown", "id": "829a6664", "metadata": {}, "source": [ "- In this section, we set up the main model architecture using the `LightningModule` from PyTorch Lightning.\n", "- We start with defining our `MultiLayerPerceptron` model in pure PyTorch, and then we use it in the `LightningModule` to get all the extra benefits that PyTorch Lightning provides." ] }, { "cell_type": "code", "execution_count": 4, "id": "c03faadc", "metadata": {}, "outputs": [], "source": [ "import torch\n", "\n", "\n", "# Regular PyTorch Module\n", "class MultiLayerPerceptron(torch.nn.Module):\n", " def __init__(self, input_size, hidden_units, num_classes):\n", " super().__init__()\n", "\n", " # num_classes is used by the corn loss function\n", " self.num_classes = num_classes\n", " \n", " # Initialize MLP layers\n", " all_layers = []\n", " for hidden_unit in hidden_units:\n", " layer = torch.nn.Linear(input_size, hidden_unit)\n", " all_layers.append(layer)\n", " all_layers.append(torch.nn.ReLU())\n", " input_size = hidden_unit\n", "\n", " output_layer = torch.nn.Linear(hidden_units[-1], num_classes)\n", " \n", " all_layers.append(output_layer)\n", " self.model = torch.nn.Sequential(*all_layers)\n", " \n", " def forward(self, x):\n", " x = self.model(x)\n", " return x" ] }, { "cell_type": "markdown", "id": "8ff4cacb", "metadata": {}, "source": [ "- In our `LightningModule` we use loggers to track mean absolute errors for both the training and validation set during training; this allows us to select the best model based on validation set performance later." ] }, { "cell_type": "code", "execution_count": 5, "id": "0c14a021", "metadata": {}, "outputs": [], "source": [ "import pytorch_lightning as pl\n", "import torchmetrics\n", "\n", "\n", "# LightningModule that receives a PyTorch model as input\n", "class LightningMLP(pl.LightningModule):\n", " def __init__(self, model, learning_rate):\n", " super().__init__()\n", "\n", " self.learning_rate = learning_rate\n", " # The inherited PyTorch module\n", " self.model = model\n", "\n", " # Save settings and hyperparameters to the log directory\n", " # but skip the model parameters\n", " self.save_hyperparameters(ignore=['model'])\n", "\n", " # Set up attributes for computing the MAE\n", " self.train_mae = torchmetrics.MeanAbsoluteError()\n", " self.valid_mae = torchmetrics.MeanAbsoluteError()\n", " self.test_mae = torchmetrics.MeanAbsoluteError()\n", " \n", " # Defining the forward method is only necessary \n", " # if you want to use a Trainer's .predict() method (optional)\n", " def forward(self, x):\n", " return self.model(x)\n", " \n", " # A common forward step to compute the loss and labels\n", " # this is used for training, validation, and testing below\n", " def _shared_step(self, batch):\n", " features, true_labels = batch\n", " logits = self(features)\n", " loss = torch.nn.functional.cross_entropy(logits, true_labels)\n", " predicted_labels = torch.argmax(logits, dim=1)\n", "\n", " return loss, true_labels, predicted_labels\n", "\n", " def training_step(self, batch, batch_idx):\n", " loss, true_labels, predicted_labels = self._shared_step(batch)\n", " self.log(\"train_loss\", loss)\n", " self.train_mae(predicted_labels, true_labels)\n", " self.log(\"train_mae\", self.train_mae, on_epoch=True, on_step=False)\n", " return loss # this is passed to the optimzer for training\n", "\n", " def validation_step(self, batch, batch_idx):\n", " loss, true_labels, predicted_labels = self._shared_step(batch)\n", " self.log(\"valid_loss\", loss)\n", " self.valid_mae(predicted_labels, true_labels)\n", " self.log(\"valid_mae\", self.valid_mae,\n", " on_epoch=True, on_step=False, prog_bar=True)\n", "\n", " def test_step(self, batch, batch_idx):\n", " loss, true_labels, predicted_labels = self._shared_step(batch)\n", " self.test_mae(predicted_labels, true_labels)\n", " self.log(\"test_mae\", self.test_mae, on_epoch=True, on_step=False)\n", "\n", " def configure_optimizers(self):\n", " optimizer = torch.optim.Adam(self.parameters(), lr=self.learning_rate)\n", " return optimizer" ] }, { "cell_type": "markdown", "id": "6ec9d753", "metadata": {}, "source": [ "## Setting up the dataset" ] }, { "cell_type": "markdown", "id": "5621120c", "metadata": {}, "source": [ "- In this section, we are going to set up our dataset.\n", "- We start by downloading and taking a look at the Cement dataset:" ] }, { "cell_type": "markdown", "id": "5311a22d", "metadata": {}, "source": [ "### Inspecting the dataset" ] }, { "cell_type": "code", "execution_count": 6, "id": "27c9ba77", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
responseV1V2V3V4V5V6V7V8
04540.00.00.0162.02.51040.0676.028
14540.00.00.0162.02.51055.0676.028
22332.5142.50.0228.00.0932.0594.0270
32332.5142.50.0228.00.0932.0594.0365
42198.6132.40.0192.00.0978.4825.5360
\n", "
" ], "text/plain": [ " response V1 V2 V3 V4 V5 V6 V7 V8\n", "0 4 540.0 0.0 0.0 162.0 2.5 1040.0 676.0 28\n", "1 4 540.0 0.0 0.0 162.0 2.5 1055.0 676.0 28\n", "2 2 332.5 142.5 0.0 228.0 0.0 932.0 594.0 270\n", "3 2 332.5 142.5 0.0 228.0 0.0 932.0 594.0 365\n", "4 2 198.6 132.4 0.0 192.0 0.0 978.4 825.5 360" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import pandas as pd\n", "import numpy as np\n", "\n", "\n", "data_df = pd.read_csv(\"https://raw.githubusercontent.com/gagolews/\"\n", " \"ordinal_regression_data/master/cement_strength.csv\")\n", "data_df[\"response\"] = data_df[\"response\"]-1 # labels should start at 0\n", "\n", "data_labels = data_df[\"response\"]\n", "data_features = data_df.loc[:, [\n", " \"V1\", \"V2\", \"V3\", \"V4\", \"V5\", \"V6\", \"V7\", \"V8\"]]\n", "\n", "data_df.head()" ] }, { "cell_type": "code", "execution_count": 7, "id": "f9611a79", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Number of features: 8\n", "Number of examples: 998\n", "Labels: [0 1 2 3 4]\n", "Label distribution: [196 310 244 152 96]\n" ] } ], "source": [ "print('Number of features:', data_features.shape[1])\n", "print('Number of examples:', data_features.shape[0])\n", "print('Labels:', np.unique(data_labels.values))\n", "print('Label distribution:', np.bincount(data_labels))" ] }, { "cell_type": "markdown", "id": "62085741", "metadata": {}, "source": [ "- Above, we can see that the dataset consists of 8 features, and there are 998 examples in total.\n", "- The labels are in range from 1 (weakest) to 5 (strongest), and we normalize them to start at zero (hence, the normalized labels are in the range 0 to 4). \n", "- Notice also that the dataset is quite imbalanced." ] }, { "cell_type": "markdown", "id": "293b9496", "metadata": {}, "source": [ "### Performance baseline" ] }, { "cell_type": "markdown", "id": "603b070d", "metadata": {}, "source": [ "- Especially for imbalanced datasets, it's quite useful to compute a performance baseline.\n", "- In classification contexts, a useful baseline is to compute the accuracy for a scenario where the model always predicts the majority class -- you want your model to be better than that!\n", "- Note that if you are intersted in a single number that minimized the dataset mean squared error (MSE), that's the mean; similary, the median is a number that minimzes the mean absolute error (MAE).\n", "- So, if we use the mean absolute error, $\\mathrm{MAE}=\\frac{1}{N} \\sum_{i=1}^{N}\\left|y_{i}-\\hat{y}_{i}\\right|$, to evaluate the model, it is useful to compute the MAE pretending the predicted label is always the median:" ] }, { "cell_type": "code", "execution_count": 8, "id": "a5625136", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Baseline MAE: 1.03\n" ] } ], "source": [ "avg_prediction = np.median(data_labels.values) # median minimizes MAE\n", "baseline_mae = np.mean(np.abs(data_labels.values - avg_prediction))\n", "print(f'Baseline MAE: {baseline_mae:.2f}')" ] }, { "cell_type": "markdown", "id": "260494f7", "metadata": {}, "source": [ "- In other words, a model that would always predict the dataset median would achieve a MAE of 1.03. A model that has an MAE of > 1 is certainly a bad model." ] }, { "cell_type": "markdown", "id": "f8155016", "metadata": {}, "source": [ "### Creating a `Dataset` class" ] }, { "cell_type": "markdown", "id": "039d15f6", "metadata": {}, "source": [ "- Next, let us set up a data loading mechanism for our model.\n", "- Note that the Cement dataset is a relatively small dataset that fits into memory quite comfortably so this may seem like overkill. However, the following steps are useful as a template since you can use those for arbitrarily-sized datatsets.\n", "- First, we define a PyTorch `Dataset` class that returns the features (inputs) and labels:" ] }, { "cell_type": "code", "execution_count": 9, "id": "6cb989de", "metadata": {}, "outputs": [], "source": [ "from torch.utils.data import Dataset\n", "\n", "\n", "class MyDataset(Dataset):\n", "\n", " def __init__(self, feature_array, label_array, dtype=np.float32):\n", " self.features = feature_array.astype(dtype)\n", " self.labels = label_array\n", "\n", " def __getitem__(self, index):\n", " inputs = self.features[index]\n", " label = self.labels[index]\n", " return inputs, label\n", "\n", " def __len__(self):\n", " return self.features.shape[0]" ] }, { "cell_type": "markdown", "id": "754c85d2", "metadata": {}, "source": [ "### Setting up a `DataModule`" ] }, { "cell_type": "markdown", "id": "3d6eb8dc", "metadata": {}, "source": [ "- There are three main ways we can prepare the dataset for Lightning. We can\n", " 1. make the dataset part of the model;\n", " 2. set up the data loaders as usual and feed them to the fit method of a Lightning Trainer -- the Trainer is introduced in the next subsection;\n", " 3. create a LightningDataModule.\n", "- Here, we are going to use approach 3, which is the most organized approach. The `LightningDataModule` consists of several self-explanatory methods as we can see below:\n" ] }, { "cell_type": "code", "execution_count": 10, "id": "91332326", "metadata": {}, "outputs": [], "source": [ "import os\n", "from sklearn.model_selection import train_test_split\n", "from sklearn.preprocessing import StandardScaler\n", "from torch.utils.data import DataLoader\n", "\n", "\n", "class DataModule(pl.LightningDataModule):\n", " def __init__(self, data_path='./'):\n", " super().__init__()\n", " self.data_path = data_path\n", " \n", " def prepare_data(self):\n", " data_df = pd.read_csv(\n", " 'https://raw.githubusercontent.com/gagolews/'\n", " 'ordinal_regression_data/master/cement_strength.csv')\n", " data_df.to_csv(\n", " os.path.join(self.data_path, 'cement_strength.csv'), index=None)\n", " return\n", "\n", " def setup(self, stage=None):\n", " data_df = pd.read_csv(\n", " os.path.join(self.data_path, 'cement_strength.csv'))\n", " data_df[\"response\"] = data_df[\"response\"]-1 # labels should start at 0\n", " self.data_labels = data_df[\"response\"]\n", " self.data_features = data_df.loc[:, [\n", " \"V1\", \"V2\", \"V3\", \"V4\", \"V5\", \"V6\", \"V7\", \"V8\"]]\n", " \n", " # Split into\n", " # 70% train, 10% validation, 20% testing\n", " \n", " X_temp, X_test, y_temp, y_test = train_test_split(\n", " self.data_features.values,\n", " self.data_labels.values,\n", " test_size=0.2,\n", " random_state=1,\n", " stratify=self.data_labels.values)\n", "\n", " X_train, X_valid, y_train, y_valid = train_test_split(\n", " X_temp,\n", " y_temp,\n", " test_size=0.1,\n", " random_state=1,\n", " stratify=y_temp)\n", " \n", " # Standardize features\n", " sc = StandardScaler()\n", " X_train_std = sc.fit_transform(X_train)\n", " X_valid_std = sc.transform(X_valid)\n", " X_test_std = sc.transform(X_test)\n", "\n", " self.train = MyDataset(X_train_std, y_train)\n", " self.valid = MyDataset(X_valid_std, y_valid)\n", " self.test = MyDataset(X_test_std, y_test)\n", "\n", " def train_dataloader(self):\n", " return DataLoader(self.train, batch_size=BATCH_SIZE,\n", " num_workers=NUM_WORKERS,\n", " drop_last=True)\n", "\n", " def val_dataloader(self):\n", " return DataLoader(self.valid, batch_size=BATCH_SIZE,\n", " num_workers=NUM_WORKERS)\n", "\n", " def test_dataloader(self):\n", " return DataLoader(self.test, batch_size=BATCH_SIZE,\n", " num_workers=NUM_WORKERS)" ] }, { "cell_type": "markdown", "id": "ce22b8f5", "metadata": {}, "source": [ "- Note that the `prepare_data` method is usually used for steps that only need to be executed once, for example, downloading the dataset; the `setup` method defines the the dataset loading -- if you run your code in a distributed setting, this will be called on each node / GPU. \n", "- Next, lets initialize the `DataModule`; we use a random seed for reproducibility (so that the data set is shuffled the same way when we re-execute this code):" ] }, { "cell_type": "code", "execution_count": 11, "id": "4c1280d9", "metadata": {}, "outputs": [], "source": [ "torch.manual_seed(1) \n", "data_module = DataModule(data_path=DATA_BASEPATH)" ] }, { "cell_type": "markdown", "id": "f060cf94", "metadata": {}, "source": [ "## Training the model using the PyTorch Lightning Trainer class" ] }, { "cell_type": "markdown", "id": "8687bf17", "metadata": {}, "source": [ "- Next, we initialize our multilayer perceptron model (here, a 2-layer MLP with 24 units in the first hidden layer, and 16 units in the second hidden layer).\n", "- We wrap the model in our `LightningMLP` so that we can use PyTorch Lightning's powerful `Trainer` API.\n", "- Also, we define a callback so that we can obtain the model with the best validation set performance after training.\n", "- Note PyTorch Lightning offers [many advanced logging services](https://pytorch-lightning.readthedocs.io/en/latest/extensions/logging.html) like Weights & Biases. However, here, we will keep things simple and use the `CSVLogger`:" ] }, { "cell_type": "code", "execution_count": 12, "id": "fc71caab", "metadata": {}, "outputs": [], "source": [ "from pytorch_lightning.callbacks import ModelCheckpoint\n", "from pytorch_lightning.loggers import CSVLogger\n", "\n", "\n", "pytorch_model = MultiLayerPerceptron(\n", " input_size=data_features.shape[1],\n", " hidden_units=(40, 20),\n", " num_classes=np.bincount(data_labels).shape[0])\n", "\n", "lightning_model = LightningMLP(\n", " model=pytorch_model,\n", " learning_rate=LEARNING_RATE)\n", "\n", "\n", "callbacks = [ModelCheckpoint(\n", " save_top_k=1, mode=\"min\", monitor=\"valid_mae\")] # save top 1 model \n", "logger = CSVLogger(save_dir=\"logs/\", name=\"mlp-corn-cement\")" ] }, { "cell_type": "markdown", "id": "dd8cdbfe", "metadata": {}, "source": [ "- Now it's time to train our model:" ] }, { "cell_type": "code", "execution_count": 13, "id": "e531b3dc", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/jovyan/conda/lib/python3.8/site-packages/pytorch_lightning/trainer/connectors/callback_connector.py:90: LightningDeprecationWarning: Setting `Trainer(progress_bar_refresh_rate=50)` is deprecated in v1.5 and will be removed in v1.7. Please pass `pytorch_lightning.callbacks.progress.TQDMProgressBar` with `refresh_rate` directly to the Trainer's `callbacks` argument instead. Or, to disable the progress bar pass `enable_progress_bar = False` to the Trainer.\n", " rank_zero_deprecation(\n", "GPU available: True, used: True\n", "TPU available: False, using: 0 TPU cores\n", "IPU available: False, using: 0 IPUs\n", "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n", "\n", " | Name | Type | Params\n", "---------------------------------------------------\n", "0 | model | MultiLayerPerceptron | 1.3 K \n", "1 | train_mae | MeanAbsoluteError | 0 \n", "2 | valid_mae | MeanAbsoluteError | 0 \n", "3 | test_mae | MeanAbsoluteError | 0 \n", "---------------------------------------------------\n", "1.3 K Trainable params\n", "0 Non-trainable params\n", "1.3 K Total params\n", "0.005 Total estimated model params size (MB)\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validation sanity check: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "/home/jovyan/conda/lib/python3.8/site-packages/pytorch_lightning/trainer/connectors/data_connector.py:258: PossibleUserWarning: The dataloader, val_dataloader 0, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 4 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n", " rank_zero_warn(\n", "/home/jovyan/conda/lib/python3.8/site-packages/pytorch_lightning/trainer/connectors/data_connector.py:258: PossibleUserWarning: The dataloader, train_dataloader, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 4 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n", " rank_zero_warn(\n", "/home/jovyan/conda/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py:1850: PossibleUserWarning: The number of training samples (5) is smaller than the logging interval Trainer(log_every_n_steps=10). Set a lower value for log_every_n_steps if you want to see logs for the training epoch.\n", " rank_zero_warn(\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "386ac3987ac84863a7845e29626e3f7b", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Training: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Validating: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "Training took 0.29 min in total.\n" ] } ], "source": [ "import time\n", "\n", "\n", "trainer = pl.Trainer(\n", " max_epochs=NUM_EPOCHS,\n", " callbacks=callbacks,\n", " progress_bar_refresh_rate=50, # recommended for notebooks\n", " accelerator=\"auto\", # Uses GPUs or TPUs if available\n", " devices=\"auto\", # Uses all available GPUs/TPUs if applicable\n", " logger=logger,\n", " deterministic=True,\n", " log_every_n_steps=10)\n", "\n", "start_time = time.time()\n", "trainer.fit(model=lightning_model, datamodule=data_module)\n", "\n", "runtime = (time.time() - start_time)/60\n", "print(f\"Training took {runtime:.2f} min in total.\")" ] }, { "cell_type": "markdown", "id": "f66903b8", "metadata": {}, "source": [ "## Evaluating the model" ] }, { "cell_type": "markdown", "id": "eb4bd0b8", "metadata": {}, "source": [ "- After training, let's plot our training MAE and validation MAE using pandas, which, in turn, uses matplotlib for plotting (you may want to consider a [more advanced logger](https://pytorch-lightning.readthedocs.io/en/latest/extensions/logging.html) that does that for you):" ] }, { "cell_type": "code", "execution_count": 14, "id": "1abca75f", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEHCAYAAACjh0HiAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAqvUlEQVR4nO3de3xU5b3v8c+PEEgghFsg3ISggshdwkWLF6itolaxXrHWS7duaqu2bo89YrtPdbf7nLbbXtzWC9t2K9VaoxWp1mJ1a0mxVVSogIAiICgICAS5DBCuv/PHM2gIM0kmsDKD6/t+veaVmTVrrflmEdZv1rOe9Sxzd0REJL6aZTuAiIhklwqBiEjMqRCIiMScCoGISMypEIiIxJwKgYhIzDWPasVm9iDwJWCduw9MM88Y4C4gH9jg7qfVt96SkhIvKytrVKZt27bRunXrRi0btVzNplyZydVckLvZlCszjc01Z86cDe7eKeWb7h7JAzgVGAYsSPN+O2AR0DP5unND1lteXu6NNWPGjEYvG7VczaZcmcnVXO65m025MtPYXMBsT7NfjaxpyN1nAhvrmOUrwFPu/kFy/nVRZRERkfSyeY6gL9DezCrNbI6ZXZnFLCIisWUe4RATZlYGPOspzhGY2T3AcOB0oBB4FTjH3d9NMe9EYCJAaWlpeUVFRaPyJBIJioqKGrVs1HI1m3JlJldzQe5mU67MNDbX2LFj57j78FTvRXayuAFWEU4QbwO2mdlMYAhwUCFw9weABwCGDx/uY8aMadQHVlZW0thlo5ar2ZQrM7maC3I3W2VlJaNHj2bVqlVUV1dnO84n2rZtS0FBQbZjHKS+XAUFBfTo0YP8/PwGrzObheBp4B4zaw60AEYBv8hiHhHJklWrVtGmTRvKysows2zHAWDr1q20adMm2zEOUlcud6eqqopVq1bRu3fvBq8zyu6jjwFjgBIzWwXcTugmirtPdve3zezPwHxgH/Brd18QVR4RyV3V1dU5VQSOVGZGx44dWb9+fUbLRVYI3P2yBsxzJ3BnVBlE5MihInB4NGY7xufK4o8WcfSy30D1lmwnERHJKfEpBJvep+fKp2D9O9lOIiKSU+JTCDr1Cz/XvZ3dHCKSczZt2sR9992X8XJnn302mzZtyni5q6++mieffDLj5aISn0LQrhd7m7XUEYGIHCRdIdi7d2+dy02fPp127dpFlKrpZLP7aNNq1oztrXrQRkcEIjnt3/64kEWrD++5vP7dirn93AFp3580aRLLli1j6NCh5OfnU1RURElJCQsXLmTRokWcf/75rFy5kurqar797W8zceJEAMrKypg9ezaJRIKzzjqLk08+mVdeeYXu3bvz9NNPU1hYWG+2l156iVtuuYU9e/YwYsQI7r//flq2bMmkSZN45plnaN68OWeccQY//elP+f3vf8/tt99Ofn4+bdu2ZebMmYdl+8SnEADbWvdUIRCRg/z4xz9mwYIFzJ07l8rKSs455xxmzZrFoEGDAHjwwQfp0KEDO3bsYMSIEVx44YV07NjxgHUsWbKExx57jF/96ldccsklTJ06la9+9at1fm51dTVXX301L730En379uXKK6/k/vvv58orr2TatGm88847mNknzU8/+MEPmDZtGscdd1yjmqTSiV0h4KMZsONjKGyf7TgikkJd39ybysiRI6k53P3dd9/NtGnTAFi5ciVLliw5qBD07t2boUOHAlBeXs6KFSvq/ZzFixfTu3dv+vbtC8BVV13Fvffeyw033EBBQQHXXnst55xzDl/60pcAGD16NN/4xje47LLLuOCCCw79F02KzzkCkoUAYJ3OE4hIejXH+6+srOTFF1/k1VdfZd68eZxwwgkph8Jo2bLlJ8/z8vLYs2dPvZ+Tbqy35s2b8/rrr3PhhRfyhz/8gXHjxgEwefJk/vVf/5WVK1cydOhQqqqqMv3VUopVIdje6qjwZL2ah0TkU23atGHr1q0p39u8eTPt27enVatWvPPOO8yaNeuwfW6/fv1YsWIFS5cuBeCRRx7htNNOI5FIsHnzZs4++2zuuusu5s6dC8CyZcsYMWIEP/jBDygpKWHlypWHJUesmoaqCzpBfmsdEYjIATp27Mjo0aMZOHAghYWFlJaWfvLeuHHjmDx5MoMHD+a4447jxBNPPGyfW1BQwEMPPcTFF1/8ycni6667jo0bNzJ+/Hiqq6txd37xizAM23e+8x0WL16MmXH66aczZMiQw5IjVoUAawad++mIQEQO8rvf/e6A1/uPEFq2bMlzzz2Xcpn95wFKSkpYsODTodJuueWWOj9rypQpnzw//fTTefPNNw94v2vXrrz++usHLffUU09FMhherJqGAOh0vI4IRERqiF8h6NwPtq2D7XXdRVNE5NBdf/31DB069IDHQw89lO1YB4lX0xCEIwIIQ02Ujc5uFhH5TLv33nuzHaFB4nlEADpPICKSFL9CUNwdWhbrPIGISFL8CoEZdDpOg8+JiCTFrxBAGJJ63aJspxCRI1RRUREAq1ev5qKLLko5z5gxY5g9e3badZSVlbFhw4ZI8mUqnoWg8/GwvQoSmd3XU0Skpm7duuXUfQUaK7JCYGYPmtk6M6vzhvRmNsLM9ppZ6rIahf03qVHzkIgAt9566wH3I7jjjjv40Y9+xOmnn86wYcMYNGgQTz/99EHLrVixgoEDBwKwY8cOJkyYwODBg7n00kvZsWNHgz//5z//OQMHDmTgwIHcddddAGzbto1zzjmHIUOGMHDgQB5//HEAbr/9dvr378/gwYPrvXCtoaLsPjoFuAd4ON0MZpYH/AR4PsIcBysJI/1RtQR6n9KkHy0i9XhuEqx96/Cus8sgOOvHad+eMGECN910E9/85jcBeOKJJ3jyySeZNGkSxcXFbNiwgRNPPJHzzjsv7c3h77//flq1asX8+fOZP38+w4YNa1C0OXPm8NBDD/Haa6/h7owaNYrTTjuN9957j27duvGnP/0JCGMebdy4kT/+8Y+8++67BwxPfagiOyJw95lAfVdt3QhMBdZFlSOl4u7QvBA2LG3SjxWR3HTCCSewbt06Vq9ezbx582jfvj1dunThu9/9LoMHD+YLX/gCH374IR999FHadcycOfOT+w8MHjyYwYMHN+iz//a3v/HlL3+Z1q1bU1RUxAUXXMDLL7/MoEGDePHFF7n11lt5+eWXadu2LcXFxZ8MT/3UU0/RqlWrw/L7Z+2CMjPrDnwZ+Dwwokk/vFkz6HhsOCIQkdxSxzf3KF100UU8+eSTrF27lgkTJvDEE0+wfv165syZQ35+PmVlZSmHn64p3dFCXdINRd23b1/mzJnD9OnTue222zjjjDP4/ve/z4wZM3j99depqKjgnnvu4S9/+UvGn1lbNq8svgu41d331rfxzGwiMBGgtLSUysrKRn1gIpH4ZNn++4pps3I+rzVyXYdbzWy5RLkyk6u5IHezJRIJ2rZtm3YY6KZy7rnncuONN1JVVcVzzz3H1KlTadeuHdXV1bzwwgu8//77JBKJT3Ju3bqVRCLBvn372Lp1K6NGjWLKlCkMHz6cRYsWMX/+fLZt25b293J3EokE5eXlfOMb3+D666/H3Zk6dSoPPPAA7777Lu3bt2f8+PHk5eXx6KOPsmbNGhKJBKeccgoDBgxg6NChKddfXV2d2b+1u0f2AMqABWneWw6sSD4ShOah8+tbZ3l5uTfWjBkzPn3x0r+739HOfXd1o9d3OB2QLYcoV2ZyNZd77mabMWOGL1q0KNsx3N194MCBPmbMGHd3X758uZ944oleXl7u11xzjffr18+XL1/u7u6tW7f+ZJ4BAwa4u/v27dv90ksv9UGDBvkVV1zhJ510kr/xxhtpP6tXr16+fv16d3f/2c9+5gMGDPABAwb4L37xC3d3//Of/+yDBg3yIUOG+PDhw/2NN97w1atX+7Bhw3zQoEE+cOBAnzJlSsp1p9qewGxPs1/N2hGBu/fe/9zMpgDPuvsfmixASR/wfbBx+afDTohIrL311qcnqTt27Mirr76acr5EIgGEawH2Dz9dWFhIRUVFgz+r5q0sb775Zm6++eYD3j/zzDM588wzD1qusrLysA9DHVkhMLPHgDFAiZmtAm4H8gHcfXJUn9tgHY8NP6uWqBCISKxFVgjc/bIM5r06qhxp7S8EG3TCWESiM2rUKHbu3HnAtEceeYRBgwZlKdHB4jcM9X4FxVDUBarUhVREovPaa69lO0K94jnExH4lfXREIJIjPE03SslMY7ZjvAuBriUQyQkFBQVUVVWpGBwid6eqqoqCgoKMlotv0xCEI4IdH8O2KmjdMdtpRGKrR48erFq1ivXrc2cgyOrq6ox3qE2hvlwFBQX06NEjo3XGuxB07BN+Vi1RIRDJovz8fHr37l3/jE2osrKSE044IdsxDhJFrng3DZWo55CISLwLQbtekNdC5wlEJNbiXQia5UGHozUKqYjEWrwLAajnkIjEngpBSZ8w3tDePdlOIiKSFSoEHfvAvt2w6f1sJxERyQoVgpJkF9L1i7ObQ0QkS1QI9t/Ift2i7OYQEckSFYKCYmjXU4VARGJLhQCg8wD4aGG2U4iIZIUKAUDpgHB18Z6d9c8rIvIZo0IAUNoffC9seDfbSUREmpwKAYSmIVDzkIjEkgoBhKuL81qoEIhILEVWCMzsQTNbZ2YL0rx/uZnNTz5eMbMhUWWpV15z6HScCoGIxFKURwRTgHF1vL8cOM3dBwM/BB6IMEv9SgfB2rdAd0gSkZiJrBC4+0xgYx3vv+LuHydfzgIyu6XO4dZtKGxbB1vXZDWGiEhTy5VzBNcAz2U1Qdeh4efqudlMISLS5CzKm0WbWRnwrLsPrGOescB9wMnuXpVmnonARIDS0tLyioqKRuVJJBIUFRWlfK/Z3p2c8vIE3u91MSt6f6VR6z8UdWXLJuXKTK7mgtzNplyZaWyusWPHznH34SnfdPfIHkAZsKCO9wcDy4C+DV1neXm5N9aMGTPqnuGeUe6/vbjR6z8U9WbLEuXKTK7mcs/dbMqVmcbmAmZ7mv1q1pqGzKwn8BRwhbvnxpVc3YbCmrnZTiEi0qSi7D76GPAqcJyZrTKza8zsOjO7LjnL94GOwH1mNtfMZkeVpcG6DoXER7BFJ4xFJD6aR7Vid7+snvevBa6N6vMbpdvQ8HP1m1DcNatRRESaSq70GsoNXQaBNVPzkIjEigpBTS1aQ8lx6kIqIrGiQlCbThiLSMyoENSmE8YiEjMqBLXtP2GsowIRiQkVgtr2nzDWeQIRiQkVgtpatIaSvjoiEJHYUCFIpdsJ8OEcDUktIrGgQpBKzxNh23qoWpbtJCIikVMhSKXX6PDz/b9nN4eISBNQIUil47HQuhO8/0q2k4iIRE6FIBUz6PU5FQIRiQUVgnR6jYbNH8CmD7KdREQkUioE6fQ8KfzUUYGIfMapEKRTOhBalcDSF7OdREQkUioE6TRrBn2+GArBvr3ZTiMiEhkVgrr0+SLs+DhcXCYi8hmlQlCXYz4PlgdLXsh2EhGRyKgQ1KWwPRw1Ct59PttJREQiE+XN6x80s3VmtiDN+2Zmd5vZUjObb2bDospySI4bB2vnw8fvZzuJiEgkojwimAKMq+P9s4A+ycdE4P4IszRe//Hh56Kns5tDRCQikRUCd58JbKxjlvHAwx7MAtqZWdeo8jRa+7Jw17JFf8hyEBGRaJhHONSymZUBz7r7wBTvPQv82N3/lnz9EnCru89OMe9EwlEDpaWl5RUVFY3Kk0gkKCoqyni5oz6YyjHvPcysUQ9QXVjaqM+uT2OzRU25MpOruSB3sylXZhqba+zYsXPcfXjKN909sgdQBixI896fgJNrvH4JKK9vneXl5d5YM2bMaNyCVe+5317s/rf/bPRn16fR2SKmXJnJ1VzuuZtNuTLT2FzAbE+zX81mr6FVwFE1XvcAVmcpS9069FbzkIh8ZmWzEDwDXJnsPXQisNnd12QxT90GnB8uLFPvIRH5jGke1YrN7DFgDFBiZquA24F8AHefDEwHzgaWAtuBr0WV5bDofz68eEfoPTT6W9lOIyKfNRuWwvp3oE1XWDsPEuvCvVHeehLWvw3lX4MR10Ty0ZEVAne/rJ73Hbg+qs8/7Dr0hq5DQvOQCoGINNSWNVDcNdwDfd0i6HR8mP7eDLBmYYe/6GlYPB1I0XmnsD107g8v3h6GxS8677BHjKwQfCYN+HI4Kvh4RehWKiJS246PYcmLMPBCePMR+OO34LLHYcuH8KebYfCloSi89cSny7TuDKd+B/qeCVvXQqfjoLhbOEIo6Qst24Sm6VYlMG/5YY+sQpCJAReEQrBwGpz8L9lOIyK56Nl/CfuIxFqYNTlMe+47sHMrFHeH+Y+HaWO+C2WjoUVr6DIkjHhcW/fyFM9VCLKrfS/oMQIWTFUhEIm7HZvg4+VhJ770RVgxE9r1CkWgoB288K9hvtE3wd/vCgNYXv0n2Lgc9u0JHVByhApBpgZeCH+eBOvfhU59s51GRJrappWw4En4211QvQkKO8COGoMotO8NX50K/3UqdD4evnAHNC8Ibf2lA8Ijx6gQZKr/+fDn22B+BZz+/WynEZEo7NxK4fbVoS1/727Y+B6smQf/+A28//cwzzGfD2ORLftLaLYZ+lVYNA2OOhE6HgPXvRyODMxg7G1Z/XXqo0KQqeKu0O8cmP0gnPK/QvueiBzZdm0P9x3xvaH75pP/xKita2D+JNiZCNMhNP2cfjscfx6UHBumlV/96XpGXPvp8w5HN1n8Q6VC0Bif+xa88yy8+VsY9fVspxGRdLZtgK1roE230J6/8jXY8G7os59YG/rp79oWeuTs3v7pcm2PYsmx/0yftrtDj56SvlDSB7oMgmZ52ft9ItKgQmBmrYEd7r7PzPoC/YDn3H13pOlyVc9R4fDvlXvCRR7NW2Q7kchny5p50CwfSvsf/N6u7aEP/lGjwhH5W09Cs+ahj/2bj4RmmS6DQt/8TR8cvHxhe+jYJ7TfVy2D5i3hhCvg+C+F9Xz4Dxh8CR/OXkSfMWMi/1VzQUOPCGYCp5hZe8LgcLOBS4HLowqW8069BR69CN74FZx05FwXJ5Jr2m5aCFMfhYLicNHmvj0w/Tvg+8IOet+e0Ad/d3WYZ/WbsG09tGgDBW1hy6pPV3b0mLBzXz4ztOGP/Hroj79ldWjW7XUytKlnBOFen0s+WRTVr5xzGloIzN23m9k1wC/d/T/M7M0og+W8Y78QHpU/gUGXQFGnbCcSaRr79oYdc9vkmJGbV4Z29cS60LW6uBt06gcti8K3900fwNq3wk575xaY+2j45t2xD+QXMHTBNChsG3b8b/w6rLP3qaGNfc4UaN0pXMCZXwiJj5InZi8PPXe2bYDz7wv98/Oah/n27Q3NPC3bZGkDHXkaXAjM7CTCEcD+wS7ifX7BDM78Edx/Evzlh3De3dlOJHJo9u2D5ZWha/Sgi6B1yYHvrZwFsx+CJc9D9eawI2+WF65+zW8Ne3fBvga0FpedEorCxvdg2wbWdhlL16/9BloUhYJRtQT6nRuaXMf9BPILUq+nf5qhFprlqQhkqKE785uA24Bp7r7QzI4GZkSW6kjRqW849Jx1XxgMquuQbCcSabi9u+GDV2H7xrAzf/NR2JxsU3/x9vBNfOcWaNsTtq6G7VXQsi30PzeMl7N4evgWf8b/DcOuNG8Jw/8JdiXCRVO7toVv8W26hjb7XYlQLNr1PCDG4spKuu7fcXcdHB77pSsCclg1qBC4+1+BvwKYWTNgg7tr5DWA0/53uKbguUnwtenhSEEkm3Ymwk67uFvoIbOnGjocTfHmt+G552D+E5DfKjSf1LwQ6ugx8MU7QrPO7IfCjrxFq9C002UgHD02nFDd32X6czekz5DqS1HL3LvblwQN7TX0O+A6YC8wB2hrZj939zujDHdEKGwXLiz747dD++bIf852IokT99BMU5BsY1/0NEy/JRSCWoZB6Ilz/Lnh2zuE5+17Q6uOB55EPeenTRJfckNDm4b6u/sWM7uccB+BWwkFQYUAYNhV8PYfw9givU8NIweKRGX3jtDF8f1XYOFTYWjjotLQq2bn5nA3vbHfha0fhX7yLVpB1VIWrKlm4Dlfh1Ydsv0bSI5paCHIN7N84HzgHnffbWbR3fX+SGMG4+8LJ44f/ypc80LoqyxyqHYmwg5/0/uhHX7la7B67qcnZXuMgLHfCyde81qELyL9zw89aGrZUFmpIiApNbQQ/BewApgHzDSzXsCWqEIdkdqUwiWPwMPj4Ykr4fInPz38FqnNHda9HcatadMFqpaGK9V7jITuw8JQxZs/hG3rQj96CDv6bsPCdSs9T4KjRmrHLodFQ08W3w3U7B/5vpmNjSbSEaxsNIy/B6Z9PRwZXPKIej3E0dKXQht9QVuY+7vQht++DPbsDM93bIT1iw88UQuhCCx8Cub9DkoHwrGfh6IuyebGfqEdP8U3fZFD1dCTxW0J9xw+NTnpr8APgM31LDcO+E8gD/i1u/84xXp/C/RMZvmpuz+UyS+Qc4ZMCG24z94EFV+BCY+GLnRyZHKHDUvCBYOWF5pp8gtCP/WPVwAWduhv/xFadaTf+iqorPx0+cIOobvk2vmhr31BcRiR8vhzwzf/o8fAtqqwztIB4aKsrWtDd0v1QJMm0tCvFw8CC4BLkq+vAB4CLki3gJnlAfcCXwRWAW+Y2TPuXvO67euBRe5+rpl1Ahab2aPuvivD3yO3DP9auHLymRvhsQkw4XcapfRIsa0qDEC2Zl4YlGzF38ONwyH8m+5vpqmtYx9Y+xal2zeGUWkHXBB26GUn139UWPO2p0Wdw0OkCTW0EBzj7hfWeP1vZja3nmVGAkvd/T0AM6sAxnPgAB4OtDEzA4qAjUCa/2lHmGFXhB3H09+Eh8+Hrzyu9txcsbs6DFq2a1vobbNuEax6A1bNDiNU7lfYIYw6efZPw20GdyVCX3ocqrdAh97h5uOWF0am3LeXV16azujTk1e8dhmYlV9PJFMNLQQ7zOxkd/8bgJmNBnbUs0x3YGWN16uAUbXmuQd4BlgNtAEudfd9DcyU+4ZeFo4Epl4D//1FuOThnLw70WeKexgHZ+PyMNDYlg/DY/OHYUyc6s2h/b7mkMMQrn7tXg7lV0H34dDthMwvgMprzu4WxYfvdxFpIuZefy9QMxsCPAy0TU76GLjK3efXsczFwJnufm3y9RXASHe/scY8FwGjgZuBY4D/AYa4+5Za65oITAQoLS0tr6ioaPAvWFMikaCoqOmvbmy7aSH9F91J8z3bWNLn66zt+oWcyVafXM21betmjtq1jGb7dlJQvZ4OG+eQv3srLXZtouWuAy+m2t28iJ0tO1Jd0Jnd+W3Ym9eaqo7l7GrRnha7NrK9VU92FpSk+aTM5Or2gtzNplyZaWyusWPHznH34anea2ivoXnAEDMrTr7eYmY3AWkLAeEI4Kgar3sQvvnX9DXgxx6q0VIzW06418HrtT7/AeABgOHDh/uYRo4RXllZSWOXPTRj4AsXw9Rr6Lf4l/RrsRbO/H8HtAVnL1vdsp7LPTThVG8Og569+zzs28OOpS9TWL3u0/k69YOSY8KJ2B4jwjhQxd2huBv5LVqTT2h73K9HRHGzvr3qkKvZlCszUeTKqC9arW/qNwN31TH7G0AfM+sNfAhMAL5Sa54PgNOBl82sFDgOeC+TTEeMos5wxR9g5p0w86dhhzbq62HQurgPYb1mHuzZFfrTL3gyDIPQbShU/jgMilbzBG3rTlDQluqCzhSO/3kYHqFlG2jfK2vxRY50h9Ipuc6+be6+x8xuAJ4ndB99MDly6XXJ9ycDPwSmmNlbyfXd6u4bDiFTbmuWB2MmwcCL4KU7QkF45Zcw9HLa7jka9pwUj4vQ9u2FhdPC1bCrZodhjVNp1RFO/GYYDrmgbRguoedJ0CyPeZWVjDl+TJPGFvmsOpRCUO/JBXefThibqOa0yTWerwbOOIQMR6aSY+HS34b+6a/cDW8+wgl7d8GCH4arRctODRendRl85I/YuGt76JWz4d3QRz6xLvTYWZfsPFbYHj7/f0Kvm49XQL8vhUKxclZ4rp5WIpGrsxCY2VZS7/AN0FVSh6qkD5z3S/jiD3nrT//FoKJNsPxlmPHvyRksdF/sOjj87HhsaALJaxmGGM6lneTePbBxWdjBr3sbPloYfn68PIyKuV9+K+hwDFz8m7CjT3elbKe+TZNbROouBO6u2/w0hcJ2VJWMgv0ngLZvhJWvw5q5YYCxD2bBW78/eLlWJWHYguYtQmHoMjicNC3uljxR2jV0izyczU3uoRtmzZ39urdhw+Jw0xEIfes7HBO6yg66KOTqfHzos3+kH+GIfAZp4JJc1KoDHDcuPPbbtT184960MuxwP14R2tjzC0Mx2LwynICe++jB62vdqUZx6BbuNdvpuPBo1yucu6ht9w7Yu5vC7WvgnemhOWf1m2Gnvyvx6XzFPcJO/tjPQ+f+4XlJXw2rIXIEUSE4UrRoFcaf6TIo/Tzu4daCW9YkL6RaXeOiqtXhTlMfvAo7Pv50GWsGLYvDydiC4vB80wehsJC8AvB1QpNOt2Ew9CvJHX5/6NwvLCciRzQVgs8Ss+QOvW3YSaezY1M4Ub3+nTDOffWW0E9//6PHiHCzneYteeeD9fT73Fnhqts49GgSiSEVgjgqbAdHjQiPeqzdXUm/Xp+LPpOIZE2zbAcQEZHsUiEQEYk5FQIRkZhTIRARiTkVAhGRmFMhEBGJORUCEZGYUyEQEYk5FQIRkZhTIRARiTkVAhGRmFMhEBGJORUCEZGYi7QQmNk4M1tsZkvNbFKaecaY2VwzW2hmf40yj4iIHCyyYajNLA+4F/gisAp4w8yecfdFNeZpB9wHjHP3D8ysc1R5REQktSiPCEYCS939PXffBVQA42vN8xXgKXf/AMDd10WYR0REUoiyEHQHVtZ4vSo5raa+QHszqzSzOWZ2ZYR5REQkBXP3aFZsdjFwprtfm3x9BTDS3W+sMc89wHDgdKAQeBU4x93frbWuicBEgNLS0vKKiopGZUokEhQVFTVq2ajlajblykyu5oLczaZcmWlsrrFjx85x9+Ep33T3SB7AScDzNV7fBtxWa55JwB01Xv83cHFd6y0vL/fGmjFjRqOXjVquZlOuzORqLvfczaZcmWlsLmC2p9mvRtk09AbQx8x6m1kLYALwTK15ngZOMbPmZtYKGAW8HWEmERGpJbJeQ+6+x8xuAJ4H8oAH3X2hmV2XfH+yu79tZn8G5gP7gF+7+4KoMomIyMEiKwQA7j4dmF5r2uRar+8E7owyh4iIpKcri0VEYk6FQEQk5lQIRERiToVARCTmVAhERGJOhUBEJOZUCEREYk6FQEQk5lQIRERiToVARCTmVAhERGJOhUBEJOZUCEREYk6FQEQk5lQIRERiToVARCTmVAhERGJOhUBEJOZUCEREYi7SQmBm48xssZktNbNJdcw3wsz2mtlFUeYREZGDRVYIzCwPuBc4C+gPXGZm/dPM9xPg+aiyiIhIelEeEYwElrr7e+6+C6gAxqeY70ZgKrAuwiwiIpJGlIWgO7CyxutVyWmfMLPuwJeByRHmEBGROpi7R7Nis4uBM9392uTrK4CR7n5jjXl+D/zM3WeZ2RTgWXd/MsW6JgITAUpLS8srKioalSmRSFBUVNSoZaOWq9mUKzO5mgtyN5tyZaaxucaOHTvH3YenfNPdI3kAJwHP13h9G3BbrXmWAyuSjwSheej8utZbXl7ujTVjxoxGLxu1XM2mXJnJ1VzuuZtNuTLT2FzAbE+zX22ecVlpuDeAPmbWG/gQmAB8pVYR6r3/eY0jgj9EmElERGqJrBC4+x4zu4HQGygPeNDdF5rZdcn3dV5ARCQHRHlEgLtPB6bXmpayALj71VFmERGR1HRlsYhIzKkQiIjEnAqBiEjMqRCIiMScCoGISMypEIiIxJwKgYhIzKkQiIjEnAqBiEjMqRCIiMScCoGISMypEIiIxJwKgYhIzKkQiIjEnAqBiEjMqRCIiMScCoGISMypEIiIxJwKgYhIzEVaCMxsnJktNrOlZjYpxfuXm9n85OMVMxsSZR4RETlYZIXAzPKAe4GzgP7AZWbWv9Zsy4HT3H0w8EPggajyiIhIalEeEYwElrr7e+6+C6gAxtecwd1fcfePky9nAT0izCMiIilEWQi6AytrvF6VnJbONcBzEeYREZEUzN2jWbHZxcCZ7n5t8vUVwEh3vzHFvGOB+4CT3b0qxfsTgYkApaWl5RUVFY3KlEgkKCoqatSyUcvVbMqVmVzNBbmbTbky09hcY8eOnePuw1O+6e6RPICTgOdrvL4NuC3FfIOBZUDfhqy3vLzcG2vGjBmNXjZquZpNuTKTq7ncczebcmWmsbmA2Z5mvxpl09AbQB8z621mLYAJwDM1ZzCznsBTwBXu/m6EWUREJI3mUa3Y3feY2Q3A80Ae8KC7LzSz65LvTwa+D3QE7jMzgD2e7tBFREQiEVkhAHD36cD0WtMm13h+LXBtlBlERKRuurJYRCTmVAhERGJOhUBEJOZUCEREYk6FQEQk5lQIRERiToVARCTmVAhERGJOhUBEJOZUCEREYk6FQEQk5lQIRERiToVARCTmVAhERGIusltVRsXM1gPvN3LxEmDDYYxzOOVqNuXKTK7mgtzNplyZaWyuXu7eKdUbR1whOBRmNjtXb3yTq9mUKzO5mgtyN5tyZSaKXGoaEhGJORUCEZGYi1sheCDbAeqQq9mUKzO5mgtyN5tyZeaw54rVOQIRETlY3I4IRESkltgUAjMbZ2aLzWypmU3KYo6jzGyGmb1tZgvN7NvJ6XeY2YdmNjf5ODsL2VaY2VvJz5+dnNbBzP7HzJYkf7bPQq7jamyXuWa2xcxuysY2M7MHzWydmS2oMS3tNjKz25J/c4vN7MwmznWnmb1jZvPNbJqZtUtOLzOzHTW22+QmzpX2362ptlcd2R6vkWuFmc1NTm+SbVbH/iHavzF3/8w/gDxgGXA00AKYB/TPUpauwLDk8zbAu0B/4A7glixvpxVASa1p/wFMSj6fBPwkB/4t1wK9srHNgFOBYcCC+rZR8t91HtAS6J38G8xrwlxnAM2Tz39SI1dZzfmysL1S/rs15fZKl63W+z8Dvt+U26yO/UOkf2NxOSIYCSx19/fcfRdQAYzPRhB3X+Pu/0g+3wq8DXTPRpYGGg/8Jvn8N8D52YsCwOnAMndv7EWFh8TdZwIba01Ot43GAxXuvtPdlwNLCX+LTZLL3V9w9z3Jl7OAHlF8dqa56tBk26u+bGZmwCXAY1F9fppM6fYPkf6NxaUQdAdW1ni9ihzY+ZpZGXAC8Fpy0g3Jw/gHs9EEAzjwgpnNMbOJyWml7r4Gwh8p0DkLuWqawIH/ObO9zSD9Nsqlv7t/Ap6r8bq3mb1pZn81s1OykCfVv1suba9TgI/cfUmNaU26zWrtHyL9G4tLIbAU07LaXcrMioCpwE3uvgW4HzgGGAqsIRyWNrXR7j4MOAu43sxOzUKGtMysBXAe8PvkpFzYZnXJib87M/sesAd4NDlpDdDT3U8AbgZ+Z2bFTRgp3b9bTmyvpMs48AtHk26zFPuHtLOmmJbxNotLIVgFHFXjdQ9gdZayYGb5hH/kR939KQB3/8jd97r7PuBXRHhInI67r07+XAdMS2b4yMy6JnN3BdY1da4azgL+4e4fQW5ss6R02yjrf3dmdhXwJeByTzYqJ5sRqpLP5xDalfs2VaY6/t2yvr0AzKw5cAHw+P5pTbnNUu0fiPhvLC6F4A2gj5n1Tn6rnAA8k40gybbH/wbedvef15jetcZsXwYW1F424lytzazN/ueEE40LCNvpquRsVwFPN2WuWg74lpbtbVZDum30DDDBzFqaWW+gD/B6U4Uys3HArcB57r69xvROZpaXfH50Mtd7TZgr3b9bVrdXDV8A3nH3VfsnNNU2S7d/IOq/sajPgufKAzibcAZ+GfC9LOY4mXDoNh+Ym3ycDTwCvJWc/gzQtYlzHU3ofTAPWLh/GwEdgZeAJcmfHbK03VoBVUDbGtOafJsRCtEaYDfh29g1dW0j4HvJv7nFwFlNnGspof14/9/Z5OS8Fyb/jecB/wDObeJcaf/dmmp7pcuWnD4FuK7WvE2yzerYP0T6N6Yri0VEYi4uTUMiIpKGCoGISMypEIiIxJwKgYhIzKkQiIjEnAqBSC1mttcOHO30sI1WmxzFMlvXO4ik1DzbAURy0A53H5rtECJNRUcEIg2UHJ/+J2b2evJxbHJ6LzN7KTmI2ktm1jM5vdTCfQDmJR+fS64qz8x+lRxv/gUzK8zaLyWCCoFIKoW1moYurfHeFncfCdwD3JWcdg/wsLsPJgzsdndy+t3AX919CGHc+4XJ6X2Ae919ALCJcNWqSNboymKRWsws4e5FKaavAD7v7u8lBwZb6+4dzWwDYZiE3cnpa9y9xMzWAz3cfWeNdZQB/+PufZKvbwXy3f3fm+BXE0lJRwQimfE0z9PNk8rOGs/3onN1kmUqBCKZubTGz1eTz18hjGgLcDnwt+Tzl4BvAJhZXhOP+S/SYPomInKwQkvetDzpz+6+vwtpSzN7jfAl6rLktG8BD5rZd4D1wNeS078NPGBm1xC++X+DMNqlSE7ROQKRBkqeIxju7huynUXkcFLTkIhIzOmIQEQk5nREICIScyoEIiIxp0IgIhJzKgQiIjGnQiAiEnMqBCIiMff/AdXtfE+eYoaHAAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAA+p0lEQVR4nO3dd3hUZdr48e8zk8mkTHqHAEnoVUooKmBiBfUVXVFRFhXXZW27uu6+q1t+6r4v++rq7lp2VRa7LhoVGxYQQSIo0oKU0AkghJpQEia9PL8/nkkhJCEJmZmQuT/XNRdnzjlzzp2TcO556lFaa4QQQvgui7cDEEII4V2SCIQQwsdJIhBCCB8niUAIIXycJAIhhPBxft4OoLWio6N1UlJSmz5bVFREcHBw+wbUDjpqXNBxY5O4Wkfiap3OGFdWVla+1jqm0Y1a63PqNWLECN1WS5YsafNn3amjxqV1x41N4modiat1OmNcwBrdxH3VbVVDSqlXlVJHlFLZZ9hvpFKqSik12V2xCCGEaJo72wheByY0t4NSygr8FfjSjXEIIYRohtsSgdZ6KXDsDLv9EvgAOOKuOIQQQjTPa43FSqmuwHXAxcBIb8UhhPCOiooKHA4HW7Zs8XYopwkLCztn4woICCAxMRGbzdbi4yrtxrmGlFJJwGda60GNbHsf+LvWeoVS6nXXfnObOM4MYAZAXFzciIyMjDbF43Q6cTgcbfqsO3XUuKDjxiZxtU5HjMvhcBATE0NERARKKW+Hc4qqqiqsVqu3wzjNmeLSWlNQUMDhw4dxOp2nbEtPT8/SWqc2+UF3vYAkILuJbbuBPa6XE1M9dO2Zjim9hjyro8YmcbVOR4xr8+bNuqCgwNthNKqwsNDbITSqJXFVV1frzZs3n7aeZnoNea1qSGudXLNcr0TwsbfiEUJ4XkcrCXQGbbmmbksESql3gDQgWimVCzwK2AC01rPcdd4zeev7PYQG2pg0tKu3QhBCiA7FbYlAa31zK/a93V1xNPTOqn10CQ+QRCCEEC4+N9eQ3WahrLLa22EIIbzsxIkTvPDCC63+3JVXXsmJEyfaPyAv8rlE4G+VRCCEaDoRVFVVNfu5L774gvDwcDdF5R3n3KRzZ8tus1JQUuHtMIQQ9fz5001sPlDYrscc0CWUR/9rYJPbH374YXJychg6dCg2mw2Hw0FCQgLr1q1j5cqVXHvttezbt4/S0lLuv/9+ZsyYAUBSUhJr1qzB6XQyceJExo4dy/Lly+natSuffPIJgYGBjZ4vLS2NYcOGkZWVRV5eHm+++SaPP/44Gzdu5KabbmLmzJkATZ534cKF/OlPf6KyspKePXvy2muvtVuXYJ8sEZRLiUAIn/fEE0/Qs2dP1q1bx1NPPcWqVav4y1/+wubNmwF49dVXycrKYs2aNTz33HMcPXr0tGPs2LGDe++9l02bNhEeHs4HH3zQ7Dn9/f1ZunQpd911F5MmTeL5558nOzub119/vfb4jZ03Pz+fmTNnMm/ePNauXUtqair/+Mc/2u1a+F6JwM9CeWXzRT8hhGc1983dU0aNGkVycm2vdp577jk++ugjAPbt28eOHTuIioo65TPJyckMHToUgBEjRrBnz55mz3HNNdcAMHjwYAYOHEhCQgIAKSkp7Nu3j6ioqEbPm5+fz+bNm7n88suxWCyUl5dz/vnnt8ePDfhoIpA2AiFEQ/Xn+V+2bBmLFi3i+++/JygoiLS0NEpLS0/7jN1ur122Wq2UlJQ0e46a/S0WyymftVgsVFZWkpmZ2eh5tdZcdtllzJ49m5CQkLP9UU/je1VDflI1JISAkJAQTp482ei2wsJCIiIiCAoKYuvWraxYscIjMRUUFDR63jFjxvDdd9+Rk5MDQHFxMdu3b2+380qJQAjhk6KiorjwwgsZNGgQgYGBxMXF1W679NJLeeONNxgyZAh9+/ZlzJgxHolpwoQJzJo167TzxsTE8Prrr3PHHXdQWVkJwMyZM+nTp0+7nNfnEoGUCIQQNd5+++1G19vtdubPn9/otpp2gOjoaLKz65679dvf/rbZc2VmZtYup6WlkZaW1ui2ps578cUX880330jVUHvw97NQXiWJQAghavhcicDuZ6WqWlNZVY2f1efyoBDCze69916+++67U9bdf//9TJ8+3UsRnZnPJQJ/P3PzL5dEIIRwg+eff97bIbSaz90J7TWJQNoJhBAC8MFEUFMikJ5DQghh+F4isEqJQAgh6vO5RGC3med9lsk0E0IIAfhgIqgpEUjVkBCiNWpm+jxw4ACTJ09udJ+0tDTWrFnjybDahc8lArtNqoaEEG3XpUsX5s6d6+0w2pXPdR+1S4lAiI5n/sNwaGP7HjN+MEx8osnNDz30ED169OCee+4B4LHHHkMpxdKlSzl69ChVVVXMnDmTSZMmnfK5PXv2cPXVV5OdnU1JSQnTp09n8+bN9O/f/4yTzjkcDu69914WLVpEREQE//d//8fvfvc79u7dyzPPPMM111zDnj17mDZtGkVFRQD861//4oILLgDg2Wef5ZNPPqGsrIzrrruOP//5z2dzhWr5XInAX7qPCiGAKVOm8O6779a+f++995g+fTofffQRy5YtY8mSJfzmN79Ba93kMV588UWCgoLYsGEDf/zjH8nKymr2nEVFRaSlpZGVlUVISAh/+tOf+Oqrr/joo4945JFHAIiNjeWrr75i7dq1vPvuu/zqV78CzINpcnJyWLVqFevWrSMrK4ulS5e2w5XwxRKBX01jsSQCITqMZr65u8uwYcM4cuQIBw4cIC8vj4iICBISEvj1r39NZmYmfn5+7N+/n8OHDxMfH9/oMZYuXVp7ox4yZAhDhgxp9pz+/v5MmDABMM8ksNvt2Gw2Bg8eXDuHUUVFBffddx/r1q3DarXWzjK6cOFCvv76a4YNGwaA0+lkx44djB8//qyvhc8lAikRCCFqTJ48mblz53Lo0CGmTJnCnDlzyMvLY+nSpURGRpKUlNTocwjqU0q1+Hw2m612//rPJKh5HgHA008/TVxcHOvXr6e6upqAgAAAtNY8+OCD3H///W35UZvltqohpdSrSqkjSqnsJrZPVUptcL2WK6XOc1cs9dWOLD7DA6qFEJ3flClTyMjIYO7cuUyePJmCggJiY2Ox2WwsWbKEH3/8sdnPjx8/njlz5gCQnZ3Nhg0bzjqmgoICEhISsFgsvPXWW1S57lVXXHEFb731Fk6nE4D9+/dz5MiRsz4fuLeN4HVgQjPbdwMXaa2HAP8LzHZjLLVqRxZXSIlACF83cOBATp48SdeuXUlISGDq1KmsWbOGiy66iDlz5tCvX79mP3/33XfjdDoZMmQITz75JKNGjTrrmO655x7eeOMNxowZw/bt22ufnHb55Zdzww03cP755zN48GAmT57c5IN1WsttVUNa66VKqaRmti+v93YFkOiuWOqrP+mcEEJs3FjXWyk6Oprvv/+ekydPnjbvf8038aSkpNrnEAQGBpKRkdHic9UcA0wvpca29e7d+5SSxeOPP167fM899/DQQw+1+HwtpZprET/rg5tE8JnWetAZ9vst0E9rfWcT22cAMwDi4uJGtObC1+d0OrEGBHP3omJu6uvPxGRbm47T3pxOZ+1glY6mo8YmcbVOR4wrLCyM5ORkrFart0M5TVVV1Tkd186dOykoKDhlXXp6epbWOrWx/b3eWKyUSgd+Boxtah+t9WxcVUepqam6/pN9WiMzM5Pzx46DRQvonpRMWlqvNh2nvWVmZtLWn8ndOmpsElfrdMS4tmzZgtVqdcsTt85WYyWC1hg9ejRlZWWnrHvrrbcYPHiwR+IKCAio7V3UEl5NBEqpIcDLwESt9VFPnFOmmBCi43BnjYQ3rVy50mvnbss19dqAMqVUd+BDYJrWersHz4u/n0UmnRPCywICAigoKOi0ycAbtNYcPXq0tstpS7mtRKCUegdIA6KVUrnAo4ANQGs9C3gEiAJecPWrrWyq/qq92a3yAHshvC0xMZH169ef0oDaUZSWlrb6ZuoJLYkrICCAxMTW9b1xZ6+hm8+w/U6g0cZhd7PbJBEI4W02mw2n00lqqke+/7VKZmZmq+rYPcVdcfncXENg2gmkjUAIIQyfTAR2m1VKBEII4eKTicCUCKSxWAghwFcTgZ+0EQghRA2fTAR2P4tMMSGEEC4+mQj8/Swy6ZwQQrj4bCKQEoEQQhg+mQjsUiIQQohaPpkI/P2sUiIQQggXn0oElqoyqK42jcXSa0gIIQBfSgQb5zJu2U1wYo9MOieEEPX4TiII745CQ952mWJCCCHq8Z1EEN3H/Ju/DbtNEoEQQtTwnUQQGE6ZfwTkbaudhlrmQRdCCF9KBEBxUKJJBDbzzM+KKkkEQgjhY4mgG+Rvx9+iAKTBWAgh8LFEUBScCGWFhFflA0gXUiGEwMcSQXFQNwCiS/cAyKAyIYTA5xKBeY5nRPEeAJlmQggh8LFEUO4fAQFhRBTtNu+lRCCEEL6VCFAKYvoTVrgNgJJyaSwWQgi3JQKl1KtKqSNKqewmtiul1HNKqZ1KqQ1KqeHuiuUUXYYSWrAFC9VsyD3hkVMKIURH5s4SwevAhGa2TwR6u14zgBfdGEudLsOwVJYwLuIYX2894pFTCiFER+a2RKC1Xgoca2aXScCb2lgBhCulEtwVT62EoQBcG5vH8pyjUj0khPB5yp3TLCilkoDPtNaDGtn2GfCE1vpb1/vFwENa6zWN7DsDU2ogLi5uREZGRpvicTqdOIIDGbfsFrLDL2bSgZ/ywHA7Q2P92nS89uJ0OnE4HF6NoSkdNTaJq3UkrtbpjHGlp6dnaa1TG9vmzTugamRdo1lJaz0bmA2Qmpqq09LS2nTCzMxM0tLSYPcwBlfnE+RvJd8/jrS0wW06XnupjasD6qixSVytI3G1jq/F5c1eQ7lAt3rvE4EDHjlzl2FYDm1kRLcQNu4v9MgphRCio/JmIpgH3OrqPTQGKNBaH/TImROGQmUJw4Ly2X+8xCOnFEKIjsptVUNKqXeANCBaKZULPArYALTWs4AvgCuBnUAxMN1dsZwmMgWAXv7HyHf6UVpRRYBrRlIhhPA1bksEWuubz7BdA/e66/zNCokDoKtfIRDLgRMlpMR0vIYhIYTwBN8aWVzDYRJBrDoOQK5UDwkhfJhvJgI/OwRFEVF1FID9JyQRCCF8l28mAoCQBILK8rBalDQYCyF8mg8ngngszkPEhwZIiUAI4dN8NxE44uHkYbpGBEqJQAjh03w3EYTEg/Mw3cL8pUQghPBpvp0IdBW9HKUcLCihQh5SI4TwUT6cCMxEpyn2k1RrOFRQ6uWAhBDCO3w+EXS1mbmGpHpICOGrfDgRmEFlcZhBZdJgLITwVb6bCFyji8NlUJkQwsf5biKw2iA4BlvxYaIddikRCCF8lu8mAjA9hwoPkhgRKCUCIYTP8u1EEDcIcleTGG4n93ixt6MRQgiv8O1EkJIOJccY7r+PAydKqa523/ObhRCio/LxRHARAOeV/0B5VTX5zjIvBySEEJ7n24kgJB5iB5BcuAqAXGknEEL4IN9OBAAp6YTnr8VOufQcEkL4JEkEKRdhqSpjmGWn9BwSQvgkSQTxgwEY7H9ASgRCCJ8kiSAkAfxDOM9+WLqQCiF8klsTgVJqglJqm1Jqp1Lq4Ua2hymlPlVKrVdKbVJKTXdnPE0ECTF96an2c1BmIBVC+CC3JQKllBV4HpgIDABuVkoNaLDbvcBmrfV5QBrwd6WUv7tialJMXxIr93GoUBKBEML3uLNEMArYqbXepbUuBzKASQ320UCIUkoBDuAYUOnGmBoX3YeQyqNUFx+ntKLK46cXQghvUlq7ZzStUmoyMEFrfafr/TRgtNb6vnr7hADzgH5ACHCT1vrzRo41A5gBEBcXNyIjI6NNMTmdThwOx2nro/JXMzh7Jj8pe4ybxp5HXLBnm06aiqsj6KixSVytI3G1TmeMKz09PUtrndroRq21W17ADcDL9d5PA/7ZYJ/JwNOAAnoBu4HQ5o47YsQI3VZLlixpfMPRHK0fDdX//Yff6OU789t8/LZqMq4OoKPGJnG1jsTVOp0xLmCNbuK+6s6vvrlAt3rvE4EDDfaZDnzoinOnKxH0c2NMjQvvQbXVTi91gMPSTiCE8DHuTASrgd5KqWRXA/AUTDVQfXuBSwCUUnFAX2CXG2NqnMWKjupNb5UrPYeEED7HbYlAa10J3Ad8CWwB3tNab1JK3aWUusu12/8CFyilNgKLgYe01vnuiqk51viBDLDslRKBEMLn+DW3USkVqrUubGJbd6313uY+r7X+AviiwbpZ9ZYPAJe3PFw3ShhK3IZ3KTqaCwz0djRCCOExZyoRZNYsKKUWN9j2cXsH41VdhgEQenyTlwMRQgjPOlMiUPWWI5vZdu6LH0w1injnFm9HIoQQHnWmRKCbWG7s/bnN7uBYYBIpFTuorKr2djRCCOExzbYRALFKqQcx3/5rlnG9j3FrZF5QGDGQwcXfkucsIyEs0NvhCCGER5ypRPASZsSvo95yzfuX3Rua51XEDSVWnSD/4I/eDkUIITym2RKB1vrPTW1TSo1s/3C8y6/bCPgBqncthX6eH9cmhBDe0KpxBEqpAUqp/1FK7QBedFNMXhOcMoZ91THE7XzP26EIIYTHnDERKKV6KKUeVkqtB94C7gEu001NXnQOi3QE8E5VOvHHVkP+Tm+HI4QQHtFsIlBKLccMCLMBk7XWI4CTWus9HojN4/z9LMy3XUoVVsh6zdvhCCGER5ypRJCHaRyOo66XUOfqNtqAcsSxOXgkbF/g7VCEEMIjmk0EWutJwGBgLfBnpdRuIEIpNcoTwXlDlMOfXao7HP8RquUhNUKIzu+MbQRa6wKt9ata68uAMcCjwDNKqX1uj84LooLt5FTGQHUFFDacNVsIITqfVvUa0lof1lo/p7W+ABjrppi8Ksrhz/byKPPm+B6vxiKEEJ5wptlHGz4/oKFr2jGWDiHKYWdpSSTYgeO7IXmct0MSQgi3OtMUE+cD+4B3gJV0tonmGhHt8OeAjkJb/FBSIhBC+IAzJYJ44DLgZuAW4HPgHa11p52rOSrYThVWKhxd8ZdEIITwAWfqNVSltV6gtb4N01C8E8hUSv3SI9F5QZTDH4Ci4G5wbLeXoxFCCPc7U4kApZQduApTKkgCngM+dG9Y3hPtSgQn7F2JOPyVl6MRQgj3O1Nj8RvAIGA+8GetdbZHovKiqGA7AHl+CSSXHIPSAggI83JUQgjhPmcqEUwDioA+wK+Uqm0rVoDWWoe6MTavCAu0YbUoDlrizIrjeyDhPK/GJIQQ7nSmaahbNc6gM7BYFFHB/uyplkQghPANbr3RK6UmKKW2KaV2KqUebmKfNKXUOqXUJqXUN+6Mp6WiHHZ2VESbN9JgLITo5M7YWNxWSikr8Dym+2kusFopNU9rvbnePuHAC8AErfVepVSsu+JpjWiHP7nFlRAUJaOLhRCdnjtLBKOAnVrrXVrrciADmNRgn1uAD7XWewG01kfcGE+LJUYEsSvPiQ5PkkQghOj0lNbumVVaKTUZ803/Ttf7acBorfV99fZ5BvOsg4GY6a6f1Vq/2cixZgAzAOLi4kZkZGS0KSan04nD4TjjfktzK3g1u5xvurxAbPFOVo75d5vO195xeUNHjU3iah2Jq3U6Y1zp6elZTT5QTGvtlhdwA/ByvffTgH822OdfwAogGIgGdgB9mjvuiBEjdFstWbKkRfttO1Soezz0md78n99q/ViE1pUVbT5ne8blDR01NomrdSSu1umMcQFrdBP3VXdWDeUC3eq9TwQazuucCyzQWhdprfOBpYDXu+j0jHHgsPuxpTQSdBUU5no7JCGEcBt3JoLVQG+lVLJSyh+YAjSczfQTYJxSyk8pFQSMBra4MaYWsVoUQxLDWF3gGiYhPYeEEJ2Y23oNaa0rlVL3AV8CVuBVrfUmpdRdru2ztNZblFILgA1ANaYqqUOMXh7aLZxPl4aCP9JgLITo1NyWCAC01l8AXzRYN6vB+6eAp9wZR1uc1y2cWdXhVFv8sRyXEoEQovPyuZHDLTW8ewTVWCgI6CIlAiFEpyaJoAkxIXaSo4PJ1bGQtw3c1M1WCCG8TRJBM0YlRfJxyVDI2wrL/ubtcIQQwi0kETRjZHIkr5ReRGHv6+Drv8Ceb70dkhBCtDtJBM0YlRQJKD5LehiCo+H7570dkhBCtDtJBM3oFhlIXKidFXtLYNhPYfsCKNjv7bCEEKJdSSJohlKKMSlRfLM9j8IBt4CuhrWnTYUkhBDnNEkEZ3B3Wk9Ollbwt9Xl0PNikwiqKr0dlhBCtBtJBGfQLz6UaWN68J8VP7K/181w8gDsWNj4zs48OLAOCmRuIiHEuUMSQQvcf2kfNPDByUHgiIes107f6cA6eGYwzL4I/jkCSo57OkwhhGgTSQQtEBnsT9+4ENbkOmH4NNjxFWx4D7Z+bl6b50HGVPNEs8tnQmWpdDUVQpwz3DrXUGcyrHsEn204QPW107B8+wx8+PNTd/ALhDvmQ9wgyHwCcpZA///ySqxCCNEakghaaHj3cN5ZtZecikh6/+oHKDl26g4hXcARY5aTxsKuJZ4PUggh2kASQQsN7xEBwNq9x+k9sjuEd2t655R0M+bg+I8Q0cNDEQohRNtIImih5KhgwgJtrP3xBDeN7N78zilp5t8v/wDRvc3ywJ9AwhC3xiiEEG0hiaCFLBbFsO7hZO1tQW+gmL7QNdV0M92xEKoqTOPxnYvcH6gQQrSS9BpqhbG9otl5xMnu/KLmd1QKfr4Y/l+eeV0+E3JXw6EO8fA1IYQ4hSSCVrhycAIAn2840LoPDr0FrPbGxx8IIYSXSSJohS7hgaT2iOCzDQdb98GgSBh4LfwwB167Cta9febPlByHeb+EvO1tilUIIVpKEkErXT0kga2HTrJu34nWffDCB6D7aHAehk/uhZ2Lm98/6w0zr9E7N0FJK88lhBCtIImgla4cnECQv5Vrn/+O6a+toqq6hY+wjBsAt34Cv/gGYvqbkcjPDoVnh5K6+oFTp7eurjbVSJE94cQ++OwBN/wkQghhSCJopdjQAL58YDwzxqewZFsen29sZTWRfzDckgFDboTEkZCYSnDRj7Dmlbp9dmfC8T2Q9nsY9XMzjUV5cXv+GEIIUcutiUApNUEptU0ptVMp9XAz+41USlUppSa7M5720i0yiIcm9KNXrIN/Lt5BdUtLBTXCu8M1z8H1L8H1L3M0agT88B/TzbS6Cr59BgIjzRQVvS6FqnL4cblbfhYhhHBbIlBKWYHngYnAAOBmpdSAJvb7K/Clu2JxB6tF8cuLe7HjiJO3V+09q2MdTLjCtB1snAtf/y/s/gYu/hPYAqDHBabHkUxZIYRwE3cOKBsF7NRa7wJQSmUAk4DNDfb7JfABMNKNsbjF1UO68O7qffy/T7LRWjPt/KQ2Hedo1HAITYSP7zIrht8GqXeYZVugaWTeldkuMQshRENK61ZWa7T0wKaaZ4LW+k7X+2nAaK31ffX26Qq8DVwMvAJ8prWe28ixZgAzAOLi4kZkZGS0KSan04nD4WjTZ5tSXqV5fl0ZG/KqePT8AJLCrG2KK14fIbRwC1XWYI7EXoi22Gq3d/9xLim73+K7C96gwj8ce2k+torCU45RZo+mwj+06ZNoTVDxPizVlRQHJVJt9W9xbO19zdqDxNU6ElfrdMa40tPTs7TWqY1tc2eJQDWyrmHWeQZ4SGtdpVRju7s+pPVsYDZAamqqTktLa1NAmZmZtPWzzRlzYQUX/y2TefsD+OCaC2juZ2kqrtS0q2vf92+4w8EI+PdbXGhZD13GwDs/N89Prs/fAT/7yvROasznv4HVL5vl826B615scWzuuGZnS+JqHYmrdXwtLncmglyg/hSdiUDDIbmpQIbrxhkNXKmUqtRaf+zGuNpdaICN313Rj999sIF3Vu3jltFnmJSuteKHwLBp8O0/wBZknnmQVq/tvboSvvhvyLgZLv8LqAZNP4c2miQw4nYzJiH7A7jiL2agmxDC57kzEawGeiulkoH9wBTglvo7aK2Ta5aVUq9jqoY+dmNMbjN5RCLz1h/gkU+ySQgLIL1fbPsdXCm46u+Qtw2O7YIpc0zPo/oc8fDG1fDu1MaP0fMSuOof5hibPzajmy+4r/F9hRA+xW2JQGtdqZS6D9MbyAq8qrXepJS6y7V9lrvO7Q0Wi+LFnw7n5pdWcM+ctSx4YBw9ooLb7wR+drj9c6gohsDw07d3Hw0PbISTh07fpiwQOwAsVlN11G00rH7JHLO+5PFm5lQhhE9x6zTUWusvgC8arGs0AWitb3dnLJ4QEmDj5VtHctk/vuH3H25kzp2jW91e0Cw/f/NqMoB48zqT0XfB3OnwxW9PXR87EO7+zpRAhBA+Q55H0M7iwwL4/ZX9+cNHG3nl293cOS7F2yGdbtBPoGc6VFXWrcueCwseNtNldxvlvdiEEB4nU0y4wZSR3bi0fxwzP9/CrG9yvB1O4wIjzDOWa17Dfmp6Hq2RqbKF8DVSInADi0XxwtThPPjeOp6Yv5VRyZEM7x7h7bCaZw+BwTfA+ndAV9Wu7nf4MEQchvNugm0LzAjoEbed/vnqalj82OltFJE9YfxvTfuEEKJDkkTgJv5+Fv56/RCW5xzlucU7eH36OVDdMuYe2Ps97FtZuyqiqAA+yoSjO+Hbp6G6wnRhHXLDqZ/N+Rq+exZCu4LVNRhOV8OGd6GqDC55xHM/hxCiVSQRuFGw3Y87xyXz5IJtrNt3gqHdwr0dUvNi+sC9K09ZtXLxQsbnPA5Ln4TIFHDEwbz7YO0bkJgKlzxqGpezXoPgGPjVulMbtD+9H5b9HeIHw8Dr6tZXV8OiR+DAulNjCIqEiU9B2UlY+Ccod0K/q2DM3bD6FZNcRv389NhLTphBc87D5r3VBhc9BN3HtMeVEaJTkzYCN7v1/CQig/25/bVVzFn5I86yyjN/qAOptvqbcQtDpsDNGXDjW9D7cnOD/vZpWPY3KDwA2+bD0Kmn92qa+KTprvrxPac+sznzcVj+T9MdVlfXvbYvhPemwTtTYO9yKD5qGrE/uRc+fxC+/CMUH2sQZBV88DPY/EndcQ5vhnduNtN5CyGaJSUCN3PY/XjvF2P4w4fZ/PGjbP48bzPXDevKQxP7ERncsvl+vC60C/zk33Xvb3oLtIYPZ8DXf4FVL5l2hcbaDvzscOObMDsNMm6BGZmw51tTwhj6U5j0r1O7q2Z/AHPvAIufeZBPt9Hw5iQzTXdMP8jbCuvfoceejfDcr81nKsuhMBeufgZSp5t1R3PgpXT490VNj6BWFlMdNvJn7XCRhDh3SSLwgF6xIWTMGMOqPcf4fMNB3lm1ly83H2L6Bcncen4Pb4fXNkqZZyo4Yk11TMJ5puqoMSHxcNN/4LWJ8PaN5tt611QzWrrhmIVB15uH8ASGQ9JYs+7GN+H75031UMZUyHyC5LJC6DEWQhPMPokj65IAQFRPuOV988CfhvMy1TiaY6qTwntA70vP6nIIcS6TROAhFotiTEoUY1KimDqmO3+dv5WnF23n30tzGJug6DO0hC7hgd4Os3VsgWbOopZITIWrnzZVPI44U6qwBTS+7/Bpp74PjoZLHzXLqdPh47s5FnEekbd+AtZm/oS7jzavppQXwSuXw5zJ0MLZWM9kfHU1LGtQ42q1mevU+3KYc4Ppqjvm7nY5nxDtQRKBF/SLD+W16aPYeqiQf3+zi0/W7efrJ5dwzdAuzBifQt+4kPYdkdxRDPsp+AVA3EBT3dQWg2+A6ko2H41kbHNJoCX8g2Hq+5D1OlSWnd2xXHL37qV79wbzQO1bCZ89CFG9IH8bLPg9hHUzDx1y58R/Wpv2k6auU3VV27r1tvVznZHWdaXa+stN6aDXThKBF/WLD+Xpm4ZyYegxsivieHf1Pj5cu5/QAD9GJUfxi4tSGJnUyWYIHXyWTyO12mD4rVRmZrZLOIR2gfQ/tM+xgF2ZmXRvOE1waSG8fIlJAj95Cb57rm5ywFEz4Mqn2u38taqr4f3b4MhmuGPh6dvztptJCi/6HYy8s+XHXfsmfPUo3PGl6WXmy3K+hnenwX2r4dhu+M/1cN+q0yeErHE0B168EG5+x4zs70AkEXQA0YEWHps4kPsv6c2nGw6w9dBJFmQf4oZZ3zOiRwTTL0wirW8sDrv8us5JAaFw26eQvwOSx0HPi00Pp9zVsGo2hCVCylneGCJTwO4wJZu8bbDxfdgyzzSIz70dR+R1cND1paK60jT0Ow+bpDTiDig9AQHhYLFAyXGwh57+zXXvClOyqa4w05pf+SQUHYXgKLO95ETdhIhNLZ/rtIaifDMaf+di03tu5yLTI66yxDxbvGEicB4xbWnbvzT7rHjRJILiY3UlwuJjZrS/Uqcul5yAgDC3z/8ld5YOJCLYn1tdj7v8f1cN4P2sfcxeuov73v4Bfz8L1w7twkV9YjlYUMLALmGM6BGBn0VhsXTCaqTOpv6EgMHRpqfSiNvNTeWrdhhsF94dps83jekH15l1w35qGtQ/vovU3Ushq97+Fj9TGlk1G5Y/B988aW5OF/8/026SmGqqzWqSQeEB8+03vBtE94H1GeZm+PVMuOafpsrj8wdh6lyTUF69HNL/aKrD3r8Nrvs3DLnx7H9Ob1s3xyTD+9fXjYHJWQKHXV2jD/wA502p2z83C16+GKZ+UPfc8R0LTTJY8HsY9xvocT7MuRFG/8KMmXnzWhh6Mwy/HV6/0uxz0e/c+mNJIuigAv2t3Hp+EreM6k7Wj8eZt/4Ac7NyeW9N7in7WZQZq/CHK/vj7yfDQs4pFqvpTbX7G6iqaPtxSk+Ym9Pzo00D+IQnILo3JKeZ9oHo3mR/v4hBgwbVfSaqpylFZH8Aix41o8W3fmaeja2rIWex6VHV5wqz/9KnzJiP2+aZsR3bF5gkYAsy5wbzuZX/Nt9yqyth8f+YNiFdbW58Q1w9xqJ7144+t1SVQv5OiO7V9p+/vZQXQeFBiO6FparMVJ81rP7a+oUZKZ+zGA6uN+t2LDQlAzDJQWuTjLsMg+3zzfrv/wX7VkGfCaZksOBhsAWbcTgrgk036xUvmKo3q838m/0hVJaa7tljf103Yt8NJBF0cH5WC6NTohidEsVvLu/L/uMlxIXZWfvjcbYdcrLveDGvL9/DguxDhAfZuHpIAjPG95SkcK6wBdTdbM+GssDHd5tv9A17JCWmkh/jhP5pp38u9WemRHDbZ6ar7Yb3zPiNTR+a6p+smkkIlUlasf3NjS5+sCkF3PIuvHGNOX/Pi81nrP7mcahHNsHJwzD0LjP4cOGfzCDCQZPh+pehqpzz1j8C3+WYqrMeF5z9dWirynJ46zrYnwXTPmLwxpnwbbYpFfVydS2uqoQ9y8zy6leg/CQkjatblzTOfH7j+/Dhz+Hmd01pAepKA8OmmRv6vtXwsy9NFV3+DrhzEXz2a1OiuHOxSc67l5mpWRb/D2z9HAZe67YfXxLBOSQy2L92ENqEQQlMcH3Bu7R/HPPW7+dYUTl/W7idD9fu5660nlzWPw6LRbFy11EGdAklMSLIi9ELtxp6C/S6zFTXtEba703iCIqErsPhsv8xVVc9LjBJosrVmyooqq7uWylTDWW1m5Hkd39n1hcfNYmgqgzOv8dUIVWWmsSxYpZJAo54M+V5YDgU5RFWuM10J37vVjMliFKgrNDvavOz7F1hxnmEJpibZ0i8qZ4CqCgxN8tel5gSyK5M8yQ+tGnITUk3pa6di81Dl6z+5pt8jwtMSWbnYtO92B4C839nenc54uDNa4nQVWZ57h2miisi2dTVlxVCYCQcWGtiGPuASQTBMeZ3sGcZLPqz2fbt02a/oVPNZI4o00bU61IzANM/2CTgiiLTJvDTD83xgyJhyttQWmDOueY1k5AlEYjmTBgUz4RBpv55ydYjPPnlNn43dwNg/l9pDSF2P351SW925TvpFx/K1NHd8bNKqaFTaW0SANM4XNNgqZRJAjXLcQOa/pw9pG7ZP7ju3wGTTEKIH2zW1TwFL3W6qUKZvgC+/INJGMCeHjeRdPVv4dUrTn1Q0qrZJjG8f7upwrr8f00bRWgXMzo9KMqUgDZ9BGMfNA3f6+bA6LvNTXbVbPPtOyjSTIY4aLJJTJn/B32vNMlg4Z9Msuh3lbnRjn0QzrsZXpvAvshxdPvJYyau+a76+fghgIJxD5rP+gWY6rfu55snAHYZbvYrzIXQRNi3wrwf9lNTrVZaaG7s9dV/2JTVr+53YbHWLQ+/FZb+zbQnuYkkgk4mvV8saX1jWLn7GNn7CzhZWsnQ7uH8Y+F2/vLFFgJsFkor9vGfFT8yMjmS8b1juHxAnDQ4i/Yx+dXG11/xf3D5THOD+8ls895iZc+qDSTF9IEHN5uJBsFUr2TcYhqZI3vCiR/N+4gkM8352zea5LDpI4jpD9/+w3wupj+sfLFu+Ye36paz59Ytb/vCvGL6myqbXUtMaeriP5n4frONnGXf0S0yGe7fYL6Zf3y3KU10GQYDf2ISQfxgc/O+/XNAAdrU+1dXwJT/wOx0U/LomgrXj3bt0wajfm5KZzW9s9xAEkEnpFTdKOYaF/SMYldeEb1iHSzcdJjXvtvNZ+sP8PbKvaREBzNjfAoDuoRysKCUruGBpMQEE+Qvfx6ilZoaLFVT5VOz3LD0Ygs0L4C+E81khateMnX0u5fCd8+YOveD6+DTB0xD7YjbzX4ZU82352v+ZR7Bqixw/SvwyT3mJn7Tf0xp4/iPpk3jq0fh0AbTw2npU6Y65/qX62Kv3yhrCzCvya+YOa+G3ARhXaH3FXVToNT/mQdea7redhkGw6bWVZ+djUD3P8tE/qf7CLuflf4JoQBcNSSBq4YkUFWtmZ99kFnf5PDwhxtP+0yPqCCu7VHFRVpTUFJBWKCtc454Fh3PqJ/XTTceMa1u2pHoXqcPSvzp3LrlKXPqlq9/uW75mn/WLV/5ZN3yZX9uWTyBEfCLpXXvp77X+H7XvlC3POn5lh27A5BE4MOsFsXVQ7pw1eAEVu4+xonichLCAtl/ooScI07mZx/i2bXFvJfzNQcLSrl+eCIzrx1EoH/HGyIvhGg7tyYCpdQE4FnACrystX6iwfapwEOut07gbq31enfGJE5XU5VU4zzXA3R+Pj6FB15ZRGVgGJf0j2XOyr1sPljIi1OH0yMqCK2RtgUhOgG3JQKllBV4HrgMyAVWK6Xmaa0319ttN3CR1vq4UmoiMBtoZrpI4UkBNitT+tlJS0sF4JL+cTyQsY6Jzy7DoqCiSpMUHURKtIOBXUK5pH8c/RM66YR5QnRi7iwRjAJ2aq13ASilMoBJQG0i0Fovr7f/CiDRjfGIs5TeN5bPfjmWZxfvwGH3w+5nISeviO2HT/Ll5kP8/avtJEYEcuXgBG67IImu59q02kL4KKW1ds+BlZoMTNBa3+l6Pw0YrbW+r4n9fwv0q9m/wbYZwAyAuLi4ERkZGW2Kyel04nA42vRZd+qocUHLYyso06w7UsnaI1VszK9CAYkhFnqGWbiml41we/uOWeio10ziah2Jq3XOJq709PQsrXVqY9vcWSJorH6g0ayjlEoHfgaMbWy71no2ptqI1NRUndZwmt8WyszMpK2fdaeOGhe0LrZJrn9zjxczZ+VesvcX8O2uY6zOq2Bot3DsfhaSooJJTYpkfJ/os+qe2lGvmcTVOhJX67grLncmglygW733icCBhjsppYYALwMTtdZH3RiP8JDEiCAemtAPgF15Tv62cBuHCkrJO1nFsh35vPztbux+Fsb1jmZkUiQxIXZyj5cwqGsoF/eL83L0QvgedyaC1UBvpVQysB+YAtxSfwelVHfgQ2Ca1nq7G2MRXpIS4+CFqSNq31dUVbN69zEWbj7Moi2HWbTlyCn7X9QnhnG9o0mJCaZnjIPEiCCs0jNJCLdyWyLQWlcqpe4DvsR0H31Va71JKXWXa/ss4BEgCnjB1dOksqk6LNE52KwWLugVzQW9onnsmoEUlFSQd7KMuFA7b6/cy0vLdvHN9rza/f39LCRHBdMjKqh2RlU/iyKsvIJBzjKiHXZv/ShCdBpuHUegtf4C+KLBuln1lu8EWvGcPNHZhAXaCAs0Q/p/cVFPfnFRT44VlbMrz0lOnpNdeUXk5DnZc7SIymrTxFRUVsnhwnLe2fY114/oyozxPUmODvbmjyHEOU1GFosOx0y3HUlqE89r1lrzn8+WsLUyhvezcslYvY+L+8YyYVA8Vw5OIFge6SlEq8j/GHHOUUrRLcTCtLTBPHBpH177bjcf/bCfxVuPMPPzLVwxMA67n5X4sAB6xjjoFRtMSrRDRkEL0QRJBOKcFhNi53cT+vHfV/Rl7d7jzF66i8VbjlClNSeK6x7/GO2wM75PNH3iQrjmvC50kcFuQtSSRCA6BaUUI3pE8u9pddVJJ0sr2J1fxLZDJ8ncnsfS7fl8uHY/s77J4ZcX92bLwUKOF5UTGmjjltHdGdlEVZQQnZ0kAtFphQTYGJIYzpDEcG5INUNacvKc3DtnLf/72WbCg2wkRgSStfc4H/2wnz5xDlKTIjlSWEpMiJ2UaAdzs3KJCLbVjosoKKnA38/CyKRIbPKEN9FJSCIQPqVnjIOP772QXXlF9Ilz4Ge1UFJexdy1uXyx4SCfrjtAl/BAVuw6hrOskoFdQtl+2Ml1Lyw/5ThdwgIY1j2CfceLqazSOAL86BnjgMIK1i/awXc5+YQF2hjYJZS0vrGUV1bj72dhaLdwjhWVszvfyfDuETJBn+gQJBEInxNgszKgS2jt+0B/K9PG9GDamB6168oqq9h/vITk6GBOFFcwP/sQsSF2IoL9yTtZyuvL97BxfwE9ooKw+1k5XlzOguyDHC+uQG3bzuCuYRwrKmfxlsM8s2hH7XEv6hPD+twTnCiu4JJ+sfRPCMVZVsn4PtF0jwwm2G4lIUzaL4RnSSIQohF2PyspMWZyr4hgf24Z3f2U7RMGJTT6uU8XLuHCCy8kMtg8nvCos4zlOUcJCfAje38B//x6J0MSwxjbK4ZZ3+SQuT0Pm1Xx+vI9tcfoHWuqqGJD7Ow7Xozdz0JsSAD7T5TgZ1F0jwrC31o3uG50ShRdwgPZd6yYympNXKhdkoloFUkEQrSjEH9VmwQAohx2/uu8LgCk9Y3ljrHJBNqsKKWYMT4Fq0Wh0azefZxjxeXknSzj662HmZ99kBPFFcSHBlBeVc2xonJiQ+xUa02+s/yMcQxICGVw1zACbBZyj5cQVFFOYcQBDhWUcHG/WHrGONidX0RCWCCB/lb25BcRbPcjJkRGavsiSQRCeFD9GVfrP/JzbO/o2uWfjU0GqG1XaLhcXF5JlWuU9cnSSpZuz+NESQVJrmk4dh5xsmRrHou3Hqa0opr4sAByjlTwac4PADz15TZ6x4aw+WAhXcMDGdEjgnnrD6AUDO4aRp+4EEIDbGZ6j+ggUmIcxLomBuweGUS3yCAqqqqxKiVjMzoJSQRCdFA1N/6Gy/WTSUiAjSmjTq22urhfHDPG9zxl3acLl9Bz0AhCA/14+qsdbD5YyH9f0Zd56w7w+caDzBifQrC/H9/vymfp9jxKyqsoq6ymvKr6lONYFIzoEcGmA4WEBdq4/YIkhveIYO2Px/lq82HS+8USYLOyIPsgY3vFcNWQBOx+FrqGB2KxKE4UlxMWaEMpxfGicsqr3PM8FNE6kgiE8AEh/qq2gfzvN55Xu37G+BROFFfUVgndT+/abVXVmgMnStiZ5ySvsIwu4YF8uzOfzG1HmDS0K7vynDw+f2vt/inRwTz15bba5acXbefpRWZS4bhQO2GBNrYfdtIvPoQeUUEs3HwYNAzYtIxHrh7A6HrPzRaeJYlACB9ms1qabBewWhTdXFVBNcb2jubhif1q3+87VszOI05iQ+0M7BLGtkMnqaiqZlDXMHYcPkn2gQKKy6v4dkc+haUVTBiUwPyNB1m+8ygzxqVw+MA+1hyt4KbZK0iODiYlOpiUmGBCAmy15+gTF0L/hBC+23mUfGcZNquFpKggwoP8CfK3MrhrmNurqLTWnbqrryQCIUSbNUwUfeNDapd7x4XQO868nzq6rmvug5f1qV3OzDzM47eO443v97Ah9wS78opYtjOf8spTq6Sa0yfOQUSQPxtyCxjeI5zLB8QzqGsYH/2QS3llNTeP6k5VtWZXXhH7jhcTG2KnZ4yD7lFB5DtNA31VtSb3eDEAk0ckkp1fxa5vdzNxcDzPL9nJFxsP8cClvSkqq+KHvcf5yfCuWC0WNu4vIDE8kIhg/9oeXd0jg/CzKDYdKOT15XtYkH2IgV1M431xRRXLduShNUwZ2Y09R4spLKkgrW/saQk50GYlMSKQDfsLyD1ezPjeMbjr0cKSCIQQXhXob+Wui+raNLTWuNrCqayuJmvPcXYccXJBzyhSYhyUVFSxJ7+Ik6WV5B4v5s3vf6SgpIJrh3Vl1e6jPDpvEwB2PwtWi+K9Nbmtiufx+VtdjfGb+Z/PNgMm2TzyiTluZLC/qdZqgp9FERpo41hROQE2C1cMjGfrwZPMWbkXP4tiZHIkztJK/rZwO6EBfoQE2Jo9Xo0nF2zjsh5+pKe36sdpEUkEQogORSmF1VULY7VYax9kVMNh92NQ1zDXu6ja6UNq5OQ5Wb/vBOP7xGCzWPhqy2Eig230jHHQNTyQfGc5OXlO9h4rJirYn/iwACxK0SU8kCMnS3lvdS76xH5uvHQ0767ex9he0VzSP5Zvd+YTEeRP/4RQMrcdqZ1q5HBhKSdLKymrrGJPfjE5eU6OnCxjVFIklw2II6Jed+L69p8oITbEjp9FsTu/iOLyqlO2nyytZO+xInrFOugeGUzmtiMU5LrnQY6SCIQQnUrPGIeZ7sNl8ojEU7bHhwUQHxbAhY18NjLYn0f+awCZmUfonxDKY9cMrN02rndM7fIl/euerd0jqu6hSCN6tHziwq71ZsBNqRdvfef3rGtAvyG1G5nOnBYfvzVk1iwhhPBxkgiEEMLHSSIQQggfJ4lACCF8nFsTgVJqglJqm1Jqp1Lq4Ua2K6XUc67tG5RSw90ZjxBCiNO5LREopazA88BEYABws1JqQIPdJgK9Xa8ZwIvuikcIIUTj3FkiGAXs1Frv0lqXAxnApAb7TALe1MYKIFwp1fhE70IIIdxCuWvIslJqMjBBa32n6/00YLTW+r56+3wGPKG1/tb1fjHwkNZ6TYNjzcCUGIiLixuRkZHRppicTicOR+P9db2po8YFHTc2iat1JK7W6YxxpaenZ2mtUxvb5s4BZY3N0NQw67RkH7TWs4HZAEqpvPT09B/bGFM0kN/Gz7pTR40LOm5sElfrSFyt0xnj6tHUBncmglyg/tjvROBAG/Y5hdY6prntzVFKrWkqI3pTR40LOm5sElfrSFyt42txubONYDXQWymVrJTyB6YA8xrsMw+41dV7aAxQoLU+6MaYhBBCNOC2EoHWulIpdR/wJWAFXtVab1JK3eXaPgv4ArgS2AkUA9PdFY8QQojGuXXSOa31F5ibff11s+ota+Bed8bQwGwPnqs1Ompc0HFjk7haR+JqHZ+Ky229hoQQQpwbZIoJIYTwcZIIhBDCx/lMIjjTvEcejKObUmqJUmqLUmqTUup+1/rHlFL7lVLrXK8rvRDbHqXURtf517jWRSqlvlJK7XD9G+HhmPrWuybrlFKFSqkHvHG9lFKvKqWOKKWy661r8voopX7v+nvbppS6wsNxPaWU2uqaw+sjpVS4a32SUqqk3nWb1eSB3RNXk783L1+vd+vFtEcptc613pPXq6l7g/v/xrTWnf6F6bWUA6QA/sB6YICXYkkAhruWQ4DtmLmYHgN+6+XrtAeIbrDuSeBh1/LDwF+9/Hs8hBkY4/HrBYwHhgPZZ7o+rt/pesAOJLv+/qwejOtywM+1/Nd6cSXV388L16vR35u3r1eD7X8HHvHC9Wrq3uD2vzFfKRG0ZN4jj9BaH9Rar3UtnwS2AF29EUsLTQLecC2/AVzrvVC4BMjRWrd1ZPlZ0VovBY41WN3U9ZkEZGity7TWuzFdpEd5Ki6t9UKtdaXr7QrMYE2PauJ6NcWr16uGUkoBNwLvuOPczWnm3uD2vzFfSQRdgX313ufSAW6+SqkkYBiw0rXqPldR/lVPV8G4aGChUirLNb8TQJx2DfJz/RvrhbhqTOHU/6Devl7Q9PXpSH9zdwDz671PVkr9oJT6Rik1zgvxNPZ76yjXaxxwWGu9o946j1+vBvcGt/+N+UoiaNGcRp6klHIAHwAPaK0LMVNw9wSGAgcxxVNPu1BrPRwzPfi9SqnxXoihUcqMTr8GeN+1qiNcr+Z0iL85pdQfgUpgjmvVQaC71noY8CDwtlIq1IMhNfV76xDXC7iZU79sePx6NXJvaHLXRta16Zr5SiJo9ZxG7qSUsmF+0XO01h8CaK0Pa62rtNbVwEu4qVjcHK31Ade/R4CPXDEcVq6pwV3/HvF0XC4TgbVa68OuGL1+vVyauj5e/5tTSt0GXA1M1a5KZVc1wlHXchamXrmPp2Jq5vfWEa6XH/AT4N2adZ6+Xo3dG/DA35ivJIKWzHvkEa46yFeALVrrf9RbX/85DNcB2Q0/6+a4gpVSITXLmMbGbMx1us21223AJ56Mq55Tvql5+3rV09T1mQdMUUrZlVLJmIcvrfJUUEqpCcBDwDVa6+J662OUeWgUSqkUV1y7PBhXU783r14vl0uBrVrr3JoVnrxeTd0b8MTfmCdawzvCCzOn0XZMRv+jF+MYiym+bQDWuV5XAm8BG13r5wEJHo4rBdMDYT2wqeYaAVHAYmCH699IL1yzIOAoEFZvncevFyYRHQQqMN/Gftbc9QH+6Pp72wZM9HBcOzH1xzV/Y7Nc+17v+v2uB9YC/+XhuJr8vXnzernWvw7c1WBfT16vpu4Nbv8bkykmhBDCx/lK1ZAQQogmSCIQQggfJ4lACCF8nCQCIYTwcZIIhBDCx0kiEKIBpVSVOnXG03abrdY1m6W3xjwI0Si3PqpSiHNUidZ6qLeDEMJTpEQgRAu55qn/q1JqlevVy7W+h1JqsWsitcVKqe6u9XHKPAtgvet1getQVqXUS6455xcqpQK99kMJgSQCIRoT2KBq6KZ62wq11qOAfwHPuNb9C3hTaz0EM7nbc671zwHfaK3Pw8x/v8m1vjfwvNZ6IHACM3pVCK+RkcVCNKCUcmqtHY2s3wNcrLXe5Zoc7JDWOkoplY+ZKqHCtf6g1jpaKZUHJGqty+odIwn4Smvd2/X+IcCmtZ7pgR9NiEZJiUCI1tFNLDe1T2PK6i1XIW11wsskEQjROjfV+/d71/JyzIy2AFOBb13Li4G7AZRSVg/P+y9Ei8k3ESFOF6hcDy93WaC1rulCaldKrcR8ibrZte5XwKtKqf8G8oDprvX3A7OVUj/DfPO/GzPrpRAdirQRCNFCrjaCVK11vrdjEaI9SdWQEEL4OCkRCCGEj5MSgRBC+DhJBEII4eMkEQghhI+TRCCEED5OEoEQQvi4/w8O84tffDu1rgAAAABJRU5ErkJggg==\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "metrics = pd.read_csv(f\"{trainer.logger.log_dir}/metrics.csv\")\n", "\n", "aggreg_metrics = []\n", "agg_col = \"epoch\"\n", "for i, dfg in metrics.groupby(agg_col):\n", " agg = dict(dfg.mean())\n", " agg[agg_col] = i\n", " aggreg_metrics.append(agg)\n", "\n", "df_metrics = pd.DataFrame(aggreg_metrics)\n", "df_metrics[[\"train_loss\", \"valid_loss\"]].plot(\n", " grid=True, legend=True, xlabel='Epoch', ylabel='Loss')\n", "df_metrics[[\"train_mae\", \"valid_mae\"]].plot(\n", " grid=True, legend=True, xlabel='Epoch', ylabel='MAE')" ] }, { "cell_type": "markdown", "id": "4d73be89", "metadata": {}, "source": [ "- As we can see from the loss plot above, the model starts overfitting pretty quickly; however the validation set MAE keeps improving. Based on the MAE plot, we can see that the best model, based on the validation set MAE, may be around epoch 175.\n", "- The `trainer` saved this model automatically for us, we which we can load from the checkpoint via the `ckpt_path='best'` argument; below we use the `trainer` instance to evaluate the best model on the test set:" ] }, { "cell_type": "code", "execution_count": 15, "id": "d60a0ce3", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Restoring states from the checkpoint path at logs/mlp-corn-cement/version_2/checkpoints/epoch=155-step=779.ckpt\n", "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n", "Loaded model weights from checkpoint at logs/mlp-corn-cement/version_2/checkpoints/epoch=155-step=779.ckpt\n", "/home/jovyan/conda/lib/python3.8/site-packages/pytorch_lightning/trainer/connectors/data_connector.py:258: PossibleUserWarning: The dataloader, test_dataloader 0, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 4 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n", " rank_zero_warn(\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "88b0f73bc1154f74898737527129654f", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Testing: 0it [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
       "┃        Test metric        ┃       DataLoader 0        ┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
       "│         test_mae          │    0.28999999165534973    │\n",
       "└───────────────────────────┴───────────────────────────┘\n",
       "
\n" ], "text/plain": [ "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n", "┃\u001b[1m \u001b[0m\u001b[1m Test metric \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m DataLoader 0 \u001b[0m\u001b[1m \u001b[0m┃\n", "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n", "│\u001b[36m \u001b[0m\u001b[36m test_mae \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m 0.28999999165534973 \u001b[0m\u001b[35m \u001b[0m│\n", "└───────────────────────────┴───────────────────────────┘\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/plain": [ "[{'test_mae': 0.28999999165534973}]" ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "trainer.test(model=lightning_model, datamodule=data_module, ckpt_path='best')" ] }, { "cell_type": "markdown", "id": "ec7d93a1", "metadata": {}, "source": [ "- The MAE of our model is quite good, especially compared to the 1.03 MAE baseline earlier." ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.12" } }, "nbformat": 4, "nbformat_minor": 5 }