{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn.functional as F\n",
    "import matplotlib.pyplot as plt # for making figures\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read in all the words\n",
    "words = open('names.txt', 'r').read().splitlines()\n",
    "words[:8]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# build the vocabulary of characters and mappings to/from integers\n",
    "chars = sorted(list(set(''.join(words))))\n",
    "stoi = {s:i+1 for i,s in enumerate(chars)}\n",
    "stoi['.'] = 0\n",
    "itos = {i:s for s,i in stoi.items()}\n",
    "print(itos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# build the dataset\n",
    "\n",
    "block_size = 3 # context length: how many characters do we take to predict the next one?\n",
    "X, Y = [], []\n",
    "for w in words:\n",
    "  \n",
    "  #print(w)\n",
    "  context = [0] * block_size\n",
    "  for ch in w + '.':\n",
    "    ix = stoi[ch]\n",
    "    X.append(context)\n",
    "    Y.append(ix)\n",
    "    #print(''.join(itos[i] for i in context), '--->', itos[ix])\n",
    "    context = context[1:] + [ix] # crop and append\n",
    "  \n",
    "X = torch.tensor(X)\n",
    "Y = torch.tensor(Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X.shape, X.dtype, Y.shape, Y.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 769,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([182441, 3]) torch.Size([182441])\n",
      "torch.Size([22902, 3]) torch.Size([22902])\n",
      "torch.Size([22803, 3]) torch.Size([22803])\n"
     ]
    }
   ],
   "source": [
    "# build the dataset\n",
    "block_size = 3 # context length: how many characters do we take to predict the next one?\n",
    "\n",
    "def build_dataset(words):  \n",
    "  X, Y = [], []\n",
    "  for w in words:\n",
    "\n",
    "    #print(w)\n",
    "    context = [0] * block_size\n",
    "    for ch in w + '.':\n",
    "      ix = stoi[ch]\n",
    "      X.append(context)\n",
    "      Y.append(ix)\n",
    "      #print(''.join(itos[i] for i in context), '--->', itos[ix])\n",
    "      context = context[1:] + [ix] # crop and append\n",
    "\n",
    "  X = torch.tensor(X)\n",
    "  Y = torch.tensor(Y)\n",
    "  print(X.shape, Y.shape)\n",
    "  return X, Y\n",
    "\n",
    "import random\n",
    "random.seed(42)\n",
    "random.shuffle(words)\n",
    "n1 = int(0.8*len(words))\n",
    "n2 = int(0.9*len(words))\n",
    "\n",
    "Xtr, Ytr = build_dataset(words[:n1])\n",
    "Xdev, Ydev = build_dataset(words[n1:n2])\n",
    "Xte, Yte = build_dataset(words[n2:])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "C = torch.randn((27, 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "emb = C[X]\n",
    "emb.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "W1 = torch.randn((6, 100))\n",
    "b1 = torch.randn(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "h = torch.tanh(emb.view(-1, 6) @ W1 + b1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "h"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "h.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "W2 = torch.randn((100, 27))\n",
    "b2 = torch.randn(27)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "logits = h @ W2 + b2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "logits.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "counts = logits.exp()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prob = counts / counts.sum(1, keepdims=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prob.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss = -prob[torch.arange(32), Y].log().mean()\n",
    "loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ------------ now made respectable :) ---------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 780,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([182441, 3]), torch.Size([182441]))"
      ]
     },
     "execution_count": 780,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Xtr.shape, Ytr.shape # dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 790,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = torch.Generator().manual_seed(2147483647) # for reproducibility\n",
    "C = torch.randn((27, 10), generator=g)\n",
    "W1 = torch.randn((30, 200), generator=g)\n",
    "b1 = torch.randn(200, generator=g)\n",
    "W2 = torch.randn((200, 27), generator=g)\n",
    "b2 = torch.randn(27, generator=g)\n",
    "parameters = [C, W1, b1, W2, b2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 791,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11897"
      ]
     },
     "execution_count": 791,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(p.nelement() for p in parameters) # number of parameters in total"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 792,
   "metadata": {},
   "outputs": [],
   "source": [
    "for p in parameters:\n",
    "  p.requires_grad = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 793,
   "metadata": {},
   "outputs": [],
   "source": [
    "lre = torch.linspace(-3, 0, 1000)\n",
    "lrs = 10**lre"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 794,
   "metadata": {},
   "outputs": [],
   "source": [
    "lri = []\n",
    "lossi = []\n",
    "stepi = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 795,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(200000):\n",
    "  \n",
    "  # minibatch construct\n",
    "  ix = torch.randint(0, Xtr.shape[0], (32,))\n",
    "  \n",
    "  # forward pass\n",
    "  emb = C[Xtr[ix]] # (32, 3, 10)\n",
    "  h = torch.tanh(emb.view(-1, 30) @ W1 + b1) # (32, 200)\n",
    "  logits = h @ W2 + b2 # (32, 27)\n",
    "  loss = F.cross_entropy(logits, Ytr[ix])\n",
    "  #print(loss.item())\n",
    "  \n",
    "  # backward pass\n",
    "  for p in parameters:\n",
    "    p.grad = None\n",
    "  loss.backward()\n",
    "  \n",
    "  # update\n",
    "  #lr = lrs[i]\n",
    "  lr = 0.1 if i < 100000 else 0.01\n",
    "  for p in parameters:\n",
    "    p.data += -lr * p.grad\n",
    "\n",
    "  # track stats\n",
    "  #lri.append(lre[i])\n",
    "  stepi.append(i)\n",
    "  lossi.append(loss.log10().item())\n",
    "\n",
    "#print(loss.item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 796,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7feda5f10250>]"
      ]
     },
     "execution_count": 796,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAD4CAYAAADmWv3KAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAArgklEQVR4nO3deXxU1f3/8dfJwr5D2JGw76AQQQRUVllq+bm1Ur9VUb/UfsWqba24U1esVat1q3WrWre60iKKgLIIggFZpAiEgBBAkrAvQkhyfn/MkpnJLDdhkskN7+fjwYOZe8/c+5k7k88995xzzxhrLSIiUv0kJToAERGpGErwIiLVlBK8iEg1pQQvIlJNKcGLiFRTKYnacbNmzWx6enqidi8i4korVqzIt9amOSmbsASfnp5OZmZmonYvIuJKxpjvnZZVE42ISDWlBC8iUk0pwYuIVFNK8CIi1ZQSvIhINaUELyJSTSnBi4hUUzETvDHmJWNMrjHm2xjlzjTGFBljLolfeKVt3H2Ix+ZsIP/w8YrcjYiI6zmpwb8CjI1WwBiTDDwMfBqHmKLatPswT87PYu+RgorelYiIq8VM8NbahcDeGMVuAN4DcuMRlIiInLyTboM3xrQBLgSeO/lwnNMPUYmIRBePTta/ALdaa4tiFTTGTDHGZBpjMvPy8sq1M2PK9TIRkVNOPCYbywDeMp7M2wwYb4wptNZ+GFrQWvs88DxARkbGSdXBLarCi4hEc9IJ3lrbwffYGPMK8J9wyT1eVIEXEXEmZoI3xrwJnAc0M8bkAPcAqQDW2kptdxcREediJnhr7SSnG7PWXnVS0ZSBOllFRKJz3Z2s6mQVEXHGdQneRzV4EZHoXJjgVYUXEXHChQleRESccG2C1zh4EZHoXJfg1ckqIuKM6xK8jzpZRUSic12CVwVeRMQZ1yV4ERFxRgleRKSacl2CN+plFRFxxHUJ3kedrCIi0bkuwav+LiLijOsSvIiIOOPaBK87WUVEonNdglcfq4iIM65L8CIi4oxrE7xG0YiIROe6BK8mGhERZ1yX4H1UgRcRic51Cd5oJLyIiCOuS/AiIuJMzARvjHnJGJNrjPk2wvrLjTFrvP+WGGP6xT/M0qx6WUVEonJSg38FGBtl/RbgXGttX+A+4Pk4xBWZWmhERBxJiVXAWrvQGJMeZf2SgKdfAW3jEFdMqr+LiEQX7zb4a4DZkVYaY6YYYzKNMZl5eXnl2oEq8CIizsQtwRtjhuNJ8LdGKmOtfd5am2GtzUhLS4vXrkVEJIyYTTROGGP6Ai8A46y1e+KxzVjUxyoiEt1J1+CNMacB7wO/tNZuPPmQYu6vonchIlItxKzBG2PeBM4DmhljcoB7gFQAa+1zwN1AU+AZb/IttNZmVFTAJVSFFxGJxskomkkx1l8LXBu3iGJQ/V1ExBndySoiUk25NsGrk1VEJDrXJXj1sYqIOOO6BO+jCryISHSuS/CaLlhExBnXJXgREXHGtQlenawiItG5LsGrk1VExBnXJXgREXHGtQlev+gkIhKd6xK8WmhERJxxXYL3Uf1dRCQ61yb4Vdv3JzoEEZEqzXUJfv+PJwCYMfu7BEciIlK1uS7Bi4iIM65L8OpkFRFxxn0JXhleRMQR1yV41eFFRJxxXYJf8f3eRIcgIuIKrkvwuYeOJzoEERFXcF2CVwONiIgzMRO8MeYlY0yuMebbCOuNMeZJY0yWMWaNMaZ//MMM2l9Fbl5EpNpwUoN/BRgbZf04oIv33xTg2ZMPKzKldxERZ2ImeGvtQiBaz+ZE4FXr8RXQyBjTKl4BlqIMLyLiSDza4NsA2wOe53iXlWKMmWKMyTTGZObl5ZVrZ/pNVhERZ+KR4MNl3LCTPVprn7fWZlhrM9LS0sq3M+V3ERFH4pHgc4B2Ac/bAjvjsF0RETkJ8UjwM4ErvKNpzgIOWGt3xWG7YakCLyLiTEqsAsaYN4HzgGbGmBzgHiAVwFr7HPAxMB7IAo4CkysqWBERcS5mgrfWToqx3gLXxy2iGNQGLyLijOvuZE1ShhcRccR1CV5ERJxxXYJXBV5ExBnXJXiNoxERccZ1CV41eBERZ1yX4EVExBnXJXgbdhIEEREJ5boEryYaERFn3JfgEx2AiIhLuC7Bi4iIM0rwIiLVlOsSvNrgRUSccV2C37n/WKJDEBFxBdcl+P1HCxIdgoiIK7guwTeqUyPRIYiIuILrEvxP+rZKdAgiIq7gugSv+eBFRJxxXYIPtCX/SKJDEBGpslyX4AMr8IeOnUhcICIiVZzrEryIiDjj6gSvmSVFRCJzlOCNMWONMRuMMVnGmGlh1jc0xvzbGLPaGLPOGDM5/qH69+V/rPwuIhJZzARvjEkGngbGAT2BScaYniHFrgf+a63tB5wHPGqMqZAB64FjaKyq8CIiETmpwQ8Esqy12dbaAuAtYGJIGQvUN57qdT1gL1AY10jDeCdze0XvQkTEtZwk+DZAYCbN8S4L9BTQA9gJrAVutNYWh27IGDPFGJNpjMnMy8srV8CBo2g0L42ISGROEny4O4tC20bOB1YBrYHTgaeMMQ1Kvcja5621GdbajLS0tDKG6gtGbfAiIk44SfA5QLuA523x1NQDTQbetx5ZwBage3xCjExt8CIikTlJ8F8DXYwxHbwdp5cBM0PKbANGAhhjWgDdgOx4BuoT2ERTWKQELyISSUqsAtbaQmPMVOBTIBl4yVq7zhhznXf9c8B9wCvGmLV4mnRutdbmV2DcACzN3lPRuxARca2YCR7AWvsx8HHIsucCHu8ExsQ3NBERORmuu5NVc0mKiDjjvgSvDC8i4ojrEryIiDjjwgSvKryIiBOuS/C1Ul0XsohIQrguW3ZtUT/RIYiIuILrEryIiDjjugSvUTQiIs64LsGLiIgzrk/wmnBMRCQ81yf4w8cr/HdFRERcyXUJ3mgcvIiII65L8KG+2bY/0SGIiFRJrk/wV7y0PNEhiIhUSa5L8OGGSY57YhHHThRVfjAiIlWY6xJ8OOt3HeS7Hw4lOgwRkSqlWiR4gH9lbk90CCIiVYrrEnykMTT/XLatUuMQEanqXJfgRUTEGdcl+IZ1UhMdgoiIK7guwddMSU50CCIiruAowRtjxhpjNhhjsowx0yKUOc8Ys8oYs84YsyC+YcbP59/lkj5tFtv3Hk10KCIiFSpmgjfGJANPA+OAnsAkY0zPkDKNgGeAn1prewGXxj/U2NbmHIhZ5t2VOQCs2r6/gqMREUksJzX4gUCWtTbbWlsAvAVMDCnzC+B9a+02AGttbnzDdOaWd1eTPm0W63bGTvSag1JEqjsnCb4NEDjIPMe7LFBXoLEx5gtjzApjzBXhNmSMmWKMyTTGZObl5ZUv4ih8Nzt9+M0OjhcWhb27VVOVicipwkmCD5cTQyvAKcAAYAJwPnCXMaZrqRdZ+7y1NsNam5GWllbmYMvizPvn0v2uTyKu/3rL3grdv4hIojlJ8DlAu4DnbYGdYcp8Yq09Yq3NBxYC/eITYtkt27KXg8c888T/dd6moHXGO5nNa199X+lxiYhUJicJ/mugizGmgzGmBnAZMDOkzEfAMGNMijGmDjAIWB/fUJ1bE9DZ+uhnGxMVhohIQsVM8NbaQmAq8CmepP2OtXadMeY6Y8x13jLrgU+ANcBy4AVr7bcVF3bZTHk10/9YbfAicqpIcVLIWvsx8HHIsudCnj8CPBK/0OJnzn93+x//cPBYmV//0Oz1vLFsG2unnx+0/GhBIXVqODqEIiKVznV3spaXb7bJ5WE6VwuLisk9eIwZs78L+xuvf1uQzaFjwcv/vXonPe/+lP/uPFgxAZfR3iMFunlLRIKcMgn+lnfXsGl38Jzxfe75FIB7Zq5j4IPzeG7BZp6Y62mzT582i9++syrstpZl7+GGN78B4NsdBzh2ooiiYs/AoqzcQ+w7UhAzngNHT5T3rZRytKCQ/vd9xrA/fR52/Y79PzLXexWzZHM+K7fti9u+RaTqOmUSPMDoxxcGPT/kra0HTjV8vLCYW99dA8D7K3cElfcl8Z8//5V/2fR/r6P7XZ/wk78uBmDUYwsZ98SiqHEs37KXfvfOYc66H8r5ToJNfvnrqOsv+OtirvX2Q/zi78u46Jkljre990hB2KuaivDE3E30me456a7cto9Za3ZFLZ978Bg/FoT/Ja/iYsujczaQf/h43OMUcYtTKsGHc9EzXwY937b3KG9H+PGQFxdnk3H/3KBlR70JZv2ug7y7wjMNQrh2/qWb9zBvvacWvdo7TcLtH6wtd9y7DvzIm8u38eLiLSwLaXay1vL+yhyOF3pi2+vgiiLUzv0/kj5tFv3v+4xzIlwZhMZzxOGJYNaaXWET7+NzN/qbwi56ZgnXv7Ey6nYGPjiPSX//Kuy6pdl7+Ov8LKa9V/5jLOJ2p3wP4cpt+4Oef7Eh+A7b9Gmz/I8f/Pi7qNv6/b9WR1znS0RbZ0zwL8s/XMDSzXsY3Kkp4DlJ+Gr/lw5oy58u6UtW7mFqpCTRvmndoO0Nfmh+xH199t/d/Pad1Szfspcp53SMGnMkSzfv8T92coIY/NB8erRqwOwbh0Utd+DoCa5/YyV92jTk3zcMLVdsgSLNKVTovdryneTKK/fQMbbv/ZEB7Ruf1HZEEuGUr8FXlPRpsygsKiZ92iweDxiLf9XLy3nPO+EZeBL/5JeXAwQ17fxrRQ7GGEY/vpBzH/kiaNu7DvwYdd/7f/S077/19XZGPBp5Ys9vtu3jnYCrlazcwyzaVP4pJNbvKulwXvH9Xj5ataNUmYKiYsBzhVCRrI3PbEPjn1jExc86b9ISqUqU4CtQ5ztmA/BEwN20X2zIK/UD4Z9vyAubkMLVTud/tztq7f2GN79xPJPahc8s4Q/vrmHfkQL+Om8Tox5bwC9fXB6x/Irv95Gzz9lInYufXcqNb60qtfzAj56rgT1HCthz+DhvLd9WqlP6hwNlH8oK8OdPNwRdcUHJncvllX+47M1bIlXFKd9EU1WEa0v++d+W+h+HJq5I/r16J12a1wu7LnAbgY/PuO+zUmVnrg6ejeLaf2Qy19uH4Gtmyso9xIzZG2hat0bEeN5Yto1JA9v5E+34Jxb712U8MBdrYdbaXbx2zSD/8rMemhdxe9E89XmW/3G4c9yR44UMfXg+T1x2Bj+eKOJXr62ga4t6TL+gF33bNWLT7kPUTEnmqc838eRlZ5CSrPpPdXX3R9+Sd+g4z/7PgESHUqGU4KuIr7JLj88/Xlhcrm09FofpGRZsDG6q8SV3gPdW5PDDwWM88umGmNu5/YO11EpNYu763dw2roe/iQbAd9ESrZb8zBdZPPLpBjbeP47CIsu+owW0blQ76j7zDnmuDAAWbsxj75ECmtStQVbuYfYdPcFv31nl3+fG3Yf5xQvLOK9bGl9syKN1w1rsPHCM34w8TPeWDWK+v3gqKCzm7cztXD7wNJKSKvae64LCYpKTDMkVvJ+q6tWlp8ZcVK5M8B2a1WVL/pFEh1FtXfpc9Dbn30XpTD52ooiNIfcb/PYdT/mP14YfFrp+10Hu+Sj8zBZ/+sRzEunibe4CuGN8D87s0CRiDE9/nsWiTfn+5/29Vyi+K41wJxRf53rgCSiecg8d48DRE3RpUT9imac+z+LJeZuonZrMJQPaVkgcPl3vnE1G+8a8++uzy72NomJLkjn5ZrBosnIPk960TpW9mso9eIxV2/czplfLRIcSVtU8apJQX28t/41Q3e/6hJ8+9WXsgiH+UYYa1QMfr+f/PV2yjw0/HGJxQEI/XljkH74aaI+D0UDl7ZstLrY8NX8T+4+G38fZD80vdR9GKF9fhNPhptFkbt3LxKe/jDqKKPN7Z59zVu7hUv0kxwuL6HT7x9z41qqwv7tQHtZa3li2zb+9rflHGPXYAh6ZE/tKMVF+9relTHlthf8emarGlQn+1LyolEjO/8tC/ufFZf7nby4Pfx+DE76TQGFR8B/srgOe+wICh48GWpyVz5/nbOSOD8NfiRSGJIAZs7+j252z/Qn4qpeX+6ewPnTsBFm5h8qd6A8fL+SS55ayevv+Ule6B4+dYGqM+wt8svMOk3/4OKMeW1Dq5OS7wWzm6p10v+sTrnnla77dcYDBD83jrgjHIJZPvv2B2z9Y629izPPeK7EiRoXj9g/WMuLRL/zPcw8e49xHPuf7PWW7yn9p8RbSp83ihIOruPRps7j2H5ls3eMZdFBVc5IrE3yVPZpSbdwzc13Qc9/IpUl//4r8w8fJ2Xc0aORTgbe/xJf4PvxmR9R5ip5bsJnjhcV0u/MTcg8dC7r/4s9zNjLqsYX08k6lEUlRseV4YRGvLd3K1oBE3jvgdaFXJK9/9T3/CbhDONpw0hGPLmDIDM/7Dr0xzYT8Ec77Lpef/HUxuw4c47Wvvqe42HLff/4btik1O+9wqWY8KLmzfI+3Cc3J1dSH3+zgjWXbyM474h96O3P1Tr7fc5R/LClbO7vvxBJ6RbL/aAEj/vwFv3xxGenTZvHFBs8vkgb2S72+zPOeo8k7dDxmmXhzZYJXfpeKtuL7ffzh3fB9DRn3z2Xow58HdTIvzvI0EfmGkd709irGP+m5r6EwpEYY+pvBvjugI7HWsm7nAfIPH2fDD4c4eOwEW/KP8H//XEG3Oz/hro/WcUmEfhNfkjx2IvxPWE5+5WuWbM6PWNsN7ej/aNUOz30YMf4Is/MP8+LiLf6puqfPXEfXOz39KCMeXcCYxxfy5LxNQU0bkTbpa+IPN2XGTW+v8j8OvIoDKCz2xL7vSAF3ffgtxwuL+POnG4Leq7XWf6XkO9ltzT9K7sFjjHj0C7bvPcr873LJzj/i79d5PcyPBd390Tr+s9Zz4rzk2SX+E+Oy7D18lb2HXQd+5MwH5vLk/E2lXluRXNnJOrxbczbnbUl0GFLNvZMZPfE+88Vm/jC2O1AyW+nG3YdLlVsd8AM04Ya7Pj0/q9Qyn+Jiy98WZvPwJyV3UXdKq8vmvOCE7Os4Dk3iy7fsoWfrBnS/6xMa1Eph8pAOQeu/2JDnv3rwDX8tLrY8/XnpmF5YlM39s9bTvmkdZk6NfhfyzNWeZGeB0++dw/4wk+s99tlG9h4pYNX2/fz6vE584R25Zb2DXD/03ij39dZ9LN28h0l//4oXrsigcd1U+p/WuFTnbnbeEdbvOkjOPk9N/tWl33PvxN5c9OwStuQf4ZN1P5B36Dizvy25gun3xzkcPFbIq1cP5Ij36uuCpxbzs4y2ZOcdYdifPufeib2C9hN67H2Oek8Uvr6NfUcK/PNWvXzVmQD8Ze4m/jJ3E9kPjq/wkVIAJl53/JVVRkaGzczMjF0wjMc+28iT8yr3TCgSSY9WDdiSf5hjJzw1xg+vH+LvBJ5947CYk8/Fy9YZE8i4f26p5pStMyY4uo9i64wJTHk1M+j3EyJZM30MfafPiVmuTaPa7Ai4a7lPm4as3XEgyiucuXRAWx65tF/Y9zWhTytmrY0+UV1FWTt9DH3CHJfOzeuRlVty8t/0wDhSyzkyyBizwlqb4aSsK2vwaqKRqiRwigYgaIRPZSV38Ny7EG4SN6ftvmc/NI+dDu8i/rODeyCAoOQOxCW5g2cqj10RYo3XqJ7yCJfcgaDkXplc2QafVIHjbkXc6sqXwk8z0fH2j8MuD+U0uUPVuFHI1+8Rat53uZUcSdk5+c2IeHBpgk90BCIi5TfwwfJNx1FWrkzwF1fwXX4iItWBKxN8rLlIRETEYYI3xow1xmwwxmQZY6ZFKXemMabIGHNJ/EIUEZHyiJngjTHJwNPAOKAnMMkY0zNCuYeB6LffiYhIpXBSgx8IZFlrs621BcBbwMQw5W4A3gOqfhe2iMgpwEmCbwMEzt6U413mZ4xpA1wIPBdtQ8aYKcaYTGNMZl5e+X8aTkREYnOS4MMNSgy9c+IvwK3W2qh3GFhrn7fWZlhrM9LS0hyGKCIi5eHkTtYcoF3A87bAzpAyGcBb3rkhmgHjjTGF1toP4xGkiIiUnZME/zXQxRjTAdgBXAb8IrCAtdY/g5Ex5hXgP0ruIiKJFbOJxlpbCEzFMzpmPfCOtXadMeY6Y8x1FR1gJNcM7RC7kIjIKczRZGPW2o+Bj0OWhe1QtdZedfJhxdanTcPK2I2IiGu58k5WKJkzWkREwnNvgld+FxGJSgleRKSacm+CT3QAIiJVnHsTvKrwIiJRuTbBi4hIdK5N8D1aNUh0CCIiVZprE3zvNg1ZM31MosMQEamyXJvgARrUSk10CCIiVZarE7yIiESmBC8iUk0pwYuIVFOuT/Bzbj7H//i6czslMBIRkarF9Qm+a4v6/sc3jeqSwEhERKoW1yf4QLVSkxMdgohIlVGtEjzA1hkTGNC+caLDEBFJuGqX4EVExOOUTPC9WmuaAxGp/qplgo81mmbWb4ax5aHxlRSNiEhiVMsEP7pnC765a3TQsn9dN7jM27licPug5y0a1IxavlNa3TLvQ0SkolTLBA/QuG4Nts6Y4H9+ZnqTiGVH92zBa9cM5IP/OztoeaPaqbx+zSAA2jWpzctXDWR8n5b+9ZcPOo1/XD3Q/3ze786LGpPvBNGmUW3H7yOclSEnL7d4/ZpB/OrcjokOQ+SU4SjBG2PGGmM2GGOyjDHTwqy/3BizxvtviTGmX/xDje7sTk1jlnnm8v5cGVIrB/j7FRkM65LGGaeVHn3TroknGRsMPVs34JnLB/jXPXBhH87tmhZ1n2/+71n+x/+5YRj3TuzF578/z79sSOemUbfxmxGdSy2rkeLsvFyvZkqpZYEnvZMx8fTWZX7N0C7NSEkycdm/iMQWM1MYY5KBp4FxQE9gkjGmZ0ixLcC51tq+wH3A8/EONJrND47317TD8Q2bHN+nFX+c2Btw+JuuxpT5t1+vHtKB4d1KEvbggBNPWv2aXDE4PShB105NCboK6NjM08zz8MV92DpjAr8d0y3sfpbdPpKlt42IGsvDF/clNdlQIzn4Y75tXPeIr/ks4M7gUJMGnuZ//MRlZ0TddyRXDE4v1+tEpOycVAUHAlnW2mxrbQHwFjAxsIC1dom1dp/36VdA2/iGGV1ykiEpQs3wy2kjeO2agWHXARiHFUqn5e6+oCcvT468v9KCzyDzf38eW2dM4OdnnlaqZOfm9QCokZxEiwa1aNWwpKnnvG7hrwLWTj+/1Lz5k4d08D/u06ZhqX089rN+/L+QGvrkIencNj74xDCkc+mrpuW3j/Q/viHM1UeLBrXCxulzzdAOEdeFu5qJ5d1y9L2IVBelr+FLawNsD3ieA0SuLsM1wOxwK4wxU4ApAKedVjqBVYRY7d1ntGsUdnmzejW5dEBbiopP/rdf59x8Dtl5R8Ku810hDO3cjDU5+6Nu57Obz6Gw2JIaUCNfMm0ERcWW6TPXhX1NuLt7a6Qk8ezl/Vm/6yBfZe8tFc9F/duyJb8k3tX3jKFh7dJz7/uagJ6cdAZFxcW0b1qX5gEJ3HdCmtCnFY/+zFmr3RWD23PXT3ry1PxN/HnORgDSm9Zh656jTOjbmifnZwHwv8M68PdFWyJuZ830MTSolcrxwiJH+xWpjpzU4MPVXcNmPWPMcDwJ/tZw6621z1trM6y1GWlp0duuK1pSkmHm1CG8cnXp2rYxkHnnKNo1qUOjOp7ENrpHi6AyGWW4W7Zri/qM7d0yaNmSaSPolFaX+y/0NBm9fu0g1kw/P+zrbx7V1RuXCUruAK0b1aZdkzr0DxOPDf8xATCuTyt+O6YbY3p53lf7pnV4ZfKZ/iuhywe1p1frBiy/fWTY5B6oRrLhwjPa0j+kD8N38kpOMkEnmkhXGwD1vT/iMnVEybxCxnv5lJJc8lW8Y0JJK+GKO0ex8JbhdEqrS01v81dqUtnHD1x4RhseuqhPmV9XWSb0bUX2g+N57ZqB/u+EhBfu6rIqmTl1SKXsx0kNPgdoF/C8LbAztJAxpi/wAjDOWrsnPuFVrL5tG5VaNm1c96BOz0Z1avD1HaNoUreGf9ma6WP8iSSShy7qQ9cW9SKub92odsxRNz43jurCjTEmUvv1uZ0Y17slIx5d4GibPo3qeN7XgPaNOa9bc//ylg1rMes3w0qV/93orpzZIfKIJIArB7dnTK+WHC3w1J4Djx1AUpT2rtCyUFLDiNQf0rReTZrW84xiOlpQyNb8o9Su4Tmh1ExJ5saRXRjbuyWFRZaPVu3g3ZU57D96Imgb9Wqm8Pvzu9GmUW1ue39txPiGdm7G4qz8iOsr0s2jupKUZBjWJY1hXdJ4fO7GMr3+rI5NePznp/PCoi28uDjy1Q94KjmRjvfF/dvy3soc+rRpyDOX9+eDb3bw2GdliyWSlCRDYZSr5rT6Nck7dDxo2YS+rZi1ZlfQslvO786XWV/GJaZ4+eNPe3HPzHXUTk0Om3sqgpNqztdAF2NMB2NMDeAyYGZgAWPMacD7wC+ttfH5pBPkunM7lfpB77T6NUkOaONvUCuVmiklNdJFfxjOoj8MD3rNpIGnMaB99EQYT0lJho5pwScUJx3EI7o3p3XDWvzqHGdTLd8wsgtndWzqfy1A5+b1g8r8cWJvhnRuxqgezXngwt7cOja47T7cSJplt48sNcLn6iEdmDq8c8A1ZPAb+nlGO+6b2CtoWZ0aKfQMuVP55tFd6dGqAX3aNuTOn/QMmmLa59s/nu9vzrv7J6FjCEq8fm1w62Rqcun38vJVZ0Z8fVlMHd6ZmVOHMOfmc9g6Y4K/ySvUa9cMZGjnZjG3d/mg9rRqWDvoSijUglvO47v7xvLilRn+Zf+8dhAT+rbyP/eNLBveLY12Terwm5EllY+/TjqDxbcG/y2UxR0TekRcd9/EXmG/0x2alr7/JNyyQMO6RD5esZp1f3lW6ZF4sXw5bQSXZlRq1yTgIMFbawuBqcCnwHrgHWvtOmPMdcaY67zF7gaaAs8YY1YZYzIrLOIqqF2TOrRrUifRYZRS28Hsmk3q1mDJbSPp1rJ+zLKhfpbRjrXTx0RMPMYYLh/U3l+b9rn/wt4M69KMRy8taZcP1/l69wU9+f353YJq8E9OOoMXrvAkn4cv6csvyzEqp3n9WtwxPnIiuXpoB/8+YhnUoSlbZ0xg0kDPRe643i0Z3r151JFK0cz6zVB+Meg0sh8cz+/P70bfto2CpsQOFDg66m+/HMAjl/QtVeafASekC/p5O86jnPjbN61LrdRkzutacjU3pHMz7rnAc9KbPCQdE7bV1qN+rRTaNi75W/jdaE9TktNhtaN7tgi7fOuMCd7P2hP8GwHvK9w04Q3rpDL/d+c63s87vyrpjF/0h+FRpx5vHXACWH3PGOp4v9/hBghsnTGBrTMm0KZRbWqlJNO7TQMe//npEbcdb06aaLDWfgx8HLLsuYDH1wLXxjc0KY/kJENRseWeC3oyskfJH+nr1wxic97huO7LGONvMy+L5vVr8Zp3WOvv/rXa0X58ftqv7OPvo2nbuDavhumHGRUh0QBsemAcxdYyfeZ/meod2TOiewveXL7d//xX53aiWb2ajt7fjSO7kHvoGH3aNKJX64Y8eKGzfoBBHZuwaFM+1kLdmilcmtGOdzK38/XWfQzr0oxFm/I5PcIggkCXDGhL03o1yNn7o39Z6Ki05vVrsfH+caQmG56c5+noDjxPnNM1jYUb80pt2/fRtW0cXCu+fngnnv58M+D5bq7cto8L+rWOOcoqvWld8g8X0Kt1Q3q3acCJQktKcvh6auAV7aSBp/Hm8m3+Jp7QK4GBAc2OSUmGm0Z1pXZqMg/N/i7iewJoWDuVBbcMZ/fBY3zwzQ7/8iXTRgSdCHzb/c8NpZs9K5KjBC/usfqeMRQV21Ido0O7NGNolMtSNzj58UwlfPcn/OmSvqWatsIZ1qWZv2/G19Ed2CE7umcLNj0wLqgT3JesBnVowrItwaOVAo3s0fyk2mSLw7RbTB3emX9MHkhSkuGTm4aRHJCVAq8IVt41Omy/Rzi++zfCdaF0SqvLwo15Qdvq17YhFw9oy79W5DBp4Gk0rlOD+2etp2Ozutxyfnd/gg/9bk4b150ZYRIrwAtXZvDNtv00rJMalCxfvDKDujVT2LbnKF3C9H09eGFv7p3YCwMUWcsby7aVKvPR9UNYsrmk+/BX53ZiVM8WjIzRr5VWvyZp9WuyNOC1To9pRVOCr2bC3b1alQ3vlhZUQw8nVidrefRu07BMd/XePr5Hqb6ZUKEjnHyjmJLD9DlMOacjzy/M9pQr5/sa36cVizbl07FZSUILbD7x1cK7twyO+6L+bWhWvyb7jhRETUS3jevOgR9PRFwfGPdt43owqkcL/4lq+e0jqV8rldo1kllwy3B/vPfPWh+xycnnunM7RUzwjerUYHj35qWWj/SOcvP1D4XyjEDzjsYCJp7ehndX5LBu50F/mX7tGtEv5IqnU8DJ33eVYoAnLjudXq2D7yG5emgHWjeqzfg+LWN+pyuLu7KBVDtObgrz/a1EG/ZZUVo0qMnbUwaT3qzsE8n5EmDgqKFOaXXZnHeEn2W05avsPazJOVDu2C47sx0XntEmaAjqTaO78KtXV9AjypTYxpiYU2yApwYb9vVhltVISWJIQEdv8zBNLa0b1eaNaweVSqLhvPfrs5m7fjd1UpPpEuOEUB5N6tZg1m+GkT5tluPXdGtRz98MNfH0NqXWJyeZoM7oqkAJXqo8X600njV4Jz696RxaNqhFwzpl72cIFFiZ69e2EZvzjlCvZirndEljTc4B0upHn6U08nZNqRvZzu7UjLV/DH8/Rbykeptqoo3GieRsB6N9wDNst6r+MlsVqZw7ogQvVd7gTk3ZsPuQ/6azylKekUWBfE0Rlwxoy6JNnrHzD17Uh6uGpNOyYS1uHt2V/zmrPS0bRu9YrGquOjudfUcKHA+tjeSRS/rGHI9/sjo0q8v/Dju5GUynnNORvm0bsmrb/vgEVYmU4KXKu2NCD64Y3D5o7h03aNmwlr+d/8a3VgGeqSN87dTJScZ1yR087+G2KMNMnbo0ox2XZrSLXfAkBM7cWl63e9/rWm9zWrQb9aoaJXip8lKTkxyNdBGpSNeP6Mzh44VcPqjsNzolihK8iJyy6tZI5ubRzub1aVArlQcc3qNQVSjBi1SScNMaSGKtu3dsokOoUErwIpXgzgk9GNYlsTOoyqlHCV6kElx7kiM5RMqj2v7otojIqU4JXkSkmlKCFxGpppTgRUSqKSV4EZFqSgleRKSaUoIXEammlOBFRKopYyt7km3fjo3JA74v58ubAflxDCdeqmpcUHVjU1xlo7jKpjrG1d5a6+i26IQl+JNhjMm01jr72ftKVFXjgqobm+IqG8VVNqd6XGqiERGpppTgRUSqKbcm+OcTHUAEVTUuqLqxKa6yUVxlc0rH5co2eBERic2tNXgREYlBCV5EpLqy1rrqHzAW2ABkAdMqYPvtgM+B9cA64Ebv8unADmCV99/4gNfc5o1nA3B+wPIBwFrvuicpaRKrCbztXb4MSHcY21bv9lYBmd5lTYDPgE3e/xtXZlxAt4Bjsgo4CNyUqOMFvATkAt8GLKuUYwRc6d3HJuBKB3E9AnwHrAE+ABp5l6cDPwYcu+cqOa5K+ezKEdfbATFtBVZV5vEicm5I+Pcr4t9DPJNjRf8DkoHNQEegBrAa6BnnfbQC+nsf1wc2Aj29X/rfhynf0xtHTaCDN75k77rlwGDAALOBcd7l/+f7EgKXAW87jG0r0Cxk2Z/wnuiAacDDlR1XyOfzA9A+UccLOAfoT3BiqPBjhOePPNv7f2Pv48Yx4hoDpHgfPxwQV3pguZD3VxlxVfhnV564QmJ5FLi7Mo8XkXNDwr9fEf8eypMEE/XPe0A+DXh+G3BbBe/zI2B0lC99UAzAp944WwHfBSyfBPwtsIz3cQqeO9qMg1i2UjrBbwBaBXwBN1R2XAHbGgN86X2csONFyB98ZRyjwDLedX8DJkWLK2TdhcA/o5WrrLgq47M7mePlff12oEsijleY3FAlvl/h/rmtDb4Nng/WJ8e7rEIYY9KBM/BcKgFMNcasMca8ZIxpHCOmNt7H4WL1v8ZaWwgcAJo6CMkCc4wxK4wxU7zLWlhrd3m3tQtonoC4fC4D3gx4nujj5VMZx+hkv5tX46nJ+XQwxnxjjFlgjBkWsO/KiquiP7uTOV7DgN3W2k0Byyr1eIXkhir7/XJbgjdhltkK2ZEx9YD3gJustQeBZ4FOwOnALjyXiNFiihZred/HEGttf2AccL0x5pwoZSszLowxNYCfAv/yLqoKxyuWeMZyMsfuDqAQ+Kd30S7gNGvtGcBvgTeMMQ0qMa7K+OxO5jOdRHBFolKPV5jcEEnCj5fbEnwOno4On7bAznjvxBiTiucD/Ke19n0Aa+1ua22RtbYY+DswMEZMOd7H4WL1v8YYkwI0BPbGistau9P7fy6eTrmBwG5jTCvvtlrh6Ziq1Li8xgErrbW7vTEm/HgFqIxjVK7vpjHmSuAnwOXWe+1trT1urd3jfbwCT9tt18qKq5I+u/IerxTgIjwdkb54K+14hcsNVOHvV4W1XVfEPzxtUtl4Oix8nay94rwPA7wK/CVkeauAxzcDb3kf9yK4IyWbko6Ur4GzKOlIGe9dfj3BHSnvOIirLlA/4PESPCOKHiG4g+dPlRlXQHxvAZOrwvGidJtyhR8jPJ1fW/B0gDX2Pm4SI66xwH+BtJByaQFxdMQzoqVJJcZV4Z9deeIKOGYLEnG8iJwbqsT3K+zfwskkw0T8A8bj6b3eDNxRAdsfiufSZw0Bw8SA1/AMa1oDzAz5I7jDG88GvL3h3uUZwLfedU9RMhSqFp6mjCw8vekdHcTV0ftlWY1niNYd3uVNgXl4hk7NC/zQKyMu7+vqAHuAhgHLEnK88Fy67wJO4Kn1XFNZxwhPO3qW999kB3Fl4WlX9X3PfH/YF3s/49XASuCCSo6rUj67ssblXf4KcF1I2Uo5XkTODQn/fkX6p6kKRESqKbe1wYuIiENK8CIi1ZQSvIhINaUELyJSTSnBi4hUU0rwIiLVlBK8iEg19f8B+0l3dDcvrEIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(stepi, lossi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 797,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(2.1260, grad_fn=<NllLossBackward0>)"
      ]
     },
     "execution_count": 797,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "emb = C[Xtr] # (32, 3, 2)\n",
    "h = torch.tanh(emb.view(-1, 30) @ W1 + b1) # (32, 100)\n",
    "logits = h @ W2 + b2 # (32, 27)\n",
    "loss = F.cross_entropy(logits, Ytr)\n",
    "loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 798,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(2.1701, grad_fn=<NllLossBackward0>)"
      ]
     },
     "execution_count": 798,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "emb = C[Xdev] # (32, 3, 2)\n",
    "h = torch.tanh(emb.view(-1, 30) @ W1 + b1) # (32, 100)\n",
    "logits = h @ W2 + b2 # (32, 27)\n",
    "loss = F.cross_entropy(logits, Ydev)\n",
    "loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 710,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAewAAAHSCAYAAAAuWvi9AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAA5WElEQVR4nO3dfXxU133n8e+ZBwkxAgtsC2RBjC1jO7UVgk2AOLYjShIb5cE4D128aYJDGhZt0thN02660KR+5cW23d3sOt64yuKEhiTbqGkTEupAnZhGsR2XB2MDAhyebBwBQsQGAZKFNA9n/xiN0Eh3pJEYzdx75/N+vfRCM/dezT3M1Xx1zz33d4y1VgAAwN0Chd4BAAAwMgIbAAAPILABAPAAAhsAAA8gsAEA8AACGwAADwgVegeGc9VVV9lZs2aNeruuri5FIpHc75CL+L2Nfm+f5P82+r19Em30A7e1b9euXa9ba692WubqwJ41a5ZeeOGFUW/X3Nysurq63O+Qi/i9jX5vn+T/Nvq9fRJt9AO3tc8Y81qmZXSJAwDgAQQ2AAAeQGADAOABBDYAAB5AYAMA4AEENgAAHkBgAwDgAQQ2AAAeQGADAOABBDYAAB5AYAOAC8XiCZ2/GFU8YQu9K3AJV9cSB4Bi0hOLa3NLmxqbj+rw6U6FAkaxhNWNleVaVVej+toqlYaChd5NFAiBDQAusLu1Qw+u36FoPKGu3rgkKRpPnl0fbO/Umo379MimA9qwYr7mzKwo4J6iUOgSB4AC29PaoQfWbVNHd7Q/rAfr6o2rozuqZeu2aU9rR353EK5AYANAAfXE4lq+foe6o85BPVh3NLl+Tyy79eEfdIkDQAFtbmlTNJ5Ie+6Pf/8GLX17tU6e69bZrl61nDivJ559pX95NJ7QlpZTWjq3Ot+7iwLiDBsACqix+WhaN3ht9RVacut01T/2rFZ9b5dqZ1QM2aarN67G5iN53Eu4AWfYAFAg8YTV4dOdac+9Y9YU/eJAu3piCfVI2vpyu+O2h053Kp6wCgZMHvYUbsAZNgAUSFdvTKFBgWtMdgEcChh19cbGY7fgUgQ2ABRIpCSk2KDCKDuPndHit05TaSigiSVBLbq50nHbWMIqUkInaTHh3QaAAgkGjGZXlutQ+6Vu8b3Hz+npl9u1+aG7dOJst1qOn9OFi9Eh295YWU53eJHhDBsACqihrkaRkvTqZeueeUWLv/YrrfzeC7r+6ohaTpxLWx4pCaqh7oZ87iZcgDNsACig+toqPbLpgKRLI8X/+sO1ml1ZrtJQUD968bj2nzyftk04GNCS2ul53lMUGoENAAVUGgpqw4r5WrZuW3/xlIeadmdcvyycXJ+a4sWHLnEAKLA5MyvUtHKhKsrCQ7rHUyIlQVWUhdW0ciG1xIsUZ9gA4AJzZlZo++rF2tJySo3NR3QobbauSWqoq9GS2umcWRcxAhsAXKI0FNTSudVaOrda8YRVV29MkZIQo8EhicAGAFcKBowmTwgXejfgIjm5hm2MWW+MOW2M2ZdheZ0x5pwxZnff15dz8boAABSLXJ1hf0fSNyR9d5h1nrXWfiBHrwcAQFHJyRm2tfYZSWdy8bMAAMBQ+byt653GmD3GmC3GmFvy+LoAAHiesdaOvFY2P8iYWZKetNbe6rBssqSEtbbTGFMv6evW2tkZfs5KSSsladq0abc3NTWNel86OztVXl4+6u28xO9t9Hv7JP+30e/tk2ijH7itfYsWLdplrZ3ntCwvge2w7jFJ86y1rw+33rx58+wLL7ww6n1pbm5WXV3dqLfzEr+30e/tk/zfRr+3T6KNfuC29hljMgZ2XrrEjTHTTd8kr8aY+X2v+0Y+XhsAAD/IyShxY8wPJNVJusoYc1zSVySFJcla+01JH5XUYIyJSeqWtMzm6tQeAIAikJPAttY+MMLybyh52xcAABgDJv8AAMADCGwAADyAwAYAwAMIbAAAPIDABgDAAwhsAAA8gMAGAMADCGwAADyAwAYAwAMIbAAAPIDABgDAAwhsAAA8gMAGAMADCGwAADyAwAYAwAMIbAAAPIDABgDAAwhsAAA8gMAGAMADCGwAADyAwAYAwAMIbAAAPIDABgDAAwhsAAA8gMAGAMADCGwAADyAwAYAwAMIbAAAPIDABgDAAwhsAAA8gMAGAMADCGwAADyAwAYAwAMIbAAAPIDABgDAAwhsAAA8gMAGAMADCGwAADyAwAYAwAMIbAAAPIDABgDAAwhsAAA8gMAGAMADCGwAADyAwAYAwAMIbAAAPIDABgDAAwhsAAA8gMAGAMADCGwAADyAwAYAwAMIbAAAPIDABgDAAwhsAAA8gMAGAMADCGwAADyAwAYAwAMIbAAAPIDABgDAAwhsAAA8gMAGAMADCGwAADyAwAYAwAMIbAAAPIDABgDAAwhsAAA8gMAGAMADCGwAADyAwAYAwAMIbAAAPIDABgDAAwhsAAA8gMAGAMADCGwAADyAwAYAwAMIbAAAPIDABjBELJ7Q+YtRxRO20LsCoE+o0DsAwB16YnFtbmlTY/NRHT7dqVDAKJawurGyXKvqalRfW6XSULDQuwkULQIbgHa3dujB9TsUjSfU1RuXJEXjybPrg+2dWrNxnx7ZdEAbVszXnJkVBdxToHjRJQ4UuT2tHXpg3TZ1dEf7w3qwrt64OrqjWrZum/a0duR3BwFIIrCBotYTi2v5+h3qjl4K6hlTyvTUw3c7rt8dTa7fE3MOdgDjh8AGitjmljZF44lRbRONJ7Sl5dQ47RGATAhsoIg1Nh917AYPBYy+9rE52vLQXfq7j9+mCeFLHxVdvXE1Nh/J524CEIENFK14wurw6U7HZTWV5fqHHb/Vkq8/q86emD6xcFba8kOnO7nlC8iznAS2MWa9Mea0MWZfhuXGGPOYMeaIMWavMea2XLwugLHr6o0pFDCOy050dGvXa2clSRtfOqF3zJqStjwUMOrqjY37PgK4JFdn2N+RdO8wy5dImt33tVJSY45eF8AYRUpCimU4S7Y2/fnBa8USVpES7goF8ikngW2tfUbSmWFWuU/Sd23SNkkVxpiqXLw2gLEJBoxmV5Y7LpsxZaJue0uFJOlDc67RzmPpv943VpYrmOHsHMD4yNc17GpJrQMeH+97DkABNdTVKFIytHrZ4fYL+shtM7TlobtUMTGs7297rX9ZpCSohrob8rmbACSZwV1fY/5BxsyS9KS19laHZT+T9NfW2uf6Hm+V9OfW2l0O665Usttc06ZNu72pqWnU+9LZ2anycuczB7/wexv93j7JHW20kl5uOz+qAWTBgNFbqyZrpPNrN7RvvNFG73Nb+xYtWrTLWjvPaVm+LkIdlzRzwOMZkk46rWitXSdpnSTNmzfP1tXVjfrFmpubNZbtvMTvbfR7+yT3tHFqa4eWrduWVjwlk7JwUE0rF2ZVntQt7RtPtNH7vNS+fHWJb5L0yb7R4gslnbPWtuXptQEMY87MCjWtXKiKsrBj97iU7AavKAtnHdYAci8nZ9jGmB9IqpN0lTHmuKSvSApLkrX2m5I2S6qXdETSm5I+lYvXBZAbc2ZWaPvqxdrSckqNzUd0KG22rklqqKvRktrpzNYFFFBOAtta+8AIy62kz+bitQCMj9JQUEvnVmvp3GrFE1ZdvTFFSkKMBgdcghspAQwRDBhNnhAu9G4AGIDSpABcKRZP6PzFKCVQgT6cYQNwjZ5YXJtb2tTYfFSH066jl2tVXY3qa6u4jo6iRWADcIXdrR16cP0OReOJ/hnEovHk2fXB9k6t2bhPj2w6oA0r5jNSHUWJLnF4Hl2n3rentUMPrNumju5of1g/eMcsPf2Fd+vR//B2SclpPTu6o1q2bpv2tHYUbmeBAuEMG55E16l/9MTiWr5+x5DCLZ9YeK2W//0OHT/bnfZ8dzS5/vbVi3mPUVQ4w4bn7G7t0IK1W7Vm4z4dau+UtcmuU2svdZ0uWLuVszCP2NzSpmg8kfbc2qW3aubUifrW8nn69J3XDdkmGk9oS8upfO0i4AoENjzFqet0MLpOvaWx+eiQ93L1T/bp9IWLemDdNn37uVeHbNPVG1dj85F87SLgCgQ2PCNT12kmqa7Tnlh26yP/4gmrw6c7x7TtodOdjFtAUeEaNjzDqet0xpQybfjUfO1u7dDvXTNZr77epS/8cLcuRpPrpbpOl85lNlc36uqNKRQw/aPBRyMUMOrqjVHgBUWDM2x4hlPXqSTVVJbrH3b8Vku+/qw6e2L6xMJZ/cvoOnW3SElIsTGeJUfjVr/Y304PCooGgQ1PGK7r9ERHt3a9dlaStPGlE3rHrClpy+k6da9gwGh25djnIv7yTxlgiOJBYMMTUl2nTpJzywx4PGh5qusU7tRQV+M4reedf/tLnX0zOuy2DDBEMSGw4QnDdZ3OmDJRt72lQpL0oTnXaOexM2nLYwmrSAnDNdyqvrZK4eDlfRQxwBDFgMCGJwzXdXq4/YI+ctsMbXnoLlVMDOv7215LW35jZTlTRLpYaSioDSvmqyzsXATlP919vR68Y5Yk6S8/8Fb9w2cWSJLuqLlS/7uvCprEvdnwPwIbnpGp6zRhk/ftLvn6s2r4/ov9I8QlKVISVEPdDfncTYzBnJkValq5UBVl4SHv8Y5Xz+gds6ZKkmqrKzSxJKRQwOgds6Zq56uXelMYYAi/I7DhGWPpOg0HA1pSO32c9gi5NGdmhbavXqyv3ndr2vMtJ86ptvoKRUqC6o0l9NJvz+ptM67QO2ZN1Y5Blz8YYAg/I7DhGU5dp8fPduueR59xXL8snFyfetPeURoK6j23TFM4eOkSRixhdbzjTX1s3kzt+u1Z7Xj1jBZef6WuvXKijgy6c4ABhvAzAhueMlzXaUqkJKiKsrCaVi5kGkYPchpguOPVM/rM3ddrx6tvaOexM/r4wmt1oO38kG0ZYAg/I7DhOamu07X31+qmaeUyRgoHjYyRbpo2SWvvr9X21YsJa49yGmC449UzqpxUqhdf69Drnb3qica149UzQ7ZlgCH8jD9F4UmloaCWzq3W0rnViiesunpjipSE+LD2iYa6Gq3ZuK+/st3zR9/Q7NVb+pf//td+NWQbBhjC7zjDhucFA0aTJ4QJax9hgCEwFIENwHVGujd7MAYYohgQ2ABciQGGQDquYcP1YvGE3ozGuUZdhFIDDLe0nFJj8xEdOt2pUMAolrC6sXKSGupqtKR2umfPrDm2MRoENlzJStr40nE1Nh/V4bQP6XKtqqtRfW2VZz+kMTp+G2DYE4trc0sbxzZGjS5xuM7u1g693HZeazbu06H2TlmbnPvYWulge6fWbGRKxWLl9QGGu1s7tGDtVo5tjAmBDVfZ09qhB9Zt6zuTcp55iSkV4UWpY7ujO8qxjTEhsOEaPbHkFInd0eymSGRKRXgFxzZygcCGa2xuaVM0nhh5xQGYUhFe4HRsf+nem/WHC6/tf/zwe2brj+66rv8xxzYGI7DhGo3NRx27Cj98W7W2PHSXtjx0l/7XH8xJW8aUivACp2P7X/ae1AffVtX/+P21Vdq8t63/Mcc2BmOUOFwhnrA6PGjmJUmaXVmuzy66QR9tfF5n34zqirLwkHVSUyp6dSAS/C3Tsb3/5HldWV6qykmlurK8ROe6ozp57mLaOhzbGIjAhit09cYUChhF4+mzNN1xw1Xa0nJKZ9+MSpLOdUeHbJuaUnHyhKFhDhRapmNbSnaV19dW6epJpfqXAWfXKRzbGIgucbiC05SKkmQkWQ19fiCmVISbZTq2Jelf9pzUB+dcoyW3TtfmlqGBzbGNgQhsuILTlIqS9Osjr+v9tVWqmJg8w3DqEmdKRbhZpmNbkg6f7lSkNKj28z363YWeIcs5tjEQgQ3XaKirGVIz+vDpTj3+yyP6x5Xv1JaH7tJffuCtacuZUhFe4HRsp9z76LN64IltQ57n2MZg9LXANeprq/TIpgOS0kfT/ujFE/rRiycct2FKRXhBpmN7OBzbGIwzbLgGUyrCrzi2kQsENlwlNaViMGCYUhG+wnShuFx0icN15sys0JmqyVp782xfTqmI4uX36UIxvghsuJKRfDWlIpDit+lCkT8ENlwvNaUi4Dcc2xgNrmEDAOABBDYAAB5AYAMA4AEENgAAHkBgAwDgAQQ2AAAeQGADBRSLJ3T+YlTxDNMvAkAK92EDedYTi2tzS5teP92pT63ZMqDSVblW1dWovraKSlcAhuAMG8ij3a0dWrB2q9Zs3KeL0bislaJxK2ulg+2dWrNxnxas3ao9rR2F3lUALkNgA3myp7VDD6zbpo7uqLp645oUKdNTD9+dtk5Xb1wd3VEtW7eN0AaQhsAG8qAnFtfy9TvUHc1uPuTuaHL9nlj28ycD8DcCG8iDzS1tisYTGZfPnFqmn33+Tr1txhX9z0XjCW1pOZWP3QPgAQQ2kAeNzUfV1et8tnz9VRF98w9v15/9017tPX6u//mu3rgam4/kaxcBuByjxIFxFk9YHT7d6bhsaqRET3xynlZ9f5fjOodOdyqesEy9CIAzbGC8dfXGFMoQuBcuRnXyXLfmzZriuDwUMOrqjY3n7gHwCAIbGGeRkpBiGQqjRONWK7+7Sx++bYY+NOeaIctjCavSYIDiKgDoEgfGWzBgNLuyXIfanbvFu6Nxffo7O/W9Ty9QdzSuXxxo718WDgR085f/leIqADjDBvKhoa5GkZL0gL3Q1a17Hn1GknT+Ykz3Pf7rtLCWpN54guIqACQR2EBe1NdWKRy8vF+3h98zW5+563qKqwBFisBGUSj0JBuloaA2rJivsnDuurEprgIUF65hw7dSk2w0Nh/V4dOdBb8OPGdmhZpWLtTy9Tv6iqiMPPr7s4tu0Idvq1Zbx0Wd6epRy4nzactTxVWWzq0ep70G4BacYcOXBk6ycai90zXXgefMrND21Yu19v5aTQgHZYwUDhoZI5UM6jK/tXqyPjinSu9/7Fmt+v4uvW1GxZCfR3EVoHgQ2PCdwZNsTJ4Q0h8uvDZtnUJeBy4NBbV0brVmV5bryNp67frL9+rgV5comkgvXTp/1lQ9tb9dF6MJdfbE9PTL7Y4/L1VcBYC/EdjwFadJNiaXhfWJQYGdUujrwMGA0eQJYV2MxZ2Lq9iRg5jiKkBxILDhK06TbPyXe2/WtVdO1ObP36m/WHLzkG3cMMmGU3GV7a+e0ftuma7SUECRkqAWv3Wa47axhFWkhOEogN/xWw5fcZpk42//9Te6cdok1T/2nOM2qevAhRy45VRcZf/J83pyb5s2P3SXTpzt1s5jZxy3vbGynFrjQBEgsOEbw02yMRI3TLLRUFejNRv3pf3B8fgvj+jxX2YeVBYpCaqh7oZ87B6AAqNLHL4x3CQbI3HDdeCxFFcJBwNaUjt9nPYIgJsQ2PCNTJNsdPbEFCkd/n5rN1wHHm1xlbJwcn1qigPFgcCGb6SuAw/W8WZUu147q6cevttx0JnknuvAqeIqFWXhIbXHUyIlQVWUhdW0cqHmzKzI7w4CKBiuYcNXnK4DS9JDTbszbuO268Cp4ipbWk6psfmIDqVVaZukhroaLamdzpk1UGQIbPhKfW2VHtl0QFL291W78TpwqrjK0rnViiesunpjipSEXNELAKAw6BKHr/jxOnCquAphDRQ3Ahu+4/XrwIWeWQyAO9ElDl/y2nVgt80sBsB9CGz4lleuA+9u7dCDfVNupgbLRePJs+vUzGKPbDqgDSvmu643AED+0CWOouDW68CDZxZzUsiZxQC4B4ENFIiVhswsNpxCzywGoLDoEgcK5Fx3NG1msY8veIs+vuAtkqRJE8I6frZbDzyxLW2b1MxihZyoBEBhcIYNFMjvLvSkdYP/v+2/Vf1jz+lD3/i12s5d1Leee2XINqmZxQAUHwIbKIB4wupihq7wr3zwFv370de19eXTjstTM4sBKC4ENlAAXb0xGTN0ANxHb5+h6illenTr4YzbumFmMQD5xzVsoAAiJSFZm36WfGv1ZH3mruv1sf/7vOwwJ9BumFkMQP7l5AzbGHOvMeagMeaIMeZLDsvrjDHnjDG7+76+nIvXRXHxUwWwYMBowqDyqcvfOUsVE8Nq+sxCbf78nfqbj9Q6buuWmcUA5Ndl/5lujAlKelzSeyUdl7TTGLPJWntg0KrPWms/cLmvh+Li5wpgV08qVaQk3j/w7M/+ee+I27htZjEA+ZOLfrX5ko5Ya1+RJGNMk6T7JA0ObGBU/F4B7IqysMJBK6/PLAYgP8zg62ij/gHGfFTSvdbaP+p7/AlJC6y1nxuwTp2kHyl5Bn5S0hettfsz/LyVklZK0rRp025vamoa9T51dnaqvLx81Nt5id/beP7CBbVesEpkcXwGjNH1V0eynqHLLTo7OxUsLdMrv+vyZTv9foxKtNEP3Na+RYsW7bLWznNaloszbKeLaYM/fV6UdK21ttMYUy/pJ5JmO/0wa+06Seskad68ebaurm7UO9Tc3KyxbOclfm5jTyyub//Tz/Q/9mYfTBVlcW1fXeep7vHUe7intUPLB/UkDBQpCSocDHiuJ8HPx2gKbfQ+L7UvF4POjkuaOeDxDCXPovtZa89bazv7vt8sKWyMuSoHrw0f2tzSNuwoaSepCmBelJpZbO39tbppWrmMkcJBI2Okm6ZN0tr7a7V99WJPhTWA3MvFGfZOSbONMddJOiFpmaT/OHAFY8x0Se3WWmuMma/kHwpv5OC14UONzUf1wcr0xC4LB/X4x29T1RUTFDBG/+ffDuvJvW39y1MVwLxastMrM4sBKJzLDmxrbcwY8zlJT0kKSlpvrd1vjFnVt/ybkj4qqcEYE5PULWmZvdyL5/CleMLq8OlOqTL9+XffdLXaz1/Uiu/slCRNKh166KYqgHk95FIziwHAQDmpvtDXzb150HPfHPD9NyR9IxevBX/r6o0p5BC4B09d0Or6t+pL996srb9p185jZ4esk6oARtgB8CNKk8JVIiUhxRwKo7z6epc+8H+e029OXdCf33uzPr946L3IVAAD4Gd8usFVggGj2ZXlkjrSnq+cVKpz3VH9ZPcJvdkb00dvnzFkWyqAIRZP6M1onOv/8CUCG67TUFejky+/mPbczdMn6S/q3yprraJxqzU/2Ze2nApgxcvP1fCAgQhsuE59bZW+/Zv05545/Lqe+fqzGbehAlhx8ns1PGAgrmHDdUpDQV13VfYVvcrCQW1YMZ+zqCKzp7VDD6zbpo7uaH9Y73/knrR1unrj6uiOatm6bdrT2lGAvQRyh8CGK5WFg2pauVAVZWFFSpyDOFISVEVZWE0rF3L2VGR6YnEtX79D3dHs6rB3R5Pr98Syr9sOuA2BDdeiAhgy2dzSpmg8MaptvFwND5C4hg2XowIYnDQ2H3Wsuz4cr1fDAzjDhmekKoC5Jaxj8YTOX4wq7nDfOMZPfzW8MUhVwwO8iDNsYBS4hajwUtXwUqPBR4NqePAyAhvIErcQuUOmanjZoBoevIwucSALTrcQDcYtRPlxqRre6FEND15GYAMj4BYi92moq3G83e+WrzyVcRuq4cHrCGxgBE63EL1txhXa8tBdKg0FVBYO6ud/crdunHbprI9biMZXfW2VwsHRfXxRDS/3GHiZX1zMAUbgdAvR3uPn9PTL7frT992kCeGAfvLSCR1qvzRymVuIxldpKFndbtm6bVn1fFANL3cYeFk4nGEDwxjuFqLHth7WXbOv0tuqr9A3f3V0yHJuIRpfc2ZWUA0vz3a3dmjB2q1as3GfDrV3ytrkwEtrLw28XLB2K2M4xgmBDQwjdQuRk4qyEk0sCSpSGnI8o0jdQoTxQzW8/GHgZeHRJQ4MY7hbiP7bh2v1tZ8f0sypE/WlJTfrK5v2py3nFqL8oBre+BvrwMvtqxfTPZ5DfJoAw0jdQjTw+rQkffi2ZDhs2nNSASP9uOEOvbPmSv370Tf61+EWovxLVcNDbg0ceDljSpk2fGq+dr52RnNnTtHLbef1T7ta9SfvuVFXlpfq4aaXtOf4uf6Bl4zjyB26xIERON1C9OMXT2jV93dJkhJWWvp3z6eFNbcQwU8GD7y89sqJ+vvnjunerz+jmqvLdd/bq/XRb/67/tvml/XZRcnjPjXwErlDYAMj8MItRNxeg/E0eOBl69luHWy/IGulQ6cv6NdHXpck/ebUec2YUta/HgMvc4sucWAEbr2FiNtrkA8Ja4fUbu+NXapLYK3tf2ytFAxc+uOW2u25xRk2kAW33ULE7TXIl4Ax1G53CQIbyJJbbiHi9hrkG7Xb3YE/fYBRKPQtRNxeg0JoqKvRmo371NUb1/Gz3brn0Wf6l33xn/b2fz9wGQMvc48zbGCMUrcQ5fMMwqmuuRMzYJeoa47L5YWBl8WAwAY8xKmuuZS8N/bpL7xbX73vVv3s83fqmisujdTl9hpcrtTAy7Jwdr001G4fHwQ24BHD1TWXpOuviujHLx7X+x97Tic6utOWcXsNLpfbBl4WI65hAx6Rqms+8PaagU50dOulDAPMuL0GuZAaeLml5ZQam4/oUNrthJPUUFejJbXTObMeJwQ24BHD1TWXpDczjBiXuL0GuVPogZfFjC5xwCNSdc3HgttrMB4KMfCymBHYgIc41TUfCbfXAP5AYAMekun2msH3xg7E7TWAPxDYgIdwew1QvAhswGO4vQYoTgwbBTyI22uA4kNgAx7F7TVAcSGwAR9I3V4DwL+4hg0AgAcQ2AAAeEDRBHYsntD5i1EmQAAAeJKvr2H3xOLa3NKmxuajOpw2irZcq+pqVF9bxShaAIAn+Dawd7d26MH1OxSNJ/rnD07NcnSwvVNrNu7TI5sOaMOK+dynCgBwPV92iXdH43pg3TZ1dEf7w3qwrt64OrqjWrZum/ZkmJIQAAC38F1g98TievX1LnVHM081OFB3NK7l63eoJ5bd+gAAFILvAntzS5vsKMeVReMJbWk5NT47BABADvgusBubjyoxKLHXfeJ2/cvn7tTP/+RuPTB/5pBtunrjamw+kq9dBABg1Hw16CyesDp8ulOqTH/+z/55r851R1UaCmjT5+7Uln2n1PFmNG2dQ6c7FU9YyjoCAFzJV4Hd1RtTyCFwP/WuWbrnluR8wFUVE3TdlRG99GZH2jqhgFFXb4zyjgAAV/JVYEdKQooNKoyy8PqpetcNV+n+v/u1LkYTalq5UKXhoVcCYgmrSImv/jsAAD7iq2vYwYDR7MrytOcmTQjrXHdUF6MJ1Vwd0dwM91zfWFlOdzgAwLV8FdiS1FBXo4C5FLy/Ovg7hQJGWx66S1947016yeGe60hJUA11N+RxLwEAGB3f9QHX11bp27+59Lg3ntCDf79z2G3CwYCW1E4f5z0DAGDsfHeGXRoK6rqrIioLZ1cjvCwc1IYV86kpDgBwNd8FtpQM4aaVC1VRFlakxDmIIyVBVZSF1bRyIbXEAQCu57su8ZQ5Myu0ffVibWk5pcbmIzqUNlvXJDXU1WhJ7XTOrAEAnuDbwJaS3eNL51Zr6dxqxRNWXb0xRUpCjAYHAHiOrwN7oGDAUBQFAOBZvryGDQCA3xDYAJBHsXhC5y9GFU+MclpBFL2i6RIHgELpicW1uaVNjc1HdThtAGy5VtXVqL62igGwGBGBDQDjaHdrhx5cv0PReEJdvXFJUjSePLs+2N6pNRv36ZFNB7RhxXxuMcWw6BIHgHGyp7VDD6zbpo7uqKZESvTUw3cPWaerN66O7qiWrdumPQ6lk4EUAhsAxkFPLK7l63eoOxrPav3uaHL9nlh266P4ENgAMA42t7QpGk+kPRcMGP31h2v18z+5W99dMV+lofSP4Gg8oS0tp/K5m/AQAhsAxkFj89H+a9Yps66cqO/9+2t63/9+RucvRrXk1qq05V29cTU2H8nnbsJDCGwAyLF4wurw6c4hz7ee7daBtvOSpH0nzmnG1LIh6xw63cktX3BEYANAjnX1xhRyKIHcG7vURR5PyHGdUMCoqzc2rvsHbyKwPYSCC4A3REpCio3x9zSWsIqUcMcthuKocLlMBRf+61yrsy8dp+AC4ELBgNHsynIdah/aLT6SGyvLmaAIjghsFxuu4MLFaJyCC4CLNdTVaM3Gff2/u8fPduueR5/pX/7Es68M2SZSElRD3Q1520d4C13iLjWw4MLgkaYpFFwA3Ku+tkrh4Og+YsPBgJbUTh+nPYLXEdguRMEFwPtKQ0FtWDFfZeHsLlmVhZPrc4kLmRDYLuRUcEGSPn3ndXrq4bv11MN3a85N16cto+AC4D5zZlaoaeVCVZSFFSlxDuJISVAVZWE1rVzIpS0Mi2vYLuRUcOHW6sn62LwZWvr4r2WM9OwX7tAt13Ro/8nkPZ2pggtL51YXYpcBZDBnZoW2r16sLS2n1Nh8RIfSZuuapIa6Gi2pnc6ZNUZEYLtMpoIL75g1VU/tb+/vJj/a2qZ3zJraH9jSpYILjDAF3KU0FNTSudVaOrda8YRVV29MkZIQv6sYFbrEXSZTwYVsfq0puAC4XzBgNHlCmLDGqBHYLpOp4ML2V8/ofb83TRPCAZWFg6qZWaWdx86krUPBBcB9KHiEXOHT3WUyFVzYf/K8/nnXcf30s3cmHx85ltYdLlFwAXCLTAWPbqws16q6GgoeYUwIbBcaXHAh5dvPvapvP/eqJOlPa2Ma+PZRcAFwh+EKHh1s76TgEcaMLnEXouAC4E0UPMJ4IrBdiIILgPcMV/DoRw13DHmOgkcYLQLbpSi4AHhLpoJHkvSRxucdn6fgEUaDwHaxVMGFtffX6qZp5TJGCgeNjJEmhINae3+ttq9eTFgDLuBU8Chl/yP3OD6fKngEZCMng86MMfdK+rqkoKRvWWv/ZtBy07e8XtKbkh601r6Yi9f2u0wFF5595leqo6rZmMTiCb0ZjVO4AjmTqeBRNih4hGxddmAbY4KSHpf0XknHJe00xmyy1h4YsNoSSbP7vhZIauz7F6OQKriA0eM2G4ynVMGj1Gjw0UgVPOJ3GyPJRZf4fElHrLWvWGt7JTVJum/QOvdJ+q5N2iapwhhTlYPXBka0u7VDC9Zu1ZqN+3SovVPWJm+zsfbSbTYL1m5lxG6B+KGwSKaCR9mg4JF35fvYzcVRUi2pdcDj4xp69uy0TrWkthy8PpBR6jab4aYqTV53jGvZum0M4MsTv/V4ZCp4lA0KHnlLIY9dY+3l/WVgjPmYpHustX/U9/gTkuZba/94wDo/k/TX1trn+h5vlfTn1tpdDj9vpaSVkjRt2rTbm5qaRr1PnZ2dKi8vH0tzPMPvbcxF+6ykl9vOj+qv32DA6K1Vk7Oq3X65ivU97I7G9errXbJWSjh8/gRMcmDldVdFsr61sVAGtrGjO6oTZ7sd2/Sf/uD9+r8//NmQ5wPGqHpKmSrK3NsdXqzHqZN8HLuLFi3aZa2d57QsF2fYxyXNHPB4hqSTY1hHkmStXSdpnSTNmzfP1tXVjXqHmpubNZbtvMTvbcxF+za+dFyP/9vQinHDiZQEtfbm2XmZprQY38M9rR1qWLdN3dGRP8zKwlE1rbzd1T0eA9vYE4trwdqt6uiOpq1TMTGs+zt79bWWoR+3FWVhbV/9+67uTSjG49SJG47dXFzD3ilptjHmOmNMiaRlkjYNWmeTpE+apIWSzllr6Q7HuBp8m80X3nujPvWuWf2Pv/i+m/TgHbPStuE2m/EzXGERJ14rLOJU8KhyUql+3HCHnnj21SHrU/DIO9xy7F52YFtrY5I+J+kpSS9L+qG1dr8xZpUxZlXfapslvSLpiKQnJP3ny31dYDhOt9n88IVWfeS2GZIkY6QPzqnST3afGLJt6jYb5NZwhUUy8VphkcEFj05f6NHvf+1X2vD8sf51KHjkPU7H7mcX3aCtf/puff/TC/TYsrfrM3ddn7Z8PI7dnAxNtNZuVjKUBz73zQHfW0mfzcVrAdlwus3m+NlunX2zV7dcM1lXlZdq/8nz6ngzOmRbbrMZH06FRWZMKdN3PjVfO4+d0e3XTtGpcxf1me++oJ5Y8sMx1eORj0sUuZIqeLSl5ZQam4/oUNrApElqqKvRktrpnFl7yOBj99bqyfrgnCq9/7FnFQoE9OQf36mWE+mzJ47Hscu9BPClTLfZ/OPOVn309hm6urxUP3yh1WFLbrMZD8MVFpl15UR9/gcv6S9+3KJv/Me5WnJres+HFwuLZCp45KU2IMnp2J0/a6qe2t+ui9GEpISefrndcdtcH7uUJoUvpW6zGeyp/ad0941X620zKvTMod85bsttNrmX6vFw0nq2Wwfakmcn+06c04ypZWnLUz0eXpUqeMQx5U0Zj90s7rDK9bFLYMO3GupqhkycEo1bbTv6hn7WclJOl6mZV3x8DFdYpDd26dpgPKEhH470eKCQnI7d7a+e0ftuma7SUECRkqAWv3Wa47a5PnYJbPiW07zixkhz31Khf9zp3B3OvOLjI1OPRzbo8UAhOR27+0+e15N727T5obvU+Ie3a+exM47b5vrYJbDhW4Nvs7mhsly/+uIi/froGzr2xptD1uc2m/Hl1OMxEno84AZOx+7jvzyixV/7lT65fodOdnQP2WY8jl0CG7428Dabto5u3f0/fqm1P3s5bR1us8kPpx6P42e7dc+jz/Q/fuLZV/To04f7H9PjATdwOnZHMh7HLoEN3xtuXvGbpk1iXvE8cSosMhx6POAWIx27jz59WE88+0r/4/E6dhnJgaLAbTbukOrxWL5+h6LxhGPZ2EhJUOFgQBtWzOePKLiGG45dAhtFh3nFC4vCIvCqQh+7BDaAvKPHA15VyGOXwAZQUPR4wKvyfewy6AwAAA8gsAEA8AACGwAADyCwAQDwAAIbAAAPILABAPAAAhsAAA8gsAEA8AACGwAADyCwAQDwAAI7S7F4QucvRhVP2ELvCgCgCFFLfBg9sbg2t7SpsfmoDqfNylKuVXU1qq+tYkYhAEBeENgZ7G7t0IOD5j2NxpNn1wfbO7Vm4z49sukAc/YCAPKCLnEHe1o79MC6berojjpOUi5JXb1xdXRHtWzdNu1p7cjvDgIAig6BPUhPLK7l63eoO+oc1IN1R5Pr98SyWx8AgLGgS3yQzS1tisYTac8tfXu1HnzXLJUEjXa3dmjNT/Zp4NizaDyhLS2ntHRudZ73FgBQLDjDHqSx+WhaN3jN1eX6wJwqfbTxedU/9pziCQ0J5q7euBqbj+R7VwEARYQz7AHiCavDpzvTnnvXDVeqtvoKbfrcuyRJpeGg3ujqGbLtodOdiiesggGTl30FABQXAnuArt6YQgHTPxpckowx+tGu4/rvTx0cdttQwKirN6bJE8LjvZsAgCJEl/gAkZKQYoMKo/z6yOtaUlulKyMlkqQrysKqrigbsm0sYRUp4e8fAMD4IGEGCAaMZleW61D7pW7xI6c79bWfH9T3Pj1fxhjF4lZf/uk+nejoTtv2xspyusMBAOOGwB6koa5GazbuSxt49uTeNj25ty3jNpGSoBrqbsjH7gEAihRd4oPU11YpHBzdf0s4GNCS2unjtEcAABDYQ5SGgtqwYr7KwtnVCC8LJ9enpjgAYDwR2A7mzKxQ08qFqigLK1LiHMSRkqAqysJqWrmQWuIAgHHHNewM5sys0PbVi7Wl5ZQam4/oUNpsXZPUUFejJbXTObMGAOQFgT2M0lBQS+dWa+ncasUTVl29MUVKQowGBwDkHYGdpWDAUBQFAFAwXMMGAMADCGwAADyAwAYAwAMIbAAAPIDABgDAAwhsAAA8gMAGAMADCGwAADyAwAYAwAMIbAAAPIDABgDAAwhsAAA8gMAGAMADCGwAADyAwAYAwAMIbAAAPIDABgDAAwhsAAA8gMAGAMADCGwAADyAwAYAwAMIbAAAPIDABgDAAwhsAAA8gMAGAMADCOwiEIsndP5iVPGELfSuAADGKFToHcD46InFtbmlTY3NR3X4dKdCAaNYwurGynKtqqtRfW2VSkPBQu8mACBLBLYP7W7t0IPrdygaT6irNy5JisaTZ9cH2zu1ZuM+PbLpgDasmK85MysKuKcAgGzRJe4ze1o79MC6berojvaHtST9qOGO/u+7euPq6I5q2bpt2tPaUYC9BACMFoHtIz2xuJav36HuaHzIso80Pj/kue5ocv2e2ND1AQDuQmD7yOaWNkXjCcdl+x+5x/H5aDyhLS2nxnO3AAA5QGD7SGPz0bRu8Gx09cbV2HxknPYIAJArBLZPxBNWh093jmnbQ6c7ueULAFyOwPaJrt6YQgEzpm1DAaOu3liO9wgAkEsEtk9ESkKKjfEsOZawipRwhx8AuBmB7RPBgNHsyvIxbXtjZbmCYzw7BwDkB4HtIw11NYqUOFcvu+UrTzk+HykJqqHuhvHcLSAvKMELv6Mf1Efqa6v0yKYDkrIfKR4OBrSkdvr47RQwjijBi2LCGbaPlIaC2rBivsrC2X1AlYWT6/OBBi/a3dqhBWu3as3GfTrU3ilrkyV4rb1UgnfB2q1U84NvENg+M2dmhZpWLlRFWThj93ikJKiKsrCaVi6kljg8KVMJ3oEowQu/IbB9aM7MCm1fvVhr76/VTdPKZYwUDhoZI900bZLW3l+r7asXE9bwpOFK8DqhBC/8gmvYPlUaCmrp3GotnVuteMKqqzemSEmI0eDwvOFK8GaSKsG7dG71OO0VMP44wy4CwYDR5Alhwhq+QAleFCsCG4BnUIIXxYzABuAZlOBFMbusa9jGmKmS/lHSLEnHJP2Btfasw3rHJF1Q8gbhmLV23uW8LoDiRAleFLPLPcP+kqSt1trZkrb2Pc5kkbX27YQ1gLEaqQTv3z/4DlVOKnVcRgleeN3lBvZ9kjb0fb9B0tLL/HkAMKzhSvB+6js7dfpCz5DnKcELP7jcwJ5mrW2TpL5/KzOsZyX93Bizyxiz8jJfE0ARq6+tUjg4uo8uSvDCD4y1w18PMsY8LcnpSF8taYO1tmLAumettVMcfsY11tqTxphKSb+Q9MfW2mcyvN5KSSsladq0abc3NTVl25Z+nZ2dKi8f28xVAyWsVcC4swstV210K7+3T/J/G8ezfd3RuF75XZcSI3x+SVLAGF1/dSTrkr2j4ff3UPJ/G93WvkWLFu3KdOl4xMAejjHmoKQ6a22bMaZKUrO19qYRtvkrSZ3W2v850s+fN2+efeGFF0a9X83Nzaqrqxv1dl6aSGCsbfQKv7dP8n8bx7t9e1o7tHz9DkXjCcf7siMlQYWDAW1YMX/cqvr5/T2U/N9Gt7XPGJMxsC+3S3yTpOV93y+X9FOHF48YYyalvpf0Pkn7LvN1c46JBABvoQQvis3l3uPwN5J+aIz5tKTfSvqYlOwCl/Qta229pGmSNppk13JI0j9Ya//1Ml83p1ITCQxXmzj5F3xcy9ZtY9IMwCUowYticlmBba19Q9Jih+dPSqrv+/4VSXMu53XG01gnEti+erFruscBXCrBC/hV0Vc6c5pIYMaUMj318N39jz9z1/V6+D2z+x+nJhIAACBfij6wmUgAAOAFRR3YTCQAAPCKog7sTBMJxOJWA58uDQ/9b2IiAQBAPhV1YGeaSOD1zh5dWV6qiolhlQQDWnzz0AJuTCQAAMinok6c1EQCh9rTu8VjCavHth7WT/7zu9R69k0d/d3QbnMmEgAA5FNRB7aUnEhgzcZ9Qwaefef5Y/rO88cct2EiAQBAvhV1l7jERAIAAG8o+sAuDQW1YcX8rCcGKAsn16doCgAgn4o+sKVkTeKmlQtVURbOOM9upCSoirIwZUkBAAVR9NewU1ITCWxpOaXG5iM6lDZb1yQ11NVoSe10zqwBAAVBYA/ARAIAALcisDNgIgEAgJtwDRsAAA8gsAEA8AACGwAADyCwAQDwAAIbAAAPILABAPAAAhsAAA8gsAEA8AACGwAADyCwAQDwAGOtLfQ+ZGSM+Z2k18aw6VWSXs/x7riN39vo9/ZJ/m+j39sn0UY/cFv7rrXWXu20wNWBPVbGmBestfMKvR/jye9t9Hv7JP+30e/tk2ijH3ipfXSJAwDgAQQ2AAAe4NfAXlfoHcgDv7fR7+2T/N9Gv7dPoo1+4Jn2+fIaNgAAfuPXM2wAAHzFs4FtjJlqjPmFMeZw379THNa5yRize8DXeWPMw33L/soYc2LAsvq8N2IE2bSxb71jxpiWvna8MNrtCyXL93CmMeaXxpiXjTH7jTEPDVjmyvfQGHOvMeagMeaIMeZLDsuNMeaxvuV7jTG3ZbutW2TRxo/3tW2vMeZ5Y8ycAcscj1c3yaJ9dcaYcwOOvS9nu61bZNHGPxvQvn3GmLgxZmrfMi+8h+uNMaeNMfsyLPfe76G11pNfkv67pC/1ff8lSX87wvpBSaeUvMdNkv5K0hcL3Y5ctFHSMUlXXe7/kRvbJ6lK0m1930+SdEjS77n1Pew7zo5Kul5SiaQ9qf0dsE69pC2SjKSFkrZnu60bvrJs4x2SpvR9vyTVxuGOV7d8Zdm+OklPjmVbN3yNdj8lfVDSv3nlPezbx7sl3SZpX4blnvs99OwZtqT7JG3o+36DpKUjrL9Y0lFr7VgKsRTKaNuY6+3H24j7Z61ts9a+2Pf9BUkvS6rO1w6OwXxJR6y1r1hreyU1KdnOge6T9F2btE1ShTGmKstt3WDE/bTWPm+tPdv3cJukGXnex8txOe+Db97DQR6Q9IO87FmOWGufkXRmmFU893vo5cCeZq1tk5If6pIqR1h/mYYecJ/r6wpZ77bu4j7ZttFK+rkxZpcxZuUYti+UUe2fMWaWpLmStg942m3vYbWk1gGPj2voHxiZ1slmWzcY7X5+WskzmZRMx6tbZNu+dxpj9hhjthhjbhnltoWW9X4aYyZKulfSjwY87fb3MBue+z0MFXoHhmOMeVrSdIdFq0f5c0okfUjSXwx4ulHSV5U88L4q6WuSVoxtT8cuR218l7X2pDGmUtIvjDG/6fvrsuBy+B6WK/mB8bC19nzf0654DwcxDs8NvhUj0zrZbOsGWe+nMWaRkoF954CnXXu89smmfS8qeXmts2/sxE8kzc5yWzcYzX5+UNKvrbUDz1bd/h5mw3O/h64ObGvtezItM8a0G2OqrLVtfd0Yp4f5UUskvWitbR/ws/u/N8Y8IenJXOzzaOWijdbak33/njbGbFSyS+cZSaP5PxoXuWifMSasZFj/P2vtjwf8bFe8h4MclzRzwOMZkk5muU5JFtu6QTZtlDHmbZK+JWmJtfaN1PPDHK9uMWL7BvzRKGvtZmPM3xljrspmW5cYzX4O6Z30wHuYDc/9Hnq5S3yTpOV93y+X9NNh1h1y/aUvIFLul+Q4krDARmyjMSZijJmU+l7S+3SpLaP5PyqEbNpnJH1b0svW2v81aJkb38OdkmYbY67r69lZpmQ7B9ok6ZN9o1QXSjrXd0kgm23dYMT9NMa8RdKPJX3CWntowPPDHa9ukU37pvcdmzLGzFfys/SNbLZ1iaz20xhzhaR3a8Dvpkfew2x47/ew0KPexvol6UpJWyUd7vt3at/z10jaPGC9iUr+Il0xaPvvSWqRtFfJN6Oq0G0aSxuVHMm4p+9rv6TVI23vlq8s23enkt1ReyXt7vuqd/N7qOTo00NKjjRd3ffcKkmr+r43kh7vW94iad5w27rxK4s2fkvS2QHv2QsjHa9u+sqifZ/r2/89Sg6qu8Nv72Hf4wclNQ3azivv4Q8ktUmKKnk2/Wmv/x5S6QwAAA/wcpc4AABFg8AGAMADCGwAADyAwAYAwAMIbAAAPIDABgDAAwhsAAA8gMAGAMAD/j+d5wZTiS2b2AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# visualize dimensions 0 and 1 of the embedding matrix C for all characters\n",
    "plt.figure(figsize=(8,8))\n",
    "plt.scatter(C[:,0].data, C[:,1].data, s=200)\n",
    "for i in range(C.shape[0]):\n",
    "    plt.text(C[i,0].item(), C[i,1].item(), itos[i], ha=\"center\", va=\"center\", color='white')\n",
    "plt.grid('minor')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# training split, dev/validation split, test split\n",
    "# 80%, 10%, 10%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 805,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 3, 10])"
      ]
     },
     "execution_count": 805,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "context = [0] * block_size\n",
    "C[torch.tensor([context])].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 820,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "carmahela.\n",
      "jhovi.\n",
      "kimrin.\n",
      "thil.\n",
      "halanna.\n",
      "jazhien.\n",
      "amerynci.\n",
      "aqui.\n",
      "nellara.\n",
      "chaiiv.\n",
      "kaleigh.\n",
      "ham.\n",
      "joce.\n",
      "quinton.\n",
      "lilea.\n",
      "jamilio.\n",
      "jeron.\n",
      "jaryni.\n",
      "jace.\n",
      "chrudeley.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "# sample from the model\n",
    "g = torch.Generator().manual_seed(2147483647 + 10)\n",
    "\n",
    "for _ in range(20):\n",
    "    \n",
    "    out = []\n",
    "    context = [0] * block_size # initialize with all ...\n",
    "    while True:\n",
    "      emb = C[torch.tensor([context])] # (1,block_size,d)\n",
    "      h = torch.tanh(emb.view(1, -1) @ W1 + b1)\n",
    "      logits = h @ W2 + b2\n",
    "      probs = F.softmax(logits, dim=1)\n",
    "      ix = torch.multinomial(probs, num_samples=1, generator=g).item()\n",
    "      context = context[1:] + [ix]\n",
    "      out.append(ix)\n",
    "      if ix == 0:\n",
    "        break\n",
    "    \n",
    "    print(''.join(itos[i] for i in out))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}