{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "name": "ResNet18_MNIST_Train", "provenance": [], "collapsed_sections": [], "authorship_tag": "ABX9TyM+8cvSUvYDVU2JoPqs679O", "include_colab_link": true }, "kernelspec": { "name": "python3", "display_name": "Python 3" }, "accelerator": "GPU" }, "cells": [ { "cell_type": "markdown", "metadata": { "id": "view-in-github", "colab_type": "text" }, "source": [ "\"Open" ] }, { "cell_type": "markdown", "metadata": { "id": "USQT9gpmntzy" }, "source": [ "#### ResNet18 모델 정의 및 인스턴스 초기화" ] }, { "cell_type": "code", "metadata": { "id": "iIDEevihnpsd" }, "source": [ "import torch\n", "import torch.nn as nn\n", "import torch.nn.functional as F\n", "import torch.backends.cudnn as cudnn\n", "import torch.optim as optim\n", "import os\n", "\n", "\n", "# ResNet18을 위해 최대한 간단히 수정한 BasicBlock 클래스 정의\n", "class BasicBlock(nn.Module):\n", " def __init__(self, in_planes, planes, stride=1):\n", " super(BasicBlock, self).__init__()\n", "\n", " # 3x3 필터를 사용 (너비와 높이를 줄일 때는 stride 값 조절)\n", " self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)\n", " self.bn1 = nn.BatchNorm2d(planes) # 배치 정규화(batch normalization)\n", "\n", " # 3x3 필터를 사용 (패딩을 1만큼 주기 때문에 너비와 높이가 동일)\n", " self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False)\n", " self.bn2 = nn.BatchNorm2d(planes) # 배치 정규화(batch normalization)\n", "\n", " self.shortcut = nn.Sequential() # identity인 경우\n", " if stride != 1: # stride가 1이 아니라면, Identity mapping이 아닌 경우\n", " self.shortcut = nn.Sequential(\n", " nn.Conv2d(in_planes, planes, kernel_size=1, stride=stride, bias=False),\n", " nn.BatchNorm2d(planes)\n", " )\n", "\n", " def forward(self, x):\n", " out = F.relu(self.bn1(self.conv1(x)))\n", " out = self.bn2(self.conv2(out))\n", " out += self.shortcut(x) # (핵심) skip connection\n", " out = F.relu(out)\n", " return out\n", "\n", "\n", "# ResNet 클래스 정의\n", "class ResNet(nn.Module):\n", " def __init__(self, block, num_blocks, num_classes=10):\n", " super(ResNet, self).__init__()\n", " self.in_planes = 64\n", "\n", " # 64개의 3x3 필터(filter)를 사용\n", " self.conv1 = nn.Conv2d(1, 64, kernel_size=3, stride=1, padding=1, bias=False)\n", " self.bn1 = nn.BatchNorm2d(64)\n", " self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1)\n", " self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2)\n", " self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2)\n", " self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2)\n", " self.linear = nn.Linear(512, num_classes)\n", "\n", " def _make_layer(self, block, planes, num_blocks, stride):\n", " strides = [stride] + [1] * (num_blocks - 1)\n", " layers = []\n", " for stride in strides:\n", " layers.append(block(self.in_planes, planes, stride))\n", " self.in_planes = planes # 다음 레이어를 위해 채널 수 변경\n", " return nn.Sequential(*layers)\n", "\n", " def forward(self, x):\n", " out = F.relu(self.bn1(self.conv1(x)))\n", " out = self.layer1(out)\n", " out = self.layer2(out)\n", " out = self.layer3(out)\n", " out = self.layer4(out)\n", " out = F.avg_pool2d(out, 4)\n", " out = out.view(out.size(0), -1)\n", " out = self.linear(out)\n", " return out\n", "\n", "\n", "# ResNet18 함수 정의\n", "def ResNet18():\n", " return ResNet(BasicBlock, [2, 2, 2, 2])" ], "execution_count": null, "outputs": [] }, { "cell_type": "markdown", "metadata": { "id": "w-_otYXNOphJ" }, "source": [ "#### 데이터셋(Dataset) 다운로드 및 불러오기" ] }, { "cell_type": "code", "metadata": { "id": "OuPNq8xxnyQM" }, "source": [ "import torchvision\n", "import torchvision.transforms as transforms\n", "\n", "transform_train = transforms.Compose([\n", " transforms.ToTensor(),\n", "])\n", "\n", "transform_test = transforms.Compose([\n", " transforms.ToTensor(),\n", "])\n", "\n", "train_dataset = torchvision.datasets.MNIST(root='./data', train=True, download=True, transform=transform_train)\n", "test_dataset = torchvision.datasets.MNIST(root='./data', train=False, download=True, transform=transform_test)\n", "\n", "train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=128, shuffle=True, num_workers=4)\n", "test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=100, shuffle=False, num_workers=4)" ], "execution_count": null, "outputs": [] }, { "cell_type": "markdown", "metadata": { "id": "ijRg44G4PTFX" }, "source": [ "#### 환경 설정 및 학습(Training) 함수 정의" ] }, { "cell_type": "code", "metadata": { "id": "5adk4WGbPRoE" }, "source": [ "device = 'cuda'\n", "\n", "net = ResNet18()\n", "net = net.to(device)\n", "net = torch.nn.DataParallel(net)\n", "cudnn.benchmark = True\n", "\n", "learning_rate = 0.01\n", "file_name = 'resnet18_mnist.pt'\n", "\n", "criterion = nn.CrossEntropyLoss()\n", "optimizer = optim.SGD(net.parameters(), lr=learning_rate, momentum=0.9, weight_decay=0.0002)\n", "\n", "\n", "def train(epoch):\n", " print('\\n[ Train epoch: %d ]' % epoch)\n", " net.train()\n", " train_loss = 0\n", " correct = 0\n", " total = 0\n", " for batch_idx, (inputs, targets) in enumerate(train_loader):\n", " inputs, targets = inputs.to(device), targets.to(device)\n", " optimizer.zero_grad()\n", "\n", " benign_outputs = net(inputs)\n", " loss = criterion(benign_outputs, targets)\n", " loss.backward()\n", "\n", " optimizer.step()\n", " train_loss += loss.item()\n", " _, predicted = benign_outputs.max(1)\n", "\n", " total += targets.size(0)\n", " correct += predicted.eq(targets).sum().item()\n", " \n", " if batch_idx % 100 == 0:\n", " print('\\nCurrent batch:', str(batch_idx))\n", " print('Current benign train accuracy:', str(predicted.eq(targets).sum().item() / targets.size(0)))\n", " print('Current benign train loss:', loss.item())\n", "\n", " print('\\nTotal benign train accuarcy:', 100. * correct / total)\n", " print('Total benign train loss:', train_loss)\n", "\n", "\n", "def test(epoch):\n", " print('\\n[ Test epoch: %d ]' % epoch)\n", " net.eval()\n", " loss = 0\n", " correct = 0\n", " total = 0\n", "\n", " for batch_idx, (inputs, targets) in enumerate(test_loader):\n", " inputs, targets = inputs.to(device), targets.to(device)\n", " total += targets.size(0)\n", "\n", " outputs = net(inputs)\n", " loss += criterion(outputs, targets).item()\n", "\n", " _, predicted = outputs.max(1)\n", " correct += predicted.eq(targets).sum().item()\n", "\n", " print('\\nTest accuarcy:', 100. * correct / total)\n", " print('Test average loss:', loss / total)\n", "\n", " state = {\n", " 'net': net.state_dict()\n", " }\n", " if not os.path.isdir('checkpoint'):\n", " os.mkdir('checkpoint')\n", " torch.save(state, './checkpoint/' + file_name)\n", " print('Model Saved!')\n", "\n", "\n", "def adjust_learning_rate(optimizer, epoch):\n", " lr = learning_rate\n", " if epoch >= 5:\n", " lr /= 10\n", " for param_group in optimizer.param_groups:\n", " param_group['lr'] = lr" ], "execution_count": null, "outputs": [] }, { "cell_type": "markdown", "metadata": { "id": "yeWOjfJwPVh2" }, "source": [ "#### 학습(Training) 진행\n", "\n", "* MNIST 데이터셋에 대하여 전체 10 epoch로 99.5% test accuracy를 시연할 수 있습니다." ] }, { "cell_type": "code", "metadata": { "id": "s2rvjJ_fPWKJ", "outputId": "895900bb-5237-4d21-a8ee-b5279ecd7bea", "colab": { "base_uri": "https://localhost:8080/", "height": 1000 } }, "source": [ "for epoch in range(0, 10):\n", " adjust_learning_rate(optimizer, epoch)\n", " train(epoch)\n", " test(epoch)" ], "execution_count": null, "outputs": [ { "output_type": "stream", "text": [ "\n", "[ Train epoch: 0 ]\n", "\n", "Current batch: 0\n", "Current benign train accuracy: 0.109375\n", "Current benign train loss: 2.457585334777832\n", "\n", "Current batch: 100\n", "Current benign train accuracy: 0.9921875\n", "Current benign train loss: 0.04392915591597557\n", "\n", "Current batch: 200\n", "Current benign train accuracy: 0.984375\n", "Current benign train loss: 0.05753253400325775\n", "\n", "Current batch: 300\n", "Current benign train accuracy: 0.984375\n", "Current benign train loss: 0.06585847586393356\n", "\n", "Current batch: 400\n", "Current benign train accuracy: 0.9921875\n", "Current benign train loss: 0.04449915513396263\n", "\n", "Total benign train accuarcy: 96.08666666666667\n", "Total benign train loss: 58.918862423161045\n", "\n", "[ Test epoch: 0 ]\n", "\n", "Test accuarcy: 98.89\n", "Test average loss: 0.0003533052503829822\n", "Model Saved!\n", "\n", "[ Train epoch: 1 ]\n", "\n", "Current batch: 0\n", "Current benign train accuracy: 0.984375\n", "Current benign train loss: 0.042657215148210526\n", "\n", "Current batch: 100\n", "Current benign train accuracy: 1.0\n", "Current benign train loss: 0.009428155608475208\n", "\n", "Current batch: 200\n", "Current benign train accuracy: 1.0\n", "Current benign train loss: 0.007753262296319008\n", "\n", "Current batch: 300\n", "Current benign train accuracy: 0.984375\n", "Current benign train loss: 0.0654078871011734\n", "\n", "Current batch: 400\n", "Current benign train accuracy: 1.0\n", "Current benign train loss: 0.00916474498808384\n", "\n", "Total benign train accuarcy: 99.285\n", "Total benign train loss: 11.052289012935944\n", "\n", "[ Test epoch: 1 ]\n", "\n", "Test accuarcy: 99.01\n", "Test average loss: 0.0003090664865274448\n", "Model Saved!\n", "\n", "[ Train epoch: 2 ]\n", "\n", "Current batch: 0\n", "Current benign train accuracy: 1.0\n", "Current benign train loss: 0.0018334293272346258\n", "\n", "Current batch: 100\n", "Current benign train accuracy: 1.0\n", "Current benign train loss: 0.008323779329657555\n", "\n", "Current batch: 200\n", "Current benign train accuracy: 1.0\n", "Current benign train loss: 0.007550822105258703\n", "\n", "Current batch: 300\n", "Current benign train accuracy: 0.9921875\n", "Current benign train loss: 0.00899261049926281\n", "\n", "Current batch: 400\n", "Current benign train accuracy: 0.9921875\n", "Current benign train loss: 0.022175714373588562\n", "\n", "Total benign train accuarcy: 99.60166666666667\n", "Total benign train loss: 6.371826991671696\n", "\n", "[ Test epoch: 2 ]\n", "\n", "Test accuarcy: 99.22\n", "Test average loss: 0.00024446971391153057\n", "Model Saved!\n", "\n", "[ Train epoch: 3 ]\n", "\n", "Current batch: 0\n", "Current benign train accuracy: 1.0\n", "Current benign train loss: 0.005485222674906254\n", "\n", "Current batch: 100\n", "Current benign train accuracy: 1.0\n", "Current benign train loss: 0.008616755716502666\n", "\n", "Current batch: 200\n", "Current benign train accuracy: 1.0\n", "Current benign train loss: 0.0008379603968933225\n", "\n", "Current batch: 300\n", "Current benign train accuracy: 0.9921875\n", "Current benign train loss: 0.03002006560564041\n", "\n", "Current batch: 400\n", "Current benign train accuracy: 0.9921875\n", "Current benign train loss: 0.009678397327661514\n", "\n", "Total benign train accuarcy: 99.73666666666666\n", "Total benign train loss: 4.193828289353405\n", "\n", "[ Test epoch: 3 ]\n", "\n", "Test accuarcy: 99.32\n", "Test average loss: 0.00020407700923424273\n", "Model Saved!\n", "\n", "[ Train epoch: 4 ]\n", "\n", "Current batch: 0\n", "Current benign train accuracy: 1.0\n", "Current benign train loss: 0.001628458732739091\n", "\n", "Current batch: 100\n", "Current benign train accuracy: 0.9921875\n", "Current benign train loss: 0.009311182424426079\n", "\n", "Current batch: 200\n", "Current benign train accuracy: 1.0\n", "Current benign train loss: 0.00027747839340008795\n", "\n", "Current batch: 300\n", "Current benign train accuracy: 1.0\n", "Current benign train loss: 0.0009844534797593951\n", "\n", "Current batch: 400\n", "Current benign train accuracy: 1.0\n", "Current benign train loss: 0.005588078405708075\n", "\n", "Total benign train accuarcy: 99.85833333333333\n", "Total benign train loss: 2.157053777445981\n", "\n", "[ Test epoch: 4 ]\n", "\n", "Test accuarcy: 99.49\n", "Test average loss: 0.00017496016312088615\n", "Model Saved!\n", "\n", "[ Train epoch: 5 ]\n", "\n", "Current batch: 0\n", "Current benign train accuracy: 1.0\n", "Current benign train loss: 0.003467789851129055\n", "\n", "Current batch: 100\n", "Current benign train accuracy: 1.0\n", "Current benign train loss: 0.0008459117962047458\n", "\n", "Current batch: 200\n", "Current benign train accuracy: 1.0\n", "Current benign train loss: 0.00043548387475311756\n", "\n", "Current batch: 300\n", "Current benign train accuracy: 1.0\n", "Current benign train loss: 0.0018839928088709712\n", "\n", "Current batch: 400\n", "Current benign train accuracy: 1.0\n", "Current benign train loss: 0.0005425138515420258\n", "\n", "Total benign train accuarcy: 99.96\n", "Total benign train loss: 0.9033363241396728\n", "\n", "[ Test epoch: 5 ]\n", "\n", "Test accuarcy: 99.57\n", "Test average loss: 0.00014637001545461317\n", "Model Saved!\n", "\n", "[ Train epoch: 6 ]\n", "\n", "Current batch: 0\n", "Current benign train accuracy: 1.0\n", "Current benign train loss: 0.0004341896856203675\n", "\n", "Current batch: 100\n", "Current benign train accuracy: 1.0\n", "Current benign train loss: 0.002498171990737319\n", "\n", "Current batch: 200\n", "Current benign train accuracy: 1.0\n", "Current benign train loss: 0.0013761294540017843\n", "\n", "Current batch: 300\n", "Current benign train accuracy: 1.0\n", "Current benign train loss: 0.000202844719751738\n", "\n", "Current batch: 400\n", "Current benign train accuracy: 1.0\n", "Current benign train loss: 0.0008309237309731543\n", "\n", "Total benign train accuarcy: 99.99166666666666\n", "Total benign train loss: 0.4757085992023349\n", "\n", "[ Test epoch: 6 ]\n", "\n", "Test accuarcy: 99.53\n", "Test average loss: 0.0001422469148723849\n", "Model Saved!\n", "\n", "[ Train epoch: 7 ]\n", "\n", "Current batch: 0\n", "Current benign train accuracy: 1.0\n", "Current benign train loss: 0.0015783451963216066\n", "\n", "Current batch: 100\n", "Current benign train accuracy: 1.0\n", "Current benign train loss: 0.0007505526300519705\n", "\n", "Current batch: 200\n", "Current benign train accuracy: 1.0\n", "Current benign train loss: 9.658330964157358e-05\n", "\n", "Current batch: 300\n", "Current benign train accuracy: 1.0\n", "Current benign train loss: 0.0004911551368422806\n", "\n", "Current batch: 400\n", "Current benign train accuracy: 1.0\n", "Current benign train loss: 0.0015762506518512964\n", "\n", "Total benign train accuarcy: 99.99833333333333\n", "Total benign train loss: 0.36925776132920873\n", "\n", "[ Test epoch: 7 ]\n", "\n", "Test accuarcy: 99.54\n", "Test average loss: 0.00014345818043552753\n", "Model Saved!\n", "\n", "[ Train epoch: 8 ]\n", "\n", "Current batch: 0\n", "Current benign train accuracy: 1.0\n", "Current benign train loss: 0.00039367773570120335\n", "\n", "Current batch: 100\n", "Current benign train accuracy: 1.0\n", "Current benign train loss: 0.00025876687141135335\n", "\n", "Current batch: 200\n", "Current benign train accuracy: 1.0\n", "Current benign train loss: 0.0009910550434142351\n", "\n", "Current batch: 300\n", "Current benign train accuracy: 1.0\n", "Current benign train loss: 0.0031274878419935703\n", "\n", "Current batch: 400\n", "Current benign train accuracy: 1.0\n", "Current benign train loss: 0.0006479751318693161\n", "\n", "Total benign train accuarcy: 99.99833333333333\n", "Total benign train loss: 0.3608664751664037\n", "\n", "[ Test epoch: 8 ]\n", "\n", "Test accuarcy: 99.55\n", "Test average loss: 0.0001472517613638047\n", "Model Saved!\n", "\n", "[ Train epoch: 9 ]\n", "\n", "Current batch: 0\n", "Current benign train accuracy: 1.0\n", "Current benign train loss: 0.002792907180264592\n", "\n", "Current batch: 100\n", "Current benign train accuracy: 1.0\n", "Current benign train loss: 0.000621031504124403\n", "\n", "Current batch: 200\n", "Current benign train accuracy: 1.0\n", "Current benign train loss: 0.0006191624561324716\n", "\n", "Current batch: 300\n", "Current benign train accuracy: 1.0\n", "Current benign train loss: 0.0001497315097367391\n", "\n", "Current batch: 400\n", "Current benign train accuracy: 1.0\n", "Current benign train loss: 0.0004953081370331347\n", "\n", "Total benign train accuarcy: 100.0\n", "Total benign train loss: 0.2829050283471588\n", "\n", "[ Test epoch: 9 ]\n", "\n", "Test accuarcy: 99.54\n", "Test average loss: 0.0001376936424314863\n", "Model Saved!\n" ], "name": "stdout" } ] } ] }