{"id":30128,"date":"2024-10-28T03:19:43","date_gmt":"2024-10-28T03:19:43","guid":{"rendered":"http:\/\/atmokpo.com\/w\/?p=30128"},"modified":"2024-11-26T06:49:48","modified_gmt":"2024-11-26T06:49:48","slug":"%eb%94%a5%eb%9f%ac%eb%8b%9d-%ed%8c%8c%ec%9d%b4%ed%86%a0%ec%b9%98-%ea%b0%95%ec%a2%8c-%ed%8a%b8%eb%9e%9c%ec%8a%a4%ed%8f%ac%eb%a8%b8-%ec%96%b4%ed%85%90%ec%85%98","status":"publish","type":"post","link":"https:\/\/atmokpo.com\/w\/30128\/","title":{"rendered":"\ub525\ub7ec\ub2dd \ud30c\uc774\ud1a0\uce58 \uac15\uc88c, \ud2b8\ub79c\uc2a4\ud3ec\uba38 \uc5b4\ud150\uc158"},"content":{"rendered":"

<\/p>\n

\ub525\ub7ec\ub2dd\uc740 \ucd5c\uadfc \uba87 \ub144\uac04 \uc778\uacf5\uc9c0\ub2a5(AI) \ubd84\uc57c\uc758 \ud601\uc2e0\uc744 \uac00\uc838\uc628 \ud575\uc2ec \uae30\uc220\uc785\ub2c8\ub2e4. \ub2e4\uc591\ud55c \ub525\ub7ec\ub2dd \ubaa8\ub378 \uc911\uc5d0\uc11c\ub3c4 \ud2b8\ub79c\uc2a4\ud3ec\uba38(Transformer)\ub294 \uc790\uc5f0\uc5b4 \ucc98\ub9ac(NLP) \ubd84\uc57c\uc5d0\uc11c \uadf8 \uc131\ub2a5\uc774 \ub450\ub4dc\ub7ec\uc9c0\uba70, \ub9ce\uc740 \uc5f0\uad6c\uc790\ub4e4\uc5d0\uac8c \uc8fc\ubaa9\ubc1b\uace0 \uc788\uc2b5\ub2c8\ub2e4. \uc774 \uae00\uc5d0\uc11c\ub294 \ud30c\uc774\ud1a0\uce58(PyTorch) \ud504\ub808\uc784\uc6cc\ud06c\ub97c \uc774\uc6a9\ud558\uc5ec \ud2b8\ub79c\uc2a4\ud3ec\uba38 \uc544\ud0a4\ud14d\ucc98\uc640 \uc5b4\ud150\uc158 \uba54\ucee4\ub2c8\uc998\uc5d0 \ub300\ud574 \uae4a\uc774 \uc788\ub294 \uc124\uba85\uacfc \uc2e4\uc2b5 \ucf54\ub4dc\ub97c \uc81c\uacf5\ud558\uaca0\uc2b5\ub2c8\ub2e4.<\/p>\n

1. \ud2b8\ub79c\uc2a4\ud3ec\uba38\ub780 \ubb34\uc5c7\uc778\uac00?<\/h2>\n

\ud2b8\ub79c\uc2a4\ud3ec\uba38\ub294 2017\ub144 Google\uc758 Vaswani \uc678 \uc5f0\uad6c\uc790\ub4e4\uc5d0 \uc758\ud574 \uc81c\uc548\ub41c \ubaa8\ub378\ub85c, \uae30\uc874\uc758 RNN\uc774\ub098 LSTM\uacfc \uac19\uc740 \uc21c\ud658 \uc2e0\uacbd\ub9dd \uad6c\uc870\uc758 \ud55c\uacc4\ub97c \uadf9\ubcf5\ud558\uae30 \uc704\ud574 \uc124\uacc4\ub418\uc5c8\uc2b5\ub2c8\ub2e4. \ud2b8\ub79c\uc2a4\ud3ec\uba38\ub294 \uc804\uccb4 \uc785\ub825 \uc2dc\ud000\uc2a4\ub97c \ud55c \ubc88\uc5d0 \ucc98\ub9ac\ud560 \uc218 \uc788\uc5b4 \ubcd1\ub82c\ud654\uac00 \uc6a9\uc774\ud558\uba70, \ub354 \uae34 \uc758\uc874 \uad00\uacc4\ub97c \ud559\uc2b5\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n

1.1 \ud2b8\ub79c\uc2a4\ud3ec\uba38\uc758 \uad6c\uc870<\/h3>\n

\ud2b8\ub79c\uc2a4\ud3ec\uba38\ub294 \ub450 \uac00\uc9c0 \uc8fc\uc694 \uad6c\uc131 \uc694\uc18c\ub85c \uc774\ub8e8\uc5b4\uc838 \uc788\uc2b5\ub2c8\ub2e4: \uc778\ucf54\ub354\uc640 \ub514\ucf54\ub354. \uc778\ucf54\ub354\ub294 \uc785\ub825 \uc2dc\ud000\uc2a4\ub97c \ubc1b\uc544\ub4e4\uc774\uace0, \ub514\ucf54\ub354\ub294 \uc778\ucf54\ub354\uc758 \ucd9c\ub825\uc744 \uae30\ubc18\uc73c\ub85c \ucd9c\ub825 \uc2dc\ud000\uc2a4\ub97c \uc0dd\uc131\ud569\ub2c8\ub2e4. \uc5ec\uae30\uc11c \uc911\uc694\ud55c \ubd80\ubd84\uc774 \uc5b4\ud150\uc158 \uba54\ucee4\ub2c8\uc998\uc785\ub2c8\ub2e4.<\/p>\n

2. \uc5b4\ud150\uc158 \uba54\ucee4\ub2c8\uc998<\/h2>\n

\uc5b4\ud150\uc158\uc740 \uc785\ub825 \uc2dc\ud000\uc2a4 \ub0b4\uc5d0\uc11c \ud2b9\uc815 \ubd80\ubd84\uc5d0 \uc9d1\uc911\ud558\ub3c4\ub85d \ud558\ub294 \uba54\ucee4\ub2c8\uc998\uc785\ub2c8\ub2e4. \uc989, \uac01 \ub2e8\uc5b4(\ub610\ub294 \uc785\ub825 \ubca1\ud130)\uac00 \ub2e4\ub978 \ub2e8\uc5b4\uc640\uc758 \uad00\uacc4\ub97c \uace0\ub824\ud558\uc5ec \uac00\uc911\uce58\ub97c \ub450\uc5b4 \uc815\ubcf4\ub97c \ucd94\ucd9c\ud569\ub2c8\ub2e4. \uc5b4\ud150\uc158\uc740 \uae30\ubcf8\uc801\uc73c\ub85c \uc138 \uac00\uc9c0 \uc694\uc18c\ub85c \uad6c\uc131\ub429\ub2c8\ub2e4: \ucffc\ub9ac(Query), \ud0a4(Key), \uac12(Value).<\/p>\n

2.1 \uc5b4\ud150\uc158 \uc2a4\ucf54\uc5b4<\/h3>\n

\uc5b4\ud150\uc158 \uc2a4\ucf54\uc5b4\ub294 \ucffc\ub9ac\uc640 \ud0a4 \uac04\uc758 \ub0b4\uc801(dot product)\uc73c\ub85c \uacc4\uc0b0\ub429\ub2c8\ub2e4. \uc774 \uc2a4\ucf54\uc5b4\ub294 \uc785\ub825 \uc2dc\ud000\uc2a4 \ub0b4 \uac01 \ub2e8\uc5b4\uac00 \ud604\uc7ac \ub2e8\uc5b4\uc5d0 \uc5bc\ub9c8\ub098 \uc601\ud5a5\uc744 \ubbf8\uce58\ub294\uc9c0\ub97c \ub098\ud0c0\ub0c5\ub2c8\ub2e4.<\/p>\n

2.2 \uc18c\ud504\ud2b8\ub9e5\uc2a4 \ud568\uc218<\/h3>\n

\uc5b4\ud150\uc158 \uc2a4\ucf54\uc5b4\ub97c \uc815\uaddc\ud654\ud558\uae30 \uc704\ud574 \uc18c\ud504\ud2b8\ub9e5\uc2a4 \ud568\uc218\ub97c \uc0ac\uc6a9\ud558\uc5ec \uac00\uc911\uce58\ub97c \uad6c\ud569\ub2c8\ub2e4. \uc774\ub97c \ud1b5\ud574 \ubaa8\ub4e0 \uac00\uc911\uce58\ub294 0\uacfc 1 \uc0ac\uc774\uc758 \uac12\uc744 \uac00\uc9c0\uba70, \uc804\uccb4 \uac00\uc911\uce58\uc758 \ud569\uc740 1\uc774 \ub429\ub2c8\ub2e4.<\/p>\n

2.3 \uc5b4\ud150\uc158 \uc5f0\uc0b0<\/h3>\n

\uac00\uc911\uce58\uac00 \uacb0\uc815\ub418\uba74, \uc774\ub97c \uac12(Value)\uc5d0 \uacf1\ud558\uc5ec \ucd5c\uc885 \uc5b4\ud150\uc158 \ucd9c\ub825\uc744 \uc0dd\uc131\ud569\ub2c8\ub2e4. \ucd5c\uc885 \ucd9c\ub825\uc740 \uac00\uc911\uce58\uac00 \uc801\uc6a9\ub41c \uac12\ub4e4\uc758 \uac00\uc0b0\uc73c\ub85c \uc774\ub8e8\uc5b4\uc9d1\ub2c8\ub2e4.<\/p>\n

3. \ud30c\uc774\ud1a0\uce58\ub85c \ud2b8\ub79c\uc2a4\ud3ec\uba38 \uad6c\ud604\ud558\uae30<\/h2>\n

\uc774\uc81c \ud30c\uc774\ud1a0\uce58\ub97c \uc0ac\uc6a9\ud558\uc5ec \ud2b8\ub79c\uc2a4\ud3ec\uba38\uc640 \uc5b4\ud150\uc158 \uba54\ucee4\ub2c8\uc998\uc744 \uad6c\ud604\ud574 \ubcf4\uaca0\uc2b5\ub2c8\ub2e4. \uc544\ub798\uc758 \ucf54\ub4dc\ub294 \uae30\ubcf8\uc801\uc778 \uc5b4\ud150\uc158 \ubaa8\ub4c8\uc744 \uad6c\ud604\ud55c \uc608\uc81c\uc785\ub2c8\ub2e4.<\/p>\n

3.1 \ud544\uc694\ud55c \ub77c\uc774\ube0c\ub7ec\ub9ac \uc124\uce58<\/h3>\n
!pip install torch torchvision<\/code><\/pre>\n
3.2 \uc5b4\ud150\uc158 \ud074\ub798\uc2a4 \uad6c\ud604<\/h3>\n
\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nclass ScaledDotProductAttention(nn.Module):\n    def __init__(self):\n        super(ScaledDotProductAttention, self).__init__()\n\n    def forward(self, query, key, value, mask=None):\n        # \ucffc\ub9ac\uc640 \ud0a4\uc758 \ub0b4\uc801 \uacc4\uc0b0\n        scores = torch.matmul(query, key.transpose(-2, -1)) \/ (key.size(-1) ** 0.5)\n\n        # \ub9c8\uc2a4\ud06c\uac00 \uc8fc\uc5b4\uc9c4 \uacbd\uc6b0 \ub9c8\uc2a4\ud0b9\n        if mask is not None:\n            scores.masked_fill_(mask == 0, -1e9)\n\n        # \uc18c\ud504\ud2b8\ub9e5\uc2a4 \ud568\uc218\ub85c \uc815\uaddc\ud654\n        attn_weights = F.softmax(scores, dim=-1)\n\n        # \uac00\uc911\uce58\ub97c \uac12\uc5d0 \uacf1\ud558\ub294 \uc2dd\uc73c\ub85c \uc5b4\ud150\uc158 \ucd9c\ub825 \uacc4\uc0b0\n        output = torch.matmul(attn_weights, value)\n        return output, attn_weights\n    <\/code><\/pre>\n
3.3 \ud2b8\ub79c\uc2a4\ud3ec\uba38 \uc778\ucf54\ub354 \uad6c\ud604<\/h3>\n
\nclass TransformerEncoder(nn.Module):\n    def __init__(self, embed_size, heads, num_layers, drop_out):\n        super(TransformerEncoder, self).__init__()\n        self.embed_size = embed_size\n        self.heads = heads\n        self.num_layers = num_layers\n        self.drop_out = drop_out\n\n        self.attention = ScaledDotProductAttention()\n        self.linear = nn.Linear(embed_size, embed_size)\n        self.dropout = nn.Dropout(drop_out)\n        self.norm = nn.LayerNorm(embed_size)\n\n    def forward(self, x, mask):\n        for _ in range(self.num_layers):\n            attention_output, _ = self.attention(x, x, x, mask)\n            x = self.norm(x + self.dropout(attention_output))\n            x = self.norm(x + self.dropout(self.linear(x)))\n        return x\n    <\/code><\/pre>\n
4. \ubaa8\ub378 \ud559\uc2b5 \ubc0f \ud3c9\uac00<\/h2>\n
\ud2b8\ub79c\uc2a4\ud3ec\uba38 \uc778\ucf54\ub354\ub97c \uad6c\ud604\ud55c \ud6c4, \uc2e4\uc81c \ub370\uc774\ud130\ub97c \uc0ac\uc6a9\ud558\uc5ec \ubaa8\ub378\uc744 \ud559\uc2b5\uc2dc\ud0a4\uace0 \ud3c9\uac00\ud558\ub294 \ubc29\ubc95\uc5d0 \ub300\ud574 \uc124\uba85\ud569\ub2c8\ub2e4.<\/p>\n
4.1 \ub370\uc774\ud130 \uc900\ube44<\/h3>\n
\ubaa8\ub378\uc744 \ud559\uc2b5\uc2dc\ud0a4\uae30 \uc704\ud574\uc11c \uba3c\uc800 \ud6c8\ub828 \ub370\uc774\ud130\ub97c \uc900\ube44\ud574\uc57c \ud569\ub2c8\ub2e4. \uc77c\ubc18\uc801\uc73c\ub85c \ud14d\uc2a4\ud2b8 \ub370\uc774\ud130\uc640 \uac19\uc774 \uc2dc\ud000\uc2a4 \ud615\ud0dc\uc758 \ub370\uc774\ud130\ub97c \uc0ac\uc6a9\ud569\ub2c8\ub2e4.<\/p>\n
4.2 \ubaa8\ub378 \ucd08\uae30\ud654<\/h3>\n
\nembed_size = 256  # \uc784\ubca0\ub529 \ucc28\uc6d0\nheads = 8  # \uc5b4\ud150\uc158 \ud5e4\ub4dc \uac1c\uc218\nnum_layers = 6  # \uc778\ucf54\ub354 \uce35 \uc218\ndrop_out = 0.1  # \ub4dc\ub86d\uc544\uc6c3 \ube44\uc728\n\nmodel = TransformerEncoder(embed_size, heads, num_layers, drop_out)\n    <\/code><\/pre>\n
4.3 \uc190\uc2e4 \ud568\uc218 \ubc0f \uc635\ud2f0\ub9c8\uc774\uc800 \uc124\uc815<\/h3>\n
\noptimizer = torch.optim.Adam(model.parameters(), lr=0.0001)\nloss_fn = nn.CrossEntropyLoss()\n    <\/code><\/pre>\n
4.4 \ud6c8\ub828 \ub8e8\ud504<\/h3>\n
\nfor epoch in range(num_epochs):\n    model.train()\n    total_loss = 0\n    for batch in train_loader:\n        optimizer.zero_grad()\n        output = model(batch['input'], batch['mask'])\n        loss = loss_fn(output.view(-1, output.size(-1)), batch['target'])\n        loss.backward()\n        optimizer.step()\n        total_loss += loss.item()\n    print(f\"Epoch: {epoch+1}, Loss: {total_loss\/len(train_loader)}\")\n    <\/code><\/pre>\n
4.5 \ud3c9\uac00 \ubc0f \ud14c\uc2a4\ud2b8<\/h3>\n
\ud6c8\ub828\uc774 \uc644\ub8cc\ub41c \ud6c4, \ubaa8\ub378\uc744 \ud3c9\uac00\ud558\uc5ec \uc131\ub2a5\uc744 \uce21\uc815\ud569\ub2c8\ub2e4. \uc77c\ubc18\uc801\uc73c\ub85c \ud14c\uc2a4\ud2b8 \ub370\uc774\ud130\uc5d0 \ub300\ud574 \uc815\ud655\ub3c4, \uc815\ubc00\ub3c4, \uc7ac\ud604\uc728 \ub4f1\uc758 \uc9c0\ud45c\ub97c \uc0ac\uc6a9\ud569\ub2c8\ub2e4.<\/p>\n
5. \uacb0\ub860<\/h2>\n
\uc774\ubc88 \uae00\uc5d0\uc11c\ub294 \ud2b8\ub79c\uc2a4\ud3ec\uba38 \uc544\ud0a4\ud14d\ucc98\uc640 \uc5b4\ud150\uc158 \uba54\ucee4\ub2c8\uc998\uc5d0 \ub300\ud574 \uc124\uba85\ud558\uace0, \ud30c\uc774\ud1a0\uce58\ub97c \uc774\uc6a9\ud558\uc5ec \uc774\ub97c \uad6c\ud604\ud558\ub294 \uacfc\uc815\uc744 \ubcf4\uc5ec\uc8fc\uc5c8\uc2b5\ub2c8\ub2e4. \ud2b8\ub79c\uc2a4\ud3ec\uba38 \ubaa8\ub378\uc740 \uace0\uc131\ub2a5\uc758 \uc790\uc5f0\uc5b4 \ucc98\ub9ac \ubaa8\ub378\uc744 \uad6c\ucd95\ud558\ub294 \ub370 \uc720\uc6a9\ud558\uba70, \ub2e4\uc591\ud55c \ubd84\uc57c\uc5d0\uc11c \ud65c\uc6a9\ub418\uace0 \uc788\uc2b5\ub2c8\ub2e4. \ud559\uc2b5 \ub370\uc774\ud130\uc640 \ubaa8\ub378 \ud558\uc774\ud37c\ud30c\ub77c\ubbf8\ud130\uc5d0 \ub530\ub77c \uc131\ub2a5\uc774 \ud06c\uac8c \ub2ec\ub77c\uc9c8 \uc218 \uc788\uc73c\ubbc0\ub85c, \uc5ec\ub7ec \uac00\uc9c0 \uc2e4\ud5d8\uc744 \ud1b5\ud574 \ucd5c\uc801\uc758 \uc870\ud569\uc744 \ucc3e\uc544\uac00\ub294 \uacfc\uc815\uc774 \uc911\uc694\ud569\ub2c8\ub2e4.<\/p>\n
\ud2b8\ub79c\uc2a4\ud3ec\uba38\ub294 \ud604\uc7ac NLP \ubaa8\ub378\ub9c1\uc5d0 \ud601\uc2e0\uc801\uc778 \uae30\uc5ec\ub97c \ud558\uace0 \uc788\uc73c\uba70, \uc55e\uc73c\ub85c\ub3c4 \ub2e4\uc591\ud55c \uc5f0\uad6c \uacb0\uacfc\ub97c \ud1b5\ud574 \ub354\uc6b1 \ubc1c\uc804\ud560 \uac83\uc73c\ub85c \uae30\ub300\ub429\ub2c8\ub2e4. \ub2e4\uc74c \uae00\uc5d0\uc11c\ub294 \uc790\uc5f0\uc5b4 \ucc98\ub9ac\uc5d0\uc11c\uc758 \ud2b8\ub79c\uc2a4\ud3ec\uba38 \ubaa8\ub378\uc758 \ud65c\uc6a9 \uc0ac\ub840\uc5d0 \ub300\ud574 \ub2e4\ub8f0 \uc608\uc815\uc785\ub2c8\ub2e4. \ub9ce\uc740 \uad00\uc2ec \ubd80\ud0c1\ub4dc\ub9bd\ub2c8\ub2e4.<\/p>\n