\ubcf8 \uac15\uc88c\uc5d0\uc11c\ub294 \ub525 \ub7ec\ub2dd\uc744 \ud65c\uc6a9\ud55c \uc790\uc5f0\uc5b4 \ucc98\ub9ac(Natural Language Processing, NLP) \ubd84\uc57c \uc911 \ud558\ub098\uc778 \uac1c\uccb4\uba85 \uc778\uc2dd(Named Entity Recognition, NER)\uc5d0 \ub300\ud574 \uc54c\uc544\ubcf4\uaca0\uc2b5\ub2c8\ub2e4. \ud2b9\ud788, \ud55c\uad6d\uc5b4 \ucc98\ub9ac\uc5d0 \uc801\ud569\ud55c KoBERT<\/strong> \ubaa8\ub378\uc744 \ud65c\uc6a9\ud558\uc5ec \uac1c\uccb4\uba85 \uc778\uc2dd\uc758 \uae30\ubcf8 \uac1c\ub150\uacfc \uad6c\ud604 \ubc29\ubc95\uc5d0 \ub300\ud574 \uc790\uc138\ud788 \uc124\uba85\ud558\uaca0\uc2b5\ub2c8\ub2e4.<\/p>\n \uc790\uc5f0\uc5b4 \ucc98\ub9ac\ub294 \ucef4\ud4e8\ud130\uac00 \uc778\uac04\uc758 \uc5b8\uc5b4\ub97c \uc774\ud574\ud558\uace0 \uc0dd\uc131\ud558\ub294 \uae30\uc220\uc744 \uc758\ubbf8\ud569\ub2c8\ub2e4. \uc774\ub294 \uc5b8\uc5b4\uc758 \uc758\ubbf8, \ubb38\ubc95 \ubc0f \uc791\uc6a9\uc744 \ubd84\uc11d\ud558\uc5ec \ucef4\ud4e8\ud130\uac00 \uc774\ub97c \uc774\ud574\ud560 \uc218 \uc788\ub3c4\ub85d \ub9cc\ub4dc\ub294 \uacfc\uc815\uc785\ub2c8\ub2e4. \uc790\uc5f0\uc5b4 \ucc98\ub9ac\uc758 \uc8fc\uc694 \uc751\uc6a9 \ubd84\uc57c\ub85c\ub294 \uae30\uacc4 \ubc88\uc5ed, \uac10\uc815 \ubd84\uc11d, \uc9c8\ubb38 \ub2f5\ubcc0 \uc2dc\uc2a4\ud15c, \uac1c\uccb4\uba85 \uc778\uc2dd \ub4f1\uc774 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n \uac1c\uccb4\uba85 \uc778\uc2dd(Named Entity Recognition, NER)\uc740 \ud14d\uc2a4\ud2b8\uc5d0\uc11c \uc0ac\ub78c, \uc7a5\uc18c, \uc870\uc9c1, \ub0a0\uc9dc \ub4f1\uacfc \uac19\uc740 \uace0\uc720 \uba85\uc0ac\ub97c \uc2dd\ubcc4\ud558\uace0 \ubd84\ub958\ud558\ub294 \uae30\uc220\uc785\ub2c8\ub2e4. \uc608\ub97c \ub4e4\uc5b4 “\uc774\uc21c\uc2e0\uc740 \ud55c\uc0b0\ub3c4 \ub300\ucca9\uc5d0\uc11c \ud070 \uc2b9\ub9ac\ub97c \uac70\ub450\uc5c8\ub2e4”\ub77c\ub294 \ubb38\uc7a5\uc5d0\uc11c “\uc774\uc21c\uc2e0”\uc740 \uc778\ubb3c, “\ud55c\uc0b0\ub3c4”\ub294 \uc7a5\uc18c\ub85c \uc778\uc2dd\ub429\ub2c8\ub2e4. NER\uc740 \uc815\ubcf4 \ucd94\ucd9c, \uac80\uc0c9 \uc5d4\uc9c4, \ubb38\uc11c \uc694\uc57d \ub4f1 \uc5ec\ub7ec \ubd84\uc57c\uc5d0\uc11c \ud575\uc2ec\uc801\uc778 \uc5ed\ud560\uc744 \ud569\ub2c8\ub2e4.<\/p>\n KoBERT\ub294 Google\uc758 BERT \ubaa8\ub378\uc744 \ud55c\uad6d\uc5b4\uc5d0 \ub9de\uac8c \uc7ac\ud559\uc2b5\ud55c \ubaa8\ub378\uc785\ub2c8\ub2e4. BERT(Bidirectional Encoder Representations from Transformers)\ub294 \uc790\uc5f0\uc5b4 \ucc98\ub9ac\uc5d0\uc11c \uac00\uc7a5 \uc778\uae30 \uc788\ub294 \ubaa8\ub378 \uc911 \ud558\ub098\ub85c, \ubb38\ub9e5\uc744 \uc774\ud574\ud558\ub294\ub370 \ub9e4\uc6b0 \uac15\ub825\ud569\ub2c8\ub2e4. KoBERT\ub294 \ud55c\uad6d\uc5b4\uc758 \ud2b9\uc131\uc744 \ubc18\uc601\ud558\uc5ec \ud55c\uad6d\uc5b4 \ub370\uc774\ud130\uc14b\uc73c\ub85c \ud559\uc2b5\ub418\uc5c8\uc73c\uba70, \ub2e8\uc5b4\uc758 \uc758\ubbf8\ub97c \ubcf4\ub2e4 \uc815\ud655\ud558\uac8c \ud30c\uc545\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n BERT\ub294 Transformer \uad6c\uc870\ub97c \uae30\ubc18\uc73c\ub85c \ud558\uba70, \uc591\ubc29\ud5a5\uc73c\ub85c \ubb38\ub9e5\uc744 \uc774\ud574\ud569\ub2c8\ub2e4. \uc774\ub294 \ubaa8\ub378\uc774 \uc785\ub825\ub41c \ubb38\uc7a5\uc758 \uc55e\ub4a4\ub97c \ub3d9\uc2dc\uc5d0 \uace0\ub824\ud558\uc5ec \ub9e5\ub77d\uc744 \ub354 \uc798 \uc774\ud574\ud560 \uc218 \uc788\uac8c \ud569\ub2c8\ub2e4. BERT\ub294 \ub2e4\uc74c \ub450 \uac00\uc9c0 \uc791\uc5c5\uc744 \ud1b5\ud574 \ud559\uc2b5\ub429\ub2c8\ub2e4:<\/p>\n \uc774\uc81c KoBERT\ub97c \uc774\uc6a9\ud558\uc5ec \uac1c\uccb4\uba85 \uc778\uc2dd\uc744 \uad6c\ud604\ud558\ub294 \uacfc\uc815\uc744 \ub2e8\uacc4\ubcc4\ub85c \uc124\uba85\ud558\uaca0\uc2b5\ub2c8\ub2e4. \uc2e4\uc2b5\uc744 \uc704\ud574 Python\uacfc Hugging Face\uc758 Transformers \ub77c\uc774\ube0c\ub7ec\ub9ac\ub97c \uc0ac\uc6a9\ud560 \uac83\uc785\ub2c8\ub2e4.<\/p>\n \uac1c\uccb4\uba85 \uc778\uc2dd\uc744 \uc704\ud574 \ud559\uc2b5\ud560 \ub370\uc774\ud130\uc14b\uc744 \uc900\ube44\ud574\uc57c \ud569\ub2c8\ub2e4. \uc6b0\ub9ac\ub294 \uacf5\uac1c\ub41c NER \ub370\uc774\ud130\uc14b\uc778 ‘Korean NER Dataset’\uc744 \uc0ac\uc6a9\ud560 \uac83\uc785\ub2c8\ub2e4. \uc774 \ub370\uc774\ud130\uc14b\uc5d0\ub294 \ubb38\uc7a5\uacfc \uac01 \ub2e8\uc5b4\uc758 \uac1c\uccb4\uba85 \ud0dc\uadf8\uac00 \ud3ec\ud568\ub418\uc5b4 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n \uc608\ub97c \ub4e4\uc5b4:<\/p>\n \ub2e4\uc74c\uc73c\ub85c KoBERT \ubaa8\ub378\uc744 \ubd88\ub7ec\uc635\ub2c8\ub2e4. Hugging Face\uc758 Transformers \ub77c\uc774\ube0c\ub7ec\ub9ac\ub97c \ud1b5\ud574 \uc27d\uac8c \uc0ac\uc6a9\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n \ubaa8\ub378\uc5d0 \uc785\ub825\ud558\uae30 \uc704\ud574 \ub370\uc774\ud130\ub97c \uc804\ucc98\ub9ac\ud574\uc57c \ud569\ub2c8\ub2e4. \ud14d\uc2a4\ud2b8\ub97c \ud1a0\ud070\ud654\ud558\uace0, \ud0dc\uadf8\ub97c \uc778\ucf54\ub529\ud558\ub294 \uacfc\uc815\uc744 \ud3ec\ud568\ud569\ub2c8\ub2e4.<\/p>\n \uc804\ucc98\ub9ac\ub41c \ub370\uc774\ud130\ub97c \uc0ac\uc6a9\ud558\uc5ec \ubaa8\ub378\uc744 \ud559\uc2b5\ud569\ub2c8\ub2e4. PyTorch\ub97c \uc0ac\uc6a9\ud558\uc5ec \uc190\uc2e4 \ud568\uc218\uc640 \uc635\ud2f0\ub9c8\uc774\uc800\ub97c \uc815\uc758\ud558\uace0 \ubaa8\ub378\uc744 \ud559\uc2b5\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n \ud559\uc2b5\uc774 \uc644\ub8cc\ub41c \ud6c4, \uac80\uc99d \ub370\uc774\ud130\ub97c \uc0ac\uc6a9\ud558\uc5ec \ubaa8\ub378\uc758 \uc131\ub2a5\uc744 \ud3c9\uac00\ud569\ub2c8\ub2e4. \ud3c9\uac00 \uc9c0\ud45c\ub85c\ub294 \uc815\ud655\ub3c4(Accuracy), \uc815\ubc00\ub3c4(Precision), \uc7ac\ud604\uc728(Recall) \ub4f1\uc744 \ud65c\uc6a9\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n \ud559\uc2b5\ub41c \ubaa8\ub378\uc744 \uc0ac\uc6a9\ud558\uc5ec \uc0c8\ub85c\uc6b4 \ubb38\uc7a5\uc5d0\uc11c \uac1c\uccb4\uba85\uc744 \uc778\uc2dd\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4. \ud14d\uc2a4\ud2b8\ub97c \uc785\ub825\ud558\uba74, \uac01 \ub2e8\uc5b4\uc5d0 \ub300\ud55c \uac1c\uccb4\uba85 \ud0dc\uadf8\ub97c \uc608\uce21\ud558\ub294 \uacfc\uc815\uc744 \ud3ec\ud568\ud569\ub2c8\ub2e4.<\/p>\n \uc774\ubc88 \uac15\uc88c\uc5d0\uc11c\ub294 KoBERT\ub97c \uc774\uc6a9\ud55c \uac1c\uccb4\uba85 \uc778\uc2dd\uc758 \uae30\ubcf8 \uac1c\ub150\ubd80\ud130 \uad6c\ud604 \ubc29\ubc95\uae4c\uc9c0 \uc54c\uc544\ubcf4\uc558\uc2b5\ub2c8\ub2e4. KoBERT\uc758 \uac15\ub825\ud55c \uc131\ub2a5 \ub355\ubd84\uc5d0 \uc790\uc5f0\uc5b4 \ucc98\ub9ac \ubd84\uc57c\uc5d0\uc11c\uc758 \uac1c\uccb4\uba85 \uc778\uc2dd \uc791\uc5c5\uc744 \ud6a8\uc728\uc801\uc73c\ub85c \uc218\ud589\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4. \uc774\ub7ec\ud55c \uae30\uc220\uc740 \uc5ec\ub7ec \ube44\uc988\ub2c8\uc2a4 \ubc0f \uc5f0\uad6c \ubd84\uc57c\uc5d0\uc11c \ub110\ub9ac \ud65c\uc6a9\ub420 \uc218 \uc788\uc73c\uba70, \ud55c\uad6d\uc5b4 \ub370\uc774\ud130\uc5d0\uc11c\ub3c4 \uc6b0\uc218\ud55c \uc131\ub2a5\uc744 \ubc1c\ud718\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n \uc790\uc5f0\uc5b4 \ucc98\ub9ac\uc640 \uad00\ub828\ub41c \uc5ec\ub7ec \uc790\ub8cc\ub4e4\uc774 \uc788\uc73c\uba70, \ub2e4\uc591\ud55c \ub3c4\uba54\uc778\uc5d0 \ub9de\ub294 \ubaa8\ub378\uc744 \ud559\uc2b5\ud560 \uc218 \uc788\ub294 \ub9ac\uc18c\uc2a4\uac00 \ub9ce\uc2b5\ub2c8\ub2e4. \ub2e4\uc74c\uc740 \uc774\ub97c \uc704\ud55c \uba87 \uac00\uc9c0 \ucd94\ucc9c \uc790\ub8cc\uc785\ub2c8\ub2e4:<\/p>\n KoBERT \ubc0f \uac1c\uccb4\uba85 \uc778\uc2dd \uae30\uc220\uc744 \uae30\ubc18\uc73c\ub85c \ub354\uc6b1 \ubc1c\uc804\ub41c \uc2dc\uc2a4\ud15c\uc744 \uac1c\ubc1c\ud558\ub294 \uac83\uc774 \uc911\uc694\ud55c \uc5f0\uad6c \ubc29\ud5a5\uc774 \ub420 \uac83\uc785\ub2c8\ub2e4. \ub610\ud55c, \ub354 \ub9ce\uc740 \uc5b8\uc5b4\uc5d0 \ub300\ud574 \uc9c1\uc811\uc801\uc73c\ub85c \uc801\uc6a9\ud560 \uc218 \uc788\ub294 \ub2e4\uad6d\uc5b4 \ubaa8\ub378\uc758 \ud559\uc2b5\uacfc \uac1c\ubc1c\ub3c4 \ud765\ubbf8\ub85c\uc6b4 \uc5f0\uad6c \uc8fc\uc81c\uc785\ub2c8\ub2e4.<\/p>\n \uc774\ubc88 \uac15\uc88c\uc5d0 \ub300\ud574 \uad81\uae08\ud55c \uc810\uc774 \uc788\uc73c\uc2dc\ub2e4\uba74 \ub313\uae00\ub85c \ub9d0\uc500\ud574 \uc8fc\uc138\uc694. \uc801\uadf9\uc801\uc73c\ub85c \ub2f5\ubcc0\ud574\ub4dc\ub9ac\uaca0\uc2b5\ub2c8\ub2e4!<\/p>\n <\/body><\/p>\n","protected":false},"excerpt":{"rendered":"1. \uc790\uc5f0\uc5b4 \ucc98\ub9ac(NLP)\ub780?<\/h2>\n
1.1 \uac1c\uccb4\uba85 \uc778\uc2dd(NER)\uc774\ub780?<\/h3>\n
2. KoBERT \uc18c\uac1c<\/h2>\n
2.1 BERT\uc758 \uae30\ubcf8 \uad6c\uc870<\/h3>\n
\n
3. KoBERT\ub97c \uc774\uc6a9\ud55c NER \uad6c\ud604\ud558\uae30<\/h2>\n
3.1 \ud658\uacbd \uc124\uc815<\/h3>\n
!pip install transformers\n!pip install torch\n!pip install numpy\n!pip install pandas\n!pip install sklearn<\/code><\/pre>\n3.2 \ub370\uc774\ud130 \uc900\ube44<\/h3>\n
\uc774\uc21c\uc2e0 B-PER\n\uc740 O\n\ud55c\uc0b0\ub3c4 B-LOC\n\ub300\ucca9 O\n\uc5d0\uc11c O\n\ud070 O\n\uc2b9\ub9ac O\n\ub97c O\n\uac70\ub450\uc5c8\ub2e4 O<\/code><\/pre>\n3.3 KoBERT \ubaa8\ub378 \ubd88\ub7ec\uc624\uae30<\/h3>\n
from transformers import BertTokenizer, BertForTokenClassification\nimport torch\n\n# KoBERT \ubaa8\ub378\uacfc \ud1a0\ud06c\ub098\uc774\uc800 \ubd88\ub7ec\uc624\uae30\ntokenizer = BertTokenizer.from_pretrained('monologg\/kobert')\nmodel = BertForTokenClassification.from_pretrained('monologg\/kobert', num_labels=len(tag2id))<\/code><\/pre>\n3.4 \ub370\uc774\ud130 \uc804\ucc98\ub9ac<\/h3>\n
def encode_tags(tags, max_len):\n return [tag2id[tag] for tag in tags] + [tag2id['O']] * (max_len - len(tags))\n\n# \uc608\uc2dc \ub370\uc774\ud130\nsentences = [\"\uc774\uc21c\uc2e0\uc740 \ud55c\uc0b0\ub3c4 \ub300\ucca9\uc5d0\uc11c \ud070 \uc2b9\ub9ac\ub97c \uac70\ub450\uc5c8\ub2e4\"]\ntags = [[\"B-PER\", \"O\", \"B-LOC\", \"O\", \"O\", \"O\", \"O\", \"O\", \"O\"]]\n\n# \uc9d5\uacfc \uc124\uacc4\ninput_ids = []\nattention_masks = []\nlabels = []\n\nfor sentence, tag in zip(sentences, tags):\n encoded = tokenizer.encode_plus(\n sentence,\n add_special_tokens=True,\n max_length=128,\n pad_to_max_length=True,\n return_attention_mask=True,\n )\n input_ids.append(encoded['input_ids'])\n attention_masks.append(encoded['attention_mask'])\n labels.append(encode_tags(tag, 128))<\/code><\/pre>\n3.5 \ubaa8\ub378 \ud559\uc2b5<\/h3>\n
from sklearn.model_selection import train_test_split\n\n# \ud6c8\ub828 \ub370\uc774\ud130\uc640 \uac80\uc99d \ub370\uc774\ud130\ub85c \ubd84\ud560\ntrain_inputs, validation_inputs, train_labels, validation_labels = train_test_split(input_ids, labels, test_size=0.1)\n\n# \ubaa8\ub378 \ud559\uc2b5 \ubc0f \ud3c9\uac00 \ucf54\ub4dc...<\/code><\/pre>\n3.6 \ubaa8\ub378 \ud3c9\uac00<\/h3>\n
from sklearn.metrics import classification_report\n\n# \ubaa8\ub378 \uc608\uce21 \ucf54\ub4dc...\npredictions = model(validation_inputs)\npredicted_labels = ...\n\n# \ud3c9\uac00 \uc9c0\ud45c \ucd9c\ub825\nprint(classification_report(validation_labels, predicted_labels))<\/code><\/pre>\n3.7 \ubaa8\ub378 \ud65c\uc6a9\ud558\uae30<\/h3>\n
def predict_entities(sentence):\n encoded = tokenizer.encode_plus(sentence, return_tensors='pt')\n with torch.no_grad():\n output = model(**encoded)\n logits = output[0]\n predictions = torch.argmax(logits, dim=2)\n return predictions<\/code><\/pre>\n4. \uacb0\ub860<\/h2>\n
5. \ucc38\uace0 \uc790\ub8cc<\/h2>\n
\n
6. \ucd94\uac00\uc801\uc778 \ud559\uc2b5 \uc790\ub8cc<\/h2>\n
\n
7. \ud5a5\ud6c4 \uc5f0\uad6c \ubc29\ud5a5<\/h2>\n
8. Q&A<\/h2>\n