\ub525\ub7ec\ub2dd\uacfc \uc790\uc5f0\uc5b4 \ucc98\ub9ac(Natural Language Processing, NLP)\uc758 \ubc1c\uc804\uc5d0 \ub530\ub77c, \ud14d\uc2a4\ud2b8 \ub370\uc774\ud130\ub97c \ucc98\ub9ac\ud558\uace0 \ubd84\uc11d\ud558\ub294 \ubc29\ubc95\uc774 \ub2e4\uc591\ud574\uc84c\uc2b5\ub2c8\ub2e4. \uc774\ubc88 \ud3ec\uc2a4\ud305\uc5d0\uc11c\ub294 \ud5c8\uae45\ud398\uc774\uc2a4(Hugging Face) \ub77c\uc774\ube0c\ub7ec\ub9ac\ub97c \ud65c\uc6a9\ud558\uc5ec \ubaa8\ub354\ub098(Moderna)\uc640 \uad00\ub828\ub41c COVID-19 \uc815\ubcf4\ub97c \uc704\ud0a4\ud53c\ub514\uc544\uc5d0\uc11c \uac00\uc838\uc624\ub294 \ubc29\ubc95\uc5d0 \ub300\ud574 \uc790\uc138\ud788 \uc124\uba85\ud558\uaca0\uc2b5\ub2c8\ub2e4. \ud5c8\uae45\ud398\uc774\uc2a4 \ud2b8\ub79c\uc2a4\ud3ec\uba38(Transformers)\ub294 \uc790\uc5f0\uc5b4 \ucc98\ub9ac \uc791\uc5c5\uc5d0\uc11c \ub9ce\uc774 \uc0ac\uc6a9\ub418\ub294 \uc0ac\uc804 \ud559\uc2b5\ub41c \ubaa8\ub378\ub4e4\uc744 \uc81c\uacf5\ud558\uba70, \uc774\ub97c \ud1b5\ud574 \uc0ac\uc6a9\uc790\ub294 \uc190\uc27d\uac8c \ud14d\uc2a4\ud2b8 \ub370\uc774\ud130\ub97c \ubd84\uc11d\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n
\ud5c8\uae45\ud398\uc774\uc2a4\ub294 NLP \ubaa8\ub378\uc744 \uc27d\uac8c \uc0ac\uc6a9\ud560 \uc218 \uc788\ub3c4\ub85d \ub2e4\uc591\ud55c \uc0ac\uc804 \ud559\uc2b5\ub41c \ubaa8\ub378\uacfc \ud234\uc744 \uc81c\uacf5\ud558\ub294 \ud50c\ub7ab\ud3fc\uc785\ub2c8\ub2e4. \ud2b9\ud788 Transformers<\/strong> \ub77c\uc774\ube0c\ub7ec\ub9ac\ub294 BERT, GPT-2, T5 \ub4f1 \ub2e4\uc591\ud55c \ucd5c\uc2e0 \ud2b8\ub79c\uc2a4\ud3ec\uba38 \ubaa8\ub378\ub4e4\uc744 \ud3ec\ud568\ud558\uace0 \uc788\uc73c\uba70, \uc774\ub7ec\ud55c \ubaa8\ub378\uc744 \ud65c\uc6a9\ud558\uc5ec \uc790\uc5f0\uc5b4 \ucc98\ub9ac \uc791\uc5c5\uc744 \ubcf4\ub2e4 \uc6a9\uc774\ud558\uac8c \uc218\ud589\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n \ud5c8\uae45\ud398\uc774\uc2a4 \ub77c\uc774\ube0c\ub7ec\ub9ac\ub97c \uc0ac\uc6a9\ud558\uae30 \uc704\ud574 Python \ud658\uacbd\uc744 \uc124\uc815\ud574\uc57c \ud569\ub2c8\ub2e4. \uc544\ub798\uc758 \uba85\ub839\uc5b4\ub85c \ud544\uc218 \ub77c\uc774\ube0c\ub7ec\ub9ac\ub97c \uc124\uce58\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n \uc704\ud0a4\ud53c\ub514\uc544\uc5d0\uc11c \ubaa8\ub354\ub098\uc640 COVID-19\uc5d0 \uad00\ub828\ub41c \uc815\ubcf4\ub97c \uac00\uc838\uc624\uae30 \uc704\ud574 wikipedia-api<\/strong>\ub97c \uc0ac\uc6a9\ud558\uaca0\uc2b5\ub2c8\ub2e4. \uc774 \ub77c\uc774\ube0c\ub7ec\ub9ac\ub294 \uc704\ud0a4\ud53c\ub514\uc544 \ud398\uc774\uc9c0\ub97c \uc27d\uac8c \uac80\uc0c9\ud558\uace0 \ucf58\ud150\uce20\ub97c \uac00\uc838\uc624\ub294 \uae30\ub2a5\uc744 \uc81c\uacf5\ud569\ub2c8\ub2e4.<\/p>\n \uc544\ub798 \ucf54\ub4dc\ub294 \ubaa8\ub354\ub098\uc640 COVID-19\uc5d0 \ub300\ud55c \uc815\ubcf4\ub97c \uac80\uc0c9\ud558\uace0, \uadf8 \ub0b4\uc6a9\uc744 \ucd9c\ub825\ud558\ub294 \uac04\ub2e8\ud55c \uc608\uc81c\uc785\ub2c8\ub2e4.<\/p>\n \uc704 \ucf54\ub4dc\ub97c \uc2e4\ud589\ud558\uba74 \ubaa8\ub354\ub098\uc758 \uc704\ud0a4\ud53c\ub514\uc544 \ud398\uc774\uc9c0\uc5d0\uc11c \ub0b4\uc6a9\uc744 \uac00\uc838\uc62c \uc218 \uc788\uc2b5\ub2c8\ub2e4. \uc774\uc81c COVID-19\uc640 \uad00\ub828\ub41c \uc815\ubcf4\ub97c \ucd94\uac00\ub85c \ud655\uc778\ud574\ubcf4\uaca0\uc2b5\ub2c8\ub2e4.<\/p>\n \ub9c8\ucc2c\uac00\uc9c0\ub85c COVID-19\uc5d0 \ub300\ud55c \uc815\ubcf4\ub97c \uc704\ud0a4\ud53c\ub514\uc544\uc5d0\uc11c \uac00\uc838\uc624\ub294 \ucf54\ub4dc\ub294 \ub2e4\uc74c\uacfc \uac19\uc2b5\ub2c8\ub2e4.<\/p>\n \uac00\uc838\uc628 \uc704\ud0a4\ud53c\ub514\uc544 \ud14d\uc2a4\ud2b8\ub294 \ubaa8\ub378\uc5d0 \uc785\ub825\ud558\uae30 \uc804\uc5d0 \uc804\ucc98\ub9ac \uacfc\uc815\uc744 \uac70\uccd0\uc57c \ud569\ub2c8\ub2e4. \uc774 \uacfc\uc815\uc5d0\uc11c\ub294 \ubd88\ud544\uc694\ud55c \ubb38\uc790\ub098 \uae30\ud638\ub97c \uc81c\uac70\ud558\uace0, \ud544\uc694\ud55c \uc815\ubcf4\ub97c \uc815\ub9ac\ud558\ub294 \uc791\uc5c5\uc774 \ud3ec\ud568\ub429\ub2c8\ub2e4.<\/p>\n \uc544\ub798\uc758 \ucf54\ub4dc\ub294 \uac00\uc838\uc628 \ud14d\uc2a4\ud2b8\uc5d0\uc11c \ubd88\ud544\uc694\ud55c \ubb38\uc790\ub97c \uc81c\uac70\ud558\uace0 \ub9ac\uc2a4\ud2b8 \ud615\ud0dc\ub85c \uc815\ub9ac\ud558\ub294 \ubc29\ubc95\uc744 \ubcf4\uc5ec\uc90d\ub2c8\ub2e4.<\/p>\n \uac00\uc838\uc628 \ub370\uc774\ud130\ub97c \ubd84\uc11d\ud558\uae30 \uc704\ud574 \ud5c8\uae45\ud398\uc774\uc2a4 \ud2b8\ub79c\uc2a4\ud3ec\uba38\ub97c \uc0ac\uc6a9\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4. \uc5ec\uae30\uc11c\ub294 \uc804\ucc98\ub9ac\ub41c \ud14d\uc2a4\ud2b8\ub97c BERT \ubaa8\ub378\uc5d0 \uc785\ub825\ud558\uc5ec \ud2b9\uc9d5\uc744 \ucd94\ucd9c\ud558\ub294 \ubc29\ubc95\uc744 \uc0b4\ud3b4\ubcf4\uaca0\uc2b5\ub2c8\ub2e4.<\/p>\n \ud5c8\uae45\ud398\uc774\uc2a4\uc758 BERT \ubaa8\ub378\uc744 \uc0ac\uc6a9\ud558\uc5ec \uc804\ucc98\ub9ac\ub41c \ud14d\uc2a4\ud2b8\uc758 \ud2b9\uc9d5\uc744 \ucd94\ucd9c\ud574\ubcf4\uaca0\uc2b5\ub2c8\ub2e4. \uc544\ub798\uc758 \ucf54\ub4dc\ub97c \ucc38\uace0\ud558\uc138\uc694.<\/p>\n \uc774\uc81c COVID-19 \uc815\ubcf4\ub97c \ubc14\ud0d5\uc73c\ub85c \ubb38\uc11c \uc694\uc57d\uc744 \ud574\ubcf4\uaca0\uc2b5\ub2c8\ub2e4. \ud5c8\uae45\ud398\uc774\uc2a4 \ub77c\uc774\ube0c\ub7ec\ub9ac\uc758 GPT-2<\/strong> \ubaa8\ub378\uc744 \uc0ac\uc6a9\ud558\uc5ec \uc694\uc57d\uc744 \uc0dd\uc131\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n \uc774\ubc88 \ud3ec\uc2a4\ud305\uc5d0\uc11c\ub294 \ud5c8\uae45\ud398\uc774\uc2a4 \ud2b8\ub79c\uc2a4\ud3ec\uba38\ub97c \ud65c\uc6a9\ud558\uc5ec \uc704\ud0a4\ud53c\ub514\uc544\uc5d0\uc11c \ubaa8\ub354\ub098\uc640 COVID-19\uc5d0 \uad00\ub828\ub41c \uc815\ubcf4\ub97c \uac00\uc838\uc624\uace0, \uc774\ub97c \uc804\ucc98\ub9ac \ubc0f \ubd84\uc11d\ud558\ub294 \uacfc\uc815\uc744 \uc0b4\ud3b4\ubcf4\uc558\uc2b5\ub2c8\ub2e4. \ud5c8\uae45\ud398\uc774\uc2a4\ub294 \ucd5c\uc2e0 \uc790\uc5f0\uc5b4 \ucc98\ub9ac \ubaa8\ub378\uc744 \uc27d\uac8c \uc774\uc6a9\ud560 \uc218 \uc788\ub294 \ud6cc\ub96d\ud55c \ub3c4\uad6c\uc774\uba70, \uc774\ub97c \ud1b5\ud574 \ud14d\uc2a4\ud2b8 \ub370\uc774\ud130\ub97c \ubcf4\ub2e4 \ud6a8\uacfc\uc801\uc73c\ub85c \ud65c\uc6a9\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4. \uc55e\uc73c\ub85c\ub3c4 \ub2e4\uc591\ud55c NLP \uc791\uc5c5\uc744 \ud1b5\ud574 \ub370\uc774\ud130 \ubd84\uc11d \uae30\uc220\uc744 \ud55c\uce35 \ub354 \ubc1c\uc804\uc2dc\ud0ac \uc218 \uc788\uc744 \uac83\uc785\ub2c8\ub2e4.<\/p>\n \ub610\ud55c, \ud5c8\uae45\ud398\uc774\uc2a4\ub294 \ucee4\ubba4\ub2c8\ud2f0\uc640\uc758 \ud611\uc5c5\uc744 \ud1b5\ud574 \uc0c8\ub85c\uc6b4 \ubaa8\ub378\uacfc \ub370\uc774\ud130\uc14b\uc774 \uacc4\uc18d\ud574\uc11c \uc5c5\ub370\uc774\ud2b8\ub418\uace0 \uc788\uc73c\ubbc0\ub85c, \uc9c0\uc18d\uc801\uc778 \ud559\uc2b5\uacfc \ud65c\uc6a9\uc744 \uad8c\uc7a5\ud569\ub2c8\ub2e4. \uc55e\uc73c\ub85c \uc5ec\ub7ec\ubd84\uc774 \ub2e4\uc591\ud55c NLP \uc791\uc5c5\uc5d0 \ub3c4\uc804\ud558\uc5ec \ub354 \ud070 \uc131\uacfc\ub97c \uc5bb\uae30\ub97c \ubc14\ub78d\ub2c8\ub2e4.<\/p>\n1.1 \ud5c8\uae45\ud398\uc774\uc2a4\uc758 \uc8fc\uc694 \ud2b9\uc9d5<\/h3>\n
\n
2. \uc124\uce58 \ubc0f \ud658\uacbd \uc124\uc815<\/h2>\n
pip install transformers wikipedia-api<\/code><\/pre>\n3. \uc704\ud0a4\ud53c\ub514\uc544\uc5d0\uc11c \uc815\ubcf4 \uac00\uc838\uc624\uae30<\/h2>\n
3.1 \uc704\ud0a4\ud53c\ub514\uc544 \ub370\uc774\ud130 \uac00\uc838\uc624\uae30 \uc608\uc81c<\/h3>\n
import wikipediaapi\n\n # \uc704\ud0a4\ud53c\ub514\uc544 API \ucd08\uae30\ud654\n wiki_wiki = wikipediaapi.Wikipedia('ko')\n\n # \"\ubaa8\ub354\ub098\" \ud398\uc774\uc9c0 \uac00\uc838\uc624\uae30\n page = wiki_wiki.page(\"\ubaa8\ub354\ub098\")\n\n # \ud398\uc774\uc9c0 \ub0b4\uc6a9 \ucd9c\ub825\n if page.exists():\n print(\"\uc81c\ubaa9: \", page.title)\n print(\"\ub0b4\uc6a9 \uc694\uc57d: \", page.summary[0:1000]) # \uccab 1000\uc790 \ucd9c\ub825\n else:\n print(\"\ud398\uc774\uc9c0\uac00 \uc874\uc7ac\ud558\uc9c0 \uc54a\uc2b5\ub2c8\ub2e4.\")<\/code><\/pre>\n3.2 COVID-19 \uad00\ub828 \uc815\ubcf4 \uac00\uc838\uc624\uae30<\/h3>\n
# \"COVID-19\" \ud398\uc774\uc9c0 \uac00\uc838\uc624\uae30\n covid_page = wiki_wiki.page(\"COVID-19\")\n\n # \ud398\uc774\uc9c0 \ub0b4\uc6a9 \ucd9c\ub825\n if covid_page.exists():\n print(\"\uc81c\ubaa9: \", covid_page.title)\n print(\"\ub0b4\uc6a9 \uc694\uc57d: \", covid_page.summary[0:1000]) # \uccab 1000\uc790 \ucd9c\ub825\n else:\n print(\"\ud398\uc774\uc9c0\uac00 \uc874\uc7ac\ud558\uc9c0 \uc54a\uc2b5\ub2c8\ub2e4.\")<\/code><\/pre>\n4. \ud14d\uc2a4\ud2b8 \uc804\ucc98\ub9ac<\/h2>\n
4.1 \uc804\ucc98\ub9ac \uacfc\uc815<\/h3>\n
import re\n\n def preprocess_text(text):\n # \ud2b9\uc218\ubb38\uc790 \uc81c\uac70\n text = re.sub(r'[^\uac00-\ud7a3a-zA-Z0-9\\s]', '', text)\n # \uc5ec\ub7ec \uacf5\ubc31\uc744 \ud558\ub098\ub85c\n text = re.sub(r'\\s+', ' ', text)\n return text.strip()\n\n # \uc804\ucc98\ub9ac \uc608\uc81c\n processed_text_moderna = preprocess_text(page.summary)\n processed_text_covid = preprocess_text(covid_page.summary)\n\n print(\"\uc804\ucc98\ub9ac\ub41c \ubaa8\ub354\ub098 \ud14d\uc2a4\ud2b8: \", processed_text_moderna)\n print(\"\uc804\ucc98\ub9ac\ub41c COVID-19 \ud14d\uc2a4\ud2b8: \", processed_text_covid)<\/code><\/pre>\n5. \ud5c8\uae45\ud398\uc774\uc2a4 \ud2b8\ub79c\uc2a4\ud3ec\uba38\ub85c \uc815\ubcf4 \ubd84\uc11d\ud558\uae30<\/h2>\n
5.1 BERT \ubaa8\ub378 \uc0ac\uc6a9\ud558\uae30<\/h3>\n
from transformers import BertTokenizer, BertModel\n import torch\n\n # BERT \ubaa8\ub378\uacfc \ud1a0\ud06c\ub098\uc774\uc800 \ub85c\ub4dc\n model_name = 'bert-base-multilingual-cased'\n tokenizer = BertTokenizer.from_pretrained(model_name)\n model = BertModel.from_pretrained(model_name)\n\n # \ud14d\uc2a4\ud2b8 \ud1a0\ud070\ud654 \ubc0f \ud150\uc11c\ub85c \ubcc0\ud658\n inputs = tokenizer(processed_text_moderna, return_tensors='pt', padding=True, truncation=True)\n\n # \ubaa8\ub378\uc5d0 \uc785\ub825\ud558\uace0 \ud2b9\uc9d5 \ucd94\ucd9c\n with torch.no_grad():\n outputs = model(**inputs)\n \n # \ud2b9\uc9d5 \ubca1\ud130\n embeddings = outputs.last_hidden_state\n print(\"\uc784\ubca0\ub529 \ud06c\uae30: \", embeddings.shape)<\/code><\/pre>\n6. \uc2e4\uc2b5 \uc608\uc81c: COVID-19 \uad00\ub828 \ubb38\uc11c \uc694\uc57d\ud558\uae30<\/h2>\n
6.1 GPT-2 \ubaa8\ub378\ub85c \uc694\uc57d\ud558\uae30<\/h3>\n
from transformers import GPT2Tokenizer, GPT2LMHeadModel\n\n # GPT-2 \ubaa8\ub378\uacfc \ud1a0\ud06c\ub098\uc774\uc800 \ub85c\ub4dc\n gpt2_model = GPT2LMHeadModel.from_pretrained('gpt2')\n gpt2_tokenizer = GPT2Tokenizer.from_pretrained('gpt2')\n\n # \uc694\uc57d\uc744 \uc704\ud55c \ud14d\uc2a4\ud2b8 \uc785\ub825\n input_text = \"COVID-19\ub294 SARS-CoV-2\uc5d0 \uc758\ud574 \ubc1c\uc0dd\ud558\uba70...\"\n input_ids = gpt2_tokenizer.encode(input_text, return_tensors='pt')\n\n # \uc694\uc57d \uc0dd\uc131\n summary_ids = gpt2_model.generate(input_ids, max_length=50, num_beams=5, early_stopping=True)\n summary = gpt2_tokenizer.decode(summary_ids[0], skip_special_tokens=True)\n\n print(\"\uc0dd\uc131\ub41c \uc694\uc57d: \", summary)<\/code><\/pre>\n\uacb0\ub860<\/h2>\n
\ucc38\uace0 \ubb38\ud5cc<\/h2>\n