\ub525\ub7ec\ub2dd \ubd84\uc57c\uc5d0\uc11c \uc790\uc5f0\uc5b4 \ucc98\ub9ac(NLP)\ub294 \ub9e4\uc6b0 \uc911\uc694\ud55c \uc5ed\ud560\uc744 \ud558\uace0 \uc788\uc73c\uba70, \ud5c8\uae45\ud398\uc774\uc2a4(Hugging Face)\ub294 \uc774\ub7ec\ud55c \ubd84\uc57c\uc5d0\uc11c \uac00\uc7a5 \ub110\ub9ac \uc0ac\uc6a9\ub418\ub294 \ub77c\uc774\ube0c\ub7ec\ub9ac \uc911 \ud558\ub098\uc785\ub2c8\ub2e4. \uc774\ubc88 \uac15\uc88c\uc5d0\uc11c\ub294 \ud5c8\uae45\ud398\uc774\uc2a4\uc758 \ud2b8\ub80c\uc2a4\ud3ec\uba38(transformer) \ub77c\uc774\ube0c\ub7ec\ub9ac \uc911 \ud5c8\uae45\ud398\uc774\uc2a4 \ud2b8\ub80c\uc2a4\ud3ec\uba38 \ub77c\uc774\ube0c\ub7ec\ub9ac\ub294 \ub2e4\uc591\ud55c \uc790\uc5f0\uc5b4 \ucc98\ub9ac \ubaa8\ub378\uc744 \uc27d\uac8c \uc0ac\uc6a9\ud560 \uc218 \uc788\ub3c4\ub85d \uc9c0\uc6d0\ud558\ub294 \ud30c\uc774\uc36c \ud328\ud0a4\uc9c0\uc785\ub2c8\ub2e4. \uc774 \ub77c\uc774\ube0c\ub7ec\ub9ac\ub97c \ud1b5\ud574 \uc0ac\uc804 \ud6c8\ub828\ub41c \ubaa8\ub378\uc744 \ub85c\ub4dc\ud558\uace0, \ub370\uc774\ud130 \uc804\ucc98\ub9ac \ubc0f \ubaa8\ub378 \ucd94\ub860\uc744 \uc27d\uac8c \ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n \ud1a0\ud06c\ub098\uc774\uc800\ub294 \uc785\ub825\ub41c \ud14d\uc2a4\ud2b8\ub97c \ud1a0\ud070(token) \ub2e8\uc704\ub85c \ubd84\ub9ac\ud558\ub294 \uc5ed\ud560\uc744 \ud569\ub2c8\ub2e4. \ud1a0\ud070\uc740 \ub2e8\uc5b4, \ubd80\ubd84 \ub2e8\uc5b4, \ubb38\uc790 \ub4f1 \ub2e4\uc591\ud55c \ud615\ud0dc\uc77c \uc218 \uc788\uc73c\uba70, \ubaa8\ub378\uc774 \uc774\ud574\ud560 \uc218 \uc788\ub294 \ud615\ud0dc\ub85c \ub370\uc774\ud130\ub97c \ubcc0\ud658\ud558\ub294 \uc911\uc694\ud55c \uc5ed\ud560\uc744 \ud569\ub2c8\ub2e4. \ud5c8\uae45\ud398\uc774\uc2a4\uc758 \ud1a0\ud06c\ub098\uc774\uc800\ub294 \uc774\ub7ec\ud55c \uc791\uc5c5\uc744 \uc790\ub3d9\uc73c\ub85c \ucc98\ub9ac\ud574 \uc8fc\uba70, \uc0ac\uc804 \ud6c8\ub828\ub41c \ubaa8\ub378\uacfc \ud568\uaed8 \uc0ac\uc6a9\ub420 \uc218 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n \ud5c8\uae45\ud398\uc774\uc2a4\uc5d0\uc11c\ub294 \ub2e4\uc591\ud55c \uc885\ub958\uc758 \ud1a0\ud06c\ub098\uc774\uc800\ub97c \uc9c0\uc6d0\ud569\ub2c8\ub2e4:<\/p>\n \ud5c8\uae45\ud398\uc774\uc2a4 \ub77c\uc774\ube0c\ub7ec\ub9ac\ub97c \uc0ac\uc6a9\ud558\uae30 \uc704\ud574 \ud544\uc694\ud55c \ud328\ud0a4\uc9c0\ub97c \uc124\uce58\ud569\ub2c8\ub2e4. \uc544\ub798 \uba85\ub839\uc5b4\ub97c \uc0ac\uc6a9\ud558\uc5ec \uc774\uc81c \ud1a0\ud06c\ub098\uc774\uc800\ub97c \uc0ac\uc6a9\ud558\uc5ec \uc785\ub825 \ud14d\uc2a4\ud2b8\uc758 \ube48\ub3c4\uc218\ub97c \uacc4\uc0b0\ud574\ubcf4\uaca0\uc2b5\ub2c8\ub2e4. \ub2e4\uc74c\uc740 \ucf54\ub4dc \uc608\uc81c\uc785\ub2c8\ub2e4:<\/p>\n \uc704\uc758 \ucf54\ub4dc\uc5d0\uc11c\ub294 \uc704 \ucf54\ub4dc\uc758 \uc2e4\ud589 \uacb0\uacfc\ub294 \uac01 \ud1a0\ud070\uacfc \ube48\ub3c4\uc218\ub97c \ud3ec\ud568\ud55c \ud1a0\ud070\uc758 \ube48\ub3c4\uc218\ub97c \uae30\ubc18\uc73c\ub85c \ucd94\uac00\uc801\uc73c\ub85c \ub2e4\uc74c\uacfc \uac19\uc740 \ubd84\uc11d \uc791\uc5c5\uc744 \uc218\ud589\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4:<\/p>\n \uc774\uc81c \uc870\uae08 \ub354 \ubcf5\uc7a1\ud55c \uc608\uc81c\ub85c \ub4e4\uc5b4\uac00 \ubcf4\uaca0\uc2b5\ub2c8\ub2e4. \uc5ec\ub7ec \ubb38\uc7a5\uc73c\ub85c \uc774\ub8e8\uc5b4\uc9c4 \ubb38\uc11c\uc5d0\uc11c \ub2e8\uc5b4 \ube48\ub3c4\uc218\ub97c \uacc4\uc0b0\ud574\ubcf4\uaca0\uc2b5\ub2c8\ub2e4. \uae30\ubcf8\uc801\uc73c\ub85c \uc81c\uacf5\ub418\ub294 \uc5ec\ub7ec \ubb38\uc7a5\uc744 \uc0ac\uc6a9\ud558\ub418, \uac01 \ubb38\uc7a5\uc774 \uc758\ubbf8 \uc788\uac8c \ucc98\ub9ac\ub420 \uc218 \uc788\ub3c4\ub85d \ud569\uccd0\ubcf4\uaca0\uc2b5\ub2c8\ub2e4.<\/p>\n \uc774\ubc88 \uac15\uc88c\uc5d0\uc11c\ub294 \ud5c8\uae45\ud398\uc774\uc2a4\uc758 \ud1a0\ud06c\ub098\uc774\uc800\ub97c \uc0ac\uc6a9\ud558\uc5ec \ubb38\uc7a5\uc758 \ube48\ub3c4\uc218\ub97c \uacc4\uc0b0\ud558\ub294 \ubc29\ubc95\uc5d0 \ub300\ud574 \ubc30\uc6e0\uc2b5\ub2c8\ub2e4. \uc774\ub97c \ud1b5\ud574 \uc790\uc5f0\uc5b4 \ucc98\ub9ac \ubd84\uc57c\uc5d0\uc11c \ud14d\uc2a4\ud2b8 \ub370\uc774\ud130\uc758 \uc758\ubbf8\ub97c \ub354 \uae4a\uc774 \uc774\ud574\ud560 \uc218 \uc788\ub294 \ud1a0\ub300\ub97c \ub9c8\ub828\ud588\uc2b5\ub2c8\ub2e4.<\/p>\n \uc55e\uc73c\ub85c\ub294 \ub354 \ub2e4\uc591\ud55c NLP \uae30\uc220\uacfc \ubaa8\ub378\uc744 \ud65c\uc6a9\ud558\uc5ec \uc2e4\uc81c \ub370\uc774\ud130\ub97c \ubd84\uc11d\ud558\uace0, \ud1b5\uacc4\uc801 \uc815\ubcf4\ub97c \uae30\ubc18\uc73c\ub85c \uae30\uacc4 \ud559\uc2b5 \ubaa8\ub378\uc744 \uad6c\ucd95\ud558\ub294 \ub4f1\uc758 \uc791\uc5c5\uc744 \uc9c4\ud589\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n \ub354 \uc54c\uace0 \uc2f6\uc73c\uc2dc\ub2e4\uba74 \ub2e4\uc74c\uc758 \uc790\ub8cc\ub97c \ucc38\uace0\ud558\uc2dc\uae30 \ubc14\ub78d\ub2c8\ub2e4:<\/p>\n \uc5ec\ub7ec\ubd84\uc758 \ub525\ub7ec\ub2dd \ud559\uc2b5 \uc5ec\uc815\uc5d0 \ub3c4\uc6c0\uc774 \ub418\uae38 \ubc14\ub78d\ub2c8\ub2e4!<\/p>\n <\/body><\/p>\n","protected":false},"excerpt":{"rendered":"\ud1a0\ud06c\ub098\uc774\uc800(tokenizer)<\/code>\ub97c \ud65c\uc6a9\ud558\uc5ec \ud14d\uc2a4\ud2b8 \ub370\uc774\ud130\ub97c \ucc98\ub9ac\ud558\uace0, \uac01 \ub2e8\uc5b4\uc758 \ube48\ub3c4\uc218\ub97c \ud569\uc0b0\ud558\ub294 \ubc29\ubc95\uc5d0 \ub300\ud574 \uc790\uc138\ud788 \uc54c\uc544\ubcf4\uaca0\uc2b5\ub2c8\ub2e4.<\/p>\n1. \ud5c8\uae45\ud398\uc774\uc2a4 \ud2b8\ub80c\uc2a4\ud3ec\uba38 \ub77c\uc774\ube0c\ub7ec\ub9ac \uc18c\uac1c<\/h2>\n
2. \ud1a0\ud06c\ub098\uc774\uc800\ub780?<\/h2>\n
2.1. \ud1a0\ud06c\ub098\uc774\uc800\uc758 \uc885\ub958<\/h3>\n
\n
BertTokenizer<\/code>: BERT \ubaa8\ub378\uc5d0 \ucd5c\uc801\ud654\ub41c \ud1a0\ud06c\ub098\uc774\uc800<\/li>\nGPT2Tokenizer<\/code>: GPT-2 \ubaa8\ub378\uc5d0 \ucd5c\uc801\ud654\ub41c \ud1a0\ud06c\ub098\uc774\uc800<\/li>\nRobertaTokenizer<\/code>: RoBERTa \ubaa8\ub378\uc5d0 \ucd5c\uc801\ud654\ub41c \ud1a0\ud06c\ub098\uc774\uc800<\/li>\nT5Tokenizer<\/code>: T5 \ubaa8\ub378\uc5d0 \ucd5c\uc801\ud654\ub41c \ud1a0\ud06c\ub098\uc774\uc800<\/li>\n<\/ul>\n3. \ud658\uacbd \uc124\uc815<\/h2>\n
transformers<\/code>\uc640 torch<\/code>\ub97c \uc124\uce58\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4:<\/p>\npip install transformers torch<\/code><\/pre>\n4. \ud1a0\ud06c\ub098\uc774\uc800 \uc0ac\uc6a9 \uc608\uc81c<\/h2>\n
4.1. \ucf54\ub4dc \uc608\uc81c<\/h3>\n
from transformers import BertTokenizer\nfrom collections import Counter\n\n# BERT Tokenizer \ub85c\ub4dc\ntokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n\n# \ubd84\uc11d\ud560 \ubb38\uc7a5 \ub9ac\uc2a4\ud2b8\nsentences = [\n \"Hey, how are you?\",\n \"I am fine, thank you!\",\n \"How about you?\"\n]\n\n# \ud1a0\ud070 \ube48\ub3c4\uc218 \uacc4\uc0b0\ndef get_token_frequency(sentences):\n tokens = []\n for sentence in sentences:\n # \ubb38\uc7a5\uc744 \ud1a0\ud070\ud654\ud569\ub2c8\ub2e4.\n encoded_tokens = tokenizer.encode(sentence, add_special_tokens=True)\n # \ud1a0\ud070\uc744 \ub9ac\uc2a4\ud2b8\uc5d0 \ucd94\uac00\ud569\ub2c8\ub2e4.\n tokens.extend(encoded_tokens)\n \n # \ud1a0\ud070 \ube48\ub3c4\uc218 \uacc4\uc0b0\n token_counts = Counter(tokens)\n return token_counts\n\n# \ube48\ub3c4\uc218 \ucd9c\ub825\ntoken_frequencies = get_token_frequency(sentences)\nprint(token_frequencies)\n<\/code><\/pre>\n4.2. \ucf54\ub4dc \uc124\uba85<\/h3>\n
BertTokenizer<\/code>\ub97c \uc0ac\uc6a9\ud558\uc5ec \uac01 \ubb38\uc7a5\uc744 \ud1a0\ud070\ud654\ud558\uace0, \uac01 \ud1a0\ud070\uc758 \ube48\ub3c4\uc218\ub97c \uacc4\uc0b0\ud569\ub2c8\ub2e4.<\/p>\n\n
from transformers import BertTokenizer<\/code>: \ud5c8\uae45\ud398\uc774\uc2a4\uc5d0\uc11c \uc81c\uacf5\ud558\ub294 BERT \ud1a0\ud06c\ub098\uc774\uc800\ub97c \uc784\ud3ec\ud2b8\ud569\ub2c8\ub2e4.<\/li>\nCounter<\/code>: \uac01 \ud1a0\ud070\uc758 \ube48\ub3c4\uc218\ub97c \uc138\uae30 \uc704\ud574 collections<\/code> \ubaa8\ub4c8\uc758 Counter<\/code> \ud074\ub798\uc2a4\ub97c \uc0ac\uc6a9\ud569\ub2c8\ub2e4.<\/li>\ntokenizer.encode(sentence, add_special_tokens=True)<\/code>: \uc785\ub825 \ubb38\uc7a5\uc744 \ud1a0\ud070\ud654\ud558\uba70, BERT\uc640 \uac19\uc740 \ubaa8\ub378\uc5d0\uc11c \uc0ac\uc6a9\ud560 \uc218 \uc788\ub3c4\ub85d \ud2b9\ubcc4\ud55c \ud1a0\ud070\uc744 \ucd94\uac00\ud569\ub2c8\ub2e4.<\/li>\nCounter(tokens)<\/code>: \ud1a0\ud070\uc758 \ube48\ub3c4\uc218\ub97c \uacc4\uc0b0\ud558\uc5ec \ubc18\ud658\ud569\ub2c8\ub2e4.<\/li>\n<\/ul>\n5. \uacb0\uacfc \ubd84\uc11d<\/h2>\n
Counter<\/code> \uac1d\uccb4\uc785\ub2c8\ub2e4. \uc774\ub97c \ud1b5\ud574 \uac01 \ud1a0\ud070\uc774 \uc5bc\ub9c8\ub098 \uc790\uc8fc \ub4f1\uc7a5\ud588\ub294\uc9c0\ub97c \ud655\uc778\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4. \ud544\uc694\ud55c \uacbd\uc6b0, \ud2b9\uc815 \ud1a0\ud070\uc758 \ube48\ub3c4\uc218\ub9cc \ud544\ud130\ub9c1\ud574\uc11c \ucd9c\ub825\ud560 \uc218\ub3c4 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n5.1. \ucd94\uac00 \ubd84\uc11d<\/h3>\n
\n
6. \uc2e4\uc2b5: \ubb38\uc11c\uc758 \ube48\ub3c4\uc218 \ubd84\uc11d<\/h2>\n
document = \"\"\"\n Natural Language Processing (NLP) is a fascinating field.\n It encompasses understanding, interpreting, and generating human language.\n With the help of deep learning and specialized models like BERT and GPT, we can perform various NLP tasks efficiently.\n The Hugging Face library offers pre-trained models that simplify the implementation of NLP.\n \"\"\"\n \n# \ubb38\uc11c\uc758 \ube48\ub3c4\uc218 \uacc4\uc0b0 \ubc0f \ucd9c\ub825\ntoken_frequencies_document = get_token_frequency([document])\nprint(token_frequencies_document)\n<\/code><\/pre>\n7. \uc694\uc57d \ubc0f \uacb0\ub860<\/h2>\n
8. \ucc38\uace0 \uc790\ub8cc<\/h2>\n
\n