{"id":29622,"date":"2024-10-28T01:55:47","date_gmt":"2024-10-28T01:55:47","guid":{"rendered":"http:\/\/atmokpo.com\/w\/?p=29622"},"modified":"2024-11-26T06:52:01","modified_gmt":"2024-11-26T06:52:01","slug":"%ed%97%88%ea%b9%85%ed%8e%98%ec%9d%b4%ec%8a%a4-%ed%8a%b8%eb%a0%8c%ec%8a%a4%ed%8f%ac%eb%a8%b8-%ed%99%9c%ec%9a%a9%ea%b0%95%ec%a2%8c-m2m100-%ed%95%9c%ea%b5%ad%ec%96%b4-%ed%85%8d%ec%8a%a4%ed%8a%b8","status":"publish","type":"post","link":"https:\/\/atmokpo.com\/w\/29622\/","title":{"rendered":"\ud5c8\uae45\ud398\uc774\uc2a4 \ud2b8\ub80c\uc2a4\ud3ec\uba38 \ud65c\uc6a9\uac15\uc88c, M2M100 \ud55c\uad6d\uc5b4 \ud14d\uc2a4\ud2b8 \ud1a0\ud06c\ub098\uc774\uc9d5"},"content":{"rendered":"

<\/p>\n

\ub525\ub7ec\ub2dd\uc758 \uae09\uc18d\ud55c \ubc1c\uc804\uacfc \ud568\uaed8 \uc790\uc5f0\uc5b4 \ucc98\ub9ac(NLP) \ubd84\uc57c\ub294 \ub180\ub77c\uc6b4 \ubcc0\ud654\uc758 \uc911\uac04\uc5d0 \uc788\uc2b5\ub2c8\ub2e4. \ud2b9\ud788, \ucd5c\uadfc\uc5d0\ub294 \ub2e4\uc591\ud55c \uc0ac\uc804 \ud6c8\ub828\ub41c \ubaa8\ub378\uc744 \uc81c\uacf5\ud558\ub294 \ud5c8\uae45\ud398\uc774\uc2a4(Hugging Face)<\/strong> \ub77c\uc774\ube0c\ub7ec\ub9ac\uac00 \uc8fc\ubaa9\ubc1b\uace0 \uc788\uc2b5\ub2c8\ub2e4. \uc624\ub298\uc740 \uadf8\uc911\uc5d0\uc11c\ub3c4 M2M100<\/strong> \ubaa8\ub378\uc744 \uc0ac\uc6a9\ud558\uc5ec \ud55c\uad6d\uc5b4 \ud14d\uc2a4\ud2b8\ub97c \ud1a0\ud06c\ub098\uc774\uc9d5\ud558\ub294 \ubc29\ubc95\uc744 \uc18c\uac1c\ud558\uaca0\uc2b5\ub2c8\ub2e4.<\/p>\n

1. M2M100 \ubaa8\ub378 \uc18c\uac1c<\/h2>\n

M2M100\uc740 Facebook AI<\/strong>\uc5d0\uc11c \uac1c\ubc1c\ud55c \ubaa8\ub378\ub85c, \ub2e4\uad6d\uc5b4 \ubc88\uc5ed\uc744 \uc704\ud55c \uc0ac\uc804 \ud6c8\ub828\ub41c \ubcc0\uc555\uae30(Transformer) \ubaa8\ub378\uc785\ub2c8\ub2e4. \uc774 \ubaa8\ub378\uc740 100\uac1c \uc774\uc0c1\uc758 \uc5b8\uc5b4\ub97c \uc9c0\uc6d0\ud558\uba70, \ub2e4\uc591\ud55c \uc5b8\uc5b4 \uac04\uc758 \ubc88\uc5ed\uc744 \uc704\ud574 \uc124\uacc4\ub418\uc5c8\uc2b5\ub2c8\ub2e4. M2M100\uc740 \ud2b9\ud788 \ubb38\uc7a5 \uc218\uc900\uc5d0\uc11c\uc758 \ubc88\uc5ed \uc131\ub2a5\uc774 \ub6f0\uc5b4\ub098\uba70, \ud55c\uad6d\uc5b4\uc640 \uac19\uc740 \uc800\ubd84\ub7c9 \uc5b8\uc5b4\uc5d0\uc11c\ub3c4 \uc720\uc6a9\ud558\uac8c \ud65c\uc6a9\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n

2. \ud5c8\uae45\ud398\uc774\uc2a4 \ub77c\uc774\ube0c\ub7ec\ub9ac \uc124\uce58<\/h2>\n

\ubaa8\ub378\uc744 \uc0ac\uc6a9\ud558\uae30 \uc704\ud574\uc11c\ub294 \uba3c\uc800 \ud5c8\uae45\ud398\uc774\uc2a4 \ud2b8\ub79c\uc2a4\ud3ec\uba38 \ub77c\uc774\ube0c\ub7ec\ub9ac<\/strong>\ub97c \uc124\uce58\ud574\uc57c \ud569\ub2c8\ub2e4. \uc774\ub294 pip\ub97c \ud1b5\ud574 \uac04\ub2e8\ud558\uac8c \uc124\uce58\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n

pip install transformers<\/code><\/pre>\n
3. \ud1a0\ud06c\ub098\uc774\uc800(Tokenization)\ub780?<\/h2>\n
\ud1a0\ud06c\ub098\uc774\uc9d5\uc740 \uc785\ub825 \ubb38\uc7a5\uc744 \uac1c\ubcc4 \ub2e8\uc704(\ud1a0\ud070)\ub85c \ub098\ub204\ub294 \uacfc\uc815\uc785\ub2c8\ub2e4. \uc790\uc5f0\uc5b4 \ucc98\ub9ac \ubaa8\ub378\uc740 \ud14d\uc2a4\ud2b8\ub97c \uc9c1\uc811 \ucc98\ub9ac\ud560 \uc218 \uc5c6\uae30 \ub54c\ubb38\uc5d0, \ud14d\uc2a4\ud2b8\ub97c \uc22b\uc790\ub85c \ubcc0\ud658\ud558\uc5ec \ubaa8\ub378\uc5d0 \uc785\ub825\ud560 \uc218 \uc788\uc5b4\uc57c \ud569\ub2c8\ub2e4. \uc774\ub7ec\ud55c \uc791\uc5c5\uc744 \uc218\ud589\ud558\ub294 \uac83\uc774 \ubc14\ub85c ‘\ud1a0\ud06c\ub098\uc774\uc9d5’\uc785\ub2c8\ub2e4.<\/p>\n
4. M2M100\uc758 \ud1a0\ud06c\ub098\uc774\uc800 \uc0ac\uc6a9\ud558\uae30<\/h2>\n
\uc774\uc81c M2M100 \ubaa8\ub378\uc744 \uc0ac\uc6a9\ud558\uc5ec \ud55c\uad6d\uc5b4 \ud14d\uc2a4\ud2b8\ub97c \ud1a0\ud06c\ub098\uc774\uc9d5\ud558\ub294 \ubc29\ubc95\uc744 \uc0b4\ud3b4\ubcf4\uaca0\uc2b5\ub2c8\ub2e4. \uc544\ub798\uc758 \ucf54\ub4dc\ub97c \uc2e4\ud589\ud558\uc5ec \ud1a0\ud06c\ub098\uc774\uc800\ub97c \ub85c\ub4dc\ud558\uace0, \uc608\uc2dc \ud55c\uad6d\uc5b4 \ubb38\uc7a5\uc744 \ud1a0\ud06c\ub098\uc774\uc9d5\ud574 \ubcf4\uaca0\uc2b5\ub2c8\ub2e4.<\/p>\n
from transformers import M2M100Tokenizer\n\n# M2M100 \ud1a0\ud06c\ub098\uc774\uc800 \ub85c\ub4dc\ntokenizer = M2M100Tokenizer.from_pretrained(\"facebook\/m2m100_418M\")\n\n# \ud55c\uad6d\uc5b4 \ud14d\uc2a4\ud2b8 \uc785\ub825\ntext = \"\uc548\ub155\ud558\uc138\uc694! \ub525\ub7ec\ub2dd\uc744 \ubc30\uc6b0\uace0 \uc788\uc2b5\ub2c8\ub2e4.\"\n\n# \ud14d\uc2a4\ud2b8 \ud1a0\ud06c\ub098\uc774\uc9d5\ntokens = tokenizer.tokenize(text)\n\n# \ud1a0\ud070\ud654 \uacb0\uacfc \ucd9c\ub825\nprint(\"\ud1a0\ud06c\ub098\uc774\uc9d5 \uacb0\uacfc:\", tokens)<\/code><\/pre>\n
4.1 \ucf54\ub4dc \uc124\uba85<\/h3>\n
\uc704 \ucf54\ub4dc\ub294 M2M100Tokenizer<\/code> \ud074\ub798\uc2a4\ub97c \uc0ac\uc6a9\ud558\uc5ec M2M100 \ubaa8\ub378\uc744 \uc704\ud55c \ud1a0\ud06c\ub098\uc774\uc800\ub97c \ub85c\ub4dc\ud558\uace0 \uc788\uc2b5\ub2c8\ub2e4. \ub2e4\uc74c\uc73c\ub85c, \ud55c\uad6d\uc5b4 \ubb38\uc7a5\uc744 \uc785\ub825\ud558\uc5ec tokenizer.tokenize()<\/code> \uba54\uc11c\ub4dc\ub97c \ud638\ucd9c\ud568\uc73c\ub85c\uc368 \ud574\ub2f9 \ubb38\uc7a5\uc744 \ud1a0\ud070\ud654\ud569\ub2c8\ub2e4.<\/p>\n
5. \ud1a0\ud06c\ub098\uc774\uc9d5 \ucd9c\ub825 \uacb0\uacfc \ud574\uc11d<\/h2>\n
\ud1a0\ud06c\ub098\uc774\uc9d5\uc758 \ucd9c\ub825 \uacb0\uacfc\ub294 \ubaa8\ub378\uc774 \uc774\ud574\ud560 \uc218 \uc788\ub3c4\ub85d \ubcc0\ud658\ub41c \ud1a0\ud070 \ub9ac\uc2a4\ud2b8\uc785\ub2c8\ub2e4. \uc608\ub97c \ub4e4\uc5b4, \ud55c\uad6d\uc5b4 \ubb38\uc7a5 “\uc548\ub155\ud558\uc138\uc694! \ub525\ub7ec\ub2dd\uc744 \ubc30\uc6b0\uace0 \uc788\uc2b5\ub2c8\ub2e4.”\uc758 \uacbd\uc6b0, \ud1a0\ud06c\ub098\uc774\uc9d5 \ud6c4\uc5d0\ub3c4 \uac01 \ub2e8\uc5b4\uc758 \uc758\ubbf8\ub97c \uc798 \ubcf4\uc874\ud558\uba74\uc11c \ubaa8\ub378\uc5d0 \uc785\ub825\ud560 \uc218 \uc788\ub294 \ud615\uc2dd\uc73c\ub85c \ubcc0\ud658\ub429\ub2c8\ub2e4.<\/p>\n
5.1 \uc608\uc2dc \ucd9c\ub825<\/h3>\n
\uc608\uc0c1\ub41c \ucd9c\ub825:\n\ud1a0\ud06c\ub098\uc774\uc9d5 \uacb0\uacfc: ['\uc548\ub155', '##\ud558', '##\uc138', '##\uc694', '!', '\ub525', '##\ub7ec', '##\ub2dd', '\uc744', '\ubc30', '##\uc6b0', '\uace0', '\uc788', '##\uc2b5', '##\ub2c8', '##\ub2e4', '.']<\/code><\/pre>\n
\ucd9c\ub825\ub41c \uacb0\uacfc\ub97c \ubcf4\uba74, \uc785\ub825\ub41c \ubb38\uc7a5\uc774 \uc5ec\ub7ec \uac1c\uc758 \ud1a0\ud070\uc73c\ub85c \ub098\ub204\uc5b4\uc84c\uc74c\uc744 \ud655\uc778\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4. ‘\uc548\ub155\ud558\uc138\uc694’\ub294 \ub450 \uac1c\uc758 \ud1a0\ud070\uc73c\ub85c \ub098\ub204\uc5b4\uc84c\uc73c\uba70, \uac01\uac01\uc758 \ud615\ud0dc\uc18c\uc5d0 \ud574\ub2f9\ud558\ub294 ‘\uc548\ub155’, ‘\ud558\uc138’\ub85c \uad6c\ubd84\ub418\uc5c8\uc2b5\ub2c8\ub2e4.<\/p>\n
6. \ucd94\uac00 \uc635\uc158: \ud1a0\ud06c\ub098\uc774\uc800\uc758 \ub2e4\uc591\ud55c \uae30\ub2a5<\/h2>\n
\ud1a0\ud06c\ub098\uc774\uc800\ub294 \ub2e8\uc21c\ud55c \ud1a0\ud06c\ub098\uc774\uc9d5\ubfd0\ub9cc \uc544\ub2c8\ub77c, \ud2b9\ubcc4\ud55c \uae30\ud638\ub098 \uc704\uce58 \uc815\ubcf4\ub97c \ucc98\ub9ac\ud558\ub294 \uae30\ub2a5\ub3c4 \uc81c\uacf5\ud569\ub2c8\ub2e4. \uba87 \uac00\uc9c0 \ucd94\uac00 \uae30\ub2a5\uc744 \uc0b4\ud3b4\ubcf4\uaca0\uc2b5\ub2c8\ub2e4.<\/p>\n
6.1 \ud328\ub529(Padding)<\/h3>\n
\ubaa8\ub378\uc5d0 \uc785\ub825\ud560 \ud14d\uc2a4\ud2b8\uc758 \uae38\uc774\ub97c \ub3d9\uc77c\ud558\uac8c \ub9de\ucd94\uae30 \uc704\ud574 \ud328\ub529\uc744 \ucd94\uac00\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4. \uc544\ub798\uc758 \ucf54\ub4dc\ub97c \ucc38\uace0\ud558\uc2ed\uc2dc\uc624.<\/p>\n
# \uc5ec\ub7ec \uac1c\uc758 \ubb38\uc7a5 \uc900\ube44\ntexts = [\"\uc548\ub155\ud558\uc138\uc694! \ub525\ub7ec\ub2dd\uc744 \ubc30\uc6b0\uace0 \uc788\uc2b5\ub2c8\ub2e4.\", \"\uc774\ubc88 \uac15\uc88c\ub294 \ud5c8\uae45\ud398\uc774\uc2a4\ub97c \ud65c\uc6a9\ud569\ub2c8\ub2e4.\"]\n\n# \ubb38\uc7a5\uc744 \ud1a0\ud06c\ub098\uc774\uc9d5\ud558\uace0 \ud328\ub529\nencoded_inputs = tokenizer(texts, padding=True, truncation=True, return_tensors=\"pt\")\n\nprint(\"\ud328\ub529\ub41c \uc785\ub825:\", encoded_inputs)<\/code><\/pre>\n
6.2 \ub514\ucf54\ub529(Decoding)<\/h3>\n
\ud1a0\ud06c\ub098\uc774\uc9d5\ub41c \uacb0\uacfc\ub97c \ub2e4\uc2dc \uc6d0\ub798\uc758 \ud14d\uc2a4\ud2b8\ub85c \ubcc0\ud658\ud558\ub294 \uae30\ub2a5\ub3c4 \uc788\uc2b5\ub2c8\ub2e4. \uc774\ub97c \ud1b5\ud574 \ubaa8\ub378\uc758 \ucd9c\ub825\uc744 \uc27d\uac8c \ud655\uc778\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n
# \ud1a0\ud06c\ub098\uc774\uc9d5\ub41c \ud14d\uc2a4\ud2b8 \ub514\ucf54\ub529\ninput_ids = encoded_inputs['input_ids'][0]\ndecoded_text = tokenizer.decode(input_ids)\n\nprint(\"\ub514\ucf54\ub529\ub41c \ud14d\uc2a4\ud2b8:\", decoded_text)<\/code><\/pre>\n
7. \ub9c8\ubb34\ub9ac<\/h2>\n
\ubcf8 \uac15\uc88c\uc5d0\uc11c\ub294 Hugging Face\uc758 M2M100 \ubaa8\ub378\uc744 \ud65c\uc6a9\ud558\uc5ec \ud55c\uad6d\uc5b4 \ud14d\uc2a4\ud2b8\ub97c \ud6a8\uacfc\uc801\uc73c\ub85c \ud1a0\ud06c\ub098\uc774\uc9d5\ud558\ub294 \ubc29\ubc95\uc744 \uc18c\uac1c\ud588\uc2b5\ub2c8\ub2e4. M2M100 \ubaa8\ub378\uc740 \ub2e4\uc591\ud55c \uc5b8\uc5b4 \uac04\uc758 \ubc88\uc5ed \uc131\ub2a5\uc774 \ub6f0\uc5b4\ub098\uba70, \ud55c\uad6d\uc5b4\uc640 \uac19\uc740 \uc800\ubd84\ub7c9\uc758 \uc5b8\uc5b4\uc5d0\uc11c\ub3c4 \uc6b0\uc218\ud55c \uc131\ub2a5\uc744 \ubcf4\uc785\ub2c8\ub2e4. \uc774\ub97c \ud1b5\ud574 \uc790\uc5f0\uc5b4 \ucc98\ub9ac \uc560\ud50c\ub9ac\ucf00\uc774\uc158\uc5d0\uc11c \ud6a8\uc728\uc801\uc73c\ub85c \ubaa8\ub378\uc744 \ud65c\uc6a9\ud560 \uc218 \uc788\uc744 \uac83\uc785\ub2c8\ub2e4.<\/p>\n
8. \ucc38\uace0 \uc790\ub8cc<\/h2>\n