\uc790\uc5f0\uc5b4 \ucc98\ub9ac(Natural Language Processing, NLP)\ub294 \uc778\uacf5\uc9c0\ub2a5(AI)\uacfc \uba38\uc2e0\ub7ec\ub2dd(ML)\uc758 \ud55c \ubd84\uc57c\ub85c, \ucef4\ud4e8\ud130\uac00 \uc790\uc5f0\uc5b4\ub97c \uc774\ud574\ud558\uace0 \ud574\uc11d\ud558\uac8c \ud558\uc5ec \uc778\uac04\uacfc\uc758 \uc0c1\ud638\uc791\uc6a9\uc744 \uac00\ub2a5\ud558\uac8c \ud558\ub294 \uae30\uc220\uc785\ub2c8\ub2e4. \ud604\ub300\uc758 NLP\ub294 \ub9ce\uc740 \ubc1c\uc804\uc744 \uc774\ub8e8\uc5c8\uace0, \ud2b9\ud788 \ub525 \ub7ec\ub2dd\uc758 \ub3c4\uc785\uc740 \uc774\ub7ec\ud55c \ubc1c\uc804\uc5d0 \ud070 \uc5ed\ud560\uc744 \ud558\uc600\uc2b5\ub2c8\ub2e4. \ub610\ud55c, \uc784\ubca0\ub529(Embedding)\uc740 \ub2e8\uc5b4, \ubb38\uc7a5, \ub610\ub294 \ubb38\uc11c\uc640 \uac19\uc740 \ube44\uc815\ud615 \ub370\uc774\ud130\ub97c \uace0\ucc28\uc6d0\uc758 \ubca1\ud130\ub85c \ubcc0\ud658\ud558\ub294 \uacfc\uc815\uc73c\ub85c, \ud14d\uc2a4\ud2b8 \ub370\uc774\ud130\uc758 \uc758\ubbf8\uc801 \uc720\uc0ac\uc131\uc744 \ube44\uad50\ud558\uace0 \ud0d0\uc0c9\ud558\ub294 \ub370 \uc720\uc6a9\ud569\ub2c8\ub2e4. \ubcf8 \uae00\uc5d0\uc11c\ub294 \ub525 \ub7ec\ub2dd\uc744 \uc774\uc6a9\ud55c \uc790\uc5f0\uc5b4 \ucc98\ub9ac \uae30\uc220\uacfc Faiss\ub97c \ud65c\uc6a9\ud55c \uc784\ubca0\ub529 \uac80\uc0c9\uae30 \uad6c\ucd95 \ubc29\ubc95\uc5d0 \ub300\ud574 \uc0c1\uc138\ud788 \uc124\uba85\ud558\uaca0\uc2b5\ub2c8\ub2e4.<\/p>\n
\uc790\uc5f0\uc5b4 \ucc98\ub9ac\uc758 \ubaa9\ud45c\ub294 \ud14d\uc2a4\ud2b8 \ub370\uc774\ud130\ub97c \uc774\ud574\ud558\uace0 \uc774\ub97c \ud65c\uc6a9\ud558\uc5ec \ub2e4\uc591\ud55c \uc791\uc5c5\uc744 \uc218\ud589\ud558\ub294 \uac83\uc785\ub2c8\ub2e4. NLP\uc758 \uc8fc\uc694 \uc791\uc5c5\uc73c\ub85c\ub294 \ub2e4\uc74c\uacfc \uac19\uc740 \uac83\ub4e4\uc774 \uc788\uc2b5\ub2c8\ub2e4:<\/p>\n
\ub525 \ub7ec\ub2dd\uc740 \uc778\uacf5\uc9c0\ub2a5\uc758 \ud558\uc704 \ubd84\uc57c\ub85c, \uc778\uacf5 \uc2e0\uacbd\ub9dd(Artificial Neural Networks)\uc744 \uc0ac\uc6a9\ud558\uc5ec \uace0\ucc28\uc6d0 \ub370\uc774\ud130\uc758 \ud328\ud134\uc744 \ud559\uc2b5\ud558\ub294 \ubc29\ubc95\uc785\ub2c8\ub2e4. \uc804\ud1b5\uc801\uc778 NLP \uae30\uc220\uc5d0\uc11c\uc758 \ud55c\uacc4\ub97c \uadf9\ubcf5\ud558\uae30 \uc704\ud574 \ud14d\uc2a4\ud2b8 \ub370\uc774\ud130\ub97c \ubca1\ud130\ub85c \ubcc0\ud658\ud558\uace0, \uc774\ub97c \uae30\ubc18\uc73c\ub85c \ub2e4\uc591\ud55c \ub525 \ub7ec\ub2dd \ubaa8\ub378\uc744 \uc0ac\uc6a9\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n
\uc790\uc5f0\uc5b4 \ucc98\ub9ac\uc5d0 \uc0ac\uc6a9\ub418\ub294 \ub525 \ub7ec\ub2dd \ubaa8\ub378\uc5d0\ub294 \uc5ec\ub7ec \uac00\uc9c0\uac00 \uc788\uc73c\uba70, \ub300\ud45c\uc801\uc73c\ub85c \ub2e4\uc74c\uacfc \uac19\uc740 \ubaa8\ub378\ub4e4\uc774 \uc788\uc2b5\ub2c8\ub2e4:<\/p>\n
\ubcc0\ud658\uae30 \ubaa8\ub378\uc740 \uc140\ud504 \uc5b4\ud150\uc158 \uba54\ucee4\ub2c8\uc998\uc744 \ud1b5\ud574 \uc785\ub825 \ub370\uc774\ud130\uc758 \uac01 \uc694\uc18c \uac04\uc758 \uc0c1\ub300\uc801 \uad00\uacc4\ub97c \ucd5c\ub300\ud55c \ud65c\uc6a9\ud558\uc5ec \ud6a8\uc728\uc801\uc73c\ub85c \ud14d\uc2a4\ud2b8\ub97c \ucc98\ub9ac\ud569\ub2c8\ub2e4. \uc774\ub294 \ub354 \ub098\uc740 \uc131\ub2a5\uc744 \ubc1c\ud718\ud558\ub3c4\ub85d \ud558\uba70, \uc5ec\ub7ec NLP \ud0dc\uc2a4\ud06c\uc5d0\uc11c \uc6b0\uc218\ud55c \uacb0\uacfc\ub97c \ubcf4\uc5ec\uc8fc\uace0 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n
\uc784\ubca0\ub529\uc740 \ud14d\uc2a4\ud2b8 \ub370\uc774\ud130\ub97c \uace0\ucc28\uc6d0 \ubca1\ud130\ub85c \ubcc0\ud658\ud558\uc5ec \uc758\ubbf8\uc801 \uc720\uc0ac\uc131\uc744 \ube44\uad50\ud560 \uc218 \uc788\ub294 \ubc29\ubc95\uc785\ub2c8\ub2e4. \uc774\ub7ec\ud55c \ubca1\ud130\ud654\uc758 \ubaa9\uc801\uc740 \uae30\uacc4 \ud559\uc2b5 \ubaa8\ub378\uc774 \ubd84\ub958, \uad70\uc9d1\ud654, \uac80\uc0c9 \ub4f1\uc758 \uc791\uc5c5\uc744 \uc218\ud589\ud560 \uc218 \uc788\ub3c4\ub85d \ub370\uc774\ud130\uc758 \ubc30\uce58\ub97c \ucd5c\uc801\ud654\ud558\ub294 \uac83\uc785\ub2c8\ub2e4.<\/p>\n
Word2Vec\uacfc GloVe\ub294 \uac00\uc7a5 \ub110\ub9ac \uc0ac\uc6a9\ub418\ub294 \uc784\ubca0\ub529 \uae30\ubc95 \uc911 \ub450 \uac00\uc9c0\uc785\ub2c8\ub2e4. Word2Vec\uc740 \ub2e8\uc5b4 \uac04\uc758 \uad00\uacc4\ub97c \ud559\uc2b5\ud558\uc5ec \uc720\uc0ac\ub2e8\uc5b4\ub97c \ucc3e\ub294 \ub370 \uac15\uc810\uc744 \uac00\uc9c0\uba70, GloVe\ub294 \ud1b5\uacc4\uc801 \uc815\ubcf4\ub97c \uae30\ubc18\uc73c\ub85c \ub2e8\uc5b4\ub97c \ubca1\ud130\ub85c \ubcc0\ud658\ud569\ub2c8\ub2e4.<\/p>\n
Word2Vec\uc740 ‘Skip-Gram’ \ubc0f ‘Continuous Bag of Words (CBOW)’\ub77c\ub294 \ub450 \uac00\uc9c0 \ubaa8\ub378\uc744 \uc0ac\uc6a9\ud558\uc5ec \ub2e8\uc5b4\ub97c \ubca1\ud130\ub85c \ubcc0\ud658\ud569\ub2c8\ub2e4. \uc774 \uacfc\uc815\uc740 \ub9ce\uc740 \uc591\uc758 \ud14d\uc2a4\ud2b8 \ub370\uc774\ud130\uc5d0\uc11c \ub2e8\uc5b4 \uac04\uc758 \uad00\uacc4\ub97c \ud559\uc2b5\ud558\ub294 \ub370 \uae30\uc5ec\ud569\ub2c8\ub2e4.<\/p>\n
from gensim.models import Word2Vec\nsentences = [['\ub098\ub294', '\ub525\ub7ec\ub2dd', '\uc790\ub791\uc2a4\ub7ec\uc6cc', '\ud558\ub294', '\uc0ac\ub78c', '\uc774\ub2e4'],\n ['\uc790\uc5f0\uc5b4', '\ucc98\ub9ac', '\ub294', '\ud765\ubbf8', '\ub85c\uc6b4', '\ubd84\uc57c', '\uc774\ub514', '\ub2e4']]\nmodel = Word2Vec(sentences, vector_size=100, window=5, min_count=1, workers=4)\n<\/code><\/pre>\n3.1.2 GloVe\uc758 \uc6d0\ub9ac<\/h4>\n
GloVe\ub294 \uc804\uc5ed\uc801\uc778 \ud1b5\uacc4 \uc815\ubcf4\ub97c \ud65c\uc6a9\ud558\uc5ec \ub2e8\uc5b4 \ubca1\ud130\ub97c \uc0dd\uc131\ud569\ub2c8\ub2e4. \uc774\ub294 \ubca1\ud130 \uacf5\uac04\uc5d0\uc11c \ub2e8\uc5b4 \uac04\uc758 \uac70\ub9ac\uc640 \uad00\uacc4\ub97c \uc758\ubbf8 \uc788\uac8c \ub9cc\ub4e4\uc5b4 \uc8fc\uba70, \ub354\uc6b1 \ub9e4\ub825\uc801\uc778 \uacb0\uacfc\ub97c \ub3c4\ucd9c\ud558\uac8c \ub429\ub2c8\ub2e4.<\/p>\n
import numpy as np\nimport glove\n\nglove_model = glove.Glove(no_components=100, learning_rate=0.05)\nglove_model.fit(sentences, epochs=30, no_threads=4, verbose=True)\n<\/code><\/pre>\n4. Faiss\ub97c \uc774\uc6a9\ud55c \uc784\ubca0\ub529 \uac80\uc0c9<\/h2>\n
Faiss(Facebook AI Similarity Search)\ub294 \ud6a8\uc728\uc801\uc778 \uc720\uc0ac\uc131 \uac80\uc0c9\uc744 \uc704\ud55c \ub77c\uc774\ube0c\ub7ec\ub9ac\ub85c, \ub300\uaddc\ubaa8 \uc784\ubca0\ub529 \ubca1\ud130\ub97c \ud65c\uc6a9\ud558\uc5ec \ube60\ub974\uace0 \uc815\ud655\ud55c \uac80\uc0c9\uc744 \uac00\ub2a5\ud558\uac8c \ud569\ub2c8\ub2e4. Faiss\ub294 \uace0\ucc28\uc6d0 \ubca1\ud130 \uac80\uc0c9\uc5d0\uc11c \uc131\ub2a5 \uc800\ud558\ub97c \ubc29\uc9c0\ud558\uae30 \uc704\ud574 \ub2e4\uc591\ud55c \uc0c9\uc778 \uad6c\uc870 \ubc0f \uac70\ub9ac \uce21\uc815 \ubc29\ubc95\uc744 \uc81c\uacf5\ud569\ub2c8\ub2e4.<\/p>\n
4.1 Faiss\uc758 \ud2b9\uc9d5<\/h3>\n\n- \ub300\uaddc\ubaa8 \ub370\uc774\ud130\uc14b\uc5d0\uc11c\uc758 \ube60\ub978 \uac80\uc0c9 \uc18d\ub3c4<\/li>\n
- \ubca1\ud130 \uacf5\uac04\uc5d0\uc11c\uc758 \uadfc\uc811 \ud30c\uc545 \ubc0f \uc720\uc0ac\ub3c4 \uac80\uc0c9 \uac00\ub2a5<\/li>\n
- \ub2e4\uc591\ud55c \uc0c9\uc778 \ubc29\ubc95 \uc81c\uacf5 (Flat, IVF, HNSW \ub4f1)<\/li>\n<\/ul>\n
4.2 Faiss \uc124\uce58 \ubc0f \uc0ac\uc6a9<\/h3>\n!pip install faiss-cpu\nimport faiss\nimport numpy as np\n\n# \ub370\uc774\ud130 \uc0dd\uc131\nd = 64 # \ucc28\uc6d0\nnb = 100000 # \ub370\uc774\ud130\ubca0\uc774\uc2a4 \ubca1\ud130 \uac1c\uc218\nnq = 10000 # \ucffc\ub9ac \ubca1\ud130 \uac1c\uc218\nnp.random.seed(1234) # \ub79c\ub364 \uc2dc\ub4dc \uace0\uc815\nxb = np.random.random((nb, d)).astype('float32') # \uc608\uc81c \ub370\uc774\ud130\ubca0\uc774\uc2a4\nxq = np.random.random((nq, d)).astype('float32') # \ucffc\ub9ac \ub370\uc774\ud130\n<\/code><\/pre>\n5. \uc784\ubca0\ub529 \uac80\uc0c9\uae30 \uad6c\ucd95\ud558\uae30<\/h2>\n
\uc774\uc81c Faiss\uc640 \ub525 \ub7ec\ub2dd \uae30\ubc18\uc758 \uc784\ubca0\ub529\uc744 \uacb0\ud569\ud558\uc5ec \uc138\uba58\ud2f1 \uc11c\uce58 \uae30\uacc4\ub97c \ub9cc\ub4dc\ub294 \ubc29\ubc95\uc744 \uc0b4\ud3b4\ubd05\uc2dc\ub2e4. \ub2e4\uc74c \ub2e8\uacc4\ub85c \uc678\ubd80 \ub370\uc774\ud130\uc14b\uc744 \ud65c\uc6a9\ud558\uc5ec \uc784\ubca0\ub529 \ubca1\ud130\ub97c \uc0dd\uc131\ud558\uace0, Faiss\ub97c \uc0ac\uc6a9\ud558\uc5ec \ud574\ub2f9 \ubca1\ud130\ub97c \uac80\uc0c9\ud558\ub294 \ubc29\ubc95\uc744 \uc54c\uc544\ubcf4\uaca0\uc2b5\ub2c8\ub2e4.<\/p>\n
5.1 \ub370\uc774\ud130 \uc218\uc9d1 \ubc0f \uc900\ube44<\/h3>\n
\uc790\uc5f0\uc5b4 \ucc98\ub9ac\ub97c \uc704\ud55c \ub370\uc774\ud130\uc14b\uc740 \uc778\ud130\ub137\uc774\ub098 \uacf5\uac1c\ub41c \ub370\uc774\ud130\ubca0\uc774\uc2a4\uc5d0\uc11c \uc218\uc9d1\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4. \uc608\ub97c \ub4e4\uc5b4 \ud55c\uad6d\uc5b4 \ub274\uc2a4 \uae30\uc0ac, SNS \uac8c\uc2dc\ubb3c, \ube14\ub85c\uadf8 \uae00 \ub4f1\uc758 \uc790\ub8cc\ub97c \ud1b5\ud574 \ub2e4\uc591\ud55c \ubb38\uc11c \uc0d8\ud50c\uc744 \ud655\ubcf4\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n
5.2 \ub370\uc774\ud130 \uc804\ucc98\ub9ac<\/h3>\n
\ubc1c\uc0dd\ud55c \ub370\uc774\ud130\ub294 \ud14d\uc2a4\ud2b8 \uc804\ucc98\ub9ac\ub97c \ud1b5\ud574 NLP \ubaa8\ub378\uc5d0 \uc801\ud569\ud558\ub3c4\ub85d \ub9cc\ub4e4\uc5b4\uc57c \ud569\ub2c8\ub2e4. \uc8fc\uc694 \uc804\ucc98\ub9ac \uc808\ucc28\ub294 \ub2e4\uc74c\uacfc \uac19\uc2b5\ub2c8\ub2e4:<\/p>\n
\n- \uc18c\ubb38\uc790 \ubcc0\ud658<\/li>\n
- \uad6c\ub450\uc810 \ubc0f \ud2b9\uc218 \ubb38\uc790 \uc81c\uac70<\/li>\n
- \ubd88\uc6a9\uc5b4 \uc81c\uac70 (Stopwords)<\/li>\n
- \uc5b4\uadfc \ucd94\ucd9c (Stemming) \ub610\ub294 \ud45c\uc81c\uc5b4 \ucd94\ucd9c (Lemmatization)<\/li>\n<\/ul>\n
5.3 \uc784\ubca0\ub529 \ubca1\ud130 \uc0dd\uc131<\/h3>\n
\uc804\ucc98\ub9ac\ub41c \ub370\uc774\ud130\ub97c \uc774\uc6a9\ud574 Word2Vec \ub610\ub294 GloVe \ubaa8\ub378\uc744 \uc0ac\uc6a9\ud558\uc5ec \uac01 \ubb38\uc11c\uc5d0 \ub300\ud55c \uc784\ubca0\ub529 \ubca1\ud130\ub97c \uc0dd\uc131\ud569\ub2c8\ub2e4. \uc774\ud6c4, \uc0dd\uc131\ub41c \ubca1\ud130\ub4e4\uc740 Faiss \uc0c9\uc778\uc5d0 \ucd94\uac00\ud560 \uc900\ube44\uac00 \ub429\ub2c8\ub2e4.<\/p>\n
# \uc784\ubca0\ub529 \uc0dd\uc131 \ud6c4\nembedding_vector = model.wv['\uc790\uc5f0\uc5b4']\n<\/code><\/pre>\n5.4 Faiss \uc0c9\uc778 \ucd94\uac00 \ubc0f \uac80\uc0c9 \ucc98\ub9ac<\/h3>\n
\uc774\uc81c \uc0dd\uc131\ub41c \uc784\ubca0\ub529 \ubca1\ud130\ub97c Faiss \uc0c9\uc778\uc5d0 \ucd94\uac00\ud558\uace0 \ube60\ub978 \uac80\uc0c9\uc744 \uc2e4\ud589\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n
# Faiss \uc0c9\uc778 \uc0dd\uc131 \ubc0f \ubca1\ud130 \ucd94\uac00\nindex = faiss.IndexFlatL2(d) # L2 \uac70\ub9ac \uc0ac\uc6a9\nindex.add(xb) # \ub370\uc774\ud130\ubca0\uc774\uc2a4 \ubca1\ud130 \ucd94\uac00\n\nk = 5 # \uac00\uae4c\uc6b4 \uc774\uc6c3 \uac80\uc0c9\nD, I = index.search(xq, k) # \uac80\uc0c9 \uc218\ud589\n<\/code><\/pre>\n5.5 \uc720\uc0ac\ub3c4 \uacb0\uacfc \ud574\uc11d<\/h3>\n
Faiss\uc5d0\uc11c\uc758 \uac80\uc0c9 \uacb0\uacfc\ub85c \uc5bb\uc5b4\uc9c4 \uc778\ub371\uc2a4\uc640 \uac70\ub9ac\ub4e4\uc740 \uc0ac\uc6a9\uc790\uac00 \uc694\uccad\ud55c \ucffc\ub9ac\uc640 \uac00\uc7a5 \uc720\uc0ac\ud55c \ubb38\uc11c\ub4e4\uc744 \uc2dd\ubcc4\ud558\uac8c \ud574\uc90d\ub2c8\ub2e4. \uc774\ub97c \ud1b5\ud574 \uc0ac\uc6a9\uc790\ub294 \uc790\uc2e0\uc758 \ud544\uc694\uc5d0 \ub9de\ub294 \uc815\ubcf4\ub97c \uc27d\uac8c \ucc3e\uc744 \uc218 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n
6. \ub9c8\ubb34\ub9ac \ubc0f \uc751\uc6a9<\/h2>\n
\ub525 \ub7ec\ub2dd\uc744 \uc774\uc6a9\ud55c \uc790\uc5f0\uc5b4 \ucc98\ub9ac\uc640 Faiss\ub97c \ud65c\uc6a9\ud55c \uc784\ubca0\ub529 \uac80\uc0c9\uae30 \uad6c\ucd95\uc740 \uc790\uc5f0\uc5b4 \ub370\uc774\ud130\uc758 \uc758\ubbf8\uc801 \uc720\uc0ac\uc131\uc744 \uae30\ubc18\uc73c\ub85c \uc815\ubcf4\ub97c \ud0d0\uc0c9\ud560 \uc218 \uc788\ub294 \ub9e4\uc6b0 \ud6a8\uacfc\uc801\uc778 \ubc29\ubc95\uc785\ub2c8\ub2e4. \uc774\ub7ec\ud55c \uae30\uc220\ub4e4\uc740 \uc5ec\ub7ec \uac00\uc9c0 \ubd84\uc57c\uc5d0\uc11c \ud65c\uc6a9\ub418\uba70 \uc815\ubcf4 \uac80\uc0c9, \ucd94\ucc9c \uc2dc\uc2a4\ud15c, \uac10\uc815 \ubd84\uc11d \ub4f1 \ub110\ub9ac \ud65c\uc6a9\ub418\uace0 \uc788\uc2b5\ub2c8\ub2e4. \uc55e\uc73c\ub85c\ub3c4 \uc774 \uae30\uc220\ub4e4\uc740 \uc9c0\uc18d\uc801\uc73c\ub85c \ubc1c\uc804\ud558\uac8c \ub420 \uac83\uc774\uba70, \uc6b0\ub9ac \uc0ac\ud68c\uc758 \ub2e4\uc591\ud55c \ubb38\uc81c\ub97c \ud574\uacb0\ud558\ub294 \ub370 \uae30\uc5ec\ud560 \uac83\uc785\ub2c8\ub2e4.<\/p>\n
\ub525 \ub7ec\ub2dd\uacfc Faiss\ub97c \uc774\uc6a9\ud55c \uc138\uba58\ud2f1 \uc11c\uce58\uc758 \uad6c\ud604\uc744 \ud1b5\ud574 \ub370\uc774\ud130\uc758 \uac00\uce58\uc640 \ud65c\uc6a9 \uac00\ub2a5\uc131\uc744 \ud655\uc778\ud558\uace0, \ud5a5\ud6c4 \ub354 \ub9ce\uc740 \ub3c4\uc804\uacfc\uc81c\ub97c \ud574\uacb0\ud574 \ub098\uac00\uc2dc\uae38 \ubc14\ub78d\ub2c8\ub2e4.<\/p>\n
<\/body><\/p>\n","protected":false},"excerpt":{"rendered":"