\uc790\uc5f0\uc5b4 \ucc98\ub9ac\ub294 \uc624\ub298\ub0a0\uc758 \uc778\uacf5\uc9c0\ub2a5 \ubd84\uc57c\uc5d0\uc11c \uac00\uc7a5 \ube60\ub974\uac8c \ubc1c\uc804\ud558\uace0 \uc788\ub294 \ubd84\uc57c \uc911 \ud558\ub098\uc785\ub2c8\ub2e4. \ud2b9\ud788 \ub525 \ub7ec\ub2dd \uae30\uc220\uc758 \ubc1c\uc804\uc740 \uc790\uc5f0\uc5b4 \ucc98\ub9ac(NLP) \ubb38\uc81c\ub97c \ud574\uacb0\ud558\ub294 \ub370 \ud68d\uae30\uc801\uc778 \ubcc0\ud654\ub97c \uac00\uc838\uc654\uc2b5\ub2c8\ub2e4. \ubcf8 \uae00\uc5d0\uc11c\ub294 NLP\uc5d0\uc11c \ubc1c\uc0dd\ud560 \uc218 \uc788\ub294 \ub370\uc774\ud130 \ucc98\ub9ac \uacfc\uc815, \ud2b9\ud788 \ub370\uc774\ud130\uc758 \ubd84\ub9ac(Splitting Data)\uc5d0 \ub300\ud574 \uc0c1\uc138\ud788 \uc124\uba85\ud558\uaca0\uc2b5\ub2c8\ub2e4. \ub370\uc774\ud130 \ubd84\ub9ac\ub294 \ubaa8\ub378\uc758 \uc131\ub2a5\uc5d0 \uc911\ub300\ud55c \uc601\ud5a5\uc744 \ubbf8\uce58\ub294 \uc694\uc18c\ub85c, \uc62c\ubc14\ub978 \ubc29\ubc95\uc73c\ub85c \uc9c4\ud589\ub418\uc5b4\uc57c \ud569\ub2c8\ub2e4.<\/p>\n
\ub370\uc774\ud130 \ubd84\ub9ac\ub294 \ub370\uc774\ud130 \uacfc\ud559 \ubc0f \uae30\uacc4 \ud559\uc2b5\uc5d0\uc11c \uae30\ubcf8\uc801\uc778 \uc791\uc5c5 \uc911 \ud558\ub098\uc785\ub2c8\ub2e4. \ub370\uc774\ud130\uc758 \ud488\uc9c8\uc774 \ubaa8\ub378\uc758 \uc131\ud328\ub97c \uc88c\uc6b0\ud558\ubbc0\ub85c, \ub370\uc774\ud130\ub97c \ud6c8\ub828, \uac80\uc99d, \ud14c\uc2a4\ud2b8 \uc138\ud2b8\ub85c \ubd84\ub9ac\ud558\ub294 \uacfc\uc815\uc740 \ub9e4\uc6b0 \uc911\uc694\ud569\ub2c8\ub2e4. \ub370\uc774\ud130\uac00 \uc798 \ubd84\ub9ac\ub418\uc9c0 \uc54a\uc73c\uba74 \ubaa8\ub378\uc774 \uacfc\uc801\ud569(overfitting)\ub418\uac70\ub098 \uc77c\ubc18\ud654(generalization)\uac00 \uc2e4\ud328\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n
\uc77c\ubc18\uc801\uc73c\ub85c \uc790\uc5f0\uc5b4 \ucc98\ub9ac \ubaa8\ub378\uc744 \ud6c8\ub828\uc2dc\ud0a4\uae30 \uc704\ud574\uc11c\ub294 \ub2e4\uc74c\uacfc \uac19\uc740 \uc138 \uac00\uc9c0 \ub370\uc774\ud130 \uc138\ud2b8\ub97c \uc0ac\uc6a9\ud569\ub2c8\ub2e4:<\/p>\n
\ub370\uc774\ud130\ub97c \ubd84\ub9ac\ud558\ub294 \ubc29\ubc95\uc5d0\ub294 \uc5ec\ub7ec \uac00\uc9c0\uac00 \uc788\uc2b5\ub2c8\ub2e4. \uac00\uc7a5 \uc77c\ubc18\uc801\uc778 \ubc29\ubc95\uc73c\ub85c\ub294 \ub79c\ub364 \uc0d8\ud50c\ub9c1, \uacc4\uce35\uc801 \uc0d8\ud50c\ub9c1 \ub4f1\uc774 \uc788\uc2b5\ub2c8\ub2e4. \uc544\ub798\uc5d0\uc11c \uac01 \ubc29\ubc95\uc744 \uc0b4\ud3b4\ubcf4\uaca0\uc2b5\ub2c8\ub2e4.<\/p>\n
\ub79c\ub364 \uc0d8\ud50c\ub9c1\uc740 \uac00\uc7a5 \uac04\ub2e8\ud55c \ub370\uc774\ud130 \ubd84\ub9ac \ubc29\ubc95\uc785\ub2c8\ub2e4. \uc804\uccb4 \ub370\uc774\ud130\uc14b\uc5d0\uc11c \ubb34\uc791\uc704\ub85c \uc0d8\ud50c\uc744 \uc120\ud0dd\ud558\uc5ec \ud6c8\ub828 \uc138\ud2b8\uc640 \uac80\uc99d\/\ud14c\uc2a4\ud2b8 \uc138\ud2b8\ub85c \ub098\ub215\ub2c8\ub2e4. \uc774 \ubc29\ubc95\uc758 \uc7a5\uc810\uc740 \uad6c\ud604\uc774 \uac04\ub2e8\ud558\uace0 \ube60\ub974\ub2e4\ub294 \uc810\uc785\ub2c8\ub2e4. \ud558\uc9c0\ub9cc \ub370\uc774\ud130\uc758 \ubd84\ud3ec\uac00 \ubd88\uade0\ud615\ud55c \uacbd\uc6b0 \ubb38\uc81c\ub97c \uc77c\uc73c\ud0ac \uc218 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n
\n from sklearn.model_selection import train_test_split
\n train_data, temp_data = train_test_split(data, test_size=0.2, random_state=42)
\n val_data, test_data = train_test_split(temp_data, test_size=0.5, random_state=42)
\n <\/code>\n<\/div>\n3.2 \uacc4\uce35\uc801 \uc0d8\ud50c\ub9c1<\/h3>\n
\uacc4\uce35\uc801 \uc0d8\ud50c\ub9c1\uc740 \ub370\uc774\ud130\uc758 \ubd84\ud3ec\ub97c \uc720\uc9c0\ud558\uba74\uc11c \uc0d8\ud50c\uc744 \ucd94\ucd9c\ud558\ub294 \ubc29\ubc95\uc785\ub2c8\ub2e4. \ud074\ub77c\uc2a4(class)\uac00 \ubd88\uade0\ud615\ud558\uac8c \ubd84\ud3ec\ub418\uc5b4 \uc788\ub294 \ub370\uc774\ud130\uc14b\uc5d0\uc11c \ud2b9\ud788 \uc720\uc6a9\ud569\ub2c8\ub2e4. \uc774 \ubc29\ubc95\uc744 \uc0ac\uc6a9\ud558\uba74 \uac01 \ud074\ub798\uc2a4\uc758 \ube44\uc728\uc774 \ud6c8\ub828 \uc138\ud2b8\uc640 \uac80\uc99d\/\ud14c\uc2a4\ud2b8 \uc138\ud2b8\uc5d0 \ube44\uc2b7\ud558\uac8c \uc720\uc9c0\ub429\ub2c8\ub2e4.<\/p>\n
\n
\n from sklearn.model_selection import StratifiedShuffleSplit
\n sss = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)
\n for train_index, test_index in sss.split(data, labels):
\n train_data = data.loc[train_index]
\n test_data = data.loc[test_index]
\n <\/code>\n<\/div>\n4. \ub370\uc774\ud130 \uc804\ucc98\ub9ac\uc640 \ubd84\ub9ac<\/h2>\n
\uc790\uc5f0\uc5b4 \ucc98\ub9ac\uc5d0\uc11c\ub294 \ub370\uc774\ud130 \uc804\ucc98\ub9ac\uac00 \ud544\uc218\uc801\uc785\ub2c8\ub2e4. \uc804\ucc98\ub9ac \ub2e8\uacc4\uc5d0\uc11c \ud14d\uc2a4\ud2b8 \ub370\uc774\ud130\ub97c \uc815\ub9ac\ud558\uace0, \ubd88\uc6a9\uc5b4 \uc81c\uac70, \ud1a0\ud070\ud654 \ub4f1\uc744 \uc218\ud589\ud55c \ud6c4, \uc774 \ub370\uc774\ud130\ub97c \ud6c8\ub828, \uac80\uc99d, \ud14c\uc2a4\ud2b8 \uc138\ud2b8\ub85c \ubd84\ub9ac\ud569\ub2c8\ub2e4. \ub370\uc774\ud130 \uc804\ucc98\ub9ac \ud6c4\uc5d0 \ub370\uc774\ud130 \ubd84\ub9ac\ub97c \uc9c4\ud589\ud558\ub294 \uac83\uc774 \uc77c\ubc18\uc801\uc785\ub2c8\ub2e4.<\/p>\n
4.1 \uc804\ucc98\ub9ac \ub2e8\uacc4 \uc608\uc2dc<\/h3>\n\n
\n import pandas as pd
\n from sklearn.model_selection import train_test_split<\/p>\n # \ub370\uc774\ud130 \ub85c\ub4dc
\n data = pd.read_csv('data.csv')<\/p>\n
# \uc804\ucc98\ub9ac
\n data['text'] = data['text'].apply(lambda x: x.lower()) # \uc18c\ubb38\uc790 \ubcc0\ud658
\n data['text'] = data['text'].str.replace('[^a-zA-Z]', '') # \ud2b9\uc218 \ubb38\uc790 \uc81c\uac70<\/p>\n
# \ub370\uc774\ud130 \ubd84\ub9ac
\n train_data, temp_data = train_test_split(data, test_size=0.2, random_state=42)
\n val_data, test_data = train_test_split(temp_data, test_size=0.5, random_state=42)
\n <\/code>\n<\/div>\n
5. \ucd5c\uc801\uc758 \ub370\uc774\ud130 \ubd84\ub9ac \ube44\uc728<\/h2>\n
\ub370\uc774\ud130 \ubd84\ub9ac \ube44\uc728\uc740 \ubb38\uc81c\uc758 \ud2b9\uc131\uacfc \ub370\uc774\ud130\uc758 \uc591\uc5d0 \ub530\ub77c \ub2ec\ub77c\uc9c8 \uc218 \uc788\uc2b5\ub2c8\ub2e4. \uc77c\ubc18\uc801\uc73c\ub85c\ub294 \ud6c8\ub828 \uc138\ud2b8:\uac80\uc99d \uc138\ud2b8:\ud14c\uc2a4\ud2b8 \uc138\ud2b8\ub97c 70:15:15 \ub610\ub294 80:10:10 \ube44\uc728\ub85c \ub098\ub204\ub294 \uac83\uc774 \uc77c\ubc18\uc801\uc785\ub2c8\ub2e4. \uadf8\ub7ec\ub098 \ub370\uc774\ud130\uc758 \uc591\uc774 \uc801\uac70\ub098 \ubd88\uade0\ud615\ud55c \uacbd\uc6b0, \uc774\ub7ec\ud55c \ube44\uc728\uc744 \uc870\uc815\ud574\uc57c \ud560 \uc218\ub3c4 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n
\ub370\uc774\ud130 \uc138\ud2b8\ub97c \ub098\ub204\ub294 \uacfc\uc815\uc5d0\uc11c \ud558\uc774\ud37c\ud30c\ub77c\ubbf8\ud130 \ud29c\ub2dd \ub4f1\uc744 \uace0\ub824\ud558\uc5ec \uac80\uc99d \uc138\ud2b8\uc758 \ud06c\uae30\ub97c \uc870\uc815\ud558\ub294 \uac83\uc774 \uc88b\uc2b5\ub2c8\ub2e4. \ubaa8\ub378\uc774 \ucd5c\uc120\uc758 \uc131\ub2a5\uc744 \ubc1c\ud718\ud560 \uc218 \uc788\ub3c4\ub85d \ud558\uae30 \uc704\ud574\uc11c\ub294 \uc801\uc808\ud55c \ub370\uc774\ud130 \ubd84\ub9ac\uac00 \ud544\uc218\uc801\uc785\ub2c8\ub2e4.<\/p>\n
6. \uacb0\ub860<\/h2>\n
\ub525 \ub7ec\ub2dd \uae30\ubc18\uc758 \uc790\uc5f0\uc5b4 \ucc98\ub9ac \ubaa8\ub378\uc744 \ud6c8\ub828\uc2dc\ud0a4\uae30 \uc704\ud574\uc11c\ub294 \ub370\uc774\ud130 \ubd84\ub9ac\uac00 \ud544\uc218\uc801\uc785\ub2c8\ub2e4. \ud2b9\ud788, \ub370\uc774\ud130\uc758 \ubd84\ub9ac\ub294 \ubaa8\ub378\uc758 \uc131\ub2a5 \ud5a5\uc0c1\uc5d0 \uc9c1\uc811\uc801\uc778 \uc601\ud5a5\uc744 \ubbf8\uce58\ub294 \uc694\uc18c\uc785\ub2c8\ub2e4. \ub530\ub77c\uc11c, \ub2e4\uc591\ud55c \ubc29\ubc95\ub860\uc744 \ud1b5\ud574 \uc801\uc808\ud55c \ub370\uc774\ud130 \ubd84\ub9ac \ubc29\ubc95\uc744 \uc120\ud0dd\ud558\uace0, \uac01 \uc138\ud2b8\uc758 \ud2b9\uc131\uc744 \uc774\ud574\ud558\ub294 \uac83\uc774 \uc911\uc694\ud569\ub2c8\ub2e4. \uadf8 \uacb0\uacfc\ub85c \ubcf4\ub2e4 \uc2e0\ub8b0\uc131 \uc788\ub294 \uc77c\ubc18\ud654 \ubaa8\ub378\uc744 \uad6c\ucd95\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n
\ucd94\uac00 \uc815\ubcf4<\/h2>\n
\uc790\uc5f0\uc5b4 \ucc98\ub9ac\uc5d0\uc11c \ub370\uc774\ud130 \ubd84\ub9ac\uc5d0 \ub300\ud574 \ub354 \uae4a\uc774 \uc54c\uace0 \uc2f6\uc73c\uc2dc\ub2e4\uba74, \ub2e4\uc74c\uc758 \uc790\ub8cc\ub97c \ucc38\uace0\ud558\uc2dc\uae30 \ubc14\ub78d\ub2c8\ub2e4:<\/p>\n