{"id":29371,"date":"2024-10-27T16:18:06","date_gmt":"2024-10-27T16:18:06","guid":{"rendered":"http:\/\/atmokpo.com\/w\/?p=29371"},"modified":"2024-11-26T06:52:53","modified_gmt":"2024-11-26T06:52:53","slug":"%eb%a8%b8%ec%8b%a0%eb%9f%ac%eb%8b%9d-%eb%b0%8f-%eb%94%a5%eb%9f%ac%eb%8b%9d-%ec%95%8c%ea%b3%a0%eb%a6%ac%ec%a6%98-%ed%8a%b8%eb%a0%88%ec%9d%b4%eb%94%a9-%ed%85%90%ec%84%9c%ed%94%8c%eb%a1%9c2%eb%a5%bc","status":"publish","type":"post","link":"https:\/\/atmokpo.com\/w\/29371\/","title":{"rendered":"\uba38\uc2e0\ub7ec\ub2dd \ubc0f \ub525\ub7ec\ub2dd \uc54c\uace0\ub9ac\uc998 \ud2b8\ub808\uc774\ub529, \ud150\uc11c\ud50c\ub85c2\ub97c \uc774\uc6a9\ud55c DDQN\uc758 \uad6c\ud604"},"content":{"rendered":"

<\/p>\n

\n<\/header>\n
\n

1. \uc11c\ub860<\/h2>\n

\n \uae08\uc735 \uc2dc\uc7a5\uc758 \ubcf5\uc7a1\uc131\uacfc \ubcc0\ub3d9\uc131\uc73c\ub85c \uc778\ud574 \ud2b8\ub808\uc774\ub529 \uc804\ub7b5\uc740 \ub0a0\ub85c \ubc1c\uc804\ud558\uace0 \uc788\uc2b5\ub2c8\ub2e4. \ud2b9\ud788 \uba38\uc2e0\ub7ec\ub2dd\uacfc \ub525\ub7ec\ub2dd \uae30\uc220\uc774 \ud2b8\ub808\uc774\ub529 \uc804\ub7b5\uc5d0 \uc801\uc6a9\ub418\uba74\uc11c, \ud22c\uc790\uc790\ub4e4\uc740 \uadf8 \uc5b4\ub290 \ub54c\ubcf4\ub2e4\ub3c4 \ub9ce\uc740 \ub370\uc774\ud130\uc640 \uc815\ubcf4\ub97c \ud65c\uc6a9\ud558\uc5ec \ucd5c\uc801\uc758 \uc758\uc0ac\uacb0\uc815\uc744 \ub0b4\ub9b4 \uc218 \uc788\uac8c \ub418\uc5c8\uc2b5\ub2c8\ub2e4. \ubcf8 \uac15\uc88c\uc5d0\uc11c\ub294 \uac15\ud654 \ud559\uc2b5 \uae30\ubc95\uc778 DDQN(\ub354\ube14 \ub525 Q-\ub124\ud2b8\uc6cc\ud06c)\uc744 \uc774\uc6a9\ud55c \uc54c\uace0\ub9ac\uc998 \ud2b8\ub808\uc774\ub529 \uc2dc\uc2a4\ud15c\uc744 \uad6c\ud604\ud558\ub294 \ubc29\ubc95\uc5d0 \ub300\ud574 \uc0b4\ud3b4\ubcf4\uaca0\uc2b5\ub2c8\ub2e4. \uc774 \uacfc\uc815\uc5d0\uc11c\ub294 \ud150\uc11c\ud50c\ub85c2 \ub77c\uc774\ube0c\ub7ec\ub9ac\ub97c \uc0ac\uc6a9\ud558\uc5ec DDQN\uc744 \uad6c\ud604\ud558\uace0, \uc2e4\uc81c \uc8fc\uc2dd \uac70\ub798 \ub370\uc774\ud130\uc5d0 \uc801\uc6a9\ud558\ub294 \ubc29\ubc95\uc744 \uc18c\uac1c\ud560 \uac83\uc785\ub2c8\ub2e4.\n <\/p>\n<\/section>\n

\n

2. DDQN (Double Deep Q-Network) \uac1c\uc694<\/h2>\n

\n DDQN\uc740 Q-\ub7ec\ub2dd(\uac15\ud654 \ud559\uc2b5\uc758 \uc77c\uc885)\uc758 \ud55c \ubcc0\ud615\uc73c\ub85c, \uae30\uc874\uc758 DQN(Deep Q-Network)\uc758 \ud55c\uacc4\uc810\uc744 \ubcf4\uc644\ud558\uae30 \uc704\ud574 \uace0\uc548\ub418\uc5c8\uc2b5\ub2c8\ub2e4. DQN\uc740 \ucd5c\ub300 \ubcf4\uc0c1\uc744 \ucc3e\uae30 \uc704\ud574 \ud55c \uac00\uc9c0 Q\uac12\ub97c \uc774\uc6a9\ud558\ub294\ub370, \uc774\ub85c \uc778\ud574 \uacfc\uc801\ud569(overestimation) \ubb38\uc81c\ub97c \uacaa\uac8c \ub429\ub2c8\ub2e4. DDQN\uc740 Q\uac12\uc744 \ub450 \uac1c\uc758 \uc2e0\uacbd\ub9dd\uc73c\ub85c \uad6c\uc131\ud558\uc5ec \uc774 \ubb38\uc81c\ub97c \ud574\uacb0\ud569\ub2c8\ub2e4.\n <\/p>\n

\n DDQN\uc758 \uad6c\uc870\ub294 \uae30\uc874 DQN\ub4e4\uacfc \ube44\uc2b7\ud558\uc9c0\ub9cc, \ub450 \uac00\uc9c0 \ub124\ud2b8\uc6cc\ud06c\u2014\uc8fc \ub124\ud2b8\uc6cc\ud06c\uc640 \ub300\uc0c1 \ub124\ud2b8\uc6cc\ud06c\u2014\ub97c \ud1b5\ud574 \uc561\uc158\uc758 \ucd5c\uc801 \uac12\uc744 \ub354\uc6b1 \uc815\ud655\ud558\uac8c \ud3c9\uac00\ud569\ub2c8\ub2e4. \uc774\ub807\uac8c \ud568\uc73c\ub85c\uc368 \ubcf4\ub2e4 \uc548\uc815\uc801\uc778 \ud559\uc2b5 \uacfc\uc815\uc744 \uc720\uc9c0\ud558\uace0, \ub354 \ub098\uc740 \uacb0\uacfc\ub97c \uc81c\uacf5\ud569\ub2c8\ub2e4. \uc774\ub7ec\ud55c DDQN\uc758 \uc7a5\uc810\uc73c\ub85c \uc778\ud574 \uae08\uc735 \uc2dc\uc7a5\uc5d0\uc11c \ud6a8\uacfc\uc801\uc73c\ub85c \ud65c\uc6a9\ub420 \uc218 \uc788\uc2b5\ub2c8\ub2e4.\n <\/p>\n<\/section>\n

\n

3. \ud658\uacbd \uc124\uc815<\/h2>\n

3.1. \ud544\uc694\ud55c \ub77c\uc774\ube0c\ub7ec\ub9ac \uc124\uce58\ud558\uae30<\/h3>\n

\n \uc6b0\ub9ac\uc758 \uba38\uc2e0\ub7ec\ub2dd \ubaa8\ub378\uc744 \uad6c\ucd95\ud558\uae30 \uc704\ud574 \uba87 \uac00\uc9c0 \ub77c\uc774\ube0c\ub7ec\ub9ac\ub97c \uc124\uce58\ud574\uc57c \ud569\ub2c8\ub2e4. \uc8fc\ub85c \uc0ac\uc6a9\ub420 \ub77c\uc774\ube0c\ub7ec\ub9ac\ub294 \uc544\ub798\uc640 \uac19\uc2b5\ub2c8\ub2e4:\n <\/p>\n

pip install numpy pandas matplotlib tensorflow gym<\/code><\/pre>\n
3.2. \uac70\ub798 \ub370\uc774\ud130 \uc218\uc9d1\ud558\uae30<\/h3>\n
\n            DDQN \ubaa8\ub378\uc744 \ud559\uc2b5\uc2dc\ud0a4\uae30 \uc704\ud574\uc11c\ub294 \uc801\uc808\ud55c \uc8fc\uc2dd \uac70\ub798 \ub370\uc774\ud130\uac00 \ud544\uc694\ud569\ub2c8\ub2e4. Yahoo Finance, Alpha Vantage, Quandl \ub4f1 \ub2e4\uc591\ud55c \ub370\uc774\ud130 \uc18c\uc2a4\ub97c \ud65c\uc6a9\ud574 \ub370\uc774\ud130\ub97c \uc218\uc9d1\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4. \uc608\ub97c \ub4e4\uc5b4, \uce5c\uc219\ud55c yfinance<\/code> \ub77c\uc774\ube0c\ub7ec\ub9ac\ub97c \uc0ac\uc6a9\ud558\uc5ec \ub370\uc774\ud130\ub97c \uc218\uc9d1\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4.\n        <\/p>\n
import yfinance as yf\ndata = yf.download(\"AAPL\", start=\"2010-01-01\", end=\"2020-01-01\")<\/code><\/pre>\n<\/section>\n
\n
4. DDQN \ubaa8\ub378 \uad6c\ud604\ud558\uae30<\/h2>\n
4.1. \ud658\uacbd \uc124\uc815<\/h3>\n
\n            DDQN\uc744 \uad6c\ud604\ud558\uae30 \uc704\ud55c \ud658\uacbd\uc744 \uc124\uc815\ud574\ubd05\uc2dc\ub2e4. OpenAI\uc758 Gym \ub77c\uc774\ube0c\ub7ec\ub9ac\ub97c \ud1b5\ud574 \ud658\uacbd\uc744 \uad6c\ud604\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4. \uae30\ubcf8\uc801\uc778 \uad6c\uc870\ub294 \uc544\ub798\uc640 \uac19\uc2b5\ub2c8\ub2e4:\n        <\/p>\n
import gym\n\nclass StockTradingEnv(gym.Env):\n    def __init__(self, data):\n        super(StockTradingEnv, self).__init__()\n        self.data = data\n        self.current_step = 0\n        self.action_space = gym.spaces.Discrete(3) # Hold, Buy, Sell\n        self.observation_space = gym.spaces.Box(low=0, high=1, shape=(1, len(data.columns)), dtype=np.float32)\n\n    def reset(self):\n        self.current_step = 0\n        return self.data.iloc[self.current_step].values\n\n    def step(self, action):\n        ...\n<\/code><\/pre>\n
4.2. DQN \ub124\ud2b8\uc6cc\ud06c \uad6c\uc131\ud558\uae30<\/h3>\n
\n            DQN \ub124\ud2b8\uc6cc\ud06c\ub294 \uc785\ub825\uce35, \uc740\ub2c9\uce35, \ucd9c\ub825\uce35\uc73c\ub85c \uad6c\uc131\ub429\ub2c8\ub2e4. \uc544\ub798 \ucf54\ub4dc\ub294 \uae30\ubcf8\uc801\uc778 DQN \ub124\ud2b8\uc6cc\ud06c\uc758 \uad6c\uc870\ub97c \ubcf4\uc5ec\uc90d\ub2c8\ub2e4:\n        <\/p>\n
import tensorflow as tf\n\ndef create_model(state_size, action_size):\n    model = tf.keras.Sequential()\n    model.add(tf.keras.layers.Dense(24, input_dim=state_size, activation='relu'))\n    model.add(tf.keras.layers.Dense(24, activation='relu'))\n    model.add(tf.keras.layers.Dense(action_size, activation='linear'))\n    model.compile(loss='mse', optimizer=tf.keras.optimizers.Adam(learning_rate=0.001))\n    return model<\/code><\/pre>\n
4.3. DDQN \ud559\uc2b5 \ub8e8\ud504 \uad6c\ucd95\ud558\uae30<\/h3>\n
\n            DDQN\uc744 \ud559\uc2b5\ud558\uae30 \uc704\ud55c \ub8e8\ud504\ub97c \uad6c\uc131\ud569\ub2c8\ub2e4. \uacbd\ud5d8 \ub9ac\ud50c\ub808\uc774(Experience Replay)\uc640 \ud0c0\uac9f \ub124\ud2b8\uc6cc\ud06c \uc5c5\ub370\uc774\ud2b8\uc640 \uac19\uc740 DDQN\uc758 \uc911\uc694\ud55c \uac1c\ub150\uc744 \ud3ec\ud568\ud569\ub2c8\ub2e4.\n        <\/p>\n
import random\nfrom collections import deque\n\nclass Agent:\n    def __init__(self, state_size, action_size):\n        self.state_size = state_size\n        self.action_size = action_size\n        self.memory = deque(maxlen=2000)\n        self.gamma = 0.95  # discount rate\n        self.epsilon = 1.0  # exploration rate\n        self.epsilon_min = 0.01\n        self.epsilon_decay = 0.995\n        self.model = create_model(state_size, action_size)\n        self.target_model = create_model(state_size, action_size)\n\n    def act(self, state):\n        ...\n    \n    def replay(self, batch_size):\n        ...\n        \n    def update_target_model(self):\n        self.target_model.set_weights(self.model.get_weights())\n<\/code><\/pre>\n<\/section>\n
\n
5. \ubaa8\ub378 \ud3c9\uac00 \ubc0f \ucd5c\uc801\ud654<\/h2>\n
5.1. \uc131\uacfc \ud3c9\uac00<\/h3>\n
\n            DDQN \ubaa8\ub378\uc758 \uc131\uacfc\ub97c \ud3c9\uac00\ud558\uae30 \uc704\ud574 \uc218\uc775\ub960, \uc0e4\ud504 \ube44\uc728 \ub4f1\uc758 \uae08\uc735 \uc9c0\ud45c\ub97c \uc0ac\uc6a9\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4. \uc2e4\uc81c\ub85c \ubaa8\ub378\uc744 \uc0dd\uc131\ud55c \ud6c4, \uc544\ub798\uc640 \uac19\uc740 \uc9c0\ud45c\ub4e4\uc744 \ud1b5\ud574 \ud22c\uc790 \uc131\uacfc\ub97c \ubd84\uc11d\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4.\n        <\/p>\n
def evaluate_model(model, test_data):\n    ...\n<\/code><\/pre>\n
5.2. \ud558\uc774\ud37c\ud30c\ub77c\ubbf8\ud130 \ud29c\ub2dd<\/h3>\n
\n            \ubaa8\ub378\uc758 \uc131\ub2a5\uc744 \uadf9\ub300\ud654\ud558\uae30 \uc704\ud574\uc11c\ub294 \ud558\uc774\ud37c\ud30c\ub77c\ubbf8\ud130 \ud29c\ub2dd\uc774 \ud544\uc218\uc801\uc785\ub2c8\ub2e4. \ub79c\ub364 \uc11c\uce58, \uadf8\ub9ac\ub4dc \uc11c\uce58\uc640 \uac19\uc740 \uae30\ubc95\uc744 \ud1b5\ud574 \ucd5c\uc801\uc758 \ud558\uc774\ud37c\ud30c\ub77c\ubbf8\ud130\ub97c \ucc3e\uc544\ubcf4\uc138\uc694.\n        <\/p>\n
from sklearn.model_selection import ParameterGrid\n\nparams = {'batch_size': [32, 64], 'epsilon_decay': [0.995, 0.99]}\ngrid_search = ParameterGrid(params)\nfor param in grid_search:\n    ...\n<\/code><\/pre>\n<\/section>\n
\n
6. \uacb0\ub860<\/h2>\n
\n            \ubcf8 \uac15\uc88c\uc5d0\uc11c\ub294 \uba38\uc2e0\ub7ec\ub2dd \ubc0f \ub525\ub7ec\ub2dd \uae30\ubc18\uc758 \uc54c\uace0\ub9ac\uc998 \ud2b8\ub808\uc774\ub529 \uc2dc\uc2a4\ud15c\uc744 \uad6c\ud604\ud558\uae30 \uc704\ud574 DDQN\uc744 \ud65c\uc6a9\ud558\ub294 \ubc29\ubc95\uc744 \uc124\uba85\ud588\uc2b5\ub2c8\ub2e4. DDQN\uc740 \uc8fc\uc2dd \uac70\ub798\uc640 \uac19\uc740 \ubcf5\uc7a1\ud55c \ud658\uacbd\uc5d0\uc11c \ud6a8\uacfc\uc801\uc778 \uc804\ub7b5\uc744 \ucc3e\uae30 \uc704\ud574 \uc720\uc6a9\ud558\uac8c \uc0ac\uc6a9\ub420 \uc218 \uc788\uc2b5\ub2c8\ub2e4. \uc55e\uc73c\ub85c\ub3c4 \uae08\uc735 \ubd84\uc57c\uc5d0\uc11c\uc758 \uc778\uacf5\uc9c0\ub2a5 \uc801\uc6a9 \uac00\ub2a5\uc131\uc740 \ubb34\uad81\ubb34\uc9c4\ud558\ubbc0\ub85c, \uc9c0\uc18d\uc801\uc73c\ub85c \uc5f0\uad6c\ud558\uace0 \uc2e4\ud5d8\ud574 \ub098\uac00\uae38 \ubc14\ub78d\ub2c8\ub2e4.\n        <\/p>\n
\n            \uc774 \uac15\uc88c\uac00 \uc5ec\ub7ec\ubd84\uc774 DDQN\uc744 \ud1b5\ud574 \uae08\uc735 \uc2dc\uc7a5\uc5d0\uc11c \ub354\uc6b1 \ud6a8\uacfc\uc801\uc778 \ud2b8\ub808\uc774\ub529 \uc804\ub7b5\uc744 \uac1c\ubc1c\ud558\ub294 \ub370 \ub3c4\uc6c0\uc774 \ub418\uae30\ub97c \ubc14\ub78d\ub2c8\ub2e4. \ucd94\uac00\uc801\uc778 \uc9c8\ubb38\uc774\ub098 \ub3c4\uc6c0\uc774 \ud544\uc694\ud558\ub2e4\uba74 \uc5b8\uc81c\ub4e0\uc9c0 \uc5f0\ub77d\ud574 \uc8fc\uc138\uc694.\n        <\/p>\n<\/section>\n