\ub525 \uac15\ud654\ud559\uc2b5\uc740 \uac15\ud654\ud559\uc2b5\uacfc \ub525\ub7ec\ub2dd\uc744 \uacb0\ud569\ud55c \ud615\ud0dc\ub85c, \ubcf5\uc7a1\ud55c \ud658\uacbd\uc5d0\uc11c \ucd5c\uc801\uc758 \ud589\ub3d9\uc744 \ucc3e\uae30 \uc704\ud574 \uc2e0\uacbd\ub9dd\uc744 \uc774\uc6a9\ud569\ub2c8\ub2e4. \uc774 \uae00\uc5d0\uc11c\ub294 \ub525 Q-\ub7ec\ub2dd(Deep Q-Learning, DQN)\uc758 \uac1c\ub150\uacfc \uc54c\uace0\ub9ac\uc998\uc744 \uc124\uba85\ud558\uace0, \uc774\uc5d0 \ub300\ud55c \uac04\ub2e8\ud55c \uad6c\ud604\uc744 \ud1b5\ud574 DQN\uc758 \uc791\ub3d9 \ubc29\uc2dd\uc5d0 \ub300\ud574 \uc774\ud574\ud560 \uc218 \uc788\ub3c4\ub85d \ud558\uaca0\uc2b5\ub2c8\ub2e4.<\/p>\n<\/div>\n
\uac15\ud654\ud559\uc2b5\uc740 \uc5d0\uc774\uc804\ud2b8\uac00 \ud658\uacbd\uacfc \uc0c1\ud638\uc791\uc6a9\ud558\uba74\uc11c \ucd5c\ub300\uc758 \ubcf4\uc0c1\uc744 \uc5bb\uae30 \uc704\ud55c \ucd5c\uc801\uc758 \uc815\ucc45\uc744 \ud559\uc2b5\ud558\ub294 \uacfc\uc815\uc785\ub2c8\ub2e4. \uc5ec\uae30\uc11c \uc5d0\uc774\uc804\ud2b8\ub294 \ud589\ub3d9\uc744 \ucde8\ud558\uace0, \ud658\uacbd\uc740 \uadf8\uc5d0 \ub300\ud55c \ubcf4\uc0c1\uc744 \ubc18\ud658\ud569\ub2c8\ub2e4. \uac15\ud654\ud559\uc2b5\uc758 \uc8fc\uc694 \uc694\uc18c\ub294 \ub2e4\uc74c\uacfc \uac19\uc2b5\ub2c8\ub2e4:<\/p>\n
\ub525 Q-\ub7ec\ub2dd\uc740 Q-\ub7ec\ub2dd\uc758 \ud55c\uacc4\ub97c \uadf9\ubcf5\ud558\uae30 \uc704\ud574 \ub525\ub7ec\ub2dd\uc744 \ud65c\uc6a9\ud558\uc5ec Q-\uac12\uc744 \uadfc\uc0ac\ud569\ub2c8\ub2e4. \uc77c\ubc18\uc801\uc778 Q-\ub7ec\ub2dd\uc740 Q-\ud14c\uc774\ube14\uc744 \uc0ac\uc6a9\ud574 \ubaa8\ub4e0 \uc0c1\ud0dc-\ud589\ub3d9 \uc30d\uc5d0 \ub300\ud574 \uac12\uc744 \uc5c5\ub370\uc774\ud2b8\ud558\uc9c0\ub9cc, \uc0c1\ud0dc \uacf5\uac04\uc774 \ucee4\uc9c0\ub294 \uacbd\uc6b0 \uc774\ub97c \uc800\uc7a5\ud558\ub294 \uac83\uc774 \ube44\ud6a8\uc728\uc801\uc785\ub2c8\ub2e4. DQN\uc740 \uc2e0\uacbd\ub9dd\uc744 \uc0ac\uc6a9\ud558\uc5ec \uc0c1\ud0dc\uc5d0 \ub300\ud55c Q-\uac12\uc744 \uadfc\uc0ac\ud568\uc73c\ub85c\uc368 \uc774 \ubb38\uc81c\ub97c \ud574\uacb0\ud569\ub2c8\ub2e4.<\/p>\n
DQN\uc740 \ub2e4\uc74c\uacfc \uac19\uc740 \uc8fc\uc694 \uad6c\uc131 \uc694\uc18c\ub85c \uc774\ub8e8\uc5b4\uc838 \uc788\uc2b5\ub2c8\ub2e4:<\/p>\n
DQN \uc54c\uace0\ub9ac\uc998\uc740 \ub2e4\uc74c \ub2e8\uacc4\ub85c \uad6c\uc131\ub429\ub2c8\ub2e4:<\/p>\n
Q(s, a) = r + \u03b3 * max(Q(s', a'))<\/code>, \uc5ec\uae30\uc11c \u03b3<\/code>\ub294 \ud560\uc778 \uacc4\uc218\uc785\ub2c8\ub2e4.<\/li>\n- \uc77c\uc815 \uc8fc\uae30\ub85c \ud0c0\uac9f \ub124\ud2b8\uc6cc\ud06c\ub97c \uc5c5\ub370\uc774\ud2b8\ud569\ub2c8\ub2e4.<\/li>\n
- \uc5d0\ud53c\uc18c\ub4dc\ub97c \ubc18\ubcf5\ud569\ub2c8\ub2e4.<\/li>\n<\/ol>\n<\/div>\n\n
4. DQN \uad6c\ud604 \uc608\uc81c<\/h2>\n
\uc774\uc81c DQN\uc744 \uad6c\ud604\ud574 \ubcf4\uaca0\uc2b5\ub2c8\ub2e4. \uc774 \uc608\uc81c\uc5d0\uc11c\ub294 OpenAI\uc758 Gym \ub77c\uc774\ube0c\ub7ec\ub9ac\ub97c \uc0ac\uc6a9\ud558\uc5ec \uac04\ub2e8\ud55c \ud658\uacbd\uc744 \uc124\uc815\ud558\uace0 DQN \uc5d0\uc774\uc804\ud2b8\ub97c \ud559\uc2b5\uc2dc\ud0b5\ub2c8\ub2e4.<\/p>\n
4.1 \ud544\uc694 \ub77c\uc774\ube0c\ub7ec\ub9ac \uc124\uce58<\/h3>\npip install gym keras numpy<\/code><\/pre>\n4.2 DQN \ud074\ub798\uc2a4 \uad6c\ud604<\/h3>\nimport random\nimport numpy as np\nimport gym\nfrom keras.models import Sequential\nfrom keras.layers import Dense\nfrom keras.optimizers import Adam\n\nclass DQNAgent:\n def __init__(self, state_size, action_size):\n self.state_size = state_size\n self.action_size = action_size\n self.memory = []\n self.gamma = 0.95 # \ud560\uc778 \uacc4\uc218\n self.epsilon = 1.0 # \ud0d0\uc0c9 \ud655\ub960\n self.epsilon_min = 0.01\n self.epsilon_decay = 0.995\n self.model = self._build_model()\n\n def _build_model(self):\n model = Sequential()\n model.add(Dense(24, input_dim=self.state_size, activation='relu'))\n model.add(Dense(24, activation='relu'))\n model.add(Dense(self.action_size, activation='linear'))\n model.compile(loss='mse', optimizer=Adam(lr=0.001))\n return model\n\n def remember(self, state, action, reward, next_state, done):\n self.memory.append((state, action, reward, next_state, done))\n\n def act(self, state):\n if np.random.rand() <= self.epsilon:\n return random.randrange(self.action_size)\n act_values = self.model.predict(state)\n return np.argmax(act_values[0])\n\n def replay(self, batch_size):\n minibatch = random.sample(self.memory, batch_size)\n for state, action, reward, next_state, done in minibatch:\n target = reward\n if not done:\n target += self.gamma * np.amax(self.model.predict(next_state)[0])\n target_f = self.model.predict(state)\n target_f[0][action] = target\n self.model.fit(state, target_f, epochs=1, verbose=0)\n if self.epsilon > self.epsilon_min:\n self.epsilon *= self.epsilon_decay<\/code><\/pre>\n4.3 \ud559\uc2b5 \ud568\uc218<\/h3>\ndef train_dqn(episodes):\n for e in range(episodes):\n state = env.reset()\n state = np.reshape(state, [1, state_size])\n for time in range(500):\n action = agent.act(state)\n next_state, reward, done, _ = env.step(action)\n reward = reward if not done else -10\n next_state = np.reshape(next_state, [1, state_size])\n agent.remember(state, action, reward, next_state, done)\n state = next_state\n if done:\n print(f\"Episode: {e}\/{episodes}, score: {time}, e: {agent.epsilon:.2}\")\n break\n if len(agent.memory) > batch_size:\n agent.replay(batch_size)<\/code><\/pre>\n4.4 \uba54\uc778 \ud568\uc218<\/h3>\nif __name__ == \"__main__\":\n env = gym.make('CartPole-v1')\n state_size = env.observation_space.shape[0]\n action_size = env.action_space.n\n agent = DQNAgent(state_size, action_size)\n batch_size = 32\n train_dqn(1000)<\/code><\/pre>\n4.5 \ucf54\ub4dc \uc124\uba85<\/h3>\n
\uc704\uc758 \ucf54\ub4dc\ub294 DQN \uc54c\uace0\ub9ac\uc998\uc744 \uad6c\ud604\ud55c \uac83\uc785\ub2c8\ub2e4. \uc8fc\uc694 \uad6c\uc131 \uc694\uc18c\ub97c \uc0b4\ud3b4\ubcf4\uba74:<\/p>\n
\n- DQNAgent \ud074\ub798\uc2a4<\/strong>: \uc5d0\uc774\uc804\ud2b8\uc758 \ub3d9\uc791 \ubc0f \uae30\uc5b5\uc744 \uad00\ub9ac\ud569\ub2c8\ub2e4.<\/li>\n
- _build_model \uba54\uc11c\ub4dc<\/strong>: Q-\uac12\uc744 \ucd94\uc815\ud558\ub294 \uc2e0\uacbd\ub9dd\uc744 \uc0dd\uc131\ud569\ub2c8\ub2e4.<\/li>\n
- act \uba54\uc11c\ub4dc<\/strong>: \u03b5-greedy \uc804\ub7b5\uc744 \uc0ac\uc6a9\ud558\uc5ec \ud589\ub3d9\uc744 \uc120\ud0dd\ud569\ub2c8\ub2e4.<\/li>\n
- replay \uba54\uc11c\ub4dc<\/strong>: \uacbd\ud5d8\uc744 \ud559\uc2b5\ud569\ub2c8\ub2e4.<\/li>\n<\/ul>\n<\/div>\n\n
5. DQN\uc758 \ud55c\uacc4\uc640 \uac1c\uc120 \ubc29\ubc95<\/h2>\n
DQN\uc740 \uba87 \uac00\uc9c0 \ud55c\uacc4\uac00 \uc788\uc2b5\ub2c8\ub2e4:<\/p>\n
\n- Q-\uac12 \ud568\uc218\ub97c \uadfc\uc0ac\ud558\ub294\ub370 \uc788\uc5b4 \uacfc\uc801\ud569\uc758 \uc704\ud5d8\uc774 \uc788\uc2b5\ub2c8\ub2e4.<\/li>\n
- \ud0d0\uc0c9-\uac1c\ubc1c \uc0ac\uc774\uc758 \uade0\ud615\uc744 \ub9de\ucd94\ub294 \uac83\uc774 \uc5b4\ub835\uc2b5\ub2c8\ub2e4.<\/li>\n
- \uc0c1\ud0dc-\ud589\ub3d9 \uacf5\uac04\uc774 \ub9e4\uc6b0 \ud070 \uacbd\uc6b0 \uc131\ub2a5\uc774 \ub5a8\uc5b4\uc9c8 \uc218 \uc788\uc2b5\ub2c8\ub2e4.<\/li>\n<\/ul>\n
\uc774\ub7ec\ud55c \ud55c\uacc4\ub97c \uadf9\ubcf5\ud558\uae30 \uc704\ud55c \uba87 \uac00\uc9c0 \ubc29\ubc95\uc740 \ub2e4\uc74c\uacfc \uac19\uc2b5\ub2c8\ub2e4:<\/p>\n
\n- \uc6b0\uc120 \uc21c\uc704 \uacbd\ud5d8 \uc7ac\ud50c\ub808\uc774 (Prioritized Experience Replay)<\/strong>: \ub354 \uc911\uc694\ud55c \uacbd\ud5d8\uc744 \uc6b0\uc120\uc801\uc73c\ub85c \uc0d8\ud50c\ub9c1\ud558\uc5ec \ud559\uc2b5\ud569\ub2c8\ub2e4.<\/li>\n
- \ub354\ube14 \ub525 Q-\ub7ec\ub2dd (Double DQN)<\/strong>: \ud0c0\uac9f \ub124\ud2b8\uc6cc\ud06c\uc640 \uc8fc \ub124\ud2b8\uc6cc\ud06c\uc758 \uc5c5\ub370\uc774\ud2b8\ub97c \ubd84\ub9ac\ud558\uc5ec \uacfc\uc801\ud569\uc744 \uc904\uc785\ub2c8\ub2e4.<\/li>\n
- \ub4c0\uc5bc \ub525 Q-\ub7ec\ub2dd (Dueling DQN)<\/strong>: Q-\uac12\uc744 \uc0c1\ud0dc \uac00\uce58\uc640 Advantage \ud568\uc218\ub97c \ubd84\ub9ac\ud558\uc5ec \uacc4\uc0b0\ud558\ub294 \ubc29\ubc95\uc785\ub2c8\ub2e4.<\/li>\n<\/ul>\n<\/div>\n