\n \ub525 \ud050 \ub7ec\ub2dd(Deep Q-Learning)\uc740 \uac15\ud654\ud559\uc2b5(Reinforcement Learning) \ubd84\uc57c\uc5d0\uc11c \ub9e4\uc6b0 \uc911\uc694\ud55c \uc54c\uace0\ub9ac\uc998 \uc911 \ud558\ub098\uc785\ub2c8\ub2e4.
\n \uc2ec\uce35 \uc2e0\uacbd\ub9dd\uc744 \uc0ac\uc6a9\ud558\uc5ec \uc5d0\uc774\uc804\ud2b8\uac00 \ucd5c\uc801\uc758 \ud589\ub3d9(action)\uc744 \uc120\ud0dd\ud558\ub3c4\ub85d \ud559\uc2b5\uc2dc\ud0b5\ub2c8\ub2e4. \ubcf8 \uac15\uc88c\uc5d0\uc11c\ub294 \ud30c\uc774\ud1a0\uce58(PyTorch) \ub77c\uc774\ube0c\ub7ec\ub9ac\ub97c \ud65c\uc6a9\ud558\uc5ec
\n \ub525 \ud050 \ub7ec\ub2dd \uc54c\uace0\ub9ac\uc998\uc744 \uad6c\ud604\ud558\uace0 \uc774\ud574\ud558\ub294 \ub370 \ud544\uc694\ud55c \uae30\ubcf8 \uac1c\ub150\ub4e4\uc744 \uc790\uc138\ud788 \uc0b4\ud3b4\ubcf4\uaca0\uc2b5\ub2c8\ub2e4.\n <\/p>\n
\n \uac15\ud654 \ud559\uc2b5\uc740 \uc5d0\uc774\uc804\ud2b8\uac00 \ud658\uacbd\uacfc \uc0c1\ud638\uc791\uc6a9\ud558\uc5ec \ubcf4\uc0c1(reward)\uc744 \ucd5c\ub300\ud654\ud558\ub294 \ud589\ub3d9\uc744 \ud559\uc2b5\ud558\ub294 \ubc29\ubc95\uc785\ub2c8\ub2e4.
\n \uc5d0\uc774\uc804\ud2b8\ub294 \uc0c1\ud0dc(state)\ub97c \uad00\ucc30\ud558\uace0, \uac00\ub2a5\ud55c \ud589\ub3d9\uc744 \uc120\ud0dd\ud558\uba70, \uadf8\ub85c \uc778\ud574 \ud658\uacbd\uc5d0\uc11c \ubcc0\ud654\ub97c \uacbd\ud5d8\ud569\ub2c8\ub2e4.
\n \uc774\ub7ec\ud55c \uacfc\uc815\uc740 \ub2e4\uc74c\uacfc \uac19\uc740 \uad6c\uc131\uc694\uc18c\ub85c \uc774\ub8e8\uc5b4\uc838 \uc788\uc2b5\ub2c8\ub2e4.\n <\/p>\n
\n Q-Learning\uc740 \uac15\ud654\ud559\uc2b5\uc758 \ud55c \ud615\ud0dc\ub85c, \uc5d0\uc774\uc804\ud2b8\uac00 \uc5b4\ub5a4 \uc0c1\ud0dc\uc5d0\uc11c \ud2b9\uc815 \ud589\ub3d9\uc744 \ucde8\ud588\uc744 \ub54c \uc608\uc0c1\ub418\ub294 \ubcf4\uc0c1\uc744
\n \ud559\uc2b5\ud558\ub294 \uc54c\uace0\ub9ac\uc998\uc785\ub2c8\ub2e4. Q-Learning\uc758 \ud575\uc2ec\uc740 Q-\uac12\uc744 \uc5c5\ub370\uc774\ud2b8\ud558\ub294 \uac83\uc785\ub2c8\ub2e4. Q-\uac12\uc740 \uc0c1\ud0dc-\ud589\ub3d9 \uc30d\uc758
\n \uc7a5\uae30\uc801\uc778 \ubcf4\uc0c1\uc744 \ub098\ud0c0\ub0c5\ub2c8\ub2e4. Q-\uac12\uc740 \ub2e4\uc74c\uacfc \uac19\uc740 \ubca8\ub9cc \ubc29\uc815\uc2dd\uc73c\ub85c \uc5c5\ub370\uc774\ud2b8\ub429\ub2c8\ub2e4.\n <\/p>\n
\nQ(s, a) \u2190 Q(s, a) + \u03b1[r + \u03b3 max Q(s’, a’) – Q(s, a)]<\/strong>\n<\/p>\n \n \uc5ec\uae30\uc11c \u03b1\ub294 \ud559\uc2b5\ub960, \u03b3\ub294 \ud560\uc778\uacc4\uc218, s\ub294 \ud604\uc7ac \uc0c1\ud0dc, s’\ub294 \ub2e4\uc74c \uc0c1\ud0dc\uc785\ub2c8\ub2e4. \n \ub525 \ud050 \ub7ec\ub2dd(Deep Q-Learning)\uc740 \uc2ec\uce35 \uc2e0\uacbd\ub9dd\uc744 \uc0ac\uc6a9\ud558\uc5ec Q-\uac12\uc744 \uadfc\uc0ac\ud558\ub294 \ubc29\uc2dd\uc785\ub2c8\ub2e4. \n DQN\uc740 \uc774 \ub450 \uac00\uc9c0 \uae30\uc220\uc744 \ud65c\uc6a9\ud558\uc5ec \ud559\uc2b5\uc758 \uc548\uc815\uc131\uacfc \uc131\ub2a5\uc744 \uac1c\uc120\ud569\ub2c8\ub2e4.\n <\/p>\n \n \uc774\uc81c \ud30c\uc774\uc36c\uacfc \ud30c\uc774\ud1a0\uce58\ub97c \uc0ac\uc6a9\ud558\uc5ec DQN\uc744 \uad6c\ud604\ud558\uae30 \uc704\ud574 \ud544\uc694\ud55c \ud328\ud0a4\uc9c0\ub97c \uc124\uce58\ud574\ubcf4\uaca0\uc2b5\ub2c8\ub2e4. \n \uc544\ub798\ub294 DQN \ud074\ub798\uc2a4\uc758 \uae30\ubcf8\uc801\uc778 \uace8\uaca9\uacfc \ud658\uacbd \uc124\uc815 \ucf54\ub4dc\uc785\ub2c8\ub2e4. \uac04\ub2e8\ud55c \uc608\ub85c OpenAI\uc758 Gym\uc5d0\uc11c \uc81c\uacf5\ud558\ub294 CartPole \ud658\uacbd\uc744 \uc0ac\uc6a9\ud558\uaca0\uc2b5\ub2c8\ub2e4.\n <\/p>\n \n DQN \uc54c\uace0\ub9ac\uc998\uc740 \ubcf5\uc7a1\ud55c \uc0c1\ud0dc \uacf5\uac04\uc744 \uac00\uc9c4 \ubb38\uc81c\uc5d0\uc11c \ud6a8\uacfc\uc801\uc73c\ub85c \ub3d9\uc791\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4. \n \ud5a5\ud6c4 \uac1c\uc120 \ubc29\uc548\uc73c\ub85c\ub294 \ub354 \ubcf5\uc7a1\ud55c \ud658\uacbd\uc5d0\uc11c\uc758 \uc2e4\ud5d8, \ub2e4\uc591\ud55c \ud558\uc774\ud37c\ud30c\ub77c\ubbf8\ud130 \uc870\uc815,
\n Q-Learning\uc740 \uc8fc\ub85c \ud14c\uc774\ube14 \ud615\uc2dd\uc73c\ub85c Q-\uac12\uc744 \uc800\uc7a5\ud558\uc9c0\ub9cc, \uc0c1\ud0dc \uacf5\uac04\uc774 \ud06c\uac70\ub098 \uc5f0\uc18d\uc801\uc77c \ub54c\ub294
\n \ub525\ub7ec\ub2dd\uc744 \ud1b5\ud574 Q-\uac12\uc744 \uadfc\uc0ac\ud574\uc57c \ud569\ub2c8\ub2e4.\n <\/p>\n4. \ub525 \ud050 \ub7ec\ub2dd (DQN)<\/h2>\n
\n DQN\uc740 \ub2e4\uc74c\uacfc \uac19\uc740 \uc8fc\uc694 \uad6c\uc131 \uc694\uc18c\ub97c \uac16\uc2b5\ub2c8\ub2e4.\n <\/p>\n\n
5. \ud658\uacbd \uc124\uc815<\/h2>\n
\n \uc544\ub798\uc640 \uac19\uc774 pip\ub97c \uc774\uc6a9\ud558\uc5ec \ud544\uc694\ud55c \ub77c\uc774\ube0c\ub7ec\ub9ac\ub97c \uc124\uce58\ud558\uaca0\uc2b5\ub2c8\ub2e4.\n <\/p>\n\n
\n pip install torch torchvision numpy matplotlib gym\n <\/code>\n <\/pre>\n6. DQN \uad6c\ud604\ud558\uae30<\/h2>\n
6.1 DQN \ud074\ub798\uc2a4 \uc815\uc758<\/h3>\n
\n
\n import torch\n import torch.nn as nn\n import torch.optim as optim\n import numpy as np\n import random\n \n class DQN(nn.Module):\n def __init__(self, state_size, action_size):\n super(DQN, self).__init__()\n self.fc1 = nn.Linear(state_size, 128)\n self.fc2 = nn.Linear(128, 128)\n self.fc3 = nn.Linear(128, action_size)\n\n def forward(self, x):\n x = torch.relu(self.fc1(x))\n x = torch.relu(self.fc2(x))\n return self.fc3(x)\n <\/code>\n <\/pre>\n6.2 \ud658\uacbd \uc124\uc815 \ubc0f \ud558\uc774\ud37c\ud30c\ub77c\ubbf8\ud130<\/h3>\n
\n
\n import gym\n \n # \ud658\uacbd\uacfc \ud558\uc774\ud37c\ud30c\ub77c\ubbf8\ud130 \uc124\uc815\n env = gym.make('CartPole-v1')\n state_size = env.observation_space.shape[0]\n action_size = env.action_space.n\n learning_rate = 0.001\n gamma = 0.99\n epsilon = 1.0\n epsilon_decay = 0.995\n epsilon_min = 0.01\n num_episodes = 1000\n replay_memory = []\n replay_memory_size = 2000\n <\/code>\n <\/pre>\n6.3 \ud559\uc2b5 \ub8e8\ud504<\/h3>\n
\n
\n def train_dqn():\n model = DQN(state_size, action_size)\n optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n criterion = nn.MSELoss()\n \n for episode in range(num_episodes):\n state = env.reset()\n state = np.reshape(state, [1, state_size])\n done = False\n total_reward = 0\n \n while not done:\n if np.random.rand() <= epsilon:\n action = np.random.randint(action_size)\n else:\n q_values = model(torch.FloatTensor(state))\n action = torch.argmax(q_values).item()\n\n next_state, reward, done, _ = env.step(action)\n total_reward += reward\n next_state = np.reshape(next_state, [1, state_size])\n \n if done:\n reward = -1\n\n replay_memory.append((state, action, reward, next_state, done))\n if len(replay_memory) > replay_memory_size:\n replay_memory.pop(0)\n\n if len(replay_memory) > 32:\n minibatch = random.sample(replay_memory, 32)\n for m_state, m_action, m_reward, m_next_state, m_done in minibatch:\n target = m_reward\n if not m_done:\n target += gamma * torch.max(model(torch.FloatTensor(m_next_state))).item()\n target_f = model(torch.FloatTensor(m_state))\n target_f[m_action] = target\n optimizer.zero_grad()\n loss = criterion(model(torch.FloatTensor(m_state)), target_f)\n loss.backward()\n optimizer.step()\n\n state = next_state\n\n global epsilon\n if epsilon > epsilon_min:\n epsilon *= epsilon_decay\n \n print(f\"Episode: {episode}\/{num_episodes}, Total Reward: {total_reward}\")\n \n train_dqn()\n <\/code>\n <\/pre>\n7. \uacb0\uacfc \ubc0f \ub9c8\ubb34\ub9ac<\/h2>\n
\n \uc774 \ucf54\ub4dc \uc608\uc81c\uc5d0\uc11c \uc6b0\ub9ac\ub294 CartPole \ud658\uacbd\uc744 \uc0ac\uc6a9\ud558\uc5ec DQN\uc744 \ud559\uc2b5\uc2dc\ucf30\uc2b5\ub2c8\ub2e4.
\n \ud559\uc2b5\uc774 \uc9c4\ud589\ub428\uc5d0 \ub530\ub77c \uc5d0\uc774\uc804\ud2b8\ub294 \uc810\uc810 \ub354 \ub098\uc740 \uc131\ub2a5\uc744 \ubcf4\uc774\uac8c \ub420 \uac83\uc785\ub2c8\ub2e4.\n <\/p>\n
\n \ud14c\ud06c\ub2c9\uc744 \uc870\ud569\ud55c \ub2e4\uc591\ud55c \uc804\ub7b5\uc801 \uc811\uadfc \ubc29\uc2dd \ub4f1\uc774 \uc788\uc2b5\ub2c8\ub2e4.
\n \uc774 \uac15\uc88c\uc5d0\uc11c \ub2e4\ub8ec \ub0b4\uc6a9\uc774 \uc5ec\ub7ec\ubd84\uc758 \ub525\ub7ec\ub2dd \ubc0f \uac15\ud654\ud559\uc2b5 \uc774\ud574\uc5d0 \ub3c4\uc6c0\uc774 \ub418\uae30\ub97c \ubc14\ub78d\ub2c8\ub2e4!\n <\/p>\n8. \ucc38\uace0 \ubb38\ud5cc<\/h2>\n