\n \ubcf8 \uac15\uc88c\uc5d0\uc11c\ub294 \ub525\ub7ec\ub2dd\uc758 \uae30\ucd08 \uc9c0\uc2dd\uacfc \ud568\uaed8 \ub9c8\ub974\ucf54\ud504 \ubcf4\uc0c1 \ud504\ub85c\uc138\uc2a4(Markov Decision Process, MDP)\uc5d0 \ub300\ud574
\n \uc54c\uc544\ubcf4\uace0, \uc774\ub97c \ud30c\uc774\ud1a0\uce58(Pytorch)\ub97c \ud65c\uc6a9\ud558\uc5ec \uad6c\ud604\ud558\ub294 \ubc29\ubc95\uc5d0 \ub300\ud574 \uc124\uba85\ud569\ub2c8\ub2e4. MDP\ub294 \uac15\ud654\ud559\uc2b5
\n \ubd84\uc57c\uc5d0\uc11c \uc911\uc694\ud55c \uac1c\ub150\uc73c\ub85c, \ubaa9\ud45c\ub97c \ub2ec\uc131\ud558\uae30 \uc704\ud55c \ucd5c\uc801\uc758 \ud589\ub3d9\uc744 \ucc3e\ub294 \ub370 \uc911\uc694\ud55c \uc218\ud559\uc801 \ubaa8\ub378\uc785\ub2c8\ub2e4.\n <\/p>\n
1. \ub9c8\ub974\ucf54\ud504 \ubcf4\uc0c1 \ud504\ub85c\uc138\uc2a4\ub780?<\/h2>\n
\n \ub9c8\ub974\ucf54\ud504 \ubcf4\uc0c1 \ud504\ub85c\uc138\uc2a4(MDP)\ub294 \uc5b4\ub5a4 \ud658\uacbd\uc5d0\uc11c \uc5d0\uc774\uc804\ud2b8(\ud589\ub3d9\ud558\ub294 \uc8fc\uccb4)\uac00
\n \ucd5c\uc801\uc758 \uacb0\uc815\uc744 \ub0b4\ub9ac\uae30 \uc704\ud574 \uace0\ub824\ud574\uc57c \ud560 \uc694\uc18c\ub4e4\uc744 \uc815\uc758\ud558\ub294 \uc218\ud559\uc801 \ud504\ub808\uc784\uc6cc\ud06c\uc785\ub2c8\ub2e4. MDP\ub294
\n \ub2e4\uc74c\uc758 5\uac00\uc9c0 \uc8fc\uc694 \uc694\uc18c\ub85c \uad6c\uc131\ub429\ub2c8\ub2e4:\n <\/p>\n
- \n
- \uc0c1\ud0dc \uc9d1\ud569 (S)<\/strong>: \ud658\uacbd\uc758 \ubaa8\ub4e0 \uac00\ub2a5\ud55c \uc0c1\ud0dc\ub97c \ub098\ud0c0\ub0b4\ub294 \uc9d1\ud569\uc785\ub2c8\ub2e4.<\/li>\n
- \ud589\ub3d9 \uc9d1\ud569 (A)<\/strong>: \uac01 \uc0c1\ud0dc\uc5d0\uc11c \uc5d0\uc774\uc804\ud2b8\uac00 \ucde8\ud560 \uc218 \uc788\ub294 \uac00\ub2a5\ud55c \ud589\ub3d9\uc758 \uc9d1\ud569\uc785\ub2c8\ub2e4.<\/li>\n
- \uc804\ud658 \ud655\ub960 (P)<\/strong>: \ud604\uc7ac \uc0c1\ud0dc\uc5d0\uc11c \ud2b9\uc815 \ud589\ub3d9\uc744 \ucde8\ud588\uc744 \ub54c, \ub2e4\uc74c \uc0c1\ud0dc\ub85c \uc804\ud658\ub420 \ud655\ub960\uc744 \ub098\ud0c0\ub0c5\ub2c8\ub2e4.<\/li>\n
- \ubcf4\uc0c1 \ud568\uc218 (R)<\/strong>: \ud2b9\uc815 \uc0c1\ud0dc\uc5d0\uc11c \ud2b9\uc815 \ud589\ub3d9\uc744 \ud1b5\ud574 \uc5bb\ub294 \ubcf4\uc0c1\uc744 \uc815\uc758\ud569\ub2c8\ub2e4.<\/li>\n
- \ud560\uc778\uc728 (\u03b3)<\/strong>: \ubbf8\ub798 \ubcf4\uc0c1\uc774 \ud604\uc7ac \ubcf4\uc0c1\uc5d0 \ube44\ud574 \uc5bc\ub9c8\ub098 \uc911\uc694\ud55c\uc9c0\ub97c \uacb0\uc815\ud558\ub294 \uac12\uc785\ub2c8\ub2e4.<\/li>\n<\/ul>\n<\/section>\n
\n 2. \ub9c8\ub974\ucf54\ud504 \ubcf4\uc0c1 \ud504\ub85c\uc138\uc2a4\uc758 \uc218\ud559\uc801 \uc815\uc758<\/h2>\n
\n MDP\ub294 \uc77c\ubc18\uc801\uc73c\ub85c \ud29c\ud50c (S, A, P, R, \u03b3)\ub85c \uc815\uc758\ub418\uba70, \uc5d0\uc774\uc804\ud2b8\ub294 \uc774 \uc815\ubcf4\ub97c \ubc14\ud0d5\uc73c\ub85c \uc815\ucc45(\ubcf4\ub2e4 \ub098\uc740 \ud589\ub3d9\uc744 \uc120\ud0dd\ud558\ub294 \uaddc\uce59)\uc744 \ubc30\uc6c1\ub2c8\ub2e4. MDP\uc758 \ubaa9\ud45c\ub294 \uc7a5\uae30\uc801\uc778 \ubcf4\uc0c1(Reward)\uc744 \ucd5c\ub300\ud654\ud558\ub294 \ucd5c\uc801\uc758 \uc815\ucc45\uc744 \ucc3e\ub294 \uac83\uc785\ub2c8\ub2e4.\n <\/p>\n
\uc0c1\ud0dc\uc640 \ud589\ub3d9\uc758 \uc0c1\uad00\uad00\uacc4<\/h3>\n
\n \uac01 \uc0c1\ud0dc s \u2208 S\uc5d0\uc11c \ud589\ub3d9 a \u2208 A\ub97c \ucde8\ud588\uc744 \ub54c, \ub2e4\uc74c \uc0c1\ud0dc s’ \u2208 S\ub85c \uc804\uc774\ub420 \ud655\ub960\uc740 P(s’|s, a)\ub85c \ub098\ud0c0\ub0c5\ub2c8\ub2e4. \ubcf4\uc0c1 \ud568\uc218\ub294 R(s, a)\ub85c \ud45c\ud604\ub418\uba70, \uc774\ub294 \uc5d0\uc774\uc804\ud2b8\uac00 \uc0c1\ud0dc s\uc5d0\uc11c \ud589\ub3d9 a\ub97c \ucde8\ud560 \ub54c \uc5bb\ub294 \uc989\uac01\uc801\uc778 \ubcf4\uc0c1\uc744 \ub098\ud0c0\ub0c5\ub2c8\ub2e4.\n <\/p>\n
\uc815\ucc45 \u03c0<\/h3>\n
\n \uc815\ucc45 \u03c0\ub294 \uc0c1\ud0dc s\uc5d0\uc11c \uc5b4\ub5a4 \ud589\ub3d9 a\ub97c \ucde8\ud560 \ud655\ub960\uc744 \uc815\uc758\ud569\ub2c8\ub2e4. \uc774\ub97c \ud1b5\ud574 \uc5d0\uc774\uc804\ud2b8\ub294 \uc8fc\uc5b4\uc9c4 \uc0c1\ud0dc\uc5d0\uc11c \ucd5c\uc801\uc758 \ud589\ub3d9\uc744 \uc120\ud0dd\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4.\n <\/p>\n<\/section>\n
\n 3. \ud30c\uc774\ud1a0\uce58\ub85c MDP \uad6c\ud604\ud558\uae30<\/h2>\n
\n \uc774\uc81c \ub9c8\ub974\ucf54\ud504 \ubcf4\uc0c1 \ud504\ub85c\uc138\uc2a4\ub97c \ud30c\uc774\ud1a0\uce58\ub85c \uad6c\ud604\ud574\ubcf4\uaca0\uc2b5\ub2c8\ub2e4. \uc544\ub798 \ucf54\ub4dc\ub294 MDP\ub97c \uc815\uc758\ud558\uace0,
\n \uc5d0\uc774\uc804\ud2b8\uac00 \ucd5c\uc801\uc758 \uc815\ucc45\uc744 \ud559\uc2b5\ud558\ub294 \uacfc\uc815\uc744 \ubcf4\uc5ec\uc90d\ub2c8\ub2e4. \uc774 \uc608\uc81c\uc5d0\uc11c\ub294 \uac04\ub2e8\ud55c \uadf8\ub9ac\ub4dc \ud658\uacbd\uc5d0\uc11c\uc758
\n MDP\ub97c \uc0ac\uc6a9\ud558\uc5ec \uc5d0\uc774\uc804\ud2b8\uac00 \ubaa9\ud45c \uc9c0\uc810\uc5d0 \ub3c4\ub2ec\ud558\ub294 \uacfc\uc815\uc744 \uc2dc\ubbac\ub808\uc774\uc158\ud569\ub2c8\ub2e4.\n <\/p>\n\ud544\uc694\ud55c \ub77c\uc774\ube0c\ub7ec\ub9ac \uc124\uce58\ud558\uae30<\/h3>\n
\n
\n pip install torch numpy matplotlib\n <\/code>\n <\/pre>\n\ucf54\ub4dc \uc608\uc81c<\/h3>\n
\n
\nimport numpy as np\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport matplotlib.pyplot as plt\n\n# \ud658\uacbd \uc815\uc758\nclass GridWorld:\n def __init__(self, grid_size):\n self.grid_size = grid_size\n self.state = (0, 0) # \ucd08\uae30 \uc0c1\ud0dc\n self.goal = (grid_size - 1, grid_size - 1) # \ubaa9\ud45c \uc0c1\ud0dc\n self.actions = [(0, 1), (0, -1), (1, 0), (-1, 0)] # \uc624\ub978\ucabd, \uc67c\ucabd, \uc544\ub798, \uc704\n\n def step(self, action):\n next_state = (self.state[0] + action[0], self.state[1] + action[1])\n # \uacbd\uacc4\ub97c \ucd08\uacfc\ud558\uba74 \uc0c1\ud0dc\ub97c \ubcc0\uacbd\ud558\uc9c0 \uc54a\uc74c\n if 0 <= next_state[0] < self.grid_size and 0 <= next_state[1] < self.grid_size:\n self.state = next_state\n \n # \ubcf4\uc0c1 \ubc0f \uc644\ub8cc \uc870\uac74\n if self.state == self.goal:\n return self.state, 1, True # \ubaa9\ud45c \ub3c4\ub2ec\n return self.state, 0, False\n\n def reset(self):\n self.state = (0, 0)\n return self.state\n\n# Q-\ub124\ud2b8\uc6cc\ud06c \uc815\uc758\nclass QNetwork(nn.Module):\n def __init__(self, input_dim, output_dim):\n super(QNetwork, self).__init__()\n self.fc1 = nn.Linear(input_dim, 24) # \uccab \ubc88\uc9f8 \uc740\ub2c9\uce35\n self.fc2 = nn.Linear(24, 24) # \ub450 \ubc88\uc9f8 \uc740\ub2c9\uce35\n self.fc3 = nn.Linear(24, output_dim) # \ucd9c\ub825\uce35\n\n def forward(self, x):\n x = nn.functional.relu(self.fc1(x))\n x = nn.functional.relu(self.fc2(x))\n return self.fc3(x)\n\n# Q-learning \ud559\uc2b5\uae30\nclass QLearningAgent:\n def __init__(self, state_space, action_space):\n self.q_network = QNetwork(state_space, action_space)\n self.optimizer = optim.Adam(self.q_network.parameters(), lr=0.001)\n self.criterion = nn.MSELoss()\n self.gamma = 0.99 # \ud560\uc778\uc728\n self.epsilon = 1.0 # \ud0d0\ud5d8 \ube44\uc728\n self.epsilon_min = 0.01\n self.epsilon_decay = 0.995\n\n def choose_action(self, state):\n if np.random.rand() <= self.epsilon:\n return np.random.randint(0, 4) # \ubb34\uc791\uc704 \ud589\ub3d9\n q_values = self.q_network(torch.FloatTensor(state)).detach().numpy()\n return np.argmax(q_values) # \ucd5c\uc801 \ud589\ub3d9 \ubc18\ud658\n\n def train(self, state, action, reward, next_state, done):\n target = reward\n if not done:\n target = reward + self.gamma * np.max(self.q_network(torch.FloatTensor(next_state)).detach().numpy())\n \n target_f = self.q_network(torch.FloatTensor(state)).detach().numpy()\n target_f[action] = target\n\n # \ud559\uc2b5\n self.optimizer.zero_grad()\n output = self.q_network(torch.FloatTensor(state))\n loss = self.criterion(output, torch.FloatTensor(target_f))\n loss.backward()\n self.optimizer.step()\n\n # \ud0d0\ud5d8 \ube44\uc728 \uac10\uc18c\n if self.epsilon > self.epsilon_min:\n self.epsilon *= self.epsilon_decay\n\n# \uba54\uc778 \ub8e8\ud504\ndef main():\n env = GridWorld(grid_size=5)\n agent = QLearningAgent(state_space=2, action_space=4)\n episodes = 1000\n rewards = []\n\n for episode in range(episodes):\n state = env.reset()\n done = False\n total_reward = 0\n \n while not done:\n action = agent.choose_action(state)\n next_state, reward, done = env.step(env.actions[action])\n agent.train(state, action, reward, next_state, done)\n state = next_state\n total_reward += reward\n \n rewards.append(total_reward)\n\n # \uacb0\uacfc \uc2dc\uac01\ud654\n plt.plot(rewards)\n plt.xlabel('Episode')\n plt.ylabel('Reward')\n plt.title('Training Rewards over Episodes')\n plt.show()\n\nif __name__ == \"__main__\":\n main()\n <\/code>\n <\/pre>\n<\/section>\n\n 4. \ucf54\ub4dc \uc124\uba85<\/h2>\n
\n \uc704\uc758 \ucf54\ub4dc\ub294 5×5\uc758 \uadf8\ub9ac\ub4dc \ud658\uacbd\uc5d0\uc11c\uc758 MDP\ub97c \uad6c\ud604\ud55c \uc608\uc81c\uc785\ub2c8\ub2e4.
\n GridWorld<\/strong> \ud074\ub798\uc2a4\ub294 \uc5d0\uc774\uc804\ud2b8\uac00 \uc6c0\uc9c1\uc77c \uc218 \uc788\ub294 \uadf8\ub9ac\ub4dc \ud658\uacbd\uc744 \uc815\uc758\ud569\ub2c8\ub2e4. \uc5d0\uc774\uc804\ud2b8\ub294 \uc81c\uacf5\ub41c
\n \ud589\ub3d9 \uc9d1\ud569\uc744 \uae30\ubc18\uc73c\ub85c \uc774\ub3d9\ud558\uba70, \ubaa9\ud45c \uc9c0\uc810\uc5d0 \ub3c4\ub2ec\ud560 \ub54c \ubcf4\uc0c1\uc744 \ubc1b\uc2b5\ub2c8\ub2e4.\n <\/p>\n\nQNetwork<\/strong> \ud074\ub798\uc2a4\ub294 Q-\ub7ec\ub2dd\uc5d0\uc11c \uc0ac\uc6a9\ud558\ub294 \uc2ec\uce35 \uc2e0\uacbd\ub9dd \ubaa8\ub378\uc744 \uc815\uc758\ud569\ub2c8\ub2e4.
\n \uc785\ub825\uc73c\ub85c \uc0c1\ud0dc \ucc28\uc6d0\uc744 \ubc1b\uace0, \ucd9c\ub825\uc73c\ub85c \uac01 \ud589\ub3d9\uc5d0 \ub300\ud55c Q-\uac12\uc744 \ubc18\ud658\ud569\ub2c8\ub2e4.
\n QLearningAgent<\/strong> \ud074\ub798\uc2a4\ub294 \uc2e4\uc81c \uac15\ud654 \ud559\uc2b5\uc758 \ud559\uc2b5 \uacfc\uc815\uc744 \uc218\ud589\ud558\ub294 \uc5d0\uc774\uc804\ud2b8\ub97c
\n \ub098\ud0c0\ub0c5\ub2c8\ub2e4. \uc774 \uc5d0\uc774\uc804\ud2b8\ub294 \uc815\ucc45\uc744 \uc0ac\uc6a9\ud558\uc5ec \ud589\ub3d9\uc744 \uc120\ud0dd\ud558\uace0, Q-\uac12\uc744 \uc5c5\ub370\uc774\ud2b8\ud569\ub2c8\ub2e4.\n <\/p>\n\nmain<\/strong> \ud568\uc218\uc5d0\uc11c\ub294 \ud658\uacbd\uc744 \ucd08\uae30\ud654\ud558\uace0, \uc5d0\ud53c\uc18c\ub4dc\ub97c \uc2e4\ud589\ud558\ub294 \uc8fc \ub8e8\ud504\uac00 \ud3ec\ud568\ub418\uc5b4 \uc788\uc2b5\ub2c8\ub2e4.
\n \uac01 \uc5d0\ud53c\uc18c\ub4dc\uc5d0\uc11c \uc5d0\uc774\uc804\ud2b8\ub294 \uc8fc\uc5b4\uc9c4 \uc0c1\ud0dc\uc5d0 \ub300\ud574 \ud589\ub3d9\uc744 \uc120\ud0dd\ud558\uace0, \ud658\uacbd\uc758 \ub2e4\uc74c \uc0c1\ud0dc\ub97c \ud1b5\ud574 \ubcf4\uc0c1\uc744
\n \ubc1b\uc73c\uba70, \uc774\ub97c \ud559\uc2b5\ud569\ub2c8\ub2e4. \ud6c8\ub828\uc774 \uc644\ub8cc\ub418\uba74 \ubcf4\uc0c1\uc744 \uc2dc\uac01\ud654\ud558\uc5ec \uc5d0\uc774\uc804\ud2b8\uc758 \uc131\uacfc\ub97c \ud655\uc778\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4.\n <\/p>\n<\/section>\n\n 5. \ud559\uc2b5 \uacb0\uacfc \ubd84\uc11d<\/h2>\n
\n \ud559\uc2b5 \uacfc\uc815\uc744 \uc0b4\ud3b4\ubcf4\uba74, \uc5d0\uc774\uc804\ud2b8\uac00 \ud658\uacbd\uc744 \ud0d0\uc0c9\ud558\uba74\uc11c \ub9f5\uc744 \ud6a8\uacfc\uc801\uc73c\ub85c \ud0d0\uc0c9\ud558\uc5ec \ubaa9\ud45c\uc5d0 \ub3c4\ub2ec\ud558\uac8c \ub429\ub2c8\ub2e4.
\n \ubcf4\uc0c1\uc758 \ucd94\uc138\ub97c \uc2dc\uac01\ud654\ud55c \uadf8\ub798\ud504\ub97c \ud1b5\ud574 \ud559\uc2b5\uc774 \uc9c4\ud589\ub428\uc5d0 \ub530\ub77c \ubcf4\uc0c1\uc774 \uc5b4\ub5bb\uac8c \ubcc0\ud654\ud558\ub294\uc9c0\ub97c \ud655\uc778\ud560 \uc218
\n \uc788\uc2b5\ub2c8\ub2e4. \uc774\uc0c1\uc801\uc778 \uacbd\uc6b0, \uc5d0\uc774\uc804\ud2b8\ub294 \uc810\ucc28 \ub192\uc740 \ubcf4\uc0c1\uc744 \uc5bb\ub294 \ubc29\ud5a5\uc73c\ub85c \ud559\uc2b5\ud558\uac8c \ub429\ub2c8\ub2e4.\n <\/p>\n<\/section>\n\n 6. \uacb0\ub860 \ubc0f \ud5a5\ud6c4 \ubc29\ud5a5<\/h2>\n
\n \ubcf8 \uac15\uc88c\uc5d0\uc11c\ub294 \ub525\ub7ec\ub2dd, \ud30c\uc774\ud1a0\uce58, \uadf8\ub9ac\uace0 \ub9c8\ub974\ucf54\ud504 \ubcf4\uc0c1 \ud504\ub85c\uc138\uc2a4\uc758 \uae30\ucd08 \uac1c\ub150\ub4e4\uc5d0 \ub300\ud574
\n \uc124\uba85\ud558\uc600\uc2b5\ub2c8\ub2e4. \ud30c\uc774\ud1a0\uce58\ub97c \uc0ac\uc6a9\ud558\uc5ec MDP\ub97c \uad6c\ud604\ud558\ub294 \uacfc\uc815\uacfc \uac19\uc774 \uc2e4\uc2b5\uc744 \ud1b5\ud574 \uad00\ub828 \uac1c\ub150\uc744
\n \ubcf4\ub2e4 \uae4a\uc774 \uc774\ud574\ud560 \uc218 \uc788\uc5c8\uc2b5\ub2c8\ub2e4. \uac15\ud654\ud559\uc2b5\uc740 \ub9e4\uc6b0 \uad11\ubc94\uc704\ud55c \ubd84\uc57c\uc774\uba70, \ub2e4\uc591\ud55c \uc54c\uace0\ub9ac\uc998\uacfc
\n \uc801\uc6a9 \uac00\ub2a5\ud55c \ud658\uacbd\ub4e4\uc774 \uc874\uc7ac\ud569\ub2c8\ub2e4.
\n \ud5a5\ud6c4\uc5d0\ub294 \ub354 \ubcf5\uc7a1\ud55c \ud658\uacbd \ubc0f \ub2e4\uc591\ud55c \uc815\ucc45 \ud559\uc2b5 \uc54c\uace0\ub9ac\uc998(\uc608: DQN, Policy Gradients \ub4f1)\uc744
\n \ub2e4\ub8e8\ub294 \uc2ec\ud654 \uac15\uc88c\ub97c \uc9c4\ud589\ud560 \uc608\uc815\uc785\ub2c8\ub2e4.\n <\/p>\n<\/section>\n<\/article>\n<\/body><\/p>\n","protected":false},"excerpt":{"rendered":"
- \ud589\ub3d9 \uc9d1\ud569 (A)<\/strong>: \uac01 \uc0c1\ud0dc\uc5d0\uc11c \uc5d0\uc774\uc804\ud2b8\uac00 \ucde8\ud560 \uc218 \uc788\ub294 \uac00\ub2a5\ud55c \ud589\ub3d9\uc758 \uc9d1\ud569\uc785\ub2c8\ub2e4.<\/li>\n