MDP = (S, A, P, R, \u03b3)\ub85c \uc815\uc758\ub429\ub2c8\ub2e4.<\/li>\n<\/ul>\n\uc5ec\uae30\uc11c, \uac01 \uc694\uc18c\uc5d0 \ub300\ud574 \uc880 \ub354 \uad6c\uccb4\uc801\uc73c\ub85c \uc124\uba85\ud558\uaca0\uc2b5\ub2c8\ub2e4:<\/p>\n
\uc0c1\ud0dc \uacf5\uac04 (S)<\/h3>\n
\uc0c1\ud0dc \uacf5\uac04\uc740 \uc5d0\uc774\uc804\ud2b8\uac00 \ucc98\ud560 \uc218 \uc788\ub294 \ubaa8\ub4e0 \uc0c1\ud0dc\uc758 \uc9d1\ud569\uc785\ub2c8\ub2e4. \uc608\ub97c \ub4e4\uc5b4, \ubc14\ub451 \uac8c\uc784\uc758 \uc0c1\ud0dc \uacf5\uac04\uc740 \ubaa8\ub4e0 \uac00\ub2a5\ud55c \ubc14\ub451\ud310\uc758 \uc0c1\ud0dc\uac00 \ub420 \uc218 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n
\ud589\ub3d9 \uacf5\uac04 (A)<\/h3>\n
\ud589\ub3d9 \uacf5\uac04\uc740 \uc5d0\uc774\uc804\ud2b8\uc758 \uc0c1\ud0dc\uc5d0 \ub530\ub77c \uc120\ud0dd\ud560 \uc218 \uc788\ub294 \ubaa8\ub4e0 \ud589\ub3d9\uc744 \ud3ec\ud568\ud569\ub2c8\ub2e4. \uc608\ub97c \ub4e4\uc5b4, \ubc14\ub451 \uac8c\uc784\uc5d0\uc11c \uc5d0\uc774\uc804\ud2b8\ub294 \ud2b9\uc815 \uc704\uce58\uc5d0 \ub3cc\uc744 \ub193\ub294 \ud589\ub3d9\uc744 \ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n
\uc804\uc774 \ud655\ub960 (P)<\/h3>\n
\uc804\uc774 \ud655\ub960\uc740 \ud604\uc7ac \uc0c1\ud0dc\uc640 \uc120\ud0dd\ud55c \ud589\ub3d9\uc5d0 \ub530\ub77c \ub2e4\uc74c \uc0c1\ud0dc\ub85c \uc804\uc774\ub420 \ud655\ub960\uc744 \ub098\ud0c0\ub0c5\ub2c8\ub2e4. \uc774\ub294 \uc218\ud559\uc801\uc73c\ub85c \ub2e4\uc74c\uacfc \uac19\uc774 \ud45c\ud604\ub429\ub2c8\ub2e4:<\/p>\n
P(s', r | s, a)<\/code><\/pre>\n\uc5ec\uae30\uc11c, s'<\/code>\ub294 \ub2e4\uc74c \uc0c1\ud0dc, r<\/code>\ub294 \ubcf4\uc0c1, s<\/code>\ub294 \ud604\uc7ac \uc0c1\ud0dc, a<\/code>\ub294 \uc120\ud0dd\ud55c \ud589\ub3d9\uc785\ub2c8\ub2e4.<\/p>\n\ubcf4\uc0c1 \ud568\uc218 (R)<\/h3>\n
\ubcf4\uc0c1 \ud568\uc218\ub294 \uc5d0\uc774\uc804\ud2b8\uac00 \ud2b9\uc815 \uc0c1\ud0dc\uc5d0\uc11c \ud2b9\uc815 \ud589\ub3d9\uc744 \uc120\ud0dd\ud588\uc744 \ub54c \uc8fc\uc5b4\uc9c4 \ubcf4\uc0c1\uc744 \ub098\ud0c0\ub0c5\ub2c8\ub2e4. \ubcf4\uc0c1\uc740 \uc5d0\uc774\uc804\ud2b8\uc758 \ubaa9\ud45c\ub97c \uc815\uc758\ud558\ub294 \uc911\uc694\ud55c \uc694\uc18c\uc785\ub2c8\ub2e4.<\/p>\n
\ud560\uc778\uc728 (\u03b3)<\/h3>\n
\ud560\uc778\uc728 \u03b3 (0 \u2264 \u03b3 < 1)<\/code>\uc740 \ubbf8\ub798\uc758 \ubcf4\uc0c1\uc5d0 \ub300\ud55c \ud604\uc7ac \uac00\uce58\uc758 \uc601\ud5a5\uc744 \ubc18\uc601\ud569\ub2c8\ub2e4. \u03b3<\/code>\uac00 0\uc5d0 \uac00\uae4c\uc6b8\uc218\ub85d \uc5d0\uc774\uc804\ud2b8\ub294 \uac00\uae4c\uc6b4 \ubbf8\ub798\uc758 \ubcf4\uc0c1\uc5d0 \ub354 \uc911\uc810\uc744 \ub450\uace0, 1\uc5d0 \uac00\uae4c\uc6b8\uc218\ub85d \uba3c \ubbf8\ub798\uc758 \ubcf4\uc0c1\uc5d0 \ub354 \uc911\uc810\uc744 \ub461\ub2c8\ub2e4.<\/p>\n3. MDP\uc758 \uc608\uc81c<\/h2>\n
\uc774\uc81c MDP\uc758 \uac1c\ub150\uc744 \uc774\ud574\ud588\uc73c\ubbc0\ub85c \uc608\uc81c\ub97c \ud1b5\ud574 \uc774\ub97c \uac15\ud654 \ud559\uc2b5 \ubb38\uc81c\uc5d0 \uc5b4\ub5bb\uac8c \uc801\uc6a9\ud560 \uc218 \uc788\ub294\uc9c0 \uc0b4\ud3b4\ubcf4\uaca0\uc2b5\ub2c8\ub2e4. \ub2e4\uc74c\uc5d0\ub294 \uac04\ub2e8\ud55c MDP \uc608\uc81c\ub97c \ud1b5\ud574 \ud6c8\ub828\ub41c \uac15\ud654 \ud559\uc2b5 \uc5d0\uc774\uc804\ud2b8\ub97c \ub9cc\ub4e4 \uac83\uc785\ub2c8\ub2e4.<\/p>\n
3.1 \uac04\ub2e8\ud55c \uadf8\ub9ac\ub4dc \uc6d4\ub4dc \uc608\uc81c<\/h3>\n
\uadf8\ub9ac\ub4dc \uc6d4\ub4dc\ub294 4×4 \uaca9\uc790\ub85c \uad6c\uc131\ub41c \uc138\uacc4\ub97c \ubaa8\ub378\ub9c1\ud55c \uac83\uc785\ub2c8\ub2e4. \uc5d0\uc774\uc804\ud2b8\ub294 \uac01 \uaca9\uc790\uc5d0 \uc704\uce58\ud558\uba70, \ud2b9\uc815 \ud589\ub3d9(\uc704, \uc544\ub798, \uc67c\ucabd, \uc624\ub978\ucabd)\uc744 \ud1b5\ud574 \uc774\ub3d9\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4. \uc5d0\uc774\uc804\ud2b8\uc758 \ubaa9\ud45c\ub294 \uc624\ub978\ucabd \ud558\ub2e8 \uad6c\uc5ed(\ubaa9\ud45c \uc704\uce58)\uc5d0 \ub3c4\ub2ec\ud558\ub294 \uac83\uc785\ub2c8\ub2e4.<\/p>\n
\uc0c1\ud0dc \ubc0f \ud589\ub3d9 \uc815\uc758<\/h4>\n
\uc774 \uadf8\ub9ac\ub4dc \uc6d4\ub4dc\uc5d0\uc11c:<\/p>\n
\n- \uc0c1\ud0dc: 0\uc5d0\uc11c 15\uae4c\uc9c0\uc758 \uc22b\uc790\ub85c \uac01 \uaca9\uc790\ub97c \ud45c\ud604 (4×4 \uaca9\uc790)<\/li>\n
- \ud589\ub3d9: \uc704(0), \uc544\ub798(1), \uc67c\ucabd(2), \uc624\ub978\ucabd(3)<\/li>\n<\/ul>\n
\ubcf4\uc0c1 \uc815\uc758<\/h4>\n
\uc5d0\uc774\uc804\ud2b8\uac00 \ubaa9\ud45c \uc0c1\ud0dc\uc5d0 \ub3c4\ub2ec\ud558\uba74 \ubcf4\uc0c1 +1\uc744 \uc8fc\uace0, \ub2e4\ub978 \uc0c1\ud0dc\uc5d0\uc11c\ub294 0\uc744 \uc90d\ub2c8\ub2e4.<\/p>\n
4. \ud30c\uc774\ud1a0\uce58\ub85c MDP \uad6c\ud604\ud558\uae30<\/h2>\n
\uc774\uc81c \ud30c\uc774\ud1a0\uce58\ub97c \uc0ac\uc6a9\ud558\uc5ec \uac15\ud654 \ud559\uc2b5 \uc5d0\uc774\uc804\ud2b8\ub97c \uad6c\ud604\ud574 \ubcf4\uaca0\uc2b5\ub2c8\ub2e4. \uae30\ubcf8\uc801\uc73c\ub85c Q-\ub7ec\ub2dd \uc54c\uace0\ub9ac\uc998\uc744 \uc0ac\uc6a9\ud560 \uc608\uc815\uc785\ub2c8\ub2e4.<\/p>\n
4.1 \ud658\uacbd \ucd08\uae30\ud654<\/h3>\n
\uc6b0\uc120, \uadf8\ub9ac\ub4dc \uc6d4\ub4dc\ub97c \uc0dd\uc131\ud558\ub294 \ud074\ub798\uc2a4\ub97c \uc815\uc758\ud569\uc2dc\ub2e4:<\/p>\n
import numpy as np\n\nclass GridWorld:\n def __init__(self, grid_size=4):\n self.grid_size = grid_size\n self.state = 0\n self.goal_state = grid_size * grid_size - 1\n self.actions = [0, 1, 2, 3] # \uc704, \uc544\ub798, \uc67c\ucabd, \uc624\ub978\ucabd\n self.rewards = np.zeros((grid_size * grid_size,))\n self.rewards[self.goal_state] = 1 # \ubaa9\ud45c \uc9c0\uc810\uc5d0 \ub3c4\ub2ec \uc2dc\uc758 \ubcf4\uc0c1\n\n def reset(self):\n self.state = 0 # \uc2dc\uc791 \uc0c1\ud0dc\n return self.state\n\n def step(self, action):\n x, y = divmod(self.state, self.grid_size)\n if action == 0 and x > 0: # \uc704\n x -= 1\n elif action == 1 and x < self.grid_size - 1: # \uc544\ub798\n x += 1\n elif action == 2 and y > 0: # \uc67c\ucabd\n y -= 1\n elif action == 3 and y < self.grid_size - 1: # \uc624\ub978\ucabd\n y += 1\n self.state = x * self.grid_size + y\n return self.state, self.rewards[self.state]\n<\/code><\/pre>\n4.2 Q-\ub7ec\ub2dd \uc54c\uace0\ub9ac\uc998 \uad6c\ud604<\/h3>\n
Q-\ub7ec\ub2dd\uc744 \ud1b5\ud574 \uc5d0\uc774\uc804\ud2b8\ub97c \ud6c8\ub828\uc2dc\ud0a4\uaca0\uc2b5\ub2c8\uc2b5\ub2c8\ub2e4. \ub2e4\uc74c\uc740 Q-\ub7ec\ub2dd \uc54c\uace0\ub9ac\uc998\uc744 \uad6c\ud604\ud558\ub294 \ucf54\ub4dc\uc785\ub2c8\ub2e4:<\/p>\n
import torch\nimport torch.nn as nn\nimport torch.optim as optim\n\nclass QNetwork(nn.Module):\n def __init__(self, state_size, action_size):\n super(QNetwork, self).__init__()\n self.fc1 = nn.Linear(state_size, 24)\n self.fc2 = nn.Linear(24, 24)\n self.fc3 = nn.Linear(24, action_size)\n\n def forward(self, x):\n x = torch.relu(self.fc1(x))\n x = torch.relu(self.fc2(x))\n return self.fc3(x)\n\ndef train_agent(episodes, max_steps):\n env = GridWorld()\n state_size = env.grid_size * env.grid_size\n action_size = len(env.actions)\n \n q_network = QNetwork(state_size, action_size)\n optimizer = optim.Adam(q_network.parameters(), lr=0.001)\n criterion = nn.MSELoss()\n\n for episode in range(episodes):\n state = env.reset()\n total_reward = 0\n for step in range(max_steps):\n state_tensor = torch.eye(state_size)[state]\n q_values = q_network(state_tensor)\n \n action = np.argmax(q_values.detach().numpy()) # epsilon-greedy \uc815\ucc45\n next_state, reward = env.step(action)\n total_reward += reward\n \n next_state_tensor = torch.eye(state_size)[next_state]\n target = reward + 0.99 * torch.max(q_network(next_state_tensor)).detach()\n loss = criterion(q_values[action], target)\n \n optimizer.zero_grad()\n loss.backward()\n optimizer.step()\n\n if next_state == env.goal_state:\n break\n \n state = next_state\n print(f\"Episode {episode+1}: Total Reward = {total_reward}\")\n<\/code><\/pre>\n5. \ub9c8\ubb34\ub9ac<\/h2>\n
\uc774\ubc88 \ud3ec\uc2a4\ud305\uc5d0\uc11c\ub294 \ub9c8\ub974\ucf54\ud504 \uacb0\uc815 \uacfc\uc815(MDP)\uc758 \uac1c\ub150\uacfc \uc774\ub97c \ud30c\uc774\ud1a0\uce58\ub85c \uad6c\ud604\ud558\ub294 \ubc29\ubc95\uc5d0 \ub300\ud574 \uc54c\uc544\ubcf4\uc558\uc2b5\ub2c8\ub2e4. MDP\ub294 \uac15\ud654 \ud559\uc2b5\uc758 \uae30\ucd08\uac00 \ub418\ub294 \uc911\uc694\ud55c \ud504\ub808\uc784\uc6cc\ud06c\uc774\uba70, \uc2e4\uc81c \uac15\ud654 \ud559\uc2b5 \ubb38\uc81c\ub97c \ud574\uacb0\ud558\ub294 \ub370 \uc788\uc5b4 \ud544\uc218\uc801\uc73c\ub85c \uc774\ud574\ud574\uc57c \ud560 \uac1c\ub150\uc785\ub2c8\ub2e4. \uc5f0\uc2b5\uc744 \ud1b5\ud574 MDP\uc640 \uac15\ud654 \ud559\uc2b5\uc744 \ub354 \uae4a\uc774 \uc774\ud574\ud558\uae30\ub97c \ubc14\ub78d\ub2c8\ub2e4.<\/p>\n
\ucd94\uac00\uc801\uc73c\ub85c, \ub354 \ubcf5\uc7a1\ud55c MDP \ubb38\uc81c\uc640 \ud559\uc2b5 \uc54c\uace0\ub9ac\uc998\ub4e4\uc744 \ub2e4\ub8e8\uc5b4 \ubcf4\uc2dc\uae30\ub97c \uad8c\uc7a5\ud569\ub2c8\ub2e4. \ud30c\uc774\ud1a0\uce58\uc640 \uac19\uc740 \ub3c4\uad6c\ub97c \uc0ac\uc6a9\ud558\uba74\uc11c \ub2e4\uc591\ud55c \ud658\uacbd\uc744 \uad6c\ud604\ud558\uace0, \uc5d0\uc774\uc804\ud2b8\ub97c \ud6c8\ub828\uc2dc\ud0a4\uba70, \uc790\uc2e0\ub9cc\uc758 \uac15\ud654 \ud559\uc2b5 \ubaa8\ub378\uc744 \ub9cc\ub4e4\uc5b4 \ubcf4\uc2dc\uae30 \ubc14\ub78d\ub2c8\ub2e4.<\/p>\n