R:<\/strong> \ubcf4\uc0c1 \ud568\uc218 (Reward function)<\/li>\n<\/ul>\n1.2 \ubca8\ub9cc \ubc29\uc815\uc2dd<\/h3>\n
\ubca8\ub9cc \ubc29\uc815\uc2dd\uc740 \ud2b9\uc815 \uc0c1\ud0dc s<\/code>\uc5d0\uc11c \ucd5c\uc801\uc758 \ud589\ub3d9\uc744 \uc120\ud0dd\ud560 \ub54c, \ud604\uc7ac \uc0c1\ud0dc\uc758 \uac00\uce58\ub294 \ub2e4\uc74c\uc758 \uc2dd\uc73c\ub85c \ud45c\ud604\ub429\ub2c8\ub2e4:<\/p>\nV(s) = max_a [R(s,a) + \u03b3 * \u03a3 P(s'|s,a) * V(s')]<\/code><\/pre>\n\uc5ec\uae30\uc11c:<\/p>\n
\nV(s)<\/code>\ub294 \uc0c1\ud0dc s<\/code>\uc758 \uac00\uce58<\/li>\na<\/code>\ub294 \uac00\ub2a5\ud55c \ud589\ub3d9<\/li>\n\u03b3<\/code>\ub294 \ud560\uc778 \uc778\uc790 (0 \u2264 \u03b3 < 1)<\/li>\nP(s'|s,a)<\/code>\ub294 \uc0c1\ud0dc s<\/code>\uc5d0\uc11c \ud589\ub3d9 a<\/code>\ub97c \ucde8\ud588\uc744 \ub54c \ub2e4\uc74c \uc0c1\ud0dc s'<\/code>\ub85c \uc804\uc774\ub420 \ud655\ub960<\/li>\nR(s,a)<\/code>\ub294 \ud604\uc7ac \uc0c1\ud0dc\uc5d0\uc11c \ud589\ub3d9 a<\/code>\ub97c \ucde8\ud588\uc744 \ub54c\uc758 \ubcf4\uc0c1<\/li>\n<\/ul>\n2. \ubca8\ub9cc \ucd5c\uc801 \ubc29\uc815\uc2dd\uacfc \ub525\ub7ec\ub2dd<\/h2>\n
\ub525\ub7ec\ub2dd\uc744 \uac15\ud654\ud559\uc2b5\uc5d0 \uc811\ubaa9\ud560 \ub54c, \uc8fc\ub85c Q-\ub7ec\ub2dd(Q-learning)\uacfc \uac19\uc740 \uae30\ubc95\uc744 \uc0ac\uc6a9\ud558\uc5ec \ubca8\ub9cc \ubc29\uc815\uc2dd\uc744 \uadfc\uc0ac\ud654\ud569\ub2c8\ub2e4. \uc5ec\uae30\uc11c Q-\ud568\uc218\ub294 \ud2b9\uc815 \uc0c1\ud0dc\uc5d0\uc11c \ud2b9\uc815 \ud589\ub3d9\uc744 \ucde8\ud588\uc744 \ub54c \uae30\ub300\ub418\ub294 \ubcf4\uc0c1\uc744 \ub098\ud0c0\ub0c5\ub2c8\ub2e4.<\/p>\n
2.1 Q-\ud559\uc2b5\uc758 \ubca8\ub9cc \ubc29\uc815\uc2dd<\/h3>\n
Q-\ud559\uc2b5\uc758 \uacbd\uc6b0 \ubca8\ub9cc \ubc29\uc815\uc2dd\uc740 \ub2e4\uc74c\uacfc \uac19\uc774 \ud45c\ud604\ub429\ub2c8\ub2e4:<\/p>\n
Q(s,a) = R(s,a) + \u03b3 * max_a' Q(s',a')<\/code><\/pre>\n3. \ud30c\uc774\uc36c\uacfc \ud30c\uc774\ud1a0\uce58\ub85c \ubca8\ub9cc \ubc29\uc815\uc2dd \uad6c\ud604\ud558\uae30<\/h2>\n
\uc774\ubc88 \uc139\uc158\uc5d0\uc11c\ub294 \ud30c\uc774\ud1a0\uce58\ub97c \uc0ac\uc6a9\ud558\uc5ec \uac04\ub2e8\ud55c Q-\ud559\uc2b5 \uc5d0\uc774\uc804\ud2b8\ub97c \uad6c\ud604\ud558\ub294 \ubc29\ubc95\uc744 \uc0b4\ud3b4\ubcf4\uaca0\uc2b5\ub2c8\ub2e4.<\/p>\n
3.1 \ud658\uacbd \uc900\ube44\ud558\uae30<\/h3>\n
\uc6b0\uc120, \ud544\uc694\ud55c \ub77c\uc774\ube0c\ub7ec\ub9ac\ub97c \uc124\uce58\ud569\ub2c8\ub2e4. \ub2e4\uc74c\uacfc \uac19\uc740 \ub77c\uc774\ube0c\ub7ec\ub9ac\uac00 \ud544\uc694\ud569\ub2c8\ub2e4:<\/p>\n
pip install torch numpy gym<\/code><\/pre>\n3.2 Q-\ub124\ud2b8\uc6cc\ud06c \uc815\uc758\ud558\uae30<\/h3>\n
\ub2e4\uc74c\uc73c\ub85c Q-\ub124\ud2b8\uc6cc\ud06c\ub97c \uc815\uc758\ud569\ub2c8\ub2e4. \uc774\ub294 \ud30c\uc774\ud1a0\uce58\uc758 \uc2e0\uacbd\ub9dd(neural network)\uc744 \uc774\uc6a9\ud574 \uad6c\ud604\ud560 \uac83\uc785\ub2c8\ub2e4.<\/p>\n
import torch\nimport torch.nn as nn\nimport numpy as np\n\nclass QNetwork(nn.Module):\n def __init__(self, state_dim, action_dim):\n super(QNetwork, self).__init__()\n self.fc1 = nn.Linear(state_dim, 64)\n self.fc2 = nn.Linear(64, 64)\n self.fc3 = nn.Linear(64, action_dim)\n\n def forward(self, x):\n x = torch.relu(self.fc1(x))\n x = torch.relu(self.fc2(x))\n return self.fc3(x)<\/code><\/pre>\n3.3 \uc5d0\uc774\uc804\ud2b8 \ud074\ub798\uc2a4 \uc815\uc758\ud558\uae30<\/h3>\n
Q-\ud559\uc2b5 \uc54c\uace0\ub9ac\uc998\uc744 \uc218\ud589\ud560 \uc5d0\uc774\uc804\ud2b8 \ud074\ub798\uc2a4\ub97c \uc815\uc758\ud558\uaca0\uc2b5\ub2c8\ub2e4.<\/p>\n
class Agent:\n def __init__(self, state_dim, action_dim, learning_rate=0.001, gamma=0.99):\n self.action_dim = action_dim\n self.gamma = gamma\n self.q_network = QNetwork(state_dim, action_dim)\n self.optimizer = torch.optim.Adam(self.q_network.parameters(), lr=learning_rate)\n\n def choose_action(self, state, epsilon):\n if np.random.rand() < epsilon: # explore\n return np.random.choice(self.action_dim)\n else: # exploit\n state_tensor = torch.FloatTensor(state)\n with torch.no_grad():\n q_values = self.q_network(state_tensor)\n return torch.argmax(q_values).item()\n\n def learn(self, state, action, reward, next_state, done):\n state_tensor = torch.FloatTensor(state)\n next_state_tensor = torch.FloatTensor(next_state)\n\n q_values = self.q_network(state_tensor)\n target = reward + (1-done) * self.gamma * torch.max(self.q_network(next_state_tensor))\n\n loss = nn.MSELoss()(q_values[action], target)\n\n self.optimizer.zero_grad()\n loss.backward()\n self.optimizer.step()<\/code><\/pre>\n3.4 \ud6c8\ub828 \uacfc\uc815 \uc815\uc758\ud558\uae30<\/h3>\n
\uc774\uc81c \uc5d0\uc774\uc804\ud2b8\ub97c \ud6c8\ub828\uc2dc\ud0a4\ub294 \uacfc\uc815\uc744 \uc815\uc758\ud569\ub2c8\ub2e4. OpenAI\uc758 Gym \ub77c\uc774\ube0c\ub7ec\ub9ac\ub97c \uc0ac\uc6a9\ud574 \uac04\ub2e8\ud55c \ud658\uacbd\uc744 \uc124\uc815\ud569\ub2c8\ub2e4.<\/p>\n
import gym\n\ndef train_agent(episodes=1000):\n env = gym.make('CartPole-v1')\n agent = Agent(state_dim=4, action_dim=2)\n\n for episode in range(episodes):\n state = env.reset()\n done = False\n total_reward = 0\n epsilon = max(0.1, 1.0 - episode \/ 500) # epsilon-greedy \ucee4\ub2e4\ub780 \ubcc0\ub3d9\uc131\uc744 \uc8fc\uae30 \uc704\ud568\n\n while not done:\n action = agent.choose_action(state, epsilon)\n next_state, reward, done, _ = env.step(action)\n agent.learn(state, action, reward, next_state, done)\n state = next_state\n total_reward += reward\n\n print(f'Episode: {episode}, Total Reward: {total_reward}')\n\n env.close()\n\n# \ud6c8\ub828 \uc2dc\uc791\ntrain_agent()<\/code><\/pre>\n4. \uacb0\uacfc \ubd84\uc11d \ubc0f \uacb0\ub860<\/h2>\n
\ud6c8\ub828\uc774 \uc644\ub8cc\ub41c \ud6c4, \uc5d0\uc774\uc804\ud2b8\uac00 CartPole \ud658\uacbd\uc5d0\uc11c \uc798 \uc218\ud589\ud558\ub294\uc9c0 \uc2dc\uac01\ud654\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4. \uad50\uc721 \uacfc\uc815 \ub3d9\uc548, \uc5d0\uc774\uc804\ud2b8\uac00 \uc5b4\ub5bb\uac8c \ud589\ub3d9\ud558\uace0 \uc131\ub2a5\uc744 \uac1c\uc120\ud558\ub294\uc9c0\ub97c \uad00\ucc30\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4. \ubca8\ub9cc \ucd5c\uc801 \ubc29\uc815\uc2dd\uc744 \ud1b5\ud574 \uac15\uc870\ub41c \ucd5c\uc801\uc758 \uacbd\ub85c\ub97c \ub530\ub978\ub2e4\ub294 \uac1c\ub150\uc740 \ub525\ub7ec\ub2dd\uacfc \ud568\uaed8 \uc0ac\uc6a9\ub420 \ub54c \ub354\uc6b1 \uac15\ub825\ud569\ub2c8\ub2e4.<\/p>\n
\uc774\ubc88 \uac15\uc88c\uc5d0\uc11c\ub294 \ubca8\ub9cc \ucd5c\uc801 \ubc29\uc815\uc2dd\uc758 \uac1c\ub150\uc744 \uc774\ud574\ud558\uace0, \uc774\ub97c \ud30c\uc774\ud1a0\uce58\uc640 \ud568\uaed8 \ud65c\uc6a9\ud558\uc5ec Q-\ud559\uc2b5 \uc5d0\uc774\uc804\ud2b8\ub97c \uad6c\ud604\ud558\ub294 \uacfc\uc815\uc744 \uc0b4\ud3b4\ubcf4\uc558\uc2b5\ub2c8\ub2e4. \ubca8\ub9cc \ubc29\uc815\uc2dd\uc740 \uac15\ud654\ud559\uc2b5\uc758 \uadfc\ubcf8\uc801\uc778 \uc6d0\ub9ac\ub85c, \uc5ec\ub7ec \uc751\uc6a9 \ubd84\uc57c\uc5d0\uc11c \uc911\uc694\ud558\uac8c \ud65c\uc6a9\ub418\uace0 \uc788\uc2b5\ub2c8\ub2e4. \uc55e\uc73c\ub85c\uc758 \ub525\ub7ec\ub2dd \ubc0f \uac15\ud654\ud559\uc2b5 \uc5ec\uc815\uc5d0 \ud070 \ub3c4\uc6c0\uc774 \ub418\uae38 \ubc14\ub78d\ub2c8\ub2e4.<\/p>\n
<\/p>\n