\uac15\ud654\ud559\uc2b5(RL)\uc5d0\uc11c\ub294 \uc5d0\uc774\uc804\ud2b8\uac00 \ud658\uacbd\uacfc \uc0c1\ud638\uc791\uc6a9\ud558\uba70 \ucd5c\uc801\uc758 \uc815\ucc45\uc744 \ud559\uc2b5\ud558\uac8c \ub429\ub2c8\ub2e4. \uadf8\ub7ec\ub098 \ubb38\uc81c \uacf5\uac04\uc774 \ub300\uaddc\ubaa8 \uc0c1\ud0dc \uacf5\uac04\uc744 \uac00\uc9c4 \uacbd\uc6b0, Q-\ud14c\uc774\ube14 \uae30\ubc18\uc758 \uc804\ud1b5\uc801\uc778 \ubc29\ubc95\uc73c\ub85c\ub294 \ud544\ud130\ub9c1, \uba54\ubaa8\ub9ac \uc694\uad6c \uc0ac\ud56d, \uadf8\ub9ac\uace0 \uc77c\ubc18\ud654 \ubb38\uc81c\ub85c \uc778\ud574 \ud55c\uacc4\uc5d0 \ubd80\ub52a\ud788\uac8c \ub429\ub2c8\ub2e4. \uc774\ub7ec\ud55c \uacbd\uc6b0, \ud568\uc218 \uadfc\uc0ac(Function Approximation) \uae30\ubc95\uc744 \uc774\uc6a9\ud558\uc5ec \uc0c1\ud0dc \uacf5\uac04\uc744 \uc555\ucd95\ud558\uace0, \uc77c\ubc18\ud654\ub97c \ud1b5\ud574 \ud559\uc2b5 \ud6a8\uc728\uc744 \ub192\uc774\uace0, \uc624\ubc84\ud53c\ud305\uc744 \ubc29\uc9c0\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n
\ud568\uc218 \uadfc\uc0ac\ub780 \uc0c1\ud0dc \ub610\ub294 \uc0c1\ud0dc-\ud589\ub3d9 \uac12 \ud568\uc218\ub97c \uadfc\uc0ac\ud558\uae30 \uc704\ud574 \ub2e4\uc591\ud55c \ud568\uc218 \ud615\ud0dc(\uc608: \uc120\ud615 \ud568\uc218, \ube44\uc120\ud615 \ud568\uc218)\ub97c \uc0ac\uc6a9\ud558\ub294 \uae30\ubc95\uc785\ub2c8\ub2e4. \ub354 \ub098\uc544\uac00, \ube44\uc5b4 \uc788\ub294 Q-\ud14c\uc774\ube14 \ub300\uc2e0 \uc801\uc808\ud55c \ud568\uc218\ub85c \ud45c\ud604\ud558\uc5ec \uba54\ubaa8\ub9ac\uc640 \ucc98\ub9ac \uc2dc\uac04\uc774 \uc808\uc57d\ub420 \uc218 \uc788\uc2b5\ub2c8\ub2e4. \uc774\ub7ec\ud55c \uae30\ubc95\uc740 \ub300\uaddc\ubaa8 \ub610\ub294 \uc5f0\uc18d \uc0c1\ud0dc \uacf5\uac04\uc5d0\uc11c \ud2b9\ud788 \uc720\uc6a9\ud569\ub2c8\ub2e4.<\/p>\n
\uc120\ud615 \ud568\uc218 \uadfc\uc0ac\ub294 \uac00\uc7a5 \ub2e8\uc21c\ud55c \ud615\ud0dc\ub85c, \uc8fc\uc5b4\uc9c4 \uc785\ub825\uc5d0 \ub300\ud574 \uac00\uc911\uce58\ub97c \uacf1\ud558\uace0 \uadf8 \uacb0\uacfc\ub97c \ub354\ud558\uc5ec \ucd5c\uc885 \ucd9c\ub825\uc744 \uacc4\uc0b0\ud569\ub2c8\ub2e4. \uc218\uc2dd\uc73c\ub85c \ud45c\ud604\ud558\uba74 \ub2e4\uc74c\uacfc \uac19\uc2b5\ub2c8\ub2e4:<\/p>\n
Q(s, a) = \u03b80<\/sub> + \u03b81<\/sub> * f1<\/sub>(s, a) + \u03b82<\/sub> * f2<\/sub>(s, a) + … + \u03b8N<\/sub> * fN<\/sub>(s, a)<\/p>\n \uc5ec\uae30\uc11c \u03b8i<\/sub>\ub294 \uac00\uc911\uce58(coefficient)\uc774\uace0, fi<\/sub>\ub294 \ud2b9\uc9d5 \ud568\uc218(feature function)\uc785\ub2c8\ub2e4.<\/p>\n \ube44\uc120\ud615 \ud568\uc218 \uadfc\uc0ac\ub294 \uc2e0\uacbd\ub9dd\uacfc \uac19\uc740 \uad6c\uc870\ub97c \ud1b5\ud574 \ubcf5\uc7a1\ud55c \ud589\ub3d9\uc744 \uadfc\uc0ac\ud558\ub294 \ubc29\ubc95\uc785\ub2c8\ub2e4. \uc2ec\uce35 \uc2e0\uacbd\ub9dd\uc740 \ud2b9\ud788 \uac15\ud654 \ud559\uc2b5\uc5d0\uc11c \ud070 \uc131\uacfc\ub97c \ubcf4\uc5ec\uc8fc\uc5c8\uc73c\uba70, \uc5ec\uae30\uc11c Q-\uac12\uc744 \uadfc\uc0ac\ud558\ub294 \uc8fc\uc694 \uad6c\uc131 \uc694\uc18c\ub85c \uc791\uc6a9\ud569\ub2c8\ub2e4.<\/p>\n \ud568\uc218 \uadfc\uc0ac\ub97c \ud1b5\ud574 \ud559\uc2b5\ud558\ub294 \uacbd\uc6b0, \uc77c\ubc18\ud654(Generalization)\uc640 \uc624\ubc84\ud53c\ud305(Overfitting) \ubb38\uc81c\uc5d0 \uc8fc\ubaa9\ud574\uc57c \ud569\ub2c8\ub2e4. \uc77c\ubc18\ud654\ub294 \ubaa8\ub378\uc774 \ubcf4\uc9c0 \ubabb\ud55c \ub370\uc774\ud130\uc5d0 \ub300\ud574\uc11c\ub3c4 \uc798 \uc608\uce21\ud560 \uc218 \uc788\ub3c4\ub85d \ud558\ub294 \uac83\uc774\uba70, \uc624\ubc84\ud53c\ud305\uc740 \ud6c8\ub828 \ub370\uc774\ud130\uc5d0 \ub108\ubb34 \ud2b9\ud654\ub418\uc5b4 \uc2e4\uc81c \ub370\uc774\ud130\uc5d0 \ub300\ud55c \uc131\ub2a5\uc744 \ub5a8\uc5b4\ub728\ub9ac\ub294 \ud604\uc0c1\uc785\ub2c8\ub2e4.<\/p>\n \uc77c\ubc18\ud654 \ubb38\uc81c\ub97c \ud574\uacb0\ud558\uae30 \uc704\ud55c \uba87 \uac00\uc9c0 \uae30\ubc95\uc774 \uc788\uc2b5\ub2c8\ub2e4:<\/p>\n \uc624\ubc84\ud53c\ud305\uc744 \ubc29\uc9c0\ud558\uae30 \uc704\ud55c \uae30\uc220\ub85c\ub294 \ub2e4\uc74c\uacfc \uac19\uc740 \ubc29\ubc95\ub4e4\uc774 \uc0ac\uc6a9\ub429\ub2c8\ub2e4:<\/p>\n \uac15\ud654\ud559\uc2b5\uc5d0\uc11c \ub300\uaddc\ubaa8 \uc0c1\ud0dc \uacf5\uac04 \ubb38\uc81c\ub97c \ud574\uacb0\ud558\uae30 \uc704\ud574 \ud568\uc218 \uadfc\uc0ac\ub97c \uc0ac\uc6a9\ud558\ub294 \uad6c\uccb4\uc801\uc778 \ubc29\ubc95\uc744 \ub2e4\ub8f9\ub2c8\ub2e4. \uc774\ub97c \uc704\ud574 OpenAI Gym\uc758 CartPole \uc608\uc81c\ub97c \ud1b5\ud574 \uc124\uba85\ud569\ub2c8\ub2e4.<\/p>\n \ud568\uc218 \uadfc\uc0ac\ub97c \ud65c\uc6a9\ud55c \ub300\uaddc\ubaa8 \uc0c1\ud0dc \uacf5\uac04 \ucc98\ub9ac\ub294 \uac15\ud654\ud559\uc2b5\uc5d0\uc11c \ub9e4\uc6b0 \uc911\uc694\ud55c \uae30\uc220\uc785\ub2c8\ub2e4. \uc62c\ubc14\ub978 \uc77c\ubc18\ud654\uc640 \uc624\ubc84\ud53c\ud305 \ubc29\uc9c0 \uae30\ubc95\uc744 \uc0ac\uc6a9\ud558\uba74, \ud6a8\uc728\uc801\uc778 \ud559\uc2b5\uacfc \uc131\ub2a5 \ud5a5\uc0c1\uc5d0 \uae30\uc5ec\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4. \uc774 \uae00\uc5d0\uc11c \ub2e4\ub8ec \uae30\ubc95\ub4e4\uc744 \ud1b5\ud574 \ub354 \ubcf5\uc7a1\ud55c \uac15\ud654\ud559\uc2b5 \ubb38\uc81c\ub97c \ud574\uacb0\ud560 \uc218 \uc788\uae30\ub97c \ubc14\ub78d\ub2c8\ub2e4.<\/p>\n2.2 \ube44\uc120\ud615 \ud568\uc218 \uadfc\uc0ac<\/h3>\n
3. \uc77c\ubc18\ud654\uc640 \uc624\ubc84\ud53c\ud305<\/h2>\n
3.1 \uc77c\ubc18\ud654 \uae30\ubc95<\/h3>\n
\n
3.2 \uc624\ubc84\ud53c\ud305 \ubc29\uc9c0 \uae30\ubc95<\/h3>\n
\n
4. \ud568\uc218 \uadfc\uc0ac\ub97c \ud1b5\ud55c \ub300\uaddc\ubaa8 \uc0c1\ud0dc \uacf5\uac04 \ucc98\ub9ac<\/h2>\n
4.1 \uc608\uc81c \ucf54\ub4dc<\/h3>\n
import numpy as np\nimport gym\nfrom keras.models import Sequential\nfrom keras.layers import Dense\nfrom keras.optimizers import Adam\n\nclass DQNAgent:\n def __init__(self, state_size, action_size):\n self.state_size = state_size\n self.action_size = action_size\n self.memory = []\n self.gamma = 0.95 # \ud560\uc778\uc728\n self.epsilon = 1.0 # \ud0d0\uc0ac \ube44\uc728\n self.epsilon_min = 0.01\n self.epsilon_decay = 0.995\n self.model = self._build_model()\n\n def _build_model(self):\n model = Sequential()\n model.add(Dense(24, input_dim=self.state_size, activation='relu'))\n model.add(Dense(24, activation='relu'))\n model.add(Dense(self.action_size, activation='linear'))\n model.compile(loss='mse', optimizer=Adam(lr=0.001))\n return model\n\n def remember(self, state, action, reward, next_state, done):\n self.memory.append((state, action, reward, next_state, done))\n\n def act(self, state):\n if np.random.rand() <= self.epsilon:\n return np.random.choice(self.action_size)\n act_values = self.model.predict(state)\n return np.argmax(act_values[0]) # \ud589\ub3d9 \uc120\ud0dd\n\n def replay(self, batch_size):\n minibatch = np.random.choice(self.memory, batch_size)\n for state, action, reward, next_state, done in minibatch:\n target = reward\n if not done:\n target = (reward + self.gamma * np.amax(self.model.predict(next_state)[0]))\n target_f = self.model.predict(state)\n target_f[0][action] = target\n self.model.fit(state, target_f, epochs=1, verbose=0)\n if self.epsilon > self.epsilon_min:\n self.epsilon *= self.epsilon_decay\n\n# \ud658\uacbd \uc124\uc815\nenv = gym.make('CartPole-v1')\nstate_size = env.observation_space.shape[0]\naction_size = env.action_space.n\nagent = DQNAgent(state_size, action_size)\nepisodes = 1000\n\n# \uc5d0\uc774\uc804\ud2b8 \ud6c8\ub828\nfor e in range(episodes):\n state = env.reset()\n state = np.reshape(state, [1, state_size])\n for time in range(500):\n action = agent.act(state)\n next_state, reward, done, _ = env.step(action)\n reward = reward if not done else -10\n next_state = np.reshape(next_state, [1, state_size])\n agent.remember(state, action, reward, next_state, done)\n state = next_state\n if done:\n print(f\"episode: {e + 1}\/{episodes}, score: {time}\")\n break\n if len(agent.memory) > 32:\n agent.replay(32)<\/code><\/pre>\n5. \uacb0\ub860<\/h2>\n