\uac15\ud654 \ud559\uc2b5(Reinforcement Learning)\uc740 \uae30\uacc4 \ud559\uc2b5\uc758 \ud55c \ubd84\uc57c\ub85c, \uc5d0\uc774\uc804\ud2b8\uac00 \ud658\uacbd\uacfc \uc0c1\ud638\uc791\uc6a9\ud558\uba74\uc11c \ubcf4\uc0c1\uc744 \ucd5c\ub300\ud654\ud558\uae30 \uc704\ud55c \uc815\ucc45\uc744 \ud559\uc2b5\ud558\ub294 \ubc29\ubc95\uc785\ub2c8\ub2e4. \uac15\ud654 \ud559\uc2b5\uc758 \uc5ec\ub7ec \uac00\uc9c0 \uc54c\uace0\ub9ac\uc998 \uc911\uc5d0\uc11c \uac00\uc7a5 \uc720\uba85\ud55c \ub450 \uac00\uc9c0\ub294 Q-\ub7ec\ub2dd(Q-Learning)\uacfc SARSA(State-Action-Reward-State-Action)\uc785\ub2c8\ub2e4. \uc774 \uac15\uc88c\uc5d0\uc11c\ub294 \uc774 \ub450 \uc54c\uace0\ub9ac\uc998\uc758 \uc6d0\ub9ac, \ub3d9\uc791 \ubc29\uc2dd, \uadf8\ub9ac\uace0 \ucc28\uc774\uc810\uc5d0 \ub300\ud574 \uc790\uc138\ud788 \uc124\uba85\ud558\uaca0\uc2b5\ub2c8\ub2e4.<\/p>\n
Q-\ub7ec\ub2dd\uc740 \uc815\ucc45\uc5d0 \ub3c5\ub9bd\uc801\uc778 \uac00\uce58 \uae30\ubc18 \ubc29\ubc95\ub860\uc73c\ub85c, \uc5d0\uc774\uc804\ud2b8\uac00 \uac01 \uc0c1\ud0dc\uc5d0\uc11c \uc5b4\ub5a4 \ud589\ub3d9\uc744 \ucde8\ud588\uc744 \ub54c \uae30\ub300\ud560 \uc218 \uc788\ub294 \ubcf4\uc0c1\uc744 \ucd94\uc815\ud569\ub2c8\ub2e4. Q-\ub7ec\ub2dd\uc758 \ud575\uc2ec\uc740 Q-\uac12(Q-value)\uc744 \uc5c5\ub370\uc774\ud2b8\ud558\uc5ec \ucd5c\uc801\uc758 \uc815\ucc45\uc744 \ud559\uc2b5\ud558\ub294 \uac83\uc785\ub2c8\ub2e4.<\/p>\n
Q-\ub7ec\ub2dd \uc54c\uace0\ub9ac\uc998\uc740 \ub2e4\uc74c\uacfc \uac19\uc740 \uad6c\uc870\ub97c \uac00\uc9c0\uace0 \uc788\uc2b5\ub2c8\ub2e4:<\/p>\n
\n Q(s, a) \u2190 Q(s, a) + \u03b1[R + \u03b3 max Q(s', a') - Q(s, a)]
\n <\/code>\n<\/li>\n<\/ul>\n<\/ol>\nQ-\uac12 \uc5c5\ub370\uc774\ud2b8 \uacfc\uc815 \uc124\uba85<\/h3>\n
Q-\uac12 \uc5c5\ub370\uc774\ud2b8 \uc2dd\uc5d0\uc11c \uac01 \uc694\uc18c\uc758 \uc758\ubbf8\ub294 \ub2e4\uc74c\uacfc \uac19\uc2b5\ub2c8\ub2e4:<\/p>\n
\nQ(s, a)<\/code>: \ud604\uc7ac \uc0c1\ud0dc s\uc5d0\uc11c \ud589\ub3d9 a\ub97c \ucde8\ud588\uc744 \ub54c\uc758 Q-\uac12.<\/li>\n\u03b1<\/code>: \ud559\uc2b5\ub960(0 < \u03b1 \u2264 1), \uc0c8\ub85c\uc6b4 \uc815\ubcf4\uc5d0 \ub300\ud55c \uac00\uc911\uce58.<\/li>\nR<\/code>: \ud604\uc7ac \uc0c1\ud0dc\uc5d0\uc11c \ud589\ub3d9 a\ub97c \ucde8\ud588\uc744 \ub54c\uc758 \ubcf4\uc0c1.<\/li>\n\u03b3<\/code>: \ud560\uc778\uc728(0 \u2264 \u03b3 < 1), \ubbf8\ub798 \ubcf4\uc0c1\uc5d0 \ub300\ud55c \uc911\uc694\uc131.<\/li>\nmax Q(s', a')<\/code>: \ub2e4\uc74c \uc0c1\ud0dc s’\uc5d0\uc11c \uac00\ub2a5\ud55c \ud589\ub3d9 a’ \uc911 \uac00\uc7a5 \ub192\uc740 Q-\uac12.<\/li>\n<\/ul>\nQ-\ub7ec\ub2dd \uc608\uc81c \ucf54\ub4dc<\/h3>\n\nimport numpy as np\nimport gym\n\n# \ud658\uacbd \uc0dd\uc131\nenv = gym.make('Taxi-v3')\n\n# Q-\ub7ec\ub2dd \ub9e4\uac1c\ubcc0\uc218\nalpha = 0.1 # \ud559\uc2b5\ub960\ngamma = 0.9 # \ud560\uc778\uc728\nepsilon = 0.1 # epsilon-greedy\ub97c \uc704\ud55c \ud0d0\ud5d8 \ube44\uc728\n\n# Q-\ud14c\uc774\ube14 \ucd08\uae30\ud654\nQ = np.zeros([env.observation_space.n, env.action_space.n])\n\n# \ud559\uc2b5 \uacfc\uc815\nfor episode in range(1000):\n state = env.reset()\n done = False\n \n while not done:\n # \ud0d0\ud5d8 \ub610\ub294 \uc774\uc6a9\n if np.random.rand() < epsilon:\n action = env.action_space.sample() # \ubb34\uc791\uc704 \ud589\ub3d9 \uc120\ud0dd\n else:\n action = np.argmax(Q[state]) # \ud604\uc7ac Q-\uac12 \uae30\ubc18 \ud589\ub3d9 \uc120\ud0dd\n \n # \ud658\uacbd\uc5d0\uc11c \ud589\ub3d9 \uc218\ud589\n next_state, reward, done, _ = env.step(action)\n \n # Q-\uac12 \uc5c5\ub370\uc774\ud2b8\n Q[state, action] += alpha * (reward + gamma * np.max(Q[next_state]) - Q[state, action])\n \n # \uc0c1\ud0dc \uc5c5\ub370\uc774\ud2b8\n state = next_state\n\nprint(\"Q-\ud14c\uc774\ube14:\\n\", Q)<\/code><\/pre>\n<\/div>\nSARSA(\uc0c1\ud0dc-\ud589\ub3d9-\ubcf4\uc0c1-\uc0c1\ud0dc-\ud589\ub3d9)<\/h2>\nSARSA\uc758 \uac1c\ub150<\/h3>\n
SARSA\ub294 Q-\ub7ec\ub2dd\uacfc \uc720\uc0ac\ud558\uc9c0\ub9cc, \uc815\ucc45\uc5d0 \uc758\uc874\ud558\ub294 \ubc29\ubc95\uc785\ub2c8\ub2e4. SARSA\ub294 \uc5d0\uc774\uc804\ud2b8\uc758 \uc2e4\uc81c \ud589\ub3d9\uc744 \uae30\ubc18\uc73c\ub85c Q-\uac12\uc744 \uc5c5\ub370\uc774\ud2b8\ud558\uc5ec, \ud604\uc7ac \uc815\ucc45\uc744 \ub530\ub77c \ud559\uc2b5\ud569\ub2c8\ub2e4.<\/p>\n
SARSA \uc54c\uace0\ub9ac\uc998\uc758 \uad6c\uc870<\/h3>\n
SARSA \uc54c\uace0\ub9ac\uc998\ub3c4 Q-\ub7ec\ub2dd\uacfc \uc720\uc0ac\ud55c \uad6c\uc870\ub97c \uac00\uc9c0\uace0 \uc788\uc9c0\ub9cc, \ud589\ub3d9 \uc120\ud0dd\uacfc Q-\uac12 \uc5c5\ub370\uc774\ud2b8\uc5d0\uc11c \ucc28\uc774\uac00 \uc788\uc2b5\ub2c8\ub2e4:<\/p>\n
\n- \ucd08\uae30\ud654: Q-\uac12 \ucd08\uae30\ud654 (Q(s, a) = 0 \ub610\ub294 \ubb34\uc791\uc704).<\/li>\n
- \ubc18\ubcf5: \uc544\ub798 \ub2e8\uacc4\ub4e4\uc744 \uc218\ud589\ud569\ub2c8\ub2e4:<\/li>\n
\n- \ud604\uc7ac \uc0c1\ud0dc\uc5d0\uc11c \ud589\ub3d9 \uc120\ud0dd (\u03b5-greedy \uc815\ucc45).<\/li>\n
- \ud589\ub3d9 \uc218\ud589 \ud6c4 \ubcf4\uc0c1 \ubc0f \ub2e4\uc74c \uc0c1\ud0dc\ub97c \uad00\ucc30.<\/li>\n
- \ub2e4\uc74c \uc0c1\ud0dc\uc5d0\uc11c \ub2e4\uc74c \ud589\ub3d9 \uc120\ud0dd.<\/li>\n
- Q-\uac12 \uc5c5\ub370\uc774\ud2b8:
\n
\n Q(s, a) \u2190 Q(s, a) + \u03b1[R + \u03b3 Q(s', a') - Q(s, a)]
\n <\/code>\n<\/li>\n<\/ul>\n<\/ol>\nSARSA\uc758 Q-\uac12 \uc5c5\ub370\uc774\ud2b8 \uacfc\uc815 \uc124\uba85<\/h3>\n
SARSA\uc5d0\uc11c Q-\uac12 \uc5c5\ub370\uc774\ud2b8 \uc2dd\uc740 \ub2e4\uc74c\uacfc \uac19\uc774 \ub420 \uc218 \uc788\uc2b5\ub2c8\ub2e4:<\/p>\n
\nQ(s, a)<\/code>: \ud604\uc7ac \uc0c1\ud0dc\uc640 \ud589\ub3d9\uc758 Q-\uac12.<\/li>\nR<\/code>: \ud589\ub3d9 \uc218\ud589 \ud6c4 \ubc1b\uc740 \ubcf4\uc0c1.<\/li>\nQ(s', a')<\/code>: \ub2e4\uc74c \uc0c1\ud0dc\uc5d0\uc11c \uc120\ud0dd\ud55c \ud589\ub3d9\uc758 Q-\uac12.<\/li>\n<\/ul>\nSARSA \uc608\uc81c \ucf54\ub4dc<\/h3>\n\nimport numpy as np\nimport gym\n\n# \ud658\uacbd \uc0dd\uc131\nenv = gym.make('Taxi-v3')\n\n# SARSA \ub9e4\uac1c\ubcc0\uc218\nalpha = 0.1 # \ud559\uc2b5\ub960\ngamma = 0.9 # \ud560\uc778\uc728\nepsilon = 0.1 # epsilon-greedy \ud0d0\ud5d8 \ube44\uc728\n\n# Q-\ud14c\uc774\ube14 \ucd08\uae30\ud654\nQ = np.zeros([env.observation_space.n, env.action_space.n])\n\n# \ud559\uc2b5 \uacfc\uc815\nfor episode in range(1000):\n state = env.reset()\n action = env.action_space.sample() # \uccab \ud589\ub3d9 \ubb34\uc791\uc704 \uc120\ud0dd\n done = False\n \n while not done:\n # \ud658\uacbd\uc5d0\uc11c \ud589\ub3d9 \uc218\ud589\n next_state, reward, done, _ = env.step(action)\n \n # \ub2e4\uc74c \uc0c1\ud0dc\uc5d0\uc11c\uc758 \ud589\ub3d9 \uc120\ud0dd\n next_action = env.action_space.sample() if np.random.rand() < epsilon else np.argmax(Q[next_state])\n \n # Q-\uac12 \uc5c5\ub370\uc774\ud2b8\n Q[state, action] += alpha * (reward + gamma * Q[next_state, next_action] - Q[state, action])\n \n # \uc0c1\ud0dc\uc640 \ud589\ub3d9 \uc5c5\ub370\uc774\ud2b8\n state = next_state\n action = next_action\n\nprint(\"SARSA\uc758 Q-\ud14c\uc774\ube14:\\n\", Q)<\/code><\/pre>\n<\/div>\nQ-\ub7ec\ub2dd\uacfc SARSA\uc758 \ucc28\uc774\uc810<\/h2>\n
Q-\ub7ec\ub2dd\uacfc SARSA\ub294 \uae30\ubcf8\uc801\uc778 \uad6c\uc870\ub294 \uc720\uc0ac\ud558\uc9c0\ub9cc, Q-\uac12 \uc5c5\ub370\uc774\ud2b8 \ubc29\uc2dd\uc5d0\uc11c \uc911\uc694\ud55c \ucc28\uc774\uac00 \uc788\uc2b5\ub2c8\ub2e4:<\/p>\n
\n- \uc815\ucc45\uc758 \uc758\uc874\uc131:<\/strong> Q-\ub7ec\ub2dd\uc740 \uc815\ucc45\uc5d0 \ubb34\uad00\ud558\uac8c \ud559\uc2b5\ud558\uace0, \ubbf8\ub798\uc758 \ucd5c\uc801 \ud589\ub3d9\uc744 \uace0\ub824\ud558\uc5ec \uc5c5\ub370\uc774\ud2b8\ud558\ub294 \ubc18\uba74, SARSA\ub294 \uc2e4\uc81c\ub85c \uc120\ud0dd\ub41c \ud589\ub3d9\uc744 \uae30\ubc18\uc73c\ub85c Q-\uac12\uc744 \uc5c5\ub370\uc774\ud2b8\ud569\ub2c8\ub2e4. \uc774\ub294 SARSA\uac00 \ud0d0\ud5d8 \ube44\uc728\uc774\ub098 \ud604\uc7ac \uc815\ucc45\uc758 \uc601\ud5a5\uc744 \ub354 \ub9ce\uc774 \ubc1b\ub294\ub2e4\ub294 \uac83\uc744 \uc758\ubbf8\ud569\ub2c8\ub2e4.<\/li>\n
- \ud0d0\ud5d8 \ub300 \uc774\uc6a9:<\/strong> Q-\ub7ec\ub2dd\uc5d0\uc11c\ub294 \ub2e4\uc74c \uc0c1\ud0dc\uc5d0\uc11c \uac00\ub2a5\ud55c \ucd5c\ub300 Q-\uac12\uc744 \uc0ac\uc6a9\ud558\ub294 \ubc18\uba74, SARSA\uc5d0\uc11c\ub294 \uc2e4\uc81c\ub85c \uc120\ud0dd\ub41c \ub2e4\uc74c \ud589\ub3d9\uc758 Q-\uac12\uc744 \uc0ac\uc6a9\ud569\ub2c8\ub2e4. \ub530\ub77c\uc11c Q-\ub7ec\ub2dd\uc740 \ub354 \uacf5\uaca9\uc801\uc778 \ubc29\ubc95\uc774\uba70, SARSA\ub294 \ub354 \ubcf4\uc218\uc801\uc778 \ubc29\ubc95\uc785\ub2c8\ub2e4.<\/li>\n
- \uc218\ub834\uc131:<\/strong> \ub450 \uc54c\uace0\ub9ac\uc998 \ubaa8\ub450 \uc801\uc808\ud55c \uc870\uac74 \ud558\uc5d0 \uc218\ub834\uc131\uc744 \uac00\uc9c0\uc9c0\ub9cc, Q-\ub7ec\ub2dd\uc740 \ucd5c\uc801 \uc815\ucc45\uc744 \ub354\uc6b1 \ube60\ub974\uac8c \ucc3e\ub294 \uacbd\ud5a5\uc774 \uc788\uc2b5\ub2c8\ub2e4. \ubc18\uba74, SARSA\ub294 \ubcf4\ub2e4 \uc548\uc815\uc801\uc778 \ud559\uc2b5\uc744 \uc81c\uacf5\ud569\ub2c8\ub2e4.<\/li>\n<\/ul>\n
\uacb0\ub860<\/h2>\n
Q-\ub7ec\ub2dd\uacfc SARSA\ub294 \uac15\ud654\ud559\uc2b5\uc5d0\uc11c \uc911\uc694\ud55c \ub450 \uac00\uc9c0 \uc54c\uace0\ub9ac\uc998\uc73c\ub85c, \uac01\uac01\uc758 \uc7a5\ub2e8\uc810\uc774 \uc788\uc2b5\ub2c8\ub2e4. Q-\ub7ec\ub2dd\uc740 \ubcf4\ub2e4 \ub192\uc740 \uc218\uc775\uc744 \ubaa9\ud45c\ub85c \ud55c \uacf5\uaca9\uc801\uc778 \uc811\uadfc \ubc29\uc2dd\uc744 \uc81c\uacf5\ud558\ub294 \ubc18\uba74, SARSA\ub294 \uc815\ucc45\uc5d0 \ub530\ub77c \uc548\uc815\uc801\uc778 \ud559\uc2b5\uc744 \uc81c\uacf5\ud569\ub2c8\ub2e4. \uc0ac\uc6a9\uc790\ub294 \uac01 \ubb38\uc81c\uc758 \ud2b9\uc131\uc5d0 \ub530\ub77c \uc801\uc808\ud55c \uc54c\uace0\ub9ac\uc998\uc744 \uc120\ud0dd\ud558\uc5ec \uc0ac\uc6a9\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n
\uc774 \uae00\uc744 \ud1b5\ud574 Q-\ub7ec\ub2dd\uacfc SARSA\uc758 \uae30\ubcf8 \uac1c\ub150\uacfc \ucf54\ub4dc \uc608\uc81c\ub97c \uc774\ud574\ud558\ub294 \ub370 \ub3c4\uc6c0\uc774 \ub418\uc168\uae30\ub97c \ubc14\ub78d\ub2c8\ub2e4. \ub2e4\uc74c \uac15\uc88c\uc5d0\uc11c\ub294 \ub354 \ubc1c\uc804\ub41c \uc54c\uace0\ub9ac\uc998\uc774\ub098 \ub525 \uac15\ud654\ud559\uc2b5\uc5d0 \ub300\ud574 \ub2e4\ub8f0 \uc608\uc815\uc785\ub2c8\ub2e4.<\/p>\n
<\/body><\/p>\n","protected":false},"excerpt":{"rendered":"