\uac15\ud654\ud559\uc2b5(Deep Reinforcement Learning)\uc5d0\uc11c \uc5d0\uc774\uc804\ud2b8\ub294 \ud658\uacbd\uacfc \uc0c1\ud638\uc791\uc6a9\ud558\uba70 \ud559\uc2b5\ud569\ub2c8\ub2e4. \uc774 \uacfc\uc815\uc5d0\uc11c \uc5d0\uc774\uc804\ud2b8\ub294 \ubcf4\uc0c1\uc744 \ucd5c\ub300\ud654\ud558\uae30 \uc704\ud574 \ucd5c\uc801\uc758 \ud589\ub3d9\uc744 \uc120\ud0dd\ud574\uc57c \ud569\ub2c8\ub2e4. \uadf8\ub7ec\ub098, \uc5d0\uc774\uc804\ud2b8\uc758 \ud589\ub3d9\uacfc \ubcf4\uc0c1\uc740 \ub9e4\uc6b0 \ub2e4\uc591\ud558\uace0 \ubb34\uc791\uc704\uc801\uc77c \uc218 \uc788\uc73c\uba70, \uc774\ub85c \uc778\ud574 \ud45c\ubcf8(sample) \ud6a8\uc728\uc131\uc774 \ub5a8\uc5b4\uc9c0\uace0 \ud559\uc2b5\uc774 \ubc29\ud574\ubc1b\uc744 \uc218 \uc788\uc2b5\ub2c8\ub2e4. \uc774\ub97c \ud574\uacb0\ud558\uae30 \uc704\ud574 \uc0ac\uc6a9\ud558\ub294 \uae30\ubc95\uc774 \ubc14\ub85c \uacbd\ud5d8 \uc7ac\uc0dd(Experience Replay)<\/strong>\uc785\ub2c8\ub2e4.<\/p>\n \uacbd\ud5d8 \uc7ac\uc0dd\uc740 \uc5d0\uc774\uc804\ud2b8\uac00 \ud658\uacbd\uc5d0\uc11c \uc5bb\uc740 \uacfc\uac70\uc758 \uacbd\ud5d8\uc744 \uc800\uc7a5\ud558\uace0, \uc774 \uacbd\ud5d8\uc744 \ubbf8\ub2c8 \ubc30\uce58\ub85c \uc0d8\ud50c\ub9c1\ud558\uc5ec \ud559\uc2b5\ud558\ub294 \uae30\ubc95\uc785\ub2c8\ub2e4. \uc774 \ubc29\uc2dd\uc740 \ub2e4\uc74c\uacfc \uac19\uc740 \uc7a5\uc810\uc744 \uc81c\uacf5\ud569\ub2c8\ub2e4:<\/p>\n \uacbd\ud5d8 \uc7ac\uc0dd\uc740 \ub2e4\uc74c\uacfc \uac19\uc740 \ub450 \uac00\uc9c0 \uc8fc\uc694 \uad6c\uc131 \uc694\uc18c\ub85c \uc774\ub8e8\uc5b4\uc838 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n \uacbd\ud5d8 \uc800\uc7a5\uc18c\ub294 \uc5d0\uc774\uc804\ud2b8\uac00 \ud658\uacbd\uc5d0\uc11c \uc0c1\ud638\uc791\uc6a9 \ud558\uba74\uc11c \uc5bb\uc740 \uacbd\ud5d8(\uc0c1\ud0dc, \ud589\ub3d9, \ubcf4\uc0c1, \ub2e4\uc74c \uc0c1\ud0dc)\uc744 \uc800\uc7a5\ud558\ub294 \uacf3\uc785\ub2c8\ub2e4. \uc774 \uc800\uc7a5\uc18c\ub294 FIFO(\uc120\uc785\uc120\ucd9c) \uad6c\uc870\ub85c \uad6c\uc131\ub418\uc5b4, \uc0c8\ub85c\uc6b4 \uacbd\ud5d8\uc774 \ucd94\uac00\ub420 \ub54c \uc624\ub798\ub41c \uacbd\ud5d8\uc740 \uc0ad\uc81c\ub429\ub2c8\ub2e4.<\/p>\n \uacbd\ud5d8 \uc800\uc7a5\uc18c\uc5d0 \uc800\uc7a5\ub41c \uacbd\ud5d8\ub4e4 \uc911 \ubb34\uc791\uc704\ub85c \uc0d8\ud50c\uc744 \uc120\ud0dd\ud558\uc5ec \ubbf8\ub2c8 \ubc30\uce58\ub97c \uad6c\uc131\ud569\ub2c8\ub2e4. \uc120\ud0dd\ub41c \uacbd\ud5d8\ub4e4\uc740 \uc5d0\uc774\uc804\ud2b8\uc758 Q-\uac12\uc744 \uc5c5\ub370\uc774\ud2b8\ud558\ub294 \ub370 \uc0ac\uc6a9\ub429\ub2c8\ub2e4.<\/p>\n \uadf8\ub7fc \uacbd\ud5d8 \uc7ac\uc0dd\uc744 \ud30c\uc774\uc36c\uc73c\ub85c \uc5b4\ub5bb\uac8c \uad6c\ud604\ud558\ub294\uc9c0 \uc0b4\ud3b4\ubcf4\uaca0\uc2b5\ub2c8\ub2e4. \uc544\ub798\ub294 \uac04\ub2e8\ud55c \uacbd\ud5d8 \uc7ac\uc0dd \ud074\ub798\uc2a4\uc758 \uc608\uc81c\uc785\ub2c8\ub2e4.<\/p>\n \uc774 \ucf54\ub4dc\ub294 \ud30c\uc774\uc36c\uc758 \uc5d0\uc774\uc804\ud2b8\uac00 \ud658\uacbd\uacfc \uc0c1\ud638\uc791\uc6a9\uc744 \ud1b5\ud574 \uacbd\ud5d8\uc744 \uc5bb\uc744 \ub54c\ub9c8\ub2e4 \uacbd\ud5d8 \uc800\uc7a5\uc18c\uc5d0\uc11c \ub79c\ub364\ud558\uac8c \ubbf8\ub2c8 \ubc30\uce6d\ub41c \uacbd\ud5d8\uc744 \uc0d8\ud50c\ub9c1\ud558\uc5ec Q-\uac12\uc744 \uc5c5\ub370\uc774\ud2b8\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4. \uc608\ub97c \ub4e4\uc5b4:<\/p>\n \uacbd\ud5d8 \uc7ac\uc0dd \uc678\uc5d0\ub3c4 \uc0d8\ud50c \ud6a8\uc728\uc131\uc744 \ub192\uc774\uae30 \uc704\ud55c \uc5ec\ub7ec \uae30\ubc95\ub4e4\uc774 \uc874\uc7ac\ud569\ub2c8\ub2e4. \uc5ec\uae30\uc11c\ub294 \uba87 \uac00\uc9c0 \uae30\ubc95\uc744 \uc18c\uac1c\ud558\uaca0\uc2b5\ub2c8\ub2e4.<\/p>\n \uc6b0\uc120 \uacbd\ud5d8 \uc7ac\uc0dd\uc740 \uacbd\ud5d8 \uc800\uc7a5\uc18c\uc5d0\uc11c \uc0d8\ud50c\ub9c1\ud560 \ub54c, \ub354 \uc911\uc694\ud55c \uacbd\ud5d8\uc744 \uc6b0\uc120\uc801\uc73c\ub85c \uc120\ud0dd\ud558\ub294 \ubc29\ubc95\uc785\ub2c8\ub2e4. \uc774\ub294 \ub09c\uc774\ub3c4\uac00 \ub192\uc740 \uacbd\ud5d8\uc774\ub098 \uc5d0\uc774\uc804\ud2b8\uac00 \uc798\ubabb\ub41c \ud589\ub3d9\uc744 \ud588\uc744 \ub54c\uc758 \uacbd\ud5d8\uc5d0 \ub354 \ub192\uc740 \uc6b0\uc120\uc21c\uc704\ub97c \ubd80\uc5ec\ud568\uc73c\ub85c\uc368, \ube60\ub974\uac8c \ud559\uc2b5\ud560 \uc218 \uc788\ub3c4\ub85d \ub3d5\uc2b5\ub2c8\ub2e4.<\/p>\n \ub354\ube14 DQN\uc740 Q-\uac12\uc758 \uacfc\ucd94\uc815 \ubb38\uc81c\ub97c \ud574\uacb0\ud558\uae30 \uc704\ud574 \ub450 \uac1c\uc758 Q-\ub124\ud2b8\uc6cc\ud06c\ub97c \uc0ac\uc6a9\ud558\ub294 \uae30\ubc95\uc785\ub2c8\ub2e4. \ud55c \ub124\ud2b8\uc6cc\ud06c\ub294 \ud589\ub3d9\uc744 \uc120\ud0dd\ud558\uace0, \ub2e4\ub978 \ub124\ud2b8\uc6cc\ud06c\ub294 Q-\uac12\uc744 \ud3c9\uac00\ud558\ub294 \uc5ed\ud560\uc744 \ud569\ub2c8\ub2e4.<\/p>\n \uc790\uc8fc\uc801 \ud0d0\ud5d8\uc740 \uc5d0\uc774\uc804\ud2b8\uac00 \ub79c\ub364\ud558\uac8c \ud589\ub3d9\uc744 \uc120\ud0dd\ud560 \ud655\ub960\uc744 \uc810\ucc28 \uc904\uc5ec \uac00\uba70 \ucd5c\uc801\uc758 \ud589\ub3d9\uc744 \ucc3e\ub294 \ubc29\ubc95\uc785\ub2c8\ub2e4. \ucd08\uae30\uc5d0\ub294 \uc5d0\uc774\uc804\ud2b8\uac00 \uc5ec\ub7ec \ud658\uacbd\uc744 \ud0d0\ud5d8\ud560 \uc218 \uc788\ub3c4\ub85d \ub192\uc740 \ud655\ub960\ub85c \ub79c\ub364 \ud589\ub3d9\uc744 \ucde8\ud558\ub418, \ud559\uc2b5\uc774 \uc774\ub8e8\uc5b4\uc9c8\uc218\ub85d \uc774 \ud655\ub960\uc744 \uc904\uc5ec \uc810\uc810 \ub354 \ucd5c\uc801\uc758 \ud589\ub3d9\uc744 \uc120\ud0dd\ud558\ub3c4\ub85d \ud569\ub2c8\ub2e4.<\/p>\n \uacbd\ud5d8 \uc7ac\uc0dd\uc740 \uac15\ud654 \ud559\uc2b5\uc5d0\uc11c \uc0d8\ud50c \ud6a8\uc728\uc131\uc744 \ub192\uc774\uace0 \uc548\uc815\uc801\uc778 \ud559\uc2b5\uc744 \uc9c0\uc6d0\ud558\ub294 \uc544\uc8fc \uc911\uc694\ud55c \uae30\ubc95\uc785\ub2c8\ub2e4. \uc774\ub97c \ud1b5\ud574 \uc5d0\uc774\uc804\ud2b8\ub294 \ub354 \uc801\uc740 \ub370\uc774\ud130\ub85c\ub3c4 \ud6a8\uacfc\uc801\uc73c\ub85c \ud559\uc2b5\ud560 \uc218 \uc788\uc73c\uba70, \ub2e4\uc591\ud55c \uacbd\ud5d8\uc744 \ud6a8\uacfc\uc801\uc73c\ub85c \ud65c\uc6a9\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4. \ub610\ud55c, \uc6b0\uc120 \uacbd\ud5d8 \uc7ac\uc0dd\uc774\ub098 \ub354\ube14 DQN\uacfc \uac19\uc740 \ucd94\uac00\uc801\uc778 \uae30\ubc95\ub4e4\uc744 \ud65c\uc6a9\ud568\uc73c\ub85c\uc368 \ub354\uc6b1 \ud5a5\uc0c1\ub41c \uc131\ub2a5\uc744 \uae30\ub300\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n \uac15\ud654 \ud559\uc2b5\uc758 \uc138\uacc4\uc5d0\uc11c \uacbd\ud5d8 \uc7ac\uc0dd\uc740 \uc5d0\uc774\uc804\ud2b8\uc758 \uc131\uacf5\uc801\uc778 \ud559\uc2b5\uc744 \uc704\ud55c \uae30\ubcf8\uae30\uc774\uc790 \ud575\uc2ec \uae30\uc220\uc785\ub2c8\ub2e4. \uc774\ub97c \ud1b5\ud574 \uc2ec\uce35 \uac15\ud654 \ud559\uc2b5\uc774 \ub354\uc6b1 \ubc1c\uc804\ud558\uace0, \ub2e4\uc591\ud55c \uc2e4\uc81c \ubb38\uc81c \ud574\uacb0\uc5d0\ub3c4 \uc801\uc6a9\ub420 \uc218 \uc788\uae30\ub97c \uae30\ub300\ud569\ub2c8\ub2e4.<\/p>\n <\/body><\/p>\n","protected":false},"excerpt":{"rendered":"1. \uacbd\ud5d8 \uc7ac\uc0dd\uc758 \uac1c\ub150<\/h2>\n
\n
2. \uacbd\ud5d8 \uc7ac\uc0dd\uc758 \uad6c\uc870<\/h2>\n
2.1. \uacbd\ud5d8 \uc800\uc7a5\uc18c(Replay Buffer)<\/h3>\n
2.2. \ubbf8\ub2c8 \ubc30\uce58 \uc0d8\ud50c\ub9c1<\/h3>\n
3. \uacbd\ud5d8 \uc7ac\uc0dd \uad6c\ud604<\/h2>\n
import random\nfrom collections import deque\n\nclass ReplayBuffer:\n def __init__(self, max_size):\n self.buffer = deque(maxlen=max_size) # \uacbd\ud5d8 \uc800\uc7a5\uc18c \ucd08\uae30\ud654\n\n def add(self, experience):\n self.buffer.append(experience) # \uacbd\ud5d8 \ucd94\uac00\n\n def sample(self, batch_size):\n return random.sample(self.buffer, batch_size) # \ub79c\ub364\ud558\uac8c \ubbf8\ub2c8 \ubc30\uce58 \uc0d8\ud50c\ub9c1\n\n def size(self):\n return len(self.buffer) # \uc800\uc7a5\uc18c \ud06c\uae30 \ud655\uc778\n<\/code><\/pre>\ndeque<\/code> \ubaa8\ub4c8\uc744 \uc0ac\uc6a9\ud558\uc5ec \ud6a8\uc728\uc801\uc778 \uacbd\ud5d8 \uc800\uc7a5\uc18c\ub97c \uad6c\ud604\ud569\ub2c8\ub2e4. \uacbd\ud5d8\uc740 \ud29c\ud50c\uc758 \ud615\ud0dc\ub85c (\uc0c1\ud0dc, \ud589\ub3d9, \ubcf4\uc0c1, \ub2e4\uc74c \uc0c1\ud0dc)\ub85c \uad6c\uc131\ub429\ub2c8\ub2e4.<\/p>\n3.1. \uacbd\ud5d8 \ucd94\uac00<\/h3>\n
add<\/code> \uba54\uc11c\ub4dc\ub97c \ud1b5\ud574 \uacbd\ud5d8\uc744 \ucd94\uac00\ud569\ub2c8\ub2e4. \uc608\ub97c \ub4e4\uc5b4:<\/p>\nbuffer = ReplayBuffer(max_size=10000)\n\n# \uc0c8\ub85c\uc6b4 \uacbd\ud5d8\uc744 \uc5d0\uc774\uc804\ud2b8\uc5d0\uc11c \uc5bb\uc74c\nstate, action, reward, next_state = get_experience_from_environment()\n\nbuffer.add((state, action, reward, next_state)) # \uacbd\ud5d8 \ucd94\uac00\n<\/code><\/pre>\n3.2. \ubbf8\ub2c8 \ubc30\uce58 \uc0d8\ud50c\ub9c1 \ubc0f \uc5c5\ub370\uc774\ud2b8<\/h3>\n
batch_size = 32\nif buffer.size() > batch_size:\n batch = buffer.sample(batch_size) # \ubbf8\ub2c8 \ubc30\uce58 \uc0d8\ud50c\ub9c1\n\n for state, action, reward, next_state in batch:\n # Q-\uac12 \uc5c5\ub370\uc774\ud2b8 \ub85c\uc9c1\n q_value = update_q_value(state, action, reward, next_state) # Q-\uac12 \uc5c5\ub370\uc774\ud2b8 \ud568\uc218\n<\/code><\/pre>\n4. \uc0d8\ud50c \ud6a8\uc728\uc131\uc744 \ub192\uc774\ub294 \uae30\ud0c0 \uae30\ubc95\ub4e4<\/h2>\n
4.1. \uc6b0\uc120 \uacbd\ud5d8 \uc7ac\uc0dd (Prioritized Experience Replay)<\/h3>\n
class PrioritizedReplayBuffer:\n def __init__(self, max_size):\n self.buffer = []\n self.priorities = []\n self.max_size = max_size\n\n def add(self, experience, priority):\n if len(self.buffer) >= self.max_size:\n self.buffer.pop(0)\n self.priorities.pop(0)\n\n self.buffer.append(experience)\n self.priorities.append(priority) # \uacbd\ud5d8 \ucd94\uac00 \uc2dc \uc6b0\uc120\uc21c\uc704 \uc800\uc7a5\n\n def sample(self, batch_size):\n probabilities = [p for p in self.priorities] # \uc0d8\ud50c\ub9c1 \ud655\ub960\n \n # \uc0d8\ud50c\ub9c1\n batch = random.choices(self.buffer, k=batch_size, weights=probabilities)\n \n return batch\n<\/code><\/pre>\n4.2. \ub354\ube14 DQN(Double DQN)<\/h3>\n
4.3. \uc790\uc8fc\uc801 \ud0d0\ud5d8(Epsilon-Greedy Exploration)<\/h3>\n
5. \uacb0\ub860<\/h2>\n