通過點擊滑鼠左鍵、右鍵控制滑塊的左右移動來接住小球，如果球碰到底面，則遊戲結束

主要python代碼如下(遊戲本身的代碼省略，此處主要關注演算法代碼)：

#Game的定義類，此處Game是什麼不重要，只要提供執行Action的方法，獲取當前遊戲區域像素的方法即可
class Game(object):
def __init__(self): #Game初始化
# action是MOVE_STAY、MOVE_LEFT、MOVE_RIGHT
# ai控制棒子左右移動；返回遊戲界面像素數和對應的獎勵。(像素->獎勵->強化棒子往獎勵高的方向移動)
def step(self, action):
?
# learning_rate
LEARNING_RATE = 0.99
# 跟新梯度
INITIAL_EPSILON = 1.0
FINAL_EPSILON = 0.05
# 測試觀測次數
EXPLORE = 500000
OBSERVE = 500
# 記憶經驗大小
REPLAY_MEMORY = 500000
# 每次訓練取出的記錄數
BATCH = 100
# 輸出層神經元數。代表3種操作-MOVE_STAY:[1, 0, 0] MOVE_LEFT:[0, 1, 0] MOVE_RIGHT:[0, 0, 1]
output = 3 # MOVE_STAY:[1, 0, 0] MOVE_LEFT:[0, 1, 0] MOVE_RIGHT:[0, 0, 1]
input_image = tf.placeholder("float", [None, 80, 100, 4]) # 遊戲像素
action = tf.placeholder("float", [None, output]) # 操作
#定義CNN-卷積神經網路
def convolutional_neural_network(input_image):
weights = {w_conv1:tf.Variable(tf.zeros([8, 8, 4, 32])),
w_conv2:tf.Variable(tf.zeros([4, 4, 32, 64])),
w_conv3:tf.Variable(tf.zeros([3, 3, 64, 64])),
w_fc4:tf.Variable(tf.zeros([3456, 784])),
w_out:tf.Variable(tf.zeros([784, output]))}
?
biases = {b_conv1:tf.Variable(tf.zeros([32])),
b_conv2:tf.Variable(tf.zeros([64])),
b_conv3:tf.Variable(tf.zeros([64])),
b_fc4:tf.Variable(tf.zeros([784])),
b_out:tf.Variable(tf.zeros([output]))}
?
conv1 = tf.nn.relu(tf.nn.conv2d(input_image, weights[w_conv1], strides = [1, 4, 4, 1], padding = "VALID") + biases[b_conv1])
conv2 = tf.nn.relu(tf.nn.conv2d(conv1, weights[w_conv2], strides = [1, 2, 2, 1], padding = "VALID") + biases[b_conv2])
conv3 = tf.nn.relu(tf.nn.conv2d(conv2, weights[w_conv3], strides = [1, 1, 1, 1], padding = "VALID") + biases[b_conv3])
conv3_flat = tf.reshape(conv3, [-1, 3456])
fc4 = tf.nn.relu(tf.matmul(conv3_flat, weights[w_fc4]) + biases[b_fc4])
?
output_layer = tf.matmul(fc4, weights[w_out]) + biases[b_out]
return output_layer
?
#訓練神經網路
def train_neural_network(input_image):
predict_action = convolutional_neural_network(input_image)
?
argmax = tf.placeholder("float", [None, output])
gt = tf.placeholder("float", [None])
?
action = tf.reduce_sum(tf.mul(predict_action, argmax), reduction_indices = 1)
cost = tf.reduce_mean(tf.square(action - gt))
optimizer = tf.train.AdamOptimizer(1e-6).minimize(cost)
?
game = Game()
D = deque()
?
_, image = game.step(MOVE_STAY)
image = cv2.cvtColor(cv2.resize(image, (100, 80)), cv2.COLOR_BGR2GRAY)
ret, image = cv2.threshold(image, 1, 255, cv2.THRESH_BINARY)
input_image_data = np.stack((image, image, image, image), axis = 2)
#print ("IMG2:%s" %input_image_data)
?
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
?
saver = tf.train.Saver()
?
n = 0
epsilon = INITIAL_EPSILON
while True:
#print("InputImageData:", input_image_data)
action_t = predict_action.eval(feed_dict = {input_image : [input_image_data]})[0]
?
argmax_t = np.zeros([output], dtype=np.int)
if(random.random() <= INITIAL_EPSILON):
maxIndex = random.randrange(output)
else:
maxIndex = np.argmax(action_t)
argmax_t[maxIndex] = 1
if epsilon > FINAL_EPSILON:
epsilon -= (INITIAL_EPSILON - FINAL_EPSILON) / EXPLORE
?
reward, image = game.step(list(argmax_t))
?
image = cv2.cvtColor(cv2.resize(image, (100, 80)), cv2.COLOR_BGR2GRAY)
ret, image = cv2.threshold(image, 1, 255, cv2.THRESH_BINARY)
image = np.reshape(image, (80, 100, 1))
input_image_data1 = np.append(image, input_image_data[:, :, 0:3], axis = 2)
?
D.append((input_image_data, argmax_t, reward, input_image_data1))
?
if len(D) > REPLAY_MEMORY:
D.popleft()
?
if n > OBSERVE:
minibatch = random.sample(D, BATCH)
input_image_data_batch = [d[0] for d in minibatch]
argmax_batch = [d[1] for d in minibatch]
reward_batch = [d[2] for d in minibatch]
input_image_data1_batch = [d[3] for d in minibatch]
?
gt_batch = []
?
out_batch = predict_action.eval(feed_dict = {input_image : input_image_data1_batch})
?
for i in range(0, len(minibatch)):
gt_batch.append(reward_batch[i] + LEARNING_RATE * np.max(out_batch[i]))
?
print("gt_batch:", gt_batch, "argmax:", argmax_batch)
optimizer.run(feed_dict = {gt : gt_batch, argmax : argmax_batch, input_image : input_image_data_batch})
?
input_image_data = input_image_data1
n = n+1
print(n, "epsilon:", epsilon, " " ,"action:", maxIndex, " " ,"reward:", reward)
?
train_neural_network(input_image)


6. 總結

說到這裡，相信你已經能對強化學習有了一個大致的瞭解。接下來的事情，應該是如何把這項技術應用到我們的工作中，讓它發揮出應有的價值。

問答 什麼時候使用某種強化學習演算法？ 相關閱讀 使用 Q-Learning 實現 FlappyBird AI 強化學習總結 一文學習基於蒙特卡羅的強化學習方法 【每日課程推薦】機器學習實戰！快速入門在線廣告業務及CTR相應知識

此文已由作者授權騰訊雲+社區發布，更多原文請點擊

搜索關注公眾號「雲加社區」，第一時間獲取技術乾貨，關注後回復1024 送你一份技術課程大禮包！

海量技術實踐經驗，盡在雲加社區！