在《Q-learning简明实例》中我们介绍了Q-learning算法的简单例子，从中我们可以总结出Q-learning算法的基本思想
本次选择的经验得分 = 本次选择的反馈得分 + 本次选择后场景的历史最佳经验得分
其中反馈得分是单个步骤的价值分值（固定的分值），经验得分是完成目标的学习分值（动态的分值）。
简明实例的Java实现如下
package com.coshaho.learn.qlearning; import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Random; /**
*
* QLearning.java Create on 2017年9月4日 下午10:08:49
*
* 类功能说明: QLearning简明例子实现
*
* Copyright: Copyright(c) 2013
* Company: COSHAHO
* @Version 1.0
* @Author coshaho
*/
public class QLearning
{
FeedbackMatrix R = new FeedbackMatrix(); ExperienceMatrix Q = new ExperienceMatrix(); public static void main(String[] args)
{
QLearning ql = new QLearning(); for(int i = 0; i < 500; i++)
{
Random random = new Random();
int x = random.nextInt(100) % 6; System.out.println("第" + i + "次学习, 初始房间是" + x);
ql.learn(x);
System.out.println();
}
} public void learn(int x)
{
do
{
// 随机选择一个联通的房间进入
int y = chooseRandomRY(x); // 获取以进入的房间为起始点的历史最佳得分
int qy = getMaxQY(y); // 计算此次移动的得分
int value = calculateNewQ(x, y, qy);
Q.set(x, y, value);
x = y;
}
// 走出房间则学习结束
while(5 != x); Q.print();
} public int chooseRandomRY(int x)
{
int[] qRow = R.getRow(x);
List<Integer> yValues = new ArrayList<Integer>();
for(int i = 0; i < qRow.length; i++)
{
if(qRow[i] >= 0)
{
yValues.add(i);
}
} Random random = new Random();
int i = random.nextInt(yValues.size()) % yValues.size();
return yValues.get(i);
} public int getMaxQY(int x)
{
int[] qRow = Q.getRow(x);
int length = qRow.length;
List<YAndValue> yValues = new ArrayList<YAndValue>();
for(int i = 0; i < length; i++)
{
YAndValue yv = new YAndValue(i, qRow[i]);
yValues.add(yv);
} Collections.sort(yValues);
int num = 1;
int value = yValues.get(0).getValue();
for(int i = 1; i < length; i++)
{
if(yValues.get(i).getValue() == value)
{
num = i + 1;
}
else
{
break;
}
} Random random = new Random();
int i = random.nextInt(num) % num;
return yValues.get(i).getY();
} // Q(x,y) = R(x,y) + 0.8 * max(Q(y,i))
public int calculateNewQ(int x, int y, int qy)
{
return (int) (R.get(x, y) + 0.8 * Q.get(y, qy));
} public static class YAndValue implements Comparable<YAndValue>
{
int y;
int value; public int getY() {
return y;
}
public void setY(int y) {
this.y = y;
}
public int getValue() {
return value;
}
public void setValue(int value) {
this.value = value;
}
public YAndValue(int y, int value)
{
this.y = y;
this.value = value;
}
public int compareTo(YAndValue o)
{
return o.getValue() - this.value;
}
}
} package com.coshaho.learn.qlearning; /**
*
* FeedbackMatrix.java Create on 2017年9月4日 下午9:52:41
*
* 类功能说明: 反馈矩阵
*
* Copyright: Copyright(c) 2013
* Company: COSHAHO
* @Version 1.0
* @Author coshaho
*/
public class FeedbackMatrix
{
public int get(int x, int y)
{
return R[x][y];
} public int[] getRow(int x)
{
return R[x];
} private static int[][] R = new int[6][6];
static
{
R[0][0] = -1;
R[0][1] = -1;
R[0][2] = -1;
R[0][3] = -1;
R[0][4] = 0;
R[0][5] = -1; R[1][0] = -1;
R[1][1] = -1;
R[1][2] = -1;
R[1][3] = 0;
R[1][4] = -1;
R[1][5] = 100; R[2][0] = -1;
R[2][1] = -1;
R[2][2] = -1;
R[2][3] = 0;
R[2][4] = -1;
R[2][5] = -1; R[3][0] = -1;
R[3][1] = 0;
R[3][2] = 0;
R[3][3] = -1;
R[3][4] = 0;
R[3][5] = -1; R[4][0] = 0;
R[4][1] = -1;
R[4][2] = -1;
R[4][3] = 0;
R[4][4] = -1;
R[4][5] = 100; R[5][0] = -1;
R[5][1] = 0;
R[5][2] = -1;
R[5][3] = -1;
R[5][4] = 0;
R[5][5] = 100;
}
} package com.coshaho.learn.qlearning; /**
*
* ExperienceMatrix.java Create on 2017年9月4日 下午10:03:08
*
* 类功能说明: 经验矩阵
*
* Copyright: Copyright(c) 2013
* Company: COSHAHO
* @Version 1.0
* @Author coshaho
*/
public class ExperienceMatrix
{
public int get(int x, int y)
{
return Q[x][y];
} public int[] getRow(int x)
{
return Q[x];
} public void set(int x, int y, int value)
{
Q[x][y] = value;
} public void print()
{
for(int i = 0; i < 6; i++)
{
for(int j = 0; j < 6; j++)
{
String s = Q[i][j] + " ";
if(Q[i][j] < 10)
{
s = s + " ";
}
else if(Q[i][j] < 100)
{
s = s + " ";
}
System.out.print(s);
}
System.out.println();
}
} private static int[][] Q = new int[6][6];
static
{
Q[0][0] = 0;
Q[0][1] = 0;
Q[0][2] = 0;
Q[0][3] = 0;
Q[0][4] = 0;
Q[0][5] = 0; Q[1][0] = 0;
Q[1][1] = 0;
Q[1][2] = 0;
Q[1][3] = 0;
Q[1][4] = 0;
Q[1][5] = 0; Q[2][0] = 0;
Q[2][1] = 0;
Q[2][2] = 0;
Q[2][3] = 0;
Q[2][4] = 0;
Q[2][5] = 0; Q[3][0] = 0;
Q[3][1] = 0;
Q[3][2] = 0;
Q[3][3] = 0;
Q[3][4] = 0;
Q[3][5] = 0; Q[4][0] = 0;
Q[4][1] = 0;
Q[4][2] = 0;
Q[4][3] = 0;
Q[4][4] = 0;
Q[4][5] = 0; Q[5][0] = 0;
Q[5][1] = 0;
Q[5][2] = 0;
Q[5][3] = 0;
Q[5][4] = 0;
Q[5][5] = 0;
}
}
经过500次计算得到如下结果
第499次学习, 初始房间是1
0 0 0 0 396 0
0 0 0 316 0 496
0 0 0 316 0 0
0 396 252 0 396 0
316 0 0 316 0 496
0 396 0 0 396 496
此时，我们从任意一个房间进入，每次选取最高分值步骤移动，总可以找到最短的逃离路径。