Add File
This commit is contained in:
@@ -0,0 +1,298 @@
|
|||||||
|
package org.dromara.easyai.gameRobot;
|
||||||
|
|
||||||
|
import org.dromara.easyai.matrixTools.Matrix;
|
||||||
|
|
||||||
|
import java.util.*;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author lidapeng
|
||||||
|
* @description 动态规划
|
||||||
|
* @date 10:25 上午 2022/9/12
|
||||||
|
*/
|
||||||
|
public class DynamicProgramming {
|
||||||
|
private final List<DynamicState> dynamicStateList = new ArrayList<>();//状态集合
|
||||||
|
private final Map<Integer, Action> actionMap = new HashMap<>();//动作列表
|
||||||
|
private final List<Integer> bestStrategy = new ArrayList<>();//最佳策略
|
||||||
|
private float gaMa = 0.5F; //贴现因子
|
||||||
|
private float valueTh = 0.0001f;//价值阈值
|
||||||
|
private int maxTimes = 500;//策略改进最大迭代次数
|
||||||
|
|
||||||
|
public void setMaxTimes(int maxTimes) {
|
||||||
|
this.maxTimes = maxTimes;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setValueTh(float valueTh) {
|
||||||
|
this.valueTh = valueTh;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setGaMa(float gaMa) {
|
||||||
|
this.gaMa = gaMa;
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<DynamicState> getDynamicStateList() {
|
||||||
|
return dynamicStateList;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Map<Integer, Action> getActionMap() {
|
||||||
|
return actionMap;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void gameStart() {//遍历所有状态
|
||||||
|
for (DynamicState dynamicState : dynamicStateList) {
|
||||||
|
if (!dynamicState.isFinish()) {
|
||||||
|
dynamicState.add();//被执行次数+1
|
||||||
|
Map<Integer, List<DynamicState>> sonStatesMap = dynamicState.getSonStatesMap();//动作-子状态集合 被执行的时候需要修改
|
||||||
|
int[] stateId = dynamicState.getStateId();
|
||||||
|
for (Map.Entry<Integer, Action> actionEntry : actionMap.entrySet()) {
|
||||||
|
Action action = actionEntry.getValue();
|
||||||
|
int actionId = action.getActionId();//动作id
|
||||||
|
List<int[]> stateList = action.action(stateId);
|
||||||
|
for (int[] myStateId : stateList) {
|
||||||
|
DynamicState state = getStateByStateId(myStateId);//经过动作产生的新状态
|
||||||
|
//产生一个新的动作-子状态合集的元素
|
||||||
|
if (sonStatesMap.containsKey(actionId)) {//在动作集合里
|
||||||
|
List<DynamicState> dynamicStates = sonStatesMap.get(actionId);
|
||||||
|
if (!isHere(dynamicStates, state.getStateId())) {
|
||||||
|
dynamicStates.add(state);
|
||||||
|
}
|
||||||
|
} else {//创建动作-子状态集合
|
||||||
|
List<DynamicState> dynamicStateList = new ArrayList<>();
|
||||||
|
dynamicStateList.add(state);
|
||||||
|
sonStatesMap.put(actionId, dynamicStateList);
|
||||||
|
}
|
||||||
|
Map<Integer, Integer> profitMap = state.getProfitMap();//该状态的收益集合，主键是收益，值是次数 被执行的时候需要修改
|
||||||
|
state.add();
|
||||||
|
//产生一个新的收益
|
||||||
|
int profit = action.getProfit(stateId);
|
||||||
|
if (profitMap.containsKey(profit)) {
|
||||||
|
profitMap.put(profit, profitMap.get(profit) + 1);
|
||||||
|
} else {
|
||||||
|
profitMap.put(profit, 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public Matrix getValueMatrix() throws Exception {//获取价值矩阵
|
||||||
|
int size = dynamicStateList.size();
|
||||||
|
int maxX = 0, maxY = 0;
|
||||||
|
for (int i = 0; i < size; i++) {
|
||||||
|
DynamicState dynamicState = dynamicStateList.get(i);
|
||||||
|
int[] stateId = dynamicState.getStateId();
|
||||||
|
int x = stateId[0];
|
||||||
|
int y = stateId[1];
|
||||||
|
if (x > maxX) {
|
||||||
|
maxX = x;
|
||||||
|
}
|
||||||
|
if (y > maxY) {
|
||||||
|
maxY = y;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Matrix matrix = new Matrix(maxY + 1, maxX + 1);
|
||||||
|
for (int i = 0; i < size; i++) {
|
||||||
|
DynamicState dynamicState = dynamicStateList.get(i);
|
||||||
|
int[] stateId = dynamicState.getStateId();
|
||||||
|
float value = dynamicState.getValue();
|
||||||
|
matrix.setNub(stateId[1], stateId[0], value);
|
||||||
|
}
|
||||||
|
return matrix;
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<ValueFunction> getValueFunction() {//获取价值函数
|
||||||
|
List<ValueFunction> valueFunctions = new ArrayList<>();
|
||||||
|
int size = dynamicStateList.size();
|
||||||
|
for (int i = 0; i < size; i++) {
|
||||||
|
DynamicState dynamicState = dynamicStateList.get(i);
|
||||||
|
ValueFunction valueFunction = new ValueFunction();
|
||||||
|
valueFunction.setStateId(dynamicState.getStateId());
|
||||||
|
valueFunction.setValue(dynamicState.getValue());
|
||||||
|
valueFunctions.add(valueFunction);
|
||||||
|
}
|
||||||
|
return valueFunctions;
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<Integer> getBestAction(int[] stateId) {//根据当前环境获取策略
|
||||||
|
List<Integer> actions = new ArrayList<>();
|
||||||
|
DynamicState state = getStateByStateId(stateId);//当前的环境
|
||||||
|
if (state != null) {
|
||||||
|
Map<Integer, List<DynamicState>> sonStatesMap = state.getSonStatesMap();
|
||||||
|
float maxValue = 0;//最大价值
|
||||||
|
boolean isFirstOne = true;
|
||||||
|
for (Map.Entry<Integer, List<DynamicState>> entry : sonStatesMap.entrySet()) {
|
||||||
|
List<DynamicState> sonStates = entry.getValue();//子状态
|
||||||
|
float maxValue2 = 0;//actionId 的最大价值
|
||||||
|
boolean isFirstTwo = true;
|
||||||
|
for (DynamicState dynamicState : sonStates) {
|
||||||
|
float myValue = dynamicState.getValue();
|
||||||
|
if (myValue > maxValue2 || isFirstTwo) {
|
||||||
|
isFirstTwo = false;
|
||||||
|
maxValue2 = myValue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (maxValue2 > maxValue || isFirstOne) {
|
||||||
|
isFirstOne = false;
|
||||||
|
maxValue = maxValue2;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
//筛选等价值策略
|
||||||
|
for (Map.Entry<Integer, List<DynamicState>> entry : sonStatesMap.entrySet()) {
|
||||||
|
int actionId = entry.getKey();//动作id
|
||||||
|
List<DynamicState> sonStates = entry.getValue();//子状态
|
||||||
|
for (DynamicState dynamicState : sonStates) {
|
||||||
|
if (dynamicState.getValue() == maxValue) {
|
||||||
|
actions.add(actionId);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return actions;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void strategyStudy() throws Exception {//策略学习
|
||||||
|
//记录当前最佳策略
|
||||||
|
int times = 0;
|
||||||
|
boolean isDifferent = true;//策略是否不同
|
||||||
|
do {
|
||||||
|
int size = dynamicStateList.size();
|
||||||
|
for (int i = 0; i < size; i++) {
|
||||||
|
DynamicState dynamicState = dynamicStateList.get(i);
|
||||||
|
if (!dynamicState.isFinish()) {
|
||||||
|
int actionId = getBestStrategyByPro(dynamicState);
|
||||||
|
dynamicState.setBestActionId(actionId);//通过概率获取的当前状态最佳策略
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (times > 0) {
|
||||||
|
isDifferent = compareStrategy();
|
||||||
|
}
|
||||||
|
if (isDifferent) {//新老策略不同，重新评估策略
|
||||||
|
updateBestStrategy();//更新新策略
|
||||||
|
strategyEvaluation();
|
||||||
|
}
|
||||||
|
times++;
|
||||||
|
} while (isDifferent && times < maxTimes);
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean isHere(List<DynamicState> dynamicStates, int[] stateId) {
|
||||||
|
boolean isHere = false;
|
||||||
|
for (DynamicState dynamicState : dynamicStates) {
|
||||||
|
if (Arrays.equals(dynamicState.getStateId(), stateId)) {
|
||||||
|
isHere = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return isHere;
|
||||||
|
}
|
||||||
|
|
||||||
|
private DynamicState getStateByStateId(int[] stateId) {
|
||||||
|
DynamicState state = null;
|
||||||
|
for (DynamicState dynamicState : dynamicStateList) {
|
||||||
|
if (Arrays.equals(dynamicState.getStateId(), stateId)) {
|
||||||
|
state = dynamicState;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return state;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void updateBestStrategy() {//更新最佳策略
|
||||||
|
int size = dynamicStateList.size();
|
||||||
|
for (int i = 0; i < size; i++) {
|
||||||
|
DynamicState dynamicState = dynamicStateList.get(i);
|
||||||
|
if (!dynamicState.isFinish()) {
|
||||||
|
bestStrategy.add(dynamicState.getBestActionId());
|
||||||
|
} else {
|
||||||
|
bestStrategy.add(0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean compareStrategy() {//比较新老策略
|
||||||
|
int size = dynamicStateList.size();
|
||||||
|
boolean isDifferent = false;
|
||||||
|
for (int i = 0; i < size; i++) {
|
||||||
|
DynamicState dynamicState = dynamicStateList.get(i);
|
||||||
|
int actionId = bestStrategy.get(i);
|
||||||
|
if (dynamicState.getBestActionId() != actionId) {
|
||||||
|
isDifferent = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return isDifferent;
|
||||||
|
}
|
||||||
|
|
||||||
|
private int getBestStrategyByPro(DynamicState dynamicState) throws Exception {//通过概率获取当前状态下的最佳策略
|
||||||
|
Map<Integer, List<DynamicState>> sonStatesMap = dynamicState.getSonStatesMap();//动作-子状态集合
|
||||||
|
float maxValue = 0;//最大价值
|
||||||
|
boolean isFirst = true;
|
||||||
|
int bestActionId = 0;//最佳动作
|
||||||
|
for (Map.Entry<Integer, List<DynamicState>> entry : sonStatesMap.entrySet()) {
|
||||||
|
int actionId = entry.getKey();//动作id
|
||||||
|
float value = getValueByAction(dynamicState, actionId);//该动作的价值
|
||||||
|
if (value > maxValue || isFirst) {
|
||||||
|
isFirst = false;
|
||||||
|
maxValue = value;
|
||||||
|
bestActionId = actionId;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
//返回最佳动作
|
||||||
|
return bestActionId;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void strategyEvaluation() throws Exception {//策略评估
|
||||||
|
float maxSub;//最大差值
|
||||||
|
do {
|
||||||
|
maxSub = 0;
|
||||||
|
for (DynamicState dynamicState : dynamicStateList) {//当前状态
|
||||||
|
if (!dynamicState.isFinish()) {//非终结态
|
||||||
|
float sub = valueEvaluation(dynamicState);//返回一个差
|
||||||
|
if (sub > maxSub) {
|
||||||
|
maxSub = sub;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} while (maxSub >= valueTh);
|
||||||
|
}
|
||||||
|
|
||||||
|
private float getValueByAction(DynamicState dynamicState, int actionId) throws Exception {//通过动作获取价值
|
||||||
|
Map<Integer, List<DynamicState>> sonStatesMap = dynamicState.getSonStatesMap();//动作-子状态集合
|
||||||
|
List<DynamicState> sonStateListByAction = sonStatesMap.get(actionId);//当前状态最优策略动作下的子状态集合
|
||||||
|
if (sonStateListByAction == null) {
|
||||||
|
throw new Exception("该状态无下一步动作！可能该状态属于终结态，但并没有设置为终结态！");
|
||||||
|
}
|
||||||
|
float number = dynamicState.getNumber();//当前状态被执行的次数即所有子状态被执行的总数
|
||||||
|
float updateValue = 0;//更新价值
|
||||||
|
for (DynamicState sonState : sonStateListByAction) {
|
||||||
|
Map<Integer, Integer> profitMap = sonState.getProfitMap();//子状态收益集合
|
||||||
|
float sonNumber = sonState.getNumber();//该子状态执行的次数即该子状态所有收益被执行的总次数
|
||||||
|
float sonPro = sonNumber / number;//当前子状态在当前最优策略动作下被执行的概率
|
||||||
|
float value = sonState.getValue() * gaMa;//该子状态的价值
|
||||||
|
//先对r求和
|
||||||
|
float sigmaR = 0;
|
||||||
|
for (Map.Entry<Integer, Integer> entryProfit : profitMap.entrySet()) {
|
||||||
|
float profit = entryProfit.getKey();//--获取profit的收益
|
||||||
|
float profitNumber = entryProfit.getValue();//--获取profit收益的次数
|
||||||
|
float profitPro = (profitNumber / sonNumber) * sonPro;//在当前策略动作下产生的环境，产生profit收益的概率
|
||||||
|
float v = (value + profit) * profitPro;//价值
|
||||||
|
sigmaR = sigmaR + v;
|
||||||
|
}
|
||||||
|
updateValue = updateValue + sigmaR;
|
||||||
|
}
|
||||||
|
return updateValue;
|
||||||
|
}
|
||||||
|
|
||||||
|
private float valueEvaluation(DynamicState dynamicState) throws Exception {//价值评估
|
||||||
|
float myValue = dynamicState.getValue();//当前价值
|
||||||
|
int bestActionId = dynamicState.getBestActionId();//当前最优策略选择的动作
|
||||||
|
float updateValue = getValueByAction(dynamicState, bestActionId);
|
||||||
|
dynamicState.setValue(updateValue);//更新价值
|
||||||
|
return (float)Math.abs(myValue - updateValue);
|
||||||
|
}
|
||||||
|
//策略迭代
|
||||||
|
//状态是当前环境的向量主键与价值函数
|
||||||
|
//动作（包含动作的规则）是执行完成一个动作之后返回一个新的状态
|
||||||
|
//策略决定执行什么动作，动作执行结束之后的收益是多少
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user