Add File
This commit is contained in:
@@ -0,0 +1,208 @@
|
||||
package org.dromara.easyai.rnnJumpNerveCenter;
|
||||
|
||||
|
||||
import org.dromara.easyai.matrixTools.Matrix;
|
||||
import org.dromara.easyai.config.SentenceConfig;
|
||||
import org.dromara.easyai.entity.TypeMapping;
|
||||
import org.dromara.easyai.entity.WordBack;
|
||||
import org.dromara.easyai.function.Tanh;
|
||||
import org.dromara.easyai.i.OutBack;
|
||||
import org.dromara.easyai.naturalLanguage.word.WordEmbedding;
|
||||
import org.dromara.easyai.rnnJumpNerveEntity.MyWordFeature;
|
||||
import org.dromara.easyai.rnnJumpNerveEntity.SensoryNerve;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
public class RRNerveManager {
|
||||
private final WordEmbedding wordEmbedding;
|
||||
private final Map<Integer, Integer> mapping = new HashMap<>();//主键是真实id,值是映射识别用id
|
||||
private NerveJumpManager typeNerveManager;//类别网络
|
||||
private int typeNub;//分类数量
|
||||
private int vectorDimension;//特征纵向维度
|
||||
private int maxFeatureLength;//特征最长长度
|
||||
private float studyPoint;//词向量学习学习率
|
||||
private boolean showLog;//是否输出学习数据
|
||||
private int minLength;//最小长度
|
||||
private float trustPowerTh = 0;//可信阈值
|
||||
private int rzModel;//正则模式
|
||||
private float rzParam;//正则系数
|
||||
|
||||
public RRNerveManager(WordEmbedding wordEmbedding) {
|
||||
this.wordEmbedding = wordEmbedding;
|
||||
}
|
||||
|
||||
public void init(SentenceConfig config) throws Exception {
|
||||
if (config.getTypeNub() > 0) {
|
||||
this.trustPowerTh = config.getTrustPowerTh();
|
||||
this.minLength = config.getMinLength();
|
||||
this.typeNub = config.getTypeNub();
|
||||
this.vectorDimension = config.getWordVectorDimension();
|
||||
this.maxFeatureLength = config.getMaxWordLength();
|
||||
this.studyPoint = config.getWeStudyPoint();
|
||||
this.showLog = config.isShowLog();
|
||||
this.rzModel = config.getRzModel();
|
||||
this.rzParam = config.getParam();
|
||||
initNerveManager();
|
||||
} else {
|
||||
throw new Exception("分类种类数量必须大于0");
|
||||
}
|
||||
}
|
||||
|
||||
private void initNerveManager() throws Exception {
|
||||
typeNerveManager = new NerveJumpManager(vectorDimension, vectorDimension, typeNub, maxFeatureLength - 1, new Tanh(), false,
|
||||
studyPoint, rzModel, rzParam);
|
||||
typeNerveManager.initRnn(true, showLog, true, false, 0);
|
||||
}
|
||||
|
||||
private int getMappingType(int key) {//通过自增主键查找原映射
|
||||
int id = 0;
|
||||
for (Map.Entry<Integer, Integer> entry : mapping.entrySet()) {
|
||||
if (entry.getValue() == key) {
|
||||
id = entry.getKey();
|
||||
break;
|
||||
}
|
||||
}
|
||||
return id;
|
||||
}
|
||||
|
||||
private int balance(Map<Integer, List<String>> model) {//强行均衡
|
||||
int maxNumber = 300;
|
||||
int index = 1;
|
||||
for (Map.Entry<Integer, List<String>> entry : model.entrySet()) {//查找最大数量
|
||||
mapping.put(entry.getKey(), index);
|
||||
if (entry.getValue().size() > maxNumber) {
|
||||
maxNumber = entry.getValue().size();
|
||||
}
|
||||
index++;
|
||||
}
|
||||
for (Map.Entry<Integer, List<String>> entry : model.entrySet()) {
|
||||
int size = entry.getValue().size();
|
||||
if (maxNumber > size) {
|
||||
int times = maxNumber / size - 1;//循环几次
|
||||
int sub = maxNumber % size;//余数
|
||||
List<String> list = entry.getValue();
|
||||
List<String> otherList = new ArrayList<>(list);
|
||||
for (int i = 0; i < times; i++) {
|
||||
list.addAll(otherList);
|
||||
}
|
||||
list.addAll(otherList.subList(0, sub));
|
||||
}
|
||||
}
|
||||
return maxNumber;
|
||||
}
|
||||
|
||||
private void studyNerve(long eventId, List<SensoryNerve> sensoryNerves, List<Float> featureList, Matrix rnnMatrix, Map<Integer, Float> E, boolean isStudy, OutBack convBack, int[] storeys) throws Exception {
|
||||
if (sensoryNerves.size() == featureList.size()) {
|
||||
for (int i = 0; i < sensoryNerves.size(); i++) {
|
||||
sensoryNerves.get(i).postMessage(eventId, featureList.get(i), isStudy, E, convBack, rnnMatrix, storeys, 0);
|
||||
}
|
||||
} else {
|
||||
throw new Exception("1size not equals,feature size:" + featureList.size() + "," +
|
||||
"sensorySize:" + sensoryNerves.size());
|
||||
}
|
||||
}
|
||||
|
||||
public int getType(String sentence, long eventID) throws Exception {//进行理解
|
||||
if (sentence.length() > maxFeatureLength) {
|
||||
sentence = sentence.substring(0, maxFeatureLength);
|
||||
}
|
||||
MyWordFeature myWordFeature = wordEmbedding.getEmbedding(sentence, eventID, false);
|
||||
List<Float> featureList = myWordFeature.getFirstFeatureList();
|
||||
Matrix featureMatrix = myWordFeature.getFeatureMatrix();
|
||||
int[] storeys = new int[featureMatrix.getX()];
|
||||
for (int i = 0; i < storeys.length; i++) {
|
||||
storeys[i] = i;
|
||||
}
|
||||
WordBack wordBack = new WordBack();//trustPowerTh
|
||||
studyNerve(eventID, typeNerveManager.getSensoryNerves(), featureList, featureMatrix, null, false, wordBack, storeys);
|
||||
if (wordBack.getOut() > trustPowerTh) {
|
||||
return getMappingType(wordBack.getId());
|
||||
} else {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
public void insertModel(RandomModel randomModel) throws Exception {
|
||||
typeNerveManager.insertModelParameter(randomModel.getTypeModelParameter());
|
||||
List<TypeMapping> typeMappings = randomModel.getTypeMappings();
|
||||
mapping.clear();
|
||||
for (TypeMapping typeMapping : typeMappings) {
|
||||
mapping.put(typeMapping.getType(), typeMapping.getMapping());
|
||||
}
|
||||
}
|
||||
|
||||
public RandomModel getModel() throws Exception {
|
||||
RandomModel randomModel = new RandomModel();
|
||||
randomModel.setTypeModelParameter(typeNerveManager.getModelParameter());
|
||||
List<TypeMapping> typeMappings = new ArrayList<>();
|
||||
randomModel.setTypeMappings(typeMappings);
|
||||
for (Map.Entry<Integer, Integer> entry : mapping.entrySet()) {
|
||||
TypeMapping typeMapping = new TypeMapping();
|
||||
typeMapping.setType(entry.getKey());
|
||||
typeMapping.setMapping(entry.getValue());
|
||||
typeMappings.add(typeMapping);
|
||||
}
|
||||
return randomModel;
|
||||
}
|
||||
|
||||
public RandomModel studyType(Map<Integer, List<String>> model) throws Exception {
|
||||
int maxNumber = balance(model);//平衡样本
|
||||
for (int i = 0; i < maxFeatureLength; i++) {//第一阶段学习
|
||||
System.out.println("1第：" + (i + 1) + "次。共:" + maxFeatureLength + "次");
|
||||
myStudy(maxNumber, model, i + 1);
|
||||
}
|
||||
return getModel();
|
||||
}
|
||||
|
||||
private void myStudy(int maxNumber, Map<Integer, List<String>> model, int time) throws Exception {
|
||||
int index = 0;
|
||||
Map<Integer, Float> E = new HashMap<>();
|
||||
do {
|
||||
for (Map.Entry<Integer, List<String>> entry : model.entrySet()) {
|
||||
System.out.println("index======" + index + "," + time + "次");
|
||||
E.clear();
|
||||
List<String> sentence = entry.getValue();
|
||||
int key = mapping.get(entry.getKey());
|
||||
E.put(key, 1f);
|
||||
String word = sentence.get(index);
|
||||
if (word.length() > maxFeatureLength) {
|
||||
word = word.substring(0, maxFeatureLength);
|
||||
}
|
||||
randomTypeStudy(wordEmbedding.getEmbedding(word, 1, false), E);
|
||||
}
|
||||
index++;
|
||||
} while (index < maxNumber);
|
||||
}
|
||||
|
||||
private void randomTypeStudy(MyWordFeature myWordFeature, Map<Integer, Float> E) throws Exception {
|
||||
Matrix featureMatrix = myWordFeature.getFeatureMatrix();
|
||||
List<Float> firstFeatureList = myWordFeature.getFirstFeatureList();
|
||||
int len = featureMatrix.getX();//文字长度
|
||||
Random random = new Random();
|
||||
if (len > 1) {//长度大于1才可以进行训练
|
||||
int[] storeys;
|
||||
if (len < minLength) {
|
||||
storeys = new int[len];
|
||||
for (int i = 0; i < len; i++) {
|
||||
storeys[i] = i;
|
||||
}
|
||||
} else {
|
||||
List<Integer> list = new ArrayList<>();
|
||||
for (int i = 1; i < len; i++) {
|
||||
list.add(i);
|
||||
}
|
||||
int myLen = (int) (minLength + (float)Math.random() * (len - minLength + 1));
|
||||
storeys = new int[myLen];
|
||||
for (int i = 1; i < myLen; i++) {
|
||||
int index = random.nextInt(list.size());
|
||||
storeys[i] = list.get(index);
|
||||
list.remove(index);
|
||||
}
|
||||
Arrays.sort(storeys);
|
||||
}
|
||||
studyNerve(1, typeNerveManager.getSensoryNerves(), firstFeatureList, featureMatrix
|
||||
, E, true, null, storeys);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user