Add File
This commit is contained in:
@@ -0,0 +1,250 @@
|
|||||||
|
package org.dromara.easyai.transFormer.seflAttention;
|
||||||
|
|
||||||
|
import org.dromara.easyai.i.OutBack;
|
||||||
|
import org.dromara.easyai.matrixTools.Matrix;
|
||||||
|
import org.dromara.easyai.matrixTools.MatrixList;
|
||||||
|
import org.dromara.easyai.matrixTools.MatrixOperation;
|
||||||
|
import org.dromara.easyai.transFormer.CodecBlock;
|
||||||
|
import org.dromara.easyai.transFormer.FirstDecoderBlock;
|
||||||
|
import org.dromara.easyai.transFormer.model.LayNormModel;
|
||||||
|
import org.dromara.easyai.transFormer.nerve.HiddenNerve;
|
||||||
|
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Random;
|
||||||
|
|
||||||
|
public class LayNorm {//残差与归一化
|
||||||
|
private MultiSelfAttention multiSelfAttention;//多头
|
||||||
|
private final CodecBlock myEncoderBlock;
|
||||||
|
private final int featureDimension;//特征维度
|
||||||
|
private List<HiddenNerve> hiddenNerves;//第一层隐层
|
||||||
|
private final int type;//类别层模型需要保存
|
||||||
|
private final Map<Long, MatrixList> reMatrixMap = new HashMap<>();
|
||||||
|
private final FirstDecoderBlock firstDecoderBlock;
|
||||||
|
private Matrix bTa;//模型需要保存
|
||||||
|
private Matrix power;//模型需要保存
|
||||||
|
private Matrix myNormData;//第一步归一化后的数据
|
||||||
|
private final float study;//学习率
|
||||||
|
private Matrix myFinalError;//从FNN传来的总误差
|
||||||
|
private int number;//记录fnn传来的误差次数
|
||||||
|
private final MatrixOperation matrixOperation;
|
||||||
|
private final boolean encoder;
|
||||||
|
private final int depth;
|
||||||
|
|
||||||
|
public LayNormModel getModel() throws Exception {
|
||||||
|
LayNormModel layNormModel = new LayNormModel();
|
||||||
|
layNormModel.setbTa(bTa.getMatrix());
|
||||||
|
layNormModel.setPower(power.getMatrix());
|
||||||
|
return layNormModel;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void insertModel(LayNormModel layNormModel) throws Exception {
|
||||||
|
insertPower(layNormModel.getPower(), power);
|
||||||
|
insertPower(layNormModel.getbTa(), bTa);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void insertPower(float[][] modelPower, Matrix power) throws Exception {
|
||||||
|
for (int i = 0; i < power.getX(); i++) {
|
||||||
|
for (int j = 0; j < power.getY(); j++) {
|
||||||
|
power.setNub(i, j, modelPower[i][j]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public LayNorm(int type, int featureDimension, CodecBlock myEncoderBlock, FirstDecoderBlock firstDecoderBlock
|
||||||
|
, float study, int coreNumber, boolean encoder, int depth) throws Exception {
|
||||||
|
this.study = study;
|
||||||
|
this.myEncoderBlock = myEncoderBlock;
|
||||||
|
this.encoder = encoder;
|
||||||
|
this.depth = depth;
|
||||||
|
this.type = type;
|
||||||
|
this.featureDimension = featureDimension;
|
||||||
|
this.firstDecoderBlock = firstDecoderBlock;
|
||||||
|
matrixOperation = new MatrixOperation(coreNumber);
|
||||||
|
bTa = new Matrix(1, featureDimension);
|
||||||
|
power = new Matrix(featureDimension, featureDimension);
|
||||||
|
Random random = new Random();
|
||||||
|
float sh = 1;
|
||||||
|
if (!encoder && depth == 1) {
|
||||||
|
sh = featureDimension * featureDimension;
|
||||||
|
}
|
||||||
|
for (int i = 0; i < featureDimension; i++) {
|
||||||
|
float value = random.nextFloat() / sh;
|
||||||
|
bTa.setNub(0, i, value);
|
||||||
|
}
|
||||||
|
for (int i = 0; i < featureDimension; i++) {
|
||||||
|
for (int j = 0; j < featureDimension; j++) {
|
||||||
|
float value = random.nextFloat() / sh;
|
||||||
|
power.setNub(i, j, value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private Matrix back(Matrix errorMatrix, Matrix myData) throws Exception {
|
||||||
|
Matrix subPower = matrixOperation.matrixMulPd(errorMatrix, myData, power, false);
|
||||||
|
Matrix sub = matrixOperation.matrixMulPd(errorMatrix, myData, power, true);
|
||||||
|
power = matrixOperation.add(subPower, power);
|
||||||
|
float n = (float) Math.sqrt(sub.getY());
|
||||||
|
float nt = -n / (n - 1);
|
||||||
|
Matrix subMatrix = new Matrix(1, sub.getY());
|
||||||
|
for (int i = 0; i < sub.getY(); i++) {
|
||||||
|
float subValue = sub.getNumber(0, i) * study;
|
||||||
|
float value = subValue * n + subMatrix.getNumber(0, i);
|
||||||
|
subMatrix.setNub(0, i, value);
|
||||||
|
for (int j = 0; j < sub.getY(); j++) {
|
||||||
|
if (i != j) {
|
||||||
|
float otherValue = subValue * nt + subMatrix.getNumber(0, j);
|
||||||
|
subMatrix.setNub(0, j, otherValue);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return subMatrix;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void backErrorFromFNN(Matrix errorMatrix, long eventID, Matrix allError) throws Exception {//从fnn传下来的误差类别为1
|
||||||
|
number++;
|
||||||
|
if (myFinalError == null) {
|
||||||
|
myFinalError = errorMatrix;
|
||||||
|
} else {
|
||||||
|
myFinalError = matrixOperation.add(myFinalError, errorMatrix);
|
||||||
|
}
|
||||||
|
if (number == featureDimension) {
|
||||||
|
number = 0;
|
||||||
|
Matrix error = myFinalError.getSonOfMatrix(0, 0, myFinalError.getX(), myFinalError.getY() - 1);
|
||||||
|
myFinalError = null;
|
||||||
|
Matrix myError = matrixOperation.add(error, allError);//残差误差与FNN回传误差相加
|
||||||
|
backErrorFromLine(myError, eventID);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void backLastError(Matrix errorMatrix) throws Exception {//作为编码器最后一层接收误差
|
||||||
|
if (myFinalError == null) {
|
||||||
|
myFinalError = errorMatrix;
|
||||||
|
} else {
|
||||||
|
myFinalError = matrixOperation.add(myFinalError, errorMatrix);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void encoderBackStart(long eventID) throws Exception {
|
||||||
|
Matrix error = myFinalError.copy();
|
||||||
|
myFinalError = null;
|
||||||
|
backErrorFromLine(error, eventID);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void backErrorFromLine(Matrix errorMatrix, long eventID) throws Exception {//从线性层后传，类别为2
|
||||||
|
matrixOperation.mathMul(errorMatrix, study);
|
||||||
|
int x = errorMatrix.getX();
|
||||||
|
MatrixList errorMatrixList = null;
|
||||||
|
for (int i = 0; i < x; i++) {
|
||||||
|
Matrix error = errorMatrix.getRow(i);
|
||||||
|
Matrix myData = myNormData.getRow(i);
|
||||||
|
bTa = matrixOperation.add(error, bTa);//更新bTa
|
||||||
|
Matrix myRowError = back(error, myData);
|
||||||
|
if (i == 0) {
|
||||||
|
errorMatrixList = new MatrixList(myRowError, true);
|
||||||
|
} else {
|
||||||
|
errorMatrixList.add(myRowError);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Matrix myError = errorMatrixList.getMatrix();
|
||||||
|
//本模块误差处理完毕继续向后传播 2返回到fnn,1返回注意力
|
||||||
|
if (type == 2) {//返回fnn
|
||||||
|
int size = hiddenNerves.size();
|
||||||
|
for (int i = 0; i < size; i++) {
|
||||||
|
hiddenNerves.get(i).receiveErrorMatrix(myError.getColumn(i), eventID, myError);
|
||||||
|
}
|
||||||
|
} else {//误差返回注意力
|
||||||
|
multiSelfAttention.backError(myError, eventID);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
public void addNorm(Matrix feature, Matrix outMatrix, long eventID, boolean isStudy
|
||||||
|
, OutBack outBack, List<Integer> E, Matrix encoderFeature, boolean outAllPro) throws Exception {//残差及归一化
|
||||||
|
Matrix myMatrix = matrixOperation.add(feature, outMatrix);//残差相加
|
||||||
|
Matrix out = layNorm(myMatrix, isStudy);
|
||||||
|
// if (!encoder && depth > 1 && type == 1) {
|
||||||
|
// System.out.println(feature);
|
||||||
|
// System.out.println(outMatrix);
|
||||||
|
// }
|
||||||
|
if (type == 1) {
|
||||||
|
if (myEncoderBlock != null) {
|
||||||
|
sendHiddenParameter(out, eventID, isStudy, outBack, E, encoderFeature, outAllPro);//发送线性第一层
|
||||||
|
} else if (firstDecoderBlock != null) {//解码器第一层//输出
|
||||||
|
firstDecoderBlock.sendOutputMatrix(eventID, out, isStudy, outBack, E, outAllPro);
|
||||||
|
}
|
||||||
|
} else {//输出矩阵
|
||||||
|
myEncoderBlock.sendOutputMatrix(eventID, out, isStudy, outBack, E, encoderFeature, outAllPro);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void addNormFromNerve(long eventID, boolean isStudy, Matrix parameter, Matrix allFeature,
|
||||||
|
OutBack outBack, List<Integer> E, Matrix encoderFeature, boolean outAllPro) throws Exception {
|
||||||
|
MatrixList matrixFeature;
|
||||||
|
if (reMatrixMap.containsKey(eventID)) {
|
||||||
|
matrixFeature = reMatrixMap.get(eventID);
|
||||||
|
matrixFeature.add(parameter);
|
||||||
|
} else {
|
||||||
|
matrixFeature = new MatrixList(parameter, false);
|
||||||
|
reMatrixMap.put(eventID, matrixFeature);
|
||||||
|
}
|
||||||
|
if (matrixFeature.getY() == featureDimension) {//执行残差
|
||||||
|
reMatrixMap.remove(eventID);
|
||||||
|
addNorm(matrixFeature.getMatrix(), allFeature, eventID, isStudy, outBack, E, encoderFeature, outAllPro);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void sendHiddenParameter(Matrix feature, long eventId, boolean isStudy
|
||||||
|
, OutBack outBack, List<Integer> E, Matrix encoderFeature, boolean outAllPro) throws Exception {//hiddenNerves
|
||||||
|
for (HiddenNerve hiddenNerve : hiddenNerves) {
|
||||||
|
hiddenNerve.receive(feature, eventId, isStudy, outBack, E, encoderFeature, outAllPro);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private Matrix norm(Matrix row) throws Exception {
|
||||||
|
Matrix result = new Matrix(1, row.getY());
|
||||||
|
float avg = row.getAVG();//平均值
|
||||||
|
float sd = matrixOperation.getSdByMatrix(row, avg, 0.0000001f);//标准差
|
||||||
|
for (int i = 0; i < row.getY(); i++) {
|
||||||
|
float value = (row.getNumber(0, i) - avg) / sd;
|
||||||
|
result.setNub(0, i, value);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Matrix layNorm(Matrix feature, boolean isStudy) throws Exception {//进行归一化
|
||||||
|
int x = feature.getX();
|
||||||
|
MatrixList normMatrixList = null;
|
||||||
|
MatrixList outMatrixList = null;
|
||||||
|
for (int i = 0; i < x; i++) {
|
||||||
|
Matrix normData = norm(feature.getRow(i));//back时候需要
|
||||||
|
if (isStudy) {
|
||||||
|
if (i == 0) {
|
||||||
|
normMatrixList = new MatrixList(normData, true);
|
||||||
|
} else {
|
||||||
|
normMatrixList.add(normData);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Matrix want = matrixOperation.add(matrixOperation.mulMatrix(normData, power), bTa);
|
||||||
|
if (i == 0) {
|
||||||
|
outMatrixList = new MatrixList(want, true);
|
||||||
|
} else {
|
||||||
|
outMatrixList.add(want);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (isStudy) {
|
||||||
|
myNormData = normMatrixList.getMatrix();
|
||||||
|
}
|
||||||
|
return outMatrixList.getMatrix();
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setHiddenNerves(List<HiddenNerve> hiddenNerves) {
|
||||||
|
this.hiddenNerves = hiddenNerves;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setMultiSelfAttention(MultiSelfAttention multiSelfAttention) {
|
||||||
|
this.multiSelfAttention = multiSelfAttention;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user