From 1bb0766f7543b5b2ad6c7fad1708b9fd4542c2c3 Mon Sep 17 00:00:00 2001 From: inter Date: Thu, 4 Sep 2025 14:09:18 +0800 Subject: [PATCH] Add File --- .../transFormer/seflAttention/LayNorm.java | 250 ++++++++++++++++++ 1 file changed, 250 insertions(+) create mode 100644 src/main/java/org/dromara/easyai/transFormer/seflAttention/LayNorm.java diff --git a/src/main/java/org/dromara/easyai/transFormer/seflAttention/LayNorm.java b/src/main/java/org/dromara/easyai/transFormer/seflAttention/LayNorm.java new file mode 100644 index 0000000..19cd7d1 --- /dev/null +++ b/src/main/java/org/dromara/easyai/transFormer/seflAttention/LayNorm.java @@ -0,0 +1,250 @@ +package org.dromara.easyai.transFormer.seflAttention; + +import org.dromara.easyai.i.OutBack; +import org.dromara.easyai.matrixTools.Matrix; +import org.dromara.easyai.matrixTools.MatrixList; +import org.dromara.easyai.matrixTools.MatrixOperation; +import org.dromara.easyai.transFormer.CodecBlock; +import org.dromara.easyai.transFormer.FirstDecoderBlock; +import org.dromara.easyai.transFormer.model.LayNormModel; +import org.dromara.easyai.transFormer.nerve.HiddenNerve; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; + +public class LayNorm {//残差与归一化 + private MultiSelfAttention multiSelfAttention;//多头 + private final CodecBlock myEncoderBlock; + private final int featureDimension;//特征维度 + private List hiddenNerves;//第一层隐层 + private final int type;//类别层模型需要保存 + private final Map reMatrixMap = new HashMap<>(); + private final FirstDecoderBlock firstDecoderBlock; + private Matrix bTa;//模型需要保存 + private Matrix power;//模型需要保存 + private Matrix myNormData;//第一步归一化后的数据 + private final float study;//学习率 + private Matrix myFinalError;//从FNN传来的总误差 + private int number;//记录fnn传来的误差次数 + private final MatrixOperation matrixOperation; + private final boolean encoder; + private final int depth; + + public LayNormModel getModel() throws Exception { + LayNormModel layNormModel = new LayNormModel(); + layNormModel.setbTa(bTa.getMatrix()); + layNormModel.setPower(power.getMatrix()); + return layNormModel; + } + + public void insertModel(LayNormModel layNormModel) throws Exception { + insertPower(layNormModel.getPower(), power); + insertPower(layNormModel.getbTa(), bTa); + } + + private void insertPower(float[][] modelPower, Matrix power) throws Exception { + for (int i = 0; i < power.getX(); i++) { + for (int j = 0; j < power.getY(); j++) { + power.setNub(i, j, modelPower[i][j]); + } + } + } + + public LayNorm(int type, int featureDimension, CodecBlock myEncoderBlock, FirstDecoderBlock firstDecoderBlock + , float study, int coreNumber, boolean encoder, int depth) throws Exception { + this.study = study; + this.myEncoderBlock = myEncoderBlock; + this.encoder = encoder; + this.depth = depth; + this.type = type; + this.featureDimension = featureDimension; + this.firstDecoderBlock = firstDecoderBlock; + matrixOperation = new MatrixOperation(coreNumber); + bTa = new Matrix(1, featureDimension); + power = new Matrix(featureDimension, featureDimension); + Random random = new Random(); + float sh = 1; + if (!encoder && depth == 1) { + sh = featureDimension * featureDimension; + } + for (int i = 0; i < featureDimension; i++) { + float value = random.nextFloat() / sh; + bTa.setNub(0, i, value); + } + for (int i = 0; i < featureDimension; i++) { + for (int j = 0; j < featureDimension; j++) { + float value = random.nextFloat() / sh; + power.setNub(i, j, value); + } + } + } + + private Matrix back(Matrix errorMatrix, Matrix myData) throws Exception { + Matrix subPower = matrixOperation.matrixMulPd(errorMatrix, myData, power, false); + Matrix sub = matrixOperation.matrixMulPd(errorMatrix, myData, power, true); + power = matrixOperation.add(subPower, power); + float n = (float) Math.sqrt(sub.getY()); + float nt = -n / (n - 1); + Matrix subMatrix = new Matrix(1, sub.getY()); + for (int i = 0; i < sub.getY(); i++) { + float subValue = sub.getNumber(0, i) * study; + float value = subValue * n + subMatrix.getNumber(0, i); + subMatrix.setNub(0, i, value); + for (int j = 0; j < sub.getY(); j++) { + if (i != j) { + float otherValue = subValue * nt + subMatrix.getNumber(0, j); + subMatrix.setNub(0, j, otherValue); + } + } + } + return subMatrix; + } + + public void backErrorFromFNN(Matrix errorMatrix, long eventID, Matrix allError) throws Exception {//从fnn传下来的误差类别为1 + number++; + if (myFinalError == null) { + myFinalError = errorMatrix; + } else { + myFinalError = matrixOperation.add(myFinalError, errorMatrix); + } + if (number == featureDimension) { + number = 0; + Matrix error = myFinalError.getSonOfMatrix(0, 0, myFinalError.getX(), myFinalError.getY() - 1); + myFinalError = null; + Matrix myError = matrixOperation.add(error, allError);//残差误差与FNN回传误差相加 + backErrorFromLine(myError, eventID); + } + } + + public void backLastError(Matrix errorMatrix) throws Exception {//作为编码器最后一层接收误差 + if (myFinalError == null) { + myFinalError = errorMatrix; + } else { + myFinalError = matrixOperation.add(myFinalError, errorMatrix); + } + } + + public void encoderBackStart(long eventID) throws Exception { + Matrix error = myFinalError.copy(); + myFinalError = null; + backErrorFromLine(error, eventID); + } + + public void backErrorFromLine(Matrix errorMatrix, long eventID) throws Exception {//从线性层后传，类别为2 + matrixOperation.mathMul(errorMatrix, study); + int x = errorMatrix.getX(); + MatrixList errorMatrixList = null; + for (int i = 0; i < x; i++) { + Matrix error = errorMatrix.getRow(i); + Matrix myData = myNormData.getRow(i); + bTa = matrixOperation.add(error, bTa);//更新bTa + Matrix myRowError = back(error, myData); + if (i == 0) { + errorMatrixList = new MatrixList(myRowError, true); + } else { + errorMatrixList.add(myRowError); + } + } + Matrix myError = errorMatrixList.getMatrix(); + //本模块误差处理完毕继续向后传播 2返回到fnn,1返回注意力 + if (type == 2) {//返回fnn + int size = hiddenNerves.size(); + for (int i = 0; i < size; i++) { + hiddenNerves.get(i).receiveErrorMatrix(myError.getColumn(i), eventID, myError); + } + } else {//误差返回注意力 + multiSelfAttention.backError(myError, eventID); + } + + } + + public void addNorm(Matrix feature, Matrix outMatrix, long eventID, boolean isStudy + , OutBack outBack, List E, Matrix encoderFeature, boolean outAllPro) throws Exception {//残差及归一化 + Matrix myMatrix = matrixOperation.add(feature, outMatrix);//残差相加 + Matrix out = layNorm(myMatrix, isStudy); +// if (!encoder && depth > 1 && type == 1) { +// System.out.println(feature); +// System.out.println(outMatrix); +// } + if (type == 1) { + if (myEncoderBlock != null) { + sendHiddenParameter(out, eventID, isStudy, outBack, E, encoderFeature, outAllPro);//发送线性第一层 + } else if (firstDecoderBlock != null) {//解码器第一层//输出 + firstDecoderBlock.sendOutputMatrix(eventID, out, isStudy, outBack, E, outAllPro); + } + } else {//输出矩阵 + myEncoderBlock.sendOutputMatrix(eventID, out, isStudy, outBack, E, encoderFeature, outAllPro); + } + } + + public void addNormFromNerve(long eventID, boolean isStudy, Matrix parameter, Matrix allFeature, + OutBack outBack, List E, Matrix encoderFeature, boolean outAllPro) throws Exception { + MatrixList matrixFeature; + if (reMatrixMap.containsKey(eventID)) { + matrixFeature = reMatrixMap.get(eventID); + matrixFeature.add(parameter); + } else { + matrixFeature = new MatrixList(parameter, false); + reMatrixMap.put(eventID, matrixFeature); + } + if (matrixFeature.getY() == featureDimension) {//执行残差 + reMatrixMap.remove(eventID); + addNorm(matrixFeature.getMatrix(), allFeature, eventID, isStudy, outBack, E, encoderFeature, outAllPro); + } + } + + private void sendHiddenParameter(Matrix feature, long eventId, boolean isStudy + , OutBack outBack, List E, Matrix encoderFeature, boolean outAllPro) throws Exception {//hiddenNerves + for (HiddenNerve hiddenNerve : hiddenNerves) { + hiddenNerve.receive(feature, eventId, isStudy, outBack, E, encoderFeature, outAllPro); + } + } + + private Matrix norm(Matrix row) throws Exception { + Matrix result = new Matrix(1, row.getY()); + float avg = row.getAVG();//平均值 + float sd = matrixOperation.getSdByMatrix(row, avg, 0.0000001f);//标准差 + for (int i = 0; i < row.getY(); i++) { + float value = (row.getNumber(0, i) - avg) / sd; + result.setNub(0, i, value); + } + return result; + } + + private Matrix layNorm(Matrix feature, boolean isStudy) throws Exception {//进行归一化 + int x = feature.getX(); + MatrixList normMatrixList = null; + MatrixList outMatrixList = null; + for (int i = 0; i < x; i++) { + Matrix normData = norm(feature.getRow(i));//back时候需要 + if (isStudy) { + if (i == 0) { + normMatrixList = new MatrixList(normData, true); + } else { + normMatrixList.add(normData); + } + } + Matrix want = matrixOperation.add(matrixOperation.mulMatrix(normData, power), bTa); + if (i == 0) { + outMatrixList = new MatrixList(want, true); + } else { + outMatrixList.add(want); + } + } + if (isStudy) { + myNormData = normMatrixList.getMatrix(); + } + return outMatrixList.getMatrix(); + } + + public void setHiddenNerves(List hiddenNerves) { + this.hiddenNerves = hiddenNerves; + } + + public void setMultiSelfAttention(MultiSelfAttention multiSelfAttention) { + this.multiSelfAttention = multiSelfAttention; + } + +}