Merge remote-tracking branch 'origin/master'

mec-biz/src/main/java/com/ym/mec/biz/dal/dto/MusicPitchDetailDto.java

@@ -20,6 +20,11 @@ public class MusicPitchDetailDto {
     @ApiModelProperty("频率Hz")
     private float frequency;
 
+    private float avgFrequency;
+
+    @ApiModelProperty("分贝")
+    private float decibel;
+
     @ApiModelProperty("小节数")
     private int measureIndex;
 
@@ -29,6 +34,22 @@ public class MusicPitchDetailDto {
     @ApiModelProperty("不需要评分")
     private Boolean dontEvaluating;
 
+    public float getDecibel() {
+        return decibel;
+    }
+
+    public void setDecibel(float decibel) {
+        this.decibel = decibel;
+    }
+
+    public float getAvgFrequency() {
+        return avgFrequency;
+    }
+
+    public void setAvgFrequency(float avgFrequency) {
+        this.avgFrequency = avgFrequency;
+    }
+
     public Boolean getDontEvaluating() {
         return dontEvaluating;
     }
@@ -72,6 +93,12 @@ public class MusicPitchDetailDto {
         this.frequency = frequency;
     }
 
+    public MusicPitchDetailDto(int timeStamp, float frequency, double decibel) {
+        this.timeStamp = timeStamp;
+        this.frequency = frequency;
+        this.decibel = (float) decibel;
+    }
+
     public int getTimeStamp() {
         return timeStamp;
     }

mec-biz/src/main/java/com/ym/mec/biz/dal/dto/SoundCompareHelper.java

@@ -41,8 +41,18 @@ public class SoundCompareHelper {
     @ApiModelProperty(value = "偏移时间量，解决客户端录音播放不同步导致的声音留白")
     private int offsetTime;
 
+    private List<MusicPitchDetailDto> musicXmlInfos;
+
     private byte[] preDataArray = new byte[0];
 
+    public List<MusicPitchDetailDto> getMusicXmlInfos() {
+        return musicXmlInfos;
+    }
+
+    public void setMusicXmlInfos(List<MusicPitchDetailDto> musicXmlInfos) {
+        this.musicXmlInfos = musicXmlInfos;
+    }
+
     public Map<Integer, Float> getMusicalNotePitchMap() {
         return musicalNotePitchMap;
     }

mec-biz/src/main/java/com/ym/mec/biz/handler/WebSocketHandler.java

@@ -1,8 +1,10 @@
 package com.ym.mec.biz.handler;
 
 import be.tarsos.dsp.AudioDispatcher;
+import be.tarsos.dsp.SilenceDetector;
 import be.tarsos.dsp.io.jvm.AudioDispatcherFactory;
 import be.tarsos.dsp.pitch.PitchProcessor;
+import be.tarsos.dsp.util.PitchConverter;
 import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONObject;
 import com.ym.mec.biz.dal.dto.MusicPitchDetailDto;
@@ -46,12 +48,13 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
     public static final Map<String, WebSocketSession> WS_CLIENTS = new ConcurrentHashMap<>();
 
     private final BigDecimal oneHundred = new BigDecimal(100);
-    private final float simpleRate = 44200;
-    private final int simpleSize = 512;
+    private final float simpleRate = 44100;
+    private final int simpleSize = 1024;
     private final int overlap = 256;
 
     private final AudioFormat audioFormat = new AudioFormat(simpleRate, 16, 1, true, false);
     private static final PitchProcessor.PitchEstimationAlgorithm algo = PitchProcessor.PitchEstimationAlgorithm.FFT_YIN;
+    private final SilenceDetector silenceDetecor = new SilenceDetector();
 
     private final String tmpDir = FileUtils.getTempDirectoryPath() + "/soundCompare/";
 
@@ -97,6 +100,7 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
             case SoundSocketService.MUSIC_XML:
                 userSoundInfoMap.put(phone, new SoundCompareHelper());
                 List<MusicPitchDetailDto> musicXmlInfos = JSON.parseArray(bodyObject.getString("musicXmlInfos"), MusicPitchDetailDto.class);
+                userSoundInfoMap.get(phone).setMusicXmlInfos(musicXmlInfos);
                 musicXmlInfos = musicXmlInfos.stream().filter(m->!m.getDontEvaluating()).collect(Collectors.toList());
                 userSoundInfoMap.get(phone).setMusicScoreId(bodyObject.getInteger("id"));
                 userSoundInfoMap.get(phone).setMeasureXmlInfoMap(musicXmlInfos.stream().collect(Collectors.groupingBy(MusicPitchDetailDto::getMeasureIndex)));
@@ -158,11 +162,19 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
 
         List<MusicPitchDetailDto> recordInfo = new ArrayList<>();
         AudioDispatcher dispatcher = AudioDispatcherFactory.fromByteArray(message.getPayload().array(), audioFormat, simpleSize, overlap);
+        dispatcher.addAudioProcessor(silenceDetecor);
         dispatcher.addAudioProcessor(new PitchProcessor(algo, simpleRate, simpleSize, (pitchDetectionResult, audioEvent) -> {
             int timeStamp = (int) (userSoundInfoMap.get(phone).getMeasureStartTime() + audioEvent.getTimeStamp()*1000);
             float pitch = pitchDetectionResult.getPitch();
-//            LOGGER.info("频率：{}, {}", timeStamp, pitch);
-            recordInfo.add(new MusicPitchDetailDto(timeStamp, pitch));
+            double cents = 0;
+            if (pitch > 0) {
+                cents = PitchConverter.hertzToAbsoluteCent(pitch);
+            }
+//            LOGGER.info("时间：{}, 频率：{}, 分贝：{}, 音分：{}", timeStamp, pitch, silenceDetecor.currentSPL(), cents);
+//            if (silenceDetecor.currentSPL() < -66){
+//                pitch = -1;
+//            }
+            recordInfo.add(new MusicPitchDetailDto(timeStamp, pitch, silenceDetecor.currentSPL()));
         }));
         dispatcher.run();
         if(Objects.isNull(userSoundInfoMap.get(phone).getAccessFile())){
@@ -200,6 +212,7 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
         exception.printStackTrace();
         LOGGER.info("发生了错误,移除客户端: {}", phone);
         WS_CLIENTS.remove(phone);
+        userSoundInfoMap.remove(phone);
         createHeader(phone);
     }
 
@@ -209,6 +222,7 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
         String phone = session.getPrincipal().getName().split(":")[1];
         LOGGER.info("{}离线", phone);
         WS_CLIENTS.remove(phone);
+        userSoundInfoMap.remove(phone);
         createHeader(phone);
     }
 
@@ -253,7 +267,6 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
             userSoundInfoMap.get(phone).setAccessFile(null);
         }
 //        userSoundInfoMap.get(phone).setRecordMeasurePithInfo(null);
-        LOGGER.info("评分数据：{}", JSON.toJSONString(userSoundInfoMap.get(phone)));
         userSoundInfoMap.remove(phone);
     }
 
@@ -277,38 +290,35 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
             //最低有效频率
             float minValidFrequency = 20;
 
-            //音准匹配数量
-            float intonationNum = 0;
-            //音准匹配误差范围
-            float intonationErrRange = 15;
-            //音准有效阈值
-            float intonationValidDuty = 0.2f;
+            //音准分数
+            float intonationScore = 0;
 
             //节奏匹配数量
             float cadenceNum = 0;
-            //节奏匹配误差范围
-            float cadenceErrRange = 30;
             //节奏有效阈值
-            float cadenceValidDuty = 0.2f;
+            float cadenceValidDuty = 0.01f;
 
-            //完整性数量
-            float integrityNum = 0;
             //完整性误差范围
             float integrityRange = 30;
-            //完整性有效阈值
-            float integrityValidDuty = 0.5f;
+            //完整性分数
+            float integrityScore = 0;
+
 
             int totalCompareNum = userSoundInfoMap.get(phone).getMeasureXmlInfoMap().get(measureIndex).size();
 
             for (int i = 0; i < userSoundInfoMap.get(phone).getMeasureXmlInfoMap().get(measureIndex).size(); i++) {
                 MusicPitchDetailDto musicXmlInfo = userSoundInfoMap.get(phone).getMeasureXmlInfoMap().get(measureIndex).get(i);
+
                 int ot5 = (int) (musicXmlInfo.getDuration()*0.1);
-                int halfOt = (int) (musicXmlInfo.getDuration()*0.35);
-                int startTimeStamp = musicXmlInfo.getTimeStamp() + userSoundInfoMap.get(phone).getOffsetTime() - ot5;
-                int endTimeStamp = musicXmlInfo.getTimeStamp()  + userSoundInfoMap.get(phone).getOffsetTime() + musicXmlInfo.getDuration() + ot5;
+                int startTimeStamp = musicXmlInfo.getTimeStamp() + userSoundInfoMap.get(phone).getOffsetTime() + ot5;
+                int endTimeStamp = musicXmlInfo.getTimeStamp()  + userSoundInfoMap.get(phone).getOffsetTime() + musicXmlInfo.getDuration() - ot5;
+
+                int preMeasureEndTimeStamp = startTimeStamp;
+                List<MusicPitchDetailDto> ms = userSoundInfoMap.get(phone).getMusicXmlInfos().stream().filter(m -> m.getMusicalNotesIndex() == musicXmlInfo.getMusicalNotesIndex() - 1).collect(Collectors.toList());
+                if(!CollectionUtils.isEmpty(ms)){
+                    preMeasureEndTimeStamp = ms.get(0).getEndTimeStamp() + userSoundInfoMap.get(phone).getOffsetTime();
+                }
 
-                //时间范围内有效音准数量
-                float recordValidIntonationNum = 0;
                 //时间范围内有效节奏数量
                 float cadenceValidNum = 0;
                 //时间范围内有效音频数量
@@ -333,7 +343,7 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
                     if(newMeasure){
                         break;
                     }
-                    if(recordInfo.getTimeStamp()<startTimeStamp||recordInfo.getTimeStamp()>(startTimeStamp+halfOt)){
+                    if(recordInfo.getTimeStamp()<preMeasureEndTimeStamp||recordInfo.getTimeStamp()>startTimeStamp){
                         continue;
                     }
                     if(Math.abs(recordInfo.getFrequency()-preMusicalNotesPitch)>10){
@@ -346,7 +356,9 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
                     }
                 }
 
-                List<Integer> musicalNotesPitchs = new ArrayList<>();
+//                List<Float> musicalNotesPitchs = new ArrayList<>();
+//                List<Float> decibels = new ArrayList<>();
+                List<MusicPitchDetailDto> measureSoundPitchInfos = new ArrayList<>();
 
                 for (int j = 0; j < userSoundInfoMap.get(phone).getRecordMeasurePithInfo().size(); j++) {
                     MusicPitchDetailDto recordInfo = userSoundInfoMap.get(phone).getRecordMeasurePithInfo().get(j);
@@ -354,12 +366,14 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
                     if(recordInfo.getTimeStamp()<startTimeStamp||recordInfo.getTimeStamp()>endTimeStamp){
                         continue;
                     }
-                    musicalNotesPitchs.add((int) recordInfo.getFrequency());
+//                    musicalNotesPitchs.add(recordInfo.getFrequency());
+//                    decibels.add(recordInfo.getDecibel());
+                    measureSoundPitchInfos.add(recordInfo);
+                    compareNum++;
                     if(!newMeasure){
                         continue;
                     }
 //                    LOGGER.info("{}频率({}-{})：{}, {}", recordInfo.getTimeStamp(), startTimeStamp, endTimeStamp, musicXmlInfo.getFrequency(), recordInfo.getFrequency());
-                    compareNum++;
                     //如果在最低有效频率以下则跳过
                     if(recordInfo.getFrequency()<minValidFrequency&&musicXmlInfo.getFrequency()!=-1){
                         continue;
@@ -372,65 +386,101 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
                     if(Math.abs(recordInfo.getFrequency()-musicXmlInfo.getFrequency())<=integrityRange){
                         integrityValidNum++;
                     }
-                    //如果频率差值在音准误差范围内
-                    if(Math.abs(recordInfo.getFrequency()-musicXmlInfo.getFrequency())<=intonationErrRange){
-                        recordValidIntonationNum++;
-                    }
                 }
 
                 //非正常频率次数
                 int errPitchNum = 0;
+                //分贝变化次数
+                int decibelChangeNum = 0;
 
-                if(CollectionUtils.isEmpty(musicalNotesPitchs)){
+                if(CollectionUtils.isEmpty(measureSoundPitchInfos)){
                     userSoundInfoMap.get(phone).getMusicalNotePitchMap().put(musicXmlInfo.getMusicalNotesIndex(), (float) 0);
                 }else{
-                    Map<Integer, Long> collect = musicalNotesPitchs.stream().collect(Collectors.groupingBy(Integer::intValue, Collectors.counting()));
+                    Map<Integer, Long> collect = measureSoundPitchInfos.stream().map(pitch -> (int)pitch.getFrequency()).collect(Collectors.groupingBy(Integer::intValue, Collectors.counting()));
                     //出现次数最多的频率
                     Integer pitch = collect.entrySet().stream().max(Comparator.comparing(e -> e.getValue())).get().getKey();
+                    //当前频率
+                    double cf = -1;
+                    //频率持续数量
+                    int fnum = 0;
+                    //是否演奏中
                     boolean ing = false;
+                    //当前分贝
+                    double cd = 0;
+                    //分贝持续数量
                     int dnum = 0;
-                    for (Integer musicalNotesPitch : musicalNotesPitchs) {
-                        if (!ing && Math.abs(musicalNotesPitch - pitch) <= 20){
-                            ing = true;
-                            continue;
+                    for (MusicPitchDetailDto musicalNotesPitch : measureSoundPitchInfos) {
+                        //计算频率断层次数
+                        if (Math.abs(musicalNotesPitch.getFrequency() - cf) > 20){
+                            fnum ++;
                         }
-                        if (ing && Math.abs(musicalNotesPitch - pitch) > 20){
-                            dnum++;
+                        if (fnum>=5){
+                            cf = musicalNotesPitch.getFrequency();
+                            fnum = 0;
+                            if (cf != -1){
+                                errPitchNum ++;
+                                ing = true;
+                                cd = musicalNotesPitch.getDecibel();
+                            }
                         }
-                        if (dnum>=2){
+                        //计算声音大小断层册数
+                        if(ing && Math.abs(musicalNotesPitch.getDecibel() - cd) > 5){
+                            dnum ++;
+                        }
+                        if (dnum > 2){
+                            cd = musicalNotesPitch.getDecibel();
                             dnum = 0;
-                            errPitchNum++;
-                            ing  = false;
+                            decibelChangeNum++;
                         }
                     }
                     userSoundInfoMap.get(phone).getMusicalNotePitchMap().put(musicXmlInfo.getMusicalNotesIndex(), (float) pitch);
                 }
 
-                //有效音频占比
-                float integrityDuty = integrityValidNum/compareNum;
-                //有效音高占比
-                float intonationDuty = recordValidIntonationNum/compareNum;
                 //有效节奏占比
                 float cadenceDuty = cadenceValidNum/compareNum;
-                //如果错误频率达到一定数值，则当前小节无效
-                if(errPitchNum>=3){
+                //如果频率出现断层或这个音量出现断层，则当前音符节奏无效
+                if(errPitchNum>=2 || decibelChangeNum>1){
                     cadenceDuty = 0;
                 }
                 //节奏
                 if(cadenceDuty>=cadenceValidDuty){
                     cadenceNum++;
-                    if(intonationDuty>=intonationValidDuty){
-                        intonationNum++;
+                }
+                //音准
+                if (!CollectionUtils.isEmpty(measureSoundPitchInfos)){
+                    Double avgPitch = measureSoundPitchInfos.stream().filter(pitch -> Math.abs((pitch.getFrequency()-musicXmlInfo.getFrequency()))<5).collect(Collectors.averagingDouble(pitch -> pitch.getFrequency()));
+                    //音分
+                    double recordCents = 0;
+                    if (avgPitch > 0){
+                        recordCents = PitchConverter.hertzToAbsoluteCent(avgPitch);
                     }
-                    if(integrityDuty>=integrityValidDuty){
-                        integrityNum++;
+                    double cents = PitchConverter.hertzToAbsoluteCent(musicXmlInfo.getFrequency());
+                    double score = 100 - Math.round(Math.abs(cents - recordCents)) + 3;
+                    if (score < 0){
+                        score = 0;
+                    }else if(score > 100){
+                        score = 100;
                     }
+                    intonationScore += score;
+                    musicXmlInfo.setAvgFrequency(avgPitch.floatValue());
+                }
+                //完成度
+                if(integrityValidNum>0){
+                    integrityValidNum = integrityValidNum + (float) (compareNum * 0.05);
+                }
+                if(integrityValidNum > compareNum){
+                    integrityValidNum = compareNum;
                 }
+                float integrityDuty = integrityValidNum/compareNum;
+                integrityScore += integrityDuty;
             }
 
-            intonation = new BigDecimal(intonationNum).divide(new BigDecimal(totalCompareNum), CommonConstants.DECIMAL_PLACE, BigDecimal.ROUND_DOWN).multiply(oneHundred).setScale(0, BigDecimal.ROUND_HALF_UP);
-            cadence = new BigDecimal(cadenceNum).divide(new BigDecimal(totalCompareNum), CommonConstants.DECIMAL_PLACE, BigDecimal.ROUND_DOWN).multiply(oneHundred).setScale(0, BigDecimal.ROUND_HALF_UP);
-            integrity = new BigDecimal(integrityNum).divide(new BigDecimal(totalCompareNum), CommonConstants.DECIMAL_PLACE, BigDecimal.ROUND_DOWN).multiply(oneHundred).setScale(0, BigDecimal.ROUND_HALF_UP);
+            BigDecimal measureNum = new BigDecimal(totalCompareNum);
+
+            intonation = new BigDecimal(intonationScore).divide(measureNum, CommonConstants.DECIMAL_PLACE, BigDecimal.ROUND_DOWN).setScale(0, BigDecimal.ROUND_UP);
+            cadence = new BigDecimal(cadenceNum).divide(measureNum, CommonConstants.DECIMAL_PLACE, BigDecimal.ROUND_DOWN).multiply(oneHundred).setScale(0, BigDecimal.ROUND_UP);
+            integrity = new BigDecimal(integrityScore).divide(measureNum, CommonConstants.DECIMAL_PLACE, BigDecimal.ROUND_DOWN).multiply(oneHundred).setScale(0, BigDecimal.ROUND_UP);
+
         } catch (ArithmeticException e){
             LOGGER.info("无musicXml信息");
         }
@@ -650,6 +700,8 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
 
         //存储评分数据
         sysMusicCompareRecordService.saveMusicCompareData(phone, userSoundInfoMap.get(phone).getMusicScoreId(), userSoundInfoMap.get(phone).getUserMeasureScoreMap());
+
+        LOGGER.info("评分数据：{}", JSON.toJSONString(userSoundInfoMap.get(phone)));
     }
 
     /**
@@ -671,8 +723,8 @@ public class WebSocketHandler extends AbstractWebSocketHandler {
         webSocketInfo.setHeader(header);
         Map<String, Object> result = new HashMap<>();
 //        BigDecimal score = intonation.multiply(new BigDecimal(0.5)).add(cadence.multiply(new BigDecimal(0.5))).setScale(0, BigDecimal.ROUND_HALF_UP);
-        BigDecimal score = intonation.add(cadence).add(integrity).divide(new BigDecimal(3), CommonConstants.DECIMAL_PLACE, BigDecimal.ROUND_DOWN).setScale(0, BigDecimal.ROUND_HALF_UP);
-//        BigDecimal score = intonation.setScale(0, BigDecimal.ROUND_HALF_UP);
+        BigDecimal score = intonation.add(cadence).add(integrity).divide(new BigDecimal(3), CommonConstants.DECIMAL_PLACE, BigDecimal.ROUND_DOWN).setScale(0, BigDecimal.ROUND_UP);
+//        BigDecimal score = integrity.setScale(0, BigDecimal.ROUND_HALF_UP);
         result.put("score", score);
         result.put("intonation", intonation);
         result.put("cadence", cadence);