Merge branch 'music_score' of http://git.dayaedu.com/yonge/mec

 Conflicts:

	mec-biz/src/main/java/com/ym/mec/biz/dal/dto/SoundCompareHelper.java

	mec-biz/src/main/java/com/ym/mec/biz/service/impl/SoundCompareHandler.java

mec-biz/src/main/java/com/ym/mec/biz/dal/config/NioAudioInputStream.java

@@ -17,6 +17,8 @@ public class NioAudioInputStream implements TarsosDSPAudioInputStream {
     private RandomAccessFile randomAccessFile;
     private AudioFormat format;
 
+    private long position = 44;
+
     public NioAudioInputStream() {
     }
 
@@ -32,7 +34,12 @@ public class NioAudioInputStream implements TarsosDSPAudioInputStream {
 
     @Override
     public int read(byte[] b, int off, int len) throws IOException {
-        return randomAccessFile.read(b, off, len);
+        randomAccessFile.seek(position);
+        int read = randomAccessFile.read(b, off, len);
+        if(read>0){
+            position += read;
+        }
+        return read;
     }
 
     @Override

mec-biz/src/main/java/com/ym/mec/biz/dal/config/SoundCompareConfig.java

@@ -35,7 +35,7 @@ public class SoundCompareConfig {
     /**
      * @describe 有效分贝大小
      */
-    public int validDb = -65;
+    public int validDb = 30;
     /**
      * @describe 有效频率
      */

mec-biz/src/main/java/com/ym/mec/biz/dal/dto/SoundCompareHelper.java

@@ -1,6 +1,7 @@
 package com.ym.mec.biz.dal.dto;
 
 import be.tarsos.dsp.AudioEvent;
+import be.tarsos.dsp.AudioRunDispatcher;
 import be.tarsos.dsp.SilenceDetector;
 import be.tarsos.dsp.pitch.PitchDetectionHandler;
 import be.tarsos.dsp.pitch.PitchDetectionResult;
@@ -58,10 +59,10 @@ public class SoundCompareHelper implements PitchDetectionHandler {
     @ApiModelProperty(value = "播放速度")
     private int speed = 90;
 
-    @ApiModelProperty(value = "xml中每个小节的音符详情")
+    @ApiModelProperty(value = "小节xml信息字典")
     private Map<Integer, List<MusicPitchDetailDto>> measureXmlInfoMap = new HashMap<>();
 
-    @ApiModelProperty(value = "xml中每小节开始时间、结束时间")
+    @ApiModelProperty(value = "小节结束时间字典")
     private Map<Integer, MusicPitchDetailDto> measureEndTime = new HashMap<>();
 
 //    @ApiModelProperty(value = "录音音频信息")
@@ -72,7 +73,7 @@ public class SoundCompareHelper implements PitchDetectionHandler {
 
     private Map<Integer, Map<String, Object>> userMeasureScoreMap = new HashMap<>();
 
-    @ApiModelProperty(value = "xml中每个音符频率")
+    @ApiModelProperty(value = "音符频率字典")
     private Map<Integer, Float> musicalNotePitchMap = new HashMap<>();
 
     @ApiModelProperty(value = "每个音符评测结果")
@@ -81,13 +82,10 @@ public class SoundCompareHelper implements PitchDetectionHandler {
     @ApiModelProperty(value = "偏移时间量，解决客户端录音播放不同步导致的声音留白")
     private int offsetTime = -1;
 
-    /** xml中每个音符详情 */
     private List<MusicPitchDetailDto> musicXmlInfos;
 
-    /** 第一个音符前有多少个字节 */
     private long firstMeasureStartBytes = 0;
 
-    /** 文件的总字节数长度 */
     private long recordBytes = 0;
 
     private String clientId;
@@ -112,34 +110,28 @@ public class SoundCompareHelper implements PitchDetectionHandler {
      */
     public SilenceDetector silenceDetector = new SilenceDetector();
 
-    /**
-     * xml中每个音符详情 
-     * @return
-     */
+    private AudioRunDispatcher audioRunDispatcher;
+
+    public AudioRunDispatcher getAudioRunDispatcher() {
+        return audioRunDispatcher;
+    }
+
+    public void setAudioRunDispatcher(AudioRunDispatcher audioRunDispatcher) {
+        this.audioRunDispatcher = audioRunDispatcher;
+    }
+
     public List<MusicPitchDetailDto> getMusicXmlInfos() {
         return musicXmlInfos;
     }
 
-    /**
-     * xml中每个音符详情 
-     * @return
-     */
     public void setMusicXmlInfos(List<MusicPitchDetailDto> musicXmlInfos) {
         this.musicXmlInfos = musicXmlInfos;
     }
 
-    /**
-     * xml中每个音符对应的频率
-     * @return
-     */
     public Map<Integer, Float> getMusicalNotePitchMap() {
         return musicalNotePitchMap;
     }
 
-    /**
-     * xml中每个音符对应的频率
-     * @return
-     */
     public void setMusicalNotePitchMap(Map<Integer, Float> musicalNotePitchMap) {
         this.musicalNotePitchMap = musicalNotePitchMap;
     }
@@ -232,10 +224,6 @@ public class SoundCompareHelper implements PitchDetectionHandler {
         this.musicScoreId = musicScoreId;
     }
 
-    /**
-     * 录音文件
-     * @return
-     */
     public RandomAccessFile getAccessFile() {
         return accessFile;
     }
@@ -256,10 +244,6 @@ public class SoundCompareHelper implements PitchDetectionHandler {
         this.recordFilePath = recordFilePath;
     }
 
-    /**
-     * 录音文件
-     * @return
-     */
     public void setAccessFile(RandomAccessFile accessFile) {
         this.accessFile = accessFile;
     }
@@ -280,114 +264,58 @@ public class SoundCompareHelper implements PitchDetectionHandler {
         this.measureStartTime = measureStartTime;
     }
 
-    /**
-     * xml中每个小节的音符详情
-     * @return
-     */
     public Map<Integer, List<MusicPitchDetailDto>> getMeasureXmlInfoMap() {
         return measureXmlInfoMap;
     }
 
-    /**
-     * xml中每个小节的音符详情
-     * @return
-     */
     public void setMeasureXmlInfoMap(Map<Integer, List<MusicPitchDetailDto>> measureXmlInfoMap) {
         this.measureXmlInfoMap = measureXmlInfoMap;
     }
 
-    /**
-     * xml中每小节开始时间、结束时间
-     * @return
-     */
     public Map<Integer, MusicPitchDetailDto> getMeasureEndTime() {
         return measureEndTime;
     }
 
-    /**
-     * xml中每小节开始时间、结束时间
-     * @param measureEndTime
-     */
     public void setMeasureEndTime(Map<Integer, MusicPitchDetailDto> measureEndTime) {
         this.measureEndTime = measureEndTime;
     }
 
-    /**
-     * 每个音符的评测结果
-     * @return
-     */
     public List<MusicalNotesPlayStatDto> getMusicalNotesPlayStats() {
         return musicalNotesPlayStats;
     }
 
-    /**
-     * 每个音符的评测结果
-     * @return
-     */
     public void setMusicalNotesPlayStats(List<MusicalNotesPlayStatDto> musicalNotesPlayStats) {
         this.musicalNotesPlayStats = musicalNotesPlayStats;
     }
 
-    /**
-     * 小节累积得分（节奏、音准、完整度）
-     * @return
-     */
     public Map<String, BigDecimal> getUserScoreMap() {
         return userScoreMap;
     }
 
-    /**
-     * 小节累积得分（节奏、音准、完整度）
-     * @return
-     */
     public void setUserScoreMap(Map<String, BigDecimal> userScoreMap) {
         this.userScoreMap = userScoreMap;
     }
 
-    /**
-     * 第一个音符前有多少个字节
-     * @return
-     */
     public long getFirstMeasureStartBytes() {
         return firstMeasureStartBytes;
     }
 
-    /**
-     * 第一个音符前有多少个字节
-     * @param firstMeasureStartBytes
-     */
     public void setFirstMeasureStartBytes(long firstMeasureStartBytes) {
         this.firstMeasureStartBytes = firstMeasureStartBytes;
     }
 
-    /**
-     * 文件的总字节数长度
-     * @return
-     */
     public long getRecordBytes() {
         return recordBytes;
     }
 
-    /**
-     * 文件的总字节数长度
-     * @param recordBytes
-     */
     public void setRecordBytes(long recordBytes) {
         this.recordBytes = recordBytes;
     }
 
-    /**
-     * 小节得分结果
-     * @return
-     */
     public Map<Integer, Map<String, Object>> getUserMeasureScoreMap() {
         return userMeasureScoreMap;
     }
 
-    /**
-     * 小节得分结果
-     * @return
-     */
     public void setUserMeasureScoreMap(Map<Integer, Map<String, Object>> userMeasureScoreMap) {
         this.userMeasureScoreMap = userMeasureScoreMap;
     }
@@ -406,17 +334,21 @@ public class SoundCompareHelper implements PitchDetectionHandler {
         float pitch = pitchDetectionResult.getPitch();
         int decibel = (int) (100 - Math.abs(silenceDetector.currentSPL()));
 
+        byte byteOne = audioEvent.getByteBuffer()[0];
+        byte byteTwo = audioEvent.getByteBuffer()[1];
+
+        double amplitude = Math.abs(((byteOne&0x000000FF)|(byteTwo<<8)));
+
         if(decibel <= SoundCompareHandler.soundCompareConfig.validDb){
             pitch = -1;
             decibel = 0;
         }
 
-        SoundCompareHandler.LOGGER.info("时间：{}, 频率：{}, 分贝：{}", timeStamp, pitch, decibel);
+        SoundCompareHandler.LOGGER.info("时间：{}, 振幅：{}, 频率：{}, 分贝：{}", timeStamp, amplitude, pitch, decibel);
 
-        //上次的频率与本次的频率相差10hz或分贝相差10
         if(currPitchInfos.size()>0&&(Math.abs(currPitchInfos.get(currPitchInfos.size()-1).getFrequency()-pitch)>10||Math.abs(currPitchInfos.get(currPitchInfos.size()-1).getDecibel()-decibel)>10)){
-            Double avgPitch = currPitchInfos.stream().skip(1).collect(Collectors.averagingDouble(MusicPitchDetailDto::getFrequency));
-            Double avgDb = currPitchInfos.stream().skip(1).collect(Collectors.averagingDouble(MusicPitchDetailDto::getDecibel));
+            Double avgPitch = currPitchInfos.stream().collect(Collectors.averagingDouble(MusicPitchDetailDto::getFrequency));
+            Double avgDb = currPitchInfos.stream().collect(Collectors.averagingDouble(MusicPitchDetailDto::getDecibel));
 
             MusicPitchDetailDto measureDetail = new MusicPitchDetailDto(currPitchInfos.get(0).getTimeStamp(), avgPitch.floatValue(), avgDb);
             measureDetail.setEndTimeStamp(timeStamp);
@@ -439,7 +371,7 @@ public class SoundCompareHelper implements PitchDetectionHandler {
                         break;
                     }
                 }
-                musicXmlInfos.forEach(e->e.setTimeStamp(e.getTimeStamp()+offsetTime));//??????
+                musicXmlInfos.forEach(e->e.setTimeStamp(e.getTimeStamp()+offsetTime));
                 for (Map.Entry<Integer, MusicPitchDetailDto> musicPitchDetailDtoEntry : measureEndTime.entrySet()) {
                     musicPitchDetailDtoEntry.getValue().setTimeStamp(musicPitchDetailDtoEntry.getValue().getTimeStamp() + offsetTime);
                     musicPitchDetailDtoEntry.getValue().setEndTimeStamp(musicPitchDetailDtoEntry.getValue().getEndTimeStamp() + offsetTime);

mec-biz/src/main/java/com/ym/mec/biz/service/impl/SoundCompareHandler.java

@@ -1,12 +1,14 @@
 package com.ym.mec.biz.service.impl;
 
 import be.tarsos.dsp.AudioDispatcher;
+import be.tarsos.dsp.AudioRunDispatcher;
 import be.tarsos.dsp.io.jvm.AudioDispatcherFactory;
 import be.tarsos.dsp.pitch.PitchProcessor;
 import be.tarsos.dsp.util.PitchConverter;
 import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONObject;
 import com.alibaba.fastjson.serializer.SerializerFeature;
+import com.ym.mec.biz.dal.config.NioAudioInputStream;
 import com.ym.mec.biz.dal.config.SoundCompareConfig;
 import com.ym.mec.biz.dal.dao.SysMusicScoreAccompanimentDao;
 import com.ym.mec.biz.dal.dto.*;
@@ -153,6 +155,12 @@ public class SoundCompareHandler implements WebSocketEventHandler {
                     userSoundInfoMap.get(phone).setAccessFile(new RandomAccessFile(file, "rw"));
                     userSoundInfoMap.get(phone).getAccessFile().seek(44);
                     userSoundInfoMap.get(phone).setRecordFilePath(file.getAbsolutePath());
+
+                    AudioRunDispatcher dispatcher = new AudioRunDispatcher(new NioAudioInputStream(userSoundInfoMap.get(phone).getAccessFile(), soundCompareConfig.audioFormat), soundCompareConfig.simpleSize, 0);
+                    dispatcher.addAudioProcessor(userSoundInfoMap.get(phone).silenceDetector);
+                    dispatcher.addAudioProcessor(new PitchProcessor(soundCompareConfig.algo, soundCompareConfig.simpleRate, soundCompareConfig.simpleSize, userSoundInfoMap.get(phone)));
+//                    new Thread(dispatcher, phone).start();
+                    userSoundInfoMap.get(phone).setAudioRunDispatcher(dispatcher);
                 } catch (IOException e) {
                     throw new BizException("文件创建失败：", e);
                 }
@@ -163,7 +171,6 @@ public class SoundCompareHandler implements WebSocketEventHandler {
                 }
                 try {
                     if(!CollectionUtils.isEmpty(userSoundInfoMap.get(phone).getMeasureEndTime())){
-                    	//这是最后播放的一小节吗？
                         Integer lastMeasureIndex = userSoundInfoMap.get(phone).getMeasureEndTime().keySet().stream().min(Integer::compareTo).get();
                         double recordTime = userSoundInfoMap.get(phone).getAccessFile().length()/(soundCompareConfig.audioFormat.getFrameSize()*soundCompareConfig.audioFormat.getFrameRate())*1000;
                         //如果结束时时长大于某小节，则此小节需要评分
@@ -212,26 +219,31 @@ public class SoundCompareHandler implements WebSocketEventHandler {
         if(!userSoundInfoMap.containsKey(phone)){
             return;
         }
+        //读取波形数据
+//        for (int i = 0; i < message.getPayload().array().length; i+=2) {
+//            System.out.println((double) ((message.getPayload().array()[i]&0x000000FF)|(message.getPayload().array()[i+1]<<8))/32767);
+//        }
         userSoundInfoMap.get(phone).setRecordBytes(userSoundInfoMap.get(phone).getRecordBytes()+message.getPayloadLength());
         if(userSoundInfoMap.get(phone).getRecordBytes()<userSoundInfoMap.get(phone).getFirstMeasureStartBytes()){
             return;
         }
-        //待优化：节拍器占用的字节未被完全剔除
         try {
             if(Objects.nonNull(userSoundInfoMap.get(phone).getAccessFile())){
+                userSoundInfoMap.get(phone).getAccessFile().seek(userSoundInfoMap.get(phone).getAccessFile().length());
                 userSoundInfoMap.get(phone).getAccessFile().write(message.getPayload().array());
             }else{
                 return;
             }
 
-            AudioDispatcher dispatcher = AudioDispatcherFactory.fromByteArray(message.getPayload().array(), soundCompareConfig.audioFormat, soundCompareConfig.simpleSize, soundCompareConfig.overlap);
-            dispatcher.addAudioProcessor(userSoundInfoMap.get(phone).silenceDetector);
-            dispatcher.addAudioProcessor(new PitchProcessor(soundCompareConfig.algo, soundCompareConfig.simpleRate, soundCompareConfig.simpleSize, userSoundInfoMap.get(phone)));
-            dispatcher.run();
+            userSoundInfoMap.get(phone).getAudioRunDispatcher().run();
+
+//            AudioDispatcher dispatcher = AudioDispatcherFactory.fromByteArray(message.getPayload().array(), soundCompareConfig.audioFormat, soundCompareConfig.simpleSize, soundCompareConfig.overlap);
+//            dispatcher.addAudioProcessor(userSoundInfoMap.get(phone).silenceDetector);
+//            dispatcher.addAudioProcessor(new PitchProcessor(soundCompareConfig.algo, soundCompareConfig.simpleRate, soundCompareConfig.simpleSize, userSoundInfoMap.get(phone)));
+//            dispatcher.run();
 
-            //待优化：文件大小需要去掉头信息（44）
             double recordTime = userSoundInfoMap.get(phone).getAccessFile().length()/(soundCompareConfig.audioFormat.getFrameSize()*soundCompareConfig.audioFormat.getFrameRate())*1000;
-            userSoundInfoMap.get(phone).setMeasureStartTime(recordTime);
+//            userSoundInfoMap.get(phone).setMeasureStartTime(recordTime);
 
             for (Map.Entry<Integer, MusicPitchDetailDto> userMeasureEndTimeMapEntry : userSoundInfoMap.get(phone).getMeasureEndTime().entrySet()) {
                 if(recordTime>(userMeasureEndTimeMapEntry.getValue().getEndTimeStamp()+100)){
@@ -244,7 +256,7 @@ public class SoundCompareHandler implements WebSocketEventHandler {
                     break;
                 }
             }
-        } catch (UnsupportedAudioFileException | IOException e) {
+        } catch (IOException e) {
             throw new BizException("{}评分异常：{}", phone, e);
         }
     }
@@ -292,6 +304,7 @@ public class SoundCompareHandler implements WebSocketEventHandler {
                 randomAccessFile.seek(0);
                 randomAccessFile.write(WavHeader.getWaveHeader(randomAccessFile.length(), (long) soundCompareConfig.audioFormat.getFrameRate(), soundCompareConfig.audioFormat.getSampleSizeInBits()));
                 randomAccessFile.close();
+                userSoundInfoMap.get(phone).getAudioRunDispatcher().stop();
             } catch (IOException e) {
                 e.printStackTrace();
             }
@@ -356,7 +369,6 @@ public class SoundCompareHandler implements WebSocketEventHandler {
             int totalCompareNum = userSoundInfoMap.get(phone).getMeasureXmlInfoMap().get(measureIndex).size();
 
             for (int i = 0; i < userSoundInfoMap.get(phone).getMeasureXmlInfoMap().get(measureIndex).size(); i++) {
-            	//获取音符信息
                 MusicPitchDetailDto musicXmlInfo = userSoundInfoMap.get(phone).getMeasureXmlInfoMap().get(measureIndex).get(i);
 
                 int startTimeStamp = musicXmlInfo.getTimeStamp();
@@ -365,8 +377,7 @@ public class SoundCompareHandler implements WebSocketEventHandler {
                 int ot5 = (int) (musicXmlInfo.getDuration()*0.22<70?70:musicXmlInfo.getDuration()*0.22);
                 int rightTimeRange = ot5>200?200:ot5;
 
-                // 待优化：结束时间应该是加偏移量
-                List<MusicPitchDetailDto> recordPitchs = userSoundInfoMap.get(phone).getRecordMeasurePitchInfos().stream().filter(m -> m.getTimeStamp()>=startTimeStamp-rightTimeRange && m.getTimeStamp() < endTimeStamp+rightTimeRange).collect(Collectors.toList());
+                List<MusicPitchDetailDto> recordPitchs = userSoundInfoMap.get(phone).getRecordMeasurePitchInfos().stream().filter(m -> m.getTimeStamp()>=startTimeStamp-rightTimeRange && m.getTimeStamp() < endTimeStamp-rightTimeRange).collect(Collectors.toList());
 
                 boolean cadenceRight = false;
                 boolean intonationRight = false;
@@ -411,10 +422,10 @@ public class SoundCompareHandler implements WebSocketEventHandler {
 
                     intonationScore += score;
                     musicXmlInfo.setAvgFrequency(avgPitch);
-                    intonationRight = score>70; //大于70分音准才正确？？？？？
+                    intonationRight = score>70;
 
                     integrityScore += integrityDuty;
-                    integrityRight = integrityDuty>0.7;//大于70%完整性才正确？？？？
+                    integrityRight = integrityDuty>0.7;
                 }
 
                 //如果当前音符不需要演奏

mec-common/common-core/pom.xml

@@ -73,9 +73,9 @@
 
 
 		<dependency>
-			<groupId>com.github.st-h</groupId>
+			<groupId>com.tarsos</groupId>
 			<artifactId>TarsosDSP</artifactId>
-			<version>2.4.1</version>
+			<version>0.0.1</version>
 		</dependency>
 	</dependencies>
 </project>