mec/audio-analysis/src/main/java/com/yonge/netty/server/service/AudioCompareHandler.java

package com.yonge.netty.server.service;

import io.netty.channel.Channel;

import java.io.File;
import java.math.BigDecimal;
import java.text.SimpleDateFormat;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.stream.Collectors;

import javax.sound.sampled.AudioFormat;

import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.JSONPath;
import com.ym.mec.auth.api.client.SysUserFeignService;
import com.ym.mec.biz.dal.entity.SysMusicCompareRecord;
import com.ym.mec.biz.dal.enums.DeviceTypeEnum;
import com.ym.mec.biz.dal.enums.FeatureType;
import com.ym.mec.biz.dal.enums.HeardLevelEnum;
import com.ym.mec.biz.service.SysMusicCompareRecordService;
import com.ym.mec.thirdparty.storage.StoragePluginContext;
import com.ym.mec.thirdparty.storage.provider.KS3StoragePlugin;
import com.ym.mec.util.upload.UploadUtil;
import com.yonge.audio.analysis.AudioFloatConverter;
import com.yonge.audio.utils.ArrayUtil;
import com.yonge.netty.dto.SectionAnalysis;
import com.yonge.netty.dto.UserChannelContext;
import com.yonge.netty.dto.WebSocketResponse;
import com.yonge.netty.entity.MusicXmlBasicInfo;
import com.yonge.netty.entity.MusicXmlNote;
import com.yonge.netty.server.handler.NettyChannelManager;
import com.yonge.netty.server.handler.message.MessageHandler;
import com.yonge.netty.server.processor.WaveformWriter;

@Component
public class AudioCompareHandler implements MessageHandler {
	
	private static final Logger LOGGER = LoggerFactory.getLogger(AudioCompareHandler.class);

	@Autowired
	private UserChannelContextService userChannelContextService;

	@Autowired
	private NettyChannelManager nettyChannelManager;
	
	@Autowired
	private SysMusicCompareRecordService sysMusicCompareRecordService;
	
    @Autowired
    private SysUserFeignService sysUserFeignService;

    @Autowired
    private StoragePluginContext storagePluginContext;

	/**
	 * @describe 采样率
	 */
	private float sampleRate = 44100;

	/**
	 * 每个采样大小（Bit）
	 */
	private int bitsPerSample = 16;

	/**
	 * 通道数
	 */
	private int channels = 1;

	/**
	 * @describe 采样大小
	 */
	private int bufferSize = 1024 * 2;

	private boolean signed = true;

	private boolean bigEndian = false;

	private AudioFormat audioFormat = new AudioFormat(sampleRate, bitsPerSample, channels, signed, bigEndian);

	private AudioFloatConverter converter = AudioFloatConverter.getConverter(audioFormat);

	private String tmpFileDir = "/mdata/soundCompare/";

	private SimpleDateFormat sdf = new SimpleDateFormat("yyMMddHHmmSS");
	
	@Override
	public String getAction() {
		return "SOUND_COMPARE";
	}

	@Override
	public boolean handleTextMessage(String user, Channel channel, String jsonMsg) {
		
		String command = (String) JSONPath.extract(jsonMsg, "$.header.commond");

		JSONObject dataObj = (JSONObject) JSONPath.extract(jsonMsg, "$.body");
		
		UserChannelContext channelContext = userChannelContextService.getChannelContext(channel);
		
		MusicXmlBasicInfo musicXmlBasicInfo = null;

		switch (command) {
		case "musicXml": // 同步music xml信息
			
			musicXmlBasicInfo = JSONObject.toJavaObject(dataObj, MusicXmlBasicInfo.class);
			
			userChannelContextService.remove(channel);

			channelContext = new UserChannelContext();
			
			channelContext.setHandlerSwitch(false);

			channelContext.getSongMusicXmlMap().put(musicXmlBasicInfo.getExamSongId(), musicXmlBasicInfo);
			channelContext.init(musicXmlBasicInfo, audioFormat.getSampleRate(), bufferSize / 2);
			channelContext.setUser(user);
			
			userChannelContextService.register(channel, channelContext);

			break;
		case "recordStart": // 开始评测

			// 清空缓存信息
			channelContext.resetUserInfo();
			
			channelContext.setHandlerSwitch(false);
			
			musicXmlBasicInfo = channelContext.getMusicXmlBasicInfo(null);

			if (musicXmlBasicInfo != null) {
				Date date = new Date();
				SysMusicCompareRecord sysMusicCompareRecord = new SysMusicCompareRecord(FeatureType.CLOUD_STUDY_EVALUATION);
				sysMusicCompareRecord.setCreateTime(date);
				sysMusicCompareRecord.setUserId(Integer.parseInt(user));
				sysMusicCompareRecord.setSysMusicScoreId(musicXmlBasicInfo.getExamSongId());
				sysMusicCompareRecord.setBehaviorId(musicXmlBasicInfo.getBehaviorId());
				sysMusicCompareRecord.setClientId(musicXmlBasicInfo.getClientId());
				sysMusicCompareRecord.setDeviceType(DeviceTypeEnum.valueOf(musicXmlBasicInfo.getPlatform()));
				sysMusicCompareRecord.setSpeed(musicXmlBasicInfo.getSpeed());
				sysMusicCompareRecord.setPartIndex(musicXmlBasicInfo.getPartIndex());
				
				//SysUser sysUser = sysUserFeignService.queryUserById(sysMusicCompareRecord.getUserId());
				sysMusicCompareRecord.setTenantId(musicXmlBasicInfo.getTenantId());
				
				MusicXmlNote musicXmlNote = musicXmlBasicInfo.getMusicXmlInfos().stream().max(Comparator.comparing(MusicXmlNote::getTimeStamp)).get();
				sysMusicCompareRecord.setSourceTime((float) ((musicXmlNote.getTimeStamp()+musicXmlNote.getDuration())/1000));
				sysMusicCompareRecordService.insert(sysMusicCompareRecord,musicXmlBasicInfo.getCampId());
				channelContext.setRecordId(sysMusicCompareRecord.getId());
			}
			break;
		case "recordEnd": // 结束评测
		case "recordCancel": // 取消评测
			if (channelContext == null) {
				return false;
			}
			
			channelContext.setHandlerSwitch(false);

			WaveformWriter waveFileProcessor = channelContext.getWaveFileProcessor();
			if (waveFileProcessor != null) {
				// 写文件头
				waveFileProcessor.processingFinished();
			}

			if (StringUtils.equals(command, "recordEnd")) {
				// 生成评测报告
				Map<String, Object> params = new HashMap<String, Object>();

				Map<String, Integer> scoreMap = channelContext.evaluateForMusic();
				for (Entry<String, Integer> entry : scoreMap.entrySet()) {
					params.put(entry.getKey(), entry.getValue());
				}
				
				//保存评测结果
				Long recordId = channelContext.getRecordId();
				SysMusicCompareRecord sysMusicCompareRecord = sysMusicCompareRecordService.get(recordId);
				if(sysMusicCompareRecord != null){
					musicXmlBasicInfo = channelContext.getMusicXmlBasicInfo(null);
					
					if (scoreMap != null && scoreMap.size() > 1) {
						sysMusicCompareRecord.setScore(new BigDecimal(scoreMap.get("score")));
						sysMusicCompareRecord.setIntonation(new BigDecimal(scoreMap.get("intonation")));
						sysMusicCompareRecord.setIntegrity(new BigDecimal(scoreMap.get("integrity")));
						sysMusicCompareRecord.setCadence(new BigDecimal(scoreMap.get("cadence")));
						sysMusicCompareRecord.setPlayTime(scoreMap.get("playTime") / 1000);
						
						LOGGER.info("Score:{} Intonation:{} Integrity:{} Cadence:{}", sysMusicCompareRecord.getScore(),sysMusicCompareRecord.getIntonation(),sysMusicCompareRecord.getIntegrity(),sysMusicCompareRecord.getCadence());
					}
					sysMusicCompareRecord.setFeature(FeatureType.CLOUD_STUDY_EVALUATION);

		            String url = null;
		            try {
		                String folder = UploadUtil.getFileFloder();
		                url = storagePluginContext.asyncUploadFile(KS3StoragePlugin.PLUGIN_NAME,"cloud-coach/" + folder, waveFileProcessor.getFile(), true);
		            } catch (Exception e) {
		                LOGGER.error("录音文件上传失败：{}", e);
		            }
					sysMusicCompareRecord.setRecordFilePath(url);
					//sysMusicCompareRecord.setVideoFilePath(videoFilePath);

					Map<String, Object> scoreData = new HashMap<>();
					List<SectionAnalysis> sectionAnalysisList = channelContext.getDoneSectionAnalysisList();
					sectionAnalysisList = sectionAnalysisList.stream().filter(t -> t.isIngore() == false).collect(Collectors.toList());
					scoreData.put("userMeasureScore", sectionAnalysisList.stream().collect(Collectors.toMap(SectionAnalysis :: getIndex, t -> t)));

					Map<String, Object> musicalNotesPlayStats = new HashMap<>();
					musicalNotesPlayStats.put("detailId", musicXmlBasicInfo.getDetailId());
					musicalNotesPlayStats.put("examSongId", musicXmlBasicInfo.getExamSongId());
					musicalNotesPlayStats.put("xmlUrl", musicXmlBasicInfo.getXmlUrl());
					
					musicalNotesPlayStats.put("notesData", channelContext.getDoneNoteAnalysisList().stream().filter(t -> t.isIgnore() == false).collect(Collectors.toList()));
					scoreData.put("musicalNotesPlayStats", musicalNotesPlayStats);
					sysMusicCompareRecord.setScoreData(JSON.toJSONString(scoreData));
					
					sysMusicCompareRecord.setHeardLevel(HeardLevelEnum.valueOf(channelContext.getHardLevel().name()));
					
					sysMusicCompareRecordService.saveMusicCompareData(sysMusicCompareRecord);
				}
				
				int totalPlayTimeOfCurrentDate = sysMusicCompareRecordService.queryCurrentDatePlayTimeByUserId(sysMusicCompareRecord.getUserId());
				
				params.put("totalPlayTimeOfCurrentDate", totalPlayTimeOfCurrentDate);
				
				WebSocketResponse<Map<String, Object>> resp = new WebSocketResponse<Map<String, Object>>("overall", params);

				nettyChannelManager.sendTextMessage(user, resp);
			}

			// 清空缓存信息
			channelContext.resetUserInfo();

			break;
		case "audioPlayStart": // ???
			
			Integer offsetTime = dataObj.getInteger("offsetTime");
			if(offsetTime != null){
				channelContext.setOffsetMS(offsetTime);
				channelContext.setHandlerSwitch(true);
			}

			break;
		case "videoUpload": // 上传音频
			SysMusicCompareRecord musicCompareRecord = null;
			if (dataObj.containsKey("recordId")) {
				musicCompareRecord = sysMusicCompareRecordService.get(dataObj.getLong("recordId"));
			}
			if (Objects.nonNull(musicCompareRecord) && dataObj.containsKey("filePath")) {
				musicCompareRecord.setVideoFilePath(dataObj.getString("filePath"));
				sysMusicCompareRecordService.update(musicCompareRecord);
			} else {
				musicCompareRecord.setVideoFilePath(musicCompareRecord.getRecordFilePath());
				sysMusicCompareRecordService.update(musicCompareRecord);
			}
			
			break;

		default:
			// 非法请求
			break;
		}
		return true;
	}

	@Override
	public boolean handleBinaryMessage(String user, Channel channel, byte[] datas) {
		
		UserChannelContext channelContext = userChannelContextService.getChannelContext(channel);

		if (channelContext == null) {
			return false;
		}

		// 写录音文件
		WaveformWriter waveFileProcessor = channelContext.getWaveFileProcessor();
		if (waveFileProcessor == null) {
			File file = new File(tmpFileDir + user + "_" + sdf.format(new Date()) + ".wav");
			waveFileProcessor = new WaveformWriter(file.getAbsolutePath());
			channelContext.setWaveFileProcessor(waveFileProcessor);
		}
		waveFileProcessor.process(datas);
		
		/*datas = channelContext.skipMetronome(datas);

		if (datas.length == 0) {
			return false;
		}*/

		channelContext.setChannelBufferBytes(ArrayUtil.mergeByte(channelContext.getChannelBufferBytes(), datas));

		int totalLength = channelContext.getChannelBufferBytes().length;
		
		if (channelContext.getHandlerSwitch() == false) {
			return false;
		}
		
		if (channelContext.getOffsetMS() + channelContext.getBeatDuration() > 0) {
			int beatByteLength = (int) (audioFormat.getSampleRate() * audioFormat.getSampleSizeInBits() / 8 * (channelContext.getOffsetMS() + channelContext.getBeatDuration()) / 1000);
			
			if(totalLength > beatByteLength){
				if(beatByteLength % 2 != 0){
					LOGGER.debug("**************奇数*****************");
					beatByteLength--;
				}
				channelContext.setChannelBufferBytes(ArrayUtil.extractByte(channelContext.getChannelBufferBytes(), beatByteLength, totalLength - 1));
				
				LOGGER.debug("--------Length:{}  Times[{} + {}]:{}--------", waveFileProcessor.getFile().length() - channelContext.getChannelBufferBytes().length, channelContext.getOffsetMS() , channelContext.getBeatDuration(),(waveFileProcessor.getFile().length() - channelContext.getChannelBufferBytes().length) * 1000 /audioFormat.getSampleRate()/2);
				
				channelContext.setOffsetMS(0);
				channelContext.setBeatDuration(0);
			}else{
				return false;
			}
		}
		
		totalLength = channelContext.getChannelBufferBytes().length;
		

		while (totalLength >= bufferSize) {
			byte[] bufferData = ArrayUtil.extractByte(channelContext.getChannelBufferBytes(), 0, bufferSize - 1);

			if (bufferSize != totalLength) {
				channelContext.setChannelBufferBytes(ArrayUtil.extractByte(channelContext.getChannelBufferBytes(), bufferSize, totalLength - 1));
			} else {
				channelContext.setChannelBufferBytes(new byte[0]);
			}

			float[] sampleFloats = new float[bufferSize / 2];

			converter.toFloatArray(bufferData, sampleFloats);

			channelContext.handle(sampleFloats, audioFormat);

			MusicXmlBasicInfo musicXmlBasicInfo = channelContext.getMusicXmlBasicInfo(null);
			int sectionIndex = channelContext.getEvaluatingSectionIndex().get();

			// 评分
			int score = channelContext.evaluateForSection(sectionIndex, musicXmlBasicInfo.getSubjectId());
			if (score >= 0) {

				Map<String, Object> params = new HashMap<String, Object>();
				params.put("score", score);
				params.put("measureIndex", sectionIndex);
				params.put("measureRenderIndex", channelContext.getCurrentMusicSection(null, sectionIndex).getMeasureRenderIndex());

				WebSocketResponse<Map<String, Object>> resp = new WebSocketResponse<Map<String, Object>>("measureScore", params);

				nettyChannelManager.sendTextMessage(user, resp);
			}

			totalLength = channelContext.getChannelBufferBytes().length;
		}

		return true;
	}

}