package admin.server; import admin.config.SpeechToTextConfig; import admin.model.SendSseModel; import admin.model.WebSocketModel; import admin.model.python.PythonApiReq; import admin.modules.chatgpt.domain.ChatgptRoleRecords; import admin.modules.chatgpt.domain.ChatgptRoleSession; import admin.modules.chatgpt.service.ChatgptRoleRecordsService; import admin.modules.chatgpt.service.ChatgptRoleSessionService; import admin.service.MaxkbOpenApiService; import admin.service.PythonOpenApiClient; import admin.util.*; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import lombok.extern.slf4j.Slf4j; import okhttp3.HttpUrl; import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.ResponseBody; import org.springframework.stereotype.Component; import javax.annotation.Resource; import javax.crypto.Mac; import javax.crypto.spec.SecretKeySpec; import javax.websocket.*; import javax.websocket.server.PathParam; import javax.websocket.server.ServerEndpoint; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.net.URL; import java.nio.ByteBuffer; import java.nio.charset.Charset; import java.text.SimpleDateFormat; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Stream; /** * @author lj * @date 2025/2/20 19:46 */ @ServerEndpoint("/webSocket/voice/{memberId}") @Component @Slf4j public class VoiceWebSocketServer { private static SpeechToTextConfig voiceConfig; private static MaxkbOpenApiService maxkbOpenApiService; private static ChatgptRoleRecordsService chatgptRoleRecordsService; private static ChatgptRoleSessionService chatgptRoleSessionService; private static PythonOpenApiClient pythonOpenApiClient; @Resource public void setVoiceConfig(SpeechToTextConfig speechToTextConfig){ voiceConfig = speechToTextConfig; } @Resource public void setChatgptRoleRecordsService(ChatgptRoleRecordsService chatgptRoleRecordsService){ this.chatgptRoleRecordsService = chatgptRoleRecordsService; } @Resource public void setPythonOpenApiClient(PythonOpenApiClient pythonOpenApiClient){ this.pythonOpenApiClient = pythonOpenApiClient; } @Resource public void setChatgptRoleSessionService(ChatgptRoleSessionService chatgptRoleSessionService){ this.chatgptRoleSessionService = chatgptRoleSessionService; } @Resource public void setMaxkbOpenApiService(MaxkbOpenApiService maxkbOpenApiService){ VoiceWebSocketServer.maxkbOpenApiService = maxkbOpenApiService; } private static ConcurrentHashMap sessionPools = new ConcurrentHashMap<>(); //建立连接成功调用 @OnOpen public void onOpen(Session session, @PathParam(value = "memberId") String memberId) { String[] split = memberId.split("__"); //1.h5还是web端 String mark = split[0]; //2.城市名称 String name= split[1]; //3.sessionId String sessionId = split[2]; //4.存储不同的连接 if (mark.equals(WebSocketUtil.H5)){ H5ConnectUtil.add(name); // popUpEvent(name,"webOnOpen"); }else { WebConnectUtil.put(sessionId,name,session); } //5.获取最新的会话id String maxSessionId = chatgptRoleRecordsService.queryMaxSessionId(); WebSocketModel webSocketModel = new WebSocketModel(); webSocketModel.setType("text"); webSocketModel.setAction("replaySessionId"); webSocketModel.setContent(maxSessionId); webSocketModel.setSessionId(maxSessionId); //6.返回给前端最新的sessionId String msg = JSON.toJSONString(webSocketModel); try { session.getBasicRemote().sendText(msg); } catch (IOException e) { throw new RuntimeException(e); } } //关闭连接时调用 @OnClose public void onClose(Session session, CloseReason closeReason,@PathParam(value = "memberId") String memberId) { String[] split = memberId.split("__"); //1.h5还是web端 String mark = split[0]; //2.城市名称 String name= split[1]; //3.sessionId String sessionId = split[2]; //存储不同的连接 if (mark.equals(WebSocketUtil.H5)){ H5ConnectUtil.remove(name); popUpEvent(name,"webOnClose"); }else { WebConnectUtil.remove(sessionId); } } /** * web弹窗事件 */ private void popUpEvent(String name, String action) { WebSocketModel webSocketModel = new WebSocketModel(); webSocketModel.setType("text"); webSocketModel.setAction(action); String msg = JSONObject.toJSONString(webSocketModel); List webMap = WebConnectUtil.getWebMap(name); if (!webMap.isEmpty()){ webMap.forEach(i->{ try { i.getBasicRemote().sendText(msg); } catch (IOException e) { throw new RuntimeException(e); } }); } } //收到客户端信息 @OnMessage(maxMessageSize = 10*1024*1024) public void onTextMessage(Session session,String message,@PathParam(value = "memberId") String memberId) throws IOException { // session.setMaxBinaryMessageBufferSize(50 * 1024 * 1024); log.info("message:{}", message); WebSocketModel webSocketModel = JSON.parseObject(message, WebSocketModel.class); //1.如果问题不存在则创建问题 ChatgptRoleSession byId = chatgptRoleSessionService.getById(webSocketModel.getSessionId()); if (byId == null && "sendText".equals(webSocketModel.getAction())) { ChatgptRoleSession chatgptRoleSession = new ChatgptRoleSession(); chatgptRoleSession.setId(Integer.valueOf(webSocketModel.getSessionId())); chatgptRoleSession.setName(webSocketModel.getData()); chatgptRoleSession.setRoleDescId(10001); chatgptRoleSession.setUserId(ScreenEnum.getByUserName(webSocketModel.getName())); chatgptRoleSessionService.save(chatgptRoleSession); //创建问题记录 ChatgptRoleRecords chatgptRoleRecords = new ChatgptRoleRecords(); chatgptRoleRecords.setMessage(webSocketModel.getData()); chatgptRoleRecords.setReceiverId(10001); chatgptRoleRecords.setSessionId(Integer.valueOf(webSocketModel.getSessionId())); chatgptRoleRecords.setSenderId(ScreenEnum.getByUserName(webSocketModel.getName())); chatgptRoleRecords.setCreateTime(DateUtil.getNowTimestamp()); chatgptRoleRecords.setIsAsk(1); chatgptRoleRecordsService.save(chatgptRoleRecords); //如果是H5则发送给大屏消息 sendMsgToWeb(memberId,webSocketModel.getData()); } else if (byId != null && "sendText".equals(webSocketModel.getAction())) { ChatgptRoleRecords chatgptRoleRecords = new ChatgptRoleRecords(); chatgptRoleRecords.setMessage(webSocketModel.getData()); chatgptRoleRecords.setReceiverId(10001); chatgptRoleRecords.setSessionId(Integer.valueOf(webSocketModel.getSessionId())); chatgptRoleRecords.setSenderId(ScreenEnum.getByUserName(webSocketModel.getName())); chatgptRoleRecords.setIsAsk(1); chatgptRoleRecords.setCreateTime(DateUtil.getNowTimestamp()); chatgptRoleRecordsService.save(chatgptRoleRecords); //如果是H5则发送给大屏消息 sendMsgToWeb(memberId,webSocketModel.getData()); } // 文字聊天 if ("sendText".equals(webSocketModel.getAction())) { String msg = webSocketModel.getData(); if (msg != null && !msg.isEmpty()) { ResponseEnum showType = getShowType(msg); SendSseModel sendSseModel = new SendSseModel(msg, webSocketModel.getSessionId(), session, webSocketModel.getName(), memberId.split("__")[0].equals(WebSocketUtil.H5),showType); maxkbOpenApiService.talk(sendSseModel); } // 心跳 }else if("heartbeat".equals(webSocketModel.getAction())){ WebSocketModel webSocketModel1 = new WebSocketModel(); webSocketModel1.setAction("heartbeat_ack"); webSocketModel1.setType("text"); webSocketModel1.setContent(""); String msg = JSON.toJSONString(webSocketModel1); sendMessage(session,msg); }else if ("startAnalysis".equals(webSocketModel.getAction())){ PythonApiReq pythonApiReq = new PythonApiReq(); pythonApiReq.setQuestion(webSocketModel.getData()); String analysisData = pythonOpenApiClient.getAnalysisData(pythonApiReq); char[] charArray = analysisData.toCharArray(); for (int i=0;i map1 = WebConnectUtil.getMap(); List webMap = WebConnectUtil.getWebMap(name); if (!webMap.isEmpty()){ webMap.forEach(i->{ try { Map map = new HashMap<>(); // 用于标识是h5端问的,web收到收展示问的内容 map.put("action","askReply"); map.put("type","text"); map.put("content",content); String msg = JSON.toJSONString(map); i.getBasicRemote().sendText(msg); } catch (IOException e) { throw new RuntimeException(e); } }); } } } //错误时调用 @OnError public void onError(Session session, Throwable throwable) { System.out.println("发生错误"); for (Map.Entry map : sessionPools.entrySet()) { Session value = map.getValue(); if (value.equals(session)) { sessionPools.remove(map.getKey()); } } throwable.printStackTrace(); } //发送消息 public void sendMessage(Session session, String message) { if (session != null) { synchronized (session) { try { session.getBasicRemote().sendText(message); } catch (IOException e) { throw new RuntimeException(e); } } } } /** * 处理音频数据 * @param session * @param data */ private void handleAudioChunk(Session session, ByteBuffer data) { //1.音频转文本 speechToText(data); //2.向ai提问获取答案 //3.文本转语音 // textToSpeech(); sendMessage(session, null); } public ResponseEnum getShowType(String msg){ if (msg.contains("汤波")){ return ResponseEnum.ZK_TB; }else if (msg.contains("李乐")){ return ResponseEnum.ZK_LL; }else if (msg.contains("吕泊岚")){ return ResponseEnum.ZK_LBL; }else if (msg.contains("孙巍巍")){ return ResponseEnum.ZK_SWW; } else if (msg.contains("刘照琳")){ return ResponseEnum.ZK_LZL; } else if (msg.contains("钟华")){ return ResponseEnum.ZK_ZH; } else if (msg.contains("李海青")){ return ResponseEnum.ZK_LHQ; } else if (msg.contains("冯涛")){ return ResponseEnum.ZK_FT; } else if (msg.contains("刘瑞华")){ return ResponseEnum.ZK_LRH; } else if (msg.contains("中科慧居") && (msg.contains("运营平台"))){ return ResponseEnum.ZK_SMH; } else if (msg.contains("中科慧居") && (msg.contains("小程序") || msg.contains("客户端小程序"))){ return ResponseEnum.ZK_SMH; } else if (msg.contains("中科慧居") && (msg.contains("手持机") || msg.contains("手持机收费终端"))){ return ResponseEnum.ZK_SMH; } else if (msg.contains("中科慧居") && (msg.contains("超级魔盒") || msg.contains("物联网超级魔盒"))){ return ResponseEnum.ZK_SMH; }else if (msg.contains("中科慧居") && (msg.contains("无线高位视频") || msg.contains("ETC系列设备"))){ return ResponseEnum.ZK_WHETC; }else if (msg.contains("中科慧居") &&(msg.contains("无线视频桩") || msg.contains("视频桩"))){ return ResponseEnum.ZK_WVP; }else if (msg.contains("中科慧居") && (msg.contains("无线路牙机") || msg.contains("路牙机"))){ return ResponseEnum.ZK_WCC; }else if (msg.contains("中科慧居") && msg.contains("巡检车")){ return ResponseEnum.ZK_IRV; }else if (msg.contains("中科慧居") && (msg.contains("太阳能平板地锁") || msg.contains("地锁"))){ return ResponseEnum.ZK_SPPL; }else if (msg.contains("中科慧居") && msg.contains("立体车库")){ return ResponseEnum.ZK_APS; }else if (msg.contains("中科慧居") && msg.contains("自助洗车一体化")){ return ResponseEnum.ZK_ISCW; }else if (msg.contains("中科慧居") && msg.contains("停充储一体化")){ return ResponseEnum.ZK_IPCS; } else if (msg.contains("中科慧居") && msg.contains("产品介绍")){ return ResponseEnum.ZK_PRODUCT; }else if (msg.contains("中科慧居") && msg.contains("人员介绍")){ return ResponseEnum.ZK_USER; }else if (msg.contains("中科慧居") && msg.contains("软件介绍")){ return ResponseEnum.ZK_SOFTWARE; }else if (msg.contains("中科慧居") && msg.contains("简介")){ return ResponseEnum.ZK_PROFILE; }else if (msg.contains( ResponseEnum.PIE_CHART.getDesc())){ return ResponseEnum.PIE_CHART; }else if (msg.contains( ResponseEnum.BAR_CHART.getDesc())){ return ResponseEnum.BAR_CHART; }else if (msg.contains( ResponseEnum.TREND_CHART.getDesc())){ return ResponseEnum.TREND_CHART; } return null; } /** * 语音转文本 * @param data 语音数据 */ private void speechToText(ByteBuffer data) { byte[] bytes = new byte[data.remaining()]; ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); try { byteArrayOutputStream.write(bytes); String authUrl = null; authUrl = getAuthUrl(voiceConfig.getHostUrl(), voiceConfig.getApiKey(), voiceConfig.getApiSecret()); OkHttpClient client = new OkHttpClient.Builder().build(); String url = authUrl.toString().replace("http://", "ws://").replace("https://", "wss://"); Request request = new Request.Builder().url(url).build(); ResponseBody body = client.newCall(request).execute().body(); System.out.println(body.string()); // WebSocket webSocket = client.newWebSocket(request, new ResponseData()); } catch (Exception e) { throw new RuntimeException(e); } } public String getAuthUrl(String hostUrl, String apiKey, String apiSecret) throws Exception { URL url = new URL(hostUrl); SimpleDateFormat format = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss z", Locale.US); format.setTimeZone(TimeZone.getTimeZone("GMT")); String date = format.format(new Date()); StringBuilder builder = new StringBuilder("host: ").append(url.getHost()).append("\n").// append("date: ").append(date).append("\n").// append("GET ").append(url.getPath()).append(" HTTP/1.1"); Charset charset = Charset.forName("UTF-8"); Mac mac = Mac.getInstance("hmacsha256"); SecretKeySpec spec = new SecretKeySpec(apiSecret.getBytes(charset), "hmacsha256"); mac.init(spec); byte[] hexDigits = mac.doFinal(builder.toString().getBytes(charset)); String sha = Base64.getEncoder().encodeToString(hexDigits); String authorization = String.format("api_key=\"%s\", algorithm=\"%s\", headers=\"%s\", signature=\"%s\"", apiKey, "hmac-sha256", "host date request-line", sha); HttpUrl httpUrl = HttpUrl.parse("https://" + url.getHost() + url.getPath()).newBuilder().// addQueryParameter("authorization", Base64.getEncoder().encodeToString(authorization.getBytes(charset))).// addQueryParameter("date", date).// addQueryParameter("host", url.getHost()).// build(); return httpUrl.toString(); } }