Skip to content

feat: Support autoplay audio tag #1982

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
liuruibin merged 1 commit into from
Jan 6, 2025
+91 −34
Merged

feat: Support autoplay audio tag #1982

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 45 additions & 17 deletions ui/src/components/ai-chat/component/operation-button/ChatOperationButton.vue
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@
</span>
</div>
<!-- 先渲染，不然不能播放 -->
<audio ref="audioPlayer" controls hidden="hidden"></audio>
<audio ref="audioPlayer" v-for="item in audioList" :key="item" controls hidden="hidden"></audio>
</template>
<script setup lang="ts">
import { onMounted, ref } from 'vue'
Expand Down Expand Up @@ -110,11 +110,13 @@ const props = withDefaults(

const emit = defineEmits(['update:data', 'regeneration'])

const audioPlayer = ref<HTMLAudioElement | null>(null)
const audioPlayer = ref<HTMLAudioElement[] | null>([])
const audioPlayerStatus = ref(false)
const buttonData = ref(props.data)
const loading = ref(false)
const utterance = ref<SpeechSynthesisUtterance | null>(null)
const audioList = ref<string[]>([])
const currentAudioIndex = ref(0)

function regeneration() {
emit('regeneration')
Expand Down Expand Up @@ -170,35 +172,56 @@ const playAnswerText = (text: string) => {
text = markdownToPlainText(text)
// console.log(text)
audioPlayerStatus.value = true
if (props.tts_type === 'BROWSER') {
if (text !== utterance.value?.text) {
// 分割成多份
audioList.value = text.split(/(<audio[^>]*><\/audio>)/)
playAnswerTextPart()
}

const playAnswerTextPart = () => {
// console.log(audioList.value, currentAudioIndex.value)
if (currentAudioIndex.value === audioList.value.length) {
audioPlayerStatus.value = false
currentAudioIndex.value = 0
return
}
if (audioList.value[currentAudioIndex.value].includes('<audio')) {
if (audioPlayer.value) {
audioPlayer.value[currentAudioIndex.value].src = audioList.value[currentAudioIndex.value].match(/src="([^"]*)"/)?.[1] || ''
audioPlayer.value[currentAudioIndex.value].play() // 自动播放音频
audioPlayer.value[currentAudioIndex.value].onended = () => {
currentAudioIndex.value += 1
playAnswerTextPart()
}
}
} else if (props.tts_type === 'BROWSER') {
if (audioList.value[currentAudioIndex.value] !== utterance.value?.text) {
window.speechSynthesis.cancel()
}
if (window.speechSynthesis.paused) {
window.speechSynthesis.resume()
return
}
// 创建一个新的 SpeechSynthesisUtterance 实例
utterance.value = new SpeechSynthesisUtterance(text)
utterance.value = new SpeechSynthesisUtterance(audioList.value[currentAudioIndex.value])
utterance.value.onend = () => {
audioPlayerStatus.value = false
utterance.value = null
currentAudioIndex.value += 1
playAnswerTextPart()
}
utterance.value.onerror = () => {
audioPlayerStatus.value = false
utterance.value = null
}
// 调用浏览器的朗读功能
window.speechSynthesis.speak(utterance.value)
}
if (props.tts_type === 'TTS') {
} else if (props.tts_type === 'TTS') {
// 恢复上次暂停的播放
if (audioPlayer.value?.src) {
audioPlayer.value?.play()
if (audioPlayer.value && audioPlayer.value[currentAudioIndex.value]?.src) {
audioPlayer.value[currentAudioIndex.value].play()
return
}
applicationApi
.postTextToSpeech((props.applicationId as string) || (id as string), { text: text }, loading)
.postTextToSpeech((props.applicationId as string) || (id as string), { text: audioList.value[currentAudioIndex.value] }, loading)
.then(async (res: any) => {
if (res.type === 'application/json') {
const text = await res.text()
Expand All @@ -219,11 +242,12 @@ const playAnswerText = (text: string) => {
// link.click()

// 检查 audioPlayer 是否已经引用了 DOM 元素
if (audioPlayer.value instanceof HTMLAudioElement) {
audioPlayer.value.src = url
audioPlayer.value.play() // 自动播放音频
audioPlayer.value.onended = () => {
audioPlayerStatus.value = false
if (audioPlayer.value) {
audioPlayer.value[currentAudioIndex.value].src = url
audioPlayer.value[currentAudioIndex.value].play() // 自动播放音频
audioPlayer.value[currentAudioIndex.value].onended = () => {
currentAudioIndex.value += 1
playAnswerTextPart()
}
} else {
console.error('audioPlayer.value is not an instance of HTMLAudioElement')
Expand All @@ -238,7 +262,11 @@ const playAnswerText = (text: string) => {
const pausePlayAnswerText = () => {
audioPlayerStatus.value = false
if (props.tts_type === 'TTS') {
audioPlayer.value?.pause()
if (audioPlayer.value) {
audioPlayer.value?.forEach((item) => {
item.pause()
})
}
}
if (props.tts_type === 'BROWSER') {
window.speechSynthesis.pause()
Expand Down
63 changes: 46 additions & 17 deletions ui/src/components/ai-chat/component/operation-button/LogOperationButton.vue
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@
<EditContentDialog ref="EditContentDialogRef" @refresh="refreshContent" />
<EditMarkDialog ref="EditMarkDialogRef" @refresh="refreshMark" />
<!-- 先渲染，不然不能播放 -->
<audio ref="audioPlayer" controls hidden="hidden"></audio>
<audio ref="audioPlayer" v-for="item in audioList" :key="item" controls hidden="hidden"></audio>
</div>
</div>
</template>
Expand Down Expand Up @@ -88,14 +88,16 @@ const props = defineProps({

const emit = defineEmits(['update:data'])

const audioPlayer = ref<HTMLAudioElement | null>(null)
const audioPlayer = ref<HTMLAudioElement[] | null>(null)

const EditContentDialogRef = ref()
const EditMarkDialogRef = ref()

const buttonData = ref(props.data)
const loading = ref(false)
const utterance = ref<SpeechSynthesisUtterance | null>(null)
const audioList = ref<string[]>([])
const currentAudioIndex = ref(0)

function editContent(data: any) {
EditContentDialogRef.value.open(data)
Expand Down Expand Up @@ -149,35 +151,56 @@ const playAnswerText = (text: string) => {
text = markdownToPlainText(text)
// console.log(text)
audioPlayerStatus.value = true
if (props.tts_type === 'BROWSER') {
if (text !== utterance.value?.text) {
// 分割成多份
audioList.value = text.split(/(<audio[^>]*><\/audio>)/)
playAnswerTextPart()
}

const playAnswerTextPart = () => {
// console.log(audioList.value, currentAudioIndex.value)
if (currentAudioIndex.value === audioList.value.length) {
audioPlayerStatus.value = false
currentAudioIndex.value = 0
return
}
if (audioList.value[currentAudioIndex.value].includes('<audio')) {
if (audioPlayer.value) {
audioPlayer.value[currentAudioIndex.value].src = audioList.value[currentAudioIndex.value].match(/src="([^"]*)"/)?.[1] || ''
audioPlayer.value[currentAudioIndex.value].play() // 自动播放音频
audioPlayer.value[currentAudioIndex.value].onended = () => {
currentAudioIndex.value += 1
playAnswerTextPart()
}
}
} else if (props.tts_type === 'BROWSER') {
if (audioList.value[currentAudioIndex.value] !== utterance.value?.text) {
window.speechSynthesis.cancel()
}
if (window.speechSynthesis.paused) {
window.speechSynthesis.resume()
return
}
// 创建一个新的 SpeechSynthesisUtterance 实例
utterance.value = new SpeechSynthesisUtterance(text)
utterance.value = new SpeechSynthesisUtterance(audioList.value[currentAudioIndex.value])
utterance.value.onend = () => {
audioPlayerStatus.value = false
utterance.value = null
currentAudioIndex.value += 1
playAnswerTextPart()
}
utterance.value.onerror = () => {
audioPlayerStatus.value = false
utterance.value = null
}
// 调用浏览器的朗读功能
window.speechSynthesis.speak(utterance.value)
}
if (props.tts_type === 'TTS') {
} else if (props.tts_type === 'TTS') {
// 恢复上次暂停的播放
if (audioPlayer.value?.src) {
audioPlayer.value?.play()
if (audioPlayer.value && audioPlayer.value[currentAudioIndex.value]?.src) {
audioPlayer.value[currentAudioIndex.value].play()
return
}
applicationApi
.postTextToSpeech(id || (props.applicationId as string), { text: text }, loading)
.postTextToSpeech((props.applicationId as string) || (id as string), { text: audioList.value[currentAudioIndex.value] }, loading)
.then(async (res: any) => {
if (res.type === 'application/json') {
const text = await res.text()
Expand All @@ -198,11 +221,12 @@ const playAnswerText = (text: string) => {
// link.click()

// 检查 audioPlayer 是否已经引用了 DOM 元素
if (audioPlayer.value instanceof HTMLAudioElement) {
audioPlayer.value.src = url
audioPlayer.value.play() // 自动播放音频
audioPlayer.value.onended = () => {
audioPlayerStatus.value = false
if (audioPlayer.value) {
audioPlayer.value[currentAudioIndex.value].src = url
audioPlayer.value[currentAudioIndex.value].play() // 自动播放音频
audioPlayer.value[currentAudioIndex.value].onended = () => {
currentAudioIndex.value += 1
playAnswerTextPart()
}
} else {
console.error('audioPlayer.value is not an instance of HTMLAudioElement')
Expand All @@ -217,13 +241,18 @@ const playAnswerText = (text: string) => {
const pausePlayAnswerText = () => {
audioPlayerStatus.value = false
if (props.tts_type === 'TTS') {
audioPlayer.value?.pause()
if (audioPlayer.value) {
audioPlayer.value?.forEach((item) => {
item.pause()
})
}
}
if (props.tts_type === 'BROWSER') {
window.speechSynthesis.pause()
}
}


function refreshMark() {
buttonData.value.improve_paragraph_id_list = []
emit('update:data', buttonData.value)
Expand Down
Loading