Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions openless-all/app/src-tauri/src/asr/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ pub mod local;
pub mod mimo;
pub mod pcm;
pub mod qwen_realtime;
pub mod stepfun_realtime;
pub mod volcengine;
pub mod wav;
pub mod whisper;
Expand All @@ -22,6 +23,7 @@ pub use dashscope_multimodal::DashScopeMultimodalASR;
pub use elevenlabs::ElevenLabsBatchASR;
pub use mimo::MimoBatchASR;
pub use qwen_realtime::{Qwen3RealtimeASR, Qwen3RealtimeCredentials};
pub use stepfun_realtime::{StepfunRealtimeASR, StepfunRealtimeCredentials};
pub use volcengine::{VolcengineCredentials, VolcengineStreamingASR};
pub use whisper::WhisperBatchASR;

Expand Down
3 changes: 2 additions & 1 deletion openless-all/app/src-tauri/src/asr/qwen_realtime.rs
Original file line number Diff line number Diff line change
Expand Up @@ -576,7 +576,8 @@ fn drain_audio_chunks(buffer: &mut Vec<u8>) -> Vec<Vec<u8>> {
}

/// VAD 句段拼接:CJK 之间直接相连;拉丁词之间补空格,避免英文句段黏连。
fn join_segments(segments: &[String]) -> String {
/// `stepfun_realtime` 的多句段收尾复用同一套拼接逻辑,故 `pub(crate)`。
pub(crate) fn join_segments(segments: &[String]) -> String {
let mut joined = String::new();
for seg in segments.iter().map(|s| s.trim()) {
if seg.is_empty() {
Expand Down
1,089 changes: 1,089 additions & 0 deletions openless-all/app/src-tauri/src/asr/stepfun_realtime.rs

Large diffs are not rendered by default.

82 changes: 82 additions & 0 deletions openless-all/app/src-tauri/src/asr/whisper.rs
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,9 @@ pub struct WhisperBatchASR {
verbose_json: bool,
/// 请求体编码方式。默认 `Multipart`,OpenRouter 走 `OpenRouterJson`。
request_format: AsrRequestFormat,
/// 一等 `hotwords` 参数(JSON 数组字符串)。StepFun 等厂商不认 `prompt`
/// (静默忽略),但提供专门的热词字段——用它词典才真正生效。空 = 不发。
hotwords: Vec<String>,
buffer: Mutex<Vec<u8>>,
}

Expand All @@ -72,6 +75,7 @@ impl WhisperBatchASR {
max_chunk_duration_ms,
verbose_json,
request_format: AsrRequestFormat::Multipart,
hotwords: Vec::new(),
buffer: Mutex::new(Vec::new()),
}
}
Expand All @@ -83,6 +87,13 @@ impl WhisperBatchASR {
self
}

/// 设置一等热词列表(默认空 = 不发)。仅 Multipart 编码下生效;与 `prompt`
/// 二选一由 wiring 决定(见 coordinator 的 `whisper_uses_hotwords`)。
pub fn with_hotwords(mut self, hotwords: Vec<String>) -> Self {
self.hotwords = hotwords;
self
}

/// Stop collecting audio, encode the buffer as WAV, and POST to the
/// Whisper transcriptions endpoint.
///
Expand Down Expand Up @@ -172,6 +183,20 @@ impl WhisperBatchASR {
}
}

// 一等热词(StepFun 形状:可解析的 JSON 数组字符串,如
// `["热词1","热词2"]`)。空白词条过滤后再编码,全空则不发该字段。
let hotwords: Vec<&str> = self
.hotwords
.iter()
.map(|w| w.trim())
.filter(|w| !w.is_empty())
.collect();
if !hotwords.is_empty() {
if let Ok(encoded) = serde_json::to_string(&hotwords) {
form = form.text("hotwords", encoded);
}
}

client
.post(&url)
.header("Authorization", format!("Bearer {}", self.api_key))
Expand Down Expand Up @@ -833,6 +858,63 @@ mod tests {
server.join().unwrap();
}

#[tokio::test]
async fn hotwords_sent_as_json_array_field_without_prompt() {
// StepFun 形状:词典走一等 `hotwords`(JSON 数组字符串)而非 `prompt`;
// 空白词条过滤后编码。
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
listener.set_nonblocking(true).unwrap();
let addr = listener.local_addr().unwrap();
let server = thread::spawn(move || {
let deadline = Instant::now() + Duration::from_secs(5);
let mut stream = loop {
match listener.accept() {
Ok((stream, _)) => break stream,
Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => {
assert!(
Instant::now() < deadline,
"timed out waiting for ASR test request"
);
thread::sleep(Duration::from_millis(10));
}
Err(err) => panic!("accept ASR test request failed: {err}"),
}
};
stream.set_nonblocking(false).unwrap();
stream
.set_read_timeout(Some(Duration::from_secs(5)))
.unwrap();
let request = read_http_request(&mut stream);
let request_text = String::from_utf8_lossy(&request);
assert!(request_text.starts_with("POST /audio/transcriptions HTTP/1.1"));
assert!(request_text.contains(r#"name="hotwords""#));
assert!(request_text.contains(r#"["阶跃星辰","OpenLess"]"#));
assert!(!request_text.contains(r#"name="prompt""#));
write_json_response(&mut stream, r#"{"text":"hotwords ok"}"#);
});
let base_url = format!("http://{}", addr);

let asr = WhisperBatchASR::new(
"key".to_string(),
base_url,
"stepaudio-2.5-asr".to_string(),
None,
None,
false,
)
.with_hotwords(vec![
"阶跃星辰".to_string(),
" ".to_string(),
"OpenLess".to_string(),
]);
let pcm = vec![0u8; 32_000 * 2];
asr.consume_pcm_chunk(&pcm);

let transcript = asr.transcribe().await.unwrap();
assert_eq!(transcript.text, "hotwords ok");
server.join().unwrap();
}

fn start_whisper_test_server(texts: Vec<&'static str>) -> (String, thread::JoinHandle<()>) {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
listener.set_nonblocking(true).unwrap();
Expand Down
85 changes: 83 additions & 2 deletions openless-all/app/src-tauri/src/commands/providers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -270,6 +270,18 @@ async fn validate_asr_provider() -> Result<(), String> {
if active_asr == crate::asr::elevenlabs::PROVIDER_ID {
return validate_elevenlabs_asr_provider().await;
}
// StepFun 一入口双协议:`*-stream` 模型走实时 WS 验证,其余走批式
// /audio/transcriptions(与 build 侧 resolve_effective_asr_provider 同判据)。
if active_asr == "stepfun" || active_asr == crate::asr::stepfun_realtime::PROVIDER_ID {
let model = CredentialsVault::get(CredentialAccount::AsrModel)
.map_err(|e| e.to_string())?
.unwrap_or_default();
if active_asr == crate::asr::stepfun_realtime::PROVIDER_ID
|| crate::coordinator::stepfun_model_is_stream(&model)
{
return validate_stepfun_realtime_asr_provider().await;
}
}

let config = read_openai_provider_config("asr")?;
let model = CredentialsVault::get(CredentialAccount::AsrModel)
Expand All @@ -279,6 +291,43 @@ async fn validate_asr_provider() -> Result<(), String> {
validate_asr_transcription(&config, model.trim()).await
}

/// StepFun 实时 WS 验证:真连 + session.update + 500ms 静音 + 收尾。
/// 协议无 finish 事件,收尾走静音帧 + 宽限期(纯静音会话以空文本成功返回,
/// 见 stepfun_realtime 模块注释),全程 ~2s。
async fn validate_stepfun_realtime_asr_provider() -> Result<(), String> {
let api_key = CredentialsVault::get(CredentialAccount::AsrApiKey)
.map_err(|e| e.to_string())?
.unwrap_or_default();
if api_key.trim().is_empty() {
return Err("API Key 为空".to_string());
}
let endpoint = CredentialsVault::get(CredentialAccount::AsrEndpoint)
.map_err(|e| e.to_string())?
.unwrap_or_default();
let model = CredentialsVault::get(CredentialAccount::AsrModel)
.map_err(|e| e.to_string())?
.filter(|s| !s.trim().is_empty())
.unwrap_or_else(|| crate::asr::stepfun_realtime::DEFAULT_MODEL.to_string());
let asr = std::sync::Arc::new(crate::asr::StepfunRealtimeASR::new(
crate::asr::StepfunRealtimeCredentials {
api_key,
endpoint,
model,
prompt: None,
},
));
asr.open_session().await.map_err(|e| e.to_string())?;
crate::asr::AudioConsumer::consume_pcm_chunk(
&*asr,
&vec![0u8; crate::asr::stepfun_realtime::TARGET_AUDIO_CHUNK_BYTES * 5],
);
asr.send_last_frame().await.map_err(|e| e.to_string())?;
asr.await_final_result()
.await
.map(|_| ())
.map_err(|e| e.to_string())
}

async fn validate_mimo_asr_provider() -> Result<(), String> {
let config = read_openai_provider_config("asr")?;
let model = CredentialsVault::get(CredentialAccount::AsrModel)
Expand Down Expand Up @@ -588,6 +637,16 @@ async fn validate_asr_transcription(config: &ProviderConfig, model: &str) -> Res
};
let status = response.status();
if !status.is_success() {
// 探针音频是纯静音,有的厂商(StepFun)对无语音内容直接 400
// "no speech found"。走到这一步说明鉴权(错 key 是 401)和模型名
// (错模型是 404 model_invalid)都已通过、转写管线是通的——这类
// 内容拒收判为验证成功,避免对静音敏感的厂商恒报假阴性。
if status.as_u16() == 400 {
let body = response.text().await.unwrap_or_default();
if asr_error_is_no_speech_rejection(&body) {
return Ok(());
}
}
return Err(format!("providerHttpStatus:{}", status.as_u16()));
}
if let Some(len) = response.content_length() {
Expand All @@ -612,6 +671,13 @@ async fn validate_asr_transcription(config: &ProviderConfig, model: &str) -> Res
Ok(())
}

/// 400 应答体是否是「音频里没有语音」类内容拒收(而非参数错误)。
/// 只匹配语义明确的措辞,宁可漏判(用户看到 400 后实测仍可用)也不误判
/// 真正的参数错误为成功。
fn asr_error_is_no_speech_rejection(body: &str) -> bool {
body.to_ascii_lowercase().contains("no speech")
}

pub(crate) fn asr_transcriptions_url(base_url: &str) -> Result<String, String> {
let parsed = reqwest::Url::parse(base_url.trim()).map_err(|_| "endpointInvalid".to_string())?;

Expand Down Expand Up @@ -836,11 +902,26 @@ mod tests {
// LLM 路径的 SSRF 校验。read_openai_provider_config 依赖凭据库无法纯单测,这里直接对它调用
// 的校验器锁定 ASR 形态 endpoint 的拒绝/放行契约。
use super::{
fetch_provider_models, models_url, provider_llm_error_message, provider_log_context,
provider_request_error_message, sanitized_provider_destination, ProviderConfig,
asr_error_is_no_speech_rejection, fetch_provider_models, models_url,
provider_llm_error_message, provider_log_context, provider_request_error_message,
sanitized_provider_destination, ProviderConfig,
};
use crate::endpoint_security::validate_http_endpoint;

#[test]
fn silence_probe_content_rejection_is_not_a_credential_error() {
// StepFun 对静音探针的实测应答(2026-07):鉴权/模型都通过,只是探针
// 音频没有语音内容——不能报成凭据错误。
assert!(asr_error_is_no_speech_rejection(
r#"{"error":{"message":"no speech found","type":"request_params_invalid"}}"#
));
// 真正的参数错误不能被误判成功。
assert!(!asr_error_is_no_speech_rejection(
r#"{"error":{"message":"Request param: response_format is invalid","type":"input_invalid"}}"#
));
assert!(!asr_error_is_no_speech_rejection(""));
}

#[test]
fn provider_destination_redacts_userinfo_query_and_fragment() {
let raw = "https://alice:password@example.com:8443/v1/models?api_key=query-secret#private-fragment";
Expand Down
Loading
Loading