Add XiaoZhi WebSocket server plugin for realtime voice conversations

BotSharp.sln

@@ -149,6 +149,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.ImageHandle
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.FuzzySharp", "src\Plugins\BotSharp.Plugin.FuzzySharp\BotSharp.Plugin.FuzzySharp.csproj", "{E7C243B9-E751-B3B4-8F16-95C76CA90D31}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.XiaoZhi", "src\Plugins\BotSharp.Plugin.XiaoZhi\BotSharp.Plugin.XiaoZhi.csproj", "{A8E1D737-6C21-49DE-B241-CD5C8D9BF979}"
+EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.MMPEmbedding", "src\Plugins\BotSharp.Plugin.MMPEmbedding\BotSharp.Plugin.MMPEmbedding.csproj", "{394B858B-9C26-B977-A2DA-8CC7BE5914CB}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.Membase", "src\Plugins\BotSharp.Plugin.Membase\BotSharp.Plugin.Membase.csproj", "{13223C71-9EAC-9835-28ED-5A4833E6F915}"
@@ -633,6 +635,14 @@ Global
 		{E7C243B9-E751-B3B4-8F16-95C76CA90D31}.Release|Any CPU.Build.0 = Release|Any CPU
 		{E7C243B9-E751-B3B4-8F16-95C76CA90D31}.Release|x64.ActiveCfg = Release|Any CPU
 		{E7C243B9-E751-B3B4-8F16-95C76CA90D31}.Release|x64.Build.0 = Release|Any CPU
+		{A8E1D737-6C21-49DE-B241-CD5C8D9BF979}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{A8E1D737-6C21-49DE-B241-CD5C8D9BF979}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{A8E1D737-6C21-49DE-B241-CD5C8D9BF979}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{A8E1D737-6C21-49DE-B241-CD5C8D9BF979}.Debug|x64.Build.0 = Debug|Any CPU
+		{A8E1D737-6C21-49DE-B241-CD5C8D9BF979}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{A8E1D737-6C21-49DE-B241-CD5C8D9BF979}.Release|Any CPU.Build.0 = Release|Any CPU
+		{A8E1D737-6C21-49DE-B241-CD5C8D9BF979}.Release|x64.ActiveCfg = Release|Any CPU
+		{A8E1D737-6C21-49DE-B241-CD5C8D9BF979}.Release|x64.Build.0 = Release|Any CPU
 		{394B858B-9C26-B977-A2DA-8CC7BE5914CB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{394B858B-9C26-B977-A2DA-8CC7BE5914CB}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{394B858B-9C26-B977-A2DA-8CC7BE5914CB}.Debug|x64.ActiveCfg = Debug|Any CPU
@@ -721,6 +731,7 @@ Global
 		{FC63C875-E880-D8BB-B8B5-978AB7B62983} = {51AFE054-AE99-497D-A593-69BAEFB5106F}
 		{242F2D93-FCCE-4982-8075-F3052ECCA92C} = {51AFE054-AE99-497D-A593-69BAEFB5106F}
 		{E7C243B9-E751-B3B4-8F16-95C76CA90D31} = {51AFE054-AE99-497D-A593-69BAEFB5106F}
+		{A8E1D737-6C21-49DE-B241-CD5C8D9BF979} = {51AFE054-AE99-497D-A593-69BAEFB5106F}
 		{394B858B-9C26-B977-A2DA-8CC7BE5914CB} = {4F346DCE-087F-4368-AF88-EE9C720D0E69}
 		{13223C71-9EAC-9835-28ED-5A4833E6F915} = {53E7CD86-0D19-40D9-A0FA-AB4613837E89}
 	EndGlobalSection

Directory.Packages.props

@@ -9,6 +9,7 @@
     <PackageVersion Include="Google_GenerativeAI" Version="3.4.1" />
     <PackageVersion Include="Google_GenerativeAI.Live" Version="3.4.1" />
     <PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
+    <PackageVersion Include="OpusSharp" Version="1.5.7" />
     <PackageVersion Include="Polly" Version="8.4.2" />
     <PackageVersion Include="SharpFuzz" Version="2.2.0" />
     <PackageVersion Include="SharpHook" Version="5.3.9" />

src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs

@@ -4,6 +4,7 @@
 using BotSharp.Plugin.AzureOpenAI.Providers.Chat;
 using BotSharp.Plugin.AzureOpenAI.Providers.Embedding;
 using BotSharp.Plugin.AzureOpenAI.Providers.Image;
+using BotSharp.Plugin.AzureOpenAI.Providers.Realtime;
 using BotSharp.Plugin.AzureOpenAI.Providers.Text;
 using Microsoft.Extensions.Configuration;
 
@@ -32,5 +33,6 @@ public void RegisterDI(IServiceCollection services, IConfiguration config)
         services.AddScoped<ITextEmbedding, TextEmbeddingProvider>();
         services.AddScoped<IImageCompletion, ImageCompletionProvider>();
         services.AddScoped<IAudioTranscription, AudioCompletionProvider>();
+        services.AddScoped<IRealTimeCompletion, RealTimeCompletionProvider>();
     }
 }

src/Plugins/BotSharp.Plugin.AzureOpenAI/Models/Realtime/ConversationItemCreated.cs

@@ -0,0 +1,34 @@
+namespace BotSharp.Plugin.AzureOpenAI.Models.Realtime;
+
+public class ConversationItemCreated : ServerEventResponse
+{
+    [JsonPropertyName("item")]
+    public ConversationItemBody Item { get; set; } = new();
+}
+
+public class ConversationItemBody
+{
+    [JsonPropertyName("id")]
+    public string Id { get; set; } = null!;
+
+    [JsonPropertyName("type")]
+    public string Type { get; set; } = null!;
+
+    [JsonPropertyName("role")]
+    public string Role { get; set;} = null!;
+
+    [JsonPropertyName("content")]
+    public ConversationItemContent[] Content { get; set; } = [];
+}
+
+public class ConversationItemContent
+{
+    [JsonPropertyName("type")]
+    public string Type { get; set; } = null!;
+
+    [JsonPropertyName("transcript")]
+    public string Transcript { get; set; } = null!;
+
+    [JsonPropertyName("audio")]
+    public string Audio { get; set; } = null!;
+}

src/Plugins/BotSharp.Plugin.AzureOpenAI/Models/Realtime/RealtimeSessionBody.cs

@@ -0,0 +1,89 @@
+namespace BotSharp.Plugin.AzureOpenAI.Models.Realtime;
+
+public class RealtimeSessionBody
+{
+    [JsonPropertyName("id")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string Id { get; set; } = null!;
+
+    [JsonPropertyName("object")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string Object { get; set; } = null!;
+
+    [JsonPropertyName("model")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string Model { get; set; } = null!;
+
+    [JsonPropertyName("temperature")]
+    public float Temperature { get; set; } = 0.8f;
+
+    [JsonPropertyName("modalities")]
+    public string[] Modalities { get; set; } = ["audio", "text"];
+
+    [JsonPropertyName("input_audio_format")]
+    public string InputAudioFormat { get; set; } = null!;
+
+    [JsonPropertyName("output_audio_format")]
+    public string OutputAudioFormat { get; set; } = null!;
+
+    [JsonPropertyName("input_audio_transcription")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public InputAudioTranscription? InputAudioTranscription { get; set; }
+
+    [JsonPropertyName("instructions")]
+    public string Instructions { get; set; } = "You are a friendly assistant.";
+
+    [JsonPropertyName("voice")]
+    public string Voice { get; set; } = "sage";
+
+    [JsonPropertyName("max_response_output_tokens")]
+    public int MaxResponseOutputTokens { get; set; } = 512;
+
+    [JsonPropertyName("tool_choice")]
+    public string ToolChoice { get; set; } = "auto";
+
+    [JsonPropertyName("tools")]
+    public FunctionDef[] Tools { get; set; } = [];
+
+    [JsonPropertyName("turn_detection")]
+    public RealtimeSessionTurnDetection? TurnDetection { get; set; } = new();
+
+    [JsonPropertyName("input_audio_noise_reduction")]
+    public InputAudioNoiseReduction InputAudioNoiseReduction { get; set; } = new();
+}
+
+public class RealtimeSessionTurnDetection
+{
+    [JsonPropertyName("interrupt_response")]
+    public bool InterruptResponse { get; set; } = true;
+
+    /// <summary>
+    /// server_vad, semantic_vad
+    /// </summary>
+    [JsonPropertyName("type")]
+    public string Type { get; set; } = "semantic_vad";
+
+    [JsonPropertyName("eagerness")]
+    public string Eagerness { get;set; } = "auto";
+}
+
+public class InputAudioTranscription
+{
+    [JsonPropertyName("model")]
+    public string Model { get; set; } = "gpt-4o-transcribe";
+
+    [JsonPropertyName("language")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? Language { get; set; }
+
+    [JsonPropertyName("prompt")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? Prompt { get; set; }
+}
+
+public class InputAudioNoiseReduction
+{
+    [JsonPropertyName("type")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string Type { get; set; } = "far_field";
+}

src/Plugins/BotSharp.Plugin.AzureOpenAI/Models/Realtime/RealtimeSessionRequest.cs

@@ -0,0 +1,31 @@
+namespace BotSharp.Plugin.AzureOpenAI.Models.Realtime;
+
+public class RealtimeSessionCreationRequest
+{
+    [JsonPropertyName("model")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string Model { get; set; } = null!;
+
+    [JsonPropertyName("modalities")]
+    public string[] Modalities { get; set; } = ["audio", "text"];
+
+    [JsonPropertyName("instructions")]
+    public string Instructions { get; set; } = null!;
+
+    [JsonPropertyName("tool_choice")]
+    public string ToolChoice { get; set; } = "auto";
+
+    [JsonPropertyName("tools")]
+    public FunctionDef[] Tools { get; set; } = [];
+
+    [JsonPropertyName("turn_detection")]
+    public RealtimeSessionTurnDetection TurnDetection { get; set; } = new();
+}
+
+/// <summary>
+/// https://learn.microsoft.com/en-us/azure/ai-services/openai/realtime-audio-reference
+/// </summary>
+public class RealtimeSessionUpdateRequest : RealtimeSessionBody
+{
+
+}

src/Plugins/BotSharp.Plugin.AzureOpenAI/Models/Realtime/RealtimeSessionUpdate.cs

@@ -0,0 +1,13 @@
+using BotSharp.Abstraction.Realtime.Sessions;
+
+namespace BotSharp.Plugin.AzureOpenAI.Models.Realtime;
+
+public class RealtimeSessionUpdate
+{
+    /// <summary>
+    /// Optional client-generated ID used to identify this event.
+    /// </summary>
+    public string EventId { get; set; } = null!;
+    public string Type { get; set; } = "session.update";
+    public RealtimeSession Session { get; set; } = null!;
+}

src/Plugins/BotSharp.Plugin.AzureOpenAI/Models/Realtime/ResponseAudioDelta.cs

@@ -0,0 +1,19 @@
+namespace BotSharp.Plugin.AzureOpenAI.Models.Realtime;
+
+public class ResponseAudioDelta : ServerEventResponse
+{
+    [JsonPropertyName("response_id")]
+    public string ResponseId { get; set; } = null!;
+
+    [JsonPropertyName("item_id")]
+    public string ItemId { get; set; } = null!;
+
+    [JsonPropertyName("output_index")]
+    public int OutputIndex { get; set; }
+
+    [JsonPropertyName("content_index")]
+    public int ContentIndex { get; set; }
+
+    [JsonPropertyName("delta")]
+    public string? Delta { get; set; }
+}

src/Plugins/BotSharp.Plugin.AzureOpenAI/Models/Realtime/ResponseAudioTranscript.cs

@@ -0,0 +1,19 @@
+namespace BotSharp.Plugin.AzureOpenAI.Models.Realtime;
+
+public class ResponseAudioTranscript : ServerEventResponse
+{
+    [JsonPropertyName("response_id")]
+    public string ResponseId { get; set; } = null!;
+
+    [JsonPropertyName("item_id")]
+    public string ItemId { get; set; } = null!;
+
+    [JsonPropertyName("output_index")]
+    public int OutputIndex { get; set; }
+
+    [JsonPropertyName("content_index")]
+    public int ContentIndex { get; set; }
+
+    [JsonPropertyName("transcript")]
+    public string? Transcript { get; set; }
+}