From 87c34e4951a7bd5c9c8252c2676258432a14287c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 23 Nov 2025 10:38:33 +0000 Subject: [PATCH 01/39] Initial plan From 504e5bf65f9e36f4825a50cb7dd4d5f529ad651b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 23 Nov 2025 10:40:56 +0000 Subject: [PATCH 02/39] Initial plan for native OpenWebUI citations support Co-authored-by: owndev <69784886+owndev@users.noreply.github.com> --- filters/vertex_ai_search_tool.py | 1 - pipelines/google/google_gemini.py | 14 ++++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/filters/vertex_ai_search_tool.py b/filters/vertex_ai_search_tool.py index b135351..8be1534 100644 --- a/filters/vertex_ai_search_tool.py +++ b/filters/vertex_ai_search_tool.py @@ -41,4 +41,3 @@ def inlet(self, body: dict) -> dict: "vertex_ai_search enabled but vertex_rag_store not provided in params or VERTEX_AI_RAG_STORE env var" ) return body - diff --git a/pipelines/google/google_gemini.py b/pipelines/google/google_gemini.py index b60797f..21b3548 100644 --- a/pipelines/google/google_gemini.py +++ b/pipelines/google/google_gemini.py @@ -426,7 +426,7 @@ async def _build_image_generation_contents( { "index": i + 1, "label": ( - f"Image {i+1}" if self.valves.IMAGE_ADD_LABELS else str(i + 1) + f"Image {i + 1}" if self.valves.IMAGE_ADD_LABELS else str(i + 1) ), "reused": reused_flags[i], "origin": "history" if reused_flags[i] else "current", @@ -1429,11 +1429,15 @@ def _configure_generation( or os.getenv("VERTEX_AI_RAG_STORE") ) if vertex_rag_store: - self.log.debug(f"Enabling Vertex AI Search grounding: {vertex_rag_store}") + self.log.debug( + f"Enabling Vertex AI Search grounding: {vertex_rag_store}" + ) gen_config_params.setdefault("tools", []).append( types.Tool( retrieval=types.Retrieval( - vertex_ai_search=types.VertexAISearch(datastore=vertex_rag_store) + vertex_ai_search=types.VertexAISearch( + datastore=vertex_rag_store + ) ) ) ) @@ -1470,7 +1474,9 @@ def _format_grounding_chunks_as_sources( "uri": getattr(context, "uri", None), }, "document": [getattr(context, "chunk_text", None) or ""], - "metadata": [{"source": getattr(context, "title", None) or "Document"}], + "metadata": [ + {"source": getattr(context, "title", None) or "Document"} + ], } ) elif hasattr(chunk, "web") and chunk.web: From d817d1ebb3b90b2e235ec50e927a964d2b4acea3 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 23 Nov 2025 10:43:20 +0000 Subject: [PATCH 03/39] Add native OpenWebUI citations support to Azure AI Foundry pipeline Co-authored-by: owndev <69784886+owndev@users.noreply.github.com> --- pipelines/azure/azure_ai_foundry.py | 181 +++++++++++++++++++++++++++- 1 file changed, 175 insertions(+), 6 deletions(-) diff --git a/pipelines/azure/azure_ai_foundry.py b/pipelines/azure/azure_ai_foundry.py index b8a841c..bd80c47 100644 --- a/pipelines/azure/azure_ai_foundry.py +++ b/pipelines/azure/azure_ai_foundry.py @@ -203,6 +203,12 @@ class Valves(BaseModel): description="If True, enhance Azure AI Search responses with better citation formatting and source content display.", ) + # Enable native OpenWebUI citations (structured events and fields) + AZURE_AI_OPENWEBUI_CITATIONS: bool = Field( + default=bool(os.getenv("AZURE_AI_OPENWEBUI_CITATIONS", True)), + description="If True, emit native OpenWebUI citation events for streaming responses and attach openwebui_citations field for non-streaming responses. Enables citation cards and UI in OpenWebUI frontend.", + ) + def __init__(self): self.valves = self.Valves() self.name: str = f"{self.valves.AZURE_AI_PIPELINE_PREFIX}:" @@ -335,15 +341,152 @@ def get_azure_ai_data_sources(self) -> Optional[List[Dict[str, Any]]]: log.error(f"Error parsing AZURE_AI_DATA_SOURCES: {e}") return None + def _extract_citations_from_response( + self, response_data: Dict[str, Any] + ) -> Optional[List[Dict[str, Any]]]: + """ + Extract citations from an Azure AI response (streaming or non-streaming). + + Args: + response_data: Response data from Azure AI (can be a delta or full message) + + Returns: + List of citation objects, or None if no citations found + """ + if not isinstance(response_data, dict): + return None + + # Try multiple possible locations for citations + citations = None + + # Check in choices[0].delta.context.citations (streaming) + if "choices" in response_data and response_data["choices"]: + choice = response_data["choices"][0] + if ( + "delta" in choice + and isinstance(choice["delta"], dict) + and "context" in choice["delta"] + and "citations" in choice["delta"]["context"] + ): + citations = choice["delta"]["context"]["citations"] + + # Check in choices[0].message.context.citations (non-streaming) + elif ( + "message" in choice + and isinstance(choice["message"], dict) + and "context" in choice["message"] + and "citations" in choice["message"]["context"] + ): + citations = choice["message"]["context"]["citations"] + + return citations if citations and isinstance(citations, list) else None + + def _normalize_citation_for_openwebui( + self, citation: Dict[str, Any], index: int + ) -> Dict[str, Any]: + """ + Normalize an Azure citation object to OpenWebUI citation format. + + Args: + citation: Azure citation object + index: Citation index (1-based) + + Returns: + Normalized citation in OpenWebUI format + """ + # Get title with fallback to filepath or url + title = citation.get("title", "") + if not title or not title.strip(): + filepath = citation.get("filepath", "") + if filepath and filepath.strip(): + title = filepath + else: + url = citation.get("url", "") + if url and url.strip(): + title = url + else: + title = "Unknown Document" + + doc_id = f"doc{index}" + + # Build normalized citation structure + normalized = { + "id": doc_id, + "token": doc_id, + "title": title, + "document": [citation.get("content", "")], + "metadata": [citation.get("metadata", {})], + "source": { + "name": title, + }, + } + + # Add optional fields if available + if citation.get("url"): + normalized["url"] = citation["url"] + normalized["source"]["url"] = citation["url"] + + if citation.get("filepath"): + normalized["filepath"] = citation["filepath"] + + if citation.get("content"): + normalized["preview"] = citation["content"] + + if citation.get("chunk_id") is not None: + normalized["chunk_id"] = citation["chunk_id"] + + if citation.get("score") is not None: + normalized["score"] = citation["score"] + + return normalized + + async def _emit_openwebui_citation_events( + self, + citations: List[Dict[str, Any]], + __event_emitter__, + ) -> None: + """ + Emit OpenWebUI citation events for each citation. + + Args: + citations: List of Azure citation objects + __event_emitter__: Event emitter function + """ + if not __event_emitter__ or not citations: + return + + log = logging.getLogger("azure_ai._emit_openwebui_citation_events") + + for i, citation in enumerate(citations, 1): + if not isinstance(citation, dict): + continue + + try: + normalized = self._normalize_citation_for_openwebui(citation, i) + + # Emit citation event + await __event_emitter__( + { + "type": "citation", + "data": normalized, + } + ) + + log.debug(f"Emitted citation event for {normalized['id']}") + + except Exception as e: + log.warning(f"Failed to emit citation event for citation {i}: {e}") + def enhance_azure_search_response(self, response: Dict[str, Any]) -> Dict[str, Any]: """ Enhances Azure AI Search responses by improving citation display and adding source content. + Also attaches openwebui_citations field for native OpenWebUI citation support. Args: response: The original response from Azure AI Returns: - Enhanced response with better citation formatting + Enhanced response with better citation formatting and optional openwebui_citations field """ if not isinstance(response, dict): return response @@ -380,11 +523,11 @@ def enhance_azure_search_response(self, response: Dict[str, Any]) -> Dict[str, A "chunk_id": citation.get("chunk_id", "0"), } - # Enhance the content with better citation display + # Enhance the content with better citation display (if enabled) enhanced_content = content - # Add citation section at the end - if citation_details: + # Add citation section at the end (if markdown/HTML citations are enabled) + if self.valves.AZURE_AI_ENHANCE_CITATIONS and citation_details: citation_section = self._format_citation_section( citations, content, for_streaming=False ) @@ -396,6 +539,19 @@ def enhance_azure_search_response(self, response: Dict[str, Any]) -> Dict[str, A # Add enhanced citation info to context for API consumers context["enhanced_citations"] = citation_details + # Add native OpenWebUI citations field (if enabled) + if self.valves.AZURE_AI_OPENWEBUI_CITATIONS: + openwebui_citations = [] + for i, citation in enumerate(citations, 1): + if not isinstance(citation, dict): + continue + normalized = self._normalize_citation_for_openwebui(citation, i) + openwebui_citations.append(normalized) + + # Attach to response root level for OpenWebUI frontend + if openwebui_citations: + response["openwebui_citations"] = openwebui_citations + return response except Exception as e: @@ -694,6 +850,15 @@ async def stream_processor_with_citations( f"Successfully extracted {len(citations_data)} citations from stream" ) + # Emit native OpenWebUI citation events immediately if enabled + if ( + self.valves.AZURE_AI_OPENWEBUI_CITATIONS + and __event_emitter__ + ): + await self._emit_openwebui_citation_events( + citations_data, __event_emitter__ + ) + except json.JSONDecodeError: # Skip invalid JSON continue @@ -709,8 +874,12 @@ async def stream_processor_with_citations( log.debug("End of stream detected") break - # After the stream ends, add citations if we found any - if citations_data and not citations_added: + # After the stream ends, add markdown/HTML citations if we found any and it's enabled + if ( + citations_data + and not citations_added + and self.valves.AZURE_AI_ENHANCE_CITATIONS + ): log.info("Adding citation summary at end of stream...") # Pass the accumulated response content to filter citations From bdef4afe944cd698d12f9db630edeac32f7fabb0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 23 Nov 2025 10:45:59 +0000 Subject: [PATCH 04/39] Add documentation for native OpenWebUI citations feature Co-authored-by: owndev <69784886+owndev@users.noreply.github.com> --- README.md | 3 + docs/azure-ai-citations.md | 202 ++++++++++++++++++++++++++++ pipelines/azure/azure_ai_foundry.py | 3 +- 3 files changed, 207 insertions(+), 1 deletion(-) create mode 100644 docs/azure-ai-citations.md diff --git a/README.md b/README.md index 5a63f18..3cf57c1 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,7 @@ The functions include a built-in encryption mechanism for sensitive information: - Enables interaction with **Azure OpenAI** and other **Azure AI** models. - Supports Azure Search integration for enhanced document retrieval. +- **Native OpenWebUI Citations Support** 🎯: Rich citation cards, source previews, and inline citation correlations for Azure AI Search responses (Azure OpenAI only). - Supports multiple Azure AI models selection via the `AZURE_AI_MODEL` environment variable (e.g. `gpt-4o;gpt-4o-mini`). - Customizable pipeline display with configurable prefix via `AZURE_AI_PIPELINE_PREFIX`. - Azure AI Search / RAG integration with enhanced collapsible citation display (Azure OpenAI only). @@ -112,6 +113,8 @@ The functions include a built-in encryption mechanism for sensitive information: 🔗 [Learn More About Azure AI](https://azure.microsoft.com/en-us/solutions/ai) +📖 [Azure AI Citations Documentation](./docs/azure-ai-citations.md) + ### **2. [N8N Pipeline](./pipelines/n8n/n8n.py)** > [!TIP] diff --git a/docs/azure-ai-citations.md b/docs/azure-ai-citations.md new file mode 100644 index 0000000..9811e7e --- /dev/null +++ b/docs/azure-ai-citations.md @@ -0,0 +1,202 @@ +# Azure AI Foundry Pipeline - Native OpenWebUI Citations + +This document describes the native OpenWebUI citation support in the Azure AI Foundry Pipeline, which enables rich citation cards and source previews in the OpenWebUI frontend. + +## Overview + +The Azure AI Foundry Pipeline now supports **native OpenWebUI citations** for Azure AI Search (RAG) responses. This feature enables the OpenWebUI frontend to display: + +- **Citation cards** with source information +- **Source previews** with content snippets +- **Inline citation correlations** linking `[doc1]`, `[doc2]` markers to their sources +- **Interactive citation UI** with clickable sources + +## Features + +### Dual Citation Modes + +The pipeline supports two modes for displaying citations: + +1. **Native OpenWebUI Citations** (new): Structured citation events and fields for frontend consumption +2. **Markdown/HTML Citations** (existing): Collapsible HTML details with formatted citation information + +Both modes can be enabled simultaneously or independently via configuration. + +### Configuration Options + +| Environment Variable | Default | Description | +|---------------------|---------|-------------| +| `AZURE_AI_OPENWEBUI_CITATIONS` | `true` | Enable native OpenWebUI citation events and fields | +| `AZURE_AI_ENHANCE_CITATIONS` | `true` | Enable markdown/HTML citation display (collapsible sections) | + +### How It Works + +#### Streaming Responses + +When Azure AI Search returns citations in a streaming response: + +1. The pipeline detects citations in the SSE (Server-Sent Events) stream +2. **If `AZURE_AI_OPENWEBUI_CITATIONS` is enabled**: Citation events are emitted immediately via `__event_emitter__` +3. **If `AZURE_AI_ENHANCE_CITATIONS` is enabled**: A formatted markdown/HTML citation section is appended at the end of the stream + +#### Non-Streaming Responses + +When Azure AI Search returns citations in a non-streaming response: + +1. The pipeline extracts citations from the response +2. **If `AZURE_AI_OPENWEBUI_CITATIONS` is enabled**: An `openwebui_citations` field is attached to the response root +3. **If `AZURE_AI_ENHANCE_CITATIONS` is enabled**: The response content is enhanced with a formatted citation section + +## Citation Format + +### OpenWebUI Citation Event Structure + +Citation events follow the OpenWebUI specification: + +```python +{ + "type": "citation", + "data": { + "id": "doc1", # Unique identifier (matches inline tokens) + "token": "doc1", # Token for correlation (e.g., [doc1]) + "title": "Document Title", # Source name/title + "document": ["..."], # Content array + "metadata": [{}], # Metadata array + "source": { + "name": "Document Title", + "url": "https://..." # Optional + }, + "url": "https://...", # Optional: document URL + "filepath": "/path/to/file", # Optional: file path + "preview": "Content snippet...", # Optional: content preview + "chunk_id": "chunk-123", # Optional: chunk identifier + "score": 0.95 # Optional: relevance score + } +} +``` + +### Azure Citation Format (Input) + +Azure AI Search returns citations in this format: + +```python +{ + "title": "Document Title", + "content": "Full or partial content", + "url": "https://...", + "filepath": "/path/to/file", + "chunk_id": "chunk-123", + "score": 0.95, + "metadata": {} +} +``` + +The pipeline automatically converts Azure citations to OpenWebUI format. + +## Usage Examples + +### Basic Setup with Native Citations + +```python +# Enable native OpenWebUI citations (default) +AZURE_AI_OPENWEBUI_CITATIONS=true + +# Optionally disable markdown/HTML citations if you only want native citations +AZURE_AI_ENHANCE_CITATIONS=false +``` + +### Both Citation Modes Enabled (Default) + +```python +# Enable both native and markdown/HTML citations (default) +AZURE_AI_OPENWEBUI_CITATIONS=true +AZURE_AI_ENHANCE_CITATIONS=true +``` + +This configuration provides: +- Native citation cards in the OpenWebUI frontend +- Markdown/HTML citation section as fallback for non-supported clients + +### Only Markdown/HTML Citations (Legacy) + +```python +# Disable native citations, use only markdown/HTML +AZURE_AI_OPENWEBUI_CITATIONS=false +AZURE_AI_ENHANCE_CITATIONS=true +``` + +## Implementation Details + +### Helper Functions + +The pipeline includes three new helper functions: + +1. **`_extract_citations_from_response()`**: Extracts citations from Azure responses +2. **`_normalize_citation_for_openwebui()`**: Converts Azure citations to OpenWebUI format +3. **`_emit_openwebui_citation_events()`**: Emits citation events via `__event_emitter__` + +### Title Fallback Logic + +The pipeline uses intelligent title fallback: + +1. Use `title` field if available +2. Fallback to `filepath` if title is empty +3. Fallback to `url` if both title and filepath are empty +4. Fallback to `"Unknown Document"` if all are empty + +This ensures every citation has a meaningful display name. + +### Streaming Citation Emission + +Citations are emitted **as soon as they are detected** in the stream, ensuring: +- Low latency for citation display +- Frontend can start rendering citations while content is still streaming +- No waiting for the complete response + +### Backward Compatibility + +The implementation maintains full backward compatibility: + +- Existing markdown/HTML citation display continues to work +- No breaking changes to the API +- Both citation modes can be enabled simultaneously +- Default configuration enables both modes + +## Troubleshooting + +### Citations Not Appearing + +**Problem**: Citations don't appear in the OpenWebUI frontend + +**Solutions**: +1. Verify `AZURE_AI_OPENWEBUI_CITATIONS=true` is set +2. Check that Azure AI Search is properly configured (`AZURE_AI_DATA_SOURCES`) +3. Ensure you're using an Azure OpenAI endpoint (not a generic Azure AI endpoint) +4. Check browser console for errors + +### Citation Cards vs. Markdown Section + +**Problem**: Seeing both citation cards and markdown section + +**Solution**: This is the default behavior. To show only citation cards: +```bash +AZURE_AI_OPENWEBUI_CITATIONS=true +AZURE_AI_ENHANCE_CITATIONS=false +``` + +### Missing Citation Metadata + +**Problem**: Some citation fields (URL, filepath, score) are missing + +**Solution**: These fields are optional. Azure AI Search may not return all fields depending on your index configuration. The pipeline gracefully handles missing fields. + +## References + +- [OpenWebUI Pipelines Citation Feature Discussion](https://github.com/open-webui/pipelines/issues/229) +- [OpenWebUI Event Emitter Documentation](https://docs.openwebui.com/features/plugin/development/events) +- [Azure AI Search Documentation](https://learn.microsoft.com/en-us/azure/search/) + +## Version History + +- **v2.6.0**: Added native OpenWebUI citations support +- **v2.5.x**: Markdown/HTML citation display only diff --git a/pipelines/azure/azure_ai_foundry.py b/pipelines/azure/azure_ai_foundry.py index bd80c47..c416e72 100644 --- a/pipelines/azure/azure_ai_foundry.py +++ b/pipelines/azure/azure_ai_foundry.py @@ -4,7 +4,7 @@ author_url: https://github.com/owndev/ project_url: https://github.com/owndev/Open-WebUI-Functions funding_url: https://github.com/sponsors/owndev -version: 2.5.2 +version: 2.6.0 license: Apache License 2.0 description: A pipeline for interacting with Azure AI services, enabling seamless communication with various AI models via configurable headers and robust error handling. This includes support for Azure OpenAI models as well as other Azure AI models by dynamically managing headers and request configurations. Azure AI Search (RAG) integration is only supported with Azure OpenAI endpoints. features: @@ -16,6 +16,7 @@ - Predefined models for easy access. - Encrypted storage of sensitive API keys - Azure AI Search / RAG integration with enhanced citation display (Azure OpenAI only) + - Native OpenWebUI citations support with structured events and citation cards (Azure OpenAI only) """ from typing import ( From bac7435652f8c5f82e71d322bed23b90a0af3777 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 23 Nov 2025 10:48:36 +0000 Subject: [PATCH 05/39] Fix code review issues: env var parsing and type annotations Co-authored-by: owndev <69784886+owndev@users.noreply.github.com> --- pipelines/azure/azure_ai_foundry.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pipelines/azure/azure_ai_foundry.py b/pipelines/azure/azure_ai_foundry.py index c416e72..04e3e28 100644 --- a/pipelines/azure/azure_ai_foundry.py +++ b/pipelines/azure/azure_ai_foundry.py @@ -200,13 +200,13 @@ class Valves(BaseModel): # Enable enhanced citation display for Azure AI Search responses AZURE_AI_ENHANCE_CITATIONS: bool = Field( - default=bool(os.getenv("AZURE_AI_ENHANCE_CITATIONS", True)), + default=os.getenv("AZURE_AI_ENHANCE_CITATIONS", "true").lower() == "true", description="If True, enhance Azure AI Search responses with better citation formatting and source content display.", ) # Enable native OpenWebUI citations (structured events and fields) AZURE_AI_OPENWEBUI_CITATIONS: bool = Field( - default=bool(os.getenv("AZURE_AI_OPENWEBUI_CITATIONS", True)), + default=os.getenv("AZURE_AI_OPENWEBUI_CITATIONS", "true").lower() == "true", description="If True, emit native OpenWebUI citation events for streaming responses and attach openwebui_citations field for non-streaming responses. Enables citation cards and UI in OpenWebUI frontend.", ) @@ -444,7 +444,7 @@ def _normalize_citation_for_openwebui( async def _emit_openwebui_citation_events( self, citations: List[Dict[str, Any]], - __event_emitter__, + __event_emitter__: Optional[callable], ) -> None: """ Emit OpenWebUI citation events for each citation. @@ -480,7 +480,7 @@ async def _emit_openwebui_citation_events( def enhance_azure_search_response(self, response: Dict[str, Any]) -> Dict[str, Any]: """ - Enhances Azure AI Search responses by improving citation display and adding source content. + Enhance Azure AI Search responses by improving citation display and adding source content. Also attaches openwebui_citations field for native OpenWebUI citation support. Args: From 21973ca53e804ca097bafab3610372b9e582cea8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 23 Nov 2025 10:51:30 +0000 Subject: [PATCH 06/39] Address final code review feedback: type annotations and efficiency improvements Co-authored-by: owndev <69784886+owndev@users.noreply.github.com> --- pipelines/azure/azure_ai_foundry.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/pipelines/azure/azure_ai_foundry.py b/pipelines/azure/azure_ai_foundry.py index 04e3e28..d4998e3 100644 --- a/pipelines/azure/azure_ai_foundry.py +++ b/pipelines/azure/azure_ai_foundry.py @@ -29,6 +29,7 @@ Any, AsyncIterator, Set, + Callable, ) from urllib.parse import urlparse from fastapi.responses import StreamingResponse @@ -444,14 +445,14 @@ def _normalize_citation_for_openwebui( async def _emit_openwebui_citation_events( self, citations: List[Dict[str, Any]], - __event_emitter__: Optional[callable], + __event_emitter__: Optional[Callable[..., Any]], ) -> None: """ Emit OpenWebUI citation events for each citation. Args: citations: List of Azure citation objects - __event_emitter__: Event emitter function + __event_emitter__: Event emitter callable for sending citation events """ if not __event_emitter__ or not citations: return @@ -481,13 +482,17 @@ async def _emit_openwebui_citation_events( def enhance_azure_search_response(self, response: Dict[str, Any]) -> Dict[str, Any]: """ Enhance Azure AI Search responses by improving citation display and adding source content. - Also attaches openwebui_citations field for native OpenWebUI citation support. + Modifies the response in-place and returns it. + + The function performs the following enhancements: + 1. If AZURE_AI_ENHANCE_CITATIONS is True: Appends a formatted markdown/HTML citation section + 2. If AZURE_AI_OPENWEBUI_CITATIONS is True: Attaches an 'openwebui_citations' array at root level Args: - response: The original response from Azure AI + response: The original response from Azure AI (modified in-place) Returns: - Enhanced response with better citation formatting and optional openwebui_citations field + The enhanced response with better citation formatting and optional openwebui_citations field """ if not isinstance(response, dict): return response @@ -540,8 +545,8 @@ def enhance_azure_search_response(self, response: Dict[str, Any]) -> Dict[str, A # Add enhanced citation info to context for API consumers context["enhanced_citations"] = citation_details - # Add native OpenWebUI citations field (if enabled) - if self.valves.AZURE_AI_OPENWEBUI_CITATIONS: + # Add native OpenWebUI citations field (if enabled and citations exist) + if self.valves.AZURE_AI_OPENWEBUI_CITATIONS and citations: openwebui_citations = [] for i, citation in enumerate(citations, 1): if not isinstance(citation, dict): From 4747deaf8a6d92845353a39f2046708677e3aaba Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 23 Nov 2025 10:54:15 +0000 Subject: [PATCH 07/39] Refactor: Add helper function for boolean env parsing and simplify title fallback Co-authored-by: owndev <69784886+owndev@users.noreply.github.com> --- pipelines/azure/azure_ai_foundry.py | 37 ++++++++++++++++++----------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/pipelines/azure/azure_ai_foundry.py b/pipelines/azure/azure_ai_foundry.py index d4998e3..f43899b 100644 --- a/pipelines/azure/azure_ai_foundry.py +++ b/pipelines/azure/azure_ai_foundry.py @@ -125,6 +125,20 @@ def __get_pydantic_core_schema__( # Helper functions +def get_bool_env(env_var: str, default: str = "true") -> bool: + """ + Parse a boolean environment variable. + + Args: + env_var: The environment variable name + default: The default value as a string ("true" or "false") + + Returns: + Boolean value parsed from the environment variable + """ + return os.getenv(env_var, default).lower() == "true" + + async def cleanup_response( response: Optional[aiohttp.ClientResponse], session: Optional[aiohttp.ClientSession], @@ -201,13 +215,13 @@ class Valves(BaseModel): # Enable enhanced citation display for Azure AI Search responses AZURE_AI_ENHANCE_CITATIONS: bool = Field( - default=os.getenv("AZURE_AI_ENHANCE_CITATIONS", "true").lower() == "true", + default=get_bool_env("AZURE_AI_ENHANCE_CITATIONS"), description="If True, enhance Azure AI Search responses with better citation formatting and source content display.", ) # Enable native OpenWebUI citations (structured events and fields) AZURE_AI_OPENWEBUI_CITATIONS: bool = Field( - default=os.getenv("AZURE_AI_OPENWEBUI_CITATIONS", "true").lower() == "true", + default=get_bool_env("AZURE_AI_OPENWEBUI_CITATIONS"), description="If True, emit native OpenWebUI citation events for streaming responses and attach openwebui_citations field for non-streaming responses. Enables citation cards and UI in OpenWebUI frontend.", ) @@ -396,18 +410,13 @@ def _normalize_citation_for_openwebui( Returns: Normalized citation in OpenWebUI format """ - # Get title with fallback to filepath or url - title = citation.get("title", "") - if not title or not title.strip(): - filepath = citation.get("filepath", "") - if filepath and filepath.strip(): - title = filepath - else: - url = citation.get("url", "") - if url and url.strip(): - title = url - else: - title = "Unknown Document" + # Get title with fallback chain: title → filepath → url → "Unknown Document" + title = ( + citation.get("title", "").strip() + or citation.get("filepath", "").strip() + or citation.get("url", "").strip() + or "Unknown Document" + ) doc_id = f"doc{index}" From d078d6bc5b5d61e0d58a802ecc7413a153e64923 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 26 Nov 2025 08:20:09 +0000 Subject: [PATCH 08/39] Fix citation format to match OpenWebUI specification with distances for relevance Co-authored-by: owndev <69784886+owndev@users.noreply.github.com> --- docs/azure-ai-citations.md | 32 +++---- pipelines/azure/azure_ai_foundry.py | 140 ++++++++++++++++------------ 2 files changed, 90 insertions(+), 82 deletions(-) diff --git a/docs/azure-ai-citations.md b/docs/azure-ai-citations.md index 9811e7e..40fa324 100644 --- a/docs/azure-ai-citations.md +++ b/docs/azure-ai-citations.md @@ -6,9 +6,9 @@ This document describes the native OpenWebUI citation support in the Azure AI Fo The Azure AI Foundry Pipeline now supports **native OpenWebUI citations** for Azure AI Search (RAG) responses. This feature enables the OpenWebUI frontend to display: -- **Citation cards** with source information +- **Citation cards** with source information and relevance scores - **Source previews** with content snippets -- **Inline citation correlations** linking `[doc1]`, `[doc2]` markers to their sources +- **Relevance percentage** displayed on citation cards - **Interactive citation UI** with clickable sources ## Features @@ -17,7 +17,7 @@ The Azure AI Foundry Pipeline now supports **native OpenWebUI citations** for Az The pipeline supports two modes for displaying citations: -1. **Native OpenWebUI Citations** (new): Structured citation events and fields for frontend consumption +1. **Native OpenWebUI Citations** (new): Structured citation events emitted via `__event_emitter__` for frontend consumption 2. **Markdown/HTML Citations** (existing): Collapsible HTML details with formatted citation information Both modes can be enabled simultaneously or independently via configuration. @@ -26,7 +26,7 @@ Both modes can be enabled simultaneously or independently via configuration. | Environment Variable | Default | Description | |---------------------|---------|-------------| -| `AZURE_AI_OPENWEBUI_CITATIONS` | `true` | Enable native OpenWebUI citation events and fields | +| `AZURE_AI_OPENWEBUI_CITATIONS` | `true` | Enable native OpenWebUI citation events | | `AZURE_AI_ENHANCE_CITATIONS` | `true` | Enable markdown/HTML citation display (collapsible sections) | ### How It Works @@ -44,37 +44,29 @@ When Azure AI Search returns citations in a streaming response: When Azure AI Search returns citations in a non-streaming response: 1. The pipeline extracts citations from the response -2. **If `AZURE_AI_OPENWEBUI_CITATIONS` is enabled**: An `openwebui_citations` field is attached to the response root +2. **If `AZURE_AI_OPENWEBUI_CITATIONS` is enabled**: Citation events are emitted via `__event_emitter__` 3. **If `AZURE_AI_ENHANCE_CITATIONS` is enabled**: The response content is enhanced with a formatted citation section ## Citation Format ### OpenWebUI Citation Event Structure -Citation events follow the OpenWebUI specification: +Citation events follow the official OpenWebUI specification (see [OpenWebUI Events Documentation](https://docs.openwebui.com/features/plugin/development/events#source-or-citation-and-code-execution)): ```python { "type": "citation", "data": { - "id": "doc1", # Unique identifier (matches inline tokens) - "token": "doc1", # Token for correlation (e.g., [doc1]) - "title": "Document Title", # Source name/title - "document": ["..."], # Content array - "metadata": [{}], # Metadata array - "source": { - "name": "Document Title", - "url": "https://..." # Optional - }, - "url": "https://...", # Optional: document URL - "filepath": "/path/to/file", # Optional: file path - "preview": "Content snippet...", # Optional: content preview - "chunk_id": "chunk-123", # Optional: chunk identifier - "score": 0.95 # Optional: relevance score + "document": ["Document content 1", "Document content 2", ...], # Content from each citation + "metadata": [{"source": "https://..."}, ...], # Metadata with source URLs + "source": {"name": "Source Name"}, # Display name for the source + "distances": [0.95, 0.87, ...] # Relevance scores (displayed as percentage) } } ``` +The `distances` array contains relevance scores from Azure AI Search, which OpenWebUI displays as a percentage on the citation cards. + ### Azure Citation Format (Input) Azure AI Search returns citations in this format: diff --git a/pipelines/azure/azure_ai_foundry.py b/pipelines/azure/azure_ai_foundry.py index f43899b..774eda2 100644 --- a/pipelines/azure/azure_ai_foundry.py +++ b/pipelines/azure/azure_ai_foundry.py @@ -403,6 +403,9 @@ def _normalize_citation_for_openwebui( """ Normalize an Azure citation object to OpenWebUI citation format. + The format follows OpenWebUI's official citation event structure: + https://docs.openwebui.com/features/plugin/development/events#source-or-citation-and-code-execution + Args: citation: Azure citation object index: Citation index (1-based) @@ -418,36 +421,25 @@ def _normalize_citation_for_openwebui( or "Unknown Document" ) - doc_id = f"doc{index}" + # Build source URL for metadata + source_url = citation.get("url") or citation.get("filepath") or "" + + # Build metadata with source information + metadata_entry = {"source": source_url} + if citation.get("metadata"): + metadata_entry.update(citation.get("metadata", {})) - # Build normalized citation structure + # Build normalized citation structure matching OpenWebUI format exactly normalized = { - "id": doc_id, - "token": doc_id, - "title": title, "document": [citation.get("content", "")], - "metadata": [citation.get("metadata", {})], - "source": { - "name": title, - }, + "metadata": [metadata_entry], + "source": {"name": title}, } - # Add optional fields if available - if citation.get("url"): - normalized["url"] = citation["url"] - normalized["source"]["url"] = citation["url"] - - if citation.get("filepath"): - normalized["filepath"] = citation["filepath"] - - if citation.get("content"): - normalized["preview"] = citation["content"] - - if citation.get("chunk_id") is not None: - normalized["chunk_id"] = citation["chunk_id"] - + # Add distances array for relevance score (OpenWebUI uses this for percentage display) if citation.get("score") is not None: - normalized["score"] = citation["score"] + # Azure AI Search returns relevance scores - convert to distance format + normalized["distances"] = [citation["score"]] return normalized @@ -457,7 +449,10 @@ async def _emit_openwebui_citation_events( __event_emitter__: Optional[Callable[..., Any]], ) -> None: """ - Emit OpenWebUI citation events for each citation. + Emit OpenWebUI citation events for citations. + + Emits a single citation event with all citations as arrays in the data fields, + following the OpenWebUI citation event format. Args: citations: List of Azure citation objects @@ -468,40 +463,65 @@ async def _emit_openwebui_citation_events( log = logging.getLogger("azure_ai._emit_openwebui_citation_events") - for i, citation in enumerate(citations, 1): - if not isinstance(citation, dict): - continue + try: + # Build combined citation data with all citations + all_documents = [] + all_metadata = [] + all_distances = [] + source_name = None + + for i, citation in enumerate(citations, 1): + if not isinstance(citation, dict): + continue - try: normalized = self._normalize_citation_for_openwebui(citation, i) - # Emit citation event - await __event_emitter__( - { - "type": "citation", - "data": normalized, - } - ) + # Collect documents, metadata, and distances from each citation + all_documents.extend(normalized.get("document", [])) + all_metadata.extend(normalized.get("metadata", [])) + + if "distances" in normalized: + all_distances.extend(normalized.get("distances", [])) + + # Use the first citation's source name, or aggregate if needed + if source_name is None and "source" in normalized: + source_name = normalized["source"].get("name", "Source") + + # Build the combined citation event + citation_event = { + "type": "citation", + "data": { + "document": all_documents, + "metadata": all_metadata, + "source": {"name": source_name or "Azure AI Search"}, + }, + } + + # Add distances if we have any scores + if all_distances: + citation_event["data"]["distances"] = all_distances - log.debug(f"Emitted citation event for {normalized['id']}") + # Emit the citation event + await __event_emitter__(citation_event) - except Exception as e: - log.warning(f"Failed to emit citation event for citation {i}: {e}") + log.debug(f"Emitted citation event with {len(all_documents)} documents") + + except Exception as e: + log.warning(f"Failed to emit citation events: {e}") def enhance_azure_search_response(self, response: Dict[str, Any]) -> Dict[str, Any]: """ Enhance Azure AI Search responses by improving citation display and adding source content. Modifies the response in-place and returns it. - The function performs the following enhancements: - 1. If AZURE_AI_ENHANCE_CITATIONS is True: Appends a formatted markdown/HTML citation section - 2. If AZURE_AI_OPENWEBUI_CITATIONS is True: Attaches an 'openwebui_citations' array at root level + If AZURE_AI_ENHANCE_CITATIONS is True, appends a formatted markdown/HTML citation section + to the response content. Args: response: The original response from Azure AI (modified in-place) Returns: - The enhanced response with better citation formatting and optional openwebui_citations field + The enhanced response with better citation formatting """ if not isinstance(response, dict): return response @@ -554,19 +574,6 @@ def enhance_azure_search_response(self, response: Dict[str, Any]) -> Dict[str, A # Add enhanced citation info to context for API consumers context["enhanced_citations"] = citation_details - # Add native OpenWebUI citations field (if enabled and citations exist) - if self.valves.AZURE_AI_OPENWEBUI_CITATIONS and citations: - openwebui_citations = [] - for i, citation in enumerate(citations, 1): - if not isinstance(citation, dict): - continue - normalized = self._normalize_citation_for_openwebui(citation, i) - openwebui_citations.append(normalized) - - # Attach to response root level for OpenWebUI frontend - if openwebui_citations: - response["openwebui_citations"] = openwebui_citations - return response except Exception as e: @@ -1345,12 +1352,21 @@ async def pipe( request.raise_for_status() # Enhance Azure Search responses with better citation display - if ( - isinstance(response, dict) - and self.valves.AZURE_AI_DATA_SOURCES - and self.valves.AZURE_AI_ENHANCE_CITATIONS - ): - response = self.enhance_azure_search_response(response) + # Call this when either citation mode is enabled + if isinstance(response, dict) and self.valves.AZURE_AI_DATA_SOURCES: + if ( + self.valves.AZURE_AI_ENHANCE_CITATIONS + or self.valves.AZURE_AI_OPENWEBUI_CITATIONS + ): + response = self.enhance_azure_search_response(response) + + # Emit native OpenWebUI citation events for non-streaming responses + if self.valves.AZURE_AI_OPENWEBUI_CITATIONS and __event_emitter__: + citations = self._extract_citations_from_response(response) + if citations: + await self._emit_openwebui_citation_events( + citations, __event_emitter__ + ) # Send completion status update if __event_emitter__: From c4d7d5cee28ace4104d75483dca850bdb407a782 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 26 Nov 2025 08:21:42 +0000 Subject: [PATCH 09/39] Fix code review comments on citation format Co-authored-by: owndev <69784886+owndev@users.noreply.github.com> --- pipelines/azure/azure_ai_foundry.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipelines/azure/azure_ai_foundry.py b/pipelines/azure/azure_ai_foundry.py index 774eda2..6cdf18b 100644 --- a/pipelines/azure/azure_ai_foundry.py +++ b/pipelines/azure/azure_ai_foundry.py @@ -438,7 +438,7 @@ def _normalize_citation_for_openwebui( # Add distances array for relevance score (OpenWebUI uses this for percentage display) if citation.get("score") is not None: - # Azure AI Search returns relevance scores - convert to distance format + # Wrap score in distances array as required by OpenWebUI format normalized["distances"] = [citation["score"]] return normalized @@ -483,7 +483,7 @@ async def _emit_openwebui_citation_events( if "distances" in normalized: all_distances.extend(normalized.get("distances", [])) - # Use the first citation's source name, or aggregate if needed + # Use the first citation's source name for the combined event if source_name is None and "source" in normalized: source_name = normalized["source"].get("name", "Source") From 6c30f6aed0c4e4fab3d56cc9fc5958cd3049f0ae Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 26 Nov 2025 08:45:10 +0000 Subject: [PATCH 10/39] Fix citation structure: emit separate events per source with unique names Co-authored-by: owndev <69784886+owndev@users.noreply.github.com> --- docs/azure-ai-citations.md | 20 +++++--- pipelines/azure/azure_ai_foundry.py | 79 +++++++++++------------------ 2 files changed, 43 insertions(+), 56 deletions(-) diff --git a/docs/azure-ai-citations.md b/docs/azure-ai-citations.md index 40fa324..e102e69 100644 --- a/docs/azure-ai-citations.md +++ b/docs/azure-ai-citations.md @@ -44,28 +44,34 @@ When Azure AI Search returns citations in a streaming response: When Azure AI Search returns citations in a non-streaming response: 1. The pipeline extracts citations from the response -2. **If `AZURE_AI_OPENWEBUI_CITATIONS` is enabled**: Citation events are emitted via `__event_emitter__` +2. **If `AZURE_AI_OPENWEBUI_CITATIONS` is enabled**: Individual citation events are emitted via `__event_emitter__` for each source 3. **If `AZURE_AI_ENHANCE_CITATIONS` is enabled**: The response content is enhanced with a formatted citation section ## Citation Format ### OpenWebUI Citation Event Structure -Citation events follow the official OpenWebUI specification (see [OpenWebUI Events Documentation](https://docs.openwebui.com/features/plugin/development/events#source-or-citation-and-code-execution)): +Each citation is emitted as a separate event to ensure all sources appear in the UI. Citation events follow the official OpenWebUI specification (see [OpenWebUI Events Documentation](https://docs.openwebui.com/features/plugin/development/events#source-or-citation-and-code-execution)): ```python { "type": "citation", "data": { - "document": ["Document content 1", "Document content 2", ...], # Content from each citation - "metadata": [{"source": "https://..."}, ...], # Metadata with source URLs - "source": {"name": "Source Name"}, # Display name for the source - "distances": [0.95, 0.87, ...] # Relevance scores (displayed as percentage) + "document": ["Document content..."], # Content from this citation + "metadata": [{"source": "https://..."}], # Metadata with source URL + "source": { + "name": "[doc1] Document Title", # Unique name with index + "url": "https://..." # Source URL if available + }, + "distances": [0.95] # Relevance score (displayed as percentage) } } ``` -The `distances` array contains relevance scores from Azure AI Search, which OpenWebUI displays as a percentage on the citation cards. +Key points: +- Each source document gets its own citation event +- The `source.name` includes the doc index (`[doc1]`, `[doc2]`, etc.) to prevent grouping +- The `distances` array contains relevance scores from Azure AI Search, which OpenWebUI displays as a percentage on the citation cards ### Azure Citation Format (Input) diff --git a/pipelines/azure/azure_ai_foundry.py b/pipelines/azure/azure_ai_foundry.py index 6cdf18b..d0df8d5 100644 --- a/pipelines/azure/azure_ai_foundry.py +++ b/pipelines/azure/azure_ai_foundry.py @@ -401,7 +401,7 @@ def _normalize_citation_for_openwebui( self, citation: Dict[str, Any], index: int ) -> Dict[str, Any]: """ - Normalize an Azure citation object to OpenWebUI citation format. + Normalize an Azure citation object to OpenWebUI citation event format. The format follows OpenWebUI's official citation event structure: https://docs.openwebui.com/features/plugin/development/events#source-or-citation-and-code-execution @@ -411,15 +411,18 @@ def _normalize_citation_for_openwebui( index: Citation index (1-based) Returns: - Normalized citation in OpenWebUI format + Complete citation event object with type and data fields """ # Get title with fallback chain: title → filepath → url → "Unknown Document" - title = ( + # Add index to make each source unique and prevent grouping + base_title = ( citation.get("title", "").strip() or citation.get("filepath", "").strip() or citation.get("url", "").strip() or "Unknown Document" ) + # Make title unique by appending doc index if there could be duplicates + title = f"[doc{index}] {base_title}" # Build source URL for metadata source_url = citation.get("url") or citation.get("filepath") or "" @@ -429,19 +432,27 @@ def _normalize_citation_for_openwebui( if citation.get("metadata"): metadata_entry.update(citation.get("metadata", {})) - # Build normalized citation structure matching OpenWebUI format exactly - normalized = { + # Build normalized citation data structure matching OpenWebUI format exactly + citation_data = { "document": [citation.get("content", "")], "metadata": [metadata_entry], "source": {"name": title}, } + # Add URL to source if available + if source_url: + citation_data["source"]["url"] = source_url + # Add distances array for relevance score (OpenWebUI uses this for percentage display) if citation.get("score") is not None: # Wrap score in distances array as required by OpenWebUI format - normalized["distances"] = [citation["score"]] + citation_data["distances"] = [citation["score"]] - return normalized + # Return complete citation event structure + return { + "type": "citation", + "data": citation_data, + } async def _emit_openwebui_citation_events( self, @@ -451,8 +462,9 @@ async def _emit_openwebui_citation_events( """ Emit OpenWebUI citation events for citations. - Emits a single citation event with all citations as arrays in the data fields, - following the OpenWebUI citation event format. + Emits one citation event per source document, following the OpenWebUI + citation event format. Each citation is emitted separately to ensure + all sources appear in the UI. Args: citations: List of Azure citation objects @@ -463,51 +475,20 @@ async def _emit_openwebui_citation_events( log = logging.getLogger("azure_ai._emit_openwebui_citation_events") - try: - # Build combined citation data with all citations - all_documents = [] - all_metadata = [] - all_distances = [] - source_name = None - - for i, citation in enumerate(citations, 1): - if not isinstance(citation, dict): - continue + for i, citation in enumerate(citations, 1): + if not isinstance(citation, dict): + continue + try: normalized = self._normalize_citation_for_openwebui(citation, i) - # Collect documents, metadata, and distances from each citation - all_documents.extend(normalized.get("document", [])) - all_metadata.extend(normalized.get("metadata", [])) - - if "distances" in normalized: - all_distances.extend(normalized.get("distances", [])) - - # Use the first citation's source name for the combined event - if source_name is None and "source" in normalized: - source_name = normalized["source"].get("name", "Source") - - # Build the combined citation event - citation_event = { - "type": "citation", - "data": { - "document": all_documents, - "metadata": all_metadata, - "source": {"name": source_name or "Azure AI Search"}, - }, - } + # Emit citation event for this individual source + await __event_emitter__(normalized) - # Add distances if we have any scores - if all_distances: - citation_event["data"]["distances"] = all_distances + log.debug(f"Emitted citation event for doc{i}") - # Emit the citation event - await __event_emitter__(citation_event) - - log.debug(f"Emitted citation event with {len(all_documents)} documents") - - except Exception as e: - log.warning(f"Failed to emit citation events: {e}") + except Exception as e: + log.warning(f"Failed to emit citation event for citation {i}: {e}") def enhance_azure_search_response(self, response: Dict[str, Any]) -> Dict[str, Any]: """ From 43a0e64c6ca2dc16dda36dcd8ae42172721c3756 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 26 Nov 2025 08:46:31 +0000 Subject: [PATCH 11/39] Fix comments per code review feedback Co-authored-by: owndev <69784886+owndev@users.noreply.github.com> --- pipelines/azure/azure_ai_foundry.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pipelines/azure/azure_ai_foundry.py b/pipelines/azure/azure_ai_foundry.py index d0df8d5..af60c1a 100644 --- a/pipelines/azure/azure_ai_foundry.py +++ b/pipelines/azure/azure_ai_foundry.py @@ -414,14 +414,13 @@ def _normalize_citation_for_openwebui( Complete citation event object with type and data fields """ # Get title with fallback chain: title → filepath → url → "Unknown Document" - # Add index to make each source unique and prevent grouping base_title = ( citation.get("title", "").strip() or citation.get("filepath", "").strip() or citation.get("url", "").strip() or "Unknown Document" ) - # Make title unique by appending doc index if there could be duplicates + # Always prefix title with doc index to ensure uniqueness and prevent grouping title = f"[doc{index}] {base_title}" # Build source URL for metadata From 7360c4c1ce4a5398f9bd9b62ad17833f445b2e71 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 26 Nov 2025 09:05:49 +0000 Subject: [PATCH 12/39] Add comprehensive debug logging and fix distances field for citation events Co-authored-by: owndev <69784886+owndev@users.noreply.github.com> --- pipelines/azure/azure_ai_foundry.py | 82 +++++++++++++++++++++++++---- 1 file changed, 71 insertions(+), 11 deletions(-) diff --git a/pipelines/azure/azure_ai_foundry.py b/pipelines/azure/azure_ai_foundry.py index af60c1a..620356b 100644 --- a/pipelines/azure/azure_ai_foundry.py +++ b/pipelines/azure/azure_ai_foundry.py @@ -369,7 +369,10 @@ def _extract_citations_from_response( Returns: List of citation objects, or None if no citations found """ + log = logging.getLogger("azure_ai._extract_citations_from_response") + if not isinstance(response_data, dict): + log.debug(f"Response data is not a dict: {type(response_data)}") return None # Try multiple possible locations for citations @@ -385,6 +388,9 @@ def _extract_citations_from_response( and "citations" in choice["delta"]["context"] ): citations = choice["delta"]["context"]["citations"] + log.info( + f"Found {len(citations) if citations else 0} citations in delta.context.citations" + ) # Check in choices[0].message.context.citations (non-streaming) elif ( @@ -394,8 +400,27 @@ def _extract_citations_from_response( and "citations" in choice["message"]["context"] ): citations = choice["message"]["context"]["citations"] + log.info( + f"Found {len(citations) if citations else 0} citations in message.context.citations" + ) + else: + log.debug( + f"No citations found in response. Choice keys: {choice.keys() if isinstance(choice, dict) else 'not a dict'}" + ) + else: + log.debug(f"No choices in response. Response keys: {response_data.keys()}") + + if citations and isinstance(citations, list): + log.info(f"Extracted {len(citations)} citations from response") + # Log first citation structure for debugging + if citations: + log.info( + f"First citation structure: {json.dumps(citations[0], default=str)[:500]}" + ) + return citations - return citations if citations and isinstance(citations, list) else None + log.debug("No valid citations found in response") + return None def _normalize_citation_for_openwebui( self, citation: Dict[str, Any], index: int @@ -413,6 +438,8 @@ def _normalize_citation_for_openwebui( Returns: Complete citation event object with type and data fields """ + log = logging.getLogger("azure_ai._normalize_citation_for_openwebui") + # Get title with fallback chain: title → filepath → url → "Unknown Document" base_title = ( citation.get("title", "").strip() @@ -431,9 +458,12 @@ def _normalize_citation_for_openwebui( if citation.get("metadata"): metadata_entry.update(citation.get("metadata", {})) + # Get document content + content = citation.get("content", "") + # Build normalized citation data structure matching OpenWebUI format exactly citation_data = { - "document": [citation.get("content", "")], + "document": [content], "metadata": [metadata_entry], "source": {"name": title}, } @@ -443,16 +473,31 @@ def _normalize_citation_for_openwebui( citation_data["source"]["url"] = source_url # Add distances array for relevance score (OpenWebUI uses this for percentage display) - if citation.get("score") is not None: - # Wrap score in distances array as required by OpenWebUI format - citation_data["distances"] = [citation["score"]] + # Always include distances to ensure relevance is shown (use 0 if score not available) + score = citation.get("score") + if score is not None: + citation_data["distances"] = [float(score)] + else: + # Default to 0 if no score to ensure the distances field is present + citation_data["distances"] = [0.0] - # Return complete citation event structure - return { + # Build complete citation event structure + citation_event = { "type": "citation", "data": citation_data, } + # Log the normalized citation for debugging + log.info( + f"Normalized citation {index}: title='{title}', " + f"content_length={len(content)}, " + f"url='{source_url}', " + f"score={score}, " + f"event={json.dumps(citation_event, default=str)[:500]}" + ) + + return citation_event + async def _emit_openwebui_citation_events( self, citations: List[Dict[str, Any]], @@ -469,25 +514,40 @@ async def _emit_openwebui_citation_events( citations: List of Azure citation objects __event_emitter__: Event emitter callable for sending citation events """ - if not __event_emitter__ or not citations: + log = logging.getLogger("azure_ai._emit_openwebui_citation_events") + + if not __event_emitter__: + log.warning("No __event_emitter__ provided, cannot emit citation events") return - log = logging.getLogger("azure_ai._emit_openwebui_citation_events") + if not citations: + log.info("No citations to emit") + return + + log.info(f"Emitting {len(citations)} citation events via __event_emitter__") + emitted_count = 0 for i, citation in enumerate(citations, 1): if not isinstance(citation, dict): + log.warning(f"Citation {i} is not a dict, skipping: {type(citation)}") continue try: normalized = self._normalize_citation_for_openwebui(citation, i) # Emit citation event for this individual source + log.info( + f"Emitting citation event {i}/{len(citations)}: {normalized.get('data', {}).get('source', {}).get('name', 'unknown')}" + ) await __event_emitter__(normalized) + emitted_count += 1 - log.debug(f"Emitted citation event for doc{i}") + log.info(f"Successfully emitted citation event for doc{i}") except Exception as e: - log.warning(f"Failed to emit citation event for citation {i}: {e}") + log.exception(f"Failed to emit citation event for citation {i}: {e}") + + log.info(f"Finished emitting {emitted_count}/{len(citations)} citation events") def enhance_azure_search_response(self, response: Dict[str, Any]) -> Dict[str, Any]: """ From 76fec626a92371cac67ff603b1e681f64bb7c8cc Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 26 Nov 2025 09:08:27 +0000 Subject: [PATCH 13/39] Optimize logging to avoid JSON serialization when INFO logging is disabled Co-authored-by: owndev <69784886+owndev@users.noreply.github.com> --- pipelines/azure/azure_ai_foundry.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/pipelines/azure/azure_ai_foundry.py b/pipelines/azure/azure_ai_foundry.py index 620356b..b694754 100644 --- a/pipelines/azure/azure_ai_foundry.py +++ b/pipelines/azure/azure_ai_foundry.py @@ -412,8 +412,8 @@ def _extract_citations_from_response( if citations and isinstance(citations, list): log.info(f"Extracted {len(citations)} citations from response") - # Log first citation structure for debugging - if citations: + # Log first citation structure for debugging (only if INFO logging is enabled) + if citations and log.isEnabledFor(logging.INFO): log.info( f"First citation structure: {json.dumps(citations[0], default=str)[:500]}" ) @@ -487,14 +487,15 @@ def _normalize_citation_for_openwebui( "data": citation_data, } - # Log the normalized citation for debugging - log.info( - f"Normalized citation {index}: title='{title}', " - f"content_length={len(content)}, " - f"url='{source_url}', " - f"score={score}, " - f"event={json.dumps(citation_event, default=str)[:500]}" - ) + # Log the normalized citation for debugging (only if INFO logging is enabled) + if log.isEnabledFor(logging.INFO): + log.info( + f"Normalized citation {index}: title='{title}', " + f"content_length={len(content)}, " + f"url='{source_url}', " + f"score={score}, " + f"event={json.dumps(citation_event, default=str)[:500]}" + ) return citation_event From 6888955fa287e1def0c7e81dbd2472df25d62414 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 26 Nov 2025 10:34:05 +0000 Subject: [PATCH 14/39] Fix NoneType error when citation fields are null Co-authored-by: owndev <69784886+owndev@users.noreply.github.com> --- pipelines/azure/azure_ai_foundry.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/pipelines/azure/azure_ai_foundry.py b/pipelines/azure/azure_ai_foundry.py index b694754..f3b71ba 100644 --- a/pipelines/azure/azure_ai_foundry.py +++ b/pipelines/azure/azure_ai_foundry.py @@ -441,25 +441,30 @@ def _normalize_citation_for_openwebui( log = logging.getLogger("azure_ai._normalize_citation_for_openwebui") # Get title with fallback chain: title → filepath → url → "Unknown Document" + # Handle None values explicitly since dict.get() returns None if key exists but value is None + title_raw = citation.get("title") or "" + filepath_raw = citation.get("filepath") or "" + url_raw = citation.get("url") or "" + base_title = ( - citation.get("title", "").strip() - or citation.get("filepath", "").strip() - or citation.get("url", "").strip() + title_raw.strip() + or filepath_raw.strip() + or url_raw.strip() or "Unknown Document" ) # Always prefix title with doc index to ensure uniqueness and prevent grouping title = f"[doc{index}] {base_title}" # Build source URL for metadata - source_url = citation.get("url") or citation.get("filepath") or "" + source_url = url_raw or filepath_raw # Build metadata with source information metadata_entry = {"source": source_url} if citation.get("metadata"): metadata_entry.update(citation.get("metadata", {})) - # Get document content - content = citation.get("content", "") + # Get document content (handle None values) + content = citation.get("content") or "" # Build normalized citation data structure matching OpenWebUI format exactly citation_data = { From f67f7ea4befaac560746c9f26cfda5eb46d943ad Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 26 Nov 2025 10:51:31 +0000 Subject: [PATCH 15/39] Remove get_bool_env, change defaults, remove doc prefix from titles, add markdown links for citations Co-authored-by: owndev <69784886+owndev@users.noreply.github.com> --- pipelines/azure/azure_ai_foundry.py | 64 +++++++++++++++++++++-------- 1 file changed, 46 insertions(+), 18 deletions(-) diff --git a/pipelines/azure/azure_ai_foundry.py b/pipelines/azure/azure_ai_foundry.py index f3b71ba..23ec534 100644 --- a/pipelines/azure/azure_ai_foundry.py +++ b/pipelines/azure/azure_ai_foundry.py @@ -125,18 +125,6 @@ def __get_pydantic_core_schema__( # Helper functions -def get_bool_env(env_var: str, default: str = "true") -> bool: - """ - Parse a boolean environment variable. - - Args: - env_var: The environment variable name - default: The default value as a string ("true" or "false") - - Returns: - Boolean value parsed from the environment variable - """ - return os.getenv(env_var, default).lower() == "true" async def cleanup_response( @@ -215,13 +203,13 @@ class Valves(BaseModel): # Enable enhanced citation display for Azure AI Search responses AZURE_AI_ENHANCE_CITATIONS: bool = Field( - default=get_bool_env("AZURE_AI_ENHANCE_CITATIONS"), - description="If True, enhance Azure AI Search responses with better citation formatting and source content display.", + default=False, + description="If True, enhance Azure AI Search responses with better citation formatting and source content display (markdown/HTML).", ) # Enable native OpenWebUI citations (structured events and fields) AZURE_AI_OPENWEBUI_CITATIONS: bool = Field( - default=get_bool_env("AZURE_AI_OPENWEBUI_CITATIONS"), + default=True, description="If True, emit native OpenWebUI citation events for streaming responses and attach openwebui_citations field for non-streaming responses. Enables citation cards and UI in OpenWebUI frontend.", ) @@ -452,8 +440,8 @@ def _normalize_citation_for_openwebui( or url_raw.strip() or "Unknown Document" ) - # Always prefix title with doc index to ensure uniqueness and prevent grouping - title = f"[doc{index}] {base_title}" + # Use base title directly without prefix for OpenWebUI citation cards + title = base_title # Build source URL for metadata source_url = url_raw or filepath_raw @@ -607,6 +595,11 @@ def enhance_azure_search_response(self, response: Dict[str, Any]) -> Dict[str, A # Enhance the content with better citation display (if enabled) enhanced_content = content + # Convert [docX] references to markdown links + enhanced_content = self._convert_doc_refs_to_markdown_links( + enhanced_content, citations + ) + # Add citation section at the end (if markdown/HTML citations are enabled) if self.valves.AZURE_AI_ENHANCE_CITATIONS and citation_details: citation_section = self._format_citation_section( @@ -820,7 +813,7 @@ async def stream_processor_with_citations( all_chunks.append(chunk) # Log chunk for debugging (only first 200 chars to avoid spam) - log.debug(f"Processing chunk: {chunk_str[:200]}...") + # log.debug(f"Processing chunk: {chunk_str[:200]}...") # Extract content from delta messages to build the full response content try: @@ -1034,6 +1027,41 @@ def _extract_referenced_citations(self, content: str) -> Set[int]: # Convert to integers and return as a set return {int(match) for match in matches} + def _convert_doc_refs_to_markdown_links( + self, content: str, citations: List[Dict[str, Any]] + ) -> str: + """ + Convert [docX] references in content to markdown links if the document has a URL. + + Args: + content: The response content containing [docX] references + citations: List of citation objects with URLs + + Returns: + Content with [docX] references converted to markdown links + """ + if not citations: + return content + + # Build a mapping of doc index to URL + doc_urls = {} + for i, citation in enumerate(citations, 1): + if isinstance(citation, dict): + url = citation.get("url") or citation.get("filepath") or "" + if url: + doc_urls[i] = url + + # Replace [docX] with markdown links where URL is available + def replace_doc_ref(match: re.Match) -> str: + doc_num = int(match.group(1)) + if doc_num in doc_urls: + # Use standard markdown link format: [text](url) + return f"[doc{doc_num}]({doc_urls[doc_num]})" + return match.group(0) # Keep original if no URL + + pattern = r"\[doc(\d+)\]" + return re.sub(pattern, replace_doc_ref, content) + def _format_citation_section( self, citations: List[Dict[str, Any]], From 0e1a58c6413b1a4fcd5970a9e0620c1c91a0cf99 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 26 Nov 2025 11:05:01 +0000 Subject: [PATCH 16/39] Fix: use stream_processor_with_citations when AZURE_AI_OPENWEBUI_CITATIONS is enabled and add [docX] prefix to titles Co-authored-by: owndev <69784886+owndev@users.noreply.github.com> --- pipelines/azure/azure_ai_foundry.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pipelines/azure/azure_ai_foundry.py b/pipelines/azure/azure_ai_foundry.py index 23ec534..b61d489 100644 --- a/pipelines/azure/azure_ai_foundry.py +++ b/pipelines/azure/azure_ai_foundry.py @@ -440,8 +440,8 @@ def _normalize_citation_for_openwebui( or url_raw.strip() or "Unknown Document" ) - # Use base title directly without prefix for OpenWebUI citation cards - title = base_title + # Include [docX] prefix in OpenWebUI citation card titles for document identification + title = f"[doc{index}] {base_title}" # Build source URL for metadata source_url = url_raw or filepath_raw @@ -1392,9 +1392,9 @@ async def pipe( sse_headers.pop("Content-Length", None) # Use enhanced stream processor if Azure AI Search is configured and citations are enabled - if ( - self.valves.AZURE_AI_DATA_SOURCES - and self.valves.AZURE_AI_ENHANCE_CITATIONS + if self.valves.AZURE_AI_DATA_SOURCES and ( + self.valves.AZURE_AI_ENHANCE_CITATIONS + or self.valves.AZURE_AI_OPENWEBUI_CITATIONS ): stream_processor = self.stream_processor_with_citations else: From 7283f1c2b568e816c6ae0c086f4473eb4e6bbf49 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 26 Nov 2025 11:13:02 +0000 Subject: [PATCH 17/39] Remove _convert_doc_refs_to_markdown_links function as requested Co-authored-by: owndev <69784886+owndev@users.noreply.github.com> --- pipelines/azure/azure_ai_foundry.py | 40 ----------------------------- 1 file changed, 40 deletions(-) diff --git a/pipelines/azure/azure_ai_foundry.py b/pipelines/azure/azure_ai_foundry.py index b61d489..7816028 100644 --- a/pipelines/azure/azure_ai_foundry.py +++ b/pipelines/azure/azure_ai_foundry.py @@ -595,11 +595,6 @@ def enhance_azure_search_response(self, response: Dict[str, Any]) -> Dict[str, A # Enhance the content with better citation display (if enabled) enhanced_content = content - # Convert [docX] references to markdown links - enhanced_content = self._convert_doc_refs_to_markdown_links( - enhanced_content, citations - ) - # Add citation section at the end (if markdown/HTML citations are enabled) if self.valves.AZURE_AI_ENHANCE_CITATIONS and citation_details: citation_section = self._format_citation_section( @@ -1027,41 +1022,6 @@ def _extract_referenced_citations(self, content: str) -> Set[int]: # Convert to integers and return as a set return {int(match) for match in matches} - def _convert_doc_refs_to_markdown_links( - self, content: str, citations: List[Dict[str, Any]] - ) -> str: - """ - Convert [docX] references in content to markdown links if the document has a URL. - - Args: - content: The response content containing [docX] references - citations: List of citation objects with URLs - - Returns: - Content with [docX] references converted to markdown links - """ - if not citations: - return content - - # Build a mapping of doc index to URL - doc_urls = {} - for i, citation in enumerate(citations, 1): - if isinstance(citation, dict): - url = citation.get("url") or citation.get("filepath") or "" - if url: - doc_urls[i] = url - - # Replace [docX] with markdown links where URL is available - def replace_doc_ref(match: re.Match) -> str: - doc_num = int(match.group(1)) - if doc_num in doc_urls: - # Use standard markdown link format: [text](url) - return f"[doc{doc_num}]({doc_urls[doc_num]})" - return match.group(0) # Keep original if no URL - - pattern = r"\[doc(\d+)\]" - return re.sub(pattern, replace_doc_ref, content) - def _format_citation_section( self, citations: List[Dict[str, Any]], From f9407a7db0bf40ccef0b4f7958bcb45011e0dc10 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 26 Nov 2025 11:30:06 +0000 Subject: [PATCH 18/39] Add detailed debug logging for citation events to help troubleshoot [docX] prefix issue Co-authored-by: owndev <69784886+owndev@users.noreply.github.com> --- pipelines/azure/azure_ai_foundry.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pipelines/azure/azure_ai_foundry.py b/pipelines/azure/azure_ai_foundry.py index 7816028..9c3d714 100644 --- a/pipelines/azure/azure_ai_foundry.py +++ b/pipelines/azure/azure_ai_foundry.py @@ -529,9 +529,15 @@ async def _emit_openwebui_citation_events( try: normalized = self._normalize_citation_for_openwebui(citation, i) + # Log the full citation JSON for debugging + log.debug( + f"Full citation event JSON for doc{i}: {json.dumps(normalized, default=str)}" + ) + # Emit citation event for this individual source + source_name = normalized.get("data", {}).get("source", {}).get("name", "unknown") log.info( - f"Emitting citation event {i}/{len(citations)}: {normalized.get('data', {}).get('source', {}).get('name', 'unknown')}" + f"Emitting citation event {i}/{len(citations)} with source.name='{source_name}'" ) await __event_emitter__(normalized) emitted_count += 1 From 24d3ef95e9fe6a3cebd0794468e18bba857c5cc3 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 26 Nov 2025 12:13:15 +0000 Subject: [PATCH 19/39] Fix: use [docX] title in metadata.source field for proper OpenWebUI display Co-authored-by: owndev <69784886+owndev@users.noreply.github.com> --- pipelines/azure/azure_ai_foundry.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pipelines/azure/azure_ai_foundry.py b/pipelines/azure/azure_ai_foundry.py index 9c3d714..e5d842f 100644 --- a/pipelines/azure/azure_ai_foundry.py +++ b/pipelines/azure/azure_ai_foundry.py @@ -447,7 +447,9 @@ def _normalize_citation_for_openwebui( source_url = url_raw or filepath_raw # Build metadata with source information - metadata_entry = {"source": source_url} + # Use title with [docX] prefix as metadata source for OpenWebUI display + # The UI may extract display name from metadata.source rather than source.name + metadata_entry = {"source": title, "url": source_url} if citation.get("metadata"): metadata_entry.update(citation.get("metadata", {})) From 52b007ce173d6f0e58a13b194bdad5ce5b074013 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 26 Nov 2025 12:30:14 +0000 Subject: [PATCH 20/39] Fix: Filter OpenWebUI citations to only show documents referenced in response content Co-authored-by: owndev <69784886+owndev@users.noreply.github.com> --- pipelines/azure/azure_ai_foundry.py | 57 +++++++++++++++++++++++------ 1 file changed, 45 insertions(+), 12 deletions(-) diff --git a/pipelines/azure/azure_ai_foundry.py b/pipelines/azure/azure_ai_foundry.py index e5d842f..29f6833 100644 --- a/pipelines/azure/azure_ai_foundry.py +++ b/pipelines/azure/azure_ai_foundry.py @@ -498,6 +498,7 @@ async def _emit_openwebui_citation_events( self, citations: List[Dict[str, Any]], __event_emitter__: Optional[Callable[..., Any]], + content: str = "", ) -> None: """ Emit OpenWebUI citation events for citations. @@ -506,9 +507,12 @@ async def _emit_openwebui_citation_events( citation event format. Each citation is emitted separately to ensure all sources appear in the UI. + Only emits citations that are actually referenced in the content (e.g., [doc1], [doc2]). + Args: citations: List of Azure citation objects __event_emitter__: Event emitter callable for sending citation events + content: The response content (used to filter only referenced citations) """ log = logging.getLogger("azure_ai._emit_openwebui_citation_events") @@ -520,10 +524,25 @@ async def _emit_openwebui_citation_events( log.info("No citations to emit") return - log.info(f"Emitting {len(citations)} citation events via __event_emitter__") + # Extract which citations are actually referenced in the content + referenced_indices = self._extract_referenced_citations(content) + + # If we couldn't find any references, include all citations (backward compatibility) + if not referenced_indices: + referenced_indices = set(range(1, len(citations) + 1)) + log.debug(f"No [docX] references found in content, including all {len(citations)} citations") + else: + log.info(f"Found {len(referenced_indices)} referenced citations: {sorted(referenced_indices)}") + + log.info(f"Emitting citation events for {len(referenced_indices)} referenced citations via __event_emitter__") emitted_count = 0 for i, citation in enumerate(citations, 1): + # Skip citations that are not referenced in the content + if i not in referenced_indices: + log.debug(f"Skipping citation {i} - not referenced in content") + continue + if not isinstance(citation, dict): log.warning(f"Citation {i} is not a dict, skipping: {type(citation)}") continue @@ -549,7 +568,7 @@ async def _emit_openwebui_citation_events( except Exception as e: log.exception(f"Failed to emit citation event for citation {i}: {e}") - log.info(f"Finished emitting {emitted_count}/{len(citations)} citation events") + log.info(f"Finished emitting {emitted_count}/{len(referenced_indices)} citation events") def enhance_azure_search_response(self, response: Dict[str, Any]) -> Dict[str, Any]: """ @@ -913,15 +932,8 @@ async def stream_processor_with_citations( log.info( f"Successfully extracted {len(citations_data)} citations from stream" ) - - # Emit native OpenWebUI citation events immediately if enabled - if ( - self.valves.AZURE_AI_OPENWEBUI_CITATIONS - and __event_emitter__ - ): - await self._emit_openwebui_citation_events( - citations_data, __event_emitter__ - ) + # Note: OpenWebUI citation events are emitted after the stream ends + # to filter only citations referenced in the response content except json.JSONDecodeError: # Skip invalid JSON @@ -938,6 +950,18 @@ async def stream_processor_with_citations( log.debug("End of stream detected") break + # After the stream ends, emit OpenWebUI citation events if enabled + if ( + citations_data + and self.valves.AZURE_AI_OPENWEBUI_CITATIONS + and __event_emitter__ + ): + log.info("Emitting OpenWebUI citation events at end of stream...") + # Filter to only citations referenced in the response content + await self._emit_openwebui_citation_events( + citations_data, __event_emitter__, response_content + ) + # After the stream ends, add markdown/HTML citations if we found any and it's enabled if ( citations_data @@ -1406,8 +1430,17 @@ async def pipe( if self.valves.AZURE_AI_OPENWEBUI_CITATIONS and __event_emitter__: citations = self._extract_citations_from_response(response) if citations: + # Get response content for filtering + response_content = "" + if ( + isinstance(response, dict) + and "choices" in response + and response["choices"] + ): + message = response["choices"][0].get("message", {}) + response_content = message.get("content", "") await self._emit_openwebui_citation_events( - citations, __event_emitter__ + citations, __event_emitter__, response_content ) # Send completion status update From a95bf0147c546c0d3ff926b3ab3e0730c5e7898b Mon Sep 17 00:00:00 2001 From: owndev <69784886+owndev@users.noreply.github.com> Date: Wed, 26 Nov 2025 13:34:39 +0100 Subject: [PATCH 21/39] fix(azure_ai_foundry.py): Update citation title --- pipelines/azure/azure_ai_foundry.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/pipelines/azure/azure_ai_foundry.py b/pipelines/azure/azure_ai_foundry.py index 29f6833..a049bc1 100644 --- a/pipelines/azure/azure_ai_foundry.py +++ b/pipelines/azure/azure_ai_foundry.py @@ -441,7 +441,7 @@ def _normalize_citation_for_openwebui( or "Unknown Document" ) # Include [docX] prefix in OpenWebUI citation card titles for document identification - title = f"[doc{index}] {base_title}" + title = f"[doc{index}] - {base_title}" # Build source URL for metadata source_url = url_raw or filepath_raw @@ -530,11 +530,17 @@ async def _emit_openwebui_citation_events( # If we couldn't find any references, include all citations (backward compatibility) if not referenced_indices: referenced_indices = set(range(1, len(citations) + 1)) - log.debug(f"No [docX] references found in content, including all {len(citations)} citations") + log.debug( + f"No [docX] references found in content, including all {len(citations)} citations" + ) else: - log.info(f"Found {len(referenced_indices)} referenced citations: {sorted(referenced_indices)}") + log.info( + f"Found {len(referenced_indices)} referenced citations: {sorted(referenced_indices)}" + ) - log.info(f"Emitting citation events for {len(referenced_indices)} referenced citations via __event_emitter__") + log.info( + f"Emitting citation events for {len(referenced_indices)} referenced citations via __event_emitter__" + ) emitted_count = 0 for i, citation in enumerate(citations, 1): @@ -556,7 +562,9 @@ async def _emit_openwebui_citation_events( ) # Emit citation event for this individual source - source_name = normalized.get("data", {}).get("source", {}).get("name", "unknown") + source_name = ( + normalized.get("data", {}).get("source", {}).get("name", "unknown") + ) log.info( f"Emitting citation event {i}/{len(citations)} with source.name='{source_name}'" ) @@ -568,7 +576,9 @@ async def _emit_openwebui_citation_events( except Exception as e: log.exception(f"Failed to emit citation event for citation {i}: {e}") - log.info(f"Finished emitting {emitted_count}/{len(referenced_indices)} citation events") + log.info( + f"Finished emitting {emitted_count}/{len(referenced_indices)} citation events" + ) def enhance_azure_search_response(self, response: Dict[str, Any]) -> Dict[str, Any]: """ From 9358ddb4895ed3cfdffecc655d6048d823191374 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 1 Dec 2025 11:20:17 +0000 Subject: [PATCH 22/39] Add support for Azure AI Search relevance scores (original_search_score, rerank_score) Co-authored-by: owndev <69784886+owndev@users.noreply.github.com> --- pipelines/azure/azure_ai_foundry.py | 273 +++++++++++++++++++++------- 1 file changed, 203 insertions(+), 70 deletions(-) diff --git a/pipelines/azure/azure_ai_foundry.py b/pipelines/azure/azure_ai_foundry.py index a049bc1..aa87ed8 100644 --- a/pipelines/azure/azure_ai_foundry.py +++ b/pipelines/azure/azure_ai_foundry.py @@ -213,6 +213,12 @@ class Valves(BaseModel): description="If True, emit native OpenWebUI citation events for streaming responses and attach openwebui_citations field for non-streaming responses. Enables citation cards and UI in OpenWebUI frontend.", ) + # Enable relevance scores from Azure AI Search + AZURE_AI_INCLUDE_SEARCH_SCORES: bool = Field( + default=True, + description="If True, automatically add 'include_contexts' with 'all_retrieved_documents' to Azure AI Search requests to get relevance scores (original_search_score and rerank_score). This enables relevance percentage display in citation cards.", + ) + def __init__(self): self.valves = self.Valves() self.name: str = f"{self.valves.AZURE_AI_PIPELINE_PREFIX}:" @@ -326,22 +332,51 @@ def get_azure_ai_data_sources(self) -> Optional[List[Dict[str, Any]]]: Builds Azure AI data sources configuration from the AZURE_AI_DATA_SOURCES environment variable. Only works with Azure OpenAI endpoints: https://.openai.azure.com/openai/deployments//chat/completions?api-version=2025-01-01-preview + If AZURE_AI_INCLUDE_SEARCH_SCORES is enabled, automatically adds 'include_contexts' + with 'all_retrieved_documents' to get relevance scores from Azure AI Search. + Returns: List containing Azure AI data source configuration, or None if not configured. """ if not self.valves.AZURE_AI_DATA_SOURCES: return None + log = logging.getLogger("azure_ai.get_azure_ai_data_sources") + try: data_sources = json.loads(self.valves.AZURE_AI_DATA_SOURCES) - if isinstance(data_sources, list): - return data_sources - else: + if not isinstance(data_sources, list): # If it's a single object, wrap it in a list - return [data_sources] + data_sources = [data_sources] + + # If AZURE_AI_INCLUDE_SEARCH_SCORES is enabled, add include_contexts + if self.valves.AZURE_AI_INCLUDE_SEARCH_SCORES: + for source in data_sources: + if ( + isinstance(source, dict) + and source.get("type") == "azure_search" + and "parameters" in source + ): + params = source["parameters"] + # Get or create include_contexts list + include_contexts = params.get("include_contexts", []) + if not isinstance(include_contexts, list): + include_contexts = [include_contexts] + + # Add 'citations' and 'all_retrieved_documents' if not present + if "citations" not in include_contexts: + include_contexts.append("citations") + if "all_retrieved_documents" not in include_contexts: + include_contexts.append("all_retrieved_documents") + + params["include_contexts"] = include_contexts + log.debug( + f"Added include_contexts to Azure Search: {include_contexts}" + ) + + return data_sources except json.JSONDecodeError as e: # Log error and return None if JSON parsing fails - log = logging.getLogger("azure_ai.get_azure_ai_data_sources") log.error(f"Error parsing AZURE_AI_DATA_SOURCES: {e}") return None @@ -351,6 +386,10 @@ def _extract_citations_from_response( """ Extract citations from an Azure AI response (streaming or non-streaming). + Supports both 'citations' and 'all_retrieved_documents' response structures. + When include_contexts includes 'all_retrieved_documents', the response contains + additional score fields like 'original_search_score' and 'rerank_score'. + Args: response_data: Response data from Azure AI (can be a delta or full message) @@ -366,34 +405,45 @@ def _extract_citations_from_response( # Try multiple possible locations for citations citations = None - # Check in choices[0].delta.context.citations (streaming) + # Check in choices[0].delta.context or choices[0].message.context if "choices" in response_data and response_data["choices"]: choice = response_data["choices"][0] - if ( - "delta" in choice - and isinstance(choice["delta"], dict) - and "context" in choice["delta"] - and "citations" in choice["delta"]["context"] - ): - citations = choice["delta"]["context"]["citations"] - log.info( - f"Found {len(citations) if citations else 0} citations in delta.context.citations" - ) + context = None + + # Get context from delta (streaming) or message (non-streaming) + if "delta" in choice and isinstance(choice["delta"], dict): + context = choice["delta"].get("context") + elif "message" in choice and isinstance(choice["message"], dict): + context = choice["message"].get("context") + + if context and isinstance(context, dict): + # Try citations first + if "citations" in context: + citations = context["citations"] + log.info( + f"Found {len(citations) if citations else 0} citations in context.citations" + ) - # Check in choices[0].message.context.citations (non-streaming) - elif ( - "message" in choice - and isinstance(choice["message"], dict) - and "context" in choice["message"] - and "citations" in choice["message"]["context"] - ): - citations = choice["message"]["context"]["citations"] - log.info( - f"Found {len(citations) if citations else 0} citations in message.context.citations" - ) + # If all_retrieved_documents is present, merge score data into citations + if "all_retrieved_documents" in context: + all_docs = context["all_retrieved_documents"] + log.debug( + f"Found {len(all_docs) if all_docs else 0} all_retrieved_documents" + ) + + # If we have both citations and all_retrieved_documents, + # try to merge score data from all_retrieved_documents into citations + if citations and all_docs: + self._merge_score_data(citations, all_docs, log) + elif all_docs and not citations: + # Use all_retrieved_documents as citations if no citations found + citations = all_docs + log.info( + f"Using {len(citations)} all_retrieved_documents as citations" + ) else: log.debug( - f"No citations found in response. Choice keys: {choice.keys() if isinstance(choice, dict) else 'not a dict'}" + f"No context found in response. Choice keys: {choice.keys() if isinstance(choice, dict) else 'not a dict'}" ) else: log.debug(f"No choices in response. Response keys: {response_data.keys()}") @@ -410,6 +460,64 @@ def _extract_citations_from_response( log.debug("No valid citations found in response") return None + def _merge_score_data( + self, + citations: List[Dict[str, Any]], + all_docs: List[Dict[str, Any]], + log: logging.Logger, + ) -> None: + """ + Merge score data from all_retrieved_documents into citations. + + When include_contexts includes 'all_retrieved_documents', Azure returns + additional documents with score fields. This method attempts to match + them with citations and copy over the score data. + + Args: + citations: List of citation objects to update (modified in place) + all_docs: List of all_retrieved_documents with score data + log: Logger instance + """ + # Build a lookup map by content or filepath to match documents + doc_scores = {} + for doc in all_docs: + # Try to match by chunk_id, filepath, or content hash + key = None + if doc.get("chunk_id"): + key = doc["chunk_id"] + elif doc.get("filepath"): + key = doc["filepath"] + elif doc.get("content"): + # Use first 100 chars of content as a key + key = doc["content"][:100] if len(doc.get("content", "")) > 100 else doc.get("content") + + if key: + doc_scores[key] = { + "original_search_score": doc.get("original_search_score"), + "rerank_score": doc.get("rerank_score"), + } + + # Match citations with score data + matched = 0 + for citation in citations: + key = None + if citation.get("chunk_id"): + key = citation["chunk_id"] + elif citation.get("filepath"): + key = citation["filepath"] + elif citation.get("content"): + key = citation["content"][:100] if len(citation.get("content", "")) > 100 else citation.get("content") + + if key and key in doc_scores: + scores = doc_scores[key] + if scores.get("original_search_score") is not None: + citation["original_search_score"] = scores["original_search_score"] + if scores.get("rerank_score") is not None: + citation["rerank_score"] = scores["rerank_score"] + matched += 1 + + log.debug(f"Merged score data for {matched}/{len(citations)} citations") + def _normalize_citation_for_openwebui( self, citation: Dict[str, Any], index: int ) -> Dict[str, Any]: @@ -468,10 +576,38 @@ def _normalize_citation_for_openwebui( citation_data["source"]["url"] = source_url # Add distances array for relevance score (OpenWebUI uses this for percentage display) - # Always include distances to ensure relevance is shown (use 0 if score not available) - score = citation.get("score") - if score is not None: - citation_data["distances"] = [float(score)] + # Priority: rerank_score > original_search_score > score + # rerank_score is the semantic reranker score (if enabled in Azure AI Search) + # original_search_score is the BM25/keyword search score + # score is a legacy field for backward compatibility + rerank_score = citation.get("rerank_score") + original_search_score = citation.get("original_search_score") + legacy_score = citation.get("score") + + # Prefer rerank_score (semantic ranker), then original_search_score, then legacy score + if rerank_score is not None: + # Reranker scores are typically 0-4 range, normalize to 0-1 for display + # Azure AI Search semantic reranker returns scores in 0-4 range + normalized_score = min(float(rerank_score) / 4.0, 1.0) + citation_data["distances"] = [normalized_score] + log.debug( + f"Using rerank_score {rerank_score} -> normalized {normalized_score}" + ) + elif original_search_score is not None: + # Original search scores can vary widely, use as-is if <= 1, otherwise normalize + score_val = float(original_search_score) + if score_val > 1.0: + # Normalize high scores (BM25 scores can be > 1) + normalized_score = min(score_val / 100.0, 1.0) + else: + normalized_score = score_val + citation_data["distances"] = [normalized_score] + log.debug( + f"Using original_search_score {original_search_score} -> {normalized_score}" + ) + elif legacy_score is not None: + citation_data["distances"] = [float(legacy_score)] + log.debug(f"Using legacy score {legacy_score}") else: # Default to 0 if no score to ensure the distances field is present citation_data["distances"] = [0.0] @@ -488,7 +624,8 @@ def _normalize_citation_for_openwebui( f"Normalized citation {index}: title='{title}', " f"content_length={len(content)}, " f"url='{source_url}', " - f"score={score}, " + f"rerank_score={rerank_score}, original_search_score={original_search_score}, " + f"distances={citation_data['distances']}, " f"event={json.dumps(citation_event, default=str)[:500]}" ) @@ -874,9 +1011,9 @@ async def stream_processor_with_citations( except Exception as e: log.debug(f"Exception while processing chunk: {e}") - # Look for citations in any part of the response - if "citations" in chunk_str.lower() and not citations_data: - log.debug("Found 'citations' in chunk, attempting to parse...") + # Look for citations or all_retrieved_documents in any part of the response + if ("citations" in chunk_str.lower() or "all_retrieved_documents" in chunk_str.lower()) and not citations_data: + log.debug("Found 'citations' or 'all_retrieved_documents' in chunk, attempting to parse...") # Try to extract citation data from the current buffer try: @@ -894,54 +1031,50 @@ async def stream_processor_with_citations( # Check multiple possible locations for citations citations_found = None + all_docs_found = None if ( isinstance(response_data, dict) and "choices" in response_data ): for choice in response_data["choices"]: - # Check in delta.context.citations - if ( - "delta" in choice - and isinstance( - choice["delta"], dict - ) - and "context" in choice["delta"] - and "citations" - in choice["delta"]["context"] - ): - citations_found = choice["delta"][ - "context" - ]["citations"] - log.debug( - f"Found citations in delta.context: {len(citations_found)} citations" - ) + context = None + # Get context from delta or message + if "delta" in choice and isinstance(choice["delta"], dict): + context = choice["delta"].get("context") + elif "message" in choice and isinstance(choice["message"], dict): + context = choice["message"].get("context") + + if context and isinstance(context, dict): + # Check for citations + if "citations" in context: + citations_found = context["citations"] + log.debug( + f"Found citations in context: {len(citations_found)} citations" + ) + # Check for all_retrieved_documents + if "all_retrieved_documents" in context: + all_docs_found = context["all_retrieved_documents"] + log.debug( + f"Found all_retrieved_documents in context: {len(all_docs_found)} docs" + ) break - # Check in message.context.citations - elif ( - "message" in choice - and isinstance( - choice["message"], dict - ) - and "context" in choice["message"] - and "citations" - in choice["message"]["context"] - ): - citations_found = choice["message"][ - "context" - ]["citations"] - log.debug( - f"Found citations in message.context: {len(citations_found)} citations" - ) - break + # Merge score data if we have both + if citations_found and all_docs_found: + self._merge_score_data(citations_found, all_docs_found, log) - # Store the first valid citations we find + # Use citations if found, otherwise use all_retrieved_documents if citations_found and not citations_data: citations_data = citations_found log.info( f"Successfully extracted {len(citations_data)} citations from stream" ) + elif all_docs_found and not citations_data: + citations_data = all_docs_found + log.info( + f"Using {len(citations_data)} all_retrieved_documents as citations" + ) # Note: OpenWebUI citation events are emitted after the stream ends # to filter only citations referenced in the response content From a09daee583014372ee797849e201d863dc23a9af Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 1 Dec 2025 11:43:48 +0000 Subject: [PATCH 23/39] Fix score matching: use title as primary key, add multiple matching strategies Co-authored-by: owndev <69784886+owndev@users.noreply.github.com> --- pipelines/azure/azure_ai_foundry.py | 112 +++++++++++++++++++++------- 1 file changed, 83 insertions(+), 29 deletions(-) diff --git a/pipelines/azure/azure_ai_foundry.py b/pipelines/azure/azure_ai_foundry.py index aa87ed8..c7fe9fe 100644 --- a/pipelines/azure/azure_ai_foundry.py +++ b/pipelines/azure/azure_ai_foundry.py @@ -478,45 +478,99 @@ def _merge_score_data( all_docs: List of all_retrieved_documents with score data log: Logger instance """ - # Build a lookup map by content or filepath to match documents - doc_scores = {} + # Build multiple lookup maps to maximize matching chances + # all_retrieved_documents may have different keys than citations + doc_scores_by_title = {} + doc_scores_by_filepath = {} + doc_scores_by_content = {} + doc_scores_by_chunk_id = {} + for doc in all_docs: - # Try to match by chunk_id, filepath, or content hash - key = None - if doc.get("chunk_id"): - key = doc["chunk_id"] - elif doc.get("filepath"): - key = doc["filepath"] - elif doc.get("content"): - # Use first 100 chars of content as a key - key = doc["content"][:100] if len(doc.get("content", "")) > 100 else doc.get("content") - - if key: - doc_scores[key] = { - "original_search_score": doc.get("original_search_score"), - "rerank_score": doc.get("rerank_score"), - } + scores = { + "original_search_score": doc.get("original_search_score"), + "rerank_score": doc.get("rerank_score"), + } + + # Only store if we have at least one score + if scores["original_search_score"] is None and scores["rerank_score"] is None: + continue - # Match citations with score data + # Index by title + if doc.get("title"): + doc_scores_by_title[doc["title"]] = scores + + # Index by filepath + if doc.get("filepath"): + doc_scores_by_filepath[doc["filepath"]] = scores + + # Index by chunk_id (may include title as prefix for uniqueness) + if doc.get("chunk_id") is not None: + chunk_key = doc.get("chunk_id") + # Also try with title prefix for uniqueness + if doc.get("title"): + chunk_key = f"{doc['title']}_{doc['chunk_id']}" + doc_scores_by_chunk_id[str(doc["chunk_id"])] = scores + doc_scores_by_chunk_id[chunk_key] = scores + + # Index by content prefix (first 100 chars) + if doc.get("content"): + content_key = doc["content"][:100] if len(doc.get("content", "")) > 100 else doc.get("content") + doc_scores_by_content[content_key] = scores + + log.debug( + f"Built score lookup: by_title={len(doc_scores_by_title)}, " + f"by_filepath={len(doc_scores_by_filepath)}, " + f"by_chunk_id={len(doc_scores_by_chunk_id)}, " + f"by_content={len(doc_scores_by_content)}" + ) + + # Match citations with score data using multiple strategies matched = 0 for citation in citations: - key = None - if citation.get("chunk_id"): - key = citation["chunk_id"] - elif citation.get("filepath"): - key = citation["filepath"] - elif citation.get("content"): - key = citation["content"][:100] if len(citation.get("content", "")) > 100 else citation.get("content") - - if key and key in doc_scores: - scores = doc_scores[key] + scores = None + + # Try matching by title first (most reliable) + if not scores and citation.get("title"): + scores = doc_scores_by_title.get(citation["title"]) + if scores: + log.debug(f"Matched citation by title: {citation['title']}") + + # Try matching by filepath + if not scores and citation.get("filepath"): + scores = doc_scores_by_filepath.get(citation["filepath"]) + if scores: + log.debug(f"Matched citation by filepath: {citation['filepath']}") + + # Try matching by chunk_id with title prefix + if not scores and citation.get("chunk_id") is not None: + chunk_key = str(citation["chunk_id"]) + if citation.get("title"): + chunk_key_with_title = f"{citation['title']}_{citation['chunk_id']}" + scores = doc_scores_by_chunk_id.get(chunk_key_with_title) + if not scores: + scores = doc_scores_by_chunk_id.get(chunk_key) + if scores: + log.debug(f"Matched citation by chunk_id: {citation['chunk_id']}") + + # Try matching by content prefix + if not scores and citation.get("content"): + content_key = citation["content"][:100] if len(citation.get("content", "")) > 100 else citation.get("content") + scores = doc_scores_by_content.get(content_key) + if scores: + log.debug(f"Matched citation by content prefix") + + if scores: if scores.get("original_search_score") is not None: citation["original_search_score"] = scores["original_search_score"] if scores.get("rerank_score") is not None: citation["rerank_score"] = scores["rerank_score"] matched += 1 + log.debug( + f"Citation scores: original={scores.get('original_search_score')}, " + f"rerank={scores.get('rerank_score')}" + ) - log.debug(f"Merged score data for {matched}/{len(citations)} citations") + log.info(f"Merged score data for {matched}/{len(citations)} citations") def _normalize_citation_for_openwebui( self, citation: Dict[str, Any], index: int From e7c30504ab0d40bcd2564476c88c688a85201f30 Mon Sep 17 00:00:00 2001 From: owndev <69784886+owndev@users.noreply.github.com> Date: Mon, 1 Dec 2025 14:06:41 +0100 Subject: [PATCH 24/39] refactor: Improve readability of score and content key checks in Pipe class --- pipelines/azure/azure_ai_foundry.py | 71 +++++++++++++++++++++-------- 1 file changed, 53 insertions(+), 18 deletions(-) diff --git a/pipelines/azure/azure_ai_foundry.py b/pipelines/azure/azure_ai_foundry.py index c7fe9fe..ea97f35 100644 --- a/pipelines/azure/azure_ai_foundry.py +++ b/pipelines/azure/azure_ai_foundry.py @@ -492,7 +492,10 @@ def _merge_score_data( } # Only store if we have at least one score - if scores["original_search_score"] is None and scores["rerank_score"] is None: + if ( + scores["original_search_score"] is None + and scores["rerank_score"] is None + ): continue # Index by title @@ -514,7 +517,11 @@ def _merge_score_data( # Index by content prefix (first 100 chars) if doc.get("content"): - content_key = doc["content"][:100] if len(doc.get("content", "")) > 100 else doc.get("content") + content_key = ( + doc["content"][:100] + if len(doc.get("content", "")) > 100 + else doc.get("content") + ) doc_scores_by_content[content_key] = scores log.debug( @@ -554,7 +561,11 @@ def _merge_score_data( # Try matching by content prefix if not scores and citation.get("content"): - content_key = citation["content"][:100] if len(citation.get("content", "")) > 100 else citation.get("content") + content_key = ( + citation["content"][:100] + if len(citation.get("content", "")) > 100 + else citation.get("content") + ) scores = doc_scores_by_content.get(content_key) if scores: log.debug(f"Matched citation by content prefix") @@ -748,9 +759,9 @@ async def _emit_openwebui_citation_events( normalized = self._normalize_citation_for_openwebui(citation, i) # Log the full citation JSON for debugging - log.debug( - f"Full citation event JSON for doc{i}: {json.dumps(normalized, default=str)}" - ) + # log.debug( + # f"Full citation event JSON for doc{i}: {json.dumps(normalized, default=str)}" + # ) # Emit citation event for this individual source source_name = ( @@ -1066,8 +1077,13 @@ async def stream_processor_with_citations( log.debug(f"Exception while processing chunk: {e}") # Look for citations or all_retrieved_documents in any part of the response - if ("citations" in chunk_str.lower() or "all_retrieved_documents" in chunk_str.lower()) and not citations_data: - log.debug("Found 'citations' or 'all_retrieved_documents' in chunk, attempting to parse...") + if ( + "citations" in chunk_str.lower() + or "all_retrieved_documents" in chunk_str.lower() + ) and not citations_data: + log.debug( + "Found 'citations' or 'all_retrieved_documents' in chunk, attempting to parse..." + ) # Try to extract citation data from the current buffer try: @@ -1094,21 +1110,38 @@ async def stream_processor_with_citations( for choice in response_data["choices"]: context = None # Get context from delta or message - if "delta" in choice and isinstance(choice["delta"], dict): - context = choice["delta"].get("context") - elif "message" in choice and isinstance(choice["message"], dict): - context = choice["message"].get("context") - - if context and isinstance(context, dict): + if "delta" in choice and isinstance( + choice["delta"], dict + ): + context = choice["delta"].get( + "context" + ) + elif "message" in choice and isinstance( + choice["message"], dict + ): + context = choice["message"].get( + "context" + ) + + if context and isinstance( + context, dict + ): # Check for citations if "citations" in context: - citations_found = context["citations"] + citations_found = context[ + "citations" + ] log.debug( f"Found citations in context: {len(citations_found)} citations" ) # Check for all_retrieved_documents - if "all_retrieved_documents" in context: - all_docs_found = context["all_retrieved_documents"] + if ( + "all_retrieved_documents" + in context + ): + all_docs_found = context[ + "all_retrieved_documents" + ] log.debug( f"Found all_retrieved_documents in context: {len(all_docs_found)} docs" ) @@ -1116,7 +1149,9 @@ async def stream_processor_with_citations( # Merge score data if we have both if citations_found and all_docs_found: - self._merge_score_data(citations_found, all_docs_found, log) + self._merge_score_data( + citations_found, all_docs_found, log + ) # Use citations if found, otherwise use all_retrieved_documents if citations_found and not citations_data: From f1d84e8614653bb3b37fce165190574b0dc7843b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 3 Dec 2025 07:35:07 +0000 Subject: [PATCH 25/39] Add [docX] to markdown link conversion and enhanced score debugging Co-authored-by: owndev <69784886+owndev@users.noreply.github.com> --- pipelines/azure/azure_ai_foundry.py | 75 +++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/pipelines/azure/azure_ai_foundry.py b/pipelines/azure/azure_ai_foundry.py index ea97f35..274451c 100644 --- a/pipelines/azure/azure_ai_foundry.py +++ b/pipelines/azure/azure_ai_foundry.py @@ -219,6 +219,12 @@ class Valves(BaseModel): description="If True, automatically add 'include_contexts' with 'all_retrieved_documents' to Azure AI Search requests to get relevance scores (original_search_score and rerank_score). This enables relevance percentage display in citation cards.", ) + # Enable [docX] to markdown link conversion + AZURE_AI_LINK_CITATIONS: bool = Field( + default=True, + description="If True, convert [doc1], [doc2], etc. references in the response content to clickable markdown links pointing to the document URL.", + ) + def __init__(self): self.valves = self.Valves() self.name: str = f"{self.valves.AZURE_AI_PIPELINE_PREFIX}:" @@ -491,11 +497,19 @@ def _merge_score_data( "rerank_score": doc.get("rerank_score"), } + log.debug( + f"Processing all_retrieved_document: title='{doc.get('title')}', " + f"chunk_id='{doc.get('chunk_id')}', " + f"original_search_score={scores['original_search_score']}, " + f"rerank_score={scores['rerank_score']}" + ) + # Only store if we have at least one score if ( scores["original_search_score"] is None and scores["rerank_score"] is None ): + log.debug(f"Skipping doc with no scores: {doc.get('title')}") continue # Index by title @@ -834,6 +848,12 @@ def enhance_azure_search_response(self, response: Dict[str, Any]) -> Dict[str, A # Enhance the content with better citation display (if enabled) enhanced_content = content + # Convert [docX] references to markdown links (if enabled) + if self.valves.AZURE_AI_LINK_CITATIONS and citations: + enhanced_content = self._convert_doc_refs_to_links( + enhanced_content, citations + ) + # Add citation section at the end (if markdown/HTML citations are enabled) if self.valves.AZURE_AI_ENHANCE_CITATIONS and citation_details: citation_section = self._format_citation_section( @@ -1286,6 +1306,61 @@ def _extract_referenced_citations(self, content: str) -> Set[int]: # Convert to integers and return as a set return {int(match) for match in matches} + def _convert_doc_refs_to_links( + self, content: str, citations: List[Dict[str, Any]] + ) -> str: + """ + Convert [docX] references in the content to markdown links pointing to the document URL. + + This replaces plain [doc1], [doc2], etc. references with clickable markdown links + like [[doc1]](https://example.com/doc.pdf) when the cited document has a URL. + + Args: + content: The response content containing citation references + citations: List of citation objects with URL information + + Returns: + Content with [docX] references converted to markdown links + """ + if not content or not citations: + return content + + log = logging.getLogger("azure_ai._convert_doc_refs_to_links") + + # Build a mapping of doc index to URL + doc_urls = {} + for i, citation in enumerate(citations, 1): + if not isinstance(citation, dict): + continue + # Get URL from citation (prefer url, then filepath) + url = citation.get("url") or citation.get("filepath") or "" + if url and url.strip(): + doc_urls[i] = url.strip() + + if not doc_urls: + log.debug("No URLs found in citations, skipping link conversion") + return content + + # Replace [docX] references with markdown links + def replace_doc_ref(match): + doc_num = int(match.group(1)) + if doc_num in doc_urls: + # Convert [doc1] to [[doc1]](url) + return f"[[doc{doc_num}]]({doc_urls[doc_num]})" + return match.group(0) # Return unchanged if no URL + + pattern = r"\[doc(\d+)\]" + converted_content = re.sub(pattern, replace_doc_ref, content) + + # Log how many conversions were made + converted_count = len( + re.findall(r"\[\[doc\d+\]\]\(", converted_content) + ) + if converted_count > 0: + log.info(f"Converted {converted_count} [docX] references to markdown links") + + return converted_content + def _format_citation_section( self, citations: List[Dict[str, Any]], From da20a8b7a45be440314f29f755c840ea363e8639 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 3 Dec 2025 07:37:34 +0000 Subject: [PATCH 26/39] Refactor: Extract DOC_REF_PATTERN constant, optimize conversion counting Co-authored-by: owndev <69784886+owndev@users.noreply.github.com> --- pipelines/azure/azure_ai_foundry.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/pipelines/azure/azure_ai_foundry.py b/pipelines/azure/azure_ai_foundry.py index 274451c..a651672 100644 --- a/pipelines/azure/azure_ai_foundry.py +++ b/pipelines/azure/azure_ai_foundry.py @@ -145,6 +145,9 @@ async def cleanup_response( class Pipe: + # Regex pattern for matching [docX] citation references + DOC_REF_PATTERN = r"\[doc(\d+)\]" + # Environment variables for API key, endpoint, and optional model class Valves(BaseModel): # Custom prefix for pipeline display name @@ -1299,9 +1302,8 @@ def _extract_referenced_citations(self, content: str) -> Set[int]: Returns: Set of citation indices that are referenced (e.g., {1, 2, 7, 8, 9}) """ - # Find all [docN] references in the content - pattern = r"\[doc(\d+)\]" - matches = re.findall(pattern, content) + # Find all [docN] references in the content using class constant + matches = re.findall(self.DOC_REF_PATTERN, content) # Convert to integers and return as a set return {int(match) for match in matches} @@ -1341,23 +1343,24 @@ def _convert_doc_refs_to_links( log.debug("No URLs found in citations, skipping link conversion") return content + # Track conversion count during replacement + conversion_count = 0 + # Replace [docX] references with markdown links def replace_doc_ref(match): + nonlocal conversion_count doc_num = int(match.group(1)) if doc_num in doc_urls: + conversion_count += 1 # Convert [doc1] to [[doc1]](url) return f"[[doc{doc_num}]]({doc_urls[doc_num]})" return match.group(0) # Return unchanged if no URL - pattern = r"\[doc(\d+)\]" - converted_content = re.sub(pattern, replace_doc_ref, content) + converted_content = re.sub(self.DOC_REF_PATTERN, replace_doc_ref, content) # Log how many conversions were made - converted_count = len( - re.findall(r"\[\[doc\d+\]\]\(", converted_content) - ) - if converted_count > 0: - log.info(f"Converted {converted_count} [docX] references to markdown links") + if conversion_count > 0: + log.info(f"Converted {conversion_count} [docX] references to markdown links") return converted_content From 8acb84e0c5dee75f4bc65821be3f197a7f170aaf Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 3 Dec 2025 07:58:24 +0000 Subject: [PATCH 27/39] Remove AZURE_AI_LINK_CITATIONS valve and _convert_doc_refs_to_links method, fix score normalization Co-authored-by: owndev <69784886+owndev@users.noreply.github.com> --- pipelines/azure/azure_ai_foundry.py | 84 ++++------------------------- 1 file changed, 11 insertions(+), 73 deletions(-) diff --git a/pipelines/azure/azure_ai_foundry.py b/pipelines/azure/azure_ai_foundry.py index a651672..fc28b0d 100644 --- a/pipelines/azure/azure_ai_foundry.py +++ b/pipelines/azure/azure_ai_foundry.py @@ -222,12 +222,6 @@ class Valves(BaseModel): description="If True, automatically add 'include_contexts' with 'all_retrieved_documents' to Azure AI Search requests to get relevance scores (original_search_score and rerank_score). This enables relevance percentage display in citation cards.", ) - # Enable [docX] to markdown link conversion - AZURE_AI_LINK_CITATIONS: bool = Field( - default=True, - description="If True, convert [doc1], [doc2], etc. references in the response content to clickable markdown links pointing to the document URL.", - ) - def __init__(self): self.valves = self.Valves() self.name: str = f"{self.valves.AZURE_AI_PIPELINE_PREFIX}:" @@ -668,18 +662,24 @@ def _normalize_citation_for_openwebui( # Prefer rerank_score (semantic ranker), then original_search_score, then legacy score if rerank_score is not None: - # Reranker scores are typically 0-4 range, normalize to 0-1 for display - # Azure AI Search semantic reranker returns scores in 0-4 range - normalized_score = min(float(rerank_score) / 4.0, 1.0) + # Azure AI Search reranker scores are typically already 0-1 + # Use as-is if <= 1, normalize if > 1 (some models may return 0-4 range) + score_val = float(rerank_score) + if score_val > 1.0: + # Normalize scores > 1 (some semantic rerankers use 0-4 range) + normalized_score = min(score_val / 4.0, 1.0) + else: + normalized_score = score_val citation_data["distances"] = [normalized_score] log.debug( f"Using rerank_score {rerank_score} -> normalized {normalized_score}" ) elif original_search_score is not None: - # Original search scores can vary widely, use as-is if <= 1, otherwise normalize + # Original search scores can vary widely (BM25 scores can be > 1) + # Use as-is if <= 1, otherwise normalize score_val = float(original_search_score) if score_val > 1.0: - # Normalize high scores (BM25 scores can be > 1) + # Normalize high scores (BM25 scores can be much greater than 1) normalized_score = min(score_val / 100.0, 1.0) else: normalized_score = score_val @@ -851,12 +851,6 @@ def enhance_azure_search_response(self, response: Dict[str, Any]) -> Dict[str, A # Enhance the content with better citation display (if enabled) enhanced_content = content - # Convert [docX] references to markdown links (if enabled) - if self.valves.AZURE_AI_LINK_CITATIONS and citations: - enhanced_content = self._convert_doc_refs_to_links( - enhanced_content, citations - ) - # Add citation section at the end (if markdown/HTML citations are enabled) if self.valves.AZURE_AI_ENHANCE_CITATIONS and citation_details: citation_section = self._format_citation_section( @@ -1308,62 +1302,6 @@ def _extract_referenced_citations(self, content: str) -> Set[int]: # Convert to integers and return as a set return {int(match) for match in matches} - def _convert_doc_refs_to_links( - self, content: str, citations: List[Dict[str, Any]] - ) -> str: - """ - Convert [docX] references in the content to markdown links pointing to the document URL. - - This replaces plain [doc1], [doc2], etc. references with clickable markdown links - like [[doc1]](https://example.com/doc.pdf) when the cited document has a URL. - - Args: - content: The response content containing citation references - citations: List of citation objects with URL information - - Returns: - Content with [docX] references converted to markdown links - """ - if not content or not citations: - return content - - log = logging.getLogger("azure_ai._convert_doc_refs_to_links") - - # Build a mapping of doc index to URL - doc_urls = {} - for i, citation in enumerate(citations, 1): - if not isinstance(citation, dict): - continue - # Get URL from citation (prefer url, then filepath) - url = citation.get("url") or citation.get("filepath") or "" - if url and url.strip(): - doc_urls[i] = url.strip() - - if not doc_urls: - log.debug("No URLs found in citations, skipping link conversion") - return content - - # Track conversion count during replacement - conversion_count = 0 - - # Replace [docX] references with markdown links - def replace_doc_ref(match): - nonlocal conversion_count - doc_num = int(match.group(1)) - if doc_num in doc_urls: - conversion_count += 1 - # Convert [doc1] to [[doc1]](url) - return f"[[doc{doc_num}]]({doc_urls[doc_num]})" - return match.group(0) # Return unchanged if no URL - - converted_content = re.sub(self.DOC_REF_PATTERN, replace_doc_ref, content) - - # Log how many conversions were made - if conversion_count > 0: - log.info(f"Converted {conversion_count} [docX] references to markdown links") - - return converted_content - def _format_citation_section( self, citations: List[Dict[str, Any]], From 3ecdbc75b5373e4b8caa994178dd7ba246c09621 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 3 Dec 2025 08:00:54 +0000 Subject: [PATCH 28/39] Improve code comments for score normalization logic Co-authored-by: owndev <69784886+owndev@users.noreply.github.com> --- pipelines/azure/azure_ai_foundry.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pipelines/azure/azure_ai_foundry.py b/pipelines/azure/azure_ai_foundry.py index fc28b0d..b00427f 100644 --- a/pipelines/azure/azure_ai_foundry.py +++ b/pipelines/azure/azure_ai_foundry.py @@ -662,11 +662,11 @@ def _normalize_citation_for_openwebui( # Prefer rerank_score (semantic ranker), then original_search_score, then legacy score if rerank_score is not None: - # Azure AI Search reranker scores are typically already 0-1 - # Use as-is if <= 1, normalize if > 1 (some models may return 0-4 range) + # Azure AI Search semantic rerankers typically return scores in 0-1 range. + # However, some Cohere rerankers (via Azure AI) may use 0-4 range. + # We normalize to 0-1: use as-is if <= 1, divide by 4 if > 1. score_val = float(rerank_score) if score_val > 1.0: - # Normalize scores > 1 (some semantic rerankers use 0-4 range) normalized_score = min(score_val / 4.0, 1.0) else: normalized_score = score_val @@ -675,11 +675,11 @@ def _normalize_citation_for_openwebui( f"Using rerank_score {rerank_score} -> normalized {normalized_score}" ) elif original_search_score is not None: - # Original search scores can vary widely (BM25 scores can be > 1) - # Use as-is if <= 1, otherwise normalize + # BM25/keyword search scores vary based on term frequency and document collection. + # Typical BM25 scores in Azure AI Search range from ~0 to ~50 but can go higher. + # We normalize to 0-1: use as-is if <= 1, otherwise apply heuristic /100 cap. score_val = float(original_search_score) if score_val > 1.0: - # Normalize high scores (BM25 scores can be much greater than 1) normalized_score = min(score_val / 100.0, 1.0) else: normalized_score = score_val From 5b030d17b245e9fc5487bb574cd50ec59cad39ee Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 3 Dec 2025 08:19:13 +0000 Subject: [PATCH 29/39] Fix score selection to use filter_reason field per Azure documentation Co-authored-by: owndev <69784886+owndev@users.noreply.github.com> --- pipelines/azure/azure_ai_foundry.py | 139 ++++++++++++++++------------ 1 file changed, 81 insertions(+), 58 deletions(-) diff --git a/pipelines/azure/azure_ai_foundry.py b/pipelines/azure/azure_ai_foundry.py index b00427f..7ae7345 100644 --- a/pipelines/azure/azure_ai_foundry.py +++ b/pipelines/azure/azure_ai_foundry.py @@ -476,6 +476,11 @@ def _merge_score_data( additional documents with score fields. This method attempts to match them with citations and copy over the score data. + Copies: + - original_search_score: BM25/keyword search score + - rerank_score: Semantic reranker score (if enabled) + - filter_reason: Indicates which score is relevant ("score" or "rerank") + Args: citations: List of citation objects to update (modified in place) all_docs: List of all_retrieved_documents with score data @@ -483,39 +488,41 @@ def _merge_score_data( """ # Build multiple lookup maps to maximize matching chances # all_retrieved_documents may have different keys than citations - doc_scores_by_title = {} - doc_scores_by_filepath = {} - doc_scores_by_content = {} - doc_scores_by_chunk_id = {} + doc_data_by_title = {} + doc_data_by_filepath = {} + doc_data_by_content = {} + doc_data_by_chunk_id = {} for doc in all_docs: - scores = { + doc_data = { "original_search_score": doc.get("original_search_score"), "rerank_score": doc.get("rerank_score"), + "filter_reason": doc.get("filter_reason"), } log.debug( f"Processing all_retrieved_document: title='{doc.get('title')}', " f"chunk_id='{doc.get('chunk_id')}', " - f"original_search_score={scores['original_search_score']}, " - f"rerank_score={scores['rerank_score']}" + f"original_search_score={doc_data['original_search_score']}, " + f"rerank_score={doc_data['rerank_score']}, " + f"filter_reason={doc_data['filter_reason']}" ) # Only store if we have at least one score if ( - scores["original_search_score"] is None - and scores["rerank_score"] is None + doc_data["original_search_score"] is None + and doc_data["rerank_score"] is None ): log.debug(f"Skipping doc with no scores: {doc.get('title')}") continue # Index by title if doc.get("title"): - doc_scores_by_title[doc["title"]] = scores + doc_data_by_title[doc["title"]] = doc_data # Index by filepath if doc.get("filepath"): - doc_scores_by_filepath[doc["filepath"]] = scores + doc_data_by_filepath[doc["filepath"]] = doc_data # Index by chunk_id (may include title as prefix for uniqueness) if doc.get("chunk_id") is not None: @@ -523,8 +530,8 @@ def _merge_score_data( # Also try with title prefix for uniqueness if doc.get("title"): chunk_key = f"{doc['title']}_{doc['chunk_id']}" - doc_scores_by_chunk_id[str(doc["chunk_id"])] = scores - doc_scores_by_chunk_id[chunk_key] = scores + doc_data_by_chunk_id[str(doc["chunk_id"])] = doc_data + doc_data_by_chunk_id[chunk_key] = doc_data # Index by content prefix (first 100 chars) if doc.get("content"): @@ -533,63 +540,66 @@ def _merge_score_data( if len(doc.get("content", "")) > 100 else doc.get("content") ) - doc_scores_by_content[content_key] = scores + doc_data_by_content[content_key] = doc_data log.debug( - f"Built score lookup: by_title={len(doc_scores_by_title)}, " - f"by_filepath={len(doc_scores_by_filepath)}, " - f"by_chunk_id={len(doc_scores_by_chunk_id)}, " - f"by_content={len(doc_scores_by_content)}" + f"Built score lookup: by_title={len(doc_data_by_title)}, " + f"by_filepath={len(doc_data_by_filepath)}, " + f"by_chunk_id={len(doc_data_by_chunk_id)}, " + f"by_content={len(doc_data_by_content)}" ) # Match citations with score data using multiple strategies matched = 0 for citation in citations: - scores = None + doc_data = None # Try matching by title first (most reliable) - if not scores and citation.get("title"): - scores = doc_scores_by_title.get(citation["title"]) - if scores: + if not doc_data and citation.get("title"): + doc_data = doc_data_by_title.get(citation["title"]) + if doc_data: log.debug(f"Matched citation by title: {citation['title']}") # Try matching by filepath - if not scores and citation.get("filepath"): - scores = doc_scores_by_filepath.get(citation["filepath"]) - if scores: + if not doc_data and citation.get("filepath"): + doc_data = doc_data_by_filepath.get(citation["filepath"]) + if doc_data: log.debug(f"Matched citation by filepath: {citation['filepath']}") # Try matching by chunk_id with title prefix - if not scores and citation.get("chunk_id") is not None: + if not doc_data and citation.get("chunk_id") is not None: chunk_key = str(citation["chunk_id"]) if citation.get("title"): chunk_key_with_title = f"{citation['title']}_{citation['chunk_id']}" - scores = doc_scores_by_chunk_id.get(chunk_key_with_title) - if not scores: - scores = doc_scores_by_chunk_id.get(chunk_key) - if scores: + doc_data = doc_data_by_chunk_id.get(chunk_key_with_title) + if not doc_data: + doc_data = doc_data_by_chunk_id.get(chunk_key) + if doc_data: log.debug(f"Matched citation by chunk_id: {citation['chunk_id']}") # Try matching by content prefix - if not scores and citation.get("content"): + if not doc_data and citation.get("content"): content_key = ( citation["content"][:100] if len(citation.get("content", "")) > 100 else citation.get("content") ) - scores = doc_scores_by_content.get(content_key) - if scores: - log.debug(f"Matched citation by content prefix") - - if scores: - if scores.get("original_search_score") is not None: - citation["original_search_score"] = scores["original_search_score"] - if scores.get("rerank_score") is not None: - citation["rerank_score"] = scores["rerank_score"] + doc_data = doc_data_by_content.get(content_key) + if doc_data: + log.debug("Matched citation by content prefix") + + if doc_data: + if doc_data.get("original_search_score") is not None: + citation["original_search_score"] = doc_data["original_search_score"] + if doc_data.get("rerank_score") is not None: + citation["rerank_score"] = doc_data["rerank_score"] + if doc_data.get("filter_reason") is not None: + citation["filter_reason"] = doc_data["filter_reason"] matched += 1 log.debug( - f"Citation scores: original={scores.get('original_search_score')}, " - f"rerank={scores.get('rerank_score')}" + f"Citation scores: original={doc_data.get('original_search_score')}, " + f"rerank={doc_data.get('rerank_score')}, " + f"filter_reason={doc_data.get('filter_reason')}" ) log.info(f"Merged score data for {matched}/{len(citations)} citations") @@ -652,47 +662,59 @@ def _normalize_citation_for_openwebui( citation_data["source"]["url"] = source_url # Add distances array for relevance score (OpenWebUI uses this for percentage display) - # Priority: rerank_score > original_search_score > score - # rerank_score is the semantic reranker score (if enabled in Azure AI Search) - # original_search_score is the BM25/keyword search score - # score is a legacy field for backward compatibility + # Azure AI Search returns filter_reason to indicate which score type is relevant: + # - filter_reason not present or "score": use original_search_score (BM25/keyword) + # - filter_reason "rerank": use rerank_score (semantic reranker) + # Reference: https://learn.microsoft.com/en-us/azure/ai-foundry/openai/references/on-your-data + filter_reason = citation.get("filter_reason") rerank_score = citation.get("rerank_score") original_search_score = citation.get("original_search_score") legacy_score = citation.get("score") - # Prefer rerank_score (semantic ranker), then original_search_score, then legacy score - if rerank_score is not None: + normalized_score = 0.0 + + # Select score based on filter_reason as per Azure documentation + if filter_reason == "rerank" and rerank_score is not None: + # Document filtered by rerank score - use rerank_score # Azure AI Search semantic rerankers typically return scores in 0-1 range. # However, some Cohere rerankers (via Azure AI) may use 0-4 range. - # We normalize to 0-1: use as-is if <= 1, divide by 4 if > 1. score_val = float(rerank_score) if score_val > 1.0: normalized_score = min(score_val / 4.0, 1.0) else: normalized_score = score_val - citation_data["distances"] = [normalized_score] log.debug( - f"Using rerank_score {rerank_score} -> normalized {normalized_score}" + f"Using rerank_score (filter_reason=rerank): {rerank_score} -> {normalized_score}" ) elif original_search_score is not None: + # filter_reason not present, "score", or rerank_score unavailable - use original_search_score # BM25/keyword search scores vary based on term frequency and document collection. # Typical BM25 scores in Azure AI Search range from ~0 to ~50 but can go higher. - # We normalize to 0-1: use as-is if <= 1, otherwise apply heuristic /100 cap. score_val = float(original_search_score) if score_val > 1.0: normalized_score = min(score_val / 100.0, 1.0) else: normalized_score = score_val - citation_data["distances"] = [normalized_score] log.debug( - f"Using original_search_score {original_search_score} -> {normalized_score}" + f"Using original_search_score (filter_reason={filter_reason}): {original_search_score} -> {normalized_score}" + ) + elif rerank_score is not None: + # Fallback to rerank_score if available but filter_reason doesn't indicate it + score_val = float(rerank_score) + if score_val > 1.0: + normalized_score = min(score_val / 4.0, 1.0) + else: + normalized_score = score_val + log.debug( + f"Using rerank_score (fallback): {rerank_score} -> {normalized_score}" ) elif legacy_score is not None: - citation_data["distances"] = [float(legacy_score)] - log.debug(f"Using legacy score {legacy_score}") + normalized_score = float(legacy_score) + log.debug(f"Using legacy score: {legacy_score}") else: - # Default to 0 if no score to ensure the distances field is present - citation_data["distances"] = [0.0] + log.debug("No score available, using default 0.0") + + citation_data["distances"] = [normalized_score] # Build complete citation event structure citation_event = { @@ -706,6 +728,7 @@ def _normalize_citation_for_openwebui( f"Normalized citation {index}: title='{title}', " f"content_length={len(content)}, " f"url='{source_url}', " + f"filter_reason={filter_reason}, " f"rerank_score={rerank_score}, original_search_score={original_search_score}, " f"distances={citation_data['distances']}, " f"event={json.dumps(citation_event, default=str)[:500]}" From 0c39ca53dfbe0c55c8aaf17d21a16016de9e2e9e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 3 Dec 2025 08:21:55 +0000 Subject: [PATCH 30/39] Fix code review issues: remove unused variable, add explicit score handling Co-authored-by: owndev <69784886+owndev@users.noreply.github.com> --- pipelines/azure/azure_ai_foundry.py | 32 +++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/pipelines/azure/azure_ai_foundry.py b/pipelines/azure/azure_ai_foundry.py index 7ae7345..791cf1f 100644 --- a/pipelines/azure/azure_ai_foundry.py +++ b/pipelines/azure/azure_ai_foundry.py @@ -526,12 +526,12 @@ def _merge_score_data( # Index by chunk_id (may include title as prefix for uniqueness) if doc.get("chunk_id") is not None: - chunk_key = doc.get("chunk_id") - # Also try with title prefix for uniqueness - if doc.get("title"): - chunk_key = f"{doc['title']}_{doc['chunk_id']}" + # Store by plain chunk_id doc_data_by_chunk_id[str(doc["chunk_id"])] = doc_data - doc_data_by_chunk_id[chunk_key] = doc_data + # Also store by title-prefixed chunk_id for uniqueness + if doc.get("title"): + chunk_key_with_title = f"{doc['title']}_{doc['chunk_id']}" + doc_data_by_chunk_id[chunk_key_with_title] = doc_data # Index by content prefix (first 100 chars) if doc.get("content"): @@ -673,7 +673,9 @@ def _normalize_citation_for_openwebui( normalized_score = 0.0 - # Select score based on filter_reason as per Azure documentation + # Select score based on filter_reason as per Azure documentation: + # - filter_reason="rerank": Document filtered by rerank score threshold, use rerank_score + # - filter_reason="score" or not present: Document filtered by/passed original search score, use original_search_score if filter_reason == "rerank" and rerank_score is not None: # Document filtered by rerank score - use rerank_score # Azure AI Search semantic rerankers typically return scores in 0-1 range. @@ -686,8 +688,10 @@ def _normalize_citation_for_openwebui( log.debug( f"Using rerank_score (filter_reason=rerank): {rerank_score} -> {normalized_score}" ) - elif original_search_score is not None: - # filter_reason not present, "score", or rerank_score unavailable - use original_search_score + elif ( + filter_reason is None or filter_reason == "score" + ) and original_search_score is not None: + # filter_reason is "score" or not present - use original_search_score # BM25/keyword search scores vary based on term frequency and document collection. # Typical BM25 scores in Azure AI Search range from ~0 to ~50 but can go higher. score_val = float(original_search_score) @@ -698,8 +702,18 @@ def _normalize_citation_for_openwebui( log.debug( f"Using original_search_score (filter_reason={filter_reason}): {original_search_score} -> {normalized_score}" ) + elif original_search_score is not None: + # Fallback for unknown filter_reason values - use original_search_score + score_val = float(original_search_score) + if score_val > 1.0: + normalized_score = min(score_val / 100.0, 1.0) + else: + normalized_score = score_val + log.debug( + f"Using original_search_score (fallback, filter_reason={filter_reason}): {original_search_score} -> {normalized_score}" + ) elif rerank_score is not None: - # Fallback to rerank_score if available but filter_reason doesn't indicate it + # Fallback to rerank_score if available but filter_reason doesn't match score_val = float(rerank_score) if score_val > 1.0: normalized_score = min(score_val / 4.0, 1.0) From 3442a3f6741eb2015190edba9d1e219bb791b4c5 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 3 Dec 2025 08:45:40 +0000 Subject: [PATCH 31/39] Convert [docX] references to tags for OpenWebUI citation linking Co-authored-by: owndev <69784886+owndev@users.noreply.github.com> --- pipelines/azure/azure_ai_foundry.py | 134 +++++++++++++++++++++++++++- 1 file changed, 133 insertions(+), 1 deletion(-) diff --git a/pipelines/azure/azure_ai_foundry.py b/pipelines/azure/azure_ai_foundry.py index 791cf1f..aa258e9 100644 --- a/pipelines/azure/azure_ai_foundry.py +++ b/pipelines/azure/azure_ai_foundry.py @@ -750,6 +750,67 @@ def _normalize_citation_for_openwebui( return citation_event + def _convert_doc_refs_to_source_tags( + self, content: str, citations: List[Dict[str, Any]] + ) -> str: + """ + Convert [docX] references in content to OpenWebUI tags for citation linking. + + OpenWebUI uses text tags to create clickable + citation links in the response. This method converts Azure's [doc1], [doc2], etc. + references to this format. + + Reference: https://github.com/open-webui/open-webui/blob/main/backend/open_webui/utils/middleware.py#L1518 + + Args: + content: The response content containing [docX] references + citations: List of citation objects with title, url, etc. + + Returns: + Content with [docX] references converted to tags + """ + if not content or not citations: + return content + + log = logging.getLogger("azure_ai._convert_doc_refs_to_source_tags") + + # Build a mapping of citation index to source name + citation_names = {} + for i, citation in enumerate(citations, 1): + if isinstance(citation, dict): + # Get title with fallback chain + title = citation.get("title") or "" + filepath = citation.get("filepath") or "" + url = citation.get("url") or "" + + source_name = ( + title.strip() or filepath.strip() or url.strip() or f"Document {i}" + ) + citation_names[i] = source_name + + def replace_doc_ref(match): + """Replace [docX] with [docX]""" + doc_num = int(match.group(1)) + source_name = citation_names.get(doc_num, f"Document {doc_num}") + # Escape special characters in source_name for HTML attribute + escaped_name = ( + source_name.replace("&", "&") + .replace('"', """) + .replace("<", "<") + .replace(">", ">") + ) + return f'[doc{doc_num}]' + + # Replace all [docX] references + converted = re.sub(self.DOC_REF_PATTERN, replace_doc_ref, content) + + # Count conversions for logging + original_count = len(re.findall(self.DOC_REF_PATTERN, content)) + if original_count > 0: + log.info(f"Converted {original_count} [docX] references to tags") + + return converted + async def _emit_openwebui_citation_events( self, citations: List[Dict[str, Any]], @@ -888,6 +949,13 @@ def enhance_azure_search_response(self, response: Dict[str, Any]) -> Dict[str, A # Enhance the content with better citation display (if enabled) enhanced_content = content + # Convert [docX] references to tags for OpenWebUI citation linking + # Reference: https://github.com/open-webui/open-webui/blob/main/backend/open_webui/utils/middleware.py#L1518 + if self.valves.AZURE_AI_OPENWEBUI_CITATIONS: + enhanced_content = self._convert_doc_refs_to_source_tags( + enhanced_content, citations + ) + # Add citation section at the end (if markdown/HTML citations are enabled) if self.valves.AZURE_AI_ENHANCE_CITATIONS and citation_details: citation_section = self._format_citation_section( @@ -1228,7 +1296,71 @@ async def stream_processor_with_citations( except Exception as parse_error: log.debug(f"Error parsing citations from chunk: {parse_error}") - # Always yield the original chunk first + # Convert [docX] references to tags in the chunk content + # This enables OpenWebUI to link citations in streaming responses + # Reference: https://github.com/open-webui/open-webui/blob/main/backend/open_webui/utils/middleware.py#L1518 + if self.valves.AZURE_AI_OPENWEBUI_CITATIONS and "[doc" in chunk_str: + try: + # Parse and modify each SSE data line + modified_lines = [] + chunk_lines = chunk_str.split("\n") + + for line in chunk_lines: + if line.startswith("data: ") and line.strip() != "data: [DONE]": + json_str = line[6:].strip() + if json_str and json_str != "[DONE]": + try: + data = json.loads(json_str) + if ( + isinstance(data, dict) + and "choices" in data + and data["choices"] + ): + modified = False + for choice in data["choices"]: + if ( + "delta" in choice + and "content" in choice["delta"] + ): + content_val = choice["delta"]["content"] + if "[doc" in content_val: + # Convert [docX] to tag inline + # Format: [docX] + def replace_ref(m): + doc_num = m.group(1) + return f'[doc{doc_num}]' + + choice["delta"]["content"] = re.sub( + self.DOC_REF_PATTERN, + replace_ref, + content_val, + ) + modified = True + + if modified: + modified_lines.append(f"data: {json.dumps(data)}") + else: + modified_lines.append(line) + else: + modified_lines.append(line) + except json.JSONDecodeError: + modified_lines.append(line) + else: + modified_lines.append(line) + else: + modified_lines.append(line) + + # Reconstruct the chunk with modified content + modified_chunk_str = "\n".join(modified_lines) + if modified_chunk_str != chunk_str: + log.debug("Converted [docX] references to tags in streaming chunk") + chunk = modified_chunk_str.encode("utf-8") + + except Exception as convert_err: + log.debug(f"Error converting [docX] to source tags: {convert_err}") + # Fall through to yield original chunk + + # Yield the (possibly modified) chunk yield chunk # Check if this is the end of the stream From 1be8cc7f8fae45b63274debfe246802edf9b1441 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 3 Dec 2025 08:48:40 +0000 Subject: [PATCH 32/39] Refactor: Extract _format_source_tag helper, optimize streaming conversion Co-authored-by: owndev <69784886+owndev@users.noreply.github.com> --- pipelines/azure/azure_ai_foundry.py | 85 ++++++++++++++++++++++------- 1 file changed, 65 insertions(+), 20 deletions(-) diff --git a/pipelines/azure/azure_ai_foundry.py b/pipelines/azure/azure_ai_foundry.py index aa258e9..9431261 100644 --- a/pipelines/azure/azure_ai_foundry.py +++ b/pipelines/azure/azure_ai_foundry.py @@ -750,6 +750,33 @@ def _normalize_citation_for_openwebui( return citation_event + def _format_source_tag( + self, doc_num: int, source_name: Optional[str] = None + ) -> str: + """ + Format a tag for a [docX] reference. + + Creates an OpenWebUI-compatible source tag that enables citation linking. + + Args: + doc_num: The document number (1-based) + source_name: Optional source name/title for the document + + Returns: + Formatted tag string + """ + if source_name: + # Escape special characters in source_name for HTML attribute + escaped_name = ( + source_name.replace("&", "&") + .replace('"', """) + .replace("<", "<") + .replace(">", ">") + ) + return f'[doc{doc_num}]' + else: + return f'[doc{doc_num}]' + def _convert_doc_refs_to_source_tags( self, content: str, citations: List[Dict[str, Any]] ) -> str: @@ -791,15 +818,8 @@ def _convert_doc_refs_to_source_tags( def replace_doc_ref(match): """Replace [docX] with [docX]""" doc_num = int(match.group(1)) - source_name = citation_names.get(doc_num, f"Document {doc_num}") - # Escape special characters in source_name for HTML attribute - escaped_name = ( - source_name.replace("&", "&") - .replace('"', """) - .replace("<", "<") - .replace(">", ">") - ) - return f'[doc{doc_num}]' + source_name = citation_names.get(doc_num) + return self._format_source_tag(doc_num, source_name) # Replace all [docX] references converted = re.sub(self.DOC_REF_PATTERN, replace_doc_ref, content) @@ -1299,8 +1319,24 @@ async def stream_processor_with_citations( # Convert [docX] references to tags in the chunk content # This enables OpenWebUI to link citations in streaming responses # Reference: https://github.com/open-webui/open-webui/blob/main/backend/open_webui/utils/middleware.py#L1518 + chunk_modified = False if self.valves.AZURE_AI_OPENWEBUI_CITATIONS and "[doc" in chunk_str: try: + # Build citation names map if we have citations data + citation_names = {} + if citations_data: + for i, cit in enumerate(citations_data, 1): + if isinstance(cit, dict): + title = cit.get("title") or "" + filepath = cit.get("filepath") or "" + url = cit.get("url") or "" + citation_names[i] = ( + title.strip() + or filepath.strip() + or url.strip() + or None + ) + # Parse and modify each SSE data line modified_lines = [] chunk_lines = chunk_str.split("\n") @@ -1316,7 +1352,7 @@ async def stream_processor_with_citations( and "choices" in data and data["choices"] ): - modified = False + line_modified = False for choice in data["choices"]: if ( "delta" in choice @@ -1324,21 +1360,28 @@ async def stream_processor_with_citations( ): content_val = choice["delta"]["content"] if "[doc" in content_val: - # Convert [docX] to tag inline - # Format: [docX] + # Convert [docX] to tag + # Use _format_source_tag with name if available + def replace_ref(m): - doc_num = m.group(1) - return f'[doc{doc_num}]' + doc_num = int(m.group(1)) + source_name = citation_names.get( + doc_num + ) + return self._format_source_tag( + doc_num, source_name + ) choice["delta"]["content"] = re.sub( self.DOC_REF_PATTERN, replace_ref, content_val, ) - modified = True + line_modified = True - if modified: + if line_modified: modified_lines.append(f"data: {json.dumps(data)}") + chunk_modified = True else: modified_lines.append(line) else: @@ -1350,10 +1393,12 @@ def replace_ref(m): else: modified_lines.append(line) - # Reconstruct the chunk with modified content - modified_chunk_str = "\n".join(modified_lines) - if modified_chunk_str != chunk_str: - log.debug("Converted [docX] references to tags in streaming chunk") + # Reconstruct the chunk only if something was modified + if chunk_modified: + modified_chunk_str = "\n".join(modified_lines) + log.debug( + "Converted [docX] references to tags in streaming chunk" + ) chunk = modified_chunk_str.encode("utf-8") except Exception as convert_err: From 5f106faaf154c28d979a35e6c79368e91fa4cbb3 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 3 Dec 2025 08:52:02 +0000 Subject: [PATCH 33/39] Refactor: Extract _build_citation_names_map, move replace_ref outside loops Co-authored-by: owndev <69784886+owndev@users.noreply.github.com> --- pipelines/azure/azure_ai_foundry.py | 74 ++++++++++++++--------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/pipelines/azure/azure_ai_foundry.py b/pipelines/azure/azure_ai_foundry.py index 9431261..312584d 100644 --- a/pipelines/azure/azure_ai_foundry.py +++ b/pipelines/azure/azure_ai_foundry.py @@ -750,6 +750,34 @@ def _normalize_citation_for_openwebui( return citation_event + def _build_citation_names_map( + self, citations: Optional[List[Dict[str, Any]]] + ) -> Dict[int, Optional[str]]: + """ + Build a mapping of citation indices to source names. + + Args: + citations: List of citation objects with title, filepath, url, etc. + + Returns: + Dict mapping 1-based citation index to source name (or None if no name available) + """ + citation_names: Dict[int, Optional[str]] = {} + if not citations: + return citation_names + + for i, citation in enumerate(citations, 1): + if isinstance(citation, dict): + # Get title with fallback chain + title = citation.get("title") or "" + filepath = citation.get("filepath") or "" + url = citation.get("url") or "" + + source_name = title.strip() or filepath.strip() or url.strip() or None + citation_names[i] = source_name + + return citation_names + def _format_source_tag( self, doc_num: int, source_name: Optional[str] = None ) -> str: @@ -802,18 +830,7 @@ def _convert_doc_refs_to_source_tags( log = logging.getLogger("azure_ai._convert_doc_refs_to_source_tags") # Build a mapping of citation index to source name - citation_names = {} - for i, citation in enumerate(citations, 1): - if isinstance(citation, dict): - # Get title with fallback chain - title = citation.get("title") or "" - filepath = citation.get("filepath") or "" - url = citation.get("url") or "" - - source_name = ( - title.strip() or filepath.strip() or url.strip() or f"Document {i}" - ) - citation_names[i] = source_name + citation_names = self._build_citation_names_map(citations) def replace_doc_ref(match): """Replace [docX] with [docX]""" @@ -1322,20 +1339,14 @@ async def stream_processor_with_citations( chunk_modified = False if self.valves.AZURE_AI_OPENWEBUI_CITATIONS and "[doc" in chunk_str: try: - # Build citation names map if we have citations data - citation_names = {} - if citations_data: - for i, cit in enumerate(citations_data, 1): - if isinstance(cit, dict): - title = cit.get("title") or "" - filepath = cit.get("filepath") or "" - url = cit.get("url") or "" - citation_names[i] = ( - title.strip() - or filepath.strip() - or url.strip() - or None - ) + # Build citation names map using shared helper + citation_names = self._build_citation_names_map(citations_data) + + # Define replacement function once, outside inner loops + def replace_ref(m): + doc_num = int(m.group(1)) + source_name = citation_names.get(doc_num) + return self._format_source_tag(doc_num, source_name) # Parse and modify each SSE data line modified_lines = [] @@ -1361,17 +1372,6 @@ async def stream_processor_with_citations( content_val = choice["delta"]["content"] if "[doc" in content_val: # Convert [docX] to tag - # Use _format_source_tag with name if available - - def replace_ref(m): - doc_num = int(m.group(1)) - source_name = citation_names.get( - doc_num - ) - return self._format_source_tag( - doc_num, source_name - ) - choice["delta"]["content"] = re.sub( self.DOC_REF_PATTERN, replace_ref, From d4c792ae2e5e9234c2b6d4fff4d3e87d5473dae4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 3 Dec 2025 08:54:52 +0000 Subject: [PATCH 34/39] Use html.escape for proper XSS protection in source tag names Co-authored-by: owndev <69784886+owndev@users.noreply.github.com> --- pipelines/azure/azure_ai_foundry.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/pipelines/azure/azure_ai_foundry.py b/pipelines/azure/azure_ai_foundry.py index 312584d..bea5592 100644 --- a/pipelines/azure/azure_ai_foundry.py +++ b/pipelines/azure/azure_ai_foundry.py @@ -42,6 +42,7 @@ import logging import base64 import hashlib +import html import re from pydantic_core import core_schema @@ -794,13 +795,8 @@ def _format_source_tag( Formatted tag string """ if source_name: - # Escape special characters in source_name for HTML attribute - escaped_name = ( - source_name.replace("&", "&") - .replace('"', """) - .replace("<", "<") - .replace(">", ">") - ) + # Use html.escape for proper HTML entity encoding (handles all special chars) + escaped_name = html.escape(source_name, quote=True) return f'[doc{doc_num}]' else: return f'[doc{doc_num}]' From 2a9d802d153908def6a853f8b47903776b9f5cb1 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 3 Dec 2025 09:16:33 +0000 Subject: [PATCH 35/39] Convert [docX] to markdown links with document URLs instead of tags Co-authored-by: owndev <69784886+owndev@users.noreply.github.com> --- pipelines/azure/azure_ai_foundry.py | 97 ++++++++++++++--------------- 1 file changed, 46 insertions(+), 51 deletions(-) diff --git a/pipelines/azure/azure_ai_foundry.py b/pipelines/azure/azure_ai_foundry.py index bea5592..432760a 100644 --- a/pipelines/azure/azure_ai_foundry.py +++ b/pipelines/azure/azure_ai_foundry.py @@ -42,7 +42,6 @@ import logging import base64 import hashlib -import html import re from pydantic_core import core_schema @@ -751,96 +750,94 @@ def _normalize_citation_for_openwebui( return citation_event - def _build_citation_names_map( + def _build_citation_urls_map( self, citations: Optional[List[Dict[str, Any]]] ) -> Dict[int, Optional[str]]: """ - Build a mapping of citation indices to source names. + Build a mapping of citation indices to document URLs. Args: citations: List of citation objects with title, filepath, url, etc. Returns: - Dict mapping 1-based citation index to source name (or None if no name available) + Dict mapping 1-based citation index to URL (or None if no URL available) """ - citation_names: Dict[int, Optional[str]] = {} + citation_urls: Dict[int, Optional[str]] = {} if not citations: - return citation_names + return citation_urls for i, citation in enumerate(citations, 1): if isinstance(citation, dict): - # Get title with fallback chain - title = citation.get("title") or "" - filepath = citation.get("filepath") or "" + # Get URL with fallback to filepath url = citation.get("url") or "" + filepath = citation.get("filepath") or "" - source_name = title.strip() or filepath.strip() or url.strip() or None - citation_names[i] = source_name + citation_url = url.strip() or filepath.strip() or None + citation_urls[i] = citation_url - return citation_names + return citation_urls - def _format_source_tag( - self, doc_num: int, source_name: Optional[str] = None + def _format_citation_link( + self, doc_num: int, url: Optional[str] = None ) -> str: """ - Format a tag for a [docX] reference. + Format a markdown link for a [docX] reference. - Creates an OpenWebUI-compatible source tag that enables citation linking. + If a URL is available, creates a clickable markdown link. + Otherwise, returns the original [docX] reference. Args: doc_num: The document number (1-based) - source_name: Optional source name/title for the document + url: Optional URL for the document Returns: - Formatted tag string + Formatted markdown link string or original [docX] reference """ - if source_name: - # Use html.escape for proper HTML entity encoding (handles all special chars) - escaped_name = html.escape(source_name, quote=True) - return f'[doc{doc_num}]' + if url: + # Create markdown link: [[doc1]](url) + return f"[[doc{doc_num}]]({url})" else: - return f'[doc{doc_num}]' + # No URL available, keep original reference + return f"[doc{doc_num}]" - def _convert_doc_refs_to_source_tags( + def _convert_doc_refs_to_links( self, content: str, citations: List[Dict[str, Any]] ) -> str: """ - Convert [docX] references in content to OpenWebUI tags for citation linking. - - OpenWebUI uses text tags to create clickable - citation links in the response. This method converts Azure's [doc1], [doc2], etc. - references to this format. + Convert [docX] references in content to markdown links with document URLs. - Reference: https://github.com/open-webui/open-webui/blob/main/backend/open_webui/utils/middleware.py#L1518 + If a citation has a URL, [doc1] becomes [[doc1]](url). This creates clickable + links to the source documents in the response. Args: content: The response content containing [docX] references citations: List of citation objects with title, url, etc. Returns: - Content with [docX] references converted to tags + Content with [docX] references converted to markdown links """ if not content or not citations: return content - log = logging.getLogger("azure_ai._convert_doc_refs_to_source_tags") + log = logging.getLogger("azure_ai._convert_doc_refs_to_links") - # Build a mapping of citation index to source name - citation_names = self._build_citation_names_map(citations) + # Build a mapping of citation index to URL + citation_urls = self._build_citation_urls_map(citations) def replace_doc_ref(match): - """Replace [docX] with [docX]""" + """Replace [docX] with [[docX]](url) if URL available""" doc_num = int(match.group(1)) - source_name = citation_names.get(doc_num) - return self._format_source_tag(doc_num, source_name) + url = citation_urls.get(doc_num) + return self._format_citation_link(doc_num, url) # Replace all [docX] references converted = re.sub(self.DOC_REF_PATTERN, replace_doc_ref, content) # Count conversions for logging original_count = len(re.findall(self.DOC_REF_PATTERN, content)) + linked_count = sum(1 for i in range(1, len(citations) + 1) if citation_urls.get(i)) if original_count > 0: - log.info(f"Converted {original_count} [docX] references to tags") + log.info(f"Converted {original_count} [docX] references to markdown links ({linked_count} with URLs)") return converted @@ -982,10 +979,9 @@ def enhance_azure_search_response(self, response: Dict[str, Any]) -> Dict[str, A # Enhance the content with better citation display (if enabled) enhanced_content = content - # Convert [docX] references to tags for OpenWebUI citation linking - # Reference: https://github.com/open-webui/open-webui/blob/main/backend/open_webui/utils/middleware.py#L1518 + # Convert [docX] references to markdown links for citation linking if self.valves.AZURE_AI_OPENWEBUI_CITATIONS: - enhanced_content = self._convert_doc_refs_to_source_tags( + enhanced_content = self._convert_doc_refs_to_links( enhanced_content, citations ) @@ -1329,20 +1325,19 @@ async def stream_processor_with_citations( except Exception as parse_error: log.debug(f"Error parsing citations from chunk: {parse_error}") - # Convert [docX] references to tags in the chunk content - # This enables OpenWebUI to link citations in streaming responses - # Reference: https://github.com/open-webui/open-webui/blob/main/backend/open_webui/utils/middleware.py#L1518 + # Convert [docX] references to markdown links in the chunk content + # This creates clickable links to source documents in streaming responses chunk_modified = False if self.valves.AZURE_AI_OPENWEBUI_CITATIONS and "[doc" in chunk_str: try: - # Build citation names map using shared helper - citation_names = self._build_citation_names_map(citations_data) + # Build citation URLs map using shared helper + citation_urls = self._build_citation_urls_map(citations_data) # Define replacement function once, outside inner loops def replace_ref(m): doc_num = int(m.group(1)) - source_name = citation_names.get(doc_num) - return self._format_source_tag(doc_num, source_name) + url = citation_urls.get(doc_num) + return self._format_citation_link(doc_num, url) # Parse and modify each SSE data line modified_lines = [] @@ -1367,7 +1362,7 @@ def replace_ref(m): ): content_val = choice["delta"]["content"] if "[doc" in content_val: - # Convert [docX] to tag + # Convert [docX] to markdown link choice["delta"]["content"] = re.sub( self.DOC_REF_PATTERN, replace_ref, @@ -1393,12 +1388,12 @@ def replace_ref(m): if chunk_modified: modified_chunk_str = "\n".join(modified_lines) log.debug( - "Converted [docX] references to tags in streaming chunk" + "Converted [docX] references to markdown links in streaming chunk" ) chunk = modified_chunk_str.encode("utf-8") except Exception as convert_err: - log.debug(f"Error converting [docX] to source tags: {convert_err}") + log.debug(f"Error converting [docX] to markdown links: {convert_err}") # Fall through to yield original chunk # Yield the (possibly modified) chunk From 7e88c19028a726e8014dbbffa2e4ab08eadc4656 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 3 Dec 2025 09:43:39 +0000 Subject: [PATCH 36/39] Remove AZURE_AI_ENHANCE_CITATIONS and AZURE_AI_OPENWEBUI_CITATIONS valves, simplify code Co-authored-by: owndev <69784886+owndev@users.noreply.github.com> --- pipelines/azure/azure_ai_foundry.py | 252 ++-------------------------- 1 file changed, 17 insertions(+), 235 deletions(-) diff --git a/pipelines/azure/azure_ai_foundry.py b/pipelines/azure/azure_ai_foundry.py index 432760a..9c24379 100644 --- a/pipelines/azure/azure_ai_foundry.py +++ b/pipelines/azure/azure_ai_foundry.py @@ -4,7 +4,7 @@ author_url: https://github.com/owndev/ project_url: https://github.com/owndev/Open-WebUI-Functions funding_url: https://github.com/sponsors/owndev -version: 2.6.0 +version: 2.7.0 license: Apache License 2.0 description: A pipeline for interacting with Azure AI services, enabling seamless communication with various AI models via configurable headers and robust error handling. This includes support for Azure OpenAI models as well as other Azure AI models by dynamically managing headers and request configurations. Azure AI Search (RAG) integration is only supported with Azure OpenAI endpoints. features: @@ -15,8 +15,9 @@ - Compatible with Azure OpenAI and other Azure AI models. - Predefined models for easy access. - Encrypted storage of sensitive API keys - - Azure AI Search / RAG integration with enhanced citation display (Azure OpenAI only) - - Native OpenWebUI citations support with structured events and citation cards (Azure OpenAI only) + - Azure AI Search / RAG integration with native OpenWebUI citations (Azure OpenAI only) + - Automatic [docX] to markdown link conversion for clickable citations + - Relevance scores from Azure AI Search displayed in citation cards """ from typing import ( @@ -204,18 +205,6 @@ class Valves(BaseModel): description='JSON configuration for data_sources field (for Azure AI Search / RAG). Example: \'[{"type":"azure_search","parameters":{"endpoint":"https://xxx.search.windows.net","index_name":"your-index","authentication":{"type":"api_key","key":"your-key"}}}]\'', ) - # Enable enhanced citation display for Azure AI Search responses - AZURE_AI_ENHANCE_CITATIONS: bool = Field( - default=False, - description="If True, enhance Azure AI Search responses with better citation formatting and source content display (markdown/HTML).", - ) - - # Enable native OpenWebUI citations (structured events and fields) - AZURE_AI_OPENWEBUI_CITATIONS: bool = Field( - default=True, - description="If True, emit native OpenWebUI citation events for streaming responses and attach openwebui_citations field for non-streaming responses. Enables citation cards and UI in OpenWebUI frontend.", - ) - # Enable relevance scores from Azure AI Search AZURE_AI_INCLUDE_SEARCH_SCORES: bool = Field( default=True, @@ -929,17 +918,14 @@ async def _emit_openwebui_citation_events( def enhance_azure_search_response(self, response: Dict[str, Any]) -> Dict[str, Any]: """ - Enhance Azure AI Search responses by improving citation display and adding source content. + Enhance Azure AI Search responses by converting [docX] references to markdown links. Modifies the response in-place and returns it. - If AZURE_AI_ENHANCE_CITATIONS is True, appends a formatted markdown/HTML citation section - to the response content. - Args: response: The original response from Azure AI (modified in-place) Returns: - The enhanced response with better citation formatting + The enhanced response with markdown links for citations """ if not isinstance(response, dict): return response @@ -961,43 +947,12 @@ def enhance_azure_search_response(self, response: Dict[str, Any]) -> Dict[str, A citations = context["citations"] content = message["content"] - # Create citation mappings - citation_details = {} - for i, citation in enumerate(citations, 1): - if not isinstance(citation, dict): - continue - - doc_ref = f"[doc{i}]" - citation_details[doc_ref] = { - "title": citation.get("title", "Unknown Document"), - "content": citation.get("content", ""), - "url": citation.get("url"), - "filepath": citation.get("filepath"), - "chunk_id": citation.get("chunk_id", "0"), - } - - # Enhance the content with better citation display (if enabled) - enhanced_content = content - - # Convert [docX] references to markdown links for citation linking - if self.valves.AZURE_AI_OPENWEBUI_CITATIONS: - enhanced_content = self._convert_doc_refs_to_links( - enhanced_content, citations - ) - - # Add citation section at the end (if markdown/HTML citations are enabled) - if self.valves.AZURE_AI_ENHANCE_CITATIONS and citation_details: - citation_section = self._format_citation_section( - citations, content, for_streaming=False - ) - enhanced_content += citation_section + # Convert [docX] references to markdown links + enhanced_content = self._convert_doc_refs_to_links(content, citations) # Update the message content message["content"] = enhanced_content - # Add enhanced citation info to context for API consumers - context["enhanced_citations"] = citation_details - return response except Exception as e: @@ -1189,8 +1144,6 @@ async def stream_processor_with_citations( full_response_buffer = "" response_content = "" # Track the actual response content citations_data = None - citations_added = False - all_chunks = [] async for chunk in content: chunk_str = chunk.decode("utf-8", errors="ignore") @@ -1328,7 +1281,7 @@ async def stream_processor_with_citations( # Convert [docX] references to markdown links in the chunk content # This creates clickable links to source documents in streaming responses chunk_modified = False - if self.valves.AZURE_AI_OPENWEBUI_CITATIONS and "[doc" in chunk_str: + if "[doc" in chunk_str: try: # Build citation URLs map using shared helper citation_urls = self._build_citation_urls_map(citations_data) @@ -1404,59 +1357,14 @@ def replace_ref(m): log.debug("End of stream detected") break - # After the stream ends, emit OpenWebUI citation events if enabled - if ( - citations_data - and self.valves.AZURE_AI_OPENWEBUI_CITATIONS - and __event_emitter__ - ): + # After the stream ends, emit OpenWebUI citation events + if citations_data and __event_emitter__: log.info("Emitting OpenWebUI citation events at end of stream...") # Filter to only citations referenced in the response content await self._emit_openwebui_citation_events( citations_data, __event_emitter__, response_content ) - # After the stream ends, add markdown/HTML citations if we found any and it's enabled - if ( - citations_data - and not citations_added - and self.valves.AZURE_AI_ENHANCE_CITATIONS - ): - log.info("Adding citation summary at end of stream...") - - # Pass the accumulated response content to filter citations - citation_section = self._format_citation_section( - citations_data, response_content, for_streaming=True - ) - if citation_section: - # Convert escaped newlines to actual newlines for display - display_section = citation_section.replace("\\n", "\n") - - # Send the citation section in smaller, safer chunks - # Split by lines and send each as a separate SSE event - lines = display_section.split("\n") - - for line in lines: - # Escape quotes and backslashes for JSON - safe_line = line.replace("\\", "\\\\").replace('"', '\\"') - # Create a simple SSE event - sse_event = f'data: {{"choices":[{{"delta":{{"content":"{safe_line}\\n"}}}}]}}\n\n' - yield sse_event.encode("utf-8") - - citations_added = True - log.info("Citation summary successfully added to stream") - - # If we didn't find citations in the stream but detected citation references, - # try one more time with the full buffer - elif not citations_data and "[doc" in full_response_buffer: - log.warning( - "Found [doc] references but no citation data - attempting final parse..." - ) - # This is a fallback for cases where citation detection failed - fallback_message = "\\n\\n
\\n⚠️ Citations Processing Issue\\n\\nThe response contains citation references [doc1], [doc2], etc., but the citation details could not be extracted from the streaming response.\\n\\n
\\n" - fallback_sse = f'data: {{"choices":[{{"delta":{{"content":"{fallback_message}"}}}}]}}\n\n' - yield fallback_sse.encode("utf-8") - # Send completion status update when streaming is done if __event_emitter__: await __event_emitter__( @@ -1507,124 +1415,6 @@ def _extract_referenced_citations(self, content: str) -> Set[int]: # Convert to integers and return as a set return {int(match) for match in matches} - def _format_citation_section( - self, - citations: List[Dict[str, Any]], - content: str = "", - for_streaming: bool = False, - ) -> str: - """ - Creates a formatted citation section using collapsible details elements. - Only includes citations that are actually referenced in the content. - - Args: - citations: List of citation objects - content: The response content (used to filter only referenced citations) - for_streaming: If True, format for streaming (with escaping), else for regular response - - Returns: - Formatted citation section with HTML details elements - """ - if not citations: - return "" - - # Extract which citations are actually referenced in the content - referenced_indices = self._extract_referenced_citations(content) - - # If we couldn't find any references, include all citations (backward compatibility) - if not referenced_indices: - referenced_indices = set(range(1, len(citations) + 1)) - - # Collect only referenced citation details - citation_entries = [] - - for i, citation in enumerate(citations, 1): - # Skip citations that are not referenced in the content - if i not in referenced_indices: - continue - - if not isinstance(citation, dict): - continue - - doc_ref = f"[doc{i}]" - - # Get title with fallback to filepath or url - title = citation.get("title", "") - # Check if title is empty (not just None) and use alternatives - if not title or not title.strip(): - # Try filepath first - filepath = citation.get("filepath", "") - if filepath and filepath.strip(): - title = filepath - else: - # Try url next - url = citation.get("url", "") - if url and url.strip(): - title = url - else: - # Final fallback - title = "Unknown Document" - - content_text = citation.get("content", "") - filepath = citation.get("filepath", "") - url = citation.get("url", "") - chunk_id = citation.get("chunk_id", "") - - # Build individual citation details - citation_info = [] - - # Show filepath if available and not empty - if filepath and filepath.strip(): - citation_info.append(f"📁 **File:** `{filepath}`") - # Show URL if available, not empty, and no filepath was shown - elif url and url.strip(): - citation_info.append(f"🔗 **URL:** {url}") - - # Show chunk_id if available and not empty - if chunk_id is not None and str(chunk_id).strip(): - citation_info.append(f"📄 **Chunk ID:** {chunk_id}") - - # Add full content if available - if content_text and str(content_text).strip(): - try: - # Clean content for display - clean_content = str(content_text).strip() - if for_streaming: - # Additional escaping for streaming - clean_content = clean_content.replace("\\", "\\\\").replace( - '"', '\\"' - ) - - citation_info.append("**Content:**") - citation_info.append(f"> {clean_content}") - except Exception: - citation_info.append("**Content:** [Content unavailable]") - - # Create collapsible details for individual citation - if for_streaming: - # For streaming, we need to escape newlines - citation_content = "\\n".join(citation_info) - citation_entry = f"
\\n{doc_ref} - {title}\\n\\n{citation_content}\\n\\n
" - else: - citation_content = "\n".join(citation_info) - citation_entry = f"
\n{doc_ref} - {title}\n\n{citation_content}\n\n
" - - citation_entries.append(citation_entry) - - # Only create the section if we have citations to show - if not citation_entries: - return "" - - # Combine all citations into main collapsible section - if for_streaming: - all_citations = "\\n\\n".join(citation_entries) - result = f"\\n\\n
\\n📚 Sources and References\\n\\n{all_citations}\\n\\n
\\n" - else: - all_citations = "\n\n".join(citation_entries) - result = f"\n\n
\n📚 Sources and References\n\n{all_citations}\n\n
\n" - - return result - async def stream_processor( self, content: aiohttp.StreamReader, @@ -1836,11 +1626,8 @@ async def pipe( sse_headers["Content-Type"] = "text/event-stream" sse_headers.pop("Content-Length", None) - # Use enhanced stream processor if Azure AI Search is configured and citations are enabled - if self.valves.AZURE_AI_DATA_SOURCES and ( - self.valves.AZURE_AI_ENHANCE_CITATIONS - or self.valves.AZURE_AI_OPENWEBUI_CITATIONS - ): + # Use enhanced stream processor if Azure AI Search is configured + if self.valves.AZURE_AI_DATA_SOURCES: stream_processor = self.stream_processor_with_citations else: stream_processor = self.stream_processor @@ -1870,17 +1657,12 @@ async def pipe( request.raise_for_status() - # Enhance Azure Search responses with better citation display - # Call this when either citation mode is enabled + # Enhance Azure Search responses with citation linking and emit citation events if isinstance(response, dict) and self.valves.AZURE_AI_DATA_SOURCES: - if ( - self.valves.AZURE_AI_ENHANCE_CITATIONS - or self.valves.AZURE_AI_OPENWEBUI_CITATIONS - ): - response = self.enhance_azure_search_response(response) + response = self.enhance_azure_search_response(response) - # Emit native OpenWebUI citation events for non-streaming responses - if self.valves.AZURE_AI_OPENWEBUI_CITATIONS and __event_emitter__: + # Emit OpenWebUI citation events for non-streaming responses + if __event_emitter__: citations = self._extract_citations_from_response(response) if citations: # Get response content for filtering From 99920c566893c2361a0ac63a5ef9471f7585a43c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 3 Dec 2025 10:00:25 +0000 Subject: [PATCH 37/39] Fix all_chunks undefined error and revert version to 2.6.0 Co-authored-by: owndev <69784886+owndev@users.noreply.github.com> --- pipelines/azure/azure_ai_foundry.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pipelines/azure/azure_ai_foundry.py b/pipelines/azure/azure_ai_foundry.py index 9c24379..bfeb083 100644 --- a/pipelines/azure/azure_ai_foundry.py +++ b/pipelines/azure/azure_ai_foundry.py @@ -4,7 +4,7 @@ author_url: https://github.com/owndev/ project_url: https://github.com/owndev/Open-WebUI-Functions funding_url: https://github.com/sponsors/owndev -version: 2.7.0 +version: 2.6.0 license: Apache License 2.0 description: A pipeline for interacting with Azure AI services, enabling seamless communication with various AI models via configurable headers and robust error handling. This includes support for Azure OpenAI models as well as other Azure AI models by dynamically managing headers and request configurations. Azure AI Search (RAG) integration is only supported with Azure OpenAI endpoints. features: @@ -1148,7 +1148,6 @@ async def stream_processor_with_citations( async for chunk in content: chunk_str = chunk.decode("utf-8", errors="ignore") full_response_buffer += chunk_str - all_chunks.append(chunk) # Log chunk for debugging (only first 200 chars to avoid spam) # log.debug(f"Processing chunk: {chunk_str[:200]}...") From b4b882bc86fb9fb7fa633ff1496daa278537ef89 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 3 Dec 2025 10:14:40 +0000 Subject: [PATCH 38/39] Update Azure AI documentation to reflect citation changes Co-authored-by: owndev <69784886+owndev@users.noreply.github.com> --- docs/azure-ai-citations.md | 144 ++++++++++++++++++----------------- docs/azure-ai-integration.md | 84 +++++++------------- 2 files changed, 99 insertions(+), 129 deletions(-) diff --git a/docs/azure-ai-citations.md b/docs/azure-ai-citations.md index e102e69..26401a6 100644 --- a/docs/azure-ai-citations.md +++ b/docs/azure-ai-citations.md @@ -4,30 +4,31 @@ This document describes the native OpenWebUI citation support in the Azure AI Fo ## Overview -The Azure AI Foundry Pipeline now supports **native OpenWebUI citations** for Azure AI Search (RAG) responses. This feature enables the OpenWebUI frontend to display: +The Azure AI Foundry Pipeline supports **native OpenWebUI citations** for Azure AI Search (RAG) responses. This feature is **automatically enabled** when you configure Azure AI Search data sources (`AZURE_AI_DATA_SOURCES`). The OpenWebUI frontend will display: - **Citation cards** with source information and relevance scores - **Source previews** with content snippets -- **Relevance percentage** displayed on citation cards -- **Interactive citation UI** with clickable sources +- **Relevance percentage** displayed on citation cards (requires `AZURE_AI_INCLUDE_SEARCH_SCORES=true`) +- **Clickable `[docX]` references** that link directly to document URLs +- **Interactive citation UI** with expandable source details ## Features -### Dual Citation Modes +### Automatic Citation Support -The pipeline supports two modes for displaying citations: +When Azure AI Search is configured, the pipeline automatically: -1. **Native OpenWebUI Citations** (new): Structured citation events emitted via `__event_emitter__` for frontend consumption -2. **Markdown/HTML Citations** (existing): Collapsible HTML details with formatted citation information - -Both modes can be enabled simultaneously or independently via configuration. +1. Emits citation events via `__event_emitter__` for the OpenWebUI frontend +2. Converts `[docX]` references in the response to clickable markdown links +3. Filters citations to only show documents actually referenced in the response +4. Extracts relevance scores from Azure Search when available ### Configuration Options | Environment Variable | Default | Description | |---------------------|---------|-------------| -| `AZURE_AI_OPENWEBUI_CITATIONS` | `true` | Enable native OpenWebUI citation events | -| `AZURE_AI_ENHANCE_CITATIONS` | `true` | Enable markdown/HTML citation display (collapsible sections) | +| `AZURE_AI_DATA_SOURCES` | `""` | JSON configuration for Azure AI Search (required for citations) | +| `AZURE_AI_INCLUDE_SEARCH_SCORES` | `true` | Enable relevance score extraction from Azure Search | ### How It Works @@ -36,16 +37,17 @@ Both modes can be enabled simultaneously or independently via configuration. When Azure AI Search returns citations in a streaming response: 1. The pipeline detects citations in the SSE (Server-Sent Events) stream -2. **If `AZURE_AI_OPENWEBUI_CITATIONS` is enabled**: Citation events are emitted immediately via `__event_emitter__` -3. **If `AZURE_AI_ENHANCE_CITATIONS` is enabled**: A formatted markdown/HTML citation section is appended at the end of the stream +2. `[docX]` references in each chunk are converted to markdown links with document URLs +3. After the stream ends, citation events are emitted via `__event_emitter__` +4. Citations are filtered to only include documents referenced in the response #### Non-Streaming Responses When Azure AI Search returns citations in a non-streaming response: -1. The pipeline extracts citations from the response -2. **If `AZURE_AI_OPENWEBUI_CITATIONS` is enabled**: Individual citation events are emitted via `__event_emitter__` for each source -3. **If `AZURE_AI_ENHANCE_CITATIONS` is enabled**: The response content is enhanced with a formatted citation section +1. The pipeline extracts citations from the response context +2. `[docX]` references in the content are converted to markdown links +3. Individual citation events are emitted via `__event_emitter__` for each referenced source ## Citation Format @@ -91,74 +93,71 @@ Azure AI Search returns citations in this format: The pipeline automatically converts Azure citations to OpenWebUI format. -## Usage Examples +## Usage -### Basic Setup with Native Citations +### Basic Setup -```python -# Enable native OpenWebUI citations (default) -AZURE_AI_OPENWEBUI_CITATIONS=true +Configure Azure AI Search to enable citation support: -# Optionally disable markdown/HTML citations if you only want native citations -AZURE_AI_ENHANCE_CITATIONS=false +```bash +# Azure AI Search configuration (required for citations) +AZURE_AI_DATA_SOURCES='[{"type":"azure_search","parameters":{"endpoint":"https://YOUR-SEARCH-SERVICE.search.windows.net","index_name":"YOUR-INDEX-NAME","authentication":{"type":"api_key","key":"YOUR-SEARCH-API-KEY"}}}]' + +# Enable relevance scores (default: true) +AZURE_AI_INCLUDE_SEARCH_SCORES=true ``` -### Both Citation Modes Enabled (Default) +### Clickable Document Links -```python -# Enable both native and markdown/HTML citations (default) -AZURE_AI_OPENWEBUI_CITATIONS=true -AZURE_AI_ENHANCE_CITATIONS=true +The pipeline automatically converts `[docX]` references to clickable markdown links: + +```markdown +# Input from Azure AI +The answer can be found in [doc1] and [doc2]. + +# Output (converted by pipeline) +The answer can be found in [[doc1]](https://example.com/doc1.pdf) and [[doc2]](https://example.com/doc2.pdf). ``` -This configuration provides: -- Native citation cards in the OpenWebUI frontend -- Markdown/HTML citation section as fallback for non-supported clients +This works for both streaming and non-streaming responses. -### Only Markdown/HTML Citations (Legacy) +### Relevance Scores -```python -# Disable native citations, use only markdown/HTML -AZURE_AI_OPENWEBUI_CITATIONS=false -AZURE_AI_ENHANCE_CITATIONS=true -``` +When `AZURE_AI_INCLUDE_SEARCH_SCORES=true` (default), the pipeline: + +1. Automatically adds `include_contexts: ["citations", "all_retrieved_documents"]` to Azure Search requests +2. Extracts scores based on the `filter_reason` field: + - `filter_reason="rerank"` → uses `rerank_score` + - `filter_reason="score"` or not present → uses `original_search_score` +3. Displays the score as a percentage on citation cards ## Implementation Details ### Helper Functions -The pipeline includes three new helper functions: +The pipeline includes these helper functions for citation processing: 1. **`_extract_citations_from_response()`**: Extracts citations from Azure responses 2. **`_normalize_citation_for_openwebui()`**: Converts Azure citations to OpenWebUI format 3. **`_emit_openwebui_citation_events()`**: Emits citation events via `__event_emitter__` +4. **`_merge_score_data()`**: Matches citations with score data from `all_retrieved_documents` +5. **`_build_citation_urls_map()`**: Builds mapping of citation indices to URLs +6. **`_format_citation_link()`**: Creates markdown links for `[docX]` references +7. **`_convert_doc_refs_to_links()`**: Converts all `[docX]` references in content to markdown links ### Title Fallback Logic The pipeline uses intelligent title fallback: 1. Use `title` field if available -2. Fallback to `filepath` if title is empty -3. Fallback to `url` if both title and filepath are empty -4. Fallback to `"Unknown Document"` if all are empty +2. Fallback to filename extracted from `filepath` or `url` +3. Fallback to `"Unknown Document"` if all are empty This ensures every citation has a meaningful display name. -### Streaming Citation Emission - -Citations are emitted **as soon as they are detected** in the stream, ensuring: -- Low latency for citation display -- Frontend can start rendering citations while content is still streaming -- No waiting for the complete response - -### Backward Compatibility - -The implementation maintains full backward compatibility: +### Citation Filtering -- Existing markdown/HTML citation display continues to work -- No breaking changes to the API -- Both citation modes can be enabled simultaneously -- Default configuration enables both modes +Citations are filtered to only show documents that are actually referenced in the response content. For example, if Azure returns 5 citations but the response only references `[doc1]` and `[doc3]`, only those 2 citations will appear in the UI. ## Troubleshooting @@ -167,34 +166,37 @@ The implementation maintains full backward compatibility: **Problem**: Citations don't appear in the OpenWebUI frontend **Solutions**: -1. Verify `AZURE_AI_OPENWEBUI_CITATIONS=true` is set -2. Check that Azure AI Search is properly configured (`AZURE_AI_DATA_SOURCES`) -3. Ensure you're using an Azure OpenAI endpoint (not a generic Azure AI endpoint) -4. Check browser console for errors +1. Check that Azure AI Search is properly configured (`AZURE_AI_DATA_SOURCES`) +2. Ensure you're using an Azure OpenAI endpoint (not a generic Azure AI endpoint) +3. Verify the response contains `[docX]` references +4. Check browser console and server logs for errors -### Citation Cards vs. Markdown Section +### Relevance Scores Showing 0% -**Problem**: Seeing both citation cards and markdown section +**Problem**: All citation cards show 0% relevance -**Solution**: This is the default behavior. To show only citation cards: -```bash -AZURE_AI_OPENWEBUI_CITATIONS=true -AZURE_AI_ENHANCE_CITATIONS=false -``` +**Solutions**: +1. Verify `AZURE_AI_INCLUDE_SEARCH_SCORES=true` is set +2. Check that your Azure Search index supports scoring +3. Enable DEBUG logging to see the raw score values from Azure -### Missing Citation Metadata +### Links Not Working -**Problem**: Some citation fields (URL, filepath, score) are missing +**Problem**: `[docX]` references are not clickable -**Solution**: These fields are optional. Azure AI Search may not return all fields depending on your index configuration. The pipeline gracefully handles missing fields. +**Solutions**: +1. Ensure citations have valid `url` or `filepath` fields +2. Check that the document URL is accessible +3. Verify the markdown link format is being generated correctly ## References - [OpenWebUI Pipelines Citation Feature Discussion](https://github.com/open-webui/pipelines/issues/229) - [OpenWebUI Event Emitter Documentation](https://docs.openwebui.com/features/plugin/development/events) - [Azure AI Search Documentation](https://learn.microsoft.com/en-us/azure/search/) +- [Azure On Your Data API Reference](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/references/on-your-data) ## Version History -- **v2.6.0**: Added native OpenWebUI citations support -- **v2.5.x**: Markdown/HTML citation display only +- **v2.6.0**: Major refactor - removed `AZURE_AI_ENHANCE_CITATIONS` and `AZURE_AI_OPENWEBUI_CITATIONS` valves; citation support is now always enabled when `AZURE_AI_DATA_SOURCES` is configured; added clickable `[docX]` markdown links; improved score extraction using `filter_reason` field +- **v2.5.x**: Dual citation modes (OpenWebUI events + markdown/HTML) diff --git a/docs/azure-ai-integration.md b/docs/azure-ai-integration.md index 6d0782b..1f9766b 100644 --- a/docs/azure-ai-integration.md +++ b/docs/azure-ai-integration.md @@ -60,8 +60,9 @@ AZURE_AI_ENDPOINT="https://.openai.azure.com/openai/deployments/ -📚 Sources and References - -
-[doc1] - README.md - -📁 **File:** `README.md` -📄 **Chunk ID:** 0 -**Content:** -> environment variable. The token can be used to authenticate the workflow when accessing GitHub resources... - -
- -
-[doc2] - Documentation.md +# Enhanced response (with clickable links) +**Docker container actions** are a type of GitHub Actions [[doc1]](https://example.com/README.md)... +``` -📁 **File:** `Documentation.md` -📄 **Chunk ID:** 1 -**Content:** -> Docker container actions contain all their dependencies in the container and are therefore very consistent... +**Citation Card Features:** -
+- **Source information** with `[docX]` prefix for easy identification +- **Relevance percentage** displayed on citation cards (requires `AZURE_AI_INCLUDE_SEARCH_SCORES=true`) +- **Document preview** with content snippets +- **Clickable links** to source documents when URLs are available +- **Streaming support** with links converted inline as content streams - -``` +**Relevance Score Selection:** -**Enhanced Citation Features:** +The pipeline uses the `filter_reason` field from Azure Search to select the appropriate score: +- `filter_reason="rerank"` → uses `rerank_score` +- `filter_reason="score"` or not present → uses `original_search_score` -- **Collapsible interface** with expandable sections for clean presentation -- **Two-level organization** - main sources section and individual document details -- **Complete content display** - full document content, not just previews -- **Document references** with clear [doc1], [doc2] labels for easy cross-referencing -- **Source metadata** including file paths, URLs, and chunk IDs for precise tracking -- **Streaming support** with citations properly formatted for both streaming and non-streaming responses -- **Space efficient** - collapsed by default to avoid overwhelming the main response +For more details, see the [Azure AI Citations Documentation](azure-ai-citations.md). > [!TIP] > To use **Azure OpenAI** and other **Azure AI** models **simultaneously**, you can use the following URL: `https://.services.ai.azure.com/models/chat/completions?api-version=2024-05-01-preview` From f927f16908f28d165677ce6fd226d3d481418d8b Mon Sep 17 00:00:00 2001 From: owndev <69784886+owndev@users.noreply.github.com> Date: Thu, 11 Dec 2025 08:55:11 +0100 Subject: [PATCH 39/39] refactor: citation handling and normalization in Azure AI pipeline - Updated regex pattern for [docX] citations to use compiled regex for performance. - Enhanced environment variable handling for boolean flags. - Introduced BM25 and rerank score normalization factors for relevance display. - Improved logging for citation conversion and error handling. --- pipelines/azure/azure_ai_foundry.py | 154 ++++++++++++++++++++-------- 1 file changed, 112 insertions(+), 42 deletions(-) diff --git a/pipelines/azure/azure_ai_foundry.py b/pipelines/azure/azure_ai_foundry.py index bfeb083..0a87638 100644 --- a/pipelines/azure/azure_ai_foundry.py +++ b/pipelines/azure/azure_ai_foundry.py @@ -147,7 +147,7 @@ async def cleanup_response( class Pipe: # Regex pattern for matching [docX] citation references - DOC_REF_PATTERN = r"\[doc(\d+)\]" + DOC_REF_PATTERN = re.compile(r"\[doc(\d+)\]") # Environment variables for API key, endpoint, and optional model class Valves(BaseModel): @@ -182,19 +182,26 @@ class Valves(BaseModel): # Switch for sending model name in request body AZURE_AI_MODEL_IN_BODY: bool = Field( - default=os.getenv("AZURE_AI_MODEL_IN_BODY", False), + default=bool( + os.getenv("AZURE_AI_MODEL_IN_BODY", "false").lower() == "true" + ), description="If True, include the model name in the request body instead of as a header.", ) # Flag to indicate if predefined Azure AI models should be used USE_PREDEFINED_AZURE_AI_MODELS: bool = Field( - default=os.getenv("USE_PREDEFINED_AZURE_AI_MODELS", False), + default=bool( + os.getenv("USE_PREDEFINED_AZURE_AI_MODELS", "false").lower() == "true" + ), description="Flag to indicate if predefined Azure AI models should be used.", ) # If True, use Authorization header with Bearer token instead of api-key header. USE_AUTHORIZATION_HEADER: bool = Field( - default=bool(os.getenv("AZURE_AI_USE_AUTHORIZATION_HEADER", False)), + default=bool( + os.getenv("AZURE_AI_USE_AUTHORIZATION_HEADER", "false").lower() + == "true" + ), description="Set to True to use Authorization header with Bearer token instead of api-key header.", ) @@ -207,10 +214,30 @@ class Valves(BaseModel): # Enable relevance scores from Azure AI Search AZURE_AI_INCLUDE_SEARCH_SCORES: bool = Field( - default=True, + default=bool( + os.getenv("AZURE_AI_INCLUDE_SEARCH_SCORES", "true").lower() == "true" + ), description="If True, automatically add 'include_contexts' with 'all_retrieved_documents' to Azure AI Search requests to get relevance scores (original_search_score and rerank_score). This enables relevance percentage display in citation cards.", ) + # BM25 score normalization factor for relevance percentage display + # BM25 scores are unbounded and vary by collection. This value is used to normalize + # scores to 0-1 range: normalized = min(score / BM25_SCORE_MAX, 1.0) + # See: https://learn.microsoft.com/en-us/azure/search/index-ranking-similarity + BM25_SCORE_MAX: float = Field( + default=float(os.getenv("AZURE_AI_BM25_SCORE_MAX", "100.0")), + description="Normalization divisor for BM25 search scores (0-1 range). Adjust based on your index characteristics. Default 100.0 is suitable for typical collections; higher values (e.g., 200.0) reduce saturation for large documents.", + ) + + # Rerank score normalization factor for relevance percentage display + # Cohere rerankers via Azure return 0-4, most others 0-1. This value normalizes + # scores above 1.0 to 0-1 range: normalized = min(score / RERANK_SCORE_MAX, 1.0) + # See: https://learn.microsoft.com/en-us/azure/search/semantic-ranking + RERANK_SCORE_MAX: float = Field( + default=float(os.getenv("AZURE_AI_RERANK_SCORE_MAX", "4.0")), + description="Normalization divisor for rerank scores (0-1 range). Use 4.0 for Cohere rerankers, 1.0 for standard semantic rerankers.", + ) + def __init__(self): self.valves = self.Valves() self.name: str = f"{self.valves.AZURE_AI_PIPELINE_PREFIX}:" @@ -579,7 +606,9 @@ def _merge_score_data( if doc_data: if doc_data.get("original_search_score") is not None: - citation["original_search_score"] = doc_data["original_search_score"] + citation["original_search_score"] = doc_data[ + "original_search_score" + ] if doc_data.get("rerank_score") is not None: citation["rerank_score"] = doc_data["rerank_score"] if doc_data.get("filter_reason") is not None: @@ -667,49 +696,56 @@ def _normalize_citation_for_openwebui( # - filter_reason="score" or not present: Document filtered by/passed original search score, use original_search_score if filter_reason == "rerank" and rerank_score is not None: # Document filtered by rerank score - use rerank_score - # Azure AI Search semantic rerankers typically return scores in 0-1 range. - # However, some Cohere rerankers (via Azure AI) may use 0-4 range. + # Cohere rerankers via Azure AI return scores in 0-4 range (source: Azure AI Search documentation) + # Most semantic rerankers return 0-1, so we normalize 0-4 range down to 0-1 for consistency. + # Reference: https://learn.microsoft.com/en-us/azure/search/semantic-ranking score_val = float(rerank_score) if score_val > 1.0: - normalized_score = min(score_val / 4.0, 1.0) + normalized_score = min(score_val / self.valves.RERANK_SCORE_MAX, 1.0) else: normalized_score = score_val log.debug( - f"Using rerank_score (filter_reason=rerank): {rerank_score} -> {normalized_score}" + f"Using rerank_score (filter_reason=rerank): {rerank_score} -> {normalized_score} " + f"(normalized via {self.valves.RERANK_SCORE_MAX})" ) elif ( filter_reason is None or filter_reason == "score" ) and original_search_score is not None: # filter_reason is "score" or not present - use original_search_score - # BM25/keyword search scores vary based on term frequency and document collection. - # Typical BM25 scores in Azure AI Search range from ~0 to ~50 but can go higher. + # BM25 scores are unbounded and vary by collection size and term distribution. + # We normalize by dividing by BM25_SCORE_MAX to produce a value in 0-1 range. + # This preserves relative ranking without hard-capping high-relevance documents. + # Reference: https://learn.microsoft.com/en-us/azure/search/index-ranking-similarity score_val = float(original_search_score) if score_val > 1.0: - normalized_score = min(score_val / 100.0, 1.0) + normalized_score = min(score_val / self.valves.BM25_SCORE_MAX, 1.0) else: normalized_score = score_val log.debug( - f"Using original_search_score (filter_reason={filter_reason}): {original_search_score} -> {normalized_score}" + f"Using original_search_score (filter_reason={filter_reason}): {original_search_score} -> {normalized_score} " + f"(normalized via {self.valves.BM25_SCORE_MAX})" ) elif original_search_score is not None: # Fallback for unknown filter_reason values - use original_search_score score_val = float(original_search_score) if score_val > 1.0: - normalized_score = min(score_val / 100.0, 1.0) + normalized_score = min(score_val / self.valves.BM25_SCORE_MAX, 1.0) else: normalized_score = score_val log.debug( - f"Using original_search_score (fallback, filter_reason={filter_reason}): {original_search_score} -> {normalized_score}" + f"Using original_search_score (fallback, filter_reason={filter_reason}): {original_search_score} -> {normalized_score} " + f"(normalized via {self.valves.BM25_SCORE_MAX})" ) elif rerank_score is not None: # Fallback to rerank_score if available but filter_reason doesn't match score_val = float(rerank_score) if score_val > 1.0: - normalized_score = min(score_val / 4.0, 1.0) + normalized_score = min(score_val / self.valves.RERANK_SCORE_MAX, 1.0) else: normalized_score = score_val log.debug( - f"Using rerank_score (fallback): {rerank_score} -> {normalized_score}" + f"Using rerank_score (fallback): {rerank_score} -> {normalized_score} " + f"(normalized via {self.valves.RERANK_SCORE_MAX})" ) elif legacy_score is not None: normalized_score = float(legacy_score) @@ -766,9 +802,7 @@ def _build_citation_urls_map( return citation_urls - def _format_citation_link( - self, doc_num: int, url: Optional[str] = None - ) -> str: + def _format_citation_link(self, doc_num: int, url: Optional[str] = None) -> str: """ Format a markdown link for a [docX] reference. @@ -824,9 +858,13 @@ def replace_doc_ref(match): # Count conversions for logging original_count = len(re.findall(self.DOC_REF_PATTERN, content)) - linked_count = sum(1 for i in range(1, len(citations) + 1) if citation_urls.get(i)) + linked_count = sum( + 1 for i in range(1, len(citations) + 1) if citation_urls.get(i) + ) if original_count > 0: - log.info(f"Converted {original_count} [docX] references to markdown links ({linked_count} with URLs)") + log.info( + f"Converted {original_count} [docX] references to markdown links ({linked_count} with URLs)" + ) return converted @@ -1144,6 +1182,13 @@ async def stream_processor_with_citations( full_response_buffer = "" response_content = "" # Track the actual response content citations_data = None + citation_urls = {} # Pre-allocate citation URLs map + + # Pre-define the replacement function outside the loop to avoid repeated creation + def replace_ref(m, urls_map): + doc_num = int(m.group(1)) + url = urls_map.get(doc_num) + return self._format_citation_link(doc_num, url) async for chunk in content: chunk_str = chunk.decode("utf-8", errors="ignore") @@ -1259,11 +1304,23 @@ async def stream_processor_with_citations( # Use citations if found, otherwise use all_retrieved_documents if citations_found and not citations_data: citations_data = citations_found + # Build citation URLs map once when citations are found + citation_urls = ( + self._build_citation_urls_map( + citations_data + ) + ) log.info( f"Successfully extracted {len(citations_data)} citations from stream" ) elif all_docs_found and not citations_data: citations_data = all_docs_found + # Build citation URLs map once when citations are found + citation_urls = ( + self._build_citation_urls_map( + citations_data + ) + ) log.info( f"Using {len(citations_data)} all_retrieved_documents as citations" ) @@ -1280,23 +1337,23 @@ async def stream_processor_with_citations( # Convert [docX] references to markdown links in the chunk content # This creates clickable links to source documents in streaming responses chunk_modified = False - if "[doc" in chunk_str: + if "[doc" in chunk_str and citation_urls: try: - # Build citation URLs map using shared helper - citation_urls = self._build_citation_urls_map(citations_data) - - # Define replacement function once, outside inner loops - def replace_ref(m): - doc_num = int(m.group(1)) - url = citation_urls.get(doc_num) - return self._format_citation_link(doc_num, url) - # Parse and modify each SSE data line modified_lines = [] chunk_lines = chunk_str.split("\n") for line in chunk_lines: - if line.startswith("data: ") and line.strip() != "data: [DONE]": + # Early exit: skip lines without [doc references + if "[doc" not in line: + modified_lines.append(line) + continue + + # Process only SSE data lines + if ( + line.startswith("data: ") + and line.strip() != "data: [DONE]" + ): json_str = line[6:].strip() if json_str and json_str != "[DONE]": try: @@ -1307,23 +1364,34 @@ def replace_ref(m): and data["choices"] ): line_modified = False + # Process choices until we find and modify content for choice in data["choices"]: if ( "delta" in choice and "content" in choice["delta"] ): - content_val = choice["delta"]["content"] + content_val = choice["delta"][ + "content" + ] if "[doc" in content_val: - # Convert [docX] to markdown link - choice["delta"]["content"] = re.sub( - self.DOC_REF_PATTERN, - replace_ref, - content_val, + # Convert [docX] to markdown link using pre-compiled pattern + # Use lambda to pass citation_urls to pre-defined function + choice["delta"]["content"] = ( + self.DOC_REF_PATTERN.sub( + lambda m: replace_ref( + m, citation_urls + ), + content_val, + ) ) line_modified = True + # Early exit: content found and modified + break if line_modified: - modified_lines.append(f"data: {json.dumps(data)}") + modified_lines.append( + f"data: {json.dumps(data)}" + ) chunk_modified = True else: modified_lines.append(line) @@ -1345,7 +1413,9 @@ def replace_ref(m): chunk = modified_chunk_str.encode("utf-8") except Exception as convert_err: - log.debug(f"Error converting [docX] to markdown links: {convert_err}") + log.debug( + f"Error converting [docX] to markdown links: {convert_err}" + ) # Fall through to yield original chunk # Yield the (possibly modified) chunk