Feature/speech managed identity

application/single_app/config.py

@@ -64,6 +64,7 @@
 from io import BytesIO
 from typing import List
 
+import azure.cognitiveservices.speech as speechsdk
 from azure.cosmos import CosmosClient, PartitionKey, exceptions
 from azure.cosmos.exceptions import CosmosResourceNotFoundError
 from azure.core.credentials import AzureKeyCredential

application/single_app/functions_documents.py

@@ -4765,6 +4765,24 @@ def _split_audio_file(input_path: str, chunk_seconds: int = 540) -> List[str]:
  print(f"Produced {len(chunks)} WAV chunks: {chunks}")
  return chunks
 
+# Azure Speech SDK helper to get speech config with fresh token
+def _get_speech_config(settings, endpoint: str, locale: str):
+ """Get speech config with fresh token"""
+ if settings.get("speech_service_authentication_type") == "managed_identity":
+ credential = DefaultAzureCredential()
+ token = credential.get_token(cognitive_services_scope)
+ speech_config = speechsdk.SpeechConfig(endpoint=endpoint)
+
+ # Set the authorization token AFTER creating the config
+ speech_config.authorization_token = token.token
+ else:
+ key = settings.get("speech_service_key", "")
+ speech_config = speechsdk.SpeechConfig(endpoint=endpoint, subscription=key)
+
+ speech_config.speech_recognition_language = locale
+ print(f"[Debug] Speech config obtained successfully", flush=True)
+ return speech_config
+
 def process_audio_document(
  document_id: str,
  user_id: str,
@@ -4804,32 +4822,65 @@ def process_audio_document(
  # 3) transcribe each WAV chunk
  settings = get_settings()
  endpoint = settings.get("speech_service_endpoint", "").rstrip('/')
- key = settings.get("speech_service_key", "")
  locale = settings.get("speech_service_locale", "en-US")
- url = f"{endpoint}/speechtotext/transcriptions:transcribe?api-version=2024-11-15"
 
  all_phrases: List[str] = []
- for idx, chunk_path in enumerate(chunk_paths, start=1):
- update_callback(current_file_chunk=idx, status=f"Transcribing chunk {idx}/{len(chunk_paths)}…")
- print(f"Transcribing WAV chunk: {chunk_path}")
-
- with open(chunk_path, 'rb') as audio_f:
- files = {
- 'audio': (os.path.basename(chunk_path), audio_f, 'audio/wav'),
- 'definition': (None, json.dumps({'locales':[locale]}), 'application/json')
- }
- headers = {'Ocp-Apim-Subscription-Key': key}
- resp = requests.post(url, headers=headers, files=files)
- try:
- resp.raise_for_status()
- except Exception as e:
- print(f"[Error] HTTP error for {chunk_path}: {e}")
- raise
 
- result = resp.json()
- phrases = result.get('combinedPhrases', [])
- print(f"Received {len(phrases)} phrases")
- all_phrases += [p.get('text','').strip() for p in phrases if p.get('text')]
+ # Fast Transcription API not yet available in sovereign clouds, so use SDK
+ if AZURE_ENVIRONMENT in ("usgovernment", "custom"):
+ for idx, chunk_path in enumerate(chunk_paths, start=1):
+ print(f"[Debug] Transcribing chunk {idx}: {chunk_path}")
+
+ # Get fresh config (tokens expire after ~1 hour)
+ speech_config = _get_speech_config(settings, endpoint, locale)
+
+ audio_config = speechsdk.AudioConfig(filename=chunk_path)
+ speech_recognizer = speechsdk.SpeechRecognizer(
+ speech_config=speech_config,
+ audio_config=audio_config
+ )
+
+ result = speech_recognizer.recognize_once()
+ if result.reason == speechsdk.ResultReason.RecognizedSpeech:
+ print(f"[Debug] Recognized: {result.text}")
+ all_phrases.append(result.text)
+ elif result.reason == speechsdk.ResultReason.NoMatch:
+ print(f"[Warning] No speech in {chunk_path}")
+ elif result.reason == speechsdk.ResultReason.Canceled:
+ print(f"[Error] {result.cancellation_details.reason}: {result.cancellation_details.error_details}")
+ raise RuntimeError(f"Transcription canceled for {chunk_path}: {result.cancellation_details.error_details}")
+
+ else:
+ # Use the fast-transcription API if not in sovereign or custom cloud
+ url = f"{endpoint}/speechtotext/transcriptions:transcribe?api-version=2024-11-15"
+ for idx, chunk_path in enumerate(chunk_paths, start=1):
+ update_callback(current_file_chunk=idx, status=f"Transcribing chunk {idx}/{len(chunk_paths)}…")
+ print(f"[Debug] Transcribing WAV chunk: {chunk_path}")
+
+ with open(chunk_path, 'rb') as audio_f:
+ files = {
+ 'audio': (os.path.basename(chunk_path), audio_f, 'audio/wav'),
+ 'definition': (None, json.dumps({'locales':[locale]}), 'application/json')
+ }
+ if settings.get("speech_service_authentication_type") == "managed_identity":
+ credential = DefaultAzureCredential()
+ token = credential.get_token(cognitive_services_scope)
+ headers = {'Authorization': f'Bearer {token.token}'}
+ else:
+ key = settings.get("speech_service_key", "")
+ headers = {'Ocp-Apim-Subscription-Key': key}
+
+ resp = requests.post(url, headers=headers, files=files)
+ try:
+ resp.raise_for_status()
+ except Exception as e:
+ print(f"[Error] HTTP error for {chunk_path}: {e}")
+ raise
+
+ result = resp.json()
+ phrases = result.get('combinedPhrases', [])
+ print(f"[Debug] Received {len(phrases)} phrases")
+ all_phrases += [p.get('text','').strip() for p in phrases if p.get('text')]
 
  # 4) cleanup WAV chunks
  for p in chunk_paths:

application/single_app/requirements.txt

@@ -52,4 +52,5 @@ cython
 pyyaml==6.0.2
 aiohttp==3.12.15
 html2text==2025.4.15
-matplotlib==3.10.7
+matplotlib==3.10.7
+azure-cognitiveservices-speech==1.47.0

application/single_app/route_frontend_admin_settings.py

@@ -668,6 +668,7 @@ def is_valid_url(url):
  'speech_service_endpoint': form_data.get('speech_service_endpoint', '').strip(),
  'speech_service_location': form_data.get('speech_service_location', '').strip(),
  'speech_service_locale': form_data.get('speech_service_locale', '').strip(),
+ 'speech_service_authentication_type': form_data.get('speech_service_authentication_type', 'key'),
  'speech_service_key': form_data.get('speech_service_key', '').strip(),
 
  'metadata_extraction_model': form_data.get('metadata_extraction_model', '').strip(),

application/single_app/static/js/admin/admin_settings.js

@@ -1654,6 +1654,15 @@ function setupToggles() {
  });
  }
 
+ const speechAuthType = document.getElementById('speech_service_authentication_type');
+ if (speechAuthType) {
+ speechAuthType.addEventListener('change', function () {
+ document.getElementById('speech_service_key_container').style.display =
+ (this.value === 'key') ? 'block' : 'none';
+ markFormAsModified();
+ });
+ }
+
  const officeAuthType = document.getElementById('office_docs_authentication_type');
  const connStrGroup = document.getElementById('office_docs_storage_conn_str_group');
  const urlGroup = document.getElementById('office_docs_storage_url_group');
@@ -3113,28 +3122,41 @@ function handleTabNavigation(stepNumber) {
  5: 'ai-models-tab', // Embedding settings (now in AI Models tab)
  6: 'search-extract-tab', // AI Search settings
  7: 'search-extract-tab', // Document Intelligence settings
- 8: 'workspaces-tab', // Video support
- 9: 'workspaces-tab', // Audio support
+ 8: 'search-extract-tab', // Video support
+ 9: 'search-extract-tab', // Audio support
  10: 'safety-tab', // Content safety
- 11: 'system-tab', // User feedback and archiving (renamed from other-tab)
+ 11: 'safety-tab', // User feedback and archiving (changed from system-tab)
  12: 'citation-tab' // Enhanced Citations and Image Generation
  };
 
  // Activate the appropriate tab
  const tabId = stepToTab[stepNumber];
  if (tabId) {
- const tab = document.getElementById(tabId);
- if (tab) {
- // Use bootstrap Tab to show the tab
- const bootstrapTab = new bootstrap.Tab(tab);
- bootstrapTab.show();
-
- // Scroll to the relevant section after a small delay to allow tab to switch
- setTimeout(() => {
- // For tabs that need to jump to specific sections
- scrollToRelevantSection(stepNumber, tabId);
- }, 300);
+ // Check if we're using sidebar navigation or tab navigation
+ const sidebarToggle = document.getElementById('admin-settings-toggle');
+
+ if (sidebarToggle) {
+ // Using sidebar navigation - call showAdminTab function
+ const tabName = tabId.replace('-tab', ''); // Remove '-tab' suffix
+ if (typeof showAdminTab === 'function') {
+ showAdminTab(tabName);
+ } else if (typeof window.showAdminTab === 'function') {
+ window.showAdminTab(tabName);
+ }
+ } else {
+ // Using Bootstrap tabs
+ const tab = document.getElementById(tabId);
+ if (tab) {
+ // Use bootstrap Tab to show the tab
+ const bootstrapTab = new bootstrap.Tab(tab);
+ bootstrapTab.show();
+ }
  }
+
+ // Scroll to the relevant section after a small delay to allow tab to switch
+ setTimeout(() => {
+ scrollToRelevantSection(stepNumber, tabId);
+ }, 300);
  }
 }
 
@@ -3148,23 +3170,50 @@ function scrollToRelevantSection(stepNumber, tabId) {
  let targetElement = null;
 
  switch (stepNumber) {
+ case 1: // App title and logo
+ targetElement = document.getElementById('branding-section');
+ break;
+ case 2: // GPT settings
+ targetElement = document.getElementById('gpt-configuration');
+ break;
+ case 3: // GPT model selection
+ targetElement = document.getElementById('gpt_models_list')?.closest('.mb-3');
+ break;
  case 4: // Workspaces toggle section
- targetElement = document.getElementById('enable_user_workspace')?.closest('.card');
+ targetElement = document.getElementById('personal-workspaces-section');
+ break;
+ case 5: // Embedding settings
+ targetElement = document.getElementById('embeddings-configuration');
+ break;
+ case 6: // AI Search settings
+ targetElement = document.getElementById('azure-ai-search-section');
+ break;
+ case 7: // Document Intelligence settings
+ targetElement = document.getElementById('document-intelligence-section');
  break;
  case 8: // Video file support
  targetElement = document.getElementById('enable_video_file_support')?.closest('.form-group');
  break;
  case 9: // Audio file support
  targetElement = document.getElementById('enable_audio_file_support')?.closest('.form-group');
  break;
+ case 10: // Content safety
+ targetElement = document.getElementById('content-safety-section');
+ break;
+ case 11: // User feedback and archiving
+ targetElement = document.getElementById('user-feedback-section');
+ break;
+ case 12: // Enhanced citations and image generation
+ targetElement = document.getElementById('enhanced-citations-section');
+ break;
  default:
  // For other steps, no specific scrolling
  break;
  }
 
  // If we found a target element, scroll to it
  if (targetElement) {
- targetElement.scrollIntoView({ behavior: 'smooth', block: 'center' });
+ targetElement.scrollIntoView({ behavior: 'smooth', block: 'start' });
  }
 }
 
@@ -3302,9 +3351,14 @@ function isStepComplete(stepNumber) {
 
  // Otherwise check settings
  const speechEndpoint = document.getElementById('speech_service_endpoint')?.value;
- const speechKey = document.getElementById('speech_service_key')?.value;
+ const authType = document.getElementById('speech_service_authentication_type').value;
+ const key = document.getElementById('speech_service_key').value;
 
- return speechEndpoint && speechKey;
+ if (!speechEndpoint || (authType === 'key' && !key)) {
+ return false;
+ } else {
+ return true;
+ }
 
  case 10: // Content safety - always complete (optional)
  case 11: // User feedback and archiving - always complete (optional)

application/single_app/templates/admin_settings.html

@@ -369,18 +369,24 @@ <h4>9. Enable Audio File Support</h4>
  <strong>Required:</strong> Audio support configuration is required if workspaces are enabled.
  </div>
 
- <p class="mb-3">Use the <strong>Audio Support</strong> settings in the Workspaces tab to enable and configure audio file processing.</p>
+ <p class="mb-3">Use the <strong>Audio Support</strong> settings on the Search & Extract tab to enable and configure audio file processing.</p>
 
  <ul class="list-group mb-4">
  <li class="list-group-item d-flex justify-content-between align-items-center">
  <span>Speech Service Endpoint</span>
  <span id="audio-endpoint-badge" class="badge bg-danger">Required</span>
  </li>
  <li class="list-group-item d-flex justify-content-between align-items-center">
- <span>Speech Service Key</span>
- <span id="audio-key-badge" class="badge bg-danger">Required</span>
+ <span>Authentication</span>
+ <span id="speech-auth-badge" class="badge bg-danger">Required</span>
  </li>
  </ul>
+
+ <div class="alert alert-info">
+ <strong>Note:</strong> If using Managed Identity authentication ensure the Service Principal has been assigned the Cognitive Services Speech Contributor role on the Azure Speech Service and that the Speech endpoint is configured with a custom domain name.
+
+ <i class="bi bi-info-circle ms-2" data-bs-toggle="tooltip" title="When a Cognitive Service has a custom domain name, the endpoint will typically look like https://<resource-name>.cognitiveservices.azure.<com or us> instead of https://<location>.cognitiveservices.azure.<com or us>"></i>
+ </div>
  </div>
 
  <!-- Step 10: Content Safety -->
@@ -2831,7 +2837,7 @@ <h5>Speech Service Settings</h5>
  <input type="text" class="form-control"
  id="speech_service_endpoint" name="speech_service_endpoint"
  value="{{ settings.speech_service_endpoint or '' }}"
- placeholder="https://<location>.cognitiveservices.azure.<com or us>/">
+ placeholder="https://<location or custom domain>.cognitiveservices.azure.<com or us>/">
  </div>
 
  <div class="mb-3">
@@ -2849,17 +2855,41 @@ <h5>Speech Service Settings</h5>
  </div>
 
  <div class="mb-3">
- <label for="speech_service_key" class="form-label">API Key</label>
+ <label for="speech_service_authentication_type" class="form-label">
+ Authentication Type
+ </label>
+ <select class="form-select" id="speech_service_authentication_type" name="speech_service_authentication_type">
+ <option value="key" {% if settings.speech_service_authentication_type == 'key' or not settings.speech_service_authentication_type %}selected{% endif %}>
+ Key
+ </option>
+ <option value="managed_identity" {% if settings.speech_service_authentication_type == 'managed_identity' %}selected{% endif %}>
+ Managed Identity
+ </option>
+ </select>
+ </div>
+ <div class="mb-3" id="speech_service_key_container" {% if settings.speech_service_authentication_type == 'managed_identity' %}style="display: none;"{% endif %}>
+ <label for="speech_service_key" class="form-label">
+ API Key
+ </label>
  <div class="input-group">
- <input type="password" class="form-control"
- id="speech_service_key" name="speech_service_key"
- value="{{ settings.speech_service_key or '' }}">
- <button type="button" class="btn btn-outline-secondary"
- id="toggle_speech_service_key">Show</button>
+ <input
+ type="password"
+ class="form-control"
+ id="speech_service_key"
+ name="speech_service_key"
+ value="{{ settings.speech_service_key or '' }}"
+ >
+ <button
+ type="button"
+ class="btn btn-outline-secondary"
+ id="toggle_speech_service_key"
+ >
+ Show
+ </button>
  </div>
  </div>
- </div>
 
+ </div>
  <p class="mt-2 mb-0">
  <small class="text-muted">
  <a href="#citation" onclick="switchTab(event, 'citation-tab')">