MMI-3400 (bcgov#2544)

* auto-clipper initial commit

* segment refine

* update segmentation and labels, added weather/traffic/ads labels

* add documents

* Update auto clipper
Add DevOps configs

* Publish tno-core:1.0.30

* Fix UI

* Fix build

* Debug

* Debug

* Fix config

* change to batch transcription and update prompt, heuristics,etc

* update prompt amd heuristics

* Add Auto Clipper Service
segment refine
update segmentation and labels, added weather/traffic/ads labels
add documents
Add DevOps configs
Publish tno-core:1.0.30

* add missing file

* update prompt for clip summary

* remove deleted files

* update

* update station file

* update title format

* update llm temprature

* update temperature

* add category in the title

---------

Co-authored-by: Jeremy Foster <jeremy.foster@fosol.ca>

Author: AlessiaYChen

services/net/auto-clipper/AutoClipperManager.cs

@@ -1,5 +1,6 @@
 using System.Globalization;
 using System.Text;
+using System.Text.RegularExpressions;
 using Confluent.Kafka;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
@@ -584,6 +585,10 @@ private ContentModel BuildClipContentModel(ContentModel sourceContent, ClipDefin
         // Calculate clip time = parent content publish time + clip start offset
         var clipTime = sourceContent.PublishedOn?.Add(definition.Start);
         var timePrefix = clipTime?.ToString("HH:mm");
+        var clipTitle = FormatClipTitle(definition.Title);
+        var categoryLabel = FormatClipCategory(definition.Category);
+        var headlineCore = string.IsNullOrEmpty(timePrefix) ? clipTitle : $"{timePrefix} - {clipTitle}";
+        var headline = string.IsNullOrEmpty(categoryLabel) ? headlineCore : $"[{categoryLabel}] {headlineCore}";
 
         return new ContentModel
         {
@@ -601,7 +606,7 @@ private ContentModel BuildClipContentModel(ContentModel sourceContent, ClipDefin
             Byline = sourceContent.Byline,
             Status = ContentStatus.Draft,
             Uid = BaseService.GetContentHash(sourceContent.Source?.Code ?? "AutoClipper", $"{sourceContent.Uid}-clip-{clipIndex}", sourceContent.PublishedOn),
-            Headline = string.IsNullOrEmpty(timePrefix) ? definition.Title : $"{timePrefix} - {definition.Title}",
+            Headline = headline,
             Summary = $"[AutoClipper:{definition.Category}]\n{clipSummary}",
             Body = transcriptBody,
             SourceUrl = sourceContent.SourceUrl,
@@ -650,6 +655,24 @@ private static string BuildTranscriptDocument(IReadOnlyList<TimestampedTranscrip
         return sb.ToString().Trim();
     }
 
+    private static readonly Regex CamelCaseBoundaryRegex = new("(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])", RegexOptions.Compiled);
+
+    private static string FormatClipTitle(string? title)
+    {
+        if (string.IsNullOrWhiteSpace(title)) return "Clip";
+        var normalized = title.Trim();
+        normalized = normalized.Replace('_', ' ').Replace('-', ' ');
+        normalized = CamelCaseBoundaryRegex.Replace(normalized, " ");
+        normalized = Regex.Replace(normalized, "\\s+", " ").Trim();
+        return normalized;
+    }
+
+    private static string FormatClipCategory(string? category)
+    {
+        if (string.IsNullOrWhiteSpace(category)) return string.Empty;
+        return Regex.Replace(category.Trim(), "\\s+", " ");
+    }
+
     private static string FormatTimestamp(TimeSpan value)
     {
         var hours = (int)Math.Floor(value.TotalHours);
@@ -684,4 +707,3 @@ private static string FormatTimestamp(TimeSpan value)
     }
     #endregion
 }
-

services/net/auto-clipper/Config/Stations/CKNW.yml

@@ -1,4 +1,4 @@
-# Station specific configuration for CKNW. Adjust as needed.
+# Station-specific configuration for CKNW.
 name: CKNW
 sample_rate: 16000
 transcription:
@@ -19,16 +19,26 @@ text:
     "(?i)coming up": Promo
   llm_segmentation: true
   llm_model: gpt-5.1-chat
-  llm_temperature: 1
+  llm_temperature: 1 # Set to 0.0 for deterministic JSON and stable indices.
   system_prompt: |
     You are a Broadcast Structure Parser. Your ONLY job is to detect segment transitions.
-    Output MUST be a single, raw JSON object.
-    CRITICAL: Start your response with '{' and end with '}'.
+    Output MUST be a single, raw JSON object. 
+    CRITICAL: Start your response with '{' and end with '}'. 
     DO NOT use markdown, backticks, or "```json" blocks. No introductory or closing text.
   max_stories: 15
   llm_prompt: |
     Identify every point in the transcript where the topic or segment type changes.
-
+    
+    # SUMMARY RULES
+    1. **Prioritize Anchor Leads**: For News, derive the summary from the anchor's introduction or the first three sentences of the report.
+    2. **Active Voice**: Use active journalistic voice (e.g., "Surrey Council rejects housing proposal" NOT "A report about a meeting").
+    3. **Category Formulas**:
+       - News: [Subject] [Action] (e.g., "Abbotsford shooting victim’s son calls for urgent investigation").
+       - Traffic: [Location] [Incident/Status] (e.g., "Highway 99 northbound blocked at Hwy 17A due to crash").
+       - Weather: [Condition] + [High Temp] (e.g., "Mix of sun and cloud with a high of 9 degrees").
+       - Ad: [Business Name] + [Offer/Service] (e.g., "McDonald's features Egg McMuffin with Hollandaise sauce").
+    4. **One Sentence Only**: Summaries MUST be a single, concise sentence.
+    
     # STRUCTURAL RULES (To Prevent Bundling)
     1. **The Sign-off Rule**: Phrases like "Global News," "CBC News," or "Reporting live" followed by a name mark the END of a segment. The very next sentence MUST be a new boundary.
     2. **The Handoff Rule**: When an anchor introduces a reporter (e.g., "As Joshua reports..."), the boundary starts at the ANCHOR'S introduction line.
@@ -42,7 +52,7 @@ text:
           "index": [Sentence Number],
           "category": "News | Traffic | Weather | Ad | Promo",
           "title": "[Short Slug]",
-          "summary": "[One Sentence Recap]",
+          "summary": "[Journalistic Summary Sentence]",
           "score": 0.95
         }
       ]
@@ -61,7 +71,7 @@ heuristics:
       weight: 0.65
       category: Promo
       note: Signals a hard break/transition
-
+    
     # --- New Anchor-Specific Patterns ---
     - pattern: "(?i)Ian Wilson"
       weight: 0.85
@@ -75,7 +85,6 @@ heuristics:
       weight: 0.60
       category: News
       note: Primary News anchor reset
-
     # --- Service Cues ---
     - pattern: "(?i)traffic update"
       weight: 0.6

tools/auto-clipper-harness/.env.sample

@@ -8,9 +8,12 @@ AUTOCLIP_HARNESS_BATCH_ENDPOINT=
 AUTOCLIP_HARNESS_BATCH_VERSION=v3.2
 AUTOCLIP_HARNESS_BATCH_POLL_SECONDS=10
 AUTOCLIP_HARNESS_BATCH_TIMEOUT_MINUTES=45
+# Provide either the full chat-completions endpoint or the base resource URL.
 AUTOCLIP_HARNESS_LLM_URL=https://your-resource.openai.azure.com
 AUTOCLIP_HARNESS_LLM_KEY=
-AUTOCLIP_HARNESS_LLM_DEPLOYMENT=
-AUTOCLIP_HARNESS_LLM_VERSION=2024-02-15-preview
+AUTOCLIP_HARNESS_LLM_DEPLOYMENT=gpt-4o-mini
+AUTOCLIP_HARNESS_LLM_MODEL=gpt-4o-mini
+AUTOCLIP_HARNESS_LLM_VERSION=2024-07-18
 AUTOCLIP_HARNESS_LANGUAGE=en-US
 AUTOCLIP_HARNESS_MAX_STORIES=5
+

tools/auto-clipper-harness/Program.cs

@@ -1,4 +1,5 @@
 using System.Text;
+using System.Linq;
 using System.Text.RegularExpressions;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
@@ -38,17 +39,25 @@
 var stationOptions = Options.Create(new AutoClipperOptions { StationConfigPath = stationConfigPath });
 var stationConfiguration = new StationConfigurationService(stationOptions, loggerFactory.CreateLogger<StationConfigurationService>());
 var stationProfile = stationConfiguration.GetProfile(stationCode);
-
 var language = args.Length > 1
     ? args[1]
     : Environment.GetEnvironmentVariable("AUTOCLIP_HARNESS_LANGUAGE")
         ?? (!string.IsNullOrWhiteSpace(stationProfile.Transcription.Language) ? stationProfile.Transcription.Language : "en-US");
 var sampleRate = int.TryParse(Environment.GetEnvironmentVariable("AUTOCLIP_HARNESS_SAMPLE_RATE"), out var sr)
     ? sr
     : (stationProfile.Transcription.SampleRate > 0 ? stationProfile.Transcription.SampleRate : 16000);
-
 var audioNormalizer = new AudioNormalizer(loggerFactory.CreateLogger<AudioNormalizer>());
 var workingFile = await audioNormalizer.NormalizeAsync(input, sampleRate);
+var llmBaseUrl = RequireEnv("AUTOCLIP_HARNESS_LLM_URL").Trim();
+var llmDeployment = Environment.GetEnvironmentVariable("AUTOCLIP_HARNESS_LLM_DEPLOYMENT");
+var llmVersion = Environment.GetEnvironmentVariable("AUTOCLIP_HARNESS_LLM_VERSION") ?? "2024-07-18";
+var llmModel = Environment.GetEnvironmentVariable("AUTOCLIP_HARNESS_LLM_MODEL");
+var llmEndpoint = BuildLlmEndpointUri(llmBaseUrl, llmDeployment, llmVersion);
+var defaultModel = string.IsNullOrWhiteSpace(llmModel)
+    ? (!string.IsNullOrWhiteSpace(llmDeployment) ? llmDeployment : "gpt-4o-mini")
+    : llmModel;
+
+
 
 var options = Options.Create(new AutoClipperOptions
 {
@@ -60,21 +69,21 @@
     AzureSpeechBatchTimeoutMinutes = int.TryParse(Environment.GetEnvironmentVariable("AUTOCLIP_HARNESS_BATCH_TIMEOUT_MINUTES"), out var batchTimeoutMinutes) ? batchTimeoutMinutes : 45,
     AzureSpeechStorageConnectionString = RequireEnv("AUTOCLIP_HARNESS_STORAGE_CONNECTION_STRING"),
     AzureSpeechStorageContainer = RequireEnv("AUTOCLIP_HARNESS_STORAGE_CONTAINER"),
-    AzureSpeechStorageSasExpiryMinutes = int.TryParse(Environment.GetEnvironmentVariable("AUTOCLIP_HARNESS_STORAGE_SAS_MINUTES"), out var sasMinutes) ? sasMinutes : 180,
-    LlmApiUrl = new Uri(RequireEnv("AUTOCLIP_HARNESS_LLM_URL")),
+    AzureSpeechStorageSasExpiryMinutes = int.TryParse(Environment.GetEnvironmentVariable("AUTOCLIP_HARNESS_STORAGE_SAS_MINUTES"), out var sasMinutes) ? sasMinutes : 180,    LlmApiUrl = llmEndpoint,
+
     LlmApiKey = RequireEnv("AUTOCLIP_HARNESS_LLM_KEY"),
+
+    LlmDefaultModel = defaultModel,
     LlmPrompt = Environment.GetEnvironmentVariable("AUTOCLIP_HARNESS_PROMPT")
         ?? (string.IsNullOrWhiteSpace(stationProfile.Text.LlmPrompt) ? string.Empty : stationProfile.Text.LlmPrompt),
     MaxStoriesFromClip = int.TryParse(Environment.GetEnvironmentVariable("AUTOCLIP_HARNESS_MAX_STORIES"), out var maxStories) ? maxStories : 5,
     VolumePath = Path.GetDirectoryName(Path.GetFullPath(input)) ?? ".",
     DefaultTranscriptLanguage = stationProfile.Transcription.Language ?? "en-US"
 });
-
 var speechLogger = loggerFactory.CreateLogger<AzureSpeechTranscriptionService>();
 var llmLogger = loggerFactory.CreateLogger<ClipSegmentationService>();
 var speechService = new AzureSpeechTranscriptionService(new HttpClient(), options, speechLogger);
 var llmService = new ClipSegmentationService(new HttpClient(), options, llmLogger);
-
 var transcriptionRequest = new SpeechTranscriptionRequest
 {
     Language = language,
@@ -86,12 +95,10 @@
 Console.WriteLine($"[HARNESS] Transcribing {workingFile} ...");
 var segments = await speechService.TranscribeAsync(workingFile, transcriptionRequest, CancellationToken.None);
 Console.WriteLine($"[HARNESS] Received {segments.Count} transcript segments");
-
 var fullTranscriptBody = BuildTranscriptDocument(segments);
 var fullTranscriptPath = Path.Combine(outputDir, "transcript_full.txt");
 await File.WriteAllTextAsync(fullTranscriptPath, fullTranscriptBody ?? string.Empty);
 Console.WriteLine($"[HARNESS] Full transcript -> {fullTranscriptPath}");
-
 var segmentationSettings = BuildSegmentationSettings(stationProfile);
 Console.WriteLine("[HARNESS] Asking LLM for clip definitions ...");
 var promptDebugPath = Path.Combine(outputDir, "llm_prompt_debug.txt");
@@ -101,7 +108,6 @@
     .OrderBy(c => c.Start)
     .ToArray();
 Console.WriteLine($"[HARNESS] LLM returned {clipDefinitions.Length} clip candidates");
-
 var index = 1;
 foreach (var definition in clipDefinitions)
 {
@@ -292,6 +298,19 @@ static async Task<string> CreateClipFileAsync(string srcFile, string outputDir,
 
 
 
+
+static Uri BuildLlmEndpointUri(string baseUrl, string? deployment, string apiVersion)
+{
+    if (string.IsNullOrWhiteSpace(baseUrl))
+        throw new InvalidOperationException("AUTOCLIP_HARNESS_LLM_URL must be set.");
+    if (baseUrl.Contains("/chat/completions", StringComparison.OrdinalIgnoreCase))
+        return new Uri(baseUrl);
+    if (string.IsNullOrWhiteSpace(deployment))
+        throw new InvalidOperationException("AUTOCLIP_HARNESS_LLM_DEPLOYMENT must be set when using a base LLM URL.");
+    var version = string.IsNullOrWhiteSpace(apiVersion) ? "2024-07-18" : apiVersion;
+    return new Uri($"{baseUrl.TrimEnd('/')}/openai/deployments/{deployment}/chat/completions?api-version={version}");
+}
+
 static ClipSegmentationSettings BuildSegmentationSettings(StationProfile profile)
 {
     return new ClipSegmentationSettings
@@ -373,3 +392,4 @@ static IReadOnlyList<TimestampedTranscript> ExtractTranscriptRange(IReadOnlyList
 
 
 
+