[Text Analytics] Add support for script detection (Azure#32197)

- Added `Script` property to `DetectedLanguage`.
- Added `ScriptKind` enum.

Author: joseharriaga

sdk/textanalytics/Azure.AI.TextAnalytics/CHANGELOG.md

@@ -9,6 +9,8 @@
 - Added `WellKnownFhirVersion` and `HealthcareDocumentType` enums.
 - Added `TextAnalyticsActions.ExtractSummaryActions` to perform extractive summarization in a batch of actions.
 - Added `TextAnalyticsClient.StartExtractSummary` and `StartExtractSummaryAsync` to perform extractive summarization on a collection of documents.
+- Added `Script` property to `DetectedLanguage`.
+- Added `ScriptKind` enum.
 
 ### Breaking Changes
 

sdk/textanalytics/Azure.AI.TextAnalytics/api/Azure.AI.TextAnalytics.netstandard2.0.cs

@@ -224,6 +224,7 @@ public readonly partial struct DetectedLanguage
         public double ConfidenceScore { get { throw null; } }
         public string Iso6391Name { get { throw null; } }
         public string Name { get { throw null; } }
+        public Azure.AI.TextAnalytics.ScriptKind? Script { get { throw null; } }
         public System.Collections.Generic.IReadOnlyCollection<Azure.AI.TextAnalytics.TextAnalyticsWarning> Warnings { get { throw null; } }
     }
     public partial class DetectLanguageInput : Azure.AI.TextAnalytics.TextAnalyticsInput
@@ -925,6 +926,23 @@ internal RecognizePiiEntitiesResultCollection() : base (default(System.Collectio
         public Azure.AI.TextAnalytics.TextDocumentBatchStatistics Statistics { get { throw null; } }
     }
     [System.Runtime.InteropServices.StructLayoutAttribute(System.Runtime.InteropServices.LayoutKind.Sequential)]
+    public readonly partial struct ScriptKind : System.IEquatable<Azure.AI.TextAnalytics.ScriptKind>
+    {
+        private readonly object _dummy;
+        private readonly int _dummyPrimitive;
+        public ScriptKind(string value) { throw null; }
+        public static Azure.AI.TextAnalytics.ScriptKind Latin { get { throw null; } }
+        public bool Equals(Azure.AI.TextAnalytics.ScriptKind other) { throw null; }
+        [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)]
+        public override bool Equals(object obj) { throw null; }
+        [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)]
+        public override int GetHashCode() { throw null; }
+        public static bool operator ==(Azure.AI.TextAnalytics.ScriptKind left, Azure.AI.TextAnalytics.ScriptKind right) { throw null; }
+        public static implicit operator Azure.AI.TextAnalytics.ScriptKind (string value) { throw null; }
+        public static bool operator !=(Azure.AI.TextAnalytics.ScriptKind left, Azure.AI.TextAnalytics.ScriptKind right) { throw null; }
+        public override string ToString() { throw null; }
+    }
+    [System.Runtime.InteropServices.StructLayoutAttribute(System.Runtime.InteropServices.LayoutKind.Sequential)]
     public readonly partial struct SentenceOpinion
     {
         private readonly object _dummy;
@@ -1237,6 +1255,8 @@ public static partial class TextAnalyticsModelFactory
         public static Azure.AI.TextAnalytics.ClassifyDocumentResult ClassifyDocumentResult(string id, Azure.AI.TextAnalytics.TextAnalyticsError error) { throw null; }
         public static Azure.AI.TextAnalytics.ClassifyDocumentResult ClassifyDocumentResult(string id, Azure.AI.TextAnalytics.TextDocumentStatistics statistics, Azure.AI.TextAnalytics.ClassificationCategoryCollection documentClassificationCollection, System.Collections.Generic.IEnumerable<Azure.AI.TextAnalytics.TextAnalyticsWarning> warnings = null) { throw null; }
         public static Azure.AI.TextAnalytics.ClassifyDocumentResultCollection ClassifyDocumentResultCollection(System.Collections.Generic.IEnumerable<Azure.AI.TextAnalytics.ClassifyDocumentResult> classificationResultList, Azure.AI.TextAnalytics.TextDocumentBatchStatistics statistics, string projectName, string deploymentName) { throw null; }
+        public static Azure.AI.TextAnalytics.DetectedLanguage DetectedLanguage(string name, string iso6391Name, double confidenceScore, Azure.AI.TextAnalytics.ScriptKind script, System.Collections.Generic.IList<Azure.AI.TextAnalytics.TextAnalyticsWarning> warnings = null) { throw null; }
+        [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)]
         public static Azure.AI.TextAnalytics.DetectedLanguage DetectedLanguage(string name, string iso6391Name, double confidenceScore, System.Collections.Generic.IList<Azure.AI.TextAnalytics.TextAnalyticsWarning> warnings = null) { throw null; }
         public static Azure.AI.TextAnalytics.DetectLanguageResult DetectLanguageResult(string id, Azure.AI.TextAnalytics.TextAnalyticsError error) { throw null; }
         public static Azure.AI.TextAnalytics.DetectLanguageResult DetectLanguageResult(string id, Azure.AI.TextAnalytics.TextDocumentStatistics statistics, Azure.AI.TextAnalytics.DetectedLanguage detectedLanguage) { throw null; }

sdk/textanalytics/Azure.AI.TextAnalytics/src/DetectedLanguage.cs

@@ -13,15 +13,16 @@ namespace Azure.AI.TextAnalytics
     public readonly struct DetectedLanguage
     {
         internal DetectedLanguage(DetectedLanguageInternal language, IList<TextAnalyticsWarning> warnings)
-            : this(language.Name, language.Iso6391Name, language.ConfidenceScore, warnings)
+            : this(language.Name, language.Iso6391Name, language.ConfidenceScore, language.Script, warnings)
         {
         }
 
-        internal DetectedLanguage(string name, string iso6391Name, double confidenceScore, IList<TextAnalyticsWarning> warnings)
+        internal DetectedLanguage(string name, string iso6391Name, double confidenceScore, ScriptKind? script, IList<TextAnalyticsWarning> warnings)
         {
             Name = name;
             Iso6391Name = iso6391Name;
             ConfidenceScore = confidenceScore;
+            Script = script;
             Warnings = new ReadOnlyCollection<TextAnalyticsWarning>(warnings);
         }
 
@@ -43,6 +44,12 @@ internal DetectedLanguage(string name, string iso6391Name, double confidenceScor
         /// </summary>
         public double ConfidenceScore { get; }
 
+        /// <summary>
+        /// Gets the non-native script of the detected language, if applicable
+        /// (for example, "Latin" in the case of romanized Hindi).
+        /// </summary>
+        public ScriptKind? Script { get; }
+
         /// <summary>
         /// Gets the warnings encountered while processing the document.
         /// </summary>

sdk/textanalytics/Azure.AI.TextAnalytics/src/Generated/Models/DetectedLanguageInternal.Serialization.cs

@@ -0,0 +1,12 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using Azure.Core;
+
+namespace Azure.AI.TextAnalytics
+{
+    [CodeGenModel("ScriptKind")]
+    public readonly partial struct ScriptKind
+    {
+    }
+}

sdk/textanalytics/Azure.AI.TextAnalytics/src/Generated/Models/DetectedLanguageInternal.cs

@@ -216,12 +216,28 @@ public static AnalyzeSentimentResultCollection AnalyzeSentimentResultCollection(
         /// <param name="confidenceScore">Sets the <see cref="DetectedLanguage.ConfidenceScore"/> property.</param>
         /// <param name="warnings">Sets the <see cref="DetectedLanguage.Warnings"/> property.</param>
         /// <returns>A new instance of <see cref="TextAnalytics.DetectedLanguage"/> for mocking purposes.</returns>
+        [EditorBrowsable(EditorBrowsableState.Never)]
         public static DetectedLanguage DetectedLanguage(string name, string iso6391Name, double confidenceScore, IList<TextAnalyticsWarning> warnings = default)
         {
             warnings ??= new List<TextAnalyticsWarning>();
             return new DetectedLanguage(new DetectedLanguageInternal(name, iso6391Name, confidenceScore, default), warnings);
         }
 
+        /// <summary>
+        /// Initializes a new instance of <see cref="TextAnalytics.DetectedLanguage"/> for mocking purposes.
+        /// </summary>
+        /// <param name="name">Sets the <see cref="DetectedLanguage.Name"/> property.</param>
+        /// <param name="iso6391Name">Sets the <see cref="DetectedLanguage.Iso6391Name"/> property.</param>
+        /// <param name="confidenceScore">Sets the <see cref="DetectedLanguage.ConfidenceScore"/> property.</param>
+        /// <param name="script">Sets the <see cref="DetectedLanguage.Script"/> property.</param>
+        /// <param name="warnings">Sets the <see cref="DetectedLanguage.Warnings"/> property.</param>
+        /// <returns>A new instance of <see cref="TextAnalytics.DetectedLanguage"/> for mocking purposes.</returns>
+        public static DetectedLanguage DetectedLanguage(string name, string iso6391Name, double confidenceScore, ScriptKind script, IList<TextAnalyticsWarning> warnings = default)
+        {
+            warnings ??= new List<TextAnalyticsWarning>();
+            return new DetectedLanguage(new DetectedLanguageInternal(name, iso6391Name, confidenceScore, script), warnings);
+        }
+
         /// <summary>
         /// Initializes a new instance of <see cref="TextAnalytics.DetectLanguageResult"/> for mocking purposes.
         /// </summary>

sdk/textanalytics/Azure.AI.TextAnalytics/src/Generated/Models/ScriptKind.cs

@@ -130,7 +130,7 @@ internal static TextAnalyticsOperationStatus ConvertToTextAnalyticsOperationStat
         internal static DetectedLanguage ConvertToDetectedLanguage(Legacy.DocumentLanguage documentLanguage)
         {
             var detected = documentLanguage.DetectedLanguage;
-            return new DetectedLanguage(detected.Name, detected.Iso6391Name, detected.ConfidenceScore, ConvertToWarnings(documentLanguage.Warnings));
+            return new DetectedLanguage(detected.Name, detected.Iso6391Name, detected.ConfidenceScore, default, ConvertToWarnings(documentLanguage.Warnings));
         }
 
         internal static DetectLanguageResultCollection ConvertToDetectLanguageResultCollection(Legacy.LanguageResult results, IDictionary<string, int> idToIndexMap)

sdk/textanalytics/Azure.AI.TextAnalytics/src/ScriptKind.cs

@@ -3,6 +3,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Threading.Tasks;
 using Azure.Core.TestFramework;
 using NUnit.Framework;
@@ -266,11 +267,39 @@ public void DetectLanguageBatchDisableServiceLogsThrows()
             Assert.AreEqual("TextAnalyticsRequestOptions.DisableServiceLogs is not available in API version v3.0. Use service API version v3.1 or newer.", ex.Message);
         }
 
+        [RecordedTest]
+        [ServiceVersion(Min = TextAnalyticsClientOptions.ServiceVersion.V2022_10_01_Preview)]
+        public async Task DetectLanguageBatchWithScriptTest()
+        {
+            TextAnalyticsClient client = GetClient();
+
+            // BUGBUG: The only model version that currently supports script detection is 2022-04-10-preview, which is even older
+            // than the latest GA API version (i.e., 2022-05-01). Ideally, we shouldn't have to pin to such an old version.
+            // See https://github.com/Azure/azure-sdk-for-net/issues/32234.
+            TextAnalyticsRequestOptions options = new TextAnalyticsRequestOptions() { ModelVersion = "2022-04-10-preview" };
+
+            DetectLanguageResultCollection results = await client.DetectLanguageBatchAsync(new List<DetectLanguageInput>() {
+                new("1", "What is your name?"),
+                new("2", "Tumhara naam kya hai?")
+            }, options: options);
+
+            DetectLanguageResult result1 = results.Where(result => result.Id == "1").FirstOrDefault();
+            Assert.IsNotNull(result1);
+            Assert.IsNull(result1.PrimaryLanguage.Script);
+
+            DetectLanguageResult result2 = results.Where(result => result.Id == "2").FirstOrDefault();
+            Assert.IsNotNull(result2);
+            Assert.AreEqual(ScriptKind.Latin, result2.PrimaryLanguage.Script);
+
+            ValidateBatchDocumentsResult(results);
+        }
+
         private void ValidateInDocumenResult(DetectedLanguage language)
         {
             Assert.That(language.Name, Is.Not.Null.And.Not.Empty);
             Assert.That(language.Iso6391Name, Is.Not.Null.And.Not.Empty);
             Assert.GreaterOrEqual(language.ConfidenceScore, 0.0);
+            Assert.LessOrEqual(language.ConfidenceScore, 1.0);
             Assert.IsNotNull(language.Warnings);
         }