chore: 调整生成逻辑; 调整资源嵌入逻辑;

Author: stratosblue

.gitignore

@@ -1,4 +1,4 @@
-## Ignore Visual Studio temporary files, build results, and
+## Ignore Visual Studio temporary files, build results, and
 ## files generated by popular Visual Studio add-ons.
 ##
 ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore
@@ -360,4 +360,6 @@ MigrationBackup/
 .ionide/
 
 # Fody - auto-generated XML schema
-FodyWeavers.xsd
+FodyWeavers.xsd
+
+/src/LanguageIdentification/Resources/langid-model-data

Directory.Build.props

@@ -1,7 +1,5 @@
 <Project>
   <PropertyGroup>
-    <TargetFrameworks>net8.0;net9.0;netstandard2.0</TargetFrameworks>
-
     <Nullable>enable</Nullable>
     <LangVersion>latest</LangVersion>
     <IsPackable>false</IsPackable>

src/LanguageIdentification/LanguageIdentification.csproj

@@ -1,6 +1,8 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
-    <VersionPrefix>1.1.2</VersionPrefix>
+    <TargetFrameworks>net8.0;net9.0;netstandard2.0</TargetFrameworks>
+
+    <VersionPrefix>1.2.0</VersionPrefix>
     <IsPackable>true</IsPackable>
     <Description>.NET Port of Language Identification Library for langid-java. 移植自langid-java的语言识别库。</Description>
 
@@ -18,26 +20,23 @@
     <GenerateDocumentationFile>true</GenerateDocumentationFile>
 
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+    
+    <IsAotCompatible>true</IsAotCompatible>
+    <InvariantGlobalization>true</InvariantGlobalization>
+    <NoWarn>$(NoWarn);NETSDK1210</NoWarn>
   </PropertyGroup>
 
   <ItemGroup Condition="'$(TargetFramework)' == 'netstandard2.0'">
     <PackageReference Include="System.Memory" Version="4.5.5" />
   </ItemGroup>
 
+  <ItemGroup>
+    <EmbeddedResource Include="Resources\langid-model-data" />
+  </ItemGroup>
+  
   <ItemGroup>
     <None Include="..\..\readme.md" Link="readme.md" Pack="true" PackagePath="/" />
 
-    <Compile Update="Resource.Designer.cs">
-      <DesignTime>True</DesignTime>
-      <AutoGen>True</AutoGen>
-      <DependentUpon>Resource.resx</DependentUpon>
-    </Compile>
-
-    <EmbeddedResource Update="Resource.resx">
-      <Generator>ResXFileCodeGenerator</Generator>
-      <LastGenOutput>Resource.Designer.cs</LastGenOutput>
-    </EmbeddedResource>
-
     <AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleToAttribute">
       <_Parameter1>LanguageIdentification.Test</_Parameter1>
     </AssemblyAttribute>
@@ -56,4 +55,12 @@
   <PropertyGroup Condition="'$(GITHUB_ACTIONS)' == 'true'">
     <ContinuousIntegrationBuild>true</ContinuousIntegrationBuild>
   </PropertyGroup>
+
+  <!--生成文件-->
+  <Target Name="_LanguageIdentificationDataGenerate" BeforeTargets="DispatchToInnerBuilds">
+    <Exec Command="dotnet run --project ../../tool/ModelDataGenerator/ModelDataGenerator.csproj -c Release -e OUTPUT_DIR=&quot;$(MSBuildThisFileDirectory)Resources&quot;" ContinueOnError="true">
+      <Output TaskParameter="ExitCode" PropertyName="LMDGenerateErrorCode" />
+    </Exec>
+    <Error Condition="'$(LMDGenerateErrorCode)' != '0'" Text="langid-model-data generate failed." />
+  </Target>
 </Project>

src/LanguageIdentification/LanguageIdentificationModel.cs

@@ -5,6 +5,7 @@
 using System.IO;
 using System.IO.Compression;
 using System.Linq;
+using System.Reflection;
 using System.Runtime.Serialization;
 using System.Text;
 
@@ -64,13 +65,15 @@ public static LanguageIdentificationModel Default
     {
         get
         {
-            if (s_defaultModelReference.TryGetTarget(out var defaultModel))
+            if (s_defaultModelReference.TryGetTarget(out var defaultModel)
+                && defaultModel is not null)
             {
                 return defaultModel;
             }
             lock (s_defaultModelReference)
             {
-                if (s_defaultModelReference.TryGetTarget(out defaultModel))
+                if (s_defaultModelReference.TryGetTarget(out defaultModel)
+                    && defaultModel is not null)
                 {
                     return defaultModel;
                 }
@@ -289,8 +292,13 @@ public bool Equals(T[]? x, T[]? y)
                    && Enumerable.SequenceEqual(x, y);
         }
 
-        public int GetHashCode(T[] obj)
+        /// <inheritdoc/>
+        public int GetHashCode(T[]? obj)
         {
+            if (obj is null)
+            {
+                return 0;
+            }
             var result = int.MaxValue;
             foreach (var item in obj)
             {
@@ -314,8 +322,10 @@ public int GetHashCode(T[] obj)
     /// <returns></returns>
     private static LanguageIdentificationModel InternalLoadDefaultModel()
     {
-        using var stream = new MemoryStream(Resource.ModelData);
-        using var gzipStream = new GZipStream(stream, CompressionMode.Decompress);
+        using var modelDataStream = Assembly.GetExecutingAssembly().GetManifestResourceStream("LanguageIdentification.Resources.langid-model-data")
+                                    ?? throw new InvalidOperationException("Can not load langid-model-data in embedded resource. The referenced library is invalid.");
+
+        using var gzipStream = new GZipStream(modelDataStream, CompressionMode.Decompress);
         using var bufferedStream = new BufferedStream(gzipStream);
 
         return Deserialize(bufferedStream);

src/LanguageIdentification/Resource.Designer.cs

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFrameworks>net9.0</TargetFrameworks>
+    <TargetFramework>net9.0</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>

src/LanguageIdentification/Resource.resx

@@ -1,7 +1,8 @@
 using System;
+using System.Diagnostics;
 using System.IO;
 using System.IO.Compression;
-
+using System.Reflection;
 using LanguageIdentification;
 
 #pragma warning disable CS8602 // 解引用可能出现空引用。
@@ -14,7 +15,11 @@ internal class Program
 
     private static void Main(string[] args)
     {
-        using var sourceZipStream = File.OpenRead("langid.zip");
+        Console.WriteLine("Start Generate langid-model-data");
+
+        var stopwatch = Stopwatch.StartNew();
+
+        using var sourceZipStream = File.OpenRead(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "langid.zip"));
         using var zipArchive = new ZipArchive(sourceZipStream, ZipArchiveMode.Read);
         var zipArchiveEntry = zipArchive.GetEntry("langid.json");
         using var sourceStream = zipArchiveEntry.Open();
@@ -29,7 +34,17 @@ private static void Main(string[] args)
 
         memoryStream.Seek(0, SeekOrigin.Begin);
 
-        using var targetStream = File.OpenWrite("langid-model-data");
+        var outputDir = Environment.GetEnvironmentVariable("OUTPUT_DIR");
+        if (!string.IsNullOrEmpty(outputDir))
+        {
+            if (!Directory.Exists(outputDir))
+            {
+                Directory.CreateDirectory(outputDir);
+            }
+        }
+
+        using var targetStream = File.OpenWrite(Path.Combine(outputDir ?? string.Empty, "langid-model-data"));
+        targetStream.SetLength(0);
         using var targetGzipStream = new GZipStream(targetStream, CompressionLevel.SmallestSize);
 
         memoryStream.CopyTo(targetGzipStream);
@@ -46,7 +61,9 @@ private static void Main(string[] args)
             throw new Exception("SerializeObject not equals SourceObject generate fail.");
         }
 
-        Console.WriteLine("Generation complete");
+        stopwatch.Stop();
+
+        Console.WriteLine($"langid-model-data Generation complete. During {stopwatch.Elapsed}");
     }
 
     #endregion Private 方法