|
| 1 | +// Type 'Program' can be sealed because it has no subtypes in its containing assembly and is not externally visible |
| 2 | +#pragma warning disable CA1852 |
| 3 | +using System; |
| 4 | +using System.Collections.Generic; |
| 5 | +using System.Diagnostics; |
| 6 | +using System.IO; |
| 7 | +using System.Linq; |
| 8 | +using System.Net.Http; |
| 9 | +using System.Reflection; |
| 10 | +using System.Runtime.CompilerServices; |
| 11 | +using BenchmarkDotNet.Environments; |
| 12 | +using nietras.LargeLanguageModel; |
| 13 | +using nietras.SeparatedValues; |
| 14 | +[assembly: System.Runtime.InteropServices.ComVisible(false)] |
| 15 | + |
| 16 | +Action<string> log = t => { Console.WriteLine(t); Trace.WriteLine(t); }; |
| 17 | + |
| 18 | +var location = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); |
| 19 | +var dataDirectory = Path.Combine(location!, "../../../"); |
| 20 | + |
| 21 | +log($"{Environment.Version} args: {args.Length}"); |
| 22 | + |
| 23 | +var name = args?.Length > 0 ? args[0] : LlmFactory.DefaultName; |
| 24 | +log($"Llm '{name}'"); |
| 25 | +var llm = LlmFactory.NameToCreate[name](); |
| 26 | + |
| 27 | +// Download the model and tokenizer files if they don't exist |
| 28 | +DownloadBinaryFilesIfNotExists(Gpt2.FileNames, Gpt2.RemoteUrl, dataDirectory, log); |
| 29 | + |
| 30 | +// Log to file too for reference |
| 31 | +var logFilePath = Path.Combine(dataDirectory, $"{name}.log"); |
| 32 | +using var logWriter = new StreamWriter(logFilePath); |
| 33 | +Action<string> newLog = t => { log(t); logWriter.WriteLine(t); }; |
| 34 | + |
| 35 | +const int steps = 10; |
| 36 | +var meanStep_ms = Gpt2.VerifyTrain(dataDirectory, llm, steps, newLog); |
| 37 | +var boardName = nameof(Gpt2.VerifyTrain); |
| 38 | +//Gpt2.Infer(dataDirectory, llm, newLog); |
| 39 | +//Gpt2.Train(dataDirectory, llm); |
| 40 | + |
| 41 | +var processorNameInDirectory = GetProcessorName(); |
| 42 | +log(processorNameInDirectory); |
| 43 | + |
| 44 | +var sourceDirectory = GetSourceDirectory(); |
| 45 | +var benchmarksDirectory = $"{sourceDirectory}/../../benchmarks/"; |
| 46 | +var directory = $"{benchmarksDirectory}{processorNameInDirectory}"; |
| 47 | +if (!Directory.Exists(directory)) { Directory.CreateDirectory(directory); } |
| 48 | + |
| 49 | +var filePathBoard = Path.Combine(directory, $"{boardName}-Board.csv"); |
| 50 | +var filePathBoardMarkdown = Path.Combine(directory, $"{boardName}-Board.md"); |
| 51 | + |
| 52 | +var (colNames, sortedBoard) = UpdateBoardCsv(name, meanStep_ms, filePathBoard); |
| 53 | + |
| 54 | +WriteBoardMarkdown(colNames, sortedBoard, filePathBoardMarkdown); |
| 55 | + |
| 56 | +static void WriteBoardMarkdown(string[] colNames, IReadOnlyList<string[]> sortedCols, |
| 57 | + string filePathBoardMarkdown) |
| 58 | +{ |
| 59 | + using var writer = new StreamWriter(filePathBoardMarkdown); |
| 60 | + writer.WriteLine($"|{string.Join("|", colNames.Select(c => c.PadLeft(16)))}|"); |
| 61 | + writer.WriteLine($"|{string.Join("|", colNames.Select(_ => ":").Select(c => c.PadLeft(16, '-')))}|"); |
| 62 | + foreach (var cols in sortedCols) |
| 63 | + { |
| 64 | + writer.WriteLine($"|{string.Join("|", cols.Select(c => c.PadLeft(16)))}|"); |
| 65 | + } |
| 66 | +} |
| 67 | + |
| 68 | +static void DownloadBinaryFilesIfNotExists( |
| 69 | + IReadOnlyList<string> fileNames, Func<string, string> toUrl, |
| 70 | + string dataDirectory, Action<string>? log) |
| 71 | +{ |
| 72 | + foreach (var fileName in fileNames) |
| 73 | + { |
| 74 | + var filePath = Path.Combine(dataDirectory, fileName); |
| 75 | + filePath = Path.GetFullPath(filePath); |
| 76 | + if (!File.Exists(filePath)) |
| 77 | + { |
| 78 | + var url = toUrl(fileName); |
| 79 | + log?.Invoke($"Downloading '{url}' to '{filePath}'"); |
| 80 | + using var client = new HttpClient(); |
| 81 | + // Download the file |
| 82 | + var source = client.GetStreamAsync(url).Result; |
| 83 | + using var destination = new FileStream(filePath, FileMode.Create); |
| 84 | + source.CopyTo(destination); |
| 85 | + } |
| 86 | + } |
| 87 | +} |
| 88 | + |
| 89 | +static string GetSourceDirectory([CallerFilePath] string filePath = "") => |
| 90 | + Path.GetDirectoryName(filePath)!; |
| 91 | + |
| 92 | +static (string[] colNames, string[][] Cols) UpdateBoardCsv( |
| 93 | + string name, double mean_ms, string filePathBoard) |
| 94 | +{ |
| 95 | + const string colNameName = "Name"; |
| 96 | + const string colNameMean = "Mean [ms]"; |
| 97 | + |
| 98 | + string[] colNames = [colNameName, colNameMean]; //, "StdDev [ms]", "Allocated [KB]"]; |
| 99 | + |
| 100 | + var value = (mean_ms, (string[])[name, mean_ms.ToString("F0")]); |
| 101 | + |
| 102 | + var nameToCols = File.Exists(filePathBoard) |
| 103 | + ? ReadNameToCols(filePathBoard, colNameName, colNameMean, colNames) |
| 104 | + : []; |
| 105 | + nameToCols[name] = value; |
| 106 | + |
| 107 | + using var writerBoard = Sep.Writer().ToFile(filePathBoard); |
| 108 | + var sorted = nameToCols.Values.OrderBy(v => v.Mean).ToArray(); |
| 109 | + foreach (var (_, cols) in sorted) |
| 110 | + { |
| 111 | + using var writeRow = writerBoard.NewRow(); |
| 112 | + writeRow[colNames].Set(cols); |
| 113 | + } |
| 114 | + return (colNames, sorted.Select(v => v.Cols).ToArray()); |
| 115 | +} |
| 116 | + |
| 117 | +static Dictionary<string, (double Mean, string[] Cols)> ReadNameToCols( |
| 118 | + string filePath, string colNameName, string colNameMean, string[] colNames) |
| 119 | +{ |
| 120 | + using var reader = Sep |
| 121 | + .Reader(o => o with { Unescape = true, DisableFastFloat = true }) |
| 122 | + .FromFile(filePath); |
| 123 | + return reader.Enumerate(r => (Name: r[colNameName].ToString(), |
| 124 | + Mean: r[colNameMean].Parse<double>(), Cols: r[colNames].ToStringsArray())) |
| 125 | + .ToDictionary(t => t.Name, t => (t.Mean, t.Cols)); |
| 126 | +} |
| 127 | + |
| 128 | +static string GetProcessorName() |
| 129 | +{ |
| 130 | + var cpuInfo = HostEnvironmentInfo.GetCurrent().CpuInfo.Value; |
| 131 | + var processorName = ProcessorBrandStringHelper.Prettify(cpuInfo); |
| 132 | + var processorNameInDirectory = processorName |
| 133 | + .Replace(" Processor", "").Replace(" CPU", "") |
| 134 | + .Replace(" Graphics", "") |
| 135 | + .Replace("/", "").Replace("\\", "") |
| 136 | + .Replace(" ", "."); |
| 137 | + // Remove iGPU info |
| 138 | + var indexOfWith = processorNameInDirectory.LastIndexOf(".w."); |
| 139 | + if (indexOfWith > 0) |
| 140 | + { processorNameInDirectory = processorNameInDirectory.Substring(0, indexOfWith); } |
| 141 | + return processorNameInDirectory; |
| 142 | +} |
0 commit comments