From 65b19def1422873d29a3ddb7bb55c6dd9f880923 Mon Sep 17 00:00:00 2001 From: Nikhitha Malkapuram Date: Wed, 30 Jul 2025 21:20:10 +0530 Subject: [PATCH 1/4] Arm support for hammerdb profile where server runs on arm and client on x64 as hammerdb doesn't itself support arm --- .../HammerDB/HammerDBClientExecutor.cs | 33 +++-- .../HammerDB/HammerDBExecutor.cs | 30 +++- .../PostgreSQLServerConfiguration.cs | 29 +++- .../PERF-POSTGRESQL-HAMMERDB-TPCC-ARM64.json | 133 ++++++++++++++++++ 4 files changed, 208 insertions(+), 17 deletions(-) create mode 100644 src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-HAMMERDB-TPCC-ARM64.json diff --git a/src/VirtualClient/VirtualClient.Actions/HammerDB/HammerDBClientExecutor.cs b/src/VirtualClient/VirtualClient.Actions/HammerDB/HammerDBClientExecutor.cs index 2ef87939b7..46d7117903 100644 --- a/src/VirtualClient/VirtualClient.Actions/HammerDB/HammerDBClientExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/HammerDB/HammerDBClientExecutor.cs @@ -171,22 +171,29 @@ private Task ExecuteWorkloadAsync(EventContext telemetryContext, CancellationTok { using (BackgroundOperations profiling = BackgroundOperations.BeginProfiling(this, cancellationToken)) { - string command = "python3"; - string script = $"{this.HammerDBPackagePath}/run-workload.py"; - - using (IProcessProxy process = await this.ExecuteCommandAsync( - command, - script + " " + this.hammerDBExecutionArguments, - this.HammerDBPackagePath, - telemetryContext, - cancellationToken)) + if (string.Equals(this.Action, "PopulateDatabase", StringComparison.OrdinalIgnoreCase)) { - if (!cancellationToken.IsCancellationRequested) + await this.PrepareSQLDatabase(telemetryContext, cancellationToken); + } + else + { + string command = "python3"; + string script = $"{this.HammerDBPackagePath}/run-workload.py"; + + using (IProcessProxy process = await this.ExecuteCommandAsync( + command, + script + " " + this.hammerDBExecutionArguments, + this.HammerDBPackagePath, + telemetryContext, + cancellationToken)) { - await this.LogProcessDetailsAsync(process, telemetryContext, "HammerDB", logToFile: true); - process.ThrowIfErrored(process.StandardError.ToString(), ErrorReason.WorkloadFailed); + if (!cancellationToken.IsCancellationRequested) + { + await this.LogProcessDetailsAsync(process, telemetryContext, "HammerDB", logToFile: true); + process.ThrowIfErrored(process.StandardError.ToString(), ErrorReason.WorkloadFailed); - this.CaptureMetrics(process, telemetryContext, cancellationToken); + this.CaptureMetrics(process, telemetryContext, cancellationToken); + } } } } diff --git a/src/VirtualClient/VirtualClient.Actions/HammerDB/HammerDBExecutor.cs b/src/VirtualClient/VirtualClient.Actions/HammerDB/HammerDBExecutor.cs index be86aa7046..e6c0a1c603 100644 --- a/src/VirtualClient/VirtualClient.Actions/HammerDB/HammerDBExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/HammerDB/HammerDBExecutor.cs @@ -160,6 +160,18 @@ public string SQLServer } } + /// + /// The action to perform. + /// + public string Action + { + get + { + this.Parameters.TryGetValue(nameof(HammerDBClientExecutor.Action), out IConvertible action); + return action?.ToString(); + } + } + /// /// Client used to communicate with the hosted instance of the /// Virtual Client API at server side. @@ -294,8 +306,11 @@ protected async Task InitializeExecutablesAsync(EventContext telemetryContext, C await this.stateManager.SaveStateAsync(nameof(HammerDBState), state, cancellationToken); } - - private async Task PrepareSQLDatabase(EventContext telemetryContext, CancellationToken cancellationToken) + + /// + /// Prepares the SQL database by executing the createDB TCL script. + /// + protected async Task PrepareSQLDatabase(EventContext telemetryContext, CancellationToken cancellationToken) { string command = "python3"; @@ -353,7 +368,16 @@ private static Task OpenFirewallPortsAsync(int port, IFirewallManager firewallMa private async Task CheckDistroSupportAsync(CancellationToken cancellationToken) { - if (this.CpuArchitecture == System.Runtime.InteropServices.Architecture.X64) + // Check architecture support based on executor type and role + // HammerDBServerExecutor in Server role: Supports both X64 and ARM64 (PostgreSQL server can run on both) + // All other scenarios: Supports only X64 (HammerDB client operations don't support ARM64) + bool isServerExecutorInServerRole = this.IsInRole(ClientRole.Server) && this.GetType() == typeof(HammerDBServerExecutor); + bool isX64 = this.CpuArchitecture == System.Runtime.InteropServices.Architecture.X64; + bool isArm64 = this.CpuArchitecture == System.Runtime.InteropServices.Architecture.Arm64; + + bool isArchitectureSupported = isX64 || (isServerExecutorInServerRole && isArm64); + + if (isArchitectureSupported) { if (this.Platform == PlatformID.Unix) { diff --git a/src/VirtualClient/VirtualClient.Dependencies/PostgreSQLServer/PostgreSQLServerConfiguration.cs b/src/VirtualClient/VirtualClient.Dependencies/PostgreSQLServer/PostgreSQLServerConfiguration.cs index cd00a1affe..1f86d45090 100644 --- a/src/VirtualClient/VirtualClient.Dependencies/PostgreSQLServer/PostgreSQLServerConfiguration.cs +++ b/src/VirtualClient/VirtualClient.Dependencies/PostgreSQLServer/PostgreSQLServerConfiguration.cs @@ -124,7 +124,23 @@ public string SharedMemoryBuffer return sharedMemoryBuffer?.ToString(); } } + + /// + /// stripedisk mount point. + /// + public string StripeDiskMountPoint + { + get + { + if (this.Parameters.TryGetValue(nameof(this.StripeDiskMountPoint), out IConvertible stripediskmountpoint) && stripediskmountpoint != null) + { + return stripediskmountpoint.ToString(); + } + return string.Empty; + } + } + /// /// Retrieves the interface to interacting with the underlying system. /// @@ -180,9 +196,20 @@ private async Task ConfigurePostgreSQLServerAsync(EventContext telemetryContext, string serverIp = (this.GetLayoutClientInstances(ClientRole.Server, false) ?? Enumerable.Empty()) .FirstOrDefault()?.IPAddress ?? IPAddress.Loopback.ToString(); + + string innoDbDirs = !string.IsNullOrEmpty(this.StripeDiskMountPoint) ? this.StripeDiskMountPoint : await this.GetPostgreSQLInnodbDirectoriesAsync(cancellationToken); string arguments = $"{this.packageDirectory}/configure-server.py --dbName {this.DatabaseName} --serverIp {serverIp} --password {this.SuperUserPassword} --port {this.Port} --inMemory {this.SharedMemoryBuffer}"; + if (innoDbDirs != null) + { + if (innoDbDirs.Split(';', StringSplitOptions.RemoveEmptyEntries).Length == 1) + { + string cleanDirectory = innoDbDirs.TrimEnd(';'); + arguments += $" --innodbDirectory {cleanDirectory}"; + } + } + using (IProcessProxy process = await this.ExecuteCommandAsync( "python3", arguments, @@ -216,7 +243,7 @@ private async Task SetupPostgreSQLDatabaseAsync(EventContext telemetryContext, C } } } - + private async Task DistributePostgreSQLDatabaseAsync(EventContext telemetryContext, CancellationToken cancellationToken) { string innoDbDirs = await this.GetPostgreSQLInnodbDirectoriesAsync(cancellationToken); diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-HAMMERDB-TPCC-ARM64.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-HAMMERDB-TPCC-ARM64.json new file mode 100644 index 0000000000..7f394b54b0 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-HAMMERDB-TPCC-ARM64.json @@ -0,0 +1,133 @@ +{ + "Description": "HammerDB PostgrSQL TPCC Database Server Performance Workload", + "MinimumExecutionInterval": "00:01:00", + "Metadata": { + "RecommendedMinimumExecutionTime": "04:00:00", + "SupportedPlatforms": "linux-x64", + "SupportedOperatingSystems": "Debian,Ubuntu" + }, + "Parameters": { + "DatabaseName": "hammerdb_tpcc", + "DiskFilter": "osdisk:false&sizegreaterthan:256g", + "Port": "5432", + "VirtualUsers": "{calculate({LogicalCoreCount})}", + "WarehouseCount": "{calculate({SystemMemoryMegabytes} * 15 / 800)}", + "SharedMemoryBuffer": "{calculate({SystemMemoryMegabytes} * 85 / 100)}" + }, + "Actions": [ + { + "Type": "HammerDBServerExecutor", + "Parameters": { + "Scenario": "ExecuteServer", + "PackageName": "hammerdb", + "DatabaseName": "$.Parameters.DatabaseName", + "Workload": "tpcc", + "SQLServer": "postgresql", + "Port": "$.Parameters.Port", + "VirtualUsers": "$.Parameters.VirtualUsers", + "WarehouseCount": "$.Parameters.WarehouseCount", + "Role": "Server" + } + }, + { + "Type": "HammerDBClientExecutor", + "Parameters": { + "Scenario": "PopulatePostgreSQLDatabase", + "Action": "PopulateDatabase", + "DatabaseName": "$.Parameters.DatabaseName", + "Workload": "tpcc", + "SQLServer": "postgresql", + "PackageName": "hammerdb", + "VirtualUsers": "$.Parameters.VirtualUsers", + "WarehouseCount": "$.Parameters.WarehouseCount", + "Port": "$.Parameters.Port", + "Role": "Client" + } + }, + { + "Type": "HammerDBClientExecutor", + "Parameters": { + "Scenario": "tpcc", + "Action": "RunTransactions", + "Workload": "tpcc", + "SQLServer": "postgresql", + "PackageName": "hammerdb", + "DatabaseName": "$.Parameters.DatabaseName", + "Port": "$.Parameters.Port", + "VirtualUsers": "$.Parameters.VirtualUsers", + "WarehouseCount": "$.Parameters.WarehouseCount", + "Role": "Client" + } + } + ], + "Dependencies": [ + { + "Type": "FormatDisks", + "Parameters": { + "Scenario": "FormatDisks", + "Role": "Server" + } + }, + { + "Type": "MountDisks", + "Parameters": { + "Scenario": "CreateMountPoints", + "Role": "Server" + } + }, + { + "Type": "LinuxPackageInstallation", + "Parameters": { + "Scenario": "InstallLinuxPackages", + "Packages": "python3" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "DownloadPostgreSQLPackage", + "BlobContainer": "packages", + "BlobName": "postgresql.14.0.0.rev2.zip", + "PackageName": "postgresql", + "Extract": true + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "DownloadHammerDBPackage", + "BlobContainer": "packages", + "BlobName": "hammerdb.4.7.0.rev1.zip", + "PackageName": "hammerdb", + "Extract": true + } + }, + { + "Type": "PostgreSQLServerInstallation", + "Parameters": { + "Scenario": "InstallPostgreSQLServer", + "Action": "InstallServer", + "PackageName": "postgresql" + } + }, + { + "Type": "PostgreSQLServerConfiguration", + "Parameters": { + "Scenario": "ConfigurePostgreSQLServer", + "Action": "ConfigureServer", + "PackageName": "postgresql", + "Role": "Server", + "DatabaseName": "$.Parameters.DatabaseName", + "Port": "$.Parameters.Port", + "SharedMemoryBuffer": "$.Parameters.SharedMemoryBuffer" + } + }, + { + "Type": "ApiServer", + "Parameters": { + "Scenario": "StartAPIServer", + "Role": "Server" + } + } + ] +} \ No newline at end of file From 5c62190b62156be66d60b6cf5244b2a517ab71f4 Mon Sep 17 00:00:00 2001 From: Nikhitha Malkapuram Date: Mon, 4 Aug 2025 17:20:18 +0530 Subject: [PATCH 2/4] updated postgresql configuration logic --- .../PostgreSQLServerConfiguration.cs | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/src/VirtualClient/VirtualClient.Dependencies/PostgreSQLServer/PostgreSQLServerConfiguration.cs b/src/VirtualClient/VirtualClient.Dependencies/PostgreSQLServer/PostgreSQLServerConfiguration.cs index 1f86d45090..91e9196003 100644 --- a/src/VirtualClient/VirtualClient.Dependencies/PostgreSQLServer/PostgreSQLServerConfiguration.cs +++ b/src/VirtualClient/VirtualClient.Dependencies/PostgreSQLServer/PostgreSQLServerConfiguration.cs @@ -197,16 +197,27 @@ private async Task ConfigurePostgreSQLServerAsync(EventContext telemetryContext, .FirstOrDefault()?.IPAddress ?? IPAddress.Loopback.ToString(); - string innoDbDirs = !string.IsNullOrEmpty(this.StripeDiskMountPoint) ? this.StripeDiskMountPoint : await this.GetPostgreSQLInnodbDirectoriesAsync(cancellationToken); - string arguments = $"{this.packageDirectory}/configure-server.py --dbName {this.DatabaseName} --serverIp {serverIp} --password {this.SuperUserPassword} --port {this.Port} --inMemory {this.SharedMemoryBuffer}"; - if (innoDbDirs != null) + string innoDbDirs = string.Empty; + if (this.Parameters.ContainsKey(nameof(this.DiskFilter)) || !string.IsNullOrEmpty(this.StripeDiskMountPoint)) { - if (innoDbDirs.Split(';', StringSplitOptions.RemoveEmptyEntries).Length == 1) + innoDbDirs = this.Parameters.ContainsKey(nameof(this.DiskFilter)) ? await this.GetPostgreSQLInnodbDirectoriesAsync(cancellationToken) : string.Empty; + innoDbDirs = !string.IsNullOrEmpty(this.StripeDiskMountPoint) ? this.StripeDiskMountPoint : innoDbDirs; + + if (!string.IsNullOrEmpty(innoDbDirs)) { - string cleanDirectory = innoDbDirs.TrimEnd(';'); - arguments += $" --innodbDirectory {cleanDirectory}"; + if (innoDbDirs.Split(';', StringSplitOptions.RemoveEmptyEntries).Length == 1) + { + string cleanDirectory = innoDbDirs.TrimEnd(';'); + arguments += $" --innodbDirectory {cleanDirectory}"; + } + else + { + throw new WorkloadException( + "Multiple InnoDB directories detected. Please specify a single directory for PostgreSQL configuration when using the DiskFilter parameter explicitly for PostgreSQLServerConfiguration.", + ErrorReason.InvalidProfileDefinition); + } } } From b1b9cd8ba1cbc1bae021e12629bcd59fd0223af6 Mon Sep 17 00:00:00 2001 From: Nikhitha Malkapuram Date: Mon, 4 Aug 2025 21:25:35 +0530 Subject: [PATCH 3/4] a fix for arm64 hammerdb --- .../HammerDB/HammerDBExecutor.cs | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/VirtualClient/VirtualClient.Actions/HammerDB/HammerDBExecutor.cs b/src/VirtualClient/VirtualClient.Actions/HammerDB/HammerDBExecutor.cs index e6c0a1c603..9bff07051f 100644 --- a/src/VirtualClient/VirtualClient.Actions/HammerDB/HammerDBExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/HammerDB/HammerDBExecutor.cs @@ -229,21 +229,25 @@ await this.CheckDistroSupportAsync(cancellationToken) .ConfigureAwait(false); await HammerDBExecutor.OpenFirewallPortsAsync(this.Port, this.SystemManager.FirewallManager, cancellationToken); - - DependencyPath hammerDBPackage = await this.GetPlatformSpecificPackageAsync(this.PackageName, cancellationToken); - this.HammerDBPackagePath = hammerDBPackage.Path; - - await this.InitializeExecutablesAsync(telemetryContext, cancellationToken); - + this.InitializeApiClients(cancellationToken); - await this.Logger.LogMessageAsync($"{this.TypeName}.ConfigureHammerDBFile", telemetryContext.Clone(), async () => + // Only load HammerDB package for x64 executors. + if (this.CpuArchitecture == System.Runtime.InteropServices.Architecture.X64) { - if (!cancellationToken.IsCancellationRequested) + DependencyPath hammerDBPackage = await this.GetPlatformSpecificPackageAsync(this.PackageName, cancellationToken); + this.HammerDBPackagePath = hammerDBPackage.Path; + + await this.InitializeExecutablesAsync(telemetryContext, cancellationToken); + + await this.Logger.LogMessageAsync($"{this.TypeName}.ConfigureHammerDBFile", telemetryContext.Clone(), async () => { - await this.ConfigureCreateHammerDBFile(telemetryContext, cancellationToken); - } - }); + if (!cancellationToken.IsCancellationRequested) + { + await this.ConfigureCreateHammerDBFile(telemetryContext, cancellationToken); + } + }); + } if (this.IsMultiRoleLayout()) { @@ -252,7 +256,7 @@ await this.Logger.LogMessageAsync($"{this.TypeName}.ConfigureHammerDBFile", tele this.ThrowIfLayoutClientIPAddressNotFound(layoutIPAddress); this.ThrowIfRoleNotSupported(clientInstance.Role); - } + } } /// From 889af8531296248562a80c36a83043997ae025b0 Mon Sep 17 00:00:00 2001 From: Nikhitha Malkapuram Date: Tue, 5 Aug 2025 00:38:04 +0530 Subject: [PATCH 4/4] updating postgresql and hammerdb versions --- .../PERF-POSTGRESQL-HAMMERDB-TPCC-ARM64.json | 133 ------------------ .../PERF-POSTGRESQL-HAMMERDB-TPCC.json | 4 +- 2 files changed, 2 insertions(+), 135 deletions(-) delete mode 100644 src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-HAMMERDB-TPCC-ARM64.json diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-HAMMERDB-TPCC-ARM64.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-HAMMERDB-TPCC-ARM64.json deleted file mode 100644 index 7f394b54b0..0000000000 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-HAMMERDB-TPCC-ARM64.json +++ /dev/null @@ -1,133 +0,0 @@ -{ - "Description": "HammerDB PostgrSQL TPCC Database Server Performance Workload", - "MinimumExecutionInterval": "00:01:00", - "Metadata": { - "RecommendedMinimumExecutionTime": "04:00:00", - "SupportedPlatforms": "linux-x64", - "SupportedOperatingSystems": "Debian,Ubuntu" - }, - "Parameters": { - "DatabaseName": "hammerdb_tpcc", - "DiskFilter": "osdisk:false&sizegreaterthan:256g", - "Port": "5432", - "VirtualUsers": "{calculate({LogicalCoreCount})}", - "WarehouseCount": "{calculate({SystemMemoryMegabytes} * 15 / 800)}", - "SharedMemoryBuffer": "{calculate({SystemMemoryMegabytes} * 85 / 100)}" - }, - "Actions": [ - { - "Type": "HammerDBServerExecutor", - "Parameters": { - "Scenario": "ExecuteServer", - "PackageName": "hammerdb", - "DatabaseName": "$.Parameters.DatabaseName", - "Workload": "tpcc", - "SQLServer": "postgresql", - "Port": "$.Parameters.Port", - "VirtualUsers": "$.Parameters.VirtualUsers", - "WarehouseCount": "$.Parameters.WarehouseCount", - "Role": "Server" - } - }, - { - "Type": "HammerDBClientExecutor", - "Parameters": { - "Scenario": "PopulatePostgreSQLDatabase", - "Action": "PopulateDatabase", - "DatabaseName": "$.Parameters.DatabaseName", - "Workload": "tpcc", - "SQLServer": "postgresql", - "PackageName": "hammerdb", - "VirtualUsers": "$.Parameters.VirtualUsers", - "WarehouseCount": "$.Parameters.WarehouseCount", - "Port": "$.Parameters.Port", - "Role": "Client" - } - }, - { - "Type": "HammerDBClientExecutor", - "Parameters": { - "Scenario": "tpcc", - "Action": "RunTransactions", - "Workload": "tpcc", - "SQLServer": "postgresql", - "PackageName": "hammerdb", - "DatabaseName": "$.Parameters.DatabaseName", - "Port": "$.Parameters.Port", - "VirtualUsers": "$.Parameters.VirtualUsers", - "WarehouseCount": "$.Parameters.WarehouseCount", - "Role": "Client" - } - } - ], - "Dependencies": [ - { - "Type": "FormatDisks", - "Parameters": { - "Scenario": "FormatDisks", - "Role": "Server" - } - }, - { - "Type": "MountDisks", - "Parameters": { - "Scenario": "CreateMountPoints", - "Role": "Server" - } - }, - { - "Type": "LinuxPackageInstallation", - "Parameters": { - "Scenario": "InstallLinuxPackages", - "Packages": "python3" - } - }, - { - "Type": "DependencyPackageInstallation", - "Parameters": { - "Scenario": "DownloadPostgreSQLPackage", - "BlobContainer": "packages", - "BlobName": "postgresql.14.0.0.rev2.zip", - "PackageName": "postgresql", - "Extract": true - } - }, - { - "Type": "DependencyPackageInstallation", - "Parameters": { - "Scenario": "DownloadHammerDBPackage", - "BlobContainer": "packages", - "BlobName": "hammerdb.4.7.0.rev1.zip", - "PackageName": "hammerdb", - "Extract": true - } - }, - { - "Type": "PostgreSQLServerInstallation", - "Parameters": { - "Scenario": "InstallPostgreSQLServer", - "Action": "InstallServer", - "PackageName": "postgresql" - } - }, - { - "Type": "PostgreSQLServerConfiguration", - "Parameters": { - "Scenario": "ConfigurePostgreSQLServer", - "Action": "ConfigureServer", - "PackageName": "postgresql", - "Role": "Server", - "DatabaseName": "$.Parameters.DatabaseName", - "Port": "$.Parameters.Port", - "SharedMemoryBuffer": "$.Parameters.SharedMemoryBuffer" - } - }, - { - "Type": "ApiServer", - "Parameters": { - "Scenario": "StartAPIServer", - "Role": "Server" - } - } - ] -} \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-HAMMERDB-TPCC.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-HAMMERDB-TPCC.json index 616dc6d4a8..40e1541aec 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-HAMMERDB-TPCC.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-HAMMERDB-TPCC.json @@ -71,7 +71,7 @@ "Parameters": { "Scenario": "DownloadPostgreSQLPackage", "BlobContainer": "packages", - "BlobName": "postgresql.14.0.0.rev2.zip", + "BlobName": "postgresql.16.0.0.rev2.zip", "PackageName": "postgresql", "Extract": true } @@ -81,7 +81,7 @@ "Parameters": { "Scenario": "DownloadHammerDBPackage", "BlobContainer": "packages", - "BlobName": "hammerdb.4.7.0.rev1.zip", + "BlobName": "hammerdb.4.12.0.rev2.zip", "PackageName": "hammerdb", "Extract": true }