MicrosoftHPCServerTools/AzureNodeAggressiveScaler.ps1 at master · istairbn/MicrosoftHPCServerTools · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
<#        .Synopsis        This script automatically scales Azure Nodes

        .Parameter Scheduler
        Determines the scheduler used - defaults to the environment variable                .Example        Azure Node Balancer.ps1        .Notes                 .Link        www.excelian.com#>    [CmdletBinding()]Param(    [Parameter(Mandatory=$False,ValueFromPipelineByPropertyName=$True)]
    [string]
    $Scheduler = $env:CCP_SCHEDULER,

    [Parameter (Mandatory=$False,ValueFromPipelineByPropertyName=$True)]
    [String[]]
    $jobTemplates,

    [Parameter (Mandatory=$False)]
    [ValidateRange(0,[Int]::MaxValue)]
    [Int]
    $InitialNodeGrowth=6,

    [Parameter (Mandatory=$False,ValueFromPipelineByPropertyName=$True)]
    [String[]]
    $ExcludedNodes = @(),

    [Parameter (Mandatory=$False,ValueFromPipelineByPropertyName=$True)]
    [String[]]
    #$NodeGroup = @("AzureNodes","ComputeNodes"),
    $NodeGroup = @("AzureNodes"),

    [Parameter (Mandatory=$False)]
    [ValidateRange(0,[Int]::MaxValue)]
    [Int]
    $NodeGrowth=3,

    [Parameter (Mandatory=$False)]
    [ValidateRange(0,[Int]::MaxValue)]
    [Int]
    $TemplateSwitchNodeGrowth=4,

    [Parameter (Mandatory=$False)]
    [ValidateRange(0,[Int]::MaxValue)]
    [Int]
    $CallQueueThreshold=2000,

    [Parameter (Mandatory=$False)]
    [bool]
    $UndeployAzure=$True,

    [Parameter (Mandatory=$False)]
    [bool]
    $SwitchInternalNodeTemplates=$True,

    [Parameter (Mandatory=$False)]
    [ValidateRange(0,[Int]::MaxValue)]
    [Int]
    $Sleep=30,

    [Parameter(Mandatory=$False,ValueFromPipelineByPropertyName=$True)]
    [int]
    $AcceptableJTUtilisation = 70,

    [Parameter(Mandatory=$False,ValueFromPipelineByPropertyName=$True)]
    [int]
    $AcceptableNodeUtilisation = 30,

    [Parameter(Mandatory=$False,ValueFromPipelineByPropertyName=$True)]
    [int]
    $UnacceptableNodeUtilisation = 20,

    [Parameter (Mandatory=$False)]
    [ValidateRange(0,100)]
    [Int]
    $TemplateUtilisationThreshold = 80,

    [Parameter (Mandatory=$False)]
    [ValidateRange(0,[Int]::MaxValue)]
    [Int]
    $NumOfQueuedJobsToGrowThreshold=1,

    [Parameter (Mandatory=$False)]
    [ValidateRange(0,[Int]::MaxValue)]
    [Int]
    $GridMinsRemainingThreshold= 20,

    [Parameter (Mandatory=$false,ValueFromPipelineByPropertyName=$True)]
    [String[]]
    $NodeTemplates,

    [Parameter (Mandatory=$False,ValueFromPipelineByPropertyName=$True)]
    [String[]]
    $ExcludedNodeTemplates = @(),

    [Parameter (Mandatory=$False,ValueFromPipelineByPropertyName=$True)]
    [String[]]
    $ExcludedGroups = @(),

    [Parameter (Mandatory=$False,ValueFromPipelineByPropertyName=$True)]
    [int]
    $ShrinkThreshold = 6,

    [Parameter (Mandatory=$False)]
    [bool]
    $Logging=$False,

    [Parameter (Mandatory=$False)]
    [String]
    $LogFilePrefix="AzureNodeBalancer"
)

Try{
    Import-Module -Name .\MicrosoftHPCServerTools.psm1  -Force -ErrorAction SilentlyContinue
    Import-Module -Name .\lib\MicrosoftHPCServerTools.psm1  -Force -ErrorAction SilentlyContinue
    Import-Module -Name .\deployed-bundles\HPCHybridAutoScalerApp-2.0\lib\MicrosoftHPCServerTools.psm1 -Force -ErrorAction SilentlyContinue
    Add-PSSnapin Microsoft.hpc
}Catch [System.Exception]{    Write-LogError $Error.ToString()    $Error.Clear()}

$elapsed = [System.Diagnostics.Stopwatch]::StartNew()
Write-Output "Starting Autoscaling"
$PreviousIdleNodeCount = 0
Write-Verbose "Scheduler:$Scheduler
        jobTemplates:$jobTemplates
        InitialNodeGrowth:$InitialNodeGrowth
        ExcludedNodes:$ExcludedNodes
        NodeGroup:$NodeGroup
        NodeGrowth:$NodeGrowth
        CallQueueThreshold:$CallQueueThreshold
        UndeployAzure:$UndeployAzure
        AcceptableJTUtilisation: $AcceptableJTUtilisation
        AcceptableNodeUtilisation: $AcceptableNodeUtilisation
        UnacceptableNodeUtilisation: $UnacceptableNodeUtilisation
        SwitchInternalNodeTemplates:$SwitchInternalNodeTemplates
        Sleep:$Sleep
        NumOfQueuedJobsToGrowThreshold:$NumOfQueuedJobsToGrowThreshold
        GridMinsRemainingThreshold:$GridMinsRemainingThreshold
        NodeTemplates:$NodeTemplates
        ExcludedNodeTemplates:$ExcludedNodeTemplates
        ExcludedGroups:$ExcludedGroups
        ShrinkThreshold:$ShrinkThreshold
        Logging:$Logging
        TemplateUtilisationThreshold:$TemplateUtilisationThreshold
        LogFilePrefix:$LogFilePrefix"

$NodesToRemoveMap = @{}

While($elapsed.Elapsed.Minutes -lt 30){
    Write-Verbose "Elapsed"
    Write-Verbose $Elapsed.Elapsed.Minutes

    $ActiveJobs = Get-HPCClusterActiveJobs -Scheduler $Scheduler -Logging $Logging -LogFilePrefix $LogFilePrefix -jobTemplates $JobTemplates

    If($ActiveJobs -ne $Null){
        $Count = $ActiveJobs.Count
    }
    Else{
        $Count = 0
    }

    Write-LogInfo "Jobs:$Count Scheduler:$Scheduler" -Logging $Logging -LogFilePrefix $LogFilePrefix

    If($Count -ne 0){
        Invoke-HPCClusterAzureAutoScaleUp -Scheduler $Scheduler -InitialNodeGrowth $InitialNodeGrowth `
        -ExcludedNodes $ExcludedNodes -ExcludedNodeTemplates $ExcludedNodeTemplates -NodeGrowth $NodeGrowth `
        -CallQueueThreshold $CallQueueThreshold -GridMinsRemainingThreshold $GridMinsRemainingThreshold -NodeTemplates $NodeTemplates `
        -Logging $Logging -LogFilePrefix $LogFilePrefix -NodeGroup $NodeGroup -TemplateUtilisationThreshold $TemplateUtilisationThreshold -JobQueueThreshold $NumOfQueuedJobsToGrowThreshold
    }

    Else{
        Write-LogInfo "Action:NOTHING No Growth Required"
    }

    $NodesToRemoveMap = Get-HPCClusterNodesToRemoveByUtilisation -Scheduler $Scheduler -ExcludedNodes $ExcludedNodes `
    -NodeGroup $NodeGroup -NodeTemplates $NodeTemplates -ExcludedNodeTemplates $ExcludedNodeTemplates -ExcludedGroups $ExcludedGroups `
    -Logging $Logging -NodesToRemoveMap $NodesToRemoveMap -AcceptableJTUtilisation $AcceptableJTUtilisation -AcceptableNodeUtilisation $AcceptableNodeUtilisation `
    -UnacceptableNodeUtilisation $UnacceptableNodeUtilisation -jobTemplates $jobTemplates

    $NodesToTurnOffline = Get-HPCClusterNodesMappedToRemove -Map $NodesToRemoveMap -Threshold $ShrinkThreshold

    If(@($NodesToTurnOffline).count -ne 0){
        ForEach($Node in $NodesToTurnOffline){
            $NodesToRemoveMap.Remove($Node)
        }
        $idleNodes = @()
        $idleNodes = Get-HpcNode -Name $NodesToTurnOffline
        Write-LogInfo -Logging $Logging -LogFilePrefix $LogFilePrefix -message "ShrinkThreshold Exceeded Nodes:$NodesToTurnOffline"
        Set-HPCClusterNodesUndeployedOrOffline -idleNodes $idleNodes -LogFilePrefix $LogFilePrefix -Logging $Logging -Scheduler $Scheduler
    }
    Else{
        Write-LogInfo -Logging $Logging -LogFilePrefix $LogFilePrefix -message "ShrinkThreshold $ShrinkThreshold not exceeded"
    }
    sleep $Sleep
}
Write-Verbose "Time has elapsed. Restarting"