Skip to content

Commit 23fdc35

Browse files
laozclaozc
committed
Use WMI to implement iSCSI API to reduce PowerShell overhead
1 parent 0c25f10 commit 23fdc35

File tree

2 files changed

+165
-91
lines changed

2 files changed

+165
-91
lines changed

‎integrationtests/iscsi_ps_scripts.go‎

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -42,14 +42,14 @@ $ProgressPreference = "SilentlyContinue"
4242
$targetName = "%s"
4343
4444
# Get local IPv4 (e.g. 10.30.1.15, not 127.0.0.1)
45-
$address = $(Get-NetIPAddress | Where-Object { $_.InterfaceAlias -eq "Ethernet" -and $_.AddressFamily -eq "IPv4" }).IPAddress
45+
$address = $(Get-NetIPAddress | Where-Object { $_.InterfaceAlias -eq "%s" -and $_.AddressFamily -eq "IPv4" }).IPAddress
4646
4747
# Create virtual disk in RAM
48-
New-IscsiVirtualDisk -Path "ramdisk:scratch-${targetName}.vhdx" -Size 100MB | Out-Null
48+
New-IscsiVirtualDisk -Path "ramdisk:scratch-${targetName}.vhdx" -Size 100MB -ComputerName $env:computername | Out-Null
4949
5050
# Create a target that allows all initiator IQNs and map a disk to the new target
51-
$target = New-IscsiServerTarget -TargetName $targetName -InitiatorIds @("Iqn:*")
52-
Add-IscsiVirtualDiskTargetMapping -TargetName $targetName -DevicePath "ramdisk:scratch-${targetName}.vhdx" | Out-Null
51+
$target = New-IscsiServerTarget -TargetName $targetName -InitiatorIds @("Iqn:*") -ComputerName $env:computername
52+
Add-IscsiVirtualDiskTargetMapping -TargetName $targetName -DevicePath "ramdisk:scratch-${targetName}.vhdx" -ComputerName $env:computername | Out-Null
5353
5454
$output = @{
5555
"iqn" = "$($target.TargetIqn)"
@@ -68,7 +68,7 @@ $username = "%s"
6868
$password = "%s"
6969
$securestring = ConvertTo-SecureString -String $password -AsPlainText -Force
7070
$chap = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList ($username, $securestring)
71-
Set-IscsiServerTarget -TargetName $targetName -EnableChap $true -Chap $chap
71+
Set-IscsiServerTarget -TargetName $targetName -EnableChap $true -Chap $chap -ComputerName $env:computername
7272
`
7373

7474
func setChap(targetName string, username string, password string) error {
@@ -92,7 +92,7 @@ $securestring = ConvertTo-SecureString -String $password -AsPlainText -Force
9292
9393
# Windows initiator does not uses the username for mutual authentication
9494
$chap = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList ($username, $securestring)
95-
Set-IscsiServerTarget -TargetName $targetName -EnableReverseChap $true -ReverseChap $chap
95+
Set-IscsiServerTarget -TargetName $targetName -EnableReverseChap $true -ReverseChap $chap -ComputerName $env:computername
9696
`
9797

9898
func setReverseChap(targetName string, password string) error {
@@ -131,8 +131,8 @@ Get-IscsiTarget | Disconnect-IscsiTarget -Confirm:$false
131131
Get-IscsiTargetPortal | Remove-IscsiTargetPortal -confirm:$false
132132
133133
# Clean target
134-
Get-IscsiServerTarget | Remove-IscsiServerTarget
135-
Get-IscsiVirtualDisk | Remove-IscsiVirtualDisk
134+
Get-IscsiServerTarget -ComputerName $env:computername | Remove-IscsiServerTarget
135+
Get-IscsiVirtualDisk -ComputerName $env:computername | Remove-IscsiVirtualDisk
136136
137137
# Stop iSCSI initiator
138138
Get-Service "MsiSCSI" | Stop-Service
@@ -173,7 +173,12 @@ func runPowershellScript(script string) (string, error) {
173173
}
174174

175175
func setupEnv(targetName string) (*IscsiSetupConfig, error) {
176-
script := fmt.Sprintf(IscsiEnvironmentSetupScript, targetName)
176+
ethernetName := "Ethernet"
177+
if val, ok := os.LookupEnv("ETHERNET_NAME"); ok {
178+
ethernetName = val
179+
}
180+
181+
script := fmt.Sprintf(IscsiEnvironmentSetupScript, targetName, ethernetName)
177182
out, err := runPowershellScript(script)
178183
if err != nil {
179184
return nil, fmt.Errorf("failed setting up environment. err=%v", err)

‎pkg/os/iscsi/api.go‎

Lines changed: 151 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,13 @@
11
package iscsi
22

33
import (
4-
"encoding/json"
54
"fmt"
5+
"strconv"
6+
"strings"
67

7-
"github.com/kubernetes-csi/csi-proxy/pkg/utils"
8+
"github.com/kubernetes-csi/csi-proxy/pkg/cim"
9+
"github.com/microsoft/wmi/server2019/root/microsoft/windows/storage"
10+
"k8s.io/klog/v2"
811
)
912

1013
// Implements the iSCSI OS API calls. All code here should be very simple
@@ -18,119 +21,178 @@ func New() APIImplementor {
1821
return APIImplementor{}
1922
}
2023

24+
func parseTargetPortal(instance *storage.MSFT_iSCSITargetPortal) (string, uint32, error) {
25+
portalAddress, err := instance.GetPropertyTargetPortalAddress()
26+
if err != nil {
27+
return "", 0, fmt.Errorf("failed parsing target portal address %v. err: %w", instance, err)
28+
}
29+
30+
portalPort, err := instance.GetProperty("TargetPortalPortNumber")
31+
if err != nil {
32+
return "", 0, fmt.Errorf("failed parsing target portal port number %v. err: %w", instance, err)
33+
}
34+
35+
return portalAddress, uint32(portalPort.(int32)), nil
36+
}
37+
2138
func (APIImplementor) AddTargetPortal(portal *TargetPortal) error {
22-
cmdLine := fmt.Sprintf(
23-
`New-IscsiTargetPortal -TargetPortalAddress ${Env:iscsi_tp_address} ` +
24-
`-TargetPortalPortNumber ${Env:iscsi_tp_port}`)
25-
out, err := utils.RunPowershellCmd(cmdLine, fmt.Sprintf("iscsi_tp_address=%s", portal.Address),
26-
fmt.Sprintf("iscsi_tp_port=%d", portal.Port))
39+
existing, err := cim.QueryISCSITargetPortal(portal.Address, portal.Port, nil)
40+
if cim.IgnoreNotFound(err) != nil {
41+
return err
42+
}
43+
44+
if existing != nil {
45+
klog.V(2).Infof("target portal at (%s:%d) already exists", portal.Address, portal.Port)
46+
return nil
47+
}
48+
49+
_, err = cim.NewISCSITargetPortal(portal.Address, portal.Port, nil, nil, nil, nil)
2750
if err != nil {
28-
return fmt.Errorf("error adding target portal. cmd %s, output: %s, err: %v", cmdLine, string(out), err)
51+
return fmt.Errorf("error adding target portal at (%s:%d). err: %v", portal.Address, portal.Port, err)
2952
}
3053

3154
return nil
3255
}
3356

3457
func (APIImplementor) DiscoverTargetPortal(portal *TargetPortal) ([]string, error) {
35-
// ConvertTo-Json is not part of the pipeline because powershell converts an
36-
// array with one element to a single element
37-
cmdLine := fmt.Sprintf(
38-
`ConvertTo-Json -InputObject @(Get-IscsiTargetPortal -TargetPortalAddress ` +
39-
`${Env:iscsi_tp_address} -TargetPortalPortNumber ${Env:iscsi_tp_port} | ` +
40-
`Get-IscsiTarget | Select-Object -ExpandProperty NodeAddress)`)
41-
out, err := utils.RunPowershellCmd(cmdLine, fmt.Sprintf("iscsi_tp_address=%s", portal.Address),
42-
fmt.Sprintf("iscsi_tp_port=%d", portal.Port))
58+
instance, err := cim.QueryISCSITargetPortal(portal.Address, portal.Port, nil)
4359
if err != nil {
44-
return nil, fmt.Errorf("error discovering target portal. cmd: %s, output: %s, err: %w", cmdLine, string(out), err)
60+
return nil, err
4561
}
4662

47-
var iqns []string
48-
err = json.Unmarshal(out, &iqns)
63+
targets, err := cim.ListISCSITargetsByTargetPortalWithFilters(nil, []*storage.MSFT_iSCSITargetPortal{instance})
4964
if err != nil {
50-
return nil, fmt.Errorf("failed parsing iqn list. cmd: %s output: %s, err: %w", cmdLine, string(out), err)
65+
return nil, err
66+
}
67+
68+
var iqns []string
69+
for _, target := range targets {
70+
iqn, err := target.GetProperty("NodeAddress")
71+
if err != nil {
72+
return nil, fmt.Errorf("failed parsing node address of target %v to target portal at (%s:%d). err: %w", target, portal.Address, portal.Port, err)
73+
}
74+
75+
iqns = append(iqns, iqn.(string))
5176
}
5277

5378
return iqns, nil
5479
}
5580

5681
func (APIImplementor) ListTargetPortals() ([]TargetPortal, error) {
57-
cmdLine := fmt.Sprintf(
58-
`ConvertTo-Json -InputObject @(Get-IscsiTargetPortal | ` +
59-
`Select-Object TargetPortalAddress, TargetPortalPortNumber)`)
60-
61-
out, err := utils.RunPowershellCmd(cmdLine)
82+
instances, err := cim.ListISCSITargetPortals([]string{"TargetPortalAddress", "TargetPortalPortNumber"})
6283
if err != nil {
63-
return nil, fmt.Errorf("error listing target portals. cmd %s, output: %s, err: %w", cmdLine, string(out), err)
84+
return nil, err
6485
}
6586

6687
var portals []TargetPortal
67-
err = json.Unmarshal(out, &portals)
68-
if err != nil {
69-
return nil, fmt.Errorf("failed parsing target portal list. cmd: %s output: %s, err: %w", cmdLine, string(out), err)
88+
for _, instance := range instances {
89+
address, port, err := parseTargetPortal(instance)
90+
if err != nil {
91+
return nil, fmt.Errorf("failed parsing target portal %v. err: %w", instance, err)
92+
}
93+
94+
portals = append(portals, TargetPortal{
95+
Address: address,
96+
Port: port,
97+
})
7098
}
7199

72100
return portals, nil
73101
}
74102

75103
func (APIImplementor) RemoveTargetPortal(portal *TargetPortal) error {
76-
cmdLine := fmt.Sprintf(
77-
`Get-IscsiTargetPortal -TargetPortalAddress ${Env:iscsi_tp_address} ` +
78-
`-TargetPortalPortNumber ${Env:iscsi_tp_port} | Remove-IscsiTargetPortal ` +
79-
`-Confirm:$false`)
104+
instance, err := cim.QueryISCSITargetPortal(portal.Address, portal.Port, nil)
105+
if err != nil {
106+
return err
107+
}
80108

81-
out, err := utils.RunPowershellCmd(cmdLine, fmt.Sprintf("iscsi_tp_address=%s", portal.Address),
82-
fmt.Sprintf("iscsi_tp_port=%d", portal.Port))
109+
address, port, err := parseTargetPortal(instance)
83110
if err != nil {
84-
return fmt.Errorf("error removing target portal. cmd %s, output: %s, err: %w", cmdLine, string(out), err)
111+
return fmt.Errorf("failed to parse target portal %v. error: %v", instance, err)
112+
}
113+
114+
result, err := instance.InvokeMethodWithReturn("Remove",
115+
nil,
116+
nil,
117+
int(port),
118+
address,
119+
)
120+
if result != 0 || err != nil {
121+
return fmt.Errorf("error removing target portal at (%s:%d). result: %d, err: %w", address, port, result, err)
85122
}
86123

87124
return nil
88125
}
89126

90127
func (APIImplementor) ConnectTarget(portal *TargetPortal, iqn string,
91128
authType string, chapUser string, chapSecret string) error {
92-
// Not using InputObject as Connect-IscsiTarget's InputObject does not work.
93-
// This is due to being a static WMI method together with a bug in the
94-
// powershell version of the API.
95-
cmdLine := fmt.Sprintf(
96-
`Connect-IscsiTarget -TargetPortalAddress ${Env:iscsi_tp_address}` +
97-
` -TargetPortalPortNumber ${Env:iscsi_tp_port} -NodeAddress ${Env:iscsi_target_iqn}` +
98-
` -AuthenticationType ${Env:iscsi_auth_type}`)
129+
target, err := cim.QueryISCSITarget(portal.Address, portal.Port, iqn, nil)
130+
if err != nil {
131+
return err
132+
}
99133

100-
if chapUser != "" {
101-
cmdLine += ` -ChapUsername ${Env:iscsi_chap_user}`
134+
connected, err := target.GetPropertyIsConnected()
135+
if err != nil {
136+
return err
102137
}
103138

104-
if chapSecret != "" {
105-
cmdLine += ` -ChapSecret ${Env:iscsi_chap_secret}`
139+
if connected {
140+
klog.V(2).Infof("target %s from target portal at (%s:%d) is connected.", iqn, portal.Address, portal.Port)
141+
return nil
106142
}
107143

108-
out, err := utils.RunPowershellCmd(cmdLine, fmt.Sprintf("iscsi_tp_address=%s", portal.Address),
109-
fmt.Sprintf("iscsi_tp_port=%d", portal.Port),
110-
fmt.Sprintf("iscsi_target_iqn=%s", iqn),
111-
fmt.Sprintf("iscsi_auth_type=%s", authType),
112-
fmt.Sprintf("iscsi_chap_user=%s", chapUser),
113-
fmt.Sprintf("iscsi_chap_secret=%s", chapSecret))
144+
targetAuthType := strings.ToUpper(strings.ReplaceAll(authType, "_", ""))
145+
146+
result, _, err := cim.ConnectISCSITarget(portal.Address, portal.Port, iqn, targetAuthType, &chapUser, &chapSecret)
114147
if err != nil {
115-
return fmt.Errorf("error connecting to target portal. cmd %s, output: %s, err: %w", cmdLine, string(out), err)
148+
return fmt.Errorf("error connecting to target portal. result: %d, err: %w", result, err)
116149
}
117150

118151
return nil
119152
}
120153

121154
func (APIImplementor) DisconnectTarget(portal *TargetPortal, iqn string) error {
122-
// Using InputObject instead of pipe to verify input is not empty
123-
cmdLine := fmt.Sprintf(
124-
`Disconnect-IscsiTarget -InputObject (Get-IscsiTargetPortal ` +
125-
`-TargetPortalAddress ${Env:iscsi_tp_address} -TargetPortalPortNumber ${Env:iscsi_tp_port} ` +
126-
` | Get-IscsiTarget | Where-Object { $_.NodeAddress -eq ${Env:iscsi_target_iqn} }) ` +
127-
`-Confirm:$false`)
155+
target, err := cim.QueryISCSITarget(portal.Address, portal.Port, iqn, nil)
156+
if err != nil {
157+
return err
158+
}
159+
160+
connected, err := target.GetPropertyIsConnected()
161+
if err != nil {
162+
return fmt.Errorf("error query connected of target %s from target portal at (%s:%d). err: %w", iqn, portal.Address, portal.Port, err)
163+
}
164+
165+
if !connected {
166+
klog.V(2).Infof("target %s from target portal at (%s:%d) is not connected.", iqn, portal.Address, portal.Port)
167+
return nil
168+
}
128169

129-
out, err := utils.RunPowershellCmd(cmdLine, fmt.Sprintf("iscsi_tp_address=%s", portal.Address),
130-
fmt.Sprintf("iscsi_tp_port=%d", portal.Port),
131-
fmt.Sprintf("iscsi_target_iqn=%s", iqn))
170+
// get session
171+
session, err := cim.QueryISCSISessionByTarget(target, nil)
132172
if err != nil {
133-
return fmt.Errorf("error disconnecting from target portal. cmd %s, output: %s, err: %w", cmdLine, string(out), err)
173+
return fmt.Errorf("error query session of target %s from target portal at (%s:%d). err: %w", iqn, portal.Address, portal.Port, err)
174+
}
175+
176+
sessionIdentifier, err := session.GetPropertySessionIdentifier()
177+
if err != nil {
178+
return fmt.Errorf("error query session identifier of target %s from target portal at (%s:%d). err: %w", iqn, portal.Address, portal.Port, err)
179+
}
180+
181+
persistent, err := session.GetPropertyIsPersistent()
182+
if err != nil {
183+
return fmt.Errorf("error query session persistency of target %s from target portal at (%s:%d). err: %w", iqn, portal.Address, portal.Port, err)
184+
}
185+
186+
if persistent {
187+
result, err := session.InvokeMethodWithReturn("Unregister")
188+
if err != nil {
189+
return fmt.Errorf("error unregister session on target %s from target portal at (%s:%d). result: %d, err: %w", iqn, portal.Address, portal.Port, result, err)
190+
}
191+
}
192+
193+
result, err := target.InvokeMethodWithReturn("Disconnect", sessionIdentifier)
194+
if err != nil {
195+
return fmt.Errorf("error disconnecting target %s from target portal at (%s:%d). result: %d, err: %w", iqn, portal.Address, portal.Port, result, err)
134196
}
135197

136198
return nil
@@ -139,36 +201,43 @@ func (APIImplementor) DisconnectTarget(portal *TargetPortal, iqn string) error {
139201
func (APIImplementor) GetTargetDisks(portal *TargetPortal, iqn string) ([]string, error) {
140202
// Converting DiskNumber to string for compatibility with disk api group
141203
// Not using pipeline in order to validate that items are non-empty
142-
cmdLine := fmt.Sprintf(
143-
`$ErrorActionPreference = "Stop"; ` +
144-
`$tp = Get-IscsiTargetPortal -TargetPortalAddress ${Env:iscsi_tp_address} -TargetPortalPortNumber ${Env:iscsi_tp_port}; ` +
145-
`$t = $tp | Get-IscsiTarget | Where-Object { $_.NodeAddress -eq ${Env:iscsi_target_iqn} }; ` +
146-
`$c = Get-IscsiConnection -IscsiTarget $t; ` +
147-
`$ids = $c | Get-Disk | Select -ExpandProperty Number | Out-String -Stream; ` +
148-
`ConvertTo-Json -InputObject @($ids)`)
204+
target, err := cim.QueryISCSITarget(portal.Address, portal.Port, iqn, nil)
205+
if err != nil {
206+
return nil, err
207+
}
149208

150-
out, err := utils.RunPowershellCmd(cmdLine, fmt.Sprintf("iscsi_tp_address=%s", portal.Address),
151-
fmt.Sprintf("iscsi_tp_port=%d", portal.Port),
152-
fmt.Sprintf("iscsi_target_iqn=%s", iqn))
209+
connected, err := target.GetPropertyIsConnected()
153210
if err != nil {
154-
return nil, fmt.Errorf("error getting target disks. cmd %s, output: %s, err: %w", cmdLine, string(out), err)
211+
return nil, fmt.Errorf("error query connected of target %s from target portal at (%s:%d). err: %w", iqn, portal.Address, portal.Port, err)
155212
}
156213

157-
var ids []string
158-
err = json.Unmarshal(out, &ids)
214+
if !connected {
215+
klog.V(2).Infof("target %s from target portal at (%s:%d) is not connected.", iqn, portal.Address, portal.Port)
216+
return nil, nil
217+
}
218+
219+
disks, err := cim.ListDisksByTarget(target, []string{})
220+
159221
if err != nil {
160-
return nil, fmt.Errorf("error parsing iqn target disks. cmd: %s output: %s, err: %w", cmdLine, string(out), err)
222+
return nil, fmt.Errorf("error getting target disks on target %s from target portal at (%s:%d). err: %w", iqn, portal.Address, portal.Port, err)
161223
}
162224

225+
var ids []string
226+
for _, disk := range disks {
227+
number, err := disk.GetProperty("Number")
228+
if err != nil {
229+
return nil, fmt.Errorf("error getting number of disk %v on target %s from target portal at (%s:%d). err: %w", disk, iqn, portal.Address, portal.Port, err)
230+
}
231+
232+
ids = append(ids, strconv.Itoa(int(number.(int32))))
233+
}
163234
return ids, nil
164235
}
165236

166237
func (APIImplementor) SetMutualChapSecret(mutualChapSecret string) error {
167-
cmdLine := `Set-IscsiChapSecret -ChapSecret ${Env:iscsi_mutual_chap_secret}`
168-
out, err := utils.RunPowershellCmd(cmdLine, fmt.Sprintf("iscsi_mutual_chap_secret=%s", mutualChapSecret))
238+
result, _, err := cim.InvokeCimMethod(cim.WMINamespaceStorage, "MSFT_iSCSISession", "SetCHAPSecret", map[string]interface{}{"ChapSecret": mutualChapSecret})
169239
if err != nil {
170-
return fmt.Errorf("error setting mutual chap secret. cmd %s,"+
171-
" output: %s, err: %v", cmdLine, string(out), err)
240+
return fmt.Errorf("error setting mutual chap secret. result: %d, err: %v", result, err)
172241
}
173242

174243
return nil

0 commit comments

Comments
 (0)