diff --git a/controllers/openstackmachine_controller.go b/controllers/openstackmachine_controller.go
index d54fecf39a..725b24b11d 100644
--- a/controllers/openstackmachine_controller.go
+++ b/controllers/openstackmachine_controller.go
@@ -556,7 +556,10 @@ func openStackMachineSpecToOpenStackServerSpec(openStackMachineSpec *infrav1.Ope
 			}
 			serverPort.FixedIPs = clusterSubnets
 		}
-		if len(serverPort.SecurityGroups) == 0 && defaultSecGroup != nil {
+		// Only inject the default SG when portSecurity is not disabled,
+		// there are no SGs passed by user and defaultSecGroup is set
+		portSecurityDisabled := serverPort.DisablePortSecurity != nil && *serverPort.DisablePortSecurity
+		if !portSecurityDisabled && len(serverPort.SecurityGroups) == 0 && defaultSecGroup != nil {
 			serverPort.SecurityGroups = []infrav1.SecurityGroupParam{
 				{
 					ID: defaultSecGroup,
diff --git a/controllers/openstackmachine_controller_test.go b/controllers/openstackmachine_controller_test.go
index ecc8595a0c..78bf6f6936 100644
--- a/controllers/openstackmachine_controller_test.go
+++ b/controllers/openstackmachine_controller_test.go
@@ -346,6 +346,34 @@ func TestOpenStackMachineSpecToOpenStackServerSpec(t *testing.T) {
 				UserDataRef: userData,
 			},
 		},
+		{
+			name: "Explicit port with disablePortSecurity",
+			spec: &infrav1.OpenStackMachineSpec{
+				Flavor: ptr.To(flavorName),
+				Image:  image,
+				Ports: []infrav1.PortOpts{{
+					Network: &infrav1.NetworkParam{ID: ptr.To(networkUUID)},
+					ResolvedPortSpecFields: infrav1.ResolvedPortSpecFields{
+						DisablePortSecurity: ptr.To(true),
+					},
+				}},
+			},
+			cluster: openStackClusterNetworkWithoutID,
+			want: &infrav1alpha1.OpenStackServerSpec{
+				Flavor:      ptr.To(flavorName),
+				IdentityRef: identityRef,
+				Image:       image,
+				Ports: []infrav1.PortOpts{{
+					Network:        &infrav1.NetworkParam{ID: ptr.To(networkUUID)},
+					SecurityGroups: nil,
+					ResolvedPortSpecFields: infrav1.ResolvedPortSpecFields{
+						DisablePortSecurity: ptr.To(true),
+					},
+				}},
+				Tags:        tags,
+				UserDataRef: userData,
+			},
+		},
 	}
 	for i := range tests {
 		tt := tests[i]