Skip to content

Commit 1e19da8

Browse files
Peter Zijlstrasuryasaimadhu
Peter Zijlstra
authored and
suryasaimadhu
committed
x86/speculation: Add eIBRS + Retpoline options
Thanks to the chaps at VUsec it is now clear that eIBRS is not sufficient, therefore allow enabling of retpolines along with eIBRS. Add spectre_v2=eibrs, spectre_v2=eibrs,lfence and spectre_v2=eibrs,retpoline options to explicitly pick your preferred means of mitigation. Since there's new mitigations there's also user visible changes in /sys/devices/system/cpu/vulnerabilities/spectre_v2 to reflect these new mitigations. [ bp: Massage commit message, trim error messages, do more precise eIBRS mode checking. ] Co-developed-by: Josh Poimboeuf <[email protected]> Signed-off-by: Josh Poimboeuf <[email protected]> Signed-off-by: Peter Zijlstra (Intel) <[email protected]> Signed-off-by: Borislav Petkov <[email protected]> Reviewed-by: Patrick Colp <[email protected]> Reviewed-by: Thomas Gleixner <[email protected]>
1 parent d45476d commit 1e19da8

File tree

2 files changed

+99
-38
lines changed

2 files changed

+99
-38
lines changed

‎arch/x86/include/asm/nospec-branch.h

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -190,7 +190,9 @@ enum spectre_v2_mitigation {
190190
SPECTRE_V2_NONE,
191191
SPECTRE_V2_RETPOLINE,
192192
SPECTRE_V2_LFENCE,
193-
SPECTRE_V2_IBRS_ENHANCED,
193+
SPECTRE_V2_EIBRS,
194+
SPECTRE_V2_EIBRS_RETPOLINE,
195+
SPECTRE_V2_EIBRS_LFENCE,
194196
};
195197

196198
/* The indirect branch speculation control variants */

‎arch/x86/kernel/cpu/bugs.c

Lines changed: 96 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -665,6 +665,9 @@ enum spectre_v2_mitigation_cmd {
665665
SPECTRE_V2_CMD_RETPOLINE,
666666
SPECTRE_V2_CMD_RETPOLINE_GENERIC,
667667
SPECTRE_V2_CMD_RETPOLINE_LFENCE,
668+
SPECTRE_V2_CMD_EIBRS,
669+
SPECTRE_V2_CMD_EIBRS_RETPOLINE,
670+
SPECTRE_V2_CMD_EIBRS_LFENCE,
668671
};
669672

670673
enum spectre_v2_user_cmd {
@@ -737,6 +740,13 @@ spectre_v2_parse_user_cmdline(enum spectre_v2_mitigation_cmd v2_cmd)
737740
return SPECTRE_V2_USER_CMD_AUTO;
738741
}
739742

743+
static inline bool spectre_v2_in_eibrs_mode(enum spectre_v2_mitigation mode)
744+
{
745+
return (mode == SPECTRE_V2_EIBRS ||
746+
mode == SPECTRE_V2_EIBRS_RETPOLINE ||
747+
mode == SPECTRE_V2_EIBRS_LFENCE);
748+
}
749+
740750
static void __init
741751
spectre_v2_user_select_mitigation(enum spectre_v2_mitigation_cmd v2_cmd)
742752
{
@@ -804,7 +814,7 @@ spectre_v2_user_select_mitigation(enum spectre_v2_mitigation_cmd v2_cmd)
804814
*/
805815
if (!boot_cpu_has(X86_FEATURE_STIBP) ||
806816
!smt_possible ||
807-
spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED)
817+
spectre_v2_in_eibrs_mode(spectre_v2_enabled))
808818
return;
809819

810820
/*
@@ -826,7 +836,9 @@ static const char * const spectre_v2_strings[] = {
826836
[SPECTRE_V2_NONE] = "Vulnerable",
827837
[SPECTRE_V2_RETPOLINE] = "Mitigation: Retpolines",
828838
[SPECTRE_V2_LFENCE] = "Mitigation: LFENCE",
829-
[SPECTRE_V2_IBRS_ENHANCED] = "Mitigation: Enhanced IBRS",
839+
[SPECTRE_V2_EIBRS] = "Mitigation: Enhanced IBRS",
840+
[SPECTRE_V2_EIBRS_LFENCE] = "Mitigation: Enhanced IBRS + LFENCE",
841+
[SPECTRE_V2_EIBRS_RETPOLINE] = "Mitigation: Enhanced IBRS + Retpolines",
830842
};
831843

832844
static const struct {
@@ -840,6 +852,9 @@ static const struct {
840852
{ "retpoline,amd", SPECTRE_V2_CMD_RETPOLINE_LFENCE, false },
841853
{ "retpoline,lfence", SPECTRE_V2_CMD_RETPOLINE_LFENCE, false },
842854
{ "retpoline,generic", SPECTRE_V2_CMD_RETPOLINE_GENERIC, false },
855+
{ "eibrs", SPECTRE_V2_CMD_EIBRS, false },
856+
{ "eibrs,lfence", SPECTRE_V2_CMD_EIBRS_LFENCE, false },
857+
{ "eibrs,retpoline", SPECTRE_V2_CMD_EIBRS_RETPOLINE, false },
843858
{ "auto", SPECTRE_V2_CMD_AUTO, false },
844859
};
845860

@@ -877,15 +892,29 @@ static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void)
877892

878893
if ((cmd == SPECTRE_V2_CMD_RETPOLINE ||
879894
cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE ||
880-
cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC) &&
895+
cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC ||
896+
cmd == SPECTRE_V2_CMD_EIBRS_LFENCE ||
897+
cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) &&
881898
!IS_ENABLED(CONFIG_RETPOLINE)) {
882-
pr_err("%s selected but not compiled in. Switching to AUTO select\n", mitigation_options[i].option);
899+
pr_err("%s selected but not compiled in. Switching to AUTO select\n",
900+
mitigation_options[i].option);
901+
return SPECTRE_V2_CMD_AUTO;
902+
}
903+
904+
if ((cmd == SPECTRE_V2_CMD_EIBRS ||
905+
cmd == SPECTRE_V2_CMD_EIBRS_LFENCE ||
906+
cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) &&
907+
!boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) {
908+
pr_err("%s selected but CPU doesn't have eIBRS. Switching to AUTO select\n",
909+
mitigation_options[i].option);
883910
return SPECTRE_V2_CMD_AUTO;
884911
}
885912

886-
if ((cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE) &&
913+
if ((cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE ||
914+
cmd == SPECTRE_V2_CMD_EIBRS_LFENCE) &&
887915
!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
888-
pr_err("%s selected, but CPU doesn't have a serializing LFENCE. Switching to AUTO select\n", mitigation_options[i].option);
916+
pr_err("%s selected, but CPU doesn't have a serializing LFENCE. Switching to AUTO select\n",
917+
mitigation_options[i].option);
889918
return SPECTRE_V2_CMD_AUTO;
890919
}
891920

@@ -894,6 +923,25 @@ static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void)
894923
return cmd;
895924
}
896925

926+
static enum spectre_v2_mitigation __init spectre_v2_select_retpoline(void)
927+
{
928+
if (!IS_ENABLED(CONFIG_RETPOLINE)) {
929+
pr_err("Kernel not compiled with retpoline; no mitigation available!");
930+
return SPECTRE_V2_NONE;
931+
}
932+
933+
if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD ||
934+
boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) {
935+
if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
936+
pr_err("LFENCE not serializing, switching to generic retpoline\n");
937+
return SPECTRE_V2_RETPOLINE;
938+
}
939+
return SPECTRE_V2_LFENCE;
940+
}
941+
942+
return SPECTRE_V2_RETPOLINE;
943+
}
944+
897945
static void __init spectre_v2_select_mitigation(void)
898946
{
899947
enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline();
@@ -914,49 +962,60 @@ static void __init spectre_v2_select_mitigation(void)
914962
case SPECTRE_V2_CMD_FORCE:
915963
case SPECTRE_V2_CMD_AUTO:
916964
if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) {
917-
mode = SPECTRE_V2_IBRS_ENHANCED;
918-
/* Force it so VMEXIT will restore correctly */
919-
x86_spec_ctrl_base |= SPEC_CTRL_IBRS;
920-
wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
921-
goto specv2_set_mode;
965+
mode = SPECTRE_V2_EIBRS;
966+
break;
922967
}
923-
if (IS_ENABLED(CONFIG_RETPOLINE))
924-
goto retpoline_auto;
968+
969+
mode = spectre_v2_select_retpoline();
925970
break;
971+
926972
case SPECTRE_V2_CMD_RETPOLINE_LFENCE:
927-
if (IS_ENABLED(CONFIG_RETPOLINE))
928-
goto retpoline_lfence;
973+
mode = SPECTRE_V2_LFENCE;
929974
break;
975+
930976
case SPECTRE_V2_CMD_RETPOLINE_GENERIC:
931-
if (IS_ENABLED(CONFIG_RETPOLINE))
932-
goto retpoline_generic;
977+
mode = SPECTRE_V2_RETPOLINE;
933978
break;
979+
934980
case SPECTRE_V2_CMD_RETPOLINE:
935-
if (IS_ENABLED(CONFIG_RETPOLINE))
936-
goto retpoline_auto;
981+
mode = spectre_v2_select_retpoline();
982+
break;
983+
984+
case SPECTRE_V2_CMD_EIBRS:
985+
mode = SPECTRE_V2_EIBRS;
986+
break;
987+
988+
case SPECTRE_V2_CMD_EIBRS_LFENCE:
989+
mode = SPECTRE_V2_EIBRS_LFENCE;
990+
break;
991+
992+
case SPECTRE_V2_CMD_EIBRS_RETPOLINE:
993+
mode = SPECTRE_V2_EIBRS_RETPOLINE;
937994
break;
938995
}
939-
pr_err("Spectre mitigation: kernel not compiled with retpoline; no mitigation available!");
940-
return;
941996

942-
retpoline_auto:
943-
if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD ||
944-
boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) {
945-
retpoline_lfence:
946-
if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
947-
pr_err("Spectre mitigation: LFENCE not serializing, switching to generic retpoline\n");
948-
goto retpoline_generic;
949-
}
950-
mode = SPECTRE_V2_LFENCE;
997+
if (spectre_v2_in_eibrs_mode(mode)) {
998+
/* Force it so VMEXIT will restore correctly */
999+
x86_spec_ctrl_base |= SPEC_CTRL_IBRS;
1000+
wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
1001+
}
1002+
1003+
switch (mode) {
1004+
case SPECTRE_V2_NONE:
1005+
case SPECTRE_V2_EIBRS:
1006+
break;
1007+
1008+
case SPECTRE_V2_LFENCE:
1009+
case SPECTRE_V2_EIBRS_LFENCE:
9511010
setup_force_cpu_cap(X86_FEATURE_RETPOLINE_LFENCE);
1011+
fallthrough;
1012+
1013+
case SPECTRE_V2_RETPOLINE:
1014+
case SPECTRE_V2_EIBRS_RETPOLINE:
9521015
setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
953-
} else {
954-
retpoline_generic:
955-
mode = SPECTRE_V2_RETPOLINE;
956-
setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
1016+
break;
9571017
}
9581018

959-
specv2_set_mode:
9601019
spectre_v2_enabled = mode;
9611020
pr_info("%s\n", spectre_v2_strings[mode]);
9621021

@@ -982,7 +1041,7 @@ static void __init spectre_v2_select_mitigation(void)
9821041
* the CPU supports Enhanced IBRS, kernel might un-intentionally not
9831042
* enable IBRS around firmware calls.
9841043
*/
985-
if (boot_cpu_has(X86_FEATURE_IBRS) && mode != SPECTRE_V2_IBRS_ENHANCED) {
1044+
if (boot_cpu_has(X86_FEATURE_IBRS) && !spectre_v2_in_eibrs_mode(mode)) {
9861045
setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW);
9871046
pr_info("Enabling Restricted Speculation for firmware calls\n");
9881047
}
@@ -1691,7 +1750,7 @@ static ssize_t tsx_async_abort_show_state(char *buf)
16911750

16921751
static char *stibp_state(void)
16931752
{
1694-
if (spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED)
1753+
if (spectre_v2_in_eibrs_mode(spectre_v2_enabled))
16951754
return "";
16961755

16971756
switch (spectre_v2_user_stibp) {

0 commit comments

Comments
 (0)