@@ -83,10 +83,6 @@ struct idle_cpu {
83
83
84
84
static const struct idle_cpu * icpu ;
85
85
static struct cpuidle_device __percpu * intel_idle_cpuidle_devices ;
86
- static int intel_idle (struct cpuidle_device * dev ,
87
- struct cpuidle_driver * drv , int index );
88
- static void intel_idle_s2idle (struct cpuidle_device * dev ,
89
- struct cpuidle_driver * drv , int index );
90
86
static struct cpuidle_state * cpuidle_state_table ;
91
87
92
88
/*
@@ -112,6 +108,81 @@ static struct cpuidle_state *cpuidle_state_table;
112
108
#define flg2MWAIT (flags ) (((flags) >> 24) & 0xFF)
113
109
#define MWAIT2flg (eax ) ((eax & 0xFF) << 24)
114
110
111
+ /**
112
+ * intel_idle - Ask the processor to enter the given idle state.
113
+ * @dev: cpuidle device of the target CPU.
114
+ * @drv: cpuidle driver (assumed to point to intel_idle_driver).
115
+ * @index: Target idle state index.
116
+ *
117
+ * Use the MWAIT instruction to notify the processor that the CPU represented by
118
+ * @dev is idle and it can try to enter the idle state corresponding to @index.
119
+ *
120
+ * If the local APIC timer is not known to be reliable in the target idle state,
121
+ * enable one-shot tick broadcasting for the target CPU before executing MWAIT.
122
+ *
123
+ * Optionally call leave_mm() for the target CPU upfront to avoid wakeups due to
124
+ * flushing user TLBs.
125
+ *
126
+ * Must be called under local_irq_disable().
127
+ */
128
+ static __cpuidle int intel_idle (struct cpuidle_device * dev ,
129
+ struct cpuidle_driver * drv , int index )
130
+ {
131
+ struct cpuidle_state * state = & drv -> states [index ];
132
+ unsigned long eax = flg2MWAIT (state -> flags );
133
+ unsigned long ecx = 1 ; /* break on interrupt flag */
134
+ bool uninitialized_var (tick );
135
+ int cpu = smp_processor_id ();
136
+
137
+ /*
138
+ * leave_mm() to avoid costly and often unnecessary wakeups
139
+ * for flushing the user TLB's associated with the active mm.
140
+ */
141
+ if (state -> flags & CPUIDLE_FLAG_TLB_FLUSHED )
142
+ leave_mm (cpu );
143
+
144
+ if (!static_cpu_has (X86_FEATURE_ARAT ) && !lapic_timer_always_reliable ) {
145
+ /*
146
+ * Switch over to one-shot tick broadcast if the target C-state
147
+ * is deeper than C1.
148
+ */
149
+ if ((eax >> MWAIT_SUBSTATE_SIZE ) & MWAIT_CSTATE_MASK ) {
150
+ tick = true;
151
+ tick_broadcast_enter ();
152
+ } else {
153
+ tick = false;
154
+ }
155
+ }
156
+
157
+ mwait_idle_with_hints (eax , ecx );
158
+
159
+ if (!static_cpu_has (X86_FEATURE_ARAT ) && tick )
160
+ tick_broadcast_exit ();
161
+
162
+ return index ;
163
+ }
164
+
165
+ /**
166
+ * intel_idle_s2idle - Ask the processor to enter the given idle state.
167
+ * @dev: cpuidle device of the target CPU.
168
+ * @drv: cpuidle driver (assumed to point to intel_idle_driver).
169
+ * @index: Target idle state index.
170
+ *
171
+ * Use the MWAIT instruction to notify the processor that the CPU represented by
172
+ * @dev is idle and it can try to enter the idle state corresponding to @index.
173
+ *
174
+ * Invoked as a suspend-to-idle callback routine with frozen user space, frozen
175
+ * scheduler tick and suspended scheduler clock on the target CPU.
176
+ */
177
+ static __cpuidle void intel_idle_s2idle (struct cpuidle_device * dev ,
178
+ struct cpuidle_driver * drv , int index )
179
+ {
180
+ unsigned long eax = flg2MWAIT (drv -> states [index ].flags );
181
+ unsigned long ecx = 1 ; /* break on interrupt flag */
182
+
183
+ mwait_idle_with_hints (eax , ecx );
184
+ }
185
+
115
186
/*
116
187
* States are indexed by the cstate number,
117
188
* which is also the index into the MWAIT hint array.
@@ -891,81 +962,6 @@ static struct cpuidle_state dnv_cstates[] = {
891
962
.enter = NULL }
892
963
};
893
964
894
- /**
895
- * intel_idle - Ask the processor to enter the given idle state.
896
- * @dev: cpuidle device of the target CPU.
897
- * @drv: cpuidle driver (assumed to point to intel_idle_driver).
898
- * @index: Target idle state index.
899
- *
900
- * Use the MWAIT instruction to notify the processor that the CPU represented by
901
- * @dev is idle and it can try to enter the idle state corresponding to @index.
902
- *
903
- * If the local APIC timer is not known to be reliable in the target idle state,
904
- * enable one-shot tick broadcasting for the target CPU before executing MWAIT.
905
- *
906
- * Optionally call leave_mm() for the target CPU upfront to avoid wakeups due to
907
- * flushing user TLBs.
908
- *
909
- * Must be called under local_irq_disable().
910
- */
911
- static __cpuidle int intel_idle (struct cpuidle_device * dev ,
912
- struct cpuidle_driver * drv , int index )
913
- {
914
- struct cpuidle_state * state = & drv -> states [index ];
915
- unsigned long eax = flg2MWAIT (state -> flags );
916
- unsigned long ecx = 1 ; /* break on interrupt flag */
917
- bool uninitialized_var (tick );
918
- int cpu = smp_processor_id ();
919
-
920
- /*
921
- * leave_mm() to avoid costly and often unnecessary wakeups
922
- * for flushing the user TLB's associated with the active mm.
923
- */
924
- if (state -> flags & CPUIDLE_FLAG_TLB_FLUSHED )
925
- leave_mm (cpu );
926
-
927
- if (!static_cpu_has (X86_FEATURE_ARAT ) && !lapic_timer_always_reliable ) {
928
- /*
929
- * Switch over to one-shot tick broadcast if the target C-state
930
- * is deeper than C1.
931
- */
932
- if ((eax >> MWAIT_SUBSTATE_SIZE ) & MWAIT_CSTATE_MASK ) {
933
- tick = true;
934
- tick_broadcast_enter ();
935
- } else {
936
- tick = false;
937
- }
938
- }
939
-
940
- mwait_idle_with_hints (eax , ecx );
941
-
942
- if (!static_cpu_has (X86_FEATURE_ARAT ) && tick )
943
- tick_broadcast_exit ();
944
-
945
- return index ;
946
- }
947
-
948
- /**
949
- * intel_idle_s2idle - Ask the processor to enter the given idle state.
950
- * @dev: cpuidle device of the target CPU.
951
- * @drv: cpuidle driver (assumed to point to intel_idle_driver).
952
- * @index: Target idle state index.
953
- *
954
- * Use the MWAIT instruction to notify the processor that the CPU represented by
955
- * @dev is idle and it can try to enter the idle state corresponding to @index.
956
- *
957
- * Invoked as a suspend-to-idle callback routine with frozen user space, frozen
958
- * scheduler tick and suspended scheduler clock on the target CPU.
959
- */
960
- static __cpuidle void intel_idle_s2idle (struct cpuidle_device * dev ,
961
- struct cpuidle_driver * drv , int index )
962
- {
963
- unsigned long eax = flg2MWAIT (drv -> states [index ].flags );
964
- unsigned long ecx = 1 ; /* break on interrupt flag */
965
-
966
- mwait_idle_with_hints (eax , ecx );
967
- }
968
-
969
965
static const struct idle_cpu idle_cpu_nehalem = {
970
966
.state_table = nehalem_cstates ,
971
967
.auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE ,
0 commit comments