11use crate :: base;
22use crate :: traits:: * ;
3+ use rustc_hir:: def_id:: DefId ;
4+ use rustc_index:: vec:: Idx ;
35use rustc_middle:: mir;
46use rustc_middle:: mir:: interpret:: ErrorHandled ;
7+ use rustc_middle:: mir:: visit:: MutVisitor ;
8+ use rustc_middle:: mir:: visit:: Visitor ;
59use rustc_middle:: ty:: layout:: { FnAbiOf , HasTyCtxt , TyAndLayout } ;
10+ use rustc_middle:: ty:: TyCtxt ;
611use rustc_middle:: ty:: { self , Instance , Ty , TyCtxt , TypeFoldable , TypeVisitableExt } ;
12+ use rustc_span:: DUMMY_SP ;
713use rustc_target:: abi:: call:: { FnAbi , PassMode } ;
814
915use std:: iter;
@@ -43,6 +49,9 @@ pub struct FunctionCx<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> {
4349
4450 fn_abi : & ' tcx FnAbi < ' tcx , Ty < ' tcx > > ,
4551
52+ // Used to replace call terminators with a call to a thread local shim.
53+ call_thread_local_shims : Vec < ( mir:: Operand < ' tcx > , DefId ) > ,
54+
4655 /// When unwinding is initiated, we have to store this personality
4756/// value somewhere so that we can load it and re-use it in the
4857/// resume instruction. The personality is (afaik) some kind of
@@ -142,6 +151,112 @@ impl<'a, 'tcx, V: CodegenObject> LocalRef<'tcx, V> {
142151 }
143152}
144153
154+ struct FindThreadLocal ( bool ) ;
155+
156+ impl < ' tcx > Visitor < ' tcx > for FindThreadLocal {
157+ fn visit_rvalue ( & mut self , rvalue : & mir:: Rvalue < ' tcx > , location : mir:: Location ) {
158+ if let mir:: Rvalue :: ThreadLocalRef ( ..) = rvalue {
159+ self . 0 = true ;
160+ }
161+ self . super_rvalue ( rvalue, location) ;
162+ }
163+ }
164+
165+ struct ReplaceThreadLocal < ' tcx > {
166+ tcx : TyCtxt < ' tcx > ,
167+ local_start : usize ,
168+ list : Vec < DefId > ,
169+ }
170+
171+ impl < ' tcx > MutVisitor < ' tcx > for ReplaceThreadLocal < ' tcx > {
172+ fn tcx ( & self ) -> TyCtxt < ' tcx > {
173+ self . tcx
174+ }
175+
176+ fn visit_rvalue ( & mut self , rvalue : & mut mir:: Rvalue < ' tcx > , location : mir:: Location ) {
177+ if let mir:: Rvalue :: ThreadLocalRef ( def_id) = * rvalue {
178+ if self . tcx . is_in_upstream_dylib ( def_id. krate ) {
179+ * rvalue = mir:: Rvalue :: Use ( mir:: Operand :: Copy ( mir:: Place {
180+ local : mir:: Local :: new ( self . local_start + self . list . len ( ) ) ,
181+ projection : self . tcx . intern_place_elems ( & [ ] ) ,
182+ } ) ) ;
183+ self . list . push ( def_id) ;
184+ }
185+ }
186+ self . super_rvalue ( rvalue, location) ;
187+ }
188+ }
189+
190+ // Convert thread local references to thread local function shims if necessary
191+ fn convert_tls_rvalues < ' tcx > (
192+ tcx : TyCtxt < ' tcx > ,
193+ mir : & mut & ' tcx mir:: Body < ' tcx > ,
194+ ) -> Vec < ( mir:: Operand < ' tcx > , DefId ) > {
195+ if tcx. sess . target . dll_tls_export {
196+ // The target supports DLL TLS exports. We don't need to do anything
197+ return Vec :: new ( ) ;
198+ }
199+
200+ // Fast path to look for any thread locals
201+ let mut visitor = FindThreadLocal ( false ) ;
202+ visitor. visit_body ( & mir) ;
203+ if !visitor. 0 {
204+ return Vec :: new ( ) ;
205+ }
206+
207+ // Don't modify shims
208+ if let ty:: InstanceDef :: ThreadLocalShim ( ..) = mir. source . instance {
209+ return Vec :: new ( ) ;
210+ }
211+
212+ let mut result = Vec :: new ( ) ;
213+ let mut body = mir. clone ( ) ;
214+
215+ let mut visitor =
216+ ReplaceThreadLocal { tcx, local_start : mir. local_decls . len ( ) , list : Vec :: new ( ) } ;
217+ visitor. visit_body ( & mut body) ;
218+
219+ for ( i, & def_id) in visitor. list . iter ( ) . enumerate ( ) {
220+ let ty = mir:: Rvalue :: ThreadLocalRef ( def_id) . ty ( & IndexVec :: new ( ) , tcx) ;
221+ body. local_decls . push ( mir:: LocalDecl :: new ( ty, DUMMY_SP ) ) ;
222+ let local = mir:: Local :: new ( visitor. local_start + i) ;
223+ let place = mir:: Place { local, projection : tcx. intern_place_elems ( & [ ] ) } ;
224+ let func = mir:: Operand :: Copy ( place) ;
225+
226+ result. push ( ( func. clone ( ) , def_id) ) ;
227+
228+ let blocks = body. basic_blocks . as_mut ( ) ;
229+
230+ let new_entry = mir:: BasicBlock :: new ( blocks. len ( ) ) ;
231+
232+ let entry = std:: mem:: replace (
233+ & mut blocks[ mir:: BasicBlock :: new ( 0 ) ] ,
234+ mir:: BasicBlockData {
235+ statements : Vec :: new ( ) ,
236+ terminator : Some ( mir:: Terminator {
237+ source_info : mir:: SourceInfo :: outermost ( DUMMY_SP ) ,
238+ kind : mir:: TerminatorKind :: Call {
239+ func,
240+ args : Vec :: new ( ) ,
241+ destination : place,
242+ target : Some ( new_entry) ,
243+ cleanup : None ,
244+ from_hir_call : false ,
245+ fn_span : DUMMY_SP ,
246+ } ,
247+ } ) ,
248+ is_cleanup : false ,
249+ } ,
250+ ) ;
251+
252+ blocks. push ( entry) ;
253+ }
254+
255+ * mir = tcx. arena . alloc ( body) ;
256+
257+ result
258+ }
259+
145260///////////////////////////////////////////////////////////////////////////
146261
147262#[ instrument( level = "debug" , skip( cx) ) ]
@@ -153,7 +268,9 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
153268
154269 let llfn = cx. get_fn ( instance) ;
155270
156- let mir = cx. tcx ( ) . instance_mir ( instance. def ) ;
271+ let mut mir = cx. tcx ( ) . instance_mir ( instance. def ) ;
272+
273+ let call_thread_local_shims = convert_tls_rvalues ( cx. tcx ( ) , & mut mir) ;
157274
158275 let fn_abi = cx. fn_abi_of_instance ( instance, ty:: List :: empty ( ) ) ;
159276 debug ! ( "fn_abi: {:?}" , fn_abi) ;
@@ -183,6 +300,7 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
183300 llfn,
184301 fn_abi,
185302 cx,
303+ call_thread_local_shims,
186304 personality_slot : None ,
187305 cached_llbbs,
188306 unreachable_block : None ,
0 commit comments