relibc/ld_so/
tcb.rs

1use alloc::vec::Vec;
2use core::{
3    cell::UnsafeCell,
4    mem,
5    ops::{Deref, DerefMut},
6    ptr, slice,
7    sync::atomic::AtomicBool,
8};
9use generic_rt::GenericTcb;
10
11use crate::{
12    header::sys_mman,
13    ld_so::linker::Linker,
14    platform::{Dlmalloc, Pal, Sys},
15    pthread::{OsTid, Pthread},
16    sync::{mutex::Mutex, waitval::Waitval},
17};
18
19use super::linker::DlError;
20
21#[repr(C)]
22#[derive(Debug, Clone)]
23pub struct Master {
24    /// Pointer to initial data
25    pub ptr: *const u8,
26    /// Length of initial data in bytes
27    pub image_size: usize,
28    pub segment_size: usize,
29    /// Offset in TLS to copy initial data to
30    pub offset: usize,
31}
32
33impl Master {
34    /// The initial data for this TLS region
35    pub unsafe fn data(&self) -> &'static [u8] {
36        unsafe { slice::from_raw_parts(self.ptr, self.image_size) }
37    }
38}
39
40#[cfg(target_os = "linux")]
41pub type OsSpecific = ();
42
43#[cfg(target_os = "redox")]
44pub type OsSpecific = redox_rt::signal::RtSigarea;
45
46#[derive(Debug)]
47#[repr(C)]
48// FIXME: Only return &Tcb, and use interior mutability, since it contains the Pthread struct
49pub struct Tcb {
50    pub generic: GenericTcb<OsSpecific>,
51    /// Pointer to a list of initial TLS data
52    pub masters_ptr: *mut Master,
53    /// Size of the masters list in bytes (multiple of `mem::size_of::<Master>()`)
54    pub masters_len: usize,
55    /// Index of last copied Master
56    pub num_copied_masters: usize,
57    /// Pointer to dynamic linker
58    pub linker_ptr: *const Mutex<Linker>,
59    /// pointer to rust memory allocator structure
60    pub mspace: *const Mutex<Dlmalloc>,
61    /// Underlying pthread_t struct, pthread_self() returns &self.pthread
62    pub pthread: Pthread,
63
64    // Dynamic TLS Vector
65    pub dtv_ptr: *mut *mut u8,
66    // Number of DTV entries.
67    pub dtv_len: usize,
68}
69
70#[cfg(target_os = "redox")]
71const _: () = {
72    if mem::size_of::<Tcb>() > syscall::PAGE_SIZE {
73        panic!("too large TCB!");
74    }
75};
76
77impl Tcb {
78    /// Create a new TCB
79    ///
80    /// `size` is the size of the TLS in bytes.
81    #[allow(unsafe_op_in_unsafe_fn)]
82    pub unsafe fn new(size: usize) -> Result<&'static mut Self, DlError> {
83        let page_size = Sys::getpagesize();
84        let (_abi_page, tls, tcb_page) = Self::os_new(size.next_multiple_of(page_size))?;
85
86        let tcb_ptr = tcb_page.as_mut_ptr().cast::<Self>();
87        ptr::write(
88            tcb_ptr,
89            Self {
90                generic: GenericTcb {
91                    tls_end: tls.as_mut_ptr().add(tls.len()),
92                    tls_len: tls.len(),
93                    tcb_ptr: tcb_ptr.cast(),
94                    tcb_len: tcb_page.len(),
95                    os_specific: OsSpecific::default(),
96                },
97                masters_ptr: ptr::null_mut(),
98                masters_len: 0,
99                num_copied_masters: 0,
100                linker_ptr: ptr::null(),
101                mspace: ptr::null(),
102                pthread: Pthread {
103                    waitval: Waitval::new(),
104                    flags: Default::default(),
105                    has_enabled_cancelation: AtomicBool::new(false),
106                    has_queued_cancelation: AtomicBool::new(false),
107                    stack_base: core::ptr::null_mut(),
108                    stack_size: 0,
109                    os_tid: UnsafeCell::new(OsTid::default()),
110                },
111
112                dtv_ptr: ptr::null_mut(),
113                dtv_len: 0,
114            },
115        );
116
117        Ok(&mut *tcb_ptr)
118    }
119
120    /// Get the current TCB
121    pub unsafe fn current() -> Option<&'static mut Self> {
122        unsafe { Some(&mut *GenericTcb::<OsSpecific>::current_ptr()?.cast()) }
123    }
124
125    /// A slice for all of the TLS data
126    pub unsafe fn tls(&self) -> Option<&'static mut [u8]> {
127        if self.tls_end.is_null() || self.tls_len == 0 {
128            None
129        } else {
130            unsafe {
131                let tls_start = self.tls_end.sub(self.tls_len);
132                Some(slice::from_raw_parts_mut(tls_start, self.tls_len))
133            }
134        }
135    }
136
137    /// The initial images for TLS
138    pub fn masters(&self) -> Option<&'static mut [Master]> {
139        if self.masters_ptr.is_null() || self.masters_len == 0 {
140            None
141        } else {
142            Some(unsafe {
143                slice::from_raw_parts_mut(
144                    self.masters_ptr,
145                    self.masters_len / mem::size_of::<Master>(),
146                )
147            })
148        }
149    }
150
151    /// Copy data from masters
152    pub unsafe fn copy_masters(&mut self) -> Result<(), DlError> {
153        //TODO: Complain if masters or tls exist without the other
154        if let Some(tls) = unsafe { self.tls() }
155            && let Some(masters) = self.masters()
156        {
157            for master in masters
158                .iter()
159                .skip(self.num_copied_masters)
160                .filter(|master| master.image_size != 0)
161            {
162                let range = if cfg!(any(target_arch = "x86", target_arch = "x86_64")) {
163                    // x86{_64} TLS layout is backwards
164                    self.tls_len - master.offset..self.tls_len - master.offset + master.image_size
165                } else {
166                    master.offset..master.offset + master.image_size
167                };
168                if let Some(tls_data) = tls.get_mut(range) {
169                    let data = unsafe { master.data() };
170                    #[cfg(feature = "trace_tls")]
171                    log::trace!(
172                        "tls master: {:p}, {:#x}: {:p}, {:#x}",
173                        data.as_ptr(),
174                        data.len(),
175                        tls_data.as_mut_ptr(),
176                        tls_data.len()
177                    );
178                    tls_data.copy_from_slice(data);
179                } else {
180                    return Err(DlError::Malformed);
181                }
182            }
183            self.num_copied_masters = masters.len();
184        }
185
186        Ok(())
187    }
188
189    /// The initial images for TLS
190    pub unsafe fn append_masters(&mut self, mut new_masters: Vec<Master>) {
191        if self.masters_ptr.is_null() {
192            self.masters_ptr = new_masters.as_mut_ptr();
193            self.masters_len = new_masters.len() * mem::size_of::<Master>();
194            mem::forget(new_masters);
195        } else {
196            // XXX: [`Vec::from_raw_parts`] cannot be used here as the masters were originally
197            // allocated by the ld.so allocator and that would violate that function's invariants.
198            let mut masters = self.masters().unwrap().to_vec();
199            masters.extend(new_masters);
200
201            self.masters_ptr = masters.as_mut_ptr();
202            self.masters_len = masters.len() * mem::size_of::<Master>();
203            mem::forget(masters);
204        }
205    }
206
207    /// Activate TLS
208    pub unsafe fn activate(
209        &mut self,
210        #[cfg(target_os = "redox")] thr_fd: Option<redox_rt::proc::FdGuardUpper>,
211    ) {
212        unsafe {
213            Self::os_arch_activate(
214                &self.os_specific,
215                self.tls_end as usize,
216                self.tls_len,
217                #[cfg(target_os = "redox")]
218                thr_fd,
219            )
220        };
221    }
222
223    pub fn setup_dtv(&mut self, n: usize) {
224        if self.dtv_ptr.is_null() {
225            let mut dtv = vec![ptr::null_mut(); n];
226
227            if let Some(masters) = self.masters() {
228                for (i, master) in masters.iter().enumerate() {
229                    let tls = unsafe { self.tls().unwrap() };
230                    let offset = if cfg!(any(target_arch = "x86", target_arch = "x86_64")) {
231                        // x86{_64} TLS layout is backwards
232                        self.tls_len - master.offset
233                    } else {
234                        master.offset
235                    };
236
237                    dtv[i] = unsafe { tls.as_mut_ptr().add(offset) };
238                }
239            }
240
241            let (ptr, len, _) = dtv.into_raw_parts();
242
243            self.dtv_ptr = ptr;
244            self.dtv_len = len;
245        } else {
246            // Resize DTV.
247            //
248            // XXX: [`Vec::from_raw_parts`] cannot be used here as the DTV was originally allocated
249            // by the ld.so allocator and that would violate that function's invariants.
250            let mut dtv = self.dtv_mut().to_vec();
251            dtv.resize(n, ptr::null_mut());
252
253            let (ptr, len, _) = dtv.into_raw_parts();
254            self.dtv_ptr = ptr;
255            self.dtv_len = len;
256        }
257    }
258
259    pub fn dtv_mut(&mut self) -> &'static mut [*mut u8] {
260        if self.dtv_len != 0 {
261            unsafe { slice::from_raw_parts_mut(self.dtv_ptr, self.dtv_len) }
262        } else {
263            &mut []
264        }
265    }
266
267    /// Mapping with correct flags for TCB and TLS
268    #[allow(unsafe_op_in_unsafe_fn)]
269    unsafe fn map(size: usize) -> Result<&'static mut [u8], DlError> {
270        let ptr = Sys::mmap(
271            ptr::null_mut(),
272            size,
273            sys_mman::PROT_READ | sys_mman::PROT_WRITE,
274            sys_mman::MAP_ANONYMOUS | sys_mman::MAP_PRIVATE,
275            -1,
276            0,
277        )
278        .map_err(|_| DlError::Oom)?;
279
280        ptr::write_bytes(ptr.cast::<u8>(), 0, size);
281        Ok(slice::from_raw_parts_mut(ptr.cast::<u8>(), size))
282    }
283
284    /// OS specific code to create a new TLS and TCB - Linux and Redox
285    ///
286    /// Memory layout:
287    ///
288    /// ```text
289    /// 0          page_size                   size       (size + page_size * 2)
290    /// |----------|---------------------------|----------|
291    /// +++++++++++++++++++++++++++++++++++++++++++++++++++
292    /// | ABI Page | TLS                       | TCB Page |
293    /// +++++++++++++++++++++++++++++++++++++++++++++++++++
294    ///     ^ $tp (aarch64)                    ^ $tp (x86_64)
295    /// ```
296    ///
297    /// `$tp` refers to the architecture specific thread pointer.
298    ///
299    /// **Note**: On x86{_64}, the TLS layout is backwards (i.e. the first byte of the TLS is at
300    /// the end of the TLS region).
301    ///
302    /// ABI page layout for aarch64:
303    /// ```text
304    /// 0                     4096
305    /// +---------------------+
306    /// | ABI Page            |
307    /// +---------------------+
308    ///                     ^
309    ///                     |
310    ///                     +-------> (page_size - 16): pointer to the start of the TCB page
311    /// ```
312    ///
313    /// ABI page layout for riscv64:
314    ///
315    /// ```text
316    /// 0                     4096
317    /// +---------------------+
318    /// | ABI Page            |
319    /// +---------------------+
320    ///                      ^
321    ///                      |
322    ///                      +-------> (page_size - 8): pointer to the start of the TCB page
323    /// ```
324    ///
325    /// For x86_64, the ABI page is not used.
326    #[cfg(any(target_os = "linux", target_os = "redox"))]
327    unsafe fn os_new(
328        size: usize,
329    ) -> Result<(&'static mut [u8], &'static mut [u8], &'static mut [u8]), DlError> {
330        let page_size = Sys::getpagesize();
331        let abi_tls_tcb = unsafe { Self::map(page_size + size + page_size)? };
332        let (abi, tls_tcb) = abi_tls_tcb.split_at_mut(page_size);
333        let (tls, tcb) = tls_tcb.split_at_mut(size);
334        Ok((abi, tls, tcb))
335    }
336
337    /// OS and architecture specific code to activate TLS - Linux x86_64
338    #[cfg(all(target_os = "linux", target_arch = "x86_64"))]
339    unsafe fn os_arch_activate(_os: &(), tls_end: usize, _tls_len: usize) {
340        const ARCH_SET_FS: usize = 0x1002;
341        unsafe {
342            syscall!(ARCH_PRCTL, ARCH_SET_FS, tls_end);
343        }
344    }
345
346    #[cfg(all(target_os = "linux", target_arch = "aarch64"))]
347    unsafe fn os_arch_activate(_os: &(), tls_end: usize, tls_len: usize) {
348        // Uses ABI page
349        let abi_ptr = tls_end - tls_len - 16;
350        unsafe {
351            core::ptr::write(abi_ptr as *mut usize, tls_end);
352            core::arch::asm!(
353                "msr tpidr_el0, {}",
354                in(reg) abi_ptr,
355            );
356        }
357    }
358
359    #[cfg(target_os = "redox")]
360    unsafe fn os_arch_activate(
361        os: &OsSpecific,
362        tls_end: usize,
363        tls_len: usize,
364        thr_fd: Option<redox_rt::proc::FdGuardUpper>,
365    ) {
366        unsafe {
367            if let Some(thr_fd) = thr_fd {
368                os.thr_fd.get().write(Some(thr_fd));
369            }
370            redox_rt::tcb_activate(os, tls_end, tls_len)
371        }
372    }
373}
374
375impl Deref for Tcb {
376    type Target = GenericTcb<OsSpecific>;
377
378    fn deref(&self) -> &Self::Target {
379        &self.generic
380    }
381}
382impl DerefMut for Tcb {
383    fn deref_mut(&mut self) -> &mut Self::Target {
384        &mut self.generic
385    }
386}
relibc/ld_so/tcb.rs

relibc/ld_so/
tcb.rs
