1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140

use std::{
    marker::PhantomData,
    sync::atomic::{AtomicU64, Ordering},
};

use crate::Object;

/// The raw invariant pointer, containing just a 64-bit packed FOT entry and offset.
#[repr(transparent)]
pub struct InvPtr<T> {
    raw: u64,
    _pd: PhantomData<T>,
}

impl<T> !Unpin for InvPtr<T> {}

impl<T> Object<T> {
    /// Get a raw pointer into an object given an offset.
    #[inline]
    pub fn raw_lea<P>(&self, off: usize) -> *const P {
        self.slot.raw_lea(off)
    }

    /// Get a raw mutable pointer into an object given an offset.
    #[inline]
    pub fn raw_lea_mut<P>(&self, off: usize) -> *mut P {
        self.slot.raw_lea_mut(off)
    }
}

fn ipoffset(raw: u64) -> u64 {
    raw & 0x0000ffffffffffff
}

fn ipfote(raw: u64) -> u64 {
    (raw & !0x0000ffffffffffff) >> 48
}

impl<Target> InvPtr<Target> {
    /// Read the invariant pointer into its raw parts.
    ///
    /// # Safety
    /// See this crate's base documentation ([Isolation Safety](crate)).
    pub unsafe fn parts_unguarded(&self) -> (usize, u64) {
        let raw = self.raw;
        (ipfote(raw) as usize, ipoffset(raw))
    }

    /// Read the invariant pointer into its raw parts.
    pub fn parts(&mut self) -> (usize, u64) {
        let raw = self.raw;
        (ipfote(raw) as usize, ipoffset(raw))
    }

    /// Construct an InvPtr from an FOT entry and an offset.
    pub fn from_parts(fote: usize, off: u64) -> Self {
        Self {
            raw: (fote << 48) as u64 | (off & 0x0000ffffffffffff),
            _pd: PhantomData,
        }
    }

    /// Check if an invariant pointer is null.
    ///
    /// # Safety
    /// See this crate's base documentation ([Isolation Safety](crate)).
    pub unsafe fn is_null_unguarded(&self) -> bool {
        self.raw == 0
    }

    /// Check if an invariant pointer is null.
    pub fn is_null(&mut self) -> bool {
        self.raw == 0
    }

    /// Construct a null raw pointer.
    pub fn null() -> Self {
        Self {
            raw: 0,
            _pd: PhantomData,
        }
    }

    /// Get a reference to the inner raw 64 bits of the invariant pointer.
    ///
    /// # Safety
    /// See this crate's base documentation ([Isolation Safety](crate)). Additionally, the caller is
    /// expected to maintain the correct semantics of invariant pointers.
    pub unsafe fn raw_inner(&mut self) -> *mut u64 {
        &mut self.raw as *mut u64
    }
}

/// An atomic invariant pointer. Allows reading through an immutable reference without unsafe.
#[repr(transparent)]
pub struct AtomicInvPtr<T> {
    raw: AtomicU64,
    _pd: PhantomData<T>,
}

impl<T> !Unpin for AtomicInvPtr<T> {}

impl<Target> AtomicInvPtr<Target> {
    /// Read the invariant pointer into its raw parts.
    pub fn parts(&self) -> (usize, u64) {
        let raw = self.raw.load(Ordering::SeqCst);
        (ipfote(raw) as usize, ipoffset(raw))
    }

    /// Construct an InvPtr from an FOT entry and an offset.
    pub fn from_parts(fote: usize, off: u64) -> Self {
        Self {
            raw: AtomicU64::new((fote << 48) as u64 | (off & 0x0000ffffffffffff)),
            _pd: PhantomData,
        }
    }

    /// Check if an invariant pointer is null.
    pub fn is_null(&self) -> bool {
        let raw = self.raw.load(Ordering::SeqCst);
        raw == 0
    }

    /// Construct a null raw pointer.
    pub fn null() -> Self {
        Self {
            raw: AtomicU64::new(0),
            _pd: PhantomData,
        }
    }

    /// Get a reference to the inner raw 64 bits of the invariant pointer.
    ///
    /// # Safety
    /// See this crate's base documentation ([Isolation Safety](crate)). Additionally, the caller is
    /// expected to maintain the correct semantics of invariant pointers.
    pub unsafe fn inner(&mut self) -> *mut AtomicU64 {
        &mut self.raw as *mut AtomicU64
    }
}