twizzler_driver/dma/
region.rs

1use core::{marker::PhantomData, ops::Range};
2use std::sync::Arc;
3
4use twizzler::{
5    error::{GenericError, ResourceError, TwzError},
6    object::RawObject,
7};
8use twizzler_abi::{
9    kso::{
10        pack_kaction_pin_start_and_len, unpack_kaction_pin_token_and_len, KactionCmd, KactionFlags,
11        KactionGenericCmd,
12    },
13    object::NULLPAGE_SIZE,
14    syscall::{sys_kaction, PinnedPage},
15};
16
17use super::{
18    pin::PhysInfo,
19    pool::{AllocatableDmaObject, SplitPageRange},
20    Access, DeviceSync, DmaObject, DmaOptions, DmaPin, SyncMode,
21};
22use crate::arch::DMA_PAGE_SIZE;
23
24/// A region of DMA memory, represented in virtual memory as type `T`, with a particular access mode
25/// and options.
26pub struct DmaRegion<T: DeviceSync> {
27    virt: *mut u8,
28    backing: Option<(Vec<PhysInfo>, u32)>,
29    len: usize,
30    access: Access,
31    dma: Option<DmaObject>,
32    pool: Option<(Arc<AllocatableDmaObject>, SplitPageRange)>,
33    options: DmaOptions,
34    offset: usize,
35    _pd: PhantomData<T>,
36}
37
38impl<T: DeviceSync> core::fmt::Debug for DmaRegion<T> {
39    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
40        f.debug_struct("DmaRegion")
41            .field("len", &self.len)
42            .finish_non_exhaustive()
43    }
44}
45
46unsafe impl<T: DeviceSync> Send for DmaRegion<T> {}
47
48/// A region of DMA memory, represented in virtual memory as type `[T; len]`, with a particular
49/// access mode and options.
50#[derive(Debug)]
51pub struct DmaSliceRegion<T: DeviceSync> {
52    region: DmaRegion<T>,
53    len: usize,
54}
55
56impl<'a, T: DeviceSync> DmaRegion<T> {
57    pub(super) fn new(
58        len: usize,
59        access: Access,
60        options: DmaOptions,
61        offset: usize,
62        pool: Option<(Arc<AllocatableDmaObject>, SplitPageRange)>,
63    ) -> Self {
64        Self {
65            virt: unsafe {
66                (pool
67                    .as_ref()
68                    .unwrap()
69                    .0
70                    .dma_object()
71                    .object()
72                    .base_mut_ptr::<u8>())
73                .add(offset)
74                .sub(NULLPAGE_SIZE)
75            },
76            len,
77            access,
78            options,
79            backing: None,
80            pool,
81            offset,
82            _pd: PhantomData,
83            dma: None,
84        }
85    }
86
87    pub(super) fn new_with_virt(
88        len: usize,
89        access: Access,
90        options: DmaOptions,
91        offset: usize,
92        virt: *mut u8,
93        dma: DmaObject,
94    ) -> Self {
95        Self {
96            virt,
97            len,
98            access,
99            options,
100            backing: None,
101            pool: None,
102            offset,
103            _pd: PhantomData,
104            dma: Some(dma),
105        }
106    }
107
108    pub(super) fn fill(&mut self, init: T) {
109        let p = self.virt as *mut T;
110        unsafe {
111            p.write_volatile(init);
112            self.sync(SyncMode::FullCoherence);
113        }
114    }
115
116    fn dma_object(&self) -> &DmaObject {
117        if let Some(dma) = self.dma.as_ref() {
118            return dma;
119        }
120        self.pool.as_ref().unwrap().0.dma_object()
121    }
122
123    /// Calculate the number of pages this region covers.
124    pub fn nr_pages(&self) -> usize {
125        (self.len - 1) / DMA_PAGE_SIZE + 1
126    }
127
128    fn setup_backing(&mut self) -> Result<(), TwzError> {
129        if self.backing.is_some() {
130            return Ok(());
131        }
132        let mut pins = Vec::new();
133        let len = self.nr_pages();
134        pins.resize(len, PinnedPage::new(0));
135
136        // The kaction call here compresses start and len into a u64, and returns the token and len
137        // in the return u64. This is all because of the limited registers in a syscall.
138        let start = (self.offset / DMA_PAGE_SIZE) as u64;
139        let ptr = (&pins).as_ptr() as u64;
140        let res = sys_kaction(
141            KactionCmd::Generic(KactionGenericCmd::PinPages(0)),
142            Some(self.dma_object().object().id()),
143            ptr,
144            pack_kaction_pin_start_and_len(start, len).ok_or(GenericError::Other)?,
145            KactionFlags::empty(),
146        )?
147        .u64()
148        .ok_or(GenericError::Other)?;
149
150        let (token, retlen) = unpack_kaction_pin_token_and_len(res).ok_or(GenericError::Other)?;
151
152        if retlen < len {
153            return Err(ResourceError::OutOfResources.into());
154        } else if retlen > len {
155            return Err(GenericError::Other.into());
156        }
157
158        let backing: Result<Vec<_>, _> = pins
159            .iter()
160            .map(|p| p.physical_address().try_into().map(|pa| PhysInfo::new(pa)))
161            .collect();
162
163        self.backing = Some((backing.map_err(|_| GenericError::Other)?, token));
164
165        Ok(())
166    }
167
168    /// Return the number of bytes this region covers.
169    pub fn num_bytes(&self) -> usize {
170        self.len
171    }
172
173    /// Return the access direction of this region.
174    pub fn access(&self) -> Access {
175        self.access
176    }
177
178    // Determines the backing information for region. This includes acquiring physical addresses for
179    // the region and holding a pin for the pages.
180    pub fn pin(&mut self) -> Result<DmaPin<'_>, TwzError> {
181        self.setup_backing()?;
182        Ok(DmaPin::new(&self.backing.as_ref().unwrap().0))
183    }
184
185    // Synchronize the region for cache coherence.
186    pub fn sync(&self, sync: SyncMode) {
187        crate::arch::sync(self, sync, 0, self.len);
188    }
189
190    // Run a closure that takes a reference to the DMA data, ensuring coherence.
191    pub fn with<F, R>(&self, f: F) -> R
192    where
193        F: FnOnce(&T) -> R,
194    {
195        if !self.options.contains(DmaOptions::UNSAFE_MANUAL_COHERENCE) {
196            if self.access() != Access::HostToDevice {
197                self.sync(SyncMode::PostDeviceToCpu);
198            }
199        }
200        let data = unsafe { self.get() };
201        let ret = f(data);
202        ret
203    }
204
205    // Run a closure that takes a mutable reference to the DMA data, ensuring coherence.
206    pub fn with_mut<F, R>(&mut self, f: F) -> R
207    where
208        F: FnOnce(&mut T) -> R,
209    {
210        if !self.options.contains(DmaOptions::UNSAFE_MANUAL_COHERENCE) {
211            match self.access() {
212                Access::HostToDevice => self.sync(SyncMode::PreCpuToDevice),
213                Access::DeviceToHost => self.sync(SyncMode::PostDeviceToCpu),
214                Access::BiDirectional => self.sync(SyncMode::FullCoherence),
215            }
216        }
217        let data = unsafe { self.get_mut() };
218        let ret = f(data);
219        if !self.options.contains(DmaOptions::UNSAFE_MANUAL_COHERENCE) {
220            if self.access() != Access::DeviceToHost {
221                self.sync(SyncMode::PostCpuToDevice);
222            }
223        }
224        ret
225    }
226
227    /// Release any pin created for this region.
228    ///
229    /// # Safety
230    /// Caller must ensure that no device is using the information from any active pins for this
231    /// region.
232    pub unsafe fn release_pin(&mut self) {
233        if let Some((_, token)) = self.backing {
234            super::object::release_pin(self.dma_object().object().id(), token);
235            self.backing = None;
236        }
237    }
238
239    /// Get a reference to the DMA memory.
240    ///
241    /// # Safety
242    /// The caller must ensure coherence is applied.
243    #[inline]
244    pub unsafe fn get(&self) -> &T {
245        (self.virt as *const T).as_ref().unwrap()
246    }
247
248    /// Get a mutable reference to the DMA memory.
249    ///
250    /// # Safety
251    /// The caller must ensure coherence is applied.
252    #[inline]
253    pub unsafe fn get_mut(&mut self) -> &mut T {
254        (self.virt as *mut T).as_mut().unwrap()
255    }
256}
257
258impl<'a, T: DeviceSync> DmaSliceRegion<T> {
259    pub(super) fn new(
260        nrbytes: usize,
261        access: Access,
262        options: DmaOptions,
263        offset: usize,
264        len: usize,
265        pool: Option<(Arc<AllocatableDmaObject>, SplitPageRange)>,
266    ) -> Self {
267        Self {
268            region: DmaRegion::new(nrbytes, access, options, offset, pool),
269            len,
270        }
271    }
272
273    pub(super) fn new_with_virt(
274        nrbytes: usize,
275        access: Access,
276        options: DmaOptions,
277        offset: usize,
278        len: usize,
279        virt: *mut u8,
280        dma: DmaObject,
281    ) -> Self {
282        Self {
283            region: DmaRegion::new_with_virt(nrbytes, access, options, offset, virt, dma),
284            len,
285        }
286    }
287
288    pub(super) fn fill(&mut self, init: T)
289    where
290        T: Clone,
291    {
292        let p = self.region.virt as *mut T;
293        for idx in 0..self.len {
294            unsafe {
295                p.add(idx).write_volatile(init.clone());
296            }
297        }
298        self.sync(0..self.len, SyncMode::FullCoherence);
299    }
300
301    pub(super) fn fill_with(&mut self, init: impl Fn() -> T) {
302        let p = self.region.virt as *mut T;
303        for idx in 0..self.len {
304            unsafe {
305                p.add(idx).write_volatile(init());
306            }
307        }
308        self.sync(0..self.len, SyncMode::FullCoherence);
309    }
310
311    /// Return the number of bytes this region covers.
312    pub fn num_bytes(&self) -> usize {
313        self.region.len
314    }
315
316    #[inline]
317    /// Return the access direction of this region.
318    pub fn access(&self) -> Access {
319        self.region.access()
320    }
321
322    /// Return the number of elements in the slice that this region covers.
323    pub fn len(&self) -> usize {
324        self.len
325    }
326
327    // Determines the backing information for region. This includes acquiring physical addresses for
328    // the region and holding a pin for the pages.
329    #[inline]
330    pub fn pin(&mut self) -> Result<DmaPin<'_>, TwzError> {
331        self.region.pin()
332    }
333
334    // Synchronize a subslice of the region for cache coherence.
335    pub fn sync(&self, range: Range<usize>, sync: SyncMode) {
336        let start = range.start * core::mem::size_of::<T>();
337        let len = range.len() * core::mem::size_of::<T>();
338        crate::arch::sync(&self.region, sync, start, len);
339    }
340
341    // Run a closure that takes a reference to a subslice of the DMA data, ensuring coherence.
342    pub fn with<F, R>(&self, range: Range<usize>, f: F) -> R
343    where
344        F: FnOnce(&[T]) -> R,
345    {
346        if !self
347            .region
348            .options
349            .contains(DmaOptions::UNSAFE_MANUAL_COHERENCE)
350        {
351            if self.access() != Access::HostToDevice {
352                self.sync(range.clone(), SyncMode::PostDeviceToCpu);
353            }
354        }
355        let data = &unsafe { self.get() }[range];
356        let ret = f(data);
357        ret
358    }
359
360    // Run a closure that takes a mutable reference to a subslice of the DMA data, ensuring
361    // coherence.
362    pub fn with_mut<F, R>(&mut self, range: Range<usize>, f: F) -> R
363    where
364        F: FnOnce(&mut [T]) -> R,
365    {
366        if !self
367            .region
368            .options
369            .contains(DmaOptions::UNSAFE_MANUAL_COHERENCE)
370        {
371            match self.access() {
372                Access::HostToDevice => self.sync(range.clone(), SyncMode::PreCpuToDevice),
373                Access::DeviceToHost => self.sync(range.clone(), SyncMode::PostDeviceToCpu),
374                Access::BiDirectional => self.sync(range.clone(), SyncMode::FullCoherence),
375            }
376        }
377        let data = &mut unsafe { self.get_mut() }[range.clone()];
378        let ret = f(data);
379        if !self
380            .region
381            .options
382            .contains(DmaOptions::UNSAFE_MANUAL_COHERENCE)
383        {
384            if self.access() != Access::DeviceToHost {
385                self.sync(range, SyncMode::PostCpuToDevice);
386            }
387        }
388        ret
389    }
390
391    /// Release any pin created for this region.
392    ///
393    /// # Safety
394    /// Caller must ensure that no device is using the information from any active pins for this
395    /// region.
396    #[inline]
397    pub unsafe fn release_pin(&mut self) {
398        self.region.release_pin()
399    }
400
401    /// Get a reference to the DMA memory.
402    ///
403    /// # Safety
404    /// The caller must ensure coherence is applied.
405    #[inline]
406    pub unsafe fn get(&self) -> &[T] {
407        core::slice::from_raw_parts(self.region.virt as *const T, self.len)
408    }
409
410    /// Get a mutable reference to the DMA memory.
411    ///
412    /// # Safety
413    /// The caller must ensure coherence is applied.
414    #[inline]
415    pub unsafe fn get_mut(&mut self) -> &mut [T] {
416        core::slice::from_raw_parts_mut(self.region.virt as *mut T, self.len)
417    }
418}
419
420impl<'a, T: DeviceSync> Drop for DmaRegion<T> {
421    fn drop(&mut self) {
422        if let Some((_, token)) = self.backing.as_ref() {
423            self.dma_object()
424                .releasable_pins
425                .lock()
426                .unwrap()
427                .push(*token);
428        }
429
430        if let Some((ado, range)) = self.pool.take() {
431            ado.free(range);
432        }
433    }
434}
twizzler_driver/dma/region.rs

twizzler_driver/dma/
region.rs
