nyx_lite/
mem.rs

1use core::fmt;
2use std::ffi::c_void;
3use std::io::{self, Read, Seek, SeekFrom, Write};
4use std::sync::{Arc, Mutex};
5use std::{
6    iter::Peekable,
7    marker::PhantomData,
8    ops::Range,
9    sync::atomic::{AtomicU64, Ordering},
10};
11
12use crate::error::MemoryError;
13use libc::{PROT_READ, PROT_WRITE, mprotect};
14use vm_memory::bitmap::BS;
15use vm_memory::{AtomicAccess, ByteValued, GuestUsize, VolatileSlice};
16use vmm::Vmm;
17use vmm::vstate::memory::{
18    Address, Bytes, GuestAddress, GuestMemory, GuestMemoryMmap, GuestMemoryRegion, GuestRegionMmap,
19};
20
21type MResult<T> = Result<T, MemoryError>;
22pub const M_PAGE_ALIGN: u64 = 0xffff_ffff_ffff_f000;
23pub const M_PAGE_OFFSET: u64 = 0xfff;
24pub const FRAMES_PER_PAGE_TABLE: u64 = 0x1ff;
25pub const M_PTE_OFFSET: u64 = FRAMES_PER_PAGE_TABLE;
26pub const M_PTE_PADDR: u64 = 0x000f_ffff_ffff_f000;
27pub const PAGE_SIZE: u64 = 0x1000;
28pub const BIT_PTE_PRESENT: u64 = 1;
29const BIT_PTE_WRITABLE: u64 = 1 << 1;
30const BIT_PTE_USER: u64 = 1 << 2;
31const BIT_PTE_NX: u64 = 1 << 63;
32
33/// Tracks host-side writes into guest memory using a lock-free bitmap.
34#[derive(Debug)]
35pub struct HostDirtyTracker {
36    page_count: usize,
37    bits: Vec<AtomicU64>,
38}
39
40impl HostDirtyTracker {
41    pub fn new(page_count: usize) -> Self {
42        let words = page_count.div_ceil(64);
43        let mut bits = Vec::with_capacity(words);
44        bits.resize_with(words, AtomicU64::default);
45        Self { page_count, bits }
46    }
47
48    fn page_index(&self, page_addr: u64) -> Option<usize> {
49        let idx = (page_addr / PAGE_SIZE) as usize;
50        (idx < self.page_count).then_some(idx)
51    }
52
53    pub fn mark_page(&self, page_addr: u64) {
54        let Some(idx) = self.page_index(page_addr & M_PAGE_ALIGN) else {
55            return;
56        };
57        let word = idx / 64;
58        let mask = 1u64 << (idx % 64);
59        if let Some(entry) = self.bits.get(word) {
60            entry.fetch_or(mask, Ordering::Release);
61        }
62    }
63
64    pub fn mark_range_phys(&self, start: u64, len: usize) {
65        if len == 0 {
66            return;
67        }
68        let mut addr = start & M_PAGE_ALIGN;
69        let end = start.checked_add(len as u64 - 1).unwrap_or(u64::MAX) & M_PAGE_ALIGN;
70        while addr <= end {
71            self.mark_page(addr);
72            if let Some(next) = addr.checked_add(PAGE_SIZE) {
73                addr = next;
74            } else {
75                break;
76            }
77        }
78    }
79
80    pub fn take_pages(&self) -> Vec<u64> {
81        let mut pages = Vec::new();
82        for (word_idx, entry) in self.bits.iter().enumerate() {
83            let mut value = entry.swap(0, Ordering::AcqRel);
84            while value != 0 {
85                let bit = value.trailing_zeros() as usize;
86                let page_idx = word_idx * 64 + bit;
87                if page_idx < self.page_count {
88                    pages.push(page_idx as u64 * PAGE_SIZE);
89                }
90                value &= value - 1;
91            }
92        }
93        pages
94    }
95
96    pub fn snapshot_pages(&self) -> Vec<u64> {
97        let mut pages = Vec::new();
98        for (word_idx, entry) in self.bits.iter().enumerate() {
99            let mut value = entry.load(Ordering::Acquire);
100            while value != 0 {
101                let bit = value.trailing_zeros() as usize;
102                let page_idx = word_idx * 64 + bit;
103                if page_idx < self.page_count {
104                    pages.push(page_idx as u64 * PAGE_SIZE);
105                }
106                value &= value - 1;
107            }
108        }
109        pages
110    }
111
112    pub fn clear(&self) {
113        for entry in &self.bits {
114            entry.store(0, Ordering::Release);
115        }
116    }
117
118    pub fn page_count(&self) -> usize {
119        self.page_count
120    }
121}
122
123impl Default for HostDirtyTracker {
124    fn default() -> Self {
125        Self::new(0)
126    }
127}
128
129/// Page-level mapping flags for injected page table entries.
130#[derive(Debug, Clone, Copy, Eq, PartialEq)]
131pub struct PageMapping {
132    pub writable: bool,
133    pub executable: bool,
134    pub user: bool,
135}
136
137impl Default for PageMapping {
138    fn default() -> Self {
139        Self {
140            writable: true,
141            executable: true,
142            user: true,
143        }
144    }
145}
146
147pub trait PageAllocator {
148    fn alloc_page(&mut self) -> Option<GuestAddress>;
149}
150
151pub struct VecPageAllocator {
152    pages: Vec<GuestAddress>,
153}
154
155impl VecPageAllocator {
156    pub fn new(pages: Vec<GuestAddress>) -> Self {
157        Self { pages }
158    }
159}
160
161impl PageAllocator for VecPageAllocator {
162    fn alloc_page(&mut self) -> Option<GuestAddress> {
163        self.pages.pop()
164    }
165}
166
167pub trait VirtualMemoryBackend: Send + Sync {
168    fn read_virtual_bytes(&self, cr3: u64, vaddr: u64, buffer: &mut [u8]) -> MResult<usize>;
169    fn write_virtual_bytes(&self, cr3: u64, vaddr: u64, buffer: &[u8]) -> MResult<usize>;
170    fn read_virtual_u64(&self, cr3: u64, vaddr: u64) -> MResult<u64>;
171    fn write_virtual_u64(&self, cr3: u64, vaddr: u64, val: u64) -> MResult<()>;
172    fn read_phys_u64(&self, paddr: u64) -> MResult<u64>;
173    fn write_phys_u64(&self, paddr: u64, val: u64) -> MResult<()>;
174    fn resolve_vaddr(&self, cr3: u64, vaddr: u64) -> MResult<GuestAddress>;
175}
176
177impl<T> VirtualMemoryBackend for T
178where
179    T: NyxMemExtension + Send + Sync,
180{
181    fn read_virtual_bytes(&self, cr3: u64, vaddr: u64, buffer: &mut [u8]) -> MResult<usize> {
182        NyxMemExtension::read_virtual_bytes(self, cr3, vaddr, buffer)
183    }
184
185    fn write_virtual_bytes(&self, cr3: u64, vaddr: u64, buffer: &[u8]) -> MResult<usize> {
186        NyxMemExtension::write_virtual_bytes(self, cr3, vaddr, buffer)
187    }
188
189    fn read_virtual_u64(&self, cr3: u64, vaddr: u64) -> MResult<u64> {
190        NyxMemExtension::read_virtual_u64(self, cr3, vaddr)
191    }
192
193    fn write_virtual_u64(&self, cr3: u64, vaddr: u64, val: u64) -> MResult<()> {
194        NyxMemExtension::write_virtual_u64(self, cr3, vaddr, val)
195    }
196
197    fn read_phys_u64(&self, paddr: u64) -> MResult<u64> {
198        NyxMemExtension::read_phys(self, paddr)
199    }
200
201    fn write_phys_u64(&self, paddr: u64, val: u64) -> MResult<()> {
202        NyxMemExtension::write_phys(self, paddr, val)
203    }
204
205    fn resolve_vaddr(&self, cr3: u64, vaddr: u64) -> MResult<GuestAddress> {
206        NyxMemExtension::resolve_vaddr(self, cr3, vaddr)
207    }
208}
209
210#[derive(Clone)]
211pub struct LockedVmm {
212    inner: Arc<Mutex<Vmm>>,
213}
214
215impl LockedVmm {
216    pub fn new(inner: Arc<Mutex<Vmm>>) -> Self {
217        Self { inner }
218    }
219}
220
221impl VirtualMemoryBackend for LockedVmm {
222    fn read_virtual_bytes(&self, cr3: u64, vaddr: u64, buffer: &mut [u8]) -> MResult<usize> {
223        let vmm = self.inner.lock().unwrap();
224        vmm.read_virtual_bytes(cr3, vaddr, buffer)
225    }
226
227    fn write_virtual_bytes(&self, cr3: u64, vaddr: u64, buffer: &[u8]) -> MResult<usize> {
228        let vmm = self.inner.lock().unwrap();
229        vmm.write_virtual_bytes(cr3, vaddr, buffer)
230    }
231
232    fn read_virtual_u64(&self, cr3: u64, vaddr: u64) -> MResult<u64> {
233        let vmm = self.inner.lock().unwrap();
234        vmm.read_virtual_u64(cr3, vaddr)
235    }
236
237    fn write_virtual_u64(&self, cr3: u64, vaddr: u64, val: u64) -> MResult<()> {
238        let vmm = self.inner.lock().unwrap();
239        vmm.write_virtual_u64(cr3, vaddr, val)
240    }
241
242    fn read_phys_u64(&self, paddr: u64) -> MResult<u64> {
243        let vmm = self.inner.lock().unwrap();
244        vmm.read_phys(paddr)
245    }
246
247    fn write_phys_u64(&self, paddr: u64, val: u64) -> MResult<()> {
248        let vmm = self.inner.lock().unwrap();
249        vmm.write_phys(paddr, val)
250    }
251
252    fn resolve_vaddr(&self, cr3: u64, vaddr: u64) -> MResult<GuestAddress> {
253        let vmm = self.inner.lock().unwrap();
254        vmm.resolve_vaddr(cr3, vaddr)
255    }
256}
257
258#[derive(Clone)]
259/// Convenience wrapper for guest virtual memory access in a process context (CR3).
260pub struct ProcessMemory<B>
261where
262    B: VirtualMemoryBackend + Clone,
263{
264    backend: B,
265    cr3: u64,
266    host_dirty: Option<Arc<HostDirtyTracker>>,
267}
268
269impl<B> ProcessMemory<B>
270where
271    B: VirtualMemoryBackend + Clone,
272{
273    pub fn new(backend: B, cr3: u64) -> Self {
274        Self {
275            backend,
276            cr3,
277            host_dirty: None,
278        }
279    }
280
281    pub fn with_host_dirty(mut self, tracker: Arc<HostDirtyTracker>) -> Self {
282        self.host_dirty = Some(tracker);
283        self
284    }
285
286    pub fn cr3(&self) -> u64 {
287        self.cr3
288    }
289
290    pub fn resolve_vaddr(&self, vaddr: u64) -> MResult<GuestAddress> {
291        self.backend.resolve_vaddr(self.cr3, vaddr)
292    }
293
294    pub fn read_u64(&self, vaddr: u64) -> MResult<u64> {
295        self.backend.read_virtual_u64(self.cr3, vaddr)
296    }
297
298    pub fn write_u64(&self, vaddr: u64, val: u64) -> MResult<()> {
299        self.backend.write_virtual_u64(self.cr3, vaddr, val)?;
300        self.mark_host_dirty_virtual_range(vaddr, std::mem::size_of::<u64>());
301        Ok(())
302    }
303
304    pub fn read_bytes(&self, vaddr: u64, buffer: &mut [u8]) -> MResult<usize> {
305        self.backend.read_virtual_bytes(self.cr3, vaddr, buffer)
306    }
307
308    pub fn write_bytes(&self, vaddr: u64, buffer: &[u8]) -> MResult<usize> {
309        let bytes = self.backend.write_virtual_bytes(self.cr3, vaddr, buffer)?;
310        self.mark_host_dirty_virtual_range(vaddr, bytes);
311        Ok(bytes)
312    }
313
314    pub fn map_page(
315        &self,
316        vaddr: u64,
317        paddr: u64,
318        mapping: PageMapping,
319        mut allocator: Option<&mut dyn PageAllocator>,
320    ) -> MResult<()> {
321        if (vaddr & M_PAGE_OFFSET) != 0 || (paddr & M_PAGE_OFFSET) != 0 {
322            return Err(MemoryError::UnalignedAddress(vaddr));
323        }
324        let (l1, l2, l3, l4, _) = split_vaddr(vaddr);
325        let mut table = self.cr3 & M_PAGE_ALIGN;
326        let indices = [l1, l2, l3];
327
328        for index in indices {
329            let entry_addr = table + index * 8;
330            let entry = self.backend.read_phys_u64(entry_addr)?;
331            if (entry & BIT_PTE_PRESENT) == 0 {
332                let Some(allocator) = allocator.as_deref_mut() else {
333                    return Err(MemoryError::MissingPageTable(vaddr));
334                };
335                let new_page = allocator
336                    .alloc_page()
337                    .ok_or(MemoryError::PageTableAllocationFailed(vaddr))?;
338                self.zero_page(new_page.0)?;
339                let mut flags = BIT_PTE_PRESENT | BIT_PTE_WRITABLE;
340                if mapping.user {
341                    flags |= BIT_PTE_USER;
342                }
343                self.backend
344                    .write_phys_u64(entry_addr, new_page.0 | flags)?;
345                table = new_page.0;
346            } else {
347                table = entry & M_PTE_PADDR;
348            }
349        }
350
351        let mut flags = BIT_PTE_PRESENT;
352        if mapping.writable {
353            flags |= BIT_PTE_WRITABLE;
354        }
355        if mapping.user {
356            flags |= BIT_PTE_USER;
357        }
358        if !mapping.executable {
359            flags |= BIT_PTE_NX;
360        }
361        let final_entry = (paddr & M_PTE_PADDR) | flags;
362        self.backend.write_phys_u64(table + l4 * 8, final_entry)?;
363        Ok(())
364    }
365
366    pub fn unmap_page(&self, vaddr: u64) -> MResult<()> {
367        if (vaddr & M_PAGE_OFFSET) != 0 {
368            return Err(MemoryError::UnalignedAddress(vaddr));
369        }
370        let (l1, l2, l3, l4, _) = split_vaddr(vaddr);
371        let mut table = self.cr3 & M_PAGE_ALIGN;
372        let indices = [l1, l2, l3];
373        for index in indices {
374            let entry_addr = table + index * 8;
375            let entry = self.backend.read_phys_u64(entry_addr)?;
376            if (entry & BIT_PTE_PRESENT) == 0 {
377                return Err(MemoryError::MissingPageTable(vaddr));
378            }
379            table = entry & M_PTE_PADDR;
380        }
381        self.backend.write_phys_u64(table + l4 * 8, 0)?;
382        Ok(())
383    }
384
385    pub fn inject_code(
386        &self,
387        vaddr: u64,
388        paddr: u64,
389        code: &[u8],
390        mapping: PageMapping,
391        allocator: Option<&mut dyn PageAllocator>,
392    ) -> MResult<()> {
393        self.map_page(vaddr, paddr, mapping, allocator)?;
394        self.write_bytes(vaddr, code)?;
395        Ok(())
396    }
397
398    pub fn cursor(&self, vaddr: u64) -> VirtualMemoryCursor<B> {
399        VirtualMemoryCursor {
400            process: self.clone(),
401            addr: vaddr,
402        }
403    }
404
405    fn mark_host_dirty_virtual_range(&self, vaddr: u64, len: usize) {
406        let Some(tracker) = &self.host_dirty else {
407            return;
408        };
409        if len == 0 {
410            return;
411        }
412        let start = vaddr & M_PAGE_ALIGN;
413        let end = vaddr.checked_add(len as u64 - 1).unwrap_or(u64::MAX) & M_PAGE_ALIGN;
414        let mut cur = start;
415        while cur <= end {
416            if let Ok(paddr) = self.backend.resolve_vaddr(self.cr3, cur) {
417                tracker.mark_page(paddr.0);
418            }
419            if let Some(next) = cur.checked_add(PAGE_SIZE) {
420                cur = next;
421            } else {
422                break;
423            }
424        }
425    }
426
427    fn zero_page(&self, paddr: u64) -> MResult<()> {
428        let mut offset = 0u64;
429        while offset < PAGE_SIZE {
430            self.backend.write_phys_u64(paddr + offset, 0)?;
431            offset += 8;
432        }
433        Ok(())
434    }
435}
436
437/// A contiguous guest memory region exposed for sharing with the host.
438#[derive(Clone)]
439pub struct SharedMemoryRegion<B>
440where
441    B: VirtualMemoryBackend + Clone,
442{
443    process: ProcessMemory<B>,
444    addr: u64,
445    len: usize,
446}
447
448impl<B> SharedMemoryRegion<B>
449where
450    B: VirtualMemoryBackend + Clone,
451{
452    pub fn new(process: ProcessMemory<B>, addr: u64, len: usize) -> Self {
453        Self { process, addr, len }
454    }
455
456    pub fn len(&self) -> usize {
457        self.len
458    }
459
460    pub fn read_at(&self, offset: usize, buffer: &mut [u8]) -> MResult<usize> {
461        if offset >= self.len {
462            return Ok(0);
463        }
464        let max_len = (self.len - offset).min(buffer.len());
465        self.process
466            .read_bytes(self.addr + offset as u64, &mut buffer[..max_len])
467    }
468
469    pub fn write_at(&self, offset: usize, buffer: &[u8]) -> MResult<usize> {
470        if offset >= self.len {
471            return Ok(0);
472        }
473        let max_len = (self.len - offset).min(buffer.len());
474        self.process
475            .write_bytes(self.addr + offset as u64, &buffer[..max_len])
476    }
477
478    pub fn cursor(&self) -> VirtualMemoryCursor<B> {
479        self.process.cursor(self.addr)
480    }
481}
482
483/// A cursor that implements `Read`/`Write`/`Seek` over guest virtual memory.
484pub struct VirtualMemoryCursor<B>
485where
486    B: VirtualMemoryBackend + Clone,
487{
488    process: ProcessMemory<B>,
489    addr: u64,
490}
491
492impl<B> Seek for VirtualMemoryCursor<B>
493where
494    B: VirtualMemoryBackend + Clone,
495{
496    fn seek(&mut self, pos: SeekFrom) -> io::Result<u64> {
497        match pos {
498            SeekFrom::Start(offset) => self.addr = offset,
499            SeekFrom::Current(offset) => match self.addr.checked_add_signed(offset) {
500                Some(new) => self.addr = new,
501                None => {
502                    return Err(io::Error::new(
503                        io::ErrorKind::InvalidInput,
504                        "Offset caused address Over/Underflow",
505                    ));
506                }
507            },
508            SeekFrom::End(offset) => match checked_sub_signed(u64::MAX, offset) {
509                Some(new) => self.addr = new,
510                None => {
511                    return Err(io::Error::new(
512                        io::ErrorKind::InvalidInput,
513                        "Offset caused address Over/Underflow",
514                    ));
515                }
516            },
517        }
518        Ok(self.addr)
519    }
520}
521
522impl<B> Read for VirtualMemoryCursor<B>
523where
524    B: VirtualMemoryBackend + Clone,
525{
526    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
527        let bytes = self
528            .process
529            .read_bytes(self.addr, buf)
530            .map_err(|err| io::Error::new(io::ErrorKind::Other, err))?;
531        if let Some(next) = self.addr.checked_add(bytes as u64) {
532            self.addr = next;
533        }
534        Ok(bytes)
535    }
536}
537
538impl<B> Write for VirtualMemoryCursor<B>
539where
540    B: VirtualMemoryBackend + Clone,
541{
542    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
543        let bytes = self
544            .process
545            .write_bytes(self.addr, buf)
546            .map_err(|err| io::Error::new(io::ErrorKind::Other, err))?;
547        if let Some(next) = self.addr.checked_add(bytes as u64) {
548            self.addr = next;
549        }
550        Ok(bytes)
551    }
552
553    fn flush(&mut self) -> io::Result<()> {
554        Ok(())
555    }
556}
557
558pub fn read_phys<T: ByteValued + AtomicAccess>(mem: &GuestMemoryMmap, paddr: u64) -> MResult<T> {
559    let size = std::mem::size_of::<T>();
560    return mem
561        .load(GuestAddress(paddr), Ordering::Relaxed)
562        .map_err(|_| MemoryError::CantReadPhysicalPage(GuestAddress(paddr), size));
563}
564
565pub fn write_phys<T: ByteValued + AtomicAccess>(
566    mem: &GuestMemoryMmap,
567    paddr: u64,
568    val: T,
569) -> MResult<()> {
570    let size = std::mem::size_of::<T>();
571    assert!(paddr & M_PAGE_OFFSET <= PAGE_SIZE - (size as u64));
572    return mem
573        .store(val, GuestAddress(paddr), Ordering::Relaxed)
574        .map_err(|_| MemoryError::CantWritePhysicalPage(GuestAddress(paddr), size));
575}
576
577pub fn read_phys_u64(mem: &GuestMemoryMmap, paddr: u64) -> MResult<u64> {
578    return read_phys(mem, paddr);
579}
580pub fn read_phys_u8(mem: &GuestMemoryMmap, paddr: u64) -> MResult<u8> {
581    return read_phys(mem, paddr);
582}
583
584pub enum PagePermission {
585    R,
586    W,
587    RW,
588    None,
589}
590
591pub trait NyxMemExtension {
592    fn resolve_vaddr(&self, cr3: u64, vaddr: u64) -> MResult<GuestAddress>;
593    fn read_virtual<T: ByteValued + AtomicAccess>(&self, cr3: u64, vaddr: u64) -> MResult<T>;
594    fn write_virtual<T: ByteValued + AtomicAccess>(
595        &self,
596        cr3: u64,
597        vaddr: u64,
598        val: T,
599    ) -> MResult<()>;
600
601    fn read_phys<T: ByteValued + AtomicAccess>(&self, paddr: u64) -> MResult<T>;
602    fn write_phys<T: ByteValued + AtomicAccess>(&self, paddr: u64, val: T) -> MResult<()>;
603
604    fn set_physical_page_permission(&mut self, phys_addr: u64, perm: PagePermission);
605    fn set_virtual_page_permission(&mut self, cr3: u64, vaddr: u64, perm: PagePermission);
606
607    fn read_virtual_u8(&self, cr3: u64, vaddr: u64) -> MResult<u8>;
608    fn write_virtual_u8(&self, cr3: u64, vaddr: u64, val: u8) -> MResult<()>;
609    fn read_virtual_u64(&self, cr3: u64, vaddr: u64) -> MResult<u64>;
610    fn write_virtual_u64(&self, cr3: u64, vaddr: u64, val: u64) -> MResult<()>;
611    fn read_virtual_cstr(&self, cr3: u64, guest_vaddr: u64) -> Vec<u8>;
612
613    fn read_virtual_bytes(&self, cr3: u64, vaddr: u64, buffer: &mut [u8]) -> MResult<usize>;
614    fn write_virtual_bytes(&self, cr3: u64, guest_vaddr: u64, buffer: &[u8]) -> MResult<usize>;
615}
616
617pub trait GetMem {
618    fn get_mem(&self) -> &GuestMemoryMmap;
619}
620
621impl GetMem for Vmm {
622    fn get_mem(&self) -> &GuestMemoryMmap {
623        self.vm.guest_memory()
624    }
625}
626
627impl<GetMemT> NyxMemExtension for GetMemT
628where
629    GetMemT: GetMem,
630{
631    fn read_virtual_cstr(&self, cr3: u64, guest_vaddr: u64) -> Vec<u8> {
632        let mem = self.get_mem();
633        let mut res = Vec::new();
634        let mut cur_addr = guest_vaddr;
635        for pte in walk_virtual_pages(mem, cr3, guest_vaddr & M_PAGE_ALIGN, M_PAGE_ALIGN) {
636            if !pte.present() || pte.missing_page() {
637                return res;
638            }
639            let slice = mem.get_slice(pte.phys_addr(), PAGE_SIZE as usize).unwrap();
640            while cur_addr < pte.vaddrs.end {
641                let u8_char = slice
642                    .load::<u8>((cur_addr & M_PAGE_OFFSET) as usize, Ordering::Relaxed)
643                    .unwrap();
644                res.push(u8_char);
645                cur_addr += 1;
646                if u8_char == 0 {
647                    return res;
648                }
649            }
650        }
651        return res;
652    }
653
654    fn resolve_vaddr(&self, cr3: u64, vaddr: u64) -> MResult<GuestAddress> {
655        let mem = self.get_mem();
656        let paddr = resolve_vaddr(mem, cr3, vaddr)?;
657        return Ok(GuestAddress(paddr));
658    }
659
660    fn read_phys<T: ByteValued + AtomicAccess>(&self, paddr: u64) -> MResult<T> {
661        let mem = self.get_mem();
662        read_phys(mem, paddr)
663    }
664
665    fn write_phys<T: ByteValued + AtomicAccess>(&self, paddr: u64, val: T) -> MResult<()> {
666        let mem = self.get_mem();
667        write_phys(mem, paddr, val)
668    }
669
670    fn read_virtual<T: ByteValued + AtomicAccess>(&self, cr3: u64, vaddr: u64) -> MResult<T> {
671        let mem = self.get_mem();
672        let paddr = resolve_vaddr(mem, cr3, vaddr)?;
673        return read_phys(mem, paddr);
674    }
675
676    fn write_virtual<T: ByteValued + AtomicAccess>(
677        &self,
678        cr3: u64,
679        vaddr: u64,
680        value: T,
681    ) -> MResult<()> {
682        let mem = self.get_mem();
683        let paddr = resolve_vaddr(mem, cr3, vaddr)?;
684        return write_phys(mem, paddr, value);
685    }
686
687    fn read_virtual_u64(&self, cr3: u64, vaddr: u64) -> MResult<u64> {
688        return self.read_virtual(cr3, vaddr);
689    }
690
691    fn write_virtual_u64(&self, cr3: u64, vaddr: u64, val: u64) -> MResult<()> {
692        return self.write_virtual(cr3, vaddr, val);
693    }
694    fn read_virtual_u8(&self, cr3: u64, vaddr: u64) -> MResult<u8> {
695        return self.read_virtual(cr3, vaddr);
696    }
697    fn write_virtual_u8(&self, cr3: u64, vaddr: u64, val: u8) -> MResult<()> {
698        return self.write_virtual(cr3, vaddr, val);
699    }
700
701    fn read_virtual_bytes<'buffer>(
702        &self,
703        cr3: u64,
704        guest_vaddr: u64,
705        buffer: &'buffer mut [u8],
706    ) -> MResult<usize> {
707        if buffer.is_empty() {
708            return Ok(0);
709        }
710        let mem = self.get_mem();
711        let mut num_bytes = 0;
712        let start_page = guest_vaddr & M_PAGE_ALIGN;
713        let last_page = (guest_vaddr + (buffer.len() as u64) - 1) & M_PAGE_ALIGN;
714        for pte in walk_virtual_pages(mem, cr3, start_page, last_page) {
715            if !pte.present() || pte.missing_page() {
716                return Ok(num_bytes);
717            }
718            let guest_slice = mem.get_slice(pte.phys_addr(), PAGE_SIZE as usize).unwrap();
719            let page_vaddr = pte.vaddrs.start;
720            assert_eq!(guest_slice.len(), PAGE_SIZE as usize);
721            let slice_start = if page_vaddr < guest_vaddr {
722                (guest_vaddr - page_vaddr) as usize
723            } else {
724                0
725            };
726            let num_copied = guest_slice
727                .subslice(slice_start, guest_slice.len() - slice_start)
728                .unwrap()
729                .copy_to(&mut buffer[num_bytes..]);
730            num_bytes += num_copied;
731            if num_bytes >= buffer.len() {
732                break;
733            }
734        }
735        return Ok(num_bytes);
736    }
737
738    fn write_virtual_bytes(&self, cr3: u64, guest_vaddr: u64, buffer: &[u8]) -> MResult<usize> {
739        if buffer.is_empty() {
740            return Ok(0);
741        }
742        let mem = self.get_mem();
743        let mut num_bytes = 0;
744        let start_page = guest_vaddr & M_PAGE_ALIGN;
745        let last_page = (guest_vaddr + (buffer.len() as u64) - 1) & M_PAGE_ALIGN;
746        for pte in walk_virtual_pages(mem, cr3, start_page, last_page) {
747            if !pte.present() || pte.missing_page() {
748                return Ok(num_bytes);
749            }
750            let guest_slice = mem.get_slice(pte.phys_addr(), PAGE_SIZE as usize).unwrap();
751            let page_vaddr = pte.vaddrs.start;
752            assert_eq!(guest_slice.len(), PAGE_SIZE as usize);
753            let slice_start = if page_vaddr < guest_vaddr {
754                (guest_vaddr - page_vaddr) as usize
755            } else {
756                0
757            };
758            let buff_slice = &buffer[num_bytes..];
759            let guest_slice = guest_slice
760                .subslice(slice_start, guest_slice.len() - slice_start)
761                .unwrap();
762            guest_slice.copy_from(buff_slice);
763            let num_copied = buff_slice.len().min(guest_slice.len());
764            num_bytes += num_copied;
765            if num_bytes >= buffer.len() {
766                break;
767            }
768        }
769        return Ok(num_bytes);
770    }
771
772    fn set_physical_page_permission(&mut self, paddr: u64, perm: PagePermission) {
773        let page_addr = paddr & M_PAGE_ALIGN;
774        let region = self.get_mem().find_region(GuestAddress(page_addr)).unwrap();
775        assert!(region.to_region_addr(GuestAddress(page_addr)).is_some());
776        let offset = page_addr - region.start_addr().0 as u64;
777        let prot = match perm {
778            PagePermission::None => 0,
779            PagePermission::R => PROT_READ,
780            PagePermission::W => PROT_WRITE,
781            PagePermission::RW => PROT_READ | PROT_WRITE,
782        };
783        unsafe {
784            let ptr = region.as_ptr().offset(offset as isize);
785            mprotect(ptr as *mut c_void, PAGE_SIZE as usize, prot);
786        }
787    }
788
789    fn set_virtual_page_permission(&mut self, cr3: u64, vaddr: u64, perm: PagePermission) {
790        let phys_addr = self.resolve_vaddr(cr3, vaddr).unwrap();
791        self.set_physical_page_permission(phys_addr.0, perm);
792    }
793}
794
795fn read_page_table_entry(
796    mem: &GuestMemoryMmap,
797    paddr: u64,
798    offset: u64,
799) -> Result<u64, MemoryError> {
800    let entry = read_phys_u64(mem, paddr + offset * 8)?;
801    if (entry & BIT_PTE_PRESENT) != 0 {
802        return Ok(entry & M_PTE_PADDR);
803    }
804    return Err(MemoryError::PageNotPresent(GuestAddress(paddr), offset));
805}
806
807pub fn resolve_vaddr(mem: &GuestMemoryMmap, cr3: u64, vaddr: u64) -> MResult<u64> {
808    let mask = M_PAGE_ALIGN;
809    let (l1, l2, l3, l4, offset) = split_vaddr(vaddr);
810    let pml4_addr = read_page_table_entry(mem, cr3 & mask, l1)?;
811    let pdp_addr = read_page_table_entry(mem, pml4_addr, l2)?;
812    let pd_addr = read_page_table_entry(mem, pdp_addr, l3)?;
813    let pt_addr = read_page_table_entry(mem, pd_addr, l4)?;
814    let addr = pt_addr + offset;
815    return Ok(addr);
816}
817
818// Primary function to walk virtual address spaces. Will merge all adjacent
819// unmapped/invalid pages & only return leave nodes of the page tables
820pub fn walk_virtual_pages<'mem>(
821    mem: &'mem GuestMemoryMmap,
822    cr3: u64,
823    start: u64,
824    last_page: u64,
825) -> impl Iterator<Item = PTE> + 'mem {
826    MergedPTEWalker::new(PTEWalker::new(mem, cr3, start, last_page).filter(|pte| pte.level == 3))
827}
828
829// walks the page tables between start and end. Note: End is inclusive, not
830// exclusive. I.e. the last page will be starting at end. Will return all node
831// in the page tables (including not-presen & invalid phys address nodes)
832pub fn walk_page_tables(
833    mem: &GuestMemoryMmap,
834    cr3: u64,
835    start: u64,
836    last_page: u64,
837) -> PTEWalker<'_> {
838    PTEWalker::new(mem, cr3, start, last_page)
839}
840
841pub fn walk_page_tables_merged<'mem>(
842    mem: &'mem GuestMemoryMmap,
843    cr3: u64,
844    start: u64,
845    last_page: u64,
846) -> MergedPTEWalker<'mem, PTEWalker<'mem>> {
847    MergedPTEWalker::new(PTEWalker::new(mem, cr3, start, last_page))
848}
849
850pub fn split_vaddr(vaddr: u64) -> (u64, u64, u64, u64, u64) {
851    let l1 = (vaddr >> 39) & M_PTE_OFFSET;
852    let l2 = (vaddr >> 30) & M_PTE_OFFSET;
853    let l3 = (vaddr >> 21) & M_PTE_OFFSET;
854    let l4 = (vaddr >> 12) & M_PTE_OFFSET;
855    let addr = (vaddr >> 0) & M_PAGE_OFFSET;
856    return (l1, l2, l3, l4, addr);
857}
858
859pub fn join_vaddr(l1: u64, l2: u64, l3: u64, l4: u64, offset: u64) -> u64 {
860    let l1 = (l1 & M_PTE_OFFSET) << 39;
861    let l2 = (l2 & M_PTE_OFFSET) << 30;
862    let l3 = (l3 & M_PTE_OFFSET) << 21;
863    let l4 = (l4 & M_PTE_OFFSET) << 12;
864    return l1 | l2 | l3 | l4 | (offset & M_PAGE_OFFSET);
865}
866pub struct PTE {
867    pub vaddrs: Range<u64>,
868    pub val: u64,
869    pub level: u8,
870    pub missing_page: bool,
871}
872
873impl<'mem> core::fmt::Debug for PTE {
874    fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
875        fmt.debug_struct("PTE")
876            .field("level", &self.level)
877            .field("val", &format_args!("{:X?}", self.val))
878            .field("vaddrs", &format_args!("{:X?}", self.vaddrs))
879            .finish()
880    }
881}
882
883impl PTE {
884    pub fn present(&self) -> bool {
885        (self.val & BIT_PTE_PRESENT) != 0
886    }
887    pub fn phys_addr(&self) -> GuestAddress {
888        GuestAddress(self.val & M_PTE_PADDR)
889    }
890    pub fn missing_page(&self) -> bool {
891        self.missing_page
892    }
893
894    fn merge_with(&mut self, other: &PTE) -> bool {
895        let same_level = self.level == other.level;
896        let adjacent = self.vaddrs.end == other.vaddrs.start;
897        if self.present() || !same_level || !adjacent {
898            return false;
899        }
900        let both_missing_page = self.missing_page() && other.missing_page();
901        let both_not_present = !self.present() && !other.present();
902        if both_missing_page || both_not_present {
903            self.vaddrs.end = other.vaddrs.end;
904            return true;
905        }
906        return false;
907    }
908}
909
910pub struct PTEWalker<'mem> {
911    mem: &'mem GuestMemoryMmap,
912    offsets: [u64; 4],
913    bases: [u64; 4],
914    last_page: u64,
915    level: usize,
916}
917
918impl<'mem> core::fmt::Debug for PTEWalker<'mem> {
919    fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
920        fmt.debug_struct("PTEWalker")
921            .field("level", &self.level)
922            .field("offsets", &format_args!("{:?}", self.offsets))
923            .field("bases", &format_args!("{:X?}", self.bases))
924            .field("last_page", &format_args!("{:X?}", self.last_page))
925            .finish()
926    }
927}
928
929impl<'mem> Iterator for PTEWalker<'mem> {
930    type Item = PTE;
931    fn next(&mut self) -> Option<Self::Item> {
932        let res = self.iter_next();
933        return res;
934    }
935}
936
937impl<'mem> PTEWalker<'mem> {
938    pub fn new(mem: &'mem GuestMemoryMmap, cr3: u64, start: u64, last_page: u64) -> Self {
939        let (l1, l2, l3, l4, offset) = split_vaddr(start);
940        assert_eq!(offset, 0);
941        Self {
942            mem,
943            offsets: [l1, l2, l3, l4],
944            bases: [cr3 & M_PAGE_ALIGN, 0, 0, 0],
945            level: 0,
946            last_page,
947        }
948    }
949
950    fn get_cur_pte(&self) -> PTE {
951        let vaddrs = self.make_vaddr_range();
952        let level = self.level.try_into().unwrap();
953        let base = self.bases[self.level];
954        let offset = self.offsets[self.level];
955        if let Ok(val) = read_phys_u64(self.mem, base + offset * 8) {
956            PTE {
957                vaddrs,
958                val,
959                level,
960                missing_page: false,
961            }
962        } else {
963            PTE {
964                vaddrs,
965                val: 0,
966                level,
967                missing_page: true,
968            }
969        }
970    }
971
972    fn advance_cursor_up(&mut self) {
973        loop {
974            let cur = &mut self.offsets[self.level];
975            *cur += 1;
976            if *cur >= 0x1ff {
977                if self.level == 0 {
978                    return;
979                }
980                *cur = 0;
981                self.level -= 1;
982            } else {
983                return;
984            }
985        }
986    }
987
988    fn advance_cursor_down(&mut self, new_base_addr: u64) {
989        assert_eq!(new_base_addr & M_PAGE_OFFSET, 0);
990        self.level += 1;
991        //This should be imlied as advance_cursor_up set's the offset[cur] to 0 before going up one level.
992        //If we don't do this explicitly, we can initialliza the level to 0 and the offsets to a given vaddr to start iterating from that given vaddr
993        //self.offsets[self.level]=0;
994        self.bases[self.level] = new_base_addr;
995    }
996
997    fn make_vaddr_range(&self) -> Range<u64> {
998        let l1 = self.offsets[0];
999        let l2 = if self.level >= 1 { self.offsets[1] } else { 0 };
1000        let l3 = if self.level >= 2 { self.offsets[2] } else { 0 };
1001        let l4 = if self.level >= 3 { self.offsets[3] } else { 0 };
1002        let start = join_vaddr(l1, l2, l3, l4, 0);
1003        let size = [
1004            PAGE_SIZE * 512 * 512 * 512,
1005            PAGE_SIZE * 512 * 512,
1006            PAGE_SIZE * 512,
1007            PAGE_SIZE,
1008        ][self.level];
1009        start..start + size
1010    }
1011
1012    fn iter_next(&mut self) -> Option<PTE> {
1013        // It appears that we see high kernel addresses sometimes (i.e.
1014        // ffffffff823b1ad8 during the boot breakpoint vmexit). Those aren't
1015        // handled correctly right now. Investigate expected behavior
1016        if self.offsets[0] > M_PTE_OFFSET {
1017            return None;
1018        }
1019        let res = self.get_cur_pte();
1020        if res.vaddrs.start > self.last_page {
1021            return None;
1022        }
1023        if res.present() && self.level < 3 {
1024            self.advance_cursor_down(res.phys_addr().0);
1025        } else {
1026            self.advance_cursor_up();
1027        }
1028        return Some(res);
1029    }
1030}
1031
1032pub struct MergedPTEWalker<'mem, PTEIter: Iterator<Item = PTE> + 'mem> {
1033    inner: Peekable<PTEIter>,
1034    _phantom: PhantomData<&'mem PTEIter>,
1035}
1036
1037impl<'mem, BaseIter: Iterator<Item = PTE>> MergedPTEWalker<'mem, BaseIter> {
1038    pub fn new(iter: BaseIter) -> Self {
1039        let inner = iter.peekable();
1040        Self {
1041            inner,
1042            _phantom: PhantomData,
1043        }
1044    }
1045}
1046
1047impl<'mem, BaseIter: Iterator<Item = PTE>> Iterator for MergedPTEWalker<'mem, BaseIter> {
1048    type Item = PTE;
1049    fn next(&mut self) -> Option<Self::Item> {
1050        let mut cur = self.inner.next()?;
1051        while let Some(next) = self.inner.peek() {
1052            if !cur.merge_with(next) {
1053                break;
1054            }
1055            self.inner.next();
1056        }
1057        return Some(cur);
1058    }
1059}
1060
1061pub struct VirtSpace<'vm> {
1062    pub cr3: u64,
1063    pub vmm: &'vm Vmm,
1064    pub addr: u64,
1065}
1066
1067// currently not a stable feature.
1068fn checked_sub_signed(x: u64, y: i64) -> Option<u64> {
1069    return x.checked_sub(y as u64);
1070}
1071
1072impl<'vm> Seek for VirtSpace<'vm> {
1073    fn seek(&mut self, pos: SeekFrom) -> io::Result<u64> {
1074        match pos {
1075            SeekFrom::Start(offset) => self.addr = offset,
1076            SeekFrom::Current(offset) => match self.addr.checked_add_signed(offset) {
1077                Some(new) => self.addr = new,
1078                None => {
1079                    return Err(io::Error::new(
1080                        io::ErrorKind::InvalidInput,
1081                        "Offset caused address Over/Underflow",
1082                    ));
1083                }
1084            },
1085            SeekFrom::End(offset) => match checked_sub_signed(u64::MAX, offset) {
1086                Some(new) => self.addr = new,
1087                None => {
1088                    return Err(io::Error::new(
1089                        io::ErrorKind::InvalidInput,
1090                        "Offset caused address Over/Underflow",
1091                    ));
1092                }
1093            },
1094        }
1095        Ok(self.addr)
1096    }
1097}
1098
1099impl<'vm> Read for VirtSpace<'vm> {
1100    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
1101        let bytes = self
1102            .vmm
1103            .read_virtual_bytes(self.cr3, self.addr, buf)
1104            .unwrap();
1105        if let Some(next) = self.addr.checked_add(bytes as u64) {
1106            self.addr = next;
1107        }
1108        Ok(bytes)
1109    }
1110}
1111
1112impl<'vm> Write for VirtSpace<'vm> {
1113    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
1114        let bytes = self
1115            .vmm
1116            .write_virtual_bytes(self.cr3, self.addr, buf)
1117            .unwrap();
1118        if let Some(next) = self.addr.checked_add(bytes as u64) {
1119            self.addr = next;
1120        }
1121        Ok(bytes)
1122    }
1123
1124    fn flush(&mut self) -> io::Result<()> {
1125        Ok(()) // No-op for in-memory buffer
1126    }
1127}
1128
1129pub struct VirtMappedRange {
1130    cr3: u64,
1131    start: u64,
1132    end: u64,
1133    tlb: Vec<GuestAddress>,
1134}
1135
1136impl VirtMappedRange {
1137    pub fn new(cr3: u64, start_page: u64, end_page: u64) -> Self {
1138        assert!(end_page >= start_page);
1139        assert!(start_page & M_PAGE_OFFSET == 0);
1140        assert!(end_page & M_PAGE_OFFSET == 0);
1141        Self {
1142            cr3,
1143            start: start_page,
1144            end: end_page,
1145            tlb: vec![],
1146        }
1147    }
1148
1149    pub fn validate(&mut self, mem: &GuestMemoryMmap) -> Result<(), MemoryError> {
1150        self.tlb
1151            .reserve(((self.end - self.start) / PAGE_SIZE).try_into().unwrap());
1152        self.tlb.clear();
1153        for pte in walk_virtual_pages(
1154            mem,
1155            self.cr3,
1156            self.start & M_PAGE_OFFSET,
1157            self.end & M_PAGE_OFFSET,
1158        ) {
1159            if !pte.present() {
1160                return Err(MemoryError::PageNotPresent(
1161                    pte.phys_addr(),
1162                    pte.vaddrs.start,
1163                ));
1164            }
1165            if pte.missing_page() {
1166                return Err(MemoryError::CantAccessMissingPhysicalPage(pte.phys_addr()));
1167            }
1168            self.tlb.push(pte.phys_addr());
1169        }
1170        return Ok(());
1171    }
1172
1173    pub fn iter<'mem>(
1174        &'mem mut self,
1175        mem: &'mem GuestMemoryMmap,
1176    ) -> impl Iterator<
1177        Item = (
1178            u64,
1179            VolatileSlice<'mem, BS<'mem, <GuestRegionMmap as GuestMemoryRegion>::B>>,
1180        ),
1181    > + 'mem {
1182        self.tlb.iter().enumerate().map(|(i, guest_phys_addr)| {
1183        if let Some(region) = mem.find_region(*guest_phys_addr) {
1184           let start = region.to_region_addr(*guest_phys_addr).unwrap();
1185           let cap = region.len() - start.raw_value();
1186           let len = std::cmp::min(cap, PAGE_SIZE as GuestUsize);
1187           assert_eq!(len, PAGE_SIZE as GuestUsize);
1188           let volatile_slice = region.as_volatile_slice().unwrap();
1189           return (self.start+(i as u64)*PAGE_SIZE, volatile_slice)
1190        }
1191        panic!("couldn't access memory for cr3: {:x}, addr: {:x} - region for physical {:x} not found ", 
1192        self.cr3,
1193        self.start + (i as u64) * PAGE_SIZE,
1194        guest_phys_addr.raw_value());
1195    })
1196    }
1197}
1198
1199#[cfg(test)]
1200mod tests {
1201
1202    use std::collections::HashSet;
1203
1204    use vmm::vmm_config::machine_config::HugePageConfig;
1205    use vmm::vstate::memory::{self, GuestMemory, GuestMemoryRegion};
1206
1207    use super::*;
1208
1209    #[derive(Clone)]
1210    struct MemWrapper {
1211        mem: GuestMemoryMmap,
1212    }
1213
1214    impl GetMem for MemWrapper {
1215        fn get_mem(&self) -> &GuestMemoryMmap {
1216            &self.mem
1217        }
1218    }
1219
1220    #[test]
1221    fn test_process_memory_marks_host_dirty() {
1222        let mem = make_mem();
1223        let mut allocated_pages = HashSet::new();
1224        let cr3 = PAGE_SIZE * 8;
1225        allocated_pages.insert(cr3);
1226        let vaddr = 0x4000;
1227        let paddr = make_vpage(&mem, cr3, vaddr, &mut allocated_pages);
1228
1229        let total_pages = mem
1230            .iter()
1231            .map(|region| {
1232                let len = region.len() as usize;
1233                (len + PAGE_SIZE as usize - 1) / PAGE_SIZE as usize
1234            })
1235            .sum();
1236        let wrapper = MemWrapper { mem };
1237        let tracker = Arc::new(HostDirtyTracker::new(total_pages));
1238        let process = ProcessMemory::new(wrapper, cr3).with_host_dirty(tracker.clone());
1239
1240        process.write_u64(vaddr, 0xdead_beef).unwrap();
1241        let dirty_pages = tracker.snapshot_pages();
1242        assert!(dirty_pages.contains(&paddr.0));
1243    }
1244
1245    #[test]
1246    fn test_virtual_memory_cursor_advances() {
1247        let mem = make_mem();
1248        let mut allocated_pages = HashSet::new();
1249        let cr3 = PAGE_SIZE * 8;
1250        allocated_pages.insert(cr3);
1251        let vaddr = 0x4000;
1252        make_vpage(&mem, cr3, vaddr, &mut allocated_pages);
1253        make_vpage(&mem, cr3, vaddr + PAGE_SIZE, &mut allocated_pages);
1254
1255        let wrapper = MemWrapper { mem };
1256        let process = ProcessMemory::new(wrapper, cr3);
1257        let mut cursor = process.cursor(vaddr);
1258        let payload = vec![0x5a_u8; (PAGE_SIZE as usize) + 16];
1259        let written = cursor.write(&payload).unwrap();
1260        assert_eq!(written, payload.len());
1261
1262        let mut read_back = vec![0_u8; payload.len()];
1263        let mut read_cursor = process.cursor(vaddr);
1264        let read = read_cursor.read(&mut read_back).unwrap();
1265        assert_eq!(read, payload.len());
1266        assert_eq!(read_back, payload);
1267    }
1268
1269    #[test]
1270    fn test_map_page_with_allocator() {
1271        let mem = make_mem();
1272        let mut allocated_pages = HashSet::new();
1273        let cr3 = PAGE_SIZE * 8;
1274        allocated_pages.insert(cr3);
1275
1276        let target_page = allocate_page(&mem, &mut allocated_pages);
1277        let pt1 = allocate_page(&mem, &mut allocated_pages);
1278        let pt2 = allocate_page(&mem, &mut allocated_pages);
1279        let pt3 = allocate_page(&mem, &mut allocated_pages);
1280        let mut allocator = VecPageAllocator::new(vec![pt1, pt2, pt3]);
1281
1282        let wrapper = MemWrapper { mem };
1283        let process = ProcessMemory::new(wrapper, cr3);
1284        let vaddr = 0x9000;
1285        process
1286            .map_page(
1287                vaddr,
1288                target_page.0,
1289                PageMapping {
1290                    writable: true,
1291                    executable: false,
1292                    user: true,
1293                },
1294                Some(&mut allocator),
1295            )
1296            .unwrap();
1297
1298        process.write_u64(vaddr, 0x1122_3344_5566_7788).unwrap();
1299        let val = process.read_u64(vaddr).unwrap();
1300        assert_eq!(val, 0x1122_3344_5566_7788);
1301        let resolved = process.resolve_vaddr(vaddr).unwrap();
1302        assert_eq!(resolved, target_page);
1303    }
1304
1305    #[test]
1306    fn test_resolve_address() {
1307        let mem = make_mem();
1308        let fake_cr3 = PAGE_SIZE * 8;
1309        let vaddr = 0x400000;
1310        let l1_addr = fake_cr3 + PAGE_SIZE;
1311        let l2_addr = fake_cr3 + PAGE_SIZE * 2;
1312        let l3_addr = fake_cr3 + PAGE_SIZE * 3;
1313        let target_1 = PAGE_SIZE;
1314        let target_2 = PAGE_SIZE * 3;
1315        let target_3 = PAGE_SIZE * 5;
1316
1317        store(
1318            &mem,
1319            GuestAddress(fake_cr3 + 8 * ((vaddr >> 39) & M_PTE_OFFSET)),
1320            l1_addr | BIT_PTE_PRESENT,
1321        );
1322        store(
1323            &mem,
1324            GuestAddress(l1_addr + 8 * ((vaddr >> 30) & M_PTE_OFFSET)),
1325            l2_addr | BIT_PTE_PRESENT,
1326        );
1327        store(
1328            &mem,
1329            GuestAddress(l2_addr + 8 * ((vaddr >> 21) & M_PTE_OFFSET)),
1330            l3_addr | BIT_PTE_PRESENT,
1331        );
1332        store(
1333            &mem,
1334            GuestAddress(l3_addr + 8 * (((vaddr + 0 * PAGE_SIZE) >> 12) & M_PTE_OFFSET)),
1335            target_1 | BIT_PTE_PRESENT,
1336        );
1337        store(
1338            &mem,
1339            GuestAddress(l3_addr + 8 * (((vaddr + 1 * PAGE_SIZE) >> 12) & M_PTE_OFFSET)),
1340            target_2 | BIT_PTE_PRESENT,
1341        );
1342        store(
1343            &mem,
1344            GuestAddress(l3_addr + 8 * (((vaddr + 2 * PAGE_SIZE) >> 12) & M_PTE_OFFSET)),
1345            target_3 | BIT_PTE_PRESENT,
1346        );
1347        assert_eq!(resolve_vaddr(&mem, fake_cr3, vaddr).unwrap(), target_1);
1348        let walk = walk_virtual_pages(&mem, fake_cr3, vaddr, vaddr + PAGE_SIZE * 2)
1349            .map(|pte| (pte.vaddrs.start, pte.phys_addr()))
1350            .collect::<Vec<_>>();
1351        assert_eq!(
1352            walk,
1353            vec![
1354                (vaddr + 0 * PAGE_SIZE, GuestAddress(target_1)),
1355                (vaddr + 1 * PAGE_SIZE, GuestAddress(target_2)),
1356                (vaddr + 2 * PAGE_SIZE, GuestAddress(target_3)),
1357            ]
1358        );
1359    }
1360
1361    #[test]
1362    fn test_make_pages() {
1363        let mem = make_mem();
1364        let mut allocated_pages = HashSet::new();
1365        let cr3 = PAGE_SIZE * 8;
1366        allocated_pages.insert(cr3);
1367        let t1 = make_vpage(&mem, cr3, 0x41000, &mut allocated_pages);
1368        let t2 = make_vpage(&mem, cr3, 0x42000, &mut allocated_pages);
1369        let t3 = make_vpage(&mem, cr3, 0x43000, &mut allocated_pages);
1370
1371        assert_eq!(resolve_vaddr(&mem, cr3, 0x41000).unwrap(), t1.0);
1372        assert_eq!(resolve_vaddr(&mem, cr3, 0x42000).unwrap(), t2.0);
1373        assert_eq!(resolve_vaddr(&mem, cr3, 0x43000).unwrap(), t3.0);
1374        let walk = walk_virtual_pages(&mem, cr3, 0x41000, 0x41000 + PAGE_SIZE * 2)
1375            .map(|pte| (pte.vaddrs.start, pte.phys_addr()))
1376            .collect::<Vec<_>>();
1377        assert_eq!(
1378            walk,
1379            vec![
1380                (0x41000 + 0 * PAGE_SIZE, t1),
1381                (0x41000 + 1 * PAGE_SIZE, t2),
1382                (0x41000 + 2 * PAGE_SIZE, t3),
1383            ]
1384        );
1385    }
1386
1387    #[test]
1388    fn test_walk_past_boundary() {
1389        let mem = make_mem();
1390        let mut allocated_pages = HashSet::new();
1391        let cr3 = PAGE_SIZE * 8;
1392        allocated_pages.insert(cr3);
1393        let boundary = 0x8000000000;
1394        let t1 = make_vpage(&mem, cr3, boundary - PAGE_SIZE, &mut allocated_pages);
1395        let t2 = make_vpage(&mem, cr3, boundary, &mut allocated_pages);
1396        let t3 = make_vpage(&mem, cr3, boundary + PAGE_SIZE, &mut allocated_pages);
1397        let walk = walk_virtual_pages(&mem, cr3, boundary - PAGE_SIZE, boundary + PAGE_SIZE)
1398            .map(|pte| (pte.vaddrs.start, pte.phys_addr()))
1399            .collect::<Vec<_>>();
1400        assert_eq!(
1401            walk,
1402            vec![
1403                (boundary - PAGE_SIZE, t1),
1404                (boundary, t2),
1405                (boundary + PAGE_SIZE, t3),
1406            ]
1407        );
1408    }
1409
1410    #[test]
1411    fn test_walk_missing_page() {
1412        let mem = make_mem();
1413        let mut allocated_pages = HashSet::new();
1414        let cr3 = PAGE_SIZE * 8;
1415        allocated_pages.insert(cr3);
1416        let boundary = 0x8000000000;
1417        let t1 = make_vpage(&mem, cr3, boundary - PAGE_SIZE, &mut allocated_pages);
1418        // t2 is missing
1419        let t3 = make_vpage(&mem, cr3, boundary + PAGE_SIZE, &mut allocated_pages);
1420        let walk = walk_virtual_pages(&mem, cr3, boundary - PAGE_SIZE, boundary + 3 * PAGE_SIZE)
1421            .map(|pte| {
1422                (
1423                    pte.vaddrs.start,
1424                    pte.phys_addr(),
1425                    pte.present(),
1426                    pte.missing_page(),
1427                )
1428            })
1429            .collect::<Vec<_>>();
1430        assert_eq!(
1431            walk,
1432            vec![
1433                (boundary - PAGE_SIZE, t1, true, false),
1434                (boundary, GuestAddress(0), false, false),
1435                (boundary + PAGE_SIZE, t3, true, false),
1436                (boundary + PAGE_SIZE * 2, GuestAddress(0), false, false),
1437            ]
1438        );
1439    }
1440
1441    #[test]
1442    fn test_pte_walker() {
1443        let mem = make_mem();
1444        let mut allocated_pages = HashSet::new();
1445        let cr3 = PAGE_SIZE * 8;
1446        allocated_pages.insert(cr3);
1447        let boundary = 0x8000000000;
1448        let t1 = make_vpage(&mem, cr3, boundary - PAGE_SIZE, &mut allocated_pages);
1449        let t2 = make_vpage(&mem, cr3, boundary, &mut allocated_pages);
1450        let t3 = make_vpage(&mem, cr3, boundary + PAGE_SIZE, &mut allocated_pages);
1451        let last_page = 0xffffffff_ffffffff;
1452        let walk = PTEWalker::new(&mem, cr3, boundary - PAGE_SIZE, last_page)
1453            .filter(|i| i.present())
1454            .map(|i| (i.level, i.vaddrs))
1455            .collect::<Vec<_>>();
1456        let expected = [
1457            (0, 0x0..0x8000000000),
1458            (1, 0x7fc0000000..0x8000000000),
1459            (2, 0x7fffe00000..0x8000000000),
1460            (3, 0x7ffffff000..0x8000000000),
1461            (0, 0x8000000000..0x10000000000),
1462            (1, 0x8000000000..0x8040000000),
1463            (2, 0x8000000000..0x8000200000),
1464            (3, 0x8000000000..0x8000001000),
1465            (3, 0x8000001000..0x8000002000),
1466        ];
1467        assert_eq!(walk, expected);
1468
1469        let walk = PTEWalker::new(&mem, cr3, boundary - PAGE_SIZE, boundary + PAGE_SIZE)
1470            .filter(|i| i.level == 3 && i.present())
1471            .map(|i| (i.level, i.vaddrs.start, i.phys_addr()))
1472            .collect::<Vec<_>>();
1473        let expected = [
1474            (3, 0x7ffffff000, t1),
1475            (3, 0x8000000000, t2),
1476            (3, 0x8000001000, t3),
1477        ];
1478        assert_eq!(walk, expected);
1479    }
1480
1481    #[test]
1482    fn test_ptr_walker_missing_page() {
1483        let mem = make_mem();
1484        let mut allocated_pages = HashSet::new();
1485        let cr3 = PAGE_SIZE * 8;
1486        allocated_pages.insert(cr3);
1487        let boundary = 0x8000000000;
1488        let t1 = make_vpage(&mem, cr3, boundary - PAGE_SIZE, &mut allocated_pages);
1489        // t2 is missing
1490        let t3 = make_vpage(&mem, cr3, boundary + PAGE_SIZE, &mut allocated_pages);
1491
1492        let walk = PTEWalker::new(&mem, cr3, boundary - PAGE_SIZE, boundary + PAGE_SIZE)
1493            .filter(|i| i.level == 3 && i.present())
1494            .map(|i| (i.level, i.vaddrs.start, i.phys_addr()))
1495            .collect::<Vec<_>>();
1496        let expected = [(3, 0x7ffffff000, t1), (3, 0x8000001000, t3)];
1497        assert_eq!(walk, expected);
1498    }
1499
1500    #[test]
1501    fn test_ptr_walker_missing_page_table() {
1502        let mem = make_mem();
1503        let mut allocated_pages = HashSet::new();
1504        let cr3 = PAGE_SIZE * 8;
1505        allocated_pages.insert(cr3);
1506        let boundary = 0x8000000000;
1507        let t1 = make_vpage(&mem, cr3, boundary - PAGE_SIZE, &mut allocated_pages);
1508        //t2 and t3 are later invalidating by breaking the page table entry
1509        let _t2 = make_vpage(&mem, cr3, boundary, &mut allocated_pages);
1510        let _t3 = make_vpage(&mem, cr3, boundary + PAGE_SIZE, &mut allocated_pages);
1511        let t4 = make_vpage(&mem, cr3, boundary + PAGE_SIZE * 512, &mut allocated_pages);
1512
1513        let (l1, l2, _l3, l4, _offset) = split_vaddr(boundary);
1514        let b2 = read_page_table_entry(&mem, cr3, l1).unwrap();
1515        let b3 = read_page_table_entry(&mem, b2, l2).unwrap();
1516        store_page_table_entry(&mem, GuestAddress(b3), l4, PAGE_SIZE * 512); // store invalid pointer in page table
1517
1518        let walk = PTEWalker::new(&mem, cr3, boundary - PAGE_SIZE, boundary + PAGE_SIZE * 512)
1519            .filter(|i| i.present() || i.missing_page())
1520            .map(|i| (i.level, i.vaddrs.start, i.phys_addr(), i.missing_page()))
1521            .collect::<Vec<_>>();
1522
1523        let mut expected = vec![
1524            (0, 0, GuestAddress(0x2000), false),
1525            (1, 0x7fc0000000, GuestAddress(0x3000), false),
1526            (2, 0x7fffe00000, GuestAddress(0x4000), false),
1527            (3, 0x7ffffff000, t1 /*GuestAddress(0x1000)*/, false),
1528            (0, 0x8000000000, GuestAddress(0x6000), false),
1529            (1, 0x8000000000, GuestAddress(0x7000), false),
1530            (2, 0x8000000000, GuestAddress(0x200000), false),
1531        ];
1532        let mut missed_pages: Vec<(u8, u64, GuestAddress, bool)> = (boundary
1533            ..(boundary + PAGE_SIZE * 511))
1534            .step_by(PAGE_SIZE as usize)
1535            .map(|addr| (3, addr, GuestAddress(0), true))
1536            .collect();
1537        expected.append(&mut missed_pages);
1538        expected.append(&mut vec![
1539            (2, 0x8000200000, GuestAddress(0xc000), false),
1540            (3, 0x8000200000, t4, false),
1541        ]);
1542        //for (i,(real,expected)) in walk.iter().zip(expected.iter()).enumerate(){
1543        //    if real != expected {
1544        //        println!("at {} got {:X?}, expected {:X?}",i, real, expected);
1545        //        assert!(false);
1546        //    }
1547        //}
1548        assert_eq!(walk.len(), expected.len());
1549        assert_eq!(walk, expected);
1550    }
1551
1552    fn allocate_page(mem: &GuestMemoryMmap, allocated_pages: &mut HashSet<u64>) -> GuestAddress {
1553        let mut selected_page = 0;
1554        'outer: for region in mem.iter() {
1555            for page in (region.start_addr().0..region.start_addr().0 + (region.size() as u64))
1556                .step_by(PAGE_SIZE as usize)
1557            {
1558                if page != 0 && !allocated_pages.contains(&page) {
1559                    selected_page = page;
1560                    break 'outer;
1561                }
1562            }
1563        }
1564        assert_ne!(selected_page, 0);
1565        allocated_pages.insert(selected_page);
1566        return GuestAddress(selected_page);
1567    }
1568
1569    fn get_or_make_page_table_entry(
1570        mem: &GuestMemoryMmap,
1571        allocated_pages: &mut HashSet<u64>,
1572        page_table: GuestAddress,
1573        offset: u64,
1574    ) -> GuestAddress {
1575        if let Err(MemoryError::PageNotPresent(..)) =
1576            read_page_table_entry(mem, page_table.0, offset)
1577        {
1578            let new_page = allocate_page(mem, allocated_pages);
1579            store_page_table_entry(mem, page_table, offset, new_page.0);
1580        }
1581        let page = read_page_table_entry(mem, page_table.0, offset).unwrap();
1582        return GuestAddress(page);
1583    }
1584    fn make_vpage(
1585        mem: &GuestMemoryMmap,
1586        cr3: u64,
1587        vaddr: u64,
1588        allocated_pages: &mut HashSet<u64>,
1589    ) -> GuestAddress {
1590        let target_page = allocate_page(mem, allocated_pages);
1591        let (l1, l2, l3, l4, offset) = split_vaddr(vaddr);
1592        assert_eq!(offset, 0); //page aligned;
1593
1594        let l2_page = get_or_make_page_table_entry(mem, allocated_pages, GuestAddress(cr3), l1);
1595        let l3_page = get_or_make_page_table_entry(mem, allocated_pages, l2_page, l2);
1596        let l4_page = get_or_make_page_table_entry(mem, allocated_pages, l3_page, l3);
1597        store_page_table_entry(mem, l4_page, l4, target_page.0);
1598        return target_page;
1599    }
1600
1601    fn make_mem() -> GuestMemoryMmap {
1602        let page_size = PAGE_SIZE as usize;
1603
1604        let region_1_address = GuestAddress(0);
1605        let region_2_address = GuestAddress(PAGE_SIZE * 8);
1606        let mem_regions = [
1607            (region_1_address, page_size * 8),
1608            (region_2_address, page_size * 8),
1609        ];
1610        let regions = memory::anonymous(mem_regions.into_iter(), false, HugePageConfig::None)
1611            .expect("failed to allocate test memory");
1612        memory::test_utils::into_region_ext(regions)
1613    }
1614
1615    fn store(mem: &GuestMemoryMmap, addr: GuestAddress, val: u64) {
1616        mem.store(val, addr, Ordering::Relaxed).unwrap();
1617    }
1618
1619    fn store_page_table_entry(mem: &GuestMemoryMmap, paddr: GuestAddress, offset: u64, value: u64) {
1620        store(
1621            mem,
1622            GuestAddress(paddr.0 + offset * 8),
1623            value | BIT_PTE_PRESENT,
1624        );
1625    }
1626}
nyx_lite/mem.rs

nyx_lite/
mem.rs
