virt_support_aarch64emu/
translate.rs

1// Copyright (c) Microsoft Corporation.
2// Licensed under the MIT License.
3
4//! aarch64 page table walking.
5
6#![warn(missing_docs)]
7
8use crate::emulate::EmuTranslateError;
9use crate::emulate::EmuTranslateResult;
10use crate::emulate::TranslateGvaSupport;
11use crate::emulate::TranslateMode;
12use aarch64defs::Cpsr64;
13use aarch64defs::EsrEl2;
14use aarch64defs::FaultStatusCode;
15use aarch64defs::IntermPhysAddrSize;
16use aarch64defs::IssDataAbort;
17use aarch64defs::Pte;
18use aarch64defs::SctlrEl1;
19use aarch64defs::TranslationControlEl1;
20use aarch64defs::TranslationGranule0;
21use aarch64defs::TranslationGranule1;
22use guestmem::GuestMemory;
23use hvdef::HV_PAGE_SHIFT;
24use hvdef::hypercall::TranslateGvaControlFlagsArm64;
25use hvdef::hypercall::TranslateGvaResultCode;
26use thiserror::Error;
27
28/// Registers needed to walk the page table.
29#[derive(Debug, Clone)]
30pub struct TranslationRegisters {
31    /// SPSR_EL2
32    pub cpsr: Cpsr64,
33    /// SCTLR_ELx
34    pub sctlr: SctlrEl1,
35    /// TCR_ELx
36    pub tcr: TranslationControlEl1,
37    /// TTBR0_ELx
38    pub ttbr0: u64,
39    /// TTBR1_ELx
40    pub ttbr1: u64,
41    /// EsrEl2
42    pub syndrome: u64,
43
44    /// The way the processor uses to determine if an access is to encrypted
45    /// memory. This is used to enforce that page tables and executable code are
46    /// in encrypted memory.
47    pub encryption_mode: EncryptionMode,
48}
49
50/// The way the processor uses to determine if an access is to encrypted memory.
51#[derive(Debug, Copy, Clone)]
52pub enum EncryptionMode {
53    /// Memory accesses below the virtual top of memory address are encrypted.
54    Vtom(u64),
55    /// No memory is encrypted.
56    None,
57}
58
59/// Flags to control the page table walk.
60#[derive(Debug, Clone)]
61pub struct TranslateFlags {
62    /// Validate a VP in the current state can execute from this GVA.
63    pub validate_execute: bool,
64    /// Validate a VP in the current state can read from this GVA.
65    pub validate_read: bool,
66    /// Validate a VP in the current state can write to this GVA.
67    pub validate_write: bool,
68    /// The type of privilege check to perform.
69    pub privilege_check: TranslatePrivilegeCheck,
70    /// Update the page table entries' access and dirty bits as appropriate.
71    pub set_page_table_bits: bool,
72}
73
74/// The type of privilege check to perform.
75#[derive(Debug, Copy, Clone)]
76pub enum TranslatePrivilegeCheck {
77    /// No privilege checks.
78    None,
79    /// Validate user-mode access.
80    User,
81    /// Validate supervisor access.
82    Supervisor,
83    /// Validate both supervisor and user-mode access.
84    Both,
85    /// Validate according to the current privilege level.
86    CurrentPrivilegeLevel,
87}
88
89impl TranslateFlags {
90    /// Return flags based on the `HvTranslateVirtualAddress` hypercall input
91    /// flags.
92    ///
93    /// Note that not all flags are considered.
94    pub fn from_hv_flags(flags: TranslateGvaControlFlagsArm64) -> Self {
95        Self {
96            validate_execute: flags.validate_execute(),
97            validate_read: flags.validate_read(),
98            validate_write: flags.validate_write(),
99            privilege_check: if flags.pan_clear() {
100                TranslatePrivilegeCheck::None
101            } else if flags.user_access() {
102                if flags.supervisor_access() {
103                    TranslatePrivilegeCheck::Both
104                } else {
105                    TranslatePrivilegeCheck::User
106                }
107            } else if flags.supervisor_access() {
108                TranslatePrivilegeCheck::Supervisor
109            } else {
110                TranslatePrivilegeCheck::CurrentPrivilegeLevel
111            },
112            set_page_table_bits: flags.set_page_table_bits(),
113        }
114    }
115}
116
117/// Translation error.
118#[derive(Debug, Error)]
119pub enum Error {
120    /// Invalid address size
121    #[error("invalid address size at level")]
122    InvalidAddressSize(u8),
123    /// The page table flags were invalid.
124    #[error("invalid page table flags at level")]
125    InvalidPageTableFlags(u8),
126    /// A page table GPA was not mapped.
127    #[error("gpa unmapped at level")]
128    GpaUnmapped(u8),
129    /// The page was not present in the page table.
130    #[error("page not present at level")]
131    PageNotPresent(u8),
132    /// Accessing the GVA would create a privilege violation.
133    #[error("privilege violation at level")]
134    PrivilegeViolation(u8),
135}
136
137impl From<&Error> for TranslateGvaResultCode {
138    fn from(err: &Error) -> TranslateGvaResultCode {
139        match err {
140            Error::InvalidAddressSize(_) => TranslateGvaResultCode::INVALID_PAGE_TABLE_FLAGS,
141            Error::InvalidPageTableFlags(_) => TranslateGvaResultCode::INVALID_PAGE_TABLE_FLAGS,
142            Error::GpaUnmapped(_) => TranslateGvaResultCode::GPA_UNMAPPED,
143            Error::PageNotPresent(_) => TranslateGvaResultCode::PAGE_NOT_PRESENT,
144            Error::PrivilegeViolation(_) => TranslateGvaResultCode::PRIVILEGE_VIOLATION,
145        }
146    }
147}
148
149impl From<Error> for TranslateGvaResultCode {
150    fn from(err: Error) -> TranslateGvaResultCode {
151        (&err).into()
152    }
153}
154
155impl From<&Error> for EsrEl2 {
156    fn from(err: &Error) -> EsrEl2 {
157        let dfsc = match err {
158            Error::InvalidAddressSize(i) => FaultStatusCode::ADDRESS_SIZE_FAULT_LEVEL0.0 + i,
159            Error::InvalidPageTableFlags(i) => FaultStatusCode::TRANSLATION_FAULT_LEVEL0.0 + i,
160            Error::GpaUnmapped(i) => FaultStatusCode::ACCESS_FLAG_FAULT_LEVEL0.0 + i,
161            Error::PageNotPresent(i) => FaultStatusCode::ACCESS_FLAG_FAULT_LEVEL0.0 + i,
162            Error::PrivilegeViolation(i) => FaultStatusCode::PERMISSION_FAULT_LEVEL0.0 + i,
163        };
164        let data_abort = IssDataAbort::new().with_dfsc(FaultStatusCode(dfsc));
165        data_abort.into()
166    }
167}
168
169impl From<Error> for EsrEl2 {
170    fn from(err: Error) -> EsrEl2 {
171        (&err).into()
172    }
173}
174
175/// Emulates a page table walk.
176///
177/// This is suitable for implementing [`crate::emulate::EmulatorSupport::translate_gva`].
178pub fn emulate_translate_gva<T: TranslateGvaSupport>(
179    support: &mut T,
180    gva: u64,
181    mode: TranslateMode,
182) -> Result<Result<EmuTranslateResult, EmuTranslateError>, T::Error> {
183    // Always acquire the TLB lock for this path.
184    support.acquire_tlb_lock();
185
186    let registers = support.registers()?;
187    let flags = TranslateFlags {
188        validate_execute: matches!(mode, TranslateMode::Execute),
189        validate_read: matches!(mode, TranslateMode::Execute | TranslateMode::Read),
190        validate_write: matches!(mode, TranslateMode::Write),
191        privilege_check: TranslatePrivilegeCheck::CurrentPrivilegeLevel,
192        set_page_table_bits: true,
193    };
194
195    let r = match translate_gva_to_gpa(support.guest_memory(), gva, &registers, flags) {
196        Ok(gpa) => Ok(EmuTranslateResult {
197            gpa,
198            overlay_page: None,
199        }),
200        Err(err) => {
201            let mut syndrome: EsrEl2 = (&err).into();
202            let cur_syndrome: EsrEl2 = registers.syndrome.into();
203            syndrome.set_il(cur_syndrome.il());
204            Err(EmuTranslateError {
205                code: err.into(),
206                event_info: Some(syndrome),
207            })
208        }
209    };
210    Ok(r)
211}
212
213struct Aarch64PageTable {
214    pub table_address_gpa: u64,
215    pub page_shift: u64,
216    pub span_shift: u64,
217    pub level: u64,
218    pub level_width: u64,
219    pub is_hierarchical_permissions: bool,
220}
221
222fn get_root_page_table(
223    gva: u64,
224    registers: &TranslationRegisters,
225    flags: &TranslateFlags,
226) -> Result<(u64, Aarch64PageTable), Error> {
227    let use_ttbr1 = (gva & 0x00400000_00000000) != 0;
228    let (
229        root_address,
230        address_width,
231        granule_width,
232        ignore_top_byte,
233        ignore_top_byte_instruction,
234        is_hierarchical_permissions,
235    ) = if use_ttbr1 {
236        let granule_width = match registers.tcr.tg1() {
237            TranslationGranule1::TG_INVALID => return Err(Error::InvalidPageTableFlags(0)),
238            TranslationGranule1::TG_16KB => 14,
239            TranslationGranule1::TG_4KB => 12,
240            TranslationGranule1::TG_64KB => 16,
241            _ => return Err(Error::InvalidPageTableFlags(0)),
242        };
243        (
244            registers.ttbr1 & ((1 << 48) - 2),
245            registers.tcr.ttbr1_valid_address_bits(),
246            granule_width,
247            registers.tcr.tbi1() != 0,
248            registers.tcr.tbid1() != 0,
249            !registers.tcr.hpd1() != 0, // || processor does not support HPDS
250        )
251    } else {
252        let granule_width = match registers.tcr.tg0() {
253            TranslationGranule0::TG_4KB => 12,
254            TranslationGranule0::TG_64KB => 16,
255            TranslationGranule0::TG_16KB => 14,
256            _ => return Err(Error::InvalidPageTableFlags(0)),
257        };
258        (
259            registers.ttbr0 & ((1 << 48) - 2),
260            registers.tcr.ttbr0_valid_address_bits(),
261            granule_width,
262            registers.tcr.tbi0() != 0,
263            registers.tcr.tbid0() != 0,
264            !registers.tcr.hpd0() != 0, // || processor does not support HPDS
265        )
266    };
267    if !(25..=48).contains(&address_width) {
268        tracing::trace!(address_width, "Invalid TCR value");
269        return Err(Error::InvalidAddressSize(0));
270    }
271    let num_levels = (address_width - 1) / granule_width;
272    if num_levels == 0 || num_levels > 4 {
273        tracing::trace!(address_width, granule_width, "Invalid page hierarchy");
274        return Err(Error::InvalidPageTableFlags(0));
275    }
276    let ignore_top_byte =
277        ignore_top_byte && (!flags.validate_execute || ignore_top_byte_instruction);
278    let high_mask = !((1 << address_width) - 1);
279    let verify_high_bits = if use_ttbr1 {
280        // TTBR1 addresses should have all the high bits set.
281        let masked_address = gva
282            | if ignore_top_byte {
283                0xff000000_00000000
284            } else {
285                0
286            };
287        (masked_address & high_mask) == high_mask
288    } else {
289        // TTBR0 addresses should have all the high bits clear.
290        let masked_address = gva
291            & if ignore_top_byte {
292                0x00ffffff_ffffffff
293            } else {
294                0xffffffff_ffffffff
295            };
296        (masked_address & high_mask) == 0
297    };
298    if !verify_high_bits {
299        tracing::trace!(gva, address_width, "Invalid high bits");
300        return Err(Error::InvalidAddressSize(0));
301    }
302    let span_shift = granule_width + (granule_width - 3) * (num_levels - 1);
303    let level_width = address_width - span_shift;
304    Ok((
305        gva & !high_mask,
306        Aarch64PageTable {
307            table_address_gpa: root_address & !((1 << (level_width + 3)) - 1),
308            page_shift: granule_width,
309            span_shift,
310            level: num_levels - 1,
311            level_width,
312            is_hierarchical_permissions,
313        },
314    ))
315}
316
317struct PageTableWalkContext<'a> {
318    guest_memory: &'a GuestMemory,
319    flags: TranslateFlags,
320    check_user_access: bool,
321    check_supervisor_access: bool,
322    write_no_execute: bool,
323    output_size_mask: u64,
324}
325
326enum PageTableWalkResult {
327    Table(Aarch64PageTable),
328    BaseGpa(u64, u64),
329}
330
331fn get_next_page_table(
332    level: u8,
333    address: u64,
334    page_table: &Aarch64PageTable,
335    context: &PageTableWalkContext<'_>,
336    is_user_address: &mut bool,
337    is_writeable_address: &mut bool,
338    is_executable_address: &mut bool,
339) -> Result<PageTableWalkResult, Error> {
340    if page_table.table_address_gpa & context.output_size_mask != page_table.table_address_gpa {
341        tracing::trace!(
342            address,
343            level = page_table.level,
344            page_table_address = page_table.table_address_gpa,
345            "Invalid page table address"
346        );
347        return Err(Error::InvalidAddressSize(level));
348    }
349    let index_mask = (1 << page_table.level_width) - 1;
350    let pte_index = (address >> page_table.span_shift) & index_mask;
351    let pte_gpa = page_table.table_address_gpa + (pte_index << 3);
352    let mut pte_access = context
353        .guest_memory
354        .read_plain::<u64>(pte_gpa)
355        .map(Pte::from);
356    let mut pte;
357    loop {
358        pte = pte_access.map_err(|_| Error::GpaUnmapped(level))?;
359        let large_page_supported = match page_table.level {
360            3 => false,
361            2 => page_table.page_shift > 12,
362            _ => true,
363        };
364        if !pte.valid() || (!pte.not_large_page() && !large_page_supported) {
365            return Err(Error::PageNotPresent(level));
366        }
367        let next_address = pte.pfn() << HV_PAGE_SHIFT;
368        if pte.reserved_must_be_zero() != 0
369            || (next_address & context.output_size_mask) != next_address
370        {
371            return Err(Error::InvalidPageTableFlags(level));
372        }
373        if page_table.level > 0 && pte.not_large_page() {
374            if page_table.is_hierarchical_permissions {
375                *is_user_address = *is_user_address && !pte.ap_table_privileged_only();
376                *is_writeable_address = *is_writeable_address && !pte.ap_table_read_only();
377                *is_executable_address = *is_executable_address
378                    && if context.check_user_access {
379                        pte.uxn_table()
380                    } else {
381                        pte.pxn_table()
382                    };
383            }
384        } else {
385            // check permissions
386            *is_user_address = *is_user_address && pte.ap_unprivileged();
387            *is_writeable_address = *is_writeable_address && !pte.ap_read_only();
388            *is_executable_address = *is_executable_address
389                && !(if context.check_user_access {
390                    pte.user_no_execute()
391                } else {
392                    pte.privilege_no_execute()
393                });
394            if context.check_user_access {
395                if context.flags.validate_read && !*is_user_address {
396                    return Err(Error::PrivilegeViolation(level));
397                }
398                if context.write_no_execute && *is_writeable_address && *is_user_address {
399                    *is_executable_address = false;
400                }
401            } else {
402                if context.check_supervisor_access && *is_user_address {
403                    return Err(Error::PrivilegeViolation(level));
404                }
405                if *is_writeable_address && (*is_user_address || context.write_no_execute) {
406                    *is_executable_address = false;
407                }
408            }
409            if context.flags.validate_write && !*is_writeable_address
410                || context.flags.validate_execute && !*is_executable_address
411            {
412                return Err(Error::PrivilegeViolation(level));
413            }
414        }
415
416        // Update access and dirty bits.
417        let mut new_pte = pte;
418        if context.flags.set_page_table_bits {
419            new_pte.set_access_flag(true);
420            if context.flags.validate_write && new_pte.dbm() {
421                new_pte.set_ap_read_only(false);
422            }
423        }
424
425        // Access bits already set.
426        if new_pte == pte {
427            break;
428        }
429
430        let r = if !pte.not_large_page() {
431            context.guest_memory.compare_exchange(address, pte, new_pte)
432        } else {
433            context
434                .guest_memory
435                .compare_exchange(address, u64::from(pte) as u32, u64::from(new_pte) as u32)
436                .map(|r| {
437                    r.map(|n| Pte::from(n as u64))
438                        .map_err(|n| Pte::from(n as u64))
439                })
440        };
441
442        match r {
443            Ok(Ok(_)) => {
444                // Compare exchange succeeded, so continue.
445                break;
446            }
447            Ok(Err(pte)) => {
448                // Compare exchange failed. Loop around again.
449                pte_access = Ok(pte);
450                continue;
451            }
452            Err(err) => {
453                // Memory access failed. Loop around again to handle the
454                // failure consistently.
455                pte_access = Err(err);
456                continue;
457            }
458        }
459    }
460    let pfn_mask = !(1_u64 << (page_table.page_shift - HV_PAGE_SHIFT)).wrapping_sub(1);
461    let next_address = (pte.pfn() & pfn_mask) << HV_PAGE_SHIFT;
462    if page_table.level == 0 || !pte.not_large_page() {
463        Ok(PageTableWalkResult::BaseGpa(
464            next_address,
465            (1 << page_table.span_shift) - 1,
466        ))
467    } else {
468        Ok(PageTableWalkResult::Table(Aarch64PageTable {
469            table_address_gpa: next_address,
470            page_shift: page_table.page_shift,
471            span_shift: page_table.span_shift - (page_table.page_shift - 3),
472            level: page_table.level - 1,
473            level_width: page_table.page_shift - 3,
474            is_hierarchical_permissions: page_table.is_hierarchical_permissions,
475        }))
476    }
477}
478
479/// Translate a GVA by walking the processor's page tables.
480pub fn translate_gva_to_gpa(
481    guest_memory: &GuestMemory,
482    gva: u64,
483    registers: &TranslationRegisters,
484    flags: TranslateFlags,
485) -> Result<u64, Error> {
486    tracing::trace!(gva, ?registers, ?flags, "translating gva");
487
488    // If paging is disabled, just return the GVA as the GPA.
489    if !registers.sctlr.m() {
490        return Ok(gva);
491    }
492
493    // FEAT_LPA2 - Larger physical address for 4KB and 16KB translation granules
494
495    let (check_user_access, check_supervisor_access) = match flags.privilege_check {
496        TranslatePrivilegeCheck::None => (false, false),
497        TranslatePrivilegeCheck::User => (true, false),
498        TranslatePrivilegeCheck::Both => (true, true),
499        TranslatePrivilegeCheck::CurrentPrivilegeLevel if registers.cpsr.el() == 0 => (true, false),
500        TranslatePrivilegeCheck::Supervisor | TranslatePrivilegeCheck::CurrentPrivilegeLevel => {
501            (false, true)
502        }
503    };
504    let output_size = match registers.tcr.ips() {
505        IntermPhysAddrSize::IPA_32_BITS_4_GB => 32,
506        IntermPhysAddrSize::IPA_36_BITS_64_GB => 36,
507        IntermPhysAddrSize::IPA_40_BITS_1_TB => 40,
508        IntermPhysAddrSize::IPA_42_BITS_4_TB => 42,
509        IntermPhysAddrSize::IPA_44_BITS_16_TB => 44,
510        IntermPhysAddrSize::IPA_48_BITS_256_TB => 48,
511        IntermPhysAddrSize::IPA_52_BITS_4_PB => 52,
512        IntermPhysAddrSize::IPA_56_BITS_64_PB => 56,
513        _ => return Err(Error::InvalidPageTableFlags(0)),
514    };
515    let write_no_execute = registers.sctlr.wxn();
516    let walk_context = PageTableWalkContext {
517        guest_memory,
518        flags,
519        check_user_access,
520        check_supervisor_access,
521        write_no_execute,
522        output_size_mask: (1 << output_size) - 1,
523    };
524    let mut is_user_address = false;
525    let mut is_writeable_address = true;
526    let mut is_executable_address = true;
527    let (address, mut page_table) = get_root_page_table(gva, registers, &walk_context.flags)?;
528    let mut level = 1;
529    loop {
530        page_table = match get_next_page_table(
531            level,
532            address,
533            &page_table,
534            &walk_context,
535            &mut is_user_address,
536            &mut is_writeable_address,
537            &mut is_executable_address,
538        )? {
539            PageTableWalkResult::BaseGpa(base_address, mask) => {
540                break Ok(base_address + (gva & mask));
541            }
542            PageTableWalkResult::Table(next_table) => next_table,
543        };
544        level += 1;
545    }
546}
virt_support_aarch64emu/translate.rs

virt_support_aarch64emu/
translate.rs
