From 456ee51d362e5177f4eb76573bfe3e5a72e2a9fe Mon Sep 17 00:00:00 2001 From: Paul Pan Date: Fri, 12 Apr 2024 18:48:36 +0800 Subject: [PATCH] feat: add kernel pagetable support --- kernel/.cargo/config.toml | 6 + kernel/Cargo.lock | 36 ++--- kernel/Cargo.toml | 1 - kernel/src/arch/riscv/entry.rs | 85 ++++++++--- kernel/src/arch/riscv/layout.rs | 9 +- kernel/src/arch/riscv/linker32.ld | 18 ++- kernel/src/arch/riscv/linker64.ld | 20 +-- kernel/src/arch/riscv/tls.rs | 12 -- kernel/src/arch/riscv/trap.rs | 4 +- kernel/src/arch/riscv/vspace/mod.rs | 4 +- kernel/src/arch/riscv/vspace/utils.rs | 203 ++++++++++++++----------- kernel/src/drivers/serial/sifive.rs | 6 +- kernel/src/drivers/serial/uart16550.rs | 5 +- kernel/src/entry.rs | 12 +- kernel/src/plat/console.rs | 6 +- lib/vspace/src/addr.rs | 3 + 16 files changed, 243 insertions(+), 187 deletions(-) diff --git a/kernel/.cargo/config.toml b/kernel/.cargo/config.toml index 235d8cd..897e795 100644 --- a/kernel/.cargo/config.toml +++ b/kernel/.cargo/config.toml @@ -1,8 +1,14 @@ [build] target = "riscv64imac-unknown-none-elf" +[unstable] +build-std = ["core", "compiler_builtins", "alloc"] +build-std-features = ["compiler-builtins-mem"] + [target.'cfg(all(target_arch = "riscv64", target_os = "none"))'] +rustflags = ["-C", "relocation-model=static", "-C", "code-model=medium", "-C", "target-feature=+relax"] runner = "qemu-system-riscv64 -nographic -machine virt -serial mon:stdio -smp 1 -kernel " [target.'cfg(all(target_arch = "riscv32", target_os = "none"))'] +rustflags = ["-C", "relocation-model=static", "-C", "code-model=medium", "-C", "target-feature=+relax"] runner = "qemu-system-riscv32 -nographic -machine virt -serial mon:stdio -smp 1 -kernel " diff --git a/kernel/Cargo.lock b/kernel/Cargo.lock index 57404be..0ce51db 100644 --- a/kernel/Cargo.lock +++ b/kernel/Cargo.lock @@ -6,7 +6,7 @@ version = 3 name = "allocator" version = "0.1.0" dependencies = [ - "spin 0.9.8", + "spin", "static_assertions", "utils", "vspace", @@ -22,9 +22,9 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80" [[package]] name = "bit_field" @@ -77,28 +77,18 @@ dependencies = [ "bitflags 2.5.0", "cfg-if", "fdt", - "lazy_static", "log", "num-derive", "num-traits", "riscv", "sbi-rt", - "spin 0.9.8", + "spin", "static_assertions", "uart_16550", "utils", "vspace", ] -[[package]] -name = "lazy_static" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" -dependencies = [ - "spin 0.5.2", -] - [[package]] name = "lock_api" version = "0.4.11" @@ -146,9 +136,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.35" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ "proc-macro2", ] @@ -174,9 +164,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.14" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" +checksum = "80af6f9131f277a45a3fba6ce8e2258037bb0477a67e610d3c1fe046ab31de47" [[package]] name = "sbi-rt" @@ -199,12 +189,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "spin" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" - [[package]] name = "spin" version = "0.9.8" @@ -222,9 +206,9 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "syn" -version = "2.0.53" +version = "2.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7383cd0e49fff4b6b90ca5670bfd3e9d6a733b3f90c686605aa7eec8c4996032" +checksum = "44cfb93f38070beee36b3fef7d4f5a16f27751d94b187b666a5cc5e9b0d30687" dependencies = [ "proc-macro2", "quote", diff --git a/kernel/Cargo.toml b/kernel/Cargo.toml index 68b3094..2bfe870 100644 --- a/kernel/Cargo.toml +++ b/kernel/Cargo.toml @@ -41,7 +41,6 @@ vspace = { path = "../lib/vspace", default-features = false } bitflags = "2.5" cfg-if = "1.0" fdt = "0.1" -lazy_static = { version = "1.4", features = ["spin_no_std"] } log = "0.4" num-derive = "0.4" num-traits = { version = "0.2", default-features = false } diff --git a/kernel/src/arch/riscv/entry.rs b/kernel/src/arch/riscv/entry.rs index 232e5e4..4c4bb0c 100644 --- a/kernel/src/arch/riscv/entry.rs +++ b/kernel/src/arch/riscv/entry.rs @@ -1,6 +1,10 @@ -use crate::arch::layout::zero_bss; -use crate::arch::vspace::utils::{setup_kernel_paging, setup_memory}; -use allocator::RamBlock; +use crate::arch::init_early_console; +use crate::arch::layout::{mmap_phys_to_virt, zero_bss}; +use crate::arch::vspace::{setup_kernel_paging, setup_memory}; +use crate::entry::{rust_main, HART_ID}; +use crate::plat::console::mute_console; +use fdt::Fdt; +use vspace::addr::AddressOps; #[naked] #[no_mangle] @@ -25,15 +29,51 @@ unsafe extern "C" fn _start(hart_id: usize, device_tree_addr: usize) -> ! { addi gp, gp, %pcrel_lo(1b) .option pop - la tp, TSS_START - la sp, {stack} + {stack_size} - j {main} + lla t0, KERNEL_OFFSET + ld t0, 0(t0) + + # load and relocate registers + lla tp, TSS_START + lla sp, {stack} + {stack_size} + add gp, gp, t0 + add tp, tp, t0 + add sp, sp, t0 + + # TODO: riscv32 do not need it, add a function in board/ and call it + # setup early paging + lla t1, boot_page_table + srli t1, t1, 12 + li t2, 8 << 60 + or t1, t1, t2 + csrw satp, t1 + sfence.vma + + # jump to absolute address + lla t1, {main} + add t1, t1, t0 + jr t1 .section .data - .global KERNEL_OFFSET - KERNEL_OFFSET: .quad __kernel_offset - .global MMAP_OFFSET - MMAP_OFFSET: .quad MMAP_BASE_ADDRESS + .global KERNEL_OFFSET + KERNEL_OFFSET: .quad __kernel_offset + .global MMAP_OFFSET + MMAP_OFFSET: .quad MMAP_BASE_ADDRESS + + # TODO: move this into board/ + .section .temp.boot_page_table + .align 12 + boot_page_table: + # sv39 page table + # 0x00000000_00000000 -> 0x00000000 [ 0x00000000_00000000 -> 0x00000000_40000000 ] + # 0x00000000_40000000 -> 0x40000000 [ 0x00000000_40000000 -> 0x00000000_80000000 ] + # 0x00000000_80000000 -> 0x80000000 [ 0x00000000_80000000 -> 0x00000001_00000000 ] + # 0xFFFFFFD0_00000000 -> 0x80000000 [ 0xFFFFFFD0_00000000 -> 0xFFFFFFD0_40000000 ] + .quad (0x00000 << 10) | 0xf + .quad (0x40000 << 10) | 0xf + .quad (0x80000 << 10) | 0xf + .zero 8 * 317 + .quad (0x80000 << 10) | 0xf + .zero 8 * 191 ", stack_size = const STACK_SIZE, stack = sym STACK, @@ -42,24 +82,19 @@ unsafe extern "C" fn _start(hart_id: usize, device_tree_addr: usize) -> ! { ) } -extern "C" fn pre_main(hart_id: usize, fdt_addr: usize) -> ! { +unsafe fn pre_main(hart_id: usize, fdt_addr: usize) { zero_bss(); - // Don't know why, but the **fucking** rust compiler will hard-code jump table to absolute - // addresses, even if I forced compiler to use pic mode, and global spin-lock depends on it, - // so we must remap the kernel to the higher half before anything. - // Console is not available yet !! - - let fdt = unsafe { Fdt::from_ptr(fdt_addr as *const u8).unwrap() }; - let mut allocator = RamBlock::<8>::new(); - - unsafe { - setup_memory(&fdt, fdt_addr, &mut allocator); - setup_kernel_paging(&mut allocator); - } - + HART_ID.set(hart_id); crate::logging::init(); init_early_console(); - crate::entry::rust_main(hart_id, fdt, allocator); + setup_memory(fdt_addr); + setup_kernel_paging(); + + // after kernel paging, board level early console is broken (no address mapping) + mute_console(); + + let fdt = unsafe { Fdt::from_ptr(mmap_phys_to_virt(fdt_addr.into()).as_const_ptr()).unwrap() }; + rust_main(fdt); } diff --git a/kernel/src/arch/riscv/layout.rs b/kernel/src/arch/riscv/layout.rs index 84ac3f9..f63292f 100644 --- a/kernel/src/arch/riscv/layout.rs +++ b/kernel/src/arch/riscv/layout.rs @@ -43,10 +43,13 @@ pub fn zero_bss() { unsafe { clear_range( - BOOT_STACK_END.as_phys_addr().as_mut_ptr(), - BSS_END.as_phys_addr().as_mut_ptr(), + kernel_virt_to_phys(BOOT_STACK_END.as_virt_addr()).as_mut_ptr(), + kernel_virt_to_phys(BSS_END.as_virt_addr()).as_mut_ptr(), + ); + clear_range( + kernel_virt_to_phys(TBSS_START.as_virt_addr()).as_mut_ptr(), + kernel_virt_to_phys(TBSS_END.as_virt_addr()).as_mut_ptr(), ); - clear_range(TBSS_START.as_phys_addr().as_mut_ptr(), TBSS_END.as_phys_addr().as_mut_ptr()); } } diff --git a/kernel/src/arch/riscv/linker32.ld b/kernel/src/arch/riscv/linker32.ld index d4d69da..fdfe792 100644 --- a/kernel/src/arch/riscv/linker32.ld +++ b/kernel/src/arch/riscv/linker32.ld @@ -1,16 +1,17 @@ OUTPUT_ARCH(riscv) ENTRY(_start) -MMAP_BASE_ADDRESS = 0x0; -BASE_ADDRESS = 0x80200000; -PHY_BASE_ADDRESS = 0x80200000; -PAGE_SIZE = 0x1000; +MMAP_BASE_ADDRESS = 0x00000000; +BASE_ADDRESS = 0x80200000; +PHY_BASE_ADDRESS = 0x80200000; +PAGE_SIZE = 0x1000; SECTIONS { . = BASE_ADDRESS; KERNEL_START = .; __kernel_offset = . - PHY_BASE_ADDRESS; + . = ALIGN(PAGE_SIZE); .text : AT(ADDR(.text) - __kernel_offset) { TEXT_START = .; *(.text.entry) @@ -18,26 +19,27 @@ SECTIONS { TEXT_END = .; } + . = ALIGN(PAGE_SIZE); .rodata : AT(ADDR(.rodata) - __kernel_offset) { - . = ALIGN(PAGE_SIZE); RODATA_START = .; *(.rodata .rodata.*) *(.srodata .srodata.*) RODATA_END = .; } + . = ALIGN(PAGE_SIZE); .data : AT(ADDR(.data) - __kernel_offset) { - . = ALIGN(PAGE_SIZE); DATA_START = .; *(.data .data.*) + *(.temp .temp.*) PROVIDE( __global_pointer$ = . + 0x800 ); *(.sdata .sdata.*) DATA_END = .; } + . = ALIGN(PAGE_SIZE); .bss : AT(ADDR(.bss) - __kernel_offset) { - . = ALIGN(PAGE_SIZE); BSS_START = .; *(.bss.boot_stack) BOOT_STACK_END = .; @@ -47,8 +49,8 @@ SECTIONS { BSS_END = .; } + . = ALIGN(PAGE_SIZE); .tss : AT(ADDR(.tss) - __kernel_offset) { - . = ALIGN(PAGE_SIZE); TSS_START = .; . = ALIGN(8); diff --git a/kernel/src/arch/riscv/linker64.ld b/kernel/src/arch/riscv/linker64.ld index 8eff525..1758743 100644 --- a/kernel/src/arch/riscv/linker64.ld +++ b/kernel/src/arch/riscv/linker64.ld @@ -3,19 +3,20 @@ ENTRY(_start) /* We use high memory (0xFFFF....) for kernel space * For sv39 and larger layout, memory base starts from 0xFFFFFFC000000000: {1'b1, {38{1'b0}}} - * Our kernel will placed at 0xFFFFFFD000000000 (VA) and 0x80200000 (PA) - * Regions between 0x...C... and 0x...D... will be reserved for firmware starting from 0x80000000 (PA) */ + * Our kernel will placed at 0xFFFFFFD000200000 (VA) and 0x80200000 (PA) + */ MMAP_BASE_ADDRESS = 0xFFFFFFC000000000; -BASE_ADDRESS = 0xFFFFFFD000000000; -PHY_BASE_ADDRESS = 0x80200000; -PAGE_SIZE = 0x1000; +BASE_ADDRESS = 0xFFFFFFD000200000; +PHY_BASE_ADDRESS = 0x0000000080200000; +PAGE_SIZE = 0x1000; SECTIONS { . = BASE_ADDRESS; KERNEL_START = .; __kernel_offset = . - PHY_BASE_ADDRESS; + . = ALIGN(PAGE_SIZE); .text : AT(ADDR(.text) - __kernel_offset) { TEXT_START = .; *(.text.entry) @@ -23,26 +24,27 @@ SECTIONS { TEXT_END = .; } + . = ALIGN(PAGE_SIZE); .rodata : AT(ADDR(.rodata) - __kernel_offset) { - . = ALIGN(PAGE_SIZE); RODATA_START = .; *(.rodata .rodata.*) *(.srodata .srodata.*) RODATA_END = .; } + . = ALIGN(PAGE_SIZE); .data : AT(ADDR(.data) - __kernel_offset) { - . = ALIGN(PAGE_SIZE); DATA_START = .; *(.data .data.*) + *(.temp .temp.*) PROVIDE( __global_pointer$ = . + 0x800 ); *(.sdata .sdata.*) DATA_END = .; } + . = ALIGN(PAGE_SIZE); .bss : AT(ADDR(.bss) - __kernel_offset) { - . = ALIGN(PAGE_SIZE); BSS_START = .; *(.bss.boot_stack) BOOT_STACK_END = .; @@ -52,8 +54,8 @@ SECTIONS { BSS_END = .; } + . = ALIGN(PAGE_SIZE); .tss : AT(ADDR(.tss) - __kernel_offset) { - . = ALIGN(PAGE_SIZE); TSS_START = .; . = ALIGN(8); diff --git a/kernel/src/arch/riscv/tls.rs b/kernel/src/arch/riscv/tls.rs index 2706e15..0c9dc00 100644 --- a/kernel/src/arch/riscv/tls.rs +++ b/kernel/src/arch/riscv/tls.rs @@ -3,15 +3,3 @@ // https://github.com/riscv-non-isa/riscv-elf-psabi-doc // - Rust #[thread_local] feature // https://github.com/rust-lang/rust/issues/29594 - -// TODO: use TLS for per-hart data - -pub fn get_hart_id() -> usize { - let id; - unsafe { core::arch::asm!("mv {}, tp", out(reg) id) } - id -} - -pub fn set_hart_id(id: usize) { - unsafe { core::arch::asm!("mv tp, {}", in(reg) id) } -} diff --git a/kernel/src/arch/riscv/trap.rs b/kernel/src/arch/riscv/trap.rs index 2a618e8..cd80201 100644 --- a/kernel/src/arch/riscv/trap.rs +++ b/kernel/src/arch/riscv/trap.rs @@ -1,10 +1,10 @@ +use crate::entry::HART_ID; use crate::plat::console::CONSOLE; use cfg_if::cfg_if; use log::trace; use riscv::register::scause::{Interrupt as I, Trap as T}; use riscv::register::stvec::TrapMode; -use super::tls::get_hart_id; use crate::plat::timer::{Timer, TimerOps}; use crate::plat::trap::{Trap, TrapContextOps, TrapOps}; @@ -32,7 +32,7 @@ extern "C" fn kernel_trap_handler(tf: &mut TrapContext) { fn trap_handler(_tf: &mut TrapContext, _from_kernel: bool) { let scause = riscv::register::scause::read(); - trace!("[Interrupt] cpu@{} scause: {:?}", get_hart_id(), scause.cause()); + trace!("[Interrupt] cpu@{} scause: {:?}", HART_ID.get(), scause.cause()); match scause.cause() { T::Interrupt(I::SupervisorTimer) => { // TODO: refactor this diff --git a/kernel/src/arch/riscv/vspace/mod.rs b/kernel/src/arch/riscv/vspace/mod.rs index d05123b..cedc1a0 100644 --- a/kernel/src/arch/riscv/vspace/mod.rs +++ b/kernel/src/arch/riscv/vspace/mod.rs @@ -1,4 +1,6 @@ mod entry; mod table; mod traits; -pub mod utils; +mod utils; + +pub use utils::*; diff --git a/kernel/src/arch/riscv/vspace/utils.rs b/kernel/src/arch/riscv/vspace/utils.rs index 004ea7e..58d3296 100644 --- a/kernel/src/arch/riscv/vspace/utils.rs +++ b/kernel/src/arch/riscv/vspace/utils.rs @@ -2,16 +2,34 @@ use super::table::Table; use super::traits::{PhysAddrPaging, TableLevelSize}; use crate::arch::layout::*; use allocator::RamBlock; -use fdt::Fdt; +use log::{debug, info}; +use spin::Mutex; use utils::size::GIB; use vspace::addr::*; -use vspace::paging::{MapAttr, PageError::*, TableLevel, TableOps}; +use vspace::paging::PageError::*; +use vspace::paging::*; -pub unsafe fn setup_memory(fdt_addr: usize, mem: &mut RamBlock<8>) { - let fdt = unsafe { Fdt::from_ptr(fdt_addr as *const u8).unwrap() }; +#[thread_local] +static KERNEL_PAGETABLE: Mutex> = Mutex::new(None); + +static KERNEL_ALLOCATOR: Mutex> = Mutex::new(RamBlock::new()); + +#[inline] +fn alloc_page() -> PhysAddr { + KERNEL_ALLOCATOR.lock().alloc(PAGE_LAYOUT).expect("Failed to allocate page") +} + +pub unsafe fn setup_memory(fdt_addr: usize) { + info!("Setting up memory"); + let fdt = unsafe { fdt::Fdt::from_ptr(fdt_addr as *const u8).unwrap() }; + let mut mem = KERNEL_ALLOCATOR.lock(); // Add main memory regions to allocator for region in fdt.memory().regions() { + debug!( + "Adding free memory:\tstart: {:x?}, size: {:x?}", + region.starting_address, region.size + ); mem.dealloc( PhysAddr::from(region.starting_address), align_up(region.size.unwrap(), PAGE_SIZE), @@ -20,10 +38,12 @@ pub unsafe fn setup_memory(fdt_addr: usize, mem: &mut RamBlock<8>) { // Exclude memory occupied by UEFI for region in fdt.memory_reservations() { - mem.reserve( - PhysAddr::from(region.address()), - align_up(region.size(), PAGE_SIZE), + debug!( + "Reserving memory:\tstart: {:x?}, size: {:x?}", + region.address(), + region.size() ); + mem.reserve(PhysAddr::from(region.address()), align_up(region.size(), PAGE_SIZE)); } // Exclude memory occupied by OpenSBI @@ -33,6 +53,10 @@ pub unsafe fn setup_memory(fdt_addr: usize, mem: &mut RamBlock<8>) { .expect("No reserved memory found in device tree"); for child in reserved_memory { let region = child.reg().unwrap().next().unwrap(); + debug!( + "Reserving memory:\tstart: {:x?}, size: {:x?}", + region.starting_address, region.size + ); mem.reserve( PhysAddr::from(region.starting_address), align_up(region.size.unwrap(), PAGE_SIZE), @@ -40,56 +64,92 @@ pub unsafe fn setup_memory(fdt_addr: usize, mem: &mut RamBlock<8>) { } // Exclude kernel memory - let kernel_start = KERNEL_START.as_phys_addr() - KERNEL_OFFSET; - let kernel_end = (KERNEL_END.as_phys_addr() - KERNEL_OFFSET).align_up(PAGE_SIZE); + let kernel_start = kernel_virt_to_phys(KERNEL_START.as_virt_addr()); + let kernel_end = kernel_virt_to_phys(KERNEL_END.as_virt_addr()).align_up(PAGE_SIZE); + debug!("Reserving kernel memory:\tstart: {:x?}, end: {:x?}", kernel_start, kernel_end); mem.reserve(kernel_start, (kernel_end - kernel_start).as_usize()); // Exclude FDT table let fdt_addr = PhysAddr::from(fdt_addr); let fdt_size = align_up(fdt.total_size(), PAGE_SIZE); + debug!("Reserving FDT memory:\tstart: {:x?}, size: {:x?}", fdt_addr, fdt_size); mem.reserve(fdt_addr, fdt_size); } -pub unsafe fn setup_kernel_paging( - allocator: &mut RamBlock<8>, - hart_id: usize, - fdt_addr: usize, -) -> ! { - let mut alloc = || { - allocator - .alloc(PAGE_LAYOUT) - .expect("Failed to allocate page table") - }; - - let root_pt = alloc(); - let kernel_pt = unsafe { Table::new(root_pt.as_usize().into()) }; - - let mut map = |vaddr: VirtAddr, paddr: PhysAddr, attr: MapAttr, level: TableLevel| loop { - match kernel_pt.map(vaddr, paddr, attr, level) { - Ok(_) => return true, - Err(MissingEntry(missed_level)) => kernel_pt - .map( - missed_level.align(vaddr), - alloc(), - MapAttr::PAGE_TABLE, - missed_level, - ) - .expect("Failed to map miss-ed page table"), - Err(AlreadyMapped(_)) => return false, +#[inline] +fn map_one(pt: &mut Table, vaddr: VirtAddr, paddr: PhysAddr, attr: MapAttr, level: TableLevel) -> PageResult<()> { + loop { + let ret = pt.map(vaddr, paddr, attr, level); + if let Err(MissingEntry(missed_level)) = ret { + pt.map(missed_level.align(vaddr), alloc_page(), MapAttr::PAGE_TABLE, missed_level)?; + continue; } - }; + return ret; + } +} + +#[inline] +fn map_range(pt: &mut Table, from: VirtAddr, to: PhysAddr, size: usize, attr: MapAttr) -> PageResult<()> { + let mut virt_start = from; + let mut phys_start = to; + let virt_end = from + size; + let mut map_level = Table::MAX_PAGE_SIZE; + + debug!("Mapping physical memory:\t[{:X?}, {:X?}]", virt_start, virt_end,); + + while virt_start < virt_end { + let ret = map_one(pt, virt_start, phys_start, attr, map_level); + + if ret.is_ok() { + // map success, move to next region + virt_start += map_level.level_size(); + phys_start += map_level.level_size(); + + // check whether we could raise the level + if let Some(prv) = map_level.previous() + && prv.is_aligned(phys_start) + { + map_level = prv; + } + continue; + } + + // already mapped, try smaller level + match map_level.next() { + Some(next) => map_level = next, + None => return ret, + } + } + + Ok(()) +} + +pub unsafe fn map(from: VirtAddr, to: PhysAddr, size: usize, attr: MapAttr) { + let mut guard = KERNEL_PAGETABLE.lock(); + let pt = guard.as_mut().unwrap(); + map_range(pt, from, to, size, attr).expect("Failed to map memory"); +} + +pub unsafe fn setup_kernel_paging() { + info!("Setting up kernel paging"); + + let root_pt = alloc_page(); + let kernel_pt = Table::new(root_pt.as_usize().into()); macro_rules! map_section { (($($section:ident),+),$attr:expr) => { $( - for addr in (concat_idents!($section, _START).as_phys_addr()..concat_idents!($section, _END).as_phys_addr()).step_by(PAGE_SIZE) { - if !map(kernel_phys_to_virt(addr), addr, $attr, TableLevel::Level0) { + let start = concat_idents!($section, _START).as_virt_addr(); + let end = concat_idents!($section, _END).as_virt_addr(); + debug!("Mapping section {}:\t[{:X?}, {:X?}]", stringify!($section), start, end); + for addr in (start..end).step_by(PAGE_SIZE) { + let _ = map_one(kernel_pt, addr, kernel_virt_to_phys(addr), $attr, TableLevel::Level0).is_err_and(|_| { panic!( "Failed to map section: {:X?} - {:X?}", - concat_idents!($section, _START).as_phys_addr(), - concat_idents!($section, _END).as_phys_addr() + concat_idents!($section, _START).as_virt_addr(), + concat_idents!($section, _END).as_virt_addr() ); - } + }); } )+ }; @@ -103,54 +163,21 @@ pub unsafe fn setup_kernel_paging( map_section!((TSS), MapAttr::READABLE | MapAttr::WRITABLE); // map 4 GiB physical memory - // TODO: walk fdt to get all memory region? - { - #[cfg(feature = "legacy")] - let level = TableLevel::Level1; - #[cfg(not(feature = "legacy"))] - let level = TableLevel::Level2; + // TODO: walk fdt to get all memory region? put it in driver init + map_range( + kernel_pt, + mmap_phys_to_virt(PhysAddr(0)), + PhysAddr(0), + 3 * GIB - 1 + GIB, + MapAttr::READABLE | MapAttr::WRITABLE, + ) + .expect("Failed to map physical memory"); - let addr_end = PhysAddr(3 * GIB - 1 + GIB); - let mut phys_addr = PhysAddr(0); - let mut map_level = level; - - while phys_addr < addr_end { - let ok = map( - kernel_phys_to_virt(phys_addr), - phys_addr, - MapAttr::READABLE | MapAttr::WRITABLE, - map_level, - ); - - if ok || map_level.next().is_none() { - // map success or reach the end, move to next region - phys_addr += map_level.level_size(); - - // check whether we could raise the level - if let Some(prv) = map_level.previous() - && prv.is_aligned(phys_addr) - { - map_level = prv; - } - continue; - } - - // already mapped, try smaller level - map_level = map_level.next().unwrap(); - } - } - - riscv::register::satp::set(riscv::register::satp::Mode::Sv39, 0, root_pt.to_ppn()); + // setup new pagetable + debug!("Setting up new kernel pagetable"); + riscv::register::satp::set(Table::mode(), 0, root_pt.to_ppn()); riscv::asm::sfence_vma_all(); - macro_rules! remap { - ($reg:ident) => { - let $reg: usize; - core::arch::asm!(concat!("mv {}, ", stringify!($reg)), out(reg) $reg); - let $reg = kernel_phys_to_virt(PhysAddr($reg)); - core::arch::asm!(concat!("mv ", stringify!($reg), ", {}"), in(reg) $reg.as_usize()); - }; - } - - todo!("remap registers"); + // switch to virtual address + *KERNEL_PAGETABLE.lock() = Some(Table::new(mmap_phys_to_virt(root_pt))); } diff --git a/kernel/src/drivers/serial/sifive.rs b/kernel/src/drivers/serial/sifive.rs index 8d9e465..603ba54 100644 --- a/kernel/src/drivers/serial/sifive.rs +++ b/kernel/src/drivers/serial/sifive.rs @@ -1,8 +1,10 @@ +use crate::arch::layout::mmap_phys_to_virt; use crate::drivers::serial::SerialDriver; use crate::drivers::Driver; use crate::plat::console::ConsoleDevice; use core::sync::atomic::{AtomicPtr, Ordering}; use fdt::node::FdtNode; +use vspace::addr::PhysAddr; // https://static.dev.sifive.com/FU540-C000-v1.0.pdf @@ -121,7 +123,9 @@ impl Driver for UartSifive { } fn setup(fdt: FdtNode) -> Self { - let addr = fdt.reg().unwrap().next().unwrap().starting_address as *mut u32; + let addr = fdt.reg().unwrap().next().unwrap().starting_address; + let addr = unsafe { mmap_phys_to_virt(PhysAddr::from(addr)) }; + let addr: *mut u32 = addr.into(); let uart = unsafe { UartSifive { tx_data: AtomicPtr::new(addr), diff --git a/kernel/src/drivers/serial/uart16550.rs b/kernel/src/drivers/serial/uart16550.rs index f2f70c9..ab87164 100644 --- a/kernel/src/drivers/serial/uart16550.rs +++ b/kernel/src/drivers/serial/uart16550.rs @@ -1,8 +1,10 @@ +use crate::arch::layout::mmap_phys_to_virt; use crate::drivers::serial::SerialDriver; use crate::drivers::Driver; use crate::plat::console::ConsoleDevice; use fdt::node::FdtNode; use uart_16550::MmioSerialPort; +use vspace::addr::PhysAddr; pub struct Uart16550 where Uart16550: SerialDriver @@ -17,7 +19,8 @@ impl Driver for Uart16550 { fn setup(fdt: FdtNode) -> Self { let addr = fdt.reg().unwrap().next().unwrap().starting_address; - let mut serial_port = unsafe { MmioSerialPort::new(addr as usize) }; + let addr = unsafe { mmap_phys_to_virt(PhysAddr::from(addr)) }; + let mut serial_port = unsafe { MmioSerialPort::new(addr.into()) }; serial_port.init(); Uart16550 { port: serial_port } } diff --git a/kernel/src/entry.rs b/kernel/src/entry.rs index 13eed07..406477f 100644 --- a/kernel/src/entry.rs +++ b/kernel/src/entry.rs @@ -1,9 +1,7 @@ -use crate::arch::init_early_console; use crate::plat::console::{set_console, ConsoleDevice, ConsoleDriver, CONSOLE}; use crate::plat::lowlevel::{Hardware, LowLevel}; use crate::plat::timer::{Timer, TimerOps}; use crate::plat::trap::{Trap, TrapOps}; -use allocator::RamBlock; use core::cell::Cell; use fdt::Fdt; use log::{debug, error, info, warn}; @@ -11,13 +9,9 @@ use log::{debug, error, info, warn}; #[thread_local] pub static HART_ID: Cell = Cell::new(0); -// NOTE: we will call rust_main through trap (stvec), make sure it is aligned -#[repr(align(4))] -pub extern "C" fn rust_main(hart_id: usize, fdt_addr: usize, allocator: &mut RamBlock<8>) -> ! { - HART_ID.set(hart_id); - - info!("Kernel Started"); +pub fn rust_main(fdt: Fdt) -> ! { setup_console(&fdt); + info!("Kernel Started"); #[cfg(test)] { @@ -52,7 +46,7 @@ pub extern "C" fn rust_main(hart_id: usize, fdt_addr: usize, allocator: &mut Ram Hardware::shutdown(true); } -fn setup_console(fdt: &fdt::Fdt) { +fn setup_console(fdt: &Fdt) { // NOTE: ignore stdin: both stdin and stdout will go through stdout device match fdt.chosen().stdout().and_then(|stdout| ConsoleDriver::new(stdout)) { None => warn!("No stdout device found or no compatible drivers found, falling back to platform default"), diff --git a/kernel/src/plat/console.rs b/kernel/src/plat/console.rs index 537deea..8b4a810 100644 --- a/kernel/src/plat/console.rs +++ b/kernel/src/plat/console.rs @@ -22,7 +22,7 @@ macro_rules! create_console_driver { .any(|s| $dynamic_driver::compatible().contains(&s)) { debug!("Console: Using driver: {}", stringify!($dynamic_driver)); - return Some(ConsoleDriver::$dynamic_driver($dynamic_driver::setup(fdt))) + return Some(ConsoleDriver::$dynamic_driver($dynamic_driver::setup(fdt))); } )+ } @@ -105,6 +105,10 @@ pub fn set_console(driver: ConsoleDriver) { *CONSOLE.lock() = driver; } +pub fn mute_console() { + *CONSOLE.lock() = ConsoleDriver::SilenceConsole(SilenceConsole); +} + #[cfg(test)] mod tests { use super::*; diff --git a/lib/vspace/src/addr.rs b/lib/vspace/src/addr.rs index 9e75c03..7748e6e 100644 --- a/lib/vspace/src/addr.rs +++ b/lib/vspace/src/addr.rs @@ -24,6 +24,9 @@ pub trait AddressOps: Copy + Clone + Default + Eq + Ord + PartialOrd + PartialEq fn as_mut_ptr(&self) -> *mut T { self.as_usize() as *mut T } + fn as_const_ptr(&self) -> *const T { + self.as_usize() as *const T + } fn as_u32(&self) -> u32; fn as_u64(&self) -> u64; fn as_usize(&self) -> usize;