Makefile

@@ -40,9 +40,9 @@ kernel:
 	env RUSTFLAGS="$(RUSTFLAGS)" cargo build --bin kernel $(CARGO_BUILD_ARGS)
 
 clippy:
-	env RUSTFLAGS="-Dwarnings" cargo clippy -p allocator
-	env RUSTFLAGS="-Dwarnings" cargo clippy -p utils
-	env RUSTFLAGS="-Dwarnings" cargo clippy --no-deps $(CARGO_TARGET_ARGS) --manifest-path=kernel/Cargo.toml
+	cargo clippy -p allocator
+	cargo clippy -p utils
+	cargo clippy --no-deps $(CARGO_TARGET_ARGS) --manifest-path=kernel/Cargo.toml
 
 fmt:
 	cargo fmt --all
@@ -70,10 +70,4 @@ qemu: kernel
 qemu-gdb: kernel
 	$(QEMU) $(QEMU_ARGS) -s -S
 
-kernel-expand:
-	@env RUSTFLAGS="$(RUSTFLAGS)" cargo expand $(CARGO_TARGET_ARGS) --manifest-path kernel/Cargo.toml $(filter-out $@,$(MAKECMDGOALS))
-
-%:
-	@:
-
-.PHONY: kernel clippy fmt test kernel-test-dump kernel-asm build-target clean qemu qemu-gdb kernel-expand
+.PHONY: kernel clippy fmt test kernel-test-dump kernel-asm build-target clean qemu qemu-gdb

kernel/src/arch/riscv/board/virt/console.rs

@@ -6,28 +6,14 @@ impl ConsoleDevice for EarlyConsole {
     fn read(&mut self) -> u8 {
         let uart = super::UART0_BASE as *mut u8;
         let line_sts = super::UART0_LSR as *mut u8;
-
         while unsafe { line_sts.read_volatile() } & 0x01 == 0 {}
         unsafe { uart.read_volatile() }
     }
 
-    fn try_read(&mut self) -> Option<u8> {
-        let uart = super::UART0_BASE as *mut u8;
-        let line_sts = super::UART0_LSR as *mut u8;
-
-        (unsafe { line_sts.read_volatile() } & 0x01 != 0).then(|| unsafe { uart.read_volatile() })
-    }
-
     fn write(&mut self, ch: u8) {
         let uart = super::UART0_BASE as *mut u8;
         unsafe { uart.write_volatile(ch) }
     }
-
-    fn write_str(&mut self, s: &str) {
-        for ch in s.bytes() {
-            self.write(ch);
-        }
-    }
 }
 
 pub fn init_early_console() {

kernel/src/arch/riscv/vspace/utils.rs

@@ -15,7 +15,7 @@ fn alloc_callback() {
     ALLOC_COUNT.fetch_add(1, core::sync::atomic::Ordering::SeqCst);
 }
 
-#[cfg_attr(debug_assertions, tracer::trace_callback(callback = alloc_callback))]
+#[cfg_attr(debug_assertions, tracer::trace_callback(log = true, callback = alloc_callback))]
 pub fn alloc_page(_mapped_addr: VirtAddr) -> PhysAddr {
     let addr = RAM_ALLOCATOR.lock().alloc(PAGE_LAYOUT).expect("Failed to allocate page");
 

kernel/src/drivers/irq/plic.rs

@@ -2,9 +2,9 @@ use super::IrqDriver;
 use crate::drivers::Driver;
 use crate::entry::HART_ID;
 use crate::plat::irq::IrqController;
-use core::sync::atomic::{AtomicPtr, Ordering};
 use fdt::{node::FdtNode, Fdt};
 use log::trace;
+use spin::Mutex;
 
 const IRQ_OCCUPIED: usize = u32::MAX as usize; // OpenSBI will rewrite IRQ_M_EXT to this value
 const IRQ_S_EXT: usize = 9;
@@ -28,35 +28,18 @@ struct PlicCtrl {
 pub struct IrqPlic
 where IrqPlic: IrqDriver
 {
-    priority: AtomicPtr<u32>,
-    pending:  AtomicPtr<u32>,
-    enable:   AtomicPtr<u32>,
-    control:  AtomicPtr<PlicCtrl>,
+    priority: *mut u32,
+    pending:  *mut u32,
+    enable:   *mut u32,
+    control:  *mut PlicCtrl,
 
     nr_irqs: usize,
-}
-
-impl IrqPlic {
-    fn priority(&self) -> *mut u32 {
-        self.priority.load(Ordering::Relaxed)
-    }
-
-    fn pending(&self) -> *mut u32 {
-        self.pending.load(Ordering::Relaxed)
-    }
-
-    fn enable(&self) -> *mut u32 {
-        self.enable.load(Ordering::Relaxed)
-    }
-
-    fn control(&self) -> *mut PlicCtrl {
-        self.control.load(Ordering::Relaxed)
-    }
+    lock:    Mutex<()>,
 }
 
 impl Driver for IrqPlic {
     fn compatible() -> &'static [&'static str] {
-        &["sifive,plic-1.0.0", "riscv,plic0"]
+        &["riscv,plic0"]
     }
 
     fn setup(node: FdtNode, fdt: &Fdt) -> Self {
@@ -107,23 +90,19 @@ impl Driver for IrqPlic {
 
         let ctx_id = ctx_id.expect("No PLIC context found for us");
 
-        let priority = unsafe { base_address.add(PLIC_PRIORITY_OFFSET) as *mut u32 };
-        let pending = unsafe { base_address.add(PLIC_PENDING_OFFSET) as *mut u32 };
-        let enable = unsafe { base_address.add(PLIC_ENABLE_OFFSET + ctx_id * PLIC_ENABLE_STRIDE) as *mut u32 };
-        let control = unsafe { (base_address.add(PLIC_CONTROL_OFFSET) as *mut PlicCtrl).add(ctx_id) };
-
         Self {
-            priority: AtomicPtr::new(priority),
-            pending:  AtomicPtr::new(pending),
-            enable:   AtomicPtr::new(enable),
-            control:  AtomicPtr::new(control),
+            priority: unsafe { base_address.add(PLIC_PRIORITY_OFFSET) as *mut u32 },
+            pending:  unsafe { base_address.add(PLIC_PENDING_OFFSET) as *mut u32 },
+            enable:   unsafe { base_address.add(PLIC_ENABLE_OFFSET + ctx_id * PLIC_ENABLE_STRIDE) as *mut u32 },
+            control:  unsafe { (base_address.add(PLIC_CONTROL_OFFSET) as *mut PlicCtrl).add(ctx_id) },
             nr_irqs:  nr_irqs.unwrap(),
+            lock:     Mutex::new(()),
         }
     }
 }
 
 impl IrqController for IrqPlic {
-    fn enable(&mut self, irq: usize) {
+    fn enable(&self, irq: usize) {
         if irq == 0 || irq >= self.nr_irqs {
             return;
         }
@@ -131,12 +110,18 @@ impl IrqController for IrqPlic {
         let word = irq / 32;
         let bit = irq % 32;
 
-        let val = unsafe { IrqPlic::enable(self).add(word).read_volatile() };
-        let val = val | (1 << bit);
-        unsafe { IrqPlic::enable(self).add(word).write_volatile(val) };
+        {
+            let guard = self.lock.lock();
+
+            let val = unsafe { self.enable.add(word).read_volatile() };
+            let val = val | (1 << bit);
+            unsafe { self.enable.add(word).write_volatile(val) };
+
+            drop(guard);
+        }
     }
 
-    fn disable(&mut self, irq: usize) {
+    fn disable(&self, irq: usize) {
         if irq == 0 || irq >= self.nr_irqs {
             return;
         }
@@ -144,12 +129,18 @@ impl IrqController for IrqPlic {
         let word = irq / 32;
         let bit = irq % 32;
 
-        let val = unsafe { IrqPlic::enable(self).add(word).read_volatile() };
-        let val = val & !(1 << bit);
-        unsafe { IrqPlic::enable(self).add(word).write_volatile(val) };
+        {
+            let guard = self.lock.lock();
+
+            let val = unsafe { self.enable.add(word).read_volatile() };
+            let val = val & !(1 << bit);
+            unsafe { self.enable.add(word).write_volatile(val) };
+
+            drop(guard);
+        }
     }
 
-    fn is_pending(&mut self, irq: usize) -> bool {
+    fn is_pending(&self, irq: usize) -> bool {
         if irq == 0 || irq >= self.nr_irqs {
             return false;
         }
@@ -157,12 +148,12 @@ impl IrqController for IrqPlic {
         let word = irq / 32;
         let bit = irq % 32;
 
-        let val = unsafe { IrqPlic::pending(self).add(word).read_volatile() };
+        let val = unsafe { self.pending.add(word).read_volatile() };
         val & (1 << bit) != 0
     }
 
-    fn claim(&mut self) -> Option<usize> {
-        let val = unsafe { IrqPlic::control(self).as_ref().unwrap().claim.read_volatile() };
+    fn claim(&self) -> Option<usize> {
+        let val = unsafe { self.control.as_ref().unwrap().claim.read_volatile() };
 
         if val > 0 {
             Some(val as usize)
@@ -171,13 +162,13 @@ impl IrqController for IrqPlic {
         }
     }
 
-    fn complete(&mut self, irq: usize) {
+    fn complete(&self, irq: usize) {
         if irq == 0 || irq >= self.nr_irqs {
             return;
         }
 
         let val = irq as u32;
-        unsafe { IrqPlic::control(self).as_ref().unwrap().claim.write_volatile(val) };
+        unsafe { self.control.as_ref().unwrap().claim.write_volatile(val) };
     }
 }
 

kernel/src/drivers/serial/sifive.rs

@@ -171,12 +171,6 @@ impl ConsoleDevice for UartSifive {
 
         self.tx_write(n);
     }
-
-    fn write_str(&mut self, s: &str) {
-        for ch in s.bytes() {
-            self.write(ch);
-        }
-    }
 }
 
 impl SerialDriver for UartSifive {}

kernel/src/drivers/serial/uart16550.rs

@@ -2,7 +2,6 @@ use crate::arch::layout::mmap_phys_to_virt;
 use crate::drivers::serial::SerialDriver;
 use crate::drivers::Driver;
 use crate::plat::console::ConsoleDevice;
-use core::fmt::Write;
 use fdt::node::FdtNode;
 use fdt::Fdt;
 use uart_16550::MmioSerialPort;
@@ -33,17 +32,9 @@ impl ConsoleDevice for Uart16550 {
         self.port.receive()
     }
 
-    fn try_read(&mut self) -> Option<u8> {
-        Some(self.read())
-    }
-
     fn write(&mut self, byte: u8) {
         self.port.send(byte);
     }
-
-    fn write_str(&mut self, s: &str) {
-        self.port.write_str(s).unwrap()
-    }
 }
 
 impl SerialDriver for Uart16550 {}

kernel/src/entry.rs

@@ -1,11 +1,8 @@
 use crate::logging::set_log_level;
-use crate::plat::{
-    console::{set_console, ConsoleDriver},
-    irq::{set_irq_driver, IrqDriver},
-    lowlevel::{Hardware, LowLevel},
-    timer::{Timer, TimerOps},
-    trap::{Trap, TrapOps},
-};
+use crate::plat::console::{set_console, ConsoleDriver};
+use crate::plat::lowlevel::{Hardware, LowLevel};
+use crate::plat::timer::{Timer, TimerOps};
+use crate::plat::trap::{Trap, TrapOps};
 use crate::root::setup_root_server;
 use crate::scheduler::{IDLE_THREAD, SCHEDULER};
 use core::cell::Cell;
@@ -26,7 +23,6 @@ pub fn rust_main() -> ! {
 
     set_log_level(&fdt);
     setup_console(&fdt);
-    setup_intc(&fdt);
 
     info!("Kernel Started");
 
@@ -62,10 +58,3 @@ fn setup_console(fdt: &Fdt) {
         Some(driver) => set_console(driver),
     }
 }
-
-fn setup_intc(fdt: &Fdt) {
-    match fdt.all_nodes().find_map(|node| IrqDriver::new(node, fdt)) {
-        None => error!("No compatible interrupt controller found! System may not work properly"),
-        Some(driver) => set_irq_driver(driver),
-    }
-}

kernel/src/main.rs

@@ -2,10 +2,11 @@
 #![no_std]
 #![no_main]
 // Features
+#![feature(asm_const)]
+#![feature(cell_update)]
 #![feature(concat_idents)]
 #![feature(iter_array_chunks)]
 #![feature(let_chains)]
-#![feature(macro_metavar_expr)]
 #![feature(naked_functions)]
 #![feature(thread_local)]
 // Test Infrastructure

kernel/src/objects/mod.rs

@@ -14,7 +14,7 @@
 
 */
 
-use core::marker::PhantomData;
+use core::{marker::PhantomData, ptr::NonNull};
 use uapi::{
     cap::ObjectType,
     error::{SysError, SysResult},
@@ -61,7 +61,22 @@ impl<'a, T: KernelObject + ?Sized> TryFrom<&'a CapEntry> for Cap<'a, T> {
 
 impl<'a, T: KernelObject + ?Sized> Cap<'a, T> {
     pub fn append(&mut self, new: &mut CapEntry) {
-        self.cte.link.append(new)
+        let next = self.cte.link.next_raw();
+
+        // update new cap's link
+        new.link.set_prev(Some(NonNull::from(self.cte)));
+        new.link.set_next(next);
+
+        // record new cap's addr
+        let new_addr = Some(NonNull::from(new));
+
+        // update next cap's link.prev
+        if let Some(mut next) = next {
+            unsafe { next.as_mut().link.set_prev(new_addr) };
+        }
+
+        // update self's link.next
+        self.cte.link.next.set(new_addr);
     }
 }
 

kernel/src/plat/console.rs

@@ -1,19 +1,68 @@
-use super::utils::generate_driver;
 use crate::arch::EarlyConsole;
 use crate::drivers::serial::{Uart16550, UartSifive};
+use crate::drivers::Driver;
+use fdt::{node::FdtNode, Fdt};
+use log::debug;
 use spin::Mutex;
 
-generate_driver!(
-    ConsoleDriver {
-        (Uart16550, UartSifive),
-        (SilenceConsole, EarlyConsole)
-    } : ConsoleDevice {
-        fn read(&mut self) -> u8;
-        fn try_read(&mut self) -> Option<u8>;
-        fn write(&mut self, ch: u8) -> ();
-        fn write_str(&mut self, s: &str) -> ();
-    }
-);
+macro_rules! create_console_driver {
+    (($($dynamic_driver:ident),+), ($($static_driver:ident),+)) => {
+        // A VERY DIRTY hack to work around dynamic dispatch
+        pub enum ConsoleDriver {
+            $($dynamic_driver($dynamic_driver),)+
+            $($static_driver($static_driver),)+
+        }
+
+        impl ConsoleDriver {
+            pub fn new(node: FdtNode, fdt: &Fdt) -> Option<Self> {
+                if let Some(compatible) = node.compatible() {
+                    $(
+                        if compatible
+                            .all()
+                            .any(|s| $dynamic_driver::compatible().contains(&s))
+                        {
+                            debug!("Console: Using driver: {}", stringify!($dynamic_driver));
+                            return Some(ConsoleDriver::$dynamic_driver($dynamic_driver::setup(node, fdt)));
+                        }
+                    )+
+                }
+                None
+            }
+        }
+
+        impl ConsoleDevice for ConsoleDriver {
+            fn read(&mut self) -> u8 {
+                match self {
+                    $(ConsoleDriver::$dynamic_driver(driver) => driver.read(),)+
+                    $(ConsoleDriver::$static_driver(driver) => driver.read(),)+
+                }
+            }
+
+            fn try_read(&mut self) -> Option<u8> {
+                match self {
+                    $(ConsoleDriver::$dynamic_driver(driver) => driver.try_read(),)+
+                    $(ConsoleDriver::$static_driver(driver) => driver.try_read(),)+
+                }
+            }
+
+            fn write(&mut self, n: u8) {
+                match self {
+                    $(ConsoleDriver::$dynamic_driver(driver) => driver.write(n),)+
+                    $(ConsoleDriver::$static_driver(driver) => driver.write(n),)+
+                }
+            }
+
+            fn write_str(&mut self, s: &str) {
+                match self {
+                    $(ConsoleDriver::$dynamic_driver(driver) => driver.write_str(s),)+
+                    $(ConsoleDriver::$static_driver(driver) => driver.write_str(s),)+
+                }
+            }
+        }
+    };
+}
+
+create_console_driver!((Uart16550, UartSifive), (SilenceConsole, EarlyConsole));
 
 impl core::fmt::Write for ConsoleDriver {
     fn write_str(&mut self, s: &str) -> core::fmt::Result {
@@ -29,13 +78,25 @@ impl ConsoleDevice for SilenceConsole {
         0
     }
 
+    fn write(&mut self, _: u8) {}
+}
+
+pub trait ConsoleDevice {
+    /// Blocking read
+    fn read(&mut self) -> u8;
+    /// Non-blocking read
     fn try_read(&mut self) -> Option<u8> {
-        None
+        Some(self.read())
     }
 
-    fn write(&mut self, _: u8) {}
-
-    fn write_str(&mut self, _: &str) {}
+    /// Write a byte
+    fn write(&mut self, _: u8);
+    /// Write a str
+    fn write_str(&mut self, s: &str) {
+        for byte in s.bytes() {
+            self.write(byte);
+        }
+    }
 }
 
 pub static CONSOLE: Mutex<ConsoleDriver> = Mutex::new(ConsoleDriver::SilenceConsole(SilenceConsole));

kernel/src/plat/irq.rs

@@ -1,7 +1,4 @@
-use super::utils::generate_driver;
-use crate::drivers::irq::IrqPlic;
 use crate::objects::*;
-use spin::{lazy::Lazy, Mutex};
 
 const IRQ_NUM: usize = 32;
 
@@ -13,37 +10,12 @@ pub enum IrqState {
     Reserved,
 }
 
-generate_driver!(
-    IrqDriver {
-        (IrqPlic),
-        (DummyIrqDriver)
-    } : IrqController {
-        fn enable(&mut self, irq: usize) -> ();
-        fn disable(&mut self, irq: usize) -> ();
-        fn is_pending(&mut self, irq: usize) -> bool;
-        fn claim(&mut self) -> Option<usize>;
-        fn complete(&mut self, irq: usize) -> ();
-    }
-);
-
-pub struct DummyIrqDriver;
-
-impl IrqController for DummyIrqDriver {
-    fn enable(&mut self, _: usize) {}
-    fn disable(&mut self, _: usize) {}
-    fn is_pending(&mut self, _: usize) -> bool {
-        false
-    }
-    fn claim(&mut self) -> Option<usize> {
-        None
-    }
-    fn complete(&mut self, _: usize) {}
-}
-
-static IRQ_DRIVER: Mutex<IrqDriver> = Mutex::new(IrqDriver::DummyIrqDriver(DummyIrqDriver));
-
-pub fn set_irq_driver(driver: IrqDriver) {
-    *IRQ_DRIVER.lock() = driver;
+pub trait IrqController {
+    fn enable(&self, irq: usize);
+    fn disable(&self, irq: usize);
+    fn is_pending(&self, irq: usize) -> bool;
+    fn claim(&self) -> Option<usize>;
+    fn complete(&self, irq: usize);
 }
 
 pub struct IrqManager {
@@ -51,8 +23,6 @@ pub struct IrqManager {
     state:   [IrqState; IRQ_NUM],
 }
 
-pub static IRQ_MANAGER: Mutex<Lazy<IrqManager>> = Mutex::new(Lazy::new(IrqManager::new));
-
 impl IrqManager {
     pub fn new() -> Self {
         Self {
@@ -60,4 +30,5 @@ impl IrqManager {
             state:   [IrqState::Inactive; IRQ_NUM],
         }
     }
-}
+
+    }

kernel/src/plat/mod.rs

@@ -4,5 +4,3 @@ pub mod irq;
 pub mod lowlevel;
 pub mod timer;
 pub mod trap;
-
-mod utils;

kernel/src/plat/utils.rs

@@ -1,53 +0,0 @@
-macro_rules! generate_driver {
-    (
-        $name:ident {
-            ($($dynamic_driver:ident),*),($($static_driver:ident),*)
-        } : $trait:ident {
-            $(fn $fn_name:ident (&mut self $(, $arg_name:ident : $arg_ty:ty)*) -> $ret_ty:ty;)+
-        }
-    ) => {
-        pub enum $name {
-            $($dynamic_driver($dynamic_driver),)*
-            $($static_driver($static_driver),)*
-        }
-
-        pub trait $trait {
-            $(fn $fn_name(&mut self $(, $arg_name: $arg_ty)*) -> $ret_ty;)+
-        }
-
-        impl $name {
-            pub fn new(node: fdt::node::FdtNode, fdt: &fdt::Fdt) -> Option<Self> {
-                use $crate::drivers::Driver;
-                if let Some(compatible) = node.compatible() {
-                    $(
-                        if compatible
-                            .all()
-                            .any(|s| $dynamic_driver::compatible().contains(&s))
-                        {
-                            log::debug!("{}: using driver: {}", stringify!($name), stringify!($dynamic_driver));
-                            return Some($name::$dynamic_driver($dynamic_driver::setup(node, fdt)));
-                        }
-                    )*
-                }
-                None
-            }
-        }
-
-        macro_rules! expand_driver {
-            ($$self:ident, $$fn_name:ident, $$($$arg:ident),*) => {
-                match $$self {
-                    $(Self::$dynamic_driver(driver) => driver.$$fn_name($$($$arg),*),)*
-                    $(Self::$static_driver(driver) => driver.$$fn_name($$($$arg),*),)*
-                }
-            };
-        }
-
-        impl $trait for $name {
-            $(fn $fn_name(&mut self $(, $arg_name: $arg_ty)*) -> $ret_ty {
-                expand_driver!(self, $fn_name, $($arg_name),*)
-            })+
-        }
-    };
-}
-
-pub(super) use generate_driver;

kernel/src/scheduler.rs

@@ -1,8 +1,8 @@
 use crate::objects::*;
-use core::sync::atomic::AtomicPtr;
+use core::ptr::NonNull;
 use log::{error, trace};
 use spin::lazy::Lazy;
-use utils::{container_of_mut, linked_list::Link};
+use utils::{container_of, linked_list::Link};
 
 #[thread_local]
 pub static IDLE_THREAD: Lazy<TcbObject> = Lazy::new(|| {
@@ -26,11 +26,9 @@ impl Scheduler {
     }
 
     pub fn init(&self) {
-        unsafe {
-            let head = Some(AtomicPtr::new(container_of_mut!(&self.head, TcbObject, link)));
-            self.head.set_next(&head);
-            self.head.set_prev(&head);
-        }
+        let head = unsafe { Some(NonNull::from(&*container_of!(&self.head, TcbObject, link))) };
+        self.head.set_next(head);
+        self.head.set_prev(head);
     }
 
     pub fn add(&self, tcb: &TcbObject) {

lib/utils/src/container_of.rs

@@ -7,13 +7,6 @@ macro_rules! container_of {
     };
 }
 
-#[macro_export]
-macro_rules! container_of_mut {
-    ($ptr:expr, $type:ty, $field:ident) => {
-        ($ptr as *const _ as *mut u8).sub(core::mem::offset_of!($type, $field) as usize) as *mut $type
-    };
-}
-
 #[macro_export]
 macro_rules! container_of_offset {
     ($ptr:expr, $type:ty, $offset:expr) => {

lib/utils/src/linked_list.rs

@@ -1,15 +1,5 @@
 use crate::{container_of_offset, to_field_offset};
-use core::sync::atomic::{AtomicPtr, Ordering};
-
-#[macro_export]
-macro_rules! as_mut_ptr {
-    ($ref:expr => $type:ty) => {
-        $ref as *const _ as *mut $type
-    };
-    ($ref:expr) => {
-        $ref as *const _ as *mut _
-    };
-}
+use core::{cell::Cell, ptr::NonNull};
 
 #[macro_export]
 macro_rules! LinkHelperImpl {
@@ -31,8 +21,8 @@ pub trait LinkHelper: Sized {
 }
 
 pub struct Link<T: LinkHelper> {
-    pub prev: Option<AtomicPtr<T>>,
-    pub next: Option<AtomicPtr<T>>,
+    pub prev: Cell<Option<NonNull<T>>>,
+    pub next: Cell<Option<NonNull<T>>>,
 }
 
 impl<T: LinkHelper> Default for Link<T> {
@@ -44,15 +34,18 @@ impl<T: LinkHelper> Default for Link<T> {
 impl<T: LinkHelper> Clone for Link<T> {
     fn clone(&self) -> Self {
         Self {
-            prev: copy_ptr(&self.prev),
-            next: copy_ptr(&self.next),
+            prev: Cell::new(self.prev.get()),
+            next: Cell::new(self.next.get()),
         }
     }
 }
 
 impl<T: LinkHelper> Link<T> {
     pub const fn new() -> Self {
-        Self { prev: None, next: None }
+        Self {
+            prev: Cell::new(None),
+            next: Cell::new(None),
+        }
     }
 
     /// # Safety
@@ -61,75 +54,67 @@ impl<T: LinkHelper> Link<T> {
         &*(container_of_offset!(self, T, T::LINK_OFFSET))
     }
 
-    pub fn prev_raw(&self) -> &Option<AtomicPtr<T>> {
-        &self.prev
+    pub fn prev_raw(&self) -> Option<NonNull<T>> {
+        self.prev.get()
     }
 
-    pub fn next_raw(&self) -> &Option<AtomicPtr<T>> {
-        &self.next
+    pub fn next_raw(&self) -> Option<NonNull<T>> {
+        self.next.get()
     }
 
     pub fn prev(&self) -> Option<&T> {
-        self.prev_raw().as_ref().map(|p| unsafe { &*p.load(Ordering::Acquire) })
+        self.prev_raw().map(|p| unsafe { &*p.as_ptr() })
     }
 
     pub fn prev_mut(&self) -> Option<&mut T> {
-        self.prev_raw().as_ref().map(|p| unsafe { &mut *p.load(Ordering::Acquire) })
+        self.prev_raw().map(|p| unsafe { &mut *p.as_ptr() })
     }
 
     pub fn next(&self) -> Option<&T> {
-        self.next_raw().as_ref().map(|n| unsafe { &*n.load(Ordering::Acquire) })
+        self.next_raw().map(|n| unsafe { &*n.as_ptr() })
     }
 
     pub fn next_mut(&self) -> Option<&mut T> {
-        self.next_raw().as_ref().map(|n| unsafe { &mut *n.load(Ordering::Acquire) })
+        self.next_raw().map(|n| unsafe { &mut *n.as_ptr() })
     }
 
-    /// # Safety
-    /// Take care of race condition
-    pub unsafe fn set_prev(&self, prev: &Option<AtomicPtr<T>>) {
-        #[allow(invalid_reference_casting)]
-        let link = &mut *(self as *const _ as *mut Self);
-        link.prev = copy_ptr(prev);
+    pub fn set_prev(&self, prev: Option<NonNull<T>>) {
+        self.prev.set(prev);
     }
 
-    /// # Safety
-    /// Take care of race condition
-    pub unsafe fn set_next(&self, next: &Option<AtomicPtr<T>>) {
-        #[allow(invalid_reference_casting)]
-        let link = &mut *(self as *const _ as *mut Self);
-        link.next = copy_ptr(next);
+    pub fn set_next(&self, next: Option<NonNull<T>>) {
+        self.next.set(next);
     }
 
     pub fn prepend(&self, new: &T) {
         unsafe {
             // setup new's link
             new.get_link().set_prev(self.prev_raw());
-            new.get_link().set_next(&Some(AtomicPtr::new(as_mut_ptr!(self.object()))));
+            new.get_link().set_next(Some(NonNull::from(self.object())));
 
             // setup prev's link
             if let Some(prev) = self.prev_raw() {
-                load_ptr(prev).get_link().set_next(&Some(AtomicPtr::new(as_mut_ptr!(new))));
+                prev.as_ref().get_link().set_next(Some(NonNull::from(new)))
             }
 
             // setup self's link
-            self.set_prev(&Some(AtomicPtr::new(as_mut_ptr!(new))));
+            self.set_prev(Some(NonNull::from(new)));
         }
     }
 
     pub fn append(&self, new: &T) {
         unsafe {
             // setup new's link
-            new.get_link().set_prev(&Some(AtomicPtr::new(as_mut_ptr!(self.object()))));
+            new.get_link().set_prev(Some(NonNull::from(self.object())));
             new.get_link().set_next(self.next_raw());
 
             // setup next's link
             if let Some(next) = self.next_raw() {
-                load_ptr(next).get_link().set_prev(&Some(AtomicPtr::new(as_mut_ptr!(new))))
+                next.as_ref().get_link().set_prev(Some(NonNull::from(new)))
             }
 
             // setup self's link
-            self.set_next(&Some(AtomicPtr::new(as_mut_ptr!(new))));
+            self.set_next(Some(NonNull::from(new)));
         }
     }
 
@@ -137,31 +122,21 @@ impl<T: LinkHelper> Link<T> {
         unsafe {
             // setup prev's link
             if let Some(prev) = self.prev_raw() {
-                load_ptr(prev).get_link().set_next(self.next_raw())
+                prev.as_ref().get_link().set_next(self.next_raw())
             }
 
             // setup next's link
             if let Some(next) = self.next_raw() {
-                load_ptr(next).get_link().set_prev(self.prev_raw())
+                next.as_ref().get_link().set_prev(self.prev_raw())
             }
 
             // setup self's link
-            self.set_prev(&None);
-            self.set_next(&None);
+            self.set_prev(None);
+            self.set_next(None);
         }
     }
 }
 
-#[inline]
-fn copy_ptr<T>(ptr: &Option<AtomicPtr<T>>) -> Option<AtomicPtr<T>> {
-    ptr.as_ref().map(|p| AtomicPtr::new(p.load(Ordering::Acquire)))
-}
-
-#[inline]
-unsafe fn load_ptr<'a, T>(ptr: &AtomicPtr<T>) -> &'a mut T {
-    &mut *ptr.load(Ordering::Acquire)
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -173,6 +148,15 @@ mod tests {
 
     LinkHelperImpl!(Node: link);
 
+    macro_rules! as_mut_ptr {
+        ($ref:expr => $type:ty) => {
+            $ref as *const _ as *mut $type
+        };
+        ($ref:expr) => {
+            $ref as *const _ as *mut _
+        };
+    }
+
     #[test]
     fn test_linked_list() {
         let node1 = Node {
@@ -211,15 +195,9 @@ mod tests {
         {
             // check next link
             assert!(node1.link.next_raw().is_some());
-            assert_eq!(
-                node1.link.next_raw().as_ref().unwrap().load(Ordering::Acquire),
-                as_mut_ptr!(&node2)
-            );
+            assert_eq!(node1.link.next_raw().unwrap().as_ptr(), as_mut_ptr!(&node2));
             assert!(node2.link.next_raw().is_some());
-            assert_eq!(
-                node2.link.next_raw().as_ref().unwrap().load(Ordering::Acquire),
-                as_mut_ptr!(&node3)
-            );
+            assert_eq!(node2.link.next_raw().unwrap().as_ptr(), as_mut_ptr!(&node3));
             assert!(node3.link.next_raw().is_none());
         }
 
@@ -227,15 +205,9 @@ mod tests {
             // check prev link
             assert!(node1.link.prev_raw().is_none());
             assert!(node2.link.prev_raw().is_some());
-            assert_eq!(
-                node2.link.prev_raw().as_ref().unwrap().load(Ordering::Acquire),
-                as_mut_ptr!(&node1)
-            );
+            assert_eq!(node2.link.prev_raw().unwrap().as_ptr(), as_mut_ptr!(&node1));
             assert!(node3.link.prev_raw().is_some());
-            assert_eq!(
-                node3.link.prev_raw().as_ref().unwrap().load(Ordering::Acquire),
-                as_mut_ptr!(&node2)
-            );
+            assert_eq!(node3.link.prev_raw().unwrap().as_ptr(), as_mut_ptr!(&node2));
         }
 
         {
@@ -246,16 +218,10 @@ mod tests {
             assert!(node2.link.prev_raw().is_none());
 
             assert!(node1.link.next_raw().is_some());
-            assert_eq!(
-                node1.link.next_raw().as_ref().unwrap().load(Ordering::Acquire),
-                as_mut_ptr!(&node3)
-            );
+            assert_eq!(node1.link.next_raw().unwrap().as_ptr(), as_mut_ptr!(&node3));
 
             assert!(node3.link.prev_raw().is_some());
-            assert_eq!(
-                node3.link.prev_raw().as_ref().unwrap().load(Ordering::Acquire),
-                as_mut_ptr!(&node1)
-            );
+            assert_eq!(node3.link.prev_raw().unwrap().as_ptr(), as_mut_ptr!(&node1));
         }
     }
 }

uapi/src/lib.rs

@@ -1,6 +1,5 @@
 #![no_std]
 #![feature(custom_inner_attributes)]
-#![allow(dead_code)]
 
 #[macro_use]
 extern crate num_derive;