feat: kernel/entry: wire up intc setup

Makefile

@@ -40,9 +40,9 @@ kernel:
 	env RUSTFLAGS="$(RUSTFLAGS)" cargo build --bin kernel $(CARGO_BUILD_ARGS)
 
 clippy:
-	cargo clippy -p allocator
+	env RUSTFLAGS="-Dwarnings" cargo clippy -p allocator
-	cargo clippy -p utils
+	env RUSTFLAGS="-Dwarnings" cargo clippy -p utils
-	cargo clippy --no-deps $(CARGO_TARGET_ARGS) --manifest-path=kernel/Cargo.toml
+	env RUSTFLAGS="-Dwarnings" cargo clippy --no-deps $(CARGO_TARGET_ARGS) --manifest-path=kernel/Cargo.toml
 
 fmt:
 	cargo fmt --all
@@ -70,4 +70,10 @@ qemu: kernel
 qemu-gdb: kernel
 	$(QEMU) $(QEMU_ARGS) -s -S
 
-.PHONY: kernel clippy fmt test kernel-test-dump kernel-asm build-target clean qemu qemu-gdb
+kernel-expand:
+	@env RUSTFLAGS="$(RUSTFLAGS)" cargo expand $(CARGO_TARGET_ARGS) --manifest-path kernel/Cargo.toml $(filter-out $@,$(MAKECMDGOALS))
+
+%:
+	@:
+
+.PHONY: kernel clippy fmt test kernel-test-dump kernel-asm build-target clean qemu qemu-gdb kernel-expand

kernel/src/arch/riscv/board/virt/console.rs

@@ -6,14 +6,28 @@ impl ConsoleDevice for EarlyConsole {
     fn read(&mut self) -> u8 {
         let uart = super::UART0_BASE as *mut u8;
         let line_sts = super::UART0_LSR as *mut u8;
+
         while unsafe { line_sts.read_volatile() } & 0x01 == 0 {}
         unsafe { uart.read_volatile() }
     }
 
+    fn try_read(&mut self) -> Option<u8> {
+        let uart = super::UART0_BASE as *mut u8;
+        let line_sts = super::UART0_LSR as *mut u8;
+
+        (unsafe { line_sts.read_volatile() } & 0x01 != 0).then(|| unsafe { uart.read_volatile() })
+    }
+
     fn write(&mut self, ch: u8) {
         let uart = super::UART0_BASE as *mut u8;
         unsafe { uart.write_volatile(ch) }
     }
+
+    fn write_str(&mut self, s: &str) {
+        for ch in s.bytes() {
+            self.write(ch);
+        }
+    }
 }
 
 pub fn init_early_console() {

kernel/src/arch/riscv/vspace/utils.rs

@@ -15,7 +15,7 @@ fn alloc_callback() {
     ALLOC_COUNT.fetch_add(1, core::sync::atomic::Ordering::SeqCst);
 }
 
-#[cfg_attr(debug_assertions, tracer::trace_callback(log = true, callback = alloc_callback))]
+#[cfg_attr(debug_assertions, tracer::trace_callback(callback = alloc_callback))]
 pub fn alloc_page(_mapped_addr: VirtAddr) -> PhysAddr {
     let addr = RAM_ALLOCATOR.lock().alloc(PAGE_LAYOUT).expect("Failed to allocate page");
 

kernel/src/drivers/irq/plic.rs

@@ -2,9 +2,9 @@ use super::IrqDriver;
 use crate::drivers::Driver;
 use crate::entry::HART_ID;
 use crate::plat::irq::IrqController;
+use core::sync::atomic::{AtomicPtr, Ordering};
 use fdt::{node::FdtNode, Fdt};
 use log::trace;
-use spin::Mutex;
 
 const IRQ_OCCUPIED: usize = u32::MAX as usize; // OpenSBI will rewrite IRQ_M_EXT to this value
 const IRQ_S_EXT: usize = 9;
@@ -28,18 +28,35 @@ struct PlicCtrl {
 pub struct IrqPlic
 where IrqPlic: IrqDriver
 {
-    priority: *mut u32,
+    priority: AtomicPtr<u32>,
-    pending:  *mut u32,
+    pending:  AtomicPtr<u32>,
-    enable:   *mut u32,
+    enable:   AtomicPtr<u32>,
-    control:  *mut PlicCtrl,
+    control:  AtomicPtr<PlicCtrl>,
 
     nr_irqs: usize,
-    lock:    Mutex<()>,
+}
+
+impl IrqPlic {
+    fn priority(&self) -> *mut u32 {
+        self.priority.load(Ordering::Relaxed)
+    }
+
+    fn pending(&self) -> *mut u32 {
+        self.pending.load(Ordering::Relaxed)
+    }
+
+    fn enable(&self) -> *mut u32 {
+        self.enable.load(Ordering::Relaxed)
+    }
+
+    fn control(&self) -> *mut PlicCtrl {
+        self.control.load(Ordering::Relaxed)
+    }
 }
 
 impl Driver for IrqPlic {
     fn compatible() -> &'static [&'static str] {
-        &["riscv,plic0"]
+        &["sifive,plic-1.0.0", "riscv,plic0"]
     }
 
     fn setup(node: FdtNode, fdt: &Fdt) -> Self {
@@ -90,19 +107,23 @@ impl Driver for IrqPlic {
 
         let ctx_id = ctx_id.expect("No PLIC context found for us");
 
+        let priority = unsafe { base_address.add(PLIC_PRIORITY_OFFSET) as *mut u32 };
+        let pending = unsafe { base_address.add(PLIC_PENDING_OFFSET) as *mut u32 };
+        let enable = unsafe { base_address.add(PLIC_ENABLE_OFFSET + ctx_id * PLIC_ENABLE_STRIDE) as *mut u32 };
+        let control = unsafe { (base_address.add(PLIC_CONTROL_OFFSET) as *mut PlicCtrl).add(ctx_id) };
+
         Self {
-            priority: unsafe { base_address.add(PLIC_PRIORITY_OFFSET) as *mut u32 },
+            priority: AtomicPtr::new(priority),
-            pending:  unsafe { base_address.add(PLIC_PENDING_OFFSET) as *mut u32 },
+            pending:  AtomicPtr::new(pending),
-            enable:   unsafe { base_address.add(PLIC_ENABLE_OFFSET + ctx_id * PLIC_ENABLE_STRIDE) as *mut u32 },
+            enable:   AtomicPtr::new(enable),
-            control:  unsafe { (base_address.add(PLIC_CONTROL_OFFSET) as *mut PlicCtrl).add(ctx_id) },
+            control:  AtomicPtr::new(control),
             nr_irqs:  nr_irqs.unwrap(),
-            lock:     Mutex::new(()),
         }
     }
 }
 
 impl IrqController for IrqPlic {
-    fn enable(&self, irq: usize) {
+    fn enable(&mut self, irq: usize) {
         if irq == 0 || irq >= self.nr_irqs {
             return;
         }
@@ -110,18 +131,12 @@ impl IrqController for IrqPlic {
         let word = irq / 32;
         let bit = irq % 32;
 
-        {
+        let val = unsafe { IrqPlic::enable(self).add(word).read_volatile() };
-            let guard = self.lock.lock();
-
-            let val = unsafe { self.enable.add(word).read_volatile() };
         let val = val | (1 << bit);
-            unsafe { self.enable.add(word).write_volatile(val) };
+        unsafe { IrqPlic::enable(self).add(word).write_volatile(val) };
-
-            drop(guard);
-        }
     }
 
-    fn disable(&self, irq: usize) {
+    fn disable(&mut self, irq: usize) {
         if irq == 0 || irq >= self.nr_irqs {
             return;
         }
@@ -129,18 +144,12 @@ impl IrqController for IrqPlic {
         let word = irq / 32;
         let bit = irq % 32;
 
-        {
+        let val = unsafe { IrqPlic::enable(self).add(word).read_volatile() };
-            let guard = self.lock.lock();
-
-            let val = unsafe { self.enable.add(word).read_volatile() };
         let val = val & !(1 << bit);
-            unsafe { self.enable.add(word).write_volatile(val) };
+        unsafe { IrqPlic::enable(self).add(word).write_volatile(val) };
-
-            drop(guard);
-        }
     }
 
-    fn is_pending(&self, irq: usize) -> bool {
+    fn is_pending(&mut self, irq: usize) -> bool {
         if irq == 0 || irq >= self.nr_irqs {
             return false;
         }
@@ -148,12 +157,12 @@ impl IrqController for IrqPlic {
         let word = irq / 32;
         let bit = irq % 32;
 
-        let val = unsafe { self.pending.add(word).read_volatile() };
+        let val = unsafe { IrqPlic::pending(self).add(word).read_volatile() };
         val & (1 << bit) != 0
     }
 
-    fn claim(&self) -> Option<usize> {
+    fn claim(&mut self) -> Option<usize> {
-        let val = unsafe { self.control.as_ref().unwrap().claim.read_volatile() };
+        let val = unsafe { IrqPlic::control(self).as_ref().unwrap().claim.read_volatile() };
 
         if val > 0 {
             Some(val as usize)
@@ -162,13 +171,13 @@ impl IrqController for IrqPlic {
         }
     }
 
-    fn complete(&self, irq: usize) {
+    fn complete(&mut self, irq: usize) {
         if irq == 0 || irq >= self.nr_irqs {
             return;
         }
 
         let val = irq as u32;
-        unsafe { self.control.as_ref().unwrap().claim.write_volatile(val) };
+        unsafe { IrqPlic::control(self).as_ref().unwrap().claim.write_volatile(val) };
     }
 }
 

kernel/src/drivers/serial/sifive.rs

@@ -171,6 +171,12 @@ impl ConsoleDevice for UartSifive {
 
         self.tx_write(n);
     }
+
+    fn write_str(&mut self, s: &str) {
+        for ch in s.bytes() {
+            self.write(ch);
+        }
+    }
 }
 
 impl SerialDriver for UartSifive {}

kernel/src/drivers/serial/uart16550.rs

@@ -2,6 +2,7 @@ use crate::arch::layout::mmap_phys_to_virt;
 use crate::drivers::serial::SerialDriver;
 use crate::drivers::Driver;
 use crate::plat::console::ConsoleDevice;
+use core::fmt::Write;
 use fdt::node::FdtNode;
 use fdt::Fdt;
 use uart_16550::MmioSerialPort;
@@ -32,9 +33,17 @@ impl ConsoleDevice for Uart16550 {
         self.port.receive()
     }
 
+    fn try_read(&mut self) -> Option<u8> {
+        Some(self.read())
+    }
+
     fn write(&mut self, byte: u8) {
         self.port.send(byte);
     }
+
+    fn write_str(&mut self, s: &str) {
+        self.port.write_str(s).unwrap()
+    }
 }
 
 impl SerialDriver for Uart16550 {}

kernel/src/entry.rs

@@ -1,8 +1,11 @@
 use crate::logging::set_log_level;
-use crate::plat::console::{set_console, ConsoleDriver};
+use crate::plat::{
-use crate::plat::lowlevel::{Hardware, LowLevel};
+    console::{set_console, ConsoleDriver},
-use crate::plat::timer::{Timer, TimerOps};
+    irq::{set_irq_driver, IrqDriver},
-use crate::plat::trap::{Trap, TrapOps};
+    lowlevel::{Hardware, LowLevel},
+    timer::{Timer, TimerOps},
+    trap::{Trap, TrapOps},
+};
 use crate::root::setup_root_server;
 use crate::scheduler::{IDLE_THREAD, SCHEDULER};
 use core::cell::Cell;
@@ -23,6 +26,7 @@ pub fn rust_main() -> ! {
 
     set_log_level(&fdt);
     setup_console(&fdt);
+    setup_intc(&fdt);
 
     info!("Kernel Started");
 
@@ -58,3 +62,10 @@ fn setup_console(fdt: &Fdt) {
         Some(driver) => set_console(driver),
     }
 }
+
+fn setup_intc(fdt: &Fdt) {
+    match fdt.all_nodes().find_map(|node| IrqDriver::new(node, fdt)) {
+        None => error!("No compatible interrupt controller found! System may not work properly"),
+        Some(driver) => set_irq_driver(driver),
+    }
+}

kernel/src/main.rs

@@ -2,11 +2,10 @@
 #![no_std]
 #![no_main]
 // Features
-#![feature(asm_const)]
-#![feature(cell_update)]
 #![feature(concat_idents)]
 #![feature(iter_array_chunks)]
 #![feature(let_chains)]
+#![feature(macro_metavar_expr)]
 #![feature(naked_functions)]
 #![feature(thread_local)]
 // Test Infrastructure

kernel/src/objects/mod.rs

@@ -14,7 +14,7 @@
 
 */
 
-use core::{marker::PhantomData, ptr::NonNull};
+use core::marker::PhantomData;
 use uapi::{
     cap::ObjectType,
     error::{SysError, SysResult},
@@ -61,22 +61,7 @@ impl<'a, T: KernelObject + ?Sized> TryFrom<&'a CapEntry> for Cap<'a, T> {
 
 impl<'a, T: KernelObject + ?Sized> Cap<'a, T> {
     pub fn append(&mut self, new: &mut CapEntry) {
-        let next = self.cte.link.next_raw();
+        self.cte.link.append(new)
-
-        // update new cap's link
-        new.link.set_prev(Some(NonNull::from(self.cte)));
-        new.link.set_next(next);
-
-        // record new cap's addr
-        let new_addr = Some(NonNull::from(new));
-
-        // update next cap's link.prev
-        if let Some(mut next) = next {
-            unsafe { next.as_mut().link.set_prev(new_addr) };
-        }
-
-        // update self's link.next
-        self.cte.link.next.set(new_addr);
     }
 }
 

kernel/src/plat/console.rs

@@ -1,68 +1,19 @@
+use super::utils::generate_driver;
 use crate::arch::EarlyConsole;
 use crate::drivers::serial::{Uart16550, UartSifive};
-use crate::drivers::Driver;
-use fdt::{node::FdtNode, Fdt};
-use log::debug;
 use spin::Mutex;
 
-macro_rules! create_console_driver {
+generate_driver!(
-    (($($dynamic_driver:ident),+), ($($static_driver:ident),+)) => {
+    ConsoleDriver {
-        // A VERY DIRTY hack to work around dynamic dispatch
+        (Uart16550, UartSifive),
-        pub enum ConsoleDriver {
+        (SilenceConsole, EarlyConsole)
-            $($dynamic_driver($dynamic_driver),)+
+    } : ConsoleDevice {
-            $($static_driver($static_driver),)+
+        fn read(&mut self) -> u8;
+        fn try_read(&mut self) -> Option<u8>;
+        fn write(&mut self, ch: u8) -> ();
+        fn write_str(&mut self, s: &str) -> ();
     }
-
+);
-        impl ConsoleDriver {
-            pub fn new(node: FdtNode, fdt: &Fdt) -> Option<Self> {
-                if let Some(compatible) = node.compatible() {
-                    $(
-                        if compatible
-                            .all()
-                            .any(|s| $dynamic_driver::compatible().contains(&s))
-                        {
-                            debug!("Console: Using driver: {}", stringify!($dynamic_driver));
-                            return Some(ConsoleDriver::$dynamic_driver($dynamic_driver::setup(node, fdt)));
-                        }
-                    )+
-                }
-                None
-            }
-        }
-
-        impl ConsoleDevice for ConsoleDriver {
-            fn read(&mut self) -> u8 {
-                match self {
-                    $(ConsoleDriver::$dynamic_driver(driver) => driver.read(),)+
-                    $(ConsoleDriver::$static_driver(driver) => driver.read(),)+
-                }
-            }
-
-            fn try_read(&mut self) -> Option<u8> {
-                match self {
-                    $(ConsoleDriver::$dynamic_driver(driver) => driver.try_read(),)+
-                    $(ConsoleDriver::$static_driver(driver) => driver.try_read(),)+
-                }
-            }
-
-            fn write(&mut self, n: u8) {
-                match self {
-                    $(ConsoleDriver::$dynamic_driver(driver) => driver.write(n),)+
-                    $(ConsoleDriver::$static_driver(driver) => driver.write(n),)+
-                }
-            }
-
-            fn write_str(&mut self, s: &str) {
-                match self {
-                    $(ConsoleDriver::$dynamic_driver(driver) => driver.write_str(s),)+
-                    $(ConsoleDriver::$static_driver(driver) => driver.write_str(s),)+
-                }
-            }
-        }
-    };
-}
-
-create_console_driver!((Uart16550, UartSifive), (SilenceConsole, EarlyConsole));
 
 impl core::fmt::Write for ConsoleDriver {
     fn write_str(&mut self, s: &str) -> core::fmt::Result {
@@ -78,25 +29,13 @@ impl ConsoleDevice for SilenceConsole {
         0
     }
 
-    fn write(&mut self, _: u8) {}
-}
-
-pub trait ConsoleDevice {
-    /// Blocking read
-    fn read(&mut self) -> u8;
-    /// Non-blocking read
     fn try_read(&mut self) -> Option<u8> {
-        Some(self.read())
+        None
     }
 
-    /// Write a byte
+    fn write(&mut self, _: u8) {}
-    fn write(&mut self, _: u8);
+
-    /// Write a str
+    fn write_str(&mut self, _: &str) {}
-    fn write_str(&mut self, s: &str) {
-        for byte in s.bytes() {
-            self.write(byte);
-        }
-    }
 }
 
 pub static CONSOLE: Mutex<ConsoleDriver> = Mutex::new(ConsoleDriver::SilenceConsole(SilenceConsole));

kernel/src/plat/irq.rs

@@ -1,4 +1,7 @@
+use super::utils::generate_driver;
+use crate::drivers::irq::IrqPlic;
 use crate::objects::*;
+use spin::{lazy::Lazy, Mutex};
 
 const IRQ_NUM: usize = 32;
 
@@ -10,12 +13,37 @@ pub enum IrqState {
     Reserved,
 }
 
-pub trait IrqController {
+generate_driver!(
-    fn enable(&self, irq: usize);
+    IrqDriver {
-    fn disable(&self, irq: usize);
+        (IrqPlic),
-    fn is_pending(&self, irq: usize) -> bool;
+        (DummyIrqDriver)
-    fn claim(&self) -> Option<usize>;
+    } : IrqController {
-    fn complete(&self, irq: usize);
+        fn enable(&mut self, irq: usize) -> ();
+        fn disable(&mut self, irq: usize) -> ();
+        fn is_pending(&mut self, irq: usize) -> bool;
+        fn claim(&mut self) -> Option<usize>;
+        fn complete(&mut self, irq: usize) -> ();
+    }
+);
+
+pub struct DummyIrqDriver;
+
+impl IrqController for DummyIrqDriver {
+    fn enable(&mut self, _: usize) {}
+    fn disable(&mut self, _: usize) {}
+    fn is_pending(&mut self, _: usize) -> bool {
+        false
+    }
+    fn claim(&mut self) -> Option<usize> {
+        None
+    }
+    fn complete(&mut self, _: usize) {}
+}
+
+static IRQ_DRIVER: Mutex<IrqDriver> = Mutex::new(IrqDriver::DummyIrqDriver(DummyIrqDriver));
+
+pub fn set_irq_driver(driver: IrqDriver) {
+    *IRQ_DRIVER.lock() = driver;
 }
 
 pub struct IrqManager {
@@ -23,6 +51,8 @@ pub struct IrqManager {
     state:   [IrqState; IRQ_NUM],
 }
 
+pub static IRQ_MANAGER: Mutex<Lazy<IrqManager>> = Mutex::new(Lazy::new(IrqManager::new));
+
 impl IrqManager {
     pub fn new() -> Self {
         Self {
@@ -30,5 +60,4 @@ impl IrqManager {
             state:   [IrqState::Inactive; IRQ_NUM],
         }
     }
-
 }

kernel/src/plat/mod.rs

@@ -4,3 +4,5 @@ pub mod irq;
 pub mod lowlevel;
 pub mod timer;
 pub mod trap;
+
+mod utils;

kernel/src/plat/utils.rs

@@ -0,0 +1,53 @@
+macro_rules! generate_driver {
+    (
+        $name:ident {
+            ($($dynamic_driver:ident),*),($($static_driver:ident),*)
+        } : $trait:ident {
+            $(fn $fn_name:ident (&mut self $(, $arg_name:ident : $arg_ty:ty)*) -> $ret_ty:ty;)+
+        }
+    ) => {
+        pub enum $name {
+            $($dynamic_driver($dynamic_driver),)*
+            $($static_driver($static_driver),)*
+        }
+
+        pub trait $trait {
+            $(fn $fn_name(&mut self $(, $arg_name: $arg_ty)*) -> $ret_ty;)+
+        }
+
+        impl $name {
+            pub fn new(node: fdt::node::FdtNode, fdt: &fdt::Fdt) -> Option<Self> {
+                use $crate::drivers::Driver;
+                if let Some(compatible) = node.compatible() {
+                    $(
+                        if compatible
+                            .all()
+                            .any(|s| $dynamic_driver::compatible().contains(&s))
+                        {
+                            log::debug!("{}: using driver: {}", stringify!($name), stringify!($dynamic_driver));
+                            return Some($name::$dynamic_driver($dynamic_driver::setup(node, fdt)));
+                        }
+                    )*
+                }
+                None
+            }
+        }
+
+        macro_rules! expand_driver {
+            ($$self:ident, $$fn_name:ident, $$($$arg:ident),*) => {
+                match $$self {
+                    $(Self::$dynamic_driver(driver) => driver.$$fn_name($$($$arg),*),)*
+                    $(Self::$static_driver(driver) => driver.$$fn_name($$($$arg),*),)*
+                }
+            };
+        }
+
+        impl $trait for $name {
+            $(fn $fn_name(&mut self $(, $arg_name: $arg_ty)*) -> $ret_ty {
+                expand_driver!(self, $fn_name, $($arg_name),*)
+            })+
+        }
+    };
+}
+
+pub(super) use generate_driver;

kernel/src/scheduler.rs

@@ -1,8 +1,8 @@
 use crate::objects::*;
-use core::ptr::NonNull;
+use core::sync::atomic::AtomicPtr;
 use log::{error, trace};
 use spin::lazy::Lazy;
-use utils::{container_of, linked_list::Link};
+use utils::{container_of_mut, linked_list::Link};
 
 #[thread_local]
 pub static IDLE_THREAD: Lazy<TcbObject> = Lazy::new(|| {
@@ -26,9 +26,11 @@ impl Scheduler {
     }
 
     pub fn init(&self) {
-        let head = unsafe { Some(NonNull::from(&*container_of!(&self.head, TcbObject, link))) };
+        unsafe {
-        self.head.set_next(head);
+            let head = Some(AtomicPtr::new(container_of_mut!(&self.head, TcbObject, link)));
-        self.head.set_prev(head);
+            self.head.set_next(&head);
+            self.head.set_prev(&head);
+        }
     }
 
     pub fn add(&self, tcb: &TcbObject) {

lib/utils/src/container_of.rs

@@ -7,6 +7,13 @@ macro_rules! container_of {
     };
 }
 
+#[macro_export]
+macro_rules! container_of_mut {
+    ($ptr:expr, $type:ty, $field:ident) => {
+        ($ptr as *const _ as *mut u8).sub(core::mem::offset_of!($type, $field) as usize) as *mut $type
+    };
+}
+
 #[macro_export]
 macro_rules! container_of_offset {
     ($ptr:expr, $type:ty, $offset:expr) => {

lib/utils/src/linked_list.rs

@@ -1,5 +1,15 @@
 use crate::{container_of_offset, to_field_offset};
-use core::{cell::Cell, ptr::NonNull};
+use core::sync::atomic::{AtomicPtr, Ordering};
+
+#[macro_export]
+macro_rules! as_mut_ptr {
+    ($ref:expr => $type:ty) => {
+        $ref as *const _ as *mut $type
+    };
+    ($ref:expr) => {
+        $ref as *const _ as *mut _
+    };
+}
 
 #[macro_export]
 macro_rules! LinkHelperImpl {
@@ -21,8 +31,8 @@ pub trait LinkHelper: Sized {
 }
 
 pub struct Link<T: LinkHelper> {
-    pub prev: Cell<Option<NonNull<T>>>,
+    pub prev: Option<AtomicPtr<T>>,
-    pub next: Cell<Option<NonNull<T>>>,
+    pub next: Option<AtomicPtr<T>>,
 }
 
 impl<T: LinkHelper> Default for Link<T> {
@@ -34,18 +44,15 @@ impl<T: LinkHelper> Default for Link<T> {
 impl<T: LinkHelper> Clone for Link<T> {
     fn clone(&self) -> Self {
         Self {
-            prev: Cell::new(self.prev.get()),
+            prev: copy_ptr(&self.prev),
-            next: Cell::new(self.next.get()),
+            next: copy_ptr(&self.next),
         }
     }
 }
 
 impl<T: LinkHelper> Link<T> {
     pub const fn new() -> Self {
-        Self {
+        Self { prev: None, next: None }
-            prev: Cell::new(None),
-            next: Cell::new(None),
-        }
     }
 
     /// # Safety
@@ -54,67 +61,75 @@ impl<T: LinkHelper> Link<T> {
         &*(container_of_offset!(self, T, T::LINK_OFFSET))
     }
 
-    pub fn prev_raw(&self) -> Option<NonNull<T>> {
+    pub fn prev_raw(&self) -> &Option<AtomicPtr<T>> {
-        self.prev.get()
+        &self.prev
     }
 
-    pub fn next_raw(&self) -> Option<NonNull<T>> {
+    pub fn next_raw(&self) -> &Option<AtomicPtr<T>> {
-        self.next.get()
+        &self.next
     }
 
     pub fn prev(&self) -> Option<&T> {
-        self.prev_raw().map(|p| unsafe { &*p.as_ptr() })
+        self.prev_raw().as_ref().map(|p| unsafe { &*p.load(Ordering::Acquire) })
     }
 
     pub fn prev_mut(&self) -> Option<&mut T> {
-        self.prev_raw().map(|p| unsafe { &mut *p.as_ptr() })
+        self.prev_raw().as_ref().map(|p| unsafe { &mut *p.load(Ordering::Acquire) })
     }
 
     pub fn next(&self) -> Option<&T> {
-        self.next_raw().map(|n| unsafe { &*n.as_ptr() })
+        self.next_raw().as_ref().map(|n| unsafe { &*n.load(Ordering::Acquire) })
     }
 
     pub fn next_mut(&self) -> Option<&mut T> {
-        self.next_raw().map(|n| unsafe { &mut *n.as_ptr() })
+        self.next_raw().as_ref().map(|n| unsafe { &mut *n.load(Ordering::Acquire) })
     }
 
-    pub fn set_prev(&self, prev: Option<NonNull<T>>) {
+    /// # Safety
-        self.prev.set(prev);
+    /// Take care of race condition
+    pub unsafe fn set_prev(&self, prev: &Option<AtomicPtr<T>>) {
+        #[allow(invalid_reference_casting)]
+        let link = &mut *(self as *const _ as *mut Self);
+        link.prev = copy_ptr(prev);
     }
 
-    pub fn set_next(&self, next: Option<NonNull<T>>) {
+    /// # Safety
-        self.next.set(next);
+    /// Take care of race condition
+    pub unsafe fn set_next(&self, next: &Option<AtomicPtr<T>>) {
+        #[allow(invalid_reference_casting)]
+        let link = &mut *(self as *const _ as *mut Self);
+        link.next = copy_ptr(next);
     }
 
     pub fn prepend(&self, new: &T) {
         unsafe {
             // setup new's link
             new.get_link().set_prev(self.prev_raw());
-            new.get_link().set_next(Some(NonNull::from(self.object())));
+            new.get_link().set_next(&Some(AtomicPtr::new(as_mut_ptr!(self.object()))));
 
             // setup prev's link
             if let Some(prev) = self.prev_raw() {
-                prev.as_ref().get_link().set_next(Some(NonNull::from(new)))
+                load_ptr(prev).get_link().set_next(&Some(AtomicPtr::new(as_mut_ptr!(new))));
             }
 
             // setup self's link
-            self.set_prev(Some(NonNull::from(new)));
+            self.set_prev(&Some(AtomicPtr::new(as_mut_ptr!(new))));
         }
     }
 
     pub fn append(&self, new: &T) {
         unsafe {
             // setup new's link
-            new.get_link().set_prev(Some(NonNull::from(self.object())));
+            new.get_link().set_prev(&Some(AtomicPtr::new(as_mut_ptr!(self.object()))));
             new.get_link().set_next(self.next_raw());
 
             // setup next's link
             if let Some(next) = self.next_raw() {
-                next.as_ref().get_link().set_prev(Some(NonNull::from(new)))
+                load_ptr(next).get_link().set_prev(&Some(AtomicPtr::new(as_mut_ptr!(new))))
             }
 
             // setup self's link
-            self.set_next(Some(NonNull::from(new)));
+            self.set_next(&Some(AtomicPtr::new(as_mut_ptr!(new))));
         }
     }
 
@@ -122,21 +137,31 @@ impl<T: LinkHelper> Link<T> {
         unsafe {
             // setup prev's link
             if let Some(prev) = self.prev_raw() {
-                prev.as_ref().get_link().set_next(self.next_raw())
+                load_ptr(prev).get_link().set_next(self.next_raw())
             }
 
             // setup next's link
             if let Some(next) = self.next_raw() {
-                next.as_ref().get_link().set_prev(self.prev_raw())
+                load_ptr(next).get_link().set_prev(self.prev_raw())
             }
 
             // setup self's link
-            self.set_prev(None);
+            self.set_prev(&None);
-            self.set_next(None);
+            self.set_next(&None);
         }
     }
 }
 
+#[inline]
+fn copy_ptr<T>(ptr: &Option<AtomicPtr<T>>) -> Option<AtomicPtr<T>> {
+    ptr.as_ref().map(|p| AtomicPtr::new(p.load(Ordering::Acquire)))
+}
+
+#[inline]
+unsafe fn load_ptr<'a, T>(ptr: &AtomicPtr<T>) -> &'a mut T {
+    &mut *ptr.load(Ordering::Acquire)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -148,15 +173,6 @@ mod tests {
 
     LinkHelperImpl!(Node: link);
 
-    macro_rules! as_mut_ptr {
-        ($ref:expr => $type:ty) => {
-            $ref as *const _ as *mut $type
-        };
-        ($ref:expr) => {
-            $ref as *const _ as *mut _
-        };
-    }
-
     #[test]
     fn test_linked_list() {
         let node1 = Node {
@@ -195,9 +211,15 @@ mod tests {
         {
             // check next link
             assert!(node1.link.next_raw().is_some());
-            assert_eq!(node1.link.next_raw().unwrap().as_ptr(), as_mut_ptr!(&node2));
+            assert_eq!(
+                node1.link.next_raw().as_ref().unwrap().load(Ordering::Acquire),
+                as_mut_ptr!(&node2)
+            );
             assert!(node2.link.next_raw().is_some());
-            assert_eq!(node2.link.next_raw().unwrap().as_ptr(), as_mut_ptr!(&node3));
+            assert_eq!(
+                node2.link.next_raw().as_ref().unwrap().load(Ordering::Acquire),
+                as_mut_ptr!(&node3)
+            );
             assert!(node3.link.next_raw().is_none());
         }
 
@@ -205,9 +227,15 @@ mod tests {
             // check prev link
             assert!(node1.link.prev_raw().is_none());
             assert!(node2.link.prev_raw().is_some());
-            assert_eq!(node2.link.prev_raw().unwrap().as_ptr(), as_mut_ptr!(&node1));
+            assert_eq!(
+                node2.link.prev_raw().as_ref().unwrap().load(Ordering::Acquire),
+                as_mut_ptr!(&node1)
+            );
             assert!(node3.link.prev_raw().is_some());
-            assert_eq!(node3.link.prev_raw().unwrap().as_ptr(), as_mut_ptr!(&node2));
+            assert_eq!(
+                node3.link.prev_raw().as_ref().unwrap().load(Ordering::Acquire),
+                as_mut_ptr!(&node2)
+            );
         }
 
         {
@@ -218,10 +246,16 @@ mod tests {
             assert!(node2.link.prev_raw().is_none());
 
             assert!(node1.link.next_raw().is_some());
-            assert_eq!(node1.link.next_raw().unwrap().as_ptr(), as_mut_ptr!(&node3));
+            assert_eq!(
+                node1.link.next_raw().as_ref().unwrap().load(Ordering::Acquire),
+                as_mut_ptr!(&node3)
+            );
 
             assert!(node3.link.prev_raw().is_some());
-            assert_eq!(node3.link.prev_raw().unwrap().as_ptr(), as_mut_ptr!(&node1));
+            assert_eq!(
+                node3.link.prev_raw().as_ref().unwrap().load(Ordering::Acquire),
+                as_mut_ptr!(&node1)
+            );
         }
     }
 }

uapi/src/lib.rs

@@ -1,5 +1,6 @@
 #![no_std]
 #![feature(custom_inner_attributes)]
+#![allow(dead_code)]
 
 #[macro_use]
 extern crate num_derive;