add fork with copy-on-write for riscv64

Unlike x86_64/aarch64, no kernel stack has to be cloned: the user-
mode dispatcher owns the complete trap context, so fork is
intercepted in user_loop (the syscall-table entry stays as an ENOSYS
fallback), the child gets a copy of the parent's UserContext with
a0 = 0, a fresh kernel stack, and re-enters user mode through its own
user_loop_resume via fork_child_entry.

The COW machinery mirrors the other targets: RSW bit 8 marks shared
frames, mark_user_pages_copy_on_write drops the write bit before
copy_current_root_page_table deep-copies the user slots (kernel slots
stay shared), and do_cow_fault clones a frame on the first write —
or re-marks it writable in place when the last reference faults.
The frame refcounting protocol is completed on riscv64: every user
mapping (code, TLS, stack, lazily faulted pages) takes a reference
and clear_user_slots only releases frames whose count drops to zero,
which also covers exec after fork. Kernel-mode stores through
sstatus.SUM to COW pages are resolved in trap_handler.

Verified in QEMU with the fork example: fork, exec with argv/envp in
the child, waitpid, and 16k fork/exec/join cycles complete cleanly.

Author: stlankes

src/arch/riscv64/kernel/interrupts.rs

@@ -248,6 +248,28 @@ pub extern "C" fn trap_handler(tf: &mut TrapFrame) {
 		Trap::Interrupt(Interrupt::SupervisorTimer) => {
 			crate::arch::riscv64::kernel::scheduler::timer_handler();
 		}
+		// The kernel writes to user pages through `sstatus.SUM` (program
+		// loading, argv/envp setup, syscall buffers). Such a store can
+		// fault on a COW-marked or not-yet-faulted user page and must be
+		// resolved exactly like the corresponding user-mode fault.
+		#[cfg(feature = "common-os")]
+		Trap::Exception(Exception::LoadPageFault | Exception::StorePageFault) => {
+			let fault_addr = stval;
+
+			#[cfg(feature = "fork")]
+			if matches!(cause, Trap::Exception(Exception::StorePageFault))
+				&& crate::arch::riscv64::mm::paging::do_cow_fault(memory_addresses::VirtAddr::new(
+					fault_addr as u64,
+				)) {
+				return;
+			}
+
+			if !crate::arch::riscv64::kernel::do_user_page_fault(fault_addr) {
+				error!("Unhandled kernel page fault at {fault_addr:#x} ({cause:?})");
+				error!("sepc = {sepc:x}");
+				scheduler::abort();
+			}
+		}
 		cause => {
 			error!("Interrupt: {cause:?}");
 			error!("tf = {tf:x?} ");

src/arch/riscv64/kernel/mod.rs

@@ -230,6 +230,10 @@ where
 	let layout = PageLayout::from_size_align(code_size, BasePageSize::SIZE as usize).unwrap();
 	let frame_range = FrameAlloc::allocate(layout).unwrap();
 	let physaddr = PhysAddr::from(frame_range.start());
+	#[cfg(feature = "fork")]
+	for i in 0..code_size / BasePageSize::SIZE as usize {
+		crate::mm::frame_ref_inc(physaddr + i * BasePageSize::SIZE as usize);
+	}
 
 	let mut flags = PageTableEntryFlags::empty();
 	flags.normal().writable().user();
@@ -272,6 +276,10 @@ where
 		let layout = PageLayout::from_size(tls_memsz).unwrap();
 		let frame_range = FrameAlloc::allocate(layout).unwrap();
 		let physaddr = PhysAddr::from(frame_range.start());
+		#[cfg(feature = "fork")]
+		for i in 0..tls_memsz / BasePageSize::SIZE as usize {
+			crate::mm::frame_ref_inc(physaddr + i * BasePageSize::SIZE as usize);
+		}
 
 		let mut flags = PageTableEntryFlags::empty();
 		flags.normal().writable().user().execute_disable();
@@ -364,7 +372,7 @@ fn run_user(ctx: &mut trapframe::UserContext) {
 /// Returns `true` if the fault was resolved and the faulting instruction
 /// can be retried.
 #[cfg(feature = "common-os")]
-fn do_user_page_fault(fault_addr: usize) -> bool {
+pub(crate) fn do_user_page_fault(fault_addr: usize) -> bool {
 	use core::ops::Bound;
 
 	use align_address::Align;
@@ -401,6 +409,8 @@ fn do_user_page_fault(fault_addr: usize) -> bool {
 		}
 
 		paging::map::<BasePageSize>(addr, physaddr, 1, flags);
+		#[cfg(feature = "fork")]
+		crate::mm::frame_ref_inc(physaddr);
 
 		// Clear the page through the identity mapping of physical memory,
 		// so this works independently of `sstatus.SUM`.
@@ -489,9 +499,11 @@ pub(crate) fn user_loop(
 		| SSTATUS_FS_INITIAL
 		| SSTATUS_SUM;
 
-	let mut ctx = trapframe::UserContext::default();
-	ctx.sepc = entry;
-	ctx.sstatus = user_sstatus;
+	let mut ctx = trapframe::UserContext {
+		sepc: entry,
+		sstatus: user_sstatus,
+		..Default::default()
+	};
 	ctx.general.sp = stack_pointer;
 	ctx.general.tp = thread_pointer as usize;
 	ctx.general.a0 = args[0];
@@ -500,6 +512,15 @@ pub(crate) fn user_loop(
 
 	debug!("Jump to user space at {entry:#x}, stack pointer {stack_pointer:#x}");
 
+	user_loop_resume(ctx)
+}
+
+/// Service all traps the user code raises, starting (or resuming, for a
+/// fork child) user execution from `ctx`. Never returns; the process
+/// leaves through the `exit` system call (or is torn down after a fatal
+/// fault).
+#[cfg(feature = "common-os")]
+pub(crate) fn user_loop_resume(mut ctx: trapframe::UserContext) -> ! {
 	use riscv::interrupt::{Exception, Interrupt, Trap};
 	use riscv::register::{scause, stval};
 
@@ -513,13 +534,35 @@ pub(crate) fn user_loop(
 		let cause = Trap::<Interrupt, Exception>::try_from(scause.cause()).unwrap();
 
 		match cause {
-			Trap::Exception(Exception::UserEnvCall) => dispatch_syscall(&mut ctx),
+			Trap::Exception(Exception::UserEnvCall) => {
+				// `fork` needs the full user context to seed the child's
+				// user loop, so it is dispatched here instead of through
+				// the syscall table.
+				#[cfg(feature = "fork")]
+				if ctx.general.a7 == crate::syscalls::table::SYSNO_FORK {
+					ctx.sepc += 4;
+					let pid: i32 = unsafe { crate::scheduler::fork_from_user_context(&ctx) }.into();
+					ctx.general.a0 = pid as usize;
+					continue;
+				}
+
+				dispatch_syscall(&mut ctx);
+			}
 			Trap::Exception(
 				Exception::InstructionPageFault
 				| Exception::LoadPageFault
 				| Exception::StorePageFault,
 			) => {
 				let fault_addr = stval::read();
+
+				#[cfg(feature = "fork")]
+				if matches!(cause, Trap::Exception(Exception::StorePageFault))
+					&& crate::arch::riscv64::mm::paging::do_cow_fault(
+						memory_addresses::VirtAddr::new(fault_addr as u64),
+					) {
+					continue;
+				}
+
 				if !do_user_page_fault(fault_addr) {
 					error!("Unhandled user page fault at {fault_addr:#x} ({cause:?})");
 					error!("sepc = {:#x}", ctx.sepc);
@@ -565,6 +608,10 @@ pub unsafe fn jump_to_user_land(entry_point: usize, args: Vec<CString>, envs: Ve
 	let layout = PageLayout::from_size(USER_STACK_SIZE).unwrap();
 	let frame_range = FrameAlloc::allocate(layout).unwrap();
 	let phys_addr = PhysAddr::from(frame_range.start());
+	#[cfg(feature = "fork")]
+	for i in 0..USER_STACK_SIZE / BasePageSize::SIZE as usize {
+		crate::mm::frame_ref_inc(phys_addr + i * BasePageSize::SIZE as usize);
+	}
 	let mut flags = PageTableEntryFlags::empty();
 	flags.normal().writable().user().execute_disable();
 	paging::map::<BasePageSize>(

src/arch/riscv64/kernel/scheduler.rs

@@ -293,6 +293,50 @@ extern "C" fn user_task_entry(entry: usize, arg: usize, user_stack: usize, tp: u
 	crate::arch::riscv64::kernel::user_loop(entry, user_stack, tp as u64, [arg, 0, 0])
 }
 
+/// First code of a fork child, running in kernel mode on the child's
+/// fresh kernel stack. Re-enters user mode with the parent's saved
+/// register state (`a0` already patched to 0 by `fork_from_user_context`).
+#[cfg(all(feature = "common-os", feature = "fork"))]
+extern "C" fn fork_child_entry(ctx_box: usize) -> ! {
+	let ctx = unsafe {
+		alloc::boxed::Box::from_raw(core::ptr::with_exposed_provenance_mut::<
+			trapframe::UserContext,
+		>(ctx_box))
+	};
+	crate::arch::riscv64::kernel::user_loop_resume(*ctx)
+}
+
+/// Craft the initial kernel-stack frame for a fork child. When the
+/// scheduler first picks the child, `switch_to_task` pops the `State`
+/// and returns into `fork_child_entry` with the boxed `UserContext` as
+/// argument. Returns the child's initial `last_stack_pointer`.
+#[cfg(all(feature = "common-os", feature = "fork"))]
+pub(crate) fn create_fork_stack_frame(
+	stacks: &TaskStacks,
+	ctx: alloc::boxed::Box<trapframe::UserContext>,
+) -> VirtAddr {
+	unsafe {
+		// Set a marker for debugging at the very top.
+		let mut stack = stacks.get_kernel_stack() + stacks.get_kernel_stack_size() - 0x10u64;
+		*stack.as_mut_ptr::<u64>() = 0xdead_beefu64;
+
+		// Put the State structure expected by the ASM switch() function on the stack.
+		stack -= size_of::<State>();
+
+		let state = stack.as_mut_ptr::<State>();
+		state.cast::<u8>().write_bytes(0, size_of::<State>());
+
+		(*state).ra = core::mem::transmute::<
+			extern "C" fn(usize) -> !,
+			unsafe extern "C" fn(extern "C" fn(usize), usize, u64),
+		>(fork_child_entry);
+		(*state).sp = stack.as_usize();
+		(*state).a0 = alloc::boxed::Box::into_raw(ctx).expose_provenance();
+
+		stack
+	}
+}
+
 #[cfg(feature = "common-os")]
 impl Task {
 	/// Craft the initial kernel-stack frame for a new user-space thread.

src/arch/riscv64/mm/mod.rs

@@ -1,5 +1,7 @@
 pub mod paging;
 
+#[cfg(all(feature = "common-os", feature = "fork"))]
+pub use paging::{copy_current_root_page_table, prepare_mem_copy_on_write};
 #[cfg(feature = "common-os")]
 pub use paging::{create_new_root_page_table, drop_user_space};