Add Litmus tests

.gitmodules

@@ -6,3 +6,6 @@
 	path = sw/deps/cva6-sdk
 	url = https://github.com/pulp-platform/cva6-sdk.git
 	ignore = dirty
+[submodule "sw/deps/litmus-tests"]
+	path = sw/deps/litmus-tests
+	url = https://github.com/pulp-platform/CHERI-Litmus.git

cheshire.mk

@@ -11,6 +11,7 @@ VLOGAN ?= vlogan
 
 # Caution: Questasim requires this to point to the *actual* compiler install path
 CXX_PATH := $(shell which $(CXX))
+QUESTA   ?= questa-2023.4
 
 VLOG_ARGS   ?= -suppress 2583 -suppress 13314 -timescale 1ns/1ps
 VLOGAN_ARGS ?= -kdb -nc -assert svaext +v2k -timescale=1ns/1ps
@@ -63,7 +64,7 @@ chs-clean-deps:
 ######################
 
 CHS_NONFREE_REMOTE ?= git@iis-git.ee.ethz.ch:pulp-restricted/cheshire-nonfree.git
-CHS_NONFREE_COMMIT ?= 99aa8d9
+CHS_NONFREE_COMMIT ?= a111e47
 
 CHS_PHONY += chs-nonfree-init
 chs-nonfree-init:
@@ -82,17 +83,26 @@ include $(CHS_ROOT)/sw/sw.mk
 # Generate HW #
 ###############
 
+# `CHS_NUM_IRQ_HARTS` and `CHS_NUM_PLIC_SRCS` are used to generate register files.
+# They must match the corresponding SystemVerilog parameters.
+CHS_NUM_IRQ_HARTS  ?= 1
+CHS_NUM_PLIC_SRCS  ?= 58
+CHS_NUM_PLIC_PRIOW ?= 7
+
 # SoC registers
 $(CHS_ROOT)/hw/regs/cheshire_reg_pkg.sv $(CHS_ROOT)/hw/regs/cheshire_reg_top.sv: $(CHS_ROOT)/hw/regs/cheshire_regs.hjson
 	$(REGTOOL) -r $< --outdir $(dir $@)
 
 # CLINT
-CLINTCORES ?= 1
+CLINTCORES ?= $(CHS_NUM_IRQ_HARTS)
 include $(CLINTROOT)/clint.mk
 $(CLINTROOT)/.generated:
 	flock -x $@ $(MAKE) clint && touch $@
 
 # OpenTitan peripherals
+$(CHS_ROOT)/hw/rv_plic.cfg.hjson: $(CHS_ROOT)/util/gen_pliccfg.py
+	$< $(CHS_NUM_IRQ_HARTS) $(CHS_NUM_PLIC_SRCS) $(CHS_NUM_PLIC_PRIOW) > $@
+
 include $(OTPROOT)/otp.mk
 $(OTPROOT)/.generated: $(CHS_ROOT)/hw/rv_plic.cfg.hjson
 	flock -x $@ sh -c "cp $< $(dir $@)/src/rv_plic/; $(MAKE) -j1 otp" && touch $@
@@ -175,6 +185,67 @@ CHS_SIM_ALL += $(CHS_ROOT)/target/sim/models/24FC1025.v
 CHS_SIM_ALL += $(CHS_ROOT)/target/sim/vsim/compile.cheshire_soc.tcl
 CHS_SIM_ALL += $(CHS_ROOT)/target/sim/vcs/compile.cheshire_soc.sh
 
+################
+# Litmus tests #
+################
+
+LITMUS_NCORES     ?= 2
+LITMUS_DIR        := $(CHS_SW_DIR)/deps/litmus-tests
+LITMUS_BIN_DIR    := $(LITMUS_DIR)/binaries
+LITMUS_WORK_DIR   := $(CHS_ROOT)/work-litmus
+LITMUS_SIMLOG_DIR := $(LITMUS_WORK_DIR)/simlogs
+LITMUS_TEST_LIST  := $(LITMUS_WORK_DIR)/litmus-tests.list
+LITMUS_RESULTS    := $(LITMUS_WORK_DIR)/compare.log
+
+$(LITMUS_DIR)/.git:
+	cd $(CHS_ROOT) && git submodule update --init --recursive $(LITMUS_DIR)
+
+.PHONY: chs-build-litmus-tests
+chs-build-litmus-tests: $(LITMUS_DIR)/.git
+	cd $(LITMUS_DIR)/frontend; ./make.sh
+	cd $(LITMUS_DIR)/binaries; ./make-riscv.sh ../tests/ cheshire $(LITMUS_NCORES)
+
+$(LITMUS_WORK_DIR):
+	mkdir -p $(LITMUS_WORK_DIR)
+
+$(LITMUS_SIMLOG_DIR):
+	mkdir -p $(LITMUS_SIMLOG_DIR)
+
+$(LITMUS_TEST_LIST): $(LITMUS_WORK_DIR) $(LITMUS_SIMLOG_DIR)
+	@echo Generating $@ ...
+	@LITMUS_ROOT=$(LITMUS_DIR) LITMUS_WORK=$(LITMUS_WORK_DIR) $(CHS_ROOT)/util/litmus create_list
+
+$(LITMUS_SIMLOG_DIR)/%.log: $(LITMUS_BIN_DIR)/%.elf $(CHS_SIM_ALL)
+	@echo "Running test $(notdir $<) (Log file: $@)"
+	@cd target/sim/vsim && $(QUESTA) vsim -c -do "set PRELMODE 1; set BOOTMODE 0; set BINARY $<; source start.cheshire_soc.tcl; run -all" > $@ 2>&1
+	@echo "Finished test $<"
+
+.PHONY: chs-run-litmus-tests
+chs-run-litmus-tests: $(LITMUS_TEST_LIST)
+	$(eval LITMUS_TESTS_ELF = $(shell xargs printf '\n%s' < $(LITMUS_TEST_LIST)))
+	@echo Running $(words $(LITMUS_TESTS_ELF)) tests
+	@$(MAKE) $(addprefix $(LITMUS_SIMLOG_DIR)/, $(LITMUS_TESTS_ELF:.elf=.log))
+	@echo "Finished running litmus tests"
+
+.PHONY: chs-check-litmus-tests
+chs-check-litmus-tests:
+	$(eval export LITMUS_ROOT=$(LITMUS_DIR))
+	$(eval export LITMUS_WORK=$(LITMUS_WORK_DIR))
+	@echo "Parsing UART output from simulation logs.."
+	@$(CHS_ROOT)/util/litmus parse_uart
+	@echo "Patching UART logs.."
+	@$(CHS_ROOT)/util/litmus patch_uart
+	@echo "Concatenating logs in a single file.."
+	@$(CHS_ROOT)/util/litmus combine_logs
+	@echo "Comparing logs with reference model.."
+	@$(CHS_ROOT)/util/litmus check > $(LITMUS_RESULTS)
+	@echo "Done! Check '$(LITMUS_RESULTS)' file"
+
+.PHONY: chs-clean-litmus-tests
+chs-clean-litmus-tests:
+	rm -rf $(LITMUS_WORK_DIR)
+	cd $(LITMUS_DIR)/binaries; rm *.elf *.dump
+
 ###########
 # DRAMSys #
 ###########

docs/um/arch.md

@@ -185,7 +185,7 @@ Cheshire provides a flexible RISC-V interrupt architecture that can route and mu
 
 First, all internal (`intr.intn`) and external (`intr_ext_i`) interrupt sources are collected (`intr`). From here, they either pass through an *interrupt router* if enabled (`IrqRouter`) or are simply fanned out to interrupt *targets*, which may support as many or fewer interrupt sources as provided by `intr`. If a target supports fewer sources, its interrupt sources are *truncated*.
 
-Cheshire provides both a core-local interruptor (CLINT), grouping all per-core interrupts in one module, and a shared platform-level interrupt controller (PLIC). The former is used only for inter-processor and timer interrupts, while the latter is a proper interrupt target. If enabled (`Clic`), each CVA6 core also has a core-local interrupt controller (CLIC), another interrupt target. In addition to the PLIC and CLICs, any number external interrupt targets may be defined (`NumExtOutIntrTgts`) with their own number of incoming sources (`NumExtIrqHarts`).
+Cheshire provides both a core-local interruptor (CLINT), grouping all per-core interrupts in one module, and a shared platform-level interrupt controller (PLIC). The former is used only for inter-processor and timer interrupts, while the latter is a proper interrupt target. If enabled (`Clic`), each CVA6 core also has a core-local interrupt controller (CLIC), another interrupt target. In addition to the PLIC and CLICs, any number of external interrupt targets may be defined (`NumExtOutIntrTgts`) with their own number of incoming sources (`NumExtOutIntrs`).
 
 Finally, the PLIC and grouped CLINT also support allocating external harts for which to manage interrupts (`NumExtIrqHarts`), i.e. harts without interrupt controllers of themselves.
 

hw/bootrom/cheshire_bootrom.S

@@ -5,10 +5,13 @@
 // Nicole Narr <narrn@student.ethz.ch>
 // Christopher Reinwardt <creinwar@student.ethz.ch>
 // Paul Scheffler <paulsc@iis.ee.ethz.ch>
+// Enrico Zelioli <ezelioli@iis.ee.ethz.ch>
 
-// TODO: Avoid hardcoding in addresses and offsets
+#include <regs/cheshire.h>
+#include <regs/axi_llc.h>
 
-#include "smp.h"
+// The hart that the boot ROM should run on
+#define NONSMP_HART 0
 
 .section .text._start
 
@@ -47,7 +50,11 @@ _start:
     li x31, 0
 
     // Pause SMP harts
-    smp_pause(t0, t1)
+    li   t1, 0x8
+    csrw mie, t1
+    li   t0, NONSMP_HART
+    csrr t1, mhartid
+    bne  t0, t1, _wait_for_ipi
 
     // Init stack and global pointer with safe, linked values
     la sp, __stack_pointer$
@@ -57,57 +64,34 @@ _start:
     .option pop
 
     // If LLC present: Wait for end of BIST, then extend stack and set to all SPM
-    la t0, __base_regs
-    lw t0, 80(t0)   // regs.HW_FEATURES
-    andi t0, t0, 2  // regs.HW_FEATURES.llc
+    la   t0, __base_regs
+    lw   t0, CHESHIRE_HW_FEATURES_REG_OFFSET(t0)
+    andi t0, t0, (1 << CHESHIRE_HW_FEATURES_LLC_BIT)
     beqz t0, _prom_check_run
-    la t0, __base_llc
+    la   t0, __base_llc
 _wait_llc_bist:
-    lw t1, 72(t0)   // llc.BIST_STATUS_DONE_BIT
+    lw   t1, AXI_LLC_BIST_STATUS_REG_OFFSET(t0) // Check BIST status done bit
     beqz t1, _wait_llc_bist
-    li t1, -1
-    sw t1, 0(t0)    // llc.CFG_SPM_LOW
-    sw t1, 4(t0)    // llc.CFG_SPM_HIGH
-    li t1, 1
-    sw t1, 16(t0)   // llc.CFG_COMMIT
+    li   t1, -1
+    sw   t1, AXI_LLC_CFG_SPM_LOW_REG_OFFSET(t0)
+    sw   t1, AXI_LLC_CFG_SPM_HIGH_REG_OFFSET(t0)
+    li   t1, 1
+    sw   t1, AXI_LLC_COMMIT_CFG_REG_OFFSET(t0)
     // Correct stack to start at end of SPM
-    la t0, __base_regs
-    la sp, __base_spm
-    lw t0, 84(t0)   // regs.LLC_SIZE
-    add sp, sp, t0
+    la   t0, __base_regs
+    la   sp, __base_spm
+    lw   t0, CHESHIRE_LLC_SIZE_REG_OFFSET(t0)
+    add  sp, sp, t0
     addi sp, sp, -8
 
 // Enter Platform ROM if present.
 _prom_check_run:
     // Note that we have internal access to SPM here *if and only if* there is an LLC.
     la t0, __base_regs
-    lw t0, 72(t0)   // regs.PLATFORM_ROM
+    lw t0, CHESHIRE_PLATFORM_ROM_REG_OFFSET(t0)
     beqz t0, _boot
     jalr t0
 
-.global boot_next_stage
-boot_next_stage:
-    // Non-SMP hart: Write boot address into global scratch registers
-    la t0, __base_regs
-    sw a0, 16(t0)   // regs.SCRATCH[4]
-    srli a0, a0, 32
-    sw a0, 20(t0)   // regs.SCRATCH[5]
-    fence
-    // Resume SMP harts
-    smp_resume(t0, t1, t2)
-    // Load boot address from global scratch registers
-    la t0, __base_regs
-    lwu t1, 20(t0)  // regs.SCRATCH[5]
-    slli t1, t1, 32
-    lwu t0, 16(t0)  // regs.SCRATCH[4]
-    or t0, t0, t1
-    // Store hartid to a0
-    csrr a0, mhartid
-    // Jump to boot address
-    jalr ra, 0(t0)
-    // We should never get here
-    ret
-
 // Reset regs, full fence, then jump to main
 _boot:
     li t0, 0
@@ -120,10 +104,71 @@ _boot:
 .global _exit
 _exit:
     // Save the return value to scratch register 2, try `ebreak`, then wait forever
+    // Set bit 0 to signal that the execution is done.
     slli a0, a0, 1
     ori  a0, a0, 1
     la t0, __base_regs
-    sw a0, 8(t0)     // regs.SCRATCH[2]
+    sw a0, CHESHIRE_SCRATCH_2_REG_OFFSET(t0)
     ebreak
 1:  wfi
     j 1b
+
+// Move to next stage of boot
+// 1. Write the address of next stage boot loader in Cheshire's scratch registers
+// 2. Resume execution of all other harts
+.global boot_next_stage
+boot_next_stage:
+    // Non-SMP hart: write boot address into global scratch registers
+    la   t0, __base_regs
+    sw   a0, CHESHIRE_SCRATCH_4_REG_OFFSET(t0)
+    srli a0, a0, 32
+    sw   a0, CHESHIRE_SCRATCH_5_REG_OFFSET(t0)
+    fence
+    // Resume SMP harts: set CLINT IPI registers
+    // NOTE: this will cause CLINT to send IPIs to all cores, therefore also the
+    // non-smp hart will receive one. The following instructions make sure that
+    // all harts will wait until the IPI is received (WFI with global ie disabled),
+    // then clear the IPI in the CLINT and wait until all other harts are done with it.
+    la   t0, __base_clint
+    la   t2, __base_regs
+    lw   t2, CHESHIRE_NUM_INT_HARTS_REG_OFFSET(t2)
+    slli t2, t2, 2
+    add  t2, t0, t2  // t2 = CLINT_BASE + (n_harts * 4)
+1:  li   t1, 1
+    sw   t1, 0(t0)
+    addi t0, t0, 4
+    blt  t0, t2, 1b
+
+// Stall hart until IPI is raised
+_wait_for_ipi:
+    // Wait until this hart receives IPI
+    wfi
+    csrr t1, mip
+    andi t1, t1, 0x8
+    beqz t1, _wait_for_ipi
+    // Clear CLINT IPI register for this hart
+    la   t0, __base_clint
+    csrr t1, mhartid
+    slli t1, t1, 2
+    add  t1, t1, t0
+    sw   zero, 0(t1)   // *(CLINT_BASE + hart_id * 4) = 0
+    la   t2, __base_regs
+    lw   t2, CHESHIRE_NUM_INT_HARTS_REG_OFFSET(t2)
+    slli t2, t2, 2
+    add  t2, t0, t2    // t2 = CLINT_BASE + (n_harts * 4)
+    // Wait until *all* CLINT IPI registers are cleared
+1:  lw   t1, 0(t0)
+    bnez t1, 1b
+    addi t0, t0, 4
+    blt  t0, t2, 1b
+
+    // Jump to next stage
+    // Load boot address from global scratch registers
+    la   t0, __base_regs
+    lwu  t1, CHESHIRE_SCRATCH_5_REG_OFFSET(t0)
+    slli t1, t1, 32
+    lwu  t0, CHESHIRE_SCRATCH_4_REG_OFFSET(t0)
+    or   t0, t0, t1
+    csrr a0, mhartid  // Store hartid to a0
+    jalr ra, 0(t0)    // Jump to boot address
+    ret               // We should never get here