[NOT MERGE] add coverage option to collect line/toggle coverage

doc/cover.md

@@ -0,0 +1,122 @@
+# Coverage
+
+## Introduction
+
+The collection of T1 coverage involves several stages:
+
+1. Describe coverpoints using JSON and write to a file. A simple JSON format is as follows:
+tests/mlir/hello.json:
+```json
+{
+  "assert": [
+    {
+      "name": "vmv_v_i",
+      "description": "single instruction vmv.v.i"
+    }
+  ],
+  "tree": [],
+  "module": []
+}
+```
+The `assert`, `tree`, and `module` fields describe three types of coverage collection methods, each containing multiple coverpoints. The `name` is the coverpoint's name, and the `description` is its description.
+
+2. Convert the file into an assertion hierarchical file acceptable by VCS.
+This step is completed in the T1 tests' Nix script. When writing tests, use the Nix script to call `jq` to generate the corresponding assertion hierarchical file. If no file is specified, it defaults to not covering any points. For example, in T1's tests/mlir/default.nix:
+```shell
+if [ -f ${caseName}.json ]; then
+  ${jq}/bin/jq -r '[.assert[] | "+assert " + .name] + [.tree[] | "+tree " + .name] + [.module[] | "+module " + .name] | .[]' \
+      ${caseName}.json > $pname.cover
+else 
+  echo "-assert *" > $pname.cover
+fi
+```
+
+You can see the generated file when running the test:
+```shell
+nix build .#t1.blastoise.t1rocketemu.cases.mlir.hello
+```
+
+The generated assertion hierarchical file format is as follows:
+result/mlir.hello.cover:
+```
++assert vmv_v_i
+```
+
+3. Specify the types of coverage the simulator can generate using the `cm` parameter when compiling the simulator with VCS.
+
+4. Generate coverage and output the `cm.vdb` file using the `cm` parameter and hierarchical file when running the simulator with VCS.
+These parameters are specified in the `coverType` in nix/t1/run/run-vcs-emu.nix:
+```shell
+coverType = "line+cond+fsm+tgl+branch+assert";
+
+"-cm"
+"${coverType}"
+"-assert"
+"hier=${testCase}/${testCase.pname}.cover"
+```
+
+5. Generate a coverage report using URG from the `cm.vdb` file.
+nix/t1/run/default.nix:
+```shell
+${vcs-emu.snps-fhs-env}/bin/snps-fhs-env -c "urg -dir $emuOutput/*/cm.vdb -format text -metric line+cond+fsm+tgl+branch+assert -show summary"
+```
+
+## Adding New Coverpoints
+
+From the introduction, adding new coverpoints involves adding the corresponding coverpoints in the JSON file and generating the corresponding assertion hierarchical file in the Nix script. For example, to add a new `assert`:
+
+1. Add a new `assert` in the JSON file:
+tests/mlir/hello.json:
+```json
+{
+  "assert": [
+    {
+      "name": "vmv_v_i",
+      "description": "single instruction vmv.v.i"
+    },
+    {
+      "name": "vmv_v_v",
+      "description": "single instruction vmv.v.v"
+    }
+  ],
+  "tree": [],
+  "module": []
+}
+```
+
+2. Generate the corresponding assertion hierarchical file in the Nix script:
+tests/mlir/default.nix:
+```shell
+if [ -f ${caseName}.json ]; then
+  ${jq}/bin/jq -r '[.assert[] | "+assert " + .name] + [.tree[] | "+tree " + .name] + [.module[] | "+module " + .name] | .[]' \
+      ${caseName}.json > $pname.cover
+else 
+  echo "-assert *" > $pname.cover
+fi
+```
+
+3. Run the coverage for a single test case:
+```shell
+nix build .#t1.blastoise.t1rocketemu.run.mlir.hello.vcs-emu-cover-full --impure
+```
+Then view the coverage report:
+```shell
+cat ./result/urgReport/tests.txt
+```
+
+4. Get the coverage for all test cases (ensure you are on the correct branch):
+```shell
+nix build .#t1.blastoise.t1rocketemu.run._vcsEmuResult --impure
+urg -dir ./result/*/cm.vdb -full64 -dbname merged -show summary -parallel
+```
+
+5. View the coverage using Verdi:
+```shell
+verdi -cov -covdir merged.vdb
+```
+
+6. To view the coverage in HTML/text format, use URG:
+```shell
+urg -dir ./result/*/cm.vdb -full64 -format text -show summary -parallel
+cat ./urgReport/tests.txt
+```

nix/t1/conversion/sv-to-vcs-simulator.nix

@@ -8,6 +8,7 @@
 , rtl
 , vsrc
 , enableCover ? false
+, coverType ? null
 , enableTrace ? false
 , vcsLinkLibs ? [ ]
 , topModule ? null
@@ -48,7 +49,7 @@ stdenv.mkDerivation rec {
   ]
   ++ lib.optionals enableCover [
     "-cm"
-    "assert"
+    coverType
     "-cm_dir"
     "./cm"
     "-assert"

nix/t1/run/default.nix

@@ -6,6 +6,7 @@
 , vcs-emu
 , vcs-emu-rtlink
 , vcs-emu-cover
+, vcs-emu-cover-full
 , vcs-emu-trace
 , vcs-dpi-lib
 , cases
@@ -40,6 +41,7 @@ let
             verilator-emu-trace = runVerilatorEmu verilator-emu-trace case;
             vcs-emu = runVCSEmu vcs-emu-rtlink case;
             vcs-emu-cover = runVCSEmu vcs-emu-cover case;
+            vcs-emu-cover-full = runVCSEmu vcs-emu-cover-full case;
             vcs-emu-trace = runVCSEmu vcs-emu-trace case;
             vcs-prof-vcd = runFsdb2vcd (runVCSEmu vcs-emu-trace case);
           };
@@ -92,11 +94,11 @@ let
     in
     runCommand "catch-${configName}-all-emu-result-for-ci" { } script;
 
-  _vcsEmuResult = runCommand "get-vcs-emu-result" { __noChroot = true; emuOutput = _getAllResult "vcs-emu-cover"; } ''
+  _vcsEmuResult = runCommand "get-vcs-emu-result" { __noChroot = true; emuOutput = _getAllResult "vcs-emu-cover-full"; } ''
     cp -vr $emuOutput $out
     chmod -R u+w $out
 
-    ${vcs-emu.snps-fhs-env}/bin/snps-fhs-env -c "urg -dir $emuOutput/*/cm.vdb -format text -metric assert -show summary"
+    ${vcs-emu.snps-fhs-env}/bin/snps-fhs-env -c "urg -dir $emuOutput/*/cm.vdb -format text -metric line+cond+fsm+tgl+branch+assert -show summary"
     cp -vr urgReport $out/
   '';
 in

nix/t1/run/run-vcs-emu.nix

@@ -20,6 +20,7 @@ stdenvNoCC.mkDerivation (rec {
   name = "${testCase.pname}-vcs-result" + (lib.optionalString emulator.enableTrace "-trace");
   nativeBuildInputs = [ zstd jq python3 ];
   __noChroot = true;
+  coverType = "line+cond+fsm+tgl+branch+assert";
 
   passthru = {
     caseName = testCase.pname;
@@ -49,7 +50,7 @@ stdenvNoCC.mkDerivation (rec {
   ]
   ++ lib.optionals emulator.enableCover [
     "-cm"
-    "assert"
+    "${coverType}"
     "-assert"
     "hier=${testCase}/${testCase.pname}.cover"
   ]
@@ -134,7 +135,7 @@ stdenvNoCC.mkDerivation (rec {
     fi
 
     ${lib.optionalString emulator.enableCover ''
-      ${snps-fhs-env}/bin/snps-fhs-env -c "urg -dir cm.vdb -format text -metric assert -show summary"
+      ${snps-fhs-env}/bin/snps-fhs-env -c "urg -dir cm.vdb -format text -metric ${coverType} -show summary"
       # TODO: add a flag to specify 'vdb only generated in ci mode'
       cp -vr cm.vdb $out/
       cp -vr urgReport $out/

nix/t1/t1.nix

@@ -156,8 +156,14 @@ forEachTop (topName: generator: self: {
   };
   vcs-emu-cover = self.vcs-emu.override {
     enableCover = true;
+    coverType = "assert";
     mainProgram = "${topName}-vcs-cover-simulator";
   };
+  vcs-emu-cover-full = self.vcs-emu.override {
+    enableCover = true;
+    coverType = "line+cond+fsm+tgl+branch+assert";
+    mainProgram = "${topName}-vcs-cover-full-simulator";
+  };
   vcs-emu-trace = self.vcs-emu.override {
     enableTrace = true;
     mainProgram = "${topName}-vcs-trace-simulator";

script/ci/src/Main.scala

@@ -164,7 +164,8 @@ object Main:
 
       val testAttr       = emuLib.toLowerCase() match
         case "verilator" => s".#t1.$config.$top.run.$caseName.verilator-emu"
-        case "vcs"       => s".#t1.$config.$top.run.$caseName.vcs-emu-cover"
+        // TODO: should not be cover for every test case
+        case "vcs"       => s".#t1.$config.$top.run.$caseName.vcs-emu-cover-full"
         case _           => Logger.fatal(s"Invalid test type ${emuLib}")
       val testResultPath =
         try
@@ -305,25 +306,40 @@ object Main:
           Logger.info("Filtering urg report")
           val finalMdPath     = os.Path(urgReportFilePath.get, os.pwd)
           val urgAssertFile   = emuResultPath / "urgReport" / "asserts.txt"
-          val summaryHeading  = "^Summary for Cover Properties$".r
+          val urgSummaryFile  = emuResultPath / "urgReport" / "tests.txt"
+
+          val summaryHeading   = "^Total Coverage Summary $".r
+          val summaryStr =
+            os.read(urgSummaryFile)
+              .lines()
+              .dropWhile(!summaryHeading.matches(_))
+              .takeWhile(!_.trim.isEmpty)
+              .toArray
+              .mkString("\n")
+
+          val coverSummaryHeading  = "^Summary for Cover Properties$".r
           val coverSummaryStr =
             os.read(urgAssertFile)
               .lines()
-              .dropWhile(!summaryHeading.matches(_))
-              .takeWhile(_.distinct != "-")
+              .dropWhile(!coverSummaryHeading.matches(_))
+              .takeWhile(!_.trim.isEmpty)
               .toArray
               .mkString("\n")
-          val detailHeading   = "^Detail Report for Cover Properties$".r
+
+          val coverDetailHeading   = "^Detail Report for Cover Properties$".r
           val coverDetailStr  =
             os.read(urgAssertFile)
               .lines()
-              .dropWhile(!detailHeading.matches(_))
+              .dropWhile(!coverDetailHeading.matches(_))
               .toArray
               .mkString("\n")
+
           os.write.append(finalMdPath, s"### Coverage for $config \n")
           os.write.append(finalMdPath, "```text\n")
+          os.write.append(finalMdPath, summaryStr)
+          os.write.append(finalMdPath, "\n----------------------\n")
           os.write.append(finalMdPath, coverSummaryStr)
-          os.write.append(finalMdPath, "----------------------\n")
+          os.write.append(finalMdPath, "\n----------------------\n")
           os.write.append(finalMdPath, coverDetailStr)
           os.write.append(finalMdPath, "\n```\n")
 

script/emu/src/Main.scala

@@ -177,6 +177,10 @@ object Main:
       short = 'c',
       doc = "configuration name"
     ) config:   Option[String],
+    @arg(
+      name = "cover-type",
+      doc = "Type for coverage, only avaliable in vcs-emu-cover-full, Eg. assert+line+tgl"
+    ) coverType:  Option[String] = Some("assert"),
     @arg(
       name = "verbose",
       short = 'v',
@@ -214,8 +218,9 @@ object Main:
         s"No cached emulator selection nor --emu argument was provided"
       )
 
-    val isTrace = finalEmuType.get.contains("-trace")
-    val isCover = finalEmuType.get.contains("-cover")
+    val isTrace = finalEmuType.get.endsWith("-trace")
+    val isCover = finalEmuType.get.endsWith("-cover")
+    val isCoverFull = finalEmuType.get.endsWith("-cover-full")
 
     val finalConfig = tryRestoreFromCache("config", config)
     if finalConfig.isEmpty then
@@ -227,6 +232,16 @@ object Main:
       s"Using config=${BOLD}${finalConfig.get}${RESET} emulator=${BOLD}${finalEmuType.get}${RESET} case=${BOLD}$caseName${RESET}"
     )
 
+    // check the arguments of coverage
+    if isCover && !(coverType.isDefined && coverType.get.equals("assert")) then
+      Logger.fatal(s"coverType should be defined in assert")
+    if isCoverFull && !coverType.isDefined then
+      Logger.fatal(s"coverType should be defined")
+    if isCover || isCoverFull then
+      Logger.info(
+        s"Coverage type is ${BOLD}${coverType.get}${RESET}"
+      )
+
     val caseElfPath = resolveTestElfPath(finalIp.get, finalConfig.get, caseName, forceX86)
     val caseCoverPath = resolveTestCoverPath(finalIp.get, finalConfig.get, caseName, forceX86)
     val outputPath  = prepareOutputDir(outDir.getOrElse("t1-sim-result"))
@@ -240,7 +255,7 @@ object Main:
     )
       ++ optionals(timeout.isDefined, Seq(s"+t1_timeout=${timeout.getOrElse("unreachable")}"))
       ++ optionals(isTrace, Seq(s"+t1_wave_path=${outputPath / "wave.fsdb"}"))
-      ++ optionals(isCover, Seq("-cm", "assert", "-assert", s"hier=${caseCoverPath}"))
+      ++ optionals(isCover || isCoverFull, Seq("-cm", s"${coverType.get}", "-assert", s"hier=${caseCoverPath}"))
       ++ optionals(!leftOverArguments.isEmpty, leftOverArguments)
 
     if dryRun.value then return
@@ -307,7 +322,8 @@ object Main:
 
     Logger.info("Driver finished")
 
-    if isCover then
+    // check the output of coverage
+    if isCover || isCoverFull then
       if os.exists(os.pwd / "cm.vdb") then
         Logger.info(s"Coverage database saved under ${os.pwd}/cm.vdb")
 
@@ -325,7 +341,7 @@ object Main:
           "-format", 
           "text",
           "-metric",
-          "assert",
+          s"${coverType.get}",
           "-show",
           "summary"
         )