refactor: [config] move the flow config into the project config

This means the project config contains the whole project
configuration including the flow config. This can be broken
down later into smaller Pydantic objects as the fields and
types become better defined.

Signed-off-by: James McCorrie <[email protected]>

Author: machshev

src/dvsim/cli.py

@@ -22,12 +22,10 @@
 
 import argparse
 import datetime
-import json
 import os
 import random
 import sys
 import textwrap
-from collections.abc import Sequence
 from pathlib import Path
 
 from dvsim.flow.factory import make_flow
@@ -671,6 +669,8 @@ def main() -> None:
         purge=args.purge,
         dry_run=args.dry_run,
         remote=args.remote,
+        select_cfgs=args.select_cfgs,
+        args=args,
     )
     project_cfg.save()
 
@@ -701,16 +701,8 @@ def main() -> None:
 
     # Build infrastructure from hjson file and create the list of items to
     # be deployed.
-    config_data = project_cfg.load_config(
-        select_cfgs=args.select_cfgs,
-        args=args,
-    )
-    project_cfg.run_dir.mkdir(parents=True, exist_ok=True)
-    (project_cfg.run_dir / "config.json").write_text(json.dumps(config_data))
-
     flow = make_flow(
         project_cfg=project_cfg,
-        config_data=config_data,
         args=args,
     )
 

src/dvsim/flow/factory.py

@@ -4,7 +4,6 @@
 """Factory to generate a flow config."""
 
 from argparse import Namespace
-from collections.abc import Mapping
 
 from dvsim.flow.base import FlowCfg
 from dvsim.flow.cdc import CdcCfg
@@ -53,7 +52,6 @@ def _get_flow_handler_cls(flow: str) -> type[FlowCfg]:
 
 def make_flow(
     project_cfg: Project,
-    config_data: Mapping,
     args: Namespace,
 ) -> FlowCfg:
     """Make a flow config by loading the config file at path.
@@ -68,6 +66,9 @@ def make_flow(
         config file.
 
     """
+    # convert back to simple dict
+    config_data = project_cfg.config.model_dump()
+
     if "flow" not in config_data:
         msg = 'No value for the "flow" key. Are you sure this is a dvsim configuration file?'
         raise RuntimeError(

src/dvsim/project.py

@@ -29,6 +29,21 @@ class FlowConfig(BaseModel):
 
     model_config = ConfigDict(frozen=True, extra="allow")
 
+    flow: str
+    name: str
+
+
+class TopFlowConfig(BaseModel):
+    """Flow configuration data."""
+
+    model_config = ConfigDict(frozen=True, extra="allow")
+
+    flow: str
+    project: str
+    revision: str
+
+    cfgs: Mapping[Path, FlowConfig]
+
 
 class Project(BaseModel):
     """Project meta data."""
@@ -41,6 +56,7 @@ class Project(BaseModel):
     scratch_path: Path
     branch: str
     job_prefix: str
+    config: TopFlowConfig | FlowConfig
 
     logfile: Path
     run_dir: Path
@@ -55,41 +71,6 @@ def save(self) -> None:
 
         (self.run_dir / "project.json").write_text(meta_json)
 
-    def load_config(
-        self,
-        select_cfgs: Sequence[str] | None,
-        args: Namespace,
-    ) -> Mapping:
-        """Load the project configuration.
-
-        Args:
-            project_cfg: metadata about the project
-            select_cfgs: list of child config names to use from the primary config
-            args: are the arguments passed to the CLI
-
-        Returns:
-            Project configuration.
-
-        """
-        log.info("Loading primary config file: %s", self.top_cfg_path)
-
-        # load the whole project config data
-        cfg = dict(
-            load_cfg(
-                path=self.top_cfg_path,
-                path_resolution_wildcards={
-                    "proj_root": self.root_path,
-                },
-                select_cfgs=select_cfgs,
-            ),
-        )
-
-        # Tool specified on CLI overrides the file based config
-        if args.tool is not None:
-            cfg["tool"] = args.tool
-
-        return cfg
-
     @staticmethod
     def load(path: Path) -> "Project":
         """Load project meta from file."""
@@ -102,6 +83,8 @@ def init(
         proj_root: Path | None,
         scratch_root: Path | None,
         branch: str,
+        select_cfgs: Sequence[str] | None,
+        args: Namespace,
         *,
         job_prefix: str = "",
         purge: bool = False,
@@ -118,6 +101,19 @@ def init(
         This function returns the updated proj_root src and destination path. If
         --remote switch is not set, the destination path is identical to the src
         path. Likewise, if --dry-run is set.
+
+        Args:
+            args: are the arguments passed to the CLI
+            branch: version control branch
+            cfg_path: path to the top flow config
+            dry_run: do not run any jobs just go through the motions
+            job_prefix: prefix for the job name
+            proj_root: path to the project root
+            purge: bool = False,
+            remote: remote execution
+            scratch_root: path to the scratch dir
+            select_cfgs: list of child config names to use from the primary config
+
         """
         if not cfg_path.exists():
             log.fatal("Path to config file %s appears to be invalid.", cfg_path)
@@ -156,6 +152,14 @@ def init(
             cfg_path = root_path / cfg_path.relative_to(src_path)
 
         run_dir = scratch_path / branch
+
+        config = _load_flow_config(
+            top_cfg_path=cfg_path,
+            root_path=root_path,
+            select_cfgs=select_cfgs,
+            args=args,
+        )
+
         return Project(
             top_cfg_path=cfg_path,
             root_path=root_path,
@@ -165,9 +169,51 @@ def init(
             job_prefix=job_prefix,
             logfile=run_dir / "run.log",
             run_dir=run_dir,
+            config=config,
         )
 
 
+def _load_flow_config(
+    top_cfg_path: Path,
+    root_path: Path,
+    select_cfgs: Sequence[str] | None,
+    args: Namespace,
+) -> TopFlowConfig | FlowConfig:
+    """Load the project configuration.
+
+    Args:
+        top_cfg_path: path to the top level config file
+        root_path: path to the root of the project,
+        select_cfgs: list of child config names to use from the primary config
+        args: are the arguments passed to the CLI
+
+    Returns:
+        Project configuration.
+
+    """
+    log.info("Loading primary config file: %s", top_cfg_path)
+
+    # load the whole project config data
+    cfg = dict(
+        load_cfg(
+            path=top_cfg_path,
+            path_resolution_wildcards={
+                "proj_root": root_path,
+            },
+            select_cfgs=select_cfgs,
+        ),
+    )
+
+    # Tool specified on CLI overrides the file based config
+    if args.tool is not None:
+        cfg["tool"] = args.tool
+
+    if "cfgs" in cfg:
+        return TopFlowConfig.model_validate(cfg)
+
+    return FlowConfig.model_validate(cfg)
+
+
 def _network_dir_accessible_and_exists(
     path: Path,
     timeout: int = 1,

tests/test_project.py

@@ -8,27 +8,38 @@
 from pathlib import Path
 
 import pytest
-from hamcrest import assert_that, equal_to
+from hamcrest import assert_that, equal_to, instance_of
 
-from dvsim.project import Project
+from dvsim.project import FlowConfig, Project, TopFlowConfig
 
 __all__ = ()
 
 
 @pytest.mark.parametrize(
-    "data",
+    ("data", "flow_config_cls"),
     [
-        {
-            "top_cfg_path": Path("cfg_path.hjson"),
-            "root_path": Path("root_path"),
-            "src_path": Path("src_path"),
-            "branch": "branch",
-            "job_prefix": "job_prefix",
-            "logfile": Path("logfile"),
-        },
+        (
+            {
+                "top_cfg_path": Path("cfg_path.hjson"),
+                "root_path": Path("root_path"),
+                "src_path": Path("src_path"),
+                "branch": "branch",
+                "job_prefix": "job_prefix",
+                "logfile": Path("logfile"),
+                "config": {
+                    "flow": "flow",
+                    "name": "name",
+                },
+            },
+            FlowConfig,
+        ),
     ],
 )
-def test_project_meta(data: Mapping, tmp_path: Path) -> None:
+def test_project_config(
+    data: Mapping,
+    flow_config_cls: type[FlowConfig | TopFlowConfig],
+    tmp_path: Path,
+) -> None:
     """Test Project saving and loading."""
     meta = Project(
         **data,
@@ -41,3 +52,4 @@ def test_project_meta(data: Mapping, tmp_path: Path) -> None:
     loaded_meta = Project.load(path=meta.run_dir)
 
     assert_that(loaded_meta, equal_to(meta))
+    assert_that(loaded_meta.config, instance_of(flow_config_cls))