diff --git a/README.md b/README.md index da973fa..96d9ad1 100644 --- a/README.md +++ b/README.md @@ -54,3 +54,51 @@ More complete examples of projects using the ASHRAE Standard 232P framework incl - [ASHRAE Standard 205](https://github.com/open205/schema-205) (transitioning to lattice) - [ASHRAE Standard 229](https://github.com/open229/ruleset-model-description-schema) (does not use lattice...yet) +### C++ Library Code Generation + +Lattice's C++ code generation is achieved by calling the function `generate_cpp_project()`. Its only parameter is a list of submodule URLs. + +#### Translations + +Schema can be converted into C++ classes with the following mappings: + +| Object Type | C++ type | +|------------------- | -------- | +| Data Group | `struct` | +| Data Element | public data member | +| Enumerator | `enum` | +| Data Group Template | base class*| +| + + *with optional methods, including initialize() + +| Data Type | C++ type | +|------------------- | -------- | +| Integer | `int` | +| Numeric | `float` | +| String | `std::string` | +| {} | `struct` | +| <> | `enum` | +| [] | `std::vector` | +| list "(A,B)" | `std::unique_ptr`| +| + +#### Inheritance + +The code generator will assume that *Data Group* schema elements with a *Data Group Template* parameter use that template as the element's superclass. An `#include` statement for the expected superclass file is listed at the top of the schema's implemenation (cpp) file (note: see Big Ladder file naming conventions). If the superclass file is not found, the C++ generator will create a stub file for the class. If it is found, any virtual functions in the superclass will appear as overridden function stubs in the subclassed *Data Group*'s struct. Any additional code (members, methods) for the superclass itself must be provided by the **lattice** user. + +In the event that the source schema contains a *Data Element* with a "selector constraint" (i.e. a list of possible *Data Type*s combined with an associated list of possible enumerator values for the selector *Data Element*), the C++ generated code will assume that the *Data Type*s in the list all derive from a common base class, named by the *Data Group Template* of the first *Data Type* in the list. (The *Data Group Template* may be defined in a different schema than the *Data Type*.) An `#include` statement for the base class will be generated, as above. The code that populates the *Data Group* (the struct's `from_json` function) will use a conditional statement to create a new object of the correct subclass and assign it to a member `unique_ptr`, calling directly the derived class' auto-generated `from_json` functions. + +Note: If the first *Data Type* in a selector constraint list does not have a *Data Group Template* tag in its schema, the `unique_ptr`'s base class will default to "MissingType." + +#### Build information + +In addition to `.h` and `.cpp` files containing translated schema data, the code generator adds Git repository and CMake build support to the schema code, creating most of the structure necessary to test-build the schema code as a library. Necessary submodules are also downloaded: [fmt](https://github.com/fmtlib/fmt.git), [json](https://github.com/nlohmann/json), and [courier](https://github.com/bigladder/courier.git). To build a generated project, navigate to your **lattice** project's build directory, cpp subdirectory (e.g. /.lattice/cpp), and use a standard cmake build sequence: + +> cmake -B build +> +> cmake --build build --config release + + + + diff --git a/dodo.py b/dodo.py index a97f5b7..d781058 100644 --- a/dodo.py +++ b/dodo.py @@ -1,5 +1,6 @@ -from lattice import Lattice from pathlib import Path +from lattice import Lattice +from lattice.cpp.header_entry_extension_loader import load_extensions from doit.tools import create_folder @@ -29,7 +30,7 @@ def task_generate_meta_schemas(): name = Path(example.root_directory).name yield { "name": name, - "file_dep": [schema.path for schema in example.schemas] + "file_dep": [schema.file_path for schema in example.schemas] + [BASE_META_SCHEMA_PATH, CORE_SCHEMA_PATH, Path(SOURCE_PATH, "meta_schema.py")], "targets": [schema.meta_schema_path for schema in example.schemas], "actions": [(example.generate_meta_schemas, [])], @@ -44,7 +45,7 @@ def task_validate_schemas(): yield { "name": name, "task_dep": [f"generate_meta_schemas:{name}"], - "file_dep": [schema.path for schema in example.schemas] + "file_dep": [schema.file_path for schema in example.schemas] + [schema.meta_schema_path for schema in example.schemas] + [BASE_META_SCHEMA_PATH, CORE_SCHEMA_PATH, Path(SOURCE_PATH, "meta_schema.py")], "actions": [(example.validate_schemas, [])], @@ -58,7 +59,7 @@ def task_generate_json_schemas(): yield { "name": name, "task_dep": [f"validate_schemas:{name}"], - "file_dep": [schema.path for schema in example.schemas] + "file_dep": [schema.file_path for schema in example.schemas] + [schema.meta_schema_path for schema in example.schemas] + [CORE_SCHEMA_PATH, BASE_META_SCHEMA_PATH, Path(SOURCE_PATH, "schema_to_json.py")], "targets": [schema.json_schema_path for schema in example.schemas], @@ -88,7 +89,7 @@ def task_generate_markdown(): yield { "name": name, "targets": [template.markdown_output_path for template in example.doc_templates], - "file_dep": [schema.path for schema in example.schemas] + "file_dep": [schema.file_path for schema in example.schemas] + [template.path for template in example.doc_templates] + [Path(SOURCE_PATH, "docs", "grid_table.py")], "task_dep": [f"validate_schemas:{name}"], @@ -104,13 +105,13 @@ def task_generate_cpp_code(): yield { "name": name, "task_dep": [f"validate_schemas:{name}"], - "file_dep": [schema.path for schema in example.cpp_schemas] + "file_dep": [schema.file_path for schema in example.cpp_schemas] + [schema.meta_schema_path for schema in example.schemas] - + [CORE_SCHEMA_PATH, BASE_META_SCHEMA_PATH, Path(SOURCE_PATH, "header_entries.py")], - "targets": [schema.cpp_header_path for schema in example.cpp_schemas] - + [schema.cpp_source_path for schema in example.cpp_schemas] - + example.cpp_support_headers(), - "actions": [(example.generate_cpp_headers, [])], + + [CORE_SCHEMA_PATH, BASE_META_SCHEMA_PATH, Path(SOURCE_PATH, "cpp", "header_entries.py"), Path(SOURCE_PATH, "cpp", "cpp_entries.py")], + "targets": [schema.cpp_header_file_path for schema in example.cpp_schemas] + + [schema.cpp_source_file_path for schema in example.cpp_schemas] + + example.cpp_support_headers + [example.cpp_output_dir / "CMakeLists.txt", example.cpp_output_dir / "src" / "CMakeLists.txt"], + "actions": [(load_extensions, [Path(example.root_directory, "cpp", "extensions")]), (example.generate_cpp_project, [["https://github.com/nlohmann/json.git", "https://github.com/bigladder/courier.git", "https://github.com/fmtlib/fmt.git"]])], "clean": True, } @@ -122,7 +123,7 @@ def task_generate_web_docs(): yield { "name": name, "task_dep": [f"validate_schemas:{name}", f"generate_json_schemas:{name}", f"validate_example_files:{name}"], - "file_dep": [schema.path for schema in example.schemas] + "file_dep": [schema.file_path for schema in example.schemas] + [template.path for template in example.doc_templates] + [Path(SOURCE_PATH, "docs", "mkdocs_web.py")], "targets": [Path(example.web_docs_directory_path, "public")], diff --git a/examples/fan_spec/cpp/extensions/grid_var_enum.py b/examples/fan_spec/cpp/extensions/grid_var_enum.py new file mode 100644 index 0000000..1fe9077 --- /dev/null +++ b/examples/fan_spec/cpp/extensions/grid_var_enum.py @@ -0,0 +1,28 @@ +from dataclasses import dataclass +from lattice.cpp.header_entries import HeaderEntry +from lattice.cpp.header_entries import register_data_element_operation + +@dataclass +class GridVarCounterEnum(HeaderEntry): + elements_dict: dict + + def __post_init__(self): + super().__post_init__() + self.type = "enum" + self._closure = "};" + self._enumerants = list() + + for element in self.elements_dict: + self._enumerants.append(f"{element}_index") + self._enumerants.append("index_count"); + + def __str__(self): + enums = self._enumerants + tab = "\t" + entry = f"{self._level * tab}{self.type} {self._opener}\n" + entry += ",\n".join([f"{(self._level + 1) * tab}{e}" for e in enums]) + entry += f"\n{self._level * tab}{self._closure}" + return entry + +def register(): + register_data_element_operation("GridVariablesTemplate", GridVarCounterEnum) \ No newline at end of file diff --git a/examples/fan_spec/cpp/extensions/lookup_struct.py b/examples/fan_spec/cpp/extensions/lookup_struct.py new file mode 100644 index 0000000..d97cfd5 --- /dev/null +++ b/examples/fan_spec/cpp/extensions/lookup_struct.py @@ -0,0 +1,29 @@ +import re +from dataclasses import dataclass +from lattice.cpp.header_entries import DataElement, Struct +from lattice.cpp.header_entries import register_data_group_operation + +@dataclass +class LookupStruct(Struct): + """ + Special case struct for Lookup Variables. Its str overload adds a LookupStruct declaration. + """ + + def __str__(self): + """Two C++ entries share the schema child-entries, so one HeaderEntry subclass creates both.""" + struct = super().__str__() + + # Add a LookupStruct that offers a SOA access rather than AOS + tab = "\t" + struct += "\n" + struct += f"{self._level * tab}{self.type} {self.name}Struct {self._opener}\n" + for c in [ch for ch in self.child_entries if isinstance(ch, DataElement)]: + m = re.match(r'std::vector\<(.*)\>', c.type) + assert m is not None + struct += f"{(self._level+1) * tab}{m.group(1)} {c.name};\n" + struct += f"{self._level * tab}{self._closure}" + + return struct + +def register(): + register_data_group_operation("LookupVariablesTemplate", LookupStruct) \ No newline at end of file diff --git a/examples/fan_spec/examples/Fan-Continuous.RS0003.a205.json b/examples/fan_spec/examples/Fan-Continuous.RS0003.a205.json index e6ca139..279695a 100644 --- a/examples/fan_spec/examples/Fan-Continuous.RS0003.a205.json +++ b/examples/fan_spec/examples/Fan-Continuous.RS0003.a205.json @@ -1,12 +1,13 @@ { "metadata": { "schema_author": "ASHRAE_205", - "schema": "RS0003", + "schema_name": "RS0003", "schema_version": "0.2.0", + "author": "SSPC 205 Working Group", "description": "Continuous Fan", "id": "123e4567-e89b-12d3-a456-426614174000", - "timestamp": "2020-05-11T00:00Z", - "version": 1, + "time_of_creation": "2020-05-11T00:00Z", + "version": "1.0.0", "disclaimer": "Example data not to be used for simulation" }, "description": { diff --git a/examples/fan_spec/schema/ASHRAE205.schema.yaml b/examples/fan_spec/schema/ASHRAE205.schema.yaml index 4f7ec2a..2fdb026 100644 --- a/examples/fan_spec/schema/ASHRAE205.schema.yaml +++ b/examples/fan_spec/schema/ASHRAE205.schema.yaml @@ -25,15 +25,15 @@ PerformanceMapTemplate: GridVariablesTemplate: Object Type: "Data Group Template" Required Data Types: - - "[Numeric][1..]" - - "[Integer][1..]" + - "[Numeric]" + - "[Integer]" Data Elements Required: True LookupVariablesTemplate: Object Type: "Data Group Template" Required Data Types: - - "[Numeric][1..]" - - "[Integer][1..]" + - "[Numeric]" + - "[Integer]" Data Elements Required: True RepresentationSpecificationTemplate: diff --git a/examples/fan_spec/schema/RS0003.schema.yaml b/examples/fan_spec/schema/RS0003.schema.yaml index cb0ac41..9d5c273 100644 --- a/examples/fan_spec/schema/RS0003.schema.yaml +++ b/examples/fan_spec/schema/RS0003.schema.yaml @@ -217,14 +217,18 @@ SystemCurve: Data Elements: standard_air_volumetric_flow_rate: Description: "Volumetric air flow rate through an air distribution system at standard air conditions" - Data Type: "[Numeric][2..]" - Constraints: ">=0.0" + Data Type: "[Numeric]" + Constraints: + - ">=0.0" + - "[2..]" Units: "m3/s" Required: True static_pressure_difference: Description: "Static pressure difference of an air distribution system" - Data Type: "[Numeric][2..]" - Constraints: ">=0.0" + Data Type: "[Numeric]" + Constraints: + - ">=0.0" + - "[2..]" Units: "Pa" Required: True @@ -247,14 +251,18 @@ GridVariablesContinuous: Data Elements: standard_air_volumetric_flow_rate: Description: "Volumetric air flow rate through fan assembly at standard air conditions" - Data Type: "[Numeric][1..]" - Constraints: ">=0.0" + Data Type: "[Numeric]" + Constraints: + - ">=0.0" + - "[1..]" Units: "m3/s" Required: True static_pressure_difference: Description: "External static pressure across fan assembly at dry coil conditions" - Data Type: "[Numeric][1..]" - Constraints: ">=0.0" + Data Type: "[Numeric]" + Constraints: + - ">=0.0" + - "[1..]" Units: "Pa" Notes: "Any static pressure deduction (or addition) for wet coil is specified by `wet_pressure_difference` in 'assembly_components' data group" Required: True @@ -265,14 +273,18 @@ LookupVariablesContinuous: Data Elements: impeller_rotational_speed: Description: "Rotational speed of fan impeller" - Data Type: "[Numeric][1..]" - Constraints: ">=0.0" + Data Type: "[Numeric]" + Constraints: + - ">=0.0" + - "[1..]" Units: "rev/s" Required: True shaft_power: Description: "Mechanical shaft power input to fan assembly" - Data Type: "[Numeric][1..]" - Constraints: ">=0.0" + Data Type: "[Numeric]" + Constraints: + - ">=0.0" + - "[1..]" Units: "W" Notes: "Does not include the mechanical efficiency of any mechanical drive used to modify rotational speed between the motor and impeller" Required: True @@ -296,15 +308,19 @@ GridVariablesDiscrete: Data Elements: speed_number: Description: "Number indicating discrete speed of fan impeller in rank order (with 1 being the lowest speed)" - Data Type: "[Integer][1..]" - Constraints: ">=0" + Data Type: "[Integer]" + Constraints: + - ">=0.0" + - "[1..]" Units: "-" Notes: "Data shall be provided for all allowable discrete speeds or settings" Required: True static_pressure_difference: Description: "External static pressure across fan assembly at dry coil conditions" - Data Type: "[Numeric][1..]" - Constraints: ">=0.0" + Data Type: "[Numeric]" + Constraints: + - ">=0.0" + - "[1..]" Units: "Pa" Notes: "Any static pressure deduction (or addition) for wet coil is specified by `wet_pressure_difference` in 'assembly_components' data group" Required: True @@ -315,20 +331,26 @@ LookupVariablesDiscrete: Data Elements: standard_air_volumetric_flow_rate: Description: "Volumetric air flow rate through fan assembly at standard air conditions" - Data Type: "[Numeric][1..]" - Constraints: ">=0.0" + Data Type: "[Numeric]" + Constraints: + - ">=0.0" + - "[1..]" Units: "m3/s" Required: True shaft_power: Description: "Mechanical shaft power input to fan assembly" - Data Type: "[Numeric][1..]" - Constraints: ">=0.0" + Data Type: "[Numeric]" + Constraints: + - ">=0.0" + - "[1..]" Units: "W" Notes: "Does not include the mechanical efficiency of any mechanical drive used to modify rotational speed between the motor and impeller" Required: True impeller_rotational_speed: Description: "Rotational speed of fan impeller" - Data Type: "[Numeric][1..]" - Constraints: ">=0.0" + Data Type: "[Numeric]" + Constraints: + - ">=0.0" + - "[1..]" Units: "rev/s" Required: True diff --git a/examples/fan_spec/schema/RS0005.schema.yaml b/examples/fan_spec/schema/RS0005.schema.yaml index 4c58fce..9e2157d 100644 --- a/examples/fan_spec/schema/RS0005.schema.yaml +++ b/examples/fan_spec/schema/RS0005.schema.yaml @@ -103,14 +103,18 @@ GridVariables: Data Elements: shaft_power: Description: "Delivered rotational shaft power" - Data Type: "[Numeric][1..]" - Constraints: ">=0.0" + Data Type: "[Numeric]" + Constraints: + - ">=0.0" + - "[1..]" Units: "W" Required: True shaft_rotational_speed: Description: "Rotational speed of shaft" - Data Type: "[Numeric][1..]" - Constraints: ">=0.0" + Data Type: "[Numeric]" + Constraints: + - ">=0.0" + - "[1..]" Units: "rev/s" Required: True @@ -120,14 +124,20 @@ LookupVariables: Data Elements: efficiency: Description: "Efficiency of motor" - Data Type: "[Numeric][1..]" - Constraints: [">=0.0", "<=1.0"] + Data Type: "[Numeric]" + Constraints: + - ">=0.0" + - "<=1.0" + - "[1..]" Units: "-" Notes: "Defined as the ratio of mechanical shaft power to electrical input power of the motor" Required: True power_factor: Description: "Power factor of the motor" - Data Type: "[Numeric][1..]" - Constraints: [">=0.0", "<=1.0"] + Data Type: "[Numeric]" + Constraints: + - ">=0.0" + - "<=1.0" + - "[1..]" Units: "-" Required: True diff --git a/examples/fan_spec/schema/RS0006.schema.yaml b/examples/fan_spec/schema/RS0006.schema.yaml index 623b3e0..14896c6 100644 --- a/examples/fan_spec/schema/RS0006.schema.yaml +++ b/examples/fan_spec/schema/RS0006.schema.yaml @@ -105,14 +105,18 @@ GridVariables: Data Elements: output_power: Description: "Power delivered to the motor" - Data Type: "[Numeric][1..]" - Constraints: ">=0.0" + Data Type: "[Numeric]" + Constraints: + - ">=0.0" + - "[1..]" Units: "W" Required: True output_frequency: Description: "Frequency delivered to the motor" - Data Type: "[Numeric][1..]" - Constraints: ">=0.0" + Data Type: "[Numeric]" + Constraints: + - ">=0.0" + - "[1..]" Units: "Hz" Required: True @@ -122,8 +126,11 @@ LookupVariables: Data Elements: efficiency: Description: "Efficiency of drive" - Data Type: "[Numeric][1..]" - Constraints: [">=0.0","<=1.0"] + Data Type: "[Numeric]" + Constraints: + - ">=0.0" + - "<=1.0" + - "[1..]" Units: "-" Notes: ["Defined as the ratio of electrical output power (to the motor) to electrical input power (to the drive)", "Input power shall include any power required to provide active air cooling for the drive"] diff --git a/examples/fan_spec/schema/RS0007.schema.yaml b/examples/fan_spec/schema/RS0007.schema.yaml index 252fdba..0ac786a 100644 --- a/examples/fan_spec/schema/RS0007.schema.yaml +++ b/examples/fan_spec/schema/RS0007.schema.yaml @@ -99,8 +99,10 @@ GridVariables: Data Elements: output_power: Description: "Output shaft power" - Data Type: "[Numeric][1..]" - Constraints: ">=0.0" + Data Type: "[Numeric]" + Constraints: + - ">=0.0" + - "[1..]" Units: "W" Required: True @@ -110,8 +112,11 @@ LookupVariables: Data Elements: efficiency: Description: "Efficiency of drive" - Data Type: "[Numeric][1..]" - Constraints: [">=0.0","<=1.0"] + Data Type: "[Numeric]" + Constraints: + - ">=0.0" + - "<=1.0" + - "[1..]" Units: "-" Notes: "Defined as the ratio of output shaft power to input shaft power" Required: True diff --git a/examples/lookup_table/schema/LookupTable.schema.yaml b/examples/lookup_table/schema/LookupTable.schema.yaml index 6bda6b3..262d68a 100644 --- a/examples/lookup_table/schema/LookupTable.schema.yaml +++ b/examples/lookup_table/schema/LookupTable.schema.yaml @@ -24,15 +24,15 @@ LookupTableTemplate: GridVariablesTemplate: Object Type: "Data Group Template" Required Data Types: - - "[Numeric][1..]" - - "[Integer][1..]" + - "[Numeric]" + - "[Integer]" Data Elements Required: True LookupVariablesTemplate: Object Type: "Data Group Template" Required Data Types: - - "[Numeric][1..]" - - "[Integer][1..]" + - "[Numeric]" + - "[Integer]" Data Elements Required: True LookupTable: @@ -54,8 +54,10 @@ GridVariables: Data Elements: output_power: Description: "Output shaft power" - Data Type: "[Numeric][1..]" - Constraints: ">=0.0" + Data Type: "[Numeric]" + Constraints: + - ">=0.0" + - "[1..]" Units: "W" Required: True @@ -65,8 +67,11 @@ LookupVariables: Data Elements: efficiency: Description: "Efficiency of drive" - Data Type: "[Numeric][1..]" - Constraints: [">=0.0","<=1.0"] + Data Type: "[Numeric]" + Constraints: + - ">=0.0" + - "<=1.0" + - "[1..]" Units: "-" Notes: "Defined as the ratio of output shaft power to input shaft power" Required: True diff --git a/examples/time_series/schema/TimeSeries.schema.yaml b/examples/time_series/schema/TimeSeries.schema.yaml index 900b55d..97474df 100644 --- a/examples/time_series/schema/TimeSeries.schema.yaml +++ b/examples/time_series/schema/TimeSeries.schema.yaml @@ -31,9 +31,9 @@ TimeIntervals: Data Elements: id: Description: Reference identification - Data Type: ID - Constraints: ":TimeIntervals:" + Data Type: String Required: true + ID: true starting_time: Description: Timestamp indicating the beginning of the data Data Type: Timestamp @@ -47,15 +47,21 @@ TimeIntervals: Required: "if !timestamps" timestamps: Description: Array of timestamps - Data Type: "[Timestamp][1..]" + Data Type: "[Timestamp]" + Constraints: + - "[1..]" Required: "if !regular_interval" labels: Description: Informal labels describing each time interval - Data Type: "[String][1..]" + Data Type: "[String]" + Constraints: + - "[1..]" Notes: "e.g., month names for monthly intervals" notes: Description: Notes about each time interval - Data Type: "[String][1..]" + Data Type: "[String]" + Constraints: + - "[1..]" TimeSeries: Object Type: "Data Group" @@ -74,10 +80,11 @@ TimeSeries: Required: true value_time_intervals: Description: Reference to a `TimeInterval` data group associated with this time series - Data Type: Reference - Constraints: ":TimeIntervals:" + Data Type: ":TimeIntervals:" Required: true values: Description: Time series data values - Data Type: "[Numeric][1..]" + Data Type: "[Numeric]" + Constraints: + - "[1..]" Required: true diff --git a/lattice/core.schema.yaml b/lattice/core.schema.yaml index 4fb805a..7239e85 100644 --- a/lattice/core.schema.yaml +++ b/lattice/core.schema.yaml @@ -48,47 +48,37 @@ Pattern: Examples: - "CA225FB.[1-9]" -ID: - Object Type: "Data Type" - Description: "A string used to identify an instance of a data group." - JSON Schema Type: string - Examples: - - "Lobby Zone" - -Reference: - Object Type: "Data Type" - Description: "A string used to reference an identified instance of a data group." - JSON Schema Type: string - Examples: - - "Lobby Zone" - # Special String Data Types UUID: Object Type: "String Type" Description: "An effectively unique character string conforming to ITU-T Recommendation X.667 (ITU-T 2012)." - JSON Schema Pattern: "^[0-9,a-f,A-F]{8}-[0-9,a-f,A-F]{4}-[0-9,a-f,A-F]{4}-[0-9,a-f,A-F]{4}-[0-9,a-f,A-F]{12}$" + Regular Expression Pattern: "^[0-9,a-f,A-F]{8}-[0-9,a-f,A-F]{4}-[0-9,a-f,A-F]{4}-[0-9,a-f,A-F]{4}-[0-9,a-f,A-F]{12}$" Examples: - "123e4567-e89b-12d3-a456-426655440000" Date: Object Type: "String Type" Description: "A calendar date formatted per ISO 8601 (ISO 2004)" - JSON Schema Pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" + Regular Expression Pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" Examples: - "2015-04-29" Timestamp: Object Type: "String Type" Description: "Date with UTC time formatted per ISO 8601 (ISO 2004)" - JSON Schema Pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}Z$" + Regular Expression Pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}Z$" Examples: - "2016-06-29T14:35Z" +# TODO: GenericTimestamp + +# TODO: TimeDuration + Version: Object Type: "String Type" Description: "Version identifier in the form major.minor.patch as defined by Semver 2016." - JSON Schema Pattern: "^(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)(?:-((?:0|[1-9][0-9]*|[0-9]*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9][0-9]*|[0-9]*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$" + Regular Expression Pattern: "^(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)(?:-((?:0|[1-9][0-9]*|[0-9]*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9][0-9]*|[0-9]*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$" Examples: - "1.1.3" - "1.2.0-beta-92" @@ -99,41 +89,52 @@ Metadata: Object Type: "Data Group" Data Elements: schema_author: - Description: "Name of the schema author" + Description: "Name of the organization that published the schema" Data Type: "String" Required: True - Notes: "Identifies the data model where the schema is defined" - schema: + Notes: "Identifies the organization that defined the schema" + schema_name: Description: "Schema name or identifier" - Data Type: "" + Data Type: "String" Required: True Notes: "Identifies the schema used to define the data content" schema_version: - Description: "Version of the root schema this data complies with" + Description: "The version of the schema the data complies with" Data Type: "Version" Required: True - description: - Description: "Description of data content (suitable for display)" + schema_url: + Description: "The Uniform Resource Locator (url) for the schema definition and/or documentation" + Data Type: "String" + author: + Description: "Name of the entity creating the serialization" Data Type: "String" Required: True - timestamp: - Description: "Date of data publication" - Data Type: "Timestamp" - Required: True - Notes: "Date/time of publication of this data." + Notes: "Identifies the organization that created the file" id: Description: "Unique data set identifier" Data Type: "UUID" Notes: "Assigned by *data publisher* to identify this data. `id` shall remain unchanged for revisions of the same data." + description: + Description: "Description of data content (suitable for display)" + Data Type: "String" + Required: True + time_of_creation: + Description: "Timestamp indicating when the serialization was created" + Data Type: "Timestamp" + Required: True + Notes: "Updated anytime any data content is modified" version: Description: "Integer version identifier for the data" - Data Type: "Integer" - Constraints: ">=1" - Notes: "Used by *data publisher* to track revisions of the data. `data_version` shall be incremented for each data revision." - data_source: + Data Type: "Version" + Notes: + - "Used by data publisher to track revisions of the data" + - "Shall be incremented for each data revision" + source: Description: "Source(s) of the data" Data Type: "String" - Notes: "Used by *data publisher* to document methods (e.g. software and version) used to generate data. **Informative note:** `data_source` may be different from ratings source(s) included elsewhere." + Notes: + - "Used by data publisher to document methods (e.g., software and version) used to generate data" + - "**Informative note:** `source` may be different from other data source(s) included elsewhere within the data" disclaimer: Description: "Characterization of accuracy, limitations, and applicability of this data" Data Type: "String" diff --git a/lattice/cpp_entries.py b/lattice/cpp/cpp_entries.py similarity index 71% rename from lattice/cpp_entries.py rename to lattice/cpp/cpp_entries.py index 8ad99f9..f032c44 100644 --- a/lattice/cpp_entries.py +++ b/lattice/cpp/cpp_entries.py @@ -2,13 +2,13 @@ HeaderEntry, Struct, DataElement, - DataIsSetElement, + DataElementIsSetFlag, DataElementStaticMetainfo, - MemberFunctionOverride, + MemberFunctionOverrideDeclaration, ObjectSerializationDeclaration, - CalculatePerformanceOverload, + InlineDependency, ) -from .util import snake_style +from lattice.util import snake_style from collections import defaultdict @@ -31,23 +31,19 @@ def __init__(self, name, parent=None): else: self._lineage = [name] - # ............................................................................................. def _add_child_entry(self, child): self._child_entries.append(child) - # ............................................................................................. def _get_level(self, level=0): if self._parent_entry: return self._parent_entry._get_level(level + 1) else: return level - # ............................................................................................. @property def level(self): return self._get_level() - # ............................................................................................. @property def value(self): entry = self.level * "\t" + self._type + " " + self._name + " " + " " + self._opener + "\n" @@ -74,7 +70,18 @@ def __init__(self, header_entry: DataElementStaticMetainfo, parent: Implementati ] ) - # ............................................................................................. + @property + def value(self): + entry = self.level * "\t" + self._func + "\n" + return entry + + +# ------------------------------------------------------------------------------------------------- +class DependencyInitialization(ImplementationEntry): + def __init__(self, header_entry: InlineDependency, parent: ImplementationEntry = None): + super().__init__(None, parent) + self._func = f"void set_{header_entry.name} ({header_entry.type} value) {{ {header_entry.name} = value; }}" + @property def value(self): entry = self.level * "\t" + self._func + "\n" @@ -87,7 +94,6 @@ def __init__(self, header_entry, parent=None): super().__init__(None, parent) self._func = f"void {header_entry.fname}{header_entry.args}" - # ............................................................................................. @property def value(self): entry = self.level * "\t" + self._func + " " + self._opener + "\n" @@ -99,14 +105,13 @@ def value(self): # ------------------------------------------------------------------------------------------------- class MemberFunctionDefinition(ImplementationEntry): - def __init__(self, header_entry: MemberFunctionOverride, parent: ImplementationEntry = None): + def __init__(self, header_entry: MemberFunctionOverrideDeclaration, parent: ImplementationEntry = None): super().__init__(None, parent) args = header_entry.args - if hasattr(header_entry, "args_as_list"): - args = "(" + ", ".join([a.split("=")[0] for a in header_entry.args_as_list]) + ")" - self._func = f"{header_entry.ret_type} {header_entry.parent.name}::{header_entry.fname}{args}" + if hasattr(header_entry, "_f_args"): + args = "(" + ", ".join([a.split("=")[0] for a in header_entry._f_args]) + ")" + self._func = f"{header_entry._f_ret} {header_entry.parent.name}::{header_entry._f_name}{args}" - # ............................................................................................. @property def value(self): entry = self.level * "\t" + self._func + " " + self._opener + "\n" @@ -122,7 +127,6 @@ def __init__(self, name, parent=None): super().__init__(name, parent) self._func = f"void from_json(const nlohmann::json& j, {name}& x)" - # ............................................................................................. @property def value(self): entry = self.level * "\t" + self._func + " " + self._opener + "\n" @@ -137,10 +141,9 @@ class ElementSerialization(ImplementationEntry): def __init__(self, parent, header_entry: DataElement): super().__init__(header_entry.name, parent) self._func = [ - f'json_get<{header_entry.type}>(j, "{self._name}", {self._name}, {self._name}_is_set, {"true" if header_entry.is_required else "false"});' + f'json_get<{header_entry.type}>(j, logger.get(), "{self._name}", {self._name}, {self._name}_is_set, {"true" if header_entry.is_required else "false"});' ] - # ............................................................................................. @property def value(self): entry = "" @@ -154,7 +157,7 @@ class OwnedElementSerialization(ElementSerialization): def __init__(self, parent, header_entry: DataElement): super().__init__(parent, header_entry) self._func = [ - f'json_get<{header_entry.type}>(j, "{self._name}", x.{self._name}, x.{self._name}_is_set, {"true" if header_entry.is_required else "false"});' + f'json_get<{header_entry.type}>(j, logger.get(), "{self._name}", x.{self._name}, x.{self._name}_is_set, {"true" if header_entry.is_required else "false"});' ] @@ -170,7 +173,7 @@ def __init__(self, parent, header_entry: DataElement): f"if (x.{data_element} == {enum}) {{", f"\tx.{self._name} = std::make_unique<{header_entry.selector[data_element][enum]}>();", f"\tif (x.{self._name}) {{", - f'\t\tx.{self._name}->initialize(j.at("{self._name}"));', + f'\t\tfrom_json(j.at("{self._name}"), *dynamic_cast<{header_entry.selector[data_element][enum]}*>(x.{self._name}.get()));', "\t}", "}", ] @@ -199,7 +202,6 @@ def __init__(self, parent, header_entry): super().__init__(parent, header_entry) self._func = "x.initialize(j);\n" - # ............................................................................................. @property def value(self): return self.level * "\t" + self._func @@ -214,7 +216,6 @@ def __init__(self, parent, header_entry, populates_self=False): else: self._func = f"x.{self._name}.populate_performance_map(&x);\n" - # ............................................................................................. @property def value(self): return self.level * "\t" + self._func @@ -226,7 +227,6 @@ def __init__(self, name, parent): super().__init__(name, parent) self._func = [f"add_grid_axis(performance_map, {name});\n"] - # ............................................................................................. @property def value(self): entry = "" @@ -241,7 +241,6 @@ def __init__(self, name, parent): super().__init__(name, parent) self._func = [f"performance_map->finalize_grid();\n"] - # ............................................................................................. @property def value(self): entry = "" @@ -256,7 +255,6 @@ def __init__(self, name, parent): super().__init__(name, parent) self._func = [f"add_data_table(performance_map, {name});\n"] - # ............................................................................................. @property def value(self): entry = "" @@ -265,22 +263,22 @@ def value(self): return entry -# ------------------------------------------------------------------------------------------------- -class PerformanceOverloadImplementation(ElementSerialization): - def __init__(self, header_entry, parent): - super().__init__(None, None, parent, None) - self._func = [] - args = ", ".join([f"{a[1]}" for a in [arg.split(" ") for arg in header_entry.args_as_list[:-1]]]) - self._func.append(f"std::vector target {{{args}}};") - self._func.append( - "auto v = PerformanceMapBase::calculate_performance(target, performance_interpolation_method);" - ) - init_str = f"{header_entry.ret_type} s {{" - for i in range(header_entry.n_return_values): - init_str += f"v[{i}], " - init_str += "};" - self._func.append(init_str) - self._func.append("return s;") +# # ------------------------------------------------------------------------------------------------- +# class PerformanceOverloadImplementation(ElementSerialization): +# def __init__(self, header_entry, parent): +# super().__init__(None, None, parent, None) +# self._func = [] +# args = ", ".join([f"{a[1]}" for a in [arg.split(" ") for arg in header_entry.args_as_list[:-1]]]) +# self._func.append(f"std::vector target {{{args}}};") +# self._func.append( +# "auto v = PerformanceMapBase::calculate_performance(target, performance_interpolation_method);" +# ) +# init_str = f"{header_entry.ret_type} s {{" +# for i in range(header_entry.n_return_values): +# init_str += f"v[{i}], " +# init_str += "};" +# self._func.append(init_str) +# self._func.append("return s;") # ------------------------------------------------------------------------------------------------- @@ -289,7 +287,6 @@ def __init__(self, name, parent): super().__init__(name, parent) self._func = f'return "{name}";' - # ............................................................................................. @property def value(self): entry = self.level * "\t" + self._func + "\n" @@ -306,7 +303,6 @@ def __init__(self): self._implementations["lookup_variables_base"] = DataTableImplementation self._implementations["performance_map_base"] = ElementSerialization - # ............................................................................................. def __str__(self): s = "" for p in self._preamble: @@ -316,7 +312,6 @@ def __str__(self): s += "\n" return s - # ............................................................................................. def translate(self, container_class_name, header_tree): """X""" self._add_included_headers(header_tree._schema_name) @@ -327,14 +322,17 @@ def translate(self, container_class_name, header_tree): self._get_items_to_serialize(header_tree.root) - # ............................................................................................. def _get_items_to_serialize(self, header_tree): for entry in header_tree.child_entries: # Shortcut to avoid creating "from_json" entries for the main class, but create them # for all other classes. The main class relies on an "Initialize" function instead, # dealt-with in the next block with function overrides. - if isinstance(entry, Struct) and entry.name not in self._namespace._name: - # Create the "from_json" function definition (header) + if ( + isinstance(entry, Struct) + and entry.name not in self._namespace._name + and len([c for c in entry.child_entries if isinstance(c, DataElement)]) + ): + # Create the "from_json" function definition (header), only if it won't be empty s = StructSerialization(entry.name, self._namespace) for data_element_entry in [c for c in entry.child_entries if isinstance(c, DataElement)]: # In function body, create each "get_to" for individual data elements @@ -349,12 +347,14 @@ def _get_items_to_serialize(self, header_tree): # Initialize static members if isinstance(entry, DataElementStaticMetainfo): DataElementStaticInitialization(entry, self._namespace) + if isinstance(entry, InlineDependency): + DependencyInitialization(entry, self._namespace) # Initialize and Populate overrides (Currently the only Member_function_override is the Initialize override) - if isinstance(entry, MemberFunctionOverride): + if isinstance(entry, MemberFunctionOverrideDeclaration): # Create the override function definition (header) using the declaration's signature m = MemberFunctionDefinition(entry, self._namespace) # Dirty hack workaround for Name() function - if "Name" in entry.fname: + if "Name" in entry._f_name: SimpleReturnProperty(entry.parent.name, m) else: # In function body, choose element-wise ops based on the superclass @@ -362,31 +362,31 @@ def _get_items_to_serialize(self, header_tree): if "unique_ptr" in data_element_entry.type: ClassFactoryCreation(m, data_element_entry) self._preamble.append(f"#include <{data_element_entry.name}_factory.h>\n") - else: - if entry.parent.superclass == "GridVariablesBase": - GridAxisImplementation(data_element_entry.name, m) - elif entry.parent.superclass == "LookupVariablesBase": - DataTableImplementation(data_element_entry.name, m) - elif entry.parent.superclass == "PerformanceMapBase": - ElementSerialization( - data_element_entry.name, data_element_entry.type, m, data_element_entry._is_required - ) - else: - ElementSerialization( - data_element_entry.name, data_element_entry.type, m, data_element_entry._is_required - ) - if entry.parent.superclass == "PerformanceMapBase": - PerformanceMapImplementation(data_element_entry.name, m, populates_self=True) - # Special case of grid_axis_base needs a finalize function after all grid axes - # are added - if entry.parent.superclass == "GridVariablesBase": - GridAxisFinalize("", m) - if isinstance(entry, CalculatePerformanceOverload): - m = MemberFunctionDefinition(entry, self._namespace) - for data_element_entry in [c for c in entry.parent.child_entries if isinstance(c, DataElement)]: - # Build internals of Calculate_performance function - if data_element_entry.name == "grid_variables": - PerformanceOverloadImplementation(entry, m) + # else: + # if entry.parent.superclass == "GridVariablesBase": + # GridAxisImplementation(data_element_entry.name, m) + # elif entry.parent.superclass == "LookupVariablesBase": + # DataTableImplementation(data_element_entry.name, m) + # elif entry.parent.superclass == "PerformanceMapBase": + # ElementSerialization( + # data_element_entry.name, data_element_entry.type, m, data_element_entry._is_required + # ) + # else: + # ElementSerialization( + # data_element_entry.name, data_element_entry.type, m, data_element_entry._is_required + # ) + # if entry.parent.superclass == "PerformanceMapBase": + # PerformanceMapImplementation(data_element_entry.name, m, populates_self=True) + # # Special case of grid_axis_base needs a finalize function after all grid axes + # # are added + # if entry.parent.superclass == "GridVariablesBase": + # GridAxisFinalize("", m) + # if isinstance(entry, CalculatePerformanceOverload): + # m = MemberFunctionDefinition(entry, self._namespace) + # for data_element_entry in [c for c in entry.parent.child_entries if isinstance(c, DataElement)]: + # # Build internals of Calculate_performance function + # if data_element_entry.name == "grid_variables": + # PerformanceOverloadImplementation(entry, m) # Lastly, handle the special case of objects that need both serialization # and initialization (currently a bit of a hack specific to this project) if isinstance(entry, ObjectSerializationDeclaration) and entry.name in self._namespace._name: @@ -395,7 +395,6 @@ def _get_items_to_serialize(self, header_tree): else: self._get_items_to_serialize(entry) - # ............................................................................................. def _add_included_headers(self, main_header): self._preamble.clear() self._preamble.append(f"#include <{snake_style(main_header)}.h>\n#include \n") diff --git a/lattice/cpp/generate_support_headers.py b/lattice/cpp/generate_support_headers.py deleted file mode 100644 index 349c1bf..0000000 --- a/lattice/cpp/generate_support_headers.py +++ /dev/null @@ -1,23 +0,0 @@ -from jinja2 import Template -import os -import sys -from lattice.file_io import dump -from lattice.util import snake_style -from pathlib import Path - - -def support_header_pathnames(output_directory: Path): - return [ - output_directory / "-".join(snake_style(template.stem).split("_")) - for template in Path(__file__).with_name("templates").iterdir() - ] - - -def generate_support_headers(namespace_name: str, root_data_groups: list[str], output_directory: Path): - for template in Path(__file__).with_name("templates").iterdir(): - enum_info = Template(template.read_text()) - generated_file_name = "-".join(snake_style(template.stem).split("_")) - dump( - enum_info.render(namespace=namespace_name, root_objects=root_data_groups), - Path(output_directory) / generated_file_name, - ) diff --git a/lattice/cpp/header_entries.py b/lattice/cpp/header_entries.py new file mode 100644 index 0000000..3b7a491 --- /dev/null +++ b/lattice/cpp/header_entries.py @@ -0,0 +1,414 @@ +from __future__ import annotations +import re +from lattice.util import snake_style +from typing import Callable, Optional +from pathlib import Path +from dataclasses import dataclass, field + + +def remove_prefix(text, prefix): + return text[len(prefix) :] if text.startswith(prefix) else text + + +data_group_extensions: dict[str, Callable] = {} + + +def register_data_group_operation(data_group_template_name: str, header_entry: Callable): + data_group_extensions[data_group_template_name] = header_entry + + +data_element_extensions: dict[str, Callable] = {} + + +def register_data_element_operation(data_group_template_name: str, header_entry: Callable): + data_element_extensions[data_group_template_name] = header_entry + + +# ------------------------------------------------------------------------------------------------- +@dataclass() +class HeaderEntryFormat: + _opener: str = field(init=False, default="{") + _closure: str = field(init=False, default="}") + _level: int = field(init=False, default=0) + + +# ------------------------------------------------------------------------------------------------- +@dataclass +class HeaderEntry(HeaderEntryFormat): + name: str + parent: Optional[HeaderEntry] + type: str = field(init=False, default="namespace") # TODO: kw_only=True? + child_entries: list[HeaderEntry] = field(init=False, default_factory=list) + + def __post_init__(self): + if self.parent: + self.parent._add_child_entry(self) + self._level = self._get_level() + + def _add_child_entry(self, child: HeaderEntry) -> None: + self.child_entries.append(child) + + def _get_level(self, level: int = 0) -> int: + if self.parent: + return self.parent._get_level(level + 1) + else: + return level + + def __lt__(self, other): + """ + A Header_entry must be "less than" any another Header_entry that references it, i.e. + you must define a C++ value before you reference it. + """ + return self._less_than(other) + + def _less_than(self, other): + """ """ + lt = False + t = f"{other.type} {other.name}" + # \b is a "boundary" character, or specifier for a whole word + if re.search(r"\b" + self.name + r"\b", t): + return True + for c in other.child_entries: + t = f"{c.type} {c.name}" + if re.search(r"\b" + self.name + r"\b", t): + # Shortcut around checking siblings; if one child matches, then self < other + return True + else: + # Check grandchildren + lt = self._less_than(c) + return lt + + def __str__(self): + tab = "\t" + entry = f"{self._level * tab}{self.type} {self.name} {self._opener}\n" + entry += "\n".join([str(c) for c in self.child_entries]) + entry += f"\n{self._level * tab}{self._closure}" + return entry + + +# ------------------------------------------------------------------------------------------------- +@dataclass +class Typedef(HeaderEntry): + _typedef: str + + def __post_init__(self): + super().__post_init__() + self.type = "typedef" + + def __str__(self): + tab = "\t" + return f"{self._level * tab}{self.type} {self._typedef} {self.name};" + + +# ------------------------------------------------------------------------------------------------- +@dataclass +class Enumeration(HeaderEntry): + _enumerants: dict + + def __post_init__(self): + super().__post_init__() + self.type = "enum class" + self._closure = "};" + + def __str__(self): + tab = "\t" + entry = f"{self._level * tab}{self.type} {self.name} {self._opener}\n" + for e in self._enumerants: + entry += f"{(self._level + 1) * tab}{e},\n" + entry += f"{(self._level + 1) * tab}UNKNOWN\n{self._level * tab}{self._closure}" + + # Incorporate an enum_info map into this object + map_type = f"const static std::unordered_map<{self.name}, enum_info>" + entry += f"\n" f"{self._level * tab}{map_type} {self.name}_info {self._opener}\n" + for e in self._enumerants: + display_text = self._enumerants[e].get("Display Text", e) + description = self._enumerants[e].get("Description") + entry += ( + f"{(self._level + 1) * tab}" f'{{{self.name}::{e}, {{"{e}", "{display_text}", "{description}"}}}},\n' + ) + entry += f"{(self._level + 1) * tab}" f'{{{self.name}::UNKNOWN, {{"UNKNOWN", "None", "None"}}}}\n' + entry += f"{self._level * tab}{self._closure}" + + return entry + + +# ------------------------------------------------------------------------------------------------- +@dataclass +class EnumSerializationDeclaration(HeaderEntry): + """Provides shortcut marcros that populate an enumeration from json.""" + + _enumerants: dict + + def __post_init__(self): + super().__post_init__() + self.type = "NLOHMANN_JSON_SERIALIZE_ENUM" + self._opener = "(" + self.name + ", {" + self._closure = "})" + + def __str__(self): + enums_with_placeholder = ["UNKNOWN"] + (list(self._enumerants.keys())) + tab = "\t" + entry = f"{self._level * tab}{self.type} {self._opener}\n" + entry += ",\n".join([f'{(self._level + 1) * tab}{{{self.name}::{e}, "{e}"}}' for e in enums_with_placeholder]) + # for e in enums_with_placeholder: + # entry += (self._level + 1) * "\t" + # mapping = "{" + self.name + "::" + e + ', "' + e + '"}' + # entry += mapping + ",\n" + entry += f"\n{self._level * tab}{self._closure}" + return entry + + +# ------------------------------------------------------------------------------------------------- +@dataclass +class Struct(HeaderEntry): + superclass: str = "" + + def __post_init__(self): + super().__post_init__() + self.type = "struct" + self._closure = "};" + + def __str__(self): + tab = "\t" + entry = f"{self._level * tab}{self.type} {self.name}" + if self.superclass: + entry += f" : {self.superclass}" + entry += f" {self._opener}\n" + entry += "\n".join([str(c) for c in self.child_entries]) + entry += f"\n{self._level * tab}{self._closure}" + return entry + + +# ------------------------------------------------------------------------------------------------- +@dataclass +class DataElement(HeaderEntry): + _element_attributes: dict + _pod_datatypes_map: dict[str, str] + _custom_datatypes_by_location: dict[str, list[str]] + _type_finder: Callable | None = None + selector: dict[str, dict[str, str]] = field(init=False, default_factory=dict) + + def __post_init__(self): + super().__post_init__() + self._closure = ";" + self.is_required: bool = self._element_attributes.get("Required", False) # used externally + self.scoped_innertype: tuple[str, str] = ("", "") + # self.external_reference_sources: list = [] + + self._create_type_entry(self._element_attributes, self._type_finder) + + def __str__(self): + tab = "\t" + return f"{self._level * tab}{self.type} {self.name}{self._closure}" + + def _create_type_entry(self, element_attributes: dict, type_finder: Callable | None = None) -> None: + """Create type node.""" + try: + # If the type is an array, extract the surrounding [] first (using non-greedy qualifier "?") + m = re.findall(r"\[(.*?)\]", element_attributes["Data Type"]) + if m: + self.type = f"std::vector<{self._get_scoped_inner_type(m[0])}>" + else: + # If the type is oneOf a set + m = re.match(r"\((?P.*)\)", element_attributes["Data Type"]) + if m: + # Choices can only be mapped to enums, so store the mapping for future use + # Constraints (of selection type) are of the form + # selection_key(ENUM_VAL_1, ENUM_VAL_2, ENUM_VAL_3) + # They connect pairwise with Data Type of the form ({Type_1}, {Type_2}, {Type_3}) + oneof_selection_key = element_attributes["Constraints"].split("(")[0] + if type_finder: + selection_key_type = ( + self._get_scoped_inner_type( + "".join(ch for ch in type_finder(oneof_selection_key) if ch.isalnum()) + ) + + "::" + ) + else: + selection_key_type = "" + selection_types = [ + self._get_scoped_inner_type(t.strip()) + for t in m.group("comma_separated_selection_types").split(",") + ] + m_opt = re.match(r".*\((?P.*)\)", element_attributes["Constraints"]) + if not m_opt: + raise TypeError + constraints = [ + (selection_key_type + s.strip()) for s in m_opt.group("comma_separated_constraints").split(",") + ] + + # the _selector dictionary would have a form like: + # { operation_speed_control_type : { CONTINUOUS : PerformanceMapContinuous, DISCRETE : PerformanceMapDiscrete} } + self.selector[oneof_selection_key] = dict(zip(constraints, selection_types)) + + # The elements of 'types' are Data Groups that derive from a Data Group Template. + # The template is a verbatim "base class," which is what makes the selector + # polymorphism possible + self.type = f"std::unique_ptr<{type_finder(selection_types[0]) if type_finder else None}>" + else: + # 1. 'type' entry + self.type = self._get_scoped_inner_type(element_attributes["Data Type"]) + except KeyError as ke: + pass + + def _get_scoped_inner_type(self, type_str: str) -> str: + """Return the scoped cpp type described by type_str. + + First, attempt to capture enum, definition, or special string type as references; + then default to fundamental types with simple key "type". + """ + enum_or_def = r"(\{|\<)(?P.*)(\}|\>)" + inner_type: str = "" + m = re.match(enum_or_def, type_str) + if m: + inner_type = m.group("inner_type") + else: + inner_type = type_str + # Look through the references to assign a scope to the type. 'location' is generally a + # schema name; its value will be a list of matchable data object names + for location in self._custom_datatypes_by_location: + if inner_type in self._custom_datatypes_by_location[location]: + self.scoped_innertype = (f"{snake_style(location)}", inner_type) + return "_ns::".join(self.scoped_innertype) + try: + # e.g. "Numeric/Null" or "Numeric" both ok + self.scoped_innertype = ("", self._pod_datatypes_map[type_str.split("/")[0]]) + return self.scoped_innertype[1] + except KeyError: + print("Type not processed:", type_str) + return f"Type not processed: {type_str}" + + def _get_simple_minmax(self, range_str, target_dict) -> None: + """Process Range into min and max fields.""" + if range_str is not None: + ranges = range_str.split(",") + minimum = None + maximum = None + if "type" not in target_dict: + target_dict["type"] = None + for r in ranges: + try: + numerical_value = re.findall(r"[+-]?\d*\.?\d+|\d+", r)[0] + if ">" in r: + minimum = float(numerical_value) if "number" in target_dict["type"] else int(numerical_value) + mn = "exclusiveMinimum" if "=" not in r else "minimum" + target_dict[mn] = minimum + elif "<" in r: + maximum = float(numerical_value) if "number" in target_dict["type"] else int(numerical_value) + mx = "exclusiveMaximum" if "=" not in r else "maximum" + target_dict[mx] = maximum + except ValueError: + pass + + +# ------------------------------------------------------------------------------------------------- +@dataclass +class DataElementIsSetFlag(HeaderEntry): + + def __str__(self): + tab = "\t" + return f"{self._level * tab}bool {self.name}_is_set;" + + +# ------------------------------------------------------------------------------------------------- +@dataclass +class DataElementStaticMetainfo(HeaderEntry): + element: dict + metainfo_key: str + + def __post_init__(self): + super().__post_init__() + self._type_specifier = "const static" + self.type = "std::string_view" + self.init_val = self.element.get(self.metainfo_key, "") if self.metainfo_key != "Name" else self.name + self.name = self.name + "_" + self.metainfo_key.lower() + self._closure = ";" + + def __str__(self): + tab = "\t" + return f"{self._level * tab}{self._type_specifier} {self.type} {self.name}{self._closure}" + + +# ------------------------------------------------------------------------------------------------- +@dataclass +class InlineDependency(HeaderEntry): + type: str # HeaderEntry does not initialize this in __init__, but InlineDependency will + + def __post_init__(self): + super().__post_init__() + self._type_specifier = "inline" + self._closure = ";" + + def __str__(self): + tab = "\t" + return ( + f"{self._level * tab}{self._type_specifier} {self.type} {self.name}{self._closure}" + "\n" + f"{self._level * tab}void set_{self.name}({self.type} value);" + ) + + +# ------------------------------------------------------------------------------------------------- +@dataclass +class FunctionalHeaderEntry(HeaderEntry): + _f_ret: str + _f_name: str + _f_args: list[str] + + def __post_init__(self): + super().__post_init__() + self.args = f"({', '.join(self._f_args)})" + self._closure = ";" + + def __str__(self): + tab = "\t" + return f"{self._level * tab}{' '.join([self._f_ret, self._f_name])}{self.args}{self._closure}" + + +# ------------------------------------------------------------------------------------------------- +@dataclass +class MemberFunctionOverrideDeclaration(FunctionalHeaderEntry): + def __post_init__(self): + super().__post_init__() + self._closure = " override;" + + +# ------------------------------------------------------------------------------------------------- +@dataclass +class ObjectSerializationDeclaration(FunctionalHeaderEntry): + _f_ret: str = field(init=False) + _f_name: str = field(init=False) + _f_args: list[str] = field(init=False) + + def __post_init__(self): + self._f_ret = "void" + self._f_name = "from_json" + self._f_args = ["const nlohmann::json& j", f"{self.name}& x"] + super().__post_init__() + + +# ------------------------------------------------------------------------------------------------- +@dataclass +class VirtualDestructor(FunctionalHeaderEntry): + _explicit_definition: Optional[str] = None + + def __post_init__(self): + self._closure = f" = {self._explicit_definition};" if self._explicit_definition else ";" + self._f_ret = "virtual" + self._f_name = f"~{self._f_name}" + self._f_args = [] + super().__post_init__() + + +# # ------------------------------------------------------------------------------------------------- +# class CalculatePerformanceOverload(FunctionalHeaderEntry): +# def __init__(self, f_ret, f_args, name, parent, n_return_values): +# super().__init__(f_ret, "calculate_performance", "(" + ", ".join(f_args) + ")", name, parent) +# self.args_as_list = f_args +# self.n_return_values = n_return_values + +# @property +# def value(self): +# complete_decl = self._level * "\t" + "using PerformanceMapBase::calculate_performance;\n" +# complete_decl += self._level * "\t" + " ".join([self.ret_type, self.fname, self.args]) + self._closure +# return complete_decl diff --git a/lattice/cpp/header_entry_extension_loader.py b/lattice/cpp/header_entry_extension_loader.py new file mode 100644 index 0000000..b5f4764 --- /dev/null +++ b/lattice/cpp/header_entry_extension_loader.py @@ -0,0 +1,31 @@ +from importlib import util +from pathlib import Path + +"""From https://github.com/ArjanCodes/2021-plugin-architecture/blob/main/after/game/loader.py""" + + +class ModuleInterface: + """Represents a plugin interface. A plugin has a single register function.""" + + @staticmethod + def register() -> None: + """Register the necessary items in the game character factory.""" + + +def import_module(path: Path) -> ModuleInterface: + """Imports a module given a path.""" + spec = util.spec_from_file_location(path.stem, path) + try: + module = util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + except: + raise ModuleNotFoundError + + +def load_extensions(from_path: Path) -> None: + """Loads the plugins defined in the plugins list.""" + if from_path.is_dir(): + for plugin_file in [x for x in from_path.iterdir() if x.suffix == ".py"]: + plugin = import_module(plugin_file) + plugin.register() diff --git a/lattice/cpp/header_translator.py b/lattice/cpp/header_translator.py new file mode 100644 index 0000000..0a04143 --- /dev/null +++ b/lattice/cpp/header_translator.py @@ -0,0 +1,358 @@ +import pathlib +from typing import Optional, Union + +from .header_entries import * +from lattice.file_io import load, get_base_stem +from lattice.util import snake_style, hyphen_separated_lowercase_style +import lattice.cpp.support_files as support + + +def modified_insertion_sort(obj_list): + """From https://stackabuse.com/sorting-algorithms-in-python/#insertionsort""" + swapped = False + # Start on the second element as we assume the first element is sorted + for i in range(1, len(obj_list)): + item_to_insert = obj_list[i] + # And keep a reference of the index of the previous element + j = i - 1 + # Move all items of the sorted segment forward if they are larger than the item to insert + while j >= 0 and any(obj > item_to_insert for obj in obj_list[0 : j + 1]): + obj_list[j + 1] = obj_list[j] + swapped = True + j -= 1 + # Insert the item + obj_list[j + 1] = item_to_insert + return swapped + + +class HeaderTranslator: + def __init__(self): + self._references: dict[str, list[str]] = {} + self._fundamental_data_types: dict[str, str] = {} + self._derived_types = {} + self._preamble = [] + self._doxynotes = "/// @note This class has been auto-generated. Local changes will not be saved!\n" + self._epilogue = [] + self._data_group_types = ["Data Group"] + self._forward_declaration_dir: Optional[pathlib.Path] = None + self._required_base_classes = [] + + def __str__(self): + s = "\n".join([p for p in self._preamble]) + s += f"\n\n{self._doxynotes}\n{self.root}\n" + s += "\n".join([e for e in self._epilogue]) + return s + + @property + def root(self): + return self._top_namespace + + # fmt: off + def translate(self, + input_file_path: pathlib.Path, + forward_declarations_path: pathlib.Path, + output_path: pathlib.Path, + top_namespace: str): + """Translate schema into C++ header file, but store locally as a data structure.""" + self._source_dir = input_file_path.parent.resolve() + self._forward_declaration_dir = forward_declarations_path + self._schema_name = get_base_stem(input_file_path) + self._references.clear() + self._derived_types.clear() + self._fundamental_data_types.clear() + self._preamble.clear() + self._epilogue.clear() + + self._contents = load(input_file_path) + + # Load meta info first (assuming that base level tag == Schema means object type == Meta) + self._load_meta_info(self._contents["Schema"]) + self._add_include_guard(snake_style(self._schema_name)) + self._add_standard_dependency_headers(self._contents["Schema"].get("References")) + + # Create "root" node(s) + self._top_namespace = HeaderEntry(top_namespace, None) + self._namespace = HeaderEntry(f"{snake_style(self._schema_name)}_ns", self._top_namespace) + + # First, assemble typedefs + for base_level_tag in [tag for tag in self._contents if self._contents[tag]["Object Type"] == "String Type"]: + Typedef(base_level_tag, self._namespace, "std::string") + + # Second, enumerations + for base_level_tag in self._list_objects_of_type("Enumeration"): + Enumeration(base_level_tag, self._namespace, self._contents[base_level_tag]["Enumerators"]) + + # Namespace-level dependencies + InlineDependency("logger", self._namespace, "std::shared_ptr") + + # Collect member objects and their children + for base_level_tag in self._list_objects_of_type("Meta"): + s = Struct(base_level_tag, self._namespace) + d = DataElementStaticMetainfo(base_level_tag.lower(), + s, + self._contents[base_level_tag], + "Title") + d = DataElementStaticMetainfo(base_level_tag.lower(), + s, + self._contents[base_level_tag], + "Version") + d = DataElementStaticMetainfo(base_level_tag.lower(), + s, + self._contents[base_level_tag], + "Description") + for base_level_tag in self._list_objects_of_type(self._data_group_types): + data_group_template = self._contents[base_level_tag].get("Data Group Template", "") + if data_group_template in data_group_extensions: + s = data_group_extensions[data_group_template]( + base_level_tag, + self._namespace, + superclass=data_group_template + ) + else: + s = Struct( + base_level_tag, + self._namespace, + superclass=data_group_template, + ) + self._add_header_dependencies(s, output_path) + # When there is a base class, add overrides: + self._add_function_overrides(s, output_path, data_group_template) + + # Process plugin code for the entire element group, if there is any + if data_group_template in data_element_extensions: + e = data_element_extensions[data_group_template]( + data_group_template, + s, + self._contents[base_level_tag]["Data Elements"] + ) + + # Per-element processing + for data_element in self._contents[base_level_tag]["Data Elements"]: + d = DataElement( + data_element, + s, + self._contents[base_level_tag]["Data Elements"][data_element], + self._fundamental_data_types, + self._references, + self._search_nodes_for_datatype, + ) + self._add_header_dependencies(d, output_path) + for data_element in self._contents[base_level_tag]["Data Elements"]: + d = DataElementIsSetFlag(data_element, s) + for data_element in self._contents[base_level_tag]["Data Elements"]: + d = DataElementStaticMetainfo( + data_element, + s, + self._contents[base_level_tag]["Data Elements"][data_element], + "Units" + ) + for data_element in self._contents[base_level_tag]["Data Elements"]: + d = DataElementStaticMetainfo( + data_element, + s, + self._contents[base_level_tag]["Data Elements"][data_element], + "Description" + ) + for data_element in self._contents[base_level_tag]["Data Elements"]: + d = DataElementStaticMetainfo( + data_element, + s, + self._contents[base_level_tag]["Data Elements"][data_element], + "Name" + ) + + modified_insertion_sort(self._namespace.child_entries) + # PerformanceMapBase object needs sibling grid/lookup vars to be created, so parse last + # self._add_performance_overloads() + + # Final passes through dictionary in order to add elements related to serialization + for base_level_tag in self._list_objects_of_type("Enumeration"): + EnumSerializationDeclaration(base_level_tag, + self._namespace, + self._contents[base_level_tag]["Enumerators"]) + for base_level_tag in self._list_objects_of_type(self._data_group_types): + # from_json declarations are necessary in top container, as the header-declared + # objects might be included and used from elsewhere. + ObjectSerializationDeclaration(base_level_tag, self._namespace) + + + def _load_meta_info(self, schema_section): + """Store the global/common types and the types defined by any named references.""" + self._root_data_group = schema_section.get("Root Data Group") + refs: dict[str, pathlib.Path] = { + f"{self._schema_name}": self._source_dir / f"{self._schema_name}.schema.yaml", + "core": pathlib.Path(__file__).parent.with_name("core.schema.yaml"), + } + if "References" in schema_section: + for ref in schema_section["References"]: + refs.update({f"{ref}": self._source_dir / f"{ref}.schema.yaml"}) + if (self._schema_name == "core" and + self._forward_declaration_dir and + self._forward_declaration_dir.is_dir()): + for file in self._forward_declaration_dir.iterdir(): + ref = get_base_stem(file) + refs.update({ref: file}) + + for ref_file in refs: + ext_dict = load(refs[ref_file]) + # Load every explicitly listed reference and collect the base classes therein + self._data_group_types.extend( + [name for name in ext_dict if ext_dict[name]["Object Type"] == "Data Group Template"] + ) + self._references[ref_file] = [ + name for name in ext_dict if ext_dict[name]["Object Type"] in self._data_group_types + ["Enumeration"] + ] + # For every reference listed, store all the derived-class/base-class pairs as dictionaries + self._derived_types[ref_file] = {name : ext_dict[name].get("Data Group Template") for name in ext_dict if ext_dict[name].get("Data Group Template")} + + cpp_types = {"integer": "int", + "string": "std::string", + "number": "double", + "boolean": "bool"} + for base_item in [name for name in ext_dict if ext_dict[name]["Object Type"] == "Data Type"]: + self._fundamental_data_types[base_item] = cpp_types[ext_dict[base_item]["JSON Schema Type"]] + for base_item in [name for name in ext_dict if ext_dict[name]["Object Type"] == "String Type"]: + self._fundamental_data_types[base_item] = "std::string" + + # print(self._schema_name) + # print("_references", self._references) + # print("_derived_types", self._derived_types) + + # fmt: on + + def _add_include_guard(self, header_name): + """Populate the file's include guards.""" + s1 = f"#ifndef {header_name.upper()}_H_" + s2 = f"#define {header_name.upper()}_H_" + s3 = f"#endif" + self._preamble.extend([s1, s2]) + self._epilogue.append(s3) + + def _add_standard_dependency_headers(self, ref_list): + """Populate a list of #includes that every lattice-based project will need.""" + if ref_list: + includes = "" + for r in ref_list: + include = f"#include <{hyphen_separated_lowercase_style(r)}.h>" + self._preamble.append(include) + self._preamble.extend( + [ + "#include ", + "#include ", + "#include ", + "#include ", + "#include ", + ] + ) + + def _add_header_dependencies(self, header_element, generated_header_path: pathlib.Path): + """Extract the dependency name from the data_element's type for included headers.""" + if isinstance(header_element, DataElement): + if "core_ns" in header_element.type: + self._add_member_includes("core") + if "unique_ptr" in header_element.type: + m = re.search(r"\<(?P.*)\>", header_element.type) + if m: + self._add_member_includes(m.group("base_class_type"), generated_header_path) + if header_element.scoped_innertype[0]: + # This piece captures any "forward-declared" types that need to be + # processed by the DataElement type-finding mechanism before their header is known. + self._add_member_includes(header_element.scoped_innertype[0]) + elif isinstance(header_element, Struct): + if header_element.superclass: + self._add_member_includes(header_element.superclass, generated_header_path) + + def _add_member_includes(self, dependency: str, generated_base_class_path: Optional[pathlib.Path] = None): + """ + Add the dependency to the list of included headers, and generate the header file if it's a base class. + """ + header_include = f"#include <{hyphen_separated_lowercase_style(dependency)}.h>" + if header_include not in self._preamble: + self._preamble.append(header_include) + if generated_base_class_path: + # self._required_base_classes.append(dependency) + support.generate_superclass_header(dependency, generated_base_class_path) + + # fmt: off + def _add_function_overrides(self, parent_node, output_path, base_class_name): + """Get base class virtual functions to be overridden.""" + base_class = pathlib.Path(output_path) / f"{hyphen_separated_lowercase_style(base_class_name)}.h" + try: + with open(base_class) as b: + for line in b: + if base_class_name not in line: + m = re.search(r"\s*virtual\s(?P.*)\s(?P.*)\((?P.*)\)", line) + if m: + MemberFunctionOverrideDeclaration("", + parent_node, + m.group("return_type"), + m.group("name"), + m.group("arguments").split(",")) + except: + pass + # fmt: on + + def _add_performance_overloads(self, parent_node=None): + """ """ + if not parent_node: + parent_node = self.root + for entry in parent_node.child_entries: + if entry.parent and entry.superclass == "PerformanceMapBase": + for lvstruct in [ + lv + for lv in entry.parent.child_entries + if lv.superclass == "LookupVariablesBase" + and remove_prefix(lv.name, "LookupVariables") == remove_prefix(entry.name, "PerformanceMap") + ]: + f_ret = f"{lvstruct.name}Struct" + n_ret = len([c for c in lvstruct.child_entries if isinstance(c, DataElement)]) + # for each performance map, find GridVariables sibling of PerformanceMap, that has a matching name + for gridstruct in [ + gridv + for gridv in entry.parent.child_entries + if gridv.superclass == "GridVariablesBase" + and remove_prefix(gridv.name, "GridVariables") == remove_prefix(entry.name, "PerformanceMap") + ]: + f_args = list() + for ce in [c for c in gridstruct.child_entries if isinstance(c, DataElement)]: + f_args.append(" ".join(["double", ce.name])) + f_args.append("Btwxt::Method performance_interpolation_method = Btwxt::Method::LINEAR") + CalculatePerformanceOverload(f_ret, f_args, "", entry, n_ret) + else: + self._add_performance_overloads(entry) + + def _search_references_for_base_types(self, data_element): + """ + Search the pre-populated derived-class list for base class type. Used + when a Data Type requested in a selector constraint is defined in a schema's references rather than in-file. + """ + data_element_unscoped = data_element.split(":")[-1] + for reference in self._derived_types: + if self._derived_types[reference].get(data_element_unscoped) in self._data_group_types: + return self._derived_types[reference][data_element_unscoped] + return None + + def _search_nodes_for_datatype(self, data_element) -> str: + """ + If data_element exists, return its data type; else return the data group's 'data type,' which + is the Data Group Template (base class type). Hacky overload. + """ + base_class_type = self._search_references_for_base_types(data_element) + if base_class_type: + return base_class_type + else: + for listing in self._contents: + if "Data Elements" in self._contents[listing]: + # if "Data Group Template" in self._contents[listing] and listing in data_element: + # return self._contents[listing]["Data Group Template"] + + for element in self._contents[listing]["Data Elements"]: + if element == data_element and "Data Type" in self._contents[listing]["Data Elements"][element]: + return self._contents[listing]["Data Elements"][element]["Data Type"] + return "MissingType" # Placeholder base class + + def _list_objects_of_type(self, object_type_or_list: Union[str, list[str]]) -> list: + if isinstance(object_type_or_list, str): + return [tag for tag in self._contents if self._contents[tag].get("Object Type") == object_type_or_list] + elif isinstance(object_type_or_list, list): + return [tag for tag in self._contents if self._contents[tag].get("Object Type") in object_type_or_list] diff --git a/lattice/cpp/support_files.py b/lattice/cpp/support_files.py new file mode 100644 index 0000000..0074e7c --- /dev/null +++ b/lattice/cpp/support_files.py @@ -0,0 +1,73 @@ +from jinja2 import Template +import os +import sys +from lattice.file_io import dump +from lattice.util import snake_style, hyphen_separated_lowercase_style +from pathlib import Path +import lattice.cpp.header_entries as header_entries + + +def support_header_pathnames(output_directory: Path): + """Return a list of the template-generated header file names.""" + return [ + output_directory / "-".join(snake_style(template.stem).split("_")) + for template in Path(__file__).with_name("templates").iterdir() + if ".h" in template.suffixes + ] + + +def render_support_headers(namespace_name: str, output_directory: Path): + """Generate the project-specific helper headers.""" + for template in Path(__file__).with_name("templates").iterdir(): + if ".h" in template.suffixes: + header = Template(template.read_text()) + generated_file_name = "-".join(snake_style(template.stem).split("_")) + dump( + header.render(namespace=namespace_name), + Path(output_directory) / generated_file_name, + ) + + +def render_build_files(project_name: str, submodules: list, output_directory: Path): + """Generate the project-specific CMakeLists files.""" + generated_file_name = "CMakeLists.txt" + + project_cmake_file = Path(__file__).with_name("templates") / "project-cmake.txt.j2" + if project_cmake_file.exists(): + cmake_project = Template(project_cmake_file.read_text()) + dump( + cmake_project.render(project_name=project_name), + Path(output_directory) / generated_file_name, + ) + src_cmake_file = Path(__file__).with_name("templates") / "project-src-cmake.txt.j2" + if src_cmake_file.exists(): + src_cmake = Template(src_cmake_file.read_text()) + dump( + src_cmake.render(project_name=project_name), + Path(output_directory) / "src" / generated_file_name, + ) + vendor_cmake_file = Path(__file__).with_name("templates") / "project-vendor-cmake.txt.j2" + if vendor_cmake_file.exists(): + vendor_cmake = Template(vendor_cmake_file.read_text()) + submodule_names = [Path(submodule).stem for submodule in submodules] + print(submodule_names) + dump( + vendor_cmake.render(submodules=submodule_names), + Path(output_directory) / "vendor" / generated_file_name, + ) + + +def generate_superclass_header(superclass: str, output_directory: Path): + s1 = f"#ifndef {superclass.upper()}_H_" + s2 = f"#define {superclass.upper()}_H_" + s3 = f"#endif" + + class_entry = header_entries.Struct(superclass, None) + # initialize_fn = InitializeFunction(None, class_entry) + dtor = header_entries.VirtualDestructor("", class_entry, "", superclass, []) + + superclass_contents = f"{s1}\n{s2}\n{class_entry}\n{s3}" + + header = Path(output_directory / f"{hyphen_separated_lowercase_style(superclass)}.h") + if not header.exists(): + dump(superclass_contents, header) diff --git a/lattice/cpp/templates/load-object.h.j2 b/lattice/cpp/templates/load-object.h.j2 index 97f6ce1..66fb7a5 100644 --- a/lattice/cpp/templates/load-object.h.j2 +++ b/lattice/cpp/templates/load-object.h.j2 @@ -4,7 +4,7 @@ #include #include #include -#include +#include namespace {{namespace}} { @@ -53,6 +53,7 @@ namespace {{namespace}} { template void json_get(nlohmann::json j, + Courier::Courier* logger, const char *subnode, T& object, bool& object_is_set, @@ -68,18 +69,10 @@ namespace {{namespace}} { object_is_set = false; if (required) { - logger.warning(ex.what()); + logger->send_warning(ex.what()); } } } - -{% for root_object in root_objects %} - {{root_object}} create_instance(const char* instance_file, {{root_object}}& object) - { - auto j = load_json(instance_file); - j.get<{{root_object}}>(object); - } -{% endfor %} } #endif \ No newline at end of file diff --git a/lattice/cpp/templates/project-cmake.txt.j2 b/lattice/cpp/templates/project-cmake.txt.j2 new file mode 100644 index 0000000..ec0dd7e --- /dev/null +++ b/lattice/cpp/templates/project-cmake.txt.j2 @@ -0,0 +1,32 @@ +cmake_minimum_required (VERSION 3.10) +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +project({{project_name}}) + +# Set a default build type if none was specified +if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) + message(STATUS "Setting build type to 'Release' as none was specified.") + set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build." FORCE) + # Set the possible values of build type for cmake-gui + set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" + "MinSizeRel" "RelWithDebInfo") +endif() + +find_package(Git QUIET) + +include(CTest) +set(JSON_BuildTests OFF CACHE INTERNAL "") + +option(${PROJECT_NAME}_BUILD_TESTING "Build ${PROJECT_NAME} testing targets" OFF) +option(${PROJECT_NAME}_COVERAGE "Add ${PROJECT_NAME} coverage reports" OFF) + +#set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH}) +#include(compiler-flags) + +add_subdirectory(src) +add_subdirectory(vendor) + +if (${PROJECT_NAME}_BUILD_TESTING) + enable_testing() + add_subdirectory(test) +endif() diff --git a/lattice/cpp/templates/project-src-cmake.txt.j2 b/lattice/cpp/templates/project-src-cmake.txt.j2 new file mode 100644 index 0000000..886512f --- /dev/null +++ b/lattice/cpp/templates/project-src-cmake.txt.j2 @@ -0,0 +1,28 @@ +file(GLOB lib_headers "${PROJECT_SOURCE_DIR}/include/{{project_name}}/*.h") +file(GLOB lib_src "${PROJECT_SOURCE_DIR}/src/*.cpp") + +set (sources "${lib_headers}" + "${lib_src}") + +option(${PROJECT_NAME}_STATIC_LIB "Make ${PROJECT_NAME} a static library" ON) + +if (${PROJECT_NAME}_STATIC_LIB) + add_library(${PROJECT_NAME} STATIC ${sources}) + set_target_properties(${PROJECT_NAME} PROPERTIES COMPILE_FLAGS "-D${PROJECT_NAME}_STATIC_DEFINE") +else () + set(CMAKE_MACOSX_RPATH 1) + add_library(${PROJECT_NAME} SHARED ${sources}) +endif () + +target_link_libraries({{project_name}} PUBLIC courier fmt nlohmann_json) +target_include_directories({{project_name}} PUBLIC ${PROJECT_SOURCE_DIR}/include/{{project_name}}) + +target_compile_options(${PROJECT_NAME} PRIVATE + $<$:/W4> + $<$,$,$>: + -Wall -Wextra -Wpedantic> + ) + +target_compile_features(${PROJECT_NAME} PRIVATE cxx_std_17) +include(GenerateExportHeader) +generate_export_header(${PROJECT_NAME}) diff --git a/lattice/cpp/templates/project-vendor-cmake.txt.j2 b/lattice/cpp/templates/project-vendor-cmake.txt.j2 new file mode 100644 index 0000000..535be23 --- /dev/null +++ b/lattice/cpp/templates/project-vendor-cmake.txt.j2 @@ -0,0 +1,40 @@ +#include(initialize-submodules) +#initialize_submodules() + +{% for module in submodules %} + +{% if "fmt" in module %} +if (NOT TARGET fmt) + add_subdirectory(fmt) + set(FMT_INSTALL OFF CACHE BOOL "" FORCE) + mark_as_advanced(FMT_CMAKE_DIR FMT_CUDA_TEST FMT_DEBUG_POSTFIX FMT_DOC FMT_FUZZ FMT_INC_DIR FMT_INSTALL FMT_INSTALL + FMT_LIB_DIR FMT_MODULE FMT_OS FMT_PEDANTIC FMT_PKGCONFIG_DIR FMT_SYSTEM_HEADERS FMT_TEST FMT_WERROR) +endif () + +{% elif "gtest" in module %} +if (${PROJECT_NAME}_BUILD_TESTING AND NOT TARGET gtest) + + # Prevent GoogleTest from overriding our compiler/linker options + # when building with Visual Studio + set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) + set(BUILD_GTEST ON CACHE BOOL "" FORCE MARK) + set(BUILD_GMOCK ON CACHE BOOL "" FORCE) + set(INSTALL_GTEST OFF CACHE BOOL "" FORCE) + mark_as_advanced(BUILD_GTEST BUILD_GMOCK INSTALL_GTEST) + add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/googletest) + +endif () + +{% elif "nlohmann" in module %} +if (NOT TARGET nlohmann_json) + add_subdirectory(json) +endif() + +{% else %} +if (NOT TARGET {{ module }}) + add_subdirectory({{module}}) + mark_as_advanced({{module}}_BUILD_TESTING {{module}}_COVERAGE {{module}}_STATIC_LIB) +endif () + +{% endif %} +{% endfor %} diff --git a/lattice/docs/mkdocs_web.py b/lattice/docs/mkdocs_web.py index 424387c..c2e3667 100644 --- a/lattice/docs/mkdocs_web.py +++ b/lattice/docs/mkdocs_web.py @@ -1,7 +1,7 @@ """Build web documentation""" from pathlib import Path -from distutils.dir_util import copy_tree # pylint: disable=deprecated-module +from shutil import copytree # pylint: disable=deprecated-module import shutil from urllib.parse import urlparse from typing import List @@ -345,7 +345,7 @@ def make_specification_page( # Process template process_template(template_path, output_path, schema_dir=schema_dir_path) else: - copy_tree(template_path, output_path) + copytree(template_path, output_path) title = get_file_basename(template_path, depth=2) diff --git a/lattice/docs/schema_table.py b/lattice/docs/schema_table.py index 0a8a23b..0d59323 100644 --- a/lattice/docs/schema_table.py +++ b/lattice/docs/schema_table.py @@ -29,9 +29,12 @@ def process_string_types(string_types): for str_typ in string_types: new_item = deepcopy(str_typ) if "Is Regex" in new_item and new_item["Is Regex"]: - new_item["JSON Schema Pattern"] = "(Not applicable)" - new_item["JSON Schema Pattern"] = ( - new_item["JSON Schema Pattern"].replace("*", r"\*").replace(r"(?", "\n" r"(?").replace(r"-[", "\n" r"-[") + new_item["Regular Expression Pattern"] = "(Not applicable)" + new_item["Regular Expression Pattern"] = ( + new_item["Regular Expression Pattern"] + .replace("*", r"\*") + .replace(r"(?", "\n" r"(?") + .replace(r"-[", "\n" r"-[") ) new_list.append(new_item) return new_list @@ -215,7 +218,7 @@ def string_types_table(string_types, caption=None, add_training_ws=True): RETURN: string, the table in Pandoc markdown grid table format """ return create_table_from_list( - columns=["String Type", "Description", "JSON Schema Pattern", "Examples"], + columns=["String Type", "Description", "Regular Expression Pattern", "Examples"], data_list=string_types, caption=caption, add_training_ws=add_training_ws, diff --git a/lattice/file_io.py b/lattice/file_io.py index 24d0bed..8f1c865 100644 --- a/lattice/file_io.py +++ b/lattice/file_io.py @@ -56,7 +56,7 @@ def dump(content, output_file_path): elif ext in [".yaml", ".yml"]: with open(output_file_path, "w", encoding="utf-8") as out_file: yaml.dump(content, out_file, sort_keys=False) - elif ext in [".h", ".cpp"]: + elif ext in [".h", ".cpp", ".txt"]: with open(output_file_path, "w", encoding="utf-8") as src: src.write(content) src.write("\n") diff --git a/lattice/header_entries.py b/lattice/header_entries.py index 5bc144f..1fdb11c 100644 --- a/lattice/header_entries.py +++ b/lattice/header_entries.py @@ -252,7 +252,7 @@ def _get_simple_type(self, type_str): First, attempt to capture enum, definition, or special string type as references; then default to fundamental types with simple key "type". """ - enum_or_def = r"(\{|\<)(.*)(\}|\>)" + enum_or_def = r"(\{|\<|:)(.*)(\}|\>|:)" internal_type = None nested_type = None m = re.match(enum_or_def, type_str) @@ -284,11 +284,7 @@ def _get_simple_type(self, type_str): return simple_type try: - if "/" in type_str: - # e.g. "Numeric/Null" - simple_type = self._datatypes[type_str.split("/")[0]] - else: - simple_type = self._datatypes[type_str] + simple_type = self._datatypes[type_str] except KeyError: print("Type not processed:", type_str) return simple_type diff --git a/lattice/lattice.py b/lattice/lattice.py index 9dc49be..d031e8e 100644 --- a/lattice/lattice.py +++ b/lattice/lattice.py @@ -2,121 +2,22 @@ import re import warnings +import os +import subprocess from fnmatch import fnmatch from pathlib import Path from typing import List, Union from jsonschema.exceptions import RefResolutionError -from lattice.docs.process_template import process_template from .file_io import check_dir, make_dir, load, dump, get_file_basename, get_base_stem from .meta_schema import generate_meta_schema, meta_validate_file from .schema_to_json import generate_json_schema, validate_file, postvalidate_file from .docs import MkDocsWeb, DocumentFile -from .header_entries import HeaderTranslator -from .cpp_entries import CPPTranslator -from lattice.cpp.generate_support_headers import generate_support_headers, support_header_pathnames - - -class SchemaFile: # pylint:disable=R0902 - """Parse the components of a schema file.""" - - def __init__(self, path: Path) -> None: - """Open and parse source schema""" - - self.path = Path(path).absolute() - self.file_base_name = get_base_stem(self.path) - self.schema_type = self.file_base_name # Overwritten if it is actually specified - - self._content: dict = load(self.path) - self._meta_schema_path: Path = None - self._json_schema_path: Path = None - self._root_data_group: str = None - - # Check for required content - if "Schema" not in self._content: - raise Exception(f'Required "Schema" object not found in schema file, "{self.path}".') - - self.schema_author = None - if "Root Data Group" in self._content["Schema"]: - self._root_data_group = self._content["Schema"]["Root Data Group"] - self.schema_type = self._root_data_group - if self._root_data_group in self._content: - # Get metadata - if "Data Elements" not in self._content[self._root_data_group]: - raise Exception(f'Root Data Group, "{self._root_data_group}" ' 'does not contain "Data Elements".') - if "metadata" in self._content[self._root_data_group]["Data Elements"]: - self._get_schema_constraints() - else: - pass # Warning? - else: - raise Exception( - f'Root Data Group, "{self._root_data_group}", ' f'not found in schema file, "{self.path}"' - ) - - # TODO: Version? # pylint: disable=fixme - - def _get_schema_constraints(self): - """Populate instance variables from schema constraints""" - - constraints = self._content[self._root_data_group]["Data Elements"]["metadata"].get("Constraints") - data_element_pattern = "([a-z]+)(_([a-z]|[0-9])+)*" - enumerator_pattern = "([A-Z]([A-Z]|[0-9])*)(_([A-Z]|[0-9])+)*" - constraint_pattern = re.compile( - f"^(?P{data_element_pattern})=(?P{enumerator_pattern})$" - ) - if not isinstance(constraints, list): - constraints = [constraints] - for constraint in [c for c in constraints if c]: - match = constraint_pattern.match(constraint) - if match: - if match.group(1) == "schema_author": - self.schema_author = match.group(5) - else: - pass # Warning? - - if match.group(1) == "schema": - self.schema_type = match.group(5) - else: - pass # Warning? - - if match.group("data_element") == "schema": - self.schema_type = match.group("enumerator") - else: - pass # Warning? - - @property - def meta_schema_path(self) -> Path: - """Path to this SchemaFile's validating metaschema""" - return self._meta_schema_path - - @meta_schema_path.setter - def meta_schema_path(self, meta_schema_path): - self._meta_schema_path = Path(meta_schema_path).absolute() - - @property - def json_schema_path(self) -> Path: - """Path to this SchemaFile as translated JSON""" - return self._json_schema_path - - @json_schema_path.setter - def json_schema_path(self, json_schema_path): - self._json_schema_path = Path(json_schema_path).absolute() - - @property - def cpp_header_path(self): # pylint:disable=C0116 - return self._cpp_header_path - - @cpp_header_path.setter - def cpp_header_path(self, value): - self._cpp_header_path = Path(value).absolute() - - @property - def cpp_source_path(self): # pylint:disable=C0116 - return self._cpp_source_path - - @cpp_source_path.setter - def cpp_source_path(self, value): - self._cpp_source_path = Path(value).absolute() +from lattice.docs.process_template import process_template +from .cpp.header_translator import HeaderTranslator +from .cpp.cpp_entries import CPPTranslator +import lattice.cpp.support_files as support +from lattice.schema import Schema class Lattice: # pylint:disable=R0902 @@ -128,7 +29,7 @@ def __init__( self, root_directory: Path = Path.cwd(), build_directory: Union[Path, None] = None, - build_output_directory_name: Path = Path(".lattice"), + build_output_directory_name: Union[Path, None] = Path(".lattice"), build_validation: bool = True, ) -> None: """Set up file structure""" @@ -182,10 +83,10 @@ def collect_schemas(self): self.schema_directory_path = self.root_directory # Collect list of schema files - self.schemas: List[SchemaFile] = [] + self.schemas: List[Schema] = [] for file_name in sorted(list(self.schema_directory_path.iterdir())): if fnmatch(file_name, "*.schema.yaml") or fnmatch(file_name, "*.schema.yml"): - self.schemas.append(SchemaFile(file_name)) + self.schemas.append(Schema(file_name)) if len(self.schemas) == 0: raise Exception(f'No schemas found in "{self.schema_directory_path}".') @@ -196,34 +97,34 @@ def setup_meta_schemas(self): self.meta_schema_directory = Path(self.build_directory) / "meta_schema" make_dir(self.meta_schema_directory) for schema in self.schemas: - meta_schema_path = self.meta_schema_directory / f"{schema.file_base_name}.meta.schema.json" + meta_schema_path = self.meta_schema_directory / f"{schema.name}.meta.schema.json" schema.meta_schema_path = meta_schema_path def generate_meta_schemas(self): """Generate metaschemas""" for schema in self.schemas: - generate_meta_schema(Path(schema.meta_schema_path), Path(schema.path)) + generate_meta_schema(Path(schema.meta_schema_path), Path(schema.file_path)) def validate_schemas(self): """Validate source schema using metaschema file""" for schema in self.schemas: - meta_validate_file(Path(schema.path), Path(schema.meta_schema_path)) + meta_validate_file(Path(schema.file_path), Path(schema.meta_schema_path)) def setup_json_schemas(self): """Set up json_schema subdirectory""" self.json_schema_directory = Path(self.build_directory) / "json_schema" make_dir(self.json_schema_directory) for schema in self.schemas: - json_schema_path = self.json_schema_directory / f"{schema.file_base_name}.schema.json" + json_schema_path = self.json_schema_directory / f"{schema.name}.schema.json" schema.json_schema_path = json_schema_path def generate_json_schemas(self): """Generate JSON schemas""" for schema in self.schemas: - generate_json_schema(schema.path, schema.json_schema_path) + generate_json_schema(schema.file_path, schema.json_schema_path) def validate_file(self, input_path, schema_type=None): """ @@ -235,8 +136,8 @@ def validate_file(self, input_path, schema_type=None): instance = load(input_path) if schema_type is None: if "metadata" in instance: - if "schema" in instance["metadata"]: - schema_type = instance["metadata"]["schema"] + if "schema_name" in instance["metadata"]: + schema_type = instance["metadata"]["schema_name"] if schema_type is None: if len(self.schemas) > 1: @@ -249,7 +150,7 @@ def validate_file(self, input_path, schema_type=None): else: # Find corresponding schema for schema in self.schemas: - if schema.schema_type == schema_type: + if schema.schema_name == schema_type: try: validate_file(input_path, schema.json_schema_path) postvalidate_file(input_path, schema.json_schema_path) @@ -322,29 +223,48 @@ def generate_web_documentation(self): def collect_cpp_schemas(self): """Collect source schemas into list of SchemaFiles""" - self.cpp_schemas = self.schemas + [SchemaFile(Path(__file__).with_name("core.schema.yaml"))] + self.cpp_schemas = self.schemas + [Schema(Path(__file__).with_name("core.schema.yaml"))] def setup_cpp_source_files(self): """Create directories for generated CPP source""" self.cpp_output_dir = Path(self.build_directory) / "cpp" make_dir(self.cpp_output_dir) + include_dir = make_dir(self.cpp_output_dir / "include") + self._cpp_output_include_dir = make_dir(include_dir / f"{self.root_directory.name}") + self._cpp_output_src_dir = make_dir(self.cpp_output_dir / "src") for schema in self.cpp_schemas: - schema.cpp_header_path = self.cpp_output_dir / f"{schema.file_base_name.lower()}.h" - schema.cpp_source_path = self.cpp_output_dir / f"{schema.file_base_name.lower()}.cpp" + schema.cpp_header_file_path = self._cpp_output_include_dir / f"{schema.name.lower()}.h" + schema.cpp_source_file_path = self._cpp_output_src_dir / f"{schema.name.lower()}.cpp" + + def setup_cpp_repository(self, submodules: list[str]): + """Initialize the CPP output directory as a Git repo.""" + cwd = os.getcwd() + os.chdir(self.cpp_output_dir) + subprocess.run(["git", "init"], check=True) + vendor_dir = make_dir("vendor") + os.chdir(vendor_dir) + # subprocess.run(["git", "remote", "add", "-f -t", "main", "--no-tags", "origin_atheneum", "https://github.com/bigladder/atheneum.git"]) + try: + for submodule in submodules: + subprocess.run(["git", "submodule", "add", submodule], check=False) + finally: + os.chdir(cwd) + os.chdir(cwd) + @property def cpp_support_headers(self) -> list[Path]: - return support_header_pathnames(self.cpp_output_dir) + """Wrap list of template-generated headers.""" + return support.support_header_pathnames(self.cpp_output_dir) - def generate_cpp_headers(self): - """Generate CPP header and source files""" + def generate_cpp_project(self, submodules: list[str]): + """Generate CPP header files, source files, and build support files.""" h = HeaderTranslator() c = CPPTranslator() - root_groups = [] for schema in self.cpp_schemas: - h.translate(schema.path, self.root_directory.name, self.schema_directory_path) - if h._root_data_group is not None: - root_groups.append(h._root_data_group) - dump(str(h), schema.cpp_header_path) + h.translate(schema.file_path, self.schema_directory_path, self._cpp_output_include_dir, self.root_directory.name) + dump(str(h), schema.cpp_header_file_path) c.translate(self.root_directory.name, h) - dump(str(c), schema.cpp_source_path) - generate_support_headers(self.root_directory.name, root_groups, self.cpp_output_dir) + dump(str(c), schema.cpp_source_file_path) + self.setup_cpp_repository(submodules) + support.render_support_headers(self.root_directory.name, self._cpp_output_include_dir) + support.render_build_files(self.root_directory.name, submodules, self.cpp_output_dir) diff --git a/lattice/meta.schema.yaml b/lattice/meta.schema.yaml index 0bbbb9d..6720be3 100644 --- a/lattice/meta.schema.yaml +++ b/lattice/meta.schema.yaml @@ -72,7 +72,7 @@ definitions: const: String Type Description: type: string - JSON Schema Pattern: + Regular Expression Pattern: type: string Examples: type: array @@ -167,7 +167,7 @@ definitions: pattern: "**GENERATED**" DataElementAttributes: type: object - properties: + properties: # TODO: Need to allow custom attributes Description: type: string Data Type: @@ -181,6 +181,8 @@ definitions: "$ref": meta.schema.json#/definitions/Required Notes: "$ref": meta.schema.json#/definitions/Notes + ID: + type: boolean DataGroup: type: object properties: diff --git a/lattice/meta_schema.py b/lattice/meta_schema.py index 0d8b060..66b1f68 100644 --- a/lattice/meta_schema.py +++ b/lattice/meta_schema.py @@ -11,7 +11,7 @@ class MetaSchema: def __init__(self, schema_path): - with open(schema_path) as meta_schema_file: + with open(schema_path, encoding="utf-8") as meta_schema_file: uri_path = os.path.abspath(os.path.dirname(schema_path)) if os.sep != posixpath.sep: uri_path = posixpath.sep + uri_path @@ -20,7 +20,8 @@ def __init__(self, schema_path): self.validator = jsonschema.Draft7Validator(json.load(meta_schema_file), resolver=resolver) def validate(self, instance_path): - instance = load(instance_path) + with open(os.path.join(instance_path), "r", encoding="utf-8") as input_file: + instance = yaml.load(input_file, Loader=yaml.FullLoader) errors = sorted(self.validator.iter_errors(instance), key=lambda e: e.path) file_name = os.path.basename(instance_path) if len(errors) == 0: @@ -128,12 +129,11 @@ def generate_meta_schema(output_path, schema=None): meta_schema["patternProperties"][schema_patterns.type_base_names.anchored()]["allOf"].append( { "if": {"properties": {"Object Type": {"const": "Data Group"}, "Data Group Template": True}}, - "then": {"$ref": f"meta.schema.json#/definitions/DataGroup"}, + "then": {"$ref": "meta.schema.json#/definitions/DataGroup"}, } ) - for data_group_template_name in data_group_templates: - data_group_template = data_group_templates[data_group_template_name] + for data_group_template_name, data_group_template in data_group_templates.items(): meta_schema["definitions"][f"{data_group_template_name}DataElementAttributes"] = copy.deepcopy( meta_schema["definitions"]["DataElementAttributes"] ) @@ -244,9 +244,9 @@ def generate_meta_schema(output_path, schema=None): dump(meta_schema, output_path) - with open(output_path, "r") as file: + with open(output_path, "r", encoding="utf-8") as file: content = file.read() - with open(output_path, "w") as file: + with open(output_path, "w", encoding="utf-8") as file: file.writelines(content.replace("meta.schema.json", meta_schema_file_name)) return schema_patterns diff --git a/lattice/schema.py b/lattice/schema.py index 0880112..e58f019 100644 --- a/lattice/schema.py +++ b/lattice/schema.py @@ -1,7 +1,7 @@ from __future__ import ( annotations, ) # Needed for type hinting classes that are not yet fully defined -from typing import List +from typing import List, Union, Type, Dict, Any import pathlib import re @@ -18,14 +18,14 @@ def __init__(self, pattern_string: str) -> None: def __str__(self): return self.pattern.pattern - def match(self, test_string: str, anchored: bool = False): + def match(self, test_string: str, anchored: bool = False) -> Union[re.Match[str], None]: return self.pattern.match(test_string) if not anchored else self.anchored_pattern.match(test_string) def anchored(self): return self.anchored_pattern.pattern @staticmethod - def anchor(pattern_text: str): + def anchor(pattern_text: str) -> str: return f"^{pattern_text}$" @@ -37,7 +37,10 @@ def anchor(pattern_text: str): class DataType: - def __init__(self, text, parent_data_element: DataElement): + pattern: RegularExpressionPattern + value_pattern: RegularExpressionPattern + + def __init__(self, text: str, parent_data_element: DataElement): self.text = text self.parent_data_element = parent_data_element @@ -78,16 +81,14 @@ class PatternType(DataType): value_pattern = RegularExpressionPattern('".*"') -class ArrayType(DataType): - pattern = RegularExpressionPattern(r"\[(\S+)]") - - class DataGroupType(DataType): pattern = RegularExpressionPattern(rf"\{{({_type_base_names})\}}") def __init__(self, text, parent_data_element): super().__init__(text, parent_data_element) - self.data_group_name = self.pattern.match(text).group(1) + match = self.pattern.match(text) + assert match is not None + self.data_group_name = match.group(1) self.data_group = None # only valid once resolve() is called def resolve(self): @@ -103,6 +104,14 @@ class AlternativeType(DataType): pattern = RegularExpressionPattern(r"\(([^\s,]+)((, ?([^\s,]+))+)\)") +class ReferenceType(DataType): + pattern = RegularExpressionPattern(rf":{_type_base_names}:") + + +class ArrayType(DataType): + pattern = RegularExpressionPattern(rf"\[({_type_base_names}|{DataGroupType.pattern}|{EnumerationType.pattern})\]") + + _value_pattern = RegularExpressionPattern( f"(({NumericType.value_pattern})|" f"({StringType.value_pattern})|" @@ -113,6 +122,8 @@ class AlternativeType(DataType): # Constraints class Constraint: + pattern: RegularExpressionPattern + def __init__(self, text: str, parent_data_element: DataElement): self.text = text self.parent_data_element = parent_data_element @@ -154,6 +165,7 @@ def __init__(self, text: str, parent_data_element: DataElement): super().__init__(text, parent_data_element) self.pattern = parent_data_element.parent_data_group.parent_schema.schema_patterns.data_element_value_constraint match = self.pattern.match(self.text) + assert match is not None self.data_element_name = match.group(1) # TODO: Named groups? self.data_element_value = match.group(5) # TODO: Named groups? @@ -162,7 +174,7 @@ class ArrayLengthLimitsConstraint(Constraint): pattern = RegularExpressionPattern(r"\[(\d*)\.\.(\d*)\]") -_constraint_list: List[Constraint] = [ +_constraint_list: List[Type[Constraint]] = [ RangeConstraint, MultipleConstraint, SetConstraint, @@ -173,19 +185,18 @@ class ArrayLengthLimitsConstraint(Constraint): ] -def _constraint_factory(input: str, parent_data_element: DataElement) -> Constraint: +def _constraint_factory(text: str, parent_data_element: DataElement) -> Constraint: number_of_matches = 0 - match_type = None for constraint in _constraint_list: - if constraint.pattern.match(input): + if constraint.pattern.match(text): match_type = constraint number_of_matches += 1 if number_of_matches == 1: - return match_type(input, parent_data_element) + return match_type(text, parent_data_element) if number_of_matches == 0: - raise Exception(f"No matching constraint for {input}.") - raise Exception(f"Multiple matches found for constraint, {input}") + raise Exception(f"No matching constraint for {text}.") + raise Exception(f"Multiple matches found for constraint, {text}") # Required @@ -198,7 +209,8 @@ def __init__(self, name: str, data_element_dictionary: dict, parent_data_group: self.name = name self.dictionary = data_element_dictionary self.parent_data_group = parent_data_group - self.constraints = [] + self.constraints: List[Constraint] = [] + self.is_id = False for attribute in self.dictionary: if attribute == "Description": self.description = self.dictionary[attribute] @@ -214,6 +226,16 @@ def __init__(self, name: str, data_element_dictionary: dict, parent_data_group: self.required = self.dictionary[attribute] elif attribute == "Notes": self.notes = self.dictionary[attribute] + elif attribute == "ID": + self.is_id = self.dictionary[attribute] + if self.is_id: + if self.parent_data_group.id_data_element is None: + self.parent_data_group.id_data_element = self + else: + raise RuntimeError( + f"Multiple ID data elements found for Data Group '{self.parent_data_group.name}': '{self.parent_data_group.id_data_element.name}' and '{self.name}'" + ) + else: raise Exception( f'Unrecognized attribute, "{attribute}".' @@ -222,7 +244,7 @@ def __init__(self, name: str, data_element_dictionary: dict, parent_data_group: f"Data Element={self.name}" ) - def set_constraints(self, constraints_input): + def set_constraints(self, constraints_input: Union[str, List[str]]) -> None: if not isinstance(constraints_input, list): constraints_input = [constraints_input] @@ -245,7 +267,7 @@ def __init__(self, name: str, string_type_dictionary: dict, parent_schema: Schem self.name = name self.dictionary = string_type_dictionary self.parent_schema = parent_schema - self.value_pattern = self.dictionary["JSON Schema Pattern"] + self.value_pattern = self.dictionary["Regular Expression Pattern"] # Make new DataType class def init_method(self, text, parent_data_element): @@ -265,11 +287,12 @@ def init_method(self, text, parent_data_element): class DataGroup: - def __init__(self, name: str, data_group_dictionary, parent_schema: Schema): + def __init__(self, name: str, data_group_dictionary: dict, parent_schema: Schema): self.name = name self.dictionary = data_group_dictionary self.parent_schema = parent_schema self.data_elements = {} + self.id_data_element: Union[DataElement, None] = None # data element containing unique id for this data group for data_element in self.dictionary["Data Elements"]: self.data_elements[data_element] = DataElement( data_element, self.dictionary["Data Elements"][data_element], self @@ -283,14 +306,14 @@ def resolve(self): class Enumerator: pattern = EnumerationType.value_pattern - def __init__(self, name, enumerator_dictionary, parent_enumeration: Enumeration) -> None: + def __init__(self, name: str, enumerator_dictionary: dict, parent_enumeration: Enumeration): self.name = name self.dictionary = enumerator_dictionary self.parent_enumeration = parent_enumeration class Enumeration: - def __init__(self, name, enumeration_dictionary, parent_schema: Schema): + def __init__(self, name: str, enumeration_dictionary: dict, parent_schema: Schema): self.name = name self.dictionary = enumeration_dictionary self.parent_schema = parent_schema @@ -329,8 +352,8 @@ def __init__(self, schema=None): regex_base_types = core_types["Data Type"] - base_types = "|".join(regex_base_types) - base_types = RegularExpressionPattern(f"({base_types})") + base_types_string = "|".join(regex_base_types) + base_types = RegularExpressionPattern(f"({base_types_string})") string_types = core_types["String Type"] if schema: @@ -339,39 +362,41 @@ def __init__(self, schema=None): if "String Type" in schema_types: string_types += ["String Type"] - re_string_types = "|".join(string_types) - re_string_types = RegularExpressionPattern(f"({re_string_types})") + re_string_types_string = "|".join(string_types) + re_string_types = RegularExpressionPattern(f"({re_string_types_string})") self.data_group_types = DataGroupType.pattern self.enumeration_types = EnumerationType.pattern - single_type = rf"({base_types}|{re_string_types}|{self.data_group_types}|{self.enumeration_types})" + references = ReferenceType.pattern + single_type = rf"({base_types}|{re_string_types}|{self.data_group_types}|{self.enumeration_types}|{references})" alternatives = rf"\(({single_type})(,\s*{single_type})+\)" - arrays = rf"\[({single_type})\](\[\d*\.*\d*\])?" + arrays = ArrayType.pattern self.data_types = RegularExpressionPattern(f"({single_type})|({alternatives})|({arrays})") # Values self.values = RegularExpressionPattern( - f"(({self.numeric})|" f"({self.string})|" f"({self.enumerator})|" f"({self.boolean}))" + f"(({self.numeric})|({self.string})|({self.enumerator})|({self.boolean}))" ) # Constraints - alpha_array = "([A-Z]{[1-9]+})" - numeric_array = "([0-9]{[1-9]+})" self.range_constraint = RangeConstraint.pattern self.multiple_constraint = MultipleConstraint.pattern self.data_element_value_constraint = DataElementValueConstraint.pattern sets = SetConstraint.pattern reference_scope = f":{_type_base_names}:" self.selector_constraint = SelectorConstraint.pattern + array_limits = ArrayLengthLimitsConstraint.pattern + string_patterns = StringPatternConstraint.pattern self.constraints = RegularExpressionPattern( - f"({alpha_array}|{numeric_array}|{self.range_constraint})|" # pylint:disable=C0301 + f"({self.range_constraint})|" # pylint:disable=C0301 f"({self.multiple_constraint})|" f"({sets})|" f"({self.data_element_value_constraint})|" f"({reference_scope})|" f"({self.selector_constraint})|" - f"({StringPatternConstraint.pattern})" + f"({array_limits})|" + f"({string_patterns})" ) # Conditional Requirements @@ -395,7 +420,7 @@ def __init__(self, file_path: pathlib.Path, parent_schema: Schema | None = None) self.data_groups = {} self.data_group_templates = {} - self._data_type_list = [ + self._data_type_list: List[Type[DataType]] = [ IntegerType, NumericType, BooleanType, @@ -405,6 +430,7 @@ def __init__(self, file_path: pathlib.Path, parent_schema: Schema | None = None) DataGroupType, EnumerationType, AlternativeType, + ReferenceType, ] self.schema_patterns = SchemaPatterns(self.source_dictionary) @@ -445,10 +471,10 @@ def __init__(self, file_path: pathlib.Path, parent_schema: Schema | None = None) self.root_data_group = None self.metadata = None self.schema_author = None - self.schema_type = self.name + self.schema_name = self.name if self.root_data_group_name is not None: - self.schema_type = self.root_data_group_name + self.schema_name = self.root_data_group_name self.root_data_group = self.get_data_group(self.root_data_group_name) self.metadata = ( self.root_data_group.data_elements["metadata"] @@ -461,7 +487,7 @@ def __init__(self, file_path: pathlib.Path, parent_schema: Schema | None = None) if constraint.data_element_name == "schema_author": self.schema_author = constraint.data_element_value elif constraint.data_element_name == "schema": - self.schema_type = constraint.data_element_value + self.schema_name = constraint.data_element_value for data_group in self.data_groups.values(): data_group.resolve() @@ -488,7 +514,7 @@ def set_reference_schema(self, schema_name, schema_path): else: self.reference_schemas[schema_name] = Schema(schema_path, self) - def get_reference_schema(self, schema_name) -> Schema | None: + def get_reference_schema(self, schema_name: str) -> Schema | None: # TODO: verify schema has the same path too? # Search this schema first if schema_name in self.reference_schemas: @@ -500,7 +526,7 @@ def get_reference_schema(self, schema_name) -> Schema | None: return None - def get_data_group(self, data_group_name: str): + def get_data_group(self, data_group_name: str) -> DataGroup: matching_schemas = [] # 1. Search this schema first if data_group_name in self.data_groups: @@ -514,22 +540,21 @@ def get_data_group(self, data_group_name: str): return matching_schemas[0].data_groups[data_group_name] - def data_type_factory(self, input: str, parent_data_element: DataElement) -> DataType: + def data_type_factory(self, text: str, parent_data_element: DataElement) -> DataType: number_of_matches = 0 - match_type = None for data_type in self._data_type_list: - if data_type.pattern.match(input): + if data_type.pattern.match(text): match_type = data_type number_of_matches += 1 if number_of_matches == 1: - return match_type(input, parent_data_element) + return match_type(text, parent_data_element) if number_of_matches == 0: - raise Exception(f"No matching data type for {input}.") + raise Exception(f"No matching data type for {text}.") else: - raise Exception(f"Multiple matches found for data type, {input}") + raise Exception(f"Multiple matches found for data type, {text}") - def add_data_type(self, data_type: DataType): + def add_data_type(self, data_type: Type[DataType]) -> None: if data_type not in self._data_type_list: self._data_type_list.append(data_type) @@ -539,9 +564,9 @@ def add_data_type(self, data_type: DataType): def get_types(schema): """For each Object Type in a schema, map a list of Objects matching that type.""" - types = {} - for object in schema: - if schema[object]["Object Type"] not in types: - types[schema[object]["Object Type"]] = [] - types[schema[object]["Object Type"]].append(object) + types: Dict[str, Any] = {} + for object_name in schema: + if schema[object_name]["Object Type"] not in types: + types[schema[object_name]["Object Type"]] = [] + types[schema[object_name]["Object Type"]].append(object_name) return types diff --git a/lattice/schema_to_json.py b/lattice/schema_to_json.py index 01a7a16..f78c60f 100644 --- a/lattice/schema_to_json.py +++ b/lattice/schema_to_json.py @@ -25,7 +25,7 @@ class DataGroup: # pylint: disable=R0903 # Parse ellipsis range-notation e.g. '[1..]' minmax_range_type = r"(?P[0-9]*)(?P\.*)(?P[0-9]*)" - enum_or_def = r"(\{|\<)(.*)(\}|\>)" + enum_or_def = r"(\{|\<|:)(.*)(\}|\>|:)" numeric_type = r"[+-]?[0-9]*\.?[0-9]+|[0-9]+" # Any optionally signed, floating point number scope_constraint = r"^:(?P.*):" # Lattice scope constraint for ID/Reference ranged_array_type = rf"{array_type}(\[{minmax_range_type}\])?" @@ -179,11 +179,7 @@ def _get_simple_type(self, type_str, target_dict_to_append): target_dict_to_append["$ref"] = internal_type return try: - if "/" in type_str: - # e.g. "Numeric/Null" becomes a list of 'type's - target_dict_to_append["type"] = [self._types[t] for t in type_str.split("/")] - else: - target_dict_to_append["type"] = self._types[type_str] + target_dict_to_append["type"] = self._types[type_str] except KeyError: raise KeyError( f"Unknown type: {type_str} does not appear in referenced schema " @@ -428,7 +424,7 @@ def __init__(self, input_file_path: Path, forward_declaration_dir: Optional[Path { tag: { "type": "string", - "pattern": entry["JSON Schema Pattern"], + "pattern": entry["Regular Expression Pattern"], } } ) diff --git a/test/test_schema_patterns.py b/test/test_schema_patterns.py index 2aa5d01..2a0ece4 100644 --- a/test/test_schema_patterns.py +++ b/test/test_schema_patterns.py @@ -47,9 +47,8 @@ def test_data_type_pattern(): "Numeric", "[Numeric]", "{DataGroup}", - "[String][1..]", ], - invalid_examples=["Wrong"], + invalid_examples=["Wrong", "[String][1..]", "ID"], )