fix: Multiple

- Make module paths w.r.t `/` and not `drishti/`
- Ensure Python 3.8 compatibility

Author: jay-tau

drishti/handlers/handle_darshan.py

@@ -11,22 +11,24 @@
 import sys
 import time
 import typing
+from typing import Dict
 from dataclasses import dataclass, field
 from typing import List, Optional
 
 import darshan
 import darshan.backend.cffi_backend as darshanll
-import includes.config as config
-import includes.module as module
 import pandas as pd
-from includes.module import HIGH, RECOMMENDATIONS, WARN, Panel, insights_total
-
-# from includes.module import *
-from includes.parser import args
 from packaging import version
 from rich import print
 from rich.padding import Padding
 
+import drishti.includes.config as config
+import drishti.includes.module as module
+from drishti.includes.module import HIGH, RECOMMENDATIONS, WARN, Panel, insights_total
+
+# from includes.module import *
+from drishti.includes.parser import args
+
 
 def is_available(name):
     """Check whether `name` is on PATH and marked as executable."""
@@ -98,7 +100,7 @@ class AbstractDarshanTrace(abc.ABC):
 
     # Report
     modules: typing.Iterable[str]
-    name_records: dict[str, str] = field(default_factory=dict)
+    name_records: Dict[str, str] = field(default_factory=dict)
 
     max_read_offset: int = float('-inf')
     max_write_offset: int = float('-inf')
@@ -108,7 +110,7 @@ class AbstractDarshanTrace(abc.ABC):
     total_files_posix: int = 0
     total_files_mpiio: int = 0
 
-    files: dict[str, dict[str, int]] = None
+    files: Dict[str, Dict[str, int]] = None
 
     total_reads: int = 0
     total_writes: int = 0
@@ -165,7 +167,7 @@ class AbstractDarshanTrace(abc.ABC):
     # TODO: Should be a list of CB nodes for agg
     cb_nodes: Optional[int] = None
     number_of_compute_nodes: int = 0
-    hints: list[str] = dataclasses.field(default_factory=list)
+    hints: List[str] = dataclasses.field(default_factory=list)
 
     job_start: Optional[datetime.datetime] = None
     job_end: Optional[datetime.datetime] = None
@@ -346,61 +348,62 @@ def _do_something(self):
         # TODO: for trace in traces
         for trace in self.traces:
             pass
-        module.check_misaligned(self.total_operations, self.total_mem_not_aligned, self.total_file_not_aligned,
-                                self.modules, self.name_records, self.lustre_df, self.dxt_posix,
-                                self.dxt_posix_read_data) # posix alignment
+        # module.check_misaligned(self.total_operations, self.total_mem_not_aligned, self.total_file_not_aligned,
+        #                         self.modules, self.name_records, self.lustre_df, self.dxt_posix,
+        #                         self.dxt_posix_read_data) # posix alignment
 
-        module.check_traffic(self.max_read_offset, self.total_read_size, self.max_write_offset, self.total_written_size,
-                             self.dxt_posix, self.dxt_posix_read_data, self.dxt_posix_write_data) # redundant reads
+        # module.check_traffic(self.max_read_offset, self.total_read_size, self.max_write_offset, self.total_written_size,
+        #                      self.dxt_posix, self.dxt_posix_read_data, self.dxt_posix_write_data) # redundant reads
 
-        module.check_random_operation(self.read_consecutive, self.read_sequential, self.read_random, self.total_reads,
-                                      self.write_consecutive, self.write_sequential, self.write_random,
-                                      self.total_writes, self.dxt_posix,
-                                      self.dxt_posix_read_data, self.dxt_posix_write_data) # random check
+        # module.check_random_operation(self.read_consecutive, self.read_sequential, self.read_random, self.total_reads,
+        #                               self.write_consecutive, self.write_sequential, self.write_random,
+        #                               self.total_writes, self.dxt_posix,
+        #                               self.dxt_posix_read_data, self.dxt_posix_write_data) # random check
 
-        module.check_shared_small_operation(self.total_shared_reads, self.total_shared_reads_small,
-                                            self.total_shared_writes,
-                                            self.total_shared_writes_small, self.shared_files, self.report.name_records)
+        # module.check_shared_small_operation(self.total_shared_reads, self.total_shared_reads_small,
+        #                                     self.total_shared_writes,
+        #                                     self.total_shared_writes_small, self.shared_files, self.report.name_records)
 
         module.check_long_metadata(self.count_long_metadata, self.modules)
 
-        module.check_shared_data_imblance(self.posix_shared_data_imbalance_stragglers_count,
-                                          self.posix_data_straggler_files,
-                                          self.report.name_records, self.dxt_posix,
-                                          self.dxt_posix_read_data,
-                                          self.dxt_posix_write_data)
-
-        module.check_shared_time_imbalance(self.posix_stragglers_shared_file_time_imbalance_count,
-                                           self.posix_shared_time_imbalance_detected_files1, self.report.name_records)
-
-        module.check_individual_write_imbalance(self.posix_data_imbalance_count,
-                                                self.posix_shared_time_imbalance_detected_files2,
-                                                self.report.name_records, self.dxt_posix, self.dxt_posix_write_data)
+        # module.check_shared_data_imblance(self.posix_shared_data_imbalance_stragglers_count,
+        #                                   self.posix_data_straggler_files,
+        #                                   self.report.name_records, self.dxt_posix,
+        #                                   self.dxt_posix_read_data,
+        #                                   self.dxt_posix_write_data)
 
-        module.check_mpi_collective_read_operation(self.mpiio_coll_reads, self.mpiio_indep_reads,
-                                                   self.total_mpiio_read_operations,
-                                                   self.detected_files_mpi_coll_reads, self.report.name_records,
-                                                   self.dxt_mpiio)
+        # module.check_shared_time_imbalance(self.posix_stragglers_shared_file_time_imbalance_count,
+        #                                    self.posix_shared_time_imbalance_detected_files1, self.report.name_records)
 
-        module.check_mpi_collective_write_operation(self.mpiio_coll_writes, self.mpiio_indep_writes,
-                                                    self.total_mpiio_write_operations,
-                                                    self.detected_files_mpiio_coll_writes, self.report.name_records, self.dxt_mpiio)
+        # module.check_individual_write_imbalance(self.posix_data_imbalance_count,
+        #                                         self.posix_shared_time_imbalance_detected_files2,
+        #                                         self.report.name_records, self.dxt_posix, self.dxt_posix_write_data)
 
-        module.check_individual_read_imbalance(self.imbalance_count_posix_shared_time,
-                                               self.posix_shared_time_imbalance_detected_files3,
-                                               self.report.name_records, self.dxt_posix, self.dxt_posix_read_data)
+        # module.check_mpi_collective_read_operation(self.mpiio_coll_reads, self.mpiio_indep_reads,
+        #                                            self.total_mpiio_read_operations,
+        #                                            self.detected_files_mpi_coll_reads, self.report.name_records,
+        #                                            self.dxt_mpiio)
+
+        # module.check_mpi_collective_write_operation(self.mpiio_coll_writes, self.mpiio_indep_writes,
+        #                                             self.total_mpiio_write_operations,
+        #                                             self.detected_files_mpiio_coll_writes, self.report.name_records, self.dxt_mpiio)
+        #
+        # module.check_individual_read_imbalance(self.imbalance_count_posix_shared_time,
+        #                                        self.posix_shared_time_imbalance_detected_files3,
+        #                                        self.report.name_records, self.dxt_posix, self.dxt_posix_read_data)
 
         module.check_mpi_none_block_operation(self.mpiio_nb_reads, self.mpiio_nb_writes, self.has_hdf5_extension,
                                               self.modules)
 
 
 
     def _small_operation_insight(self):
-        module.check_small_operation(self.total_reads, self.total_reads_small, self.total_writes,
-                                     self.total_writes_small,
-                                     self.small_operation_detected_files,
-                                     self.modules, self.report.name_records, self.dxt_posix, self.dxt_posix_read_data,
-                                     self.dxt_posix_write_data)
+        pass
+        # module.check_small_operation(self.total_reads, self.total_reads_small, self.total_writes,
+        #                              self.total_writes_small,
+        #                              self.small_operation_detected_files,
+        #                              self.modules, self.report.name_records, self.dxt_posix, self.dxt_posix_read_data,
+        #                              self.dxt_posix_write_data)
 
 
 
@@ -921,7 +924,7 @@ def __init__(self, traces: List[DarshanTrace]):
         reports = [current_trace.report for current_trace in traces]
         self.name_records = dict()
         for report in reports:
-            self.name_records |= report.name_records
+            self.name_records.update(report.name_records) # self.name_records |= report.name_records
 
     def aggregate_traces(self):
         self.modules = set()
@@ -953,7 +956,7 @@ def aggregate_traces(self):
             self.total_files_posix += current_trace.total_size_posix
             self.total_files_mpiio += current_trace.total_files_mpiio
 
-            self.files |= current_trace.files
+            self.files.update(current_trace.files) # self.files |= current_trace.files
 
             self.total_reads += current_trace.total_reads
             self.total_writes += current_trace.total_writes

drishti/includes/config.py

@@ -3,7 +3,7 @@
 import json
 import os
 
-from includes.parser import *
+from drishti.includes.parser import *
 from rich.console import Console, Group
 from rich.padding import Padding
 from rich.panel import Panel

drishti/includes/module.py

@@ -5,7 +5,7 @@
 import time
 
 import pandas as pd
-from includes.config import *
+from drishti.includes.config import *
 from rich import box
 from rich.syntax import Syntax
 

drishti/reporter.py

@@ -3,9 +3,10 @@
 import os
 import sys
 from subprocess import call
+from typing import List, Optional
 
 # from includes.parser import * # imports {'parser', 'args', 'argparse'} # TODO: Is next line enuf
-from includes.parser import args
+from drishti.includes.parser import args
 
 '''
                          |- handler_darshan   -|
@@ -29,7 +30,7 @@ def clear():
     _ = call('clear' if os.name == 'posix' else 'cls')
 
 
-def check_log_type(paths: list[str]) -> int | None:
+def check_log_type(paths: List[str]) -> Optional[int]:
     is_darshan = True
     is_recorder = True
     multiple_logs = len(paths) > 1
@@ -70,10 +71,10 @@ def main():
     log_type = check_log_type(args.log_paths)
 
     if log_type == LOG_TYPE_DARSHAN:
-        from handlers.handle_darshan import handler
+        from drishti.handlers.handle_darshan import handler
 
     elif log_type == LOG_TYPE_RECORDER:
-        from handlers.handle_recorder import handler
+        from drishti.handlers.handle_recorder import handler
 
     handler()