Update python files & pipeline settings

Author: fredsnap

pygpt/core/profiler.py

@@ -76,8 +76,8 @@ def update(self, process):
             self.stats['cpu_perc'].append(cpu_perc) # cpu usage
             self.stats['cpu_time'].append(cpu_time) # cpu time
             self.stats['threads'].append(n_threads) # num threads
-        except Exception:
-            print('No access to process...')
+        except Exception as e:
+            print(e.args)
 
 
     def summary(self):
@@ -257,6 +257,7 @@ def __split_command_args__(command):
             token += char
     if token:
         args.append(token)
+
     return args
 
 
@@ -332,7 +333,8 @@ def profile(command, sampling_time, output, **kwargs):
     env = os.environ
     if 'env' in kwargs:
         env = kwargs['env']
-    print(env)
+    if os.environ.get('CI_PROJECT_DIR') is None:
+        print(env)
     print(command)
     proc = subprocess.Popen(command, 
                             stdout=subprocess.PIPE, 

pygpt/core/results.py

@@ -5,7 +5,7 @@
 License: GPLv3
 """
 import datetime
-import sys
+import os
 import json
 
 import matplotlib as mpl
@@ -61,9 +61,11 @@ def __init__(self, struct, results, adaptor=None):
 
     def __load_perfs__(self):
         perf_stats_file = fs.statistics.resolve(self.name+'.json')
-        with open(perf_stats_file, 'r') as stats:
-            return json.load(stats)
-        return None
+        if os.path.isfile(perf_stats_file):
+            with open(perf_stats_file, 'r') as stats:
+                return json.load(stats)
+        else:
+            return None
 
     def __load_stdout__(self):
         stdout_path = fs.outputs.resolve(f'{self.name}_gptOutput.txt')
@@ -276,7 +278,7 @@ def plots_path(self, version):
             plt.ylabel('CPU Average Time (s)')
             plt.title('CPU Average Time Historic')
             #set ticks every day
-            axis.xaxis.set_major_locator(mdates.DayLocator())
+            axis.xaxis.set_major_locator(mdates.DayLocator(interval=5))
             #set major ticks format
             axis.xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%d'))
 
@@ -292,7 +294,7 @@ def plots_path(self, version):
             plt.ylabel('Memory Average (Mb)')
             plt.title('Memory Average Historic')
             #set ticks every day
-            axis.xaxis.set_major_locator(mdates.DayLocator())
+            axis.xaxis.set_major_locator(mdates.DayLocator(interval=5))
             #set major ticks format
             axis.xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%d'))
 

pygpt/core/tools.py

@@ -91,6 +91,10 @@ def setter(self, value):
     return inner
 
 def split_args(arg):
+    """
+    DEPRECATED since it does not work on CI.
+    Useless because str.split(" ") does the job
+    """
     res = []
     curr = ''
     skip = False
@@ -117,4 +121,4 @@ def split_args(arg):
 
     if len(curr) > 0:
         res.append(curr)
-    return res
+    return res

pygpt/filter_json.py

@@ -15,6 +15,7 @@
 from core.models import TestScope
 
 __test_files__ = "JSONTestFiles.txt"
+__data_files__ = "testData.txt"
 
 # tags
 __RELEASE_TAG__ = 'release'
@@ -23,10 +24,11 @@
 __REGULAR_TAG__ = 'regular'
 
 
-def __create_test_json_list__(test_folder, scope, test_files_path):
+def __create_test_json_list__(test_folder, scope, test_files_path, data_files_path):
     """cretas files containing list of tests to execute for a given scope"""
     test_files = utils.rlist_files(test_folder, lambda f: f.endswith('.json'))
     test_list = []
+    test_data = []
     scope = TestScope.init(scope)
     for test_path in test_files:
         with open(test_path, 'r') as test_file:
@@ -36,9 +38,33 @@ def __create_test_json_list__(test_folder, scope, test_files_path):
                     if TestScope.compatibleN(scope, test['frequency']):
                         if test_path not in test_list:
                             test_list.append(test_path)
+                        if 'inputs' in test:
+                            if 'input' in test['inputs']:
+                                if test['inputs']['input'] not in test_data:
+                                    test_data.append(test['inputs']['input'])
+                            if 'input1' in test['inputs']:
+                                if test['inputs']['input1'] not in test_data:
+                                    test_data.append(test['inputs']['input1'])
+                            if 'input2' in test['inputs']:
+                                if test['inputs']['input2'] not in test_data:
+                                    test_data.append(test['inputs']['input2'])
+                            if 'input3' in test['inputs']:
+                                if test['inputs']['input3'] not in test_data:
+                                    test_data.append(test['inputs']['input3'])
+                            if 'input4' in test['inputs']:
+                                if test['inputs']['input4'] not in test_data:
+                                    test_data.append(test['inputs']['input4'])
+                            if 'input5' in test['inputs']:
+                                if test['inputs']['input5'] not in test_data:
+                                    test_data.append(test['inputs']['input5'])
+                            if 'input6' in test['inputs']:
+                                if test['inputs']['input6'] not in test_data:
+                                    test_data.append(test['inputs']['input6'])
                         # test_list += f'{test_path}\n'
     with open(test_files_path, 'w') as file:
         file.write('\n'.join(test_list))
+    with open(data_files_path, 'w') as file:
+        file.write('\n'.join(test_data))
     return True
 
 
@@ -73,8 +99,9 @@ def __main__():
         log.error("output folder does not exist")
         sys.exit(1)
     json_test_files = os.path.join(args.output_folder, __test_files__)
+    data_file = os.path.join(args.output_folder, __data_files__)
 
-    if __create_test_json_list__(args.test_folder, args.scope, json_test_files):
+    if __create_test_json_list__(args.test_folder, args.scope, json_test_files, data_file):
         log.success(f"filtered JSON created in {json_test_files}")
 
 

pygpt/snap_gpt_test.py

@@ -264,7 +264,7 @@ def __check_outputs__(test, args, properties):
                 cmd += args.java_args.split(" ")
             else:
                 cmd += utils.split_args(args.java_args)
-            cmd += [args.test_output, output_path, expected_output_path, output['outputName']]
+            cmd += [args.test_output, os.path.normpath(output_path), os.path.normpath(expected_output_path), output['outputName']]
             log.info(cmd)
             result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
             log.info(f'comparing done, result: {result.returncode}')

requirements.txt

@@ -2,4 +2,5 @@ psutil
 matplotlib
 lxml
 cryptography
-pymysql
+pymysql == 1.1.*
+fs

run_tests.bat

@@ -9,6 +9,7 @@ FOR /F "usebackq tokens=1,2* delims=" %%t in ("%CI_PROJECT_DIR%\%REPORT_DIR%\JSO
     :: Run test
     echo "Running %%t"
     set CI_PROJECT_DIR=%CI_PROJECT_DIR%
+    set JAVA_HOME=%JAVA_HOME%
     echo  java args: "%JAVA_OPTIONS% -cp %CLASSPATH%"
     echo %USERPROFILE%
     python "%CI_PROJECT_DIR%\pygpt\snap_gpt_test.py" java "%JAVA_OPTIONS% -cp %CLASSPATH%" org.esa.snap.test.TestOutput "%CI_PROJECT_DIR%\%PROPERTIES_PATH%" %SCOPE% "%CI_PROJECT_DIR%\%%t" "%CI_PROJECT_DIR%\%REPORT_DIR%\report\output" true

windows.gitlab-ci.yml

@@ -14,11 +14,12 @@ test_windows:
     SNAP_INSTALLER: snap_all_windows-9.exe
     TEST_DATA_LIST: singleTestData.txt
     JAVA_HOME: C:\\Program Files\\Eclipse Adoptium\\jdk-8.0.402.6-hotspot
+    JDK_HOME: C:\\Program Files\\Eclipse Adoptium\\jdk-8.0.402.6-hotspot
     JAVA_OPTIONS: >-
       -Dncsa.hdf.hdflib.HDFLibrary.hdflib=%SNAP_DIR%\\snap\\modules\\lib\\amd64\\jhdf.dll
       -Dncsa.hdf.hdf5lib.H5.hdf5lib=%SNAP_DIR%\\snap\modules\\lib\amd64\\jhdf5.dll
       -Dfile.encoding=Windows-1252
-    CLASSPATH: gpt-tests-executer\\target\\gpt-test-exec.jar
+    CLASSPATH: gpt-tests-executer\\target\\TestOutput.jar
     # output upload and download progress every 2 seconds
     TRANSFER_METER_FREQUENCY: "2s"
     # Use fast compression for artifacts, resulting in larger archives