Avoid redirecting cds compiler stdout

Revokes the approach of redirecting stdout for 'cds compile' commands
used in the CDS extractor. In some cases, stdout may contain more than
just the contents of the generated JSON, which creates errors that are
difficult to troubleshoot as they only occur in some environments.

Returns to the approach of using CLI arguments to instruct the CDS
compiler to output to a specified path, and adds some logic to account
for variations in CDS compiler behavior when using the '-o' or '--dest'
CLI argument. Because some CDS compiler versions say that they output
to a directory but actually output to a file, and some CDS compiler
versions actually output to a directory as specified in the CLI docs,
we trust the use of the '--dest' argument (more than we trust a redirect
of stdout to a JSON file), but we *verify* whether the output is a
regular '.cds.json' file or is, indeed, a directory.

Author: data-douser

.github/workflows/run-codeql-unit-tests-javascript.yml

@@ -96,10 +96,26 @@ jobs:
             for cds_file in $(find . -type f \( -iname '*.cds' \) -print)
             do
               echo "I am compiling $cds_file"
+              _out_path="${cds_file}.json"
               cds compile $cds_file \
-                -2 json \
                 --locations \
-                > "$cds_file.json" 2> "$cds_file.err"
+                --to json \
+                --dest "$_out_path" \
+                2> "$cds_file.err"
+              # Check if the output is a regular file or a (sub)directory, where
+              # files generated in an output directory will need to have the file
+              # extension changed from '.json' to '.cds.json', but we don't need
+              # to rename anything if the cds compiler just generated a single
+              # '.cds.json' file.
+              if [ -d "$_out_path" ]
+              then
+                for json_file in $(find "$_out_path" -type f \( -iname '*.json' \) -print)
+                do
+                  _new_path="${json_file%.json}.cds.json"
+                  echo "Renaming CDS compiler generated JSON file $json_file to $_new_path"
+                  mv "$json_file" "$_new_path"
+                done
+              fi
             done
             popd
           done

extractors/cds/tools/index-files.sh

@@ -48,16 +48,62 @@ fi
 
 echo "Processing CDS files to JSON"
 
-# Run the cds compile command on each file in the response file, outputting the compiled JSON to a file with
-# the same name
-while IFS= read -r cds_file; do
-    echo "Processing CDS file $cds_file to:"
-    if ! $cds_command compile "$cds_file" -2 json --locations > "$cds_file.json" 2> "$cds_file.err"; then
-        stderr_truncated=`grep "^\[ERROR\]" "$cds_file.err" | tail -n 4`
-        error_message=$'Could not compile the file '"$cds_file"$'.\nReported error(s):\n```\n'"$stderr_truncated"$'\n```'
-        echo "$error_message"
-        # Log an error diagnostic which appears on the status page
-        "$CODEQL_DIST/codeql" database add-diagnostic --extractor-name cds --ready-for-status-page --source-id cds/compilation-failure --source-name "Failure to compile one or more SAP CAP CDS files" --severity error --markdown-message "$error_message" --file-path "$cds_file" "$CODEQL_EXTRACTOR_CDS_WIP_DATABASE"
+# Run the cds compile command on each file in the response file in order to generate
+# the JSON data we need. Move the generated ".json" files from the temporary output
+# to the original directory and rename them to ".cds.json" so that the JS extractor
+# can distinguish them from regular JSON files / extract them as CDS files.
+while IFS= read -r _cds_file_path; do
+    # The cds compile command chooses how it outputs the JSON. If it creates output
+    # files in a directory, then it will create the directory when it runs. If it
+    # creates a single output file, then it will create the file when it runs. We
+    # create the output path by simply appending ".json" to the input file path,
+    # such that there is nothing further to do if the output is a single file.
+    _cds_compile_json_out="${_cds_file_path}.json"
+    # Remove any existing output directory to avoid conflicts.
+    if [ -d "$_cds_compile_json_out" ]; then
+        echo "WARNING:  overwriting existing temporary output path ${_cds_compile_json_out}."
+        rm -rf "$_cds_compile_json_out"
+    fi
+    _cds_file_err_path="${_cds_file_path}.err"
+    echo "Compiling JSON for source CDS file $_cds_file_path to output path: $_cds_compile_json_out"
+    if ! $cds_command compile "$_cds_file_path" --to json --dest "$_cds_compile_json_out" --locations 2> "$_cds_file_err_path"; then
+        stderr_truncated=`grep "^\[ERROR\]" "$_cds_file_err_path" | tail -n 4`
+        _error_msg=$'ERROR: Could not compile the CDS file '"$_cds_file_path"$'.\nReported error(s):\n```\n'"$stderr_truncated"$'\n```'
+        echo "$_error_msg"
+        # Log the error message as a diagnostic which appears on the Tools status page.
+        "$CODEQL_DIST/codeql" database add-diagnostic --extractor-name cds --ready-for-status-page --source-id cds/compilation-failure --source-name "Failure to compile one or more SAP CAP CDS files" --severity error --markdown-message "$_error_msg" --file-path "$_cds_file_path" "$CODEQL_EXTRACTOR_CDS_WIP_DATABASE"
+        # Continue to the next file.
+        continue
+    fi
+    # Cleanup the error file if it exists and is empty.
+    if [[ -f "$_cds_file_err_path" && -s "$_cds_file_err_path" ]]; then
+        rm -f "$_cds_file_err_path"
+    fi
+    # Replace the ".json" extension with a ".cds.json" extension in the files
+    # generated by the cds compile command. Due to inconsistencies between
+    # different versions of the cds compiler, and contrary to the command-line
+    # documentation for the "cds compile" command, the output may be generated
+    # as a single file in the current directory, or as one or more files in an
+    # output directory.
+    if [ -f "$_cds_compile_json_out" ]; then
+        echo "Compiled CDS source file from $_cds_file_path to $_cds_compile_json_out"
+    elif [ -d "$_cds_compile_json_out" ]; then
+        echo "CDS compiler generated JSON to output directory: $_cds_compile_json_out"
+        if ls "$_cds_compile_json_out"/*.json 1> /dev/null 2>&1; then
+            for _output_json_file in "$_cds_compile_json_out"/*.json; do
+                _cds_json_file_path="${_output_json_file%.json}.cds.json"
+                echo "Renaming CDS compiler generated output '.json' file to $_cds_json_file_path"
+                mv "$_output_json_file" "$_cds_json_file_path"
+            done
+        else
+            _error_msg="ERROR: Detected no '.json' files in CDS compiler output directory: $_cds_compile_json_out"
+            # Log the error message as a diagnostic which appears on the Tools status page.
+            "$CODEQL_DIST/codeql" database add-diagnostic --extractor-name cds --ready-for-status-page --source-id cds/compilation-failure --source-name "Failure to find one or more compiled SAP CAP CDS JSON files" --severity error --markdown-message "$_error_msg" --file-path "$_cds_file_path" "$CODEQL_EXTRACTOR_CDS_WIP_DATABASE"
+        fi
+    else
+        _error_msg="ERROR: Detected no output directory or file for CDS compiler output: $_cds_compile_json_out"
+        # Log the error message as a diagnostic which appears on the Tools status page.
+        "$CODEQL_DIST/codeql" database add-diagnostic --extractor-name cds --ready-for-status-page --source-id cds/compilation-failure --source-name "Failure to find one or more compiled SAP CAP CDS JSON files" --severity error --markdown-message "$_error_msg" --file-path "$_cds_file_path" "$CODEQL_EXTRACTOR_CDS_WIP_DATABASE"
     fi
 done < "$response_file"