Feat(ci): validate project fields in prs and issues

.config/dictionaries/project.dic

@@ -15,6 +15,7 @@ depgraph
 devenv
 dind
 dockerhub
+doseq
 doublecircle
 Earthfile
 Earthfiles
@@ -50,6 +51,7 @@ idents
 JDBC
 jorm
 jormungandr
+jsonlib
 junitreport
 Kroki
 kubeconfig

.github/workflows/validate-project-fields.yml

@@ -19,33 +19,31 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
   cancel-in-progress: true
 
-env:
-  AWS_REGION: eu-central-1
-  AWS_ROLE_ARN: arn:aws:iam::332405224602:role/ci
-  EARTHLY_TARGET: docker
-  ECR_REGISTRY: 332405224602.dkr.ecr.eu-central-1.amazonaws.com
-
 jobs:
   validate-project-fields:
     runs-on: ubuntu-latest
+    env:
+      # Needs a PAT Classic with (read:project)
+      GITHUB_PROJECTS_PAT: ${{ secrets.PROJECTS_PAT }}
+      GITHUB_REPOSITORY: "${{ github.repository }}"
+      GITHUB_EVENT_NUMBER: "${{ github.event.number || '0' }}"
+      PROJECT_NUMBER: 102
     steps:
-      - uses: actions/checkout@v4
+      - name: Fetch Validation Script
+        uses: actions/checkout@v4
+        with:
+          repository: input-output-hk/catalyst-ci
+          ref: feat/validate-project-fields-in-prs-and-issues-sj
+          sparse-checkout: |
+            utilities/project-fields-validator/main.py
+          sparse-checkout-cone-mode: false
 
-      - name: Setup CI
-        uses: input-output-hk/catalyst-ci/actions/setup@master
+      - name: Set up Python
+        uses: actions/setup-python@v5 
         with:
-          aws_role_arn: ${{ env.AWS_ROLE_ARN }}
-          aws_region: ${{ env.AWS_REGION }}
-          earthly_runner_secret: ${{ secrets.EARTHLY_RUNNER_SECRET }}
+          python-version: '3.13'
 
       - name: Run Project Fields Validation
-        uses: input-output-hk/catalyst-ci/actions/run@master
         if: always()
         continue-on-error: false
-        with:
-          earthfile: ./utilities/project-fields-validator
-          flags: --allow-privileged
-          targets: validate-project-fields
-          target_flags: --GITHUB_REPOSITORY="${{ github.repository }}" --GITHUB_EVENT_NUMBER="${{ github.event.number || '0' }}"
-          runner_address: ${{ secrets.EARTHLY_SATELLITE_ADDRESS }}
-          artifact: false
+        run: utilities/project-fields-validator/main.py

Justfile

@@ -16,5 +16,15 @@ check-spelling:
     earthly +clean-spelling-list
     earthly +check-spelling
 
+
+# Fix and Check Markdown files
+format-python-code:
+    ruff check --select I --fix .
+    ruff format .
+
+# Fix and Check Markdown files
+lint-python:
+    ruff check .
+
 # Pre Push Checks - intended to be run by a git pre-push hook.
-pre-push: check-markdown check-spelling
+pre-push: check-markdown check-spelling format-python-code lint-python

earthly/docs/common/macros/include.py

@@ -1,6 +1,7 @@
 import os
-import textwrap
 import re
+import textwrap
+
 
 def inc_file(env, filename, start_line=0, end_line=None, indent=None):
     """
@@ -10,7 +11,7 @@ def inc_file(env, filename, start_line=0, end_line=None, indent=None):
     project.
     indent = number of spaces to indent every line but the first.
     """
-    
+
     try:
         full_filename = os.path.join(env.project_dir, filename)
 
@@ -24,8 +25,8 @@ def inc_file(env, filename, start_line=0, end_line=None, indent=None):
         else:
             indent = " " * indent
         text = textwrap.indent(text, indent)
-        text = text[len(indent):] # First line should not be indented at all.
-        text = re.sub(r'\n$', '', text, count=1)
+        text = text[len(indent) :]  # First line should not be indented at all.
+        text = re.sub(r"\n$", "", text, count=1)
         # print(text)
         return text
     except Exception as exc:

earthly/docs/dev/local.py

@@ -2,13 +2,13 @@
 
 # cspell: words gmtime
 
+import argparse
 import subprocess
+import sys
 import time
+import urllib.request
 import webbrowser
 from dataclasses import dataclass, field
-import argparse
-import sys
-import urllib.request
 
 
 class ProcessRunError(Exception):
@@ -213,7 +213,7 @@ def main():
 
             # Open the webpage in a browser (once)
             if not browsed:
-                browsed=True
+                browsed = True
                 if not args.no_browser:
                     webbrowser.open(f"http://localhost:{docs_container.exposed_port}")
 

earthly/postgresql/scripts/std_checks.py

@@ -1,11 +1,10 @@
 #!/usr/bin/env python3
 
+import argparse
+
 import python.exec_manager as exec_manager
 import python.vendor_files_check as vendor_files_check
-import argparse
 import rich
-from rich import print
-import os
 
 # This script is run inside the `check` stage for postgres database setup
 # to perform all high level non-compilation checks.
@@ -32,7 +31,7 @@ def main():
     # Force color output in CI
     rich.reconfigure(color_system="256")
 
-    parser = argparse.ArgumentParser(description="Postgres checks processing.")
+    argparse.ArgumentParser(description="Postgres checks processing.")
 
     results = exec_manager.Results("Postgres checks")
 

earthly/postgresql/scripts/std_docs.py

@@ -2,16 +2,17 @@
 
 # cspell: words dbmigrations dbhost dbuser dbuserpw Tsvg pgsql11
 
-from typing import Optional
-import python.exec_manager as exec_manager
-import python.db_ops as db_ops
 import argparse
-import rich
-from rich import print
 import os
 import re
 from textwrap import indent
 
+import python.db_ops as db_ops
+import python.exec_manager as exec_manager
+import rich
+from rich import print
+
+
 def process_sql_files(directory):
     file_pattern = r"V(\d+)__(\w+)\.sql"
     migrations = {}
@@ -32,11 +33,12 @@ def process_sql_files(directory):
             migrations[version] = {
                 "version": version,
                 "migration_name": migration_name,
-                "sql_data": sql_data
+                "sql_data": sql_data,
             }
 
     return migrations, largest_version
 
+
 class Migrations:
     def __init__(self, args: argparse.Namespace):
         """
@@ -73,6 +75,7 @@ def create_markdown_file(self, file_path):
 
         print("Markdown file created successfully at: {}".format(file_path))
 
+
 def main():
     # Force color output in CI
     rich.reconfigure(color_system="256")
@@ -124,9 +127,7 @@ def main():
             f"-o docs/database_schema/ "
         )
         res = exec_manager.cli_run(
-            schemaspy_cmd,
-            name="Generate SchemaSpy Documentation",
-            verbose=True
+            schemaspy_cmd, name="Generate SchemaSpy Documentation", verbose=True
         )
         results.add(res)
 
@@ -135,7 +136,7 @@ def main():
             exec_manager.cli_run(
                 'echo "hide: true" > docs/database_schema/.pages',
                 name="Create .pages file",
-                verbose=True
+                verbose=True,
             )
 
         migrations.create_markdown_file("docs/migrations.md")
@@ -145,5 +146,6 @@ def main():
     if not results.ok():
         exit(1)
 
+
 if __name__ == "__main__":
-    main()
+    main()

earthly/python/Earthfile

@@ -35,9 +35,15 @@ python-base:
     # Adjust Poetry's configuration to prevent connection pool warnings.
     RUN poetry config installer.max-workers 10
 
+    # Extension we use needs rust.
+    RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
+    RUN echo 'source $HOME/.cargo/env' >> $HOME/.bashrc
+    ENV PATH="/root/.cargo/bin:${PATH}"
+
     # Install ruff for linting.
     RUN pip3 install ruff
     RUN pip3 install rich
+    RUN pip3 install third-party-imports
 
     # Universal build scripts we will always need and are not target dependent.
     COPY --dir scripts /scripts
@@ -58,9 +64,10 @@ BUILDER:
 
 CHECK:
     FUNCTION
+    ARG options
 
     # Execute the check script
-    RUN /scripts/std_checks.py
+    RUN /scripts/std_checks.py $options
 
 LINT_PYTHON:
     # Linting all Python code is done with ruff

earthly/python/scripts/std_checks.py

@@ -4,44 +4,114 @@
 import subprocess
 import sys
 
-def check_pyproject_toml():
+
+def check_pyproject_toml(stand_alone):
     # Check if 'pyproject.toml' exists in the project root.
-    if not os.path.isfile('pyproject.toml'):
+    if not os.path.isfile("pyproject.toml"):
+        if stand_alone:
+            print("pyproject.toml check passed.")
+            return True
+
         print("Error: pyproject.toml not found.")
         return False
     else:
+        if stand_alone:
+            print("Error: pyproject.toml found in standalone python module.")
+            return False
+
         print("pyproject.toml check passed.")
         return True
-
-def check_poetry_lock():
+
+
+def check_poetry_lock(stand_alone):
     # Check if 'poetry.lock' exists in the project root.
-    if not os.path.isfile('poetry.lock'):
+    if not os.path.isfile("poetry.lock"):
+        if stand_alone:
+            print("poetry.lock check passed.")
+            return True
+
         print("Error: poetry.lock not found.")
         return False
     else:
+        if stand_alone:
+            print("Error: poetry.lock found in stand alone module.")
+            return False
+
         print("poetry.lock check passed.")
         return True
 
+
+def check_lint_with_ruff():
+    # Check Python code linting issues using 'ruff'.
+    result = subprocess.run(["ruff", "check", "."], capture_output=True)
+    if result.returncode != 0:
+        print("Code linting issues found.")
+        print(result.stdout.decode())
+        return False
+    else:
+        print("Code linting check passed.")
+        return True
+
+
 def check_code_format_with_ruff():
     # Check Python code formatting and linting issues using 'ruff'.
-    result = subprocess.run(['ruff', 'check', '.'], capture_output=True)
+    result = subprocess.run(["ruff", "format", "--check", "."], capture_output=True)
     if result.returncode != 0:
-        print("Code formatting and linting issues found.")
+        print("Code formatting issues found.")
         print(result.stdout.decode())
         return False
     else:
-        print("Code formatting and linting check passed.")
+        print("Code formatting check passed.")
         return True
 
-def main():
+
+def zero_third_party_packages_found(output):
+    lines = output.split("\n")  # Split the multiline string into individual lines
+
+    if len(lines) < 2:
+        return False  # The second line doesn't exist
+    else:
+        return lines[1].startswith("Found '0' third-party package imports")
+
+
+def check_no_third_party_imports():
+    # Check No third party imports have been used
+    result = subprocess.run(["third-party-imports", "."], capture_output=True)
+    output = result.stdout.decode()
+
+    if result.returncode != 0 or not zero_third_party_packages_found(output):
+        print("Checking third party imports failed.")
+        print(output)
+        return False
+    else:
+        print("Checking third party imports passed.")
+        return True
+
+
+def main(stand_alone):
+    if stand_alone:
+        print(
+            "Checking Standalone Python files (No third party imports or poetry project)"
+        )
     checks_passed = True
     # Perform checks
-    checks_passed &= check_pyproject_toml()
-    checks_passed &= check_poetry_lock()
+
+    # These are true on python programs that require third party libraries, false otherwise
+    checks_passed &= check_pyproject_toml(stand_alone)
+    checks_passed &= check_poetry_lock(stand_alone)
+
+    # Always done
+    checks_passed &= check_lint_with_ruff()
     checks_passed &= check_code_format_with_ruff()
 
+    # Only done if the code should be able to run without third part libraries
+    if stand_alone:
+        checks_passed &= check_no_third_party_imports()
+
     if not checks_passed:
         sys.exit(1)
 
+
 if __name__ == "__main__":
-    main()
+    print(f"Current Working Directory: {os.getcwd()}")
+    main("--stand-alone" in sys.argv[1:])