Merge branch 'main' into llm-translate

Author: TsukiSama9292

.github/actions/test-template/action.yml

@@ -131,7 +131,8 @@ runs:
       env:
         GH_TOKEN: ${{ inputs.PAT }}
       run: |
-        docker build -f Dockerfile -t curator .
+        docker system prune -af
+        docker build -f ray-curator/docker/Dockerfile -t ray-curator .
 
     - name: Start container
       shell: bash
@@ -155,7 +156,7 @@ runs:
           --volume $(pwd)/NeMo-Curator:/workspace \
           --workdir /workspace \
           --volume $MNT_PATH/TestData:/home/TestData \
-          curator \
+          ray-curator \
           bash -c "sleep $(( ${{ inputs.timeout }} * 60 + 60 ))"
         RUN_TEST_EOF
         )
@@ -176,9 +177,8 @@ runs:
         docker exec -t nemo_container_${{ github.run_id }} bash -c '
           set -e
 
-          source activate /opt/conda/envs/curator
-          
-          bash tests/${{ inputs.script }}.sh
+          . /opt/venv/env.sh
+          bash ray-curator/tests/${{ inputs.script }}.sh
         '
 
         RUN_TEST_EOF
@@ -202,13 +202,13 @@ runs:
       id: check
       shell: bash
       run: |
-        docker exec nemo_container_${{ github.run_id }} coverage combine || true
-        docker exec nemo_container_${{ github.run_id }} coverage xml || true
-        docker cp nemo_container_${{ github.run_id }}:/workspace/.coverage .coverage
-        docker cp nemo_container_${{ github.run_id }}:/workspace/coverage.xml coverage.xml
+        # docker exec nemo_container_${{ github.run_id }} coverage combine || true
+        # docker exec nemo_container_${{ github.run_id }} coverage xml || true
+        # docker cp nemo_container_${{ github.run_id }}:/workspace/.coverage .coverage
+        # docker cp nemo_container_${{ github.run_id }}:/workspace/coverage.xml coverage.xml
 
-        coverage_report=coverage-${{ steps.create.outputs.coverage-prefix }}-${{ github.run_id }}-$(uuidgen)
-        echo "coverage_report=$coverage_report" >> "$GITHUB_OUTPUT"
+        # coverage_report=coverage-${{ steps.create.outputs.coverage-prefix }}-${{ github.run_id }}-$(uuidgen)
+        # echo "coverage_report=$coverage_report" >> "$GITHUB_OUTPUT"
 
         EXIT_CODE=${{ steps.run-main-script.outputs.exit_code }}
         IS_SUCCESS=$([[ "$EXIT_CODE" -eq 0 ]] && echo "true" || echo "false")
@@ -225,17 +225,17 @@ runs:
 
         exit $EXIT_CODE
 
-    - name: Test coverage
-      shell: bash -x -e -u -o pipefail {0}
-      run: |
-        docker exec -t nemo_container_${{ github.run_id }} coverage report -i
-
-    - name: Upload artifacts
-      uses: actions/upload-artifact@v4
-      if: ${{ steps.check.outputs.coverage_report != 'none' }}
-      with:
-        name: ${{ steps.check.outputs.coverage_report }}
-        path: |
-          coverage.xml
-          .coverage
-        include-hidden-files: true
+    # - name: Test coverage
+    #   shell: bash -x -e -u -o pipefail {0}
+    #   run: |
+    #     docker exec -t nemo_container_${{ github.run_id }} coverage report -i
+
+    # - name: Upload artifacts
+    #   uses: actions/upload-artifact@v4
+    #   if: ${{ steps.check.outputs.coverage_report != 'none' }}
+    #   with:
+    #     name: ${{ steps.check.outputs.coverage_report }}
+    #     path: |
+    #       coverage.xml
+    #       .coverage
+    #     include-hidden-files: true

.github/workflows/auto-labler.yml

@@ -8,8 +8,6 @@ on:
 jobs:
   cherry-pick:
     uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/[email protected]
-    with:
-      target-branches-pattern: 'r[0-9]+\.[0-9]+\.[0-9]+|ray-api'
     secrets:
       PAT: ${{ secrets.PAT }}
       SLACK_WEBHOOK_ADMIN: ${{ secrets.SLACK_WEBHOOK_ADMIN }}

.github/workflows/cherry-pick-release-commit.yml

@@ -59,6 +59,7 @@ jobs:
         uses: step-security/[email protected]
         with:
           files: |
+            ray-curator/**
             nemo_curator/**
             .github/**
             pyproject.toml
@@ -148,33 +149,35 @@ jobs:
       - name: Install NeMo-Curator and pytest
         run: |
           pip install -U pip
-          pip install --no-cache-dir wheel
-          pip install --no-cache-dir .
-          pip install --no-cache-dir pytest pytest-asyncio coverage
+          cd ray-curator
+          pip install --no-cache-dir ".[text]"
+          pip install --no-cache-dir ".[video]"
+          pip install --no-cache-dir "internvideo2-multi-modality @ git+https://github.com/suiyoubi/InternVideo.git@curator#subdirectory=InternVideo2/multi_modality"
+          pip install --no-cache-dir --group test
 
-      - name: Run tests
+      - name: Run tests (CPU)
         run: |
-          python -m coverage run --branch --source=nemo_curator --omit=nemo_curator/scripts/* -m pytest -v --cpu
+          cd ray-curator
+          python -m coverage run --branch --source=ray_curator -m pytest -v tests -m "not gpu"
 
       - name: Generate report
         id: check
         shell: bash
         run: |
+          cd ray-curator
           python -m coverage xml
           python -m coverage report
-
           coverage_report=coverage-unit-test-${{ github.run_id }}-$(uuidgen)
           echo "$coverage_report"
           echo "coverage_report=$coverage_report" >> "$GITHUB_OUTPUT"
-
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
         if: ${{ steps.check.outputs.coverage_report != 'none' }}
         with:
           name: ${{ steps.check.outputs.coverage_report }}
           path: |
-            coverage.xml
-            .coverage
+            ray-curator/coverage.xml
+            ray-curator/.coverage
           include-hidden-files: true
 
   cicd-gpu-tests:
@@ -237,18 +240,15 @@ jobs:
           # Get workflow run details and check job conclusions
           NUM_FAILED=$(gh run view $RUN_ID --json jobs -q '[.jobs[] | select(.conclusion == "failure") | .name] | length')
           NUM_CANCELLED=$(gh run view $RUN_ID --json jobs -q '[.jobs[] | select(.conclusion == "cancelled") | .name] | length')
-
           if [[ ($NUM_FAILED -eq 0 && $NUM_CANCELLED -eq 0) || $DOCS_ONLY == 'true' ]]; then
             RESULT="success"
           elif [[ $NUM_CANCELLED -gt 0 ]]; then
             RESULT="cancelled"
           else
             RESULT="failure"
           fi
-
           # Output the final status
           echo "code=$RESULT" | tee -a $GITHUB_OUTPUT
-
       - name: Checkout for GH CLI
         uses: actions/checkout@v4
 
@@ -276,13 +276,9 @@ jobs:
           issue-number: ${{ github.event.number }}
           body: |
             [🤖]: Hi @${{ github.event.pull_request.user.login }} 👋,
-
             We wanted to let you know that a [CICD pipeline](https://github.com/${{ env.REPOSITORY }}/actions/runs/${{ env.RUN_ID }}) for this PR just finished successfully.
-
             So it might be time to merge this PR or get some approvals.
-
             //cc @NVIDIA-NeMo/automation
-
       - name: "Pipeline not successful and not cancelled: Send Slack alert & create step summary"
         if: |
           steps.result.outputs.code == 'failure'
@@ -299,9 +295,7 @@ jobs:
           set -x
           pip install PyGithub
           export BRANCH_NAME=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}
-
           python .github/scripts/notify.py
-
       - name: Exit
         if: ${{ always() }}
         env:
@@ -326,7 +320,7 @@ jobs:
       && !cancelled()
     strategy:
       matrix:
-        flag: [unit-test, e2e]
+        flag: [unit-test]
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -341,7 +335,6 @@ jobs:
         if: always()
         run: |
           pip install coverage[toml]
-
           ls -al .
           ls -al coverage-*/
           coverage combine --keep $(ls coverage-*/.coverage)
@@ -365,11 +358,11 @@ jobs:
           include-hidden-files: true
 
   codecov-placeholder:
-     name: codecov/patch
-     needs: [pre-flight]
-     if: needs.pre-flight.outputs.docs_only == 'true'
-     runs-on: ubuntu-latest
-     steps:
-       - name: codecov_placeholder
-         run: |
-           echo "This is a placeholder status check for when no tests are ran but the codecov status is expected"
+    name: codecov/patch
+    needs: [pre-flight]
+    if: needs.pre-flight.outputs.docs_only == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - name: codecov_placeholder
+        run: |
+          echo "This is a placeholder status check for when no tests are ran but the codecov status is expected"

.github/workflows/cicd-main.yml

@@ -6,87 +6,62 @@ NeMo Curator's API reference provides comprehensive technical documentation for
 .. grid:: 1 2 2 2
    :gutter: 3
 
-   .. grid-item-card:: :octicon:`database;1.5em;sd-mr-1` Core Data Handling
-      :link: datasets/datasets
+   .. grid-item-card:: :octicon:`server;1.5em;sd-mr-1` Execution Backends
+      :link: backends/backends
       :link-type: doc
       :class-card: sd-border-0
 
-      **Datasets & Download**
+      **Ray-based execution backends**
 
-      Essential classes for loading, managing, and downloading training data from various sources.
+      Adapters and executors for running pipelines at scale.
 
-      :bdg-secondary:`doc-dataset` :bdg-secondary:`parallel-dataset` :bdg-secondary:`arxiv` :bdg-secondary:`commoncrawl`
+      :bdg-secondary:`ray-data` :bdg-secondary:`xenna`
 
-   .. grid-item-card:: :octicon:`filter;1.5em;sd-mr-1` Data Processing  
-      :link: filters/filters
+   .. grid-item-card:: :octicon:`workflow;1.5em;sd-mr-1` Pipeline
+      :link: pipeline/pipeline
       :link-type: doc
       :class-card: sd-border-0
 
-      **Filters & Modifiers**
+      **Orchestrate end-to-end workflows**
 
-      Tools for cleaning, filtering, and transforming text data to improve quality and remove unwanted content.
-      
-      :bdg-secondary:`classifier-filter` :bdg-secondary:`heuristic-filter` :bdg-secondary:`pii-modifier`
+      Build and run pipelines composed of processing stages.
 
-   .. grid-item-card:: :octicon:`code;1.5em;sd-mr-1` Classification & Analysis
-      :link: classifiers/classifiers  
+   .. grid-item-card:: :octicon:`stack;1.5em;sd-mr-1` Processing Stages
+      :link: stages/stages
       :link-type: doc
       :class-card: sd-border-0
 
-      **AI-Powered Analysis**
+      **Download, transform, and write data**
 
-      Advanced classification tools and image processing capabilities for content analysis and quality assessment.
+      Modular stages for download/extract, text models/classifiers, I/O, and utilities.
 
-      :bdg-secondary:`aegis` :bdg-secondary:`content-type` :bdg-secondary:`domain-classifier`
+      :bdg-secondary:`download` :bdg-secondary:`text` :bdg-secondary:`io` :bdg-secondary:`modules`
 
-   .. grid-item-card:: :octicon:`shield-check;1.5em;sd-mr-1` Privacy & Security
-      :link: pii/pii
+   .. grid-item-card:: :octicon:`tasklist;1.5em;sd-mr-1` Tasks
+      :link: tasks/tasks
       :link-type: doc
       :class-card: sd-border-0
 
-      **PII Detection & Redaction**
+      **Core data structures**
 
-      Identify and handle personally identifiable information in datasets with advanced recognition algorithms.
-      
-      :bdg-secondary:`recognizers` :bdg-secondary:`algorithms` :bdg-secondary:`redaction`
+      Document batches, file groups, and related interfaces passed between stages.
 
-   .. grid-item-card:: :octicon:`zap;1.5em;sd-mr-1` Synthetic Data
-      :link: synthetic/synthetic
+   .. grid-item-card:: :octicon:`gear;1.5em;sd-mr-1` Utilities
+      :link: utils/utils
       :link-type: doc
       :class-card: sd-border-0
 
-      **Data Generation**
+      **Helper functions**
 
-      Create high-quality synthetic training data using advanced language models and generation techniques.
-      
-      :bdg-secondary:`generator` :bdg-secondary:`nemotron` :bdg-secondary:`mixtral`
-
-   .. grid-item-card:: :octicon:`tools;1.5em;sd-mr-1` Advanced Processing
-      :link: modules/modules
-      :link-type: doc
-      :class-card: sd-border-0
-
-      **Deduplication & Modules**
-
-      Advanced processing modules including semantic deduplication, fuzzy matching, and data pipeline components.
-      
-      :bdg-secondary:`semantic-dedup` :bdg-secondary:`fuzzy-dedup` :bdg-secondary:`add-id`
+      File, performance, and operation utilities used across the pipeline.
 
 .. toctree::
    :maxdepth: 1
    :caption: API Modules
    :hidden:
 
-   datasets/datasets
-   download/download
-   filters/filters
-   modifiers/modifiers
-   modules/modules
-   classifiers/classifiers
-   image/image
-   pii/pii
-   synthetic/synthetic
-   services/services
-   nemo_run/nemo_run
+   backends/backends
+   pipeline/pipeline
+   stages/stages
    tasks/tasks
    utils/utils

docs/_templates/autodoc2_index.rst

@@ -13,7 +13,7 @@ modality: "universal"
 
 NeMo Curator is an open-source, enterprise-grade platform for scalable, privacy-aware data curation across text, image, and video modalities.
 
-NeMo Curator helps you prepare high-quality, compliant datasets for large language model (LLM) and generative artificial intelligence (AI) training. Whether you work in the cloud, on-premises, or in a hybrid environment, NeMo Curator supports your workflow.
+NeMo Curator, part of the NVIDIA NeMo software suite for managing the AI agent lifecycle, helps you prepare high-quality, compliant datasets for large language model (LLM) and generative artificial intelligence (AI) training. Whether you work in the cloud, on-premises, or in a hybrid environment, NeMo Curator supports your workflow.
 
 ## Target Users