auto_retrieve.yml

name: Automated retrieval
run-name: Automated retrieval
on:
  workflow_dispatch:
  schedule:
    - cron: '0 16 * * *'
permissions:
  contents: write

jobs:
  # This condition lets the job run even if any of the previous jobs failed
  # ${{ ! cancelled() }}

  # This condition prevents the job from running if the test job failed
  # ${{ needs.test-functions.result == 'success' }}

  test-functions:
    uses: ./.github/workflows/test_functions.yml

  build-nsfw:
    needs: test-functions
    if: ${{ ! cancelled() }}
    uses: ./.github/workflows/build_nsfw.yml

  retrieve-domains:
    needs: [test-functions, build-nsfw]
    if: ${{ ! cancelled() && needs.test-functions.result == 'success' }}
    uses: ./.github/workflows/retrieve_domains.yml
    secrets: inherit

  validate:
    needs: [test-functions, retrieve-domains]
    if: ${{ ! cancelled() && needs.test-functions.result == 'success' }}
    uses: ./.github/workflows/validate_domains.yml
    secrets: inherit

  check-dead:
    needs: [test-functions, validate]
    if: ${{ ! cancelled() && needs.test-functions.result == 'success' }}
    uses: ./.github/workflows/check_dead.yml

  check-unparked:
    needs: [test-functions, check-dead]
    if: ${{ ! cancelled() && needs.test-functions.result == 'success' }}
    uses: ./.github/workflows/check_unparked.yml

  build:
    needs: [test-functions, check-unparked]
    if: ${{ ! cancelled() && needs.test-functions.result == 'success' }}
    uses: ./.github/workflows/build_lists.yml

  prune-logs:
    needs: build
    if: ${{ ! cancelled() }}
    runs-on: ubuntu-22.04
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: main

      - name: Prune logs
        run: |
          # 250,000 is enough for a month's worth of logs.
          bash scripts/tools.sh --prune-lines config/domain_log.csv 250000
          bash scripts/tools.sh --prune-lines config/source_log.csv 10000

      - name: Push
        run: |
          git config user.email ${{ vars.GIT_EMAIL }}
          git config user.name ${{ vars.GIT_USERNAME }}
          git add .
          git diff-index --quiet HEAD || git commit -m "CI: prune logs"
          git push -q

  update-readme:
    needs: [build, prune-logs]
    # Run only if build was successful
    if: ${{ ! cancelled() && needs.build.result == 'success' }}
    uses: ./.github/workflows/update_readme.yml