+2 domains #13153
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: On List Update | |
| on: | |
| push: | |
| paths: | |
| - "list.json" | |
| - "dns/active_domains.json" | |
| - "dns/content_active.json" | |
| - "community/blocklist.json" | |
| - "community/live_blocklist.json" | |
| - "community/content_live.json" | |
| branches: | |
| - main | |
| workflow_dispatch: | |
| jobs: | |
| process: | |
| if: "!contains(github.event.head_commit.message || '', '[skip ci]')" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| permissions: | |
| contents: write | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v7 | |
| with: | |
| fetch-depth: 1 | |
| - name: Setup Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.12" | |
| - name: Install dependencies | |
| run: pip install -r requirements.txt | |
| - name: Validate JSON | |
| run: python scripts/validate_json.py | |
| - name: Clean all lists against allowlist | |
| run: python scripts/validate_and_clean.py | |
| - name: Convert JSON to TXT | |
| run: python scripts/json_to_txt.py | |
| - name: Update domain counts | |
| run: python scripts/update_counts.py | |
| - name: Build rootlist | |
| run: python scripts/build_rootlist.py | |
| - name: Build array shards | |
| run: python scripts/build_shards.py | |
| - name: Commit and push | |
| run: | | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| git config user.name "github-actions[bot]" | |
| # Heavy rootlist/ artifacts are committed twice a day by the | |
| # scheduled workflow (and immediately by purge.yml); committing | |
| # them on every list push bloats git history. | |
| git add list.json list.txt count.json \ | |
| community/blocklist.json community/blocklist.txt \ | |
| community/live_blocklist.json community/live_blocklist.txt \ | |
| community/content_live.json community/content_live.txt \ | |
| community/count.json community/live_count.json \ | |
| community/content_active_count.json \ | |
| dns/*.json dns/*.txt | |
| if git diff --staged --quiet; then | |
| echo "No changes to commit" | |
| exit 0 | |
| fi | |
| # Save processed outputs before any git reset can overwrite them | |
| git diff --staged --name-only > /tmp/staged_files.txt | |
| tar -cf /tmp/processed.tar -T /tmp/staged_files.txt | |
| for i in {1..10}; do | |
| if ! git fetch origin main; then | |
| echo "Fetch attempt $i failed, retrying in $((i * 3))s..." | |
| sleep $((i * 3)) | |
| continue | |
| fi | |
| git reset --hard origin/main | |
| # Restore our processed file versions on top of latest remote | |
| tar -xf /tmp/processed.tar | |
| git add $(cat /tmp/staged_files.txt | tr '\n' ' ') | |
| if git diff --staged --quiet; then | |
| echo "Changes already present in remote" | |
| exit 0 | |
| fi | |
| git commit -m "Update processed files [skip ci]" | |
| git push && exit 0 | |
| WAIT=$(( RANDOM % 10 + i * 3 )) | |
| echo "Push attempt $i failed, retrying in ${WAIT}s..." | |
| sleep $WAIT | |
| done | |
| exit 1 | |
| - name: Mirror to Codeberg | |
| env: | |
| CODEBERG_TOKEN: ${{ secrets.CODEBERG_TOKEN }} | |
| run: | | |
| if [ -z "$CODEBERG_TOKEN" ]; then exit 0; fi | |
| REPO_NAME="${GITHUB_REPOSITORY##*/}" | |
| BRANCH="${GITHUB_REF_NAME}" | |
| git remote add codeberg https://phishdestroy:${CODEBERG_TOKEN}@codeberg.org/phishdestroy/${REPO_NAME}.git 2>/dev/null || \ | |
| git remote set-url codeberg https://phishdestroy:${CODEBERG_TOKEN}@codeberg.org/phishdestroy/${REPO_NAME}.git | |
| if git push codeberg HEAD:${BRANCH} --force 2>/dev/null; then | |
| echo "Mirror updated (shallow push)" | |
| else | |
| echo "Shallow push rejected, unshallowing (repo history is slim now)" | |
| git fetch --unshallow 2>/dev/null || true | |
| git push codeberg HEAD:${BRANCH} --force || echo "Mirror push failed, continuing" | |
| fi |