Purge: +42 domain(s) via appeal (batch) #35
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Purge Domain | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| domain: | |
| description: "Domain to remove from all lists (added to allowlist automatically)" | |
| required: true | |
| type: string | |
| push: | |
| paths: | |
| - "allow/allowlist.json" | |
| branches: | |
| - main | |
| concurrency: | |
| group: repo-updates | |
| cancel-in-progress: false | |
| jobs: | |
| purge: | |
| if: "!contains(github.event.head_commit.message || '', '[skip ci]')" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 40 | |
| permissions: | |
| contents: write | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v7 | |
| with: | |
| fetch-depth: 1 | |
| - name: Setup Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.12" | |
| - name: Install dependencies | |
| run: pip install -r requirements.txt | |
| - name: Add domain to allowlist | |
| if: github.event.inputs.domain | |
| shell: python | |
| env: | |
| DOMAIN: ${{ github.event.inputs.domain }} | |
| run: | | |
| import json, os | |
| domain = os.environ["DOMAIN"].strip().lower() | |
| path = "allow/allowlist.json" | |
| with open(path, "r", encoding="utf-8") as f: | |
| data = json.load(f) | |
| if domain not in data: | |
| data.append(domain) | |
| data.sort() | |
| with open(path, "w", encoding="utf-8") as f: | |
| json.dump(data, f, indent=2, ensure_ascii=False) | |
| print(f"Added '{domain}' to allowlist ({len(data)} total)") | |
| else: | |
| print(f"'{domain}' already in allowlist") | |
| - name: Validate JSON | |
| run: python scripts/validate_json.py | |
| - name: Clean all lists against allowlist | |
| run: python scripts/validate_and_clean.py | |
| - name: Convert JSON to TXT | |
| run: python scripts/json_to_txt.py | |
| - name: Update domain counts | |
| run: python scripts/update_counts.py | |
| - name: Build rootlist | |
| run: python scripts/build_rootlist.py | |
| - name: Build array shards | |
| run: python scripts/build_shards.py | |
| - name: Commit and push | |
| env: | |
| INPUT_DOMAIN: ${{ github.event.inputs.domain }} | |
| run: | | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| git config user.name "github-actions[bot]" | |
| git add allow/allowlist.json list.json list.txt count.json \ | |
| community/blocklist.json community/blocklist.txt \ | |
| community/live_blocklist.json community/live_blocklist.txt \ | |
| community/content_live.txt \ | |
| community/count.json community/live_count.json \ | |
| community/content_active_count.json \ | |
| dns/*.json dns/*.txt \ | |
| rootlist/ | |
| if git diff --staged --quiet; then | |
| echo "No changes to commit" | |
| exit 0 | |
| fi | |
| if [ -n "$INPUT_DOMAIN" ]; then | |
| MSG="Purge: ${INPUT_DOMAIN} [skip ci]" | |
| else | |
| MSG="Allowlist update: clean all lists [skip ci]" | |
| fi | |
| # Preserve both present files and deletions before resetting onto a | |
| # concurrently updated main branch. A single name list breaks when a | |
| # generated shard was deleted because tar cannot archive a path that | |
| # no longer exists. | |
| git diff --staged --name-only --diff-filter=D > /tmp/deleted_files.txt | |
| git diff --staged --name-only --diff-filter=d > /tmp/existing_files.txt | |
| if [ -s /tmp/existing_files.txt ]; then | |
| tar -cf /tmp/processed.tar -T /tmp/existing_files.txt | |
| else | |
| tar -cf /tmp/processed.tar --files-from=/dev/null | |
| fi | |
| for i in {1..5}; do | |
| git fetch origin main | |
| git reset --hard origin/main | |
| if [ -s /tmp/existing_files.txt ]; then | |
| tar -xf /tmp/processed.tar | |
| fi | |
| while IFS= read -r path; do | |
| if [ -n "$path" ]; then | |
| git rm -f --ignore-unmatch -- "$path" | |
| fi | |
| done < /tmp/deleted_files.txt | |
| while IFS= read -r path; do | |
| if [ -n "$path" ]; then | |
| git add -A -- "$path" | |
| fi | |
| done < /tmp/existing_files.txt | |
| if git diff --staged --quiet; then | |
| echo "Changes already present in remote" | |
| exit 0 | |
| fi | |
| git commit -m "$MSG" | |
| git push && exit 0 | |
| echo "Push attempt $i failed, retrying in $((i * 2))s..." | |
| sleep $((i * 2)) | |
| done | |
| exit 1 | |
| - name: Mirror to Codeberg | |
| env: | |
| CODEBERG_TOKEN: ${{ secrets.CODEBERG_TOKEN }} | |
| run: | | |
| if [ -z "$CODEBERG_TOKEN" ]; then exit 0; fi | |
| REPO_NAME="${GITHUB_REPOSITORY##*/}" | |
| BRANCH="${GITHUB_REF_NAME}" | |
| git remote add codeberg https://phishdestroy:${CODEBERG_TOKEN}@codeberg.org/phishdestroy/${REPO_NAME}.git 2>/dev/null || \ | |
| git remote set-url codeberg https://phishdestroy:${CODEBERG_TOKEN}@codeberg.org/phishdestroy/${REPO_NAME}.git | |
| if git push codeberg HEAD:${BRANCH} --force 2>/dev/null; then | |
| echo "Mirror updated (shallow push)" | |
| else | |
| echo "Shallow push rejected, unshallowing (repo history is slim now)" | |
| git fetch --unshallow 2>/dev/null || true | |
| git push codeberg HEAD:${BRANCH} --force || echo "Mirror push failed, continuing" | |
| fi |