diff --git a/main.yaml b/main.yaml new file mode 100644 index 0000000..675df24 --- /dev/null +++ b/main.yaml @@ -0,0 +1,25 @@ +--- +# Main ansible notebook +- name: Configure mirrors infrastructure + hosts: all + become: true + + roles: + - role: nginx + tags: [nginx] + - mirrors + # - role: maintenance + # tags: [maintenance] + # tags: [mirrors] + # - role: prometheus + # tags: [prometheus] + # - role: router + # tags: [router] + # - role: tor + # tags: [tor] + + post_tasks: + - name: Display mirrors configuration status + ansible.builtin.debug: + msg: "Mirrors infrastructure has been successfully configured" + diff --git a/mirrors/files/distros/AlmaLinux/repository/confd/almalinux-mirror-sync.service b/mirrors/files/distros/AlmaLinux/repository/confd/almalinux-mirror-sync.service new file mode 100644 index 0000000..cb2753f --- /dev/null +++ b/mirrors/files/distros/AlmaLinux/repository/confd/almalinux-mirror-sync.service @@ -0,0 +1,9 @@ +[Unit] +Description=Sync Almalinux Mirror +After=network.target + +[Service] +Type=simple +User=mirrors +Group=mirrors +ExecStart=/usr/bin/rsync -avSH --bwlimit=2048 --exclude='.~tmp~' --delete-delay --delay-updates rsync://rsync.repo.almalinux.org/almalinux/ /srv/almalinux/ diff --git a/mirrors/files/distros/AlmaLinux/repository/confd/almalinux-mirror-sync.timer b/mirrors/files/distros/AlmaLinux/repository/confd/almalinux-mirror-sync.timer new file mode 100644 index 0000000..670928d --- /dev/null +++ b/mirrors/files/distros/AlmaLinux/repository/confd/almalinux-mirror-sync.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Timer to sync Almalinux Mirror at random minutes each hour + +[Timer] +OnCalendar=hourly +RandomizedDelaySec=3600 +Persistent=true + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/mirrors/files/distros/ArchLinux/repository/confd/archlinux-mirror-sync.service b/mirrors/files/distros/ArchLinux/repository/confd/archlinux-mirror-sync.service new file mode 100644 index 0000000..ed7497b --- /dev/null +++ b/mirrors/files/distros/ArchLinux/repository/confd/archlinux-mirror-sync.service @@ -0,0 +1,9 @@ +[Unit] +Description=Sync Arch Linux Mirror +After=network.target + +[Service] +Type=simple +User=mirrors +Group=mirrors +ExecStart=/usr/local/bin/archlinux-syncrepo.sh diff --git a/mirrors/files/distros/ArchLinux/repository/confd/archlinux-mirror-sync.timer b/mirrors/files/distros/ArchLinux/repository/confd/archlinux-mirror-sync.timer new file mode 100644 index 0000000..22a2478 --- /dev/null +++ b/mirrors/files/distros/ArchLinux/repository/confd/archlinux-mirror-sync.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Timer to sync Arch Linux Mirror at random minutes each hour + +[Timer] +OnCalendar=hourly +RandomizedDelaySec=3600 +Persistent=true + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/mirrors/files/distros/ArchLinux/repository/scripts/archlinux-syncrepo.sh b/mirrors/files/distros/ArchLinux/repository/scripts/archlinux-syncrepo.sh new file mode 100644 index 0000000..a20c906 --- /dev/null +++ b/mirrors/files/distros/ArchLinux/repository/scripts/archlinux-syncrepo.sh @@ -0,0 +1,96 @@ +#!/bin/bash +# +######## +# +# Copyright © 2014-2019 Florian Pritz +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see . +# +######## +# +# This is a simple mirroring script. To save bandwidth it first checks a +# timestamp via HTTP and only runs rsync when the timestamp differs from the +# local copy. As of 2016, a single rsync run without changes transfers roughly +# 6MiB of data which adds up to roughly 250GiB of traffic per month when rsync +# is run every minute. Performing a simple check via HTTP first can thus save a +# lot of traffic. + +# Directory where the repo is stored locally. Example: /srv/repo +target="/srv/archlinux" + +# Lockfile path +lock="/var/lock/syncrepo.lck" + +# If you want to limit the bandwidth used by rsync set this. +# Use 0 to disable the limit. +# The default unit is KiB (see man rsync /--bwlimit for more) +bwlimit=0 + +# The source URL of the mirror you want to sync from. +# If you are a tier 1 mirror use rsync.archlinux.org, for example like this: +# rsync://rsync.archlinux.org/ftp_tier1 +# Otherwise chose a tier 1 mirror from this list and use its rsync URL: +# https://www.archlinux.org/mirrors/ +source_url='rsync://mirrors.ocf.berkeley.edu/archlinux' + +# An HTTP(S) URL pointing to the 'lastupdate' file on your chosen mirror. +# If you are a tier 1 mirror use: https://rsync.archlinux.org/lastupdate +# Otherwise use the HTTP(S) URL from your chosen mirror. +lastupdate_url='https://mirrors.ocf.berkeley.edu/archlinux/lastupdate' + +#### END CONFIG + +[ ! -d "${target}" ] && mkdir -p "${target}" + +exec 9>"${lock}" +flock -n 9 || exit + +# Cleanup any temporary files from old run that might remain. +# Note: You can skip this if you have rsync newer than 3.2.3 +# not affected by https://github.com/WayneD/rsync/issues/192 +find "${target}" -name '.~tmp~' -exec rm -r {} + + +rsync_cmd() { + local -a cmd=(rsync -rlptH --safe-links --delete-delay --delay-updates + "--timeout=600" "--contimeout=60" --no-motd) + + if stty &>/dev/null; then + cmd+=(-h -v --progress) + else + cmd+=(--quiet) + fi + + if ((bwlimit>0)); then + cmd+=("--bwlimit=$bwlimit") + fi + + "${cmd[@]}" "$@" +} + + +# if we are called without a tty (cronjob) only run when there are changes +if ! tty -s && [[ -f "$target/lastupdate" ]] && diff -b <(curl -Ls "$lastupdate_url") "$target/lastupdate" >/dev/null; then + # keep lastsync file in sync for statistics generated by the Arch Linux website + rsync_cmd "$source_url/lastsync" "$target/lastsync" + exit 0 +fi + +rsync_cmd \ + --exclude='*.links.tar.gz*' \ + --exclude='/other' \ + --exclude='/sources' \ + "${source_url}" \ + "${target}" + +#echo "Last sync was $(date -d @$(cat ${target}/lastsync))" \ No newline at end of file diff --git a/mirrors/files/distros/Debian/ISO/confd/debian-cd-mirror-sync.service b/mirrors/files/distros/Debian/ISO/confd/debian-cd-mirror-sync.service new file mode 100644 index 0000000..e7c7bf8 --- /dev/null +++ b/mirrors/files/distros/Debian/ISO/confd/debian-cd-mirror-sync.service @@ -0,0 +1,10 @@ +[Unit] +Description=Sync Debian-cd Mirror +After=network.target + +[Service] +Type=simple +User=mirrors +Group=mirrors +ExecStart=/usr/bin/rsync --bwlimit=2048 --times --links --hard-links --partial --block-size=8192 --delete --archive rsync://mirrors.ocf.berkeley.edu/debian-cd/ /srv/debian-cd/ +WorkingDirectory=/home/mirrors diff --git a/mirrors/files/distros/Debian/ISO/confd/debian-cd-mirror-sync.timer b/mirrors/files/distros/Debian/ISO/confd/debian-cd-mirror-sync.timer new file mode 100644 index 0000000..e5d0661 --- /dev/null +++ b/mirrors/files/distros/Debian/ISO/confd/debian-cd-mirror-sync.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Timer to sync Debian-cd Mirror at random minutes each hour + +[Timer] +OnCalendar=hourly +RandomizedDelaySec=3600 +Persistent=true + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/mirrors/files/distros/Debian/repository/confd/debian-mirror-sync.service b/mirrors/files/distros/Debian/repository/confd/debian-mirror-sync.service new file mode 100644 index 0000000..4c6929e --- /dev/null +++ b/mirrors/files/distros/Debian/repository/confd/debian-mirror-sync.service @@ -0,0 +1,10 @@ +[Unit] +Description=Sync Debian Mirror +After=network.target + +[Service] +Type=simple +User=mirrors +Group=mirrors +ExecStart=/home/mirrors/bin/debian-mirror-sync-check.sh +WorkingDirectory=/home/mirrors diff --git a/mirrors/files/distros/Debian/repository/confd/debian-mirror-sync.timer b/mirrors/files/distros/Debian/repository/confd/debian-mirror-sync.timer new file mode 100644 index 0000000..1d51027 --- /dev/null +++ b/mirrors/files/distros/Debian/repository/confd/debian-mirror-sync.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Timer to sync Debian Mirror every 15 minutes + +[Timer] +OnCalendar=*:0/15 +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/mirrors/files/distros/Debian/repository/ftpsync/.jigdo-mirror b/mirrors/files/distros/Debian/repository/ftpsync/.jigdo-mirror new file mode 100644 index 0000000..e011a55 --- /dev/null +++ b/mirrors/files/distros/Debian/repository/ftpsync/.jigdo-mirror @@ -0,0 +1,5 @@ +jigdoDir="/srv/debian-cd/current/jigdo" +imageDir="/srv/debian-cd/current/images" +tmpDir="/srv/debian-cd/current/images" +debianMirror="file:/srv/debian" +include='.'; exclude='-1\.' diff --git a/mirrors/files/distros/Debian/repository/ftpsync/bin/ftpsync b/mirrors/files/distros/Debian/repository/ftpsync/bin/ftpsync new file mode 100644 index 0000000..525a130 --- /dev/null +++ b/mirrors/files/distros/Debian/repository/ftpsync/bin/ftpsync @@ -0,0 +1,1259 @@ +#!/usr/bin/env bash +# No, we can not deal with sh alone. + +set -e +set -u +# ERR traps should be inherited from functions too. (And command +# substitutions and subshells and whatnot, but for us the function is +# the important part here) +set -E + +# A pipeline's return status is the value of the last (rightmost) +# command to exit with a non-zero status, or zero if all commands exit +# success fully. +set -o pipefail + +# ftpsync script for Debian +# Based losely on a number of existing scripts, written by an +# unknown number of different people over the years. +# +# Copyright (C) 2008-2016 Joerg Jaspert +# Copyright (C) 2016 Peter Palfrader +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; version 2. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +VERSION="20180513" +# -*- mode:sh -*- +# vim:syn=sh +# Little common functions + +# push a mirror attached to us. +# Arguments (using an array named SIGNAL_OPTS): +# +# $MIRROR - Name for the mirror, also basename for the logfile +# $HOSTNAME - Hostname to push to +# $USERNAME - Username there +# $SSHPROTO - Protocol version, either 1 or 2. +# $SSHKEY - the ssh private key file to use for this push +# $SSHOPTS - any other option ssh accepts, passed blindly, be careful +# $PUSHLOCKOWN - own lockfile name to touch after stage1 in pushtype=staged +# $PUSHTYPE - what kind of push should be done? +# all - normal, just push once with ssh backgrounded and finish +# staged - staged. first push stage1, then wait for $PUSHLOCKs to appear, +# then push stage2 +# $PUSHARCHIVE - what archive to sync? (Multiple mirrors behind one ssh key!) +# $PUSHCB - do we want a callback? +# $PUSHKIND - whats going on? are we doing mhop push or already stage2? +# $FROMFTPSYNC - set to true if we run from within ftpsync. +# +# This function assumes that the variable LOG is set to a directory where +# logfiles can be written to. +# Additionally $PUSHLOCKS has to be defined as a set of space delimited strings +# (list of "lock"files) to wait for if you want pushtype=staged +# +# Pushes might be done in background (for type all). +signal () { + ARGS="SIGNAL_OPTS[*]" + local ${!ARGS} + + MIRROR=${MIRROR:-""} + HOSTNAME=${HOSTNAME:-""} + USERNAME=${USERNAME:-""} + SSHPROTO=${SSHPROTO:-""} + SSHKEY=${SSHKEY:-""} + SSHOPTS=${SSHOPTS:-""} + PUSHLOCKOWN=${PUSHLOCKOWN:-""} + PUSHTYPE=${PUSHTYPE:-"all"} + PUSHARCHIVE=${PUSHARCHIVE:-""} + PUSHCB=${PUSHCB:-""} + PUSHKIND=${PUSHKIND:-"all"} + FROMFTPSYNC=${FROMFTPSYNC:-"false"} + + # And now get # back to space... + SSHOPTS=${SSHOPTS/\#/ } + + # Defaults we always want, no matter what + SSH_OPTIONS="-o user=${USERNAME} -o BatchMode=yes -o ServerAliveInterval=45 -o ConnectTimeout=45 -o PasswordAuthentication=no" + + # If there are userdefined ssh options, add them. + if [[ -n ${SSH_OPTS} ]]; then + SSH_OPTIONS="${SSH_OPTIONS} ${SSH_OPTS}" + fi + + # Does this machine need a special key? + if [[ -n ${SSHKEY} ]]; then + SSH_OPTIONS="${SSH_OPTIONS} -i ${SSHKEY}" + fi + + # Does this machine have an extra own set of ssh options? + if [[ -n ${SSHOPTS} ]]; then + SSH_OPTIONS="${SSH_OPTIONS} ${SSHOPTS}" + fi + + # Set the protocol version + if [[ ${SSHPROTO} -ne 1 ]] && [[ ${SSHPROTO} -ne 2 ]] && [[ ${SSHPROTO} -ne 99 ]]; then + # Idiots, we only want 1 or 2. Cant decide? Lets force 2. + SSHPROTO=2 + fi + + if [[ -n ${SSHPROTO} ]] && [[ ${SSHPROTO} -ne 99 ]]; then + SSH_OPTIONS="${SSH_OPTIONS} -${SSHPROTO}" + fi + + date -u >> "${LOGDIR}/${MIRROR}.log" + + PUSHARGS="" + # PUSHARCHIVE empty or not, we always add the sync:archive: command to transfer. + # Otherwise, if nothing else is added, ssh -f would not work ("no command to execute") + # But ftpsync does treat "sync:archive:" as the main archive, so this works nicely. + PUSHARGS="${PUSHARGS} sync:archive:${PUSHARCHIVE}" + + # We have a callback wish, tell downstreams + if [[ -n ${PUSHCB} ]]; then + PUSHARGS="${PUSHARGS} sync:callback" + fi + # If we are running an mhop push AND our downstream is one to receive it, tell it. + if [[ mhop = ${PUSHKIND} ]] && [[ mhop = ${PUSHTYPE} ]]; then + PUSHARGS="${PUSHARGS} sync:mhop" + fi + + if [[ all = ${PUSHTYPE} ]]; then + # Default normal "fire and forget" push. We background that, we do not care about the mirrors doings + PUSHARGS1="sync:all" + signal_ssh "normal" "${MIRROR}" "${HOSTNAME}" $SSH_OPTIONS "${PUSHARGS} ${PUSHARGS1}" + elif [[ staged = ${PUSHTYPE} ]] || [[ mhop = ${PUSHTYPE} ]]; then + # Want a staged push. Fine, lets do that. Not backgrounded. We care about the mirrors doings. + # Only send stage1 if we havent already send it. When called with stage2, we already did. + if [[ stage2 != ${PUSHKIND} ]]; then + # Step1: Do a push to only sync stage1, do not background + PUSHARGS1="sync:stage1" + signal_ssh "first stage" "${MIRROR}" "${HOSTNAME}" $SSH_OPTIONS "${PUSHARGS} ${PUSHARGS1}" + touch "${PUSHLOCKOWN}" + + # Step2: Wait for all the other "lock"files to appear. + # In case we did not have all PUSHLOCKS and still continued, note it + # This is a little racy, especially if the other parts decide to do this + # at the same time, but it wont hurt more than a mail too much, so I don't care much + if ! wait_for_pushlocks ${PUSHDELAY}; then + msg "Failed to wait for all other mirrors. Failed ones are:" >> "${LOGDIR}/${MIRROR}.log" + for file in ${PUSHLOCKS}; do + if [[ ! -f ${file} ]]; then + msg "${file}" >> "${LOGDIR}/${MIRROR}.log" + log "Missing Pushlockfile ${file} after waiting for more than ${PUSHDELAY} seconds, continuing" + fi + done + fi + rm -f "${PUSHLOCKOWN}" + fi + + # Step3: It either timed out or we have all the "lock"files, do the rest + # If we are doing mhop AND are called from ftpsync - we now exit. + # That way we notify our uplink that we and all our clients are done with their + # stage1. It can then finish its own, and if all our upstreams downlinks are done, + # it will send us stage2. + # If we are not doing mhop or are not called from ftpsync, we start stage2 + if [[ true = ${FROMFTPSYNC} ]] && [[ mhop = ${PUSHKIND} ]]; then + return + else + PUSHARGS2="sync:stage2" + signal_ssh "second stage" "${MIRROR}" "${HOSTNAME}" $SSH_OPTIONS "${PUSHARGS} ${PUSHARGS2}" + fi + else + # Can't decide? Then you get nothing. + return + fi +} + +signal_ssh() { + local t=$1 + local mirror_log="${LOGDIR}/${2}.log" + local hostname=$3 + shift 3 + + msg "Sending ${t} trigger" >> $mirror_log + output=$(ssh -n $hostname "$@" 2>&1 | tee -a $mirror_log) + if [[ $? -eq 255 ]]; then + error_mailf "${t} trigger failed: $hostname" -b "$output" + else + log "${t} trigger succeeded: $hostname" + fi +} + +wait_for_pushlocks() { + local tries=0 + local found + local total + local timeout=${1}; shift + # We do not wait forever + while [[ ${tries} -lt ${timeout} ]]; do + total=0 + found=0 + for file in ${PUSHLOCKS}; do + total=$(( total + 1 )) + if [[ -f ${file} ]]; then + found=$(( found + 1 )) + fi + done + if [[ ${total} -eq ${found} ]] || [[ -f ${LOCKDIR}/all_stage1 ]]; then + break + fi + tries=$(( tries + 5 )) + sleep 5 + done + # Regardless of the state of our siblings, hitting one timeout cancels all waits + touch "${LOCKDIR}/all_stage1" + if [[ ${tries} -ge ${timeout} ]]; then + return 1 + else + return 0 + fi +} + +# callback, used by ftpsync +callback () { + # Defaults we always want, no matter what + SSH_OPTIONS="-o BatchMode=yes -o ServerAliveInterval=45 -o ConnectTimeout=45 -o PasswordAuthentication=no" + ssh -n $SSH_OPTIONS -i "$3" -o"user $1" "$2" callback:${HOSTNAME} +} + +# open log file +open_log() { + local log=$1 + shift + exec 4>&1 1>>$log +} + +# assemble log message (basically echo it together with a timestamp) +# +# Set $PROGRAM to a string to have it added to the output. +msg() { + if [[ -z "${PROGRAM}" ]]; then + echo "$(date +"%b %d %H:%M:%S") $(hostname -s) [$$] $@" + else + echo "$(date +"%b %d %H:%M:%S") $(hostname -s) ${PROGRAM}[$$]: $@" + fi +} + +# log something +log() { + msg "$@" +} + +# log the message using log() but then also send a mail +# to the address configured in MAILTO (if non-empty) +error () { + log "$@" + LOG_ERROR=1 + mailf -s "[$PROGRAM@$(hostname -s)] ERROR: $*" -b "$*" ${MAILTO} +} + +# log the message using log() but then also send a mail +# to the address configured in MAILTO (if non-empty) +error_mailf () { + local m="$1" + shift + log "$m" + LOG_ERROR=1 + mailf -s "[$PROGRAM@$(hostname -s)] ERROR: $m" "$@" ${MAILTO} +} + +# run a hook +# needs array variable HOOK setup with HOOKNR being a number an HOOKSCR +# the script to run. +hook () { + ARGS='HOOK[@]' + local "${!ARGS}" + if [[ -n ${HOOKSCR} ]]; then + log "Running hook $HOOKNR: ${HOOKSCR}" + set +e + ${HOOKSCR} + result=$? + set -e + if [[ ${result} -ne 0 ]] ; then + error "Back from hook $HOOKNR, got returncode ${result}" + else + log "Back from hook $HOOKNR, got returncode ${result}" + fi + return $result + else + return 0 + fi +} + +# Return the list of 2-stage mirrors. +get2stage() { + egrep -s '^(staged|mhop)' "${MIRRORS}" | { + while read MTYPE MLNAME MHOSTNAME MUSER MPROTO MKEYFILE; do + PUSHLOCKS="${LOCKDIR}/${MLNAME}.stage1 ${PUSHLOCKS}" + done + echo "$PUSHLOCKS" + } +} + +# Rotate logfiles +savelog() { + torotate="$1" + count=${2:-${LOGROTATE}} + while [[ ${count} -gt 0 ]]; do + prev=$(( count - 1 )) + if [[ -e ${torotate}.${prev} ]]; then + mv "${torotate}.${prev}" "${torotate}.${count}" + fi + count=$prev + done + if [[ -e ${torotate} ]]; then + mv "${torotate}" "${torotate}.0" + fi +} + +# Return rsync version +rsync_protocol() { + RSYNC_VERSION="$(${RSYNC} --version)" + RSYNC_REGEX="(protocol[ ]+version[ ]+([0-9]+))" + if [[ ${RSYNC_VERSION} =~ ${RSYNC_REGEX} ]]; then + echo ${BASH_REMATCH[2]} + fi + unset RSYNC_VERSION RSYNC_REGEX +} + +extract_trace_field() { + local field="$1" + local file="$2" + local value=$(awk -F': ' "\$1==\"$field\" {print \$2; exit}" "$file" 2>/dev/null) + [[ $value ]] || return 1 + echo $value +} + +extract_trace_field_string() { + local field="$1" + local string="$2" + local value=$(awk -F': ' "\$1==\"$field\" {print \$2; exit}" <<< "$string" 2>/dev/null) + [[ $value ]] || return 1 + echo $value +} + +extract_trace_serial() { + extract_trace_field 'Archive serial' "$1" + return $? +} + +extract_trace_serial_string() { + extract_trace_field_string 'Archive serial' "$1" + return $? +} + +# Search config files in various locations +search_config() { + local file + for i in ${CONFDIRS[@]}; do + file="$i/$1" + if [ -f "$file" ]; then + echo "$file" + return + fi + done +} + +# Read config file +read_config() { + local name=$(echo "$1" | sed -e 's/[^A-Za-z0-9._-]/_/g') + local config=$(search_config "$name") + if [ "$config" ]; then + . "$config" + CURRENT_CONFIG="$config" + return 0 + else + echo "Can't read config file ${name}!" >&2 + exit 78 # EX_CONFIG + fi +} + +# Create lock dir +create_lockdir() { + mkdir -p "$LOCKDIR" +} + +# Create log dir +create_logdir() { + mkdir -p "$LOGDIR" +} + +join_by() { + local IFS="$1" + shift + echo $* +} + +# Sends mail +# mailf [-a attachment] [-b body] [-s subject] to-addr ... +mailf() { + local boundary="==--$RANDOM--$RANDOM--$RANDOM--==" + local attachment=() + local body=() + local subject= + + OPTIND=1 + while getopts ":a:b:s:" arg; do + case $arg in + a) + attachment+=("$OPTARG") + ;; + b) + body+=("$OPTARG") + ;; + s) + subject="$OPTARG" + ;; + esac + done + shift $((OPTIND-1)) + + ( + cat < "${LOGDIR}/ftpsync.newversion" + fi + fi + else + # Remove a possible stampfile + rm -f "${LOGDIR}/ftpsync.newversion" + fi + fi +} + +######################################################################## +######################################################################## +## functions ## +######################################################################## +######################################################################## +check_commandline() { + while [[ $# -gt 0 ]]; do + case "$1" in + sync:stage1) + SYNCSTAGE1="true" + SYNCALL="false" + ;; + sync:stage2) + SYNCSTAGE2="true" + SYNCALL="false" + ;; + sync:callback) + SYNCCALLBACK="true" + ;; + sync:archive:*) + ARCHIVE=${1##sync:archive:} + ;; + sync:all) + SYNCALL="true" + ;; + sync:mhop) + SYNCMHOP="true" + ;; + *) + echo "Unknown option ${1} ignored" + ;; + esac + shift # Check next set of parameters. + done +} + +# All the stuff we want to do when we exit, no matter where +cleanup() { + rc=$? + + trap - ERR TERM HUP INT QUIT EXIT + # all done. Mail the log, exit. + + if [[ $rc -gt 0 ]]; then + log "Mirrorsync done with errors" + else + log "Mirrorsync done" + fi + + if [[ -n ${MAILTO} ]]; then + local args=() + local send= + local subject="SUCCESS" + + # In case rsync had something on stderr + if [[ -s $LOG_RSYNC_ERROR ]]; then + args+=(-a $LOG_RSYNC_ERROR -a $LOG) + subject="ERROR: rsync errors" + send=1 + # In case of direct errors + elif [[ $rc -gt 0 ]]; then + args+=(-a $LOG) + subject="ERROR" + send=1 + # In case admin want all logs + elif [[ ${ERRORSONLY} = false ]]; then + args+=(-a $LOG) + if [[ ${LOG_ERROR:-} ]]; then + subject="ERROR" + fi + send=1 + fi + if [[ $send ]]; then + # Someone wants full logs including rsync + if [[ ${FULLLOGS} = true ]]; then + args+=(-a $LOG_RSYNC) + fi + mailf "${args[@]}" -s "[${PROGRAM}@$(hostname -s)] ${subject}" ${MAILTO} + fi + fi + + savelog "${LOG_RSYNC}" + savelog "${LOG_RSYNC_ERROR}" + savelog "$LOG" > /dev/null + + rm -f "${LOCK}" + + exit $rc +} + +run_rsync() { + local t=$1 + shift + + log "Running $t:" "${_RSYNC[@]}" "$@" + + "${_RSYNC[@]}" "$@" \ + >>"${LOG_RSYNC_ERROR}" 2>&1 || return $? +} + +# Check rsyncs return value +check_rsync() { + ret=$1 + msg=$2 + + # Lets get a statistical value + if [[ -f ${LOG_RSYNC} ]]; then + SPEED=$(tail -n 2 ${LOG_RSYNC} | sed -Ene 's#.* ([0-9.,]+) bytes/sec#\1#p') + if [[ ${SPEED} ]]; then + SPEED=${SPEED%%.*} + SPEED=${SPEED//,} + SPEED=$(( SPEED / 1024 )) + log "Latest recorded rsync transfer speed: ${SPEED} KB/s" + fi + fi + + # 24 - vanished source files. Ignored, that should be the target of $UPDATEREQUIRED + # and us re-running. If it's not, uplink is broken anyways. + case "${ret}" in + 0) return 0;; + 24) return 0;; + 23) return 2;; + 30) return 2;; + *) + error "ERROR: ${msg}" + return 1 + ;; + esac +} + +function tracefile_content() { + set +e + + LC_ALL=POSIX LANG=POSIX date -u + rfc822date=$(LC_ALL=POSIX LANG=POSIX date -u -R) + echo "Date: ${rfc822date}" + echo "Date-Started: ${DATE_STARTED}" + + if [[ -e $TRACEFILE_MASTER ]]; then + echo "Archive serial: $(extract_trace_serial $TRACEFILE_MASTER || echo unknown )" + fi + + echo "Used ftpsync version: ${VERSION}" + echo "Creator: ftpsync ${VERSION}" + echo "Running on host: ${TRACEHOST}" + + if [[ ${INFO_MAINTAINER:-} ]]; then + echo "Maintainer: ${INFO_MAINTAINER}" + fi + if [[ ${INFO_SPONSOR:-} ]]; then + echo "Sponsor: ${INFO_SPONSOR}" + fi + if [[ ${INFO_COUNTRY:-} ]]; then + echo "Country: ${INFO_COUNTRY}" + fi + if [[ ${INFO_LOCATION:-} ]]; then + echo "Location: ${INFO_LOCATION}" + fi + if [[ ${INFO_THROUGHPUT:-} ]]; then + echo "Throughput: ${INFO_THROUGHPUT}" + fi + if [[ ${INFO_TRIGGER:-} ]]; then + echo "Trigger: ${INFO_TRIGGER}" + fi + + if [[ -d ${TO}/dists ]]; then + ARCH=$(find ${TO}/dists \( -name 'Packages.*' -o -name 'Sources.*' \) 2>/dev/null | + sed -Ene 's#.*/binary-([^/]+)/Packages.*#\1#p; s#.*/(source)/Sources.*#\1#p' | + sort -u | tr '\n' ' ') + if [[ $ARCH ]]; then + echo "Architectures: ${ARCH}" + fi + fi + if [[ ${ARCH_INCLUDE} ]]; then + echo "Architectures-Configuration: INCLUDE $(tr ' ' '\n' <<< ${ARCH_INCLUDE} | sort -u | tr '\n' ' ')" + elif [[ ${ARCH_EXCLUDE} ]]; then + echo "Architectures-Configuration: EXCLUDE $(tr ' ' '\n' <<< ${ARCH_EXCLUDE} | sort -u | tr '\n' ' ')" + else + echo "Architectures-Configuration: ALL" + fi + echo "Upstream-mirror: ${RSYNC_HOST:-unknown}" + echo "Rsync-Transport: ${RSYNC_TRANSPORT}" + total=0 + if [[ -e ${LOG_RSYNC} ]]; then + for bytes in $(sed -Ene 's/(^|.* )sent ([0-9]+) bytes received ([0-9]+) bytes.*/\3/p' "${LOG_RSYNC}"); do + total=$(( total + bytes )) + done + if [[ $total -gt 0 ]]; then + echo "Total bytes received in rsync: ${total}" + fi + fi + total_time=$(( STATS_TOTAL_RSYNC_TIME1 + STATS_TOTAL_RSYNC_TIME2 )) + echo "Total time spent in stage1 rsync: ${STATS_TOTAL_RSYNC_TIME1}" + echo "Total time spent in stage2 rsync: ${STATS_TOTAL_RSYNC_TIME2}" + echo "Total time spent in rsync: ${total_time}" + if [[ 0 != ${total_time} ]]; then + rate=$(( total / total_time )) + echo "Average rate: ${rate} B/s" + fi + + set -e +} + +# Write a tracefile +tracefile() { + local TRACEFILE=${1:-"${TO}/${TRACE}"} + local TRACEFILE_MASTER="${TO}/${TRACEDIR}/master" + + tracefile_content > "${TRACEFILE}.new" + mv "${TRACEFILE}.new" "${TRACEFILE}" + + { + if [[ -e ${TO}/${TRACEHIERARCHY}.mirror ]]; then + cat ${TO}/${TRACEHIERARCHY}.mirror + fi + echo "$(basename "${TRACEFILE}") ${MIRRORNAME} ${TRACEHOST} ${RSYNC_HOST:-unknown}" + } > "${TO}/${TRACEHIERARCHY}".new + mv "${TO}/${TRACEHIERARCHY}".new "${TO}/${TRACEHIERARCHY}" + cp "${TO}/${TRACEHIERARCHY}" "${TO}/${TRACEHIERARCHY}.mirror" + + (cd "${TO}/${TRACEDIR}" && ls -1rt $(find * -type f \! -name "_*")) > "${TO}/${TRACELIST}" +} + +arch_imexclude() { + local param="$1" arch="$2" + if [[ $arch = source ]]; then + _RSYNC+=( + "--filter=${param}_/dists/**/source/" + "--filter=${param}_/pool/**/*.tar.*" + "--filter=${param}_/pool/**/*.diff.*" + "--filter=${param}_/pool/**/*.dsc" + ) + else + _RSYNC+=( + "--filter=${param}_/dists/**/binary-${arch}/" + "--filter=${param}_/dists/**/installer-${arch}/" + "--filter=${param}_/dists/**/Contents-${arch}.gz" + "--filter=${param}_/dists/**/Contents-udeb-${arch}.gz" + "--filter=${param}_/dists/**/Contents-${arch}.diff/" + "--filter=${param}_/indices/**/arch-${arch}.files" + "--filter=${param}_/indices/**/arch-${arch}.list.gz" + "--filter=${param}_/pool/**/*_${arch}.deb" + "--filter=${param}_/pool/**/*_${arch}.udeb" + "--filter=${param}_/pool/**/*_${arch}.changes" + ) + fi +} + +arch_exclude() { + arch_imexclude exclude "$1" +} + +arch_include() { + arch_imexclude include "$1" +} + +# Learn which archs to include/exclude based on ARCH_EXCLUDE and ARCH_INCLUDE +# settings. +# Sets EXCLUDE (which might also have --include statements +# followed by a --exclude *_*.. +set_exclude_include_archs() { + if [[ -n "${ARCH_EXCLUDE}" ]] && [[ -n "${ARCH_INCLUDE}" ]]; then + echo >&2 "ARCH_EXCLUDE and ARCH_INCLUDE are mutually exclusive. Set only one." + exit 1 + fi + + if [[ -n "${ARCH_EXCLUDE}" ]]; then + for ARCH in ${ARCH_EXCLUDE}; do + arch_exclude ${ARCH} + done + arch_include '*' + arch_include source + elif [[ -n "${ARCH_INCLUDE}" ]]; then + local include_arch_all=false + for ARCH in ${ARCH_INCLUDE}; do + arch_include ${ARCH} + if [[ ${ARCH} != source ]]; then + include_arch_all=true + fi + done + if [[ true = ${include_arch_all} ]]; then + arch_include all + fi + arch_exclude '*' + arch_exclude source + fi +} + +######################################################################## +######################################################################## + + +# As what are we called? +NAME="$(basename $0)" + +# What should we do? +ARCHIVE= +# Do we sync stage1? +SYNCSTAGE1=false +# Do we sync stage2? +SYNCSTAGE2=false +# Do we sync all? +SYNCALL=true +# Do we have a mhop sync? +SYNCMHOP=false +# Do we callback? (May get changed later) +SYNCCALLBACK=false + +while getopts T: option; do + case $option in + T) INFO_TRIGGER=$OPTARG;; + ?) exit 64;; + esac +done +shift $(($OPTIND - 1)) + +# Now, check if we got told about stuff via ssh +if [[ -n ${SSH_ORIGINAL_COMMAND:-} ]]; then + INFO_TRIGGER=${INFO_TRIGGER:-ssh} + check_commandline ${SSH_ORIGINAL_COMMAND} +fi + +# Now, we can locally override all the above variables by just putting +# them into the .ssh/authorized_keys file forced command. +if [[ $# -gt 0 ]]; then + check_commandline "$@" +fi + +# If we have been told to do stuff for a different archive than default, +# set the name accordingly. +if [[ -n ${ARCHIVE} ]]; then + NAME="${NAME}-${ARCHIVE}" +fi + +# Now source the config for the archive we run on. +# (Yes, people can also overwrite the options above in the config file +# if they want to) +read_config "${NAME}.conf" + +create_logdir + +######################################################################## +# Config defaults # +######################################################################## +MIRRORNAME=${MIRRORNAME:-$(hostname -f)} +TO=${TO:-"/srv/mirrors/debian/"} +MAILTO=${MAILTO:-${LOGNAME:?Environment variable LOGNAME unset, please check your system or specify MAILTO}} +HUB=${HUB:-"false"} + +# Connection options +if [[ -z ${RSYNC_SOURCE:-} ]]; then + RSYNC_HOST=${RSYNC_HOST:?Missing a host to mirror from, please set RSYNC_HOST variable in ${CURRENT_CONFIG}} + RSYNC_PATH=${RSYNC_PATH:-"debian"} + RSYNC_USER=${RSYNC_USER:-""} +fi +RSYNC_PASSWORD=${RSYNC_PASSWORD:-""} +if [[ ${RSYNC_SSL:-} = true ]]; then + RSYNC_TRANSPORT=${RSYNC_TRANSPORT:-"ssl"} +else + RSYNC_TRANSPORT=${RSYNC_TRANSPORT:-"undefined"} +fi +RSYNC_SSL_PORT=${RSYNC_SSL_PORT:-"1873"} +RSYNC_SSL_CAPATH=${RSYNC_SSL_CAPATH:-"/etc/ssl/certs"} +RSYNC_SSL_METHOD=${RSYNC_SSL_METHOD:-"stunnel"} +RSYNC_PROXY=${RSYNC_PROXY:-""} + +# Include and exclude options +ARCH_INCLUDE=${ARCH_INCLUDE:-""} +ARCH_EXCLUDE=${ARCH_EXCLUDE:-""} +EXCLUDE=${EXCLUDE:-""} + +# Log options +LOG=${LOG:-"${LOGDIR}/${NAME}.log"} +ERRORSONLY=${ERRORSONLY:-"true"} +FULLLOGS=${FULLLOGS:-"false"} +LOGROTATE=${LOGROTATE:-14} +LOG_RSYNC="${LOGDIR}/rsync-${NAME}.log" +LOG_RSYNC_ERROR="${LOGDIR}/rsync-${NAME}.error" + +# Other options +LOCKTIMEOUT=${LOCKTIMEOUT:-3600} +UIPSLEEP=${UIPSLEEP:-1200} +UIPRETRIES=${UIPRETRIES:-3} +TRACEHOST=${TRACEHOST:-$(hostname -f)} +RSYNC=${RSYNC:-rsync} +RSYNC_PROTOCOL=$(rsync_protocol) +RSYNC_EXTRA=${RSYNC_EXTRA:-""} +RSYNC_BW=${RSYNC_BW:-0} +if [[ $RSYNC_PROTOCOL -ge 31 ]]; then + RSYNC_OPTIONS=${RSYNC_OPTIONS:-"-prltvHSB8192 --safe-links --chmod=D755,F644 --timeout 120 --stats --no-human-readable --no-inc-recursive"} +else + RSYNC_OPTIONS=${RSYNC_OPTIONS:-"-prltvHSB8192 --safe-links --timeout 120 --stats --no-human-readable --no-inc-recursive"} +fi +RSYNC_OPTIONS1=${RSYNC_OPTIONS1:-"--include=*.diff/ --exclude=*.diff/Index --exclude=Packages* --exclude=Sources* --exclude=Release* --exclude=InRelease --include=i18n/by-hash --exclude=i18n/* --exclude=ls-lR*"} +RSYNC_OPTIONS2=${RSYNC_OPTIONS2:-"--max-delete=40000 --delay-updates --delete --delete-delay --delete-excluded"} +CALLBACKUSER=${CALLBACKUSER:-"archvsync"} +CALLBACKHOST=${CALLBACKHOST:-"none"} +CALLBACKKEY=${CALLBACKKEY:-"none"} + +# Hooks +HOOK1=${HOOK1:-""} +HOOK2=${HOOK2:-""} +HOOK3=${HOOK3:-""} +HOOK4=${HOOK4:-""} +HOOK5=${HOOK5:-""} +######################################################################## +######################################################################## + +# used by log() and error() +PROGRAM=${PROGRAM:-"${NAME}"} + +# Our trace and lock files +LOCK_NAME="Archive-Update-in-Progress-${MIRRORNAME}" +LOCK="${TO}/${LOCK_NAME}" +UPDATEREQUIRED_NAME="Archive-Update-Required-${MIRRORNAME}" +UPDATEREQUIRED="${TO}/${UPDATEREQUIRED_NAME}" +TRACEDIR=project/trace +TRACE="${TRACEDIR}/${MIRRORNAME}" +TRACE_STAGE1="${TRACEDIR}/${MIRRORNAME}-stage1" +TRACEHIERARCHY="${TRACEDIR}/_hierarchy" +TRACELIST="${TRACEDIR}/_traces" + +_TRACE_FILES=( + "${LOCK_NAME}" + "${UPDATEREQUIRED_NAME}" + "${TRACE}" + "${TRACE_STAGE1}" + "${TRACEHIERARCHY}" + "${TRACELIST}" +) + +_RSYNC=( + $RSYNC + --quiet + --log-file "${LOG_RSYNC}" +) + +# Rsync filter rules. Used to protect various files we always want to keep, even if we otherwise delete +# excluded files +for i in ${_TRACE_FILES[@]}; do + _RSYNC+=("--filter=exclude_/${i}" "--filter=protect_/${i}") +done +_RSYNC+=( + "--filter=include_/project/" + "--filter=protect_/project/" + "--filter=include_/project/trace/" + "--filter=protect_/project/trace/" + "--filter=include_/project/trace/*" +) + +# Default rsync options for *every* rsync call +# Now add the bwlimit option. As default is 0 we always add it, rsync interprets +# 0 as unlimited, so this is safe. +_RSYNC+=(${RSYNC_EXTRA} --bwlimit=${RSYNC_BW} ${RSYNC_OPTIONS} ${EXCLUDE}) + +# collect some stats +STATS_TOTAL_RSYNC_TIME1=0 +STATS_TOTAL_RSYNC_TIME2=0 + +# The temp directory used by rsync --delay-updates is not +# world-readable remotely. Always exclude it to avoid errors. +_RSYNC+=("--exclude=.~tmp~/") + +if [[ ${RSYNC_TRANSPORT} = undefined ]]; then + : +elif [[ ${RSYNC_TRANSPORT} = ssh ]]; then + _RSYNC+=(-e "ssh") +elif [[ ${RSYNC_TRANSPORT} = ssl ]]; then + export RSYNC_SSL_PORT + export RSYNC_SSL_CAPATH + export RSYNC_SSL_METHOD + _RSYNC+=(-e "${BINDIR:+${BINDIR}/}rsync-ssl-tunnel") +else + echo "Unknown rsync transport configured (${RSYNC_TRANSPORT})" >&2 + exit 1 +fi + +# Exclude architectures defined in $ARCH_EXCLUDE +set_exclude_include_archs + +######################################################################## +# Really nothing to see below here. Only code follows. # +######################################################################## +######################################################################## +DATE_STARTED=$(LC_ALL=POSIX LANG=POSIX date -u -R) + +# Some sane defaults +cd "${BASEDIR:-}" +umask 022 + +# If we are here for the first time, create the +# destination and the trace directory +mkdir -p "${TO}/${TRACEDIR}" + +# Used to make sure we will have the archive fully and completly synced before +# we stop, even if we get multiple pushes while this script is running. +# Otherwise we can end up with a half-synced archive: +# - get a push +# - sync, while locked +# - get another push. Of course no extra sync run then happens, we are locked. +# - done. Archive not correctly synced, we don't have all the changes from the second push. +touch "${UPDATEREQUIRED}" + +# Check to see if another sync is in progress +if ! ( set -o noclobber; echo "$$" > "${LOCK}") 2> /dev/null; then + if [[ ${BASH_VERSINFO[0]} -gt 3 ]] || [[ -L /proc/self ]]; then + # We have a recent enough bash version, lets do it the easy way, + # the lock will contain the right pid, thanks to $BASHPID + if ! $(kill -0 $(< ${LOCK}) 2>/dev/null); then + # Process does either not exist or is not owned by us. + echo "$$" > "${LOCK}" + else + echo "Unable to start rsync, lock file still exists, PID $(< ${LOCK})" + exit 1 + fi + else + # Old bash, means we dont have the right pid in our lockfile + # So take a different way - guess if it is still there by comparing its age. + # Not optimal, but hey. + stamptime=$(date --reference="${LOCK}" +%s) + unixtime=$(date +%s) + difference=$(( $unixtime - $stamptime )) + if [[ ${difference} -ge ${LOCKTIMEOUT} ]]; then + # Took longer than LOCKTIMEOUT minutes? Assume it broke and take the lock + echo "$$" > "${LOCK}" + else + echo "Unable to start rsync, lock file younger than one hour" + exit 1 + fi + fi +fi + +# We want to cleanup always +trap cleanup EXIT TERM HUP INT QUIT + +# Open log and close stdin +open_log $LOG +exec 2>&1 <&- +log "Mirrorsync start" + +# Look who pushed us and note that in the log. +SSH_CONNECTION=${SSH_CONNECTION:-""} +PUSHFROM="${SSH_CONNECTION%%\ *}" +if [[ -n ${PUSHFROM} ]]; then + log "We got pushed from ${PUSHFROM}" +fi + +if [[ true = ${SYNCCALLBACK} ]]; then + if [[ none = ${CALLBACKHOST} ]] || [[ none = ${CALLBACKKEY} ]]; then + SYNCCALLBACK="false" + error "We are asked to call back, but we do not know where to and do not have a key, ignoring callback" + fi +fi + +HOOK=( + HOOKNR=1 + HOOKSCR=${HOOK1} +) +hook $HOOK + +# Now, we might want to sync from anonymous too. +# This is that deep in this script so hook1 could, if wanted, change things! +if [[ -z ${RSYNC_SOURCE:-} ]]; then + if [[ -z ${RSYNC_USER:-} ]]; then + RSYNC_SOURCE="${RSYNC_HOST}::${RSYNC_PATH}" + else + RSYNC_SOURCE="${RSYNC_USER}@${RSYNC_HOST}::${RSYNC_PATH}" + fi +fi + +_RSYNC+=("${RSYNC_SOURCE}" "$TO") + +# Now do the actual mirroring, and run as long as we have an updaterequired file. +export RSYNC_PASSWORD +export RSYNC_PROXY + +UPDATE_RETRIES=0 + +while [[ -e ${UPDATEREQUIRED} ]]; do + log "Running mirrorsync, update is required, ${UPDATEREQUIRED} exists" + + # if we want stage1 *or* all + if [[ true = ${SYNCSTAGE1} ]] || [[ true = ${SYNCALL} ]]; then + while [[ -e ${UPDATEREQUIRED} ]]; do + rm -f "${UPDATEREQUIRED}" + # Step one, sync everything except Packages/Releases + rsync_started=$(date +%s) + result=0 + run_rsync "stage1" ${RSYNC_OPTIONS1} || result=$? + rsync_ended=$(date +%s) + STATS_TOTAL_RSYNC_TIME1=$(( STATS_TOTAL_RSYNC_TIME1 + rsync_ended - rsync_started )) + + log "Back from rsync with returncode ${result}" + done + else + time1=$(extract_trace_field 'Total time spent in stage1 rsync' "${TO}/${TRACE_STAGE1}" || :) + if [[ $time1 ]]; then + STATS_TOTAL_RSYNC_TIME1="$time1" + fi + # Fake a good resultcode + result=0 + fi # Sync stage 1? + rm -f "${UPDATEREQUIRED}" + + set +e + check_rsync $result "Sync step 1 went wrong, got errorcode ${result}. Logfile: ${LOG}" + GO=$? + set -e + if [[ ${GO} -eq 2 ]] && [[ -e ${UPDATEREQUIRED} ]]; then + log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now" + elif [[ ${GO} -ne 0 ]]; then + exit 3 + fi + + HOOK=( + HOOKNR=2 + HOOKSCR=${HOOK2} + ) + hook $HOOK + + # if we want stage2 *or* all + if [[ true = ${SYNCSTAGE2} ]] || [[ true = ${SYNCALL} ]]; then + upstream_uip=false + for aupfile in "${TO}/Archive-Update-in-Progress-"*; do + case "$aupfile" in + "${TO}/Archive-Update-in-Progress-*") + error "Lock file is missing, this should not happen" + ;; + "${LOCK}") + : + ;; + *) + if [[ -f $aupfile ]]; then + # Remove the file, it will be synced again if + # upstream is still not done + rm -f "$aupfile" + else + log "AUIP file '$aupfile' is not really a file, weird" + fi + upstream_uip=true + ;; + esac + done + + if [[ true = ${upstream_uip} ]]; then + log "Upstream archive update in progress, skipping stage2" + if [[ ${UPDATE_RETRIES} -lt ${UIPRETRIES} ]]; then + log "Retrying update in ${UIPSLEEP}" + touch "${UPDATEREQUIRED}" + UPDATE_RETRIES=$(($UPDATE_RETRIES+1)) + sleep "${UIPSLEEP}" + result=0 + else + error "Update has been retried ${UPDATE_RETRIES} times, aborting" + log "Perhaps upstream is still updating or there's a stale AUIP file" + result=1 + fi + else + # We are lucky, it worked. Now do step 2 and sync again, this time including + # the packages/releases files + rsync_started=$(date +%s) + result=0 + run_rsync "stage2" ${RSYNC_OPTIONS2} || result=$? + rsync_ended=$(date +%s) + STATS_TOTAL_RSYNC_TIME2=$(( STATS_TOTAL_RSYNC_TIME2 + rsync_ended - rsync_started )) + + log "Back from rsync with returncode ${result}" + fi + else + # Fake a good resultcode + result=0 + fi # Sync stage 2? + + set +e + check_rsync $result "Sync step 2 went wrong, got errorcode ${result}. Logfile: ${LOG}" + GO=$? + set -e + if [[ ${GO} -eq 2 ]] && [[ -e ${UPDATEREQUIRED} ]]; then + log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now" + elif [[ ${GO} -ne 0 ]]; then + exit 4 + fi + + HOOK=( + HOOKNR=3 + HOOKSCR=${HOOK3} + ) + hook $HOOK +done + +# We only update our tracefile when we had a stage2 or an all sync. +# Otherwise we would update it after stage1 already, which is wrong. +if [[ true = ${SYNCSTAGE2} ]] || [[ true = ${SYNCALL} ]]; then + tracefile + if [[ true = ${SYNCALL} ]]; then + rm -f "${TO}/${TRACE_STAGE1}" + fi +elif [[ true = ${SYNCSTAGE1} ]]; then + tracefile "${TO}/${TRACE_STAGE1}" +fi + + +HOOK=( + HOOKNR=4 + HOOKSCR=${HOOK4} +) +hook $HOOK + +if [[ true = ${SYNCCALLBACK} ]]; then + set +e + callback ${CALLBACKUSER} ${CALLBACKHOST} "${CALLBACKKEY}" + set -e +fi + +# Remove the Archive-Update-in-Progress file before we push our downstreams. +rm -f "${LOCK}" + +declare -f -F send_mail_new_version > /dev/null && send_mail_new_version || : + +if [[ ${HUB} = true ]]; then + # Trigger slave mirrors if we had a push for stage2 or all, or if its mhop + if [[ true = ${SYNCSTAGE2} ]] || [[ true = ${SYNCALL} ]] || [[ true = ${SYNCMHOP} ]]; then + RUNMIRRORARGS="" + if [[ -n ${ARCHIVE} ]]; then + # We tell runmirrors about the archive we are running on. + RUNMIRRORARGS="-a ${ARCHIVE}" + fi + # We also tell runmirrors that we are running it from within ftpsync, so it can change + # the way it works with mhop based on that. + RUNMIRRORARGS="${RUNMIRRORARGS} -f" + + if [[ true = ${SYNCSTAGE1} ]]; then + # This is true when we have a mhop sync. A normal multi-stage push sending stage1 will + # not get to this point. + # So if that happens, tell runmirrors we are doing mhop + RUNMIRRORARGS="${RUNMIRRORARGS} -k mhop" + elif [[ true = ${SYNCSTAGE2} ]]; then + RUNMIRRORARGS="${RUNMIRRORARGS} -k stage2" + elif [[ true = ${SYNCALL} ]]; then + RUNMIRRORARGS="${RUNMIRRORARGS} -k all" + fi + log "Trigger slave mirrors using ${RUNMIRRORARGS}" + ${BINDIR:+${BINDIR}/}runmirrors ${RUNMIRRORARGS} + log "Trigger slave done" + + HOOK=( + HOOKNR=5 + HOOKSCR=${HOOK5} + ) + hook $HOOK + fi +fi diff --git a/mirrors/files/distros/Debian/repository/ftpsync/bin/ftpsync-cron b/mirrors/files/distros/Debian/repository/ftpsync/bin/ftpsync-cron new file mode 100644 index 0000000..66e11d3 --- /dev/null +++ b/mirrors/files/distros/Debian/repository/ftpsync/bin/ftpsync-cron @@ -0,0 +1,525 @@ +#!/usr/bin/env bash + +set -euE + +VERSION="20180513" +# -*- mode:sh -*- +# vim:syn=sh +# Little common functions + +# push a mirror attached to us. +# Arguments (using an array named SIGNAL_OPTS): +# +# $MIRROR - Name for the mirror, also basename for the logfile +# $HOSTNAME - Hostname to push to +# $USERNAME - Username there +# $SSHPROTO - Protocol version, either 1 or 2. +# $SSHKEY - the ssh private key file to use for this push +# $SSHOPTS - any other option ssh accepts, passed blindly, be careful +# $PUSHLOCKOWN - own lockfile name to touch after stage1 in pushtype=staged +# $PUSHTYPE - what kind of push should be done? +# all - normal, just push once with ssh backgrounded and finish +# staged - staged. first push stage1, then wait for $PUSHLOCKs to appear, +# then push stage2 +# $PUSHARCHIVE - what archive to sync? (Multiple mirrors behind one ssh key!) +# $PUSHCB - do we want a callback? +# $PUSHKIND - whats going on? are we doing mhop push or already stage2? +# $FROMFTPSYNC - set to true if we run from within ftpsync. +# +# This function assumes that the variable LOG is set to a directory where +# logfiles can be written to. +# Additionally $PUSHLOCKS has to be defined as a set of space delimited strings +# (list of "lock"files) to wait for if you want pushtype=staged +# +# Pushes might be done in background (for type all). +signal () { + ARGS="SIGNAL_OPTS[*]" + local ${!ARGS} + + MIRROR=${MIRROR:-""} + HOSTNAME=${HOSTNAME:-""} + USERNAME=${USERNAME:-""} + SSHPROTO=${SSHPROTO:-""} + SSHKEY=${SSHKEY:-""} + SSHOPTS=${SSHOPTS:-""} + PUSHLOCKOWN=${PUSHLOCKOWN:-""} + PUSHTYPE=${PUSHTYPE:-"all"} + PUSHARCHIVE=${PUSHARCHIVE:-""} + PUSHCB=${PUSHCB:-""} + PUSHKIND=${PUSHKIND:-"all"} + FROMFTPSYNC=${FROMFTPSYNC:-"false"} + + # And now get # back to space... + SSHOPTS=${SSHOPTS/\#/ } + + # Defaults we always want, no matter what + SSH_OPTIONS="-o user=${USERNAME} -o BatchMode=yes -o ServerAliveInterval=45 -o ConnectTimeout=45 -o PasswordAuthentication=no" + + # If there are userdefined ssh options, add them. + if [[ -n ${SSH_OPTS} ]]; then + SSH_OPTIONS="${SSH_OPTIONS} ${SSH_OPTS}" + fi + + # Does this machine need a special key? + if [[ -n ${SSHKEY} ]]; then + SSH_OPTIONS="${SSH_OPTIONS} -i ${SSHKEY}" + fi + + # Does this machine have an extra own set of ssh options? + if [[ -n ${SSHOPTS} ]]; then + SSH_OPTIONS="${SSH_OPTIONS} ${SSHOPTS}" + fi + + # Set the protocol version + if [[ ${SSHPROTO} -ne 1 ]] && [[ ${SSHPROTO} -ne 2 ]] && [[ ${SSHPROTO} -ne 99 ]]; then + # Idiots, we only want 1 or 2. Cant decide? Lets force 2. + SSHPROTO=2 + fi + + if [[ -n ${SSHPROTO} ]] && [[ ${SSHPROTO} -ne 99 ]]; then + SSH_OPTIONS="${SSH_OPTIONS} -${SSHPROTO}" + fi + + date -u >> "${LOGDIR}/${MIRROR}.log" + + PUSHARGS="" + # PUSHARCHIVE empty or not, we always add the sync:archive: command to transfer. + # Otherwise, if nothing else is added, ssh -f would not work ("no command to execute") + # But ftpsync does treat "sync:archive:" as the main archive, so this works nicely. + PUSHARGS="${PUSHARGS} sync:archive:${PUSHARCHIVE}" + + # We have a callback wish, tell downstreams + if [[ -n ${PUSHCB} ]]; then + PUSHARGS="${PUSHARGS} sync:callback" + fi + # If we are running an mhop push AND our downstream is one to receive it, tell it. + if [[ mhop = ${PUSHKIND} ]] && [[ mhop = ${PUSHTYPE} ]]; then + PUSHARGS="${PUSHARGS} sync:mhop" + fi + + if [[ all = ${PUSHTYPE} ]]; then + # Default normal "fire and forget" push. We background that, we do not care about the mirrors doings + PUSHARGS1="sync:all" + signal_ssh "normal" "${MIRROR}" "${HOSTNAME}" $SSH_OPTIONS "${PUSHARGS} ${PUSHARGS1}" + elif [[ staged = ${PUSHTYPE} ]] || [[ mhop = ${PUSHTYPE} ]]; then + # Want a staged push. Fine, lets do that. Not backgrounded. We care about the mirrors doings. + # Only send stage1 if we havent already send it. When called with stage2, we already did. + if [[ stage2 != ${PUSHKIND} ]]; then + # Step1: Do a push to only sync stage1, do not background + PUSHARGS1="sync:stage1" + signal_ssh "first stage" "${MIRROR}" "${HOSTNAME}" $SSH_OPTIONS "${PUSHARGS} ${PUSHARGS1}" + touch "${PUSHLOCKOWN}" + + # Step2: Wait for all the other "lock"files to appear. + # In case we did not have all PUSHLOCKS and still continued, note it + # This is a little racy, especially if the other parts decide to do this + # at the same time, but it wont hurt more than a mail too much, so I don't care much + if ! wait_for_pushlocks ${PUSHDELAY}; then + msg "Failed to wait for all other mirrors. Failed ones are:" >> "${LOGDIR}/${MIRROR}.log" + for file in ${PUSHLOCKS}; do + if [[ ! -f ${file} ]]; then + msg "${file}" >> "${LOGDIR}/${MIRROR}.log" + log "Missing Pushlockfile ${file} after waiting for more than ${PUSHDELAY} seconds, continuing" + fi + done + fi + rm -f "${PUSHLOCKOWN}" + fi + + # Step3: It either timed out or we have all the "lock"files, do the rest + # If we are doing mhop AND are called from ftpsync - we now exit. + # That way we notify our uplink that we and all our clients are done with their + # stage1. It can then finish its own, and if all our upstreams downlinks are done, + # it will send us stage2. + # If we are not doing mhop or are not called from ftpsync, we start stage2 + if [[ true = ${FROMFTPSYNC} ]] && [[ mhop = ${PUSHKIND} ]]; then + return + else + PUSHARGS2="sync:stage2" + signal_ssh "second stage" "${MIRROR}" "${HOSTNAME}" $SSH_OPTIONS "${PUSHARGS} ${PUSHARGS2}" + fi + else + # Can't decide? Then you get nothing. + return + fi +} + +signal_ssh() { + local t=$1 + local mirror_log="${LOGDIR}/${2}.log" + local hostname=$3 + shift 3 + + msg "Sending ${t} trigger" >> $mirror_log + output=$(ssh -n $hostname "$@" 2>&1 | tee -a $mirror_log) + if [[ $? -eq 255 ]]; then + error_mailf "${t} trigger failed: $hostname" -b "$output" + else + log "${t} trigger succeeded: $hostname" + fi +} + +wait_for_pushlocks() { + local tries=0 + local found + local total + local timeout=${1}; shift + # We do not wait forever + while [[ ${tries} -lt ${timeout} ]]; do + total=0 + found=0 + for file in ${PUSHLOCKS}; do + total=$(( total + 1 )) + if [[ -f ${file} ]]; then + found=$(( found + 1 )) + fi + done + if [[ ${total} -eq ${found} ]] || [[ -f ${LOCKDIR}/all_stage1 ]]; then + break + fi + tries=$(( tries + 5 )) + sleep 5 + done + # Regardless of the state of our siblings, hitting one timeout cancels all waits + touch "${LOCKDIR}/all_stage1" + if [[ ${tries} -ge ${timeout} ]]; then + return 1 + else + return 0 + fi +} + +# callback, used by ftpsync +callback () { + # Defaults we always want, no matter what + SSH_OPTIONS="-o BatchMode=yes -o ServerAliveInterval=45 -o ConnectTimeout=45 -o PasswordAuthentication=no" + ssh -n $SSH_OPTIONS -i "$3" -o"user $1" "$2" callback:${HOSTNAME} +} + +# open log file +open_log() { + local log=$1 + shift + exec 4>&1 1>>$log +} + +# assemble log message (basically echo it together with a timestamp) +# +# Set $PROGRAM to a string to have it added to the output. +msg() { + if [[ -z "${PROGRAM}" ]]; then + echo "$(date +"%b %d %H:%M:%S") $(hostname -s) [$$] $@" + else + echo "$(date +"%b %d %H:%M:%S") $(hostname -s) ${PROGRAM}[$$]: $@" + fi +} + +# log something +log() { + msg "$@" +} + +# log the message using log() but then also send a mail +# to the address configured in MAILTO (if non-empty) +error () { + log "$@" + LOG_ERROR=1 + mailf -s "[$PROGRAM@$(hostname -s)] ERROR: $*" -b "$*" ${MAILTO} +} + +# log the message using log() but then also send a mail +# to the address configured in MAILTO (if non-empty) +error_mailf () { + local m="$1" + shift + log "$m" + LOG_ERROR=1 + mailf -s "[$PROGRAM@$(hostname -s)] ERROR: $m" "$@" ${MAILTO} +} + +# run a hook +# needs array variable HOOK setup with HOOKNR being a number an HOOKSCR +# the script to run. +hook () { + ARGS='HOOK[@]' + local "${!ARGS}" + if [[ -n ${HOOKSCR} ]]; then + log "Running hook $HOOKNR: ${HOOKSCR}" + set +e + ${HOOKSCR} + result=$? + set -e + if [[ ${result} -ne 0 ]] ; then + error "Back from hook $HOOKNR, got returncode ${result}" + else + log "Back from hook $HOOKNR, got returncode ${result}" + fi + return $result + else + return 0 + fi +} + +# Return the list of 2-stage mirrors. +get2stage() { + egrep -s '^(staged|mhop)' "${MIRRORS}" | { + while read MTYPE MLNAME MHOSTNAME MUSER MPROTO MKEYFILE; do + PUSHLOCKS="${LOCKDIR}/${MLNAME}.stage1 ${PUSHLOCKS}" + done + echo "$PUSHLOCKS" + } +} + +# Rotate logfiles +savelog() { + torotate="$1" + count=${2:-${LOGROTATE}} + while [[ ${count} -gt 0 ]]; do + prev=$(( count - 1 )) + if [[ -e ${torotate}.${prev} ]]; then + mv "${torotate}.${prev}" "${torotate}.${count}" + fi + count=$prev + done + if [[ -e ${torotate} ]]; then + mv "${torotate}" "${torotate}.0" + fi +} + +# Return rsync version +rsync_protocol() { + RSYNC_VERSION="$(${RSYNC} --version)" + RSYNC_REGEX="(protocol[ ]+version[ ]+([0-9]+))" + if [[ ${RSYNC_VERSION} =~ ${RSYNC_REGEX} ]]; then + echo ${BASH_REMATCH[2]} + fi + unset RSYNC_VERSION RSYNC_REGEX +} + +extract_trace_field() { + local field="$1" + local file="$2" + local value=$(awk -F': ' "\$1==\"$field\" {print \$2; exit}" "$file" 2>/dev/null) + [[ $value ]] || return 1 + echo $value +} + +extract_trace_field_string() { + local field="$1" + local string="$2" + local value=$(awk -F': ' "\$1==\"$field\" {print \$2; exit}" <<< "$string" 2>/dev/null) + [[ $value ]] || return 1 + echo $value +} + +extract_trace_serial() { + extract_trace_field 'Archive serial' "$1" + return $? +} + +extract_trace_serial_string() { + extract_trace_field_string 'Archive serial' "$1" + return $? +} + +# Search config files in various locations +search_config() { + local file + for i in ${CONFDIRS[@]}; do + file="$i/$1" + if [ -f "$file" ]; then + echo "$file" + return + fi + done +} + +# Read config file +read_config() { + local name=$(echo "$1" | sed -e 's/[^A-Za-z0-9._-]/_/g') + local config=$(search_config "$name") + if [ "$config" ]; then + . "$config" + CURRENT_CONFIG="$config" + return 0 + else + echo "Can't read config file ${name}!" >&2 + exit 78 # EX_CONFIG + fi +} + +# Create lock dir +create_lockdir() { + mkdir -p "$LOCKDIR" +} + +# Create log dir +create_logdir() { + mkdir -p "$LOGDIR" +} + +join_by() { + local IFS="$1" + shift + echo $* +} + +# Sends mail +# mailf [-a attachment] [-b body] [-s subject] to-addr ... +mailf() { + local boundary="==--$RANDOM--$RANDOM--$RANDOM--==" + local attachment=() + local body=() + local subject= + + OPTIND=1 + while getopts ":a:b:s:" arg; do + case $arg in + a) + attachment+=("$OPTARG") + ;; + b) + body+=("$OPTARG") + ;; + s) + subject="$OPTARG" + ;; + esac + done + shift $((OPTIND-1)) + + ( + cat < "${LOGDIR}/ftpsync.newversion" + fi + fi + else + # Remove a possible stampfile + rm -f "${LOGDIR}/ftpsync.newversion" + fi + fi +} +export BASEDIR + +ARCHIVE=${1:-} +CONFIG="ftpsync${ARCHIVE:+-$ARCHIVE}.conf" +NAME=ftpsync-cron +PROGRAM=ftpsync-cron + +read_config "${CONFIG}" + +MAILTO=${MAILTO:-${LOGNAME:?Environment variable LOGNAME unset, please check your system or specify MAILTO}} +MIRRORNAME=${MIRRORNAME:-$(hostname -f)} +LOCK=${LOCK:-"${TO}/Archive-Update-in-Progress-${MIRRORNAME}"} +RSYNC_PATH="${RSYNC_PATH:-debian}" + +TRACE_UPSTREAM="project/trace/${RSYNC_HOST}" +TRACE="project/trace/${MIRRORNAME}" + +function run_ftpsync() { + exec "${BINDIR:+${BINDIR}/}ftpsync" -T "cron" "sync:archive:${ARCHIVE}" "$@" +} + +function cmp_trace() { + local trace_uri="http://${RSYNC_HOST}/${RSYNC_PATH}/${TRACE_UPSTREAM}" + local trace_local="${TO}/${TRACE}" + local trace ret + + if [[ -e $LOCK ]] && $(kill -0 $(< ${LOCK}) 2>/dev/null); then + return 0 + fi + + trace=$(curl --silent --fail --show-error --location "$trace_uri" 2>&1) + ret=$? + if [[ $ret != 0 ]]; then + error "Failed to retrieve $trace_uri: $trace" + exit $ret + fi + + if ! serial_remote=$(extract_trace_serial_string "$trace"); then + error "Failed to parse archive serial from $trace_uri" + exit 1 + elif ! serial_local=$(extract_trace_serial "$trace_local"); then + return 1 + elif [[ $serial_remote != $serial_local ]]; then + return 1 + fi +} + +if ! cmp_trace; then + run_ftpsync +fi diff --git a/mirrors/files/distros/Debian/repository/ftpsync/bin/rsync-ssl-tunnel b/mirrors/files/distros/Debian/repository/ftpsync/bin/rsync-ssl-tunnel new file mode 100644 index 0000000..46e2892 --- /dev/null +++ b/mirrors/files/distros/Debian/repository/ftpsync/bin/rsync-ssl-tunnel @@ -0,0 +1,113 @@ +#!/usr/bin/env bash + +# Helper to tunnel rsync through ssl on the client side. +# Example: +# $ RSYNC_SSL_METHOD=socat rsync -e ./rsync-ssl-tunnel syncproxy2.eu.debian.org:: +# debian Full Debian FTP Archive. +# debian-debug Debug packages. + +# Copyright (c) 2016 Peter Palfrader +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +set -e +set -u + +usage() { + cat <&2 ;; + esac +done +shift $(($OPTIND - 1)) + +if [[ "$#" = 0 ]]; then + echo >&2 "No arguments given." + usage 64 >&2 + exit 1 +fi +RSYNC_HOST="$1"; shift +RSYNC_SSL_PORT=${RSYNC_SSL_PORT:-"1873"} +RSYNC_SSL_CAPATH=${RSYNC_SSL_CAPATH:-"/etc/ssl/certs"} +RSYNC_SSL_METHOD=${RSYNC_SSL_METHOD:-"stunnel"} + +method_stunnel() { + skip_host_check="$1"; shift + + if ! [ "$skip_host_check" = 1 ]; then + checkhost="checkHost = ${RSYNC_HOST}" + fi + + exec stunnel -fd 3 3<&2 "Failed to exec stunnel" + exit 1 +} + +method_socat() { + exec socat - "openssl-connect:${RSYNC_HOST}:${RSYNC_SSL_PORT},capath=${RSYNC_SSL_CAPATH},keepalive,keepidle=300" + echo >&2 "Failed to exec socat." + exit 1 +} + +case ${RSYNC_SSL_METHOD:-} in + stunnel|stunnel4) + method_stunnel 0 + ;; + stunnel-old|stunnel4-old) + method_stunnel 1 + ;; + socat) + method_socat + ;; + *) + echo >&2 "Unknown method $RSYNC_SSL_METHOD." + exit 1 + ;; +esac diff --git a/mirrors/files/distros/Debian/repository/ftpsync/bin/runmirrors b/mirrors/files/distros/Debian/repository/ftpsync/bin/runmirrors new file mode 100644 index 0000000..ee163a7 --- /dev/null +++ b/mirrors/files/distros/Debian/repository/ftpsync/bin/runmirrors @@ -0,0 +1,778 @@ +#!/usr/bin/env bash + +# runmirrors script for Debian +# Based losely on existing scripts, written by an unknown number of +# different people over the years. +# +# Copyright (C) 2008-2016 Joerg Jaspert +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; version 2. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +set -e +set -u +set -o pipefail + +VERSION="20180513" +# -*- mode:sh -*- +# vim:syn=sh +# Little common functions + +# push a mirror attached to us. +# Arguments (using an array named SIGNAL_OPTS): +# +# $MIRROR - Name for the mirror, also basename for the logfile +# $HOSTNAME - Hostname to push to +# $USERNAME - Username there +# $SSHPROTO - Protocol version, either 1 or 2. +# $SSHKEY - the ssh private key file to use for this push +# $SSHOPTS - any other option ssh accepts, passed blindly, be careful +# $PUSHLOCKOWN - own lockfile name to touch after stage1 in pushtype=staged +# $PUSHTYPE - what kind of push should be done? +# all - normal, just push once with ssh backgrounded and finish +# staged - staged. first push stage1, then wait for $PUSHLOCKs to appear, +# then push stage2 +# $PUSHARCHIVE - what archive to sync? (Multiple mirrors behind one ssh key!) +# $PUSHCB - do we want a callback? +# $PUSHKIND - whats going on? are we doing mhop push or already stage2? +# $FROMFTPSYNC - set to true if we run from within ftpsync. +# +# This function assumes that the variable LOG is set to a directory where +# logfiles can be written to. +# Additionally $PUSHLOCKS has to be defined as a set of space delimited strings +# (list of "lock"files) to wait for if you want pushtype=staged +# +# Pushes might be done in background (for type all). +signal () { + ARGS="SIGNAL_OPTS[*]" + local ${!ARGS} + + MIRROR=${MIRROR:-""} + HOSTNAME=${HOSTNAME:-""} + USERNAME=${USERNAME:-""} + SSHPROTO=${SSHPROTO:-""} + SSHKEY=${SSHKEY:-""} + SSHOPTS=${SSHOPTS:-""} + PUSHLOCKOWN=${PUSHLOCKOWN:-""} + PUSHTYPE=${PUSHTYPE:-"all"} + PUSHARCHIVE=${PUSHARCHIVE:-""} + PUSHCB=${PUSHCB:-""} + PUSHKIND=${PUSHKIND:-"all"} + FROMFTPSYNC=${FROMFTPSYNC:-"false"} + + # And now get # back to space... + SSHOPTS=${SSHOPTS/\#/ } + + # Defaults we always want, no matter what + SSH_OPTIONS="-o user=${USERNAME} -o BatchMode=yes -o ServerAliveInterval=45 -o ConnectTimeout=45 -o PasswordAuthentication=no" + + # If there are userdefined ssh options, add them. + if [[ -n ${SSH_OPTS} ]]; then + SSH_OPTIONS="${SSH_OPTIONS} ${SSH_OPTS}" + fi + + # Does this machine need a special key? + if [[ -n ${SSHKEY} ]]; then + SSH_OPTIONS="${SSH_OPTIONS} -i ${SSHKEY}" + fi + + # Does this machine have an extra own set of ssh options? + if [[ -n ${SSHOPTS} ]]; then + SSH_OPTIONS="${SSH_OPTIONS} ${SSHOPTS}" + fi + + # Set the protocol version + if [[ ${SSHPROTO} -ne 1 ]] && [[ ${SSHPROTO} -ne 2 ]] && [[ ${SSHPROTO} -ne 99 ]]; then + # Idiots, we only want 1 or 2. Cant decide? Lets force 2. + SSHPROTO=2 + fi + + if [[ -n ${SSHPROTO} ]] && [[ ${SSHPROTO} -ne 99 ]]; then + SSH_OPTIONS="${SSH_OPTIONS} -${SSHPROTO}" + fi + + date -u >> "${LOGDIR}/${MIRROR}.log" + + PUSHARGS="" + # PUSHARCHIVE empty or not, we always add the sync:archive: command to transfer. + # Otherwise, if nothing else is added, ssh -f would not work ("no command to execute") + # But ftpsync does treat "sync:archive:" as the main archive, so this works nicely. + PUSHARGS="${PUSHARGS} sync:archive:${PUSHARCHIVE}" + + # We have a callback wish, tell downstreams + if [[ -n ${PUSHCB} ]]; then + PUSHARGS="${PUSHARGS} sync:callback" + fi + # If we are running an mhop push AND our downstream is one to receive it, tell it. + if [[ mhop = ${PUSHKIND} ]] && [[ mhop = ${PUSHTYPE} ]]; then + PUSHARGS="${PUSHARGS} sync:mhop" + fi + + if [[ all = ${PUSHTYPE} ]]; then + # Default normal "fire and forget" push. We background that, we do not care about the mirrors doings + PUSHARGS1="sync:all" + signal_ssh "normal" "${MIRROR}" "${HOSTNAME}" $SSH_OPTIONS "${PUSHARGS} ${PUSHARGS1}" + elif [[ staged = ${PUSHTYPE} ]] || [[ mhop = ${PUSHTYPE} ]]; then + # Want a staged push. Fine, lets do that. Not backgrounded. We care about the mirrors doings. + # Only send stage1 if we havent already send it. When called with stage2, we already did. + if [[ stage2 != ${PUSHKIND} ]]; then + # Step1: Do a push to only sync stage1, do not background + PUSHARGS1="sync:stage1" + signal_ssh "first stage" "${MIRROR}" "${HOSTNAME}" $SSH_OPTIONS "${PUSHARGS} ${PUSHARGS1}" + touch "${PUSHLOCKOWN}" + + # Step2: Wait for all the other "lock"files to appear. + # In case we did not have all PUSHLOCKS and still continued, note it + # This is a little racy, especially if the other parts decide to do this + # at the same time, but it wont hurt more than a mail too much, so I don't care much + if ! wait_for_pushlocks ${PUSHDELAY}; then + msg "Failed to wait for all other mirrors. Failed ones are:" >> "${LOGDIR}/${MIRROR}.log" + for file in ${PUSHLOCKS}; do + if [[ ! -f ${file} ]]; then + msg "${file}" >> "${LOGDIR}/${MIRROR}.log" + log "Missing Pushlockfile ${file} after waiting for more than ${PUSHDELAY} seconds, continuing" + fi + done + fi + rm -f "${PUSHLOCKOWN}" + fi + + # Step3: It either timed out or we have all the "lock"files, do the rest + # If we are doing mhop AND are called from ftpsync - we now exit. + # That way we notify our uplink that we and all our clients are done with their + # stage1. It can then finish its own, and if all our upstreams downlinks are done, + # it will send us stage2. + # If we are not doing mhop or are not called from ftpsync, we start stage2 + if [[ true = ${FROMFTPSYNC} ]] && [[ mhop = ${PUSHKIND} ]]; then + return + else + PUSHARGS2="sync:stage2" + signal_ssh "second stage" "${MIRROR}" "${HOSTNAME}" $SSH_OPTIONS "${PUSHARGS} ${PUSHARGS2}" + fi + else + # Can't decide? Then you get nothing. + return + fi +} + +signal_ssh() { + local t=$1 + local mirror_log="${LOGDIR}/${2}.log" + local hostname=$3 + shift 3 + + msg "Sending ${t} trigger" >> $mirror_log + output=$(ssh -n $hostname "$@" 2>&1 | tee -a $mirror_log) + if [[ $? -eq 255 ]]; then + error_mailf "${t} trigger failed: $hostname" -b "$output" + else + log "${t} trigger succeeded: $hostname" + fi +} + +wait_for_pushlocks() { + local tries=0 + local found + local total + local timeout=${1}; shift + # We do not wait forever + while [[ ${tries} -lt ${timeout} ]]; do + total=0 + found=0 + for file in ${PUSHLOCKS}; do + total=$(( total + 1 )) + if [[ -f ${file} ]]; then + found=$(( found + 1 )) + fi + done + if [[ ${total} -eq ${found} ]] || [[ -f ${LOCKDIR}/all_stage1 ]]; then + break + fi + tries=$(( tries + 5 )) + sleep 5 + done + # Regardless of the state of our siblings, hitting one timeout cancels all waits + touch "${LOCKDIR}/all_stage1" + if [[ ${tries} -ge ${timeout} ]]; then + return 1 + else + return 0 + fi +} + +# callback, used by ftpsync +callback () { + # Defaults we always want, no matter what + SSH_OPTIONS="-o BatchMode=yes -o ServerAliveInterval=45 -o ConnectTimeout=45 -o PasswordAuthentication=no" + ssh -n $SSH_OPTIONS -i "$3" -o"user $1" "$2" callback:${HOSTNAME} +} + +# open log file +open_log() { + local log=$1 + shift + exec 4>&1 1>>$log +} + +# assemble log message (basically echo it together with a timestamp) +# +# Set $PROGRAM to a string to have it added to the output. +msg() { + if [[ -z "${PROGRAM}" ]]; then + echo "$(date +"%b %d %H:%M:%S") $(hostname -s) [$$] $@" + else + echo "$(date +"%b %d %H:%M:%S") $(hostname -s) ${PROGRAM}[$$]: $@" + fi +} + +# log something +log() { + msg "$@" +} + +# log the message using log() but then also send a mail +# to the address configured in MAILTO (if non-empty) +error () { + log "$@" + LOG_ERROR=1 + mailf -s "[$PROGRAM@$(hostname -s)] ERROR: $*" -b "$*" ${MAILTO} +} + +# log the message using log() but then also send a mail +# to the address configured in MAILTO (if non-empty) +error_mailf () { + local m="$1" + shift + log "$m" + LOG_ERROR=1 + mailf -s "[$PROGRAM@$(hostname -s)] ERROR: $m" "$@" ${MAILTO} +} + +# run a hook +# needs array variable HOOK setup with HOOKNR being a number an HOOKSCR +# the script to run. +hook () { + ARGS='HOOK[@]' + local "${!ARGS}" + if [[ -n ${HOOKSCR} ]]; then + log "Running hook $HOOKNR: ${HOOKSCR}" + set +e + ${HOOKSCR} + result=$? + set -e + if [[ ${result} -ne 0 ]] ; then + error "Back from hook $HOOKNR, got returncode ${result}" + else + log "Back from hook $HOOKNR, got returncode ${result}" + fi + return $result + else + return 0 + fi +} + +# Return the list of 2-stage mirrors. +get2stage() { + egrep -s '^(staged|mhop)' "${MIRRORS}" | { + while read MTYPE MLNAME MHOSTNAME MUSER MPROTO MKEYFILE; do + PUSHLOCKS="${LOCKDIR}/${MLNAME}.stage1 ${PUSHLOCKS}" + done + echo "$PUSHLOCKS" + } +} + +# Rotate logfiles +savelog() { + torotate="$1" + count=${2:-${LOGROTATE}} + while [[ ${count} -gt 0 ]]; do + prev=$(( count - 1 )) + if [[ -e ${torotate}.${prev} ]]; then + mv "${torotate}.${prev}" "${torotate}.${count}" + fi + count=$prev + done + if [[ -e ${torotate} ]]; then + mv "${torotate}" "${torotate}.0" + fi +} + +# Return rsync version +rsync_protocol() { + RSYNC_VERSION="$(${RSYNC} --version)" + RSYNC_REGEX="(protocol[ ]+version[ ]+([0-9]+))" + if [[ ${RSYNC_VERSION} =~ ${RSYNC_REGEX} ]]; then + echo ${BASH_REMATCH[2]} + fi + unset RSYNC_VERSION RSYNC_REGEX +} + +extract_trace_field() { + local field="$1" + local file="$2" + local value=$(awk -F': ' "\$1==\"$field\" {print \$2; exit}" "$file" 2>/dev/null) + [[ $value ]] || return 1 + echo $value +} + +extract_trace_field_string() { + local field="$1" + local string="$2" + local value=$(awk -F': ' "\$1==\"$field\" {print \$2; exit}" <<< "$string" 2>/dev/null) + [[ $value ]] || return 1 + echo $value +} + +extract_trace_serial() { + extract_trace_field 'Archive serial' "$1" + return $? +} + +extract_trace_serial_string() { + extract_trace_field_string 'Archive serial' "$1" + return $? +} + +# Search config files in various locations +search_config() { + local file + for i in ${CONFDIRS[@]}; do + file="$i/$1" + if [ -f "$file" ]; then + echo "$file" + return + fi + done +} + +# Read config file +read_config() { + local name=$(echo "$1" | sed -e 's/[^A-Za-z0-9._-]/_/g') + local config=$(search_config "$name") + if [ "$config" ]; then + . "$config" + CURRENT_CONFIG="$config" + return 0 + else + echo "Can't read config file ${name}!" >&2 + exit 78 # EX_CONFIG + fi +} + +# Create lock dir +create_lockdir() { + mkdir -p "$LOCKDIR" +} + +# Create log dir +create_logdir() { + mkdir -p "$LOGDIR" +} + +join_by() { + local IFS="$1" + shift + echo $* +} + +# Sends mail +# mailf [-a attachment] [-b body] [-s subject] to-addr ... +mailf() { + local boundary="==--$RANDOM--$RANDOM--$RANDOM--==" + local attachment=() + local body=() + local subject= + + OPTIND=1 + while getopts ":a:b:s:" arg; do + case $arg in + a) + attachment+=("$OPTARG") + ;; + b) + body+=("$OPTARG") + ;; + s) + subject="$OPTARG" + ;; + esac + done + shift $((OPTIND-1)) + + ( + cat < "${LOGDIR}/ftpsync.newversion" + fi + fi + else + # Remove a possible stampfile + rm -f "${LOGDIR}/ftpsync.newversion" + fi + fi +} + +NAME="$(basename $0)" + +HELP="$0\n +Usage:\n\n + +1.) a single parameter with NO leading -.\n +\t This will will then be used as the addition for our configfile. Ie. \`$0 security\` will\n +\t have us look for ${NAME}-security.{conf,mirror} files.\n\n + +2.) using getopt style parameters:\n +\t -a [NAME] - Same as 1.) above, used for the config files. Default empty.\n +\t -k [TYPE] - Type of push. all, stage2, mhop. Default mhop.\n +\t -f - Run from within the mirrorscript ftpsync. Don't use from commandline!\n +\t -h - Print this help and exit +" +# If we got options, lets see if we use newstyle options, or oldstyle. If oldstyle +# it will not start with a -. If we find oldstyle we assume its only one, the config +# name we run on. +if [[ $# -gt 0 ]]; then + if [[ ${1:0:1} != - ]]; then + # Yes, does not start with a -, so use it for the config name. + CONF=${1:-""} + if [[ -n ${CONF} ]]; then + NAME="${NAME}-${CONF}" + fi + else + # Yeah well, new style, starting with - for getopts + while getopts ':a:k:fh' OPTION ; do + case $OPTION in + a) CONF="${OPTARG}" + if [[ -n ${CONF} ]]; then + NAME="${NAME}-${CONF}" + fi + ;; + k) PUSHKIND="${OPTARG}" + ;; + f) FROMFTPSYNC="true" + ;; + h) echo -e $HELP + exit 0 + ;; + + *) echo "Invalid usage" + echo -e $HELP + exit 1 + ;; + esac + done + fi +fi +# Make sure the values are always defined, even if there was no commandline option +# for them +# Default config is empty +CONF=${CONF:-""} + +# Set the default to all, if we didnt get told about it. Currently +# valid: all - normal push. mhop - multi-hop multi-stage push, this is stage1, +# stage2 - staged push, second phase. Default is mhop. +PUSHKIND=${PUSHKIND:-"mhop"} + +# If we are pushed from within ftpsync. Default false. +FROMFTPSYNC=${FROMFTPSYNC:-"false"} + +######################################################################## +# Read our config file +read_config "${NAME}.conf" + +# Make sure we have our log and lock directories +create_logdir +create_lockdir + +######################################################################## +# Config defaults # +######################################################################## +MAILTO=${MAILTO:-${LOGNAME:?Environment variable LOGNAME unset, please check your system or specify MAILTO}} +KEYFILE=${KEYFILE:-".ssh/pushmirror"} + +# Log options +LOG=${LOG:-"${LOGDIR}/${NAME}.log"} +LOGROTATE=${LOGROTATE:-14} + +# Other options +MIRRORS=${MIRRORS:-$(search_config "${NAME}.mirror")} +SSH_OPTS=${SSH_OPTS:-"-o StrictHostKeyChecking=no"} +PUSHARCHIVE=${PUSHARCHIVE:-"${CONF}"} +PUSHDELAY=${PUSHDELAY:-600} + +# Hooks +HOOK1=${HOOK1:-""} +HOOK2=${HOOK2:-""} +HOOK3=${HOOK3:-""} +######################################################################## +######################################################################## + +# Which ssh key to use? +KEYFILE_ABS=$(cd && readlink -f "$KEYFILE" || :) + +# used by log() +PROGRAM=${PROGRAM:-"${NAME}"} + +REGEX_URL='^([a-z.+]+)://(([^@/]+)@)?([^/]+)(/.*)?$' + +# start a new log +savelog "${LOG}" > /dev/null + +if ! [[ -f ${KEYFILE_ABS} ]]; then + error "SSH Key ${KEYFILE} does not exist" >> "${LOG}" + exit 5 +fi + +# Hooks + +######################################################################## + +# Some sane defaults +cd "${BASEDIR}" +umask 022 + +open_log $LOG + +trap 'log "Mirrorpush done"' EXIT + +log "Pushing leaf mirrors. Inside ftpsync: ${FROMFTPSYNC}. Pushkind: ${PUSHKIND}" + +HOOK=( + HOOKNR=1 + HOOKSCR=${HOOK1} +) +hook $HOOK + +# From here on we do *NOT* want to exit on errors. We don't want to +# stop pushing mirrors just because we can't reach one of them. +set +e + +# Built up our list of 2-stage mirrors. +PUSHLOCKS="" +PUSHLOCKS=$(get2stage) + +# In case we have it - remove. It is used to synchronize multi-stage mirroring +rm -f "${LOCKDIR}/all_stage1" + +# Now read our mirrorfile and push the mirrors defined in there. +# We use grep to easily sort out all lines having a # in front of them or are empty. +egrep -vs '^[[:space:]]*(#|$)' "${MIRRORS}" | +while read MTYPE MLNAME MHOSTNAME MUSER MSSHOPT; do + if [[ ${MTYPE} = DELAY ]]; then + # We should wait a bit. + if [ -z ${MLNAME} ]; then + MLNAME=600 + fi + if [ "${MHOSTNAME:-}" = "stage1" ]; then + log "Delay of ${MLNAME} or until staging is finished requested" + if ! wait_for_pushlocks ${MLNAME}; then + log "Staged delay ran into timeout of ${MLNAME} seconds" + else + log "Staged delay finished" + fi + else + log "Delay of ${MLNAME} requested, sleeping" + sleep ${MLNAME} + fi + continue + fi + + # If we are told we have a mhop sync to do and are called from within ftpsync, + # we will only look at staged/mhop entries and ignore the rest. + if [[ ${PUSHKIND} = mhop ]] && [[ ${FROMFTPSYNC} = true ]]; then + if [[ ${MTYPE} != staged ]] && [[ ${MTYPE} != mhop ]]; then + continue + fi + fi + + MPROTO=2 + MKEYFILE="${KEYFILE_ABS}" + SSHOPT="" + MPUSHARCHIVE=${PUSHARCHIVE} + + if [[ $MHOSTNAME =~ $REGEX_URL ]]; then + MSCHEME=${BASH_REMATCH[1]} + MUSER=${BASH_REMATCH[3]} + MHOSTNAME=${BASH_REMATCH[4]} + URLPATH=${BASH_REMATCH[5]} + + if [[ $MSCHEME == ssh+ftpsync ]]; then + if [[ $URLPATH ]]; then + MPUSHARCHIVE=${URLPATH#/} + fi + else + log "Trigger ${MLNAME}: ${MHOSTNAME} have wrong scheme, ignoring" + continue + fi + + # Now, MSSHOPT may start with a -. In that case the whole rest of the line is taken + # as a set of options to give to ssh, we pass it without doing anything with it. + # If it starts with a 1 or 2 then it will tell us about the ssh protocol version to use, + # and also means we look if there is one value more after a space. That value would then + # be the ssh keyfile we use with -i. That gives us full flexibility for all + # ssh options but doesn't destroy backwards compatibility. + # If it is empty we assume proto 2 and the default keyfile. + # + # There is one bug in here. We will give out the master keyfile, even if there is a + # "-i /bla/bla" in the options. ssh stuffs them together and presents two keys to the + # target server. In the case both keys do some actions- the first one presented wins. + # And this might not be what one wants. + # + # The only sane way to go around this, i think, is by dropping backward compability. + # Which I don't really like. + elif [[ -n ${MSSHOPT} ]]; then + # So its not empty, lets check if it starts with a - and as such is a "new-style" + # ssh options set. + if [[ ${MSSHOPT:0:1} = - ]]; then + # Yes we start with a - + SSHOPT="${MSSHOPT}" + MPROTO="99" + elif [[ ${MSSHOPT:0:1} -eq 1 ]] || [[ ${MSSHOPT:0:1} -eq 2 ]]; then + # We do seem to have oldstyle options here. + MPROTO=${MSSHOPT:0:1} + MKEYFILE=${MSSHOPT:2} + else + error "I don't know what is configured for mirror ${MLNAME}" + continue + fi + fi + + # Built our array + SIGNAL_OPTS=( + MIRROR="${MLNAME}" + HOSTNAME="${MHOSTNAME}" + USERNAME="${MUSER}" + SSHPROTO="${MPROTO}" + SSHKEY="${MKEYFILE}" + SSHOPTS="${SSHOPT/ /#}" + PUSHLOCKOWN="${LOCKDIR}/${MLNAME}.stage1" + PUSHTYPE="${MTYPE}" + PUSHARCHIVE=${MPUSHARCHIVE} + PUSHKIND=${PUSHKIND} + FROMFTPSYNC=${FROMFTPSYNC} + ) + + # And finally, push the mirror + log "Trigger ${MLNAME}" + signal "${SIGNAL_OPTS}" & + log "Trigger for ${MLNAME} done" + + HOOK=( + HOOKNR=2 + HOOKSCR=${HOOK2} + ) + hook $HOOK + set +e +done + +# If we are run from within ftpsync *and* have an mhop push to send on, we have +# to wait until the push is gone through and they all returned, or we will exit +# much too early. +# As the signal routine touches $LOCKDIR/all_stage1 when all are done, its +# easy enough just to wait for that to appear. Of course we do the same game +# with PUSHDELAY to not wait forever. +if [[ true = ${FROMFTPSYNC} ]] && [[ mhop = ${PUSHKIND} ]]; then + tries=0 + # We do not wait forever + while [[ ${tries} -lt ${PUSHDELAY} ]]; do + if [[ -f ${LOCKDIR}/all_stage1 ]]; then + break + fi + tries=$(( tries + 5 )) + sleep 5 + done + + if [[ ${tries} -ge ${PUSHDELAY} ]]; then + error "Failed to wait for our mirrors when sending mhop push down." >> "${LOG}" + fi +fi + +HOOK=( + HOOKNR=3 + HOOKSCR=${HOOK3} +) +hook $HOOK + +exit 0 diff --git a/mirrors/files/distros/Debian/repository/ftpsync/distrib/README.md b/mirrors/files/distros/Debian/repository/ftpsync/distrib/README.md new file mode 100644 index 0000000..009ded8 --- /dev/null +++ b/mirrors/files/distros/Debian/repository/ftpsync/distrib/README.md @@ -0,0 +1,264 @@ +# Archvsync + +This is the central repository for the Debian mirror scripts. The scripts +in this repository are written for the purposes of maintaining a Debian +archive mirror (and shortly, a Debian bug mirror), but they should be +easily generalizable. + + +Currently the following scripts are available: + + * ftpsync - Used to sync an archive using rsync + * runmirrors - Used to notify leaf nodes of available updates + +## Usage + +For impatient people, short usage instruction: + + * Create a dedicated user for the whole mirror. + * Create a seperate directory for the mirror, writeable by the new user. + * Place the ftpsync script in the mirror user's $HOME/bin (or just $HOME) + * Place the ftpsync.conf.sample into $HOME/etc as ftpsync.conf and edit + it to suit your system. You should at the very least change the TO= + and RSYNC_HOST lines. + * Create $HOME/log (or wherever you point $LOGDIR to) + * If only you receive an update trigger, + Setup the .ssh/authorized_keys for the mirror user and place the public key of + your upstream mirror into it. Preface it with + `no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty,command="~/bin/ftpsync",from="IPADDRESS"` + and replace $IPADDRESS with that of your upstream mirror. + * You are finished + +In order to receive different pushes or syncs from different archives, +name the config file ftpsync-$ARCHIVE.conf and call the ftpsync script +with the commandline "sync:archive:$ARCHIVE". Replace $ARCHIVE with a +sensible value. If your upstream mirror pushes you using runmirrors +bundled together with this sync script, you do not need to add the +"sync:archive" parameter to the commandline, the scripts deal with it +automatically. + +Common names used in the Debian mirror network are: +* standard debian archive: default empty $ARCHIVE (ftpsync.conf) +* debian-ports archive: ARCHIVE=ports (ftpsync-ports.conf) +* debian-security archive: ARCHIVE=security (ftpsync-security.conf) + + +## Debian mirror script minimum requirements + +As always, you may use whatever scripts you want for your Debian mirror, +but we *STRONGLY* recommend you to not invent your own. However, if you +want to be listed as a mirror it *MUST* support the following minimal +functionality: + +* Must perform a 2-stage sync + The archive mirroring must be done in 2 stages. The first rsync run + must ignore the index files. The correct exclude options for the + first rsync run are: + `--exclude Packages* --exclude Sources* --exclude Release* --exclude=InRelease --include=i18n/by-hash/** --exclude=i18n/* --exclude ls-lR*` + + The first stage must not delete any files. + + The second stage should then transfer the above excluded files and + delete files that no longer belong on the mirror. + + Rationale: If archive mirroring is done in a single stage, there will be + periods of time during which the index files will reference files not + yet mirrored. + +* Must not ignore pushes whil(e|st) running. + If a push is received during a run of the mirror sync, it MUST NOT + be ignored. The whole synchronization process must be rerun. + + Rationale: Most implementations of Debian mirror scripts will leave the + mirror in an inconsistent state in the event of a second push being + received while the first sync is still running. It is likely that in + the near future, the frequency of pushes will increase. + +* Should understand multi-stage pushes. + The script should parse the arguments it gets via ssh, and if they + contain a hint to only sync stage1 or stage2, then ONLY those steps + SHOULD be performed. + + Rationale: This enables us to coordinate the timing of the first + and second stage pushes and minimize the time during which the + archive is desynchronized. This is especially important for mirrors + that are involved in a round robin or GeoDNS setup. + + The minimum arguments the script has to understand are: + * `sync:stage1`: Only sync stage1 + * `sync:stage2`: Only sync stage2 + * `sync:all`: Do everything. Default if none of stage1/2 are present. + + There are more possible arguments, for a complete list see the + ftpsync script in our git repository. + +## ftpsync + +This script is based on the old anonftpsync script. It has been rewritten +to add flexibilty and fix a number of outstanding issues. + +Some of the advantages of the new version are: + * Nearly every aspect is configurable + * Correct support for multiple pushes + * Support for multi-stage archive synchronisations + * Support for hook scripts at various points + * Support for multiple archives, even if they are pushed using one ssh key + * Support for multi-hop, multi-stage archive synchronisations + +### Correct support for multiple pushes +When the script receives a second push while it is running and syncing +the archive it won't ignore it. Instead it will rerun the +synchronisation step to ensure the archive is correctly synchronised. + +Scripts that fail to do that risk ending up with an inconsistent archive. + +### Can do multi-stage archive synchronisations +The script can be told to only perform the first or second stage of the +archive synchronisation. + +This enables us to send all the binary packages and sources to a +number of mirrors, and then tell all of them to sync the +Packages/Release files at once. This will keep the timeframe in which +the mirrors are out of sync very small and will greatly help things like +DNS RR entries or even the planned GeoDNS setup. + +### Multi-hop, multi-stage archive synchronisations +The script can be told to perform a multi-hop multi-stage archive +synchronisation. + +This is basically the same as the multi-stage synchronisation +explained above, but enables the downstream mirror to push his own +staged/multi-hop downstreams before returning. This has the same +advantage than the multi-stage synchronisation but allows us to do +this over multiple level of mirrors. (Imagine one push going from +Europe to Australia, where then locally 3 others get updated before +stage2 is sent out. Instead of 4times transferring data from Europe to +Australia, just to have them all updated near instantly). + + +### Can run hook scripts +ftpsync currently allows 5 hook scripts to run at various points of the +mirror sync run. + +* Hook1: After lock is acquired, before first rsync +* Hook2: After first rsync, if successful +* Hook3: After second rsync, if successful +* Hook4: Right before leaf mirror triggering +* Hook5: After leaf mirror trigger (only if we have slave mirrors; HUB=true) + +Note that Hook3 and Hook4 are likely to be called directly after each other. +The difference is that Hook3 is called *every* time the second rsync +succeeds even if the mirroring needs to re-run due to a second push. +Hook4 is only executed if mirroring is completed. + +### Support for multiple archives, even if they are pushed using one ssh key +If you get multiple archives from your upstream mirror (say Debian, +Debian-Backports and Volatile), previously you had to use 3 different ssh +keys to be able to automagically synchronize them. This script can do it +all with just one key, if your upstream mirror tells you which archive. +See "Commandline/SSH options" below for further details. + +For details of all available options, please see the extensive documentation +in the sample configuration file. + +### Commandline/SSH options + +Script options may be set either on the local command line, or passed by +specifying an ssh "command". Local commandline options always have +precedence over the SSH_ORIGINAL_COMMAND ones. + +Currently this script understands the options listed below. To make them +take effect they MUST be prepended by "sync:". + +| Option | Behaviour | +|--------------|-----------| +| stage1 | Only do stage1 sync | +| stage2 | Only do stage2 sync | +| all | Do a complete sync (default) | +| mhop | Do a multi-hop sync | +| archive:foo | Sync archive foo (if the file $HOME/etc/ftpsync-foo.conf exists and is configured) | +| callback | Call back when done (needs proper ssh setup for this to work). It will always use the "command" callback:$HOSTNAME where $HOSTNAME is the one defined in config and will happen before slave mirrors are triggered. | + +So, to get the script to sync all of the archive behind bpo and call back when +it is complete, use an upstream trigger of +ssh $USER@$HOST sync:all sync:archive:bpo sync:callback + + +Mirror trace files +================== +Every mirror needs to have a 'trace' file under project/trace. +The filename has to be the full hostname (eg. hostname -f), or in the +case of a mirror participating in RR DNS (where users will never use +the hostname) the name of the DNS RR entry, eg. security.debian.org +for the security rotation.). (Note: ftp.$COUNTRY.debian.org is always +wrong, don't use that). + +The contents are defined as: + +First line is always the output of "date -u", ideally run with LANG +and LC_ALL set to POSIX. + +Lines two to the end of file follow a RFC822 style format, though the +field names can have spaces. Currently the following fields are +defined, listed in the order as output by ftpsync, though the order is +not mandantory: + +| Field | Content | Example | +|-------|---------|---------| +| `Date` | Date in RFC822 Format when the tracefile got generated, ie. end of mirror run. | `Sun, 28 Feb 2016 09:40:29 +0000` | +| `Date-Started` | As Date, but the time the mirror run started. | `Sun, 28 Feb 2016 09:33:55 +0000` | +| `Archive Serial` | Archive serial for mirror run, taken from main archives tracefile. | `2016022802` | +| `Creator` | Name and version of used software | `ftpsync 20170204` +| `Running on host` | FQDN of mirror host. This may not match the actual mirror name, its the real hostname. | `klecker.debian.org` | +| `Maintainer` | Groups and people responsible for this mirror. | `Admins ` | +| `Sponsor` | Organizations sponsoring this mirror. | `Example ` | +| `Country` | The ISO 3361-1 code of the country hosting this mirror. | `DE` | +| `Location` | The location this mirror is hosted in. | `Example` | +| `Throughput` | Available throughput for this mirror per second. | `10Gb` | +| `Trigger` | Trigger used for this run.` | `cron`, `ssh (comment)` | +| `Architectures` | List of architectures included in the mirror. | `all amd64 i386 source` | +| `Architectures-Configuration` | Architecture list as specified in config. | `ALL`, `INCLUDE amd64 i386 source`, `EXCLUDE armel` | +| `Upstream-Mirror` | From where does the mirror get its data. | `ftp-master.debian.org` | +| `Rsync-Transport` | Transport to connect to upstream used by rsync | `plain` +| `Total bytes received in rsync` | rsync --stats output, bytes received | `1109846675` | +| `Total time spent in stage1 rsync` | Seconds of runtime for stage1 | `347` | +| `Total time spent in stage2 rsync` | Seconds of runtime for stage2 | `47` | +| `Total time spent in rsync` | How long in total for rsync | `394` | +| `Average rate` | How fast did the sync go | `2816869 B/s` | + +The third line in the legacy format and the hostname for the "Running +on Host" line for the new format MUST NOT be the DNS RR name, even if +the mirror is part of it. It MUST BE the hosts own name. This is in +contrast to the filename, which SHOULD be the DNS RR name. + + +## runmirrors + +This script is used to tell leaf mirrors that it is time to synchronize +their copy of the archive. This is done by parsing a mirror list and +using ssh to "push" the leaf nodes. You can read much more about the +principle behind the +[push](http://blog.ganneff.de/blog/2007/12/29/ssh-triggers.html), +essentially it tells the receiving +end to run a pre-defined script. As the whole setup is extremely limited +and the ssh key is not usable for anything else than the pre-defined +script this is the most secure method for such an action. + +This script supports two types of pushes: The normal single stage push, +as well as the newer multi-stage push. + +The normal push, as described above, will simply push the leaf node and +then go on with the other nodes. + +The multi-staged push first pushes a mirror and tells it to only do a +stage1 sync run. Then it waits for the mirror (and all others being pushed +in the same run) to finish that run, before it tells all of the staged +mirrors to do the stage2 sync. + +This way you can do a nearly-simultaneous update of multiple hosts. +This is useful in situations where periods of desynchronization should +be kept as small as possible. Examples of scenarios where this might be +useful include multiple hosts in a DNS Round Robin entry. + +For details on the mirror list please see the documented +runmirrors.mirror.sample file. diff --git a/mirrors/files/distros/Debian/repository/ftpsync/distrib/doc/ftpsync-cron.1.md b/mirrors/files/distros/Debian/repository/ftpsync/distrib/doc/ftpsync-cron.1.md new file mode 100644 index 0000000..9f0f9d5 --- /dev/null +++ b/mirrors/files/distros/Debian/repository/ftpsync/distrib/doc/ftpsync-cron.1.md @@ -0,0 +1,36 @@ +% FTPSYNC-CRON(1) +% Debian mirror team +% ftpsync Manual + +# NAME +ftpsync-cron - Wrapper around ftpsync for use in cron(8) + +# SYNOPSIS +**ftpsync-cron** [_ARCHIVE_] + +# DESCRIPTION + +**ftpsync-cron** is part of the ftpsync suite for mirroring Debian and Debian-like +repositories of packages. As there are way too many mirrors of Debian to populate +them all from the machine that generates the archive ("ftp-master"), mirrors are +organized in a tree-shaped hierarchy. Thus, every mirror has exactly one upstream +from which it syncs, and each mirror can have any number of downstreams which in +turn sync from it. + +**ftpsync-cron** is a wrapper around **ftpsync** itself, intended to be run out +of cron at regular, frequent intervals, such as hourly. When started, it reads +the corresponding **ftpsync.conf** configuration file for the archive in +question and determines this mirror's upstream. It then fetches the upstream's +trace file via HTTP and compares it to the version on disk from the last mirror +run, thus learning whether the upstream mirror has updated. If it has, +**ftpsync** is triggered. + +# EXAMPLE + +Example use in a user crontab: + + SHELL=/bin/bash + 31 * * * * sleep $(( RANDOM %% 1800 )) && ./bin/ftpsync-cron + +# SEE ALSO +**ftpsync**(1) + diff --git a/mirrors/files/distros/Debian/repository/ftpsync/distrib/doc/ftpsync.1.md b/mirrors/files/distros/Debian/repository/ftpsync/distrib/doc/ftpsync.1.md new file mode 100644 index 0000000..c7deae5 --- /dev/null +++ b/mirrors/files/distros/Debian/repository/ftpsync/distrib/doc/ftpsync.1.md @@ -0,0 +1,53 @@ +% FTPSYNC(1) +% Debian mirror team +% ftpsync Manual + +# NAME +ftpsync - Mirror Debian and Debian-like repositories of packages + +# SYNOPSIS +**ftpsync** [_OPTION_]... [_SYNC-FLAG_]... + +# DESCRIPTION + +**ftpsync** is part of the ftpsync suite for mirroring Debian and Debian-like +repositories of packages. As there are way too many mirrors of Debian to populate +them all from the machine that generates the archive ("ftp-master"), mirrors are +organized in a tree-shaped hierarchy. Thus, every mirror has exactly one upstream +from which it syncs, and each mirror can have any number of downstreams which in +turn sync from it. + +# SYNC FLAGS + +Sync flags can be specified on the command-line or via a ssh forced command from the remote side. + +**sync:archive:_foo_** +: Select archive _foo_ for syncing. A config needs to be available. + +**sync:all** +: Do a complete sync. + +**sync:stage1** +: Only do a stage 1 sync. + +**sync:stage2** +: Only do a stage 2 sync. + +**sync:mhop** +: Do a special multi-hop sync, usually additionally to stage 1. + +**sync:callback** +: Call back when done. + +# OPTIONS + +**-T** +: Information about the used trigger. + + If undefined the following values are automatically set: + + - **ssh** if called from a ssh forced command. + - **cron** if called via **ftpsyn-cron**(1). + +# SEE ALSO +**ftpsync.conf**(5) + diff --git a/mirrors/files/distros/Debian/repository/ftpsync/distrib/doc/ftpsync.conf.5.md b/mirrors/files/distros/Debian/repository/ftpsync/distrib/doc/ftpsync.conf.5.md new file mode 100644 index 0000000..120f7ad --- /dev/null +++ b/mirrors/files/distros/Debian/repository/ftpsync/distrib/doc/ftpsync.conf.5.md @@ -0,0 +1,307 @@ +% FTPSYNC.CONF(5) +% Debian mirror team +% ftpsync Manual + +# NAME +ftpsync.conf - Configuration for ftpsync + +# DESCRIPTION + +**ftpsync** is part of the ftpsync suite for mirroring Debian and Debian-like +repositories of packages. As there are way too many mirrors of Debian to populate +them all from the machine that generates the archive ("ftp-master"), mirrors are +organized in a tree-shaped hierarchy. Thus, every mirror has exactly one upstream +from which it syncs, and each mirror can have any number of downstreams which in +turn sync from it. + +# OPTIONS + +**MIRRORNAME** +: Mirrorname. This is used for things like the trace file name and should + always be the full hostname of the mirror. + + Default: **$(hostname -f)** + +**TO** +: Destination of the mirrored files. Should be an empty directory. CAREFUL, + this directory will contain the mirror. Everything else that might have + happened to be in there WILL BE GONE after the mirror sync! + + Default: **/srv/mirrors/debian/** + +**MAILTO** +: The script can send logs (or error messages) to a mail address. + If this is unset it will default to the current user. + + Default: **$LOGNAME** + +**HUB** +: Do we have leaf mirror to signal we are done and they should sync? + If so set it to true and make sure you configure runmirrors.mirrors + and runmirrors.conf for your need. + + Default: **false** + +## Connection options + +**RSYNC_HOST** +: The host we mirror from. + +**RSYNC_PATH** +: The upstream name of the rsync share. + + You can find out what share names your upstream mirror supports by running + rsync YOURUPSTREAMSERVER:: + (You might have to export RSYNC_USER/RSYNC_PASSWORD for this to work) + + Default: **debian** + +**RSYNC_USER** +: In case we need a user to access the rsync share at our upstream host + +**RSYNC_PASSWORD** +: If we need a user we also need a password + +**RSYNC_TRANSPORT** +: Set to select the transport used for rsync. + + **ssh** + : Use rsync daemon over SSH. + This requires a **rsyncd.conf** on the server. + Additional options for the ssh client should be configured in **ssh_config**. + + **ssl** + : Use rsync daemon over SSL. + This requires a SSL wrapper (e.g. stunnel) for the rsync daemon on the server. + See the other **RSYNC_SSL** options below. + +**RSYNC_SSL_PORT** +: Default: **1873** + +**RSYNC_SSL_CAPATH** +: Default: **/etc/ssl/certs** + +**RSYNC_SSL_METHOD** +: ftpsync can use either stunnel, stunnel-old, or socat to set up the + encrypted tunnel. + + **stunnel** + : requires at least stunnel version 5.15 built aginst openssl + 1.0.2 or later such that the stunnel build supports the checkHost + service-level option. This will cause stunnel to verify both the + peer certificate's validity and that it's actually for the host we wish + to connect to. + + **stunnel-old** + : will skip the checkHost check. As such it will connect + to any peer that is able to present a valid certificate, regardless of + which name it is made out to. + + **socat** + : will verify the peer certificate name only starting with version + 1.7.3 (Debian 9.0). + + To test if things work, you can run: + + **rsync -e 'bin/rsync-ssl-tunnel -m socat -p 1873 -C /etc/ssl/certs' SERVER::** + + Default: stunnel + +**RSYNC_PROXY** +: You may establish the connection via a web proxy by setting the environment + variable **RSYNC_PROXY** to a hostname:port pair pointing to your web proxy. Note + that your web proxy's configuration must support proxy connections to port 873. + +## Mirror information options + +These options add informations about the mirror to a publicly accessible file. + +**INFO_MAINTAINER** +: Groups and people responsible for this mirror. These contacts are + used to report irregularities and problems with the mirror. They + must actually accept mail. + + Example: **INFO_MAINTAINER="Admins , Person "** + +**INFO_SPONSOR** +: Organizations sponsoring this mirror. + + Example: **INFO_SPONSOR="Example "** + +**INFO_COUNTRY** +: The ISO 3361-1 code of the country hosting this mirror. + + Example: **INFO_COUNTRY=DE** + +**INFO_LOCATION** +: The location this mirror is hosted in. + + Example: **INFO_LOCATION="Example"** + +**INFO_THROUGHPUT** +: Available throughput for this mirror per second. + + Example: **INFO_THROUGHPUT=10Gb** + +## Include and exclude options + +These options can be used to include or exclude specified architectures or other files from the mirror. + +**Notice: the source architecture needs to be included on an official/public mirror!** + +**ARCH_INCLUDE** +: If you want to include only a subset of architectures, this is for you. + Use as space seperated list of architectures in the archive you are + mirroring from, "source" counts as architecture. + + Architecture "all" will be included automatically if one binary + architecture is included. + + Mutually exclusive with **ARCH_EXCLUDE**. + + Example: **ARCH_INCLUDE="amd64 i386 source"** + +**ARCH_EXCLUDE** +: If you want to exclude an architecture, this is for you. + Use as space seperated list of architectures in the archive you are + mirroring from, "source" counts as architecture. + + Mutually exclusive with **ARCH_INCLUDE**. + + Example: **ARCH_EXCLUDE="alpha arm arm64 armel mipsel mips s390 sparc"** + +**EXCLUDE** +: If you do want to exclude files from the mirror run, put --exclude statements here. + See rsync(1) for the exact syntax, these are passed to rsync as written here. + DO NOT TRY TO EXCLUDE ARCHITECTURES OR SUITES WITH THIS, IT WILL NOT WORK! + +## Log option + +**LOGDIR** +: In which directory should logfiles end up. + + Default: **~/.local/log/ftpsync** in the package, **${BASEDIR}/log** otherwise + +**LOG** +: Name of our own logfile. + + Note that ${NAME} is set by the ftpsync script depending on the way it + is called. See README for a description of the multi-archive capability + and better always include ${NAME} in this path. + + Default: **${LOGDIR}/${NAME}.log** + +**ERRORSONLY** +: If you do want a mail about every single sync, set this to false + Everything else will only send mails if a mirror sync fails. + + Default: **true** + +**FULLLOGS** +: If you want the logs to also include output of rsync, set this to true. + Careful, the logs can get pretty big, especially if it is the first mirror run. + + Default: **false** + +**LOGROTATE** +: We do create three logfiles for every run. To save space we rotate them, this + defines how many we keep + + Default: **14** + +## Other options + +**LOCKTIMEOUT** +: Timeout for the lockfile, in case we have bash older than v4 (and no /proc) + + Default: **3600** + +**UIPSLEEP** +: Number of seconds to sleep before retrying to sync whenever upstream + is found to be updating while our update is running + + Default: **1200** + +**UIPRETRIES** +: Number of times the update operation will be retried when upstream + is found to be updating while our update is running. + Note that these are retries, so: 1st attempt + retries = total attempts + + Default: **3** + +**TRACEHOST** +: The local hostname to be written to the trace file. + + Default: **$(hostname -f)** + +**RSYNC** +: The rsync program + +**RSYNC_EXTRA** +: Extra rsync options as defined by the local admin. + There is no default by ftpsync. + + Please note that these options are added to EVERY rsync call. + Also note that these are added at the beginning of the rsync call, as + the very first set of options. + Please ensure you do not add a conflict with the usual rsync options as + shown below. + +**RSYNC_BW** +: limit I/O bandwidth. Value is KBytes per second, unset or 0 means unlimited + + +**RSYNC_OPTIONS** +: Default rsync options every rsync invocation sees. + + BE VERY CAREFUL WHEN YOU CHANGE THIS OPTION, BETTER DON'T! + + Default: See ftpsync script + +**RSYNC_OPTIONS1** +: Options the first pass gets. We do not want the Packages/Source indices + here, and we also do not want to delete any files yet. + + BE VERY CAREFUL WHEN YOU CHANGE THIS OPTION, BETTER DON'T! + + Default: See ftpsync script + +**RSYNC_OPTIONS2** +: Options the second pass gets. Now we want the Packages/Source indices too + and we also want to delete files. We also want to delete files that are + excluded. + + BE VERY CAREFUL WHEN YOU CHANGE THIS OPTION, BETTER DON'T! + + Default: See ftpsync script + +**CALLBACKUSER**, **CALLBACKHOST**, **CALLBACKKEY** +: The following three options are used in case we want to "callback" the host + we got pushed from. + +## Hooks + +Hook scripts can be run at various places during the sync. + +**HOOK1** +: After lock is acquired, before first rsync. + +**HOOK2** +: After first rsync, if successful. + +**HOOK3** +: After second rsync, if successful. + +**HOOK4** +: Right before leaf mirror triggering. + + Note that Hook3 and Hook4 are likely to be called directly after each other. + Difference is: Hook3 is called *every* time the second rsync was successful, + but even if the mirroring needs to re-run thanks to a second push. + Hook4 is only effective if we are done with mirroring. + +**HOOK5** +: After leaf mirror trigger, only if we have slave mirrors (**HUB=true**). + +# SEE ALSO +**ftpsync**(1) diff --git a/mirrors/files/distros/Debian/repository/ftpsync/distrib/doc/rsync-ssl-tunnel.1.md b/mirrors/files/distros/Debian/repository/ftpsync/distrib/doc/rsync-ssl-tunnel.1.md new file mode 100644 index 0000000..9160fcf --- /dev/null +++ b/mirrors/files/distros/Debian/repository/ftpsync/distrib/doc/rsync-ssl-tunnel.1.md @@ -0,0 +1,44 @@ +% RSYNC-SSL-TUNNEL(1) +% Debian mirror team +% ftpsync Manual + +# NAME +rsync-ssl-tunnel - Allows rsync to tunnel over TLS + +# SYNOPSIS +**rsync-ssl-tunnel** + +# DESCRIPTION + +**rsync-ssl-tunnel** is part of the ftpsync suite for mirroring Debian and Debian-like +repositories of packages. As there are way too many mirrors of Debian to populate +them all from the machine that generates the archive ("ftp-master"), mirrors are +organized in a tree-shaped hierarchy. Thus, every mirror has exactly one upstream +from which it syncs, and each mirror can have any number of downstreams which in +turn sync from it. + +# OPTIONS + +**-C** +: The CA path for certificate verification + + Default: **/etc/ssl/certs** + +**-m** +: The method used to connect to the remote host. + + Default: **stunnel** + +**-p** +: The port used to connect to the remote host. + + Default: **1873** + +# EXAMPLE + +Example use with rsync: + + rsync -e rsync-ssl-tunnel + +# SEE ALSO +**ftpsync**(1) + diff --git a/mirrors/files/distros/Debian/repository/ftpsync/distrib/doc/runmirrors.1.md b/mirrors/files/distros/Debian/repository/ftpsync/distrib/doc/runmirrors.1.md new file mode 100644 index 0000000..6e8d6ae --- /dev/null +++ b/mirrors/files/distros/Debian/repository/ftpsync/distrib/doc/runmirrors.1.md @@ -0,0 +1,21 @@ +% RUNMIRRORS(1) +% Debian mirror team +% ftpsync Manual + +# NAME +runmirrors - Trigger downstream mirrors + +# SYNOPSIS +**runmirrors** + +# DESCRIPTION + +**runmirrors** is part of the ftpsync suite for mirroring Debian and Debian-like +repositories of packages. As there are way too many mirrors of Debian to populate +them all from the machine that generates the archive ("ftp-master"), mirrors are +organized in a tree-shaped hierarchy. Thus, every mirror has exactly one upstream +from which it syncs, and each mirror can have any number of downstreams which in +turn sync from it. + +# SEE ALSO +**ftpsync**(1), **runmirrors.conf**(5), **runmirrors.mirror**(5) diff --git a/mirrors/files/distros/Debian/repository/ftpsync/distrib/doc/runmirrors.conf.5.md b/mirrors/files/distros/Debian/repository/ftpsync/distrib/doc/runmirrors.conf.5.md new file mode 100644 index 0000000..ec6fcfa --- /dev/null +++ b/mirrors/files/distros/Debian/repository/ftpsync/distrib/doc/runmirrors.conf.5.md @@ -0,0 +1,95 @@ +% RUNMIRRORS.CONF(5) +% Debian mirror team +% ftpsync Manual + +# NAME +runmirrors.conf - Configuration for runmirrors + +# DESCRIPTION + +**ftpsync** is part of the ftpsync suite for mirroring Debian and Debian-like +repositories of packages. As there are way too many mirrors of Debian to populate +them all from the machine that generates the archive ("ftp-master"), mirrors are +organized in a tree-shaped hierarchy. Thus, every mirror has exactly one upstream +from which it syncs, and each mirror can have any number of downstreams which in +turn sync from it. + +# OPTIONS + +**MAILTO** +: The script can send logs (or error messages) to a mail address. + If this is unset it will default to the current user. + + Default: **$LOGNAME** + +**KEYFILE** +: Which ssh key to use? + + Default: **~/.ssh/pushmirror** + +## Log options + +**LOGDIR** +: In which directory should logfiles end up. + + Default: **~/.local/log/ftpsync** in the package, **${BASEDIR}/log** otherwise + +**LOG** +: Name of our own logfile. + + Note that ${NAME} is set by the ftpsync script depending on the way it + is called. See README for a description of the multi-archive capability + and better always include ${NAME} in this path. + + Default: **${LOGDIR}/${NAME}.log** + +**LOGROTATE** +: We do create a logfile for every run. To save space we rotate them, this + defines how many we keep + + Default: **14** + +## Other options + +**LOCKDIR** +: Our lockfile directory. + + Default: **~/.local/lock/ftpsync** in the package, **${BASEDIR}/locks** otherwise + +**MIRRORS** +: Our mirrorfile + + Default: **${CONFDIR}/${NAME}.mirror** + +**SSH_OPTS** +: Extra ssh options we might want. *hostwide*. + By default, ignore ssh key change of leafs + + Default **"-o StrictHostKeyChecking=no"** + +**PUSHARCHIVE** +: Whats our archive name? We will also tell our leafs about it + This is usually empty, but if we are called as "runmirrors bpo" + it will default to bpo. This way one runmirrors script can serve + multiple archives, similar to what ftpsync does. + +**PUSHDELAY** +: How long to wait for mirrors to do stage1 if we have multi-stage syncing + + Default: **600** + +## Hooks + +Hook scripts can be run at various places. + +**HOOK1** +: After reading config, before doing the first real action. + +**HOOK2** +: Between two hosts to push. + +**HOOK3** +: When everything is done. + +# SEE ALSO +**runmirrors**(1) diff --git a/mirrors/files/distros/Debian/repository/ftpsync/distrib/doc/runmirrors.mirror.5.md b/mirrors/files/distros/Debian/repository/ftpsync/distrib/doc/runmirrors.mirror.5.md new file mode 100644 index 0000000..8484c73 --- /dev/null +++ b/mirrors/files/distros/Debian/repository/ftpsync/distrib/doc/runmirrors.mirror.5.md @@ -0,0 +1,20 @@ +% RUNMIRRORS.MIRROR(5) +% Debian mirror team +% ftpsync Manual + +# NAME +runmirrors.mirror - Downstream mirror list for runmirrors + +# DESCRIPTION + +**ftpsync** is part of the ftpsync suite for mirroring Debian and Debian-like +repositories of packages. As there are way too many mirrors of Debian to populate +them all from the machine that generates the archive ("ftp-master"), mirrors are +organized in a tree-shaped hierarchy. Thus, every mirror has exactly one upstream +from which it syncs, and each mirror can have any number of downstreams which in +turn sync from it. + +# OPTIONS + +# SEE ALSO +**runmirrors**(1) diff --git a/mirrors/files/distros/Debian/repository/ftpsync/etc/ftpsync.conf b/mirrors/files/distros/Debian/repository/ftpsync/etc/ftpsync.conf new file mode 100644 index 0000000..787528c --- /dev/null +++ b/mirrors/files/distros/Debian/repository/ftpsync/etc/ftpsync.conf @@ -0,0 +1,39 @@ +######################################################################## +######################################################################## +## This is a sample configuration file for the ftpsync mirror script. ## +## Only options most users may need are included. For documentation ## +## and all available options see ftpsync.conf(5). ## +######################################################################## +######################################################################## + +MIRRORNAME=`hostname -f` +TO="/srv/debian/" + +######################################################################## +## Connection options +######################################################################## + +RSYNC_HOST=debian.csail.mit.edu +RSYNC_PATH="debian" + +######################################################################## +## Mirror information options +######################################################################## + +INFO_MAINTAINER="LIDSoL team , Enrique Calderon " +INFO_COUNTRY=MX +INFO_LOCATION="FI UNAM" +INFO_THROUGHPUT=100Mb + +######################################################################## +## Include and exclude options +######################################################################## + +# ARCH_INCLUDE= +# ARCH_EXCLUDE= + +######################################################################## +## Log option +######################################################################## + +LOGDIR="${BASEDIR}/log" diff --git a/mirrors/files/distros/Debian/repository/ftpsync/etc/ftpsync.conf.sample b/mirrors/files/distros/Debian/repository/ftpsync/etc/ftpsync.conf.sample new file mode 100644 index 0000000..2445a5e --- /dev/null +++ b/mirrors/files/distros/Debian/repository/ftpsync/etc/ftpsync.conf.sample @@ -0,0 +1,44 @@ +######################################################################## +######################################################################## +## This is a sample configuration file for the ftpsync mirror script. ## +## Only options most users may need are included. For documentation ## +## and all available options see ftpsync.conf(5). ## +######################################################################## +######################################################################## + +# MIRRORNAME=`hostname -f` +# TO="/srv/mirrors/debian/" +# MAILTO="$LOGNAME" +# HUB=false + +######################################################################## +## Connection options +######################################################################## + +RSYNC_HOST= +RSYNC_PATH="debian" +# RSYNC_USER= +# RSYNC_PASSWORD= + +######################################################################## +## Mirror information options +######################################################################## + +# INFO_MAINTAINER="Admins , Person " +# INFO_SPONSOR="Example " +# INFO_COUNTRY=DE +# INFO_LOCATION="Example" +# INFO_THROUGHPUT=10Gb + +######################################################################## +## Include and exclude options +######################################################################## + +# ARCH_INCLUDE= +# ARCH_EXCLUDE= + +######################################################################## +## Log option +######################################################################## + +# LOGDIR= diff --git a/mirrors/files/distros/Debian/repository/ftpsync/etc/runmirrors.conf.sample b/mirrors/files/distros/Debian/repository/ftpsync/etc/runmirrors.conf.sample new file mode 100644 index 0000000..656a112 --- /dev/null +++ b/mirrors/files/distros/Debian/repository/ftpsync/etc/runmirrors.conf.sample @@ -0,0 +1,16 @@ +######################################################################## +######################################################################## +## This is a sample configuration file for the runmirror script. ## +## Only options most users may need are included. For documentation ## +## and all available options see runmirrors.conf(5). ## +######################################################################## +######################################################################## + +# MAILTO="$LOGNAME" +# KEYFILE=~/.ssh/pushmirror + +######################################################################## +## Log option +######################################################################## + +# LOGDIR= diff --git a/mirrors/files/distros/Debian/repository/ftpsync/etc/runmirrors.mirror.sample b/mirrors/files/distros/Debian/repository/ftpsync/etc/runmirrors.mirror.sample new file mode 100644 index 0000000..379d808 --- /dev/null +++ b/mirrors/files/distros/Debian/repository/ftpsync/etc/runmirrors.mirror.sample @@ -0,0 +1,81 @@ +# Definition of mirror hosts we push. +# One mirror per line, with the following formats: +# +# Type ShortName URL +# Type ShortName HostName User -$SOMESSHOPTION +# Type ShortName HostName User SSHProtocol SSHKeyFile +# +# The fields Type, ShortName and either URL or HostName and User are *mandantory*. +# +# Type is either all, staged or mhop, meaning: +# all - do a "normal" push. Trigger them, go on. +# This is the default for a Debian mirror. It does all the +# right things (when the mirror uses ftpsync), like staged-syncing. +# staged - do a two-stage push, waiting for them after stage 2(and all others that +# are staged) before doing stage2. +# Contrary to all, this waits for ALL downstream mirrors +# marked "staged", so most useful if they are behind a DNS RR +# and need to update dists/ at near the same time. +# mhop - send a multi-hop staged push. This will tell the mirror to initiate +# a mhop/stage1 push to its staged/mhop mirrors and then exit. +# When all mhop got back we then send the stage2 through to them. +# +# ShortName will be used as a shorthand in logfile outputs and for the logfile +# where every ssh output gets redirected to. +# +# URL can be one of following: +# - ssh+ftpsync://user@hostname +# (same as specifying the HostName and User field) +# - ssh+ftpsync://user@hostname/archive +# (override the signaled archive name via 'sync:archive:$archive argument) +# +# If no SSHKeyFile is given, the default from the config file applies. +# If SSHProtocol is empty, it will default to 2, but if you want to +# define a keyfile you HAVE TO set protocol too! +# +# With the ALTERNATIVE syntax you are able to use any special ssh option +# you want just for one special mirror. The option after the username +# then MUST start with a -, in which case the whole rest of the line is taken +# as a set of options to give to ssh, it is passed through without doing +# anything with it. +# +# There is one caveat here: Should you want to use the -i option to give +# another ssh key to use, keep in mind that the master keyfile will +# always be presented too! That is, ssh will show both keys to the other +# side and the first one presented wins. Which might not be the key you +# want. There is currently no way around this, as that would mean +# dropping backward compatibility. +# +# Backwards compatibility: +# An older runmirrors script will NOT run with a newer runmirrors.mirror file, but +# a new runmirrors can run with an old runmirrors.mirror file. This should make updates +# painless. +# +# Examples: +# all eu.puccini puccini.debian.org archvsync 2 +# +# -> will push puccini.debian.org, user archvsync, using ssh protocol 2 +# and the globally configured ssh key. +# +# all eu.puccini puccini.debian.org archvsync -p 2222 +# +# -> will do the same as above, but use port 2222 to connect to. +# +# staged eu.puccini puccini.debian.org archvsync +# staged eu.powell powell.debian.org archvsync +# +# -> will push both puccini and powell in stage1, waiting for both to +# finish stage1 before stage2 gets pushed. The username will be archvsync. +# +# staged eu.puccini puccini.debian.org archvsync +# mhop eu.powell powell.debian.org archvsync +# +# -> will do the same as above, but powell gets told about mhop and can then +# push its own staged/mhop mirrors before returning. When both returned +# then stage2 is sent to both. +# +# One special value is allowed: DELAY +# This word has to be on a line itself, followed by a space and a number. +# nothing else, not even whitespace. It will trigger a pause of $number +# seconds between the two mirrors. If no number is given it defaults to +# 600 seconds. diff --git a/mirrors/files/distros/Debian/repository/scripts/debian-mirror-sync-check.sh b/mirrors/files/distros/Debian/repository/scripts/debian-mirror-sync-check.sh new file mode 100644 index 0000000..83a3920 --- /dev/null +++ b/mirrors/files/distros/Debian/repository/scripts/debian-mirror-sync-check.sh @@ -0,0 +1,81 @@ +#!/bin/bash +# This script checks the status of the upstream Debian mirror. +# Specifically, it checks the status file in the project/trace/ +# directory in the upstream mirror and in the local mirror, +# and checks the following conditions: +# +# - If the pull from upstream mirror is less than 2 hours old, +# then the mirror is considered up-to-date, and no action is taken. +# - If the pull from upstream mirror is older than 2 hours old, and +# the local mirror is older than the upstream mirror, then a new +# pull is initiated. +# - If the pull from upstream mirror is older than 2 hours old, and +# the local mirror is newer than the upstream mirror, then the +# local mirror is considered up-to-date, and no action is taken. +# +# This script is intended to be run repeatedly by a cron job or +# a systemd timer. + +LOCAL_MIRROR="lidsol.fi-b.unam.mx" +MIRROR_DIRECTORY=/srv/debian + +# Set the upstream mirror URL +UPSTREAM_MIRROR=debian.csail.mit.edu +UPSTREAM_MIRROR_URL="https://${UPSTREAM_MIRROR}/debian/project/trace/${UPSTREAM_MIRROR}" + +function get_upstream_time() { + curl -s "${UPSTREAM_MIRROR_URL}" | head -n 1 +} + +function get_local_time() { + if [ -f ${MIRROR_DIRECTORY}/project/trace/${LOCAL_MIRROR} ]; then + cat ${MIRROR_DIRECTORY}/project/trace/${LOCAL_MIRROR} | head -n 1 + else + echo "1970-01-01 00:00:00 UTC" + fi +} + +function should_pull() { + local_mirror_time=$1 + upstream_time=$2 + current_time=$3 + # Log input of function to stderr + local_mirror_time_epoch=$(date -d "$local_mirror_time" +%s) + upstream_time_epoch=$(date -d "$upstream_time" +%s) + current_time_epoch=$(date -d "$current_time" +%s) + + # If the local mirror is more than 4 hours old, then the mirror is considered out of date + if [ $(($upstream_time_epoch - $local_mirror_time_epoch)) -gt 14400 ]; then + echo "true" + return + fi + + # If it has been less than 2 hours since the upstream mirror started updating, + # then the mirror is considered up-to-date + if [ $upstream_time_epoch -gt $(($current_time_epoch - 7200)) ]; then + # Log the operation above to stderr + echo "false" + return + fi + + # If the local mirror is older than the upstream mirror, then the local mirror + # is considered out of date + if [ $local_mirror_time_epoch -lt $upstream_time_epoch ]; then + echo "true" + else + echo "false" + fi +} + +# If this script is called with the --test flag, then it does not perform any actions +if [ "$1" == "--test" ]; then + return +fi + +if [ $(should_pull "$(get_local_time)" "$(get_upstream_time)" "$(date -u)") == "true" ]; then + echo "Local mirror is out of date, pulling from upstream mirror. Upstream date: $(get_upstream_time), Local date: $(get_local_time) - current date: $(date -u)" + cd ${MIRROR_DIRECTORY} + /home/mirrors/bin/ftpsync sync:all +else + echo "Local mirror is up to date. Upstream date: $(get_upstream_time), Local date: $(get_local_time) - current date: $(date -u)" +fi diff --git a/mirrors/files/SO/confd/mintUpdateSO.service b/mirrors/files/distros/LinuxMint/ISO/confd/mintUpdateSO.service similarity index 100% rename from mirrors/files/SO/confd/mintUpdateSO.service rename to mirrors/files/distros/LinuxMint/ISO/confd/mintUpdateSO.service diff --git a/mirrors/files/SO/confd/mintUpdateSO.timer b/mirrors/files/distros/LinuxMint/ISO/confd/mintUpdateSO.timer similarity index 100% rename from mirrors/files/SO/confd/mintUpdateSO.timer rename to mirrors/files/distros/LinuxMint/ISO/confd/mintUpdateSO.timer diff --git a/mirrors/files/SO/scripts/mintfetchSO.sh b/mirrors/files/distros/LinuxMint/ISO/scripts/mintfetchSO.sh similarity index 100% rename from mirrors/files/SO/scripts/mintfetchSO.sh rename to mirrors/files/distros/LinuxMint/ISO/scripts/mintfetchSO.sh diff --git a/mirrors/files/repository/confd/mintUpdateRP.service b/mirrors/files/distros/LinuxMint/repository/confd/mintUpdateRP.service similarity index 100% rename from mirrors/files/repository/confd/mintUpdateRP.service rename to mirrors/files/distros/LinuxMint/repository/confd/mintUpdateRP.service diff --git a/mirrors/files/repository/confd/mintUpdateRP.timer b/mirrors/files/distros/LinuxMint/repository/confd/mintUpdateRP.timer similarity index 100% rename from mirrors/files/repository/confd/mintUpdateRP.timer rename to mirrors/files/distros/LinuxMint/repository/confd/mintUpdateRP.timer diff --git a/mirrors/files/repository/scripts/mintfetchRP.sh b/mirrors/files/distros/LinuxMint/repository/scripts/mintfetchRP.sh similarity index 100% rename from mirrors/files/repository/scripts/mintfetchRP.sh rename to mirrors/files/distros/LinuxMint/repository/scripts/mintfetchRP.sh diff --git a/mirrors/handlers/main.yml b/mirrors/handlers/main.yml new file mode 100644 index 0000000..c61b050 --- /dev/null +++ b/mirrors/handlers/main.yml @@ -0,0 +1,12 @@ +--- +# Handlers for mirrors role + +- name: reload nginx + ansible.builtin.systemd_service: + name: nginx + state: reloaded + +- name: reload rsync + ansible.builtin.systemd_service: + name: rsync + state: restarted diff --git a/mirrors/tasks/almalinux.yaml b/mirrors/tasks/almalinux.yaml new file mode 100644 index 0000000..9bf2a59 --- /dev/null +++ b/mirrors/tasks/almalinux.yaml @@ -0,0 +1,33 @@ +--- +- name: Create AlmaLinux mirror directory + ansible.builtin.file: + path: /srv/almalinux + state: directory + owner: mirrors + group: mirrors + mode: "0755" + tags: + - almalinux + +- name: Copy AlmaLinux timer and service files + ansible.builtin.copy: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + owner: root + group: root + mode: "{{ item.mode }}" + loop: + - { src: "distros/AlmaLinux/repository/confd/almalinux-mirror-sync.timer", dest: "/etc/systemd/system/almalinux-mirror-sync.timer", mode: "0644" } + - { src: "distros/AlmaLinux/repository/confd/almalinux-mirror-sync.service", dest: "/etc/systemd/system/almalinux-mirror-sync.service", mode: "0644" } + tags: + - almalinux + +- name: Start AlmaLinux mirror timer service + ansible.builtin.systemd_service: + name: almalinux-mirror-sync.timer + enabled: true + state: started + daemon_reload: true + tags: + - almalinux +... diff --git a/mirrors/tasks/archlinux.yaml b/mirrors/tasks/archlinux.yaml new file mode 100644 index 0000000..7defee6 --- /dev/null +++ b/mirrors/tasks/archlinux.yaml @@ -0,0 +1,34 @@ +--- +- name: Create ArchLinux mirror directory + ansible.builtin.file: + path: /srv/archlinux + state: directory + owner: mirrors + group: mirrors + mode: "0755" + tags: + - archlinux + +- name: Copy ArchLinux timer, service and script files + ansible.builtin.copy: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + owner: root + group: root + mode: "{{ item.mode }}" + loop: + - { src: "distros/ArchLinux/repository/confd/archlinux-mirror-sync.timer", dest: "/etc/systemd/system/archlinux-mirror-sync.timer", mode: "0644" } + - { src: "distros/ArchLinux/repository/confd/archlinux-mirror-sync.service", dest: "/etc/systemd/system/archlinux-mirror-sync.service", mode: "0644" } + - { src: "distros/ArchLinux/repository/scripts/archlinux-syncrepo.sh", dest: "/usr/local/bin/archlinux-syncrepo.sh", mode: "0755" } + tags: + - archlinux + +- name: Start ArchLinux mirror timer service + ansible.builtin.systemd_service: + name: archlinux-mirror-sync.timer + enabled: true + state: started + daemon_reload: true + tags: + - archlinux +... diff --git a/mirrors/tasks/debian.yaml b/mirrors/tasks/debian.yaml new file mode 100644 index 0000000..4edf397 --- /dev/null +++ b/mirrors/tasks/debian.yaml @@ -0,0 +1,61 @@ +--- +- name: Create Debian mirror directories + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: mirrors + group: mirrors + mode: "0755" + loop: + - /srv/debian + - /srv/debian-cd + tags: + - debian + +- name: Copy ftpsync directory as /home/mirrors/ + ansible.builtin.copy: + src: files/distros/Debian/repository/ftpsync/ + dest: /home/mirrors/ + owner: mirrors + group: mirrors + mode: '0755' + +- name: Copy Debian repository timer, service and script files + ansible.builtin.copy: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + owner: root + group: root + mode: "{{ item.mode }}" + loop: + - { src: "distros/Debian/repository/confd/debian-mirror-sync.timer", dest: "/etc/systemd/system/debian-mirror-sync.timer", mode: "0644" } + - { src: "distros/Debian/repository/confd/debian-mirror-sync.service", dest: "/etc/systemd/system/debian-mirror-sync.service", mode: "0644" } + - { src: "distros/Debian/repository/scripts/debian-mirror-sync-check.sh", dest: "/home/mirrors/bin/debian-mirror-sync-check.sh", mode: "0755" } + tags: + - debian + +- name: Copy Debian ISO timer and service files + ansible.builtin.copy: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + owner: root + group: root + mode: "{{ item.mode }}" + loop: + - { src: "distros/Debian/ISO/confd/debian-cd-mirror-sync.timer", dest: "/etc/systemd/system/debian-cd-mirror-sync.timer", mode: "0644" } + - { src: "distros/Debian/ISO/confd/debian-cd-mirror-sync.service", dest: "/etc/systemd/system/debian-cd-mirror-sync.service", mode: "0644" } + tags: + - debian + +- name: Start Debian mirror timer services + ansible.builtin.systemd_service: + name: "{{ item }}" + enabled: true + state: started + daemon_reload: true + loop: + - debian-mirror-sync.timer + - debian-cd-mirror-sync.timer + tags: + - debian +... diff --git a/mirrors/tasks/linux-mint.yaml b/mirrors/tasks/linux-mint.yaml new file mode 100644 index 0000000..c300858 --- /dev/null +++ b/mirrors/tasks/linux-mint.yaml @@ -0,0 +1,41 @@ +--- + +- name: Create Linux Mint mirror directories + ansible.builtin.file: + path: /srv/linux-mint + state: directory + owner: mirrors + group: mirrors + mode: "0755" + tags: + - linux-mint + +- name: Copy linux mint timer and service files + ansible.builtin.copy: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + owner: root + group: root + mode: "{{ item.mode }}" + loop: + - { src: "distros/LinuxMint/ISO/confd/mintUpdateSO.timer", dest: "/etc/systemd/system/mintUpdateSO.timer", mode: "0644" } + - { src: "distros/LinuxMint/ISO/confd/mintUpdateSO.service", dest: "/etc/systemd/system/mintUpdateSO.service", mode: "0644" } + - { src: "distros/LinuxMint/ISO/scripts/mintfetchSO.sh", dest: "/usr/local/bin/mintfetchSO.sh", mode: "0755" } + - { src: "distros/LinuxMint/repository/confd/mintUpdateRP.timer", dest: "/etc/systemd/system/mintUpdateRP.timer", mode: "0644" } + - { src: "distros/LinuxMint/repository/confd/mintUpdateRP.service", dest: "/etc/systemd/system/mintUpdateRP.service", mode: "0644" } + - { src: "distros/LinuxMint/repository/scripts/mintfetchRP.sh", dest: "/usr/local/bin/mintfetchRP.sh", mode: "0755" } + tags: + - linux-mint + +- name: Start linux mint timer services + ansible.builtin.systemd_service: + name: "{{ item }}" + enabled: true + state: started + daemon_reload: true + loop: + - mintUpdateSO.timer + - mintUpdateRP.timer + tags: + - linux-mint +... \ No newline at end of file diff --git a/mirrors/tasks/main.yaml b/mirrors/tasks/main.yaml index 0488a27..fafecdd 100644 --- a/mirrors/tasks/main.yaml +++ b/mirrors/tasks/main.yaml @@ -1,64 +1,52 @@ --- - - - name: Copy linux mint timer service for ISO - ansible.builtin.copy: - src: files/SO/confd/mintUpdateSO.timer - dest: /etc/systemd/system/mintUpdateSO.timer - owner: root - group: root - mode: '0644' +# Main task file for mirrors role +# This file includes all mirror configuration tasks - - name: Copy linux mint service for ISO - ansible.builtin.copy: - src: files/SO/confd/mintUpdateSO.service - dest: /etc/systemd/system/mintUpdateSO.service - owner: root - group: root - mode: '0644' - - - name: Copy linux mint action files for ISO - ansible.builtin.copy: - src: files/SO/confd/scripts/mintfetchSO.sh - dest: /usr/local/bin/mintfetchSO.sh - owner: root - group: root - mode: '0755' +- name: Create common directory for all mirrors + ansible.builtin.file: + path: /srv + state: directory + owner: root + group: root + mode: "0755" + tags: + - mirrors - - name: Copy linux mint timer service for RP - ansible.builtin.copy: - src: files/repository/confd/mintUpdateRP.timer - dest: /etc/systemd/system/mintUpdateRP.timer - owner: root - group: root - mode: '0644' +- name: Ensure user mirrors exists + ansible.builtin.user: + name: mirrors + state: present + system: false + tags: + - mirrors - - name: Copy linux mint service for RP - ansible.builtin.copy: - src: files/repository/confd/mintUpdateRP.service - dest: /etc/systemd/system/mintUpdateRP.service - owner: root - group: root - mode: '0644' - - - name: Copy linux mint action files for RP - ansible.builtin.copy: - src: files/repository/confd/scripts/mintfetchRP.sh - dest: /usr/local/bin/mintfetchRP.sh - owner: root - group: root - mode: '0755' +- name: Include AlmaLinux mirror configuration + ansible.builtin.include_tasks: almalinux.yaml + tags: + - almalinux - - name: Start timer service for ISO - ansible.builtin.systemd_service: - name: mintUpdateSO.timer - enabled: true - state: started - daemon_reload: true +- name: Include ArchLinux mirror configuration + ansible.builtin.include_tasks: archlinux.yaml + tags: + - archlinux - - name: Start timer service for RP - ansible.builtin.systemd_service: - name: mintUpdateRP.timer - enabled: true - state: started - daemon_reload: true +- name: Include Debian mirror configuration + ansible.builtin.include_tasks: debian.yaml + tags: + - debian + +- name: Include Linux Mint mirror configuration + ansible.builtin.include_tasks: linux-mint.yaml + tags: + - linux-mint + +- name: Include nginx configuration for mirrors + ansible.builtin.include_tasks: nginx.yaml + tags: + - mirrors + +- name: Include rsync configuration for mirrors + ansible.builtin.include_tasks: rsync-daemon.yaml + tags: + - mirrors ... \ No newline at end of file diff --git a/mirrors/tasks/nginx.yaml b/mirrors/tasks/nginx.yaml new file mode 100644 index 0000000..b326a31 --- /dev/null +++ b/mirrors/tasks/nginx.yaml @@ -0,0 +1,23 @@ +--- + +- name: Create nginx configuration directory for mirrors + ansible.builtin.file: + path: /etc/nginx/conf.d + state: directory + owner: root + group: root + mode: "0755" + tags: + - mirrors + +- name: Deploy mirror configuration on nginx + ansible.builtin.template: + src: "mirror.site.j2" + dest: "/etc/nginx/conf.d/mirror.site" + owner: root + group: root + mode: "0644" + notify: reload nginx + tags: + - mirrors +... diff --git a/mirrors/tasks/rsync-daemon.yaml b/mirrors/tasks/rsync-daemon.yaml new file mode 100644 index 0000000..822aafc --- /dev/null +++ b/mirrors/tasks/rsync-daemon.yaml @@ -0,0 +1,21 @@ +--- + +- name: Install rsync + ansible.builtin.apt: + name: rsync + state: present + tags: + - mirrors + +- name: Load configuration of rsync in daemon-mode + ansible.builtin.template: + src: "rsyncd.conf.j2" + dest: "/etc/rsyncd.conf" + owner: root + group: root + mode: "0644" + notify: reload rsync + tags: + - mirrors + +... \ No newline at end of file diff --git a/mirrors/templates/mirror.site.j2 b/mirrors/templates/mirror.site.j2 new file mode 100644 index 0000000..76de35a --- /dev/null +++ b/mirrors/templates/mirror.site.j2 @@ -0,0 +1,22 @@ +{# This template is an nginx configuration file for + delivering the mirror#} + + {#Loop section for delivering all mirror content #} + + {% for mirror in mirrors %} + location /{{mirror.name}}/ { + alias {{mirror.path}}; + + # Directory listing + autoindex on; + autoindex_exact_size off; + autoindex_localtime on; + + # Where Nginx will log almalinux requests + access_log {{mirror.access_log}}; + error_log {{mirror.error_log}}; + + # Browsers will cache the content for as long as possible + expires max; + } + {% endfor %} diff --git a/mirrors/templates/rsyncd.conf.j2 b/mirrors/templates/rsyncd.conf.j2 new file mode 100644 index 0000000..da6da51 --- /dev/null +++ b/mirrors/templates/rsyncd.conf.j2 @@ -0,0 +1,26 @@ +# /etc/rsyncd: configuration file for rsync daemon mode +# See rsyncd.conf man page for more options. + +{# Template for setting up the configuration file of rsync in daemon mode + so that every extern mirror can rsync our mirror#} + +uid = nobody +gid = nogroup +read only = yes +list = yes + +use chroot = yes +max connections = 10 +timeout = 900 +log file = /var/log/rsyncd.log + +{#Loop for configurate every mirror#} + +{% for mirror in mirrors %} + +[{{mirror.name}}] + path = {{mirror.path}} + comment = {{mirror.comment}} + +{% endfor%} + diff --git a/mirrors/vars/main.yml b/mirrors/vars/main.yml new file mode 100644 index 0000000..2013924 --- /dev/null +++ b/mirrors/vars/main.yml @@ -0,0 +1,34 @@ +--- +# Variables for mirrors role +# Define mirror configurations + +mirrors: + - name: almalinux + path: /srv/almalinux/ + access_log: /var/log/nginx/almalinux.access.log + error_log: /var/log/nginx/almalinux.error.log + comment: AlmaLinux Mirror + + - name: archlinux + path: /srv/archlinux/ + access_log: /var/log/nginx/archlinux.access.log + error_log: /var/log/nginx/archlinux.error.log + comment: ArchLinux Mirror + + - name: debian + path: /srv/debian/ + access_log: /var/log/nginx/debian.access.log + error_log: /var/log/nginx/debian.error.log + comment: Debian Mirror + + - name: debian-cd + path: /srv/debian-cd/ + access_log: /var/log/nginx/debian-cd.access.log + error_log: /var/log/nginx/debian-cd.error.log + comment: Debian-cd Mirror + + - name: linux-mint + path: /srv/linux-mint/ + access_log: /var/log/nginx/linux-mint.access.log + error_log: /var/log/nginx/linux-mint.error.log + comment: Linux Mint Mirror diff --git a/nginx/files/conf.d/lidsol.fi-b.unam.mx.conf b/nginx/files/conf.d/lidsol.fi-b.unam.mx.conf new file mode 100644 index 0000000..dec7d41 --- /dev/null +++ b/nginx/files/conf.d/lidsol.fi-b.unam.mx.conf @@ -0,0 +1,56 @@ +map $http_upgrade $connection_upgrade { + default upgrade; + '' close; +} + +upstream grafana { + server localhost:3000; +} + +server { + server_name lidsol.fi-b.unam.mx; + listen 443 ssl; + + # Certbot + ssl_certificate /etc/letsencrypt/live/lidsol.fi-b.unam.mx/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/lidsol.fi-b.unam.mx/privkey.pem; + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; + + location / { + root /var/www/html/; + try_files $uri $uri/ =404; + } + + location /grafana/ { + rewrite ^/grafana/(.*) /$1 break; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + add_header Content-Security-Policy "default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; font-src 'self'; frame-src 'self';"; + proxy_pass http://grafana/; + } + + # Proxy Grafana Live WebSocket connections. + location /grafana/api/live/ { + rewrite ^/grafana/(.*) /$1 break; + proxy_http_version 1.1; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Host $host; + add_header Content-Security-Policy "default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; font-src 'self'; frame-src 'self';"; + proxy_pass http://grafana/; + } + + include /etc/nginx/conf.d/mirror.site; +} + +server { + server_name lidsol.fi-b.unam.mx; + listen 80; + + include /etc/nginx/conf.d/mirror.site; +} diff --git a/nginx/files/nginx.conf b/nginx/files/nginx.conf new file mode 100644 index 0000000..524c2e1 --- /dev/null +++ b/nginx/files/nginx.conf @@ -0,0 +1,72 @@ +user www-data; +worker_processes auto; +pid /run/nginx.pid; +error_log /var/log/nginx/error.log; + +events { + worker_connections 2048; + multi_accept on; +} + +http { + ## + # Basic Settings + ## + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + server_tokens off; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + ## + # SSL Settings + ## + + ssl_session_cache shared:le_nginx_SSL:10m; + ssl_session_timeout 1d; + ssl_session_tickets off; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_prefer_server_ciphers on; + ssl_ciphers "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384"; + + client_header_buffer_size 1k; + large_client_header_buffers 4 8k; + + add_header Strict-Transport-Security "max-age=15768000; includeSubDomains" always; + add_header X-Frame-Options SAMEORIGIN; + add_header X-Content-Type-Options nosniff; + add_header X-XSS-Protection "1; mode=block"; + add_header Content-Security-Policy "default-src 'self';"; + + ## + # Logging Settings + ## + + access_log /var/log/nginx/access.log; + + ## + # Gzip Settings + ## + + gzip on; + + gzip_vary on; + gzip_comp_level 6; + gzip_min_length 256; + gzip_buffers 16 8k; + gzip_http_version 1.1; + gzip_proxied any; + gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript application/atom+xml image/svg+xml; + + + ## + # Virtual Host Configs + ## + + include /etc/nginx/conf.d/*.conf; +} \ No newline at end of file diff --git a/nginx/tasks/main.yaml b/nginx/tasks/main.yaml new file mode 100644 index 0000000..03cd148 --- /dev/null +++ b/nginx/tasks/main.yaml @@ -0,0 +1,43 @@ +# Setup nginx for http(s) services +--- +- name: Install nginx, certbot and nginx plugin + ansible.builtin.apt: + name: "{{ item }}" + state: present + loop: + - nginx + - certbot + - python3-certbot-nginx + tags: + - certificate + +- name: Ensure nginx service is enabled and running + ansible.builtin.service: + name: nginx + enabled: true + state: started + +- name: Get the certificate for the domain + ansible.builtin.command: + cmd: certbot --nginx -d lidsol.fi-b.unam.mx --non-interactive --agree-tos --email lidsol-info@proton.me certonly + creates: /etc/letsencrypt/live/lidsol.fi-b.unam.mx/fullchain.pem + tags: + - certificate + +- name: Copy nginx configuration file + ansible.builtin.copy: + src: files/nginx.conf + dest: /etc/nginx/nginx.conf + mode: '0644' + + + +- name: Copy general lidsol.fi-b.unam.mx configuration file + ansible.builtin.copy: + src: files/conf.d/lidsol.fi-b.unam.mx.conf + dest: /etc/nginx/conf.d/lidsol.fi-b.unam.mx.conf + owner: root + group: root + mode: '0644' + +... \ No newline at end of file