fixes

README.md

@@ -41,17 +41,22 @@ Scripts
 
 `k-repo-sync-add-sftp.sh` – add SFTP for data replication (synchronization) from main repository and save separate config files for futher connections.
 
-`k-repo-sync.sh` – sync repositories data, from _box_main_ to _boxes_to_sync_.
+`k-repo-sync.sh` – sync repositories data, from _repo_main_ to _repo_sync_.
 
 `k-server-start.sh` – draft version for run Kopia web dashboard UI 
 
 Quick start
 ---
 
 ### Steps for first run:
-1. rub `k-prepare.sh` for instalation and initialization (it needs to run again if new remote storage's added) 
-2. create repository (main, master) with `k-repo-create-sftp.sh`
-3. add folders or files with `kopia snapshot create [source]`
+1. Edit `config` file (or use provided example):
+- `repo_main` (required), – main SFTP repository storage (syntax: user@host)
+- `repo_sync` (optional) – additional storages for main repository replication/sync (one or space separated list) 
+- `repository_folder` (reqired) – path where repository folder placed on storage
+- `KOPIA_PASSWORD` (reqired) – password for repositories encryption
+2. Run `k-prepare.sh` for instalation and initialization (it needs to run again if new remote storage's added). First time password for remote user@storage_host can be asked
+3. Create repository (main, master) with `k-repo-create-sftp.sh`
+4. Add folders or files with `kopia snapshot create [source (path to file or folder)]`
 
 ### Next:
 - add new SFTP storage's for main repository replication (sync) with `k-repo-sync-add-sftp.sh`

config

@@ -1,16 +1,16 @@
-# Hetzner StorageBox'es (one or space separated list), syntax: user@hostname
-export boxes="[email protected] [email protected]"
-export box_main="[email protected]"
-export boxes_to_sync="[email protected]"
+# Hetzner StorageBox'es (one or space separated list), syntax: user@host:port
+export repo_main="[email protected]:23"
+export repo_sync="[email protected]:23"
 
 # Folder path where repository stored on remote storage
-export repository_remote_folder="/home/kopia/json"
+export repository_folder="/home/kopia/json"
 
-# Default password what mandatory for repositories encruption
-export KOPIA_PASSWORD="1"
+# Default password what mandatory for repositories encryption
+export KOPIA_PASSWORD="pass"
 
-#export ui-username="k"
-#export ui-password="k"
+# Web UI
+export ui_username="k"
+export ui_password="k"
 
 # Other Kopia settings
 export KOPIA_CHECK_FOR_UPDATES=false

k-prepare.sh

@@ -9,18 +9,16 @@
 # Require: running from "root" or "sudo"
 # Note: operation will be skipped if already done before
 
-script_dir="$( cd "$( dirname "$0" )" && pwd )"
-source $script_dir/config
-
 set -e
 set -uo pipefail
 # set -x
 
-source config
+script_dir="$( cd "$( dirname "$0" )" && pwd )"
+source $script_dir/config
 # `config` overwrites: 
-# Hetzner StorageBox'es (one or space separated list), syntax: user@hostname
+# Hetzner StorageBox'es (one or space separated list), syntax: user@host
 # Uncomment to overwrite StorageBox'es list from 'config' file
-# boxes="[email protected] [email protected]"
+# repositories="[email protected] [email protected]"
 
 # initialization
 cd $script_dir || { echo "Error: keys directory inaccessible" && exit 1; }
@@ -34,25 +32,29 @@ knownhosts="$script_dir/keys/known_hosts"
 if [[ ! -f $(which kopia) ]]; then
   curl -s https://kopia.io/signing-key | sudo gpg --dearmor -o /usr/share/keyrings/kopia-keyring.gpg
   echo "deb [signed-by=/usr/share/keyrings/kopia-keyring.gpg] http://packages.kopia.io/apt/ stable main" | sudo tee /etc/apt/sources.list.d/kopia.list
-  apt update && apt install kopia #kopia-ui
+  apt update && apt install kopia lftp -y #kopia-ui
   { [[ -f $(which kopia) ]] && echo "Kopia installed to $(which kopia)"; } || { echo "Error: Kopia no installed, check errors" && exit 1; }
 fi
 
 # generate SFTP (SSH) key for access to StorageBox'es
 [[ -f "$script_dir"/keys/id_kopia ]] || \
-ssh-keygen -o -a 100 -t ed25519 -f keys/id_kopia -C "demo key (password-less) for access to Hetzner StorageBox for Kopia" -q -N ""
+ssh-keygen -o -a 100 -t ed25519 -f keys/id_kopia -C "demo key (password-less) for access to SFTP storage for Kopia" -q -N ""
 key=$(cat keys/id_kopia.pub)
 
 # import newly created SSH key to StorageBox'es, add hosts to known_hosts
-for box in $boxes; do 
-  authorized_keys="$script_dir"/keys/"$box"_authorized_keys
-  box_hostname=$(echo $box | sed 's/.*@//')
-  [[ -n $(grep "$box" "$script_dir/keys/known_hosts") ]] || ssh-keyscan -p 23 $box_hostname >> "$knownhosts" 2> /dev/null
-  scp -q -P 23 "$box":/home/.ssh/authorized_keys "$authorized_keys"
-  [[ -n $(grep "$key" keys/"$box"_authorized_keys) ]] && { echo "$box : key already imported" && continue; }
+repositories="${repo_main} ${repo_sync}"
+
+for repo in $repositories; do 
+  username=${repo%%@*} 
+  host=$(echo $repo | sed 's/.*@//' | sed 's/:/\t/g' | awk '{print $1}')
+  port=$(echo $repo | sed 's/.*://')
+  authorized_keys="$script_dir"/keys/"$username@$host"_authorized_keys
+  [[ -n $(grep "$host" "$script_dir/keys/known_hosts") ]] || ssh-keyscan -p $port $host >> "$knownhosts" 2> /dev/null
+  scp -q -P $port "$username@$host":/home/.ssh/authorized_keys "$authorized_keys"
+  [[ -n $(grep "$key" keys/"$username@$host"_authorized_keys) ]] && { echo "$repo : key already imported" && continue; }
   echo $(cat keys/id_kopia.pub) | tee -a "$authorized_keys"
-  echo -e "echo mkdir .ssh \n chmod 700 .ssh \n put "$authorized_keys" .ssh/authorized_keys \n chmod 600 .ssh/authorized_keys" | sftp -q -P 23 "$box" > /dev/null 2>&1
-  echo "$box : new key imported" 
+  echo -e "echo mkdir .ssh \n chmod 700 .ssh \n put "$authorized_keys" .ssh/authorized_keys \n chmod 600 .ssh/authorized_keys" | sftp -q -P $port "$username@$host" > /dev/null 2>&1
+  echo "$repo : key imported" 
 done
 
 # add scripts path to evironment 

k-repo-connect-sftp.sh

@@ -6,25 +6,29 @@ set -euo pipefail
 script_dir="$( cd "$( dirname "$0" )" && pwd )"
 source $script_dir/config
 # `config` overwrites: 
-# box_main="u281891@u281891.your-storagebox.de"
-# repository_remote_folder="/home/kopia/json"
+# repo_main="u281892@u281892.your-storagebox.de:23"
+# repository_folder="/home/kopia/json"
 
-box="$box_main"
+repo="$repo_main"
 keyfile="$script_dir/keys/id_kopia"
 knownhosts="$script_dir/keys/known_hosts"
 
-[[ -n $(grep "$box" "$script_dir/keys/known_hosts") ]] || ssh-keyscan -p 23 $(echo $box | sed 's/.*@//') >> "$knownhosts"  2> /dev/null
+username=${repo%%@*} 
+host=$(echo $repo | sed 's/.*@//' | sed 's/:/\t/g' | awk '{print $1}')
+port=$(echo $repo | sed 's/.*://')
+
+[[ -n $(grep "$host" "$script_dir/keys/known_hosts") ]] || ssh-keyscan -p $port $host >> "$knownhosts"  2> /dev/null
 
 kopia repository connect sftp \
-  --config-file "$script_dir/repositories/repo-$box.config" \
+  --config-file "$script_dir/repositories/repo-$username@$host.config" \
   --cache-directory	"$script_dir/cache/" \
-  --host $(echo $box | sed 's/.*@//') \
-  --username ${box%%@*} \
+  --username $username \
+  --host $host \
+  --port $port \
   --keyfile $keyfile \
   --known-hosts $knownhosts \
-  --port 23 \
-  --path $repository_remote_folder
+  --path $repository_folder
 
 # kopia repository validate-provider
 
-echo "Repository: $box connected and active"
+echo "Repository: $repo connected and active"

k-repo-create-sftp.sh

@@ -9,24 +9,27 @@ set -uo pipefail
 script_dir="$( cd "$( dirname "$0" )" && pwd )"
 source $script_dir/config
 # `config` overwrites: 
-# box_main="u281891@u281891.your-storagebox.de"
-# repository_remote_folder="/home/kopia/json"
+# repo_main="u281892@u281892.your-storagebox.de:23"
+# repository_folder="/home/kopia/json"
 
-box="$box_main"
+repo="$repo_main"
 keyfile="$script_dir/keys/id_kopia"
 knownhosts="$script_dir/keys/known_hosts"
 
+username=${repo%%@*} 
+host=$(echo $repo | sed 's/.*@//' | sed 's/:/\t/g' | awk '{print $1}')
+port=$(echo $repo | sed 's/.*://')
 
-  [[ -n $(grep "$box" "$script_dir/keys/known_hosts") ]] || ssh-keyscan -p 23 $(echo $box | sed 's/.*@//') >> "$knownhosts" 2> /dev/null
+[[ -n $(grep "$host" "$script_dir/keys/known_hosts") ]] || ssh-keyscan -p $port $host >> "$knownhosts"  2> /dev/null
 
-  kopia repository create sftp \
-    --config-file "$script_dir/repositories/repo-$box.config" \
-    --cache-directory	"$script_dir/cache/" \
-    --host $(echo $box | sed 's/.*@//') \
-    --username ${box%%@*} \
-    --keyfile $keyfile \
-    --known-hosts $knownhosts \
-    --port 23 \
-    --path $repository_remote_folder
+kopia repository create sftp \
+  --config-file "$script_dir/repositories/repo-$username@$host.config" \
+  --cache-directory	"$script_dir/cache/" \
+  --username $username \
+  --host $host \
+  --port $port \
+  --keyfile $keyfile \
+  --known-hosts $knownhosts \
+  --path $repository_folder
 
-  # kopia repository validate-provider
+# kopia repository validate-provider

k-repo-sync-add-sftp.sh

@@ -9,23 +9,48 @@ set -uo pipefail
 script_dir="$( cd "$( dirname "$0" )" && pwd )"
 source $script_dir/config
 # `config` overwrites: 
-# boxes_to_sync="[email protected] [email protected]"
-# repository_remote_folder="/home/kopia/json"
+# repo_sync="[email protected]:23 [email protected]:23"
+# repository_folder="/home/kopia/json"
 
 keyfile="$script_dir/keys/id_kopia"
 knownhosts="$script_dir/keys/known_hosts"
 
-for box in $boxes_to_sync; do 
-  [[ -n $(grep "$box" "$script_dir/keys/known_hosts") ]] || ssh-keyscan -p 23 $(echo $box | sed 's/.*@//') >> "$knownhosts" 2> /dev/null
+
+repo_main=$(echo $repo_main | sed 's/@/\t/g' | sed 's/:/\t/g' | awk '{print $1"@"$2}')
+kopia repository connect from-config --file "$script_dir/repositories/repo-$repo_main.config"
+
+for repo in $repo_sync; do 
+
+  username=${repo%%@*} 
+  host=$(echo $repo | sed 's/.*@//' | sed 's/:/\t/g' | awk '{print $1}')
+  port=$(echo $repo | sed 's/.*://')
+
+  [[ -n $(grep "$host" "$script_dir/keys/known_hosts") ]] || ssh-keyscan -p $port $host >> "$knownhosts"  2> /dev/null
+
+  # temporary workaround Kopia bug of `kopia repository sync-to sftp`
+  # kopia repository create sftp \
+  #   --config-file "$script_dir/repositories/repo-$username@$host.config" \
+  #   --cache-directory	"$script_dir/cache/" \
+  #   --username $username \
+  #   --host $host \
+  #   --port $port \
+  #   --keyfile $keyfile \
+  #   --known-hosts $knownhosts \
+  #   --path $repository_folder
+  # #echo -e "rm -r $repository_folder" | 
+  # eval `ssh-agent -s`
+  # ssh-add $keyfile
+  # lftp -e "rm -r $repository_folder" -p $port "$username@$host" #> /dev/null 2>&1
 
   kopia repository sync-to sftp \
-    --config-file "$script_dir/repositories/repo-$box.config" \
-    --host $(echo $box | sed 's/.*@//') \
-    --username ${box%%@*} \
+    --username $username \
+    --host $host \
+    --port $port \
     --keyfile $keyfile \
     --known-hosts $knownhosts \
-    --port 23 \
-    --path $repository_remote_folder
+    --path $repository_folder \
+    --delete
+    # --config-file "$script_dir/repositories/repo-$username@$host.config"
 
   # kopia repository validate-provider
 done

k-repo-sync.sh

@@ -2,22 +2,27 @@
 
 set -e
 set -uo pipefail
-set -x
+# set -x
 
 script_dir="$( cd "$( dirname "$0" )" && pwd )"
 source $script_dir/config
 # `config` overwrites: 
-# boxes_to_sync="[email protected] [email protected]"
-# box_main="[email protected]"
+# repo_sync="[email protected]:23 [email protected]:23"
+# repo_main="[email protected]:23"
 
 keyfile="$script_dir/keys/id_kopia"
 knownhosts="$script_dir/keys/known_hosts"
 
-kopia repository connect from-config --file "$script_dir/repositories/repo-$box_main.config"
+kopia repository connect from-config --file "$script_dir/repositories/repo-$repo_main.config"
 
-for box in $boxes_to_sync; do 
-  [[ -n $(grep "$box" "$script_dir/keys/known_hosts") ]] || ssh-keyscan -p 23 $(echo $box | sed 's/.*@//') >> "$knownhosts" > /dev/null
+for repo in $repo_sync; do 
+
+  host=$(echo $repo | sed 's/.*@//' | sed 's/:/\t/g' | awk '{print $1}')
+  port=$(echo $repo | sed 's/.*://')
+
+  [[ -n $(grep "$host" "$script_dir/keys/known_hosts") ]] || ssh-keyscan -p $port $host >> "$knownhosts"  2> /dev/null
+
+  kopia repository sync-to from-config --delete --file "$script_dir/repositories/repo-$username@$host.config"
 
-  kopia repository sync-to from-config --delete --file "$script_dir/repositories/repo-$box.config"
 done
 

k-server-start.sh

@@ -10,4 +10,4 @@ source $script_dir/config
 
 ip=$(curl -s -4 ifconfig.co)
 
-kopia server start --insecure --server-password $ui-password --server-username	$ui-username --ui --address http://$ip:1515
+kopia server start --insecure --server-password $ui_password --server-username	$ui_username --ui --address http://$ip:1515

keys/id_kopia

@@ -1,8 +1,8 @@
 -----BEGIN OPENSSH PRIVATE KEY-----
 b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
-QyNTUxOQAAACC6ojXLtcCyldcFGVajNyyI1BwQsxIiisbq+H5k6o6ykgAAAMgi9SVNIvUl
-TQAAAAtzc2gtZWQyNTUxOQAAACC6ojXLtcCyldcFGVajNyyI1BwQsxIiisbq+H5k6o6ykg
-AAAEB/9J1hoyOyEyinV3nFKhAq/GvBYTrsHd8kcBRIxk2eObqiNcu1wLKV1wUZVqM3LIjU
-HBCzEiKKxur4fmTqjrKSAAAAQ2RlbW8ga2V5IChwYXNzd29yZC1sZXNzKSBmb3IgYWNjZX
+QyNTUxOQAAACAjzKGHVmYfAAQE0oti6i28WEbce8PLRptqVx7FcQEKNwAAAMhasaJAWrGi
+QAAAAAtzc2gtZWQyNTUxOQAAACAjzKGHVmYfAAQE0oti6i28WEbce8PLRptqVx7FcQEKNw
+AAAEAarmNfO8I+vGxENvS5MP9Vi97Q9w4imOS41y1d094v0CPMoYdWZh8ABATSi2LqLbxY
+Rtx7w8tGm2pXHsVxAQo3AAAAQ2RlbW8ga2V5IChwYXNzd29yZC1sZXNzKSBmb3IgYWNjZX
 NzIHRvIEhldHpuZXIgU3RvcmFnZUJveCBmb3IgS29waWEBAg==
 -----END OPENSSH PRIVATE KEY-----

keys/id_kopia.pub

@@ -1 +1 @@
-ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILqiNcu1wLKV1wUZVqM3LIjUHBCzEiKKxur4fmTqjrKS demo key (password-less) for access to Hetzner StorageBox for Kopia
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICPMoYdWZh8ABATSi2LqLbxYRtx7w8tGm2pXHsVxAQo3 demo key (password-less) for access to Hetzner StorageBox for Kopia