From 235374c2223f59ac2a2c81eae5323c3b5babd99b Mon Sep 17 00:00:00 2001 From: Simon Cornet Date: Fri, 17 Jul 2020 22:55:07 +0200 Subject: [PATCH 1/4] [GLOBAL] Initial commit --- FreeBSD/jail/jail-backup.csh | 41 ++++++++++++++++ FreeBSD/jail/jail-update.csh | 32 ++++++++++++ FreeBSD/other/url-check.csh | 24 +++++++++ FreeBSD/storage/partition-disk-zfs.md | 22 +++++++++ FreeBSD/vm/vcreate.csh | 27 ++++++++++ FreeBSD/vm/vm-backup-single.bash | 67 +++++++++++++++++++++++++ FreeBSD/vm/vm-backup-single.csh | 71 +++++++++++++++++++++++++++ FreeBSD/vm/vm-backup.csh | 58 ++++++++++++++++++++++ Linux/url-check.sh | 25 ++++++++++ README.md | 3 ++ 10 files changed, 370 insertions(+) create mode 100644 FreeBSD/jail/jail-backup.csh create mode 100644 FreeBSD/jail/jail-update.csh create mode 100644 FreeBSD/other/url-check.csh create mode 100644 FreeBSD/storage/partition-disk-zfs.md create mode 100644 FreeBSD/vm/vcreate.csh create mode 100644 FreeBSD/vm/vm-backup-single.bash create mode 100644 FreeBSD/vm/vm-backup-single.csh create mode 100644 FreeBSD/vm/vm-backup.csh create mode 100644 Linux/url-check.sh create mode 100644 README.md diff --git a/FreeBSD/jail/jail-backup.csh b/FreeBSD/jail/jail-backup.csh new file mode 100644 index 0000000..4a1859e --- /dev/null +++ b/FreeBSD/jail/jail-backup.csh @@ -0,0 +1,41 @@ +#!/bin/csh + +## config section +set ezjail_config_dir = "/usr/local/etc/ezjail" +set backup_config_dir = "/tank/backup/jails" +set backup_date = `date "+%Y%m%d%H%M.%S"` + +# backup all jails +echo "" +echo "################ BACKUP JAILS #################" +foreach jail (`ls /usr/jails | \ + grep -v basejail | \ + grep -v ezjail_archives | \ + grep -v flavours | \ + grep -v newjail | \ + grep -v fulljail`) + + ## stopping jail + ezjail-admin stop $jail + + ## start backing up jail + jailconfig + echo "Backup jails: $jail" + + ## backup jail config + cp $ezjail_config_dir/$jail $backup_config_dir + mv $backup_config_dir/$jail "$backup_config_dir/$jail-$backup_date.conf" + + ## backup jail content + ezjail-admin archive $jail + echo "" + + ## start jail + ezjail-admin start $jail + echo "" + +end + +# report done +echo "" +echo "############### ALL DONE THANKS ###############" +echo "" diff --git a/FreeBSD/jail/jail-update.csh b/FreeBSD/jail/jail-update.csh new file mode 100644 index 0000000..7a978ed --- /dev/null +++ b/FreeBSD/jail/jail-update.csh @@ -0,0 +1,32 @@ +#!/bin/csh + +# Upgrade host packages +echo "" +echo "############ UPGRADE HOST PACKAGES ############" +echo "" +pkg update && \ +pkg upgrade -y && \ +pkg clean -ya + +# Upgrade jail packages +echo "" +echo "############ UPGRADE JAIL PACKAGES ############" +foreach jail (`ls /usr/jails | \ + grep -v basejail | \ + grep -v ezjail_archives | \ + grep -v flavours | \ + grep -v newjail | \ + grep -v fulljail`) + + echo "" + echo "Updating $jail" + pkg -j $jail update && \ + pkg -j $jail upgrade -y && \ + pkg -j $jail clean -ay + +end + +# Report done +echo "" +echo "############### ALL DONE THANKS ###############" +echo "" diff --git a/FreeBSD/other/url-check.csh b/FreeBSD/other/url-check.csh new file mode 100644 index 0000000..a9703ce --- /dev/null +++ b/FreeBSD/other/url-check.csh @@ -0,0 +1,24 @@ +#!/bin/csh -f +set MODE=$1 +set CURL=/usr/local/bin/curl + +if ($MODE == no_auth) then + set URL=$2 + set HS=$3 + set STATUS=`$CURL -L -o /dev/null -s -w "%{http_code}\n" -m 1 $URL` + + else if ($MODE == auth) then + set AUTH=$2 + set URL=$3 + set HS=$4 + set STATUS=`$CURL -L -o /dev/null -s -w "%{http_code}\n" -m 1 --user $AUTH $URL` + + else + exit 1 +endif + +if ($STATUS == 200) then + $CURL -fsS --retry 3 $HS > /dev/null + else + exit 1 +endif \ No newline at end of file diff --git a/FreeBSD/storage/partition-disk-zfs.md b/FreeBSD/storage/partition-disk-zfs.md new file mode 100644 index 0000000..4881865 --- /dev/null +++ b/FreeBSD/storage/partition-disk-zfs.md @@ -0,0 +1,22 @@ +# How to replace a ZFS drive + +``` +# Put drive offline +sudo zpool tank da3 offline +``` + +``` +sudo gpart destroy -F da7 +``` + +``` +# Partition drive +sudo gpart create -s gpt da7 +sudo gpart add -s 100M -t freebsd-swap da7 +sudo gpart add -t freebsd-zfs da7 +``` + +``` +# Replace drive in pool +sudo zpool replace -f tank 16950429292085325612 /dev/da7p2 +``` diff --git a/FreeBSD/vm/vcreate.csh b/FreeBSD/vm/vcreate.csh new file mode 100644 index 0000000..8ddd98f --- /dev/null +++ b/FreeBSD/vm/vcreate.csh @@ -0,0 +1,27 @@ +#!/bin/csh -f +# set arguments +set HOSTNAME=$1 +set IP=$2 + +if ( $3 == "" ) then + set SIZE=20G +else + set SIZE=$3 +endif + +# set static variables +set VM=/usr/local/sbin/vm +set TEMPLATE=ubuntu-small +set IMAGE=ubuntu20.img +set PUBKEY=/home/simon/.ssh/id_ed25519.pub +set NAMESERVER=192.168.10.254 +set GATEWAY=192.168.10.254 + +# create vm +$VM create \ + -t $TEMPLATE \ + -s $SIZE \ + -i $IMAGE \ + -C -k $PUBKEY \ + -n "ip=$IP/24;gateway=$GATEWAY;nameservers=$NAMESERVER" \ + $HOSTNAME diff --git a/FreeBSD/vm/vm-backup-single.bash b/FreeBSD/vm/vm-backup-single.bash new file mode 100644 index 0000000..76f242e --- /dev/null +++ b/FreeBSD/vm/vm-backup-single.bash @@ -0,0 +1,67 @@ +#!/usr/local/bin/bash + +## config section +vm=$1 +vm_dataset="zroot/vms" +vm_pid=`/usr/bin/pgrep -f "bhyve: $vm"` +backup_location="/tank/backup/vms/" +backup_date=`date "+%Y-%m-%d"` +enc_password="nDNmPPBPk7jQnwke" + +# snapshot vm +echo "" +for vm in $vm; do + + ## prepare + echo "################## PREPARE ####################" + if [ $vm_pid != 0 ] + then + (vm stop $vm > /dev/null) >& /dev/null + + ### check when vm is shutdown properly + echo "Waiting for $vm to shutdown properly" + + while [[ ${?} == 0 ]] + do + ps -p $vm_pid > /dev/null + done + fi + + echo "" + echo "################## SNAPSHOT ###################" + echo "Making a snapshot of $vm" + sleep 1 + + ## create snapshot + zfs snapshot $vm_dataset/$vm@$backup_date + + ## start VM + echo "Starting $vm" + (vm start $vm > /dev/null) >& /dev/null + +done + +# backup vm snapshot +echo "" +echo "################## BACKUP VM ##################" +for vm in $vm; do + + echo "Send snapshot to local backup destination" + sleep 1 + + ## send snapshot to backup destination + zfs send $vm_dataset/$vm@$backup_date | \ + pigz -c -p 4 | \ + openssl enc -aes-256-ctr -a -pbkdf2 -iter 1000 -salt -k $enc_password \ + > $backup_location/$vm-$backup_date.pigz.enc + + ## delete snapshot + sleep 1 + zfs destroy $vm_dataset/$vm@$backup_date + +done + +# report done +echo "" +echo "############### ALL DONE THANKS ###############" +echo "" diff --git a/FreeBSD/vm/vm-backup-single.csh b/FreeBSD/vm/vm-backup-single.csh new file mode 100644 index 0000000..ccecec9 --- /dev/null +++ b/FreeBSD/vm/vm-backup-single.csh @@ -0,0 +1,71 @@ +#!/bin/csh +## config section +set vm = $1 +set vm_dataset = "zroot/vms" +set vm_pid = `/usr/bin/pgrep -f "bhyve: $vm"` +set backup_location = "/tank/backup/vms/" +set backup_date = `date "+%Y-%m-%d"` + +# snapshot vm +echo "" +foreach vm ($vm) + + ## prepare + echo "################## PREPARE ####################" + + vm list | grep $vm ## DELETE ME + + if ( $vm_pid != 0 ) then + (vm stop $vm > /dev/null) >& /dev/null + + ### check when vm is shutdown properly + echo "Waiting for $vm to shutdown properly" + + while ({ /usr/bin/pgrep $vm_pid }) + echo "Waiting for pid $vm_pid" + sleep 2 + end + + vm list | grep $vm ## DELETE ME + + endif + + echo "" + echo "################## SNAPSHOT ###################" + echo "Making a snapshot of $vm" + sleep 1 + + ## create snapshot + zfs snapshot $vm_dataset/$vm@$backup_date + + ## start VM + echo "Starting $vm" + sleep 1 + (vm start $vm > /dev/null) >& /dev/null + +end + +# backup vm snapshot +echo "" +echo "################## BACKUP VM ##################" +foreach vm ($vm) + + echo "Send snapshot to local backup destination" + sleep 1 + + ## send snapshot to backup destination + zfs send $vm_dataset/$vm@$backup_date | \ + gzip | \ + openssl enc -aes-256-cbc -a -pbkdf2 -iter 1000 -salt -k development \ + > $backup_location/$vm-$backup_date.gz.enc + + ## delete snapshot + sleep 1 + zfs destroy $vm_dataset/$vm@$backup_date + +end + +# report done +echo "" +echo "############### ALL DONE THANKS ###############" +echo "" diff --git a/FreeBSD/vm/vm-backup.csh b/FreeBSD/vm/vm-backup.csh new file mode 100644 index 0000000..702eeb3 --- /dev/null +++ b/FreeBSD/vm/vm-backup.csh @@ -0,0 +1,58 @@ +#!/bin/csh + +## config section +set vm_dataset = "zroot/vms" +set backup_dataset = "tank/backup/bhyve" +set backup_date = `date "+%Y-%m-%d"` + +# Snapshot all VMs +echo "" +echo "################# SNAPSHOT VM #################" +foreach vm (`ls /usr/vms | \ + grep -v .config | \ + grep -v .iso | \ + grep -v .templates`) + + ## stopping vm + vm stop $vm + + ## give vm time to shut down properly + sleep 15 + + ## snapshot vm + echo "Snapshot VM: $vm" + zfs snapshot $vm_dataset/$vm@$backup_date + echo "" + + ## start VM + vm start $vm + echo "" + +end + +# Backup all VM snapshots +echo "" +echo "################## BACKUP VM ##################" +foreach vm (`ls /usr/vms | \ + grep -v .config | \ + grep -v .iso | \ + grep -v .templates`) + + ## Create ZFS dataset + echo "Creating destination dataset for $vm" + zfs create $backup_dataset/$vm-$backup_date + + ## Send snapshot to backup destination + echo "Backup VM: $vm" + zfs send $vm_dataset/$vm@$backup_date | zfs recv -F $backup_dataset/$vm-$backup_date + echo "" + + ## Delete snapshot + zfs destroy $vm_dataset/$vm@$backup_date + +end + +# report done +echo "" +echo "############### ALL DONE THANKS ###############" +echo "" \ No newline at end of file diff --git a/Linux/url-check.sh b/Linux/url-check.sh new file mode 100644 index 0000000..b5c0cce --- /dev/null +++ b/Linux/url-check.sh @@ -0,0 +1,25 @@ +#!/usr/bin/csh -f +set MODE=$1 +set CURL=/usr/bin/curl + +if ($MODE == no_auth) then + set URL=$2 + set HS=$3 + set STATUS=`$CURL -L -o /dev/null -s -w "%{http_code}\n" -m 1 $URL` + + else if ($MODE == auth) then + set AUTH=$2 + set URL=$3 + set HS=$4 + set STATUS=`$CURL -L -o /dev/null -s -w "%{http_code}\n" -m 1 --user $AUTH $URL` + + else + exit 1 +endif + +if ($STATUS == 200) then + $CURL -fsS --retry 3 $HS > /dev/null + else + exit 1 +endif + diff --git a/README.md b/README.md new file mode 100644 index 0000000..04b9a3b --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# Scripts + +Home for all my scripts \ No newline at end of file From adafb29bab810ae77b7771eb487b54f5b323576a Mon Sep 17 00:00:00 2001 From: Simon Cornet Date: Wed, 29 Jul 2020 17:37:58 +0200 Subject: [PATCH 2/4] [BACKUP] Added ZFS datastore backup script --- FreeBSD/storage/zfs-dataset-backup.bash | 35 +++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 FreeBSD/storage/zfs-dataset-backup.bash diff --git a/FreeBSD/storage/zfs-dataset-backup.bash b/FreeBSD/storage/zfs-dataset-backup.bash new file mode 100644 index 0000000..ade3a7c --- /dev/null +++ b/FreeBSD/storage/zfs-dataset-backup.bash @@ -0,0 +1,35 @@ +#!/usr/local/bin/bash + +## config section +dataset_orig=$1 +dataset_name=$(echo $dataset_orig | sed -e 's/\//-/g') +backup_location="/tank/backup/datasets" +backup_date=`date "+%Y-%m-%d"` +enc_password="supersecretstuff" + +echo "" +echo "################## SNAPSHOT ###################" +echo "Making a snapshot of $dataset_orig" + +## create snapshot +zfs snapshot $dataset_orig@$backup_date + +# backup vm snapshot +echo "" +echo "################## BACKUP VM ##################" +echo "Send snapshot to local backup destination" + +## send snapshot to backup destination +zfs send $dataset_orig@$backup_date | \ +pigz -c -p 4 | \ +openssl enc -aes-256-ctr -a -pbkdf2 -iter 1000 -salt -k $enc_password \ +> $backup_location/$dataset_name-$backup_date.pigz.enc + +## delete snapshot +#sleep 1 +zfs destroy $dataset_orig@$backup_date + +# report done +echo "" +echo "############### ALL DONE THANKS ###############" +echo "" From 2b8ce06d91d6a883c144234323cbdf565ea459e3 Mon Sep 17 00:00:00 2001 From: Simon Cornet Date: Wed, 29 Jul 2020 17:38:55 +0200 Subject: [PATCH 3/4] [BACKUP] Removed unused sleep line for zfs datastore backup script --- FreeBSD/storage/zfs-dataset-backup.bash | 1 - 1 file changed, 1 deletion(-) diff --git a/FreeBSD/storage/zfs-dataset-backup.bash b/FreeBSD/storage/zfs-dataset-backup.bash index ade3a7c..db7c752 100644 --- a/FreeBSD/storage/zfs-dataset-backup.bash +++ b/FreeBSD/storage/zfs-dataset-backup.bash @@ -26,7 +26,6 @@ openssl enc -aes-256-ctr -a -pbkdf2 -iter 1000 -salt -k $enc_password \ > $backup_location/$dataset_name-$backup_date.pigz.enc ## delete snapshot -#sleep 1 zfs destroy $dataset_orig@$backup_date # report done From 93e0bfb4f86673a87465d127972c447d7c48cfc0 Mon Sep 17 00:00:00 2001 From: Simon Cornet Date: Thu, 13 Aug 2020 15:18:28 +0200 Subject: [PATCH 4/4] [VCREATE] Added prefix variable --- FreeBSD/vm/vcreate.csh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/FreeBSD/vm/vcreate.csh b/FreeBSD/vm/vcreate.csh index 8ddd98f..b95c31c 100644 --- a/FreeBSD/vm/vcreate.csh +++ b/FreeBSD/vm/vcreate.csh @@ -16,6 +16,7 @@ set IMAGE=ubuntu20.img set PUBKEY=/home/simon/.ssh/id_ed25519.pub set NAMESERVER=192.168.10.254 set GATEWAY=192.168.10.254 +set PREFIX=24 # create vm $VM create \ @@ -23,5 +24,5 @@ $VM create \ -s $SIZE \ -i $IMAGE \ -C -k $PUBKEY \ - -n "ip=$IP/24;gateway=$GATEWAY;nameservers=$NAMESERVER" \ + -n "ip=$IP/$PREFIX;gateway=$GATEWAY;nameservers=$NAMESERVER" \ $HOSTNAME