RIGO.INFO TechBlog

Eventually I get into producing something I think is useful for others. You'll find random technical knowledge in this Blog so please be patient and use the search function. Some of these software are rather lame and old, the list is ordered by the approximate age (newest first). Unfortunately my older stuff are lost or scattered around my old CDs, not readable anymore 8(.

Remote backup of an OpenVZ host using duplicity and LVM snapshots

The following script was developed and tested under Debian Lenny.

#!/bin/bash
###################################################################
#
# This shell script creates remote incremental backups
# from all of your OpenVZ virtual environments using
# duplicity and LVM snapshots. This uses far less space than
# the vzdump utility but is not as configurable.
# 
# The script is meant to be run from cron.daily. It sends
# its output to stdout and syslog. Modify log_real() to override.
#
# GPLv3 2009 by Erno Rigo <erno_AT_rigo_DOT_info>
#
# also see: http://wwww.openvz.org/
#           http://duplicity.nongnu.org/
#
###################################################################
# script configuration section
 
# backup destination host
DEST_HOST="my.backup.host.example.com"
# username on destination (you should setup a key-based passwordless 
# ssh account for this purpose - see man ssh-keygen)
DEST_USER="root" 
# absolute path on backup destination - should start but not end with /
DEST_PATH="/BACKUPS"
 
# prefix to use everywhere - this should be an unique backup ID
PREFIX="vzbackup"
 
# OpenVZ configuration directory location - usually /etc/vz/conf 
VZCONF=/etc/vz/conf
 
# extended regular expression to filter VE config file list with
# eg.: set this to '(100|230).conf$' in order to just backup VE ID 100 and 230
# all VE-s will be backed up if you leave this empty
EXCLUDE_VE_REGEX=""
 
# LVM snapshot volume size - you should have at least 10-20% of
# your LV size available within the same VG to have snapshotting work reliably
SNAPSIZE=600M
 
# additional options for duplicity
# should at least specify --full-if-older-than
# you should setup encryption here if you need it
DUPLICITY_OPTS="--no-encryption --volsize 100 --full-if-older-than 1W"
 
# number of full backups to keep
# minimum backup window size = full-if-older-than * KEEP 
KEEP=2
 
###################################################################
# do not touch bellow this line unless
# you know what you're doing
 
# destination URL for duplicity
# note: since duplicity cannot autmatically create
# its destination directory, only ssh is supported for now
DEST="ssh://$DEST_USER@$DEST_HOST/$DEST_PATH"
 
# this is the main log function
# it sends output to stdout and the logger
log_real() {
    if [ $# -eq 0 ]; then
        parm=`cat`
    else
        parm="$@"
    fi
    echo -e "$parm" | while read line; do
        echo "+ $line" | logger -s -t "$0"
    done
}
 
# normal log message
log() {
    log_real "$@"
}
 
# fatal log message
fatal() {
    log_real "FATAL: $@"
    echo "-1" > $EXITCODEFILE
    exit -1
}
 
# initialization
do_init() {
    log $0 initializing... 
    trap "do_cleanup 2>&1 | log" exit
    TMPDIR=`mktemp -t -d $PREFIX.XXXXXXXXXX || fatal unable to mktemp`
    EXITCODEFILE=`mktemp -p "$TMPDIR" || fatal unable to mktemp`
    echo 0 > $EXITCODEFILE
}
 
# cleanup function (called from the "exit" bash trap)
do_cleanup() {
    echo "cleaning up $TMPDIR (errors bellow this line can be ignored safely)"
    umount -f "$TMPDIR/snapshot"
    [ -f "$TMPDIR/snapdev" ] && snapdev=`cat "$TMPDIR/snapdev"` && lvs "$snapdev" && do_cmd lvremove -f "$snapdev"
    rm -rf --one-file-system "$TMPDIR"
    echo "cleanup done"
}
 
# run generic shell command. logs and exits if command fails
do_cmd_real() {
    type="$1"
    shift 1
    #id=$RANDOM
    #echo "executing command id#$id: $@" >&2
    out=`mktemp -p "$TMPDIR" || fatal unable to mktemp` >&2
    case "$type" in
        echo)
            "$@" 2>&1 | tee "$out" >&2
            ;;
        noecho)
            "$@" 2>&1 > "$out"
            ;;
    esac
    ecode=$?
    [ $ecode -ne 0 ] && fatal "exit code $ecode for command: $@"
    cat "$out"
    rm -f "$out"
}
 
# normal command execution
do_cmd() {
    do_cmd_real noecho "$@"
}
 
# command execution with output logging
do_cmd_log() {
    do_cmd_real echo "$@"
}
 
# dump a specific VE (identified by VEID and config file path)
do_dumpve() {
    veid="$1"
    conf="$2"
 
    # discover environment
    private=`( VEID=$veid ; . "$conf" ; readlink -f $VE_PRIVATE )`
    dev=`do_cmd df -P -T "$private" | tail -n+2 | awk '{print $1}'`
    fstype=`do_cmd df -P -T "$private" | tail -n+2 | awk '{print $2}'`
    mountpoint=`do_cmd df -P -T "$private" | tail -n+2 | awk '{print $7}'`
    mountpoint=`readlink -f $mountpoint`
    relprivate=`echo "$private" | cut -b$(echo "$mountpoint" | wc -c)- `
    absprivate="$TMPDIR/snapshot/$relprivate"
    lv=`do_cmd lvs -o lv_name --rows --separator ":" $dev | cut -f2 -d':'`
    vg=`do_cmd lvs -o vg_name --rows --separator ":" $dev | cut -f2 -d':'`
    lvdev="/dev/$vg/$lv"
 
    # amuse user
    # echo "VE $veid config:$conf private:$private device:$dev lvdev:$lvdev mountpoint:$mountpoint relprivate:$relprivate"
    echo "VE $veid lvdev:$lvdev mountpoint:$mountpoint relprivate:$relprivate fstype:$fstype"
 
    # prepare and mount snapshot
    do_cmd lvcreate --size $SNAPSIZE --snapshot --name "$PREFIX-snapshot" $lvdev
    echo "/dev/$vg/$PREFIX-snapshot" > "$TMPDIR/snapdev"
    do_cmd mkdir -p "$TMPDIR/snapshot"
    do_cmd mount -v -t "$fstype" "/dev/$vg/$PREFIX-snapshot" "$TMPDIR/snapshot"
 
    # backup VE using duplicity
    pushd "$absprivate" >/dev/null
    mkdir -p "$TMPDIR/dirs/$PREFIX-$veid"
    do_cmd scp -r "$TMPDIR/dirs/$PREFIX-$veid" "$DEST_USER@$DEST_HOST:/$DEST_PATH/"
    do_cmd duplicity cleanup $DUPLICITY_OPTS "$DEST/$PREFIX-$veid/"
    do_cmd duplicity $DUPLICITY_OPTS ./ "$DEST/$PREFIX-$veid/"
    do_cmd duplicity remove-all-but-n-full $KEEP $DUPLICITY_OPTS "$DEST/$PREFIX-$veid/"
    popd >/dev/null
 
    # unmount and remove snapshot
    do_cmd umount -v "$TMPDIR/snapshot"
    do_cmd lvremove -f "/dev/$vg/$PREFIX-snapshot"
}
 
# find VEs to backup - call do_dumpve() for each
do_work() {
    echo "running"
 
    do_cmd find "$VZCONF" -type f | do_cmd egrep "^$VZCONF/?[0-9]{2,}.conf\$" | do_cmd egrep "$EXCLUDE_VE_REGEX" | sort -n | uniq | while read conf; do
        veid=`echo "$conf" | rev | cut -f2 -d'.' | egrep -o "^[0-9]+" | rev`
        echo "backing up VE $veid"
        do_dumpve "$veid" "$conf"
        echo "done backing up VE $veid"
    done
 
}
 
# script entry point - send everything to the log() function
{ do_init || fatal "unable to init" ;} && { do_work 2>&1 | log ;}
 
# exit with the defined exit code (defaults to 0)
ecode=`cat "$EXITCODEFILE"`
log "done, exiting with code:$ecode"
exit $ecode
 
###################################################################
# end of file
###################################################################

Read DVD Video Disc Label

Simplistic bash script to read DVD label from disc.

#!/bin/bash
dd if="$1" count=1 bs=1024 skip=32 2>/dev/null | cut -b 41-72 | sed 's/[^[:alnum:]]._ -]+//g' | sed 's/^ *//g' | rev | sed 's/^ *//g' | rev
2011-07-15 18:52 · 0 Linkbacks

Quick and dirty CA with bash and dialog

This is my quick shell script wrapper using dialog to create a new signed key with my own CA.

#!/bin/bash
 
set -e
 
hostname=`dialog --stdout --clear --inputbox "hostname (pl: www.tricon.hu)" 8 50`
hostmaster=`dialog --stdout --clear --inputbox "hostmaster e-mail (pl: hostmaster@tricon.hu)" 8 50`
 
cp reqtemplate_openssl.cnf reqs/$hostname.req.openssl.conf
rpl "|HOSTNAME|" "$hostname" reqs/$hostname.req.openssl.conf
rpl "|HOSTMASTER|" "$hostmaster" reqs/$hostname.req.openssl.conf
 
openssl req -new -nodes -out reqs/$hostname.req.pem -keyout keys/$hostname.key.pem -config reqs/$hostname.req.openssl.conf
openssl ca -out certs/$hostname.cert.pem -config ./openssl.cnf -infiles reqs/$hostname.req.pem
 
echo "Kesz, ezt a ket fajlt kell hasznalnod:"
echo "keys/$hostname.key.pem"
echo "certs/$hostname.cert.pem"
echo ""
echo "A CA cert a cacert.pem fajlban van, ezt kell telepiteni a kliensekben"
echo ""
echo "Az /etc/CA konyvtarban es alkonyvtaraiban fajlokat torolni, atnevezni TILOS!"

contents of reqtemplate_openssl.cnf follows:

#
# OpenSSL configuration file.
#

[ req ]
default_bits            = 1024                  # Size of keys
default_md              = md5                   # message digest algorithm
default_keyfile		= keys/|HOSTNAME|.key.pem
string_mask             = nombstr               # permitted characters
distinguished_name      = req_distinguished_name
req_extensions          = v3_req
prompt			= no

[ req_distinguished_name ]
countryName                     = HU
stateOrProvinceName             = Budapest
localityName                    = Budapest
organizationName                = TRICON
organizationalUnitName          = |HOSTNAME|
commonName                      = |HOSTNAME|
emailAddress                    = |HOSTMASTER|

[ v3_req ]
basicConstraints        = CA:FALSE
subjectKeyIdentifier    = hash
2011-07-15 18:52 · 0 Linkbacks

Proxy local mysql socket over the network

As an user of OpenVZ it's often my task to migrate database driven web applications to separate environments. In recent versions of OpenVZ, bind mounting of sockets between virtual environments is prohibited. I had to find a way to proxy local connections to /var/run/mysqld/mysqld.sock to the remote mysql server.

First I've created a mysql user and group:

addgroup --system mysql
adduser --system --home /var/run/mysqld --ingroup mysql mysql

Then I've installed and configured the excellent socat utility using launchtool.

db.vz is the address remote database host in the following executable launchtool configuration script

#!/usr/bin/launchtool -C
tag = mysql-proxy
daemon = yes
user = mysql
command = socat UNIX-LISTEN:/var/run/mysqld/mysqld.sock,fork,mode=0666 TCP:db.vz:3306,forever
start dir = /var/run/mysqld
wait times = 5
infinite runs = yes
stats = yes
launchtool output = syslog:mysql-proxy,LOG_DAEMON,LOG_INFO
launchtool errors = syslog:mysql-proxy,LOG_DAEMON,LOG_ERR
command output = syslog:mysql-proxy,LOG_DAEMON,LOG_INFO
command errors = syslog:mysql-proxy,LOG_DAEMON,LOG_ERR

Update: this method seems reliable but rather slow when an application continously creates lots of connections - like does PHP

2011-07-15 18:52 · 0 Linkbacks

MySQL dump to tables

A very small GAWK script that splits a mysql text dump to separate table dump files.

#!/bin/sh
# GPLv2 by Erno Rigo <mcree_AT_tricon_DOT_hu>
 
gawk '
 BEGIN { FS=" "; dest="/dev/null" }
 /^(USE)/ { dir=gensub(";","","g",$2) ; dir=gensub("`","","g",dir) ; system("mkdir "dir) }
 /^(CREATE|USE|DROP)/ { fflush(dest); dest="/dev/null" }
 /^CREATE TABLE/ { file=gensub("`","","g",$3) ; fflush(dest); dest=dir"/"file".sql" }
 { print >> dest }
'
2011-07-15 18:52 · 0 Linkbacks

Linkbacks

Use the following URL for manually sending trackbacks: http://rigo.info/lib/plugins/linkback/exe/trackback.php/en:blog
en/blog.txt · Utolsó módosítás: 2009-05-15 00:00 (külső szerkesztés)
CC Attribution-Noncommercial-Share Alike 4.0 International
www.chimeric.de Valid CSS Driven by DokuWiki do yourself a favour and use a real browser - get firefox!! Recent changes RSS feed Valid XHTML 1.0