#!/bin/sh
# Enterprise Onion Toolkit

# A NOTE TO CONTRIBUTORS: yes this is a big script, but it's also
# written to be easy to understand (and reason about) what it is
# doing; this is why there are `for` loops around `scp` rather than
# tarballing up a bunch of file and shipping them to remote machines,
# unpacking them there. This is (mostly) why there are some
# long_variable_names. I plan to keep it that way.

#rsync_flags="-n" # testing

# expected by tools and libraries
cd `dirname $0` || exit 1
export EOTK_HOME=`pwd`

# for invoking myself
prog=`basename $0`
self=$EOTK_HOME/$prog

# meta
version=1.5.0

# set project directory; this path is hard-replicated elsewhere
project_dir=$EOTK_HOME/projects.d

# mirror directory
mirrors_dir=$EOTK_HOME/mirrors.d

# onionbalance directory
ob_dir=$EOTK_HOME/onionbalance.d
ob_conf=$ob_dir/config.yaml
ob_status_sock=$ob_dir/ob-status.sock
ob_tor_conf=$ob_dir/tor.conf
ob_tor_control_sock=$ob_dir/tor-control.sock

# what are the hostnames of the remote workers?
host_list=$EOTK_HOME/eotk-workers.conf # <--------------------------- HOST FILE

# where we put the persistent, shared config
site_conf=$EOTK_HOME/eotk-site.conf

# init script name
init=eotk-init.sh
housekeeping=eotk-housekeeping.sh

# log retention
log_retain=8

# hostname for log purposes
hostname=`uname -n`

# set path
export PATH=$EOTK_HOME/opt.d:$EOTK_HOME/lib.d:$EOTK_HOME:$PATH

# ------------------------------------------------------------------

# argument stripping
flag_remote=false
flag_local=false
flag_boot=false
propagate_flags=""
while : ; do
    case "x$1" in
        x--local) ## [command] ... | runs commands locally, ignoring onionbalance if configured
            flag_local=true
            propagate_flags="$propagate_flags $1"
            shift
            ;;
        x--boot) ## [command] ... | used by startup scripts
            flag_boot=true
            propagate_flags="$propagate_flags $1"
            shift
            ;;
        x--remote) ## [command] ... | used during remote execution on workers
            flag_remote=true
            propagate_flags="$propagate_flags $1"
            shift
            ;;
        x--debug) ## [command] ... | enables execution tracing
            set -x
            shift
            ;;
        *)
            break
            ;;
    esac
done

CloudHosts() {
    # use --remote hack to forcibly stop risk of recursion...
    if $flag_remote ; then
        echo "localhost" # this will be treated as a magic sentinel
        return 0
    fi

    # perversely, --local, exactly the same
    if $flag_local ; then
        echo "localhost"
        return 0
    fi

    # if --boot, we want localhost IFF we would have it anyway...
    if $flag_boot ; then
        if [ -s $host_list ] ; then # does it contain localhost...?
            egrep '^(localhost)\b' $host_list
        else
            echo "localhost"
        fi
        return 0
    fi

    # todo: if hosts-file exists,
    # cat it (strip comments?)
    # and return
    if [ -s $host_list ] ; then
        cat $host_list
        return 0
    fi

    # else we are working just on this machine
    echo "localhost" # this will be treated as a magic sentinel
}

cloud_hosts=`CloudHosts` # saves multiple invocations
need_to_run_locally=false
for host in $cloud_hosts ; do
    test "x$host" = "xlocalhost" && need_to_run_locally=true
done

# delete dangerous stuff on boot
OnBootCleanup() {
    pidfiles=`find $EOTK_HOME -type f -name "*.pid"`
    sockfiles=`find $EOTK_HOME -type s -name "*.sock"`
    for file in $pidfiles $sockfiles ; do
        if [ -e $file ] ; then # -e => "exists"
            rm $file || exit 1
        fi
        Warn purged $file
    done
}

# ------------------------------------------------------------------

# print a formatted message to stdout
Print() {
    echo "$prog:" "$@"
}

# print a formatted message to stderr
Warn() {
    Print "$@" 1>&2
}

# compress logfiles
LogRotate() {
    SUFFIX=.bz2
    do_proj=$1 # true|false
    do_ob=$2 # true|false
    WHERE=""
    $do_proj && WHERE="$WHERE $project_dir"
    $do_ob && WHERE="$WHERE $ob_dir"
    if [ "x$WHERE" = "x" ] ; then
        Print error: LogRotate has no targets
        exit 1
    fi
    logfiles=`find $WHERE -type f -name "*.log" -print`
    for logfile in $logfiles ; do
        hi=`expr $log_retain - 1`
        dst=$logfile.$hi$SUFFIX
        if [ -f $dst ] ; then
            rm $dst || exit 1
            Print purged $dst
        fi
        while [ $hi -gt 0 ] ; do
            lo=`expr $hi - 1`
            src=$logfile.$lo$SUFFIX
            dst=$logfile.$hi$SUFFIX
            if [ -f $src ] ; then
                mv $src $dst || exit 1
                Print bumped $dst
            fi
            hi=$lo
        done
        dst=$logfile.$hi
        mv $logfile $dst || exit 1
        Print created $dst
        to_compress="$to_compress $dst"
    done
    if $do_proj ; then
        $self --local nxreload -a
        $self --local torreload -a
    fi
    if $do_ob ; then
        $self ob-restart -a
    fi
    if [ "x$to_compress" != "x" ] ; then
        Print $hostname COMPRESSING LOGS: patience please, this may take several minutes...
        ls -l $to_compress
        bzip2 -fv $to_compress
    fi
}

# essentially the projects.d folder is a little database
ListProjects() {
    if [ x$1 = x--softmap-only ] ; then
        softmap_only=true
    else
        softmap_only=false
    fi
    (
        test -d $project_dir || exit 1
        cd $project_dir || exit 1
        for d in *.d/ ; do
            if $softmap_only ; then
                test -f $d/softmap.conf || continue
            fi
            echo `basename $d .d`
        done
    )
}

# TODO(alecm) resolve potential clashes between project names and
# various other directory names ending in ".d"; maybe use ".proj"?

SpotPush() { # push any file named FOO from $EOTK_HOME and/or projects.d
    for filename in "$@" ; do # GENERATE/USE ABSOLUTE PATHS
        files=`find $project_dir -type f -name "$filename" | sort`
        # special: check top-level dir, too...
        tldfile=${EOTK_HOME}/$filename # USE ABSOLUTE PATHS
        test -f $tldfile && files="$files $tldfile"
        for host in $cloud_hosts ; do
            test "x$host" = "xlocalhost" && continue # skip self
            echo :::: push $host $filename ::::
            for filepath in $files ; do
                Print pushing $host:$filepath
                scp -p $filepath $host:$filepath || exit 1
            done
        done
    done
}

# push eotk directory to remote
DestructivePush() {
    for host in $cloud_hosts ; do
        test "x$host" = "xlocalhost" && continue # skip self
        echo :::: rnap $host ::::
        rsync $rsync_flags \
              -av \
              --delete \
              --delete-excluded \
              --exclude="*.log" \
              --exclude="*.pid" \
              --exclude="*.sock" \
              --exclude="*.yaml" \
              --exclude="*~" \
              --exclude="docs.d/" \
              --exclude="mirrors.d/" \
              --exclude="onionbalance.d/" \
              --exclude="secrets.d/" \
              --exclude=".git/" \
              --exclude=".gitignore" \
              --exclude="cached-certs" \
              --exclude="cached-microdesc*" \
              --exclude="configure*.log" \
              --exclude="eotk-workers.conf" \
              --exclude="hostname" \
              --exclude="lock" \
              --exclude="onion_service_non_anonymous" \
              --exclude="private_key" \
              --exclude="state" \
              ${EOTK_HOME}/ \
              $host:${EOTK_HOME}/
    done
}

# mirror remote back for log review, backup, etc
Mirror() {
    for host in $cloud_hosts ; do
        test "x$host" = "xlocalhost" && continue # skip self
        echo :::: mirror $host ::::
        test -d $mirrors_dir || mkdir -p $mirrors_dir || exit 1
        chmod 700 $mirrors_dir || exit 1
        rsync $rsync_flags -av \
              --delete \
              --delete-excluded \
              --exclude="cached-certs" \
              --exclude="cached-microdesc*" \
              $host:${EOTK_HOME}/ $mirrors_dir/$host/
    done
}

# run a command in the context of the local projects directory
RunLocally() {
    action=$1
    shift
    project=$1
    shift

    echo :::: $action $project $@ ::::
    sh "$project_dir/$project.d/$action.sh" "$@"
}

# $1=action, rest = project names
RunLocallyOverProjects() {
    action=$1
    shift # remaining arguments are projects

    if [ "x$1" = "x" ] ; then # test for no args
        Print error: missing project name, try: $prog projects for a list, or -a for all
        return 1
    elif [ "x$1" = "x-a" ] ; then # test for / expand the "-a" flag
        projects=`ListProjects`
    else # do what we are told
        projects="$*"
    fi

    # loop
    for project in $projects ; do
        RunLocally $action $project
    done
}

# run a command on remote machines, or possibly locally
InvokeRemotely() {
    for host in $cloud_hosts ; do
        test "x$host" = "xlocalhost" && continue # skip self
        echo :::: remote $host: $* ::::
        ssh "$host" "$self --remote $*"
    done
}

# get a config file and re/populate the projects directory with it
Configure() {
    log=configure$$.log

    # make the siteconf if it does not already exist
    test -f $site_conf || touch $site_conf || exit 1

    for file in "$@" ; do
        echo :::: configure $file ::::

        case "$file" in
            *.conf)
                : happy bunnies
                ;;

            *.tconf)
                file2=`basename $file .tconf`.conf
                if [ -s $file2 ] ; then
                    Print info: $file: using existing $file2
                else
                    Print info: Processing $file
                    Print info: Populating $file2 with onions
                    Print info: Please be patient...
                    expand-config.pl $file >$file2
                fi
                file="$file2"
                ;;

            *)
                Print error: bad config file suffix, was expecting: .conf, .tconf
                exit 1
                ;;
        esac

        if ! $EOTK_HOME/lib.d/do-configure.pl "$file" ; then
            Print error: failure processing $file: see $log
            exit 1
        fi
    done 2>$log

    Print done. logfile is $log
}

# argument 'parser' - ha!

cmd="$1" # we may need the remaining args
test $# = 0 || shift # if no cmd-arg skip shifting, fall-thru

case "$cmd" in
    version|test) ## print version information for all tools
        test -d .git && git show -s --oneline
        tor --version || Warn cannot find tor executable
        nginx -v || Warn cannot find nginx executable
        onionbalance --version || Warn cannot find onionbalance executable
        Print $version $EOTK_HOME `uname -a`
        InvokeRemotely version
        ;;

    projects|proj) ## lists all projects
        ListProjects
        ;;

    configure|config|conf) ## file ... | use config file to rebuild project directories
        Configure "$@"
        ;;

    genkey|gen) ## generate an onion certificate and print its address
        secrets_dir=secrets.d
        test -d $secrets_dir || mkdir -p $secrets_dir || exit 1
        chmod 700 $secrets_dir || exit 1
        (
            cd $secrets_dir
            generate-onion-key.sh
        )
        ;;

    # ACTIONS

    start) ## project* ... | start the cited projects
        $flag_boot && OnBootCleanup
        $need_to_run_locally && RunLocallyOverProjects start "$@"
        InvokeRemotely start "$@"
        ;;

    stop) ## project* ... | stop the cited projects
        $need_to_run_locally && RunLocallyOverProjects stop "$@"
        InvokeRemotely stop "$@"
        ;;

    restart|bounce|reload) ## project* ... | restart the cited projects
        $need_to_run_locally && RunLocallyOverProjects bounce "$@"
        InvokeRemotely bounce "$@"
        ;;

    nxreload|nx-reload|ob-nxreload) ## project* ... | live-reload nginx configs for projects
        $need_to_run_locally && RunLocallyOverProjects nxreload "$@"
        InvokeRemotely nxreload "$@"
        ;;

    torreload|tor-reload) ## project* ... | live-reload tor configs for projects
        $need_to_run_locally && RunLocallyOverProjects torreload "$@"
        InvokeRemotely torreload "$@"
        ;;

    debugon|debug-on) ## project* ... | enable service debugging for projects
        $need_to_run_locally && RunLocallyOverProjects debugon "$@"
        InvokeRemotely debugon "$@"
        ;;

    debugoff|debug-off) ## project* ... | disable service debugging for projects
        $need_to_run_locally && RunLocallyOverProjects debugoff "$@"
        InvokeRemotely debugoff "$@"
        ;;

    clean|cleanup) ## project* ... | stop and remove trash files for projects (eg: after crash, "nginx.pid exists!" and ".sock exists!" errors, etc)
        $need_to_run_locally && RunLocallyOverProjects cleanup "$@"
        InvokeRemotely cleanup "$@"
        ;;

    syntax) ## project* ... | perform nginx syntax check for projects
        $need_to_run_locally && RunLocallyOverProjects syntax "$@"
        InvokeRemotely syntax "$@"
        ;;

    harvest|onions) ## project* ... | list onions used for projects (including: used by onionbalance)
        $need_to_run_locally && RunLocallyOverProjects harvest "$@"
        InvokeRemotely harvest "$@"
        ;;

    status) ## project* ... | print eotk process status (known processes) for projects
        $self ob-ps # because common
        $need_to_run_locally && RunLocallyOverProjects status "$@"
        InvokeRemotely status "$@"
        ;;

    maps|map) ## project* ... | print onion mappings for projects (including: used by onionbalance)
        $need_to_run_locally && RunLocallyOverProjects maps "$@"
        InvokeRemotely maps "$@"
        ;;

    delete)
        Print $cmd not yet implemented, sorry.
        ;;

    # DIAGS

    ps) ## list all (probable) eotk processes on all workers
        $flag_remote || echo :::: eotk processes ::::
        ps auxww | egrep '\b(eotk|onionbalance|nginx)\b'
        # InvokeRemotely prints remote-ness banner diags
        InvokeRemotely ps
        ;;

    df) ## print filestore usage of eotk directory on all workers
        $flag_remote || echo :::: eotk filesystem space ::::
        df -akh $EOTK_HOME
        # InvokeRemotely prints remote-ness banner diags
        InvokeRemotely df
        ;;

    logsize|logsizes) ## list logfile sizes on all workers
        $flag_remote || echo :::: eotk log sizes ::::
        find $EOTK_HOME -name "*.log" -o -name "*.log.*" -ls
        # InvokeRemotely prints remote-ness banner diags
        InvokeRemotely logsize
        ;;

    # ---- ONIONBALANCE ----

    push)
        Print push is destructive and has been renamed, see the documentation
        ;;

    shutdown) ## shutdown all projects on all workers (and onionbalance)
        $self $propagate_flags ob-stop
        $self $propagate_flags stop -a
        ;;

    # this used to be called 'push' but got renamed because oops.
    # ---- DO NOT USE THIS CASUALLY, LEARN FROM MY MISTAKES ----
    ob-remote-nuke-and-push|rnap) ## DESTRUCTIVE PUSH of local project configuration to workers; THERE ARE NO SAFETY CHECKS, do not use on live system.
        $self stop -a
        DestructivePush
        ;;

    nxpush|ob-nxpush) ## push nginx config files to all workers
        # gently push all the nginx configs to the onionbalance workers; same as "spotpush nginx.conf"
        SpotPush nginx.conf
        ;;

    torpush|ob-torpush) ## push tor config files to all workers
        # gently push all the tor configs to the onionbalance workers; same as "spotpush tor.conf"
        SpotPush tor.conf
        ;;

    spotpush|ob-spotpush) ## file ... | push instances of files named <file> to all workers
        # gently push all instances of given filename, to onionbalance workers
        SpotPush "$@"
        ;;

    ob-config|obconfig|ob-configure|obconfigure)
        Print ob-configure had been rolled into ob-start
        Print please skip forward to: $prog ob-start project ...
        ;;

    ob-gather|obgather) ## onionbalance: gather all worker mappings, and build onionbalance configs; is automatically called by ob-start if needed
        # ob storage
        test -d $ob_dir || mkdir -p $ob_dir || exit 1
        chmod 700 $ob_dir # otherwise tor complains

        # sanity check that args are provided
        if [ "x$1" = "x" ] ; then # test for no args
            Print error: missing project name, try: $prog projects for a list, or -a for all "(if applicable)"
            exit 1
        elif [ "x$1" = "x-a" ] ; then # test for / expand the "-a" flag
            projects=`ListProjects --softmap-only`
        else # do what we are told
            projects="$*"
        fi

        tor_address=127.0.0.1
        tor_port=9055

        # get the mappings
        Print gathering mappings for OnionBalance for projects: $projects
        mappings=__om$$.tmp
        $self maps $projects |
            awk '/^::::/{next;} $4=="softmap"{print;}' >$mappings

        # check the mappings
        for p in $projects ; do
            n=`awk '$3=="'"$p"'"'<$mappings  | wc -l`
            if [ $n = 0 ] ; then
                Warn no mappings yet for project $p: does it exist / is it started / is it not a softmap project "?"
                rm $mappings
                exit 1
            else
                m=`awk '$3=="'"$p"'"{print $1}'<$mappings | sort -u | wc -l`
                w=`awk '$3=="'"$p"'"{print $6}'<$mappings | sort -u | wc -l`
                Print project $p contains $n mappings
                Print project $p uses $m master onions
                Print project $p uses $w worker onions
            fi
        done

        Print building OnionBalance configurations...
        (
            # notes/commentary, including onionbalance defaults, at:
            # https://media.readthedocs.org/pdf/onionbalance/latest/onionbalance.pdf
            echo LOG_LEVEL: info
            echo TOR_ADDRESS: $tor_address
            echo TOR_PORT: $tor_port
            echo STATUS_SOCKET_LOCATION: $ob_status_sock

            # REFRESH_INTERVAL How often to check for updated backend
            # hidden service descriptors. This value can be decreased
            # if your backend instance are under heavy loaded causing
            # them to rotate introduction points quickly. (default:
            # 600 seconds).
            refresh=600 # to be tuned
            echo REFRESH_INTERVAL: $refresh

            # INITIAL_DELAY How long to wait between starting
            # OnionBalance and publishing the master descrip- tor. If
            # you have more than 20 backend instances you may need to
            # wait longer for all instance descriptors to download
            # before starting (default: 45 seconds).
            echo INITIAL_DELAY: `expr $refresh / 6`

            # PUBLISH_CHECK_INTERVAL How often should to check if new
            # descriptors need to be published for the master hidden
            # service (default: 360 seconds).
            echo PUBLISH_CHECK_INTERVAL: `expr $refresh / 2`

            # DESCRIPTOR_OVERLAP_PERIOD How long to overlap hidden
            # service descriptors when changing descriptor IDs
            # (default: 3600 seconds)
            echo DESCRIPTOR_OVERLAP_PERIOD: `expr $refresh '*' 4`

            # DESCRIPTOR_UPLOAD_PERIOD How often to publish a
            # descriptor, even when the introduction points don't
            # change (default: 3600 seconds)
            echo DESCRIPTOR_UPLOAD_PERIOD: `expr $refresh '*' 4`

            # DESCRIPTOR_VALIDITY_PERIOD How long a hidden service
            # descriptor remains valid (default: 86400 seconds)
            echo DESCRIPTOR_VALIDITY_PERIOD: 86400 # 1 day

            do-obconfig.pl <$mappings
        ) > $ob_conf

        # clean up
        rm $mappings

        Print building OnionBalance-Tor configurations...
        (
            echo DataDirectory $ob_dir
            echo ControlPort unix:$ob_tor_control_sock
            echo PidFile $ob_dir/tor.pid
            echo "#" Log info file $ob_dir/tor.log
            echo Log notice file $ob_dir/tor.log
            echo SafeLogging 1
            echo HeartbeatPeriod 60 minutes
            echo RunAsDaemon 1
            echo "#" onionbalance
            # echo SocksPort unix:$ob_dir/tor-socks.sock # curl 7.38 does not like this
            echo SocksPort $tor_address:$tor_port
            echo CookieAuthentication 1
            # echo MaxClientCircuitsPending 1024
        ) > $ob_tor_conf
        ;;

    ob-start|obstart) ## start onionbalance (will gather and build configs if needed)
        # do not test for ob_dir here, it is okay to not exist

        if [ -f $ob_dir/ob.pid ] ; then
            Warn file $ob_dir/ob.pid already exists, OnionBalance may already be running.
            Warn Aborting...
            exit 1
        fi

        if [ -f $ob_dir/tor.pid ] ; then
            Warn file $ob_dir/tor.pid already exists, OnionBalance-Tor may already be running.
            Warn Aborting...
            exit 1
        fi

        # ob-gather checks args
        $self ob-gather "$@" || exit 1

        Print starting OnionBalance-Tor
        tor -f $ob_tor_conf >$ob_dir/tor-startup.log 2>&1

        Print starting OnionBalance
        onionbalance \
            -s $ob_tor_control_sock \
            -c $ob_conf </dev/null >$ob_dir/onionbalance.log 2>&1 & # bg
        ob_pid=$!
        echo $ob_pid >$ob_dir/ob.pid
        ;;

    ob-stop|obstop) ## stop onionbalance
        for pidfile in $ob_dir/ob.pid $ob_dir/tor.pid ; do
            test -s $pidfile || continue
            pid=`cat $pidfile`
            Print sending SIGTERM to $pid in $pidfile
            kill $pid
        done
        rm -f $ob_dir/ob.pid
        ;;

    ob-restart) ## restart onionbalance
        # ob-stop takes no args, but ob-start requires them; try to be consistent, therefore:
        if [ "x$1" = "x" ] ; then # test for no args
            Print error: missing project name, try: $prog projects for a list, or -a for all "(if applicable)"
            exit 1
        fi
        $self ob-stop
        $self ob-start "$@"
        ;;

    ob-ps) ## print list of onionbalance processes
        test -d $ob_dir || exit 0
        echo :::: onionbalance processes ::::
        pidfiles=`find $ob_dir -name "*.pid"`
        if [ "x$pidfiles" != "x" ] ; then
            ps -p `cat $pidfiles`
        fi
        ;;

    ob-status|obstatus) ## query onionbalance for its status
        $self ob-ps
        if [ -S $ob_status_sock ] ; then
            echo ""
            socat - unix-connect:$ob_status_sock
        fi
        ;;

    ob-maps|obmaps) ## project* ... | print onionbalance mappings for projects
        if [ "x$1" = "x" ] ; then # test for no args
            Print error: missing project name, try: $prog projects for a list, or -a for all "(if applicable)"
            exit 1
        fi
        $self maps "$@" |
            awk '/^::::/{next;} $4=="softmap"{print $1, $2}' |
            sort -k 2 -u
        ;;

    ob-watch|obwatch) ## use 'watch' to iterate 'ob-status'
        watch -n 5 $self ob-status
        ;;

    # FREEZE/BACKUP

    mirror|pull) ## pull a copy (rsync) of all workers to local 'mirrors.d' directory
        Mirror
        ;;

    backup|freeze) ## as 'mirror' but also create a datestamped compressed tarball once mirror is complete
        Mirror
        (
            ds=`date "+%Y%m%d%H%M%S"`
            cd $mirrors_dir || exit 1
            for directory in */ ; do
                test -d "$directory" || exit 1 # did */ expand?
                dir=`basename $directory` # strip trailing /
                echo :::: backup $dir ::::
                tar cf - $dir | bzip2 > $dir-$ds.tar.bz2
            done
        )
        ;;

    logrotate) ## compress logs and reload configurations on all workers
        InvokeRemotely logrotate
        # do we need to do onionbalance logs?
        if [ -d $ob_dir ] ; then
            need_to_do_ob_logs=true
        else
            need_to_do_ob_logs=false
        fi
        LogRotate $need_to_run_locally $need_to_do_ob_logs
        ;;

    script|scripts|make-script|make-scripts) ## create 'boot' and 'cron' housekeeping scripts
        for t in $init $housekeeping ; do
            lib.d/expand-template.pl templates.d/$t.txt </dev/null >$t 2>/dev/null
            chmod 755 $t
            Print created: $t
        done
        Print please read those files for installation instructions.
        ;;

    shell) ## run a shell in the eotk PATH environment
        env PS1='eotk-env$ ' ${SHELL:-sh} -i
        ;;

    help|*) ## prints this text
        Print "switches and commands:"
        echo "  * project* => supports '-a' for all"
        echo "  * synonyms are in <angle-brackets|diamond-brackets>"
        echo ""
        exec ./lib.d/explain.pl $0
        exit 1
        ;;
esac

exit 0