Merge branch 'master' of https://tree.clementinecomputing.com/clementinecomputing/popufare

busunit/scripts/common_values.sh

@@ -0,0 +1,282 @@
+#!/bin/bash
+#
+# Copyright (c) 2019 Clementine Computing LLC.
+# 
+# This file is part of PopuFare.
+# 
+# PopuFare is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# PopuFare is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+# 
+# You should have received a copy of the GNU Affero General Public License
+# along with PopuFare.  If not, see <https://www.gnu.org/licenses/>.
+#
+
+#THIS FILE IS NOT REALLY A SCRIPT, BUT RATHER A CENTRAL HOLDING TANK FOR SYSTEM CONFIG VALUES...
+#IT SHOULD BE SOURCED BY ALL OTHER SCRIPTS THAT NEED A CONTAINED VALUE, AND MAY SOME DAY BE PARSED
+#BY THE NATIVE APPLICATIONS.  THUS:  Keep it simple, in the following format:
+#
+##Comment on meaning of variable
+#VARIABLE="value"
+#
+#(variable name in all caps, no space, equals sign, quoted literal value (if there is even a remote chance that the value
+#is one that will be needed by the native app in the future)
+#
+
+BASEDIR="$HOME"
+
+#How long to sleep after a dial attempt before checking if we have a PPP session successfully constructed
+SLEEP_AFTER_DIAL="30"
+
+#How long to wait between redial attempts     
+SLEEP_BETWEEN_REDIALS="60"
+
+# How long to sleep after the GRPS session comes up before testing the SSH tunnel (attempting to connect to the 'hello' server)
+#
+SLEEP_BEFORE_TUNNEL_TEST="5"
+
+# How long to sleep after an established tunnel goes down before allowing any SSH tunnel establishment retries
+#
+SLEEP_AFTER_TUNNEL_FAILURE="60"
+
+# How long to sleep after an established tunnel has been aborted
+#
+SLEEP_AFTER_TUNNEL_ABORT="60"
+
+# How many successive ssh tunnel failures do we accept before forcing a hard termination of pppd and a modem reset and redail
+#
+MAX_FAIL_HANGUP="5"
+
+# How frequently to test an SSH tunnel that predates the ppp-dialer.sh process (thus having no right to wait for its PID)
+#
+SLEEP_MONITORING_TUNNEL="120"
+
+#  How frequently to perform SSH keepalive messages, and how many of them are allowed to drop before we tear the tunnel down
+# and wait to try again.  This gets passed to the SSH client in the following form:
+# -o ServerAliveInterval=$SSH_SERVER_ALIVE_INTERVAL -o ServerAliveCountMax=$SSH_SERVER_ALIVE_MAXDROP
+#
+SSH_SERVER_ALIVE_INTERVAL="15"
+SSH_SERVER_ALIVE_MAXDROP="3"
+
+# These are the default server parameters the system will use if there is not a configured sync server.
+#The system will check 
+SSH_DEFAULT_TARGET="$USER@example.com"
+SSH_DEFAULT_PORT="6055"
+SSH_DEFAULT_IDENTITY="$HOME/.ssh/id_rsa_bus"
+
+#This regular expression test is used to make sure we have a valid-looking FQDN or IP address
+SSH_TARGET_VALIDITY_CHECK="^[a-zA-Z][a-zA-Z0-9]*@([a-zA-Z0-9-]+\.)+([a-zA-Z0-9-]+)$"
+
+SSH_TARGET="$SSH_DEFAULT_TARGET"
+SSH_PORT="$SSH_DEFAULT_PORT"
+SSH_IDENTITY="$SSH_DEFAULT_IDENTITY"
+
+#--------------------------------------------------------------------------------
+#This is the list of forwarded ports we bring live when we set up our SSH tunnel:
+
+# 2857 -> AVLS
+AVLS_DAEMON_PORT="2857"
+
+# 7277 -> Bus Pass Daemon
+PASSDB_DAEMON_PORT="7277"
+
+# 2455 -> Billing Log Daemon
+BILLDB_DAEMON_PORT="2455"
+
+# 3556 -> "Hello" Daemon
+HELLO_DAEMON_PORT="3556"
+
+# 8377 -> Update / Version Server
+UPDATE_DAEMON_PORT="8377"
+
+
+#    These are used to talk to the "Hello" daemon on the server to verify that
+#actual data can flow before notifying the system that the tunnel is up.
+HELLO_DAEMON_MESSAGE="Hello."
+
+
+#--------------------------------------------------------------------------------
+#    This is a list of base ports on the server that we can use for the reverse-phone-home
+#functionality which forwards port BASE + client_id on the server to port 22 (sshd) on 
+#the client to allow for remote troubleshooting and manual configuration.
+
+#Enable reverse-phone-home feature
+REVERSE_PHONE_HOME="1"
+
+#    This is the base port (on the server) which the numbering for the reverse-phone-home
+#system starts at when numbered by equipment number:
+REVERSE_PHONE_HOME_EQNUM_BASE="10000"
+
+#    This is the base port (on the server) which the numbering for the reverse-phone-home
+#system starts at when numbered by serial number (wireless_ip on transition busses):
+REVERSE_PHONE_HOME_SERIALNUM_BASE="30000"
+
+#    This is the number above which an equipment number or other token will not be considered
+#as a phone home candidate on account of its likelyhood to overflow into the next port range.
+REVERSE_PHONE_HOME_MAX_TOKEN="19999"
+
+
+#--------------------------------------------------------------------------------
+#These paths lead to dropfiles in /tmp which are used to signal network state
+
+#This dropfile means we have an established GPRS/PPP session
+GPRS_DROPFILE="/tmp/network-is-up"
+#This dropfile means we have an established SSH tunnel to the server
+TUNNEL_DROPFILE="/tmp/tunnel-is-up"
+#This dropfile contains the process ID of our SSH client that's keeping the tunnel open
+SSH_TUNNEL_PIDFILE="/tmp/ssh_tunnel.pid"
+#This dropfile contains identifying information about our ethernet card, cell modem, and SIM
+NETWORK_ID_DROPFILE="/tmp/net_ids"
+
+#This dropfile becomes present to notify the update process to abort:
+UPDATE_ABORT_DROPFILE="/tmp/abort_update"
+TUNNEL_ABORT_DROPFILE="/tmp/abort_tunnel"
+
+
+#############################################################
+
+FIX_PACKAGE_PERMS="1"
+
+########################################################## Used only if FIX_PACKAGE_PERMS is nonzero ##############
+#This specifies who should own installed packages
+PACKAGE_OWNER_STRING="root:root"
+
+#These are the permissions applied to stuff in the bin directory
+PACKAGE_BIN_PERMISSIONS="755"
+#This is the egrep expression that must match to qualify something as a binary
+PACKAGE_BIN_PATTERN='(^[/]?'$BASEDIR'/bin/[a-z_A-Z0-9.]+$)|(.*\.sh$)'
+
+#These are the permissions applied to SSH related config files and directories
+PACKAGE_SSH_STRICT_FILE_PERMISSIONS="600"
+PACKAGE_SSH_MILD_FILE_PERMISSIONS="644"
+PACKAGE_SSH_DIR_PERMISSIONS="700"
+
+#this one is only used in its ALWAYS case
+PACKAGE_SYS_DIR_PERMISSIONS="755"
+
+#These are the egrep expressions that must match to qualify something as an SSH file or directory
+PACKAGE_SSH_STRICT_FILE_PATTERN="(^[/]?etc/ppp/id_rsa[^\.]*$)|(^[/]?root/\.ssh/id_rsa[^\.]*$)|(^[/]?root/\.ssh/authorized_keys)"
+PACKAGE_SSH_MILD_FILE_PATTERN="(^[/]?root/\.ssh/known_hosts)|(^[/]?etc/ppp/id_rsa[^\.]*.pub$)|(^[/]?root/\.ssh/id_rsa[^\.]*.pub)"
+PACKAGE_SSH_DIR_PATTERN="(^[/]?root[/]?$)|(^[/]?root/\.ssh[/]?$)"
+
+#These are the list of files to ALWAYS reset permissions on (whether or not they are in the update)
+ALWAYS_SSH_DIR_LIST="/root /root/.ssh"
+ALWAYS_SSH_STRICT_FILE_LIST="/root/.ssh/id_rsa /etc/ppp/id_rsa_bus /root/.ssh/authorized_keys $SYNC_PRIVATE_KEY"
+ALWAYS_SSH_MILD_FILE_LIST="/root/.ssh/known_hosts /root/.ssh/id_rsa.pub /etc/ppp/id_rsa_bus.pub /etc/ssh/sshd_config"
+ALWAYS_BIN_LIST="$BASEDIR/bin/init_bus.sh $BASEDIR/bin/update_loop.sh $BASEDIR/bin/apply_update.sh $BASEDIR/bin/fix_pkg_perm.sh $BASEDIR/bin/get_net_ids.sh"
+
+ALWAYS_SYS_DIR_LIST="/etc /etc/ssh /etc/ppp"
+
+DEFAULT_EXTRACT_PATH="/"
+EXTRACT_PATH_FILE="/tmp/pkg_extract_path"
+
+##################################################################################################################
+
+
+#These paths point to important bits of information for synchronizing
+EQUIP_NUM_FILE="$BASEDIR/config/equipnum.txt"
+SERIAL_NUM_FILE="$BASEDIR/config/serial_num"
+
+SERVER_CONFIG_DIR="$BASEDIR/config/server"
+
+SERVER_LIST_FILE="$SERVER_CONFIG_DIR/server_list"
+
+SYNC_DESC_FILE="$SERVER_CONFIG_DIR/sync_server_desc"
+SYNC_TARGET_FILE="$SERVER_CONFIG_DIR/sync_target"
+SYNC_PORT_FILE="$SERVER_CONFIG_DIR/sync_port"
+SYNC_KNOWN_HOSTS="/root/.ssh/known_hosts"
+SYNC_PRIVATE_KEY="/etc/ppp/id_rsa_client"
+
+###############################################################################################
+#     CONFIGURATION VALUES USED PRIMARILY BY THE UPDATE LOOP SCRIPT                           #
+###############################################################################################
+
+#How long for update_loop.sh to sleep after spawning the client supervisor before (potentially) respawning it
+SLEEP_AFTER_SPAWN="60"
+
+#How long to sleep between attempts to download patches from the server
+SLEEP_BETWEEN_UPDATES="60"
+
+#The directory in which the current patch state is to be found
+CHECKSUM_AND_VERSION_PATH="$BASEDIR/config/"
+
+###############################################################################################
+#     BELOW HERE, COMPOUND OPTIONS ARE ALLOWED (things that use variable substitution, etc...)#
+###############################################################################################
+
+#This looks for all environment variables that specify a port that we must forward...
+TUNNEL_PORT_LIST="`set | egrep "^[A-Z]+_DAEMON_PORT=" | cut -d '=' -f2 | egrep '^[0-9]+$' | xargs echo`"
+
+#This gathers the above list into something we can just hand to the SSH client...
+SSH_FORWARDS="`for port in $TUNNEL_PORT_LIST; do echo -n " -L$port:localhost:$port"; done`"
+
+#These are the flags to SSH that are specific to setting up the tunnel
+SSH_TUNNEL_FLAGS="-n -N -x"
+
+#These are the flags to SSH that are specific to our connection to the target host
+SSH_TARGET_OPTIONS="-o StrictHostKeyChecking=no"
+
+#These are the flags to SSH that specify the behavior of the protocol-level keepalive messages
+SSH_KEEPALIVE_OPTIONS="-o ServerAliveInterval=$SSH_SERVER_ALIVE_INTERVAL -o ServerAliveCountMax=$SSH_SERVER_ALIVE_MAXDROP"
+
+SSH_OPTIONS="$SSH_TUNNEL_FLAGS $SSH_KEEPALIVE_OPTIONS $SSH_TARGET_OPTIONS"
+SCP_OPTIONS="$SSH_KEEPALIVE_OPTIONS $SSH_TARGET_OPTIONS"
+
+# This function kills any currently open SSH or SCP sessions and puts dropfiles in /tmp telling
+#ppp-dialer.sh and update_loop.sh to refresh their Sync server parameters with the generate_ssh_targets
+#function before attempting to reconnect. 
+#
+abort_tunnel_and_update() {
+
+  touch $UPDATE_ABORT_DROPFILE
+  touch $TUNNEL_ABORT_DROPFILE
+
+  sync
+  sleep 1
+
+  killall ssh
+  killall scp
+}
+
+# This function uses the dropfiles left by unpack_server_data.sh and constructs new SSH and SCP
+#parameters suitable to connect to the specified server.  If there is no server specified, or if the
+#specified server does not pass a sanity check then the default server will be used.
+#
+function generate_ssh_targets
+{
+
+  if [ -f $SYNC_TARGET_FILE ]; then
+
+    alt_target="`cat $SYNC_TARGET_FILE`"
+    if echo "$alt_target" | egrep -q "$SSH_TARGET_VALIDITY_CHECK"; then
+
+      SSH_TARGET="`cat $SYNC_TARGET_FILE`"
+      SSH_PORT="`cat $SYNC_PORT_FILE`"
+      SSH_IDENTITY="$SYNC_PRIVATE_KEY"
+
+      echo "$0: Using sync server $SSH_TARGET port $SSH_PORT"
+    else
+
+      SSH_TARGET="$SSH_DEFAULT_SYNC_TARGET"
+      SSH_PORT="$SSH_DEFAULT_SYNC_PORT"
+      SSH_IDENTITY="$SSH_DEFAULT_IDENTITY"
+
+      echo "$0: Specified sync server name $alt_target is malformed.  Using default $SSH_DEFAULT_SYNC_TARGET port $SSH_DEFAULT_SYNC_PORT"
+    fi
+  else
+
+      SSH_TARGET="$SSH_DEFAULT_TARGET"
+      SSH_PORT="$SSH_DEFAULT_PORT"
+      SSH_IDENTITY="$SSH_DEFAULT_IDENTITY"
+
+      echo "$0: No sync server selected. Using default $SSH_DEFAULT_TARGET port $SSH_DEFAULT_PORT"
+  fi
+
+}
+

busunit/scripts/connection_tether-ppp.sh

@@ -0,0 +1,317 @@
+#!/bin/bash
+#
+# Copyright (c) 2019 Clementine Computing LLC.
+#
+# This file is part of PopuFare.
+#
+# PopuFare is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# PopuFare is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with PopuFare.  If not, see <https://www.gnu.org/licenses/>.
+#
+
+. $HOME/bin/common_values.sh 
+
+# Do this once at boot time, but do it again after a tunnel abort request...
+#
+generate_ssh_targets
+
+ssh_fail_counter=0
+
+debug_print()
+{
+  echo $@
+}
+
+#    Every once in a while pppd will report connection, and still have a ppp session open to the modem, but
+# the router on the other end will have croaked, or the underlying GPRS connection will be too unreliable, or
+# sometimes even a pppd will hang such that it needs a SIGKILL to get rid of it...  This function does that
+# and then cleans up after the dead pppd by removing its dropfiles and locks.
+#
+force_kill_pppd()
+{
+  /usr/bin/killall pppd
+  rm -rf /var/lock/*ttyGPRS
+  rm -rf $GPRS_DROPFILE
+}
+
+#    This function goes and looks at the version dropfiles for packages, gathering their package names
+# and versions to report when we check in with the server to log that we connected, and at what firmware and
+# config revision.
+#
+output_versions()
+{
+  for file in `ls $CHECKSUM_AND_VERSION_PATH/*.version`; do
+    echo -n " `echo $file | sed -r 's/^.*\/(.*)\.version$/\1/'`=`cat $file`";
+  done
+}
+
+# This function extracts a field from the network ID dropfile by name (also used by the checkin process)
+#
+output_net_ids_field()
+{
+  field="$1"
+
+  cat $NETWORK_ID_DROPFILE | grep "$field" | cut -d'=' -f2 | xargs -n1 echo -n
+}
+
+#    This function attempts to check in with the version server and report hardware and network serial numbers
+# so that us sysadmin types can see each time a unit attached to the network, which unit it was, and what software
+# it was running at the time.
+#
+perform_post_connect_checkin()
+{
+  if [ -f $SERIAL_NUM_FILE ]; then
+    busunitnum="`cat $SERIAL_NUM_FILE 2> /dev/null`"
+  fi
+
+  equipnum="`cat $EQUIP_NUM_FILE 2> /dev/null || echo 0`"
+
+  version="`output_versions`"
+  imei="`output_net_ids_field IMEI`"
+  imsi="`output_net_ids_field IMSI`"
+  mac="`output_net_ids_field ETH0`"
+
+  #     Send these gathered data to the update daemon.  The leading '#' tells the server that this is a 
+  #checkin, not an update request.
+  echo -e "#$busunitnum\t$equipnum\t$mac\t$imei\t$imsi\t$version" | nc localhost $UPDATE_DAEMON_PORT
+}
+
+#    This function generates the server->client port forwards to allow a sysadmin to log into any unit that is on
+# the network by equipment number, serial number, or bus number. The three parameters are:
+#
+# 1:   The path to the file containing the identifying number
+# 2:   The base port number on the remote server to add the identifying number to to get the server-side port that will 
+#    forward to port 22 (sshd) on the client side.
+# 3:   An optional parameter which if present is taken as a set of command line flags to cut to apply to the contents
+#    of the file specified by $1 to extract the numeric component (for instance, serial numbers may be in the form XYZ-1234
+#    in which case it's really the 1234 part we're after...
+#
+generate_reverse_phonehome_component()
+{
+  file="$1"
+  base="$2"
+  cut_cmdline="$3"
+
+  #Make sure the candidate dropfile exists
+  if [ -f "$file" ]; then
+
+    if [ -n "$cut_cmdline" ]; then
+      #Grab the desired substring
+      num="`cat $file | cut $cut_cmdline`"
+    else
+      #Grab its contents   
+      num="`cat $file`";
+    fi
+
+    #Make sure those contents are indeed numeric...
+    if (echo "$num" | egrep -q '^[0-9]+$'); then
+
+      #Make sure that number is within an acceptable range so as not to overflow
+      if [ "$num" -gt "0" -a "$num" -le "$REVERSE_PHONE_HOME_MAX_TOKEN" ]; then
+        echo -n " -R$((base + num)):localhost:22";
+      fi
+
+    fi
+
+  fi
+}
+
+#    This function calls the above component function for each identifying number we want to do a port forward based on...
+#
+generate_reverse_phonehome_string()
+{
+  #If the reverse phone home feature is disabled, return without printing any commandline args
+  if [ "$REVERSE_PHONE_HOME" -eq "0" ]; then return; fi
+
+  generate_reverse_phonehome_component $EQUIP_NUM_FILE  $REVERSE_PHONE_HOME_EQNUM_BASE
+  generate_reverse_phonehome_component $SERIAL_NUM_FILE $REVERSE_PHONE_HOME_SERIALNUM_BASE "-d- -f3"
+}
+
+
+#    This function performs teardown on a dead ssh connection, and increments the ssh connect failure counter.  If
+# that counter has reached its maximum value, we force pppd down and try a clean redial.  Otherwise, we just sleep and
+# try again.
+#
+clean_up_after_tunnel_teardown()
+{
+  #If the tunnel was intentionally aborted for the purpose of switching servers
+  if [ -f $TUNNEL_ABORT_DROPFILE ]; then
+
+    debug_print "SSH tunnel aborted, removing dropfiles..."
+
+    #Remove the dropfiles...
+    /bin/rm -f $TUNNEL_DROPFILE $SSH_TUNNEL_PIDFILE
+
+    #Generate new ssh target from server config dropfiles
+    generate_ssh_targets
+
+    /bin/rm -f $TUNNEL_ABORT_DROPFILE
+
+    #Reset the failure counter
+    ssh_fail_counter=0
+
+    #Sleep until it is time to try again
+    /bin/sleep $SLEEP_AFTER_TUNNEL_ABORT
+
+    
+  else
+    #OTHERWISE, we assume that the modem lost signal, or the router or remote server went wonky...
+
+    debug_print "SSH client dead, removing dropfiles..."
+    #Remove the dropfiles...
+    /bin/rm -f $TUNNEL_DROPFILE $SSH_TUNNEL_PIDFILE
+
+    ssh_fail_counter=$((ssh_fail_counter + 1))
+
+    if [ "$ssh_fail_counter" -ge "$MAX_FAIL_HANGUP" ]; then
+      debug_print "pppd claims to be up; tunnel failed $ssh_fail_counter times, killing pppd to force redial."
+      force_kill_pppd
+    fi
+
+    debug_print "Sleeping before any retry attempts..."
+    #Sleep before trying this again...
+    /bin/sleep $SLEEP_AFTER_TUNNEL_FAILURE
+  
+  fi
+}
+
+while true; do
+
+  # If our GRPS connection has died...
+  #
+  if [ ! -f $GPRS_DROPFILE ]; then
+    ssh_fail_counter=0
+    debug_print "Attempting to dial..."
+
+    # Try to re-"dial" our ISP 
+    #
+    /usr/bin/pon gprs
+
+    # Give the modem a minute to do its thing...
+    #
+    /usr/bin/pon gprs
+
+    # Give the modem a minute to do its thing...
+    #
+    /bin/sleep $SLEEP_AFTER_DIAL
+
+  fi
+
+  # If we've just been asked to abort the SSH tunnel
+  #
+  if [ -f $TUNNEL_ABORT_DROPFILE ]; then
+
+    # Generate new ssh target from server config dropfiles
+    #
+    generate_ssh_targets
+
+    /bin/rm -f $TUNNEL_ABORT_DROPFILE
+
+    # Reset the failure counter
+    #
+    ssh_fail_counter=0
+  fi
+
+  # If we now have an active GPRS network connection, try and bring a tunnel up...
+  #
+  if [ -f $GPRS_DROPFILE ]; then
+
+    # If we have no active tunnel already...
+    #
+    if [ ! -f $TUNNEL_DROPFILE ]; then
+      debug_print "Attempting to establish SSH tunnel... (Attempt number $((ssh_fail_counter + 1)))"
+
+      # Attempt to create our tunnel... (incliding (if REVERSE_PHONE_HOME != 0) reverse phone home support
+      #
+      ssh $SSH_OPTIONS -i $SSH_IDENTITY $SSH_FORWARDS `generate_reverse_phonehome_string` -p $SSH_PORT $SSH_TARGET &
+
+      # Remember its PID
+      #
+      echo "$!" > $SSH_TUNNEL_PIDFILE
+
+      # Wait a few seconds to allow SSH negotiations
+      #
+      /bin/sleep $SLEEP_BEFORE_TUNNEL_TEST
+      debug_print -n "Testing our new tunnel..."
+
+      # Test to see if our tunnel is really up...
+      #
+      if [ "`nc localhost $HELLO_DAEMON_PORT < /dev/null`" = $HELLO_DAEMON_MESSAGE ]; then
+        debug_print " It works."
+        ssh_fail_counter=0
+
+        debug_print "Checking in with server to report net IDs and package versions... "
+        perform_post_connect_checkin
+
+        debug_print "Touching dropfile and waiting for SSH to terminate..."
+
+        # Touch our dropfile indicating the tunnel is up...
+        #
+        /bin/touch $TUNNEL_DROPFILE
+
+        # and wait for the the SSH client process to end
+        #
+        wait `cat $SSH_TUNNEL_PIDFILE`
+
+        # Clean Up...
+        #
+        clean_up_after_tunnel_teardown
+
+      else
+        debug_print " No luck..."
+        debug_print "Issuing kill to SSH client...."
+
+        # Kill the defunct and/or too slow to use SSH client
+        #
+        /bin/kill `cat $SSH_TUNNEL_PIDFILE`
+
+        # Wait for the process to terminate
+        #
+        wait `cat $SSH_TUNNEL_PIDFILE`
+
+        # Clean Up...
+        #
+        clean_up_after_tunnel_teardown
+      fi
+
+    else
+
+      # This means we _think_ we have an SSH tunnel, but it's not one we set up...
+      #
+      debug_print -n "We seem to already have a pre-existing tunnel... Monitoring it."
+
+      # Loop and periodically test this tunnel...  When this condition fails, we're done...
+      #
+      while [ "`nc localhost $HELLO_DAEMON_PORT < /dev/null`" = $HELLO_DAEMON_MESSAGE ]; do
+
+        # Sleep for a while before testing this tunnel again...
+        #
+        /bin/sleep $SLEEP_MONITORING_TUNNEL
+
+      done
+
+      # Clean Up...
+      #
+      clean_up_after_tunnel_teardown
+    fi
+
+  else
+
+    # If we don't have an active GPRS session, that means we just failed at dialing
+    #
+    debug_print "Dialing failed...  Sleeping"
+    /bin/sleep $SLEEP_BETWEEN_REDIALS
+
+  fi
+
+done
+

busunit/scripts/connection_tether-ssh.sh

@@ -0,0 +1,268 @@
+#!/bin/bash
+#
+# Copyright (c) 2019 Clementine Computing LLC.
+#
+# This file is part of PopuFare.
+#
+# PopuFare is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# PopuFare is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with PopuFare.  If not, see <https://www.gnu.org/licenses/>.
+#
+
+. $HOME/bin/common_values.sh 
+
+# Do this once at boot time, but do it again after a tunnel abort request...
+#
+generate_ssh_targets
+
+ssh_fail_counter=0
+
+debug_print()
+{
+  echo $@
+}
+
+#    This function goes and looks at the version dropfiles for packages, gathering their package names
+# and versions to report when we check in with the server to log that we connected, and at what firmware and
+# config revision.
+#
+output_versions()
+{
+  for file in `ls $CHECKSUM_AND_VERSION_PATH/*.version`; do
+    echo -n " `echo $file | sed -r 's/^.*\/(.*)\.version$/\1/'`=`cat $file`";
+  done
+}
+
+# This function extracts a field from the network ID dropfile by name (also used by the checkin process)
+#
+output_net_ids_field()
+{
+  field="$1"
+
+  cat $NETWORK_ID_DROPFILE | grep "$field" | cut -d'=' -f2 | xargs -n1 echo -n
+}
+
+#    This function attempts to check in with the version server and report hardware and network serial numbers
+# so that us sysadmin types can see each time a unit attached to the network, which unit it was, and what software
+# it was running at the time.
+#
+perform_post_connect_checkin()
+{
+  if [ -f $SERIAL_NUM_FILE ]; then
+    busunitnum="`cat $SERIAL_NUM_FILE 2> /dev/null`"
+  fi
+
+  equipnum="`cat $EQUIP_NUM_FILE 2> /dev/null || echo 0`"
+
+  version="`output_versions`"
+  imei="`output_net_ids_field IMEI`"
+  imsi="`output_net_ids_field IMSI`"
+  mac="`output_net_ids_field ETH0`"
+
+  #     Send these gathered data to the update daemon.  The leading '#' tells the server that this is a 
+  # checkin, not an update request.
+  #
+  echo -e "#$busunitnum\t$equipnum\t$mac\t$imei\t$imsi\t$version" | nc localhost $UPDATE_DAEMON_PORT
+}
+
+#    This function generates the server->client port forwards to allow a sysadmin to log into any unit that is on
+# the network by equipment number, serial number, or bus number. The three parameters are:
+#
+# 1:   The path to the file containing the identifying number
+# 2:   The base port number on the remote server to add the identifying number to to get the server-side port that will 
+#    forward to port 22 (sshd) on the client side.
+# 3:   An optional parameter which if present is taken as a set of command line flags to cut to apply to the contents
+#    of the file specified by $1 to extract the numeric component (for instance, serial numbers may be in the form XYZ-1234
+#    in which case it's really the 1234 part we're after...
+#
+generate_reverse_phonehome_component()
+{
+  file="$1"
+  base="$2"
+  cut_cmdline="$3"
+
+  # Make sure the candidate dropfile exists
+  #
+  if [ -f "$file" ]; then
+
+    if [ -n "$cut_cmdline" ]; then
+
+      # Grab the desired substring
+      #
+      num="`cat $file | cut $cut_cmdline`"
+
+    else
+
+      # Grab its contents   
+      #
+      num="`cat $file`";
+
+    fi
+
+    # Make sure those contents are indeed numeric...
+    #
+    if (echo "$num" | egrep -q '^[0-9]+$'); then
+
+      # Make sure that number is within an acceptable range so as not to overflow
+      #
+      if [ "$num" -gt "0" -a "$num" -le "$REVERSE_PHONE_HOME_MAX_TOKEN" ]; then
+        echo -n " -R$((base + num)):localhost:22";
+      fi
+
+    fi
+
+  fi
+}
+
+#    This function calls the above component function for each identifying number we want to do a port forward based on...
+#
+generate_reverse_phonehome_string()
+{
+
+  # If the reverse phone home feature is disabled, return without printing any commandline args
+  #
+  if [ "$REVERSE_PHONE_HOME" -eq "0" ]; then return; fi
+
+  generate_reverse_phonehome_component $EQUIP_NUM_FILE  $REVERSE_PHONE_HOME_EQNUM_BASE
+  generate_reverse_phonehome_component $SERIAL_NUM_FILE $REVERSE_PHONE_HOME_SERIALNUM_BASE "-d- -f3"
+}
+
+
+#    This function performs teardown on a dead ssh connection, and increments the ssh connect failure counter.  If
+# that counter has reached its maximum value, we force pppd down and try a clean redial.  Otherwise, we just sleep and
+# try again.
+#
+clean_up_after_tunnel_teardown()
+{
+
+  # If the tunnel was intentionally aborted for the purpose of switching servers
+  #
+  if [ -f $TUNNEL_ABORT_DROPFILE ]; then
+
+    debug_print "SSH tunnel aborted, removing dropfiles..."
+
+    #Remove the dropfiles...
+    /bin/rm -f $TUNNEL_DROPFILE $SSH_TUNNEL_PIDFILE
+
+    #Generate new ssh target from server config dropfiles
+    generate_ssh_targets
+
+    /bin/rm -f $TUNNEL_ABORT_DROPFILE
+
+    #Reset the failure counter
+    ssh_fail_counter=0
+
+    #Sleep until it is time to try again
+    /bin/sleep $SLEEP_AFTER_TUNNEL_ABORT
+    
+  fi
+
+}
+
+while true; do
+
+  #If we've just been asked to abort the SSH tunnel
+  if [ -f $TUNNEL_ABORT_DROPFILE ]; then
+
+    #Generate new ssh target from server config dropfiles
+    generate_ssh_targets
+
+    /bin/rm -f $TUNNEL_ABORT_DROPFILE
+
+    # Reset the failure counter
+    #
+    ssh_fail_counter=0
+  fi
+
+  # If we have no active tunnel already...
+  #
+  if [ ! -f $TUNNEL_DROPFILE ]; then
+    debug_print "Attempting to establish SSH tunnel... (Attempt number $((ssh_fail_counter + 1)))"
+    s=`generate_reverse_phonehome_string`
+    debug_print ">> ssh $SSH_OPTIONS -i $SSH_IDENTITY $SSH_FORWARDS $s -p $SSH_PORT $SSH_TARGET "
+
+    # Attempt to create our tunnel... (incliding (if REVERSE_PHONE_HOME != 0) reverse phone home support
+    #
+    ssh $SSH_OPTIONS -i $SSH_IDENTITY $SSH_FORWARDS `generate_reverse_phonehome_string` -p $SSH_PORT $SSH_TARGET &
+
+    # Remember its PID
+    #
+    echo "$!" > $SSH_TUNNEL_PIDFILE
+
+    # Wait a few seconds to allow SSH negotiations
+    #
+    /bin/sleep $SLEEP_BEFORE_TUNNEL_TEST
+    debug_print -n "Testing our new tunnel..."
+
+    # Test to see if our tunnel is really up...
+    #
+    if [ "`nc localhost $HELLO_DAEMON_PORT < /dev/null`" = $HELLO_DAEMON_MESSAGE ]; then
+      debug_print " It works."
+      ssh_fail_counter=0
+
+      debug_print "Checking in with server to report net IDs and package versions... "
+      perform_post_connect_checkin
+
+      debug_print "Touching dropfile and waiting for SSH to terminate..."
+      # Touch our dropfile indicating the tunnel is up...
+      #
+      /bin/touch $TUNNEL_DROPFILE
+
+      # and wait for the the SSH client process to end
+      #
+      wait `cat $SSH_TUNNEL_PIDFILE`
+
+      # Clean Up...
+      #
+      clean_up_after_tunnel_teardown
+
+    else
+
+      debug_print " No luck..."
+      debug_print "Issuing kill to SSH client...."
+
+      # Kill the defunct and/or too slow to use SSH client
+      #
+      /bin/kill `cat $SSH_TUNNEL_PIDFILE`
+
+      # Wait for the process to terminate
+      #
+      wait `cat $SSH_TUNNEL_PIDFILE`
+
+      # Clean Up...
+      #
+      clean_up_after_tunnel_teardown
+
+    fi
+
+  else
+
+    # This means we _think_ we have an SSH tunnel, but it's not one we set up...
+    #
+    debug_print -n "We seem to already have a pre-existing tunnel... Monitoring it."
+
+    # Loop and periodically test this tunnel...  When this condition fails, we're done...
+    #
+    while [ "`nc localhost $HELLO_DAEMON_PORT < /dev/null`" = $HELLO_DAEMON_MESSAGE ]; do
+
+      # Sleep for a while before testing this tunnel again...
+      #
+      /bin/sleep $SLEEP_MONITORING_TUNNEL
+    done
+
+    # Clean Up...
+    #
+    clean_up_after_tunnel_teardown
+  fi
+
+done
+

busunit/scripts/connection_tether.sh

@@ -0,0 +1 @@
+connection_tether-ssh.sh

server/docker/.dockerignore

@@ -0,0 +1,2 @@
+*.tmp
+.tmp

server/docker/Dockerfile

@@ -9,12 +9,12 @@ RUN \
 
 RUN \
   apt-get install -y gcc perl \
-    git openssh-server sqlite3 \
+    git openssh-server sqlite3 libdbd-sqlite3-perl \
     default-mysql-server apache2 \
     netcat telnet wget
 
 RUN \
-  apt-get install -y vim
+  apt-get install -y libswitch-perl libdate-calc-perl
 
 RUN \
   useradd -m bus && \
@@ -22,11 +22,25 @@ RUN \
 
 RUN \
   su bus -c " cd /home/bus && \
-    git clone https://tree.clementinecomputing.com/clementinecomputing/popufare "
-
+    git clone https://tree.clementinecomputing.com/clementinecomputing/popufare && \
+    mkdir -p /home/bus/.ssh && chmod 0700 /home/bus/.ssh "
+
+## setup test ssh access
+##
+## WARNING: this MUST be changed for anything other than testing purposes
+##
+COPY ./snakeoil_id_rsa.pub /home/bus/.ssh/authorized_keys.snakeoil
+RUN su bus -c \
+  " cat /home/bus/.ssh/authorized_keys.snakeoil >> /home/bus/.ssh/authorized_keys && \
+    chmod 640 /home/bus/.ssh/authorized_keys "
+
+# non-essential but helpful for debuggin/admin
+#
 RUN \
-  apt-get install -y libswitch-perl libdate-calc-perl
+  apt-get install -y vim
 
+# main startup scripts
+#
 COPY ./startup_and_persist.sh /root/startup_and_persist.sh
 
 RUN \

server/docker/build_and_run.sh

@@ -9,7 +9,7 @@ export UPDATE_DAEMON_PORT="8377"
 
 docker build --no-cache -t popufare .
 
-docker run -d popufare \
+docker run -d \
   -p 8080:80 \
   -p 4430:443 \
   -p $SSH_DEFAULT_PORT:22 \
@@ -18,7 +18,7 @@ docker run -d popufare \
   -p $BILLDB_DAEMON_PORT:$BILLDB_DAEMON_PORT \
   -p $HELLO_DAEMON_PORT:$HELLO_DAEMON_PORT \
   -p $UPDATE_DAEMON_PORT:$UPDATE_DAEMON_PORT \
-  bus
+  popufare bus
 
 ## for interactive debugging...
 ##

server/docker/gen-snakeoil-ssh-keypair

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+ssh-keygen -t rsa -f snakeoil_id_rsa -N "" -C testing -q
+
+chmod a+r snakeoil_id_rsa
+

server/docker/run_interactive.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 
+#export uname="bus"
+export uname="root"
+
 export SSH_DEFAULT_PORT="6055"
 export AVLS_DAEMON_PORT="2857"
 export PASSDB_DAEMON_PORT="7277"
@@ -7,18 +10,31 @@ export BILLDB_DAEMON_PORT="2455"
 export HELLO_DAEMON_PORT="3556"
 export UPDATE_DAEMON_PORT="8377"
 
-#docker build --no-cache -t popufare .
-
 ## for interactive debugging...
 ##
-docker run  \
-  -p 8080:80 \
-  -p 4430:443 \
-  -p $SSH_DEFAULT_PORT:22 \
-  -p $AVLS_DAEMON_PORT:$AVLS_DAEMON_PORT \
-  -p $PASSDB_DAEMON_PORT:$PASSDB_DAEMON_PORT \
-  -p $BILLDB_DAEMON_PORT:$BILLDB_DAEMON_PORT \
-  -p $HELLO_DAEMON_PORT:$HELLO_DAEMON_PORT \
-  -p $UPDATE_DAEMON_PORT:$UPDATE_DAEMON_PORT \
-  --net=host -it popufare \
-  bash -c " su - bus "
+if [[ "$uname" == "root" ]] ; then
+
+  docker run  \
+    -p 8080:80 \
+    -p 4430:443 \
+    -p $SSH_DEFAULT_PORT:22 \
+    -p $AVLS_DAEMON_PORT:$AVLS_DAEMON_PORT \
+    -p $PASSDB_DAEMON_PORT:$PASSDB_DAEMON_PORT \
+    -p $BILLDB_DAEMON_PORT:$BILLDB_DAEMON_PORT \
+    -p $HELLO_DAEMON_PORT:$HELLO_DAEMON_PORT \
+    -p $UPDATE_DAEMON_PORT:$UPDATE_DAEMON_PORT \
+    -it popufare \
+    /bin/bash
+else
+  docker run  \
+    -p 8080:80 \
+    -p 4430:443 \
+    -p $SSH_DEFAULT_PORT:22 \
+    -p $AVLS_DAEMON_PORT:$AVLS_DAEMON_PORT \
+    -p $PASSDB_DAEMON_PORT:$PASSDB_DAEMON_PORT \
+    -p $BILLDB_DAEMON_PORT:$BILLDB_DAEMON_PORT \
+    -p $HELLO_DAEMON_PORT:$HELLO_DAEMON_PORT \
+    -p $UPDATE_DAEMON_PORT:$UPDATE_DAEMON_PORT \
+    -it popufare \
+    bash -c " service ssh start && su - bus "
+fi

server/docker/snakeoil_id_rsa

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEA0gwlzUCngi8nk++kRSkXELCn3hwxivfOPdLiKeQh+yqMSBzO
+08+AbOkPpMyRm4nqtgpXwqA5M0Q2sY+6KdbApih6Y0EyCubp+xeNSDioghLB37X6
+H0rpk47PS2lQiHokGDkxqUDVrix7LltFZTedZZmqvcnvDNfQ7H78DvDfEt2Q4Zqr
+OncZ2k/zOI77AGWMsOVEDSRfl0lCJx+88uHHZyR35Y9XrcQ9mr7YzBiiVJsEz2zL
+XNyWlJ/Z3aoWqbgw3n2L161im0v0dD1hiWcAljBUUexQK8vwhVBK5fEYjmM8ibaA
+6Xk/jb/hPZ2CPFlqZwGy46p6Y5AGUhDbnXJktwIDAQABAoIBAALMSKCgUNrqF+Pf
+mFeXgYgmYkYSKBLK1EN8kKiwMkevPGklgRfPNs28mQm/89yXjLnaD2s3SSvGp9fb
+q0YtZymPdnnl/22nv8wpdydUIeKBr3V0bh2GmO/vQhGmGH2I6vwrnt67T11fmFB4
+xMbhY8ZbsquOihGouk68Lk1gpggRR1OQba6763tbEfq0OBo2wRZD+Et4DCmFZJz8
+5hO5KuUTSo+9XkqwAyclkSFLNEUaQ32aU8WlnaIgmq8HnbyEoJ3FBbRtiVy25aQU
+yIHpdKSRlUPzLcq/jlE9gXxpsjVAD6JixHWTzIkeoLPfP8CqV1PwMe4xOrg/YJ8u
+/FUUmHkCgYEA7sMMtX4ujzTRvCeTqfgJ3AorwihsPhBgS74c5KTgnIyoXYahWwwP
+xR3qtUaXjV9z1wL2rlwmHihRuC/h3zTlhnUZwyMRqGNDO4Jvvy3AqSJXWIacRtBd
+4V4FL1Uf9uYqBxVsmfPBwXkdKhA81DwcejYy86QjoqC2OoQgVlhG350CgYEA4TZh
+BYXbZHG4eNGVU0BozDXPVhJdVY6qK2AQUKUGjscThbAtnG6XfIvdvAxMIhBEn1KG
+t9eLky7ysg+jCyMx7gi526JbRpupBXys6WlxZTpOVDJnR0CklTmGt00GUcO1k789
+oadYcvYXL0KBVu7ox6QxBYM3/JHZ1Zw0G68NJ2MCgYBwGo62DJfpz2AGmwJ/FKH0
+eaYQFJC2F1RPW+UPCccYWgaXWyYZrLEe2SwhjxKLY1ztgEu4lDWlhULRW1IPTjPS
+jNrav3WD/Y9MurKjLeU+xIwEWbe4nA7oSQ/gKHTIpUroZovGQSghIi+qV1NoRw0L
+6LTFprTAn/rPwMe6ItqsFQKBgFAqKx7sJsv6Ls/SynR32NXX0ZWxs8dgoPIPn78U
+/uCTvD86FwI323RZ1PO67N6zK7oKhimqb+RUMc9wAXlVbdBtCrtDb3oSrndliacM
+AXEm7/rIdjrGp872Sx20LSutnlSZHjcMek9xiY8o/SEUqYdmyoxLyQH+aKvAjkvM
+Y8JBAoGAQtQzgYfAbLUjrksm8GLFP3JpG5ZhKtiLPEWdUkxNxRfojSYW8krecOBH
+uFAJwxx8D201b24tjvJrLN5NkQQ+qsDCX8MG/x3QaJcBh1ysgefwQs+GXyT5uxCz
+t5jrLAVxnHIO4eYHngxcb1wIh4vBWs8Q3bP835J7rAjd3A1tFQo=
+-----END RSA PRIVATE KEY-----

server/docker/snakeoil_id_rsa.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDSDCXNQKeCLyeT76RFKRcQsKfeHDGK98490uIp5CH7KoxIHM7Tz4Bs6Q+kzJGbieq2ClfCoDkzRDaxj7op1sCmKHpjQTIK5un7F41IOKiCEsHftfofSumTjs9LaVCIeiQYOTGpQNWuLHsuW0VlN51lmaq9ye8M19DsfvwO8N8S3ZDhmqs6dxnaT/M4jvsAZYyw5UQNJF+XSUInH7zy4cdnJHflj1etxD2avtjMGKJUmwTPbMtc3JaUn9ndqhapuDDefYvXrWKbS/R0PWGJZwCWMFRR7FAry/CFUErl8RiOYzyJtoDpeT+Nv+E9nYI8WWpnAbLjqnpjkAZSENudcmS3 testing