connection tunnel scripts

busunit/scripts/common_values.sh

@@ -0,0 +1,282 @@
+#!/bin/bash
+#
+# Copyright (c) 2019 Clementine Computing LLC.
+# 
+# This file is part of PopuFare.
+# 
+# PopuFare is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# PopuFare is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+# 
+# You should have received a copy of the GNU Affero General Public License
+# along with PopuFare.  If not, see <https://www.gnu.org/licenses/>.
+#
+
+#THIS FILE IS NOT REALLY A SCRIPT, BUT RATHER A CENTRAL HOLDING TANK FOR SYSTEM CONFIG VALUES...
+#IT SHOULD BE SOURCED BY ALL OTHER SCRIPTS THAT NEED A CONTAINED VALUE, AND MAY SOME DAY BE PARSED
+#BY THE NATIVE APPLICATIONS.  THUS:  Keep it simple, in the following format:
+#
+##Comment on meaning of variable
+#VARIABLE="value"
+#
+#(variable name in all caps, no space, equals sign, quoted literal value (if there is even a remote chance that the value
+#is one that will be needed by the native app in the future)
+#
+
+BASEDIR="$HOME"
+
+#How long to sleep after a dial attempt before checking if we have a PPP session successfully constructed
+SLEEP_AFTER_DIAL="30"
+
+#How long to wait between redial attempts     
+SLEEP_BETWEEN_REDIALS="60"
+
+# How long to sleep after the GRPS session comes up before testing the SSH tunnel (attempting to connect to the 'hello' server)
+#
+SLEEP_BEFORE_TUNNEL_TEST="5"
+
+# How long to sleep after an established tunnel goes down before allowing any SSH tunnel establishment retries
+#
+SLEEP_AFTER_TUNNEL_FAILURE="60"
+
+# How long to sleep after an established tunnel has been aborted
+#
+SLEEP_AFTER_TUNNEL_ABORT="60"
+
+# How many successive ssh tunnel failures do we accept before forcing a hard termination of pppd and a modem reset and redail
+#
+MAX_FAIL_HANGUP="5"
+
+# How frequently to test an SSH tunnel that predates the ppp-dialer.sh process (thus having no right to wait for its PID)
+#
+SLEEP_MONITORING_TUNNEL="120"
+
+#  How frequently to perform SSH keepalive messages, and how many of them are allowed to drop before we tear the tunnel down
+# and wait to try again.  This gets passed to the SSH client in the following form:
+# -o ServerAliveInterval=$SSH_SERVER_ALIVE_INTERVAL -o ServerAliveCountMax=$SSH_SERVER_ALIVE_MAXDROP
+#
+SSH_SERVER_ALIVE_INTERVAL="15"
+SSH_SERVER_ALIVE_MAXDROP="3"
+
+# These are the default server parameters the system will use if there is not a configured sync server.
+#The system will check 
+SSH_DEFAULT_TARGET="$USER@example.com"
+SSH_DEFAULT_PORT="6055"
+SSH_DEFAULT_IDENTITY="$HOME/.ssh/id_rsa_bus"
+
+#This regular expression test is used to make sure we have a valid-looking FQDN or IP address
+SSH_TARGET_VALIDITY_CHECK="^[a-zA-Z][a-zA-Z0-9]*@([a-zA-Z0-9-]+\.)+([a-zA-Z0-9-]+)$"
+
+SSH_TARGET="$SSH_DEFAULT_TARGET"
+SSH_PORT="$SSH_DEFAULT_PORT"
+SSH_IDENTITY="$SSH_DEFAULT_IDENTITY"
+
+#--------------------------------------------------------------------------------
+#This is the list of forwarded ports we bring live when we set up our SSH tunnel:
+
+# 2857 -> AVLS
+AVLS_DAEMON_PORT="2857"
+
+# 7277 -> Bus Pass Daemon
+PASSDB_DAEMON_PORT="7277"
+
+# 2455 -> Billing Log Daemon
+BILLDB_DAEMON_PORT="2455"
+
+# 3556 -> "Hello" Daemon
+HELLO_DAEMON_PORT="3556"
+
+# 8377 -> Update / Version Server
+UPDATE_DAEMON_PORT="8377"
+
+
+#    These are used to talk to the "Hello" daemon on the server to verify that
+#actual data can flow before notifying the system that the tunnel is up.
+HELLO_DAEMON_MESSAGE="Hello."
+
+
+#--------------------------------------------------------------------------------
+#    This is a list of base ports on the server that we can use for the reverse-phone-home
+#functionality which forwards port BASE + client_id on the server to port 22 (sshd) on 
+#the client to allow for remote troubleshooting and manual configuration.
+
+#Enable reverse-phone-home feature
+REVERSE_PHONE_HOME="1"
+
+#    This is the base port (on the server) which the numbering for the reverse-phone-home
+#system starts at when numbered by equipment number:
+REVERSE_PHONE_HOME_EQNUM_BASE="10000"
+
+#    This is the base port (on the server) which the numbering for the reverse-phone-home
+#system starts at when numbered by serial number (wireless_ip on transition busses):
+REVERSE_PHONE_HOME_SERIALNUM_BASE="30000"
+
+#    This is the number above which an equipment number or other token will not be considered
+#as a phone home candidate on account of its likelyhood to overflow into the next port range.
+REVERSE_PHONE_HOME_MAX_TOKEN="19999"
+
+
+#--------------------------------------------------------------------------------
+#These paths lead to dropfiles in /tmp which are used to signal network state
+
+#This dropfile means we have an established GPRS/PPP session
+GPRS_DROPFILE="/tmp/network-is-up"
+#This dropfile means we have an established SSH tunnel to the server
+TUNNEL_DROPFILE="/tmp/tunnel-is-up"
+#This dropfile contains the process ID of our SSH client that's keeping the tunnel open
+SSH_TUNNEL_PIDFILE="/tmp/ssh_tunnel.pid"
+#This dropfile contains identifying information about our ethernet card, cell modem, and SIM
+NETWORK_ID_DROPFILE="/tmp/net_ids"
+
+#This dropfile becomes present to notify the update process to abort:
+UPDATE_ABORT_DROPFILE="/tmp/abort_update"
+TUNNEL_ABORT_DROPFILE="/tmp/abort_tunnel"
+
+
+#############################################################
+
+FIX_PACKAGE_PERMS="1"
+
+########################################################## Used only if FIX_PACKAGE_PERMS is nonzero ##############
+#This specifies who should own installed packages
+PACKAGE_OWNER_STRING="root:root"
+
+#These are the permissions applied to stuff in the bin directory
+PACKAGE_BIN_PERMISSIONS="755"
+#This is the egrep expression that must match to qualify something as a binary
+PACKAGE_BIN_PATTERN='(^[/]?'$BASEDIR'/bin/[a-z_A-Z0-9.]+$)|(.*\.sh$)'
+
+#These are the permissions applied to SSH related config files and directories
+PACKAGE_SSH_STRICT_FILE_PERMISSIONS="600"
+PACKAGE_SSH_MILD_FILE_PERMISSIONS="644"
+PACKAGE_SSH_DIR_PERMISSIONS="700"
+
+#this one is only used in its ALWAYS case
+PACKAGE_SYS_DIR_PERMISSIONS="755"
+
+#These are the egrep expressions that must match to qualify something as an SSH file or directory
+PACKAGE_SSH_STRICT_FILE_PATTERN="(^[/]?etc/ppp/id_rsa[^\.]*$)|(^[/]?root/\.ssh/id_rsa[^\.]*$)|(^[/]?root/\.ssh/authorized_keys)"
+PACKAGE_SSH_MILD_FILE_PATTERN="(^[/]?root/\.ssh/known_hosts)|(^[/]?etc/ppp/id_rsa[^\.]*.pub$)|(^[/]?root/\.ssh/id_rsa[^\.]*.pub)"
+PACKAGE_SSH_DIR_PATTERN="(^[/]?root[/]?$)|(^[/]?root/\.ssh[/]?$)"
+
+#These are the list of files to ALWAYS reset permissions on (whether or not they are in the update)
+ALWAYS_SSH_DIR_LIST="/root /root/.ssh"
+ALWAYS_SSH_STRICT_FILE_LIST="/root/.ssh/id_rsa /etc/ppp/id_rsa_bus /root/.ssh/authorized_keys $SYNC_PRIVATE_KEY"
+ALWAYS_SSH_MILD_FILE_LIST="/root/.ssh/known_hosts /root/.ssh/id_rsa.pub /etc/ppp/id_rsa_bus.pub /etc/ssh/sshd_config"
+ALWAYS_BIN_LIST="$BASEDIR/bin/init_bus.sh $BASEDIR/bin/update_loop.sh $BASEDIR/bin/apply_update.sh $BASEDIR/bin/fix_pkg_perm.sh $BASEDIR/bin/get_net_ids.sh"
+
+ALWAYS_SYS_DIR_LIST="/etc /etc/ssh /etc/ppp"
+
+DEFAULT_EXTRACT_PATH="/"
+EXTRACT_PATH_FILE="/tmp/pkg_extract_path"
+
+##################################################################################################################
+
+
+#These paths point to important bits of information for synchronizing
+EQUIP_NUM_FILE="$BASEDIR/config/equipnum.txt"
+SERIAL_NUM_FILE="$BASEDIR/config/serial_num"
+
+SERVER_CONFIG_DIR="$BASEDIR/config/server"
+
+SERVER_LIST_FILE="$SERVER_CONFIG_DIR/server_list"
+
+SYNC_DESC_FILE="$SERVER_CONFIG_DIR/sync_server_desc"
+SYNC_TARGET_FILE="$SERVER_CONFIG_DIR/sync_target"
+SYNC_PORT_FILE="$SERVER_CONFIG_DIR/sync_port"
+SYNC_KNOWN_HOSTS="/root/.ssh/known_hosts"
+SYNC_PRIVATE_KEY="/etc/ppp/id_rsa_client"
+
+###############################################################################################
+#     CONFIGURATION VALUES USED PRIMARILY BY THE UPDATE LOOP SCRIPT                           #
+###############################################################################################
+
+#How long for update_loop.sh to sleep after spawning the client supervisor before (potentially) respawning it
+SLEEP_AFTER_SPAWN="60"
+
+#How long to sleep between attempts to download patches from the server
+SLEEP_BETWEEN_UPDATES="60"
+
+#The directory in which the current patch state is to be found
+CHECKSUM_AND_VERSION_PATH="$BASEDIR/config/"
+
+###############################################################################################
+#     BELOW HERE, COMPOUND OPTIONS ARE ALLOWED (things that use variable substitution, etc...)#
+###############################################################################################
+
+#This looks for all environment variables that specify a port that we must forward...
+TUNNEL_PORT_LIST="`set | egrep "^[A-Z]+_DAEMON_PORT=" | cut -d '=' -f2 | egrep '^[0-9]+$' | xargs echo`"
+
+#This gathers the above list into something we can just hand to the SSH client...
+SSH_FORWARDS="`for port in $TUNNEL_PORT_LIST; do echo -n " -L$port:localhost:$port"; done`"
+
+#These are the flags to SSH that are specific to setting up the tunnel
+SSH_TUNNEL_FLAGS="-n -N -x"
+
+#These are the flags to SSH that are specific to our connection to the target host
+SSH_TARGET_OPTIONS="-o StrictHostKeyChecking=no"
+
+#These are the flags to SSH that specify the behavior of the protocol-level keepalive messages
+SSH_KEEPALIVE_OPTIONS="-o ServerAliveInterval=$SSH_SERVER_ALIVE_INTERVAL -o ServerAliveCountMax=$SSH_SERVER_ALIVE_MAXDROP"
+
+SSH_OPTIONS="$SSH_TUNNEL_FLAGS $SSH_KEEPALIVE_OPTIONS $SSH_TARGET_OPTIONS"
+SCP_OPTIONS="$SSH_KEEPALIVE_OPTIONS $SSH_TARGET_OPTIONS"
+
+# This function kills any currently open SSH or SCP sessions and puts dropfiles in /tmp telling
+#ppp-dialer.sh and update_loop.sh to refresh their Sync server parameters with the generate_ssh_targets
+#function before attempting to reconnect. 
+#
+abort_tunnel_and_update() {
+
+  touch $UPDATE_ABORT_DROPFILE
+  touch $TUNNEL_ABORT_DROPFILE
+
+  sync
+  sleep 1
+
+  killall ssh
+  killall scp
+}
+
+# This function uses the dropfiles left by unpack_server_data.sh and constructs new SSH and SCP
+#parameters suitable to connect to the specified server.  If there is no server specified, or if the
+#specified server does not pass a sanity check then the default server will be used.
+#
+function generate_ssh_targets
+{
+
+  if [ -f $SYNC_TARGET_FILE ]; then
+
+    alt_target="`cat $SYNC_TARGET_FILE`"
+    if echo "$alt_target" | egrep -q "$SSH_TARGET_VALIDITY_CHECK"; then
+
+      SSH_TARGET="`cat $SYNC_TARGET_FILE`"
+      SSH_PORT="`cat $SYNC_PORT_FILE`"
+      SSH_IDENTITY="$SYNC_PRIVATE_KEY"
+
+      echo "$0: Using sync server $SSH_TARGET port $SSH_PORT"
+    else
+
+      SSH_TARGET="$SSH_DEFAULT_SYNC_TARGET"
+      SSH_PORT="$SSH_DEFAULT_SYNC_PORT"
+      SSH_IDENTITY="$SSH_DEFAULT_IDENTITY"
+
+      echo "$0: Specified sync server name $alt_target is malformed.  Using default $SSH_DEFAULT_SYNC_TARGET port $SSH_DEFAULT_SYNC_PORT"
+    fi
+  else
+
+      SSH_TARGET="$SSH_DEFAULT_TARGET"
+      SSH_PORT="$SSH_DEFAULT_PORT"
+      SSH_IDENTITY="$SSH_DEFAULT_IDENTITY"
+
+      echo "$0: No sync server selected. Using default $SSH_DEFAULT_TARGET port $SSH_DEFAULT_PORT"
+  fi
+
+}
+

busunit/scripts/connection_tether-ppp.sh

@@ -0,0 +1,317 @@
+#!/bin/bash
+#
+# Copyright (c) 2019 Clementine Computing LLC.
+#
+# This file is part of PopuFare.
+#
+# PopuFare is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# PopuFare is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with PopuFare.  If not, see <https://www.gnu.org/licenses/>.
+#
+
+. $HOME/bin/common_values.sh 
+
+# Do this once at boot time, but do it again after a tunnel abort request...
+#
+generate_ssh_targets
+
+ssh_fail_counter=0
+
+debug_print()
+{
+  echo $@
+}
+
+#    Every once in a while pppd will report connection, and still have a ppp session open to the modem, but
+# the router on the other end will have croaked, or the underlying GPRS connection will be too unreliable, or
+# sometimes even a pppd will hang such that it needs a SIGKILL to get rid of it...  This function does that
+# and then cleans up after the dead pppd by removing its dropfiles and locks.
+#
+force_kill_pppd()
+{
+  /usr/bin/killall pppd
+  rm -rf /var/lock/*ttyGPRS
+  rm -rf $GPRS_DROPFILE
+}
+
+#    This function goes and looks at the version dropfiles for packages, gathering their package names
+# and versions to report when we check in with the server to log that we connected, and at what firmware and
+# config revision.
+#
+output_versions()
+{
+  for file in `ls $CHECKSUM_AND_VERSION_PATH/*.version`; do
+    echo -n " `echo $file | sed -r 's/^.*\/(.*)\.version$/\1/'`=`cat $file`";
+  done
+}
+
+# This function extracts a field from the network ID dropfile by name (also used by the checkin process)
+#
+output_net_ids_field()
+{
+  field="$1"
+
+  cat $NETWORK_ID_DROPFILE | grep "$field" | cut -d'=' -f2 | xargs -n1 echo -n
+}
+
+#    This function attempts to check in with the version server and report hardware and network serial numbers
+# so that us sysadmin types can see each time a unit attached to the network, which unit it was, and what software
+# it was running at the time.
+#
+perform_post_connect_checkin()
+{
+  if [ -f $SERIAL_NUM_FILE ]; then
+    busunitnum="`cat $SERIAL_NUM_FILE 2> /dev/null`"
+  fi
+
+  equipnum="`cat $EQUIP_NUM_FILE 2> /dev/null || echo 0`"
+
+  version="`output_versions`"
+  imei="`output_net_ids_field IMEI`"
+  imsi="`output_net_ids_field IMSI`"
+  mac="`output_net_ids_field ETH0`"
+
+  #     Send these gathered data to the update daemon.  The leading '#' tells the server that this is a 
+  #checkin, not an update request.
+  echo -e "#$busunitnum\t$equipnum\t$mac\t$imei\t$imsi\t$version" | nc localhost $UPDATE_DAEMON_PORT
+}
+
+#    This function generates the server->client port forwards to allow a sysadmin to log into any unit that is on
+# the network by equipment number, serial number, or bus number. The three parameters are:
+#
+# 1:   The path to the file containing the identifying number
+# 2:   The base port number on the remote server to add the identifying number to to get the server-side port that will 
+#    forward to port 22 (sshd) on the client side.
+# 3:   An optional parameter which if present is taken as a set of command line flags to cut to apply to the contents
+#    of the file specified by $1 to extract the numeric component (for instance, serial numbers may be in the form XYZ-1234
+#    in which case it's really the 1234 part we're after...
+#
+generate_reverse_phonehome_component()
+{
+  file="$1"
+  base="$2"
+  cut_cmdline="$3"
+
+  #Make sure the candidate dropfile exists
+  if [ -f "$file" ]; then
+
+    if [ -n "$cut_cmdline" ]; then
+      #Grab the desired substring
+      num="`cat $file | cut $cut_cmdline`"
+    else
+      #Grab its contents   
+      num="`cat $file`";
+    fi
+
+    #Make sure those contents are indeed numeric...
+    if (echo "$num" | egrep -q '^[0-9]+$'); then
+
+      #Make sure that number is within an acceptable range so as not to overflow
+      if [ "$num" -gt "0" -a "$num" -le "$REVERSE_PHONE_HOME_MAX_TOKEN" ]; then
+        echo -n " -R$((base + num)):localhost:22";
+      fi
+
+    fi
+
+  fi
+}
+
+#    This function calls the above component function for each identifying number we want to do a port forward based on...
+#
+generate_reverse_phonehome_string()
+{
+  #If the reverse phone home feature is disabled, return without printing any commandline args
+  if [ "$REVERSE_PHONE_HOME" -eq "0" ]; then return; fi
+
+  generate_reverse_phonehome_component $EQUIP_NUM_FILE  $REVERSE_PHONE_HOME_EQNUM_BASE
+  generate_reverse_phonehome_component $SERIAL_NUM_FILE $REVERSE_PHONE_HOME_SERIALNUM_BASE "-d- -f3"
+}
+
+
+#    This function performs teardown on a dead ssh connection, and increments the ssh connect failure counter.  If
+# that counter has reached its maximum value, we force pppd down and try a clean redial.  Otherwise, we just sleep and
+# try again.
+#
+clean_up_after_tunnel_teardown()
+{
+  #If the tunnel was intentionally aborted for the purpose of switching servers
+  if [ -f $TUNNEL_ABORT_DROPFILE ]; then
+
+    debug_print "SSH tunnel aborted, removing dropfiles..."
+
+    #Remove the dropfiles...
+    /bin/rm -f $TUNNEL_DROPFILE $SSH_TUNNEL_PIDFILE
+
+    #Generate new ssh target from server config dropfiles
+    generate_ssh_targets
+
+    /bin/rm -f $TUNNEL_ABORT_DROPFILE
+
+    #Reset the failure counter
+    ssh_fail_counter=0
+
+    #Sleep until it is time to try again
+    /bin/sleep $SLEEP_AFTER_TUNNEL_ABORT
+
+    
+  else
+    #OTHERWISE, we assume that the modem lost signal, or the router or remote server went wonky...
+
+    debug_print "SSH client dead, removing dropfiles..."
+    #Remove the dropfiles...
+    /bin/rm -f $TUNNEL_DROPFILE $SSH_TUNNEL_PIDFILE
+
+    ssh_fail_counter=$((ssh_fail_counter + 1))
+
+    if [ "$ssh_fail_counter" -ge "$MAX_FAIL_HANGUP" ]; then
+      debug_print "pppd claims to be up; tunnel failed $ssh_fail_counter times, killing pppd to force redial."
+      force_kill_pppd
+    fi
+
+    debug_print "Sleeping before any retry attempts..."
+    #Sleep before trying this again...
+    /bin/sleep $SLEEP_AFTER_TUNNEL_FAILURE
+  
+  fi
+}
+
+while true; do
+
+  # If our GRPS connection has died...
+  #
+  if [ ! -f $GPRS_DROPFILE ]; then
+    ssh_fail_counter=0
+    debug_print "Attempting to dial..."
+
+    # Try to re-"dial" our ISP 
+    #
+    /usr/bin/pon gprs
+
+    # Give the modem a minute to do its thing...
+    #
+    /usr/bin/pon gprs
+
+    # Give the modem a minute to do its thing...
+    #
+    /bin/sleep $SLEEP_AFTER_DIAL
+
+  fi
+
+  # If we've just been asked to abort the SSH tunnel
+  #
+  if [ -f $TUNNEL_ABORT_DROPFILE ]; then
+
+    # Generate new ssh target from server config dropfiles
+    #
+    generate_ssh_targets
+
+    /bin/rm -f $TUNNEL_ABORT_DROPFILE
+
+    # Reset the failure counter
+    #
+    ssh_fail_counter=0
+  fi
+
+  # If we now have an active GPRS network connection, try and bring a tunnel up...
+  #
+  if [ -f $GPRS_DROPFILE ]; then
+
+    # If we have no active tunnel already...
+    #
+    if [ ! -f $TUNNEL_DROPFILE ]; then
+      debug_print "Attempting to establish SSH tunnel... (Attempt number $((ssh_fail_counter + 1)))"
+
+      # Attempt to create our tunnel... (incliding (if REVERSE_PHONE_HOME != 0) reverse phone home support
+      #
+      ssh $SSH_OPTIONS -i $SSH_IDENTITY $SSH_FORWARDS `generate_reverse_phonehome_string` -p $SSH_PORT $SSH_TARGET &
+
+      # Remember its PID
+      #
+      echo "$!" > $SSH_TUNNEL_PIDFILE
+
+      # Wait a few seconds to allow SSH negotiations
+      #
+      /bin/sleep $SLEEP_BEFORE_TUNNEL_TEST
+      debug_print -n "Testing our new tunnel..."
+
+      # Test to see if our tunnel is really up...
+      #
+      if [ "`nc localhost $HELLO_DAEMON_PORT < /dev/null`" = $HELLO_DAEMON_MESSAGE ]; then
+        debug_print " It works."
+        ssh_fail_counter=0
+
+        debug_print "Checking in with server to report net IDs and package versions... "
+        perform_post_connect_checkin
+
+        debug_print "Touching dropfile and waiting for SSH to terminate..."
+
+        # Touch our dropfile indicating the tunnel is up...
+        #
+        /bin/touch $TUNNEL_DROPFILE
+
+        # and wait for the the SSH client process to end
+        #
+        wait `cat $SSH_TUNNEL_PIDFILE`
+
+        # Clean Up...
+        #
+        clean_up_after_tunnel_teardown
+
+      else
+        debug_print " No luck..."
+        debug_print "Issuing kill to SSH client...."
+
+        # Kill the defunct and/or too slow to use SSH client
+        #
+        /bin/kill `cat $SSH_TUNNEL_PIDFILE`
+
+        # Wait for the process to terminate
+        #
+        wait `cat $SSH_TUNNEL_PIDFILE`
+
+        # Clean Up...
+        #
+        clean_up_after_tunnel_teardown
+      fi
+
+    else
+
+      # This means we _think_ we have an SSH tunnel, but it's not one we set up...
+      #
+      debug_print -n "We seem to already have a pre-existing tunnel... Monitoring it."
+
+      # Loop and periodically test this tunnel...  When this condition fails, we're done...
+      #
+      while [ "`nc localhost $HELLO_DAEMON_PORT < /dev/null`" = $HELLO_DAEMON_MESSAGE ]; do
+
+        # Sleep for a while before testing this tunnel again...
+        #
+        /bin/sleep $SLEEP_MONITORING_TUNNEL
+
+      done
+
+      # Clean Up...
+      #
+      clean_up_after_tunnel_teardown
+    fi
+
+  else
+
+    # If we don't have an active GPRS session, that means we just failed at dialing
+    #
+    debug_print "Dialing failed...  Sleeping"
+    /bin/sleep $SLEEP_BETWEEN_REDIALS
+
+  fi
+
+done
+

busunit/scripts/connection_tether-ssh.sh

@@ -0,0 +1,268 @@
+#!/bin/bash
+#
+# Copyright (c) 2019 Clementine Computing LLC.
+#
+# This file is part of PopuFare.
+#
+# PopuFare is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# PopuFare is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with PopuFare.  If not, see <https://www.gnu.org/licenses/>.
+#
+
+. $HOME/bin/common_values.sh 
+
+# Do this once at boot time, but do it again after a tunnel abort request...
+#
+generate_ssh_targets
+
+ssh_fail_counter=0
+
+debug_print()
+{
+  echo $@
+}
+
+#    This function goes and looks at the version dropfiles for packages, gathering their package names
+# and versions to report when we check in with the server to log that we connected, and at what firmware and
+# config revision.
+#
+output_versions()
+{
+  for file in `ls $CHECKSUM_AND_VERSION_PATH/*.version`; do
+    echo -n " `echo $file | sed -r 's/^.*\/(.*)\.version$/\1/'`=`cat $file`";
+  done
+}
+
+# This function extracts a field from the network ID dropfile by name (also used by the checkin process)
+#
+output_net_ids_field()
+{
+  field="$1"
+
+  cat $NETWORK_ID_DROPFILE | grep "$field" | cut -d'=' -f2 | xargs -n1 echo -n
+}
+
+#    This function attempts to check in with the version server and report hardware and network serial numbers
+# so that us sysadmin types can see each time a unit attached to the network, which unit it was, and what software
+# it was running at the time.
+#
+perform_post_connect_checkin()
+{
+  if [ -f $SERIAL_NUM_FILE ]; then
+    busunitnum="`cat $SERIAL_NUM_FILE 2> /dev/null`"
+  fi
+
+  equipnum="`cat $EQUIP_NUM_FILE 2> /dev/null || echo 0`"
+
+  version="`output_versions`"
+  imei="`output_net_ids_field IMEI`"
+  imsi="`output_net_ids_field IMSI`"
+  mac="`output_net_ids_field ETH0`"
+
+  #     Send these gathered data to the update daemon.  The leading '#' tells the server that this is a 
+  # checkin, not an update request.
+  #
+  echo -e "#$busunitnum\t$equipnum\t$mac\t$imei\t$imsi\t$version" | nc localhost $UPDATE_DAEMON_PORT
+}
+
+#    This function generates the server->client port forwards to allow a sysadmin to log into any unit that is on
+# the network by equipment number, serial number, or bus number. The three parameters are:
+#
+# 1:   The path to the file containing the identifying number
+# 2:   The base port number on the remote server to add the identifying number to to get the server-side port that will 
+#    forward to port 22 (sshd) on the client side.
+# 3:   An optional parameter which if present is taken as a set of command line flags to cut to apply to the contents
+#    of the file specified by $1 to extract the numeric component (for instance, serial numbers may be in the form XYZ-1234
+#    in which case it's really the 1234 part we're after...
+#
+generate_reverse_phonehome_component()
+{
+  file="$1"
+  base="$2"
+  cut_cmdline="$3"
+
+  # Make sure the candidate dropfile exists
+  #
+  if [ -f "$file" ]; then
+
+    if [ -n "$cut_cmdline" ]; then
+
+      # Grab the desired substring
+      #
+      num="`cat $file | cut $cut_cmdline`"
+
+    else
+
+      # Grab its contents   
+      #
+      num="`cat $file`";
+
+    fi
+
+    # Make sure those contents are indeed numeric...
+    #
+    if (echo "$num" | egrep -q '^[0-9]+$'); then
+
+      # Make sure that number is within an acceptable range so as not to overflow
+      #
+      if [ "$num" -gt "0" -a "$num" -le "$REVERSE_PHONE_HOME_MAX_TOKEN" ]; then
+        echo -n " -R$((base + num)):localhost:22";
+      fi
+
+    fi
+
+  fi
+}
+
+#    This function calls the above component function for each identifying number we want to do a port forward based on...
+#
+generate_reverse_phonehome_string()
+{
+
+  # If the reverse phone home feature is disabled, return without printing any commandline args
+  #
+  if [ "$REVERSE_PHONE_HOME" -eq "0" ]; then return; fi
+
+  generate_reverse_phonehome_component $EQUIP_NUM_FILE  $REVERSE_PHONE_HOME_EQNUM_BASE
+  generate_reverse_phonehome_component $SERIAL_NUM_FILE $REVERSE_PHONE_HOME_SERIALNUM_BASE "-d- -f3"
+}
+
+
+#    This function performs teardown on a dead ssh connection, and increments the ssh connect failure counter.  If
+# that counter has reached its maximum value, we force pppd down and try a clean redial.  Otherwise, we just sleep and
+# try again.
+#
+clean_up_after_tunnel_teardown()
+{
+
+  # If the tunnel was intentionally aborted for the purpose of switching servers
+  #
+  if [ -f $TUNNEL_ABORT_DROPFILE ]; then
+
+    debug_print "SSH tunnel aborted, removing dropfiles..."
+
+    #Remove the dropfiles...
+    /bin/rm -f $TUNNEL_DROPFILE $SSH_TUNNEL_PIDFILE
+
+    #Generate new ssh target from server config dropfiles
+    generate_ssh_targets
+
+    /bin/rm -f $TUNNEL_ABORT_DROPFILE
+
+    #Reset the failure counter
+    ssh_fail_counter=0
+
+    #Sleep until it is time to try again
+    /bin/sleep $SLEEP_AFTER_TUNNEL_ABORT
+    
+  fi
+
+}
+
+while true; do
+
+  #If we've just been asked to abort the SSH tunnel
+  if [ -f $TUNNEL_ABORT_DROPFILE ]; then
+
+    #Generate new ssh target from server config dropfiles
+    generate_ssh_targets
+
+    /bin/rm -f $TUNNEL_ABORT_DROPFILE
+
+    # Reset the failure counter
+    #
+    ssh_fail_counter=0
+  fi
+
+  # If we have no active tunnel already...
+  #
+  if [ ! -f $TUNNEL_DROPFILE ]; then
+    debug_print "Attempting to establish SSH tunnel... (Attempt number $((ssh_fail_counter + 1)))"
+    s=`generate_reverse_phonehome_string`
+    debug_print ">> ssh $SSH_OPTIONS -i $SSH_IDENTITY $SSH_FORWARDS $s -p $SSH_PORT $SSH_TARGET "
+
+    # Attempt to create our tunnel... (incliding (if REVERSE_PHONE_HOME != 0) reverse phone home support
+    #
+    ssh $SSH_OPTIONS -i $SSH_IDENTITY $SSH_FORWARDS `generate_reverse_phonehome_string` -p $SSH_PORT $SSH_TARGET &
+
+    # Remember its PID
+    #
+    echo "$!" > $SSH_TUNNEL_PIDFILE
+
+    # Wait a few seconds to allow SSH negotiations
+    #
+    /bin/sleep $SLEEP_BEFORE_TUNNEL_TEST
+    debug_print -n "Testing our new tunnel..."
+
+    # Test to see if our tunnel is really up...
+    #
+    if [ "`nc localhost $HELLO_DAEMON_PORT < /dev/null`" = $HELLO_DAEMON_MESSAGE ]; then
+      debug_print " It works."
+      ssh_fail_counter=0
+
+      debug_print "Checking in with server to report net IDs and package versions... "
+      perform_post_connect_checkin
+
+      debug_print "Touching dropfile and waiting for SSH to terminate..."
+      # Touch our dropfile indicating the tunnel is up...
+      #
+      /bin/touch $TUNNEL_DROPFILE
+
+      # and wait for the the SSH client process to end
+      #
+      wait `cat $SSH_TUNNEL_PIDFILE`
+
+      # Clean Up...
+      #
+      clean_up_after_tunnel_teardown
+
+    else
+
+      debug_print " No luck..."
+      debug_print "Issuing kill to SSH client...."
+
+      # Kill the defunct and/or too slow to use SSH client
+      #
+      /bin/kill `cat $SSH_TUNNEL_PIDFILE`
+
+      # Wait for the process to terminate
+      #
+      wait `cat $SSH_TUNNEL_PIDFILE`
+
+      # Clean Up...
+      #
+      clean_up_after_tunnel_teardown
+
+    fi
+
+  else
+
+    # This means we _think_ we have an SSH tunnel, but it's not one we set up...
+    #
+    debug_print -n "We seem to already have a pre-existing tunnel... Monitoring it."
+
+    # Loop and periodically test this tunnel...  When this condition fails, we're done...
+    #
+    while [ "`nc localhost $HELLO_DAEMON_PORT < /dev/null`" = $HELLO_DAEMON_MESSAGE ]; do
+
+      # Sleep for a while before testing this tunnel again...
+      #
+      /bin/sleep $SLEEP_MONITORING_TUNNEL
+    done
+
+    # Clean Up...
+    #
+    clean_up_after_tunnel_teardown
+  fi
+
+done
+

busunit/scripts/connection_tether.sh

@@ -0,0 +1 @@
+connection_tether-ssh.sh