popufare/busunit/scripts/connection_tether-ppp.sh

#!/bin/bash
#
# Copyright (c) 2019 Clementine Computing LLC.
#
# This file is part of PopuFare.
#
# PopuFare is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PopuFare is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with PopuFare.  If not, see <https://www.gnu.org/licenses/>.
#

. $HOME/bin/common_values.sh 

# Do this once at boot time, but do it again after a tunnel abort request...
#
generate_ssh_targets

ssh_fail_counter=0

debug_print()
{
  echo $@
}

#    Every once in a while pppd will report connection, and still have a ppp session open to the modem, but
# the router on the other end will have croaked, or the underlying GPRS connection will be too unreliable, or
# sometimes even a pppd will hang such that it needs a SIGKILL to get rid of it...  This function does that
# and then cleans up after the dead pppd by removing its dropfiles and locks.
#
force_kill_pppd()
{
  /usr/bin/killall pppd
  rm -rf /var/lock/*ttyGPRS
  rm -rf $GPRS_DROPFILE
}

#    This function goes and looks at the version dropfiles for packages, gathering their package names
# and versions to report when we check in with the server to log that we connected, and at what firmware and
# config revision.
#
output_versions()
{
  for file in `ls $CHECKSUM_AND_VERSION_PATH/*.version`; do
    echo -n " `echo $file | sed -r 's/^.*\/(.*)\.version$/\1/'`=`cat $file`";
  done
}

# This function extracts a field from the network ID dropfile by name (also used by the checkin process)
#
output_net_ids_field()
{
  field="$1"

  cat $NETWORK_ID_DROPFILE | grep "$field" | cut -d'=' -f2 | xargs -n1 echo -n
}

#    This function attempts to check in with the version server and report hardware and network serial numbers
# so that us sysadmin types can see each time a unit attached to the network, which unit it was, and what software
# it was running at the time.
#
perform_post_connect_checkin()
{
  if [ -f $SERIAL_NUM_FILE ]; then
    busunitnum="`cat $SERIAL_NUM_FILE 2> /dev/null`"
  fi

  equipnum="`cat $EQUIP_NUM_FILE 2> /dev/null || echo 0`"

  version="`output_versions`"
  imei="`output_net_ids_field IMEI`"
  imsi="`output_net_ids_field IMSI`"
  mac="`output_net_ids_field ETH0`"

  #     Send these gathered data to the update daemon.  The leading '#' tells the server that this is a 
  #checkin, not an update request.
  echo -e "#$busunitnum\t$equipnum\t$mac\t$imei\t$imsi\t$version" | nc localhost $UPDATE_DAEMON_PORT
}

#    This function generates the server->client port forwards to allow a sysadmin to log into any unit that is on
# the network by equipment number, serial number, or bus number. The three parameters are:
#
# 1:   The path to the file containing the identifying number
# 2:   The base port number on the remote server to add the identifying number to to get the server-side port that will 
#    forward to port 22 (sshd) on the client side.
# 3:   An optional parameter which if present is taken as a set of command line flags to cut to apply to the contents
#    of the file specified by $1 to extract the numeric component (for instance, serial numbers may be in the form XYZ-1234
#    in which case it's really the 1234 part we're after...
#
generate_reverse_phonehome_component()
{
  file="$1"
  base="$2"
  cut_cmdline="$3"

  #Make sure the candidate dropfile exists
  if [ -f "$file" ]; then

    if [ -n "$cut_cmdline" ]; then
      #Grab the desired substring
      num="`cat $file | cut $cut_cmdline`"
    else
      #Grab its contents   
      num="`cat $file`";
    fi

    #Make sure those contents are indeed numeric...
    if (echo "$num" | egrep -q '^[0-9]+$'); then

      #Make sure that number is within an acceptable range so as not to overflow
      if [ "$num" -gt "0" -a "$num" -le "$REVERSE_PHONE_HOME_MAX_TOKEN" ]; then
        echo -n " -R$((base + num)):localhost:22";
      fi

    fi

  fi
}

#    This function calls the above component function for each identifying number we want to do a port forward based on...
#
generate_reverse_phonehome_string()
{
  #If the reverse phone home feature is disabled, return without printing any commandline args
  if [ "$REVERSE_PHONE_HOME" -eq "0" ]; then return; fi

  generate_reverse_phonehome_component $EQUIP_NUM_FILE  $REVERSE_PHONE_HOME_EQNUM_BASE
  generate_reverse_phonehome_component $SERIAL_NUM_FILE $REVERSE_PHONE_HOME_SERIALNUM_BASE "-d- -f3"
}


#    This function performs teardown on a dead ssh connection, and increments the ssh connect failure counter.  If
# that counter has reached its maximum value, we force pppd down and try a clean redial.  Otherwise, we just sleep and
# try again.
#
clean_up_after_tunnel_teardown()
{
  #If the tunnel was intentionally aborted for the purpose of switching servers
  if [ -f $TUNNEL_ABORT_DROPFILE ]; then

    debug_print "SSH tunnel aborted, removing dropfiles..."

    #Remove the dropfiles...
    /bin/rm -f $TUNNEL_DROPFILE $SSH_TUNNEL_PIDFILE

    #Generate new ssh target from server config dropfiles
    generate_ssh_targets

    /bin/rm -f $TUNNEL_ABORT_DROPFILE

    #Reset the failure counter
    ssh_fail_counter=0

    #Sleep until it is time to try again
    /bin/sleep $SLEEP_AFTER_TUNNEL_ABORT

    
  else
    #OTHERWISE, we assume that the modem lost signal, or the router or remote server went wonky...

    debug_print "SSH client dead, removing dropfiles..."
    #Remove the dropfiles...
    /bin/rm -f $TUNNEL_DROPFILE $SSH_TUNNEL_PIDFILE

    ssh_fail_counter=$((ssh_fail_counter + 1))

    if [ "$ssh_fail_counter" -ge "$MAX_FAIL_HANGUP" ]; then
      debug_print "pppd claims to be up; tunnel failed $ssh_fail_counter times, killing pppd to force redial."
      force_kill_pppd
    fi

    debug_print "Sleeping before any retry attempts..."
    #Sleep before trying this again...
    /bin/sleep $SLEEP_AFTER_TUNNEL_FAILURE
  
  fi
}

while true; do

  # If our GRPS connection has died...
  #
  if [ ! -f $GPRS_DROPFILE ]; then
    ssh_fail_counter=0
    debug_print "Attempting to dial..."

    # Try to re-"dial" our ISP 
    #
    /usr/bin/pon gprs

    # Give the modem a minute to do its thing...
    #
    /usr/bin/pon gprs

    # Give the modem a minute to do its thing...
    #
    /bin/sleep $SLEEP_AFTER_DIAL

  fi

  # If we've just been asked to abort the SSH tunnel
  #
  if [ -f $TUNNEL_ABORT_DROPFILE ]; then

    # Generate new ssh target from server config dropfiles
    #
    generate_ssh_targets

    /bin/rm -f $TUNNEL_ABORT_DROPFILE

    # Reset the failure counter
    #
    ssh_fail_counter=0
  fi

  # If we now have an active GPRS network connection, try and bring a tunnel up...
  #
  if [ -f $GPRS_DROPFILE ]; then

    # If we have no active tunnel already...
    #
    if [ ! -f $TUNNEL_DROPFILE ]; then
      debug_print "Attempting to establish SSH tunnel... (Attempt number $((ssh_fail_counter + 1)))"

      # Attempt to create our tunnel... (incliding (if REVERSE_PHONE_HOME != 0) reverse phone home support
      #
      ssh $SSH_OPTIONS -i $SSH_IDENTITY $SSH_FORWARDS `generate_reverse_phonehome_string` -p $SSH_PORT $SSH_TARGET &

      # Remember its PID
      #
      echo "$!" > $SSH_TUNNEL_PIDFILE

      # Wait a few seconds to allow SSH negotiations
      #
      /bin/sleep $SLEEP_BEFORE_TUNNEL_TEST
      debug_print -n "Testing our new tunnel..."

      # Test to see if our tunnel is really up...
      #
      if [ "`nc localhost $HELLO_DAEMON_PORT < /dev/null`" = $HELLO_DAEMON_MESSAGE ]; then
        debug_print " It works."
        ssh_fail_counter=0

        debug_print "Checking in with server to report net IDs and package versions... "
        perform_post_connect_checkin

        debug_print "Touching dropfile and waiting for SSH to terminate..."

        # Touch our dropfile indicating the tunnel is up...
        #
        /bin/touch $TUNNEL_DROPFILE

        # and wait for the the SSH client process to end
        #
        wait `cat $SSH_TUNNEL_PIDFILE`

        # Clean Up...
        #
        clean_up_after_tunnel_teardown

      else
        debug_print " No luck..."
        debug_print "Issuing kill to SSH client...."

        # Kill the defunct and/or too slow to use SSH client
        #
        /bin/kill `cat $SSH_TUNNEL_PIDFILE`

        # Wait for the process to terminate
        #
        wait `cat $SSH_TUNNEL_PIDFILE`

        # Clean Up...
        #
        clean_up_after_tunnel_teardown
      fi

    else

      # This means we _think_ we have an SSH tunnel, but it's not one we set up...
      #
      debug_print -n "We seem to already have a pre-existing tunnel... Monitoring it."

      # Loop and periodically test this tunnel...  When this condition fails, we're done...
      #
      while [ "`nc localhost $HELLO_DAEMON_PORT < /dev/null`" = $HELLO_DAEMON_MESSAGE ]; do

        # Sleep for a while before testing this tunnel again...
        #
        /bin/sleep $SLEEP_MONITORING_TUNNEL

      done

      # Clean Up...
      #
      clean_up_after_tunnel_teardown
    fi

  else

    # If we don't have an active GPRS session, that means we just failed at dialing
    #
    debug_print "Dialing failed...  Sleeping"
    /bin/sleep $SLEEP_BETWEEN_REDIALS

  fi

done