#!/usr/bin/perl
#H# Kevin's rsync based backup system version 0.21
#H# 
#H# This is a backup system that uses rsync as the primary tool.
#H# 
#H# command line switches:
#H# name		backup name (this is a directory containing configuration files)
#H# -v|--verbose	be verbose
#H# 
#H# See the included readme.txt file for more information including the format
#H# of the configuration directories and files.

# default settings
$BackupRoot="undefined";
$BackupScripts=`pwd`;
chomp $BackupScripts;
$BackupSettings="$BackupScripts/settings";
$DefaultNumberOldBackups=30;
$HumanReadableOutput="yes";
$BackupACLs="no";
$BackupXATTRs="no";
$UseNetworkCompression="no";
$UseSSHIdentyFile="none";
$ForceChecksum="no";
$BackupIsFAT="no";
$BackupIsNAS="no";
$CompensateForFAT="no";
$UseFilterFiles="no";
$TransferWholeFiles="no";
$UpdateFilesInPlace="yes";
$WriteFilesSparsely="no";
$IgnoreHardLinks="no";
$ArchiveMethod="link-dest";
$ExtraRsyncParams="";
$HostLevelSubvolumes="yes";
$Verbose="no";
$PurgeOnly="no";
$BackupName="error";
$RmParams="-f";
$MvParams="-f";
$BTRFSRedir=">/dev/null";
$LinkDest="";

# parse command line
foreach $Arg (@ARGV) {
  chomp $Arg;
  if ($Arg eq "-v" || $Arg eq "--verbose") { $Verbose="yes"; }
  if ($Arg =~ /^\w/) { $BackupName=$Arg; }
  if ($Arg eq "-h" || $Arg eq "--help" || $Arg eq "help") {
    open (SELF, $0);
    while (<SELF>) {
      if ($_ =~ /^#H# /) {
        $_ =~ s/^#H# //;
	print $_;
      } # if help line
    } # while help input
    exit
  } # if asking for help
} # foreach args

# setup initial rsync parameters (these will be heavily modified by settings later
$RsyncParams="--archive --one-file-system --hard-links --human-readable --inplace --numeric-ids --delete --delete-excluded";
if ($Verbose eq "yes") {
  $RsyncParams="$RsyncParams --verbose --itemize-changes --progress";
  $RmParams="$RmParams -v";
  $MvParams="$MvParams -v";
  $BTRFSRedir="";
  print "Running backup named $BackupName.\n";
} # if verbose

# make sure a backup name was specified
if ($BackupName eq "error") {
  print "Error: Cannot find backup name on command line.\n";
  exec ("$0 --help");
} # if error

# verify that settings exist
$Self=$0;
chomp $Self;
$Self =~ s/.*\///;
$Self = "$BackupScripts/$Self";
open (SELF, $Self) || die "ERROR: Cannot find backup scripts ($Self).\n";
close (SELF);
if (-d "$BackupSettings")  { } else {
  die "ERROR: Cannot find backup settings ($BackupSettings)\n";
} # if else backup settings exists

# verify the named backup
if (-d "$BackupScripts/$BackupName") {
  if (-f "$BackupScripts/$BackupName/backuptab") {
  } else {
    die "ERROR: No backuptab exists for $BackupName.\n";
  }
} else {
  die "ERROR: No backup named $BackupName exists.\n";
}

# read settings from files
if (-f "$BackupSettings/settings.pl") { do "$BackupSettings/settings.pl"; }
if (-f "$BackupScripts/$BackupName/settings.pl") { do "$BackupScripts/$BackupName/settings.pl"; }

# verify that settings are still sane...
if (-d "$BackupRoot") { } else { die "ERROR: The Backup Root ($BackupRoot) does not exist.\n"; }
if ($UseNetworkCompression eq "yes") { $RsyncParams="$RsyncParams --compress"; }
if ($HumanReadableOutput eq "no") { $RsyncParams =~ s/--human-readable//g; }
if ($BackupACLs eq "yes") { $RsyncParams = "$RsyncParams --acls"; }
if ($BackupXATTRs eq "yes") { $RsyncParams = "$RsyncParams --xattrs"; }
if ($UseSSHIdentyFile ne "none") {
  if (-f "$UseSSHIdentyFile") {
    $RsyncParams="$RsyncParams -e 'ssh -i $UseSSHIdentyFile'";
  } else {
    die "ERROR: You specified an ssh identity file that does not exist ($UseSSHIdentityFile).\n";
  }
}

if ($BackupIsNAS eq "yes") {
  # Optimize for writing backups to an NFS mount.  Note that this is a really inneficient thing to do.
  $TransferWholeFiles="yes";
  $WriteFilesSparsely="no";
  $UpdateFilesInPlace="no";
}

if ($TransferWholeFiles eq "yes") { $RsyncParams= "$RsyncParams --whole-file"; }
if ($WriteFilesSparsely eq "yes") {
  $RsyncParams="$RsyncParams --sparse";
  $UpdateFilesInPlace="no";
}
if ($UpdateFilesInPlace eq "no") { $RsyncParams =~ s/--inplace//g; }
if ($IgnoreHardLinks eq "yes") { $RsyncParams =~ s/--hard-links//g; }
if ($ForceChecksum eq "yes") { $RsyncParams="$RsyncParams --checksum"; }
if ($BackupIsFAT eq "yes") {
  print "Notice: The FAT filesystem is extremely limited.\n";
  print "The FAT filesystem does not support any archiving method, file ownerships,\n";
  print "hard links, symbolic links, or consistent timestamps.\n";
  print "Therefore it is inappropriate for use as a backup device.\n";
  print "The FAT filesystem is only useful for mirroring other FAT filesystems\n";
  print "which is beyond the scope of this program.\n";
  die "ERROR: Attempted to backup to FAT filesystem.\n";
}
if ($CompensateForFAT eq "yes") {
  # we are backing up a FAT filesystem which has unstable timestamps
  $RsyncParams="$RsyncParams --modify-window=3602";
}
if ($UseFilterFiles eq "yes") { $RsyncParams="$RsyncParams -F"; }
if ($ArchiveMethod ne "link-dest") {
  if ($ArchiveMethod ne "zfs") {
    if ($ArchiveMethod ne "btrfs") {
      die "ERROR: Unknown archive method ($ArchiveMethod) specified.\n";
    }
  }
}
if ($ExtraRsyncParams ne "") { $RsyncParams="$ExtraRsyncParams $RsyncParams"; }

# add in include and exclude files...
if (-f "$BackupSettings/includes") { $RsyncParams="$RsyncParams --include-from=$BackupSettings/includes"; }
if (-f "$BackupSettings/excludes") { $RsyncParams="$RsyncParams --exclude-from=$BackupSettings/excludes"; }
if (-f "$BackupScripts/$BackupName/includes") { $RsyncParams="$RsyncParams --include-from=$BackupScripts/$BackupName/includes"; }
if (-f "$BackupScripts/$BackupName/excludes") { $RsyncParams="$RsyncParams --exclude-from=$BackupScripts/$BackupName/excludes"; }

# gather information about other systems.
# This allows you to get things like partition tables that are not stored in files.
if (-f "$BackupScripts/$BackupName/infotab") {
  if ($Verbose eq "yes") {
    system ("getinfo.pl $BackupScripts/$BackupName/infotab --verbose");
  } else {
    system ("getinfo.pl $BackupScripts/$BackupName/infotab");
  }
}

# time to do some backups
open (BACKUPTAB, "$BackupScripts/$BackupName/backuptab");
while (<BACKUPTAB>) {
  $TabLine=$_;
  chomp $TabLine;
  $TabLine =~ s/#.*//;
  if ($TabLine !~ /^$/) {
    $Host=$TabLine;
    $Host =~ s/:.*//g;
    $BackupString=$TabLine;
    $BackupString =~ s/$Host://;
    if ($BackupString =~ /^\//) {
      # I see some kind of path to backup
      ($SourcePath, $NumArchivesWanted)=split (/:/,$BackupString,2);
      if ($NumArchivesWanted eq "d") { $NumArchivesWanted=$DefaultNumberOldBackups; }
      if ($SourcePath eq "//") {
        # I see the special case //
	$RsyncParams =~ s/--one-file-system//g;
	$SourcePath="/";
      } # if //
      if ($SourcePath eq "/*") {
        # I see the special case /*
	# The smart way to handle the special case * backup is to generate a temporary
	# backup based on the output of df then rerun this script using that backup.
	$TempBackupName=`mktemp --tmpdir=. --directory`;
	chomp $TempBackupName;
	open (DIRLIST, "ssh $Host df -lTP |");
	open (TEMPBACKUPTAB, "> $TempBackupName/backuptab");
	while (<DIRLIST>) {
	  $DFLine=$_;
	  chomp $DFLine;
	  if ($DFLine =~ /^\//) {
	    if ($DFLine !~ /tmpfs/) {
	      if ($DFLine !~ /iso9660/) {
	        if ($DFLine !~ /cd9660/) {
		  if ($DFLine !~ /squashfs/) {
		    @DF=split ($DFLine);
		    foreach $Df (@DF) {
		      $FS=$Df;
		    }
		    chomp $FS;
		    print (TEMPBACKUPTAB "$Host:$FS:$NumArchivesWanted\n");
		  }
		}
              }
	    }
	  }
	}
	close (DIRLIST);
	close (TEMPBACKUPTAB);
	$Params="";
	if ($Verbose eq "yes") { $Params="$Params --verbose"; }
	system ("$0 $Params $TempBackupName");
	system ("rm -rf $TempBackupName");
      # end of /*
      } else {
        # I have a regular path (or // has become /)
	$LocalTestHostName=$Host;
	$LocalTestHostName =~ s/\..*//;
	$RemoteTestHostName=`ssh -x $Host hostname -s`;
	chomp $RemoteTestHostName;
	if ($LocalTestHostName ne $RemoteTestHostName) {
	  print "WARNING: Skipping $SourcePath on $Host because I cannot verify ssh access to $Host ($LocalTestHostName ne $RemoteTestHostName).\n";
	} else {
	  # do the archive
	  if ($ArchiveMethod eq "zfs") { die "ZFS snapshot archiving not yet implamented.\n"; }
	  # determine the backup path with _ instead of /
	  # FSP = Fixed Source Path
	  $FSP=$SourcePath;
	  $FSP =~ s/\//_/g;
	  $Date=`date +'%Y-%m-%d.%H-%M-%S'`;
	  chomp $Date;
	  $FFSP="$FSP.$Date";
	  $FQFSP="$BackupRoot/$Host/$FFSP";
	  # determine current backup
	  $CurrentBackup="none";
	  if (-l "$BackupRoot/$Host/$FSP.current") {
	    $CurrentBackup=`readlink $BackupRoot/$Host/$FSP.current`;
	    chomp $CurrentBackup;
	    $CurrentBackup =~ s/.*\///g;
	  }
	  # handle special case 0
	  if ($NumArchivesWanted == 0) {
	    # not archiving anything
	    if ($CurrentBackup ne "none") {
	      if ($ArchiveMethod eq "link-dest") {
	        # if a backup already exists we just rename it
	        system ("rm $RmParams $BackupRoot/$Host/$FSP.current");
	        system ("mv $MvParams $BackupRoot/$Host/$CurrentBackup $BackupRoot/$Host/$FSP.incomplete");
              }
	    } else {
	      print "No existing backup found for $Host:$SourcePath.  Making a new backup.\n";
	      if ($ArchiveMethod eq "link-dest") {
	        system ("mkdir -p $BackupRoot/$Host/$FSP.incomplete");
	      } elsif ($ArchiveMethod eq "btrfs") {
	        if (-d "$BackupRoot/$Host") { }
		else {
		  if ($HostLevelSubvolumes eq "yes") {
		    system ("btrfs subvolume create $BackupRoot/$Host $BTRFSRedir")==0
		      or die ("ERROR: Error creating btrfs subvolume $BackupRoot/$Host ($?).\n");
		  } else {
		    system ("mkdir $BackupRoot/$Host");
		  }
		}
		if (-d "$BackupRoot/$Host/$FSP") { }
		else {
		  system ("btrfs subvolume create $BackupRoot/$Host/$FSP $BTRFSRedir")==0
		    or die ("Error creating btrfs subvolume $BackupRoot/$Host/$FSP ($?).\n");
		}
              }
	    }
	  } else {
	    # determine if old backups exist
	    if ($CurrentBackup eq "none") {
	      if ($Verbose eq "yes") { print "No existing backups for $Host:$SourcePath.  Making a new one.\n"; }
	      if ($ArchiveMethod eq "link-dest" ) {
	        system ("mkdir -p $BackupRoot/$Host/$FSP.incomplete");
              } elsif ($ArchiveMethod eq "btrfs") {
                if (-d "$BackupRoot/$Host") { }
                else {
                  if ($HostLevelSubvolumes eq "yes") {
                    system ("btrfs subvolume create $BackupRoot/$Host $BTRFSRedir")==0
		      or die ("ERROR: Error creating btrfs subvolume $BackupRoot/$Host ($?).\n");
                  } else {
                    system ("mkdir $BackupRoot/$Host");
                  }
                }
                if (-d "$BackupRoot/$Host/$FSP") { }
                else {
                  system ("btrfs subvolume create $BackupRoot/$Host/$FSP $BTRFSRedir")==0
		    or die ("Error creating btrfs subvolume $BackupRoot/$Host/$FSP ($?).\n");
                }
              }
	    } else {
	      # determine how many existing backups there are
	      @ BackupList=();
	      open (BACKUPLIST, "ls -d $BackupRoot/$Host/$FSP.* |");
	      while (<BACKUPLIST>) {
	        $Data=$_;
	        chomp $Data;
	        if ($Data !~ /complete$/) {
	          push (@BackupList,$Data);
	        }
	      }
	      close (BACKUPLIST);
	      sort (@BackupList);
	      $NumOldBackups=$#BackupList;
	      if ($Verbose eq "yes") { print "There are $NumOldBackups old backups of $Host:$SourcePath ($NumArchivesWanted are wanted).\n"; }
	      #$NumOldBackups++;
	      if ($NumOldBackups > $NumArchivesWanted) {
	        $TooManyBackups=$NumOldBackups-$NumArchivesWanted;
		if ($Verbose eq "yes") { print "There are $TooManyBackups too many old backups that need to be purged.\n"; }
		$DeleteCount=0;

		while ($DeleteCount != $TooManyBackups) {
		  if ($BackupList[$DeleteCount] =~ /.ToBePurged$/) {
		    if ($Verbose eq "yes") { print "$BackupList[$DeleteCount] is already tagged for purging.  Are you not using the Purge program?\n"; }
		  } else {
		    if ($Verbose eq "yes") { print "Purging $BackupList[$DeleteCount]...\n"; }
		    if ($ArchiveMethod eq "link-dest" || $ArchiveMethod eq "btrfs") {
                      $TempDeleteName="$BackupList[$DeleteCount].ToBePurged";
		      system ("mv $MvParams $BackupList[$DeleteCount] $TempDeleteName");
		    #} elsif ($ArchiveMethod eq "btrfs") {
		      #system ("btrfs subvolume delete $BackupList[$DeleteCount] $BTRFSRedir")==0
		        #or print ("Warning: unable to purge old subvolume $BackupList[$DeleteCount] ($?).\n");
		    }
		  }
		  $DeleteCount++;
		}

              }
	      # setup archive from current (link-dest or cp)
	      if ($ArchiveMethod eq "link-dest") {
	        if ($Verbose eq "yes") { print "Linking to old backup $CurrentBackup for new backup.\n"; }
		system ("mkdir -p $BackupRoot/$Host/$FSP.incomplete");
		$LinkDest="--link-dest=$BackupRoot/$Host/$CurrentBackup";
	      }
	      if ($ArchiveMethod eq "cp") {
	        if ($Verbose eq "yes") { print "Making duplicate of $CurrentBackup for the new backup\n"; }
		system ("mkdir -p $BackupRoot/$Host/$FSP.incomplete");
	        # this is the time consuming part that makes the cp method suck.
	        system ("cp -al $BackupRoot/$Host/$Current $BackupRoot/$Host/$FSP.incomplete")==0
		  or die ("ERROR: cp -al failed ($?).\n");
	      }
            }
          }
          # do the backup
          if ($ArchiveMethod eq "link-dest" || $ArchiveMethod eq "cp") {
	    $BackupTarget="$FSP.incomplete";
	  } else {
	    $BackupTarget="$FSP";
	  }
          if ($Verbose eq "yes") {
	    print ("Running: rsync $RsyncParams $Host:$SourcePath/ $BackupRoot/$Host/$BackupTarget/\n");
	  }
	  system ("rsync $RsyncParams $LinkDest $Host:$SourcePath/ $BackupRoot/$Host/$BackupTarget/") == 0
            or print "WARNING: There was a problem backing up $Host:$SourcePath ($?).\n";
          if ($? == 65280) { die "ALERT: Rsync was aborted (^C).\n"; }
          if ($? == 65281) { die "ALERT: Rsync was aborted (^C).\n"; }
	  if ($ArchiveMethod eq "link-dest" || $ArchiveMethod eq "cp") {
	    system ("mv $MvParams $BackupRoot/$Host/$FSP.incomplete $FQFSP");
	  } else {
	    system ("btrfs subvolume snapshot $BackupRoot/$Host/$FSP $BackupRoot/$Host/$FSP.$Date $BTRFSRedir")==0
	      or die ("ERROR snapshotting btrfs subvolume $BackupRoot/$Host/$FSP to $BackupRoot/$Host/$FSP.$Date ($?).\n");
	  }
	  system ("rm -f $BackupRoot/$Host/$FSP.current");
	  system ("ln -sf $FFSP $BackupRoot/$Host/$FSP.current");
        }
      }
    }
    if ($BackupString =~ /^r!/) {
      # I see a remote command to run
      $RemoteCommand=$BackupString;
      $RemoteCommand =~ s/^r!//;
      system (qq[ssh $Host "$RemoteCommand"]);
    }
    if ($BackupString =~ /^l!/) {
      # I see a local command to run
      $LocalCommand=$BackupString;
      $LocalCommand =~ s/^l!//;
      system ("$LocalCommand")==0
        or print "Warning: local command failed ($?).\n";
    }
  }
}