~dricottone/my-utils

96238abdd70e2f5452e423295e20fb14437a95d0 — Dominic Ricottone 2 years ago 1fe9fb5 
Big archival update

mktar now experimentally supports SHA256 checksums and age encryption.

mktar-batch is a version of mktar that strips out the 'clever' algorithm
detection and includes a batch processing/common passphrase mechanism.

Updated tests for mktar.
patch browse .tar.gz 
5 files changed, 507 insertions(+), 43 deletions(-)

M .gitignore
M Makefile
M archives/mktar
A archives/mktar-batch
M archives/test_mktar_tarcat.sh

M .gitignore => .gitignore +1 -2
@@ 1,2 1,1 @@
tests/static
tests/temp_*
**/test/


M Makefile => Makefile +8 -8
@@ 30,14 30,14 @@ uninstall:
	$(call subdir_make,sound,uninstall)

test: clean
	$(call subdir_make,core,clean)
	$(call subdir_make,archives,clean)
	$(call subdir_make,containers,clean)
	$(call subdir_make,display,clean)
	$(call subdir_make,emulation,clean)
	$(call subdir_make,games,clean)
	$(call subdir_make,network,clean)
	$(call subdir_make,sound,clean)
	$(call subdir_make,core,test)
	$(call subdir_make,archives,test)
	$(call subdir_make,containers,test)
	$(call subdir_make,display,test)
	$(call subdir_make,emulation,test)
	$(call subdir_make,games,test)
	$(call subdir_make,network,test)
	$(call subdir_make,sound,test)

clean:
	$(call subdir_make,core,clean)


M archives/mktar => archives/mktar +151 -27
@@ 1,15 1,17 @@
#!/bin/bash

name="mktar"
version="1.2"
version="1.3"
read -r -d '' help_message <<-EOF
	Wrapper around 'tar' for easier compression
	Usage: mktar FILES [..] [OPTIONS]
	Archive utility
	Usage: mktar [OPTIONS] INFILES
	Options:
	 -c, --compress        compress archive with gzip
	 --compress=ALGO       compress archive with [none|gzip|xz|zstd|bzip2]
	 -C, --checksum        create checksum with SHA1
	 --checksum=ALGO       create checksum with [none|sha1|sha256]
	 -e, --encrypt         encrypt files with GPG
	 --encrypt=ALGO        encrypt files with [none|gpg|age]
	 -h, --help            print this message
	 -n FILE, --name FILE  name of backup file (Default: archive.tar)
	 -q, --quiet           suppress error messages and prompts


@@ 29,6 31,12 @@ archive_fn=""
while [[ $# -gt 0 ]]; do
  case $1 in

  --compress=gz|--compress=gzip|--compress=1)
    debug_msg "Setting compress option to 1 (=gzip) (was ${compress})"
    compress=1
    shift
    ;;

  --compress=xz|--compress=2)
    debug_msg "Setting compress option to 2 (=xz) (was ${compress})"
    compress=2


@@ 47,12 55,6 @@ while [[ $# -gt 0 ]]; do
    shift
    ;;

  --compress=gz|--compress=gzip|--compress=1)
    debug_msg "Setting compress option to 1 (=gzip) (was ${compress})"
    compress=1
    shift
    ;;

  --compress=no|--compress=none|--compress=0)
    debug_msg "Setting compress option to 0 (=none) (was ${compress})"
    compress=0


@@ 70,12 72,58 @@ while [[ $# -gt 0 ]]; do
    shift
    ;;

  --checksum=sha1|--checksum=1)
    debug_msg "Setting checksum option to 1 (=sha1) (was ${checksum})"
    checksum=1
    shift
    ;;

  --checksum=sha256|--checksum=2)
    debug_msg "Setting checksum option to 2 (=sha256) (was ${checksum})"
    checksum=2
    shift
    ;;

  --checksum=no|--checksum=none|--checksum=0)
    debug_msg "Setting checksum option to 0 (=none) (was ${checksum})"
    checksum=0
    shift
    ;;

  --checksum=*)
    attempted_checksum="$(/usr/bin/printf "%s\n" "$1" | sed -e 's/^.*=//' )"
    error_msg "Unknown checksum '${attempted_checksum}'"
    ;;

  -C|--checksum)
    debug_msg "Setting checksum option to 1 (=SHA1) (was ${checksum})"
    checksum=1
    shift
    ;;

  --encrypt=gpg|--encrypt=1)
    debug_msg "Setting encrypt option to 1 (=GPG) (was ${encrypt})"
    encrypt=1
    shift
    ;;

  --encrypt=age|--encrypt=2)
    debug_msg "Setting encrypt option to 2 (=Age) (was ${encrypt})"
    encrypt=2
    shift
    ;;

  --encrypt=no|--encrypt=none|--encrypt=0)
    debug_msg "Setting encrypt option to 0 (=none) (was ${encrypt})"
    encrypt=0
    shift
    ;;

  --encrypt=*)
    attempted_encryption="$(/usr/bin/printf "%s\n" "$1" | sed -e 's/^.*=//' )"
    error_msg "Unknown encryption '${attempted_encryption}'"
    ;;

  -e|--encrypt)
    debug_msg "Setting encrypt option to 1 (=GPG) (was ${encrypt})"
    encrypt=1


@@ 123,7 171,7 @@ if [[ "${#positional[@]}" -lt 1 ]]; then
  usage_msg
fi

# determine tar action
# determine compress action
if [[ "$compress" -eq 1 ]]; then
  archive_action="tar.gz"
elif [[ "$compress" -eq 2 ]]; then


@@ 132,59 180,95 @@ elif [[ "$compress" -eq 3 ]]; then
  archive_action="tar.zst"
elif [[ "$compress" -eq 4 ]]; then
  archive_action="tar.bz2"
elif [[ -n "$archive_fn" && "$compress" -eq -1 ]]; then
elif [[ "$compress" -eq -1 ]]; then
  debug_msg "No explicit compress option; parsing archive name..."
  case "$archive_fn" in
  *.tar)
    debug_msg "Detected tarball (no compress)"
    archive_action="tar"
    ;;

  *.tar.gz)
    debug_msg "Detected gzip compress"
    archive_action="tar.gz"
    ;;

  *.tar.xz)
    debug_msg "Detected xz compress"
    archive_action="tar.xz"
    ;;

  *.tar.zst)
    debug_msg "Detected zstandard compress"
    archive_action="tar.zst"
    ;;

  *.tar.bz2)
    debug_msg "Detected bz2 compress"
    archive_action="tar.bz2"
    ;;

  *)
    debug_msg "Could not parse archive name; defaulting to no compress"
    archive_action="tar"
    ;;
  esac
else
  archive_action="tar"
fi

# determine encrypt action
if [[ "$encrypt" -eq 1 ]]; then
  archive_action="${archive_action}.gpg"
elif [[ -n "$archive_fn" && "$encrypt" -eq -1 ]]; then
elif [[ "$encrypt" -eq 2 ]]; then
  archive_action="${archive_action}.age"
elif [[ "$encrypt" -eq -1 ]]; then
  debug_msg "No explicit encrypt option; parsing archive name..."
  case "$archive_fn" in
  *.gpg)
    debug_msg "Detected gpg encrypt"
    archive_action="${archive_action}.gpg"
    ;;

  *.age)
    debug_msg "Detected age encrypt"
    archive_action="${archive_action}.age"
    ;;

  *)
    debug_msg "No explicit encrypt option and could not parse archive name; defaulting to no encrypt"
    archive_action="${archive_action}"
    ;;
  esac
else
  archive_action="${archive_action}"
fi

# determine checksum action
if [[ "$checksum" -eq 1 ]]; then
  checksum_action="sha1"
elif [[ "$checksum" -eq 2 ]]; then
  checksum_action="sha256"
fi

# output filename
if [[ -z "$archive_fn" ]]; then
  archive_fn="archive.${archive_action}"
  checksum_fn="archive.sha1"
  checksum_fn="archive.${checksum_action}"
  debug_msg "No output filename was given, defaulting to '${archive_fn}'"
else
  checksum_fn="$(fn_basename "$archive_fn").sha1"
  checksum_fn="$(fn_basename "$archive_fn").${checksum_action}"
fi

# check files
if contains "$archive_fn" "${positional[@]}"; then
  error_msg "Output file cannot also be input file"
elif [[ "$checksum" -eq 1 ]] && contains "$checksum_fn" "${positional[@]}"; then
  error_msg "Output file cannot also be input file"
  error_msg "Output file cannot also be input file ('${archive_fn}')"
elif [[ "$checksum" -ge 1 ]] && contains "$checksum_fn" "${positional[@]}"; then
  error_msg "Output file cannot also be input file ('${checksum_fn}')"
fi
if ! prompt_overwrite "$archive_fn"; then
  exit 1
elif [[ "$checksum" -eq 1 ]] && ! prompt_overwrite "$checksum_fn"; then
elif [[ "$checksum" -ge 1 ]] && ! prompt_overwrite "$checksum_fn"; then
  exit 1
fi
for target in "${positional[@]}"; do


@@ 203,7 287,13 @@ tar)
  ;;

tar.gpg)
  if ! /usr/bin/tar -c "${positional[@]}" | /usr/bin/gpg -c -o "$archive_fn"; then
  if ! /usr/bin/tar -c "${positional[@]}" | /usr/bin/gpg --symmetric --output "$archive_fn"; then
    code=1
  fi
  ;;

tar.age)
  if ! /usr/bin/tar -c "${positional[@]}" | /usr/bin/age --encrypt --passphrase --output "$archive_fn"; then
    code=1
  fi
  ;;


@@ 215,7 305,13 @@ tar.gz)
  ;;

tar.gz.gpg)
  if ! /usr/bin/tar -cz "${positional[@]}" | /usr/bin/gpg -c -o "$archive_fn"; then
  if ! /usr/bin/tar -cz "${positional[@]}" | /usr/bin/gpg --symmetric --output "$archive_fn"; then
    code=1
  fi
  ;;

tar.gz.age)
  if ! /usr/bin/tar -cz "${positional[@]}" | /usr/bin/age --encrypt --passphrase --output "$archive_fn"; then
    code=1
  fi
  ;;


@@ 227,7 323,13 @@ tar.xz)
  ;;

tar.xz.gpg)
  if ! /usr/bin/tar -cJ "${positional[@]}" | /usr/bin/gpg -c -o "$archive_fn"; then
  if ! /usr/bin/tar -cJ "${positional[@]}" | /usr/bin/gpg --symmetric --output "$archive_fn"; then
    code=1
  fi
  ;;

tar.xz.age)
  if ! /usr/bin/tar -cJ "${positional[@]}" | /usr/bin/age --encrypt --passphrase --output "$archive_fn"; then
    code=1
  fi
  ;;


@@ 239,7 341,13 @@ tar.zst)
  ;;

tar.zst.gpg)
  if ! /usr/bin/tar --zstd -c "${positional[@]}" | /usr/bin/gpg -c -o "$archive_fn"; then
  if ! /usr/bin/tar --zstd -c "${positional[@]}" | /usr/bin/gpg --symmetric --output "$archive_fn"; then
    code=1
  fi
  ;;

tar.zst.age)
  if ! /usr/bin/tar --zstd -c "${positional[@]}" | /usr/bin/age --encrypt --passphrase --output "$archive_fn"; then
    code=1
  fi
  ;;


@@ 251,18 359,34 @@ tar.bz2)
  ;;

tar.bz2.gpg)
  if ! /usr/bin/tar -cj "${positional[@]}" | /usr/bin/gpg -c -o "$archive_fn"; then
  if ! /usr/bin/tar -cj "${positional[@]}" | /usr/bin/gpg --symmetric --output "$archive_fn"; then
    code=1
  fi
  ;;

tar.bz2.age)
  if ! /usr/bin/tar -cj "${positional[@]}" | /usr/bin/age --encrypt --passphrase --output "$archive_fn"; then
    code=1
  fi
  ;;
esac

# checksum routine
if [[ "$checksum" -eq 1 ]]; then
if [[ "$checksum" -ge 1 ]]; then
  if [[ -f "$archive_fn" ]]; then
    if ! /usr/bin/sha1sum "$archive_fn" | /usr/bin/awk '{print $1}' > "$checksum_fn"; then
      code=1
    fi
    case "$checksum_action" in
    sha1)
      if ! /usr/bin/sha1sum "$archive_fn" | /usr/bin/awk '{print $1}' > "$checksum_fn"; then
        code=1
      fi
      ;;

    sha256)
      if ! /usr/bin/sha256sum "$archive_fn" | /usr/bin/awk '{print $1}' > "$checksum_fn"; then
        code=1
      fi
      ;;
    esac
  else
    error_msg "No such file '${archive_fn}'"
  fi


A archives/mktar-batch => archives/mktar-batch +318 -0
@@ 0,0 1,318 @@
#!/bin/bash

name="mktar-batch"
version="1.0"
read -r -d '' help_message <<-EOF
	Archive utility for scripting
	Usage: mktar-batch [--compress=ALGO] [--encrypt=ALSO --passphrase PASSWD] [--checksum=ALGO] [--name OUTFILE] INFILES
	Options:
	 --compress=ALGO                 compress archive with [none|gzip|xz|zstd|bzip2]
	 --checksum=ALGO                 create checksum with [none|sha1|sha256]
	 --encrypt=ALGO                  encrypt files with [none|gpg|age]
	 -h, --help                      print this message
	 -n FILE, --name FILE            name of backup file (Default: archive.tar)
	 -p PASSWD, --passphrase PASSWD  passphase for encryption
	 -v, --verbose                   show additional messages
	 -V, --version                   print version number and exit
EOF

source /usr/local/lib/mylib.bash

positional=()
verbose=0
compress=-1
encrypt=-1
checksum=-1
archive_fn=""
passphrase=""
while [[ $# -gt 0 ]]; do
  case $1 in

  --compress=no|--compress=none|--compress=0)
    debug_msg "Setting compress option to 0 (=none)"
    compress=0
    shift
    ;;

  --compress=gz|--compress=gzip|--compress=1)
    debug_msg "Setting compress option to 1 (=gzip)"
    compress=1
    shift
    ;;

  --compress=xz|--compress=2)
    debug_msg "Setting compress option to 2 (=xz)"
    compress=2
    shift
    ;;

  --compress=zst|--compress=zstd|--compress=3)
    debug_msg "Setting compress option to 3 (=zstd)"
    compress=3
    shift
    ;;

  --compress=bz2|--compress=bzip2|--compress=4)
    debug_msg "Setting compress option to 4 (=bzip2))"
    compress=4
    shift
    ;;

  --compress=*)
    attempted_compression="$(/usr/bin/printf "%s\n" "$1" | sed -e 's/^.*=//' )"
    error_msg "Bad compress option '${attempted_compression}'"
    ;;

  --checksum=no|--checksum=none|--checksum=0)
    debug_msg "Setting checksum option to 0 (=none)"
    checksum=0
    shift
    ;;

  --checksum=sha1|--checksum=1)
    debug_msg "Setting checksum option to 1 (=sha1)"
    checksum=1
    shift
    ;;

  --checksum=sha256|--checksum=2)
    debug_msg "Setting checksum option to 2 (=sha256)"
    checksum=2
    shift
    ;;

  --checksum=*)
    attempted_checksum="$(/usr/bin/printf "%s\n" "$1" | sed -e 's/^.*=//' )"
    error_msg "Bad checksum option '${attempted_checksum}'"
    ;;

  --encrypt=no|--encrypt=none|--encrypt=0)
    debug_msg "Setting encrypt option to 0 (=none)"
    encrypt=0
    shift
    ;;

  --encrypt=gpg|--encrypt=1)
    debug_msg "Setting encrypt option to 1 (=gpg)"
    encrypt=1
    shift
    ;;

  --encrypt=age|--encrypt=2)
    debug_msg "Setting encrypt option to 2 (=age)"
    encrypt=2
    shift
    ;;

  --encrypt=*)
    attempted_encryption="$(/usr/bin/printf "%s\n" "$1" | sed -e 's/^.*=//' )"
    error_msg "Bad encrypt option '${attempted_encryption}'"
    ;;

  -h|--help)
    help_msg
    shift
    ;;

  -n|--name)
    debug_msg "Setting output filename to ${2}"
    archive_fn="$2"
    shift; shift
    ;;

  -p|--passphrase)
    passphrase="$2"
    shift; shift
    ;;

  -v|--verbose)
    verbose=1
    shift
    ;;

  -V|--version)
    version_msg
    ;;

  *)
    debug_msg "Argument '${1}' added to positional array"
    positional+=("$1")
    shift
    ;;
  esac
done

# error if no input or output filenames given
if [[ "${#positional[@]}" -lt 1 ]]; then
  error_msg "No input filenames given"
elif [[ -z "$archive_fn" ]]; then
  error_msg "No output filename given"
fi

# construct archive action
if [[ "$compress" -eq 1 ]]; then
  archive_action="tar.gz"
elif [[ "$compress" -eq 2 ]]; then
  archive_action="tar.xz"
elif [[ "$compress" -eq 3 ]]; then
  archive_action="tar.zst"
elif [[ "$compress" -eq 4 ]]; then
  archive_action="tar.bz2"
else
  archive_action="tar"
fi
if [[ "$encrypt" -eq 1 ]]; then
  archive_action="${archive_action}.gpg"
elif [[ "$encrypt" -eq 2 ]]; then
  archive_action="${archive_action}.age"
fi

# construct checksum action
if [[ "$checksum" -eq 1 ]]; then
  checksum_action="sha1"
elif [[ "$checksum" -eq 2 ]]; then
  checksum_action="sha256"
fi

# construct checksum filename
checksum_fn="$(fn_basename "$archive_fn").${checksum_action}"

# check files
if contains "$archive_fn" "${positional[@]}"; then
  error_msg "Output file cannot also be input file ('${archive_fn}')"
elif [[ "$checksum" -ge 1 ]] && contains "$checksum_fn" "${positional[@]}"; then
  error_msg "Output file cannot also be input file ('${checksum_fn}')"
elif [[ -f "$archive_fn" ]]; then
  error_msg "Output file already exists ('${archive_fn}')"
elif [[ "$checksum" -ge 1 && -f "$checksum_fn" ]]; then
  error_msg "Output file already exists ('${checksum_fn}')"
fi
for target in "${positional[@]}"; do
  if [[ ! -f "$target" ]] && [[ ! -d "$target" ]]; then
    error_msg "No such file '${target}'"
  fi
done

# check passphrase
if [[ "$checksum" -ge 1 && -z "$passphrase" ]]; then
  error_msg "No passphrase given"
fi

# main routine
code=0
case "$archive_action" in
tar)
  if ! /usr/bin/tar -cf "$archive_fn" "${positional[@]}"; then
    code=1
  fi
  ;;

tar.gpg)
  if ! /usr/bin/tar -c "${positional[@]}" | /usr/bin/gpg --symmetric --batch --passphrase "$passphrase" --output "$archive_fn"; then
    code=1
  fi
  ;;

tar.age)
  if ! /usr/bin/tar -c "${positional[@]}" | /usr/bin/age --encrypt --passphrase --output "$archive_fn"; then
    code=1
  fi
  ;;

tar.gz)
  if ! /usr/bin/tar -czf "$archive_fn" "${positional[@]}"; then
    code=1
  fi
  ;;

tar.gz.gpg)
  if ! /usr/bin/tar -cz "${positional[@]}" | /usr/bin/gpg --symmetric --batch --passphrase "$passphrase" --output "$archive_fn"; then
    code=1
  fi
  ;;

tar.gz.age)
  if ! /usr/bin/tar -cz "${positional[@]}" | /usr/bin/age --encrypt --passphrase --output "$archive_fn"; then
    code=1
  fi
  ;;

tar.xz)
  if ! /usr/bin/tar -cJf "$archive_fn" "${positional[@]}"; then
    code=1
  fi
  ;;

tar.xz.gpg)
  if ! /usr/bin/tar -cJ "${positional[@]}" | /usr/bin/gpg --symmetric --batch --passphrase "$passphrase" --output "$archive_fn"; then
    code=1
  fi
  ;;

tar.xz.age)
  if ! /usr/bin/tar -cJ "${positional[@]}" | /usr/bin/age --encrypt --passphrase --output "$archive_fn"; then
    code=1
  fi
  ;;

tar.zst)
  if ! /usr/bin/tar --zstd -cf "$archive_fn" "${positional[@]}"; then
    code=1
  fi
  ;;

tar.zst.gpg)
  if ! /usr/bin/tar --zstd -c "${positional[@]}" | /usr/bin/gpg --symmetric --batch --passphrase "$passphrase" --output "$archive_fn"; then
    code=1
  fi
  ;;

tar.zst.age)
  if ! /usr/bin/tar --zstd -c "${positional[@]}" | /usr/bin/age --encrypt --passphrase --output "$archive_fn"; then
    code=1
  fi
  ;;

tar.bz2)
  if ! /usr/bin/tar -cjf "$archive_fn" "${positional[@]}"; then
    code=1
  fi
  ;;

tar.bz2.gpg)
  if ! /usr/bin/tar -cj "${positional[@]}" | /usr/bin/gpg --symmetric --batch --passphrase "$passphrase" --output "$archive_fn"; then
    code=1
  fi
  ;;

tar.bz2.age)
  if ! /usr/bin/tar -cj "${positional[@]}" | /usr/bin/age --encrypt --passphrase --output "$archive_fn"; then
    code=1
  fi
  ;;
esac

# checksum routine
if [[ "$checksum" -ge 1 ]]; then
  if [[ -f "$archive_fn" ]]; then
    case "$checksum_action" in
    sha1)
      if ! /usr/bin/sha1sum "$archive_fn" | /usr/bin/awk '{print $1}' > "$checksum_fn"; then
        code=1
      fi
      ;;

    sha256)
      if ! /usr/bin/sha256sum "$archive_fn" | /usr/bin/awk '{print $1}' > "$checksum_fn"; then
        code=1
      fi
      ;;
    esac
  else
    error_msg "No such file '${archive_fn}'"
  fi
fi

# return stored code
exit "$code"



M archives/test_mktar_tarcat.sh => archives/test_mktar_tarcat.sh +29 -6
@@ 35,25 35,48 @@ test_compression() {
  esac

  # try basic compression
  ../mktar compression_target.txt --compress="$arg_compression"
  ../mktar --compress="$arg_compression" compression_target.txt
  ../tarcat "$fn_compressed" > compression_result.txt
  if ! cmp compression_result.txt compression_target.txt >/dev/null 2>&1; then
    printf "Failure in compression tests: basic compression: '%s'\n" "$1"
    exit 1
  fi
  rm -f compression_result.txt "$fn_compressed"

  # clean up
  # try basic compression with misordered positional arguments
  ../mktar compression_target.txt --compress="$arg_compression"
  ../tarcat "$fn_compressed" > compression_result.txt
  if ! cmp compression_result.txt compression_target.txt >/dev/null 2>&1; then
    printf "Failure in compression tests: basic compression with misordered positional arguments: '%s'\n" "$1"
    exit 1
  fi
  rm -f compression_result.txt "$fn_compressed"

  # try implicit compression
  ../mktar compression_target.txt -n "$fn_compressed"
  # try implicit compression with short name flag
  ../mktar -n "$fn_compressed" compression_target.txt
  ../tarcat "$fn_compressed" > compression_result.txt
  if ! cmp compression_result.txt compression_target.txt >/dev/null 2>&1; then
    printf "Failure in compression tests: implicit compression: '%s'\n" "$1"
    printf "Failure in compression tests: implicit compression with short name flag: '%s'\n" "$1"
    exit 1
  fi
  rm -f compression_result.txt "$fn_compressed"

  # clean up
  # try implicit compression with long name flag
  ../mktar --name "$fn_compressed" compression_target.txt
  ../tarcat "$fn_compressed" > compression_result.txt
  if ! cmp compression_result.txt compression_target.txt >/dev/null 2>&1; then
    printf "Failure in compression tests: implicit compression with long name flag: '%s'\n" "$1"
    exit 1
  fi
  rm -f compression_result.txt "$fn_compressed"

  # try batch compression
  ../mktar-batch --compress="$arg_compression" --name "$fn_compressed" compression_target.txt
  ../tarcat "$fn_compressed" > compression_result.txt
  if ! cmp compression_result.txt compression_target.txt >/dev/null 2>&1; then
    printf "Failure in compression tests: basic compression: '%s'\n" "$1"
    exit 1
  fi
  rm -f compression_result.txt "$fn_compressed"
}