SELinux support for certbot-ocsp-fetcher

Signed-off-by: Tommy <contact@tommytran.io>
2024-06-29 17:53:27 -07:00
parent b5a6386526
commit ab5fb14232
2 changed files with 686 additions and 2 deletions
--- a/setup.sh
+++ b/setup.sh
@@ -53,7 +53,7 @@ chmod 644 /etc/systemd/system/nginx.service.d/override.conf
 sudo systemctl daemon-reload

 # Setup certbot-ocsp-fetcher
-unpriv curl https://raw.githubusercontent.com/GrapheneOS/infrastructure/main/certbot-ocsp-fetcher | sudo tee /usr/local/bin/certbot-ocsp-fetcher
+unpriv curl https://raw.githubusercontent.com/TommyTran732/NGINX-Configs/main/usr/local/bin/certbot-ocsp-fetcher | sudo tee /usr/local/bin/certbot-ocsp-fetcher
 ## Explicitly using /var/usrlocal/bin here because SELinux does not follow symlinks
 sudo semanage fcontext -a -t bin_t /var/usrlocal/bin/certbot-ocsp-fetcher
 sudo restorecon -Rv /var/usrlocal/bin/certbot-ocsp-fetcher
@@ -108,4 +108,4 @@ unpriv curl https://raw.githubusercontent.com/TommyTran732/NGINX-Configs/main/et
 unpriv curl https://raw.githubusercontent.com/TommyTran732/NGINX-Configs/main/etc/nginx/snippets/quic.conf | sudo tee /etc/nginx/snippets/quic.conf
 unpriv curl https://raw.githubusercontent.com/TommyTran732/NGINX-Configs/main/etc/nginx/snippets/security.conf | sudo tee /etc/nginx/snippets/security.conf
 unpriv curl https://raw.githubusercontent.com/TommyTran732/NGINX-Configs/main/etc/nginx/snippets/cross-origin-security.conf | sudo tee /etc/nginx/snippets/cross-origin-security.conf
-unpriv curl https://raw.githubusercontent.com/TommyTran732/NGINX-Configs/main/etc/nginx/snippets/universal_paths.conf | sudo tee /etc/nginx/snippets/universal_paths.conf
+unpriv curl https://raw.githubusercontent.com/TommyTran732/NGINX-Configs/main/etc/nginx/snippets/universal_paths.conf | sudo tee /etc/nginx/snippets/universal_paths.conf
--- a/usr/local/bin/certbot-ocsp-fetcher
+++ b/usr/local/bin/certbot-ocsp-fetcher
@@ -0,0 +1,684 @@
+#!/usr/bin/env bash
+
+# This file is the same as https://github.com/tomwassenberg/certbot-ocsp-fetcher
+# but with extra logic to restore SELinux context
+
+# Unofficial Bash strict mode
+set \
+  -o errexit \
+  -o errtrace \
+  -o noglob \
+  -o nounset \
+  -o pipefail
+IFS=$'\n\t'
+shopt -s inherit_errexit
+
+determine_colored_output() {
+  declare -gl COLORED_STDOUT COLORED_STDERR
+  readonly GREEN='\033[0;32m'
+  readonly RED='\033[0;31m'
+  readonly COLOR_DEFAULT='\033[0m'
+
+  if [[ -v NO_COLOR || ${TERM-} == dumb ]]; then
+    COLORED_STDOUT=false COLORED_STDERR=false
+  else
+    [[ -t 1 ]] || COLORED_STDOUT=false
+    [[ -t 2 ]] || COLORED_STDERR=false
+  fi
+
+}
+
+exit_with_error() {
+  local error_prefix=error:$'\t\t'
+
+  [[ ${COLORED_STDERR-} != false ]] &&
+    local -r COLORED_ERROR_MSG=${RED}${error_prefix}${*}${COLOR_DEFAULT}
+
+  # We will have closed file descriptor 2 unless verbosity was requested, so we
+  # will try to use FD5 (the FD that stderr was likely redirected to), and
+  # fallback to FD2 if FD5 wasn't opened yet.
+  if [[ -f /dev/fd/5 ]]; then
+    exec >&5
+  else
+    exec >&2
+  fi
+  printf '%b\n' "${COLORED_ERROR_MSG:-${error_prefix}${@}}"
+
+  exit 1
+}
+
+check_for_dependencies() {
+  if ((BASH_VERSINFO[0] == 4 && BASH_VERSINFO[1] < 3 || BASH_VERSINFO[0] < 4)); then
+    exit_with_error "${0##*/} requires Bash 4.3+."
+  fi
+
+  if ! { command -v openssl >&- &&
+    [[ $(openssl version || true) =~ ^OpenSSL\ ([[:digit:]]+)\.([[:digit:]]+) ]] &&
+    ((BASH_REMATCH[1] == 1 && BASH_REMATCH[2] >= 1 || BASH_REMATCH[1] > 1)); }; then
+    # shellcheck disable=2016
+    exit_with_error \
+      "${0##*/} requires OpenSSL 1.1.0+," \
+      'but it is not available on $PATH.'
+  fi
+}
+
+parse_cli_options() {
+  local -r cli_options="
+Usage: ${0} [-c/--certbot-dir DIRECTORY] [-f/--force-update] \\
+  [-h/--help] [-l/--no-color] [-n/--cert-name NAME[,NAME...] \\
+  [-u/--ocsp-responder URL]] [-o/--output-dir DIRECTORY] \\
+  [-q/--quiet|-v/--verbose] [-w/--no-reload-webserver]
+"
+
+  print_option_error() {
+    local reason=${1} option=${2}
+    shift 2
+    local option_error="${option}: "
+
+    case ${reason} in
+      --conflict)
+        local second_option=${1}
+        shift
+        option_error+="This option cannot be combined with the option ${second_option}."
+        ;;
+      --duplicate)
+        option_error+="This option cannot be specified multiple times."
+        ;;
+      --unknown)
+        option_error+="Invalid option."
+        ;;
+      --value)
+        option_error+="This option requires a value."
+        ;;
+      *)
+        exit 1
+        ;;
+    esac
+
+    exit_with_error "${option_error}" "${cli_options}"
+  }
+
+  declare -gl ERROR_ENCOUNTERED
+
+  declare -gi VERBOSITY=${VERBOSITY:-1}
+
+  while ((${#} > 0)); do
+    local parameter=${1}
+
+    case ${parameter} in
+      -[^-]?*)
+        set -- "-${parameter:1:1}" "-${parameter:2}" "${@:2}"
+        ;;
+      -c | --certbot-dir | --certbot-dir=?*)
+        if [[ -v CERTBOT_DIR ]]; then
+          print_option_error --duplicate "${parameter}"
+        fi
+
+        if [[ ${parameter} =~ --certbot-dir=(.+) ]]; then
+          CERTBOT_DIR=${BASH_REMATCH[1]}
+        else
+          if [[ -n ${2-} ]]; then
+            CERTBOT_DIR=${2}
+            shift
+          else
+            print_option_error --value "${parameter}"
+          fi
+        fi
+
+        CERTBOT_DIR=$(
+          realpath \
+            --canonicalize-missing \
+            --relative-base . \
+            -- "${CERTBOT_DIR}"
+          echo x
+        )
+        CERTBOT_DIR=${CERTBOT_DIR%??}
+        shift
+        ;;
+      -f | --force-update)
+        if [[ ! -v FORCE_UPDATE ]]; then
+          declare -glr FORCE_UPDATE=true
+        fi
+        shift
+        ;;
+      -h | --help)
+        {
+          printf '%s\n' certbot-ocsp-fetcher
+          printf '%s\n' "${cli_options}"
+          local absolute_tool_path
+          absolute_tool_path=$(realpath --no-symlinks -- "${0}")
+          readonly absolute_tool_path
+          cat <<EOSTRING
+
+certbot-ocsp-fetcher helps you setup OCSP stapling in nginx. The tool primes
+nginx's OCSP cache to work around nginx's flawed OCSP stapling implementation.
+The tool does this by fetching and saving OCSP responses for TLS certificates
+issued with Certbot.
+
+---
+
+Example:
+
+1. Fetch OCSP responses for all certificates managed by Certbot, and save
+them in the current working directory. This should usually be run on a
+schedule, e.g. as a cronjob or systemd timer.
+
+$ ${0}
+
+2. Add the path(s) to the resulting OCSP response(s) as the value of the
+ssl_stapling_file directive in the corresponding vhosts in Nginx. Don't
+forget to reload Nginx afterwards.
+
+3. Re-issue all certificates managed by Certbot, to add the OCSP Must-Staple
+flag to the certs and automatically run certbot-ocsp-fetcher during renewals:
+
+$ certbot renew --deploy-hook ${absolute_tool_path} --force-renewal --must-staple
+
+---
+
+See the online README for an explanation of all the CLI options:
+https://github.com/tomwassenberg/certbot-ocsp-fetcher/blob/main/README.md
+EOSTRING
+        }
+        exit
+        ;;
+      -l | --no-color)
+        readonly COLORED_STDOUT=false COLORED_STDERR=false
+        ;;
+      -n | --cert-name | --cert-name=?*)
+        if [[ ${parameter} =~ --cert-name=(.+) ]]; then
+          local cert_lineages_value=${BASH_REMATCH[1]}
+          shift
+        else
+          if [[ -n ${2-} ]]; then
+            local cert_lineages_value=${2}
+            shift 2
+          else
+            print_option_error --value "${parameter}"
+          fi
+        fi
+
+        # Loop over any lineages passed in the same value of --cert-name.
+        OLDIFS=${IFS}
+        IFS=,
+        declare -Ag CERT_LINEAGES
+        # Check if a hardcoded OCSP responder was specified for this set of
+        # lineages.
+        case ${1-} in
+          -u | --ocsp-responder)
+            if [[ -n ${2-} ]]; then
+              for lineage_name in ${cert_lineages_value}; do
+                CERT_LINEAGES["${lineage_name}"]=${2}
+              done
+              shift
+            else
+              print_option_error --value "${parameter}"
+            fi
+            shift
+            ;;
+          --ocsp-responder=?*)
+            [[ ${1} =~ --ocsp-responder=(.+) ]]
+            for lineage_name in ${cert_lineages_value}; do
+              CERT_LINEAGES["${lineage_name}"]=${BASH_REMATCH[1]}
+            done
+            shift
+            ;;
+          *)
+            # If no OCSP responder was specified, just save the lineage
+            # name as the key, with an empty value.
+            for lineage_name in ${cert_lineages_value}; do
+              CERT_LINEAGES["${lineage_name}"]=
+            done
+            ;;
+        esac
+        unset lineage_name cert_lineages_value
+        IFS=${OLDIFS}
+        ;;
+      -o | --output-dir | --output-dir=?*)
+        if [[ -v OUTPUT_DIR ]]; then
+          print_option_error --duplicate "${parameter}"
+        fi
+
+        if [[ ${parameter} =~ --output-dir=(.+) ]]; then
+          OUTPUT_DIR=${BASH_REMATCH[1]}
+        else
+          if [[ -n ${2-} ]]; then
+            OUTPUT_DIR=${2}
+            shift
+          else
+            print_option_error --value "${parameter}"
+          fi
+        fi
+
+        OUTPUT_DIR=$(
+          realpath \
+            --canonicalize-missing \
+            --relative-base . \
+            -- "${OUTPUT_DIR}"
+          echo x
+        )
+        OUTPUT_DIR=${OUTPUT_DIR%??}
+        shift
+        ;;
+      -q | --quiet)
+        if ((VERBOSITY != 1)); then
+          print_option_error --conflict "${parameter}" -v/--verbose
+        else
+          readonly VERBOSITY=0
+          shift
+        fi
+        ;;
+      -v | --verbose)
+        if ((VERBOSITY == 0)); then
+          print_option_error --conflict "${parameter}" -q/--quiet
+        else
+          VERBOSITY+=1
+          shift
+        fi
+        ;;
+      -w | --no-reload-webserver)
+        if [[ ! -v RELOAD_WEBSERVER ]]; then
+          declare -glr RELOAD_WEBSERVER=false
+        fi
+        shift
+        ;;
+      *)
+        print_option_error --unknown "${parameter}"
+        ;;
+    esac
+  done
+
+  # Respect the common "DEBUG" environment variable if set, unless the --quiet
+  # or --verbose flag has been passed as well.
+  if ((${DEBUG:-0} >= 1)) && ((VERBOSITY == 1)); then
+    # We set VERBOSITY to 0 in case of --quiet, so use the value of $DEBUG
+    # incremented with 1 to match it with $VERBOSITY.
+    VERBOSITY=$((DEBUG + 1))
+  fi
+
+  # When not parsed, the stdout and/or stderr output of all external commands
+  # we call in the script is redirected to file descriptor 3.  Depending on the
+  # desired verbosity, we redirect this file descriptor to either stderr or to
+  # /dev/null.
+  if ((VERBOSITY >= 2)); then
+    exec 3>&2
+  else
+    exec 3>/dev/null
+  fi
+
+  # First copy file descriptor 2 to a new FD, so stderr can still be used
+  # (unconditionally) in the exit_with_error function.
+  exec 5>&2
+  if ((VERBOSITY < 1)); then
+    exec 2>/dev/null
+  fi
+}
+
+# Set output directory if necessary and check if it's writeable
+prepare_output_dir() {
+  if [[ -v OUTPUT_DIR ]]; then
+    if [[ ! -e ${OUTPUT_DIR} ]]; then
+      # Don't yet fail if it's not possible to create the directory, so we can
+      # exit with a custom error down below
+      mkdir \
+        --parents \
+        -- "${OUTPUT_DIR}" || true
+    fi
+  else
+    # Use $CACHE_DIRECTORY if set (e.g. when run as a systemd service),
+    # otherwise the working directory
+    readonly OUTPUT_DIR=${CACHE_DIRECTORY:-.}
+  fi
+
+  if [[ ! -w ${OUTPUT_DIR} ]]; then
+    exit_with_error "no write access to output directory (\"${OUTPUT_DIR}\")"
+  fi
+}
+
+start_in_correct_mode() {
+  # Create temporary directory to store OCSP staple file,
+  # before having checked the certificate status in the response
+  local temp_output_dir
+  temp_output_dir=$(mktemp --directory)
+  readonly temp_output_dir
+  trap "rm -r -- ""${temp_output_dir}" EXIT
+
+  declare -A lineages_processed
+
+  # These two environment variables are set if this script is invoked by Certbot
+  if [[ ! -v RENEWED_DOMAINS || ! -v RENEWED_LINEAGE ]]; then
+    run_standalone
+  else
+    run_as_deploy_hook
+  fi
+
+  print_and_handle_result
+}
+
+# Run in "check one or all certificate lineage(s) managed by Certbot" mode
+# $1 - Path to temporary output directory
+run_standalone() {
+  printf >&2 '%s\n\n' "Running in stand-alone mode..."
+
+  readonly CERTBOT_DIR=${CERTBOT_DIR:-/etc/letsencrypt}
+
+  if [[ ! -r ${CERTBOT_DIR} || (-d ${CERTBOT_DIR}/live && ! -r ${CERTBOT_DIR}/live) ]]; then
+    exit_with_error "can't access ${CERTBOT_DIR}/live"
+  fi
+
+  # Check specific lineage if passed on CLI,
+  # or otherwise all lineages in Certbot's dir
+  if [[ -n ${!CERT_LINEAGES[*]} ]]; then
+    for lineage_name in "${!CERT_LINEAGES[@]}"; do
+      if [[ -r ${CERTBOT_DIR}/live/${lineage_name} ]]; then
+        fetch_ocsp_response \
+          --standalone \
+          "${temp_output_dir}" \
+          "${lineage_name}" \
+          "${CERT_LINEAGES["${lineage_name}"]}"
+      else
+        exit_with_error "can't access ${CERTBOT_DIR}/live/${lineage_name}"
+      fi
+    done
+  else
+    set +f
+    shopt -s nullglob
+    for lineage_dir in "${CERTBOT_DIR}"/live/*; do
+      set -f
+
+      # Skip non-directories, like Certbot's README file
+      [[ -d ${lineage_dir} ]] || continue
+
+      fetch_ocsp_response \
+        --standalone "${temp_output_dir}" "${lineage_dir##*/}"
+    done
+    unset lineage_dir
+  fi
+}
+
+# Run in deploy-hook mode, only processing the passed lineage
+# $1 - Path to temporary output directory
+run_as_deploy_hook() {
+  printf >&2 '%s\n\n' "Running as a deploy hook of Certbot..."
+
+  if [[ -v CERTBOT_DIR ]]; then
+    # The directory is already inferred from the environment variable that
+    # Certbot passes
+    exit_with_error \
+      "-c/--certbot-dir cannot be passed" \
+      "when run as Certbot hook"
+  fi
+
+  if [[ -v FORCE_UPDATE ]]; then
+    # When run as deploy hook the behavior of this flag is used by default.
+    # Therefore passing this flag would not have any effect.
+    exit_with_error \
+      "-f/--force-update cannot be passed" \
+      "when run as Certbot hook"
+  fi
+
+  if [[ -n ${!CERT_LINEAGES[*]} ]]; then
+    # The certificate lineage is already inferred from the environment
+    # variable that Certbot passes
+    exit_with_error "-n/--cert-name cannot be passed when run as Certbot hook"
+  fi
+
+  fetch_ocsp_response \
+    --deploy_hook "${temp_output_dir}" "${RENEWED_LINEAGE##*/}"
+}
+
+# Check if it's necessary to fetch a new OCSP response
+check_for_existing_ocsp_staple_file() {
+  [[ -f ${OUTPUT_DIR}/${lineage_name}.der ]] || return 1
+
+  # Validate and verify the existing local OCSP staple file
+  local existing_ocsp_response
+  set +e
+  existing_ocsp_response=$(openssl ocsp \
+    -no_nonce \
+    -issuer "${lineage_dir}/chain.pem" \
+    -cert "${lineage_dir}/cert.pem" \
+    -verify_other "${lineage_dir}/chain.pem" \
+    -respin "${OUTPUT_DIR}/${lineage_name}.der" 2>&3)
+  local -ir existing_ocsp_response_rc=${?}
+  set -e
+  readonly existing_ocsp_response
+
+  ((existing_ocsp_response_rc == 0)) || return 1
+
+  for existing_ocsp_response_line in ${existing_ocsp_response}; do
+    if [[ ${existing_ocsp_response_line} =~ ^[[:blank:]]*"This Update: "(.+)$ ]]; then
+      local -r this_update=${BASH_REMATCH[1]}
+    elif [[ ${existing_ocsp_response_line} =~ ^[[:blank:]]*"Next Update: "(.+)$ ]]; then
+      local -r next_update=${BASH_REMATCH[1]}
+    fi
+  done
+  [[ -n ${this_update-} && -n ${next_update-} ]] || return 1
+
+  # Only continue fetching OCSP response if existing response expires within
+  # half of its lifetime.
+  {
+    # The command substitutions here don't respect `set -o errexit`, but in
+    # case any of them fail, the total command still fails unless both
+    # substitutions print an integer. This seems very unlikely to occur, so
+    # let's ignore this.
+    # shellcheck disable=2312
+    local -ri response_lifetime_in_seconds=$(($(date +%s --date "${next_update}") - $(date +%s --date "${this_update}")))
+
+    # `set -o errexit` isn't respected here either, but we default to renewing
+    # the OCSP response, so this is fine.
+    # shellcheck disable=2312
+    (($(date +%s) < $(date +%s --date "${this_update}") + response_lifetime_in_seconds / 2)) || return 1
+  }
+}
+
+# Generate file used by ssl_stapling_file in nginx config of websites
+# $1 - Whether to run as a deploy hook for Certbot, or standalone
+# $2 - Path to temporary output directory
+# $3 - Name of certificate lineage
+# $4 - OCSP endpoint (if specified on command line)
+fetch_ocsp_response() {
+  local -r temp_output_dir=${2}
+  local -r lineage_name=${3}
+
+  # This validation should be revisited once
+  # https://github.com/certbot/certbot/issues/6127 is fixed.
+  if [[ ${lineage_name} =~ ($'\n')|($'\t') ]]; then
+    ERROR_ENCOUNTERED=true
+    exit_with_error \
+      "Unsupported characters encountered in the following" \
+      "lineage name: ${lineage_name}$'\n\n'" \
+      "Lineage names with embedded tabs or newlines are not supported," \
+      "because Certbot (as of version 1.18.0) does not have well-defined" \
+      'behavior on handling any "unconventional" lineage names.'
+  fi
+
+  case ${1} in
+    --standalone)
+      local -r lineage_dir=${CERTBOT_DIR}/live/${lineage_name}
+
+      # `set -o errexit` is not respected here, but in case of failure we still
+      # err on the safe side by renewing the OCSP staple file.
+      # shellcheck disable=2310
+      if [[ ${FORCE_UPDATE-} != true ]] &&
+        check_for_existing_ocsp_staple_file; then
+        lineages_processed["${lineage_name}"]="not updated"$'\t'"valid staple file on disk"
+        return
+      fi
+      ;;
+    --deploy_hook)
+      local -r lineage_dir=${RENEWED_LINEAGE}
+      ;;
+    *)
+      return 1
+      ;;
+  esac
+  shift 3
+
+  # Verify that the leaf certificate is still valid. If the certificate is
+  # expired, we don't have to request a (new) OCSP response.
+  local cert_expiry_output
+  set +e
+  cert_expiry_output=$(openssl x509 \
+    -in "${lineage_dir}/cert.pem" \
+    -checkend 0 \
+    -noout 2>&3)
+  local -ri cert_expiry_rc=${?}
+  set -e
+  if ((cert_expiry_rc != 0)); then
+    ERROR_ENCOUNTERED=true
+    lineages_processed["${lineage_name}"]="failed to update"
+    if [[ ${cert_expiry_output} == "Certificate will expire" ]]; then
+      lineages_processed["${lineage_name}"]+=$'\t'"leaf certificate expired"
+    fi
+    return
+  fi
+
+  local ocsp_endpoint
+  if [[ -n ${1-} ]]; then
+    ocsp_endpoint=${1}
+  else
+    ocsp_endpoint=$(openssl x509 \
+      -noout \
+      -ocsp_uri \
+      -in "${lineage_dir}/cert.pem" \
+      2>&3)
+  fi
+
+  # Request, verify and temporarily save the actual OCSP response,
+  # and check whether the certificate status is "good"
+  local ocsp_call_output
+  set +e
+  ocsp_call_output=$(openssl ocsp \
+    -no_nonce \
+    -url "${ocsp_endpoint}" \
+    -issuer "${lineage_dir}/chain.pem" \
+    -cert "${lineage_dir}/cert.pem" \
+    -verify_other "${lineage_dir}/chain.pem" \
+    -respout "${temp_output_dir}/${lineage_name}.der" 2>&3)
+  local -ir ocsp_call_rc=${?}
+  set -e
+  readonly ocsp_call_output=${ocsp_call_output#"${lineage_dir}"/cert.pem: }
+  local -r cert_status=${ocsp_call_output%%$'\n'*}
+
+  if [[ ${ocsp_call_rc} != 0 || ${cert_status} != good ]]; then
+    ERROR_ENCOUNTERED=true
+
+    lineages_processed["${lineage_name}"]="failed to update"
+    if ((VERBOSITY >= 2)); then
+      lineages_processed["${lineage_name}"]+=$'\t'"${ocsp_call_output//[[:space:]]/ }"
+    else
+      lineages_processed["${lineage_name}"]+=$'\t'"${cert_status}"
+    fi
+
+    return
+  fi
+
+  # If arrived here status was good, so move OCSP staple file to definitive
+  # folder
+  mv "${temp_output_dir}/${lineage_name}.der" "${OUTPUT_DIR}/"
+
+  # Restore SELinux context on SELinux systems
+  if [[ -f /usr/sbin/restorecon ]]; then
+    restorecon "${OUTPUT_DIR}/${lineage_name}.der"
+  fi 
+
+  lineages_processed["${lineage_name}"]=updated
+}
+
+print_and_handle_result() {
+  local -r header=LINEAGE$'\t'RESULT$'\t'REASON
+
+  local lineages_processed_marked_up
+  for lineage_name in "${!lineages_processed[@]}"; do
+    lineages_processed_marked_up+=$'\n'"${lineage_name}"$'\t'
+    if [[ ${COLORED_STDOUT-} != false ]]; then
+      if [[ ${lineages_processed["${lineage_name}"]} =~ ^updated ]]; then
+        lineages_processed_marked_up+=${GREEN}
+      elif [[ ${lineages_processed["${lineage_name}"]} =~ ^"failed to update" ]]; then
+        lineages_processed_marked_up+=${RED}
+      fi
+      lineages_processed_marked_up+=${lineages_processed["${lineage_name}"]}${COLOR_DEFAULT}
+    else
+      lineages_processed_marked_up+=${lineages_processed["${lineage_name}"]}
+    fi
+  done
+  unset lineage_name
+  lineages_processed_marked_up=$(sort <<<"${lineages_processed_marked_up-}")
+  readonly lineages_processed_marked_up
+
+  if [[ ${RELOAD_WEBSERVER-} != false ]]; then
+    reload_webserver
+  fi
+
+  local output=${header}${lineages_processed_marked_up-}${nginx_status-}
+
+  if ((VERBOSITY >= 1)); then
+    local output_table
+    # shellcheck disable=2016
+    output_table=$(column \
+      --output-separator $'\t' \
+      --separator $'\t' \
+      --table \
+      <<<"${output}" \
+      2>/dev/null) ||
+      output_table=$(column -s$'\t' -t <<<"${output}" 2>/dev/null) ||
+      local -r column_error=($'\n'
+        'Install the BSD utility `column` for properly formatted output.'
+        'If the version of `column` supports the `--output-separator` flag,'
+        'the output will be formatted as TSV.'
+        $'\n'
+      )
+    readonly output=${output_table:-${output}}
+    unset output_table
+
+    # Extract header to direct it to stderr
+    printf '%s\n' "${output%%$'\n'*}" >&2
+    # Remove header before printing everything else to stdout
+    [[ -n ${!lineages_processed[*]} ]] && printf '%b\n' "${output#*$'\n'}"
+
+    if [[ ${COLORED_STDERR-} != false ]]; then
+      printf %b "${RED}${column_error[*]-}${COLOR_DEFAULT}" >&2
+    else
+      printf %b "${column_error[*]-}" >&2
+    fi
+  fi
+
+  [[ ${ERROR_ENCOUNTERED-} != true ]]
+}
+
+reload_webserver() {
+  for lineage_name in "${!lineages_processed[@]}"; do
+    if [[ ${lineages_processed["${lineage_name}"]} == updated ]]; then
+      local nginx_status
+      if nginx -s reload >&3 2>&1; then
+        [[ ${COLORED_STDERR-} != false ]] && nginx_status=${GREEN}
+        # The last line includes a leading space, to workaround the lack of the
+        # `-n` flag in later versions of `column`.
+        nginx_status+=$'\n\n \t'"nginx reloaded"
+      else
+        ERROR_ENCOUNTERED=true
+        [[ ${COLORED_STDERR-} != false ]] && nginx_status=${RED}
+        nginx_status=$'\n\n \t'"nginx not reloaded"$'\t'"unable to reload nginx service, try manually"
+      fi
+      [[ ${COLORED_STDERR-} != false ]] &&
+        readonly nginx_status+=${COLOR_DEFAULT}
+      break
+    fi
+  done
+  unset lineage_name
+}
+
+main() {
+  check_for_dependencies
+
+  determine_colored_output
+
+  parse_cli_options "${@}"
+
+  prepare_output_dir
+
+  start_in_correct_mode
+}
+
+main "${@}"